hackboxen 0.1.0

data/bin/hb-install

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'hackboxen/tasks/install'
+
+Rake::Task[:install].invoke

data/bin/hb-runner

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# Figure out a bunch of paths
+RUNNER_SCRIPT=`readlink -f $0`
+HACKBOXEN_LIB_RELATIVE=`dirname ${RUNNER_SCRIPT}`/..
+HACKBOXEN_LIB=`readlink -f ${HACKBOXEN_LIB_RELATIVE}`
+
+# Set some defaults
+DEFAULT_RUBY_VERSION=ruby-1.8.7
+export TERM=dumb
+
+# Ensure hackbox directory is given.
+HACKBOX_DIR_RELATIVE="$1";
+if [ -z "$HACKBOX_DIR_RELATIVE" ]; then
+    echo "Must specify the path to a hackbox to run";
+    exit 1;
+fi
+HACKBOX_DIR=`readlink -f ${HACKBOX_DIR_RELATIVE}`
+
+# Ensure hackbox directory exists.
+if [ ! -d "$HACKBOX_DIR" ]; then
+    echo "Hackbox directory $HACKBOX_DIR does not exist";
+    exit 1;
+fi
+
+# Ensure config file exists
+HACKBOX_CONFIG="${HACKBOX_DIR}/config/config.yaml"
+if [ ! -e "$HACKBOX_CONFIG" ]; then
+    echo "Hackbox directory ${HACKBOX_DIR} does not contain a config/config.yaml file";
+    exit 1;
+fi
+
+# Find Ruby version
+REQUIRED_RUBY_VERSION=`ruby -r yaml -e "puts YAML.load_file('${HACKBOX_CONFIG}')['requires']['language']['ruby'].to_s.strip rescue nil"`
+if [ -z "$REQUIRED_RUBY_VERSION" ]; then
+    REQUIRED_RUBY_VERSION=$DEFAULT_RUBY_VERSION;
+fi
+
+# Source RVM and use chosen RVM Ruby version
+if [ -s /usr/local/rvm/scripts/rvm ] || [ -s ~/.rvm/bin/rvm ] ; then
+    [[ -s "/usr/local/rvm/scripts/rvm" ]] && . "/usr/local/rvm/scripts/rvm"
+    [[ -s "$HOME/.rvm/scripts/rvm" ]] && . "$HOME/.rvm/scripts/rvm"
+    # RVM defines RUBY_VERSION when it its invoked
+    if rvm use $REQUIRED_RUBY_VERSION; then
+	echo "Now running ${RUBY_VERSION}";
+    else
+	echo "Could not run ${RUBY_VERSION}";
+	exit 1;
+    fi
+fi
+
+# Symlink to Hackboxen Gemfile if necessary
+HACKBOX_GEMFILE="${HACKBOX_DIR}/Gemfile"
+DEFAULT_GEMFILE="${HACKBOXEN_LIB}/lib/gemfiles/Gemfile.${REQUIRED_RUBY_VERSION}.default"
+if [ ! -e "$HACKBOX_GEMFILE" ]; then
+    ln -s $DEFAULT_GEMFILE $HACKBOX_GEMFILE
+fi
+
+# Export the correct Gemfile for bundler to find once Ruby starts
+export BUNDLE_GEMFILE="$HACKBOX_GEMFILE"
+
+# Change to Hackbox Dir
+cd $HACKBOX_DIR
+
+# Remove Gemfile.lock if it exists
+# if [ -e Gemfile.lock ]; then
+#     rm Gemfile.lock;
+# fi
+
+# Something is setting RUBYOPT causing havoc w/Bundler.
+unset RUBYOPT;
+
+# Bundle install all gems into the vendor directory inside this
+# hackbox.
+if bundle install --path vendor; then
+    echo "Bundle installed";
+else
+    echo "Could not bundle install";
+    exit 1;
+fi
+
+# Make sure hackboxen/lib is on RUBYLIB
+export RUBYLIB=${HACKBOXEN_LIB}/lib:${RUBYLIB}
+
+# Run hackbox and add a sentinel at the end
+if bundle exec rake --trace; then
+    echo "<-- SUCCESSFULLY RAN HACKBOX -->";
+else
+    echo "<-- FAILED TO RUN HACKBOX -->";
+    exit 1;
+fi
+
+exit 0;

data/bin/hb-scaffold

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'hackboxen/template'
+require 'hackboxen/tasks/scaffold'
+
+Rake::Task[:scaffold].invoke

data/hackboxen.gemspec

@@ -0,0 +1,97 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{hackboxen}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["kornypoet", "Ganglion", "bollacker"]
+  s.date = %q{2011-06-30}
+  s.description = %q{A simple framework to assist in standardizing the data-munging input/output process.}
+  s.email = %q{travis@infochimps.com}
+  s.executables = ["hb-install", "hb-scaffold", "hb-runner"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.textile"
+  ]
+  s.files = [
+    "CHANGELOG.textile",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.textile",
+    "Rakefile",
+    "VERSION",
+    "bin/describe.rb",
+    "bin/hb-install",
+    "bin/hb-runner",
+    "bin/hb-scaffold",
+    "hackboxen.gemspec",
+    "lib/gemfiles/Gemfile.jruby-1.6.2.default",
+    "lib/gemfiles/Gemfile.ruby-1.8.7.default",
+    "lib/gemfiles/Gemfile.ruby-1.9.2.default",
+    "lib/hackboxen.rb",
+    "lib/hackboxen/tasks.rb",
+    "lib/hackboxen/tasks/endpoint.rb",
+    "lib/hackboxen/tasks/icss.rb",
+    "lib/hackboxen/tasks/init.rb",
+    "lib/hackboxen/tasks/install.rb",
+    "lib/hackboxen/tasks/mini.rb",
+    "lib/hackboxen/tasks/scaffold.rb",
+    "lib/hackboxen/template.rb",
+    "lib/hackboxen/template/Rakefile.erb",
+    "lib/hackboxen/template/config.yaml.erb",
+    "lib/hackboxen/template/endpoint.rb.erb",
+    "lib/hackboxen/template/icss.yaml.erb",
+    "lib/hackboxen/template/main.erb",
+    "lib/hackboxen/utils.rb",
+    "lib/hackboxen/utils/README_ConfigValidator.textile",
+    "lib/hackboxen/utils/config_validator.rb",
+    "lib/hackboxen/utils/logging.rb",
+    "lib/hackboxen/utils/paths.rb",
+    "spec/install_spec.rb"
+  ]
+  s.homepage = %q{http://github.com/infochimps/hackboxen}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{A simple framework to assist in standardizing the data-munging input/output process.}
+  s.test_files = [
+    "spec/install_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<swineherd>, [">= 0.0.4"])
+      s.add_runtime_dependency(%q<configliere>, ["= 0.4.6"])
+      s.add_runtime_dependency(%q<rake>, ["= 0.8.7"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<swineherd>, [">= 0.0.4"])
+      s.add_dependency(%q<configliere>, ["= 0.4.6"])
+      s.add_dependency(%q<rake>, ["= 0.8.7"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<swineherd>, [">= 0.0.4"])
+    s.add_dependency(%q<configliere>, ["= 0.4.6"])
+    s.add_dependency(%q<rake>, ["= 0.8.7"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/lib/gemfiles/Gemfile.jruby-1.6.2.default

@@ -0,0 +1,19 @@
+source :rubygems                                                                                                   
+
+gem 'rake'                                                                                                                   
+gem 'json'                                                                                                         
+gem 'extlib'                                                                                                       
+gem 'nokogiri'                                                                                                     
+gem 'ken'                                                                                                          
+gem 'mechanize'                                                                                                    
+gem 'right_aws'                                                                                                    
+gem 'rgeo'                                                                                                         
+gem 'configliere'                                                                                                  
+gem 'activemodel'
+gem 'yajl-ruby'
+gem 'sqlite3-ruby'
+gem 'swineherd'
+gem 'gorillib'
+gem 'icss'
+gem 'gorillib'
+

data/lib/gemfiles/Gemfile.ruby-1.8.7.default

@@ -0,0 +1,20 @@
+source :rubygems                                                                                                   
+
+gem 'rake'                                                                                                                   
+gem 'json'                                                                                                         
+gem 'extlib'                                                                                                       
+gem 'nokogiri'                                                                                                     
+gem 'ken'                                                                                                          
+gem 'shapelib'                                                                                                     
+gem 'mechanize'                                                                                                    
+gem 'right_aws'                                                                                                    
+gem 'rgeo'                                                                                                         
+gem 'configliere'                                                                                                  
+gem 'activemodel'
+gem 'yajl-ruby'
+gem 'sqlite3-ruby'
+gem 'swineherd'
+gem 'gorillib'
+gem 'icss'
+gem 'fastercsv'
+

data/lib/gemfiles/Gemfile.ruby-1.9.2.default

@@ -0,0 +1,18 @@
+source :rubygems                                                                                                   
+
+gem 'rake'                                                                                                                   
+gem 'json'                                                                                                         
+gem 'extlib'                                                                                                       
+gem 'nokogiri'                                                                                                     
+gem 'ken'                                                                                                          
+gem 'shapelib'                                                                                                     
+gem 'mechanize'                                                                                                    
+gem 'right_aws'                                                                                                    
+gem 'rgeo'                                                                                                         
+gem 'configliere'                                                                                                  
+gem 'activemodel'
+gem 'yajl-ruby'
+gem 'sqlite3-ruby'
+gem 'swineherd'
+gem 'gorillib'
+gem 'icss'

data/lib/hackboxen.rb

@@ -0,0 +1,17 @@
+INCLUDING_FILE = caller[2].gsub(/:.*$/, '')
+
+require 'rake'
+require 'swineherd'
+require 'configliere'
+require 'json'
+require 'hackboxen/utils'
+require 'hackboxen/tasks'
+
+machine_cfg = "/etc/hackbox/hackbox.yaml"
+user_cfg    = File.join(ENV['HOME'], '.hackbox/hackbox.yaml')
+hackbox_cfg = File.join(HackBoxen.find_root_dir, 'config/config.yaml')
+
+WorkingConfig.read machine_cfg if File.exists? machine_cfg
+WorkingConfig.read user_cfg    if File.exists? user_cfg
+WorkingConfig.read hackbox_cfg if File.exists? hackbox_cfg
+

data/lib/hackboxen/tasks.rb

@@ -0,0 +1,6 @@
+require 'hackboxen/tasks/init'
+require 'hackboxen/tasks/icss'
+require 'hackboxen/tasks/mini'
+require 'hackboxen/tasks/endpoint'
+
+include HackBoxen::Paths

data/lib/hackboxen/tasks/endpoint.rb

@@ -0,0 +1,16 @@
+module HackBoxen
+  namespace :hb do
+
+    desc "Copy an endpoint.rb file to the output code directory if the endpoint exists"
+    task :endpoint => ['hb:create_required_paths'] do
+      endpoint = "#{WorkingConfig[:protocol]}_endpoint.rb"
+      srcfile  = File.join(path_to(:hb_engine), endpoint)
+      if File.exists? srcfile
+        HackBoxen.current_fs.mkpath path_to(:code_dir)
+        destfile = File.join(path_to(:code_dir), endpoint)
+        HackBoxen.current_fs.cp(srcfile, destfile)
+      end
+    end
+
+  end
+end

data/lib/hackboxen/tasks/icss.rb

@@ -0,0 +1,15 @@
+module HackBoxen
+  namespace :hb do
+
+    desc "Copy the Icss file to the output directory in the filesystem specified in the WorkingConfig"
+    task :icss => ['hb:create_required_paths'] do
+      icss_yaml   = File.join(path_to(:hb_config), "#{WorkingConfig[:protocol]}.icss.yaml")
+      if File.exists? icss_yaml
+        icss      = YAML.load(File.read icss_yaml)
+        icss_json = File.join(path_to(:data_dir), "#{WorkingConfig[:protocol]}.icss.json")
+        HackBoxen.current_fs.open(File.join(icss_json), 'w') { |f| f.puts icss.to_json }
+      end
+    end
+
+  end
+end

data/lib/hackboxen/tasks/init.rb

@@ -0,0 +1,36 @@
+module HackBoxen
+  namespace :hb do
+
+    desc "Ensure all required config options are contained in the WorkingConfig"
+    task :validate_working_config do
+      WorkingConfig.resolve!
+      HackBoxen.verify_dependencies
+      # failures = HackBoxen::ConfigValidator.failed_requirements
+      # if failures.size > 0
+      #   raise "Hackbox environment fails to meet requirements:\n-- " + failures.join("\n-- ") + "\n"
+      # end
+    end
+
+    desc "Create the required output directories using the filesystem specified by the WorkingConfig"
+    task :create_required_paths => [:validate_working_config] do
+      output_dirs.each { |dir| HackBoxen.current_fs.mkpath(dir) unless HackBoxen.current_fs.exists? dir }
+    end
+
+    desc "Always save the WorkingConfig out to a file when running a hackbox"
+    task :create_working_config => [:create_required_paths] do
+      working_config = File.join(path_to(:env_dir), 'working_config.json')
+      HackBoxen.current_fs.open(working_config, 'w') { |f| f.write WorkingConfig.to_hash.to_json }
+    end
+
+    desc "Execute the main file inside of the current hackbox directory"
+    task :init => [:create_working_config] do
+      main_file     = File.join(path_to(:hb_engine), 'main')
+      sh "#{main_file} #{path_to(:hb_dataroot)} #{path_to(:data_dir)}" do |ok,res|
+        if !ok
+          puts "Processing script failed with #{res}"
+        end
+      end
+    end
+
+  end
+end

data/lib/hackboxen/tasks/install.rb

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'rake'
+require 'configliere'
+require 'ohai'
+
+Settings.use :commandline
+Settings.define :dataroot, :default => "/data/hb",          :description => "Global directory for hackbox output"
+Settings.define :coderoot, :default => "#{ENV['HOME']}/hb", :description => "Global directory for hackbox code"
+Settings.resolve!
+
+config_dir = File.join(ENV["HOME"], '.hackbox')
+config     = File.join(config_dir, 'hackbox.yaml')
+
+directory config_dir
+
+file config => [config_dir] do
+
+  # Create config hash and setup defaults
+  config_hash = {}
+  config_hash['dataroot'] = Settings[:dataroot]
+  config_hash['coderoot'] = Settings[:coderoot]
+
+  # Configure S3 setup if desired
+  s3_setup = { 'access_key'  => '', 'secret_key'  => '' }
+  print "Configure S3 Filesystem information? [Y]n "
+  unless STDIN.gets.downcase.chomp == 'n'
+    print "Access Key: "
+    s3_setup['access_key']  = STDIN.gets.chomp
+    print "Secret Key: "
+    s3_setup['secret_key']  = STDIN.gets.chomp
+  end
+  config_hash['s3_filesystem'] = s3_setup
+
+  # Add system requirements
+  sys = Ohai::System.new
+  sys.all_plugins
+  config_hash['requires'] = {
+    'machine' => sys[:kernel][:machine],
+    'os'      => sys[:os]
+  }
+
+  File.open(config, 'wb') { |f| f.puts config_hash.to_yaml }
+end
+
+desc "Create the install-level config file, ~/.hackbox/hackbox.yaml"
+task :install => [config]

data/lib/hackboxen/tasks/mini.rb

@@ -0,0 +1,47 @@
+require 'configliere'
+require 'swineherd'
+
+namespace :hb do
+
+  Opts = Configliere::Param.new.use :commandline
+  Opts.define :files, :type => Array
+  Opts.resolve!
+
+  desc "Stores a mini data set from the local filesystem into s3 for running a hackbox in mini mode"
+  task :create_mini do
+    Opts[:files].each do |input|
+      output = File.directory?(input) ? expected_mini_data + '/' : output = File.join(expected_mini_data, File.basename(input))
+      sh "s3cmd put -r #{input} #{output}"
+    end
+  end
+
+  desc "Runs a hackbox, skipping :get_data and instead pulling sample data from the previously stored mini bucket"
+  task :run_mini do
+    WorkingConfig[:dataroot] += "/mini"
+    dest = File.join(WorkingConfig[:dataroot], hackbox_name)
+    raise "Expected mini data has not yet been created" unless s3fs.exists? expected_mini_data.gsub(/s3:\/\//,'')
+    Rake::Task[:get_data].clear_actions
+    Rake::Task['hb:create_working_config'].invoke
+    Rake::Task['hb:icss'].invoke if Rake::Task[:default].prerequisites.include? 'hb:icss'
+    if WorkingConfig[:filesystem_scheme] == 'hdfs'
+      sh "hadoop fs -cp #{expected_mini_data.gsub(/s3:/, 's3n:')} #{dest}"
+    elsif WorkingConfig[:filesystem_scheme] == 'file'
+      sh "s3cmd sync #{expected_mini_data} #{dest}"
+    end
+    Rake::Task['hb:init'].invoke
+  end
+
+end
+
+def expected_mini_data
+  raise "You don't have a s3_filesystem: mini_bucket: specified in your config" unless WorkingConfig['s3_filesystem.mini_bucket']
+  File.join(WorkingConfig['s3_filesystem.mini_bucket'], hackbox_name, 'ripd')
+end
+
+def hackbox_name
+  File.join(WorkingConfig[:namespace].gsub(/\./, '/'), WorkingConfig[:protocol])
+end
+
+def s3fs
+  @s3fs ||= Swineherd::FileSystem.get(:s3, WorkingConfig['s3_filesystem.access_key'], WorkingConfig['s3_filesystem.secret_key'])
+end