hackboxen 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require 'hackboxen/tasks/install'
4
+
5
+ Rake::Task[:install].invoke
@@ -0,0 +1,93 @@
1
+ #!/bin/bash
2
+
3
+ # Figure out a bunch of paths
4
+ RUNNER_SCRIPT=`readlink -f $0`
5
+ HACKBOXEN_LIB_RELATIVE=`dirname ${RUNNER_SCRIPT}`/..
6
+ HACKBOXEN_LIB=`readlink -f ${HACKBOXEN_LIB_RELATIVE}`
7
+
8
+ # Set some defaults
9
+ DEFAULT_RUBY_VERSION=ruby-1.8.7
10
+ export TERM=dumb
11
+
12
+ # Ensure hackbox directory is given.
13
+ HACKBOX_DIR_RELATIVE="$1";
14
+ if [ -z "$HACKBOX_DIR_RELATIVE" ]; then
15
+ echo "Must specify the path to a hackbox to run";
16
+ exit 1;
17
+ fi
18
+ HACKBOX_DIR=`readlink -f ${HACKBOX_DIR_RELATIVE}`
19
+
20
+ # Ensure hackbox directory exists.
21
+ if [ ! -d "$HACKBOX_DIR" ]; then
22
+ echo "Hackbox directory $HACKBOX_DIR does not exist";
23
+ exit 1;
24
+ fi
25
+
26
+ # Ensure config file exists
27
+ HACKBOX_CONFIG="${HACKBOX_DIR}/config/config.yaml"
28
+ if [ ! -e "$HACKBOX_CONFIG" ]; then
29
+ echo "Hackbox directory ${HACKBOX_DIR} does not contain a config/config.yaml file";
30
+ exit 1;
31
+ fi
32
+
33
+ # Find Ruby version
34
+ REQUIRED_RUBY_VERSION=`ruby -r yaml -e "puts YAML.load_file('${HACKBOX_CONFIG}')['requires']['language']['ruby'].to_s.strip rescue nil"`
35
+ if [ -z "$REQUIRED_RUBY_VERSION" ]; then
36
+ REQUIRED_RUBY_VERSION=$DEFAULT_RUBY_VERSION;
37
+ fi
38
+
39
+ # Source RVM and use chosen RVM Ruby version
40
+ if [ -s /usr/local/rvm/scripts/rvm ] || [ -s ~/.rvm/bin/rvm ] ; then
41
+ [[ -s "/usr/local/rvm/scripts/rvm" ]] && . "/usr/local/rvm/scripts/rvm"
42
+ [[ -s "$HOME/.rvm/scripts/rvm" ]] && . "$HOME/.rvm/scripts/rvm"
43
+ # RVM defines RUBY_VERSION when it its invoked
44
+ if rvm use $REQUIRED_RUBY_VERSION; then
45
+ echo "Now running ${RUBY_VERSION}";
46
+ else
47
+ echo "Could not run ${RUBY_VERSION}";
48
+ exit 1;
49
+ fi
50
+ fi
51
+
52
+ # Symlink to Hackboxen Gemfile if necessary
53
+ HACKBOX_GEMFILE="${HACKBOX_DIR}/Gemfile"
54
+ DEFAULT_GEMFILE="${HACKBOXEN_LIB}/lib/gemfiles/Gemfile.${REQUIRED_RUBY_VERSION}.default"
55
+ if [ ! -e "$HACKBOX_GEMFILE" ]; then
56
+ ln -s $DEFAULT_GEMFILE $HACKBOX_GEMFILE
57
+ fi
58
+
59
+ # Export the correct Gemfile for bundler to find once Ruby starts
60
+ export BUNDLE_GEMFILE="$HACKBOX_GEMFILE"
61
+
62
+ # Change to Hackbox Dir
63
+ cd $HACKBOX_DIR
64
+
65
+ # Remove Gemfile.lock if it exists
66
+ # if [ -e Gemfile.lock ]; then
67
+ # rm Gemfile.lock;
68
+ # fi
69
+
70
+ # Something is setting RUBYOPT causing havoc w/Bundler.
71
+ unset RUBYOPT;
72
+
73
+ # Bundle install all gems into the vendor directory inside this
74
+ # hackbox.
75
+ if bundle install --path vendor; then
76
+ echo "Bundle installed";
77
+ else
78
+ echo "Could not bundle install";
79
+ exit 1;
80
+ fi
81
+
82
+ # Make sure hackboxen/lib is on RUBYLIB
83
+ export RUBYLIB=${HACKBOXEN_LIB}/lib:${RUBYLIB}
84
+
85
+ # Run hackbox and add a sentinel at the end
86
+ if bundle exec rake --trace; then
87
+ echo "<-- SUCCESSFULLY RAN HACKBOX -->";
88
+ else
89
+ echo "<-- FAILED TO RUN HACKBOX -->";
90
+ exit 1;
91
+ fi
92
+
93
+ exit 0;
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require 'hackboxen/template'
4
+ require 'hackboxen/tasks/scaffold'
5
+
6
+ Rake::Task[:scaffold].invoke
@@ -0,0 +1,97 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = %q{hackboxen}
8
+ s.version = "0.1.0"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["kornypoet", "Ganglion", "bollacker"]
12
+ s.date = %q{2011-06-30}
13
+ s.description = %q{A simple framework to assist in standardizing the data-munging input/output process.}
14
+ s.email = %q{travis@infochimps.com}
15
+ s.executables = ["hb-install", "hb-scaffold", "hb-runner"]
16
+ s.extra_rdoc_files = [
17
+ "LICENSE.txt",
18
+ "README.textile"
19
+ ]
20
+ s.files = [
21
+ "CHANGELOG.textile",
22
+ "Gemfile",
23
+ "Gemfile.lock",
24
+ "LICENSE.txt",
25
+ "README.textile",
26
+ "Rakefile",
27
+ "VERSION",
28
+ "bin/describe.rb",
29
+ "bin/hb-install",
30
+ "bin/hb-runner",
31
+ "bin/hb-scaffold",
32
+ "hackboxen.gemspec",
33
+ "lib/gemfiles/Gemfile.jruby-1.6.2.default",
34
+ "lib/gemfiles/Gemfile.ruby-1.8.7.default",
35
+ "lib/gemfiles/Gemfile.ruby-1.9.2.default",
36
+ "lib/hackboxen.rb",
37
+ "lib/hackboxen/tasks.rb",
38
+ "lib/hackboxen/tasks/endpoint.rb",
39
+ "lib/hackboxen/tasks/icss.rb",
40
+ "lib/hackboxen/tasks/init.rb",
41
+ "lib/hackboxen/tasks/install.rb",
42
+ "lib/hackboxen/tasks/mini.rb",
43
+ "lib/hackboxen/tasks/scaffold.rb",
44
+ "lib/hackboxen/template.rb",
45
+ "lib/hackboxen/template/Rakefile.erb",
46
+ "lib/hackboxen/template/config.yaml.erb",
47
+ "lib/hackboxen/template/endpoint.rb.erb",
48
+ "lib/hackboxen/template/icss.yaml.erb",
49
+ "lib/hackboxen/template/main.erb",
50
+ "lib/hackboxen/utils.rb",
51
+ "lib/hackboxen/utils/README_ConfigValidator.textile",
52
+ "lib/hackboxen/utils/config_validator.rb",
53
+ "lib/hackboxen/utils/logging.rb",
54
+ "lib/hackboxen/utils/paths.rb",
55
+ "spec/install_spec.rb"
56
+ ]
57
+ s.homepage = %q{http://github.com/infochimps/hackboxen}
58
+ s.licenses = ["MIT"]
59
+ s.require_paths = ["lib"]
60
+ s.rubygems_version = %q{1.3.7}
61
+ s.summary = %q{A simple framework to assist in standardizing the data-munging input/output process.}
62
+ s.test_files = [
63
+ "spec/install_spec.rb"
64
+ ]
65
+
66
+ if s.respond_to? :specification_version then
67
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
68
+ s.specification_version = 3
69
+
70
+ if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
71
+ s.add_runtime_dependency(%q<swineherd>, [">= 0.0.4"])
72
+ s.add_runtime_dependency(%q<configliere>, ["= 0.4.6"])
73
+ s.add_runtime_dependency(%q<rake>, ["= 0.8.7"])
74
+ s.add_development_dependency(%q<shoulda>, [">= 0"])
75
+ s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
76
+ s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
77
+ s.add_development_dependency(%q<rcov>, [">= 0"])
78
+ else
79
+ s.add_dependency(%q<swineherd>, [">= 0.0.4"])
80
+ s.add_dependency(%q<configliere>, ["= 0.4.6"])
81
+ s.add_dependency(%q<rake>, ["= 0.8.7"])
82
+ s.add_dependency(%q<shoulda>, [">= 0"])
83
+ s.add_dependency(%q<bundler>, ["~> 1.0.0"])
84
+ s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
85
+ s.add_dependency(%q<rcov>, [">= 0"])
86
+ end
87
+ else
88
+ s.add_dependency(%q<swineherd>, [">= 0.0.4"])
89
+ s.add_dependency(%q<configliere>, ["= 0.4.6"])
90
+ s.add_dependency(%q<rake>, ["= 0.8.7"])
91
+ s.add_dependency(%q<shoulda>, [">= 0"])
92
+ s.add_dependency(%q<bundler>, ["~> 1.0.0"])
93
+ s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
94
+ s.add_dependency(%q<rcov>, [">= 0"])
95
+ end
96
+ end
97
+
@@ -0,0 +1,19 @@
1
+ source :rubygems
2
+
3
+ gem 'rake'
4
+ gem 'json'
5
+ gem 'extlib'
6
+ gem 'nokogiri'
7
+ gem 'ken'
8
+ gem 'mechanize'
9
+ gem 'right_aws'
10
+ gem 'rgeo'
11
+ gem 'configliere'
12
+ gem 'activemodel'
13
+ gem 'yajl-ruby'
14
+ gem 'sqlite3-ruby'
15
+ gem 'swineherd'
16
+ gem 'gorillib'
17
+ gem 'icss'
18
+ gem 'gorillib'
19
+
@@ -0,0 +1,20 @@
1
+ source :rubygems
2
+
3
+ gem 'rake'
4
+ gem 'json'
5
+ gem 'extlib'
6
+ gem 'nokogiri'
7
+ gem 'ken'
8
+ gem 'shapelib'
9
+ gem 'mechanize'
10
+ gem 'right_aws'
11
+ gem 'rgeo'
12
+ gem 'configliere'
13
+ gem 'activemodel'
14
+ gem 'yajl-ruby'
15
+ gem 'sqlite3-ruby'
16
+ gem 'swineherd'
17
+ gem 'gorillib'
18
+ gem 'icss'
19
+ gem 'fastercsv'
20
+
@@ -0,0 +1,18 @@
1
+ source :rubygems
2
+
3
+ gem 'rake'
4
+ gem 'json'
5
+ gem 'extlib'
6
+ gem 'nokogiri'
7
+ gem 'ken'
8
+ gem 'shapelib'
9
+ gem 'mechanize'
10
+ gem 'right_aws'
11
+ gem 'rgeo'
12
+ gem 'configliere'
13
+ gem 'activemodel'
14
+ gem 'yajl-ruby'
15
+ gem 'sqlite3-ruby'
16
+ gem 'swineherd'
17
+ gem 'gorillib'
18
+ gem 'icss'
@@ -0,0 +1,17 @@
1
+ INCLUDING_FILE = caller[2].gsub(/:.*$/, '')
2
+
3
+ require 'rake'
4
+ require 'swineherd'
5
+ require 'configliere'
6
+ require 'json'
7
+ require 'hackboxen/utils'
8
+ require 'hackboxen/tasks'
9
+
10
+ machine_cfg = "/etc/hackbox/hackbox.yaml"
11
+ user_cfg = File.join(ENV['HOME'], '.hackbox/hackbox.yaml')
12
+ hackbox_cfg = File.join(HackBoxen.find_root_dir, 'config/config.yaml')
13
+
14
+ WorkingConfig.read machine_cfg if File.exists? machine_cfg
15
+ WorkingConfig.read user_cfg if File.exists? user_cfg
16
+ WorkingConfig.read hackbox_cfg if File.exists? hackbox_cfg
17
+
@@ -0,0 +1,6 @@
1
+ require 'hackboxen/tasks/init'
2
+ require 'hackboxen/tasks/icss'
3
+ require 'hackboxen/tasks/mini'
4
+ require 'hackboxen/tasks/endpoint'
5
+
6
+ include HackBoxen::Paths
@@ -0,0 +1,16 @@
1
+ module HackBoxen
2
+ namespace :hb do
3
+
4
+ desc "Copy an endpoint.rb file to the output code directory if the endpoint exists"
5
+ task :endpoint => ['hb:create_required_paths'] do
6
+ endpoint = "#{WorkingConfig[:protocol]}_endpoint.rb"
7
+ srcfile = File.join(path_to(:hb_engine), endpoint)
8
+ if File.exists? srcfile
9
+ HackBoxen.current_fs.mkpath path_to(:code_dir)
10
+ destfile = File.join(path_to(:code_dir), endpoint)
11
+ HackBoxen.current_fs.cp(srcfile, destfile)
12
+ end
13
+ end
14
+
15
+ end
16
+ end
@@ -0,0 +1,15 @@
1
+ module HackBoxen
2
+ namespace :hb do
3
+
4
+ desc "Copy the Icss file to the output directory in the filesystem specified in the WorkingConfig"
5
+ task :icss => ['hb:create_required_paths'] do
6
+ icss_yaml = File.join(path_to(:hb_config), "#{WorkingConfig[:protocol]}.icss.yaml")
7
+ if File.exists? icss_yaml
8
+ icss = YAML.load(File.read icss_yaml)
9
+ icss_json = File.join(path_to(:data_dir), "#{WorkingConfig[:protocol]}.icss.json")
10
+ HackBoxen.current_fs.open(File.join(icss_json), 'w') { |f| f.puts icss.to_json }
11
+ end
12
+ end
13
+
14
+ end
15
+ end
@@ -0,0 +1,36 @@
1
+ module HackBoxen
2
+ namespace :hb do
3
+
4
+ desc "Ensure all required config options are contained in the WorkingConfig"
5
+ task :validate_working_config do
6
+ WorkingConfig.resolve!
7
+ HackBoxen.verify_dependencies
8
+ # failures = HackBoxen::ConfigValidator.failed_requirements
9
+ # if failures.size > 0
10
+ # raise "Hackbox environment fails to meet requirements:\n-- " + failures.join("\n-- ") + "\n"
11
+ # end
12
+ end
13
+
14
+ desc "Create the required output directories using the filesystem specified by the WorkingConfig"
15
+ task :create_required_paths => [:validate_working_config] do
16
+ output_dirs.each { |dir| HackBoxen.current_fs.mkpath(dir) unless HackBoxen.current_fs.exists? dir }
17
+ end
18
+
19
+ desc "Always save the WorkingConfig out to a file when running a hackbox"
20
+ task :create_working_config => [:create_required_paths] do
21
+ working_config = File.join(path_to(:env_dir), 'working_config.json')
22
+ HackBoxen.current_fs.open(working_config, 'w') { |f| f.write WorkingConfig.to_hash.to_json }
23
+ end
24
+
25
+ desc "Execute the main file inside of the current hackbox directory"
26
+ task :init => [:create_working_config] do
27
+ main_file = File.join(path_to(:hb_engine), 'main')
28
+ sh "#{main_file} #{path_to(:hb_dataroot)} #{path_to(:data_dir)}" do |ok,res|
29
+ if !ok
30
+ puts "Processing script failed with #{res}"
31
+ end
32
+ end
33
+ end
34
+
35
+ end
36
+ end
@@ -0,0 +1,46 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+ require 'configliere'
4
+ require 'ohai'
5
+
6
+ Settings.use :commandline
7
+ Settings.define :dataroot, :default => "/data/hb", :description => "Global directory for hackbox output"
8
+ Settings.define :coderoot, :default => "#{ENV['HOME']}/hb", :description => "Global directory for hackbox code"
9
+ Settings.resolve!
10
+
11
+ config_dir = File.join(ENV["HOME"], '.hackbox')
12
+ config = File.join(config_dir, 'hackbox.yaml')
13
+
14
+ directory config_dir
15
+
16
+ file config => [config_dir] do
17
+
18
+ # Create config hash and setup defaults
19
+ config_hash = {}
20
+ config_hash['dataroot'] = Settings[:dataroot]
21
+ config_hash['coderoot'] = Settings[:coderoot]
22
+
23
+ # Configure S3 setup if desired
24
+ s3_setup = { 'access_key' => '', 'secret_key' => '' }
25
+ print "Configure S3 Filesystem information? [Y]n "
26
+ unless STDIN.gets.downcase.chomp == 'n'
27
+ print "Access Key: "
28
+ s3_setup['access_key'] = STDIN.gets.chomp
29
+ print "Secret Key: "
30
+ s3_setup['secret_key'] = STDIN.gets.chomp
31
+ end
32
+ config_hash['s3_filesystem'] = s3_setup
33
+
34
+ # Add system requirements
35
+ sys = Ohai::System.new
36
+ sys.all_plugins
37
+ config_hash['requires'] = {
38
+ 'machine' => sys[:kernel][:machine],
39
+ 'os' => sys[:os]
40
+ }
41
+
42
+ File.open(config, 'wb') { |f| f.puts config_hash.to_yaml }
43
+ end
44
+
45
+ desc "Create the install-level config file, ~/.hackbox/hackbox.yaml"
46
+ task :install => [config]
@@ -0,0 +1,47 @@
1
+ require 'configliere'
2
+ require 'swineherd'
3
+
4
+ namespace :hb do
5
+
6
+ Opts = Configliere::Param.new.use :commandline
7
+ Opts.define :files, :type => Array
8
+ Opts.resolve!
9
+
10
+ desc "Stores a mini data set from the local filesystem into s3 for running a hackbox in mini mode"
11
+ task :create_mini do
12
+ Opts[:files].each do |input|
13
+ output = File.directory?(input) ? expected_mini_data + '/' : output = File.join(expected_mini_data, File.basename(input))
14
+ sh "s3cmd put -r #{input} #{output}"
15
+ end
16
+ end
17
+
18
+ desc "Runs a hackbox, skipping :get_data and instead pulling sample data from the previously stored mini bucket"
19
+ task :run_mini do
20
+ WorkingConfig[:dataroot] += "/mini"
21
+ dest = File.join(WorkingConfig[:dataroot], hackbox_name)
22
+ raise "Expected mini data has not yet been created" unless s3fs.exists? expected_mini_data.gsub(/s3:\/\//,'')
23
+ Rake::Task[:get_data].clear_actions
24
+ Rake::Task['hb:create_working_config'].invoke
25
+ Rake::Task['hb:icss'].invoke if Rake::Task[:default].prerequisites.include? 'hb:icss'
26
+ if WorkingConfig[:filesystem_scheme] == 'hdfs'
27
+ sh "hadoop fs -cp #{expected_mini_data.gsub(/s3:/, 's3n:')} #{dest}"
28
+ elsif WorkingConfig[:filesystem_scheme] == 'file'
29
+ sh "s3cmd sync #{expected_mini_data} #{dest}"
30
+ end
31
+ Rake::Task['hb:init'].invoke
32
+ end
33
+
34
+ end
35
+
36
+ def expected_mini_data
37
+ raise "You don't have a s3_filesystem: mini_bucket: specified in your config" unless WorkingConfig['s3_filesystem.mini_bucket']
38
+ File.join(WorkingConfig['s3_filesystem.mini_bucket'], hackbox_name, 'ripd')
39
+ end
40
+
41
+ def hackbox_name
42
+ File.join(WorkingConfig[:namespace].gsub(/\./, '/'), WorkingConfig[:protocol])
43
+ end
44
+
45
+ def s3fs
46
+ @s3fs ||= Swineherd::FileSystem.get(:s3, WorkingConfig['s3_filesystem.access_key'], WorkingConfig['s3_filesystem.secret_key'])
47
+ end