hackboxen 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.textile +44 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +34 -0
- data/LICENSE.txt +20 -0
- data/README.textile +203 -0
- data/Rakefile +49 -0
- data/VERSION +1 -0
- data/bin/describe.rb +101 -0
- data/bin/hb-install +5 -0
- data/bin/hb-runner +93 -0
- data/bin/hb-scaffold +6 -0
- data/hackboxen.gemspec +97 -0
- data/lib/gemfiles/Gemfile.jruby-1.6.2.default +19 -0
- data/lib/gemfiles/Gemfile.ruby-1.8.7.default +20 -0
- data/lib/gemfiles/Gemfile.ruby-1.9.2.default +18 -0
- data/lib/hackboxen.rb +17 -0
- data/lib/hackboxen/tasks.rb +6 -0
- data/lib/hackboxen/tasks/endpoint.rb +16 -0
- data/lib/hackboxen/tasks/icss.rb +15 -0
- data/lib/hackboxen/tasks/init.rb +36 -0
- data/lib/hackboxen/tasks/install.rb +46 -0
- data/lib/hackboxen/tasks/mini.rb +47 -0
- data/lib/hackboxen/tasks/scaffold.rb +71 -0
- data/lib/hackboxen/template.rb +36 -0
- data/lib/hackboxen/template/Rakefile.erb +31 -0
- data/lib/hackboxen/template/config.yaml.erb +10 -0
- data/lib/hackboxen/template/endpoint.rb.erb +39 -0
- data/lib/hackboxen/template/icss.yaml.erb +125 -0
- data/lib/hackboxen/template/main.erb +31 -0
- data/lib/hackboxen/utils.rb +49 -0
- data/lib/hackboxen/utils/README_ConfigValidator.textile +63 -0
- data/lib/hackboxen/utils/config_validator.rb +41 -0
- data/lib/hackboxen/utils/logging.rb +39 -0
- data/lib/hackboxen/utils/paths.rb +66 -0
- data/spec/install_spec.rb +36 -0
- metadata +213 -0
data/bin/hb-install
ADDED
data/bin/hb-runner
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Figure out a bunch of paths
|
4
|
+
RUNNER_SCRIPT=`readlink -f $0`
|
5
|
+
HACKBOXEN_LIB_RELATIVE=`dirname ${RUNNER_SCRIPT}`/..
|
6
|
+
HACKBOXEN_LIB=`readlink -f ${HACKBOXEN_LIB_RELATIVE}`
|
7
|
+
|
8
|
+
# Set some defaults
|
9
|
+
DEFAULT_RUBY_VERSION=ruby-1.8.7
|
10
|
+
export TERM=dumb
|
11
|
+
|
12
|
+
# Ensure hackbox directory is given.
|
13
|
+
HACKBOX_DIR_RELATIVE="$1";
|
14
|
+
if [ -z "$HACKBOX_DIR_RELATIVE" ]; then
|
15
|
+
echo "Must specify the path to a hackbox to run";
|
16
|
+
exit 1;
|
17
|
+
fi
|
18
|
+
HACKBOX_DIR=`readlink -f ${HACKBOX_DIR_RELATIVE}`
|
19
|
+
|
20
|
+
# Ensure hackbox directory exists.
|
21
|
+
if [ ! -d "$HACKBOX_DIR" ]; then
|
22
|
+
echo "Hackbox directory $HACKBOX_DIR does not exist";
|
23
|
+
exit 1;
|
24
|
+
fi
|
25
|
+
|
26
|
+
# Ensure config file exists
|
27
|
+
HACKBOX_CONFIG="${HACKBOX_DIR}/config/config.yaml"
|
28
|
+
if [ ! -e "$HACKBOX_CONFIG" ]; then
|
29
|
+
echo "Hackbox directory ${HACKBOX_DIR} does not contain a config/config.yaml file";
|
30
|
+
exit 1;
|
31
|
+
fi
|
32
|
+
|
33
|
+
# Find Ruby version
|
34
|
+
REQUIRED_RUBY_VERSION=`ruby -r yaml -e "puts YAML.load_file('${HACKBOX_CONFIG}')['requires']['language']['ruby'].to_s.strip rescue nil"`
|
35
|
+
if [ -z "$REQUIRED_RUBY_VERSION" ]; then
|
36
|
+
REQUIRED_RUBY_VERSION=$DEFAULT_RUBY_VERSION;
|
37
|
+
fi
|
38
|
+
|
39
|
+
# Source RVM and use chosen RVM Ruby version
|
40
|
+
if [ -s /usr/local/rvm/scripts/rvm ] || [ -s ~/.rvm/bin/rvm ] ; then
|
41
|
+
[[ -s "/usr/local/rvm/scripts/rvm" ]] && . "/usr/local/rvm/scripts/rvm"
|
42
|
+
[[ -s "$HOME/.rvm/scripts/rvm" ]] && . "$HOME/.rvm/scripts/rvm"
|
43
|
+
# RVM defines RUBY_VERSION when it its invoked
|
44
|
+
if rvm use $REQUIRED_RUBY_VERSION; then
|
45
|
+
echo "Now running ${RUBY_VERSION}";
|
46
|
+
else
|
47
|
+
echo "Could not run ${RUBY_VERSION}";
|
48
|
+
exit 1;
|
49
|
+
fi
|
50
|
+
fi
|
51
|
+
|
52
|
+
# Symlink to Hackboxen Gemfile if necessary
|
53
|
+
HACKBOX_GEMFILE="${HACKBOX_DIR}/Gemfile"
|
54
|
+
DEFAULT_GEMFILE="${HACKBOXEN_LIB}/lib/gemfiles/Gemfile.${REQUIRED_RUBY_VERSION}.default"
|
55
|
+
if [ ! -e "$HACKBOX_GEMFILE" ]; then
|
56
|
+
ln -s $DEFAULT_GEMFILE $HACKBOX_GEMFILE
|
57
|
+
fi
|
58
|
+
|
59
|
+
# Export the correct Gemfile for bundler to find once Ruby starts
|
60
|
+
export BUNDLE_GEMFILE="$HACKBOX_GEMFILE"
|
61
|
+
|
62
|
+
# Change to Hackbox Dir
|
63
|
+
cd $HACKBOX_DIR
|
64
|
+
|
65
|
+
# Remove Gemfile.lock if it exists
|
66
|
+
# if [ -e Gemfile.lock ]; then
|
67
|
+
# rm Gemfile.lock;
|
68
|
+
# fi
|
69
|
+
|
70
|
+
# Something is setting RUBYOPT causing havoc w/Bundler.
|
71
|
+
unset RUBYOPT;
|
72
|
+
|
73
|
+
# Bundle install all gems into the vendor directory inside this
|
74
|
+
# hackbox.
|
75
|
+
if bundle install --path vendor; then
|
76
|
+
echo "Bundle installed";
|
77
|
+
else
|
78
|
+
echo "Could not bundle install";
|
79
|
+
exit 1;
|
80
|
+
fi
|
81
|
+
|
82
|
+
# Make sure hackboxen/lib is on RUBYLIB
|
83
|
+
export RUBYLIB=${HACKBOXEN_LIB}/lib:${RUBYLIB}
|
84
|
+
|
85
|
+
# Run hackbox and add a sentinel at the end
|
86
|
+
if bundle exec rake --trace; then
|
87
|
+
echo "<-- SUCCESSFULLY RAN HACKBOX -->";
|
88
|
+
else
|
89
|
+
echo "<-- FAILED TO RUN HACKBOX -->";
|
90
|
+
exit 1;
|
91
|
+
fi
|
92
|
+
|
93
|
+
exit 0;
|
data/bin/hb-scaffold
ADDED
data/hackboxen.gemspec
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{hackboxen}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["kornypoet", "Ganglion", "bollacker"]
|
12
|
+
s.date = %q{2011-06-30}
|
13
|
+
s.description = %q{A simple framework to assist in standardizing the data-munging input/output process.}
|
14
|
+
s.email = %q{travis@infochimps.com}
|
15
|
+
s.executables = ["hb-install", "hb-scaffold", "hb-runner"]
|
16
|
+
s.extra_rdoc_files = [
|
17
|
+
"LICENSE.txt",
|
18
|
+
"README.textile"
|
19
|
+
]
|
20
|
+
s.files = [
|
21
|
+
"CHANGELOG.textile",
|
22
|
+
"Gemfile",
|
23
|
+
"Gemfile.lock",
|
24
|
+
"LICENSE.txt",
|
25
|
+
"README.textile",
|
26
|
+
"Rakefile",
|
27
|
+
"VERSION",
|
28
|
+
"bin/describe.rb",
|
29
|
+
"bin/hb-install",
|
30
|
+
"bin/hb-runner",
|
31
|
+
"bin/hb-scaffold",
|
32
|
+
"hackboxen.gemspec",
|
33
|
+
"lib/gemfiles/Gemfile.jruby-1.6.2.default",
|
34
|
+
"lib/gemfiles/Gemfile.ruby-1.8.7.default",
|
35
|
+
"lib/gemfiles/Gemfile.ruby-1.9.2.default",
|
36
|
+
"lib/hackboxen.rb",
|
37
|
+
"lib/hackboxen/tasks.rb",
|
38
|
+
"lib/hackboxen/tasks/endpoint.rb",
|
39
|
+
"lib/hackboxen/tasks/icss.rb",
|
40
|
+
"lib/hackboxen/tasks/init.rb",
|
41
|
+
"lib/hackboxen/tasks/install.rb",
|
42
|
+
"lib/hackboxen/tasks/mini.rb",
|
43
|
+
"lib/hackboxen/tasks/scaffold.rb",
|
44
|
+
"lib/hackboxen/template.rb",
|
45
|
+
"lib/hackboxen/template/Rakefile.erb",
|
46
|
+
"lib/hackboxen/template/config.yaml.erb",
|
47
|
+
"lib/hackboxen/template/endpoint.rb.erb",
|
48
|
+
"lib/hackboxen/template/icss.yaml.erb",
|
49
|
+
"lib/hackboxen/template/main.erb",
|
50
|
+
"lib/hackboxen/utils.rb",
|
51
|
+
"lib/hackboxen/utils/README_ConfigValidator.textile",
|
52
|
+
"lib/hackboxen/utils/config_validator.rb",
|
53
|
+
"lib/hackboxen/utils/logging.rb",
|
54
|
+
"lib/hackboxen/utils/paths.rb",
|
55
|
+
"spec/install_spec.rb"
|
56
|
+
]
|
57
|
+
s.homepage = %q{http://github.com/infochimps/hackboxen}
|
58
|
+
s.licenses = ["MIT"]
|
59
|
+
s.require_paths = ["lib"]
|
60
|
+
s.rubygems_version = %q{1.3.7}
|
61
|
+
s.summary = %q{A simple framework to assist in standardizing the data-munging input/output process.}
|
62
|
+
s.test_files = [
|
63
|
+
"spec/install_spec.rb"
|
64
|
+
]
|
65
|
+
|
66
|
+
if s.respond_to? :specification_version then
|
67
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
68
|
+
s.specification_version = 3
|
69
|
+
|
70
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
71
|
+
s.add_runtime_dependency(%q<swineherd>, [">= 0.0.4"])
|
72
|
+
s.add_runtime_dependency(%q<configliere>, ["= 0.4.6"])
|
73
|
+
s.add_runtime_dependency(%q<rake>, ["= 0.8.7"])
|
74
|
+
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
75
|
+
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
76
|
+
s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
|
77
|
+
s.add_development_dependency(%q<rcov>, [">= 0"])
|
78
|
+
else
|
79
|
+
s.add_dependency(%q<swineherd>, [">= 0.0.4"])
|
80
|
+
s.add_dependency(%q<configliere>, ["= 0.4.6"])
|
81
|
+
s.add_dependency(%q<rake>, ["= 0.8.7"])
|
82
|
+
s.add_dependency(%q<shoulda>, [">= 0"])
|
83
|
+
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
84
|
+
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
85
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
86
|
+
end
|
87
|
+
else
|
88
|
+
s.add_dependency(%q<swineherd>, [">= 0.0.4"])
|
89
|
+
s.add_dependency(%q<configliere>, ["= 0.4.6"])
|
90
|
+
s.add_dependency(%q<rake>, ["= 0.8.7"])
|
91
|
+
s.add_dependency(%q<shoulda>, [">= 0"])
|
92
|
+
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
93
|
+
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
94
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
source :rubygems
|
2
|
+
|
3
|
+
gem 'rake'
|
4
|
+
gem 'json'
|
5
|
+
gem 'extlib'
|
6
|
+
gem 'nokogiri'
|
7
|
+
gem 'ken'
|
8
|
+
gem 'mechanize'
|
9
|
+
gem 'right_aws'
|
10
|
+
gem 'rgeo'
|
11
|
+
gem 'configliere'
|
12
|
+
gem 'activemodel'
|
13
|
+
gem 'yajl-ruby'
|
14
|
+
gem 'sqlite3-ruby'
|
15
|
+
gem 'swineherd'
|
16
|
+
gem 'gorillib'
|
17
|
+
gem 'icss'
|
18
|
+
gem 'gorillib'
|
19
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
source :rubygems
|
2
|
+
|
3
|
+
gem 'rake'
|
4
|
+
gem 'json'
|
5
|
+
gem 'extlib'
|
6
|
+
gem 'nokogiri'
|
7
|
+
gem 'ken'
|
8
|
+
gem 'shapelib'
|
9
|
+
gem 'mechanize'
|
10
|
+
gem 'right_aws'
|
11
|
+
gem 'rgeo'
|
12
|
+
gem 'configliere'
|
13
|
+
gem 'activemodel'
|
14
|
+
gem 'yajl-ruby'
|
15
|
+
gem 'sqlite3-ruby'
|
16
|
+
gem 'swineherd'
|
17
|
+
gem 'gorillib'
|
18
|
+
gem 'icss'
|
19
|
+
gem 'fastercsv'
|
20
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
source :rubygems
|
2
|
+
|
3
|
+
gem 'rake'
|
4
|
+
gem 'json'
|
5
|
+
gem 'extlib'
|
6
|
+
gem 'nokogiri'
|
7
|
+
gem 'ken'
|
8
|
+
gem 'shapelib'
|
9
|
+
gem 'mechanize'
|
10
|
+
gem 'right_aws'
|
11
|
+
gem 'rgeo'
|
12
|
+
gem 'configliere'
|
13
|
+
gem 'activemodel'
|
14
|
+
gem 'yajl-ruby'
|
15
|
+
gem 'sqlite3-ruby'
|
16
|
+
gem 'swineherd'
|
17
|
+
gem 'gorillib'
|
18
|
+
gem 'icss'
|
data/lib/hackboxen.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
INCLUDING_FILE = caller[2].gsub(/:.*$/, '')
|
2
|
+
|
3
|
+
require 'rake'
|
4
|
+
require 'swineherd'
|
5
|
+
require 'configliere'
|
6
|
+
require 'json'
|
7
|
+
require 'hackboxen/utils'
|
8
|
+
require 'hackboxen/tasks'
|
9
|
+
|
10
|
+
machine_cfg = "/etc/hackbox/hackbox.yaml"
|
11
|
+
user_cfg = File.join(ENV['HOME'], '.hackbox/hackbox.yaml')
|
12
|
+
hackbox_cfg = File.join(HackBoxen.find_root_dir, 'config/config.yaml')
|
13
|
+
|
14
|
+
WorkingConfig.read machine_cfg if File.exists? machine_cfg
|
15
|
+
WorkingConfig.read user_cfg if File.exists? user_cfg
|
16
|
+
WorkingConfig.read hackbox_cfg if File.exists? hackbox_cfg
|
17
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module HackBoxen
|
2
|
+
namespace :hb do
|
3
|
+
|
4
|
+
desc "Copy an endpoint.rb file to the output code directory if the endpoint exists"
|
5
|
+
task :endpoint => ['hb:create_required_paths'] do
|
6
|
+
endpoint = "#{WorkingConfig[:protocol]}_endpoint.rb"
|
7
|
+
srcfile = File.join(path_to(:hb_engine), endpoint)
|
8
|
+
if File.exists? srcfile
|
9
|
+
HackBoxen.current_fs.mkpath path_to(:code_dir)
|
10
|
+
destfile = File.join(path_to(:code_dir), endpoint)
|
11
|
+
HackBoxen.current_fs.cp(srcfile, destfile)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module HackBoxen
|
2
|
+
namespace :hb do
|
3
|
+
|
4
|
+
desc "Copy the Icss file to the output directory in the filesystem specified in the WorkingConfig"
|
5
|
+
task :icss => ['hb:create_required_paths'] do
|
6
|
+
icss_yaml = File.join(path_to(:hb_config), "#{WorkingConfig[:protocol]}.icss.yaml")
|
7
|
+
if File.exists? icss_yaml
|
8
|
+
icss = YAML.load(File.read icss_yaml)
|
9
|
+
icss_json = File.join(path_to(:data_dir), "#{WorkingConfig[:protocol]}.icss.json")
|
10
|
+
HackBoxen.current_fs.open(File.join(icss_json), 'w') { |f| f.puts icss.to_json }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module HackBoxen
|
2
|
+
namespace :hb do
|
3
|
+
|
4
|
+
desc "Ensure all required config options are contained in the WorkingConfig"
|
5
|
+
task :validate_working_config do
|
6
|
+
WorkingConfig.resolve!
|
7
|
+
HackBoxen.verify_dependencies
|
8
|
+
# failures = HackBoxen::ConfigValidator.failed_requirements
|
9
|
+
# if failures.size > 0
|
10
|
+
# raise "Hackbox environment fails to meet requirements:\n-- " + failures.join("\n-- ") + "\n"
|
11
|
+
# end
|
12
|
+
end
|
13
|
+
|
14
|
+
desc "Create the required output directories using the filesystem specified by the WorkingConfig"
|
15
|
+
task :create_required_paths => [:validate_working_config] do
|
16
|
+
output_dirs.each { |dir| HackBoxen.current_fs.mkpath(dir) unless HackBoxen.current_fs.exists? dir }
|
17
|
+
end
|
18
|
+
|
19
|
+
desc "Always save the WorkingConfig out to a file when running a hackbox"
|
20
|
+
task :create_working_config => [:create_required_paths] do
|
21
|
+
working_config = File.join(path_to(:env_dir), 'working_config.json')
|
22
|
+
HackBoxen.current_fs.open(working_config, 'w') { |f| f.write WorkingConfig.to_hash.to_json }
|
23
|
+
end
|
24
|
+
|
25
|
+
desc "Execute the main file inside of the current hackbox directory"
|
26
|
+
task :init => [:create_working_config] do
|
27
|
+
main_file = File.join(path_to(:hb_engine), 'main')
|
28
|
+
sh "#{main_file} #{path_to(:hb_dataroot)} #{path_to(:data_dir)}" do |ok,res|
|
29
|
+
if !ok
|
30
|
+
puts "Processing script failed with #{res}"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'configliere'
|
4
|
+
require 'ohai'
|
5
|
+
|
6
|
+
Settings.use :commandline
|
7
|
+
Settings.define :dataroot, :default => "/data/hb", :description => "Global directory for hackbox output"
|
8
|
+
Settings.define :coderoot, :default => "#{ENV['HOME']}/hb", :description => "Global directory for hackbox code"
|
9
|
+
Settings.resolve!
|
10
|
+
|
11
|
+
config_dir = File.join(ENV["HOME"], '.hackbox')
|
12
|
+
config = File.join(config_dir, 'hackbox.yaml')
|
13
|
+
|
14
|
+
directory config_dir
|
15
|
+
|
16
|
+
file config => [config_dir] do
|
17
|
+
|
18
|
+
# Create config hash and setup defaults
|
19
|
+
config_hash = {}
|
20
|
+
config_hash['dataroot'] = Settings[:dataroot]
|
21
|
+
config_hash['coderoot'] = Settings[:coderoot]
|
22
|
+
|
23
|
+
# Configure S3 setup if desired
|
24
|
+
s3_setup = { 'access_key' => '', 'secret_key' => '' }
|
25
|
+
print "Configure S3 Filesystem information? [Y]n "
|
26
|
+
unless STDIN.gets.downcase.chomp == 'n'
|
27
|
+
print "Access Key: "
|
28
|
+
s3_setup['access_key'] = STDIN.gets.chomp
|
29
|
+
print "Secret Key: "
|
30
|
+
s3_setup['secret_key'] = STDIN.gets.chomp
|
31
|
+
end
|
32
|
+
config_hash['s3_filesystem'] = s3_setup
|
33
|
+
|
34
|
+
# Add system requirements
|
35
|
+
sys = Ohai::System.new
|
36
|
+
sys.all_plugins
|
37
|
+
config_hash['requires'] = {
|
38
|
+
'machine' => sys[:kernel][:machine],
|
39
|
+
'os' => sys[:os]
|
40
|
+
}
|
41
|
+
|
42
|
+
File.open(config, 'wb') { |f| f.puts config_hash.to_yaml }
|
43
|
+
end
|
44
|
+
|
45
|
+
desc "Create the install-level config file, ~/.hackbox/hackbox.yaml"
|
46
|
+
task :install => [config]
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'configliere'
|
2
|
+
require 'swineherd'
|
3
|
+
|
4
|
+
namespace :hb do
|
5
|
+
|
6
|
+
Opts = Configliere::Param.new.use :commandline
|
7
|
+
Opts.define :files, :type => Array
|
8
|
+
Opts.resolve!
|
9
|
+
|
10
|
+
desc "Stores a mini data set from the local filesystem into s3 for running a hackbox in mini mode"
|
11
|
+
task :create_mini do
|
12
|
+
Opts[:files].each do |input|
|
13
|
+
output = File.directory?(input) ? expected_mini_data + '/' : output = File.join(expected_mini_data, File.basename(input))
|
14
|
+
sh "s3cmd put -r #{input} #{output}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
desc "Runs a hackbox, skipping :get_data and instead pulling sample data from the previously stored mini bucket"
|
19
|
+
task :run_mini do
|
20
|
+
WorkingConfig[:dataroot] += "/mini"
|
21
|
+
dest = File.join(WorkingConfig[:dataroot], hackbox_name)
|
22
|
+
raise "Expected mini data has not yet been created" unless s3fs.exists? expected_mini_data.gsub(/s3:\/\//,'')
|
23
|
+
Rake::Task[:get_data].clear_actions
|
24
|
+
Rake::Task['hb:create_working_config'].invoke
|
25
|
+
Rake::Task['hb:icss'].invoke if Rake::Task[:default].prerequisites.include? 'hb:icss'
|
26
|
+
if WorkingConfig[:filesystem_scheme] == 'hdfs'
|
27
|
+
sh "hadoop fs -cp #{expected_mini_data.gsub(/s3:/, 's3n:')} #{dest}"
|
28
|
+
elsif WorkingConfig[:filesystem_scheme] == 'file'
|
29
|
+
sh "s3cmd sync #{expected_mini_data} #{dest}"
|
30
|
+
end
|
31
|
+
Rake::Task['hb:init'].invoke
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
def expected_mini_data
|
37
|
+
raise "You don't have a s3_filesystem: mini_bucket: specified in your config" unless WorkingConfig['s3_filesystem.mini_bucket']
|
38
|
+
File.join(WorkingConfig['s3_filesystem.mini_bucket'], hackbox_name, 'ripd')
|
39
|
+
end
|
40
|
+
|
41
|
+
def hackbox_name
|
42
|
+
File.join(WorkingConfig[:namespace].gsub(/\./, '/'), WorkingConfig[:protocol])
|
43
|
+
end
|
44
|
+
|
45
|
+
def s3fs
|
46
|
+
@s3fs ||= Swineherd::FileSystem.get(:s3, WorkingConfig['s3_filesystem.access_key'], WorkingConfig['s3_filesystem.secret_key'])
|
47
|
+
end
|