mobilize-base 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +9 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +20 -0
- data/README.md +509 -0
- data/Rakefile +34 -0
- data/lib/mobilize-base/extensions/array.rb +22 -0
- data/lib/mobilize-base/extensions/google_drive.rb +296 -0
- data/lib/mobilize-base/extensions/hash.rb +86 -0
- data/lib/mobilize-base/extensions/object.rb +6 -0
- data/lib/mobilize-base/extensions/resque.rb +180 -0
- data/lib/mobilize-base/extensions/string.rb +94 -0
- data/lib/mobilize-base/handlers/emailer.rb +24 -0
- data/lib/mobilize-base/handlers/gdriver.rb +309 -0
- data/lib/mobilize-base/handlers/mongoer.rb +32 -0
- data/lib/mobilize-base/jobtracker.rb +208 -0
- data/lib/mobilize-base/models/dataset.rb +70 -0
- data/lib/mobilize-base/models/job.rb +253 -0
- data/lib/mobilize-base/models/requestor.rb +223 -0
- data/lib/mobilize-base/tasks/mobilize-base.rake +2 -0
- data/lib/mobilize-base/tasks.rb +43 -0
- data/lib/mobilize-base/version.rb +5 -0
- data/lib/mobilize-base.rb +76 -0
- data/lib/samples/gdrive.yml +27 -0
- data/lib/samples/jobtracker.yml +24 -0
- data/lib/samples/mongoid.yml +21 -0
- data/lib/samples/resque.yml +12 -0
- data/mobilize-base.gemspec +35 -0
- data/test/mobilize_test.rb +125 -0
- data/test/redis-test.conf +540 -0
- data/test/test_helper.rb +23 -0
- metadata +260 -0
@@ -0,0 +1,35 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "mobilize-base/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "mobilize-base"
|
7
|
+
s.version = Mobilize::Base::VERSION
|
8
|
+
s.authors = ["Cassio Paes-Leme"]
|
9
|
+
s.email = ["cpaesleme@ngmoco.com"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{Moves datasets and schedules data transfers using MongoDB, Resque and Google Docs}
|
12
|
+
s.description = %q{Manage your organization's workflows entirely through Google Docs and irb.
|
13
|
+
Mobilize schedules jobs, queues workers, sends failure notifications, and
|
14
|
+
integrates mobilize-hadoop, -http, -mysql, and -mongodb packages
|
15
|
+
to allow seamless transport of TSV and JSON data between any two endpoints. }
|
16
|
+
|
17
|
+
s.rubyforge_project = "mobilize-base"
|
18
|
+
|
19
|
+
s.files = `git ls-files`.split("\n")
|
20
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
21
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
22
|
+
s.require_paths = ["lib"]
|
23
|
+
|
24
|
+
s.add_runtime_dependency 'rake'
|
25
|
+
s.add_runtime_dependency 'bson','1.6.1'
|
26
|
+
s.add_runtime_dependency 'bson_ext','1.6.1'
|
27
|
+
s.add_runtime_dependency 'mongo', '1.6.1'
|
28
|
+
s.add_runtime_dependency "mongoid", "~>3.0.0"
|
29
|
+
s.add_runtime_dependency 'redis',"~>3.0.0"
|
30
|
+
s.add_runtime_dependency 'resque','1.21.0'
|
31
|
+
s.add_runtime_dependency 'google_drive','0.3.2'
|
32
|
+
s.add_runtime_dependency 'bluepill','0.0.60'
|
33
|
+
s.add_runtime_dependency 'popen4','0.1.2'
|
34
|
+
s.add_runtime_dependency 'actionmailer','3.1.1'
|
35
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
describe "Mobilize" do
|
4
|
+
|
5
|
+
def before
|
6
|
+
puts 'before'
|
7
|
+
|
8
|
+
end
|
9
|
+
|
10
|
+
# enqueues 4 workers on Resque
|
11
|
+
it "runs integration test" do
|
12
|
+
email = Mobilize::Gdriver.owner_email
|
13
|
+
|
14
|
+
#kill all workers
|
15
|
+
Mobilize::Jobtracker.kill_workers
|
16
|
+
|
17
|
+
puts 'enqueue 4 workers on Resque, wait 20s'
|
18
|
+
Mobilize::Jobtracker.prep_workers
|
19
|
+
sleep 20
|
20
|
+
assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
|
21
|
+
|
22
|
+
#make sure old one is deleted
|
23
|
+
Mobilize::Requestor.find_or_create_by_email(email).delete
|
24
|
+
|
25
|
+
puts "create requestor 'mobilize'"
|
26
|
+
requestor = Mobilize::Requestor.find_or_create_by_email(email)
|
27
|
+
assert requestor.email == email
|
28
|
+
|
29
|
+
puts "delete old books and datasets"
|
30
|
+
# delete any old specbooks from previous test runs
|
31
|
+
jobspec_title = requestor.jobspec_title
|
32
|
+
books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
|
33
|
+
books.each{|book| book.delete}
|
34
|
+
#delete old datasets for this specbook
|
35
|
+
Mobilize::Dataset.all.select{|d| d.name.starts_with?(jobspec_title)}.each{|d| d.delete}
|
36
|
+
|
37
|
+
puts "enqueue jobtracker, wait 60s"
|
38
|
+
Mobilize::Jobtracker.start
|
39
|
+
sleep 60
|
40
|
+
puts "jobtracker status: #{Mobilize::Jobtracker.status}"
|
41
|
+
puts "status:#{Mobilize::Jobtracker.status}" #!= 'stopped'
|
42
|
+
|
43
|
+
puts "requestor created specbook?"
|
44
|
+
books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
|
45
|
+
assert books.length == 1
|
46
|
+
|
47
|
+
puts "Jobtracker created jobspec with 'jobs' sheet?"
|
48
|
+
jobs_sheets = Mobilize::Gsheeter.find_all_by_name("#{jobspec_title}/Jobs",email)
|
49
|
+
assert jobs_sheets.length == 1
|
50
|
+
|
51
|
+
puts "add test_source data"
|
52
|
+
|
53
|
+
test_source_rows = [
|
54
|
+
["test_header","test_header2","test_header3"],
|
55
|
+
["t1"]*3,
|
56
|
+
["t2"]*3
|
57
|
+
]
|
58
|
+
|
59
|
+
book = books.first
|
60
|
+
test_source_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_source",email)
|
61
|
+
|
62
|
+
test_source_tsv = test_source_rows.map{|r| r.join("\t")}.join("\n")
|
63
|
+
test_source_sheet.write(test_source_tsv)
|
64
|
+
|
65
|
+
puts "add row to jobs sheet, wait 100s"
|
66
|
+
|
67
|
+
#delete existing Jobs from the db
|
68
|
+
Mobilize::Job.each{|j| j.delete}
|
69
|
+
|
70
|
+
jobs_sheet = jobs_sheets.first
|
71
|
+
|
72
|
+
test_job_rows = [{"name" => "test",
|
73
|
+
"active" => "true",
|
74
|
+
"schedule" => "once",
|
75
|
+
"status" => "",
|
76
|
+
"last_error" => "",
|
77
|
+
"destination_url" => "",
|
78
|
+
"read_handler" => "gsheeter",
|
79
|
+
"write_handler" => "gsheeter",
|
80
|
+
"param_source" => "test_source",
|
81
|
+
"params" => "",
|
82
|
+
"destination" => "test_destination"},
|
83
|
+
#run after the first
|
84
|
+
{"name" => "test2",
|
85
|
+
"active" => "true",
|
86
|
+
"schedule" => "after test",
|
87
|
+
"status" => "",
|
88
|
+
"last_error" => "",
|
89
|
+
"destination_url" => "",
|
90
|
+
"read_handler" => "gsheeter",
|
91
|
+
"write_handler" => "gsheeter",
|
92
|
+
"param_source" => "test_source",
|
93
|
+
"params" => "",
|
94
|
+
"destination" => "test_destination2"}
|
95
|
+
]
|
96
|
+
|
97
|
+
#update second row w details
|
98
|
+
test_job_rows.each_with_index do |r,r_i|
|
99
|
+
r.values.each_with_index do |v,v_i|
|
100
|
+
jobs_sheet[r_i+2,v_i+1] = v
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
jobs_sheet.save
|
105
|
+
|
106
|
+
puts "job row added, force enqueued requestor"
|
107
|
+
requestor.enqueue!
|
108
|
+
sleep 100
|
109
|
+
|
110
|
+
puts "jobtracker posted test source data to test destination, and checksum succeeded?"
|
111
|
+
test_destination_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_destination",email)
|
112
|
+
|
113
|
+
assert test_destination_sheet.to_tsv == test_source_sheet.to_tsv
|
114
|
+
end
|
115
|
+
|
116
|
+
after do
|
117
|
+
processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
|
118
|
+
pids = processes.map { |process| process.split(" ")[0] }
|
119
|
+
puts "Killing test redis server..."
|
120
|
+
pids.each { |pid| Process.kill("TERM", pid.to_i) }
|
121
|
+
puts "removing redis db dump file"
|
122
|
+
sleep 5
|
123
|
+
`rm -f #{$dir}/dump.rdb #{$dir}/dump-cluster.rdb`
|
124
|
+
end
|
125
|
+
end
|