mobilize-base 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,35 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "mobilize-base/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "mobilize-base"
7
+ s.version = Mobilize::Base::VERSION
8
+ s.authors = ["Cassio Paes-Leme"]
9
+ s.email = ["cpaesleme@ngmoco.com"]
10
+ s.homepage = ""
11
+ s.summary = %q{Moves datasets and schedules data transfers using MongoDB, Resque and Google Docs}
12
+ s.description = %q{Manage your organization's workflows entirely through Google Docs and irb.
13
+ Mobilize schedules jobs, queues workers, sends failure notifications, and
14
+ integrates mobilize-hadoop, -http, -mysql, and -mongodb packages
15
+ to allow seamless transport of TSV and JSON data between any two endpoints. }
16
+
17
+ s.rubyforge_project = "mobilize-base"
18
+
19
+ s.files = `git ls-files`.split("\n")
20
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
21
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
22
+ s.require_paths = ["lib"]
23
+
24
+ s.add_runtime_dependency 'rake'
25
+ s.add_runtime_dependency 'bson','1.6.1'
26
+ s.add_runtime_dependency 'bson_ext','1.6.1'
27
+ s.add_runtime_dependency 'mongo', '1.6.1'
28
+ s.add_runtime_dependency "mongoid", "~>3.0.0"
29
+ s.add_runtime_dependency 'redis',"~>3.0.0"
30
+ s.add_runtime_dependency 'resque','1.21.0'
31
+ s.add_runtime_dependency 'google_drive','0.3.2'
32
+ s.add_runtime_dependency 'bluepill','0.0.60'
33
+ s.add_runtime_dependency 'popen4','0.1.2'
34
+ s.add_runtime_dependency 'actionmailer','3.1.1'
35
+ end
@@ -0,0 +1,125 @@
1
+ require 'test_helper'
2
+
3
+ describe "Mobilize" do
4
+
5
+ def before
6
+ puts 'before'
7
+
8
+ end
9
+
10
+ # enqueues 4 workers on Resque
11
+ it "runs integration test" do
12
+ email = Mobilize::Gdriver.owner_email
13
+
14
+ #kill all workers
15
+ Mobilize::Jobtracker.kill_workers
16
+
17
+ puts 'enqueue 4 workers on Resque, wait 20s'
18
+ Mobilize::Jobtracker.prep_workers
19
+ sleep 20
20
+ assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
21
+
22
+ #make sure old one is deleted
23
+ Mobilize::Requestor.find_or_create_by_email(email).delete
24
+
25
+ puts "create requestor 'mobilize'"
26
+ requestor = Mobilize::Requestor.find_or_create_by_email(email)
27
+ assert requestor.email == email
28
+
29
+ puts "delete old books and datasets"
30
+ # delete any old specbooks from previous test runs
31
+ jobspec_title = requestor.jobspec_title
32
+ books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
33
+ books.each{|book| book.delete}
34
+ #delete old datasets for this specbook
35
+ Mobilize::Dataset.all.select{|d| d.name.starts_with?(jobspec_title)}.each{|d| d.delete}
36
+
37
+ puts "enqueue jobtracker, wait 60s"
38
+ Mobilize::Jobtracker.start
39
+ sleep 60
40
+ puts "jobtracker status: #{Mobilize::Jobtracker.status}"
41
+ puts "status:#{Mobilize::Jobtracker.status}" #!= 'stopped'
42
+
43
+ puts "requestor created specbook?"
44
+ books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
45
+ assert books.length == 1
46
+
47
+ puts "Jobtracker created jobspec with 'jobs' sheet?"
48
+ jobs_sheets = Mobilize::Gsheeter.find_all_by_name("#{jobspec_title}/Jobs",email)
49
+ assert jobs_sheets.length == 1
50
+
51
+ puts "add test_source data"
52
+
53
+ test_source_rows = [
54
+ ["test_header","test_header2","test_header3"],
55
+ ["t1"]*3,
56
+ ["t2"]*3
57
+ ]
58
+
59
+ book = books.first
60
+ test_source_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_source",email)
61
+
62
+ test_source_tsv = test_source_rows.map{|r| r.join("\t")}.join("\n")
63
+ test_source_sheet.write(test_source_tsv)
64
+
65
+ puts "add row to jobs sheet, wait 100s"
66
+
67
+ #delete existing Jobs from the db
68
+ Mobilize::Job.each{|j| j.delete}
69
+
70
+ jobs_sheet = jobs_sheets.first
71
+
72
+ test_job_rows = [{"name" => "test",
73
+ "active" => "true",
74
+ "schedule" => "once",
75
+ "status" => "",
76
+ "last_error" => "",
77
+ "destination_url" => "",
78
+ "read_handler" => "gsheeter",
79
+ "write_handler" => "gsheeter",
80
+ "param_source" => "test_source",
81
+ "params" => "",
82
+ "destination" => "test_destination"},
83
+ #run after the first
84
+ {"name" => "test2",
85
+ "active" => "true",
86
+ "schedule" => "after test",
87
+ "status" => "",
88
+ "last_error" => "",
89
+ "destination_url" => "",
90
+ "read_handler" => "gsheeter",
91
+ "write_handler" => "gsheeter",
92
+ "param_source" => "test_source",
93
+ "params" => "",
94
+ "destination" => "test_destination2"}
95
+ ]
96
+
97
+ #update second row w details
98
+ test_job_rows.each_with_index do |r,r_i|
99
+ r.values.each_with_index do |v,v_i|
100
+ jobs_sheet[r_i+2,v_i+1] = v
101
+ end
102
+ end
103
+
104
+ jobs_sheet.save
105
+
106
+ puts "job row added, force enqueued requestor"
107
+ requestor.enqueue!
108
+ sleep 100
109
+
110
+ puts "jobtracker posted test source data to test destination, and checksum succeeded?"
111
+ test_destination_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_destination",email)
112
+
113
+ assert test_destination_sheet.to_tsv == test_source_sheet.to_tsv
114
+ end
115
+
116
+ after do
117
+ processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
118
+ pids = processes.map { |process| process.split(" ")[0] }
119
+ puts "Killing test redis server..."
120
+ pids.each { |pid| Process.kill("TERM", pid.to_i) }
121
+ puts "removing redis db dump file"
122
+ sleep 5
123
+ `rm -f #{$dir}/dump.rdb #{$dir}/dump-cluster.rdb`
124
+ end
125
+ end