drudgery 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +80 -31
- data/lib/drudgery/extractors/csv_extractor.rb +8 -0
- data/lib/drudgery/job.rb +21 -33
- data/lib/drudgery/loaders/csv_loader.rb +8 -0
- data/lib/drudgery/version.rb +1 -1
- data/lib/drudgery.rb +15 -9
- data/spec/drudgery/extractors/active_record_extractor_spec.rb +39 -82
- data/spec/drudgery/extractors/csv_extractor_spec.rb +55 -87
- data/spec/drudgery/extractors/sqlite3_extractor_spec.rb +116 -171
- data/spec/drudgery/job_spec.rb +224 -321
- data/spec/drudgery/loaders/active_record_import_loader_spec.rb +30 -54
- data/spec/drudgery/loaders/active_record_loader_spec.rb +30 -60
- data/spec/drudgery/loaders/csv_loader_spec.rb +59 -70
- data/spec/drudgery/loaders/sqlite3_loader_spec.rb +37 -72
- data/spec/drudgery/manager_spec.rb +28 -24
- data/spec/drudgery/transformer_spec.rb +35 -41
- data/spec/drudgery_spec.rb +77 -48
- data/spec/spec_helper.rb +4 -3
- metadata +58 -43
- data/lib/drudgery/job_logger.rb +0 -21
- data/lib/drudgery/job_progress.rb +0 -11
- data/spec/drudgery/job_logger_spec.rb +0 -59
- data/spec/drudgery/job_progress_spec.rb +0 -19
@@ -1,204 +1,149 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 'sqlite3'
|
3
2
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
|
17
|
-
extractor.instance_variable_get('@db').must_equal db
|
18
|
-
extractor.instance_variable_get('@table').must_equal 'tablename'
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'initializes clauses hash' do
|
22
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(mock_db, 'tablename')
|
23
|
-
extractor.instance_variable_get('@clauses').must_equal({})
|
24
|
-
end
|
25
|
-
|
26
|
-
describe 'with in memory db' do
|
27
|
-
it 'sets name to sqlite3:memory.<table name>' do
|
28
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(mock_db, 'tablename')
|
29
|
-
extractor.name.must_equal 'sqlite3:memory.tablename'
|
3
|
+
module Drudgery
|
4
|
+
module Extractors
|
5
|
+
describe SQLite3Extractor do
|
6
|
+
before do
|
7
|
+
@db = SQLite3::Database.new(':memory:')
|
8
|
+
@db.execute('CREATE TABLE records (a INTEGER, b INTEGER)')
|
9
|
+
@db.execute('INSERT INTO records (a, b) VALUES (1, 2)');
|
10
|
+
@db.execute('INSERT INTO records (a, b) VALUES (3, 4)');
|
11
|
+
@db.execute('INSERT INTO records (a, b) VALUES (3, 6)');
|
12
|
+
|
13
|
+
@extractor = SQLite3Extractor.new(@db, 'records')
|
30
14
|
end
|
31
|
-
end
|
32
15
|
|
33
|
-
|
34
|
-
|
35
|
-
db = mock_db
|
36
|
-
db.expects(:database_list).returns([{ 'name' => 'main', 'file' => 'db/test.sqlite3.db' }])
|
37
|
-
|
38
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
|
39
|
-
extractor.name.must_equal 'sqlite3:test.tablename'
|
16
|
+
after do
|
17
|
+
@db.close
|
40
18
|
end
|
41
|
-
end
|
42
|
-
end
|
43
19
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
20
|
+
describe '#name' do
|
21
|
+
describe 'with file based db' do
|
22
|
+
it 'returns sqlite3:<main db name>.<table name>' do
|
23
|
+
db = SQLite3::Database.new('tmp/test.sqlite3')
|
48
24
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
end
|
54
|
-
end
|
25
|
+
@extractor = SQLite3Extractor.new(db, 'people')
|
26
|
+
@extractor.name.must_equal 'sqlite3:test.people'
|
27
|
+
end
|
28
|
+
end
|
55
29
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
30
|
+
describe 'with in memory db' do
|
31
|
+
it 'returns sqlite3:memory.<table name>' do
|
32
|
+
@extractor = SQLite3Extractor.new(@db, 'cities')
|
33
|
+
@extractor.name.must_equal 'sqlite3:memory.cities'
|
34
|
+
end
|
35
|
+
end
|
60
36
|
end
|
61
|
-
end
|
62
37
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
]
|
69
|
-
|
70
|
-
@extractor.joins(joins[0], joins[1])
|
71
|
-
@extractor.instance_variable_get('@clauses').must_equal({ :joins => joins })
|
38
|
+
describe '#select' do
|
39
|
+
it 'sets select clause with provided expressions' do
|
40
|
+
@extractor.select('id', "(first_name || ' ' || last_name) AS name", 'email')
|
41
|
+
@extractor.send(:sql).must_equal "SELECT id, (first_name || ' ' || last_name) AS name, email FROM records"
|
42
|
+
end
|
72
43
|
end
|
73
|
-
end
|
74
44
|
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
45
|
+
describe '#from' do
|
46
|
+
it 'sets from clause with provided expression' do
|
47
|
+
@extractor.from('records AS r')
|
48
|
+
@extractor.send(:sql).must_equal 'SELECT * FROM records AS r'
|
49
|
+
end
|
79
50
|
end
|
80
|
-
end
|
81
51
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
52
|
+
describe '#joins' do
|
53
|
+
it 'sets join clauses with provided clauses' do
|
54
|
+
joins = [
|
55
|
+
'JOIN table2 t2 ON t2.my_id = t1.id',
|
56
|
+
'LEFT OUTER JOIN table3 t3 ON t3.my_id = t2.id'
|
57
|
+
]
|
88
58
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
@extractor.instance_variable_get('@clauses').must_equal({ :having => 'count(*) > 1' })
|
59
|
+
@extractor.joins(joins[0], joins[1])
|
60
|
+
@extractor.send(:sql).must_equal 'SELECT * FROM records JOIN table2 t2 ON t2.my_id = t1.id LEFT OUTER JOIN table3 t3 ON t3.my_id = t2.id'
|
61
|
+
end
|
93
62
|
end
|
94
|
-
end
|
95
63
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
64
|
+
describe '#group' do
|
65
|
+
it 'sets group clause with provided expressions' do
|
66
|
+
@extractor.group('id', 'email')
|
67
|
+
@extractor.send(:sql).must_equal 'SELECT * FROM records GROUP BY id, email'
|
68
|
+
end
|
100
69
|
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
describe '#extract' do
|
105
|
-
it 'selects records from db using defined query' do
|
106
|
-
db = mock_db
|
107
|
-
db.expects(:execute).with('SELECT * FROM tablename')
|
108
|
-
db.expects(:execute).with('SELECT age, count(*) AS nr_ages FROM tablename t table2 t2 ON t2.my_id = t.id WHERE age > 10 GROUP BY age HAVING count(*) > 1 ORDER BY nr_ages')
|
109
|
-
|
110
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
|
111
|
-
extractor.extract
|
112
|
-
|
113
|
-
extractor.select('age', 'count(*) AS nr_ages')
|
114
|
-
extractor.from('tablename t')
|
115
|
-
extractor.joins('table2 t2 ON t2.my_id = t.id')
|
116
|
-
extractor.where('age > 10')
|
117
|
-
extractor.group('age')
|
118
|
-
extractor.having('count(*) > 1')
|
119
|
-
extractor.order('nr_ages')
|
120
|
-
|
121
|
-
extractor.extract
|
122
|
-
end
|
123
70
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
db.stubs(:execute).multiple_yields([record1], [record2])
|
130
|
-
|
131
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(db, 'tablename')
|
132
|
-
|
133
|
-
records = []
|
134
|
-
indexes = []
|
135
|
-
extractor.extract do |record, index|
|
136
|
-
records << record
|
137
|
-
indexes << index
|
71
|
+
describe '#where' do
|
72
|
+
it 'sets where clause with provided condition' do
|
73
|
+
@extractor.where('age >= 18 AND age < 50')
|
74
|
+
@extractor.send(:sql).must_equal 'SELECT * FROM records WHERE age >= 18 AND age < 50'
|
75
|
+
end
|
138
76
|
end
|
139
77
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
end
|
146
|
-
|
147
|
-
describe 'without stubs' do
|
148
|
-
before(:each) do
|
149
|
-
@db = SQLite3::Database.new(':memory:')
|
150
|
-
@db.execute('CREATE TABLE records (a INTEGER, b INTEGER)')
|
151
|
-
@db.execute('INSERT INTO records (a, b) VALUES (1, 2)');
|
152
|
-
@db.execute('INSERT INTO records (a, b) VALUES (3, 4)');
|
153
|
-
@db.execute('INSERT INTO records (a, b) VALUES (3, 6)');
|
154
|
-
end
|
155
|
-
|
156
|
-
after(:each) do
|
157
|
-
@db.close
|
158
|
-
end
|
159
|
-
|
160
|
-
describe '#initialize' do
|
161
|
-
it 'sets name to sqlite3:memory:records' do
|
162
|
-
extractor = Drudgery::Extractors::SQLite3Extractor.new(@db, 'records')
|
163
|
-
extractor.name.must_equal 'sqlite3:memory.records'
|
78
|
+
describe '#having' do
|
79
|
+
it 'sets having clause with provided condition' do
|
80
|
+
@extractor.having('COUNT(*) > 1')
|
81
|
+
@extractor.send(:sql).must_equal 'SELECT * FROM records HAVING COUNT(*) > 1'
|
82
|
+
end
|
164
83
|
end
|
165
|
-
end
|
166
84
|
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
records = []
|
172
|
-
indexes = []
|
173
|
-
extractor.extract do |record, index|
|
174
|
-
records << record
|
175
|
-
indexes << index
|
85
|
+
describe '#order' do
|
86
|
+
it 'sets order clause with provided expressions' do
|
87
|
+
@extractor.order('id', 'email DESC')
|
88
|
+
@extractor.send(:sql).must_equal 'SELECT * FROM records ORDER BY id, email DESC'
|
176
89
|
end
|
90
|
+
end
|
177
91
|
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
92
|
+
describe '#extract' do
|
93
|
+
describe 'with custom query' do
|
94
|
+
it 'yields each record hash and index' do
|
95
|
+
@extractor.where('a > 2')
|
96
|
+
|
97
|
+
records = []
|
98
|
+
indexes = []
|
99
|
+
@extractor.extract do |record, index|
|
100
|
+
records << record
|
101
|
+
indexes << index
|
102
|
+
end
|
103
|
+
|
104
|
+
records.must_equal([
|
105
|
+
{ 'a' => 3, 'b' => 4 },
|
106
|
+
{ 'a' => 3, 'b' => 6 }
|
107
|
+
])
|
108
|
+
|
109
|
+
indexes.must_equal [0, 1]
|
110
|
+
end
|
111
|
+
end
|
183
112
|
|
184
|
-
|
113
|
+
describe 'without custom query' do
|
114
|
+
it 'yields each record hash and index' do
|
115
|
+
records = []
|
116
|
+
indexes = []
|
117
|
+
@extractor.extract do |record, index|
|
118
|
+
records << record
|
119
|
+
indexes << index
|
120
|
+
end
|
121
|
+
|
122
|
+
records.must_equal([
|
123
|
+
{ 'a' => 1, 'b' => 2 },
|
124
|
+
{ 'a' => 3, 'b' => 4 },
|
125
|
+
{ 'a' => 3, 'b' => 6 }
|
126
|
+
])
|
127
|
+
|
128
|
+
indexes.must_equal [0, 1, 2]
|
129
|
+
end
|
130
|
+
end
|
185
131
|
end
|
186
|
-
end
|
187
132
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
133
|
+
describe '#record_count' do
|
134
|
+
describe 'with custom query' do
|
135
|
+
it 'returns count of query results' do
|
136
|
+
@extractor = SQLite3Extractor.new(@db, 'records')
|
137
|
+
@extractor.where('a > 2')
|
138
|
+
@extractor.record_count.must_equal 2
|
139
|
+
end
|
195
140
|
end
|
196
|
-
end
|
197
141
|
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
142
|
+
describe 'without custom query' do
|
143
|
+
it 'returns count of table records' do
|
144
|
+
@extractor = SQLite3Extractor.new(@db, 'records')
|
145
|
+
@extractor.record_count.must_equal 3
|
146
|
+
end
|
202
147
|
end
|
203
148
|
end
|
204
149
|
end
|