purview 1.2.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +7 -0
- data/README.md +21 -5
- data/lib/purview/connections/base.rb +0 -6
- data/lib/purview/databases/base.rb +155 -117
- data/lib/purview/databases/mysql.rb +5 -4
- data/lib/purview/databases/postgresql.rb +5 -4
- data/lib/purview/exceptions/could_not_baseline.rb +9 -0
- data/lib/purview/exceptions/could_not_sync.rb +9 -0
- data/lib/purview/exceptions/table_already_assigned.rb +17 -0
- data/lib/purview/exceptions.rb +3 -0
- data/lib/purview/indices/base.rb +39 -0
- data/lib/purview/indices/composite.rb +9 -0
- data/lib/purview/indices/simple.rb +9 -0
- data/lib/purview/indices.rb +4 -0
- data/lib/purview/loaders/base.rb +4 -3
- data/lib/purview/mixins/connection.rb +13 -5
- data/lib/purview/parsers/csv.rb +2 -2
- data/lib/purview/raw_connections/base.rb +0 -6
- data/lib/purview/tables/base.rb +21 -8
- data/lib/purview/version.rb +1 -1
- data/lib/purview.rb +1 -0
- metadata +9 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 857247110c4fa3aeb219d07aa207bc4b7017a3d0
|
4
|
+
data.tar.gz: 1348881a82f7aa07d31fbf52ead8423c2d7458eb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f77a3dd4d4bab8da4beadd7e2ab4f843e5ad9a5f0e87e66912b055dfa87f2d4237926829fe1658188f7aa6ec486a51fd13c71ef125c5049771066287eeeeb134
|
7
|
+
data.tar.gz: 2d522937e113acd589b83f4d0a0d939f136e461b1620b2181e522d8cfc9d501515865601c2677a8f337e6e21bee9ee0040a612c31c23b1144b28a382451f80bc
|
data/CHANGELOG
CHANGED
data/README.md
CHANGED
@@ -38,12 +38,27 @@ Define the `Column(s)` (available column-types: `Boolean`, `CreatedTimestamp`,
|
|
38
38
|
`UpdatedTimestamp` & `UUID` -- the `Id`, `CreatedTimestamp` & `UpdatedTimestamp`
|
39
39
|
columns are required for all tables)
|
40
40
|
```ruby
|
41
|
+
id_column = Purview::Columns::Id.new(:id),
|
42
|
+
name_column = Purview::Columns::String.new(:name, :nullable => false),
|
43
|
+
email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100),
|
44
|
+
created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at),
|
45
|
+
updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at),
|
46
|
+
|
41
47
|
columns = [
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
48
|
+
id_column,
|
49
|
+
name_column,
|
50
|
+
email_column,
|
51
|
+
created_at_column,
|
52
|
+
updated_at_column,
|
53
|
+
]
|
54
|
+
```
|
55
|
+
|
56
|
+
Define the `Indices` (availble index-types: `Composite` & `Simple`). By default
|
57
|
+
`Indices` will be added for the required column-types (`CreatedTimestamp` &
|
58
|
+
`UpdatedTimestamp`)
|
59
|
+
```ruby
|
60
|
+
indices = [
|
61
|
+
Purview::Indices::Simple.new(email_column),
|
47
62
|
]
|
48
63
|
```
|
49
64
|
|
@@ -73,6 +88,7 @@ Combine all the configuration options and instantiate the `Table`
|
|
73
88
|
```ruby
|
74
89
|
table_opts = {
|
75
90
|
:columns => columns,
|
91
|
+
:indices => indices,
|
76
92
|
:loader => loader_opts,
|
77
93
|
:parser => parser_opts,
|
78
94
|
:puller => puller_opts,
|
@@ -1,16 +1,22 @@
|
|
1
1
|
module Purview
|
2
2
|
module Databases
|
3
3
|
class Base
|
4
|
-
attr_reader :name
|
4
|
+
attr_reader :name, :tables
|
5
5
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
8
8
|
@opts = opts
|
9
|
+
@tables = Set.new.tap do |result|
|
10
|
+
(opts[:tables] || []).each do |table|
|
11
|
+
table.database = self if result.add?(table)
|
12
|
+
end
|
13
|
+
end
|
9
14
|
end
|
10
15
|
|
11
16
|
def baseline_table(table)
|
12
|
-
|
13
|
-
|
17
|
+
ensure_table_valid_for_database(table)
|
18
|
+
raise Purview::Exceptions::CouldNotBaseline.new(table) \
|
19
|
+
unless table_initialized?(table)
|
14
20
|
table_name = table_name(table)
|
15
21
|
with_context_logging("`baseline_table` for: #{table_name}") do
|
16
22
|
starting_timestamp = timestamp
|
@@ -21,62 +27,66 @@ module Purview
|
|
21
27
|
end
|
22
28
|
end
|
23
29
|
end
|
30
|
+
table_name
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_index(index, opts={})
|
34
|
+
ensure_index_valid_for_database(index)
|
35
|
+
table_opts = extract_table_options(opts)
|
36
|
+
table_name = table_name(index.table, table_opts)
|
37
|
+
index_opts = extract_index_options(opts)
|
38
|
+
index_columns = index.columns
|
39
|
+
index_name = index_name(
|
40
|
+
table_name,
|
41
|
+
index_columns,
|
42
|
+
index_opts
|
43
|
+
)
|
44
|
+
with_context_logging("`create_index` for: #{index_name}") do
|
45
|
+
with_new_or_existing_connection(opts) do |connection|
|
46
|
+
connection.execute(
|
47
|
+
create_index_sql(
|
48
|
+
table_name,
|
49
|
+
index_name,
|
50
|
+
index,
|
51
|
+
index_opts
|
52
|
+
)
|
53
|
+
)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
index_name
|
24
57
|
end
|
25
58
|
|
26
59
|
def create_table(table, opts={})
|
60
|
+
ensure_table_valid_for_database(table)
|
27
61
|
ensure_table_metadata_exists_for_table(table)
|
28
62
|
table_opts = extract_table_options(opts)
|
29
63
|
table_name = table_name(table, table_opts)
|
30
64
|
with_context_logging("`create_table` for: #{table_name}") do
|
31
|
-
|
65
|
+
with_new_or_existing_connection(opts) do |connection|
|
32
66
|
connection.execute(
|
33
|
-
|
67
|
+
send(
|
68
|
+
create_table_sql_method_name(table, table_opts),
|
34
69
|
table_name,
|
35
70
|
table,
|
36
71
|
table_opts
|
37
72
|
)
|
38
73
|
)
|
39
74
|
if table_opts[:create_indices]
|
40
|
-
table.
|
75
|
+
table.indices.each do |index|
|
41
76
|
create_index(
|
42
|
-
|
43
|
-
|
44
|
-
columns,
|
77
|
+
index,
|
78
|
+
:connection => connection,
|
45
79
|
:table => { :name => table_name }
|
46
80
|
)
|
47
81
|
end
|
48
82
|
end
|
49
83
|
end
|
50
|
-
table_name
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def create_temporary_table(connection, table, opts={})
|
55
|
-
table_opts = extract_table_options(opts)
|
56
|
-
table_name = table_name(table, table_opts)
|
57
|
-
with_context_logging("`create_temporary_table` for: #{table_name}") do
|
58
|
-
connection.execute(
|
59
|
-
create_temporary_table_sql(
|
60
|
-
table_name,
|
61
|
-
table,
|
62
|
-
table_opts
|
63
|
-
)
|
64
|
-
)
|
65
|
-
if table_opts[:create_indices]
|
66
|
-
table.indexed_columns.each do |columns|
|
67
|
-
create_index(
|
68
|
-
connection,
|
69
|
-
table,
|
70
|
-
columns,
|
71
|
-
:table => { :name => table_name }
|
72
|
-
)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
table_name
|
76
84
|
end
|
85
|
+
table_name
|
77
86
|
end
|
78
87
|
|
79
88
|
def disable_table(table)
|
89
|
+
ensure_table_valid_for_database(table)
|
80
90
|
table_name = table_name(table)
|
81
91
|
with_context_logging("`disable_table` for: #{table_name}") do
|
82
92
|
with_new_connection do |connection|
|
@@ -85,11 +95,38 @@ module Purview
|
|
85
95
|
raise Purview::Exceptions::CouldNotDisable.new(table) \
|
86
96
|
if zero?(rows_affected)
|
87
97
|
end
|
88
|
-
table_name
|
89
98
|
end
|
99
|
+
table_name
|
100
|
+
end
|
101
|
+
|
102
|
+
def drop_index(index, opts={})
|
103
|
+
ensure_index_valid_for_database(index)
|
104
|
+
table_opts = extract_table_options(opts)
|
105
|
+
table_name = table_name(index.table, table_opts)
|
106
|
+
index_opts = extract_index_options(opts)
|
107
|
+
index_columns = index.columns
|
108
|
+
index_name = index_name(
|
109
|
+
table_name,
|
110
|
+
index_columns,
|
111
|
+
index_opts
|
112
|
+
)
|
113
|
+
with_context_logging("`drop_index` for: #{index_name}") do
|
114
|
+
with_new_or_existing_connection(opts) do |connection|
|
115
|
+
connection.execute(
|
116
|
+
drop_index_sql(
|
117
|
+
table_name,
|
118
|
+
index_name,
|
119
|
+
index,
|
120
|
+
index_opts
|
121
|
+
)
|
122
|
+
)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
index_name
|
90
126
|
end
|
91
127
|
|
92
128
|
def drop_table(table, opts={})
|
129
|
+
ensure_table_valid_for_database(table)
|
93
130
|
ensure_table_metadata_absent_for_table(table)
|
94
131
|
table_opts = extract_table_options(opts)
|
95
132
|
table_name = table_name(table, table_opts)
|
@@ -103,11 +140,12 @@ module Purview
|
|
103
140
|
)
|
104
141
|
)
|
105
142
|
end
|
106
|
-
table_name
|
107
143
|
end
|
144
|
+
table_name
|
108
145
|
end
|
109
146
|
|
110
147
|
def enable_table(table, timestamp=timestamp)
|
148
|
+
ensure_table_valid_for_database(table)
|
111
149
|
table_name = table_name(table)
|
112
150
|
with_context_logging("`enable_table` for: #{table_name}") do
|
113
151
|
with_new_connection do |connection|
|
@@ -116,11 +154,12 @@ module Purview
|
|
116
154
|
raise Purview::Exceptions::CouldNotEnable.new(table) \
|
117
155
|
if zero?(rows_affected)
|
118
156
|
end
|
119
|
-
table_name
|
120
157
|
end
|
158
|
+
table_name
|
121
159
|
end
|
122
160
|
|
123
161
|
def initialize_table(table, timestamp=timestamp)
|
162
|
+
ensure_table_valid_for_database(table)
|
124
163
|
table_name = table_name(table)
|
125
164
|
with_context_logging("`initialize_table` for: #{table_name}") do
|
126
165
|
with_new_connection do |connection|
|
@@ -129,11 +168,12 @@ module Purview
|
|
129
168
|
raise Purview::Exceptions::CouldNotInitialize.new(table) \
|
130
169
|
if zero?(rows_affected)
|
131
170
|
end
|
132
|
-
table_name
|
133
171
|
end
|
172
|
+
table_name
|
134
173
|
end
|
135
174
|
|
136
175
|
def lock_table(table, timestamp=timestamp)
|
176
|
+
ensure_table_valid_for_database(table)
|
137
177
|
table_name = table_name(table)
|
138
178
|
with_context_logging("`lock_table` for: #{table_name}") do
|
139
179
|
with_new_connection do |connection|
|
@@ -142,8 +182,8 @@ module Purview
|
|
142
182
|
raise Purview::Exceptions::CouldNotLock.new(table) \
|
143
183
|
if zero?(rows_affected)
|
144
184
|
end
|
145
|
-
table_name
|
146
185
|
end
|
186
|
+
table_name
|
147
187
|
end
|
148
188
|
|
149
189
|
def sync
|
@@ -157,15 +197,18 @@ module Purview
|
|
157
197
|
end
|
158
198
|
|
159
199
|
def sync_table(table)
|
160
|
-
|
161
|
-
|
200
|
+
ensure_table_valid_for_database(table)
|
201
|
+
raise Purview::Exceptions::CouldNotSync.new(table) \
|
202
|
+
unless table_initialized?(table) && table_enabled?(table)
|
162
203
|
table_name = table_name(table)
|
163
204
|
with_context_logging("`sync_table` for: #{table_name}") do
|
164
205
|
sync_table_with_lock(table, timestamp)
|
165
206
|
end
|
207
|
+
table_name
|
166
208
|
end
|
167
209
|
|
168
210
|
def unlock_table(table)
|
211
|
+
ensure_table_valid_for_database(table)
|
169
212
|
table_name = table_name(table)
|
170
213
|
with_context_logging("`unlock_table` for: #{table_name}") do
|
171
214
|
with_new_connection do |connection|
|
@@ -174,8 +217,8 @@ module Purview
|
|
174
217
|
raise Purview::Exceptions::CouldNotUnlock.new(table) \
|
175
218
|
if zero?(rows_affected)
|
176
219
|
end
|
177
|
-
table_name
|
178
220
|
end
|
221
|
+
table_name
|
179
222
|
end
|
180
223
|
|
181
224
|
private
|
@@ -188,10 +231,6 @@ module Purview
|
|
188
231
|
|
189
232
|
public :connect
|
190
233
|
|
191
|
-
def column_names(columns)
|
192
|
-
columns.map(&:name)
|
193
|
-
end
|
194
|
-
|
195
234
|
def column_definition(column)
|
196
235
|
column.name.to_s.tap do |column_definition|
|
197
236
|
type = type(column)
|
@@ -207,13 +246,17 @@ module Purview
|
|
207
246
|
end
|
208
247
|
end
|
209
248
|
|
249
|
+
def column_names(columns)
|
250
|
+
columns.map(&:name)
|
251
|
+
end
|
252
|
+
|
210
253
|
def column_definitions(table)
|
211
|
-
[].tap do |
|
212
|
-
|
213
|
-
|
214
|
-
|
254
|
+
[].tap do |result|
|
255
|
+
result << column_definition(table.id_column)
|
256
|
+
result << column_definition(table.created_timestamp_column)
|
257
|
+
result << column_definition(table.updated_timestamp_column)
|
215
258
|
table.data_columns.each do |column|
|
216
|
-
|
259
|
+
result << column_definition(column)
|
217
260
|
end
|
218
261
|
end
|
219
262
|
end
|
@@ -222,28 +265,7 @@ module Purview
|
|
222
265
|
raise %{All "#{Base}(s)" must override the "connection_type" method}
|
223
266
|
end
|
224
267
|
|
225
|
-
def
|
226
|
-
table_opts = extract_table_options(opts)
|
227
|
-
table_name = table_name(table, table_opts)
|
228
|
-
index_opts = extract_index_options(opts)
|
229
|
-
index_name(
|
230
|
-
table_name,
|
231
|
-
columns,
|
232
|
-
index_opts
|
233
|
-
).tap do |index_name|
|
234
|
-
connection.execute(
|
235
|
-
create_index_sql(
|
236
|
-
table_name,
|
237
|
-
index_name,
|
238
|
-
table,
|
239
|
-
columns,
|
240
|
-
index_opts
|
241
|
-
)
|
242
|
-
)
|
243
|
-
end
|
244
|
-
end
|
245
|
-
|
246
|
-
def create_index_sql(table_name, index_name, table, columns, index_opts={})
|
268
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
247
269
|
raise %{All "#{Base}(s)" must override the "create_index_sql" method}
|
248
270
|
end
|
249
271
|
|
@@ -251,6 +273,10 @@ module Purview
|
|
251
273
|
raise %{All "#{Base}(s)" must override the "create_table_sql" method}
|
252
274
|
end
|
253
275
|
|
276
|
+
def create_table_sql_method_name(table, table_opts={})
|
277
|
+
"create#{table_opts[:temporary] && '_temporary'}_table_sql".to_sym
|
278
|
+
end
|
279
|
+
|
254
280
|
def create_temporary_table_sql(table_name, table, table_opts={})
|
255
281
|
raise %{All "#{Base}(s)" must override the "create_temporary_table_sql" method}
|
256
282
|
end
|
@@ -295,28 +321,7 @@ module Purview
|
|
295
321
|
raise %{All "#{Base}(s)" must override the "disable_table_sql" method}
|
296
322
|
end
|
297
323
|
|
298
|
-
def
|
299
|
-
table_opts = extract_table_options(opts)
|
300
|
-
table_name = table_name(table, table_opts)
|
301
|
-
index_opts = extract_index_options(opts)
|
302
|
-
index_name(
|
303
|
-
table_name,
|
304
|
-
columns,
|
305
|
-
index_opts
|
306
|
-
).tap do |index_name|
|
307
|
-
connection.execute(
|
308
|
-
drop_index_sql(
|
309
|
-
table_name,
|
310
|
-
index_name,
|
311
|
-
table,
|
312
|
-
columns,
|
313
|
-
index_opts
|
314
|
-
)
|
315
|
-
)
|
316
|
-
end
|
317
|
-
end
|
318
|
-
|
319
|
-
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
324
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
320
325
|
raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
|
321
326
|
end
|
322
327
|
|
@@ -328,6 +333,12 @@ module Purview
|
|
328
333
|
raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
|
329
334
|
end
|
330
335
|
|
336
|
+
def ensure_index_valid_for_database(index)
|
337
|
+
raise ArgumentError.new('Must provide an `Index`') \
|
338
|
+
unless index
|
339
|
+
ensure_table_valid_for_database(index.table)
|
340
|
+
end
|
341
|
+
|
331
342
|
def ensure_table_metadata_absent_for_table(table)
|
332
343
|
with_new_connection do |connection|
|
333
344
|
connection.execute(ensure_table_metadata_table_exists_sql)
|
@@ -354,6 +365,13 @@ module Purview
|
|
354
365
|
raise %{All "#{Base}(s)" must override the "ensure_table_metadata_table_exists_sql" method}
|
355
366
|
end
|
356
367
|
|
368
|
+
def ensure_table_valid_for_database(table)
|
369
|
+
raise ArgumentError.new('Must provide a `Table`') \
|
370
|
+
unless table
|
371
|
+
raise Purview::Exceptions::WrongDatabase.new(table) \
|
372
|
+
unless tables.include?(table)
|
373
|
+
end
|
374
|
+
|
357
375
|
def extract_index_options(opts)
|
358
376
|
opts[:index] || {}
|
359
377
|
end
|
@@ -516,24 +534,52 @@ module Purview
|
|
516
534
|
def sync_table_without_lock(table, timestamp)
|
517
535
|
last_window = nil
|
518
536
|
with_next_window(table, timestamp) do |window|
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
537
|
+
with_new_connection do |connection|
|
538
|
+
with_transaction(connection) do
|
539
|
+
table.sync(connection, window)
|
540
|
+
set_last_pulled_at_for_table(
|
541
|
+
connection,
|
542
|
+
table,
|
543
|
+
timestamp
|
544
|
+
)
|
545
|
+
set_max_timestamp_pulled_for_table(
|
546
|
+
connection,
|
547
|
+
table,
|
548
|
+
window.max
|
549
|
+
)
|
550
|
+
last_window = window
|
551
|
+
end
|
532
552
|
end
|
533
553
|
end
|
534
554
|
last_window
|
535
555
|
end
|
536
556
|
|
557
|
+
def table_disabled?(table)
|
558
|
+
!table_enabled?(table)
|
559
|
+
end
|
560
|
+
|
561
|
+
def table_enabled?(table)
|
562
|
+
with_new_connection do |connection|
|
563
|
+
!!get_enabled_at_for_table(connection, table)
|
564
|
+
end
|
565
|
+
end
|
566
|
+
|
567
|
+
def table_initialized?(table)
|
568
|
+
with_new_connection do |connection|
|
569
|
+
!!get_max_timestamp_pulled_for_table(connection, table)
|
570
|
+
end
|
571
|
+
end
|
572
|
+
|
573
|
+
def table_locked?(table)
|
574
|
+
with_new_connection do |connection|
|
575
|
+
!!get_locked_at_for_table(connection, table)
|
576
|
+
end
|
577
|
+
end
|
578
|
+
|
579
|
+
def table_unlocked?(table)
|
580
|
+
!table_locked?(table)
|
581
|
+
end
|
582
|
+
|
537
583
|
def table_metadata_enabled_at_column_definition
|
538
584
|
column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
|
539
585
|
column_definition(column)
|
@@ -587,14 +633,6 @@ module Purview
|
|
587
633
|
table_opts[:name] || table.name
|
588
634
|
end
|
589
635
|
|
590
|
-
def tables
|
591
|
-
@tables ||= Set.new.tap do |result|
|
592
|
-
(opts[:tables] || []).each do |table|
|
593
|
-
table.database = self if result.add?(table)
|
594
|
-
end
|
595
|
-
end
|
596
|
-
end
|
597
|
-
|
598
636
|
def tables_by_name
|
599
637
|
@tables_by_name ||= {}.tap do |result|
|
600
638
|
tables.each do |table|
|
@@ -7,11 +7,12 @@ module Purview
|
|
7
7
|
Purview::Connections::MySQL
|
8
8
|
end
|
9
9
|
|
10
|
-
def create_index_sql(table_name, index_name,
|
11
|
-
'CREATE
|
10
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
11
|
+
'CREATE%sINDEX %s ON %s (%s)' % [
|
12
|
+
index.unique? ? ' UNIQUE ' : ' ',
|
12
13
|
index_name,
|
13
14
|
table_name,
|
14
|
-
column_names(columns).join(', '),
|
15
|
+
column_names(index.columns).join(', '),
|
15
16
|
]
|
16
17
|
end
|
17
18
|
|
@@ -48,7 +49,7 @@ module Purview
|
|
48
49
|
]
|
49
50
|
end
|
50
51
|
|
51
|
-
def drop_index_sql(table_name, index_name,
|
52
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
52
53
|
'DROP INDEX %s' % [
|
53
54
|
index_name,
|
54
55
|
]
|
@@ -7,11 +7,12 @@ module Purview
|
|
7
7
|
Purview::Connections::PostgreSQL
|
8
8
|
end
|
9
9
|
|
10
|
-
def create_index_sql(table_name, index_name,
|
11
|
-
'CREATE
|
10
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
11
|
+
'CREATE%sINDEX %s ON %s (%s)' % [
|
12
|
+
index.unique? ? ' UNIQUE ' : ' ',
|
12
13
|
index_name,
|
13
14
|
table_name,
|
14
|
-
column_names(columns).join(', '),
|
15
|
+
column_names(index.columns).join(', '),
|
15
16
|
]
|
16
17
|
end
|
17
18
|
|
@@ -44,7 +45,7 @@ module Purview
|
|
44
45
|
]
|
45
46
|
end
|
46
47
|
|
47
|
-
def drop_index_sql(table_name, index_name,
|
48
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
48
49
|
'DROP INDEX %s' % [
|
49
50
|
index_name,
|
50
51
|
]
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Purview
|
2
|
+
module Exceptions
|
3
|
+
class TableAlreadyAssigned < Base
|
4
|
+
def initialize(index)
|
5
|
+
@index = index
|
6
|
+
end
|
7
|
+
|
8
|
+
def message
|
9
|
+
"Table already assigned for index on columns: #{index.table.column_names.join(', ')}"
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
attr_reader :index
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/purview/exceptions.rb
CHANGED
@@ -1,14 +1,17 @@
|
|
1
1
|
require 'purview/exceptions/base'
|
2
2
|
require 'purview/exceptions/base_table'
|
3
3
|
|
4
|
+
require 'purview/exceptions/could_not_baseline'
|
4
5
|
require 'purview/exceptions/could_not_disable'
|
5
6
|
require 'purview/exceptions/could_not_enable'
|
6
7
|
require 'purview/exceptions/could_not_initialize'
|
7
8
|
require 'purview/exceptions/could_not_lock'
|
9
|
+
require 'purview/exceptions/could_not_sync'
|
8
10
|
require 'purview/exceptions/could_not_unlock'
|
9
11
|
|
10
12
|
require 'purview/exceptions/database_already_assigned'
|
11
13
|
require 'purview/exceptions/no_table'
|
12
14
|
require 'purview/exceptions/no_window'
|
13
15
|
require 'purview/exceptions/rows_outside_window'
|
16
|
+
require 'purview/exceptions/table_already_assigned'
|
14
17
|
require 'purview/exceptions/wrong_database'
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Purview
|
2
|
+
module Indices
|
3
|
+
class Base
|
4
|
+
attr_reader :columns, :table
|
5
|
+
|
6
|
+
def initialize(columns, opts={})
|
7
|
+
@columns = columns
|
8
|
+
@opts = opts
|
9
|
+
end
|
10
|
+
|
11
|
+
def eql?(other)
|
12
|
+
self.class == other.class &&
|
13
|
+
columns == other.columns &&
|
14
|
+
unique == other.unique
|
15
|
+
end
|
16
|
+
|
17
|
+
def hash
|
18
|
+
columns.hash + unique.hash
|
19
|
+
end
|
20
|
+
|
21
|
+
def table=(value)
|
22
|
+
raise Purview::Exceptions::TableAlreadyAssigned.new(self) if table
|
23
|
+
@table = value
|
24
|
+
end
|
25
|
+
|
26
|
+
def unique
|
27
|
+
opts[:unique]
|
28
|
+
end
|
29
|
+
|
30
|
+
def unique?
|
31
|
+
!!unique
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
attr_reader :opts
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/purview/loaders/base.rb
CHANGED
@@ -38,10 +38,10 @@ module Purview
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def create_temporary_table(connection)
|
41
|
-
database.
|
42
|
-
connection,
|
41
|
+
database.create_table(
|
43
42
|
table,
|
44
|
-
:
|
43
|
+
:connection => connection,
|
44
|
+
:table => temporary_table_opts,
|
45
45
|
)
|
46
46
|
end
|
47
47
|
|
@@ -130,6 +130,7 @@ module Purview
|
|
130
130
|
{
|
131
131
|
:create_indices => true,
|
132
132
|
:name => table.temporary_name,
|
133
|
+
:temporary => true,
|
133
134
|
}
|
134
135
|
end
|
135
136
|
|
@@ -2,7 +2,7 @@ module Purview
|
|
2
2
|
module Mixins
|
3
3
|
module Connection
|
4
4
|
def connect
|
5
|
-
connection_type.
|
5
|
+
connection_type.connect(connection_opts)
|
6
6
|
end
|
7
7
|
|
8
8
|
def connection_opts
|
@@ -16,14 +16,22 @@ module Purview
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def with_new_connection
|
19
|
-
|
19
|
+
yield connection = connect
|
20
|
+
ensure
|
21
|
+
connection.disconnect if connection
|
20
22
|
end
|
21
23
|
|
22
|
-
def
|
23
|
-
|
24
|
-
|
24
|
+
def with_new_or_existing_connection(opts={})
|
25
|
+
if existing_connection = opts[:connection]
|
26
|
+
yield existing_connection
|
27
|
+
else
|
28
|
+
with_new_connection { |connection| yield connection }
|
25
29
|
end
|
26
30
|
end
|
31
|
+
|
32
|
+
def with_transaction(connection)
|
33
|
+
connection.with_transaction { yield }
|
34
|
+
end
|
27
35
|
end
|
28
36
|
end
|
29
37
|
end
|
data/lib/purview/parsers/csv.rb
CHANGED
@@ -3,10 +3,10 @@ module Purview
|
|
3
3
|
class CSV < Base
|
4
4
|
def parse(data)
|
5
5
|
with_context_logging("`parse` for: #{table.name}") do
|
6
|
-
[].tap do |
|
6
|
+
[].tap do |result|
|
7
7
|
headers = extract_headers(data)
|
8
8
|
extract_rows(data).each do |row|
|
9
|
-
|
9
|
+
result << build_result(headers.zip(row))
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
data/lib/purview/tables/base.rb
CHANGED
@@ -1,11 +1,16 @@
|
|
1
1
|
module Purview
|
2
2
|
module Tables
|
3
3
|
class Base
|
4
|
-
attr_reader :database, :name
|
4
|
+
attr_reader :database, :indices, :name
|
5
5
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
8
8
|
@opts = opts
|
9
|
+
@indices = Set.new.tap do |result|
|
10
|
+
((opts[:indices] || []) + default_indices).each do |index|
|
11
|
+
index.table = self if result.add?(index)
|
12
|
+
end
|
13
|
+
end
|
9
14
|
end
|
10
15
|
|
11
16
|
def columns
|
@@ -32,6 +37,10 @@ module Purview
|
|
32
37
|
columns_of_type(Purview::Columns::CreatedTimestamp).first
|
33
38
|
end
|
34
39
|
|
40
|
+
def created_timestamp_index
|
41
|
+
Purview::Indices::Simple.new(created_timestamp_column)
|
42
|
+
end
|
43
|
+
|
35
44
|
def data_columns
|
36
45
|
columns - [
|
37
46
|
created_timestamp_column,
|
@@ -49,13 +58,6 @@ module Purview
|
|
49
58
|
columns_of_type(Purview::Columns::Id).first
|
50
59
|
end
|
51
60
|
|
52
|
-
def indexed_columns
|
53
|
-
(opts[:indexed_columns] || []).tap do |indexed_columns|
|
54
|
-
indexed_columns << [created_timestamp_column]
|
55
|
-
indexed_columns << [updated_timestamp_column]
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
61
|
def sync(connection, window)
|
60
62
|
raw_data = puller.pull(window)
|
61
63
|
parser.validate(raw_data)
|
@@ -75,6 +77,10 @@ module Purview
|
|
75
77
|
columns_of_type(Purview::Columns::UpdatedTimestamp).first
|
76
78
|
end
|
77
79
|
|
80
|
+
def updated_timestamp_index
|
81
|
+
Purview::Indices::Simple.new(updated_timestamp_column)
|
82
|
+
end
|
83
|
+
|
78
84
|
def window_size
|
79
85
|
opts[:window_size] || (60 * 60)
|
80
86
|
end
|
@@ -86,6 +92,13 @@ module Purview
|
|
86
92
|
|
87
93
|
attr_reader :opts
|
88
94
|
|
95
|
+
def default_indices
|
96
|
+
[
|
97
|
+
created_timestamp_index,
|
98
|
+
updated_timestamp_index,
|
99
|
+
]
|
100
|
+
end
|
101
|
+
|
89
102
|
def extract_type_option(opts)
|
90
103
|
opts[:type]
|
91
104
|
end
|
data/lib/purview/version.rb
CHANGED
data/lib/purview.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: purview
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan W. Zaleski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-06-
|
11
|
+
date: 2015-06-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -112,16 +112,23 @@ files:
|
|
112
112
|
- lib/purview/exceptions.rb
|
113
113
|
- lib/purview/exceptions/base.rb
|
114
114
|
- lib/purview/exceptions/base_table.rb
|
115
|
+
- lib/purview/exceptions/could_not_baseline.rb
|
115
116
|
- lib/purview/exceptions/could_not_disable.rb
|
116
117
|
- lib/purview/exceptions/could_not_enable.rb
|
117
118
|
- lib/purview/exceptions/could_not_initialize.rb
|
118
119
|
- lib/purview/exceptions/could_not_lock.rb
|
120
|
+
- lib/purview/exceptions/could_not_sync.rb
|
119
121
|
- lib/purview/exceptions/could_not_unlock.rb
|
120
122
|
- lib/purview/exceptions/database_already_assigned.rb
|
121
123
|
- lib/purview/exceptions/no_table.rb
|
122
124
|
- lib/purview/exceptions/no_window.rb
|
123
125
|
- lib/purview/exceptions/rows_outside_window.rb
|
126
|
+
- lib/purview/exceptions/table_already_assigned.rb
|
124
127
|
- lib/purview/exceptions/wrong_database.rb
|
128
|
+
- lib/purview/indices.rb
|
129
|
+
- lib/purview/indices/base.rb
|
130
|
+
- lib/purview/indices/composite.rb
|
131
|
+
- lib/purview/indices/simple.rb
|
125
132
|
- lib/purview/loaders.rb
|
126
133
|
- lib/purview/loaders/base.rb
|
127
134
|
- lib/purview/loaders/mysql.rb
|