purview 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG +7 -0
- data/README.md +21 -5
- data/lib/purview/connections/base.rb +0 -6
- data/lib/purview/databases/base.rb +155 -117
- data/lib/purview/databases/mysql.rb +5 -4
- data/lib/purview/databases/postgresql.rb +5 -4
- data/lib/purview/exceptions/could_not_baseline.rb +9 -0
- data/lib/purview/exceptions/could_not_sync.rb +9 -0
- data/lib/purview/exceptions/table_already_assigned.rb +17 -0
- data/lib/purview/exceptions.rb +3 -0
- data/lib/purview/indices/base.rb +39 -0
- data/lib/purview/indices/composite.rb +9 -0
- data/lib/purview/indices/simple.rb +9 -0
- data/lib/purview/indices.rb +4 -0
- data/lib/purview/loaders/base.rb +4 -3
- data/lib/purview/mixins/connection.rb +13 -5
- data/lib/purview/parsers/csv.rb +2 -2
- data/lib/purview/raw_connections/base.rb +0 -6
- data/lib/purview/tables/base.rb +21 -8
- data/lib/purview/version.rb +1 -1
- data/lib/purview.rb +1 -0
- metadata +9 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 857247110c4fa3aeb219d07aa207bc4b7017a3d0
|
4
|
+
data.tar.gz: 1348881a82f7aa07d31fbf52ead8423c2d7458eb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f77a3dd4d4bab8da4beadd7e2ab4f843e5ad9a5f0e87e66912b055dfa87f2d4237926829fe1658188f7aa6ec486a51fd13c71ef125c5049771066287eeeeb134
|
7
|
+
data.tar.gz: 2d522937e113acd589b83f4d0a0d939f136e461b1620b2181e522d8cfc9d501515865601c2677a8f337e6e21bee9ee0040a612c31c23b1144b28a382451f80bc
|
data/CHANGELOG
CHANGED
data/README.md
CHANGED
@@ -38,12 +38,27 @@ Define the `Column(s)` (available column-types: `Boolean`, `CreatedTimestamp`,
|
|
38
38
|
`UpdatedTimestamp` & `UUID` -- the `Id`, `CreatedTimestamp` & `UpdatedTimestamp`
|
39
39
|
columns are required for all tables)
|
40
40
|
```ruby
|
41
|
+
id_column = Purview::Columns::Id.new(:id),
|
42
|
+
name_column = Purview::Columns::String.new(:name, :nullable => false),
|
43
|
+
email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100),
|
44
|
+
created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at),
|
45
|
+
updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at),
|
46
|
+
|
41
47
|
columns = [
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
48
|
+
id_column,
|
49
|
+
name_column,
|
50
|
+
email_column,
|
51
|
+
created_at_column,
|
52
|
+
updated_at_column,
|
53
|
+
]
|
54
|
+
```
|
55
|
+
|
56
|
+
Define the `Indices` (availble index-types: `Composite` & `Simple`). By default
|
57
|
+
`Indices` will be added for the required column-types (`CreatedTimestamp` &
|
58
|
+
`UpdatedTimestamp`)
|
59
|
+
```ruby
|
60
|
+
indices = [
|
61
|
+
Purview::Indices::Simple.new(email_column),
|
47
62
|
]
|
48
63
|
```
|
49
64
|
|
@@ -73,6 +88,7 @@ Combine all the configuration options and instantiate the `Table`
|
|
73
88
|
```ruby
|
74
89
|
table_opts = {
|
75
90
|
:columns => columns,
|
91
|
+
:indices => indices,
|
76
92
|
:loader => loader_opts,
|
77
93
|
:parser => parser_opts,
|
78
94
|
:puller => puller_opts,
|
@@ -1,16 +1,22 @@
|
|
1
1
|
module Purview
|
2
2
|
module Databases
|
3
3
|
class Base
|
4
|
-
attr_reader :name
|
4
|
+
attr_reader :name, :tables
|
5
5
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
8
8
|
@opts = opts
|
9
|
+
@tables = Set.new.tap do |result|
|
10
|
+
(opts[:tables] || []).each do |table|
|
11
|
+
table.database = self if result.add?(table)
|
12
|
+
end
|
13
|
+
end
|
9
14
|
end
|
10
15
|
|
11
16
|
def baseline_table(table)
|
12
|
-
|
13
|
-
|
17
|
+
ensure_table_valid_for_database(table)
|
18
|
+
raise Purview::Exceptions::CouldNotBaseline.new(table) \
|
19
|
+
unless table_initialized?(table)
|
14
20
|
table_name = table_name(table)
|
15
21
|
with_context_logging("`baseline_table` for: #{table_name}") do
|
16
22
|
starting_timestamp = timestamp
|
@@ -21,62 +27,66 @@ module Purview
|
|
21
27
|
end
|
22
28
|
end
|
23
29
|
end
|
30
|
+
table_name
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_index(index, opts={})
|
34
|
+
ensure_index_valid_for_database(index)
|
35
|
+
table_opts = extract_table_options(opts)
|
36
|
+
table_name = table_name(index.table, table_opts)
|
37
|
+
index_opts = extract_index_options(opts)
|
38
|
+
index_columns = index.columns
|
39
|
+
index_name = index_name(
|
40
|
+
table_name,
|
41
|
+
index_columns,
|
42
|
+
index_opts
|
43
|
+
)
|
44
|
+
with_context_logging("`create_index` for: #{index_name}") do
|
45
|
+
with_new_or_existing_connection(opts) do |connection|
|
46
|
+
connection.execute(
|
47
|
+
create_index_sql(
|
48
|
+
table_name,
|
49
|
+
index_name,
|
50
|
+
index,
|
51
|
+
index_opts
|
52
|
+
)
|
53
|
+
)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
index_name
|
24
57
|
end
|
25
58
|
|
26
59
|
def create_table(table, opts={})
|
60
|
+
ensure_table_valid_for_database(table)
|
27
61
|
ensure_table_metadata_exists_for_table(table)
|
28
62
|
table_opts = extract_table_options(opts)
|
29
63
|
table_name = table_name(table, table_opts)
|
30
64
|
with_context_logging("`create_table` for: #{table_name}") do
|
31
|
-
|
65
|
+
with_new_or_existing_connection(opts) do |connection|
|
32
66
|
connection.execute(
|
33
|
-
|
67
|
+
send(
|
68
|
+
create_table_sql_method_name(table, table_opts),
|
34
69
|
table_name,
|
35
70
|
table,
|
36
71
|
table_opts
|
37
72
|
)
|
38
73
|
)
|
39
74
|
if table_opts[:create_indices]
|
40
|
-
table.
|
75
|
+
table.indices.each do |index|
|
41
76
|
create_index(
|
42
|
-
|
43
|
-
|
44
|
-
columns,
|
77
|
+
index,
|
78
|
+
:connection => connection,
|
45
79
|
:table => { :name => table_name }
|
46
80
|
)
|
47
81
|
end
|
48
82
|
end
|
49
83
|
end
|
50
|
-
table_name
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def create_temporary_table(connection, table, opts={})
|
55
|
-
table_opts = extract_table_options(opts)
|
56
|
-
table_name = table_name(table, table_opts)
|
57
|
-
with_context_logging("`create_temporary_table` for: #{table_name}") do
|
58
|
-
connection.execute(
|
59
|
-
create_temporary_table_sql(
|
60
|
-
table_name,
|
61
|
-
table,
|
62
|
-
table_opts
|
63
|
-
)
|
64
|
-
)
|
65
|
-
if table_opts[:create_indices]
|
66
|
-
table.indexed_columns.each do |columns|
|
67
|
-
create_index(
|
68
|
-
connection,
|
69
|
-
table,
|
70
|
-
columns,
|
71
|
-
:table => { :name => table_name }
|
72
|
-
)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
table_name
|
76
84
|
end
|
85
|
+
table_name
|
77
86
|
end
|
78
87
|
|
79
88
|
def disable_table(table)
|
89
|
+
ensure_table_valid_for_database(table)
|
80
90
|
table_name = table_name(table)
|
81
91
|
with_context_logging("`disable_table` for: #{table_name}") do
|
82
92
|
with_new_connection do |connection|
|
@@ -85,11 +95,38 @@ module Purview
|
|
85
95
|
raise Purview::Exceptions::CouldNotDisable.new(table) \
|
86
96
|
if zero?(rows_affected)
|
87
97
|
end
|
88
|
-
table_name
|
89
98
|
end
|
99
|
+
table_name
|
100
|
+
end
|
101
|
+
|
102
|
+
def drop_index(index, opts={})
|
103
|
+
ensure_index_valid_for_database(index)
|
104
|
+
table_opts = extract_table_options(opts)
|
105
|
+
table_name = table_name(index.table, table_opts)
|
106
|
+
index_opts = extract_index_options(opts)
|
107
|
+
index_columns = index.columns
|
108
|
+
index_name = index_name(
|
109
|
+
table_name,
|
110
|
+
index_columns,
|
111
|
+
index_opts
|
112
|
+
)
|
113
|
+
with_context_logging("`drop_index` for: #{index_name}") do
|
114
|
+
with_new_or_existing_connection(opts) do |connection|
|
115
|
+
connection.execute(
|
116
|
+
drop_index_sql(
|
117
|
+
table_name,
|
118
|
+
index_name,
|
119
|
+
index,
|
120
|
+
index_opts
|
121
|
+
)
|
122
|
+
)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
index_name
|
90
126
|
end
|
91
127
|
|
92
128
|
def drop_table(table, opts={})
|
129
|
+
ensure_table_valid_for_database(table)
|
93
130
|
ensure_table_metadata_absent_for_table(table)
|
94
131
|
table_opts = extract_table_options(opts)
|
95
132
|
table_name = table_name(table, table_opts)
|
@@ -103,11 +140,12 @@ module Purview
|
|
103
140
|
)
|
104
141
|
)
|
105
142
|
end
|
106
|
-
table_name
|
107
143
|
end
|
144
|
+
table_name
|
108
145
|
end
|
109
146
|
|
110
147
|
def enable_table(table, timestamp=timestamp)
|
148
|
+
ensure_table_valid_for_database(table)
|
111
149
|
table_name = table_name(table)
|
112
150
|
with_context_logging("`enable_table` for: #{table_name}") do
|
113
151
|
with_new_connection do |connection|
|
@@ -116,11 +154,12 @@ module Purview
|
|
116
154
|
raise Purview::Exceptions::CouldNotEnable.new(table) \
|
117
155
|
if zero?(rows_affected)
|
118
156
|
end
|
119
|
-
table_name
|
120
157
|
end
|
158
|
+
table_name
|
121
159
|
end
|
122
160
|
|
123
161
|
def initialize_table(table, timestamp=timestamp)
|
162
|
+
ensure_table_valid_for_database(table)
|
124
163
|
table_name = table_name(table)
|
125
164
|
with_context_logging("`initialize_table` for: #{table_name}") do
|
126
165
|
with_new_connection do |connection|
|
@@ -129,11 +168,12 @@ module Purview
|
|
129
168
|
raise Purview::Exceptions::CouldNotInitialize.new(table) \
|
130
169
|
if zero?(rows_affected)
|
131
170
|
end
|
132
|
-
table_name
|
133
171
|
end
|
172
|
+
table_name
|
134
173
|
end
|
135
174
|
|
136
175
|
def lock_table(table, timestamp=timestamp)
|
176
|
+
ensure_table_valid_for_database(table)
|
137
177
|
table_name = table_name(table)
|
138
178
|
with_context_logging("`lock_table` for: #{table_name}") do
|
139
179
|
with_new_connection do |connection|
|
@@ -142,8 +182,8 @@ module Purview
|
|
142
182
|
raise Purview::Exceptions::CouldNotLock.new(table) \
|
143
183
|
if zero?(rows_affected)
|
144
184
|
end
|
145
|
-
table_name
|
146
185
|
end
|
186
|
+
table_name
|
147
187
|
end
|
148
188
|
|
149
189
|
def sync
|
@@ -157,15 +197,18 @@ module Purview
|
|
157
197
|
end
|
158
198
|
|
159
199
|
def sync_table(table)
|
160
|
-
|
161
|
-
|
200
|
+
ensure_table_valid_for_database(table)
|
201
|
+
raise Purview::Exceptions::CouldNotSync.new(table) \
|
202
|
+
unless table_initialized?(table) && table_enabled?(table)
|
162
203
|
table_name = table_name(table)
|
163
204
|
with_context_logging("`sync_table` for: #{table_name}") do
|
164
205
|
sync_table_with_lock(table, timestamp)
|
165
206
|
end
|
207
|
+
table_name
|
166
208
|
end
|
167
209
|
|
168
210
|
def unlock_table(table)
|
211
|
+
ensure_table_valid_for_database(table)
|
169
212
|
table_name = table_name(table)
|
170
213
|
with_context_logging("`unlock_table` for: #{table_name}") do
|
171
214
|
with_new_connection do |connection|
|
@@ -174,8 +217,8 @@ module Purview
|
|
174
217
|
raise Purview::Exceptions::CouldNotUnlock.new(table) \
|
175
218
|
if zero?(rows_affected)
|
176
219
|
end
|
177
|
-
table_name
|
178
220
|
end
|
221
|
+
table_name
|
179
222
|
end
|
180
223
|
|
181
224
|
private
|
@@ -188,10 +231,6 @@ module Purview
|
|
188
231
|
|
189
232
|
public :connect
|
190
233
|
|
191
|
-
def column_names(columns)
|
192
|
-
columns.map(&:name)
|
193
|
-
end
|
194
|
-
|
195
234
|
def column_definition(column)
|
196
235
|
column.name.to_s.tap do |column_definition|
|
197
236
|
type = type(column)
|
@@ -207,13 +246,17 @@ module Purview
|
|
207
246
|
end
|
208
247
|
end
|
209
248
|
|
249
|
+
def column_names(columns)
|
250
|
+
columns.map(&:name)
|
251
|
+
end
|
252
|
+
|
210
253
|
def column_definitions(table)
|
211
|
-
[].tap do |
|
212
|
-
|
213
|
-
|
214
|
-
|
254
|
+
[].tap do |result|
|
255
|
+
result << column_definition(table.id_column)
|
256
|
+
result << column_definition(table.created_timestamp_column)
|
257
|
+
result << column_definition(table.updated_timestamp_column)
|
215
258
|
table.data_columns.each do |column|
|
216
|
-
|
259
|
+
result << column_definition(column)
|
217
260
|
end
|
218
261
|
end
|
219
262
|
end
|
@@ -222,28 +265,7 @@ module Purview
|
|
222
265
|
raise %{All "#{Base}(s)" must override the "connection_type" method}
|
223
266
|
end
|
224
267
|
|
225
|
-
def
|
226
|
-
table_opts = extract_table_options(opts)
|
227
|
-
table_name = table_name(table, table_opts)
|
228
|
-
index_opts = extract_index_options(opts)
|
229
|
-
index_name(
|
230
|
-
table_name,
|
231
|
-
columns,
|
232
|
-
index_opts
|
233
|
-
).tap do |index_name|
|
234
|
-
connection.execute(
|
235
|
-
create_index_sql(
|
236
|
-
table_name,
|
237
|
-
index_name,
|
238
|
-
table,
|
239
|
-
columns,
|
240
|
-
index_opts
|
241
|
-
)
|
242
|
-
)
|
243
|
-
end
|
244
|
-
end
|
245
|
-
|
246
|
-
def create_index_sql(table_name, index_name, table, columns, index_opts={})
|
268
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
247
269
|
raise %{All "#{Base}(s)" must override the "create_index_sql" method}
|
248
270
|
end
|
249
271
|
|
@@ -251,6 +273,10 @@ module Purview
|
|
251
273
|
raise %{All "#{Base}(s)" must override the "create_table_sql" method}
|
252
274
|
end
|
253
275
|
|
276
|
+
def create_table_sql_method_name(table, table_opts={})
|
277
|
+
"create#{table_opts[:temporary] && '_temporary'}_table_sql".to_sym
|
278
|
+
end
|
279
|
+
|
254
280
|
def create_temporary_table_sql(table_name, table, table_opts={})
|
255
281
|
raise %{All "#{Base}(s)" must override the "create_temporary_table_sql" method}
|
256
282
|
end
|
@@ -295,28 +321,7 @@ module Purview
|
|
295
321
|
raise %{All "#{Base}(s)" must override the "disable_table_sql" method}
|
296
322
|
end
|
297
323
|
|
298
|
-
def
|
299
|
-
table_opts = extract_table_options(opts)
|
300
|
-
table_name = table_name(table, table_opts)
|
301
|
-
index_opts = extract_index_options(opts)
|
302
|
-
index_name(
|
303
|
-
table_name,
|
304
|
-
columns,
|
305
|
-
index_opts
|
306
|
-
).tap do |index_name|
|
307
|
-
connection.execute(
|
308
|
-
drop_index_sql(
|
309
|
-
table_name,
|
310
|
-
index_name,
|
311
|
-
table,
|
312
|
-
columns,
|
313
|
-
index_opts
|
314
|
-
)
|
315
|
-
)
|
316
|
-
end
|
317
|
-
end
|
318
|
-
|
319
|
-
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
324
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
320
325
|
raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
|
321
326
|
end
|
322
327
|
|
@@ -328,6 +333,12 @@ module Purview
|
|
328
333
|
raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
|
329
334
|
end
|
330
335
|
|
336
|
+
def ensure_index_valid_for_database(index)
|
337
|
+
raise ArgumentError.new('Must provide an `Index`') \
|
338
|
+
unless index
|
339
|
+
ensure_table_valid_for_database(index.table)
|
340
|
+
end
|
341
|
+
|
331
342
|
def ensure_table_metadata_absent_for_table(table)
|
332
343
|
with_new_connection do |connection|
|
333
344
|
connection.execute(ensure_table_metadata_table_exists_sql)
|
@@ -354,6 +365,13 @@ module Purview
|
|
354
365
|
raise %{All "#{Base}(s)" must override the "ensure_table_metadata_table_exists_sql" method}
|
355
366
|
end
|
356
367
|
|
368
|
+
def ensure_table_valid_for_database(table)
|
369
|
+
raise ArgumentError.new('Must provide a `Table`') \
|
370
|
+
unless table
|
371
|
+
raise Purview::Exceptions::WrongDatabase.new(table) \
|
372
|
+
unless tables.include?(table)
|
373
|
+
end
|
374
|
+
|
357
375
|
def extract_index_options(opts)
|
358
376
|
opts[:index] || {}
|
359
377
|
end
|
@@ -516,24 +534,52 @@ module Purview
|
|
516
534
|
def sync_table_without_lock(table, timestamp)
|
517
535
|
last_window = nil
|
518
536
|
with_next_window(table, timestamp) do |window|
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
537
|
+
with_new_connection do |connection|
|
538
|
+
with_transaction(connection) do
|
539
|
+
table.sync(connection, window)
|
540
|
+
set_last_pulled_at_for_table(
|
541
|
+
connection,
|
542
|
+
table,
|
543
|
+
timestamp
|
544
|
+
)
|
545
|
+
set_max_timestamp_pulled_for_table(
|
546
|
+
connection,
|
547
|
+
table,
|
548
|
+
window.max
|
549
|
+
)
|
550
|
+
last_window = window
|
551
|
+
end
|
532
552
|
end
|
533
553
|
end
|
534
554
|
last_window
|
535
555
|
end
|
536
556
|
|
557
|
+
def table_disabled?(table)
|
558
|
+
!table_enabled?(table)
|
559
|
+
end
|
560
|
+
|
561
|
+
def table_enabled?(table)
|
562
|
+
with_new_connection do |connection|
|
563
|
+
!!get_enabled_at_for_table(connection, table)
|
564
|
+
end
|
565
|
+
end
|
566
|
+
|
567
|
+
def table_initialized?(table)
|
568
|
+
with_new_connection do |connection|
|
569
|
+
!!get_max_timestamp_pulled_for_table(connection, table)
|
570
|
+
end
|
571
|
+
end
|
572
|
+
|
573
|
+
def table_locked?(table)
|
574
|
+
with_new_connection do |connection|
|
575
|
+
!!get_locked_at_for_table(connection, table)
|
576
|
+
end
|
577
|
+
end
|
578
|
+
|
579
|
+
def table_unlocked?(table)
|
580
|
+
!table_locked?(table)
|
581
|
+
end
|
582
|
+
|
537
583
|
def table_metadata_enabled_at_column_definition
|
538
584
|
column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
|
539
585
|
column_definition(column)
|
@@ -587,14 +633,6 @@ module Purview
|
|
587
633
|
table_opts[:name] || table.name
|
588
634
|
end
|
589
635
|
|
590
|
-
def tables
|
591
|
-
@tables ||= Set.new.tap do |result|
|
592
|
-
(opts[:tables] || []).each do |table|
|
593
|
-
table.database = self if result.add?(table)
|
594
|
-
end
|
595
|
-
end
|
596
|
-
end
|
597
|
-
|
598
636
|
def tables_by_name
|
599
637
|
@tables_by_name ||= {}.tap do |result|
|
600
638
|
tables.each do |table|
|
@@ -7,11 +7,12 @@ module Purview
|
|
7
7
|
Purview::Connections::MySQL
|
8
8
|
end
|
9
9
|
|
10
|
-
def create_index_sql(table_name, index_name,
|
11
|
-
'CREATE
|
10
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
11
|
+
'CREATE%sINDEX %s ON %s (%s)' % [
|
12
|
+
index.unique? ? ' UNIQUE ' : ' ',
|
12
13
|
index_name,
|
13
14
|
table_name,
|
14
|
-
column_names(columns).join(', '),
|
15
|
+
column_names(index.columns).join(', '),
|
15
16
|
]
|
16
17
|
end
|
17
18
|
|
@@ -48,7 +49,7 @@ module Purview
|
|
48
49
|
]
|
49
50
|
end
|
50
51
|
|
51
|
-
def drop_index_sql(table_name, index_name,
|
52
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
52
53
|
'DROP INDEX %s' % [
|
53
54
|
index_name,
|
54
55
|
]
|
@@ -7,11 +7,12 @@ module Purview
|
|
7
7
|
Purview::Connections::PostgreSQL
|
8
8
|
end
|
9
9
|
|
10
|
-
def create_index_sql(table_name, index_name,
|
11
|
-
'CREATE
|
10
|
+
def create_index_sql(table_name, index_name, index, index_opts={})
|
11
|
+
'CREATE%sINDEX %s ON %s (%s)' % [
|
12
|
+
index.unique? ? ' UNIQUE ' : ' ',
|
12
13
|
index_name,
|
13
14
|
table_name,
|
14
|
-
column_names(columns).join(', '),
|
15
|
+
column_names(index.columns).join(', '),
|
15
16
|
]
|
16
17
|
end
|
17
18
|
|
@@ -44,7 +45,7 @@ module Purview
|
|
44
45
|
]
|
45
46
|
end
|
46
47
|
|
47
|
-
def drop_index_sql(table_name, index_name,
|
48
|
+
def drop_index_sql(table_name, index_name, index, index_opts={})
|
48
49
|
'DROP INDEX %s' % [
|
49
50
|
index_name,
|
50
51
|
]
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Purview
|
2
|
+
module Exceptions
|
3
|
+
class TableAlreadyAssigned < Base
|
4
|
+
def initialize(index)
|
5
|
+
@index = index
|
6
|
+
end
|
7
|
+
|
8
|
+
def message
|
9
|
+
"Table already assigned for index on columns: #{index.table.column_names.join(', ')}"
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
attr_reader :index
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/purview/exceptions.rb
CHANGED
@@ -1,14 +1,17 @@
|
|
1
1
|
require 'purview/exceptions/base'
|
2
2
|
require 'purview/exceptions/base_table'
|
3
3
|
|
4
|
+
require 'purview/exceptions/could_not_baseline'
|
4
5
|
require 'purview/exceptions/could_not_disable'
|
5
6
|
require 'purview/exceptions/could_not_enable'
|
6
7
|
require 'purview/exceptions/could_not_initialize'
|
7
8
|
require 'purview/exceptions/could_not_lock'
|
9
|
+
require 'purview/exceptions/could_not_sync'
|
8
10
|
require 'purview/exceptions/could_not_unlock'
|
9
11
|
|
10
12
|
require 'purview/exceptions/database_already_assigned'
|
11
13
|
require 'purview/exceptions/no_table'
|
12
14
|
require 'purview/exceptions/no_window'
|
13
15
|
require 'purview/exceptions/rows_outside_window'
|
16
|
+
require 'purview/exceptions/table_already_assigned'
|
14
17
|
require 'purview/exceptions/wrong_database'
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Purview
|
2
|
+
module Indices
|
3
|
+
class Base
|
4
|
+
attr_reader :columns, :table
|
5
|
+
|
6
|
+
def initialize(columns, opts={})
|
7
|
+
@columns = columns
|
8
|
+
@opts = opts
|
9
|
+
end
|
10
|
+
|
11
|
+
def eql?(other)
|
12
|
+
self.class == other.class &&
|
13
|
+
columns == other.columns &&
|
14
|
+
unique == other.unique
|
15
|
+
end
|
16
|
+
|
17
|
+
def hash
|
18
|
+
columns.hash + unique.hash
|
19
|
+
end
|
20
|
+
|
21
|
+
def table=(value)
|
22
|
+
raise Purview::Exceptions::TableAlreadyAssigned.new(self) if table
|
23
|
+
@table = value
|
24
|
+
end
|
25
|
+
|
26
|
+
def unique
|
27
|
+
opts[:unique]
|
28
|
+
end
|
29
|
+
|
30
|
+
def unique?
|
31
|
+
!!unique
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
attr_reader :opts
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/purview/loaders/base.rb
CHANGED
@@ -38,10 +38,10 @@ module Purview
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def create_temporary_table(connection)
|
41
|
-
database.
|
42
|
-
connection,
|
41
|
+
database.create_table(
|
43
42
|
table,
|
44
|
-
:
|
43
|
+
:connection => connection,
|
44
|
+
:table => temporary_table_opts,
|
45
45
|
)
|
46
46
|
end
|
47
47
|
|
@@ -130,6 +130,7 @@ module Purview
|
|
130
130
|
{
|
131
131
|
:create_indices => true,
|
132
132
|
:name => table.temporary_name,
|
133
|
+
:temporary => true,
|
133
134
|
}
|
134
135
|
end
|
135
136
|
|
@@ -2,7 +2,7 @@ module Purview
|
|
2
2
|
module Mixins
|
3
3
|
module Connection
|
4
4
|
def connect
|
5
|
-
connection_type.
|
5
|
+
connection_type.connect(connection_opts)
|
6
6
|
end
|
7
7
|
|
8
8
|
def connection_opts
|
@@ -16,14 +16,22 @@ module Purview
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def with_new_connection
|
19
|
-
|
19
|
+
yield connection = connect
|
20
|
+
ensure
|
21
|
+
connection.disconnect if connection
|
20
22
|
end
|
21
23
|
|
22
|
-
def
|
23
|
-
|
24
|
-
|
24
|
+
def with_new_or_existing_connection(opts={})
|
25
|
+
if existing_connection = opts[:connection]
|
26
|
+
yield existing_connection
|
27
|
+
else
|
28
|
+
with_new_connection { |connection| yield connection }
|
25
29
|
end
|
26
30
|
end
|
31
|
+
|
32
|
+
def with_transaction(connection)
|
33
|
+
connection.with_transaction { yield }
|
34
|
+
end
|
27
35
|
end
|
28
36
|
end
|
29
37
|
end
|
data/lib/purview/parsers/csv.rb
CHANGED
@@ -3,10 +3,10 @@ module Purview
|
|
3
3
|
class CSV < Base
|
4
4
|
def parse(data)
|
5
5
|
with_context_logging("`parse` for: #{table.name}") do
|
6
|
-
[].tap do |
|
6
|
+
[].tap do |result|
|
7
7
|
headers = extract_headers(data)
|
8
8
|
extract_rows(data).each do |row|
|
9
|
-
|
9
|
+
result << build_result(headers.zip(row))
|
10
10
|
end
|
11
11
|
end
|
12
12
|
end
|
data/lib/purview/tables/base.rb
CHANGED
@@ -1,11 +1,16 @@
|
|
1
1
|
module Purview
|
2
2
|
module Tables
|
3
3
|
class Base
|
4
|
-
attr_reader :database, :name
|
4
|
+
attr_reader :database, :indices, :name
|
5
5
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
8
8
|
@opts = opts
|
9
|
+
@indices = Set.new.tap do |result|
|
10
|
+
((opts[:indices] || []) + default_indices).each do |index|
|
11
|
+
index.table = self if result.add?(index)
|
12
|
+
end
|
13
|
+
end
|
9
14
|
end
|
10
15
|
|
11
16
|
def columns
|
@@ -32,6 +37,10 @@ module Purview
|
|
32
37
|
columns_of_type(Purview::Columns::CreatedTimestamp).first
|
33
38
|
end
|
34
39
|
|
40
|
+
def created_timestamp_index
|
41
|
+
Purview::Indices::Simple.new(created_timestamp_column)
|
42
|
+
end
|
43
|
+
|
35
44
|
def data_columns
|
36
45
|
columns - [
|
37
46
|
created_timestamp_column,
|
@@ -49,13 +58,6 @@ module Purview
|
|
49
58
|
columns_of_type(Purview::Columns::Id).first
|
50
59
|
end
|
51
60
|
|
52
|
-
def indexed_columns
|
53
|
-
(opts[:indexed_columns] || []).tap do |indexed_columns|
|
54
|
-
indexed_columns << [created_timestamp_column]
|
55
|
-
indexed_columns << [updated_timestamp_column]
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
61
|
def sync(connection, window)
|
60
62
|
raw_data = puller.pull(window)
|
61
63
|
parser.validate(raw_data)
|
@@ -75,6 +77,10 @@ module Purview
|
|
75
77
|
columns_of_type(Purview::Columns::UpdatedTimestamp).first
|
76
78
|
end
|
77
79
|
|
80
|
+
def updated_timestamp_index
|
81
|
+
Purview::Indices::Simple.new(updated_timestamp_column)
|
82
|
+
end
|
83
|
+
|
78
84
|
def window_size
|
79
85
|
opts[:window_size] || (60 * 60)
|
80
86
|
end
|
@@ -86,6 +92,13 @@ module Purview
|
|
86
92
|
|
87
93
|
attr_reader :opts
|
88
94
|
|
95
|
+
def default_indices
|
96
|
+
[
|
97
|
+
created_timestamp_index,
|
98
|
+
updated_timestamp_index,
|
99
|
+
]
|
100
|
+
end
|
101
|
+
|
89
102
|
def extract_type_option(opts)
|
90
103
|
opts[:type]
|
91
104
|
end
|
data/lib/purview/version.rb
CHANGED
data/lib/purview.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: purview
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan W. Zaleski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-06-
|
11
|
+
date: 2015-06-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -112,16 +112,23 @@ files:
|
|
112
112
|
- lib/purview/exceptions.rb
|
113
113
|
- lib/purview/exceptions/base.rb
|
114
114
|
- lib/purview/exceptions/base_table.rb
|
115
|
+
- lib/purview/exceptions/could_not_baseline.rb
|
115
116
|
- lib/purview/exceptions/could_not_disable.rb
|
116
117
|
- lib/purview/exceptions/could_not_enable.rb
|
117
118
|
- lib/purview/exceptions/could_not_initialize.rb
|
118
119
|
- lib/purview/exceptions/could_not_lock.rb
|
120
|
+
- lib/purview/exceptions/could_not_sync.rb
|
119
121
|
- lib/purview/exceptions/could_not_unlock.rb
|
120
122
|
- lib/purview/exceptions/database_already_assigned.rb
|
121
123
|
- lib/purview/exceptions/no_table.rb
|
122
124
|
- lib/purview/exceptions/no_window.rb
|
123
125
|
- lib/purview/exceptions/rows_outside_window.rb
|
126
|
+
- lib/purview/exceptions/table_already_assigned.rb
|
124
127
|
- lib/purview/exceptions/wrong_database.rb
|
128
|
+
- lib/purview/indices.rb
|
129
|
+
- lib/purview/indices/base.rb
|
130
|
+
- lib/purview/indices/composite.rb
|
131
|
+
- lib/purview/indices/simple.rb
|
125
132
|
- lib/purview/loaders.rb
|
126
133
|
- lib/purview/loaders/base.rb
|
127
134
|
- lib/purview/loaders/mysql.rb
|