purview 1.1.1 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +7 -0
- data/lib/purview/columns/base.rb +2 -3
- data/lib/purview/databases/base.rb +77 -69
- data/lib/purview/databases/mysql.rb +2 -2
- data/lib/purview/databases/postgresql.rb +2 -2
- data/lib/purview/mixins/connection.rb +6 -0
- data/lib/purview/mixins/helpers.rb +8 -0
- data/lib/purview/raw_connections/jdbc/base.rb +10 -9
- data/lib/purview/tables/base.rb +2 -1
- data/lib/purview/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e6b41c5fd1454de8cf72663944dddcb35b3a2ff3
|
4
|
+
data.tar.gz: 196b5bf64e43ff4788ea413ef2eb70776a766cf7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e87b88ea28caeccaa996255a08683fcb4d5fb73d24cf50a9a79fcb0646ad66e8594a4039b3752b53f2e4b2a9d9f0c4ced8755070927f6876814883185feee35
|
7
|
+
data.tar.gz: 4beefdf566cba05e3e3153601ae6170e41846b6a5f4c8a816ea49f6b01c82501883fe389e9412e1698cded2b35302f976b3cfd823e85588af2e4cbaae08cfadc
|
data/CHANGELOG
CHANGED
data/lib/purview/columns/base.rb
CHANGED
@@ -34,9 +34,8 @@ module Purview
|
|
34
34
|
|
35
35
|
def parse(value)
|
36
36
|
blank = blank?(value)
|
37
|
-
|
38
|
-
|
39
|
-
type.parse(value)
|
37
|
+
raise %{Unexpected blank value for column: "#{name}"} if blank && !nullable?
|
38
|
+
blank ? nil : type.parse(value)
|
40
39
|
end
|
41
40
|
|
42
41
|
def primary_key
|
@@ -9,26 +9,16 @@ module Purview
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def baseline_table(table)
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
raise Purview::Exceptions::WrongDatabase.new(table) \
|
13
|
+
unless tables.include?(table)
|
14
|
+
table_name = table_name(table)
|
15
|
+
with_context_logging("`baseline_table` for: #{table_name}") do
|
16
|
+
starting_timestamp = timestamp
|
16
17
|
with_table_locked(table, starting_timestamp) do
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
with_next_window(
|
22
|
-
connection,
|
23
|
-
table,
|
24
|
-
transaction_timestamp
|
25
|
-
) do |window|
|
26
|
-
table.sync(connection, window)
|
27
|
-
last_window = window
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end while last_window.max < starting_timestamp
|
18
|
+
loop do
|
19
|
+
last_window = sync_table_without_lock(table, timestamp)
|
20
|
+
break if last_window.max > starting_timestamp
|
21
|
+
end
|
32
22
|
end
|
33
23
|
end
|
34
24
|
end
|
@@ -117,7 +107,7 @@ module Purview
|
|
117
107
|
end
|
118
108
|
end
|
119
109
|
|
120
|
-
def enable_table(table, timestamp=
|
110
|
+
def enable_table(table, timestamp=timestamp)
|
121
111
|
table_name = table_name(table)
|
122
112
|
with_context_logging("`enable_table` for: #{table_name}") do
|
123
113
|
with_new_connection do |connection|
|
@@ -130,12 +120,12 @@ module Purview
|
|
130
120
|
end
|
131
121
|
end
|
132
122
|
|
133
|
-
def initialize_table(table,
|
123
|
+
def initialize_table(table, timestamp=timestamp)
|
134
124
|
table_name = table_name(table)
|
135
125
|
with_context_logging("`initialize_table` for: #{table_name}") do
|
136
126
|
with_new_connection do |connection|
|
137
127
|
rows_affected = \
|
138
|
-
connection.execute(initialize_table_sql(table,
|
128
|
+
connection.execute(initialize_table_sql(table, timestamp)).rows_affected
|
139
129
|
raise Purview::Exceptions::CouldNotInitialize.new(table) \
|
140
130
|
if zero?(rows_affected)
|
141
131
|
end
|
@@ -143,7 +133,7 @@ module Purview
|
|
143
133
|
end
|
144
134
|
end
|
145
135
|
|
146
|
-
def lock_table(table, timestamp=
|
136
|
+
def lock_table(table, timestamp=timestamp)
|
147
137
|
table_name = table_name(table)
|
148
138
|
with_context_logging("`lock_table` for: #{table_name}") do
|
149
139
|
with_new_connection do |connection|
|
@@ -158,24 +148,23 @@ module Purview
|
|
158
148
|
|
159
149
|
def sync
|
160
150
|
with_context_logging('`sync`') do
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
with_next_window(
|
165
|
-
connection,
|
166
|
-
table,
|
167
|
-
transaction_timestamp
|
168
|
-
) do |window|
|
169
|
-
with_table_locked(table, transaction_timestamp) do
|
170
|
-
table.sync(connection, window)
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|
151
|
+
with_timestamp do |timestamp|
|
152
|
+
with_next_table(timestamp) do |table|
|
153
|
+
sync_table_with_lock(table, timestamp)
|
174
154
|
end
|
175
155
|
end
|
176
156
|
end
|
177
157
|
end
|
178
158
|
|
159
|
+
def sync_table(table)
|
160
|
+
raise Purview::Exceptions::WrongDatabase.new(table) \
|
161
|
+
unless tables.include?(table)
|
162
|
+
table_name = table_name(table)
|
163
|
+
with_context_logging("`sync_table` for: #{table_name}") do
|
164
|
+
sync_table_with_lock(table, timestamp)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
179
168
|
def unlock_table(table)
|
180
169
|
table_name = table_name(table)
|
181
170
|
with_context_logging("`unlock_table` for: #{table_name}") do
|
@@ -424,13 +413,12 @@ module Purview
|
|
424
413
|
]
|
425
414
|
end
|
426
415
|
|
427
|
-
def initialize_table_sql(table,
|
416
|
+
def initialize_table_sql(table, timestamp)
|
428
417
|
raise %{All "#{Base}(s)" must override the "initialize_table_sql" method}
|
429
418
|
end
|
430
419
|
|
431
420
|
def limit(column)
|
432
|
-
|
433
|
-
column.limit || limit_map[column.type]
|
421
|
+
limitless_types.include?(column.type) ? nil : (column.limit || limit_map[column.type])
|
434
422
|
end
|
435
423
|
|
436
424
|
def limit_map
|
@@ -448,7 +436,7 @@ module Purview
|
|
448
436
|
def next_table(connection, timestamp)
|
449
437
|
row = connection.execute(next_table_sql(timestamp)).rows[0]
|
450
438
|
table_name = row && row[table_metadata_table_name_column_name]
|
451
|
-
table_name
|
439
|
+
table_name && tables_by_name[table_name]
|
452
440
|
end
|
453
441
|
|
454
442
|
def next_table_sql(timestamp)
|
@@ -459,9 +447,10 @@ module Purview
|
|
459
447
|
min = get_max_timestamp_pulled_for_table(connection, table)
|
460
448
|
max = min + table.window_size
|
461
449
|
now = timestamp
|
462
|
-
|
463
|
-
|
464
|
-
|
450
|
+
min > now ? nil : Purview::Structs::Window.new(
|
451
|
+
:min => min,
|
452
|
+
:max => max > now ? now : max
|
453
|
+
)
|
465
454
|
end
|
466
455
|
|
467
456
|
def null_value
|
@@ -516,6 +505,35 @@ module Purview
|
|
516
505
|
raise %{All "#{Base}(s)" must override the "set_max_timestamp_pulled_for_table_sql" method}
|
517
506
|
end
|
518
507
|
|
508
|
+
def sync_table_with_lock(table, timestamp)
|
509
|
+
last_window = nil
|
510
|
+
with_table_locked(table, timestamp) do
|
511
|
+
last_window = sync_table_without_lock(table, timestamp)
|
512
|
+
end
|
513
|
+
last_window
|
514
|
+
end
|
515
|
+
|
516
|
+
def sync_table_without_lock(table, timestamp)
|
517
|
+
last_window = nil
|
518
|
+
with_next_window(table, timestamp) do |window|
|
519
|
+
with_new_transaction do |connection|
|
520
|
+
table.sync(connection, window)
|
521
|
+
set_last_pulled_at_for_table(
|
522
|
+
connection,
|
523
|
+
table,
|
524
|
+
timestamp
|
525
|
+
)
|
526
|
+
set_max_timestamp_pulled_for_table(
|
527
|
+
connection,
|
528
|
+
table,
|
529
|
+
window.max
|
530
|
+
)
|
531
|
+
last_window = window
|
532
|
+
end
|
533
|
+
end
|
534
|
+
last_window
|
535
|
+
end
|
536
|
+
|
519
537
|
def table_metadata_enabled_at_column_definition
|
520
538
|
column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
|
521
539
|
column_definition(column)
|
@@ -610,30 +628,24 @@ module Purview
|
|
610
628
|
raise %{All "#{Base}(s)" must override the "unlock_table_sql" method}
|
611
629
|
end
|
612
630
|
|
613
|
-
def with_next_table(
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
table,
|
620
|
-
timestamp
|
621
|
-
)
|
631
|
+
def with_next_table(timestamp)
|
632
|
+
with_new_connection do |connection|
|
633
|
+
table = next_table(connection, timestamp)
|
634
|
+
raise Purview::Exceptions::NoTable.new unless table
|
635
|
+
yield table
|
636
|
+
end
|
622
637
|
end
|
623
638
|
|
624
|
-
def with_next_window(
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
table,
|
635
|
-
window.max
|
636
|
-
)
|
639
|
+
def with_next_window(table, timestamp)
|
640
|
+
with_new_connection do |connection|
|
641
|
+
window = next_window(
|
642
|
+
connection,
|
643
|
+
table,
|
644
|
+
timestamp
|
645
|
+
)
|
646
|
+
raise Purview::Exceptions::NoWindow.new(table) unless window
|
647
|
+
yield window
|
648
|
+
end
|
637
649
|
end
|
638
650
|
|
639
651
|
def with_table_locked(table, timestamp)
|
@@ -642,10 +654,6 @@ module Purview
|
|
642
654
|
ensure
|
643
655
|
unlock_table(table)
|
644
656
|
end
|
645
|
-
|
646
|
-
def with_transaction(connection)
|
647
|
-
connection.with_transaction { yield Time.now.utc }
|
648
|
-
end
|
649
657
|
end
|
650
658
|
end
|
651
659
|
end
|
@@ -97,11 +97,11 @@ module Purview
|
|
97
97
|
]
|
98
98
|
end
|
99
99
|
|
100
|
-
def initialize_table_sql(table,
|
100
|
+
def initialize_table_sql(table, timestamp)
|
101
101
|
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
102
102
|
table_metadata_table_name,
|
103
103
|
table_metadata_max_timestamp_pulled_column_name,
|
104
|
-
quoted(
|
104
|
+
quoted(timestamp),
|
105
105
|
table_metadata_table_name_column_name,
|
106
106
|
quoted(table.name),
|
107
107
|
table_metadata_max_timestamp_pulled_column_name,
|
@@ -97,11 +97,11 @@ module Purview
|
|
97
97
|
]
|
98
98
|
end
|
99
99
|
|
100
|
-
def initialize_table_sql(table,
|
100
|
+
def initialize_table_sql(table, timestamp)
|
101
101
|
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
102
102
|
table_metadata_table_name,
|
103
103
|
table_metadata_max_timestamp_pulled_column_name,
|
104
|
-
quoted(
|
104
|
+
quoted(timestamp),
|
105
105
|
table_metadata_table_name_column_name,
|
106
106
|
quoted(table.name),
|
107
107
|
table_metadata_max_timestamp_pulled_column_name,
|
@@ -18,6 +18,12 @@ module Purview
|
|
18
18
|
def with_new_connection
|
19
19
|
connection_type.with_new_connection(connection_opts) { |connection| yield connection }
|
20
20
|
end
|
21
|
+
|
22
|
+
def with_new_transaction
|
23
|
+
with_new_connection do |connection|
|
24
|
+
connection.with_transaction { yield connection }
|
25
|
+
end
|
26
|
+
end
|
21
27
|
end
|
22
28
|
end
|
23
29
|
end
|
@@ -29,15 +29,16 @@ module Purview
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def extract_rows(result)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
32
|
+
if result
|
33
|
+
metadata = result.getMetaData
|
34
|
+
column_count = metadata.getColumnCount
|
35
|
+
[].tap do |rows|
|
36
|
+
while result.next
|
37
|
+
rows << {}.tap do |row|
|
38
|
+
(1..column_count).each do |index|
|
39
|
+
column_name = metadata.getColumnName(index)
|
40
|
+
row[column_name] = result.getString(column_name)
|
41
|
+
end
|
41
42
|
end
|
42
43
|
end
|
43
44
|
end
|
data/lib/purview/tables/base.rb
CHANGED
@@ -68,7 +68,7 @@ module Purview
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def temporary_name
|
71
|
-
"#{name}_#{
|
71
|
+
"#{name}_#{timestamp.to_i}"
|
72
72
|
end
|
73
73
|
|
74
74
|
def updated_timestamp_column
|
@@ -81,6 +81,7 @@ module Purview
|
|
81
81
|
|
82
82
|
private
|
83
83
|
|
84
|
+
include Purview::Mixins::Helpers
|
84
85
|
include Purview::Mixins::Logger
|
85
86
|
|
86
87
|
attr_reader :opts
|
data/lib/purview/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: purview
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan W. Zaleski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-06-
|
11
|
+
date: 2015-06-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|