purview 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +6 -4
- data/TODO +12 -55
- data/lib/purview/databases/base.rb +2 -2
- data/lib/purview/databases/mssql.rb +18 -6
- data/lib/purview/databases/mysql.rb +19 -7
- data/lib/purview/databases/postgresql.rb +18 -6
- data/lib/purview/databases/sqlite.rb +18 -6
- data/lib/purview/mixins/helpers.rb +1 -1
- data/lib/purview/pullers/base_sql.rb +1 -1
- data/lib/purview/tables/table_metadata.rb +5 -0
- data/lib/purview/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 10737dc7946dc6ed6233299ef4cc18042ed7ab1f5175478ae4a954f8e4c611a0
|
|
4
|
+
data.tar.gz: 5330e8e05814176ec27c623f5d98ec5af6186df0f2b8f4a8c702774874043fd4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7444fe8eafec51555096fe3c2f03551bf98d135173610c7c48161927f12c4229db29a2b762ad63e4e81f06cbf10263793c1cf025d83452812147dd8cd0eb0500
|
|
7
|
+
data.tar.gz: ea70373933da78d6e634baf8942765bb82597558561f2397e723e08ca9e0a7068dbfaf59c876d51ba56f39116a974af1f61d37a8bfb291f8a1c34c7bc204efe1
|
data/.gitignore
CHANGED
|
@@ -13,23 +13,25 @@
|
|
|
13
13
|
tags
|
|
14
14
|
tags.*
|
|
15
15
|
|
|
16
|
-
## Specific to RubyMotion
|
|
16
|
+
## Specific to RubyMotion
|
|
17
17
|
.dat*
|
|
18
18
|
.repl_history
|
|
19
19
|
build/
|
|
20
20
|
|
|
21
|
-
## Documentation cache and generated files
|
|
21
|
+
## Documentation cache and generated files
|
|
22
22
|
/.yardoc/
|
|
23
23
|
/_yardoc/
|
|
24
24
|
/doc/
|
|
25
25
|
/rdoc/
|
|
26
26
|
|
|
27
|
-
## Environment normalisation
|
|
27
|
+
## Environment normalisation
|
|
28
28
|
/.bundle/
|
|
29
29
|
/lib/bundler/man/
|
|
30
30
|
|
|
31
|
-
## Ruby artifacts
|
|
31
|
+
## Ruby artifacts
|
|
32
32
|
.ruby-gemset
|
|
33
33
|
.ruby-version
|
|
34
34
|
.rvmrc
|
|
35
35
|
Gemfile.lock
|
|
36
|
+
|
|
37
|
+
test.rb
|
data/TODO
CHANGED
|
@@ -1,55 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
d 1494850714 1495046648 Finish SQLite3 raw-connections
|
|
14
|
-
d 1495046665 1495046689 Fix JDBC driver configuration
|
|
15
|
-
d 1495051831 1495070248 Code-review recent changes and merge
|
|
16
|
-
d 1450000000 1450000000 Handle blank values a bit more intelligently (all blanks -> nil)
|
|
17
|
-
d 1450000000 1450000000 Deal w/ NULL values
|
|
18
|
-
d 1450000000 1450000000 Deal w/ BOOL values
|
|
19
|
-
d 1450000000 1450000000 Build out MVP for PostgreSQL loader
|
|
20
|
-
d 1450000000 1450000000 Add windowing to pulls (simulate w/ 2 different data-files if necessary)
|
|
21
|
-
d 1450000000 1450000000 Include [window] timestamps in query-string for pull[er]
|
|
22
|
-
d 1450000000 1450000000 Figure out an intelligent way to solve the chicken <-> egg problem for `table_metadata`
|
|
23
|
-
d 1450000000 1450000000 Fail fast if there are no candidate tables (`next_table`)
|
|
24
|
-
d 1450000000 1450000000 Ensure that the puller does not try to go into the future (`next_window`)
|
|
25
|
-
d 1450000000 1450000000 Add in a primitive logging facility (can be temporary)
|
|
26
|
-
d 1450000000 1450000000 Enhance `next_table_sql`
|
|
27
|
-
d 1450000000 1450000000 Lock tables (feeds) during pull, parse and load
|
|
28
|
-
d 1450000000 1450000000 Verify table-locking mechanism behaves as expected
|
|
29
|
-
d 1450000000 1450000000 Ability to drop a table
|
|
30
|
-
d 1450000000 1450000000 Reorder args: "table, table_name" to: "table_name, table"
|
|
31
|
-
d 1450000000 1450000000 Reorder args: "table, index_name, *columns" to "index_name, table, *columns"
|
|
32
|
-
d 1450000000 1450000000 Ability to create an index
|
|
33
|
-
d 1450000000 1450000000 Ability to drop an index
|
|
34
|
-
d 1450000000 1450000000 Indices on `created_at` and `updated_at` columns
|
|
35
|
-
d 1450000000 1450000000 Gracefully handle table {,un}locking in case of error
|
|
36
|
-
d 1450000000 1450000000 Add debugging code
|
|
37
|
-
d 1450000000 1450000000 Raise no-window & no-table exceptions (making them catchable upstream)
|
|
38
|
-
d 1450000000 1450000000 Minimize race-condition around table-locking (use where on UPDATE, raise if zero rows are updated)
|
|
39
|
-
d 1450000000 1450000000 Log number of delete, inserts and updates in `loader`
|
|
40
|
-
d 1450000000 1450000000 `load_temporary_table` should bomb if there is data outside the window
|
|
41
|
-
d 1450000000 1450000000 Delete from outside window where id in temporary table
|
|
42
|
-
d 1450000000 1450000000 Rename `data` to `rows` in `execute`
|
|
43
|
-
d 1450000000 1450000000 Rename `Postgres` to `PostgreSQL`
|
|
44
|
-
d 1450000000 1450000000 Finish PostgreSQL support (requires `pg` gem)
|
|
45
|
-
d 1450000000 1450000000 Finish MySQL support (requires `mysql2` gem)
|
|
46
|
-
d 1450000000 1450000000 Usage/configuration examples in README.md
|
|
47
|
-
d 1450000000 1450000000 Create SQL-puller(s)
|
|
48
|
-
d 1450000000 1450000000 Introduce [SQL] dialect(s)
|
|
49
|
-
d 1450000000 1450000000 Fix JRuby bundle/build
|
|
50
|
-
d 1450000000 1450000000 Fall back to logged in user if no [database-]username is provided
|
|
51
|
-
d 1450000000 1450000000 Create class to encapulate `table_metadata` (in order to clean up the logic in the `Database` class(es))
|
|
52
|
-
d 1450000000 1450000000 Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe create a new type of column?
|
|
53
|
-
d 1450000000 1495070826 Add support for SQLite3
|
|
54
|
-
d 1494850696 1495070857 Add TODOs from $HOME\purview-TODO
|
|
55
|
-
d 1495070065 1495072292 Finish MSSQL built-out
|
|
1
|
+
f510dd97-ffd2-11e8-9c7a-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544814215 1544814215 a Add `create` functionality to `Database(s)`
|
|
2
|
+
af4b8882-ffd8-11e8-8ecb-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816675 1544816675 a Add COPY `def copy(sql)` support to PostgreSQL connection
|
|
3
|
+
b30f4757-ffd8-11e8-afb9-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816681 1544816681 a Leverage COPY support for PostgreSQL pulls
|
|
4
|
+
b9fec1b8-ffd8-11e8-973b-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816693 1544816693 a Build out change-log, canonical, fact and aggregate tables (and related transforms)
|
|
5
|
+
bef458cc-ffd8-11e8-88b6-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816701 1544816701 a Make re-pull window configurable (do this automatically once up to current?)
|
|
6
|
+
c2c85f85-ffd8-11e8-b2c3-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816708 1544816708 a Add schema management capabilities (detect schema-deltas and suggestion modifications)
|
|
7
|
+
c70ab22b-ffd8-11e8-8c9a-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816715 1544816715 a Refactor mutator methods to be globally available (e.g. quoted -> Quote, etc.)
|
|
8
|
+
cc6c3091-ffd8-11e8-b484-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816724 1544816724 a Add parallel pull(s) support
|
|
9
|
+
d1ecf75e-ffd8-11e8-87cf-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816733 1544816733 a Add support for `Consumers`
|
|
10
|
+
683a87f8-ffe3-11e8-8e33-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544821280 1544821280 a Fix SQLite3 (it doesn't appear to work)
|
|
11
|
+
0aae0fab-ffe4-11e8-ae70-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544821553 1544821553 a Fix PostgreSQL, it doesn't appear to work
|
|
12
|
+
c3b21ac5-ffe6-11e8-802b-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544822722 1544822722 a Need to add time-zone support to adjust SQL puller results
|
|
@@ -395,7 +395,7 @@ module Purview
|
|
|
395
395
|
|
|
396
396
|
def get_table_metadata_value(connection, table, column)
|
|
397
397
|
row = connection.execute(get_table_metadata_value_sql(table, column)).rows[0]
|
|
398
|
-
raise CouldNotFindMetadataForTable.new(table) \
|
|
398
|
+
raise Purview::Exceptions::CouldNotFindMetadataForTable.new(table) \
|
|
399
399
|
unless row
|
|
400
400
|
value = row[column.name]
|
|
401
401
|
value && column.type.parse(value)
|
|
@@ -463,7 +463,7 @@ module Purview
|
|
|
463
463
|
def set_table_metadata_value(connection, table, column, value)
|
|
464
464
|
rows_affected = \
|
|
465
465
|
connection.execute(set_table_metadata_value_sql(table, column, value)).rows_affected
|
|
466
|
-
raise CouldNotUpdateMetadataForTable.new(table) \
|
|
466
|
+
raise Purview::Exceptions::CouldNotUpdateMetadataForTable.new(table) \
|
|
467
467
|
if zero?(rows_affected)
|
|
468
468
|
end
|
|
469
469
|
|
|
@@ -35,10 +35,12 @@ module Purview
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def disable_table_sql(table)
|
|
38
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
38
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
39
39
|
table_metadata_table.name,
|
|
40
40
|
table_metadata_table.enabled_at_column.name,
|
|
41
41
|
null_value,
|
|
42
|
+
table_metadata_table.last_updated_at_column.name,
|
|
43
|
+
quoted(Time.now.utc),
|
|
42
44
|
table_metadata_table.table_name_column.name,
|
|
43
45
|
quoted(table.name),
|
|
44
46
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -58,10 +60,12 @@ module Purview
|
|
|
58
60
|
end
|
|
59
61
|
|
|
60
62
|
def enable_table_sql(table, timestamp)
|
|
61
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
63
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
62
64
|
table_metadata_table.name,
|
|
63
65
|
table_metadata_table.enabled_at_column.name,
|
|
64
66
|
quoted(timestamp),
|
|
67
|
+
table_metadata_table.last_updated_at_column.name,
|
|
68
|
+
quoted(Time.now.utc),
|
|
65
69
|
table_metadata_table.table_name_column.name,
|
|
66
70
|
quoted(table.name),
|
|
67
71
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -95,10 +99,12 @@ module Purview
|
|
|
95
99
|
end
|
|
96
100
|
|
|
97
101
|
def initialize_table_sql(table, timestamp)
|
|
98
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
102
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
99
103
|
table_metadata_table.name,
|
|
100
104
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
101
105
|
quoted(timestamp),
|
|
106
|
+
table_metadata_table.last_updated_at_column.name,
|
|
107
|
+
quoted(Time.now.utc),
|
|
102
108
|
table_metadata_table.table_name_column.name,
|
|
103
109
|
quoted(table.name),
|
|
104
110
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
@@ -126,10 +132,12 @@ module Purview
|
|
|
126
132
|
end
|
|
127
133
|
|
|
128
134
|
def lock_table_sql(table, timestamp)
|
|
129
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
135
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
130
136
|
table_metadata_table.name,
|
|
131
137
|
table_metadata_table.locked_at_column.name,
|
|
132
138
|
quoted(timestamp),
|
|
139
|
+
table_metadata_table.last_updated_at_column.name,
|
|
140
|
+
quoted(Time.now.utc),
|
|
133
141
|
table_metadata_table.table_name_column.name,
|
|
134
142
|
quoted(table.name),
|
|
135
143
|
table_metadata_table.locked_at_column.name,
|
|
@@ -149,10 +157,12 @@ module Purview
|
|
|
149
157
|
end
|
|
150
158
|
|
|
151
159
|
def set_table_metadata_value_sql(table, column, value)
|
|
152
|
-
'UPDATE %s SET %s = %s WHERE %s = %s' % [
|
|
160
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
|
|
153
161
|
table_metadata_table.name,
|
|
154
162
|
column.name,
|
|
155
163
|
quoted(value),
|
|
164
|
+
table_metadata_table.last_updated_at_column.name,
|
|
165
|
+
quoted(Time.now.utc),
|
|
156
166
|
table_metadata_table.table_name_column.name,
|
|
157
167
|
quoted(table.name),
|
|
158
168
|
]
|
|
@@ -166,10 +176,12 @@ module Purview
|
|
|
166
176
|
end
|
|
167
177
|
|
|
168
178
|
def unlock_table_sql(table)
|
|
169
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
179
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
170
180
|
table_metadata_table.name,
|
|
171
181
|
table_metadata_table.locked_at_column.name,
|
|
172
182
|
null_value,
|
|
183
|
+
table_metadata_table.last_updated_at_column.name,
|
|
184
|
+
quoted(Time.now.utc),
|
|
173
185
|
table_metadata_table.table_name_column.name,
|
|
174
186
|
quoted(table.name),
|
|
175
187
|
table_metadata_table.locked_at_column.name,
|
|
@@ -39,10 +39,12 @@ module Purview
|
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
def disable_table_sql(table)
|
|
42
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
42
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
43
43
|
table_metadata_table.name,
|
|
44
44
|
table_metadata_table.enabled_at_column.name,
|
|
45
45
|
null_value,
|
|
46
|
+
table_metadata_table.last_updated_at_column.name,
|
|
47
|
+
quoted(Time.now.utc),
|
|
46
48
|
table_metadata_table.table_name_column.name,
|
|
47
49
|
quoted(table.name),
|
|
48
50
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -62,10 +64,12 @@ module Purview
|
|
|
62
64
|
end
|
|
63
65
|
|
|
64
66
|
def enable_table_sql(table, timestamp)
|
|
65
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
67
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
66
68
|
table_metadata_table.name,
|
|
67
69
|
table_metadata_table.enabled_at_column.name,
|
|
68
70
|
quoted(timestamp),
|
|
71
|
+
table_metadata_table.last_updated_at_column.name,
|
|
72
|
+
quoted(Time.now.utc),
|
|
69
73
|
table_metadata_table.table_name_column.name,
|
|
70
74
|
quoted(table.name),
|
|
71
75
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -81,7 +85,7 @@ module Purview
|
|
|
81
85
|
end
|
|
82
86
|
|
|
83
87
|
def ensure_table_metadata_exists_for_table_sql(table)
|
|
84
|
-
'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
|
|
88
|
+
'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL, NULL)' % [
|
|
85
89
|
table_metadata_table.name,
|
|
86
90
|
quoted(table.name),
|
|
87
91
|
]
|
|
@@ -95,10 +99,12 @@ module Purview
|
|
|
95
99
|
end
|
|
96
100
|
|
|
97
101
|
def initialize_table_sql(table, timestamp)
|
|
98
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
102
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
99
103
|
table_metadata_table.name,
|
|
100
104
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
101
105
|
quoted(timestamp),
|
|
106
|
+
table_metadata_table.last_updated_at_column.name,
|
|
107
|
+
quoted(Time.now.utc),
|
|
102
108
|
table_metadata_table.table_name_column.name,
|
|
103
109
|
quoted(table.name),
|
|
104
110
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
@@ -119,10 +125,12 @@ module Purview
|
|
|
119
125
|
end
|
|
120
126
|
|
|
121
127
|
def lock_table_sql(table, timestamp)
|
|
122
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
128
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
123
129
|
table_metadata_table.name,
|
|
124
130
|
table_metadata_table.locked_at_column.name,
|
|
125
131
|
quoted(timestamp),
|
|
132
|
+
table_metadata_table.last_updated_at_column.name,
|
|
133
|
+
quoted(Time.now.utc),
|
|
126
134
|
table_metadata_table.table_name_column.name,
|
|
127
135
|
quoted(table.name),
|
|
128
136
|
table_metadata_table.locked_at_column.name,
|
|
@@ -142,10 +150,12 @@ module Purview
|
|
|
142
150
|
end
|
|
143
151
|
|
|
144
152
|
def set_table_metadata_value_sql(table, column, value)
|
|
145
|
-
'UPDATE %s SET %s = %s WHERE %s = %s' % [
|
|
153
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
|
|
146
154
|
table_metadata_table.name,
|
|
147
155
|
column.name,
|
|
148
156
|
quoted(value),
|
|
157
|
+
table_metadata_table.last_updated_at_column.name,
|
|
158
|
+
quoted(Time.now.utc),
|
|
149
159
|
table_metadata_table.table_name_column.name,
|
|
150
160
|
quoted(table.name),
|
|
151
161
|
]
|
|
@@ -159,10 +169,12 @@ module Purview
|
|
|
159
169
|
end
|
|
160
170
|
|
|
161
171
|
def unlock_table_sql(table)
|
|
162
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
172
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
163
173
|
table_metadata_table.name,
|
|
164
174
|
table_metadata_table.locked_at_column.name,
|
|
165
175
|
null_value,
|
|
176
|
+
table_metadata_table.last_updated_at_column.name,
|
|
177
|
+
quoted(Time.now.utc),
|
|
166
178
|
table_metadata_table.table_name_column.name,
|
|
167
179
|
quoted(table.name),
|
|
168
180
|
table_metadata_table.locked_at_column.name,
|
|
@@ -35,10 +35,12 @@ module Purview
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def disable_table_sql(table)
|
|
38
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
38
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
39
39
|
table_metadata_table.name,
|
|
40
40
|
table_metadata_table.enabled_at_column.name,
|
|
41
41
|
null_value,
|
|
42
|
+
table_metadata_table.last_updated_at_column.name,
|
|
43
|
+
quoted(Time.now.utc),
|
|
42
44
|
table_metadata_table.table_name_column.name,
|
|
43
45
|
quoted(table.name),
|
|
44
46
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -58,10 +60,12 @@ module Purview
|
|
|
58
60
|
end
|
|
59
61
|
|
|
60
62
|
def enable_table_sql(table, timestamp)
|
|
61
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
63
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
62
64
|
table_metadata_table.name,
|
|
63
65
|
table_metadata_table.enabled_at_column.name,
|
|
64
66
|
quoted(timestamp),
|
|
67
|
+
table_metadata_table.last_updated_at_column.name,
|
|
68
|
+
quoted(Time.now.utc),
|
|
65
69
|
table_metadata_table.table_name_column.name,
|
|
66
70
|
quoted(table.name),
|
|
67
71
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -95,10 +99,12 @@ module Purview
|
|
|
95
99
|
end
|
|
96
100
|
|
|
97
101
|
def initialize_table_sql(table, timestamp)
|
|
98
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
102
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
99
103
|
table_metadata_table.name,
|
|
100
104
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
101
105
|
quoted(timestamp),
|
|
106
|
+
table_metadata_table.last_updated_at_column.name,
|
|
107
|
+
quoted(Time.now.utc),
|
|
102
108
|
table_metadata_table.table_name_column.name,
|
|
103
109
|
quoted(table.name),
|
|
104
110
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
@@ -126,10 +132,12 @@ module Purview
|
|
|
126
132
|
end
|
|
127
133
|
|
|
128
134
|
def lock_table_sql(table, timestamp)
|
|
129
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
135
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
130
136
|
table_metadata_table.name,
|
|
131
137
|
table_metadata_table.locked_at_column.name,
|
|
132
138
|
quoted(timestamp),
|
|
139
|
+
table_metadata_table.last_updated_at_column.name,
|
|
140
|
+
quoted(Time.now.utc),
|
|
133
141
|
table_metadata_table.table_name_column.name,
|
|
134
142
|
quoted(table.name),
|
|
135
143
|
table_metadata_table.locked_at_column.name,
|
|
@@ -149,10 +157,12 @@ module Purview
|
|
|
149
157
|
end
|
|
150
158
|
|
|
151
159
|
def set_table_metadata_value_sql(table, column, value)
|
|
152
|
-
'UPDATE %s SET %s = %s WHERE %s = %s' % [
|
|
160
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
|
|
153
161
|
table_metadata_table.name,
|
|
154
162
|
column.name,
|
|
155
163
|
quoted(value),
|
|
164
|
+
table_metadata_table.last_updated_at_column.name,
|
|
165
|
+
quoted(Time.now.utc),
|
|
156
166
|
table_metadata_table.table_name_column.name,
|
|
157
167
|
quoted(table.name),
|
|
158
168
|
]
|
|
@@ -166,10 +176,12 @@ module Purview
|
|
|
166
176
|
end
|
|
167
177
|
|
|
168
178
|
def unlock_table_sql(table)
|
|
169
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
179
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
170
180
|
table_metadata_table.name,
|
|
171
181
|
table_metadata_table.locked_at_column.name,
|
|
172
182
|
null_value,
|
|
183
|
+
table_metadata_table.last_updated_at_column.name,
|
|
184
|
+
quoted(Time.now.utc),
|
|
173
185
|
table_metadata_table.table_name_column.name,
|
|
174
186
|
quoted(table.name),
|
|
175
187
|
table_metadata_table.locked_at_column.name,
|
|
@@ -35,10 +35,12 @@ module Purview
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def disable_table_sql(table)
|
|
38
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
38
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
39
39
|
table_metadata_table.name,
|
|
40
40
|
table_metadata_table.enabled_at_column.name,
|
|
41
41
|
null_value,
|
|
42
|
+
table_metadata_table.last_updated_at_column.name,
|
|
43
|
+
quoted(Time.now.utc),
|
|
42
44
|
table_metadata_table.table_name_column.name,
|
|
43
45
|
quoted(table.name),
|
|
44
46
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -58,10 +60,12 @@ module Purview
|
|
|
58
60
|
end
|
|
59
61
|
|
|
60
62
|
def enable_table_sql(table, timestamp)
|
|
61
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
63
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
62
64
|
table_metadata_table.name,
|
|
63
65
|
table_metadata_table.enabled_at_column.name,
|
|
64
66
|
quoted(timestamp),
|
|
67
|
+
table_metadata_table.last_updated_at_column.name,
|
|
68
|
+
quoted(Time.now.utc),
|
|
65
69
|
table_metadata_table.table_name_column.name,
|
|
66
70
|
quoted(table.name),
|
|
67
71
|
table_metadata_table.enabled_at_column.name,
|
|
@@ -95,10 +99,12 @@ module Purview
|
|
|
95
99
|
end
|
|
96
100
|
|
|
97
101
|
def initialize_table_sql(table, timestamp)
|
|
98
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
102
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
99
103
|
table_metadata_table.name,
|
|
100
104
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
101
105
|
quoted(timestamp),
|
|
106
|
+
table_metadata_table.last_updated_at_column.name,
|
|
107
|
+
quoted(Time.now.utc),
|
|
102
108
|
table_metadata_table.table_name_column.name,
|
|
103
109
|
quoted(table.name),
|
|
104
110
|
table_metadata_table.max_timestamp_pulled_column.name,
|
|
@@ -119,10 +125,12 @@ module Purview
|
|
|
119
125
|
end
|
|
120
126
|
|
|
121
127
|
def lock_table_sql(table, timestamp)
|
|
122
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
128
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
|
|
123
129
|
table_metadata_table.name,
|
|
124
130
|
table_metadata_table.locked_at_column.name,
|
|
125
131
|
quoted(timestamp),
|
|
132
|
+
table_metadata_table.last_updated_at_column.name,
|
|
133
|
+
quoted(Time.now.utc),
|
|
126
134
|
table_metadata_table.table_name_column.name,
|
|
127
135
|
quoted(table.name),
|
|
128
136
|
table_metadata_table.locked_at_column.name,
|
|
@@ -142,10 +150,12 @@ module Purview
|
|
|
142
150
|
end
|
|
143
151
|
|
|
144
152
|
def set_table_metadata_value_sql(table, column, value)
|
|
145
|
-
'UPDATE %s SET %s = %s WHERE %s = %s' % [
|
|
153
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
|
|
146
154
|
table_metadata_table.name,
|
|
147
155
|
column.name,
|
|
148
156
|
quoted(value),
|
|
157
|
+
table_metadata_table.last_updated_at_column.name,
|
|
158
|
+
quoted(Time.now.utc),
|
|
149
159
|
table_metadata_table.table_name_column.name,
|
|
150
160
|
quoted(table.name),
|
|
151
161
|
]
|
|
@@ -159,10 +169,12 @@ module Purview
|
|
|
159
169
|
end
|
|
160
170
|
|
|
161
171
|
def unlock_table_sql(table)
|
|
162
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
172
|
+
'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
|
163
173
|
table_metadata_table.name,
|
|
164
174
|
table_metadata_table.locked_at_column.name,
|
|
165
175
|
null_value,
|
|
176
|
+
table_metadata_table.last_updated_at_column.name,
|
|
177
|
+
quoted(Time.now.utc),
|
|
166
178
|
table_metadata_table.table_name_column.name,
|
|
167
179
|
quoted(table.name),
|
|
168
180
|
table_metadata_table.locked_at_column.name,
|
|
@@ -8,6 +8,7 @@ module Purview
|
|
|
8
8
|
table_name_column,
|
|
9
9
|
enabled_at_column,
|
|
10
10
|
last_pulled_at_column,
|
|
11
|
+
last_updated_at_column,
|
|
11
12
|
locked_at_column,
|
|
12
13
|
max_timestamp_pulled_column,
|
|
13
14
|
],
|
|
@@ -23,6 +24,10 @@ module Purview
|
|
|
23
24
|
Purview::Columns::Timestamp.new(:last_pulled_at)
|
|
24
25
|
end
|
|
25
26
|
|
|
27
|
+
def last_updated_at_column
|
|
28
|
+
Purview::Columns::Timestamp.new(:last_updated_at)
|
|
29
|
+
end
|
|
30
|
+
|
|
26
31
|
def locked_at_column
|
|
27
32
|
Purview::Columns::Timestamp.new(:locked_at)
|
|
28
33
|
end
|
data/lib/purview/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: purview
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jonathan W. Zaleski
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2018-12-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: pry
|
|
@@ -243,7 +243,7 @@ requirements:
|
|
|
243
243
|
- "`bunny` or `march_hare` gem (if consuming from RabbitMQ)"
|
|
244
244
|
- "`ruby-kafka` or `jruby-kafka` gem (if consuming from Kafka)"
|
|
245
245
|
rubyforge_project:
|
|
246
|
-
rubygems_version: 2.6
|
|
246
|
+
rubygems_version: 2.7.6
|
|
247
247
|
signing_key:
|
|
248
248
|
specification_version: 4
|
|
249
249
|
summary: A framework created to simplify data-warehousing
|