purview 1.7.0 → 1.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (13) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +6 -4
  3. data/TODO +12 -55
  4. data/lib/purview/databases/base.rb +2 -2
  5. data/lib/purview/databases/mssql.rb +18 -6
  6. data/lib/purview/databases/mysql.rb +19 -7
  7. data/lib/purview/databases/postgresql.rb +18 -6
  8. data/lib/purview/databases/sqlite.rb +18 -6
  9. data/lib/purview/mixins/helpers.rb +1 -1
  10. data/lib/purview/pullers/base_sql.rb +1 -1
  11. data/lib/purview/tables/table_metadata.rb +5 -0
  12. data/lib/purview/version.rb +1 -1
  13. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 6bc53290ae44ba2ced3fbdb8acbea4b41647aaff
4
- data.tar.gz: 2d61de5e8a245706faaf749fc89b1d313a44c89b
2
+ SHA256:
3
+ metadata.gz: 10737dc7946dc6ed6233299ef4cc18042ed7ab1f5175478ae4a954f8e4c611a0
4
+ data.tar.gz: 5330e8e05814176ec27c623f5d98ec5af6186df0f2b8f4a8c702774874043fd4
5
5
  SHA512:
6
- metadata.gz: bfdbc5071deb3dcf88ff5186654b47ffb3d639ad2456ccee9fe9c28afe688095842bfbe4eefb68314f8a4a933545bd4758fbbe3ef1ab5e08a72910be367328c1
7
- data.tar.gz: df6ff884d2269b40ae6fff64e32061a67c229e2fe2f210c7c9781a8e165e15692bae000fb3da48e4713c595616ff96f638911384ef230258509e249a7de03386
6
+ metadata.gz: 7444fe8eafec51555096fe3c2f03551bf98d135173610c7c48161927f12c4229db29a2b762ad63e4e81f06cbf10263793c1cf025d83452812147dd8cd0eb0500
7
+ data.tar.gz: ea70373933da78d6e634baf8942765bb82597558561f2397e723e08ca9e0a7068dbfaf59c876d51ba56f39116a974af1f61d37a8bfb291f8a1c34c7bc204efe1
data/.gitignore CHANGED
@@ -13,23 +13,25 @@
13
13
  tags
14
14
  tags.*
15
15
 
16
- ## Specific to RubyMotion:
16
+ ## Specific to RubyMotion
17
17
  .dat*
18
18
  .repl_history
19
19
  build/
20
20
 
21
- ## Documentation cache and generated files:
21
+ ## Documentation cache and generated files
22
22
  /.yardoc/
23
23
  /_yardoc/
24
24
  /doc/
25
25
  /rdoc/
26
26
 
27
- ## Environment normalisation:
27
+ ## Environment normalisation
28
28
  /.bundle/
29
29
  /lib/bundler/man/
30
30
 
31
- ## Ruby artifacts:
31
+ ## Ruby artifacts
32
32
  .ruby-gemset
33
33
  .ruby-version
34
34
  .rvmrc
35
35
  Gemfile.lock
36
+
37
+ test.rb
data/TODO CHANGED
@@ -1,55 +1,12 @@
1
- a 1450000000 1450000000 Add COPY `def copy(sql)` support to PostgreSQL connection
2
- a 1450000000 1450000000 Leverage COPY support for PostgreSQL pulls
3
- a 1450000000 1450000000 Build out change-log tables
4
- a 1450000000 1450000000 Build out canonical, fact and aggregate tables (and related transforms)
5
- a 1450000000 1450000000 Configurable re-pull window (do this automatically once up to current?)
6
- a 1450000000 1450000000 Add schema management capabilities (detect schema-deltas and suggestion modifications)
7
- a 1450000000 1450000000 Consider refactoring mutator methods to be globally available (e.g. quoted -> Quote, etc.)
8
- a 1450000000 1450000000 Support parallel pulls?
9
- a 1450000000 1450000000 Come up w/ a deployment recommendation
10
- a 1495113996 1495130394 Add support for `Consumers`
11
- r 1450000000 1450000000 Add support for MSSQL
12
- r 1450000000 1495070870 Further encapsulate `Dialect` logic (in order to DRY `Database` classes)
13
- d 1494850714 1495046648 Finish SQLite3 raw-connections
14
- d 1495046665 1495046689 Fix JDBC driver configuration
15
- d 1495051831 1495070248 Code-review recent changes and merge
16
- d 1450000000 1450000000 Handle blank values a bit more intelligently (all blanks -> nil)
17
- d 1450000000 1450000000 Deal w/ NULL values
18
- d 1450000000 1450000000 Deal w/ BOOL values
19
- d 1450000000 1450000000 Build out MVP for PostgreSQL loader
20
- d 1450000000 1450000000 Add windowing to pulls (simulate w/ 2 different data-files if necessary)
21
- d 1450000000 1450000000 Include [window] timestamps in query-string for pull[er]
22
- d 1450000000 1450000000 Figure out an intelligent way to solve the chicken <-> egg problem for `table_metadata`
23
- d 1450000000 1450000000 Fail fast if there are no candidate tables (`next_table`)
24
- d 1450000000 1450000000 Ensure that the puller does not try to go into the future (`next_window`)
25
- d 1450000000 1450000000 Add in a primitive logging facility (can be temporary)
26
- d 1450000000 1450000000 Enhance `next_table_sql`
27
- d 1450000000 1450000000 Lock tables (feeds) during pull, parse and load
28
- d 1450000000 1450000000 Verify table-locking mechanism behaves as expected
29
- d 1450000000 1450000000 Ability to drop a table
30
- d 1450000000 1450000000 Reorder args: "table, table_name" to: "table_name, table"
31
- d 1450000000 1450000000 Reorder args: "table, index_name, *columns" to "index_name, table, *columns"
32
- d 1450000000 1450000000 Ability to create an index
33
- d 1450000000 1450000000 Ability to drop an index
34
- d 1450000000 1450000000 Indices on `created_at` and `updated_at` columns
35
- d 1450000000 1450000000 Gracefully handle table {,un}locking in case of error
36
- d 1450000000 1450000000 Add debugging code
37
- d 1450000000 1450000000 Raise no-window & no-table exceptions (making them catchable upstream)
38
- d 1450000000 1450000000 Minimize race-condition around table-locking (use where on UPDATE, raise if zero rows are updated)
39
- d 1450000000 1450000000 Log number of delete, inserts and updates in `loader`
40
- d 1450000000 1450000000 `load_temporary_table` should bomb if there is data outside the window
41
- d 1450000000 1450000000 Delete from outside window where id in temporary table
42
- d 1450000000 1450000000 Rename `data` to `rows` in `execute`
43
- d 1450000000 1450000000 Rename `Postgres` to `PostgreSQL`
44
- d 1450000000 1450000000 Finish PostgreSQL support (requires `pg` gem)
45
- d 1450000000 1450000000 Finish MySQL support (requires `mysql2` gem)
46
- d 1450000000 1450000000 Usage/configuration examples in README.md
47
- d 1450000000 1450000000 Create SQL-puller(s)
48
- d 1450000000 1450000000 Introduce [SQL] dialect(s)
49
- d 1450000000 1450000000 Fix JRuby bundle/build
50
- d 1450000000 1450000000 Fall back to logged in user if no [database-]username is provided
51
- d 1450000000 1450000000 Create class to encapulate `table_metadata` (in order to clean up the logic in the `Database` class(es))
52
- d 1450000000 1450000000 Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe create a new type of column?
53
- d 1450000000 1495070826 Add support for SQLite3
54
- d 1494850696 1495070857 Add TODOs from $HOME\purview-TODO
55
- d 1495070065 1495072292 Finish MSSQL built-out
1
+ f510dd97-ffd2-11e8-9c7a-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544814215 1544814215 a Add `create` functionality to `Database(s)`
2
+ af4b8882-ffd8-11e8-8ecb-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816675 1544816675 a Add COPY `def copy(sql)` support to PostgreSQL connection
3
+ b30f4757-ffd8-11e8-afb9-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816681 1544816681 a Leverage COPY support for PostgreSQL pulls
4
+ b9fec1b8-ffd8-11e8-973b-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816693 1544816693 a Build out change-log, canonical, fact and aggregate tables (and related transforms)
5
+ bef458cc-ffd8-11e8-88b6-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816701 1544816701 a Make re-pull window configurable (do this automatically once up to current?)
6
+ c2c85f85-ffd8-11e8-b2c3-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816708 1544816708 a Add schema management capabilities (detect schema-deltas and suggestion modifications)
7
+ c70ab22b-ffd8-11e8-8c9a-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816715 1544816715 a Refactor mutator methods to be globally available (e.g. quoted -> Quote, etc.)
8
+ cc6c3091-ffd8-11e8-b484-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816724 1544816724 a Add parallel pull(s) support
9
+ d1ecf75e-ffd8-11e8-87cf-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544816733 1544816733 a Add support for `Consumers`
10
+ 683a87f8-ffe3-11e8-8e33-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544821280 1544821280 a Fix SQLite3 (it doesn't appear to work)
11
+ 0aae0fab-ffe4-11e8-ae70-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544821553 1544821553 a Fix PostgreSQL, it doesn't appear to work
12
+ c3b21ac5-ffe6-11e8-802b-8c8590c6ab8c 00000000-0000-0000-0000-000000000000 jzaleski jzaleski 1544822722 1544822722 a Need to add time-zone support to adjust SQL puller results
data/lib/purview/databases/base.rb CHANGED
@@ -395,7 +395,7 @@ module Purview
395
395
 
396
396
  def get_table_metadata_value(connection, table, column)
397
397
  row = connection.execute(get_table_metadata_value_sql(table, column)).rows[0]
398
- raise CouldNotFindMetadataForTable.new(table) \
398
+ raise Purview::Exceptions::CouldNotFindMetadataForTable.new(table) \
399
399
  unless row
400
400
  value = row[column.name]
401
401
  value && column.type.parse(value)
@@ -463,7 +463,7 @@ module Purview
463
463
  def set_table_metadata_value(connection, table, column, value)
464
464
  rows_affected = \
465
465
  connection.execute(set_table_metadata_value_sql(table, column, value)).rows_affected
466
- raise CouldNotUpdateMetadataForTable.new(table) \
466
+ raise Purview::Exceptions::CouldNotUpdateMetadataForTable.new(table) \
467
467
  if zero?(rows_affected)
468
468
  end
469
469
 
data/lib/purview/databases/mssql.rb CHANGED
@@ -35,10 +35,12 @@ module Purview
35
35
  end
36
36
 
37
37
  def disable_table_sql(table)
38
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
38
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
39
39
  table_metadata_table.name,
40
40
  table_metadata_table.enabled_at_column.name,
41
41
  null_value,
42
+ table_metadata_table.last_updated_at_column.name,
43
+ quoted(Time.now.utc),
42
44
  table_metadata_table.table_name_column.name,
43
45
  quoted(table.name),
44
46
  table_metadata_table.enabled_at_column.name,
@@ -58,10 +60,12 @@ module Purview
58
60
  end
59
61
 
60
62
  def enable_table_sql(table, timestamp)
61
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
63
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
62
64
  table_metadata_table.name,
63
65
  table_metadata_table.enabled_at_column.name,
64
66
  quoted(timestamp),
67
+ table_metadata_table.last_updated_at_column.name,
68
+ quoted(Time.now.utc),
65
69
  table_metadata_table.table_name_column.name,
66
70
  quoted(table.name),
67
71
  table_metadata_table.enabled_at_column.name,
@@ -95,10 +99,12 @@ module Purview
95
99
  end
96
100
 
97
101
  def initialize_table_sql(table, timestamp)
98
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
102
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
99
103
  table_metadata_table.name,
100
104
  table_metadata_table.max_timestamp_pulled_column.name,
101
105
  quoted(timestamp),
106
+ table_metadata_table.last_updated_at_column.name,
107
+ quoted(Time.now.utc),
102
108
  table_metadata_table.table_name_column.name,
103
109
  quoted(table.name),
104
110
  table_metadata_table.max_timestamp_pulled_column.name,
@@ -126,10 +132,12 @@ module Purview
126
132
  end
127
133
 
128
134
  def lock_table_sql(table, timestamp)
129
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
135
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
130
136
  table_metadata_table.name,
131
137
  table_metadata_table.locked_at_column.name,
132
138
  quoted(timestamp),
139
+ table_metadata_table.last_updated_at_column.name,
140
+ quoted(Time.now.utc),
133
141
  table_metadata_table.table_name_column.name,
134
142
  quoted(table.name),
135
143
  table_metadata_table.locked_at_column.name,
@@ -149,10 +157,12 @@ module Purview
149
157
  end
150
158
 
151
159
  def set_table_metadata_value_sql(table, column, value)
152
- 'UPDATE %s SET %s = %s WHERE %s = %s' % [
160
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
153
161
  table_metadata_table.name,
154
162
  column.name,
155
163
  quoted(value),
164
+ table_metadata_table.last_updated_at_column.name,
165
+ quoted(Time.now.utc),
156
166
  table_metadata_table.table_name_column.name,
157
167
  quoted(table.name),
158
168
  ]
@@ -166,10 +176,12 @@ module Purview
166
176
  end
167
177
 
168
178
  def unlock_table_sql(table)
169
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
179
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
170
180
  table_metadata_table.name,
171
181
  table_metadata_table.locked_at_column.name,
172
182
  null_value,
183
+ table_metadata_table.last_updated_at_column.name,
184
+ quoted(Time.now.utc),
173
185
  table_metadata_table.table_name_column.name,
174
186
  quoted(table.name),
175
187
  table_metadata_table.locked_at_column.name,
data/lib/purview/databases/mysql.rb CHANGED
@@ -39,10 +39,12 @@ module Purview
39
39
  end
40
40
 
41
41
  def disable_table_sql(table)
42
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
42
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
43
43
  table_metadata_table.name,
44
44
  table_metadata_table.enabled_at_column.name,
45
45
  null_value,
46
+ table_metadata_table.last_updated_at_column.name,
47
+ quoted(Time.now.utc),
46
48
  table_metadata_table.table_name_column.name,
47
49
  quoted(table.name),
48
50
  table_metadata_table.enabled_at_column.name,
@@ -62,10 +64,12 @@ module Purview
62
64
  end
63
65
 
64
66
  def enable_table_sql(table, timestamp)
65
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
67
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
66
68
  table_metadata_table.name,
67
69
  table_metadata_table.enabled_at_column.name,
68
70
  quoted(timestamp),
71
+ table_metadata_table.last_updated_at_column.name,
72
+ quoted(Time.now.utc),
69
73
  table_metadata_table.table_name_column.name,
70
74
  quoted(table.name),
71
75
  table_metadata_table.enabled_at_column.name,
@@ -81,7 +85,7 @@ module Purview
81
85
  end
82
86
 
83
87
  def ensure_table_metadata_exists_for_table_sql(table)
84
- 'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
88
+ 'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL, NULL)' % [
85
89
  table_metadata_table.name,
86
90
  quoted(table.name),
87
91
  ]
@@ -95,10 +99,12 @@ module Purview
95
99
  end
96
100
 
97
101
  def initialize_table_sql(table, timestamp)
98
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
102
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
99
103
  table_metadata_table.name,
100
104
  table_metadata_table.max_timestamp_pulled_column.name,
101
105
  quoted(timestamp),
106
+ table_metadata_table.last_updated_at_column.name,
107
+ quoted(Time.now.utc),
102
108
  table_metadata_table.table_name_column.name,
103
109
  quoted(table.name),
104
110
  table_metadata_table.max_timestamp_pulled_column.name,
@@ -119,10 +125,12 @@ module Purview
119
125
  end
120
126
 
121
127
  def lock_table_sql(table, timestamp)
122
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
128
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
123
129
  table_metadata_table.name,
124
130
  table_metadata_table.locked_at_column.name,
125
131
  quoted(timestamp),
132
+ table_metadata_table.last_updated_at_column.name,
133
+ quoted(Time.now.utc),
126
134
  table_metadata_table.table_name_column.name,
127
135
  quoted(table.name),
128
136
  table_metadata_table.locked_at_column.name,
@@ -142,10 +150,12 @@ module Purview
142
150
  end
143
151
 
144
152
  def set_table_metadata_value_sql(table, column, value)
145
- 'UPDATE %s SET %s = %s WHERE %s = %s' % [
153
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
146
154
  table_metadata_table.name,
147
155
  column.name,
148
156
  quoted(value),
157
+ table_metadata_table.last_updated_at_column.name,
158
+ quoted(Time.now.utc),
149
159
  table_metadata_table.table_name_column.name,
150
160
  quoted(table.name),
151
161
  ]
@@ -159,10 +169,12 @@ module Purview
159
169
  end
160
170
 
161
171
  def unlock_table_sql(table)
162
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
172
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
163
173
  table_metadata_table.name,
164
174
  table_metadata_table.locked_at_column.name,
165
175
  null_value,
176
+ table_metadata_table.last_updated_at_column.name,
177
+ quoted(Time.now.utc),
166
178
  table_metadata_table.table_name_column.name,
167
179
  quoted(table.name),
168
180
  table_metadata_table.locked_at_column.name,
data/lib/purview/databases/postgresql.rb CHANGED
@@ -35,10 +35,12 @@ module Purview
35
35
  end
36
36
 
37
37
  def disable_table_sql(table)
38
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
38
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
39
39
  table_metadata_table.name,
40
40
  table_metadata_table.enabled_at_column.name,
41
41
  null_value,
42
+ table_metadata_table.last_updated_at_column.name,
43
+ quoted(Time.now.utc),
42
44
  table_metadata_table.table_name_column.name,
43
45
  quoted(table.name),
44
46
  table_metadata_table.enabled_at_column.name,
@@ -58,10 +60,12 @@ module Purview
58
60
  end
59
61
 
60
62
  def enable_table_sql(table, timestamp)
61
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
63
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
62
64
  table_metadata_table.name,
63
65
  table_metadata_table.enabled_at_column.name,
64
66
  quoted(timestamp),
67
+ table_metadata_table.last_updated_at_column.name,
68
+ quoted(Time.now.utc),
65
69
  table_metadata_table.table_name_column.name,
66
70
  quoted(table.name),
67
71
  table_metadata_table.enabled_at_column.name,
@@ -95,10 +99,12 @@ module Purview
95
99
  end
96
100
 
97
101
  def initialize_table_sql(table, timestamp)
98
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
102
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
99
103
  table_metadata_table.name,
100
104
  table_metadata_table.max_timestamp_pulled_column.name,
101
105
  quoted(timestamp),
106
+ table_metadata_table.last_updated_at_column.name,
107
+ quoted(Time.now.utc),
102
108
  table_metadata_table.table_name_column.name,
103
109
  quoted(table.name),
104
110
  table_metadata_table.max_timestamp_pulled_column.name,
@@ -126,10 +132,12 @@ module Purview
126
132
  end
127
133
 
128
134
  def lock_table_sql(table, timestamp)
129
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
135
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
130
136
  table_metadata_table.name,
131
137
  table_metadata_table.locked_at_column.name,
132
138
  quoted(timestamp),
139
+ table_metadata_table.last_updated_at_column.name,
140
+ quoted(Time.now.utc),
133
141
  table_metadata_table.table_name_column.name,
134
142
  quoted(table.name),
135
143
  table_metadata_table.locked_at_column.name,
@@ -149,10 +157,12 @@ module Purview
149
157
  end
150
158
 
151
159
  def set_table_metadata_value_sql(table, column, value)
152
- 'UPDATE %s SET %s = %s WHERE %s = %s' % [
160
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
153
161
  table_metadata_table.name,
154
162
  column.name,
155
163
  quoted(value),
164
+ table_metadata_table.last_updated_at_column.name,
165
+ quoted(Time.now.utc),
156
166
  table_metadata_table.table_name_column.name,
157
167
  quoted(table.name),
158
168
  ]
@@ -166,10 +176,12 @@ module Purview
166
176
  end
167
177
 
168
178
  def unlock_table_sql(table)
169
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
179
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
170
180
  table_metadata_table.name,
171
181
  table_metadata_table.locked_at_column.name,
172
182
  null_value,
183
+ table_metadata_table.last_updated_at_column.name,
184
+ quoted(Time.now.utc),
173
185
  table_metadata_table.table_name_column.name,
174
186
  quoted(table.name),
175
187
  table_metadata_table.locked_at_column.name,
data/lib/purview/databases/sqlite.rb CHANGED
@@ -35,10 +35,12 @@ module Purview
35
35
  end
36
36
 
37
37
  def disable_table_sql(table)
38
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
38
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
39
39
  table_metadata_table.name,
40
40
  table_metadata_table.enabled_at_column.name,
41
41
  null_value,
42
+ table_metadata_table.last_updated_at_column.name,
43
+ quoted(Time.now.utc),
42
44
  table_metadata_table.table_name_column.name,
43
45
  quoted(table.name),
44
46
  table_metadata_table.enabled_at_column.name,
@@ -58,10 +60,12 @@ module Purview
58
60
  end
59
61
 
60
62
  def enable_table_sql(table, timestamp)
61
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
63
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
62
64
  table_metadata_table.name,
63
65
  table_metadata_table.enabled_at_column.name,
64
66
  quoted(timestamp),
67
+ table_metadata_table.last_updated_at_column.name,
68
+ quoted(Time.now.utc),
65
69
  table_metadata_table.table_name_column.name,
66
70
  quoted(table.name),
67
71
  table_metadata_table.enabled_at_column.name,
@@ -95,10 +99,12 @@ module Purview
95
99
  end
96
100
 
97
101
  def initialize_table_sql(table, timestamp)
98
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
102
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
99
103
  table_metadata_table.name,
100
104
  table_metadata_table.max_timestamp_pulled_column.name,
101
105
  quoted(timestamp),
106
+ table_metadata_table.last_updated_at_column.name,
107
+ quoted(Time.now.utc),
102
108
  table_metadata_table.table_name_column.name,
103
109
  quoted(table.name),
104
110
  table_metadata_table.max_timestamp_pulled_column.name,
@@ -119,10 +125,12 @@ module Purview
119
125
  end
120
126
 
121
127
  def lock_table_sql(table, timestamp)
122
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
128
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NULL' % [
123
129
  table_metadata_table.name,
124
130
  table_metadata_table.locked_at_column.name,
125
131
  quoted(timestamp),
132
+ table_metadata_table.last_updated_at_column.name,
133
+ quoted(Time.now.utc),
126
134
  table_metadata_table.table_name_column.name,
127
135
  quoted(table.name),
128
136
  table_metadata_table.locked_at_column.name,
@@ -142,10 +150,12 @@ module Purview
142
150
  end
143
151
 
144
152
  def set_table_metadata_value_sql(table, column, value)
145
- 'UPDATE %s SET %s = %s WHERE %s = %s' % [
153
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s' % [
146
154
  table_metadata_table.name,
147
155
  column.name,
148
156
  quoted(value),
157
+ table_metadata_table.last_updated_at_column.name,
158
+ quoted(Time.now.utc),
149
159
  table_metadata_table.table_name_column.name,
150
160
  quoted(table.name),
151
161
  ]
@@ -159,10 +169,12 @@ module Purview
159
169
  end
160
170
 
161
171
  def unlock_table_sql(table)
162
- 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
172
+ 'UPDATE %s SET %s = %s, %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
163
173
  table_metadata_table.name,
164
174
  table_metadata_table.locked_at_column.name,
165
175
  null_value,
176
+ table_metadata_table.last_updated_at_column.name,
177
+ quoted(Time.now.utc),
166
178
  table_metadata_table.table_name_column.name,
167
179
  quoted(table.name),
168
180
  table_metadata_table.locked_at_column.name,
data/lib/purview/mixins/helpers.rb CHANGED
@@ -22,7 +22,7 @@ module Purview
22
22
  end
23
23
 
24
24
  def timestamp
25
- Time.now
25
+ Time.now.utc
26
26
  end
27
27
 
28
28
  def with_timestamp
data/lib/purview/pullers/base_sql.rb CHANGED
@@ -65,7 +65,7 @@ module Purview
65
65
  end
66
66
 
67
67
  def table_name
68
- opts[:table_name]
68
+ opts[:table].name.gsub(/_raw$/, '')
69
69
  end
70
70
  end
71
71
  end
data/lib/purview/tables/table_metadata.rb CHANGED
@@ -8,6 +8,7 @@ module Purview
8
8
  table_name_column,
9
9
  enabled_at_column,
10
10
  last_pulled_at_column,
11
+ last_updated_at_column,
11
12
  locked_at_column,
12
13
  max_timestamp_pulled_column,
13
14
  ],
@@ -23,6 +24,10 @@ module Purview
23
24
  Purview::Columns::Timestamp.new(:last_pulled_at)
24
25
  end
25
26
 
27
+ def last_updated_at_column
28
+ Purview::Columns::Timestamp.new(:last_updated_at)
29
+ end
30
+
26
31
  def locked_at_column
27
32
  Purview::Columns::Timestamp.new(:locked_at)
28
33
  end
data/lib/purview/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Purview
2
- VERSION = '1.7.0'
2
+ VERSION = '1.8.0'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: purview
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.0
4
+ version: 1.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan W. Zaleski
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-05-23 00:00:00.000000000 Z
11
+ date: 2018-12-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pry
@@ -243,7 +243,7 @@ requirements:
243
243
  - "`bunny` or `march_hare` gem (if consuming from RabbitMQ)"
244
244
  - "`ruby-kafka` or `jruby-kafka` gem (if consuming from Kafka)"
245
245
  rubyforge_project:
246
- rubygems_version: 2.6.10
246
+ rubygems_version: 2.7.6
247
247
  signing_key:
248
248
  specification_version: 4
249
249
  summary: A framework created to simplify data-warehousing