purview 1.0.0.beta2 → 1.0.0.beta3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -25
- data/lib/purview/databases/base.rb +173 -118
- data/lib/purview/databases/mysql.rb +31 -13
- data/lib/purview/databases/postgresql.rb +31 -13
- data/lib/purview/exceptions/database_already_assigned.rb +9 -0
- data/lib/purview/exceptions.rb +1 -0
- data/lib/purview/mixins/helpers.rb +4 -0
- data/lib/purview/pullers/base_sql.rb +3 -3
- data/lib/purview/raw_connections/base.rb +1 -1
- data/lib/purview/raw_connections/jdbc/mysql.rb +7 -1
- data/lib/purview/raw_connections/jdbc/postgres.rb +7 -1
- data/lib/purview/raw_connections/mysql2.rb +12 -6
- data/lib/purview/raw_connections/pg.rb +7 -1
- data/lib/purview/raw_connections.rb +4 -0
- data/lib/purview/tables/base.rb +4 -3
- data/lib/purview/version.rb +1 -1
- data/lib/purview.rb +1 -5
- metadata +3 -3
- data/lib/purview/helpers.rub +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e042a9a30d7596c9a7dad1ce2e121b9f9bf160f
|
4
|
+
data.tar.gz: b4a1f379840183ae0a11f411a409849ed06655b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 52be209e82452fa5ec45ce2726c516ef315cb32de3c67b2b74862def7a29fb4fd7c95ff4da270de4f8f5262b0a4aa4ad11c35e6ffbc2e27c3bce366e998a8d8e
|
7
|
+
data.tar.gz: c5768e07904cd00ae25d18adfb163ff5516d2337c2451812c1313ec5e1b183ec2ffdddcbb08b6a434607931e7ccbeaec48ea91e8f13a2d118c7b8461cf95529a
|
data/README.md
CHANGED
@@ -26,22 +26,6 @@ Load the `MySQL` client (for `PostgreSQL` simply change 'mysql2' to 'pg')
|
|
26
26
|
require 'mysql2'
|
27
27
|
```
|
28
28
|
|
29
|
-
Set the database-name (this can be anything, but it must exist)
|
30
|
-
```ruby
|
31
|
-
database_name = :data_warehouse
|
32
|
-
```
|
33
|
-
|
34
|
-
Combine all the configuration options and instantiate the `Database` (for
|
35
|
-
`PostgreSQL` simply change `MySQL` to `PostgreSQL`)
|
36
|
-
```ruby
|
37
|
-
database_opts = {}
|
38
|
-
|
39
|
-
database = Purview::Databases::MySQL.new(
|
40
|
-
database_name,
|
41
|
-
database_opts
|
42
|
-
)
|
43
|
-
```
|
44
|
-
|
45
29
|
Set the table-name (this can be anything, but it must exist)
|
46
30
|
```ruby
|
47
31
|
table_name = :users
|
@@ -93,7 +77,6 @@ Combine all the configuration options and instantiate the `Table`
|
|
93
77
|
```ruby
|
94
78
|
table_opts = {
|
95
79
|
:columns => columns,
|
96
|
-
:database => database,
|
97
80
|
:loader => loader_opts,
|
98
81
|
:parser => parser_opts,
|
99
82
|
:puller => puller_opts,
|
@@ -106,22 +89,39 @@ table = Purview::Tables::Raw.new(
|
|
106
89
|
)
|
107
90
|
```
|
108
91
|
|
92
|
+
Set the database-name (this can be anything, but it must exist)
|
93
|
+
```ruby
|
94
|
+
database_name = :data_warehouse
|
95
|
+
```
|
96
|
+
|
97
|
+
Combine all the configuration options and instantiate the `Database` (for
|
98
|
+
`PostgreSQL` simply change `MySQL` to `PostgreSQL`)
|
99
|
+
```ruby
|
100
|
+
database_opts = {
|
101
|
+
:tables => [table],
|
102
|
+
}
|
103
|
+
|
104
|
+
database = Purview::Databases::MySQL.new(
|
105
|
+
database_name,
|
106
|
+
database_opts
|
107
|
+
)
|
108
|
+
```
|
109
|
+
|
109
110
|
Add the `Table` to the `Database` (schema). In order for [the] `Table` to be
|
110
111
|
`sync[ed]` it *must* be added to [the] `Database`
|
111
112
|
```ruby
|
112
113
|
database.add_table(table)
|
113
114
|
```
|
114
115
|
|
115
|
-
Create the `Table
|
116
|
-
environments you will likely want an external process to manage the
|
117
|
-
`PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
|
116
|
+
Create the `Table` (in the DB). Recommended for testing purposes *only*. For
|
117
|
+
production environments you will likely want an external process to manage the
|
118
|
+
schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
|
118
119
|
```ruby
|
119
120
|
begin
|
120
|
-
database.create_table(
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
rescue Mysql2::Error; end
|
121
|
+
database.create_table(table)
|
122
|
+
rescue Mysql2::Error
|
123
|
+
# Swallow
|
124
|
+
end
|
125
125
|
```
|
126
126
|
|
127
127
|
Sync the `Database`. This process will select a [candidate] `Table`, pull data
|
@@ -6,60 +6,39 @@ module Purview
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
8
8
|
@opts = opts
|
9
|
-
@tables = Set.new
|
10
9
|
end
|
11
10
|
|
12
|
-
def
|
13
|
-
@tables << table
|
14
|
-
end
|
15
|
-
|
16
|
-
def create_index(connection, table, columns, opts={})
|
11
|
+
def create_table(table, opts={})
|
17
12
|
table_opts = extract_table_options(opts)
|
18
13
|
table_name = table_name(table, table_opts)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
).tap do |index_name|
|
25
|
-
connection.execute(
|
26
|
-
create_index_sql(
|
27
|
-
table_name,
|
28
|
-
index_name,
|
29
|
-
table,
|
30
|
-
columns,
|
31
|
-
index_opts
|
32
|
-
)
|
33
|
-
)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def create_table(connection, table, opts={})
|
38
|
-
table_opts = extract_table_options(opts)
|
39
|
-
table_name(table, table_opts).tap do |table_name|
|
40
|
-
connection.execute(
|
41
|
-
create_table_sql(
|
42
|
-
table_name,
|
43
|
-
table,
|
44
|
-
table_opts
|
45
|
-
)
|
46
|
-
)
|
47
|
-
if table_opts[:create_indices]
|
48
|
-
table.indexed_columns.each do |columns|
|
49
|
-
create_index(
|
50
|
-
connection,
|
14
|
+
with_context_logging("`create_table` for: #{table_name}") do
|
15
|
+
with_new_connection do |connection|
|
16
|
+
connection.execute(
|
17
|
+
create_table_sql(
|
18
|
+
table_name,
|
51
19
|
table,
|
52
|
-
|
53
|
-
:table => { :name => table_name }
|
20
|
+
table_opts
|
54
21
|
)
|
22
|
+
)
|
23
|
+
if table_opts[:create_indices]
|
24
|
+
table.indexed_columns.each do |columns|
|
25
|
+
create_index(
|
26
|
+
connection,
|
27
|
+
table,
|
28
|
+
columns,
|
29
|
+
:table => { :name => table_name }
|
30
|
+
)
|
31
|
+
end
|
55
32
|
end
|
56
33
|
end
|
34
|
+
table_name
|
57
35
|
end
|
58
36
|
end
|
59
37
|
|
60
38
|
def create_temporary_table(connection, table, opts={})
|
61
39
|
table_opts = extract_table_options(opts)
|
62
|
-
table_name(table, table_opts)
|
40
|
+
table_name = table_name(table, table_opts)
|
41
|
+
with_context_logging("`create_temporary_table` for: #{table_name}") do
|
63
42
|
connection.execute(
|
64
43
|
create_temporary_table_sql(
|
65
44
|
table_name,
|
@@ -77,89 +56,79 @@ module Purview
|
|
77
56
|
)
|
78
57
|
end
|
79
58
|
end
|
59
|
+
table_name
|
80
60
|
end
|
81
61
|
end
|
82
62
|
|
83
63
|
def disable_table(table)
|
84
|
-
|
64
|
+
table_name = table_name(table)
|
65
|
+
with_context_logging("`disable_table` for: #{table_name}") do
|
85
66
|
with_new_connection do |connection|
|
86
|
-
|
87
|
-
connection
|
88
|
-
|
89
|
-
|
90
|
-
)
|
67
|
+
rows_affected = \
|
68
|
+
connection.execute(disable_table_sql(table)).rows_affected
|
69
|
+
raise Purview::Exceptions::TableAlreadyDisabled.new(table) \
|
70
|
+
if zero?(rows_affected)
|
91
71
|
end
|
72
|
+
table_name
|
92
73
|
end
|
93
74
|
end
|
94
75
|
|
95
|
-
def drop_table(
|
96
|
-
table_opts = extract_table_options(opts)
|
97
|
-
table_name(table, table_opts).tap do |table_name|
|
98
|
-
connection.execute(
|
99
|
-
drop_table_sql(
|
100
|
-
table_name,
|
101
|
-
table,
|
102
|
-
table_opts
|
103
|
-
)
|
104
|
-
)
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def drop_index(connection, table, columns, opts={})
|
76
|
+
def drop_table(table, opts={})
|
109
77
|
table_opts = extract_table_options(opts)
|
110
78
|
table_name = table_name(table, table_opts)
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
table_name,
|
120
|
-
index_name,
|
121
|
-
table,
|
122
|
-
columns,
|
123
|
-
index_opts
|
79
|
+
with_context_logging("`drop_table` for: #{table_name}") do
|
80
|
+
with_new_connection do |connection|
|
81
|
+
connection.execute(
|
82
|
+
drop_table_sql(
|
83
|
+
table_name,
|
84
|
+
table,
|
85
|
+
table_opts
|
86
|
+
)
|
124
87
|
)
|
125
|
-
|
88
|
+
end
|
89
|
+
table_name
|
126
90
|
end
|
127
91
|
end
|
128
92
|
|
129
|
-
def enable_table(table)
|
130
|
-
|
93
|
+
def enable_table(table, timestamp=Time.now.utc)
|
94
|
+
table_name = table_name(table)
|
95
|
+
with_context_logging("`enable_table` for: #{table_name}") do
|
131
96
|
with_new_connection do |connection|
|
132
|
-
|
133
|
-
connection,
|
134
|
-
|
135
|
-
|
136
|
-
)
|
97
|
+
rows_affected = \
|
98
|
+
connection.execute(enable_table_sql(table, timestamp)).rows_affected
|
99
|
+
raise Purview::Exceptions::TableAlreadyEnabled.new(table) \
|
100
|
+
if zero?(rows_affected)
|
137
101
|
end
|
102
|
+
table_name
|
138
103
|
end
|
139
104
|
end
|
140
105
|
|
141
|
-
def lock_table(table, timestamp)
|
142
|
-
|
106
|
+
def lock_table(table, timestamp=Time.now.utc)
|
107
|
+
table_name = table_name(table)
|
108
|
+
with_context_logging("`lock_table` for: #{table_name}") do
|
143
109
|
with_new_connection do |connection|
|
144
110
|
rows_affected = \
|
145
111
|
connection.execute(lock_table_sql(table, timestamp)).rows_affected
|
146
|
-
raise Purview::Exceptions::
|
112
|
+
raise Purview::Exceptions::TableAlreadyLocked.new(table) \
|
147
113
|
if zero?(rows_affected)
|
148
114
|
end
|
115
|
+
table_name
|
149
116
|
end
|
150
117
|
end
|
151
118
|
|
152
119
|
def sync
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
table
|
120
|
+
with_context_logging('`sync`') do
|
121
|
+
with_new_connection do |connection|
|
122
|
+
with_transaction(connection) do |timestamp|
|
123
|
+
with_next_table(connection, timestamp) do |table|
|
124
|
+
with_next_window(
|
125
|
+
connection,
|
126
|
+
table,
|
127
|
+
timestamp
|
128
|
+
) do |window|
|
129
|
+
with_table_locked(table, timestamp) do
|
130
|
+
table.sync(connection, window)
|
131
|
+
end
|
163
132
|
end
|
164
133
|
end
|
165
134
|
end
|
@@ -168,13 +137,15 @@ module Purview
|
|
168
137
|
end
|
169
138
|
|
170
139
|
def unlock_table(table)
|
171
|
-
|
140
|
+
table_name = table_name(table)
|
141
|
+
with_context_logging("`unlock_table` for: #{table_name}") do
|
172
142
|
with_new_connection do |connection|
|
173
143
|
rows_affected = \
|
174
144
|
connection.execute(unlock_table_sql(table)).rows_affected
|
175
|
-
raise Purview::Exceptions::
|
145
|
+
raise Purview::Exceptions::TableAlreadyUnlocked.new(table) \
|
176
146
|
if zero?(rows_affected)
|
177
147
|
end
|
148
|
+
table_name
|
178
149
|
end
|
179
150
|
end
|
180
151
|
|
@@ -184,7 +155,7 @@ module Purview
|
|
184
155
|
include Purview::Mixins::Helpers
|
185
156
|
include Purview::Mixins::Logger
|
186
157
|
|
187
|
-
attr_reader :opts
|
158
|
+
attr_reader :opts
|
188
159
|
|
189
160
|
public :connect
|
190
161
|
|
@@ -219,13 +190,40 @@ module Purview
|
|
219
190
|
end
|
220
191
|
|
221
192
|
def connection_opts
|
222
|
-
{
|
193
|
+
{
|
194
|
+
:database => database_name,
|
195
|
+
:host => database_host,
|
196
|
+
:password => database_password,
|
197
|
+
:port => database_port,
|
198
|
+
:username => database_username,
|
199
|
+
}
|
223
200
|
end
|
224
201
|
|
225
202
|
def connection_type
|
226
203
|
raise %{All "#{Base}(s)" must override the "connection_type" method}
|
227
204
|
end
|
228
205
|
|
206
|
+
def create_index(connection, table, columns, opts={})
|
207
|
+
table_opts = extract_table_options(opts)
|
208
|
+
table_name = table_name(table, table_opts)
|
209
|
+
index_opts = extract_index_options(opts)
|
210
|
+
index_name(
|
211
|
+
table_name,
|
212
|
+
columns,
|
213
|
+
index_opts
|
214
|
+
).tap do |index_name|
|
215
|
+
connection.execute(
|
216
|
+
create_index_sql(
|
217
|
+
table_name,
|
218
|
+
index_name,
|
219
|
+
table,
|
220
|
+
columns,
|
221
|
+
index_opts
|
222
|
+
)
|
223
|
+
)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
229
227
|
def create_index_sql(table_name, index_name, table, columns, index_opts={})
|
230
228
|
raise %{All "#{Base}(s)" must override the "create_index_sql" method}
|
231
229
|
end
|
@@ -238,6 +236,26 @@ module Purview
|
|
238
236
|
raise %{All "#{Base}(s)" must override the "create_temporary_table_sql" method}
|
239
237
|
end
|
240
238
|
|
239
|
+
def database_host
|
240
|
+
opts[:host]
|
241
|
+
end
|
242
|
+
|
243
|
+
def database_name
|
244
|
+
name
|
245
|
+
end
|
246
|
+
|
247
|
+
def database_password
|
248
|
+
opts[:password]
|
249
|
+
end
|
250
|
+
|
251
|
+
def database_port
|
252
|
+
opts[:port]
|
253
|
+
end
|
254
|
+
|
255
|
+
def database_username
|
256
|
+
opts[:username]
|
257
|
+
end
|
258
|
+
|
241
259
|
def default(column)
|
242
260
|
column.default || default_map[column.type]
|
243
261
|
end
|
@@ -254,6 +272,31 @@ module Purview
|
|
254
272
|
raise %{All "#{Base}(s)" must override the "dialect_type" method}
|
255
273
|
end
|
256
274
|
|
275
|
+
def disable_table_sql(table)
|
276
|
+
raise %{All "#{Base}(s)" must override the "disable_table_sql" method}
|
277
|
+
end
|
278
|
+
|
279
|
+
def drop_index(table, columns, opts={})
|
280
|
+
table_opts = extract_table_options(opts)
|
281
|
+
table_name = table_name(table, table_opts)
|
282
|
+
index_opts = extract_index_options(opts)
|
283
|
+
index_name(
|
284
|
+
table_name,
|
285
|
+
columns,
|
286
|
+
index_opts
|
287
|
+
).tap do |index_name|
|
288
|
+
connection.execute(
|
289
|
+
drop_index_sql(
|
290
|
+
table_name,
|
291
|
+
index_name,
|
292
|
+
table,
|
293
|
+
columns,
|
294
|
+
index_opts
|
295
|
+
)
|
296
|
+
)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
257
300
|
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
258
301
|
raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
|
259
302
|
end
|
@@ -262,6 +305,10 @@ module Purview
|
|
262
305
|
raise %{All "#{Base}(s)" must override the "drop_table_sql" method}
|
263
306
|
end
|
264
307
|
|
308
|
+
def enable_table_sql(table, timestamp)
|
309
|
+
raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
|
310
|
+
end
|
311
|
+
|
265
312
|
def ensure_table_metadata_table_exists
|
266
313
|
with_new_connection do |connection|
|
267
314
|
connection.execute(ensure_table_metadata_table_exists_sql)
|
@@ -296,14 +343,14 @@ module Purview
|
|
296
343
|
dialect.false_value
|
297
344
|
end
|
298
345
|
|
299
|
-
def
|
300
|
-
row = connection.execute(
|
301
|
-
|
302
|
-
|
346
|
+
def get_enabled_at_for_table(connection, table)
|
347
|
+
row = connection.execute(get_enabled_at_for_table_sql(table)).rows[0]
|
348
|
+
timestamp = row[table_metadata_enabled_at_column_name]
|
349
|
+
timestamp ? Time.parse(timestamp) : nil
|
303
350
|
end
|
304
351
|
|
305
|
-
def
|
306
|
-
raise %{All "#{Base}(s)" must override the "
|
352
|
+
def get_enabled_at_for_table_sql(table)
|
353
|
+
raise %{All "#{Base}(s)" must override the "get_enabled_at_for_table_sql" method}
|
307
354
|
end
|
308
355
|
|
309
356
|
def get_last_pulled_at_for_table(connection, table)
|
@@ -401,12 +448,12 @@ module Purview
|
|
401
448
|
dialect.sanitized(value)
|
402
449
|
end
|
403
450
|
|
404
|
-
def
|
405
|
-
connection.execute(
|
451
|
+
def set_enabled_at_for_table(connection, table, timestamp)
|
452
|
+
connection.execute(set_enabled_at_for_table_sql(table, timestamp))
|
406
453
|
end
|
407
454
|
|
408
|
-
def
|
409
|
-
raise %{All "#{Base}(s)" must override the "
|
455
|
+
def set_enabled_at_for_table_sql(table, timestamp)
|
456
|
+
raise %{All "#{Base}(s)" must override the "set_enabled_at_for_table_sql" method}
|
410
457
|
end
|
411
458
|
|
412
459
|
def set_last_pulled_at_for_table(connection, table, timestamp)
|
@@ -433,13 +480,13 @@ module Purview
|
|
433
480
|
raise %{All "#{Base}(s)" must override the "set_max_timestamp_pulled_for_table_sql" method}
|
434
481
|
end
|
435
482
|
|
436
|
-
def
|
437
|
-
column = Purview::Columns::
|
483
|
+
def table_metadata_enabled_at_column_definition
|
484
|
+
column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
|
438
485
|
column_definition(column)
|
439
486
|
end
|
440
487
|
|
441
|
-
def
|
442
|
-
'
|
488
|
+
def table_metadata_enabled_at_column_name
|
489
|
+
'enabled_at'
|
443
490
|
end
|
444
491
|
|
445
492
|
def table_metadata_last_pulled_at_column_definition
|
@@ -482,18 +529,26 @@ module Purview
|
|
482
529
|
'table_name'
|
483
530
|
end
|
484
531
|
|
532
|
+
def table_name(table, table_opts={})
|
533
|
+
table_opts[:name] || table.name
|
534
|
+
end
|
535
|
+
|
536
|
+
def tables
|
537
|
+
@tables ||= Set.new.tap do |result|
|
538
|
+
(opts[:tables] || []).each do |table|
|
539
|
+
table.database = self if result.add?(table)
|
540
|
+
end
|
541
|
+
end
|
542
|
+
end
|
543
|
+
|
485
544
|
def tables_by_name
|
486
|
-
{}.tap do |result|
|
545
|
+
@tables_by_name ||= {}.tap do |result|
|
487
546
|
tables.each do |table|
|
488
547
|
result[table.name] = table
|
489
548
|
end
|
490
549
|
end
|
491
550
|
end
|
492
551
|
|
493
|
-
def table_name(table, table_opts={})
|
494
|
-
table_opts[:name] || table.name
|
495
|
-
end
|
496
|
-
|
497
552
|
def true_value
|
498
553
|
dialect.true_value
|
499
554
|
end
|
@@ -37,6 +37,17 @@ module Purview
|
|
37
37
|
Purview::Dialects::MySQL
|
38
38
|
end
|
39
39
|
|
40
|
+
def disable_table_sql(table)
|
41
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
42
|
+
table_metadata_table_name,
|
43
|
+
table_metadata_enabled_at_column_name,
|
44
|
+
null_value,
|
45
|
+
table_metadata_table_name_column_name,
|
46
|
+
quoted(table.name),
|
47
|
+
table_metadata_enabled_at_column_name,
|
48
|
+
]
|
49
|
+
end
|
50
|
+
|
40
51
|
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
41
52
|
'DROP INDEX %s' % [
|
42
53
|
index_name,
|
@@ -49,6 +60,17 @@ module Purview
|
|
49
60
|
]
|
50
61
|
end
|
51
62
|
|
63
|
+
def enable_table_sql(table, timestamp)
|
64
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
65
|
+
table_metadata_table_name,
|
66
|
+
table_metadata_enabled_at_column_name,
|
67
|
+
quoted(timestamp),
|
68
|
+
table_metadata_table_name_column_name,
|
69
|
+
quoted(table.name),
|
70
|
+
table_metadata_enabled_at_column_name,
|
71
|
+
]
|
72
|
+
end
|
73
|
+
|
52
74
|
def ensure_table_metadata_exists_for_table_sql(table)
|
53
75
|
'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
|
54
76
|
table_metadata_table_name,
|
@@ -60,16 +82,16 @@ module Purview
|
|
60
82
|
'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
|
61
83
|
table_metadata_table_name,
|
62
84
|
table_metadata_table_name_column_definition,
|
63
|
-
|
85
|
+
table_metadata_enabled_at_column_definition,
|
64
86
|
table_metadata_last_pulled_at_column_definition,
|
65
87
|
table_metadata_locked_at_column_definition,
|
66
88
|
table_metadata_max_timestamp_pulled_column_definition,
|
67
89
|
]
|
68
90
|
end
|
69
91
|
|
70
|
-
def
|
92
|
+
def get_enabled_at_for_table_sql(table)
|
71
93
|
'SELECT %s FROM %s WHERE %s = %s' % [
|
72
|
-
|
94
|
+
table_metadata_enabled_at_column_name,
|
73
95
|
table_metadata_table_name,
|
74
96
|
table_metadata_table_name_column_name,
|
75
97
|
quoted(table.name),
|
@@ -119,27 +141,23 @@ module Purview
|
|
119
141
|
end
|
120
142
|
|
121
143
|
def next_table_sql(timestamp)
|
122
|
-
'SELECT %s FROM %s WHERE %s
|
144
|
+
'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
|
123
145
|
table_metadata_table_name_column_name,
|
124
146
|
table_metadata_table_name,
|
125
|
-
|
126
|
-
quoted(true_value),
|
147
|
+
table_metadata_enabled_at_column_name,
|
127
148
|
table_metadata_locked_at_column_name,
|
128
149
|
table_metadata_last_pulled_at_column_name,
|
129
150
|
table_metadata_last_pulled_at_column_name,
|
130
151
|
]
|
131
152
|
end
|
132
153
|
|
133
|
-
def
|
134
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND
|
154
|
+
def set_enabled_at_for_table_sql(table, timestamp)
|
155
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
|
135
156
|
table_metadata_table_name,
|
136
|
-
|
137
|
-
quoted(
|
157
|
+
table_metadata_enabled_at_column_name,
|
158
|
+
quoted(timestamp),
|
138
159
|
table_metadata_table_name_column_name,
|
139
160
|
quoted(table.name),
|
140
|
-
table_metadata_enabled_column_name,
|
141
|
-
table_metadata_enabled_column_name,
|
142
|
-
quoted(false_value),
|
143
161
|
]
|
144
162
|
end
|
145
163
|
|
@@ -33,6 +33,17 @@ module Purview
|
|
33
33
|
Purview::Dialects::PostgreSQL
|
34
34
|
end
|
35
35
|
|
36
|
+
def disable_table_sql(table)
|
37
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
38
|
+
table_metadata_table_name,
|
39
|
+
table_metadata_enabled_at_column_name,
|
40
|
+
null_value,
|
41
|
+
table_metadata_table_name_column_name,
|
42
|
+
quoted(table.name),
|
43
|
+
table_metadata_enabled_at_column_name,
|
44
|
+
]
|
45
|
+
end
|
46
|
+
|
36
47
|
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
37
48
|
'DROP INDEX %s' % [
|
38
49
|
index_name,
|
@@ -45,6 +56,17 @@ module Purview
|
|
45
56
|
]
|
46
57
|
end
|
47
58
|
|
59
|
+
def enable_table_sql(table, timestamp)
|
60
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
61
|
+
table_metadata_table_name,
|
62
|
+
table_metadata_enabled_at_column_name,
|
63
|
+
quoted(timestamp),
|
64
|
+
table_metadata_table_name_column_name,
|
65
|
+
quoted(table.name),
|
66
|
+
table_metadata_enabled_at_column_name,
|
67
|
+
]
|
68
|
+
end
|
69
|
+
|
48
70
|
def ensure_table_metadata_exists_for_table_sql(table)
|
49
71
|
'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
|
50
72
|
table_metadata_table_name,
|
@@ -60,16 +82,16 @@ module Purview
|
|
60
82
|
'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
|
61
83
|
table_metadata_table_name,
|
62
84
|
table_metadata_table_name_column_definition,
|
63
|
-
|
85
|
+
table_metadata_enabled_at_column_definition,
|
64
86
|
table_metadata_last_pulled_at_column_definition,
|
65
87
|
table_metadata_locked_at_column_definition,
|
66
88
|
table_metadata_max_timestamp_pulled_column_definition,
|
67
89
|
]
|
68
90
|
end
|
69
91
|
|
70
|
-
def
|
92
|
+
def get_enabled_at_for_table_sql(table)
|
71
93
|
'SELECT %s FROM %s WHERE %s = %s' % [
|
72
|
-
|
94
|
+
table_metadata_enabled_at_column_name,
|
73
95
|
table_metadata_table_name,
|
74
96
|
table_metadata_table_name_column_name,
|
75
97
|
quoted(table.name),
|
@@ -122,27 +144,23 @@ module Purview
|
|
122
144
|
end
|
123
145
|
|
124
146
|
def next_table_sql(timestamp)
|
125
|
-
'SELECT %s FROM %s WHERE %s
|
147
|
+
'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
|
126
148
|
table_metadata_table_name_column_name,
|
127
149
|
table_metadata_table_name,
|
128
|
-
|
129
|
-
quoted(true_value),
|
150
|
+
table_metadata_enabled_at_column_name,
|
130
151
|
table_metadata_locked_at_column_name,
|
131
152
|
table_metadata_last_pulled_at_column_name,
|
132
153
|
table_metadata_last_pulled_at_column_name,
|
133
154
|
]
|
134
155
|
end
|
135
156
|
|
136
|
-
def
|
137
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND
|
157
|
+
def set_enabled_at_for_table_sql(table, timestamp)
|
158
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
|
138
159
|
table_metadata_table_name,
|
139
|
-
|
140
|
-
quoted(
|
160
|
+
table_metadata_enabled_at_column_name,
|
161
|
+
quoted(timestamp),
|
141
162
|
table_metadata_table_name_column_name,
|
142
163
|
quoted(table.name),
|
143
|
-
table_metadata_enabled_column_name,
|
144
|
-
table_metadata_enabled_column_name,
|
145
|
-
quoted(false_value),
|
146
164
|
]
|
147
165
|
end
|
148
166
|
|
data/lib/purview/exceptions.rb
CHANGED
@@ -2,6 +2,7 @@ require 'purview/exceptions/base'
|
|
2
2
|
require 'purview/exceptions/base_table'
|
3
3
|
|
4
4
|
require 'purview/exceptions/could_not_acquire_lock'
|
5
|
+
require 'purview/exceptions/database_already_assigned'
|
5
6
|
require 'purview/exceptions/lock_already_released'
|
6
7
|
require 'purview/exceptions/no_table'
|
7
8
|
require 'purview/exceptions/no_window'
|
@@ -18,13 +18,13 @@ module Purview
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def connection_opts
|
21
|
-
|
21
|
+
{
|
22
22
|
:database => database_name,
|
23
23
|
:host => database_host,
|
24
24
|
:password => database_password,
|
25
25
|
:port => database_port,
|
26
|
-
:username => database_username
|
27
|
-
|
26
|
+
:username => database_username,
|
27
|
+
}
|
28
28
|
end
|
29
29
|
|
30
30
|
def connection_type
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('jdbc-mysql')
|
2
|
+
|
3
|
+
if defined?(Jdbc::MySQL)
|
2
4
|
Jdbc::MySQL.load_driver
|
3
5
|
|
4
6
|
module Purview
|
@@ -10,6 +12,10 @@ if jruby? && safe_require('jdbc/mysql')
|
|
10
12
|
def engine
|
11
13
|
'mysql'
|
12
14
|
end
|
15
|
+
|
16
|
+
def username
|
17
|
+
super || ENV['USER'] || Etc.getlogin
|
18
|
+
end
|
13
19
|
end
|
14
20
|
end
|
15
21
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('jdbc-postgres')
|
2
|
+
|
3
|
+
if defined?(Jdbc::Postgres)
|
2
4
|
Jdbc::Postgres.load_driver
|
3
5
|
|
4
6
|
module Purview
|
@@ -10,6 +12,10 @@ if jruby? && safe_require('jdbc/postgres')
|
|
10
12
|
def engine
|
11
13
|
'postgresql'
|
12
14
|
end
|
15
|
+
|
16
|
+
def username
|
17
|
+
super || ENV['PGUSER'] || Etc.getlogin
|
18
|
+
end
|
13
19
|
end
|
14
20
|
end
|
15
21
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('mysql2')
|
2
|
+
|
3
|
+
if defined?(Mysql2)
|
2
4
|
module Purview
|
3
5
|
module RawConnections
|
4
6
|
class Mysql2 < Base
|
@@ -18,15 +20,19 @@ if !jruby? && safe_require('mysql2')
|
|
18
20
|
|
19
21
|
def new_connection
|
20
22
|
::Mysql2::Client.new(
|
21
|
-
|
22
|
-
:database => database,
|
23
|
-
:host => host,
|
24
|
-
:password => password,
|
23
|
+
filter_blank_values(
|
24
|
+
:database => database.to_s,
|
25
|
+
:host => host.to_s,
|
26
|
+
:password => password.to_s,
|
25
27
|
:port => port,
|
26
|
-
:username => username
|
28
|
+
:username => username.to_s
|
27
29
|
)
|
28
30
|
)
|
29
31
|
end
|
32
|
+
|
33
|
+
def username
|
34
|
+
super || ENV['USER'] || Etc.getlogin
|
35
|
+
end
|
30
36
|
end
|
31
37
|
end
|
32
38
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('pg')
|
2
|
+
|
3
|
+
if defined?(PG)
|
2
4
|
module Purview
|
3
5
|
module RawConnections
|
4
6
|
class PG < Base
|
@@ -27,6 +29,10 @@ if !jruby? && safe_require('pg')
|
|
27
29
|
)
|
28
30
|
)
|
29
31
|
end
|
32
|
+
|
33
|
+
def username
|
34
|
+
super || ENV['PGUSER'] || Etc.getlogin
|
35
|
+
end
|
30
36
|
end
|
31
37
|
end
|
32
38
|
end
|
@@ -6,3 +6,7 @@ require 'purview/raw_connections/jdbc/postgres'
|
|
6
6
|
|
7
7
|
require 'purview/raw_connections/mysql2'
|
8
8
|
require 'purview/raw_connections/pg'
|
9
|
+
|
10
|
+
if !defined?(Purview::RawConnections::MySQL) && !defined?(Purview::RawConnections::PostgreSQL)
|
11
|
+
raise 'Could not initialize raw-connections; please install and require one or more of the following gems: `jdbc-mysql`, `jdbc-postgres`, `mysql2` and/or `pg`'
|
12
|
+
end
|
data/lib/purview/tables/base.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Purview
|
2
2
|
module Tables
|
3
3
|
class Base
|
4
|
-
attr_reader :name
|
4
|
+
attr_reader :database, :name
|
5
5
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
@@ -40,8 +40,9 @@ module Purview
|
|
40
40
|
]
|
41
41
|
end
|
42
42
|
|
43
|
-
def database
|
44
|
-
|
43
|
+
def database=(value)
|
44
|
+
raise Purview::Exceptions::DatabaseAlreadyAssigned.new(self) if database
|
45
|
+
@database = value
|
45
46
|
end
|
46
47
|
|
47
48
|
def id_column
|
data/lib/purview/version.rb
CHANGED
data/lib/purview.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: purview
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan W. Zaleski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -112,11 +112,11 @@ files:
|
|
112
112
|
- lib/purview/exceptions/base.rb
|
113
113
|
- lib/purview/exceptions/base_table.rb
|
114
114
|
- lib/purview/exceptions/could_not_acquire_lock.rb
|
115
|
+
- lib/purview/exceptions/database_already_assigned.rb
|
115
116
|
- lib/purview/exceptions/lock_already_released.rb
|
116
117
|
- lib/purview/exceptions/no_table.rb
|
117
118
|
- lib/purview/exceptions/no_window.rb
|
118
119
|
- lib/purview/exceptions/rows_outside_window.rb
|
119
|
-
- lib/purview/helpers.rub
|
120
120
|
- lib/purview/loaders.rb
|
121
121
|
- lib/purview/loaders/base.rb
|
122
122
|
- lib/purview/loaders/mysql.rb
|
data/lib/purview/helpers.rub
DELETED
File without changes
|