purview 1.0.0.beta2 → 1.0.0.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +25 -25
- data/lib/purview/databases/base.rb +173 -118
- data/lib/purview/databases/mysql.rb +31 -13
- data/lib/purview/databases/postgresql.rb +31 -13
- data/lib/purview/exceptions/database_already_assigned.rb +9 -0
- data/lib/purview/exceptions.rb +1 -0
- data/lib/purview/mixins/helpers.rb +4 -0
- data/lib/purview/pullers/base_sql.rb +3 -3
- data/lib/purview/raw_connections/base.rb +1 -1
- data/lib/purview/raw_connections/jdbc/mysql.rb +7 -1
- data/lib/purview/raw_connections/jdbc/postgres.rb +7 -1
- data/lib/purview/raw_connections/mysql2.rb +12 -6
- data/lib/purview/raw_connections/pg.rb +7 -1
- data/lib/purview/raw_connections.rb +4 -0
- data/lib/purview/tables/base.rb +4 -3
- data/lib/purview/version.rb +1 -1
- data/lib/purview.rb +1 -5
- metadata +3 -3
- data/lib/purview/helpers.rub +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e042a9a30d7596c9a7dad1ce2e121b9f9bf160f
|
4
|
+
data.tar.gz: b4a1f379840183ae0a11f411a409849ed06655b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 52be209e82452fa5ec45ce2726c516ef315cb32de3c67b2b74862def7a29fb4fd7c95ff4da270de4f8f5262b0a4aa4ad11c35e6ffbc2e27c3bce366e998a8d8e
|
7
|
+
data.tar.gz: c5768e07904cd00ae25d18adfb163ff5516d2337c2451812c1313ec5e1b183ec2ffdddcbb08b6a434607931e7ccbeaec48ea91e8f13a2d118c7b8461cf95529a
|
data/README.md
CHANGED
@@ -26,22 +26,6 @@ Load the `MySQL` client (for `PostgreSQL` simply change 'mysql2' to 'pg')
|
|
26
26
|
require 'mysql2'
|
27
27
|
```
|
28
28
|
|
29
|
-
Set the database-name (this can be anything, but it must exist)
|
30
|
-
```ruby
|
31
|
-
database_name = :data_warehouse
|
32
|
-
```
|
33
|
-
|
34
|
-
Combine all the configuration options and instantiate the `Database` (for
|
35
|
-
`PostgreSQL` simply change `MySQL` to `PostgreSQL`)
|
36
|
-
```ruby
|
37
|
-
database_opts = {}
|
38
|
-
|
39
|
-
database = Purview::Databases::MySQL.new(
|
40
|
-
database_name,
|
41
|
-
database_opts
|
42
|
-
)
|
43
|
-
```
|
44
|
-
|
45
29
|
Set the table-name (this can be anything, but it must exist)
|
46
30
|
```ruby
|
47
31
|
table_name = :users
|
@@ -93,7 +77,6 @@ Combine all the configuration options and instantiate the `Table`
|
|
93
77
|
```ruby
|
94
78
|
table_opts = {
|
95
79
|
:columns => columns,
|
96
|
-
:database => database,
|
97
80
|
:loader => loader_opts,
|
98
81
|
:parser => parser_opts,
|
99
82
|
:puller => puller_opts,
|
@@ -106,22 +89,39 @@ table = Purview::Tables::Raw.new(
|
|
106
89
|
)
|
107
90
|
```
|
108
91
|
|
92
|
+
Set the database-name (this can be anything, but it must exist)
|
93
|
+
```ruby
|
94
|
+
database_name = :data_warehouse
|
95
|
+
```
|
96
|
+
|
97
|
+
Combine all the configuration options and instantiate the `Database` (for
|
98
|
+
`PostgreSQL` simply change `MySQL` to `PostgreSQL`)
|
99
|
+
```ruby
|
100
|
+
database_opts = {
|
101
|
+
:tables => [table],
|
102
|
+
}
|
103
|
+
|
104
|
+
database = Purview::Databases::MySQL.new(
|
105
|
+
database_name,
|
106
|
+
database_opts
|
107
|
+
)
|
108
|
+
```
|
109
|
+
|
109
110
|
Add the `Table` to the `Database` (schema). In order for [the] `Table` to be
|
110
111
|
`sync[ed]` it *must* be added to [the] `Database`
|
111
112
|
```ruby
|
112
113
|
database.add_table(table)
|
113
114
|
```
|
114
115
|
|
115
|
-
Create the `Table
|
116
|
-
environments you will likely want an external process to manage the
|
117
|
-
`PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
|
116
|
+
Create the `Table` (in the DB). Recommended for testing purposes *only*. For
|
117
|
+
production environments you will likely want an external process to manage the
|
118
|
+
schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
|
118
119
|
```ruby
|
119
120
|
begin
|
120
|
-
database.create_table(
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
rescue Mysql2::Error; end
|
121
|
+
database.create_table(table)
|
122
|
+
rescue Mysql2::Error
|
123
|
+
# Swallow
|
124
|
+
end
|
125
125
|
```
|
126
126
|
|
127
127
|
Sync the `Database`. This process will select a [candidate] `Table`, pull data
|
@@ -6,60 +6,39 @@ module Purview
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
8
8
|
@opts = opts
|
9
|
-
@tables = Set.new
|
10
9
|
end
|
11
10
|
|
12
|
-
def
|
13
|
-
@tables << table
|
14
|
-
end
|
15
|
-
|
16
|
-
def create_index(connection, table, columns, opts={})
|
11
|
+
def create_table(table, opts={})
|
17
12
|
table_opts = extract_table_options(opts)
|
18
13
|
table_name = table_name(table, table_opts)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
).tap do |index_name|
|
25
|
-
connection.execute(
|
26
|
-
create_index_sql(
|
27
|
-
table_name,
|
28
|
-
index_name,
|
29
|
-
table,
|
30
|
-
columns,
|
31
|
-
index_opts
|
32
|
-
)
|
33
|
-
)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def create_table(connection, table, opts={})
|
38
|
-
table_opts = extract_table_options(opts)
|
39
|
-
table_name(table, table_opts).tap do |table_name|
|
40
|
-
connection.execute(
|
41
|
-
create_table_sql(
|
42
|
-
table_name,
|
43
|
-
table,
|
44
|
-
table_opts
|
45
|
-
)
|
46
|
-
)
|
47
|
-
if table_opts[:create_indices]
|
48
|
-
table.indexed_columns.each do |columns|
|
49
|
-
create_index(
|
50
|
-
connection,
|
14
|
+
with_context_logging("`create_table` for: #{table_name}") do
|
15
|
+
with_new_connection do |connection|
|
16
|
+
connection.execute(
|
17
|
+
create_table_sql(
|
18
|
+
table_name,
|
51
19
|
table,
|
52
|
-
|
53
|
-
:table => { :name => table_name }
|
20
|
+
table_opts
|
54
21
|
)
|
22
|
+
)
|
23
|
+
if table_opts[:create_indices]
|
24
|
+
table.indexed_columns.each do |columns|
|
25
|
+
create_index(
|
26
|
+
connection,
|
27
|
+
table,
|
28
|
+
columns,
|
29
|
+
:table => { :name => table_name }
|
30
|
+
)
|
31
|
+
end
|
55
32
|
end
|
56
33
|
end
|
34
|
+
table_name
|
57
35
|
end
|
58
36
|
end
|
59
37
|
|
60
38
|
def create_temporary_table(connection, table, opts={})
|
61
39
|
table_opts = extract_table_options(opts)
|
62
|
-
table_name(table, table_opts)
|
40
|
+
table_name = table_name(table, table_opts)
|
41
|
+
with_context_logging("`create_temporary_table` for: #{table_name}") do
|
63
42
|
connection.execute(
|
64
43
|
create_temporary_table_sql(
|
65
44
|
table_name,
|
@@ -77,89 +56,79 @@ module Purview
|
|
77
56
|
)
|
78
57
|
end
|
79
58
|
end
|
59
|
+
table_name
|
80
60
|
end
|
81
61
|
end
|
82
62
|
|
83
63
|
def disable_table(table)
|
84
|
-
|
64
|
+
table_name = table_name(table)
|
65
|
+
with_context_logging("`disable_table` for: #{table_name}") do
|
85
66
|
with_new_connection do |connection|
|
86
|
-
|
87
|
-
connection
|
88
|
-
|
89
|
-
|
90
|
-
)
|
67
|
+
rows_affected = \
|
68
|
+
connection.execute(disable_table_sql(table)).rows_affected
|
69
|
+
raise Purview::Exceptions::TableAlreadyDisabled.new(table) \
|
70
|
+
if zero?(rows_affected)
|
91
71
|
end
|
72
|
+
table_name
|
92
73
|
end
|
93
74
|
end
|
94
75
|
|
95
|
-
def drop_table(
|
96
|
-
table_opts = extract_table_options(opts)
|
97
|
-
table_name(table, table_opts).tap do |table_name|
|
98
|
-
connection.execute(
|
99
|
-
drop_table_sql(
|
100
|
-
table_name,
|
101
|
-
table,
|
102
|
-
table_opts
|
103
|
-
)
|
104
|
-
)
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def drop_index(connection, table, columns, opts={})
|
76
|
+
def drop_table(table, opts={})
|
109
77
|
table_opts = extract_table_options(opts)
|
110
78
|
table_name = table_name(table, table_opts)
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
table_name,
|
120
|
-
index_name,
|
121
|
-
table,
|
122
|
-
columns,
|
123
|
-
index_opts
|
79
|
+
with_context_logging("`drop_table` for: #{table_name}") do
|
80
|
+
with_new_connection do |connection|
|
81
|
+
connection.execute(
|
82
|
+
drop_table_sql(
|
83
|
+
table_name,
|
84
|
+
table,
|
85
|
+
table_opts
|
86
|
+
)
|
124
87
|
)
|
125
|
-
|
88
|
+
end
|
89
|
+
table_name
|
126
90
|
end
|
127
91
|
end
|
128
92
|
|
129
|
-
def enable_table(table)
|
130
|
-
|
93
|
+
def enable_table(table, timestamp=Time.now.utc)
|
94
|
+
table_name = table_name(table)
|
95
|
+
with_context_logging("`enable_table` for: #{table_name}") do
|
131
96
|
with_new_connection do |connection|
|
132
|
-
|
133
|
-
connection,
|
134
|
-
|
135
|
-
|
136
|
-
)
|
97
|
+
rows_affected = \
|
98
|
+
connection.execute(enable_table_sql(table, timestamp)).rows_affected
|
99
|
+
raise Purview::Exceptions::TableAlreadyEnabled.new(table) \
|
100
|
+
if zero?(rows_affected)
|
137
101
|
end
|
102
|
+
table_name
|
138
103
|
end
|
139
104
|
end
|
140
105
|
|
141
|
-
def lock_table(table, timestamp)
|
142
|
-
|
106
|
+
def lock_table(table, timestamp=Time.now.utc)
|
107
|
+
table_name = table_name(table)
|
108
|
+
with_context_logging("`lock_table` for: #{table_name}") do
|
143
109
|
with_new_connection do |connection|
|
144
110
|
rows_affected = \
|
145
111
|
connection.execute(lock_table_sql(table, timestamp)).rows_affected
|
146
|
-
raise Purview::Exceptions::
|
112
|
+
raise Purview::Exceptions::TableAlreadyLocked.new(table) \
|
147
113
|
if zero?(rows_affected)
|
148
114
|
end
|
115
|
+
table_name
|
149
116
|
end
|
150
117
|
end
|
151
118
|
|
152
119
|
def sync
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
table
|
120
|
+
with_context_logging('`sync`') do
|
121
|
+
with_new_connection do |connection|
|
122
|
+
with_transaction(connection) do |timestamp|
|
123
|
+
with_next_table(connection, timestamp) do |table|
|
124
|
+
with_next_window(
|
125
|
+
connection,
|
126
|
+
table,
|
127
|
+
timestamp
|
128
|
+
) do |window|
|
129
|
+
with_table_locked(table, timestamp) do
|
130
|
+
table.sync(connection, window)
|
131
|
+
end
|
163
132
|
end
|
164
133
|
end
|
165
134
|
end
|
@@ -168,13 +137,15 @@ module Purview
|
|
168
137
|
end
|
169
138
|
|
170
139
|
def unlock_table(table)
|
171
|
-
|
140
|
+
table_name = table_name(table)
|
141
|
+
with_context_logging("`unlock_table` for: #{table_name}") do
|
172
142
|
with_new_connection do |connection|
|
173
143
|
rows_affected = \
|
174
144
|
connection.execute(unlock_table_sql(table)).rows_affected
|
175
|
-
raise Purview::Exceptions::
|
145
|
+
raise Purview::Exceptions::TableAlreadyUnlocked.new(table) \
|
176
146
|
if zero?(rows_affected)
|
177
147
|
end
|
148
|
+
table_name
|
178
149
|
end
|
179
150
|
end
|
180
151
|
|
@@ -184,7 +155,7 @@ module Purview
|
|
184
155
|
include Purview::Mixins::Helpers
|
185
156
|
include Purview::Mixins::Logger
|
186
157
|
|
187
|
-
attr_reader :opts
|
158
|
+
attr_reader :opts
|
188
159
|
|
189
160
|
public :connect
|
190
161
|
|
@@ -219,13 +190,40 @@ module Purview
|
|
219
190
|
end
|
220
191
|
|
221
192
|
def connection_opts
|
222
|
-
{
|
193
|
+
{
|
194
|
+
:database => database_name,
|
195
|
+
:host => database_host,
|
196
|
+
:password => database_password,
|
197
|
+
:port => database_port,
|
198
|
+
:username => database_username,
|
199
|
+
}
|
223
200
|
end
|
224
201
|
|
225
202
|
def connection_type
|
226
203
|
raise %{All "#{Base}(s)" must override the "connection_type" method}
|
227
204
|
end
|
228
205
|
|
206
|
+
def create_index(connection, table, columns, opts={})
|
207
|
+
table_opts = extract_table_options(opts)
|
208
|
+
table_name = table_name(table, table_opts)
|
209
|
+
index_opts = extract_index_options(opts)
|
210
|
+
index_name(
|
211
|
+
table_name,
|
212
|
+
columns,
|
213
|
+
index_opts
|
214
|
+
).tap do |index_name|
|
215
|
+
connection.execute(
|
216
|
+
create_index_sql(
|
217
|
+
table_name,
|
218
|
+
index_name,
|
219
|
+
table,
|
220
|
+
columns,
|
221
|
+
index_opts
|
222
|
+
)
|
223
|
+
)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
229
227
|
def create_index_sql(table_name, index_name, table, columns, index_opts={})
|
230
228
|
raise %{All "#{Base}(s)" must override the "create_index_sql" method}
|
231
229
|
end
|
@@ -238,6 +236,26 @@ module Purview
|
|
238
236
|
raise %{All "#{Base}(s)" must override the "create_temporary_table_sql" method}
|
239
237
|
end
|
240
238
|
|
239
|
+
def database_host
|
240
|
+
opts[:host]
|
241
|
+
end
|
242
|
+
|
243
|
+
def database_name
|
244
|
+
name
|
245
|
+
end
|
246
|
+
|
247
|
+
def database_password
|
248
|
+
opts[:password]
|
249
|
+
end
|
250
|
+
|
251
|
+
def database_port
|
252
|
+
opts[:port]
|
253
|
+
end
|
254
|
+
|
255
|
+
def database_username
|
256
|
+
opts[:username]
|
257
|
+
end
|
258
|
+
|
241
259
|
def default(column)
|
242
260
|
column.default || default_map[column.type]
|
243
261
|
end
|
@@ -254,6 +272,31 @@ module Purview
|
|
254
272
|
raise %{All "#{Base}(s)" must override the "dialect_type" method}
|
255
273
|
end
|
256
274
|
|
275
|
+
def disable_table_sql(table)
|
276
|
+
raise %{All "#{Base}(s)" must override the "disable_table_sql" method}
|
277
|
+
end
|
278
|
+
|
279
|
+
def drop_index(table, columns, opts={})
|
280
|
+
table_opts = extract_table_options(opts)
|
281
|
+
table_name = table_name(table, table_opts)
|
282
|
+
index_opts = extract_index_options(opts)
|
283
|
+
index_name(
|
284
|
+
table_name,
|
285
|
+
columns,
|
286
|
+
index_opts
|
287
|
+
).tap do |index_name|
|
288
|
+
connection.execute(
|
289
|
+
drop_index_sql(
|
290
|
+
table_name,
|
291
|
+
index_name,
|
292
|
+
table,
|
293
|
+
columns,
|
294
|
+
index_opts
|
295
|
+
)
|
296
|
+
)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
257
300
|
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
258
301
|
raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
|
259
302
|
end
|
@@ -262,6 +305,10 @@ module Purview
|
|
262
305
|
raise %{All "#{Base}(s)" must override the "drop_table_sql" method}
|
263
306
|
end
|
264
307
|
|
308
|
+
def enable_table_sql(table, timestamp)
|
309
|
+
raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
|
310
|
+
end
|
311
|
+
|
265
312
|
def ensure_table_metadata_table_exists
|
266
313
|
with_new_connection do |connection|
|
267
314
|
connection.execute(ensure_table_metadata_table_exists_sql)
|
@@ -296,14 +343,14 @@ module Purview
|
|
296
343
|
dialect.false_value
|
297
344
|
end
|
298
345
|
|
299
|
-
def
|
300
|
-
row = connection.execute(
|
301
|
-
|
302
|
-
|
346
|
+
def get_enabled_at_for_table(connection, table)
|
347
|
+
row = connection.execute(get_enabled_at_for_table_sql(table)).rows[0]
|
348
|
+
timestamp = row[table_metadata_enabled_at_column_name]
|
349
|
+
timestamp ? Time.parse(timestamp) : nil
|
303
350
|
end
|
304
351
|
|
305
|
-
def
|
306
|
-
raise %{All "#{Base}(s)" must override the "
|
352
|
+
def get_enabled_at_for_table_sql(table)
|
353
|
+
raise %{All "#{Base}(s)" must override the "get_enabled_at_for_table_sql" method}
|
307
354
|
end
|
308
355
|
|
309
356
|
def get_last_pulled_at_for_table(connection, table)
|
@@ -401,12 +448,12 @@ module Purview
|
|
401
448
|
dialect.sanitized(value)
|
402
449
|
end
|
403
450
|
|
404
|
-
def
|
405
|
-
connection.execute(
|
451
|
+
def set_enabled_at_for_table(connection, table, timestamp)
|
452
|
+
connection.execute(set_enabled_at_for_table_sql(table, timestamp))
|
406
453
|
end
|
407
454
|
|
408
|
-
def
|
409
|
-
raise %{All "#{Base}(s)" must override the "
|
455
|
+
def set_enabled_at_for_table_sql(table, timestamp)
|
456
|
+
raise %{All "#{Base}(s)" must override the "set_enabled_at_for_table_sql" method}
|
410
457
|
end
|
411
458
|
|
412
459
|
def set_last_pulled_at_for_table(connection, table, timestamp)
|
@@ -433,13 +480,13 @@ module Purview
|
|
433
480
|
raise %{All "#{Base}(s)" must override the "set_max_timestamp_pulled_for_table_sql" method}
|
434
481
|
end
|
435
482
|
|
436
|
-
def
|
437
|
-
column = Purview::Columns::
|
483
|
+
def table_metadata_enabled_at_column_definition
|
484
|
+
column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
|
438
485
|
column_definition(column)
|
439
486
|
end
|
440
487
|
|
441
|
-
def
|
442
|
-
'
|
488
|
+
def table_metadata_enabled_at_column_name
|
489
|
+
'enabled_at'
|
443
490
|
end
|
444
491
|
|
445
492
|
def table_metadata_last_pulled_at_column_definition
|
@@ -482,18 +529,26 @@ module Purview
|
|
482
529
|
'table_name'
|
483
530
|
end
|
484
531
|
|
532
|
+
def table_name(table, table_opts={})
|
533
|
+
table_opts[:name] || table.name
|
534
|
+
end
|
535
|
+
|
536
|
+
def tables
|
537
|
+
@tables ||= Set.new.tap do |result|
|
538
|
+
(opts[:tables] || []).each do |table|
|
539
|
+
table.database = self if result.add?(table)
|
540
|
+
end
|
541
|
+
end
|
542
|
+
end
|
543
|
+
|
485
544
|
def tables_by_name
|
486
|
-
{}.tap do |result|
|
545
|
+
@tables_by_name ||= {}.tap do |result|
|
487
546
|
tables.each do |table|
|
488
547
|
result[table.name] = table
|
489
548
|
end
|
490
549
|
end
|
491
550
|
end
|
492
551
|
|
493
|
-
def table_name(table, table_opts={})
|
494
|
-
table_opts[:name] || table.name
|
495
|
-
end
|
496
|
-
|
497
552
|
def true_value
|
498
553
|
dialect.true_value
|
499
554
|
end
|
@@ -37,6 +37,17 @@ module Purview
|
|
37
37
|
Purview::Dialects::MySQL
|
38
38
|
end
|
39
39
|
|
40
|
+
def disable_table_sql(table)
|
41
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
42
|
+
table_metadata_table_name,
|
43
|
+
table_metadata_enabled_at_column_name,
|
44
|
+
null_value,
|
45
|
+
table_metadata_table_name_column_name,
|
46
|
+
quoted(table.name),
|
47
|
+
table_metadata_enabled_at_column_name,
|
48
|
+
]
|
49
|
+
end
|
50
|
+
|
40
51
|
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
41
52
|
'DROP INDEX %s' % [
|
42
53
|
index_name,
|
@@ -49,6 +60,17 @@ module Purview
|
|
49
60
|
]
|
50
61
|
end
|
51
62
|
|
63
|
+
def enable_table_sql(table, timestamp)
|
64
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
65
|
+
table_metadata_table_name,
|
66
|
+
table_metadata_enabled_at_column_name,
|
67
|
+
quoted(timestamp),
|
68
|
+
table_metadata_table_name_column_name,
|
69
|
+
quoted(table.name),
|
70
|
+
table_metadata_enabled_at_column_name,
|
71
|
+
]
|
72
|
+
end
|
73
|
+
|
52
74
|
def ensure_table_metadata_exists_for_table_sql(table)
|
53
75
|
'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
|
54
76
|
table_metadata_table_name,
|
@@ -60,16 +82,16 @@ module Purview
|
|
60
82
|
'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
|
61
83
|
table_metadata_table_name,
|
62
84
|
table_metadata_table_name_column_definition,
|
63
|
-
|
85
|
+
table_metadata_enabled_at_column_definition,
|
64
86
|
table_metadata_last_pulled_at_column_definition,
|
65
87
|
table_metadata_locked_at_column_definition,
|
66
88
|
table_metadata_max_timestamp_pulled_column_definition,
|
67
89
|
]
|
68
90
|
end
|
69
91
|
|
70
|
-
def
|
92
|
+
def get_enabled_at_for_table_sql(table)
|
71
93
|
'SELECT %s FROM %s WHERE %s = %s' % [
|
72
|
-
|
94
|
+
table_metadata_enabled_at_column_name,
|
73
95
|
table_metadata_table_name,
|
74
96
|
table_metadata_table_name_column_name,
|
75
97
|
quoted(table.name),
|
@@ -119,27 +141,23 @@ module Purview
|
|
119
141
|
end
|
120
142
|
|
121
143
|
def next_table_sql(timestamp)
|
122
|
-
'SELECT %s FROM %s WHERE %s
|
144
|
+
'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
|
123
145
|
table_metadata_table_name_column_name,
|
124
146
|
table_metadata_table_name,
|
125
|
-
|
126
|
-
quoted(true_value),
|
147
|
+
table_metadata_enabled_at_column_name,
|
127
148
|
table_metadata_locked_at_column_name,
|
128
149
|
table_metadata_last_pulled_at_column_name,
|
129
150
|
table_metadata_last_pulled_at_column_name,
|
130
151
|
]
|
131
152
|
end
|
132
153
|
|
133
|
-
def
|
134
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND
|
154
|
+
def set_enabled_at_for_table_sql(table, timestamp)
|
155
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
|
135
156
|
table_metadata_table_name,
|
136
|
-
|
137
|
-
quoted(
|
157
|
+
table_metadata_enabled_at_column_name,
|
158
|
+
quoted(timestamp),
|
138
159
|
table_metadata_table_name_column_name,
|
139
160
|
quoted(table.name),
|
140
|
-
table_metadata_enabled_column_name,
|
141
|
-
table_metadata_enabled_column_name,
|
142
|
-
quoted(false_value),
|
143
161
|
]
|
144
162
|
end
|
145
163
|
|
@@ -33,6 +33,17 @@ module Purview
|
|
33
33
|
Purview::Dialects::PostgreSQL
|
34
34
|
end
|
35
35
|
|
36
|
+
def disable_table_sql(table)
|
37
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
|
38
|
+
table_metadata_table_name,
|
39
|
+
table_metadata_enabled_at_column_name,
|
40
|
+
null_value,
|
41
|
+
table_metadata_table_name_column_name,
|
42
|
+
quoted(table.name),
|
43
|
+
table_metadata_enabled_at_column_name,
|
44
|
+
]
|
45
|
+
end
|
46
|
+
|
36
47
|
def drop_index_sql(table_name, index_name, table, columns, index_opts={})
|
37
48
|
'DROP INDEX %s' % [
|
38
49
|
index_name,
|
@@ -45,6 +56,17 @@ module Purview
|
|
45
56
|
]
|
46
57
|
end
|
47
58
|
|
59
|
+
def enable_table_sql(table, timestamp)
|
60
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
|
61
|
+
table_metadata_table_name,
|
62
|
+
table_metadata_enabled_at_column_name,
|
63
|
+
quoted(timestamp),
|
64
|
+
table_metadata_table_name_column_name,
|
65
|
+
quoted(table.name),
|
66
|
+
table_metadata_enabled_at_column_name,
|
67
|
+
]
|
68
|
+
end
|
69
|
+
|
48
70
|
def ensure_table_metadata_exists_for_table_sql(table)
|
49
71
|
'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
|
50
72
|
table_metadata_table_name,
|
@@ -60,16 +82,16 @@ module Purview
|
|
60
82
|
'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
|
61
83
|
table_metadata_table_name,
|
62
84
|
table_metadata_table_name_column_definition,
|
63
|
-
|
85
|
+
table_metadata_enabled_at_column_definition,
|
64
86
|
table_metadata_last_pulled_at_column_definition,
|
65
87
|
table_metadata_locked_at_column_definition,
|
66
88
|
table_metadata_max_timestamp_pulled_column_definition,
|
67
89
|
]
|
68
90
|
end
|
69
91
|
|
70
|
-
def
|
92
|
+
def get_enabled_at_for_table_sql(table)
|
71
93
|
'SELECT %s FROM %s WHERE %s = %s' % [
|
72
|
-
|
94
|
+
table_metadata_enabled_at_column_name,
|
73
95
|
table_metadata_table_name,
|
74
96
|
table_metadata_table_name_column_name,
|
75
97
|
quoted(table.name),
|
@@ -122,27 +144,23 @@ module Purview
|
|
122
144
|
end
|
123
145
|
|
124
146
|
def next_table_sql(timestamp)
|
125
|
-
'SELECT %s FROM %s WHERE %s
|
147
|
+
'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
|
126
148
|
table_metadata_table_name_column_name,
|
127
149
|
table_metadata_table_name,
|
128
|
-
|
129
|
-
quoted(true_value),
|
150
|
+
table_metadata_enabled_at_column_name,
|
130
151
|
table_metadata_locked_at_column_name,
|
131
152
|
table_metadata_last_pulled_at_column_name,
|
132
153
|
table_metadata_last_pulled_at_column_name,
|
133
154
|
]
|
134
155
|
end
|
135
156
|
|
136
|
-
def
|
137
|
-
'UPDATE %s SET %s = %s WHERE %s = %s AND
|
157
|
+
def set_enabled_at_for_table_sql(table, timestamp)
|
158
|
+
'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
|
138
159
|
table_metadata_table_name,
|
139
|
-
|
140
|
-
quoted(
|
160
|
+
table_metadata_enabled_at_column_name,
|
161
|
+
quoted(timestamp),
|
141
162
|
table_metadata_table_name_column_name,
|
142
163
|
quoted(table.name),
|
143
|
-
table_metadata_enabled_column_name,
|
144
|
-
table_metadata_enabled_column_name,
|
145
|
-
quoted(false_value),
|
146
164
|
]
|
147
165
|
end
|
148
166
|
|
data/lib/purview/exceptions.rb
CHANGED
@@ -2,6 +2,7 @@ require 'purview/exceptions/base'
|
|
2
2
|
require 'purview/exceptions/base_table'
|
3
3
|
|
4
4
|
require 'purview/exceptions/could_not_acquire_lock'
|
5
|
+
require 'purview/exceptions/database_already_assigned'
|
5
6
|
require 'purview/exceptions/lock_already_released'
|
6
7
|
require 'purview/exceptions/no_table'
|
7
8
|
require 'purview/exceptions/no_window'
|
@@ -18,13 +18,13 @@ module Purview
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def connection_opts
|
21
|
-
|
21
|
+
{
|
22
22
|
:database => database_name,
|
23
23
|
:host => database_host,
|
24
24
|
:password => database_password,
|
25
25
|
:port => database_port,
|
26
|
-
:username => database_username
|
27
|
-
|
26
|
+
:username => database_username,
|
27
|
+
}
|
28
28
|
end
|
29
29
|
|
30
30
|
def connection_type
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('jdbc-mysql')
|
2
|
+
|
3
|
+
if defined?(Jdbc::MySQL)
|
2
4
|
Jdbc::MySQL.load_driver
|
3
5
|
|
4
6
|
module Purview
|
@@ -10,6 +12,10 @@ if jruby? && safe_require('jdbc/mysql')
|
|
10
12
|
def engine
|
11
13
|
'mysql'
|
12
14
|
end
|
15
|
+
|
16
|
+
def username
|
17
|
+
super || ENV['USER'] || Etc.getlogin
|
18
|
+
end
|
13
19
|
end
|
14
20
|
end
|
15
21
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('jdbc-postgres')
|
2
|
+
|
3
|
+
if defined?(Jdbc::Postgres)
|
2
4
|
Jdbc::Postgres.load_driver
|
3
5
|
|
4
6
|
module Purview
|
@@ -10,6 +12,10 @@ if jruby? && safe_require('jdbc/postgres')
|
|
10
12
|
def engine
|
11
13
|
'postgresql'
|
12
14
|
end
|
15
|
+
|
16
|
+
def username
|
17
|
+
super || ENV['PGUSER'] || Etc.getlogin
|
18
|
+
end
|
13
19
|
end
|
14
20
|
end
|
15
21
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('mysql2')
|
2
|
+
|
3
|
+
if defined?(Mysql2)
|
2
4
|
module Purview
|
3
5
|
module RawConnections
|
4
6
|
class Mysql2 < Base
|
@@ -18,15 +20,19 @@ if !jruby? && safe_require('mysql2')
|
|
18
20
|
|
19
21
|
def new_connection
|
20
22
|
::Mysql2::Client.new(
|
21
|
-
|
22
|
-
:database => database,
|
23
|
-
:host => host,
|
24
|
-
:password => password,
|
23
|
+
filter_blank_values(
|
24
|
+
:database => database.to_s,
|
25
|
+
:host => host.to_s,
|
26
|
+
:password => password.to_s,
|
25
27
|
:port => port,
|
26
|
-
:username => username
|
28
|
+
:username => username.to_s
|
27
29
|
)
|
28
30
|
)
|
29
31
|
end
|
32
|
+
|
33
|
+
def username
|
34
|
+
super || ENV['USER'] || Etc.getlogin
|
35
|
+
end
|
30
36
|
end
|
31
37
|
end
|
32
38
|
end
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
safe_require('pg')
|
2
|
+
|
3
|
+
if defined?(PG)
|
2
4
|
module Purview
|
3
5
|
module RawConnections
|
4
6
|
class PG < Base
|
@@ -27,6 +29,10 @@ if !jruby? && safe_require('pg')
|
|
27
29
|
)
|
28
30
|
)
|
29
31
|
end
|
32
|
+
|
33
|
+
def username
|
34
|
+
super || ENV['PGUSER'] || Etc.getlogin
|
35
|
+
end
|
30
36
|
end
|
31
37
|
end
|
32
38
|
end
|
@@ -6,3 +6,7 @@ require 'purview/raw_connections/jdbc/postgres'
|
|
6
6
|
|
7
7
|
require 'purview/raw_connections/mysql2'
|
8
8
|
require 'purview/raw_connections/pg'
|
9
|
+
|
10
|
+
if !defined?(Purview::RawConnections::MySQL) && !defined?(Purview::RawConnections::PostgreSQL)
|
11
|
+
raise 'Could not initialize raw-connections; please install and require one or more of the following gems: `jdbc-mysql`, `jdbc-postgres`, `mysql2` and/or `pg`'
|
12
|
+
end
|
data/lib/purview/tables/base.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Purview
|
2
2
|
module Tables
|
3
3
|
class Base
|
4
|
-
attr_reader :name
|
4
|
+
attr_reader :database, :name
|
5
5
|
|
6
6
|
def initialize(name, opts={})
|
7
7
|
@name = name
|
@@ -40,8 +40,9 @@ module Purview
|
|
40
40
|
]
|
41
41
|
end
|
42
42
|
|
43
|
-
def database
|
44
|
-
|
43
|
+
def database=(value)
|
44
|
+
raise Purview::Exceptions::DatabaseAlreadyAssigned.new(self) if database
|
45
|
+
@database = value
|
45
46
|
end
|
46
47
|
|
47
48
|
def id_column
|
data/lib/purview/version.rb
CHANGED
data/lib/purview.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: purview
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan W. Zaleski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -112,11 +112,11 @@ files:
|
|
112
112
|
- lib/purview/exceptions/base.rb
|
113
113
|
- lib/purview/exceptions/base_table.rb
|
114
114
|
- lib/purview/exceptions/could_not_acquire_lock.rb
|
115
|
+
- lib/purview/exceptions/database_already_assigned.rb
|
115
116
|
- lib/purview/exceptions/lock_already_released.rb
|
116
117
|
- lib/purview/exceptions/no_table.rb
|
117
118
|
- lib/purview/exceptions/no_window.rb
|
118
119
|
- lib/purview/exceptions/rows_outside_window.rb
|
119
|
-
- lib/purview/helpers.rub
|
120
120
|
- lib/purview/loaders.rb
|
121
121
|
- lib/purview/loaders/base.rb
|
122
122
|
- lib/purview/loaders/mysql.rb
|
data/lib/purview/helpers.rub
DELETED
File without changes
|