purview 1.0.0.beta2 → 1.0.0.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3e3e1cc20d1c3e7ff4a68a03010621b55dafa340
4
- data.tar.gz: 38e6f5c2ac6013c38cab51cad3594079ef6aaba4
3
+ metadata.gz: 6e042a9a30d7596c9a7dad1ce2e121b9f9bf160f
4
+ data.tar.gz: b4a1f379840183ae0a11f411a409849ed06655b2
5
5
  SHA512:
6
- metadata.gz: a9596c0fbcb897ae45964dcddc151f49da6bb0153e33c8f55d0fc9e199d4f838a7dd716f6b76abf6133a883c6521c50500941ce082d583c48562268bc1e946cd
7
- data.tar.gz: ba8989022708f2c1fddf49d77cd3dac2ca3570f710b2b14207c10fa1a6ca1408857948de1e1e8319e0958a2f06928c99ad3dd0dbcfb379bb579010d6c04e0243
6
+ metadata.gz: 52be209e82452fa5ec45ce2726c516ef315cb32de3c67b2b74862def7a29fb4fd7c95ff4da270de4f8f5262b0a4aa4ad11c35e6ffbc2e27c3bce366e998a8d8e
7
+ data.tar.gz: c5768e07904cd00ae25d18adfb163ff5516d2337c2451812c1313ec5e1b183ec2ffdddcbb08b6a434607931e7ccbeaec48ea91e8f13a2d118c7b8461cf95529a
data/README.md CHANGED
@@ -26,22 +26,6 @@ Load the `MySQL` client (for `PostgreSQL` simply change 'mysql2' to 'pg')
26
26
  require 'mysql2'
27
27
  ```
28
28
 
29
- Set the database-name (this can be anything, but it must exist)
30
- ```ruby
31
- database_name = :data_warehouse
32
- ```
33
-
34
- Combine all the configuration options and instantiate the `Database` (for
35
- `PostgreSQL` simply change `MySQL` to `PostgreSQL`)
36
- ```ruby
37
- database_opts = {}
38
-
39
- database = Purview::Databases::MySQL.new(
40
- database_name,
41
- database_opts
42
- )
43
- ```
44
-
45
29
  Set the table-name (this can be anything, but it must exist)
46
30
  ```ruby
47
31
  table_name = :users
@@ -93,7 +77,6 @@ Combine all the configuration options and instantiate the `Table`
93
77
  ```ruby
94
78
  table_opts = {
95
79
  :columns => columns,
96
- :database => database,
97
80
  :loader => loader_opts,
98
81
  :parser => parser_opts,
99
82
  :puller => puller_opts,
@@ -106,22 +89,39 @@ table = Purview::Tables::Raw.new(
106
89
  )
107
90
  ```
108
91
 
92
+ Set the database-name (this can be anything, but it must exist)
93
+ ```ruby
94
+ database_name = :data_warehouse
95
+ ```
96
+
97
+ Combine all the configuration options and instantiate the `Database` (for
98
+ `PostgreSQL` simply change `MySQL` to `PostgreSQL`)
99
+ ```ruby
100
+ database_opts = {
101
+ :tables => [table],
102
+ }
103
+
104
+ database = Purview::Databases::MySQL.new(
105
+ database_name,
106
+ database_opts
107
+ )
108
+ ```
109
+
109
110
  Add the `Table` to the `Database` (schema). In order for [the] `Table` to be
110
111
  `sync[ed]` it *must* be added to [the] `Database`
111
112
  ```ruby
112
113
  database.add_table(table)
113
114
  ```
114
115
 
115
- Create the `Table`. Recommended for testing purposes *only*. For production
116
- environments you will likely want an external process to manage the schema (for
117
- `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
116
+ Create the `Table` (in the DB). Recommended for testing purposes *only*. For
117
+ production environments you will likely want an external process to manage the
118
+ schema (for `PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
118
119
  ```ruby
119
120
  begin
120
- database.create_table(
121
- database.connect,
122
- table
123
- )
124
- rescue Mysql2::Error; end
121
+ database.create_table(table)
122
+ rescue Mysql2::Error
123
+ # Swallow
124
+ end
125
125
  ```
126
126
 
127
127
  Sync the `Database`. This process will select a [candidate] `Table`, pull data
@@ -6,60 +6,39 @@ module Purview
6
6
  def initialize(name, opts={})
7
7
  @name = name
8
8
  @opts = opts
9
- @tables = Set.new
10
9
  end
11
10
 
12
- def add_table(table)
13
- @tables << table
14
- end
15
-
16
- def create_index(connection, table, columns, opts={})
11
+ def create_table(table, opts={})
17
12
  table_opts = extract_table_options(opts)
18
13
  table_name = table_name(table, table_opts)
19
- index_opts = extract_index_options(opts)
20
- index_name(
21
- table_name,
22
- columns,
23
- index_opts
24
- ).tap do |index_name|
25
- connection.execute(
26
- create_index_sql(
27
- table_name,
28
- index_name,
29
- table,
30
- columns,
31
- index_opts
32
- )
33
- )
34
- end
35
- end
36
-
37
- def create_table(connection, table, opts={})
38
- table_opts = extract_table_options(opts)
39
- table_name(table, table_opts).tap do |table_name|
40
- connection.execute(
41
- create_table_sql(
42
- table_name,
43
- table,
44
- table_opts
45
- )
46
- )
47
- if table_opts[:create_indices]
48
- table.indexed_columns.each do |columns|
49
- create_index(
50
- connection,
14
+ with_context_logging("`create_table` for: #{table_name}") do
15
+ with_new_connection do |connection|
16
+ connection.execute(
17
+ create_table_sql(
18
+ table_name,
51
19
  table,
52
- columns,
53
- :table => { :name => table_name }
20
+ table_opts
54
21
  )
22
+ )
23
+ if table_opts[:create_indices]
24
+ table.indexed_columns.each do |columns|
25
+ create_index(
26
+ connection,
27
+ table,
28
+ columns,
29
+ :table => { :name => table_name }
30
+ )
31
+ end
55
32
  end
56
33
  end
34
+ table_name
57
35
  end
58
36
  end
59
37
 
60
38
  def create_temporary_table(connection, table, opts={})
61
39
  table_opts = extract_table_options(opts)
62
- table_name(table, table_opts).tap do |table_name|
40
+ table_name = table_name(table, table_opts)
41
+ with_context_logging("`create_temporary_table` for: #{table_name}") do
63
42
  connection.execute(
64
43
  create_temporary_table_sql(
65
44
  table_name,
@@ -77,89 +56,79 @@ module Purview
77
56
  )
78
57
  end
79
58
  end
59
+ table_name
80
60
  end
81
61
  end
82
62
 
83
63
  def disable_table(table)
84
- with_context_logging("`disable_table` for: #{table_name(table)}") do
64
+ table_name = table_name(table)
65
+ with_context_logging("`disable_table` for: #{table_name}") do
85
66
  with_new_connection do |connection|
86
- set_enabled_for_table(
87
- connection,
88
- table,
89
- false_value
90
- )
67
+ rows_affected = \
68
+ connection.execute(disable_table_sql(table)).rows_affected
69
+ raise Purview::Exceptions::TableAlreadyDisabled.new(table) \
70
+ if zero?(rows_affected)
91
71
  end
72
+ table_name
92
73
  end
93
74
  end
94
75
 
95
- def drop_table(connection, table, opts={})
96
- table_opts = extract_table_options(opts)
97
- table_name(table, table_opts).tap do |table_name|
98
- connection.execute(
99
- drop_table_sql(
100
- table_name,
101
- table,
102
- table_opts
103
- )
104
- )
105
- end
106
- end
107
-
108
- def drop_index(connection, table, columns, opts={})
76
+ def drop_table(table, opts={})
109
77
  table_opts = extract_table_options(opts)
110
78
  table_name = table_name(table, table_opts)
111
- index_opts = extract_index_options(opts)
112
- index_name(
113
- table_name,
114
- columns,
115
- index_opts
116
- ).tap do |index_name|
117
- connection.execute(
118
- drop_index_sql(
119
- table_name,
120
- index_name,
121
- table,
122
- columns,
123
- index_opts
79
+ with_context_logging("`drop_table` for: #{table_name}") do
80
+ with_new_connection do |connection|
81
+ connection.execute(
82
+ drop_table_sql(
83
+ table_name,
84
+ table,
85
+ table_opts
86
+ )
124
87
  )
125
- )
88
+ end
89
+ table_name
126
90
  end
127
91
  end
128
92
 
129
- def enable_table(table)
130
- with_context_logging("`enable_table` for: #{table_name(table)}") do
93
+ def enable_table(table, timestamp=Time.now.utc)
94
+ table_name = table_name(table)
95
+ with_context_logging("`enable_table` for: #{table_name}") do
131
96
  with_new_connection do |connection|
132
- set_enabled_for_table(
133
- connection,
134
- table,
135
- true_value
136
- )
97
+ rows_affected = \
98
+ connection.execute(enable_table_sql(table, timestamp)).rows_affected
99
+ raise Purview::Exceptions::TableAlreadyEnabled.new(table) \
100
+ if zero?(rows_affected)
137
101
  end
102
+ table_name
138
103
  end
139
104
  end
140
105
 
141
- def lock_table(table, timestamp)
142
- with_context_logging("`lock_table` for: #{table_name(table)}") do
106
+ def lock_table(table, timestamp=Time.now.utc)
107
+ table_name = table_name(table)
108
+ with_context_logging("`lock_table` for: #{table_name}") do
143
109
  with_new_connection do |connection|
144
110
  rows_affected = \
145
111
  connection.execute(lock_table_sql(table, timestamp)).rows_affected
146
- raise Purview::Exceptions::CouldNotAcquireLock.new(table) \
112
+ raise Purview::Exceptions::TableAlreadyLocked.new(table) \
147
113
  if zero?(rows_affected)
148
114
  end
115
+ table_name
149
116
  end
150
117
  end
151
118
 
152
119
  def sync
153
- with_new_connection do |connection|
154
- with_transaction(connection) do |timestamp|
155
- with_next_table(connection, timestamp) do |table|
156
- with_next_window(
157
- connection,
158
- table,
159
- timestamp
160
- ) do |window|
161
- with_table_locked(table, timestamp) do
162
- table.sync(connection, window)
120
+ with_context_logging('`sync`') do
121
+ with_new_connection do |connection|
122
+ with_transaction(connection) do |timestamp|
123
+ with_next_table(connection, timestamp) do |table|
124
+ with_next_window(
125
+ connection,
126
+ table,
127
+ timestamp
128
+ ) do |window|
129
+ with_table_locked(table, timestamp) do
130
+ table.sync(connection, window)
131
+ end
163
132
  end
164
133
  end
165
134
  end
@@ -168,13 +137,15 @@ module Purview
168
137
  end
169
138
 
170
139
  def unlock_table(table)
171
- with_context_logging("`unlock_table` for: #{table_name(table)}") do
140
+ table_name = table_name(table)
141
+ with_context_logging("`unlock_table` for: #{table_name}") do
172
142
  with_new_connection do |connection|
173
143
  rows_affected = \
174
144
  connection.execute(unlock_table_sql(table)).rows_affected
175
- raise Purview::Exceptions::LockAlreadyReleased.new(table) \
145
+ raise Purview::Exceptions::TableAlreadyUnlocked.new(table) \
176
146
  if zero?(rows_affected)
177
147
  end
148
+ table_name
178
149
  end
179
150
  end
180
151
 
@@ -184,7 +155,7 @@ module Purview
184
155
  include Purview::Mixins::Helpers
185
156
  include Purview::Mixins::Logger
186
157
 
187
- attr_reader :opts, :tables
158
+ attr_reader :opts
188
159
 
189
160
  public :connect
190
161
 
@@ -219,13 +190,40 @@ module Purview
219
190
  end
220
191
 
221
192
  def connection_opts
222
- { :database => name }
193
+ {
194
+ :database => database_name,
195
+ :host => database_host,
196
+ :password => database_password,
197
+ :port => database_port,
198
+ :username => database_username,
199
+ }
223
200
  end
224
201
 
225
202
  def connection_type
226
203
  raise %{All "#{Base}(s)" must override the "connection_type" method}
227
204
  end
228
205
 
206
+ def create_index(connection, table, columns, opts={})
207
+ table_opts = extract_table_options(opts)
208
+ table_name = table_name(table, table_opts)
209
+ index_opts = extract_index_options(opts)
210
+ index_name(
211
+ table_name,
212
+ columns,
213
+ index_opts
214
+ ).tap do |index_name|
215
+ connection.execute(
216
+ create_index_sql(
217
+ table_name,
218
+ index_name,
219
+ table,
220
+ columns,
221
+ index_opts
222
+ )
223
+ )
224
+ end
225
+ end
226
+
229
227
  def create_index_sql(table_name, index_name, table, columns, index_opts={})
230
228
  raise %{All "#{Base}(s)" must override the "create_index_sql" method}
231
229
  end
@@ -238,6 +236,26 @@ module Purview
238
236
  raise %{All "#{Base}(s)" must override the "create_temporary_table_sql" method}
239
237
  end
240
238
 
239
+ def database_host
240
+ opts[:host]
241
+ end
242
+
243
+ def database_name
244
+ name
245
+ end
246
+
247
+ def database_password
248
+ opts[:password]
249
+ end
250
+
251
+ def database_port
252
+ opts[:port]
253
+ end
254
+
255
+ def database_username
256
+ opts[:username]
257
+ end
258
+
241
259
  def default(column)
242
260
  column.default || default_map[column.type]
243
261
  end
@@ -254,6 +272,31 @@ module Purview
254
272
  raise %{All "#{Base}(s)" must override the "dialect_type" method}
255
273
  end
256
274
 
275
+ def disable_table_sql(table)
276
+ raise %{All "#{Base}(s)" must override the "disable_table_sql" method}
277
+ end
278
+
279
+ def drop_index(table, columns, opts={})
280
+ table_opts = extract_table_options(opts)
281
+ table_name = table_name(table, table_opts)
282
+ index_opts = extract_index_options(opts)
283
+ index_name(
284
+ table_name,
285
+ columns,
286
+ index_opts
287
+ ).tap do |index_name|
288
+ connection.execute(
289
+ drop_index_sql(
290
+ table_name,
291
+ index_name,
292
+ table,
293
+ columns,
294
+ index_opts
295
+ )
296
+ )
297
+ end
298
+ end
299
+
257
300
  def drop_index_sql(table_name, index_name, table, columns, index_opts={})
258
301
  raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
259
302
  end
@@ -262,6 +305,10 @@ module Purview
262
305
  raise %{All "#{Base}(s)" must override the "drop_table_sql" method}
263
306
  end
264
307
 
308
+ def enable_table_sql(table, timestamp)
309
+ raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
310
+ end
311
+
265
312
  def ensure_table_metadata_table_exists
266
313
  with_new_connection do |connection|
267
314
  connection.execute(ensure_table_metadata_table_exists_sql)
@@ -296,14 +343,14 @@ module Purview
296
343
  dialect.false_value
297
344
  end
298
345
 
299
- def get_enabled_for_table(connection, table)
300
- row = connection.execute(get_last_pulled_at_for_table_sql(table)).rows[0]
301
- enabled = row[table_metadata_enabled_column_name]
302
- !!(enabled =~ /\A(true|t|yes|y|1)\z/i)
346
+ def get_enabled_at_for_table(connection, table)
347
+ row = connection.execute(get_enabled_at_for_table_sql(table)).rows[0]
348
+ timestamp = row[table_metadata_enabled_at_column_name]
349
+ timestamp ? Time.parse(timestamp) : nil
303
350
  end
304
351
 
305
- def get_enabled_for_table_sql(table)
306
- raise %{All "#{Base}(s)" must override the "get_enabled_for_table_sql" method}
352
+ def get_enabled_at_for_table_sql(table)
353
+ raise %{All "#{Base}(s)" must override the "get_enabled_at_for_table_sql" method}
307
354
  end
308
355
 
309
356
  def get_last_pulled_at_for_table(connection, table)
@@ -401,12 +448,12 @@ module Purview
401
448
  dialect.sanitized(value)
402
449
  end
403
450
 
404
- def set_enabled_for_table(connection, table, enabled)
405
- connection.execute(set_enabled_for_table_sql(table, enabled))
451
+ def set_enabled_at_for_table(connection, table, timestamp)
452
+ connection.execute(set_enabled_at_for_table_sql(table, timestamp))
406
453
  end
407
454
 
408
- def set_enabled_for_table_sql(table, enabled)
409
- raise %{All "#{Base}(s)" must override the "set_enabled_for_table_sql" method}
455
+ def set_enabled_at_for_table_sql(table, timestamp)
456
+ raise %{All "#{Base}(s)" must override the "set_enabled_at_for_table_sql" method}
410
457
  end
411
458
 
412
459
  def set_last_pulled_at_for_table(connection, table, timestamp)
@@ -433,13 +480,13 @@ module Purview
433
480
  raise %{All "#{Base}(s)" must override the "set_max_timestamp_pulled_for_table_sql" method}
434
481
  end
435
482
 
436
- def table_metadata_enabled_column_definition
437
- column = Purview::Columns::Boolean.new(table_metadata_enabled_column_name)
483
+ def table_metadata_enabled_at_column_definition
484
+ column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
438
485
  column_definition(column)
439
486
  end
440
487
 
441
- def table_metadata_enabled_column_name
442
- 'enabled'
488
+ def table_metadata_enabled_at_column_name
489
+ 'enabled_at'
443
490
  end
444
491
 
445
492
  def table_metadata_last_pulled_at_column_definition
@@ -482,18 +529,26 @@ module Purview
482
529
  'table_name'
483
530
  end
484
531
 
532
+ def table_name(table, table_opts={})
533
+ table_opts[:name] || table.name
534
+ end
535
+
536
+ def tables
537
+ @tables ||= Set.new.tap do |result|
538
+ (opts[:tables] || []).each do |table|
539
+ table.database = self if result.add?(table)
540
+ end
541
+ end
542
+ end
543
+
485
544
  def tables_by_name
486
- {}.tap do |result|
545
+ @tables_by_name ||= {}.tap do |result|
487
546
  tables.each do |table|
488
547
  result[table.name] = table
489
548
  end
490
549
  end
491
550
  end
492
551
 
493
- def table_name(table, table_opts={})
494
- table_opts[:name] || table.name
495
- end
496
-
497
552
  def true_value
498
553
  dialect.true_value
499
554
  end
@@ -37,6 +37,17 @@ module Purview
37
37
  Purview::Dialects::MySQL
38
38
  end
39
39
 
40
+ def disable_table_sql(table)
41
+ 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
42
+ table_metadata_table_name,
43
+ table_metadata_enabled_at_column_name,
44
+ null_value,
45
+ table_metadata_table_name_column_name,
46
+ quoted(table.name),
47
+ table_metadata_enabled_at_column_name,
48
+ ]
49
+ end
50
+
40
51
  def drop_index_sql(table_name, index_name, table, columns, index_opts={})
41
52
  'DROP INDEX %s' % [
42
53
  index_name,
@@ -49,6 +60,17 @@ module Purview
49
60
  ]
50
61
  end
51
62
 
63
+ def enable_table_sql(table, timestamp)
64
+ 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
65
+ table_metadata_table_name,
66
+ table_metadata_enabled_at_column_name,
67
+ quoted(timestamp),
68
+ table_metadata_table_name_column_name,
69
+ quoted(table.name),
70
+ table_metadata_enabled_at_column_name,
71
+ ]
72
+ end
73
+
52
74
  def ensure_table_metadata_exists_for_table_sql(table)
53
75
  'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
54
76
  table_metadata_table_name,
@@ -60,16 +82,16 @@ module Purview
60
82
  'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
61
83
  table_metadata_table_name,
62
84
  table_metadata_table_name_column_definition,
63
- table_metadata_enabled_column_definition,
85
+ table_metadata_enabled_at_column_definition,
64
86
  table_metadata_last_pulled_at_column_definition,
65
87
  table_metadata_locked_at_column_definition,
66
88
  table_metadata_max_timestamp_pulled_column_definition,
67
89
  ]
68
90
  end
69
91
 
70
- def get_enabled_for_table_sql(table)
92
+ def get_enabled_at_for_table_sql(table)
71
93
  'SELECT %s FROM %s WHERE %s = %s' % [
72
- table_metadata_enabled_column_name,
94
+ table_metadata_enabled_at_column_name,
73
95
  table_metadata_table_name,
74
96
  table_metadata_table_name_column_name,
75
97
  quoted(table.name),
@@ -119,27 +141,23 @@ module Purview
119
141
  end
120
142
 
121
143
  def next_table_sql(timestamp)
122
- 'SELECT %s FROM %s WHERE %s = %s AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
144
+ 'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
123
145
  table_metadata_table_name_column_name,
124
146
  table_metadata_table_name,
125
- table_metadata_enabled_column_name,
126
- quoted(true_value),
147
+ table_metadata_enabled_at_column_name,
127
148
  table_metadata_locked_at_column_name,
128
149
  table_metadata_last_pulled_at_column_name,
129
150
  table_metadata_last_pulled_at_column_name,
130
151
  ]
131
152
  end
132
153
 
133
- def set_enabled_for_table_sql(table, enabled)
134
- 'UPDATE %s SET %s = %s WHERE %s = %s AND (%s IS NULL OR %s = %s)' % [
154
+ def set_enabled_at_for_table_sql(table, timestamp)
155
+ 'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
135
156
  table_metadata_table_name,
136
- table_metadata_enabled_column_name,
137
- quoted(enabled),
157
+ table_metadata_enabled_at_column_name,
158
+ quoted(timestamp),
138
159
  table_metadata_table_name_column_name,
139
160
  quoted(table.name),
140
- table_metadata_enabled_column_name,
141
- table_metadata_enabled_column_name,
142
- quoted(false_value),
143
161
  ]
144
162
  end
145
163
 
@@ -33,6 +33,17 @@ module Purview
33
33
  Purview::Dialects::PostgreSQL
34
34
  end
35
35
 
36
+ def disable_table_sql(table)
37
+ 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
38
+ table_metadata_table_name,
39
+ table_metadata_enabled_at_column_name,
40
+ null_value,
41
+ table_metadata_table_name_column_name,
42
+ quoted(table.name),
43
+ table_metadata_enabled_at_column_name,
44
+ ]
45
+ end
46
+
36
47
  def drop_index_sql(table_name, index_name, table, columns, index_opts={})
37
48
  'DROP INDEX %s' % [
38
49
  index_name,
@@ -45,6 +56,17 @@ module Purview
45
56
  ]
46
57
  end
47
58
 
59
+ def enable_table_sql(table, timestamp)
60
+ 'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
61
+ table_metadata_table_name,
62
+ table_metadata_enabled_at_column_name,
63
+ quoted(timestamp),
64
+ table_metadata_table_name_column_name,
65
+ quoted(table.name),
66
+ table_metadata_enabled_at_column_name,
67
+ ]
68
+ end
69
+
48
70
  def ensure_table_metadata_exists_for_table_sql(table)
49
71
  'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
50
72
  table_metadata_table_name,
@@ -60,16 +82,16 @@ module Purview
60
82
  'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
61
83
  table_metadata_table_name,
62
84
  table_metadata_table_name_column_definition,
63
- table_metadata_enabled_column_definition,
85
+ table_metadata_enabled_at_column_definition,
64
86
  table_metadata_last_pulled_at_column_definition,
65
87
  table_metadata_locked_at_column_definition,
66
88
  table_metadata_max_timestamp_pulled_column_definition,
67
89
  ]
68
90
  end
69
91
 
70
- def get_enabled_for_table_sql(table)
92
+ def get_enabled_at_for_table_sql(table)
71
93
  'SELECT %s FROM %s WHERE %s = %s' % [
72
- table_metadata_enabled_column_name,
94
+ table_metadata_enabled_at_column_name,
73
95
  table_metadata_table_name,
74
96
  table_metadata_table_name_column_name,
75
97
  quoted(table.name),
@@ -122,27 +144,23 @@ module Purview
122
144
  end
123
145
 
124
146
  def next_table_sql(timestamp)
125
- 'SELECT %s FROM %s WHERE %s = %s AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
147
+ 'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
126
148
  table_metadata_table_name_column_name,
127
149
  table_metadata_table_name,
128
- table_metadata_enabled_column_name,
129
- quoted(true_value),
150
+ table_metadata_enabled_at_column_name,
130
151
  table_metadata_locked_at_column_name,
131
152
  table_metadata_last_pulled_at_column_name,
132
153
  table_metadata_last_pulled_at_column_name,
133
154
  ]
134
155
  end
135
156
 
136
- def set_enabled_for_table_sql(table, enabled)
137
- 'UPDATE %s SET %s = %s WHERE %s = %s AND (%s IS NULL OR %s = %s)' % [
157
+ def set_enabled_at_for_table_sql(table, timestamp)
158
+ 'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
138
159
  table_metadata_table_name,
139
- table_metadata_enabled_column_name,
140
- quoted(enabled),
160
+ table_metadata_enabled_at_column_name,
161
+ quoted(timestamp),
141
162
  table_metadata_table_name_column_name,
142
163
  quoted(table.name),
143
- table_metadata_enabled_column_name,
144
- table_metadata_enabled_column_name,
145
- quoted(false_value),
146
164
  ]
147
165
  end
148
166
 
@@ -0,0 +1,9 @@
1
+ module Purview
2
+ module Exceptions
3
+ class DatabaseAlreadyAssigned < BaseTable
4
+ def message
5
+ "Database already assigned for table: #{table.name}"
6
+ end
7
+ end
8
+ end
9
+ end
@@ -2,6 +2,7 @@ require 'purview/exceptions/base'
2
2
  require 'purview/exceptions/base_table'
3
3
 
4
4
  require 'purview/exceptions/could_not_acquire_lock'
5
+ require 'purview/exceptions/database_already_assigned'
5
6
  require 'purview/exceptions/lock_already_released'
6
7
  require 'purview/exceptions/no_table'
7
8
  require 'purview/exceptions/no_window'
@@ -9,6 +9,10 @@ module Purview
9
9
  value.nil? ? default : value
10
10
  end
11
11
 
12
+ def filter_blank_values(hash)
13
+ hash.reject { |_, value| blank?(value) }
14
+ end
15
+
12
16
  def filter_nil_values(hash)
13
17
  hash.reject { |_, value| value.nil? }
14
18
  end
@@ -18,13 +18,13 @@ module Purview
18
18
  end
19
19
 
20
20
  def connection_opts
21
- filter_nil_values(
21
+ {
22
22
  :database => database_name,
23
23
  :host => database_host,
24
24
  :password => database_password,
25
25
  :port => database_port,
26
- :username => database_username
27
- )
26
+ :username => database_username,
27
+ }
28
28
  end
29
29
 
30
30
  def connection_type
@@ -111,7 +111,7 @@ module Purview
111
111
  end
112
112
 
113
113
  def username
114
- opts[:username] || Etc.getlogin
114
+ opts[:username]
115
115
  end
116
116
  end
117
117
  end
@@ -1,4 +1,6 @@
1
- if jruby? && safe_require('jdbc/mysql')
1
+ safe_require('jdbc-mysql')
2
+
3
+ if defined?(Jdbc::MySQL)
2
4
  Jdbc::MySQL.load_driver
3
5
 
4
6
  module Purview
@@ -10,6 +12,10 @@ if jruby? && safe_require('jdbc/mysql')
10
12
  def engine
11
13
  'mysql'
12
14
  end
15
+
16
+ def username
17
+ super || ENV['USER'] || Etc.getlogin
18
+ end
13
19
  end
14
20
  end
15
21
  end
@@ -1,4 +1,6 @@
1
- if jruby? && safe_require('jdbc/postgres')
1
+ safe_require('jdbc-postgres')
2
+
3
+ if defined?(Jdbc::Postgres)
2
4
  Jdbc::Postgres.load_driver
3
5
 
4
6
  module Purview
@@ -10,6 +12,10 @@ if jruby? && safe_require('jdbc/postgres')
10
12
  def engine
11
13
  'postgresql'
12
14
  end
15
+
16
+ def username
17
+ super || ENV['PGUSER'] || Etc.getlogin
18
+ end
13
19
  end
14
20
  end
15
21
  end
@@ -1,4 +1,6 @@
1
- if !jruby? && safe_require('mysql2')
1
+ safe_require('mysql2')
2
+
3
+ if defined?(Mysql2)
2
4
  module Purview
3
5
  module RawConnections
4
6
  class Mysql2 < Base
@@ -18,15 +20,19 @@ if !jruby? && safe_require('mysql2')
18
20
 
19
21
  def new_connection
20
22
  ::Mysql2::Client.new(
21
- filter_nil_values(
22
- :database => database,
23
- :host => host,
24
- :password => password,
23
+ filter_blank_values(
24
+ :database => database.to_s,
25
+ :host => host.to_s,
26
+ :password => password.to_s,
25
27
  :port => port,
26
- :username => username
28
+ :username => username.to_s
27
29
  )
28
30
  )
29
31
  end
32
+
33
+ def username
34
+ super || ENV['USER'] || Etc.getlogin
35
+ end
30
36
  end
31
37
  end
32
38
  end
@@ -1,4 +1,6 @@
1
- if !jruby? && safe_require('pg')
1
+ safe_require('pg')
2
+
3
+ if defined?(PG)
2
4
  module Purview
3
5
  module RawConnections
4
6
  class PG < Base
@@ -27,6 +29,10 @@ if !jruby? && safe_require('pg')
27
29
  )
28
30
  )
29
31
  end
32
+
33
+ def username
34
+ super || ENV['PGUSER'] || Etc.getlogin
35
+ end
30
36
  end
31
37
  end
32
38
  end
@@ -6,3 +6,7 @@ require 'purview/raw_connections/jdbc/postgres'
6
6
 
7
7
  require 'purview/raw_connections/mysql2'
8
8
  require 'purview/raw_connections/pg'
9
+
10
+ if !defined?(Purview::RawConnections::MySQL) && !defined?(Purview::RawConnections::PostgreSQL)
11
+ raise 'Could not initialize raw-connections; please install and require one or more of the following gems: `jdbc-mysql`, `jdbc-postgres`, `mysql2` and/or `pg`'
12
+ end
@@ -1,7 +1,7 @@
1
1
  module Purview
2
2
  module Tables
3
3
  class Base
4
- attr_reader :name
4
+ attr_reader :database, :name
5
5
 
6
6
  def initialize(name, opts={})
7
7
  @name = name
@@ -40,8 +40,9 @@ module Purview
40
40
  ]
41
41
  end
42
42
 
43
- def database
44
- opts[:database]
43
+ def database=(value)
44
+ raise Purview::Exceptions::DatabaseAlreadyAssigned.new(self) if database
45
+ @database = value
45
46
  end
46
47
 
47
48
  def id_column
@@ -1,3 +1,3 @@
1
1
  module Purview
2
- VERSION = '1.0.0.beta2'
2
+ VERSION = '1.0.0.beta3'
3
3
  end
data/lib/purview.rb CHANGED
@@ -1,11 +1,7 @@
1
- def jruby?
2
- defined?(JRUBY_VERSION)
3
- end
4
-
5
1
  def safe_require(name)
6
2
  require name
7
3
  rescue LoadError
8
- nil
4
+ false
9
5
  end
10
6
 
11
7
  require 'csv'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: purview
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0.beta2
4
+ version: 1.0.0.beta3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan W. Zaleski
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-19 00:00:00.000000000 Z
11
+ date: 2015-05-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -112,11 +112,11 @@ files:
112
112
  - lib/purview/exceptions/base.rb
113
113
  - lib/purview/exceptions/base_table.rb
114
114
  - lib/purview/exceptions/could_not_acquire_lock.rb
115
+ - lib/purview/exceptions/database_already_assigned.rb
115
116
  - lib/purview/exceptions/lock_already_released.rb
116
117
  - lib/purview/exceptions/no_table.rb
117
118
  - lib/purview/exceptions/no_window.rb
118
119
  - lib/purview/exceptions/rows_outside_window.rb
119
- - lib/purview/helpers.rub
120
120
  - lib/purview/loaders.rb
121
121
  - lib/purview/loaders/base.rb
122
122
  - lib/purview/loaders/mysql.rb
File without changes