purview 1.2.0 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e6b41c5fd1454de8cf72663944dddcb35b3a2ff3
4
- data.tar.gz: 196b5bf64e43ff4788ea413ef2eb70776a766cf7
3
+ metadata.gz: 857247110c4fa3aeb219d07aa207bc4b7017a3d0
4
+ data.tar.gz: 1348881a82f7aa07d31fbf52ead8423c2d7458eb
5
5
  SHA512:
6
- metadata.gz: 5e87b88ea28caeccaa996255a08683fcb4d5fb73d24cf50a9a79fcb0646ad66e8594a4039b3752b53f2e4b2a9d9f0c4ced8755070927f6876814883185feee35
7
- data.tar.gz: 4beefdf566cba05e3e3153601ae6170e41846b6a5f4c8a816ea49f6b01c82501883fe389e9412e1698cded2b35302f976b3cfd823e85588af2e4cbaae08cfadc
6
+ metadata.gz: f77a3dd4d4bab8da4beadd7e2ab4f843e5ad9a5f0e87e66912b055dfa87f2d4237926829fe1658188f7aa6ec486a51fd13c71ef125c5049771066287eeeeb134
7
+ data.tar.gz: 2d522937e113acd589b83f4d0a0d939f136e461b1620b2181e522d8cfc9d501515865601c2677a8f337e6e21bee9ee0040a612c31c23b1144b28a382451f80bc
data/CHANGELOG CHANGED
@@ -1,3 +1,10 @@
1
+ Release 1.3.0
2
+ -------------
3
+
4
+ * Add support for easily adding additional indices + clean-up
5
+ * DEPRECATION: `create_temporary_table` deprecated in favor of `create_table`
6
+ with the `temporary` option specified
7
+
1
8
  Release 1.2.0
2
9
  -------------
3
10
 
data/README.md CHANGED
@@ -38,12 +38,27 @@ Define the `Column(s)` (available column-types: `Boolean`, `CreatedTimestamp`,
38
38
  `UpdatedTimestamp` & `UUID` -- the `Id`, `CreatedTimestamp` & `UpdatedTimestamp`
39
39
  columns are required for all tables)
40
40
  ```ruby
41
+ id_column = Purview::Columns::Id.new(:id),
42
+ name_column = Purview::Columns::String.new(:name, :nullable => false),
43
+ email_column = Purview::Columns::String.new(:email, :nullable => false, :limit => 100),
44
+ created_at_column = Purview::Columns::CreatedTimestamp.new(:created_at),
45
+ updated_at_column = Purview::Columns::UpdatedTimestamp.new(:updated_at),
46
+
41
47
  columns = [
42
- Purview::Columns::Id.new(:id),
43
- Purview::Columns::String.new(:name, :nullable => false),
44
- Purview::Columns::String.new(:email, :nullable => false, :limit => 100),
45
- Purview::Columns::CreatedTimestamp.new(:created_at),
46
- Purview::Columns::UpdatedTimestamp.new(:updated_at),
48
+ id_column,
49
+ name_column,
50
+ email_column,
51
+ created_at_column,
52
+ updated_at_column,
53
+ ]
54
+ ```
55
+
56
+ Define the `Indices` (availble index-types: `Composite` & `Simple`). By default
57
+ `Indices` will be added for the required column-types (`CreatedTimestamp` &
58
+ `UpdatedTimestamp`)
59
+ ```ruby
60
+ indices = [
61
+ Purview::Indices::Simple.new(email_column),
47
62
  ]
48
63
  ```
49
64
 
@@ -73,6 +88,7 @@ Combine all the configuration options and instantiate the `Table`
73
88
  ```ruby
74
89
  table_opts = {
75
90
  :columns => columns,
91
+ :indices => indices,
76
92
  :loader => loader_opts,
77
93
  :parser => parser_opts,
78
94
  :puller => puller_opts,
@@ -5,12 +5,6 @@ module Purview
5
5
  new(opts)
6
6
  end
7
7
 
8
- def self.with_new_connection(opts)
9
- yield connection = connect(opts)
10
- ensure
11
- connection.disconnect if connection
12
- end
13
-
14
8
  def initialize(opts)
15
9
  @raw_connection = raw_connection_type.connect(opts)
16
10
  end
@@ -1,16 +1,22 @@
1
1
  module Purview
2
2
  module Databases
3
3
  class Base
4
- attr_reader :name
4
+ attr_reader :name, :tables
5
5
 
6
6
  def initialize(name, opts={})
7
7
  @name = name
8
8
  @opts = opts
9
+ @tables = Set.new.tap do |result|
10
+ (opts[:tables] || []).each do |table|
11
+ table.database = self if result.add?(table)
12
+ end
13
+ end
9
14
  end
10
15
 
11
16
  def baseline_table(table)
12
- raise Purview::Exceptions::WrongDatabase.new(table) \
13
- unless tables.include?(table)
17
+ ensure_table_valid_for_database(table)
18
+ raise Purview::Exceptions::CouldNotBaseline.new(table) \
19
+ unless table_initialized?(table)
14
20
  table_name = table_name(table)
15
21
  with_context_logging("`baseline_table` for: #{table_name}") do
16
22
  starting_timestamp = timestamp
@@ -21,62 +27,66 @@ module Purview
21
27
  end
22
28
  end
23
29
  end
30
+ table_name
31
+ end
32
+
33
+ def create_index(index, opts={})
34
+ ensure_index_valid_for_database(index)
35
+ table_opts = extract_table_options(opts)
36
+ table_name = table_name(index.table, table_opts)
37
+ index_opts = extract_index_options(opts)
38
+ index_columns = index.columns
39
+ index_name = index_name(
40
+ table_name,
41
+ index_columns,
42
+ index_opts
43
+ )
44
+ with_context_logging("`create_index` for: #{index_name}") do
45
+ with_new_or_existing_connection(opts) do |connection|
46
+ connection.execute(
47
+ create_index_sql(
48
+ table_name,
49
+ index_name,
50
+ index,
51
+ index_opts
52
+ )
53
+ )
54
+ end
55
+ end
56
+ index_name
24
57
  end
25
58
 
26
59
  def create_table(table, opts={})
60
+ ensure_table_valid_for_database(table)
27
61
  ensure_table_metadata_exists_for_table(table)
28
62
  table_opts = extract_table_options(opts)
29
63
  table_name = table_name(table, table_opts)
30
64
  with_context_logging("`create_table` for: #{table_name}") do
31
- with_new_connection do |connection|
65
+ with_new_or_existing_connection(opts) do |connection|
32
66
  connection.execute(
33
- create_table_sql(
67
+ send(
68
+ create_table_sql_method_name(table, table_opts),
34
69
  table_name,
35
70
  table,
36
71
  table_opts
37
72
  )
38
73
  )
39
74
  if table_opts[:create_indices]
40
- table.indexed_columns.each do |columns|
75
+ table.indices.each do |index|
41
76
  create_index(
42
- connection,
43
- table,
44
- columns,
77
+ index,
78
+ :connection => connection,
45
79
  :table => { :name => table_name }
46
80
  )
47
81
  end
48
82
  end
49
83
  end
50
- table_name
51
- end
52
- end
53
-
54
- def create_temporary_table(connection, table, opts={})
55
- table_opts = extract_table_options(opts)
56
- table_name = table_name(table, table_opts)
57
- with_context_logging("`create_temporary_table` for: #{table_name}") do
58
- connection.execute(
59
- create_temporary_table_sql(
60
- table_name,
61
- table,
62
- table_opts
63
- )
64
- )
65
- if table_opts[:create_indices]
66
- table.indexed_columns.each do |columns|
67
- create_index(
68
- connection,
69
- table,
70
- columns,
71
- :table => { :name => table_name }
72
- )
73
- end
74
- end
75
- table_name
76
84
  end
85
+ table_name
77
86
  end
78
87
 
79
88
  def disable_table(table)
89
+ ensure_table_valid_for_database(table)
80
90
  table_name = table_name(table)
81
91
  with_context_logging("`disable_table` for: #{table_name}") do
82
92
  with_new_connection do |connection|
@@ -85,11 +95,38 @@ module Purview
85
95
  raise Purview::Exceptions::CouldNotDisable.new(table) \
86
96
  if zero?(rows_affected)
87
97
  end
88
- table_name
89
98
  end
99
+ table_name
100
+ end
101
+
102
+ def drop_index(index, opts={})
103
+ ensure_index_valid_for_database(index)
104
+ table_opts = extract_table_options(opts)
105
+ table_name = table_name(index.table, table_opts)
106
+ index_opts = extract_index_options(opts)
107
+ index_columns = index.columns
108
+ index_name = index_name(
109
+ table_name,
110
+ index_columns,
111
+ index_opts
112
+ )
113
+ with_context_logging("`drop_index` for: #{index_name}") do
114
+ with_new_or_existing_connection(opts) do |connection|
115
+ connection.execute(
116
+ drop_index_sql(
117
+ table_name,
118
+ index_name,
119
+ index,
120
+ index_opts
121
+ )
122
+ )
123
+ end
124
+ end
125
+ index_name
90
126
  end
91
127
 
92
128
  def drop_table(table, opts={})
129
+ ensure_table_valid_for_database(table)
93
130
  ensure_table_metadata_absent_for_table(table)
94
131
  table_opts = extract_table_options(opts)
95
132
  table_name = table_name(table, table_opts)
@@ -103,11 +140,12 @@ module Purview
103
140
  )
104
141
  )
105
142
  end
106
- table_name
107
143
  end
144
+ table_name
108
145
  end
109
146
 
110
147
  def enable_table(table, timestamp=timestamp)
148
+ ensure_table_valid_for_database(table)
111
149
  table_name = table_name(table)
112
150
  with_context_logging("`enable_table` for: #{table_name}") do
113
151
  with_new_connection do |connection|
@@ -116,11 +154,12 @@ module Purview
116
154
  raise Purview::Exceptions::CouldNotEnable.new(table) \
117
155
  if zero?(rows_affected)
118
156
  end
119
- table_name
120
157
  end
158
+ table_name
121
159
  end
122
160
 
123
161
  def initialize_table(table, timestamp=timestamp)
162
+ ensure_table_valid_for_database(table)
124
163
  table_name = table_name(table)
125
164
  with_context_logging("`initialize_table` for: #{table_name}") do
126
165
  with_new_connection do |connection|
@@ -129,11 +168,12 @@ module Purview
129
168
  raise Purview::Exceptions::CouldNotInitialize.new(table) \
130
169
  if zero?(rows_affected)
131
170
  end
132
- table_name
133
171
  end
172
+ table_name
134
173
  end
135
174
 
136
175
  def lock_table(table, timestamp=timestamp)
176
+ ensure_table_valid_for_database(table)
137
177
  table_name = table_name(table)
138
178
  with_context_logging("`lock_table` for: #{table_name}") do
139
179
  with_new_connection do |connection|
@@ -142,8 +182,8 @@ module Purview
142
182
  raise Purview::Exceptions::CouldNotLock.new(table) \
143
183
  if zero?(rows_affected)
144
184
  end
145
- table_name
146
185
  end
186
+ table_name
147
187
  end
148
188
 
149
189
  def sync
@@ -157,15 +197,18 @@ module Purview
157
197
  end
158
198
 
159
199
  def sync_table(table)
160
- raise Purview::Exceptions::WrongDatabase.new(table) \
161
- unless tables.include?(table)
200
+ ensure_table_valid_for_database(table)
201
+ raise Purview::Exceptions::CouldNotSync.new(table) \
202
+ unless table_initialized?(table) && table_enabled?(table)
162
203
  table_name = table_name(table)
163
204
  with_context_logging("`sync_table` for: #{table_name}") do
164
205
  sync_table_with_lock(table, timestamp)
165
206
  end
207
+ table_name
166
208
  end
167
209
 
168
210
  def unlock_table(table)
211
+ ensure_table_valid_for_database(table)
169
212
  table_name = table_name(table)
170
213
  with_context_logging("`unlock_table` for: #{table_name}") do
171
214
  with_new_connection do |connection|
@@ -174,8 +217,8 @@ module Purview
174
217
  raise Purview::Exceptions::CouldNotUnlock.new(table) \
175
218
  if zero?(rows_affected)
176
219
  end
177
- table_name
178
220
  end
221
+ table_name
179
222
  end
180
223
 
181
224
  private
@@ -188,10 +231,6 @@ module Purview
188
231
 
189
232
  public :connect
190
233
 
191
- def column_names(columns)
192
- columns.map(&:name)
193
- end
194
-
195
234
  def column_definition(column)
196
235
  column.name.to_s.tap do |column_definition|
197
236
  type = type(column)
@@ -207,13 +246,17 @@ module Purview
207
246
  end
208
247
  end
209
248
 
249
+ def column_names(columns)
250
+ columns.map(&:name)
251
+ end
252
+
210
253
  def column_definitions(table)
211
- [].tap do |results|
212
- results << column_definition(table.id_column)
213
- results << column_definition(table.created_timestamp_column)
214
- results << column_definition(table.updated_timestamp_column)
254
+ [].tap do |result|
255
+ result << column_definition(table.id_column)
256
+ result << column_definition(table.created_timestamp_column)
257
+ result << column_definition(table.updated_timestamp_column)
215
258
  table.data_columns.each do |column|
216
- results << column_definition(column)
259
+ result << column_definition(column)
217
260
  end
218
261
  end
219
262
  end
@@ -222,28 +265,7 @@ module Purview
222
265
  raise %{All "#{Base}(s)" must override the "connection_type" method}
223
266
  end
224
267
 
225
- def create_index(connection, table, columns, opts={})
226
- table_opts = extract_table_options(opts)
227
- table_name = table_name(table, table_opts)
228
- index_opts = extract_index_options(opts)
229
- index_name(
230
- table_name,
231
- columns,
232
- index_opts
233
- ).tap do |index_name|
234
- connection.execute(
235
- create_index_sql(
236
- table_name,
237
- index_name,
238
- table,
239
- columns,
240
- index_opts
241
- )
242
- )
243
- end
244
- end
245
-
246
- def create_index_sql(table_name, index_name, table, columns, index_opts={})
268
+ def create_index_sql(table_name, index_name, index, index_opts={})
247
269
  raise %{All "#{Base}(s)" must override the "create_index_sql" method}
248
270
  end
249
271
 
@@ -251,6 +273,10 @@ module Purview
251
273
  raise %{All "#{Base}(s)" must override the "create_table_sql" method}
252
274
  end
253
275
 
276
+ def create_table_sql_method_name(table, table_opts={})
277
+ "create#{table_opts[:temporary] && '_temporary'}_table_sql".to_sym
278
+ end
279
+
254
280
  def create_temporary_table_sql(table_name, table, table_opts={})
255
281
  raise %{All "#{Base}(s)" must override the "create_temporary_table_sql" method}
256
282
  end
@@ -295,28 +321,7 @@ module Purview
295
321
  raise %{All "#{Base}(s)" must override the "disable_table_sql" method}
296
322
  end
297
323
 
298
- def drop_index(table, columns, opts={})
299
- table_opts = extract_table_options(opts)
300
- table_name = table_name(table, table_opts)
301
- index_opts = extract_index_options(opts)
302
- index_name(
303
- table_name,
304
- columns,
305
- index_opts
306
- ).tap do |index_name|
307
- connection.execute(
308
- drop_index_sql(
309
- table_name,
310
- index_name,
311
- table,
312
- columns,
313
- index_opts
314
- )
315
- )
316
- end
317
- end
318
-
319
- def drop_index_sql(table_name, index_name, table, columns, index_opts={})
324
+ def drop_index_sql(table_name, index_name, index, index_opts={})
320
325
  raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
321
326
  end
322
327
 
@@ -328,6 +333,12 @@ module Purview
328
333
  raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
329
334
  end
330
335
 
336
+ def ensure_index_valid_for_database(index)
337
+ raise ArgumentError.new('Must provide an `Index`') \
338
+ unless index
339
+ ensure_table_valid_for_database(index.table)
340
+ end
341
+
331
342
  def ensure_table_metadata_absent_for_table(table)
332
343
  with_new_connection do |connection|
333
344
  connection.execute(ensure_table_metadata_table_exists_sql)
@@ -354,6 +365,13 @@ module Purview
354
365
  raise %{All "#{Base}(s)" must override the "ensure_table_metadata_table_exists_sql" method}
355
366
  end
356
367
 
368
+ def ensure_table_valid_for_database(table)
369
+ raise ArgumentError.new('Must provide a `Table`') \
370
+ unless table
371
+ raise Purview::Exceptions::WrongDatabase.new(table) \
372
+ unless tables.include?(table)
373
+ end
374
+
357
375
  def extract_index_options(opts)
358
376
  opts[:index] || {}
359
377
  end
@@ -516,24 +534,52 @@ module Purview
516
534
  def sync_table_without_lock(table, timestamp)
517
535
  last_window = nil
518
536
  with_next_window(table, timestamp) do |window|
519
- with_new_transaction do |connection|
520
- table.sync(connection, window)
521
- set_last_pulled_at_for_table(
522
- connection,
523
- table,
524
- timestamp
525
- )
526
- set_max_timestamp_pulled_for_table(
527
- connection,
528
- table,
529
- window.max
530
- )
531
- last_window = window
537
+ with_new_connection do |connection|
538
+ with_transaction(connection) do
539
+ table.sync(connection, window)
540
+ set_last_pulled_at_for_table(
541
+ connection,
542
+ table,
543
+ timestamp
544
+ )
545
+ set_max_timestamp_pulled_for_table(
546
+ connection,
547
+ table,
548
+ window.max
549
+ )
550
+ last_window = window
551
+ end
532
552
  end
533
553
  end
534
554
  last_window
535
555
  end
536
556
 
557
+ def table_disabled?(table)
558
+ !table_enabled?(table)
559
+ end
560
+
561
+ def table_enabled?(table)
562
+ with_new_connection do |connection|
563
+ !!get_enabled_at_for_table(connection, table)
564
+ end
565
+ end
566
+
567
+ def table_initialized?(table)
568
+ with_new_connection do |connection|
569
+ !!get_max_timestamp_pulled_for_table(connection, table)
570
+ end
571
+ end
572
+
573
+ def table_locked?(table)
574
+ with_new_connection do |connection|
575
+ !!get_locked_at_for_table(connection, table)
576
+ end
577
+ end
578
+
579
+ def table_unlocked?(table)
580
+ !table_locked?(table)
581
+ end
582
+
537
583
  def table_metadata_enabled_at_column_definition
538
584
  column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
539
585
  column_definition(column)
@@ -587,14 +633,6 @@ module Purview
587
633
  table_opts[:name] || table.name
588
634
  end
589
635
 
590
- def tables
591
- @tables ||= Set.new.tap do |result|
592
- (opts[:tables] || []).each do |table|
593
- table.database = self if result.add?(table)
594
- end
595
- end
596
- end
597
-
598
636
  def tables_by_name
599
637
  @tables_by_name ||= {}.tap do |result|
600
638
  tables.each do |table|
@@ -7,11 +7,12 @@ module Purview
7
7
  Purview::Connections::MySQL
8
8
  end
9
9
 
10
- def create_index_sql(table_name, index_name, table, columns, index_opts={})
11
- 'CREATE INDEX %s ON %s (%s)' % [
10
+ def create_index_sql(table_name, index_name, index, index_opts={})
11
+ 'CREATE%sINDEX %s ON %s (%s)' % [
12
+ index.unique? ? ' UNIQUE ' : ' ',
12
13
  index_name,
13
14
  table_name,
14
- column_names(columns).join(', '),
15
+ column_names(index.columns).join(', '),
15
16
  ]
16
17
  end
17
18
 
@@ -48,7 +49,7 @@ module Purview
48
49
  ]
49
50
  end
50
51
 
51
- def drop_index_sql(table_name, index_name, table, columns, index_opts={})
52
+ def drop_index_sql(table_name, index_name, index, index_opts={})
52
53
  'DROP INDEX %s' % [
53
54
  index_name,
54
55
  ]
@@ -7,11 +7,12 @@ module Purview
7
7
  Purview::Connections::PostgreSQL
8
8
  end
9
9
 
10
- def create_index_sql(table_name, index_name, table, columns, index_opts={})
11
- 'CREATE INDEX %s ON %s (%s)' % [
10
+ def create_index_sql(table_name, index_name, index, index_opts={})
11
+ 'CREATE%sINDEX %s ON %s (%s)' % [
12
+ index.unique? ? ' UNIQUE ' : ' ',
12
13
  index_name,
13
14
  table_name,
14
- column_names(columns).join(', '),
15
+ column_names(index.columns).join(', '),
15
16
  ]
16
17
  end
17
18
 
@@ -44,7 +45,7 @@ module Purview
44
45
  ]
45
46
  end
46
47
 
47
- def drop_index_sql(table_name, index_name, table, columns, index_opts={})
48
+ def drop_index_sql(table_name, index_name, index, index_opts={})
48
49
  'DROP INDEX %s' % [
49
50
  index_name,
50
51
  ]
@@ -0,0 +1,9 @@
1
+ module Purview
2
+ module Exceptions
3
+ class CouldNotBaseline < BaseTable
4
+ def message
5
+ "Could not baseline table: #{table.name}"
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ module Purview
2
+ module Exceptions
3
+ class CouldNotSync < BaseTable
4
+ def message
5
+ "Could not sync table: #{table.name}"
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,17 @@
1
+ module Purview
2
+ module Exceptions
3
+ class TableAlreadyAssigned < Base
4
+ def initialize(index)
5
+ @index = index
6
+ end
7
+
8
+ def message
9
+ "Table already assigned for index on columns: #{index.table.column_names.join(', ')}"
10
+ end
11
+
12
+ private
13
+
14
+ attr_reader :index
15
+ end
16
+ end
17
+ end
@@ -1,14 +1,17 @@
1
1
  require 'purview/exceptions/base'
2
2
  require 'purview/exceptions/base_table'
3
3
 
4
+ require 'purview/exceptions/could_not_baseline'
4
5
  require 'purview/exceptions/could_not_disable'
5
6
  require 'purview/exceptions/could_not_enable'
6
7
  require 'purview/exceptions/could_not_initialize'
7
8
  require 'purview/exceptions/could_not_lock'
9
+ require 'purview/exceptions/could_not_sync'
8
10
  require 'purview/exceptions/could_not_unlock'
9
11
 
10
12
  require 'purview/exceptions/database_already_assigned'
11
13
  require 'purview/exceptions/no_table'
12
14
  require 'purview/exceptions/no_window'
13
15
  require 'purview/exceptions/rows_outside_window'
16
+ require 'purview/exceptions/table_already_assigned'
14
17
  require 'purview/exceptions/wrong_database'
@@ -0,0 +1,39 @@
1
+ module Purview
2
+ module Indices
3
+ class Base
4
+ attr_reader :columns, :table
5
+
6
+ def initialize(columns, opts={})
7
+ @columns = columns
8
+ @opts = opts
9
+ end
10
+
11
+ def eql?(other)
12
+ self.class == other.class &&
13
+ columns == other.columns &&
14
+ unique == other.unique
15
+ end
16
+
17
+ def hash
18
+ columns.hash + unique.hash
19
+ end
20
+
21
+ def table=(value)
22
+ raise Purview::Exceptions::TableAlreadyAssigned.new(self) if table
23
+ @table = value
24
+ end
25
+
26
+ def unique
27
+ opts[:unique]
28
+ end
29
+
30
+ def unique?
31
+ !!unique
32
+ end
33
+
34
+ private
35
+
36
+ attr_reader :opts
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,9 @@
1
+ module Purview
2
+ module Indices
3
+ class Composite < Base
4
+ def initialize(column_names, opts={})
5
+ super
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ module Purview
2
+ module Indices
3
+ class Simple < Base
4
+ def initialize(column, opts={})
5
+ super([column], opts)
6
+ end
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,4 @@
1
+ require 'purview/indices/base'
2
+
3
+ require 'purview/indices/composite'
4
+ require 'purview/indices/simple'
@@ -38,10 +38,10 @@ module Purview
38
38
  end
39
39
 
40
40
  def create_temporary_table(connection)
41
- database.create_temporary_table(
42
- connection,
41
+ database.create_table(
43
42
  table,
44
- :table => temporary_table_opts
43
+ :connection => connection,
44
+ :table => temporary_table_opts,
45
45
  )
46
46
  end
47
47
 
@@ -130,6 +130,7 @@ module Purview
130
130
  {
131
131
  :create_indices => true,
132
132
  :name => table.temporary_name,
133
+ :temporary => true,
133
134
  }
134
135
  end
135
136
 
@@ -2,7 +2,7 @@ module Purview
2
2
  module Mixins
3
3
  module Connection
4
4
  def connect
5
- connection_type.new(connection_opts)
5
+ connection_type.connect(connection_opts)
6
6
  end
7
7
 
8
8
  def connection_opts
@@ -16,14 +16,22 @@ module Purview
16
16
  end
17
17
 
18
18
  def with_new_connection
19
- connection_type.with_new_connection(connection_opts) { |connection| yield connection }
19
+ yield connection = connect
20
+ ensure
21
+ connection.disconnect if connection
20
22
  end
21
23
 
22
- def with_new_transaction
23
- with_new_connection do |connection|
24
- connection.with_transaction { yield connection }
24
+ def with_new_or_existing_connection(opts={})
25
+ if existing_connection = opts[:connection]
26
+ yield existing_connection
27
+ else
28
+ with_new_connection { |connection| yield connection }
25
29
  end
26
30
  end
31
+
32
+ def with_transaction(connection)
33
+ connection.with_transaction { yield }
34
+ end
27
35
  end
28
36
  end
29
37
  end
@@ -3,10 +3,10 @@ module Purview
3
3
  class CSV < Base
4
4
  def parse(data)
5
5
  with_context_logging("`parse` for: #{table.name}") do
6
- [].tap do |results|
6
+ [].tap do |result|
7
7
  headers = extract_headers(data)
8
8
  extract_rows(data).each do |row|
9
- results << build_result(headers.zip(row))
9
+ result << build_result(headers.zip(row))
10
10
  end
11
11
  end
12
12
  end
@@ -5,12 +5,6 @@ module Purview
5
5
  new(opts)
6
6
  end
7
7
 
8
- def self.with_new_connection(opts)
9
- yield connection = connect(opts)
10
- ensure
11
- connection.disconnect if connection
12
- end
13
-
14
8
  def initialize(opts)
15
9
  @opts = opts
16
10
  @raw_connection = new_connection
@@ -1,11 +1,16 @@
1
1
  module Purview
2
2
  module Tables
3
3
  class Base
4
- attr_reader :database, :name
4
+ attr_reader :database, :indices, :name
5
5
 
6
6
  def initialize(name, opts={})
7
7
  @name = name
8
8
  @opts = opts
9
+ @indices = Set.new.tap do |result|
10
+ ((opts[:indices] || []) + default_indices).each do |index|
11
+ index.table = self if result.add?(index)
12
+ end
13
+ end
9
14
  end
10
15
 
11
16
  def columns
@@ -32,6 +37,10 @@ module Purview
32
37
  columns_of_type(Purview::Columns::CreatedTimestamp).first
33
38
  end
34
39
 
40
+ def created_timestamp_index
41
+ Purview::Indices::Simple.new(created_timestamp_column)
42
+ end
43
+
35
44
  def data_columns
36
45
  columns - [
37
46
  created_timestamp_column,
@@ -49,13 +58,6 @@ module Purview
49
58
  columns_of_type(Purview::Columns::Id).first
50
59
  end
51
60
 
52
- def indexed_columns
53
- (opts[:indexed_columns] || []).tap do |indexed_columns|
54
- indexed_columns << [created_timestamp_column]
55
- indexed_columns << [updated_timestamp_column]
56
- end
57
- end
58
-
59
61
  def sync(connection, window)
60
62
  raw_data = puller.pull(window)
61
63
  parser.validate(raw_data)
@@ -75,6 +77,10 @@ module Purview
75
77
  columns_of_type(Purview::Columns::UpdatedTimestamp).first
76
78
  end
77
79
 
80
+ def updated_timestamp_index
81
+ Purview::Indices::Simple.new(updated_timestamp_column)
82
+ end
83
+
78
84
  def window_size
79
85
  opts[:window_size] || (60 * 60)
80
86
  end
@@ -86,6 +92,13 @@ module Purview
86
92
 
87
93
  attr_reader :opts
88
94
 
95
+ def default_indices
96
+ [
97
+ created_timestamp_index,
98
+ updated_timestamp_index,
99
+ ]
100
+ end
101
+
89
102
  def extract_type_option(opts)
90
103
  opts[:type]
91
104
  end
@@ -1,3 +1,3 @@
1
1
  module Purview
2
- VERSION = '1.2.0'
2
+ VERSION = '1.3.0'
3
3
  end
data/lib/purview.rb CHANGED
@@ -22,6 +22,7 @@ require 'purview/connections'
22
22
  require 'purview/databases'
23
23
  require 'purview/dialects'
24
24
  require 'purview/exceptions'
25
+ require 'purview/indices'
25
26
  require 'purview/loaders'
26
27
  require 'purview/loggers'
27
28
  require 'purview/parsers'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: purview
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan W. Zaleski
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-03 00:00:00.000000000 Z
11
+ date: 2015-06-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -112,16 +112,23 @@ files:
112
112
  - lib/purview/exceptions.rb
113
113
  - lib/purview/exceptions/base.rb
114
114
  - lib/purview/exceptions/base_table.rb
115
+ - lib/purview/exceptions/could_not_baseline.rb
115
116
  - lib/purview/exceptions/could_not_disable.rb
116
117
  - lib/purview/exceptions/could_not_enable.rb
117
118
  - lib/purview/exceptions/could_not_initialize.rb
118
119
  - lib/purview/exceptions/could_not_lock.rb
120
+ - lib/purview/exceptions/could_not_sync.rb
119
121
  - lib/purview/exceptions/could_not_unlock.rb
120
122
  - lib/purview/exceptions/database_already_assigned.rb
121
123
  - lib/purview/exceptions/no_table.rb
122
124
  - lib/purview/exceptions/no_window.rb
123
125
  - lib/purview/exceptions/rows_outside_window.rb
126
+ - lib/purview/exceptions/table_already_assigned.rb
124
127
  - lib/purview/exceptions/wrong_database.rb
128
+ - lib/purview/indices.rb
129
+ - lib/purview/indices/base.rb
130
+ - lib/purview/indices/composite.rb
131
+ - lib/purview/indices/simple.rb
125
132
  - lib/purview/loaders.rb
126
133
  - lib/purview/loaders/base.rb
127
134
  - lib/purview/loaders/mysql.rb