purview 1.0.0.beta3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6e042a9a30d7596c9a7dad1ce2e121b9f9bf160f
4
- data.tar.gz: b4a1f379840183ae0a11f411a409849ed06655b2
3
+ metadata.gz: caf3a73a4f73a939e96d27ecd75c682cb0c102d2
4
+ data.tar.gz: 1533be77323ba2b9b9acbaed18f0645320f0ee0a
5
5
  SHA512:
6
- metadata.gz: 52be209e82452fa5ec45ce2726c516ef315cb32de3c67b2b74862def7a29fb4fd7c95ff4da270de4f8f5262b0a4aa4ad11c35e6ffbc2e27c3bce366e998a8d8e
7
- data.tar.gz: c5768e07904cd00ae25d18adfb163ff5516d2337c2451812c1313ec5e1b183ec2ffdddcbb08b6a434607931e7ccbeaec48ea91e8f13a2d118c7b8461cf95529a
6
+ metadata.gz: 72681a57612c06a773b8bf7fef5eb53dde36f660905c06107ebb0e21feac0edfe2ef09ee21c0193dc2b5ab5cb70af7f0ef8c61b489b7e6b2165b9f01d7dee602
7
+ data.tar.gz: 256c71e89cffa61cd77be9c465a94beef74408ec263bd1bf9c774665b5426040beaf2a73f3f5f1c4fdcec66dcb18cd042fe8f69dba87bf8f9828d2e341997ea0
data/CHANGELOG ADDED
@@ -0,0 +1,6 @@
1
+ Release 1.0.0
2
+ -------------
3
+
4
+ - Support on JRuby, MRI, and RBX for pulling, validating, parsing and loading
5
+ CSV/TSV and SQL data (ETL)
6
+ - See README.md for full feature description
data/README.md CHANGED
@@ -21,7 +21,9 @@ Or install it yourself as:
21
21
 
22
22
  ## Usage
23
23
 
24
- Load the `MySQL` client (for `PostgreSQL` simply change 'mysql2' to 'pg')
24
+ Load the `MySQL` client (for `PostgreSQL` simply change 'mysql2' to 'pg' -- when
25
+ using this gem in a JRuby environment the 'jdbc/mysql' and/or 'jdbc/postgres'
26
+ library must be installed/available)
25
27
  ```ruby
26
28
  require 'mysql2'
27
29
  ```
@@ -124,6 +126,12 @@ rescue Mysql2::Error
124
126
  end
125
127
  ```
126
128
 
129
+ Enable the `Table` (in the DB). Once the related schema has been created the
130
+ `Table` needs to be enabled in order for to be able to be synchronized.
131
+ ```ruby
132
+ database.enable_table(table)
133
+ ```
134
+
127
135
  Sync the `Database`. This process will select a [candidate] `Table`, pull data
128
136
  from its [remote-]source and reconcile the new data against the main-table (e.g.
129
137
  perform `INSERTs`, `UPDATEs` and `DELETEs`). When multiple `Table(s)` are
@@ -9,6 +9,7 @@ module Purview
9
9
  end
10
10
 
11
11
  def create_table(table, opts={})
12
+ ensure_table_metadata_exists_for_table(table)
12
13
  table_opts = extract_table_options(opts)
13
14
  table_name = table_name(table, table_opts)
14
15
  with_context_logging("`create_table` for: #{table_name}") do
@@ -61,6 +62,7 @@ module Purview
61
62
  end
62
63
 
63
64
  def disable_table(table)
65
+ ensure_table_metadata_exists_for_table(table)
64
66
  table_name = table_name(table)
65
67
  with_context_logging("`disable_table` for: #{table_name}") do
66
68
  with_new_connection do |connection|
@@ -74,6 +76,7 @@ module Purview
74
76
  end
75
77
 
76
78
  def drop_table(table, opts={})
79
+ ensure_table_metadata_absent_for_table(table)
77
80
  table_opts = extract_table_options(opts)
78
81
  table_name = table_name(table, table_opts)
79
82
  with_context_logging("`drop_table` for: #{table_name}") do
@@ -91,6 +94,7 @@ module Purview
91
94
  end
92
95
 
93
96
  def enable_table(table, timestamp=Time.now.utc)
97
+ ensure_table_metadata_exists_for_table(table)
94
98
  table_name = table_name(table)
95
99
  with_context_logging("`enable_table` for: #{table_name}") do
96
100
  with_new_connection do |connection|
@@ -104,6 +108,7 @@ module Purview
104
108
  end
105
109
 
106
110
  def lock_table(table, timestamp=Time.now.utc)
111
+ ensure_table_metadata_exists_for_table(table)
107
112
  table_name = table_name(table)
108
113
  with_context_logging("`lock_table` for: #{table_name}") do
109
114
  with_new_connection do |connection|
@@ -137,6 +142,7 @@ module Purview
137
142
  end
138
143
 
139
144
  def unlock_table(table)
145
+ ensure_table_metadata_exists_for_table(table)
140
146
  table_name = table_name(table)
141
147
  with_context_logging("`unlock_table` for: #{table_name}") do
142
148
  with_new_connection do |connection|
@@ -309,9 +315,21 @@ module Purview
309
315
  raise %{All "#{Base}(s)" must override the "enable_table_sql" method}
310
316
  end
311
317
 
312
- def ensure_table_metadata_table_exists
318
+ def ensure_table_metadata_absent_for_table(table)
313
319
  with_new_connection do |connection|
314
320
  connection.execute(ensure_table_metadata_table_exists_sql)
321
+ connection.execute(ensure_table_metadata_absent_for_table_sql(table))
322
+ end
323
+ end
324
+
325
+ def ensure_table_metadata_absent_for_table_sql(table)
326
+ raise %{All "#{Base}(s)" must override the "ensure_table_metadata_absent_for_table_sql" method}
327
+ end
328
+
329
+ def ensure_table_metadata_exists_for_table(table)
330
+ with_new_connection do |connection|
331
+ connection.execute(ensure_table_metadata_table_exists_sql)
332
+ connection.execute(ensure_table_metadata_exists_for_table_sql(table))
315
333
  end
316
334
  end
317
335
 
@@ -323,14 +341,6 @@ module Purview
323
341
  raise %{All "#{Base}(s)" must override the "ensure_table_metadata_table_exists_sql" method}
324
342
  end
325
343
 
326
- def ensure_table_metadata_exists_for_tables
327
- with_new_connection do |connection|
328
- tables.each do |table|
329
- connection.execute(ensure_table_metadata_exists_for_table_sql(table))
330
- end
331
- end
332
- end
333
-
334
344
  def extract_index_options(opts)
335
345
  opts[:index] || {}
336
346
  end
@@ -408,8 +418,6 @@ module Purview
408
418
  end
409
419
 
410
420
  def next_table(connection, timestamp)
411
- ensure_table_metadata_table_exists
412
- ensure_table_metadata_exists_for_tables
413
421
  row = connection.execute(next_table_sql(timestamp)).rows[0]
414
422
  table_name = row && row[table_metadata_table_name_column_name]
415
423
  table_name ? tables_by_name[table_name] : nil
@@ -71,6 +71,14 @@ module Purview
71
71
  ]
72
72
  end
73
73
 
74
+ def ensure_table_metadata_absent_for_table_sql(table)
75
+ 'DELETE FROM %s WHERE %s = %s' % [
76
+ table_metadata_table_name,
77
+ table_metadata_table_name_column_name,
78
+ quoted(table.name),
79
+ ]
80
+ end
81
+
74
82
  def ensure_table_metadata_exists_for_table_sql(table)
75
83
  'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
76
84
  table_metadata_table_name,
@@ -67,6 +67,14 @@ module Purview
67
67
  ]
68
68
  end
69
69
 
70
+ def ensure_table_metadata_absent_for_table_sql(table)
71
+ 'DELETE FROM %s WHERE %s = %s' % [
72
+ table_metadata_table_name,
73
+ table_metadata_table_name_column_name,
74
+ quoted(table.name),
75
+ ]
76
+ end
77
+
70
78
  def ensure_table_metadata_exists_for_table_sql(table)
71
79
  'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
72
80
  table_metadata_table_name,
@@ -1,3 +1,3 @@
1
1
  module Purview
2
- VERSION = '1.0.0.beta3'
2
+ VERSION = '1.0.0'
3
3
  end
data/purview.gemspec CHANGED
@@ -9,8 +9,8 @@ Gem::Specification.new do |gem|
9
9
  gem.version = Purview::VERSION
10
10
  gem.authors = ['Jonathan W. Zaleski']
11
11
  gem.email = ['JonathanZaleski@gmail.com']
12
- gem.summary = 'A framework designed to simplify data warehousing'
13
- gem.description = 'Coming soon!'
12
+ gem.summary = 'A framework created to simplify data-warehousing'
13
+ gem.description = 'An easy to use configuration-driven framework created to simplify data-warehousing'
14
14
  gem.homepage = 'https://github.com/jzaleski/purview'
15
15
  gem.license = 'MIT'
16
16
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: purview
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0.beta3
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan W. Zaleski
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-26 00:00:00.000000000 Z
11
+ date: 2015-05-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -66,7 +66,7 @@ dependencies:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: '3.2'
69
- description: Coming soon!
69
+ description: An easy to use configuration-driven framework created to simplify data-warehousing
70
70
  email:
71
71
  - JonathanZaleski@gmail.com
72
72
  executables: []
@@ -75,6 +75,7 @@ extra_rdoc_files: []
75
75
  files:
76
76
  - ".gitignore"
77
77
  - ".travis.yml"
78
+ - CHANGELOG
78
79
  - Gemfile
79
80
  - LICENSE.txt
80
81
  - README.md
@@ -189,9 +190,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
189
190
  version: '0'
190
191
  required_rubygems_version: !ruby/object:Gem::Requirement
191
192
  requirements:
192
- - - ">"
193
+ - - ">="
193
194
  - !ruby/object:Gem::Version
194
- version: 1.3.1
195
+ version: '0'
195
196
  requirements:
196
197
  - "`jdbc-mysql` or `mysql2` gem"
197
198
  - "`jdbc-postgres` or `pg` gem"
@@ -199,6 +200,6 @@ rubyforge_project:
199
200
  rubygems_version: 2.4.6
200
201
  signing_key:
201
202
  specification_version: 4
202
- summary: A framework designed to simplify data warehousing
203
+ summary: A framework created to simplify data-warehousing
203
204
  test_files:
204
205
  - spec/spec_helper.rb