purview 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8379bdb0d5f4798293f48485cb0a6f89808fb0a1
-  data.tar.gz: 2264aa219ba7d50a60b2797aef55e8a335260928
+  metadata.gz: 4075095b0198b4d1a9f104e39cef70a86f678038
+  data.tar.gz: 8d87f15b33df6603d107edee2a5206f4579bca30
 SHA512:
-  metadata.gz: ce8958f9eb4556c2d3106d015fb8be223d1911e7cb8e6341e0151be5baede61410da4d098c7671579399ddecf7af6c7678b36f4a47f9d1296084738d4ef100ff
-  data.tar.gz: 5dc9a9313f0db29b31cba3b0700b09b86a819eeb4df70e59dc0eadedfd322de046c748930a9315e6ecedba0ea70d69d52327142de2cde64c732b00255d5ea964
+  metadata.gz: fe1283a266267325800140f453efe51da2be244c150a094d5d22c40d8b2e129b9a0192fdd004e44799377ec97625aa7c5621cdf24f0f6e4f46cef58ca216361d
+  data.tar.gz: 5f5bd8092e10d1f3f34b39aceb15ab4b4ccf307b0e94dead6a801b1c964818069e7775d5af24251be61670307f27555ae0993fa54c3411142048326ad9fd0d1c

data/CHANGELOG

@@ -1,3 +1,9 @@
+Release 1.1.0
+-------------
+
+* Change how `starting_timestamp` is configured
+* BUGFIX: Fix/standardize exception class/file names
+
 Release 1.0.1
 -------------
 

data/README.md

@@ -69,12 +69,6 @@ loader_opts = {
 }
 ```
 
-Configure the `starting_timestamp` (this is the min-date to pull and can vary
-between `Table(s)`)
-```ruby
-starting_timestamp = Time.parse('2012-01-01 00:00:00Z')
-```
-
 Combine all the configuration options and instantiate the `Table`
 ```ruby
 table_opts = {
@@ -82,7 +76,6 @@ table_opts = {
   :loader => loader_opts,
   :parser => parser_opts,
   :puller => puller_opts,
-  :starting_timestamp => starting_timestamp,
 }
 
 table = Purview::Tables::Raw.new(
@@ -126,8 +119,19 @@ rescue Mysql2::Error
 end
 ```
 
-Enable the `Table` (in the DB). Once the related schema has been created the
-`Table` needs to be enabled in order for to be able to be synchronized.
+Initialize the `Table` (in the DB). This process sets the `max_timestamp_pulled`
+value in the `table_metadata` table and is used by the candidate `Table`
+selection algorithm to determine which `Table` should be synchronized next (the
+least recently synchronized `Table` will be selected). This value is also used
+as the high-water mark for records pulled from its source
+```ruby
+database.initialize_table(table, timestamp)
+```
+
+Enable the `Table` (in the DB). This process sets the `enabled_at` value in the
+`table_metadata` table and is used by the candidate `Table` selection algorithm
+to determine the pool of `Table(s)` available for synchronization (to remove a
+`Table` from the pool simply execute `disable_table`)
 ```ruby
 database.enable_table(table)
 ```

data/lib/purview/databases/base.rb

@@ -67,7 +67,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(disable_table_sql(table)).rows_affected
-            raise Purview::Exceptions::TableAlreadyDisabled.new(table) \
+            raise Purview::Exceptions::CouldNotDisable.new(table) \
               if zero?(rows_affected)
           end
           table_name
@@ -98,7 +98,20 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(enable_table_sql(table, timestamp)).rows_affected
-            raise Purview::Exceptions::TableAlreadyEnabled.new(table) \
+            raise Purview::Exceptions::CouldNotEnable.new(table) \
+              if zero?(rows_affected)
+          end
+          table_name
+        end
+      end
+
+      def initialize_table(table, starting_timestamp=Time.now.utc)
+        table_name = table_name(table)
+        with_context_logging("`initialize_table` for: #{table_name}") do
+          with_new_connection do |connection|
+            rows_affected = \
+              connection.execute(initialize_table_sql(table, starting_timestamp)).rows_affected
+            raise Purview::Exceptions::CouldNotInitialize.new(table) \
               if zero?(rows_affected)
           end
           table_name
@@ -111,7 +124,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(lock_table_sql(table, timestamp)).rows_affected
-            raise Purview::Exceptions::TableAlreadyLocked.new(table) \
+            raise Purview::Exceptions::CouldNotLock.new(table) \
               if zero?(rows_affected)
           end
           table_name
@@ -144,7 +157,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(unlock_table_sql(table)).rows_affected
-            raise Purview::Exceptions::TableAlreadyUnlocked.new(table) \
+            raise Purview::Exceptions::CouldNotUnlock.new(table) \
               if zero?(rows_affected)
           end
           table_name
@@ -372,7 +385,7 @@ module Purview
       def get_max_timestamp_pulled_for_table(connection, table)
         row = connection.execute(get_max_timestamp_pulled_for_table_sql(table)).rows[0]
         timestamp = row[table_metadata_max_timestamp_pulled_column_name]
-        timestamp ? Time.parse(timestamp) : table.starting_timestamp
+        timestamp ? Time.parse(timestamp) : nil
       end
 
       def get_max_timestamp_pulled_for_table_sql(table)
@@ -386,6 +399,10 @@ module Purview
         ]
       end
 
+      def initialize_table_sql(table, starting_timestamp)
+        raise %{All "#{Base}(s)" must override the "initialize_table_sql" method}
+      end
+
       def limit(column)
         return nil if limitless_types.include?(column.type)
         column.limit || limit_map[column.type]

data/lib/purview/databases/mysql.rb

@@ -97,6 +97,17 @@ module Purview
         ]
       end
 
+      def initialize_table_sql(table, starting_timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table_name,
+          table_metadata_max_timestamp_pulled_column_name,
+          quoted(starting_timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_max_timestamp_pulled_column_name,
+        ]
+      end
+
       def get_enabled_at_for_table_sql(table)
         'SELECT %s FROM %s WHERE %s = %s' % [
           table_metadata_enabled_at_column_name,
@@ -149,10 +160,11 @@ module Purview
       end
 
       def next_table_sql(timestamp)
-        'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
+        'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
           table_metadata_table_name_column_name,
           table_metadata_table_name,
           table_metadata_enabled_at_column_name,
+          table_metadata_max_timestamp_pulled_column_name,
           table_metadata_locked_at_column_name,
           table_metadata_last_pulled_at_column_name,
           table_metadata_last_pulled_at_column_name,

data/lib/purview/databases/postgresql.rb

@@ -97,6 +97,17 @@ module Purview
         ]
       end
 
+      def initialize_table_sql(table, starting_timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table_name,
+          table_metadata_max_timestamp_pulled_column_name,
+          quoted(starting_timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_max_timestamp_pulled_column_name,
+        ]
+      end
+
       def get_enabled_at_for_table_sql(table)
         'SELECT %s FROM %s WHERE %s = %s' % [
           table_metadata_enabled_at_column_name,
@@ -152,10 +163,11 @@ module Purview
       end
 
       def next_table_sql(timestamp)
-        'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
+        'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
           table_metadata_table_name_column_name,
           table_metadata_table_name,
           table_metadata_enabled_at_column_name,
+          table_metadata_max_timestamp_pulled_column_name,
           table_metadata_locked_at_column_name,
           table_metadata_last_pulled_at_column_name,
           table_metadata_last_pulled_at_column_name,

data/lib/purview/exceptions/could_not_disable.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class CouldNotDisable < BaseTable
+      def message
+        "Could not disable table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/could_not_enable.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class CouldNotEnable < BaseTable
+      def message
+        "Could not enable table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/could_not_initialize.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class CouldNotInitialize < BaseTable
+      def message
+        "Could not initialize table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/could_not_lock.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class CouldNotLock < BaseTable
+      def message
+        "Could not lock table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/could_not_unlock.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class CouldNotUnlock < BaseTable
+      def message
+        "Could not unlock table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions.rb

@@ -1,9 +1,13 @@
 require 'purview/exceptions/base'
 require 'purview/exceptions/base_table'
 
-require 'purview/exceptions/could_not_acquire_lock'
+require 'purview/exceptions/could_not_disable'
+require 'purview/exceptions/could_not_enable'
+require 'purview/exceptions/could_not_initialize'
+require 'purview/exceptions/could_not_lock'
+require 'purview/exceptions/could_not_unlock'
+
 require 'purview/exceptions/database_already_assigned'
-require 'purview/exceptions/lock_already_released'
 require 'purview/exceptions/no_table'
 require 'purview/exceptions/no_window'
 require 'purview/exceptions/rows_outside_window'

data/lib/purview/tables/base.rb

@@ -56,10 +56,6 @@ module Purview
         end
       end
 
-      def starting_timestamp
-        opts[:starting_timestamp]
-      end
-
       def sync(connection, window)
         raw_data = puller.pull(window)
         parser.validate(raw_data)

data/lib/purview/version.rb

@@ -1,3 +1,3 @@
 module Purview
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: purview
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Jonathan W. Zaleski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-29 00:00:00.000000000 Z
+date: 2015-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -112,9 +112,12 @@ files:
 - lib/purview/exceptions.rb
 - lib/purview/exceptions/base.rb
 - lib/purview/exceptions/base_table.rb
-- lib/purview/exceptions/could_not_acquire_lock.rb
+- lib/purview/exceptions/could_not_disable.rb
+- lib/purview/exceptions/could_not_enable.rb
+- lib/purview/exceptions/could_not_initialize.rb
+- lib/purview/exceptions/could_not_lock.rb
+- lib/purview/exceptions/could_not_unlock.rb
 - lib/purview/exceptions/database_already_assigned.rb
-- lib/purview/exceptions/lock_already_released.rb
 - lib/purview/exceptions/no_table.rb
 - lib/purview/exceptions/no_window.rb
 - lib/purview/exceptions/rows_outside_window.rb

data/lib/purview/exceptions/could_not_acquire_lock.rb

@@ -1,9 +0,0 @@
-module Purview
-  module Exceptions
-    class CouldNotAcquireLock < BaseTable
-      def message
-        "Could not acquire the lock for table: #{table.name}"
-      end
-    end
-  end
-end

data/lib/purview/exceptions/lock_already_released.rb

@@ -1,9 +0,0 @@
-module Purview
-  module Exceptions
-    class LockAlreadyReleased < BaseTable
-      def message
-        "Lock already released for table: #{table.name}"
-      end
-    end
-  end
-end