purview 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1369176b107aba11f52e4a8604fdfd440969f3e1
-  data.tar.gz: 989bbb48900af7fb712c9745b505727bd4963d53
+  metadata.gz: e6b41c5fd1454de8cf72663944dddcb35b3a2ff3
+  data.tar.gz: 196b5bf64e43ff4788ea413ef2eb70776a766cf7
 SHA512:
-  metadata.gz: 96aed3e255fe0a70bbaec213852a50da2ae7ddcf81ba9633a50d45ee6979cfdcfc1ffc614a5412c9b27964934d4b62bda405bb314f829acaa009133a4228964a
-  data.tar.gz: 2a8645de4002ef60ba092c796d0099ab0ed2851bc54261cf9e494cc41999f434002fe4a04ec904d3e34a1c4b6b718f305aa9e4a7aa51971272ed5e4d5cbf47f4
+  metadata.gz: 5e87b88ea28caeccaa996255a08683fcb4d5fb73d24cf50a9a79fcb0646ad66e8594a4039b3752b53f2e4b2a9d9f0c4ced8755070927f6876814883185feee35
+  data.tar.gz: 4beefdf566cba05e3e3153601ae6170e41846b6a5f4c8a816ea49f6b01c82501883fe389e9412e1698cded2b35302f976b3cfd823e85588af2e4cbaae08cfadc

data/CHANGELOG

@@ -1,3 +1,10 @@
+Release 1.2.0
+-------------
+
+* Use system-time as opposed to assuming/forcing UTC time
+* Add `sync_table` helper to `Database` class
+* Eliminate explicit `return(s)` -- they were causing issues w/ `yield`
+
 Release 1.1.1
 -------------
 

data/lib/purview/columns/base.rb

@@ -34,9 +34,8 @@ module Purview
 
       def parse(value)
         blank = blank?(value)
-        return nil if blank && nullable?
-        raise %{Unexpected blank value for column: "#{name}"} if blank
-        type.parse(value)
+        raise %{Unexpected blank value for column: "#{name}"} if blank && !nullable?
+        blank ? nil : type.parse(value)
       end
 
       def primary_key

data/lib/purview/databases/base.rb

@@ -9,26 +9,16 @@ module Purview
       end
 
       def baseline_table(table)
-        with_context_logging('`baseline_table`') do
-          raise Purview::Exceptions::WrongDatabase.new(table) \
-            unless tables.include?(table)
-          starting_timestamp = Time.now.utc
+        raise Purview::Exceptions::WrongDatabase.new(table) \
+          unless tables.include?(table)
+        table_name = table_name(table)
+        with_context_logging("`baseline_table` for: #{table_name}") do
+          starting_timestamp = timestamp
           with_table_locked(table, starting_timestamp) do
-            last_window = nil
-            begin
-              with_new_connection do |connection|
-                with_transaction(connection) do |transaction_timestamp|
-                  with_next_window(
-                    connection,
-                    table,
-                    transaction_timestamp
-                  ) do |window|
-                    table.sync(connection, window)
-                    last_window = window
-                  end
-                end
-              end
-            end while last_window.max < starting_timestamp
+            loop do
+              last_window = sync_table_without_lock(table, timestamp)
+              break if last_window.max > starting_timestamp
+            end
           end
         end
       end
@@ -117,7 +107,7 @@ module Purview
         end
       end
 
-      def enable_table(table, timestamp=Time.now.utc)
+      def enable_table(table, timestamp=timestamp)
         table_name = table_name(table)
         with_context_logging("`enable_table` for: #{table_name}") do
           with_new_connection do |connection|
@@ -130,12 +120,12 @@ module Purview
         end
       end
 
-      def initialize_table(table, starting_timestamp=Time.now.utc)
+      def initialize_table(table, timestamp=timestamp)
         table_name = table_name(table)
         with_context_logging("`initialize_table` for: #{table_name}") do
           with_new_connection do |connection|
             rows_affected = \
-              connection.execute(initialize_table_sql(table, starting_timestamp)).rows_affected
+              connection.execute(initialize_table_sql(table, timestamp)).rows_affected
             raise Purview::Exceptions::CouldNotInitialize.new(table) \
               if zero?(rows_affected)
           end
@@ -143,7 +133,7 @@ module Purview
         end
       end
 
-      def lock_table(table, timestamp=Time.now.utc)
+      def lock_table(table, timestamp=timestamp)
         table_name = table_name(table)
         with_context_logging("`lock_table` for: #{table_name}") do
           with_new_connection do |connection|
@@ -158,24 +148,23 @@ module Purview
 
       def sync
         with_context_logging('`sync`') do
-          with_new_connection do |connection|
-            with_transaction(connection) do |transaction_timestamp|
-              with_next_table(connection, transaction_timestamp) do |table|
-                with_next_window(
-                  connection,
-                  table,
-                  transaction_timestamp
-                ) do |window|
-                  with_table_locked(table, transaction_timestamp) do
-                    table.sync(connection, window)
-                  end
-                end
-              end
+          with_timestamp do |timestamp|
+            with_next_table(timestamp) do |table|
+              sync_table_with_lock(table, timestamp)
             end
           end
         end
       end
 
+      def sync_table(table)
+        raise Purview::Exceptions::WrongDatabase.new(table) \
+          unless tables.include?(table)
+        table_name = table_name(table)
+        with_context_logging("`sync_table` for: #{table_name}") do
+          sync_table_with_lock(table, timestamp)
+        end
+      end
+
       def unlock_table(table)
         table_name = table_name(table)
         with_context_logging("`unlock_table` for: #{table_name}") do
@@ -424,13 +413,12 @@ module Purview
         ]
       end
 
-      def initialize_table_sql(table, starting_timestamp)
+      def initialize_table_sql(table, timestamp)
         raise %{All "#{Base}(s)" must override the "initialize_table_sql" method}
       end
 
       def limit(column)
-        return nil if limitless_types.include?(column.type)
-        column.limit || limit_map[column.type]
+        limitless_types.include?(column.type) ? nil : (column.limit || limit_map[column.type])
       end
 
       def limit_map
@@ -448,7 +436,7 @@ module Purview
       def next_table(connection, timestamp)
         row = connection.execute(next_table_sql(timestamp)).rows[0]
         table_name = row && row[table_metadata_table_name_column_name]
-        table_name ? tables_by_name[table_name] : nil
+        table_name && tables_by_name[table_name]
       end
 
       def next_table_sql(timestamp)
@@ -459,9 +447,10 @@ module Purview
         min = get_max_timestamp_pulled_for_table(connection, table)
         max = min + table.window_size
         now = timestamp
-        return nil if min > now
-        max = now if max > now
-        Purview::Structs::Window.new(:min => min, :max => max)
+        min > now ? nil : Purview::Structs::Window.new(
+          :min => min,
+          :max => max > now ? now : max
+        )
       end
 
       def null_value
@@ -516,6 +505,35 @@ module Purview
         raise %{All "#{Base}(s)" must override the "set_max_timestamp_pulled_for_table_sql" method}
       end
 
+      def sync_table_with_lock(table, timestamp)
+        last_window = nil
+        with_table_locked(table, timestamp) do
+          last_window = sync_table_without_lock(table, timestamp)
+        end
+        last_window
+      end
+
+      def sync_table_without_lock(table, timestamp)
+        last_window = nil
+        with_next_window(table, timestamp) do |window|
+          with_new_transaction do |connection|
+            table.sync(connection, window)
+            set_last_pulled_at_for_table(
+              connection,
+              table,
+              timestamp
+            )
+            set_max_timestamp_pulled_for_table(
+              connection,
+              table,
+              window.max
+            )
+            last_window = window
+          end
+        end
+        last_window
+      end
+
       def table_metadata_enabled_at_column_definition
         column = Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
         column_definition(column)
@@ -610,30 +628,24 @@ module Purview
         raise %{All "#{Base}(s)" must override the "unlock_table_sql" method}
       end
 
-      def with_next_table(connection, timestamp)
-        table = next_table(connection, timestamp)
-        raise Purview::Exceptions::NoTable.new unless table
-        yield table
-        set_last_pulled_at_for_table(
-          connection,
-          table,
-          timestamp
-        )
+      def with_next_table(timestamp)
+        with_new_connection do |connection|
+          table = next_table(connection, timestamp)
+          raise Purview::Exceptions::NoTable.new unless table
+          yield table
+        end
       end
 
-      def with_next_window(connection, table, timestamp)
-        window = next_window(
-          connection,
-          table,
-          timestamp
-        )
-        raise Purview::Exceptions::NoWindow.new(table) unless window
-        yield window
-        set_max_timestamp_pulled_for_table(
-          connection,
-          table,
-          window.max
-        )
+      def with_next_window(table, timestamp)
+        with_new_connection do |connection|
+          window = next_window(
+            connection,
+            table,
+            timestamp
+          )
+          raise Purview::Exceptions::NoWindow.new(table) unless window
+          yield window
+        end
       end
 
       def with_table_locked(table, timestamp)
@@ -642,10 +654,6 @@ module Purview
       ensure
         unlock_table(table)
       end
-
-      def with_transaction(connection)
-        connection.with_transaction { yield Time.now.utc }
-      end
     end
   end
 end

data/lib/purview/databases/mysql.rb

@@ -97,11 +97,11 @@ module Purview
         ]
       end
 
-      def initialize_table_sql(table, starting_timestamp)
+      def initialize_table_sql(table, timestamp)
         'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
           table_metadata_table_name,
           table_metadata_max_timestamp_pulled_column_name,
-          quoted(starting_timestamp),
+          quoted(timestamp),
           table_metadata_table_name_column_name,
           quoted(table.name),
           table_metadata_max_timestamp_pulled_column_name,

data/lib/purview/databases/postgresql.rb

@@ -97,11 +97,11 @@ module Purview
         ]
       end
 
-      def initialize_table_sql(table, starting_timestamp)
+      def initialize_table_sql(table, timestamp)
         'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
           table_metadata_table_name,
           table_metadata_max_timestamp_pulled_column_name,
-          quoted(starting_timestamp),
+          quoted(timestamp),
           table_metadata_table_name_column_name,
           quoted(table.name),
           table_metadata_max_timestamp_pulled_column_name,

data/lib/purview/mixins/connection.rb

@@ -18,6 +18,12 @@ module Purview
       def with_new_connection
         connection_type.with_new_connection(connection_opts) { |connection| yield connection }
       end
+
+      def with_new_transaction
+        with_new_connection do |connection|
+          connection.with_transaction { yield connection }
+        end
+      end
     end
   end
 end

data/lib/purview/mixins/helpers.rb

@@ -21,6 +21,14 @@ module Purview
         !blank?(value)
       end
 
+      def timestamp
+        Time.now
+      end
+
+      def with_timestamp
+        yield timestamp
+      end
+
       def zero?(value)
         Integer(value).zero?
       end

data/lib/purview/raw_connections/jdbc/base.rb

@@ -29,15 +29,16 @@ module Purview
         end
 
         def extract_rows(result)
-          return unless result
-          metadata = result.getMetaData
-          column_count = metadata.getColumnCount
-          [].tap do |rows|
-            while result.next
-              rows << {}.tap do |row|
-                (1..column_count).each do |index|
-                  column_name = metadata.getColumnName(index)
-                  row[column_name] = result.getString(column_name)
+          if result
+            metadata = result.getMetaData
+            column_count = metadata.getColumnCount
+            [].tap do |rows|
+              while result.next
+                rows << {}.tap do |row|
+                  (1..column_count).each do |index|
+                    column_name = metadata.getColumnName(index)
+                    row[column_name] = result.getString(column_name)
+                  end
                 end
               end
             end

data/lib/purview/tables/base.rb

@@ -68,7 +68,7 @@ module Purview
       end
 
       def temporary_name
-        "#{name}_#{Time.now.utc.to_i}"
+        "#{name}_#{timestamp.to_i}"
       end
 
       def updated_timestamp_column
@@ -81,6 +81,7 @@ module Purview
 
       private
 
+      include Purview::Mixins::Helpers
       include Purview::Mixins::Logger
 
       attr_reader :opts

data/lib/purview/version.rb

@@ -1,3 +1,3 @@
 module Purview
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: purview
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Jonathan W. Zaleski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-02 00:00:00.000000000 Z
+date: 2015-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler