purview 1.0.0.alpha

data/lib/purview/databases/mysql.rb

@@ -0,0 +1,207 @@
+module Purview
+  module Databases
+    class MySQL < Base
+      def false_value
+        '0'
+      end
+
+      def null_value
+        'NULL'
+      end
+
+      def true_value
+        '1'
+      end
+
+      private
+
+      def connection_opts
+        super.merge(:database => name)
+      end
+
+      def connection_type
+        Purview::Connections::MySQL
+      end
+
+      def create_index_sql(table_name, index_name, table, columns, index_opts={})
+        'CREATE INDEX %s ON %s (%s)' % [
+          index_name,
+          table_name,
+          column_names(columns).join(', '),
+        ]
+      end
+
+      def create_table_sql(table_name, table, table_opts={})
+        'CREATE TABLE %s (%s)' % [
+          table_name,
+          column_definitions(table).join(', '),
+        ]
+      end
+
+      def create_temporary_table_sql(table_name, table, table_opts={})
+        'CREATE TEMPORARY TABLE %s LIKE %s' % [
+          table_name,
+          table.name,
+        ]
+      end
+
+      def default_map
+        super.merge(Purview::Types::Timestamp => '0')
+      end
+
+      def drop_index_sql(table_name, index_name, table, columns, index_opts={})
+        'DROP INDEX %s' % [
+          index_name,
+        ]
+      end
+
+      def drop_table_sql(table_name, table, table_opts={})
+        'DROP TABLE %s' % [
+          table_name,
+        ]
+      end
+
+      def ensure_table_metadata_exists_for_table_sql(table)
+        'INSERT IGNORE INTO %s VALUES (%s, NULL, NULL, NULL, NULL)' % [
+          table_metadata_table_name,
+          quoted(table.name),
+        ]
+      end
+
+      def ensure_table_metadata_table_exists_sql
+        'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
+          table_metadata_table_name,
+          table_metadata_table_name_column_definition,
+          table_metadata_enabled_column_definition,
+          table_metadata_last_pulled_at_column_definition,
+          table_metadata_locked_at_column_definition,
+          table_metadata_max_timestamp_pulled_column_definition,
+        ]
+      end
+
+      def get_enabled_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_enabled_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def get_last_pulled_at_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_last_pulled_at_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def get_locked_at_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_locked_at_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def get_max_timestamp_pulled_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_max_timestamp_pulled_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def limit_map
+        super.merge(Purview::Types::String => 255)
+      end
+
+      def lock_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table_name,
+          table_metadata_locked_at_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_locked_at_column_name,
+        ]
+      end
+
+      def next_table_sql(timestamp)
+        'SELECT %s FROM %s WHERE %s = %s AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
+          table_metadata_table_name_column_name,
+          table_metadata_table_name,
+          table_metadata_enabled_column_name,
+          quoted(true_value),
+          table_metadata_locked_at_column_name,
+          table_metadata_last_pulled_at_column_name,
+          table_metadata_last_pulled_at_column_name,
+        ]
+      end
+
+      def set_enabled_for_table_sql(table, enabled)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND (%s IS NULL OR %s = %s)' % [
+          table_metadata_table_name,
+          table_metadata_enabled_column_name,
+          quoted(enabled),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_enabled_column_name,
+          table_metadata_enabled_column_name,
+          quoted(false_value),
+        ]
+      end
+
+      def set_last_pulled_at_for_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s' % [
+          table_metadata_table_name,
+          table_metadata_last_pulled_at_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def set_locked_at_for_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
+          table_metadata_table_name,
+          table_metadata_locked_at_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def set_max_timestamp_pulled_for_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s' % [
+          table_metadata_table_name,
+          table_metadata_max_timestamp_pulled_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def type_map
+        super.merge(
+          Purview::Types::Money => 'decimal',
+          Purview::Types::UUID => 'varchar',
+        )
+      end
+
+      def unlock_table_sql(table)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
+          table_metadata_table_name,
+          table_metadata_locked_at_column_name,
+          null_value,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_locked_at_column_name,
+        ]
+      end
+    end
+  end
+end

data/lib/purview/databases/postgresql.rb

@@ -0,0 +1,210 @@
+module Purview
+  module Databases
+    class PostgreSQL < Base
+      def false_value
+        'FALSE'
+      end
+
+      def null_value
+        'NULL'
+      end
+
+      def true_value
+        'TRUE'
+      end
+
+      private
+
+      def connection_opts
+        super.merge(:dbname => name)
+      end
+
+      def connection_type
+        Purview::Connections::PostgreSQL
+      end
+
+      def create_index_sql(table_name, index_name, table, columns, index_opts={})
+        'CREATE INDEX %s ON %s (%s)' % [
+          index_name,
+          table_name,
+          column_names(columns).join(', '),
+        ]
+      end
+
+      def create_table_sql(table_name, table, table_opts={})
+        'CREATE TABLE %s (%s)' % [
+          table_name,
+          column_definitions(table).join(', '),
+        ]
+      end
+
+      def create_temporary_table_sql(table_name, table, table_opts={})
+        'CREATE TEMPORARY TABLE %s (LIKE %s INCLUDING ALL)' % [
+          table_name,
+          table.name,
+        ]
+      end
+
+      def drop_index_sql(table_name, index_name, table, columns, index_opts={})
+        'DROP INDEX %s' % [
+          index_name,
+        ]
+      end
+
+      def drop_table_sql(table_name, table, table_opts={})
+        'DROP TABLE %s' % [
+          table_name,
+        ]
+      end
+
+      def ensure_table_metadata_exists_for_table_sql(table)
+        'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def ensure_table_metadata_table_exists_sql
+        'CREATE TABLE IF NOT EXISTS %s (%s PRIMARY KEY, %s, %s, %s, %s)' % [
+          table_metadata_table_name,
+          table_metadata_table_name_column_definition,
+          table_metadata_enabled_column_definition,
+          table_metadata_last_pulled_at_column_definition,
+          table_metadata_locked_at_column_definition,
+          table_metadata_max_timestamp_pulled_column_definition,
+        ]
+      end
+
+      def get_enabled_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_enabled_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def get_last_pulled_at_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_last_pulled_at_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def get_locked_at_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_locked_at_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def get_max_timestamp_pulled_for_table_sql(table)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          table_metadata_max_timestamp_pulled_column_name,
+          table_metadata_table_name,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def limitless_types
+        super + [
+          Purview::Types::Money,
+          Purview::Types::UUID,
+        ]
+      end
+
+      def lock_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table_name,
+          table_metadata_locked_at_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_locked_at_column_name,
+        ]
+      end
+
+      def next_table_sql(timestamp)
+        'SELECT %s FROM %s WHERE %s = %s AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
+          table_metadata_table_name_column_name,
+          table_metadata_table_name,
+          table_metadata_enabled_column_name,
+          quoted(true_value),
+          table_metadata_locked_at_column_name,
+          table_metadata_last_pulled_at_column_name,
+          table_metadata_last_pulled_at_column_name,
+        ]
+      end
+
+      def set_enabled_for_table_sql(table, enabled)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND (%s IS NULL OR %s = %s)' % [
+          table_metadata_table_name,
+          table_metadata_enabled_column_name,
+          quoted(enabled),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_enabled_column_name,
+          table_metadata_enabled_column_name,
+          quoted(false_value),
+        ]
+      end
+
+      def set_last_pulled_at_for_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s' % [
+          table_metadata_table_name,
+          table_metadata_last_pulled_at_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def set_locked_at_for_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s' % [
+          table_metadata_table_name,
+          table_metadata_locked_at_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def set_max_timestamp_pulled_for_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s' % [
+          table_metadata_table_name,
+          table_metadata_max_timestamp_pulled_column_name,
+          quoted(timestamp),
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+        ]
+      end
+
+      def type_map
+        super.merge(
+          Purview::Types::Money => 'money',
+          Purview::Types::UUID => 'uuid',
+        )
+      end
+
+      def unlock_table_sql(table)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
+          table_metadata_table_name,
+          table_metadata_locked_at_column_name,
+          null_value,
+          table_metadata_table_name_column_name,
+          quoted(table.name),
+          table_metadata_locked_at_column_name,
+        ]
+      end
+    end
+  end
+end

data/lib/purview/databases.rb

@@ -0,0 +1,3 @@
+require 'purview/databases/base'
+require 'purview/databases/mysql'
+require 'purview/databases/postgresql'

data/lib/purview/exceptions/base.rb

@@ -0,0 +1,5 @@
+module Purview
+  module Exceptions
+    class Base < StandardError; end
+  end
+end

data/lib/purview/exceptions/could_not_acquire_lock.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class CouldNotAcquireLock < Table
+      def message
+        "Could not acquire the lock for table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/lock_already_released.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class LockAlreadyReleased < Table
+      def message
+        "Lock already released for table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/no_table.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class NoTable < Base
+      def message
+        'Could not find a table'
+      end
+    end
+  end
+end

data/lib/purview/exceptions/no_window.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Exceptions
+    class NoWindow < Table
+      def message
+        "Could not find a window for table: #{table.name}"
+      end
+    end
+  end
+end

data/lib/purview/exceptions/rows_outside_window.rb

@@ -0,0 +1,18 @@
+module Purview
+  module Exceptions
+    class RowsOutsideWindow < Base
+      def initialize(table_name, count)
+        @table_name = table_name
+        @count = count
+      end
+
+      def message
+        "#{count} row(s) outside window for table: #{table_name}"
+      end
+
+      private
+
+      attr_reader :count, :table_name
+    end
+  end
+end

data/lib/purview/exceptions/table.rb

@@ -0,0 +1,13 @@
+module Purview
+  module Exceptions
+    class Table < Base
+      def initialize(table)
+        @table = table
+      end
+
+      private
+
+      attr_reader :table
+    end
+  end
+end

data/lib/purview/exceptions.rb

@@ -0,0 +1,7 @@
+require 'purview/exceptions/base'
+require 'purview/exceptions/table'
+require 'purview/exceptions/could_not_acquire_lock'
+require 'purview/exceptions/lock_already_released'
+require 'purview/exceptions/no_table'
+require 'purview/exceptions/no_window'
+require 'purview/exceptions/rows_outside_window'

data/lib/purview/loaders/base.rb

@@ -0,0 +1,154 @@
+module Purview
+  module Loaders
+    class Base
+      def initialize(opts={})
+        @opts = opts
+      end
+
+      def load(connection, rows, window)
+        with_context_logging("`load` for: #{table.name}") do
+          with_temporary_table(connection, rows, window) do |temporary_table_name|
+            update_result = \
+              connection.execute(table_update_sql(window, temporary_table_name))
+            delete_result = \
+              connection.execute(table_delete_sql(window, temporary_table_name))
+            insert_result = \
+              connection.execute(table_insert_sql(window, temporary_table_name))
+            logger.debug(
+              '%d row(s) inserted, %d row(s) updated and %d row(s) deleted in: %s' % [
+                insert_result.rows_affected,
+                update_result.rows_affected,
+                delete_result.rows_affected,
+                table.name,
+              ]
+            )
+          end
+        end
+      end
+
+      private
+
+      include Purview::Mixins::Helpers
+      include Purview::Mixins::Logger
+
+      attr_reader :opts
+
+      def count_column_name
+        'count'
+      end
+
+      def create_temporary_table(connection)
+        database.create_temporary_table(
+          connection,
+          table,
+          :table => temporary_table_opts
+        )
+      end
+
+      def database
+        table.database
+      end
+
+      def id_in_sql(temporary_table_name)
+        raise %{All "#{Base}(s)" must override the "id_in_sql" method}
+      end
+
+      def in_window_sql(window)
+        raise %{All "#{Base}(s)" must override the "in_window_sql" method}
+      end
+
+      def load_temporary_table(connection, temporary_table_name, rows, window)
+        with_context_logging("`load_temporary_table` for: #{temporary_table_name}") do
+          rows.each_slice(rows_per_slice) do |rows_slice|
+            connection.execute(
+              temporary_table_insert_sql(
+                temporary_table_name,
+                rows_slice
+              )
+            )
+          end
+        end
+      end
+
+      def not_in_window_sql(window)
+        raise %{All "#{Base}(s)" must override the "not_in_window_sql" method}
+      end
+
+      def quoted(value)
+        database.quoted(value)
+      end
+
+      def row_values(row)
+        table.column_names.map { |column_name| quoted(row[column_name]) }.join(', ')
+      end
+
+      def rows_per_slice
+        opts[:rows_per_slice] || 1000
+      end
+
+      def table
+        opts[:table]
+      end
+
+      def table_delete_sql(window, temporary_table_name)
+        raise %{All "#{Base}(s)" must override the "table_delete_sql" method}
+      end
+
+      def table_insert_sql(window, temporary_table_name)
+        raise %{All "#{Base}(s)" must override the "table_insert_sql" method}
+      end
+
+      def table_update_sql(window, temporary_table_name)
+        raise %{All "#{Base}(s)" must override the "table_update_sql" method}
+      end
+
+      def temporary_table_insert_sql(temporary_table_name, rows)
+        raise %{All "#{Base}(s)" must override the "temporary_table_insert_sql" method}
+      end
+
+      def temporary_table_opts
+        {
+          :create_indices => true,
+          :name => table.temporary_name,
+        }
+      end
+
+      def temporary_table_verify_sql(temporary_table_name, rows, window)
+        raise %{All "#{Base}(s)" must override the "temporary_table_verify_sql" method}
+      end
+
+      def verify_temporary_table(connection, temporary_table_name, rows, window)
+        with_context_logging("`verify_temporary_table` for: #{temporary_table_name}") do
+          rows_outside_window = connection.execute(
+            temporary_table_verify_sql(
+              temporary_table_name,
+              rows,
+              window
+            )
+          ).rows[0][count_column_name]
+          raise Purview::Exceptions::RowsOutsideWindow.new(temporary_table_name, rows_outside_window) \
+            unless zero?(rows_outside_window)
+        end
+      end
+
+      def with_temporary_table(connection, rows, window)
+        yield(
+          create_temporary_table(connection).tap do |temporary_table_name|
+            load_temporary_table(
+              connection,
+              temporary_table_name,
+              rows,
+              window
+            )
+            verify_temporary_table(
+              connection,
+              temporary_table_name,
+              rows,
+              window
+            )
+          end
+        )
+      end
+    end
+  end
+end