purview 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bfebf90c2dfbac7b718635c204d78fb6921a779
-  data.tar.gz: 7b2da04835cbf716b784300140e02b78d2188aa9
+  metadata.gz: 30aabf4950bc6dc20daf5b89eec19e9afe148765
+  data.tar.gz: 544c19c81308f7276a02a8b6149dd97bcd7f3312
 SHA512:
-  metadata.gz: 5ec0e4d36ea4d42dd29c3d8bf1608c6c30db990ecc4e95b0fc6ddf7c951db243f2225b5a7be117f2f11597a7467cfada79f8b90073de2c5c13c60a6ce79932d8
-  data.tar.gz: d43914f1565f2f3a6c4ec41245a4e06e00069c521fb9e892f378500640e16c6e277bc5c93595c95d98d4da34751c97e01afaf3f6e7b28390ab60d610a087b888
+  metadata.gz: 80b799628ae3f89c0eacab1cf449d1fadaf318d9f590c7408624e66849a9c527a9d7f1f280b9df03bbe54e62b496cf99d379091c0ae78aa571fdb9e6fbfe329e
+  data.tar.gz: 21600dcd05785d91a627338b14396290fed0dd3c4e18f9a116b9c9b23f25f3fd3715a48e749db0205d283ffd1a26b2764f7bf2339fb9ec90e2c0ea61d43a7708

data/CHANGELOG

@@ -1,3 +1,12 @@
+Release 1.4.0 (2015-06-10)
+--------------------------
+
+* Add the ability to rename columns during pull/parse
+* Create class to encapulate `table_metadata`
+* Standardize `Exception` class names
+* Allow `table` to be set for `Column(s)` & `Ind{ex,ices}` during instantiation
+* Allow `database` to be set for `Table(s)` during instantiation
+
 Release 1.3.1 (2015-06-05)
 --------------------------
 

data/TODO

@@ -44,6 +44,10 @@
 * Introduce [SQL] dialect(s)
 * Fix JRuby bundle/build
 * Fall back to logged in user if no [database-]username is provided
+* Create class to encapulate `table_metadata` (in order to clean up the logic
+  in the `Database` class(es))
+* Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe
+  create a new type of column?
 
 ... ONGOING TASKS ...
 
@@ -55,17 +59,16 @@
 
 * Add COPY `def copy(sql)` support to PostgreSQL connection
 * Leverage COPY support for PostgreSQL pulls
-* Create class to encapulate `table_metadata` (in order to clean up the logic
-  in the `Database` and `TableMetadata` (struct) classes)
 * Further encapsulate `Dialect` logic (in order to DRY `Database` classes)
+* Add support for MSSQL
 
 * Build out change-log tables
 * Build out canonical, fact and aggregate tables (and related transforms)
 * Configurable re-pull window (do this automatically once up to current?)
 * Add schema management capabilities (detect schema-deltas and suggestion
   modifications)
-* Consider refactoring mutator methods to be globally available (e.g. coalesced
-  -> Coalesce, quoted -> Quote, etc.)
+* Consider refactoring mutator methods to be globally available (e.g. quoted ->
+  Quote, etc.)
 
 ... CLOSED QUESTIONS ...
 
@@ -79,8 +82,6 @@
 ... OPEN QUESTIONS ...
 
 * Parallel pulls from the same "source"? Can this be done in one request?
-* Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe
-  create a new type of column?
 
 ... INTEGRATION CONSIDERATIONS ...
 

data/lib/purview/columns/base.rb

@@ -1,11 +1,12 @@
 module Purview
   module Columns
     class Base
-      attr_reader :name
+      attr_reader :name, :table
 
       def initialize(name, opts={})
         @name = name.to_sym
         @opts = default_opts.merge(opts)
+        @table = table_opt
       end
 
       def default
@@ -16,6 +17,24 @@ module Purview
         !!default
       end
 
+      def eql?(other)
+        self.class == other.class &&
+          limit == other.limit &&
+          name == other.name &&
+          nullable == other.nullable &&
+          primary_key == other.primary_key &&
+          type == other.type
+      end
+
+      def hash
+        default.hash +
+          limit.hash +
+          name.hash +
+          nullable.hash +
+          primary_key.hash +
+          type.hash
+      end
+
       def limit
         opts[:limit]
       end
@@ -25,7 +44,7 @@ module Purview
       end
 
       def nullable
-        coalesced(opts[:nullable], true)
+        [nil, true].include?(opts[:nullable])
       end
 
       def nullable?
@@ -46,8 +65,17 @@ module Purview
         !!primary_key
       end
 
+      def source_name
+        (opts[:source_name] || name).to_sym
+      end
+
+      def table=(value)
+        raise Purview::Exceptions::TableAlreadyAssignedForColumn.new(self) if table
+        @table = value
+      end
+
       def type
-        coalesced(opts[:type], Purview::Types::String)
+        opts[:type] || Purview::Types::String
       end
 
       private
@@ -59,6 +87,10 @@ module Purview
       def default_opts
         {}
       end
+
+      def table_opt
+        opts[:table]
+      end
     end
   end
 end

data/lib/purview/databases/base.rb

@@ -4,10 +4,10 @@ module Purview
       attr_reader :name, :tables
 
       def initialize(name, opts={})
-        @name = name
+        @name = name.to_sym
         @opts = opts
         @tables = Set.new.tap do |result|
-          (opts[:tables] || []).each do |table|
+          (default_tables + tables_opt).each do |table|
             table.database = self if result.add?(table)
           end
         end
@@ -15,7 +15,7 @@ module Purview
 
       def baseline_table(table)
         ensure_table_valid_for_database(table)
-        raise Purview::Exceptions::CouldNotBaseline.new(table) \
+        raise Purview::Exceptions::CouldNotBaselineTable.new(table) \
           unless table_initialized?(table)
         table_name = table_name(table)
         with_context_logging("`baseline_table` for: #{table_name}") do
@@ -32,13 +32,12 @@ module Purview
 
       def create_index(index, opts={})
         ensure_index_valid_for_database(index)
-        table_opts = extract_table_options(opts)
+        table_opts = extract_table_opts(opts)
         table_name = table_name(index.table, table_opts)
-        index_opts = extract_index_options(opts)
-        index_columns = index.columns
+        index_opts = extract_index_opts(opts)
         index_name = index_name(
           table_name,
-          index_columns,
+          index,
           index_opts
         )
         with_context_logging("`create_index` for: #{index_name}") do
@@ -59,7 +58,7 @@ module Purview
       def create_table(table, opts={})
         ensure_table_valid_for_database(table)
         ensure_table_metadata_exists_for_table(table)
-        table_opts = extract_table_options(opts)
+        table_opts = extract_table_opts(opts)
         table_name = table_name(table, table_opts)
         with_context_logging("`create_table` for: #{table_name}") do
           with_new_or_existing_connection(opts) do |connection|
@@ -92,7 +91,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(disable_table_sql(table)).rows_affected
-            raise Purview::Exceptions::CouldNotDisable.new(table) \
+            raise Purview::Exceptions::CouldNotDisableTable.new(table) \
               if zero?(rows_affected)
           end
         end
@@ -101,13 +100,12 @@ module Purview
 
       def drop_index(index, opts={})
         ensure_index_valid_for_database(index)
-        table_opts = extract_table_options(opts)
+        table_opts = extract_table_opts(opts)
         table_name = table_name(index.table, table_opts)
-        index_opts = extract_index_options(opts)
-        index_columns = index.columns
+        index_opts = extract_index_opts(opts)
         index_name = index_name(
           table_name,
-          index_columns,
+          index,
           index_opts
         )
         with_context_logging("`drop_index` for: #{index_name}") do
@@ -128,7 +126,7 @@ module Purview
       def drop_table(table, opts={})
         ensure_table_valid_for_database(table)
         ensure_table_metadata_absent_for_table(table)
-        table_opts = extract_table_options(opts)
+        table_opts = extract_table_opts(opts)
         table_name = table_name(table, table_opts)
         with_context_logging("`drop_table` for: #{table_name}") do
           with_new_connection do |connection|
@@ -151,7 +149,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(enable_table_sql(table, timestamp)).rows_affected
-            raise Purview::Exceptions::CouldNotEnable.new(table) \
+            raise Purview::Exceptions::CouldNotEnableTable.new(table) \
               if zero?(rows_affected)
           end
         end
@@ -165,7 +163,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(initialize_table_sql(table, timestamp)).rows_affected
-            raise Purview::Exceptions::CouldNotInitialize.new(table) \
+            raise Purview::Exceptions::CouldNotInitializeTable.new(table) \
               if zero?(rows_affected)
           end
         end
@@ -179,7 +177,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(lock_table_sql(table, timestamp)).rows_affected
-            raise Purview::Exceptions::CouldNotLock.new(table) \
+            raise Purview::Exceptions::CouldNotLockTable.new(table) \
               if zero?(rows_affected)
           end
         end
@@ -198,7 +196,7 @@ module Purview
 
       def sync_table(table)
         ensure_table_valid_for_database(table)
-        raise Purview::Exceptions::CouldNotSync.new(table) \
+        raise Purview::Exceptions::CouldNotSyncTable.new(table) \
           unless table_initialized?(table) && table_enabled?(table)
         table_name = table_name(table)
         with_context_logging("`sync_table` for: #{table_name}") do
@@ -213,10 +211,16 @@ module Purview
         table_name = table_name(table)
         with_context_logging("`table_metadata` for: #{table_name}") do
           with_new_connection do |connection|
-            row = connection.execute(table_metadata_sql(table)).rows.first
-            raise Purview::Exceptions::CouldNotFindTableMetadata.new(table) \
-              unless row
-            table_metadata = Purview::Structs::TableMetadata.new(row)
+            Purview::Structs::TableMetadata.new(
+              table_metadata_table.columns.reduce({}) do |memo, column|
+                memo[column.name] = get_table_metadata_value(
+                  connection,
+                  table,
+                  column
+                )
+                memo
+              end
+            )
           end
         end
         table_metadata
@@ -229,7 +233,7 @@ module Purview
           with_new_connection do |connection|
             rows_affected = \
               connection.execute(unlock_table_sql(table)).rows_affected
-            raise Purview::Exceptions::CouldNotUnlock.new(table) \
+            raise Purview::Exceptions::CouldNotUnlockTable.new(table) \
               if zero?(rows_affected)
           end
         end
@@ -239,6 +243,7 @@ module Purview
       private
 
       include Purview::Mixins::Connection
+      include Purview::Mixins::Dialect
       include Purview::Mixins::Helpers
       include Purview::Mixins::Logger
 
@@ -261,19 +266,12 @@ module Purview
         end
       end
 
-      def column_names(columns)
-        columns.map(&:name)
+      def column_names(index_or_table)
+        index_or_table.columns.map(&:name)
       end
 
-      def column_definitions(table)
-        [].tap do |result|
-          result << column_definition(table.id_column)
-          result << column_definition(table.created_timestamp_column)
-          result << column_definition(table.updated_timestamp_column)
-          table.data_columns.each do |column|
-            result << column_definition(column)
-          end
-        end
+      def column_definitions(index_or_table)
+        index_or_table.columns.map { |column| column_definition(column) }
       end
 
       def connection_type
@@ -324,8 +322,8 @@ module Purview
         {}
       end
 
-      def dialect
-        dialect_type.new
+      def default_tables
+        []
       end
 
       def dialect_type
@@ -383,66 +381,30 @@ module Purview
       def ensure_table_valid_for_database(table)
         raise ArgumentError.new('Must provide a `Table`') \
           unless table
-        raise Purview::Exceptions::WrongDatabase.new(table) \
+        raise Purview::Exceptions::WrongDatabaseForTable.new(table) \
           unless tables.include?(table)
       end
 
-      def extract_index_options(opts)
+      def extract_index_opts(opts)
         opts[:index] || {}
       end
 
-      def extract_table_options(opts)
+      def extract_table_opts(opts)
         opts[:table] || { :create_indices => true }
       end
 
-      def false_value
-        dialect.false_value
-      end
-
-      def get_enabled_at_for_table(connection, table)
-        row = connection.execute(get_enabled_at_for_table_sql(table)).rows[0]
-        timestamp = row[table_metadata_enabled_at_column_name]
-        timestamp ? Time.parse(timestamp) : nil
-      end
-
-      def get_enabled_at_for_table_sql(table)
-        raise %{All "#{Base}(s)" must override the "get_enabled_at_for_table_sql" method}
-      end
-
-      def get_last_pulled_at_for_table(connection, table)
-        row = connection.execute(get_last_pulled_at_for_table_sql(table)).rows[0]
-        timestamp = row[table_metadata_last_pulled_at_column_name]
-        timestamp ? Time.parse(timestamp) : nil
-      end
-
-      def get_last_pulled_at_for_table_sql(table)
-        raise %{All "#{Base}(s)" must override the "get_last_pulled_at_for_table_sql" method}
-      end
-
-      def get_locked_at_for_table(connection, table)
-        row = connection.execute(get_locked_at_for_table_sql(table)).rows[0]
-        timestamp = row[table_metadata_locked_at_column_name]
-        timestamp ? Time.parse(timestamp) : nil
-      end
-
-      def get_locked_at_for_table_sql(table)
-        raise %{All "#{Base}(s)" must override the "get_locked_at_for_table_sql" method}
-      end
-
-      def get_max_timestamp_pulled_for_table(connection, table)
-        row = connection.execute(get_max_timestamp_pulled_for_table_sql(table)).rows[0]
-        timestamp = row[table_metadata_max_timestamp_pulled_column_name]
-        timestamp ? Time.parse(timestamp) : nil
-      end
-
-      def get_max_timestamp_pulled_for_table_sql(table)
-        raise %{All "#{Base}(s)" must override the "get_max_timestamp_pulled_for_table_sql" method}
+      def get_table_metadata_value(connection, table, column)
+        row = connection.execute(get_table_metadata_value_sql(table, column)).rows[0]
+        raise CouldNotFindMetadataForTable.new(table) \
+          unless row
+        value = row[column.name]
+        value && column.type.parse(value)
       end
 
-      def index_name(table_name, columns, index_opts={})
+      def index_name(table_name, index, index_opts={})
         index_opts[:name] || 'index_%s_on_%s' % [
           table_name,
-          column_names(columns).join('_and_'),
+          column_names(index).join('_and_'),
         ]
       end
 
@@ -468,8 +430,8 @@ module Purview
 
       def next_table(connection, timestamp)
         row = connection.execute(next_table_sql(timestamp)).rows[0]
-        table_name = row && row[table_metadata_table_name_column_name]
-        table_name && tables_by_name[table_name]
+        value = row && row[table_metadata_table.table_name_column.name]
+        value && table_metadata_table.table_name_column.type.parse(value)
       end
 
       def next_table_sql(timestamp)
@@ -477,7 +439,11 @@ module Purview
       end
 
       def next_window(connection, table, timestamp)
-        min = get_max_timestamp_pulled_for_table(connection, table)
+        min = get_table_metadata_value(
+          connection,
+          table,
+          table_metadata_table.max_timestamp_pulled_column
+        )
         max = min + table.window_size
         now = timestamp
         min > now ? nil : Purview::Structs::Window.new(
@@ -486,10 +452,6 @@ module Purview
         )
       end
 
-      def null_value
-        dialect.null_value
-      end
-
       def nullable?(column)
         column.nullable?
       end
@@ -498,44 +460,11 @@ module Purview
         column.primary_key?
       end
 
-      def quoted(value)
-        dialect.quoted(value)
-      end
-
-      def sanitized(value)
-        dialect.sanitized(value)
-      end
-
-      def set_enabled_at_for_table(connection, table, timestamp)
-        connection.execute(set_enabled_at_for_table_sql(table, timestamp))
-      end
-
-      def set_enabled_at_for_table_sql(table, timestamp)
-        raise %{All "#{Base}(s)" must override the "set_enabled_at_for_table_sql" method}
-      end
-
-      def set_last_pulled_at_for_table(connection, table, timestamp)
-        connection.execute(set_last_pulled_at_for_table_sql(table, timestamp))
-      end
-
-      def set_last_pulled_at_for_table_sql(table, timestamp)
-        raise %{All "#{Base}(s)" must override the "set_last_pulled_at_for_table_sql" method}
-      end
-
-      def set_locked_at_for_table(connection, table, timestamp)
-        connection.execute(set_locked_at_for_table_sql(table, timestamp))
-      end
-
-      def set_locked_at_for_table_sql(table, timestamp)
-        raise %{All "#{Base}(s)" must override the "set_locked_at_for_table_sql" method}
-      end
-
-      def set_max_timestamp_pulled_for_table(connection, table, timestamp)
-        connection.execute(set_max_timestamp_pulled_for_table_sql(table, timestamp))
-      end
-
-      def set_max_timestamp_pulled_for_table_sql(table, timestamp)
-        raise %{All "#{Base}(s)" must override the "set_max_timestamp_pulled_for_table_sql" method}
+      def set_table_metadata_value(connection, table, column, value)
+        rows_affected = \
+          connection.execute(set_table_metadata_value_sql(table, column, value)).rows_affected
+        raise CouldNotUpdateMetadataForTable.new(table) \
+          if zero?(rows_affected)
       end
 
       def sync_table_with_lock(table, timestamp)
@@ -552,14 +481,16 @@ module Purview
           with_new_connection do |connection|
             with_transaction(connection) do
               table.sync(connection, window)
-              set_last_pulled_at_for_table(
+              set_table_metadata_value(
                 connection,
                 table,
+                table_metadata_table.last_pulled_at_column,
                 timestamp
               )
-              set_max_timestamp_pulled_for_table(
+              set_table_metadata_value(
                 connection,
                 table,
+                table_metadata_table.max_timestamp_pulled_column,
                 window.max
               )
               last_window = window
@@ -575,116 +506,36 @@ module Purview
 
       def table_enabled?(table)
         with_new_connection do |connection|
-          !!get_enabled_at_for_table(connection, table)
+          !!get_table_metadata_value(
+              connection,
+              table,
+              table_metadata_table.enabled_at_column
+            )
         end
       end
 
       def table_initialized?(table)
         with_new_connection do |connection|
-          !!get_max_timestamp_pulled_for_table(connection, table)
+          !!get_table_metadata_value(
+              connection,
+              table,
+              table_metadata_table.max_timestamp_pulled_column
+            )
         end
       end
 
       def table_locked?(table)
         with_new_connection do |connection|
-          !!get_locked_at_for_table(connection, table)
+          !!get_table_metadata_value(
+              connection,
+              table,
+              table_metadata_table.locked_at_column
+            )
         end
       end
 
-      def table_metadata_column_definitions
-        [
-          table_metadata_table_name_column_definition,
-          table_metadata_enabled_at_column_definition,
-          table_metadata_last_pulled_at_column_definition,
-          table_metadata_locked_at_column_definition,
-          table_metadata_max_timestamp_pulled_column_definition,
-        ]
-      end
-
-      def table_metadata_column_names
-        table_metadata_columns.map(&:name)
-      end
-
-      def table_metadata_columns
-        [
-          table_metadata_table_name_column,
-          table_metadata_enabled_at_column,
-          table_metadata_last_pulled_at_column,
-          table_metadata_locked_at_column,
-          table_metadata_max_timestamp_pulled_column,
-        ]
-      end
-
-      def table_metadata_enabled_at_column
-        Purview::Columns::Timestamp.new(table_metadata_enabled_at_column_name)
-      end
-
-      def table_metadata_enabled_at_column_definition
-        column_definition(table_metadata_enabled_at_column)
-      end
-
-      def table_metadata_enabled_at_column_name
-        'enabled_at'
-      end
-
-      def table_metadata_last_pulled_at_column
-        Purview::Columns::Timestamp.new(table_metadata_last_pulled_at_column_name)
-      end
-
-      def table_metadata_last_pulled_at_column_definition
-        column_definition(table_metadata_last_pulled_at_column)
-      end
-
-      def table_metadata_last_pulled_at_column_name
-        'last_pulled_at'
-      end
-
-      def table_metadata_locked_at_column
-        Purview::Columns::Timestamp.new(table_metadata_locked_at_column_name)
-      end
-
-      def table_metadata_locked_at_column_definition
-        column_definition(table_metadata_locked_at_column)
-      end
-
-      def table_metadata_locked_at_column_name
-        'locked_at'
-      end
-
-      def table_metadata_max_timestamp_pulled_column
-        Purview::Columns::Timestamp.new(table_metadata_max_timestamp_pulled_column_name)
-      end
-
-      def table_metadata_max_timestamp_pulled_column_definition
-        column_definition(table_metadata_max_timestamp_pulled_column)
-      end
-
-      def table_metadata_max_timestamp_pulled_column_name
-        'max_timestamp_pulled'
-      end
-
-      def table_metadata_sql(table)
-        raise %{All "#{Base}(s)" must override the "table_metadata_sql" method}
-      end
-
-      def table_metadata_table_name
-        'table_metadata'
-      end
-
-      def table_metadata_table_name_column
-        Purview::Columns::Id.new(
-          table_metadata_table_name_column_name,
-          :type => Purview::Types::String,
-          :limit => 255,
-        )
-      end
-
-      def table_metadata_table_name_column_definition
-        column_definition(table_metadata_table_name_column)
-      end
-
-      def table_metadata_table_name_column_name
-        'table_name'
+      def table_metadata_table
+        @table_metadata_table ||= Purview::Tables::TableMetadata.new(self)
       end
 
       def table_name(table, table_opts={})
@@ -703,8 +554,8 @@ module Purview
         end
       end
 
-      def true_value
-        dialect.true_value
+      def tables_opt
+        opts[:tables] || []
       end
 
       def type(column)
@@ -731,7 +582,8 @@ module Purview
       def with_next_table(timestamp)
         with_new_connection do |connection|
           table = next_table(connection, timestamp)
-          raise Purview::Exceptions::NoTable.new unless table
+          raise Purview::Exceptions::NoTable.new \
+            unless table
           yield table
         end
       end
@@ -743,7 +595,8 @@ module Purview
             table,
             timestamp
           )
-          raise Purview::Exceptions::NoWindow.new(table) unless window
+          raise Purview::Exceptions::NoWindowForTable.new(table) \
+            unless window
           yield window
         end
       end