purview 1.3.1 → 1.4.0

data/lib/purview/indices/base.rb

@@ -6,6 +6,7 @@ module Purview
       def initialize(columns, opts={})
         @columns = columns
         @opts = opts
+        @table = table_opt
       end
 
       def eql?(other)
@@ -19,7 +20,7 @@ module Purview
       end
 
       def table=(value)
-        raise Purview::Exceptions::TableAlreadyAssigned.new(self) if table
+        raise Purview::Exceptions::TableAlreadyAssignedForIndex.new(self) if table
         @table = value
       end
 
@@ -34,6 +35,10 @@ module Purview
       private
 
       attr_reader :opts
+
+      def table_opt
+        opts[:table]
+      end
     end
   end
 end

data/lib/purview/loaders/base.rb

@@ -28,6 +28,7 @@ module Purview
 
       private
 
+      include Purview::Mixins::Dialect
       include Purview::Mixins::Helpers
       include Purview::Mixins::Logger
 
@@ -49,18 +50,10 @@ module Purview
         table.database
       end
 
-      def dialect
-        dialect_type.new
-      end
-
       def dialect_type
         raise %{All "#{Base}(s)" must override the "dialect_type" method}
       end
 
-      def false_value
-        dialect.false_value
-      end
-
       def id_in_sql(temporary_table_name)
         raise %{All "#{Base}(s)" must override the "id_in_sql" method}
       end
@@ -86,14 +79,6 @@ module Purview
         raise %{All "#{Base}(s)" must override the "not_in_window_sql" method}
       end
 
-      def null_value
-        dialect.null_value
-      end
-
-      def quoted(value)
-        dialect.quoted(value)
-      end
-
       def row_values(row)
         table.column_names.map { |column_name| quoted(sanitized(row[column_name])) }.join(', ')
       end
@@ -102,10 +87,6 @@ module Purview
         opts[:rows_per_slice] || 1000
       end
 
-      def sanitized(value)
-        dialect.sanitized(value)
-      end
-
       def table
         opts[:table]
       end
@@ -138,10 +119,6 @@ module Purview
         raise %{All "#{Base}(s)" must override the "temporary_table_verify_sql" method}
       end
 
-      def true_value
-        dialect.true_value
-      end
-
       def verify_temporary_table(connection, temporary_table_name, rows, window)
         with_context_logging("`verify_temporary_table` for: #{temporary_table_name}") do
           rows_outside_window = connection.execute(
@@ -151,7 +128,7 @@ module Purview
               window
             )
           ).rows[0][count_column_name]
-          raise Purview::Exceptions::RowsOutsideWindow.new(temporary_table_name, rows_outside_window) \
+          raise Purview::Exceptions::RowsOutsideWindowForTable.new(table, rows_outside_window) \
             unless zero?(rows_outside_window)
         end
       end

data/lib/purview/mixins/dialect.rb

@@ -0,0 +1,29 @@
+module Purview
+  module Mixins
+    module Dialect
+      def dialect
+        dialect_type.new
+      end
+
+      def false_value
+        dialect.false_value
+      end
+
+      def null_value
+        dialect.null_value
+      end
+
+      def quoted(value)
+        dialect.quoted(value)
+      end
+
+      def sanitized(value)
+        dialect.sanitized(value)
+      end
+
+      def true_value
+        dialect.true_value
+      end
+    end
+  end
+end

data/lib/purview/mixins.rb

@@ -1,3 +1,4 @@
 require 'purview/mixins/connection'
+require 'purview/mixins/dialect'
 require 'purview/mixins/helpers'
 require 'purview/mixins/logger'

data/lib/purview/parsers/base.rb

@@ -3,6 +3,7 @@ module Purview
     class Base
       def initialize(opts={})
         @opts = opts
+        @table = table_opt
       end
 
       def parse(data)
@@ -17,7 +18,7 @@ module Purview
 
       include Purview::Mixins::Logger
 
-      attr_reader :opts
+      attr_reader :opts, :table
 
       def extract_headers(data)
         raise %{All "#{Base}(s)" must override the "extract_headers" method}
@@ -27,7 +28,7 @@ module Purview
         raise %{All "#{Base}(s)" must override the "extract_rows" method}
       end
 
-      def table
+      def table_opt
         opts[:table]
       end
     end

data/lib/purview/parsers/csv.rb

@@ -26,8 +26,8 @@ module Purview
       def build_result(row)
         {}.tap do |result|
           row.each do |key, value|
-            if column = table.columns_by_name[key]
-              result[key] = column.parse(value)
+            if column = table.columns_by_source_name[key]
+              result[column.name] = column.parse(value)
             else
               logger.debug(%{Unexpected column: "#{key}" in data-set})
             end
@@ -49,8 +49,18 @@ module Purview
         rows.map { |row| parse_row(row) }
       end
 
+      def map_headers(headers)
+        headers.map do |header|
+          if column = table.columns_by_source_name[header]
+            column.name
+          else
+            logger.debug(%{Could not find column with source_name: "#{header}"})
+          end
+        end
+      end
+
       def missing_columns(data)
-        table.column_names - extract_headers(data)
+        table.column_names - map_headers(extract_headers(data))
       end
 
       def parse_row(row)

data/lib/purview/pullers/base_sql.rb

@@ -10,11 +10,16 @@ module Purview
       private
 
       include Purview::Mixins::Connection
+      include Purview::Mixins::Dialect
       include Purview::Mixins::Helpers
       include Purview::Mixins::Logger
 
       def column_names
-        table.column_names
+        table.columns.map do |column|
+          name = column.name
+          source_name = column.source_name
+          source_name == name ? name : "#{source_name} AS #{name}"
+        end
       end
 
       def connection_type
@@ -41,22 +46,10 @@ module Purview
         opts[:database_username]
       end
 
-      def dialect
-        dialect_type.new
-      end
-
       def dialect_type
         raise %{All "#{BaseSQL}(s)" must override the "dialect_type" method}
       end
 
-      def false_value
-        dialect.false_value
-      end
-
-      def null_value
-        dialect.null_value
-      end
-
       def pull_sql(window)
         'SELECT %s FROM %s WHERE %s BETWEEN %s AND %s' % [
           column_names.join(', '),
@@ -67,21 +60,9 @@ module Purview
         ]
       end
 
-      def quoted(value)
-        dialect.quoted(value)
-      end
-
-      def sanitized(value)
-        dialect.sanitized(value)
-      end
-
       def table_name
         opts[:table_name]
       end
-
-      def true_value
-        dialect.true_value
-      end
     end
   end
 end

data/lib/purview/structs/table_metadata.rb

@@ -1,20 +1,6 @@
 module Purview
   module Structs
     class TableMetadata < Base
-      def initialize(row)
-        enabled_at = row.enabled_at && Time.parse(row.enabled_at)
-        last_pulled_at = row.last_pulled_at && Time.parse(row.last_pulled_at)
-        locked_at = row.locked_at && Time.parse(row.locked_at)
-        max_timestamp_pulled = row.max_timestamp_pulled && Time.parse(row.max_timestamp_pulled)
-        super(
-          :table_name => row.table_name,
-          :enabled_at => enabled_at,
-          :last_pulled_at => last_pulled_at,
-          :locked_at => locked_at,
-          :max_timestamp_pulled => max_timestamp_pulled,
-        )
-      end
-
       def diabled?
         !enabled?
       end

data/lib/purview/tables/base.rb

@@ -1,90 +1,47 @@
 module Purview
   module Tables
     class Base
-      attr_reader :database, :indices, :name
+      attr_reader :columns, :database, :indices, :name
 
       def initialize(name, opts={})
-        @name = name
+        @name = name.to_sym
         @opts = opts
+        @database = database_opt
+        @columns = Set.new.tap do |result|
+          (default_columns + columns_opt).each do |column|
+            column.table = self if result.add?(column)
+          end
+        end
         @indices = Set.new.tap do |result|
-          ((opts[:indices] || []) + default_indices).each do |index|
+          (default_indices + indices_opt).each do |index|
             index.table = self if result.add?(index)
           end
         end
       end
 
-      def columns
-        opts[:columns]
-      end
-
       def column_names
         columns.map(&:name)
       end
 
       def columns_by_name
-        {}.tap do |result|
-          columns.each do |column|
-            result[column.name] = column
-          end
+        columns.reduce({}) do |memo, column|
+          memo[column.name] = column
+          memo
         end
       end
 
-      def columns_of_type(type)
-        columns.select { |column| column.is_a?(type) }
-      end
-
-      def created_timestamp_column
-        columns_of_type(Purview::Columns::CreatedTimestamp).first
-      end
-
-      def created_timestamp_index
-        Purview::Indices::Simple.new(created_timestamp_column)
-      end
-
-      def data_columns
-        columns - [
-          created_timestamp_column,
-          id_column,
-          updated_timestamp_column,
-        ]
+      def columns_by_source_name
+        columns.reduce({}) do |memo, column|
+          memo[column.source_name] = column
+          memo
+        end
       end
 
       def database=(value)
-        raise Purview::Exceptions::DatabaseAlreadyAssigned.new(self) if database
+        raise Purview::Exceptions::DatabaseAlreadyAssignedForTable.new(self) if database
         @database = value
       end
 
-      def id_column
-        columns_of_type(Purview::Columns::Id).first
-      end
-
-      def sync(connection, window)
-        raw_data = puller.pull(window)
-        parser.validate(raw_data)
-        parsed_data = parser.parse(raw_data)
-        loader.load(
-          connection,
-          parsed_data,
-          window
-        )
-      end
-
-      def temporary_name
-        "#{name}_#{timestamp.to_i}"
-      end
-
-      def updated_timestamp_column
-        columns_of_type(Purview::Columns::UpdatedTimestamp).first
-      end
-
-      def updated_timestamp_index
-        Purview::Indices::Simple.new(updated_timestamp_column)
-      end
-
-      def window_size
-        opts[:window_size] || (60 * 60)
-      end
-
       private
 
       include Purview::Mixins::Helpers
@@ -92,59 +49,24 @@ module Purview
 
       attr_reader :opts
 
-      def default_indices
-        [
-          created_timestamp_index,
-          updated_timestamp_index,
-        ]
-      end
-
-      def extract_type_option(opts)
-        opts[:type]
+      def columns_opt
+        opts[:columns] || []
       end
 
-      def filter_type_option(opts)
-        opts.select { |key| key != :type }
+      def database_opt
+        opts[:database]
       end
 
-      def loader
-        loader_type.new(loader_opts)
+      def default_columns
+        []
       end
 
-      def loader_opts
-        merge_table_option(filter_type_option(opts[:loader]))
-      end
-
-      def loader_type
-        extract_type_option(opts[:loader])
-      end
-
-      def merge_table_option(opts)
-        { :table => self }.merge(opts)
-      end
-
-      def parser
-        parser_type.new(parser_opts)
-      end
-
-      def parser_opts
-        merge_table_option(filter_type_option(opts[:parser]))
-      end
-
-      def parser_type
-        extract_type_option(opts[:parser])
-      end
-
-      def puller
-        puller_type.new(puller_opts)
-      end
-
-      def puller_opts
-        merge_table_option(filter_type_option(opts[:puller]))
+      def default_indices
+        []
       end
 
-      def puller_type
-        extract_type_option(opts[:puller])
+      def indices_opt
+        opts[:indices] || []
       end
     end
   end

data/lib/purview/tables/base_syncable.rb

@@ -0,0 +1,113 @@
+module Purview
+  module Tables
+    class BaseSyncable < Base
+      def created_timestamp_column
+        column_from_opts_of_type(Purview::Columns::CreatedTimestamp) or raise %{Must specify a column of type: "#{Purview::Columns::CreatedTimestamp}"}
+      end
+
+      def created_timestamp_index
+        Purview::Indices::Simple.new(created_timestamp_column)
+      end
+
+      def id_column
+        column_from_opts_of_type(Purview::Columns::Id) or raise %{Must specify a column of type: "#{Purview::Columns::Id}"}
+      end
+
+      def sync(connection, window)
+        raw_data = puller.pull(window)
+        parser.validate(raw_data)
+        parsed_data = parser.parse(raw_data)
+        loader.load(
+          connection,
+          parsed_data,
+          window
+        )
+      end
+
+      def temporary_name
+        "#{name}_#{timestamp.to_i}"
+      end
+
+      def updated_timestamp_column
+        column_from_opts_of_type(Purview::Columns::UpdatedTimestamp) or raise %{Must specify a column of type: "#{Purview::Columns::UpdatedTimestamp}"}
+      end
+
+      def updated_timestamp_index
+        Purview::Indices::Simple.new(updated_timestamp_column)
+      end
+
+      def window_size
+        opts[:window_size] || (60 * 60)
+      end
+
+      private
+
+      def column_from_opts_of_type(type)
+        columns_opt.select { |column| column.is_a?(type) }.first
+      end
+
+      def default_columns
+        super + [
+          id_column,
+          created_timestamp_column,
+          updated_timestamp_column,
+        ]
+      end
+
+      def default_indices
+        super + [
+          created_timestamp_index,
+          updated_timestamp_index,
+        ]
+      end
+
+      def extract_type_opt(opts)
+        opts[:type]
+      end
+
+      def filter_type_opt(opts)
+        opts.select { |key| key != :type }
+      end
+
+      def loader
+        loader_type.new(loader_opts)
+      end
+
+      def loader_opts
+        merge_table_opt(filter_type_opt(opts[:loader]))
+      end
+
+      def loader_type
+        extract_type_opt(opts[:loader])
+      end
+
+      def merge_table_opt(opts)
+        { :table => self }.merge(opts)
+      end
+
+      def parser
+        parser_type.new(parser_opts)
+      end
+
+      def parser_opts
+        merge_table_opt(filter_type_opt(opts[:parser]))
+      end
+
+      def parser_type
+        extract_type_opt(opts[:parser])
+      end
+
+      def puller
+        puller_type.new(puller_opts)
+      end
+
+      def puller_opts
+        merge_table_opt(filter_type_opt(opts[:puller]))
+      end
+
+      def puller_type
+        extract_type_opt(opts[:puller])
+      end
+    end
+  end
+end

data/lib/purview/tables/raw.rb

@@ -1,6 +1,6 @@
 module Purview
   module Tables
-    class Raw < Base
+    class Raw < BaseSyncable
       def name
         "#{super}_raw"
       end

data/lib/purview/tables/table_metadata.rb

@@ -0,0 +1,43 @@
+module Purview
+  module Tables
+    class TableMetadata < Base
+      def initialize(database)
+        super(
+          :table_metadata,
+          :columns => [
+            table_name_column,
+            enabled_at_column,
+            last_pulled_at_column,
+            locked_at_column,
+            max_timestamp_pulled_column,
+          ],
+          :database => database
+        )
+      end
+
+      def enabled_at_column
+        Purview::Columns::Timestamp.new(:enabled_at)
+      end
+
+      def last_pulled_at_column
+        Purview::Columns::Timestamp.new(:last_pulled_at)
+      end
+
+      def locked_at_column
+        Purview::Columns::Timestamp.new(:locked_at)
+      end
+
+      def max_timestamp_pulled_column
+        Purview::Columns::Timestamp.new(:max_timestamp_pulled)
+      end
+
+      def table_name_column
+        Purview::Columns::Id.new(
+          :table_name,
+          :type => Purview::Types::String,
+          :limit => 255,
+        )
+      end
+    end
+  end
+end

data/lib/purview/tables.rb

@@ -1,2 +1,5 @@
 require 'purview/tables/base'
+require 'purview/tables/base_syncable'
+
 require 'purview/tables/raw'
+require 'purview/tables/table_metadata'

data/lib/purview/version.rb

@@ -1,3 +1,3 @@
 module Purview
-  VERSION = '1.3.1'
+  VERSION = '1.4.0'
 end