purview 1.0.0.alpha

data/lib/purview/loaders/mysql.rb

@@ -0,0 +1,81 @@
+module Purview
+  module Loaders
+    class MySQL < Base
+      private
+
+      def id_in_sql(temporary_table_name)
+        'SELECT %s FROM %s' % [
+          table.id_column.name,
+          temporary_table_name,
+        ]
+      end
+
+      def in_window_sql(window)
+        '%s BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def not_in_window_sql(window)
+        '%s NOT BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def table_delete_sql(window, temporary_table_name)
+        'DELETE FROM %s WHERE %s AND %s NOT IN (%s)' % [
+          table.name,
+          in_window_sql(window),
+          table.id_column.name,
+          id_in_sql(temporary_table_name),
+        ]
+      end
+
+      def table_insert_sql(window, temporary_table_name)
+        'INSERT INTO %s (%s) SELECT %s FROM %s t1 WHERE NOT EXISTS (SELECT 1 FROM %s t2 WHERE t1.%s = t2.%s)' % [
+          table.name,
+          table.column_names.join(', '),
+          table.column_names.join(', '),
+          temporary_table_name,
+          table.name,
+          table.id_column.name,
+          table.id_column.name,
+        ]
+      end
+
+      def table_update_sql(window, temporary_table_name)
+        'UPDATE %s t1 JOIN %s t2 ON t1.%s = t2.%s SET %s' % [
+          table.name,
+          temporary_table_name,
+          table.id_column.name,
+          table.id_column.name,
+          table.column_names.map { |column_name| "t1.#{column_name} = t2.#{column_name}" }.join(', '),
+        ]
+      end
+
+      def temporary_table_insert_sql(temporary_table_name, rows)
+        'INSERT INTO %s (%s) VALUES %s' % [
+          temporary_table_name,
+          table.column_names.join(', '),
+          rows.map { |row| "(#{row_values(row)})" }.join(', ')
+        ]
+      end
+
+      def temporary_table_opts
+        super.merge(:create_indices => false)
+      end
+
+      def temporary_table_verify_sql(temporary_table_name, rows, window)
+        'SELECT COUNT(1) %s FROM %s WHERE %s' % [
+          count_column_name,
+          temporary_table_name,
+          not_in_window_sql(window),
+        ]
+      end
+    end
+  end
+end

data/lib/purview/loaders/postgresql.rb

@@ -0,0 +1,81 @@
+module Purview
+  module Loaders
+    class PostgreSQL < Base
+      private
+
+      def id_in_sql(temporary_table_name)
+        'SELECT %s FROM %s' % [
+          table.id_column.name,
+          temporary_table_name,
+        ]
+      end
+
+      def in_window_sql(window)
+        '%s BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def not_in_window_sql(window)
+        '%s NOT BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def table_delete_sql(window, temporary_table_name)
+        'DELETE FROM %s WHERE %s AND %s NOT IN (%s)' % [
+          table.name,
+          in_window_sql(window),
+          table.id_column.name,
+          id_in_sql(temporary_table_name),
+        ]
+      end
+
+      def table_insert_sql(window, temporary_table_name)
+        'INSERT INTO %s (%s) SELECT %s FROM %s t1 WHERE NOT EXISTS (SELECT 1 FROM %s t2 WHERE t1.%s = t2.%s)' % [
+          table.name,
+          table.column_names.join(', '),
+          table.column_names.join(', '),
+          temporary_table_name,
+          table.name,
+          table.id_column.name,
+          table.id_column.name,
+        ]
+      end
+
+      def table_update_sql(window, temporary_table_name)
+        'UPDATE %s t1 SET %s FROM %s t2 WHERE t1.%s = t2.%s' % [
+          table.name,
+          table.column_names.map { |column_name| "#{column_name} = t2.#{column_name}" }.join(', '),
+          temporary_table_name,
+          table.id_column.name,
+          table.id_column.name,
+        ]
+      end
+
+      def temporary_table_insert_sql(temporary_table_name, rows)
+        'INSERT INTO %s (%s) VALUES %s' % [
+          temporary_table_name,
+          table.column_names.join(', '),
+          rows.map { |row| "(#{row_values(row)})" }.join(', ')
+        ]
+      end
+
+      def temporary_table_opts
+        super.merge(:create_indices => false)
+      end
+
+      def temporary_table_verify_sql(temporary_table_name, rows, window)
+        'SELECT COUNT(1) %s FROM %s WHERE %s' % [
+          count_column_name,
+          temporary_table_name,
+          not_in_window_sql(window),
+        ]
+      end
+    end
+  end
+end

data/lib/purview/loaders.rb

@@ -0,0 +1,3 @@
+require 'purview/loaders/base'
+require 'purview/loaders/mysql'
+require 'purview/loaders/postgresql'

data/lib/purview/loggers/base.rb

@@ -0,0 +1,99 @@
+module Purview
+  module Loggers
+    class Base
+      def initialize(opts={})
+        @opts = default_opts.merge(opts)
+      end
+
+      def debug(*args)
+        log(DEBUG_LEVEL, *args) if debug?
+      end
+
+      def error(*args)
+        log(ERROR_LEVEL, *args) if error?
+      end
+
+      def info(*args)
+        log(INFO_LEVEL, *args) if info?
+      end
+
+      def with_context_logging(*args)
+        debug(build_starting_message(*args))
+        yield.tap { |result| debug(build_finished_message(*args)) }
+      end
+
+      private
+
+      DEBUG_LEVEL = 'DEBUG'
+      ERROR_LEVEL = 'ERROR'
+      INFO_LEVEL = 'INFO'
+
+      attr_reader :opts
+
+      def build_finished_message(*args)
+        case args.length
+          when 1; "Finished #{args[0]}"
+          when 2; args[-1]
+          else; raise
+        end
+      end
+
+      def build_message(level, *args)
+        message, exception = args[0..1]
+        message_template(!!exception) % {
+          :exception => format_exception(exception),
+          :level => level,
+          :message => message,
+          :process_id => Process.pid,
+          :timestamp => Time.now.strftime('%Y-%m-%d %H:%M:%S.%L %z'),
+        }
+      end
+
+      def build_starting_message(*args)
+        case args.length
+          when 1; "Starting #{args[0]}"
+          when 2; args[0]
+          else; raise
+        end
+      end
+
+      def debug?
+        !!opts[:debug]
+      end
+
+      def default_opts
+        {
+          :debug => true,
+          :error => true,
+          :info => false,
+        }
+      end
+
+      def error?
+        !!opts[:error]
+      end
+
+      def format_exception(exception)
+        exception && exception.backtrace.map { |line| "\tfrom #{line}" }.join("\n")
+      end
+
+      def info?
+        !!opts[:info]
+      end
+
+      def log(level, *args)
+        stream.puts build_message(level, *args)
+      end
+
+      def message_template(exception)
+        "%{timestamp} %{level} (%{process_id}) %{message}".tap do |result|
+          result << ":\n%{exception}" if exception
+        end
+      end
+
+      def stream
+        opts[:stream]
+      end
+    end
+  end
+end

data/lib/purview/loggers/console.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Loggers
+    class Console < Base
+      private
+
+      def default_opts
+        super.merge(:stream => STDOUT)
+      end
+    end
+  end
+end

data/lib/purview/loggers.rb

@@ -0,0 +1,2 @@
+require 'purview/loggers/base'
+require 'purview/loggers/console'

data/lib/purview/mixins/helpers.rb

@@ -0,0 +1,21 @@
+module Purview
+  module Mixins
+    module Helpers
+      def blank?(value)
+        value.to_s.strip.length.zero?
+      end
+
+      def coalesce(value, default)
+        value.nil? ? default : value
+      end
+
+      def present?(value)
+        !blank?(value)
+      end
+
+      def zero?(value)
+        Integer(value).zero?
+      end
+    end
+  end
+end

data/lib/purview/mixins/logger.rb

@@ -0,0 +1,21 @@
+module Purview
+  module Mixins
+    module Logger
+      def logger
+       @logger ||= logger_type.new(logger_opts)
+      end
+
+      def logger_opts
+        opts[:logger] || {}
+      end
+
+      def logger_type
+        opts[:logger_type] || Purview::Loggers::Console
+      end
+
+      def with_context_logging(*args)
+        logger.with_context_logging(*args) { yield }
+      end
+    end
+  end
+end

data/lib/purview/mixins.rb

@@ -0,0 +1,2 @@
+require 'purview/mixins/helpers'
+require 'purview/mixins/logger'

data/lib/purview/parsers/base.rb

@@ -0,0 +1,39 @@
+module Purview
+  module Parsers
+    class Base
+      def initialize(opts={})
+        @opts = opts
+      end
+
+      def parse(data)
+        raise %{All "#{Base}(s)" must override the "parse" method}
+      end
+
+      def validate(data)
+        true
+      end
+
+      private
+
+      include Purview::Mixins::Logger
+
+      attr_reader :opts
+
+      def build_result(row)
+        {}.tap do |result|
+          row.each do |key, value|
+            if column = table.columns_by_name[key]
+              result[key] = column.parse(value)
+            else
+              logger.debug(%{Unexpected column: "#{key}" in data-set})
+            end
+          end
+        end
+      end
+
+      def table
+        opts[:table]
+      end
+    end
+  end
+end

data/lib/purview/parsers/csv.rb

@@ -0,0 +1,49 @@
+module Purview
+  module Parsers
+    class CSV < Base
+      def parse(data)
+        with_context_logging("`parse` for: #{table.name}") do
+          [].tap do |results|
+            headers = extract_headers(data)
+            extract_rows(data) do |row|
+              results << build_result(headers.zip(row))
+            end
+          end
+        end
+      end
+
+      def validate(data)
+        with_context_logging("`validate` for: #{table.name}") do
+          missing_columns = table.column_names - extract_headers(data)
+          raise 'Missing one or more columns: "%s"' % missing_columns.join('", "') \
+            unless missing_columns.empty?
+          true
+        end
+      end
+
+      private
+
+      def column_separator
+        ','
+      end
+
+      def extract_headers(data)
+        header_row = data.split(row_separator).first
+        parse_row(header_row).map(&:to_sym)
+      end
+
+      def extract_rows(data)
+        rows = data.split(row_separator)[1..-1]
+        rows.each { |row| yield parse_row(row) }
+      end
+
+      def parse_row(row)
+        ::CSV.parse(row, :col_sep => column_separator).first
+      end
+
+      def row_separator
+        $/
+      end
+    end
+  end
+end

data/lib/purview/parsers/tsv.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Parsers
+    class TSV < CSV
+      private
+
+      def column_separator
+        "\t"
+      end
+    end
+  end
+end

data/lib/purview/parsers.rb

@@ -0,0 +1,3 @@
+require 'purview/parsers/base'
+require 'purview/parsers/csv'
+require 'purview/parsers/tsv'

data/lib/purview/pullers/base.rb

@@ -0,0 +1,19 @@
+module Purview
+  module Pullers
+    class Base
+      def initialize(opts={})
+        @opts = opts
+      end
+
+      def pull(window)
+        raise %{All "#{Base}(s)" must override the "pull" method}
+      end
+
+      private
+
+      include Purview::Mixins::Logger
+
+      attr_reader :opts
+    end
+  end
+end

data/lib/purview/pullers/uri.rb

@@ -0,0 +1,66 @@
+module Purview
+  module Pullers
+    class URI < Base
+      def pull(window)
+        request = windowed_request(window)
+        with_context_logging("`pull` from: #{request.path}") do
+          http.request(request).body
+        end
+      end
+
+      private
+
+      def basic_auth?
+        username && password
+      end
+
+      def host
+        uri.host
+      end
+
+      def http
+        Net::HTTP.new(host, port).tap do |http|
+          if https?
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          end
+        end
+      end
+
+      def https?
+        uri.scheme == 'https'
+      end
+
+      def password
+        opts[:password]
+      end
+
+      def port
+        uri.port
+      end
+
+      def uri
+        ::URI.parse(opts[:uri])
+      end
+
+      def username
+        opts[:username]
+      end
+
+      def windowed_request(window)
+        Net::HTTP::Get.new(windowed_request_uri(window)).tap do |request|
+          if basic_auth?
+            request.basic_auth(username, password)
+          end
+        end
+      end
+
+      def windowed_request_uri(window)
+        uri.to_s.tap do |request_uri|
+          request_uri << (request_uri.include?('?') ? '&' : '?')
+          request_uri << 'ts1=%s&ts2=%s' % [window.min.to_i, window.max.to_i]
+        end
+      end
+    end
+  end
+end

data/lib/purview/pullers.rb

@@ -0,0 +1,2 @@
+require 'purview/pullers/base'
+require 'purview/pullers/uri'

data/lib/purview/refinements/object.rb

@@ -0,0 +1,5 @@
+class Object
+  def quoted
+    "'#{self}'"
+  end
+end

data/lib/purview/refinements/time.rb

@@ -0,0 +1,5 @@
+class Time
+  def quoted
+    "'#{self.strftime('%F %T')}'"
+  end
+end

data/lib/purview/refinements.rb

@@ -0,0 +1,2 @@
+require 'purview/refinements/object'
+require 'purview/refinements/time'

data/lib/purview/structs/base.rb

@@ -0,0 +1,10 @@
+module Purview
+  module Structs
+    class Base < OpenStruct
+      def method_missing(method_name, *args, &block)
+        raise NoMethodError if args.size.zero?
+        super
+      end
+    end
+  end
+end

data/lib/purview/structs/result.rb

@@ -0,0 +1,7 @@
+module Purview
+  module Structs
+    class Result < Base
+      # Helper methods/overrides
+    end
+  end
+end

data/lib/purview/structs/window.rb

@@ -0,0 +1,7 @@
+module Purview
+  module Structs
+    class Window < Base
+      # Helper methods/overrides
+    end
+  end
+end

data/lib/purview/structs.rb

@@ -0,0 +1,3 @@
+require 'purview/structs/base'
+require 'purview/structs/result'
+require 'purview/structs/window'