purview 1.0.0.alpha → 1.0.0.beta1

data/lib/purview/loaders/postgresql.rb

@@ -3,6 +3,10 @@ module Purview
     class PostgreSQL < Base
       private
 
+      def dialect_type
+        Purview::Dialects::PostgreSQL
+      end
+
       def id_in_sql(temporary_table_name)
         'SELECT %s FROM %s' % [
           table.id_column.name,

data/lib/purview/loaders.rb

@@ -1,3 +1,4 @@
 require 'purview/loaders/base'
+
 require 'purview/loaders/mysql'
 require 'purview/loaders/postgresql'

data/lib/purview/loggers.rb

@@ -1,2 +1,3 @@
 require 'purview/loggers/base'
+
 require 'purview/loggers/console'

data/lib/purview/mixins/connection.rb

@@ -0,0 +1,13 @@
+module Purview
+  module Mixins
+    module Connection
+      def connect
+        connection_type.new(connection_opts)
+      end
+
+      def with_new_connection
+        connection_type.with_new_connection(connection_opts) { |connection| yield connection }
+      end
+    end
+  end
+end

data/lib/purview/mixins/helpers.rb

@@ -5,10 +5,14 @@ module Purview
         value.to_s.strip.length.zero?
       end
 
-      def coalesce(value, default)
+      def coalesced(value, default)
         value.nil? ? default : value
       end
 
+      def filter_nil_values(hash)
+        hash.reject { |_, value| value.nil? }
+      end
+
       def present?(value)
         !blank?(value)
       end

data/lib/purview/mixins/logger.rb

@@ -6,11 +6,11 @@ module Purview
       end
 
       def logger_opts
-        opts[:logger] || {}
+        (defined?(opts) && opts[:logger]) || {}
       end
 
       def logger_type
-        opts[:logger_type] || Purview::Loggers::Console
+        (defined?(opts) && opts[:logger_type]) || Purview::Loggers::Console
       end
 
       def with_context_logging(*args)

data/lib/purview/mixins.rb

@@ -1,2 +1,3 @@
+require 'purview/mixins/connection'
 require 'purview/mixins/helpers'
 require 'purview/mixins/logger'

data/lib/purview/parsers/base.rb

@@ -10,7 +10,7 @@ module Purview
       end
 
       def validate(data)
-        true
+        raise %{All "#{Base}(s)" must override the "validate" method}
       end
 
       private
@@ -19,16 +19,12 @@ module Purview
 
       attr_reader :opts
 
-      def build_result(row)
-        {}.tap do |result|
-          row.each do |key, value|
-            if column = table.columns_by_name[key]
-              result[key] = column.parse(value)
-            else
-              logger.debug(%{Unexpected column: "#{key}" in data-set})
-            end
-          end
-        end
+      def extract_headers(data)
+        raise %{All "#{Base}(s)" must override the "extract_headers" method}
+      end
+
+      def extract_rows(data)
+        raise %{All "#{Base}(s)" must override the "extract_rows" method}
       end
 
       def table

data/lib/purview/parsers/csv.rb

@@ -5,7 +5,7 @@ module Purview
         with_context_logging("`parse` for: #{table.name}") do
           [].tap do |results|
             headers = extract_headers(data)
-            extract_rows(data) do |row|
+            extract_rows(data).each do |row|
               results << build_result(headers.zip(row))
             end
           end
@@ -14,7 +14,7 @@ module Purview
 
       def validate(data)
         with_context_logging("`validate` for: #{table.name}") do
-          missing_columns = table.column_names - extract_headers(data)
+          missing_columns = missing_columns(data)
           raise 'Missing one or more columns: "%s"' % missing_columns.join('", "') \
             unless missing_columns.empty?
           true
@@ -23,6 +23,18 @@ module Purview
 
       private
 
+      def build_result(row)
+        {}.tap do |result|
+          row.each do |key, value|
+            if column = table.columns_by_name[key]
+              result[key] = column.parse(value)
+            else
+              logger.debug(%{Unexpected column: "#{key}" in data-set})
+            end
+          end
+        end
+      end
+
       def column_separator
         ','
       end
@@ -34,7 +46,11 @@ module Purview
 
       def extract_rows(data)
         rows = data.split(row_separator)[1..-1]
-        rows.each { |row| yield parse_row(row) }
+        rows.map { |row| parse_row(row) }
+      end
+
+      def missing_columns(data)
+        table.column_names - extract_headers(data)
       end
 
       def parse_row(row)

data/lib/purview/parsers/sql.rb

@@ -0,0 +1,13 @@
+module Purview
+  module Parsers
+    class SQL < Base
+      def parse(data)
+        data.rows
+      end
+
+      def validate(data)
+        true
+      end
+    end
+  end
+end

data/lib/purview/parsers.rb

@@ -1,3 +1,5 @@
 require 'purview/parsers/base'
+
 require 'purview/parsers/csv'
+require 'purview/parsers/sql'
 require 'purview/parsers/tsv'

data/lib/purview/pullers/base.rb

@@ -14,6 +14,10 @@ module Purview
       include Purview::Mixins::Logger
 
       attr_reader :opts
+
+      def table
+        opts[:table]
+      end
     end
   end
 end

data/lib/purview/pullers/base_sql.rb

@@ -0,0 +1,97 @@
+module Purview
+  module Pullers
+    class BaseSQL < Base
+      def pull(window)
+        with_new_connection do |connection|
+          connection.execute(pull_sql(window))
+        end
+      end
+
+      private
+
+      include Purview::Mixins::Connection
+      include Purview::Mixins::Helpers
+      include Purview::Mixins::Logger
+
+      def column_names
+        table.column_names
+      end
+
+      def connection_opts
+        filter_nil_values(
+          :database => database_name,
+          :host => database_host,
+          :password => database_password,
+          :port => database_port,
+          :username => database_username
+        )
+      end
+
+      def connection_type
+        raise %{All "#{BaseSQL}(s)" must override the "connection_type" method}
+      end
+
+      def database_host
+        opts[:database_host]
+      end
+
+      def database_name
+        opts[:database_name]
+      end
+
+      def database_password
+        opts[:database_password]
+      end
+
+      def database_port
+        opts[:database_port]
+      end
+
+      def database_username
+        opts[:database_username]
+      end
+
+      def dialect
+        dialect_type.new
+      end
+
+      def dialect_type
+        raise %{All "#{BaseSQL}(s)" must override the "dialect_type" method}
+      end
+
+      def false_value
+        dialect.false_value
+      end
+
+      def null_value
+        dialect.null_value
+      end
+
+      def pull_sql(window)
+        'SELECT %s FROM %s WHERE %s BETWEEN %s AND %s' % [
+          column_names.join(', '),
+          table_name,
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def quoted(value)
+        dialect.quoted(value)
+      end
+
+      def sanitized(value)
+        dialect.sanitized(value)
+      end
+
+      def table_name
+        opts[:table_name]
+      end
+
+      def true_value
+        dialect.true_value
+      end
+    end
+  end
+end

data/lib/purview/pullers/mysql.rb

@@ -0,0 +1,15 @@
+module Purview
+  module Pullers
+    class MySQL < BaseSQL
+      private
+
+      def connection_type
+        Purview::Connections::MySQL
+      end
+
+      def dialect_type
+        Purview::Dialects::MySQL
+      end
+    end
+  end
+end

data/lib/purview/pullers/postgresql.rb

@@ -0,0 +1,15 @@
+module Purview
+  module Pullers
+    class PostgreSQL < BaseSQL
+      private
+
+      def connection_type
+        Purview::Connections::PostgreSQL
+      end
+
+      def dialect_type
+        Purview::Dialects::PostgreSQL
+      end
+    end
+  end
+end

data/lib/purview/pullers.rb

@@ -1,2 +1,6 @@
 require 'purview/pullers/base'
+require 'purview/pullers/base_sql'
+
+require 'purview/pullers/mysql'
+require 'purview/pullers/postgresql'
 require 'purview/pullers/uri'

data/lib/purview/raw_connections/base.rb

@@ -0,0 +1,118 @@
+module Purview
+  module RawConnections
+    class Base
+      def self.connect(opts)
+        new(opts)
+      end
+
+      def self.with_new_connection(opts)
+        yield connection = connect(opts)
+      ensure
+        connection.disconnect if connection
+      end
+
+      def initialize(opts)
+        @opts = opts
+        @raw_connection = new_connection
+      end
+
+      def disconnect
+        raw_connection.close
+        @raw_connection = nil
+        self
+      end
+
+      def execute(sql, opts={})
+        logger.debug("Executing: #{sql}")
+        result = execute_sql(sql, opts)
+        structify_result(result)
+      end
+
+      def with_transaction
+        execute_sql(BEGIN_TRANSACTION)
+        yield.tap { |result| execute_sql(COMMIT_TRANSACTION) }
+      rescue
+        execute_sql(ROLLBACK_TRANSACTION)
+        raise
+      end
+
+      private
+
+      include Purview::Mixins::Helpers
+      include Purview::Mixins::Logger
+
+      BEGIN_TRANSACTION = 'BEGIN'
+      COMMIT_TRANSACTION = 'COMMIT'
+      ROLLBACK_TRANSACTION = 'ROLLBACK'
+
+      attr_reader :opts, :raw_connection
+
+      def database
+        opts[:database]
+      end
+
+      def delete?(sql)
+        !!(sql.to_s =~ /\ADELETE/i)
+      end
+
+      def execute_sql(sql, opts={})
+        raise %{All "#{Base}(s)" must override the "execute_sql" method}
+      end
+
+      def extract_rows(result)
+        raise %{All "#{Base}(s)" must override the "extract_rows" method}
+      end
+
+      def extract_rows_affected(result)
+        raise %{All "#{Base}(s)" must override the "extract_rows_affected" method}
+      end
+
+      def host
+        opts[:host]
+      end
+
+      def insert?(sql)
+        !!(sql.to_s =~ /\AINSERT/i)
+      end
+
+      def new_connection
+        raise %{All "#{Base}(s)" must override the "new_connection" method}
+      end
+
+      def password
+        opts[:password]
+      end
+
+      def port
+        opts[:port]
+      end
+
+      def select?(sql)
+        !!(sql.to_s =~ /\ASELECT/i)
+      end
+
+      def structify_result(result)
+        Purview::Structs::Result.new(
+          :rows => structify_rows(extract_rows(result) || []),
+          :rows_affected => extract_rows_affected(result)
+        )
+      end
+
+      def structify_row(row)
+        Purview::Structs::Row.new(row)
+      end
+
+      def structify_rows(rows)
+        rows.map { |row| structify_row(row) }
+      end
+
+      def update?(sql)
+        !!(sql.to_s =~ /\AUPDATE/i)
+      end
+
+      def username
+        opts[:username] || Etc.getlogin
+      end
+    end
+  end
+end

data/lib/purview/raw_connections/jdbc/base.rb

@@ -0,0 +1,65 @@
+module Purview
+  module RawConnections
+    module JDBC
+      class Base < Purview::RawConnections::Base
+        private
+
+        attr_reader :last_sql, :last_statement
+
+        def delete_or_insert_or_update?(sql)
+          delete?(sql) || insert?(sql) || update?(sql)
+        end
+
+        def engine
+          raise %{All "#{Base}(s)" must override the "engine" method}
+        end
+
+        def execute_sql(sql, opts={})
+          @last_sql = sql
+          @last_statement = statement = raw_connection.createStatement
+          if select?(sql)
+            statement.executeQuery(sql)
+          elsif delete_or_insert_or_update?(sql)
+            statement.executeUpdate(sql)
+            nil
+          else
+            statement.execute(sql)
+            nil
+          end
+        end
+
+        def extract_rows(result)
+          return unless result
+          metadata = result.getMetaData
+          column_count = metadata.getColumnCount
+          [].tap do |rows|
+            while result.next
+              rows << {}.tap do |row|
+                (1..column_count).each do |index|
+                  column_name = metadata.getColumnName(index)
+                  row[column_name] = result.getString(column_name)
+                end
+              end
+            end
+          end
+        end
+
+        def extract_rows_affected(result)
+          delete_or_insert_or_update?(last_sql) ? last_statement.getUpdateCount : 0
+        end
+
+        def new_connection
+          java.sql.DriverManager.getConnection(
+            url,
+            username,
+            password
+          )
+        end
+
+        def url
+          "jdbc:#{engine}://#{host}#{port && ":#{port}"}/#{database}"
+        end
+      end
+    end
+  end
+end

data/lib/purview/raw_connections/jdbc/mysql.rb

@@ -0,0 +1,19 @@
+if defined?(JRUBY_VERSION) && (require 'jdbc/mysql' rescue nil)
+  Jdbc::MySQL.load_driver
+
+  module Purview
+    module RawConnections
+      module JDBC
+        class MySQL < Base
+          private
+
+          def engine
+            'mysql'
+          end
+        end
+      end
+    end
+  end
+
+  Purview::RawConnections::MySQL = Purview::RawConnections::JDBC::MySQL
+end

data/lib/purview/raw_connections/jdbc/postgres.rb

@@ -0,0 +1,19 @@
+if defined?(JRUBY_VERSION) && (require 'jdbc/postgres' rescue nil)
+  Jdbc::Postgres.load_driver
+
+  module Purview
+    module RawConnections
+      module JDBC
+        class Postgres < Base
+          private
+
+          def engine
+            'postgresql'
+          end
+        end
+      end
+    end
+  end
+
+  Purview::RawConnections::PostgreSQL = Purview::RawConnections::JDBC::Postgres
+end

data/lib/purview/raw_connections/jdbc.rb

@@ -0,0 +1,4 @@
+require 'purview/raw_connections/jdbc/base'
+
+require 'purview/raw_connections/jdbc/mysql'
+require 'purview/raw_connections/jdbc/postgres'

data/lib/purview/raw_connections/mysql2.rb

@@ -0,0 +1,35 @@
+if !defined?(JRUBY_VERSION) && (require 'mysql2' rescue nil)
+  module Purview
+    module RawConnections
+      class Mysql2 < Base
+        private
+
+        def execute_sql(sql, opts={})
+          raw_connection.query(sql, opts.merge(:cast => false))
+        end
+
+        def extract_rows(result)
+          result && result.to_a
+        end
+
+        def extract_rows_affected(result)
+          raw_connection.affected_rows
+        end
+
+        def new_connection
+          ::Mysql2::Client.new(
+            filter_nil_values(
+              :database => database,
+              :host => host,
+              :password => password,
+              :port => port,
+              :username => username
+            )
+          )
+        end
+      end
+    end
+  end
+
+  Purview::RawConnections::MySQL = Purview::RawConnections::Mysql2
+end

data/lib/purview/raw_connections/pg.rb

@@ -0,0 +1,35 @@
+if !defined?(JRUBY_VERSION) && (require 'pg' rescue nil)
+  module Purview
+    module RawConnections
+      class PG < Base
+        private
+
+        def execute_sql(sql, opts={})
+          raw_connection.exec(sql)
+        end
+
+        def extract_rows(result)
+          result && result.to_a
+        end
+
+        def extract_rows_affected(result)
+          result && result.cmd_tuples
+        end
+
+        def new_connection
+          ::PG.connect(
+            filter_nil_values(
+              :dbname => database,
+              :host => host,
+              :password => password,
+              :port => port,
+              :user => username
+            )
+          )
+        end
+      end
+    end
+  end
+
+  Purview::RawConnections::PostgreSQL = Purview::RawConnections::PG
+end

data/lib/purview/raw_connections.rb

@@ -0,0 +1,8 @@
+require 'purview/raw_connections/base'
+require 'purview/raw_connections/jdbc/base'
+
+require 'purview/raw_connections/jdbc/mysql'
+require 'purview/raw_connections/jdbc/postgres'
+
+require 'purview/raw_connections/mysql2'
+require 'purview/raw_connections/pg'

data/lib/purview/refinements/object.rb

@@ -2,4 +2,8 @@ class Object
   def quoted
     "'#{self}'"
   end
+
+  def sanitized
+    self.to_s.sanitized
+  end
 end

data/lib/purview/refinements/string.rb

@@ -0,0 +1,5 @@
+class String
+  def sanitized
+    self.gsub("'", "''")
+  end
+end

data/lib/purview/refinements.rb

@@ -1,2 +1,3 @@
 require 'purview/refinements/object'
+require 'purview/refinements/string'
 require 'purview/refinements/time'

data/lib/purview/structs/base.rb

@@ -1,8 +1,18 @@
 module Purview
   module Structs
     class Base < OpenStruct
-      def method_missing(method_name, *args, &block)
-        raise NoMethodError if args.size.zero?
+      def [](key)
+        key = key.to_sym unless key.is_a?(Symbol)
+        raise NoMethodError unless respond_to?(key)
+        send(key)
+      end
+
+      def []=(key, value)
+        send("#{key}=", value)
+      end
+
+      def method_missing(method, *args, &block)
+        raise NoMethodError if args.empty?
         super
       end
     end