purview 1.0.0.alpha → 1.0.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4753e561eece41d8838e916c7914ae3b3d155b3b
-  data.tar.gz: 8430637788951088580baa1f533540968974f28c
+  metadata.gz: 48f684f51dd0c4ff6b947959f5c1a3c2eb9489d9
+  data.tar.gz: 832f24cb3904d7fb527b5cabc36d75f488b494ac
 SHA512:
-  metadata.gz: 742e3ab26e898924c9054b461d41c700144859910b96ccfd2df6ac6e9d51bb2945a61f73282445f7c239c8adfeb4cc53bdf518aad195b91cc931cdeca24639bb
-  data.tar.gz: 43d8b1a2b5f6cf401a33e254a8b63169b1764fe3a88b83391a91b6068203b0bd02df339746982580864aba2a3c3f9ae0ba079fd938f31fe687d37e38b7aff987
+  metadata.gz: f5d78891cc67a18657720290c7bf5cfb8c41259b7a574eb23b0e6160c987a43cfbe513948ca0d35b3df8c4372fce790c9cb64986a83596ba1ecfad825998eb17
+  data.tar.gz: 9e4004abc1d287912deaabb8634589db52ea15fbaf6c561609e7f058a8966cbc6fdc778e48abd3780b00a17451683b8f13a471b9818cfbddf54a263d6f3b4d9d

data/.travis.yml

@@ -1,9 +1,5 @@
 language: ruby
 
-matrix:
-  allow_failures:
-    - rvm: jruby-19mode
-
 rvm:
   - 1.9.3
   - 2.0.0

data/Gemfile

@@ -1,3 +1,13 @@
 source 'https://rubygems.org'
 
+group :development do
+  if defined?(JRUBY_VERSION)
+    gem 'jdbc-mysql', '~> 5.1'
+    gem 'jdbc-postgres', '~> 9.4'
+  else
+    gem 'mysql2', '~> 0.3'
+    gem 'pg', '~> 0.18'
+  end
+end
+
 gemspec

data/README.md

@@ -61,7 +61,7 @@ columns = [
 ]
 ```
 
-Configure the `Puller` (available puller-types: `URI`)
+Configure the `Puller` (available puller-types: `MySQL`, `PostgreSQL` & `URI`)
 ```ruby
 puller_opts = {
   :type => Purview::Pullers::URI,
@@ -69,7 +69,7 @@ puller_opts = {
 }
 ```
 
-Configure the `Parser` (available parser-types: `CSV` & `TSV`)
+Configure the `Parser` (available parser-types: `CSV`, `SQL` & `TSV`)
 ```ruby
 parser_opts = {
   :type => Purview::Parsers::TSV,

data/TODO

@@ -40,6 +40,10 @@
 * Finish PostgreSQL support (requires `pg` gem)
 * Finish MySQL support (requires `mysql2` gem)
 * Usage/configuration examples in README.md
+* Create SQL-puller(s)
+* Introduce [SQL] dialect(s)
+* Fix JRuby bundle/build
+* Fall back to logged in user if no [database-]username is provided
 
 ... ONGOING TASKS ...
 
@@ -49,16 +53,17 @@
 
 ... OPEN TASKS ....
 
-* Create table class to encapulate `table_metadata` logic
+* Add COPY `def copy(sql)` support to PostgreSQL connection
+* Leverage COPY support for PostgreSQL pulls
+* Create class to encapulate `table_metadata` logic
+* Further encapsulate `Dialect` logic
 * Build out change-log tables
 * Build out canonical, fact and aggregate tables (and related transforms)
 * Configurable re-pull window (do this automatically once up to current?)
 * Add schema management capabilities (detect schema-deltas and suggestion
   modifications)
-
-... DEFERRED TASKS ...
-
-* Fix JRuby bundle/build
+* Refactor mutator methods to be globally available (e.g. coalesced -> Coalesce,
+  quoted -> Quote, etc.)
 
 ... CLOSED QUESTIONS ...
 

data/lib/purview/columns/base.rb

@@ -25,7 +25,7 @@ module Purview
       end
 
       def nullable
-        coalesce(opts[:nullable], true)
+        coalesced(opts[:nullable], true)
       end
 
       def nullable?
@@ -48,7 +48,7 @@ module Purview
       end
 
       def type
-        coalesce(opts[:type], Purview::Types::String)
+        coalesced(opts[:type], Purview::Types::String)
       end
 
       private

data/lib/purview/columns.rb

@@ -1,4 +1,5 @@
 require 'purview/columns/base'
+
 require 'purview/columns/boolean'
 require 'purview/columns/date'
 require 'purview/columns/float'

data/lib/purview/connections/base.rb

@@ -1,54 +1,40 @@
 module Purview
   module Connections
     class Base
-      def initialize(opts={})
-        @opts = opts
+      def self.connect(opts)
+        new(opts)
       end
 
-      def connect
-        @connection ||= new_connection
-        self
+      def self.with_new_connection(opts)
+        yield connection = connect(opts)
+      ensure
+        connection.disconnect if connection
+      end
+
+      def initialize(opts)
+        @raw_connection = raw_connection_type.connect(opts)
       end
 
       def disconnect
-        connection.close
-        @connection = nil
+        raw_connection.disconnect
+        @raw_connection = nil
         self
       end
 
       def execute(sql)
-        logger.debug("Executing: #{sql}")
-        result = execute_sql(sql)
-        Purview::Structs::Result.new(
-          :rows => extract_rows(result),
-          :rows_affected => extract_rows_affected(result)
-        )
+        raw_connection.execute(sql)
       end
 
       def with_transaction
-        raise %{All "#{Base}(s)" must override the "with_transaction" method}
+        raw_connection.with_transaction { yield }
       end
 
       private
 
-      include Purview::Mixins::Logger
-
-      attr_reader :opts, :connection
-
-      def execute_sql(sql)
-        raise %{All "#{Base}(s)" must override the "execute_sql" method}
-      end
-
-      def extract_rows(result)
-        raise %{All "#{Base}(s)" must override the "extract_rows" method}
-      end
-
-      def extract_rows_affected(result)
-        raise %{All "#{Base}(s)" must override the "extract_rows_affected" method}
-      end
+      attr_reader :raw_connection
 
-      def new_connection
-        raise %{All "#{Base}(s)" must override the "new_connection" method}
+      def raw_connection_type
+        raise %{All "#{Base}(s)" must override the "raw_connection_type" method}
       end
     end
   end

data/lib/purview/connections/mysql.rb

@@ -1,38 +1,10 @@
 module Purview
   module Connections
     class MySQL < Base
-      def with_transaction
-        connection.query(BEGIN_TRANSACTION)
-        yield.tap { |result| connection.query(COMMIT_TRANSACTION) }
-      rescue Mysql2::Error
-        connection.query(ROLLBACK_TRANSACTION)
-        raise
-      end
-
       private
 
-      BEGIN_TRANSACTION = 'BEGIN'
-      COMMIT_TRANSACTION = 'COMMIT'
-      ROLLBACK_TRANSACTION = 'ROLLBACK'
-
-      def execute_sql(sql)
-        connection.query(sql, query_opts)
-      end
-
-      def extract_rows(result)
-        result && result.to_a
-      end
-
-      def extract_rows_affected(result)
-        connection.affected_rows
-      end
-
-      def new_connection
-        Mysql2::Client.new(opts)
-      end
-
-      def query_opts
-        { :cast => false }
+      def raw_connection_type
+        Purview::RawConnections::MySQL
       end
     end
   end

data/lib/purview/connections/postgresql.rb

@@ -1,26 +1,10 @@
 module Purview
   module Connections
     class PostgreSQL < Base
-      def with_transaction
-        connection.transaction { yield }
-      end
-
       private
 
-      def execute_sql(sql)
-        connection.exec(sql)
-      end
-
-      def extract_rows(result)
-        result && result.to_a
-      end
-
-      def extract_rows_affected(result)
-        result && result.cmd_tuples
-      end
-
-      def new_connection
-        PG.connect(opts)
+      def raw_connection_type
+        Purview::RawConnections::PostgreSQL
       end
     end
   end

data/lib/purview/databases/base.rb

@@ -13,10 +13,6 @@ module Purview
         @tables << table
       end
 
-      def connect
-        connection.connect
-      end
-
       def create_index(connection, table, columns, opts={})
         table_opts = extract_table_options(opts)
         table_name = table_name(table, table_opts)
@@ -142,10 +138,6 @@ module Purview
         end
       end
 
-      def false_value
-        raise %{All "#{Base}(s)" must override the "false_value" method}
-      end
-
       def lock_table(table, timestamp)
         with_context_logging("`lock_table` for: #{table_name(table)}") do
           with_new_connection do |connection|
@@ -157,14 +149,6 @@ module Purview
         end
       end
 
-      def null_value
-        raise %{All "#{Base}(s)" must override the "null_value" method}
-      end
-
-      def quoted(value)
-        value.nil? ? null_value : value.quoted
-      end
-
       def sync
         with_new_connection do |connection|
           with_transaction(connection) do |timestamp|
@@ -183,10 +167,6 @@ module Purview
         end
       end
 
-      def true_value
-        raise %{All "#{Base}(s)" must override the "true_value" method}
-      end
-
       def unlock_table(table)
         with_context_logging("`unlock_table` for: #{table_name(table)}") do
           with_new_connection do |connection|
@@ -200,11 +180,14 @@ module Purview
 
       private
 
+      include Purview::Mixins::Connection
       include Purview::Mixins::Helpers
       include Purview::Mixins::Logger
 
       attr_reader :opts, :tables
 
+      public :connect
+
       def column_names(columns)
         columns.map(&:name)
       end
@@ -235,12 +218,8 @@ module Purview
         end
       end
 
-      def connection
-        connection_type.new(connection_opts)
-      end
-
       def connection_opts
-        {}
+        { :database => name }
       end
 
       def connection_type
@@ -267,6 +246,14 @@ module Purview
         {}
       end
 
+      def dialect
+        dialect_type.new
+      end
+
+      def dialect_type
+        raise %{All "#{Base}(s)" must override the "dialect_type" method}
+      end
+
       def drop_index_sql(table_name, index_name, table, columns, index_opts={})
         raise %{All "#{Base}(s)" must override the "drop_index_sql" method}
       end
@@ -305,6 +292,10 @@ module Purview
         opts[:table] || { :create_indices => true }
       end
 
+      def false_value
+        dialect.false_value
+      end
+
       def get_enabled_for_table(connection, table)
         row = connection.execute(get_last_pulled_at_for_table_sql(table)).rows[0]
         enabled = row[table_metadata_enabled_column_name]
@@ -390,6 +381,10 @@ module Purview
         Purview::Structs::Window.new(:min => min, :max => max)
       end
 
+      def null_value
+        dialect.null_value
+      end
+
       def nullable?(column)
         column.nullable?
       end
@@ -398,6 +393,14 @@ module Purview
         column.primary_key?
       end
 
+      def quoted(value)
+        dialect.quoted(value)
+      end
+
+      def sanitized(value)
+        dialect.sanitized(value)
+      end
+
       def set_enabled_for_table(connection, table, enabled)
         connection.execute(set_enabled_for_table_sql(table, enabled))
       end
@@ -491,6 +494,10 @@ module Purview
         table_opts[:name] || table.name
       end
 
+      def true_value
+        dialect.true_value
+      end
+
       def type(column)
         type_map[column.type]
       end
@@ -512,12 +519,6 @@ module Purview
         raise %{All "#{Base}(s)" must override the "unlock_table_sql" method}
       end
 
-      def with_new_connection
-        yield connection = connect
-      ensure
-        connection.disconnect if connection
-      end
-
       def with_next_table(connection, timestamp)
         table = next_table(connection, timestamp)
         raise Purview::Exceptions::NoTable.new unless table

data/lib/purview/databases/mysql.rb

@@ -1,24 +1,8 @@
 module Purview
   module Databases
     class MySQL < Base
-      def false_value
-        '0'
-      end
-
-      def null_value
-        'NULL'
-      end
-
-      def true_value
-        '1'
-      end
-
       private
 
-      def connection_opts
-        super.merge(:database => name)
-      end
-
       def connection_type
         Purview::Connections::MySQL
       end
@@ -49,6 +33,10 @@ module Purview
         super.merge(Purview::Types::Timestamp => '0')
       end
 
+      def dialect_type
+        Purview::Dialects::MySQL
+      end
+
       def drop_index_sql(table_name, index_name, table, columns, index_opts={})
         'DROP INDEX %s' % [
           index_name,

data/lib/purview/databases/postgresql.rb

@@ -1,24 +1,8 @@
 module Purview
   module Databases
     class PostgreSQL < Base
-      def false_value
-        'FALSE'
-      end
-
-      def null_value
-        'NULL'
-      end
-
-      def true_value
-        'TRUE'
-      end
-
       private
 
-      def connection_opts
-        super.merge(:dbname => name)
-      end
-
       def connection_type
         Purview::Connections::PostgreSQL
       end
@@ -45,6 +29,10 @@ module Purview
         ]
       end
 
+      def dialect_type
+        Purview::Dialects::PostgreSQL
+      end
+
       def drop_index_sql(table_name, index_name, table, columns, index_opts={})
         'DROP INDEX %s' % [
           index_name,

data/lib/purview/databases.rb

@@ -1,3 +1,4 @@
 require 'purview/databases/base'
+
 require 'purview/databases/mysql'
 require 'purview/databases/postgresql'

data/lib/purview/dialects/base.rb

@@ -0,0 +1,25 @@
+module Purview
+  module Dialects
+    class Base
+      def false_value
+        raise %{All "#{Base}(s)" must override the "false_value" method}
+      end
+
+      def null_value
+        raise %{All "#{Base}(s)" must override the "null_value" method}
+      end
+
+      def quoted(value)
+        raise %{All "#{Base}(s)" must override the "quoted" method}
+      end
+
+      def sanitized(value)
+        raise %{All "#{Base}(s)" must override the "sanitized" method}
+      end
+
+      def true_value
+        raise %{All "#{Base}(s)" must override the "true_value" method}
+      end
+    end
+  end
+end

data/lib/purview/dialects/mysql.rb

@@ -0,0 +1,25 @@
+module Purview
+  module Dialects
+    class MySQL < Base
+      def false_value
+        '0'
+      end
+
+      def null_value
+        'NULL'
+      end
+
+      def quoted(value)
+        value.nil? ? null_value : value.quoted
+      end
+
+      def sanitized(value)
+        value.nil? ? null_value : value.sanitized
+      end
+
+      def true_value
+        '1'
+      end
+    end
+  end
+end

data/lib/purview/dialects/postgresql.rb

@@ -0,0 +1,25 @@
+module Purview
+  module Dialects
+    class PostgreSQL < Base
+      def false_value
+        'FALSE'
+      end
+
+      def null_value
+        'NULL'
+      end
+
+      def quoted(value)
+        value.nil? ? null_value : value.quoted
+      end
+
+      def sanitized(value)
+        value.nil? ? null_value : value.sanitized
+      end
+
+      def true_value
+        'TRUE'
+      end
+    end
+  end
+end

data/lib/purview/dialects.rb

@@ -0,0 +1,4 @@
+require 'purview/dialects/base'
+
+require 'purview/dialects/mysql'
+require 'purview/dialects/postgresql'

data/lib/purview/exceptions/{table.rb → base_table.rb}

@@ -1,6 +1,6 @@
 module Purview
   module Exceptions
-    class Table < Base
+    class BaseTable < Base
       def initialize(table)
         @table = table
       end

data/lib/purview/exceptions/could_not_acquire_lock.rb

@@ -1,6 +1,6 @@
 module Purview
   module Exceptions
-    class CouldNotAcquireLock < Table
+    class CouldNotAcquireLock < BaseTable
       def message
         "Could not acquire the lock for table: #{table.name}"
       end

data/lib/purview/exceptions/lock_already_released.rb

@@ -1,6 +1,6 @@
 module Purview
   module Exceptions
-    class LockAlreadyReleased < Table
+    class LockAlreadyReleased < BaseTable
       def message
         "Lock already released for table: #{table.name}"
       end

data/lib/purview/exceptions/no_window.rb

@@ -1,6 +1,6 @@
 module Purview
   module Exceptions
-    class NoWindow < Table
+    class NoWindow < BaseTable
       def message
         "Could not find a window for table: #{table.name}"
       end

data/lib/purview/exceptions.rb

@@ -1,5 +1,6 @@
 require 'purview/exceptions/base'
-require 'purview/exceptions/table'
+require 'purview/exceptions/base_table'
+
 require 'purview/exceptions/could_not_acquire_lock'
 require 'purview/exceptions/lock_already_released'
 require 'purview/exceptions/no_table'

data/lib/purview/loaders/base.rb

@@ -49,6 +49,18 @@ module Purview
         table.database
       end
 
+      def dialect
+        dialect_type.new
+      end
+
+      def dialect_type
+        raise %{All "#{Base}(s)" must override the "dialect_type" method}
+      end
+
+      def false_value
+        dialect.false_value
+      end
+
       def id_in_sql(temporary_table_name)
         raise %{All "#{Base}(s)" must override the "id_in_sql" method}
       end
@@ -74,18 +86,26 @@ module Purview
         raise %{All "#{Base}(s)" must override the "not_in_window_sql" method}
       end
 
+      def null_value
+        dialect.null_value
+      end
+
       def quoted(value)
-        database.quoted(value)
+        dialect.quoted(value)
       end
 
       def row_values(row)
-        table.column_names.map { |column_name| quoted(row[column_name]) }.join(', ')
+        table.column_names.map { |column_name| quoted(sanitized(row[column_name])) }.join(', ')
       end
 
       def rows_per_slice
         opts[:rows_per_slice] || 1000
       end
 
+      def sanitized(value)
+        dialect.sanitized(value)
+      end
+
       def table
         opts[:table]
       end
@@ -117,6 +137,10 @@ module Purview
         raise %{All "#{Base}(s)" must override the "temporary_table_verify_sql" method}
       end
 
+      def true_value
+        dialect.true_value
+      end
+
       def verify_temporary_table(connection, temporary_table_name, rows, window)
         with_context_logging("`verify_temporary_table` for: #{temporary_table_name}") do
           rows_outside_window = connection.execute(

data/lib/purview/loaders/mysql.rb

@@ -3,6 +3,10 @@ module Purview
     class MySQL < Base
       private
 
+      def dialect_type
+        Purview::Dialects::MySQL
+      end
+
       def id_in_sql(temporary_table_name)
         'SELECT %s FROM %s' % [
           table.id_column.name,