purview 1.5.0 → 1.6.0

data/lib/purview/databases/postgresql.rb

@@ -114,6 +114,10 @@ module Purview
         ]
       end
 
+      def limit_map
+        super.merge(Purview::Types::String => 255)
+      end
+
       def limitless_types
         super + [
           Purview::Types::Money,

data/lib/purview/databases/sqlite.rb

@@ -0,0 +1,173 @@
+module Purview
+  module Databases
+    class SQLite < Base
+      private
+
+      def connection_type
+        Purview::Connections::SQLite
+      end
+
+      def create_index_sql(table_name, index_name, index, index_opts={})
+        'CREATE%sINDEX %s ON %s (%s)' % [
+          index.unique? ? ' UNIQUE ' : ' ',
+          index_name,
+          table_name,
+          column_names(index).join(', '),
+        ]
+      end
+
+      def create_table_sql(table_name, table, table_opts={})
+        'CREATE TABLE %s (%s)' % [
+          table_name,
+          column_definitions(table).join(', '),
+        ]
+      end
+
+      def create_temporary_table_sql(table_name, table, table_opts={})
+        'CREATE TEMPORARY TABLE %s (%s)' % [
+          table_name,
+          column_definitions(table).join(', '),
+        ]
+      end
+
+      def dialect_type
+        Purview::Dialects::SQLite
+      end
+
+      def disable_table_sql(table)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
+          table_metadata_table.name,
+          table_metadata_table.enabled_at_column.name,
+          null_value,
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+          table_metadata_table.enabled_at_column.name,
+        ]
+      end
+
+      def drop_index_sql(table_name, index_name, index, index_opts={})
+        'DROP INDEX %s' % [
+          index_name,
+        ]
+      end
+
+      def drop_table_sql(table_name, table, table_opts={})
+        'DROP TABLE %s' % [
+          table_name,
+        ]
+      end
+
+      def enable_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table.name,
+          table_metadata_table.enabled_at_column.name,
+          quoted(timestamp),
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+          table_metadata_table.enabled_at_column.name,
+        ]
+      end
+
+      def ensure_table_metadata_absent_for_table_sql(table)
+        'DELETE FROM %s WHERE %s = %s' % [
+          table_metadata_table.name,
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+        ]
+      end
+
+      def ensure_table_metadata_exists_for_table_sql(table)
+        'INSERT INTO %s (%s) SELECT %s WHERE NOT EXISTS (SELECT 1 FROM %s WHERE %s = %s)' % [
+          table_metadata_table.name,
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+          table_metadata_table.name,
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+        ]
+      end
+
+      def ensure_table_metadata_table_exists_sql
+        'CREATE TABLE IF NOT EXISTS %s (%s)' % [
+          table_metadata_table.name,
+          column_definitions(table_metadata_table).join(', '),
+        ]
+      end
+
+      def initialize_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table.name,
+          table_metadata_table.max_timestamp_pulled_column.name,
+          quoted(timestamp),
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+          table_metadata_table.max_timestamp_pulled_column.name,
+        ]
+      end
+
+      def get_table_metadata_value_sql(table, column)
+        'SELECT %s FROM %s WHERE %s = %s' % [
+          column.name,
+          table_metadata_table.name,
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+        ]
+      end
+
+      def limit_map
+        super.merge(Purview::Types::String => 255)
+      end
+
+      def lock_table_sql(table, timestamp)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NULL' % [
+          table_metadata_table.name,
+          table_metadata_table.locked_at_column.name,
+          quoted(timestamp),
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+          table_metadata_table.locked_at_column.name,
+        ]
+      end
+
+      def next_table_sql(timestamp)
+        'SELECT %s FROM %s WHERE %s IS NOT NULL AND %s IS NOT NULL AND %s IS NULL ORDER BY %s IS NULL DESC, %s LIMIT 1' % [
+          table_metadata_table.table_name_column.name,
+          table_metadata_table.name,
+          table_metadata_table.enabled_at_column.name,
+          table_metadata_table.max_timestamp_pulled_column.name,
+          table_metadata_table.locked_at_column.name,
+          table_metadata_table.last_pulled_at_column.name,
+          table_metadata_table.last_pulled_at_column.name,
+        ]
+      end
+
+      def set_table_metadata_value_sql(table, column, value)
+        'UPDATE %s SET %s = %s WHERE %s = %s' % [
+          table_metadata_table.name,
+          column.name,
+          quoted(value),
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+        ]
+      end
+
+      def type_map
+        super.merge(
+          Purview::Types::Money => 'decimal',
+          Purview::Types::UUID => 'varchar',
+        )
+      end
+
+      def unlock_table_sql(table)
+        'UPDATE %s SET %s = %s WHERE %s = %s AND %s IS NOT NULL' % [
+          table_metadata_table.name,
+          table_metadata_table.locked_at_column.name,
+          null_value,
+          table_metadata_table.table_name_column.name,
+          quoted(table.name),
+          table_metadata_table.locked_at_column.name,
+        ]
+      end
+    end
+  end
+end

data/lib/purview/dialects.rb

@@ -3,3 +3,4 @@ require 'purview/dialects/base'
 require 'purview/dialects/mssql'
 require 'purview/dialects/mysql'
 require 'purview/dialects/postgresql'
+require 'purview/dialects/sqlite'

data/lib/purview/dialects/sqlite.rb

@@ -0,0 +1,25 @@
+module Purview
+  module Dialects
+    class SQLite < Base
+      def false_value
+        '0'
+      end
+
+      def null_value
+        'NULL'
+      end
+
+      def quoted(value)
+        value.nil? ? null_value : value.quoted
+      end
+
+      def sanitized(value)
+        value.nil? ? null_value : value.sanitized
+      end
+
+      def true_value
+        '1'
+      end
+    end
+  end
+end

data/lib/purview/loaders.rb

@@ -1,4 +1,6 @@
 require 'purview/loaders/base'
 
+require 'purview/loaders/mssql'
 require 'purview/loaders/mysql'
 require 'purview/loaders/postgresql'
+require 'purview/loaders/sqlite'

data/lib/purview/loaders/mssql.rb

@@ -0,0 +1,85 @@
+module Purview
+  module Loaders
+    class MSSQL < Base
+      private
+
+      def dialect_type
+        Purview::Dialects::MSSQL
+      end
+
+      def id_in_sql(temporary_table_name)
+        'SELECT %s FROM %s' % [
+          table.id_column.name,
+          temporary_table_name,
+        ]
+      end
+
+      def in_window_sql(window)
+        '%s BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def not_in_window_sql(window)
+        '%s NOT BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def table_delete_sql(window, temporary_table_name)
+        'DELETE FROM %s WHERE %s AND %s NOT IN (%s)' % [
+          table.name,
+          in_window_sql(window),
+          table.id_column.name,
+          id_in_sql(temporary_table_name),
+        ]
+      end
+
+      def table_insert_sql(window, temporary_table_name)
+        'INSERT INTO %s (%s) SELECT %s FROM %s t1 WHERE NOT EXISTS (SELECT 1 FROM %s t2 WHERE t1.%s = t2.%s)' % [
+          table.name,
+          table.column_names.join(', '),
+          table.column_names.join(', '),
+          temporary_table_name,
+          table.name,
+          table.id_column.name,
+          table.id_column.name,
+        ]
+      end
+
+      def table_update_sql(window, temporary_table_name)
+        'UPDATE %s t1 JOIN %s t2 ON t1.%s = t2.%s SET %s' % [
+          table.name,
+          temporary_table_name,
+          table.id_column.name,
+          table.id_column.name,
+          table.column_names.map { |column_name| "t1.#{column_name} = t2.#{column_name}" }.join(', '),
+        ]
+      end
+
+      def temporary_table_insert_sql(temporary_table_name, rows)
+        'INSERT INTO %s (%s) VALUES %s' % [
+          temporary_table_name,
+          table.column_names.join(', '),
+          rows.map { |row| "(#{row_values(row)})" }.join(', ')
+        ]
+      end
+
+      def temporary_table_opts
+        super.merge(:create_indices => false)
+      end
+
+      def temporary_table_verify_sql(temporary_table_name, rows, window)
+        'SELECT COUNT(1) %s FROM %s WHERE %s' % [
+          count_column_name,
+          temporary_table_name,
+          not_in_window_sql(window),
+        ]
+      end
+    end
+  end
+end

data/lib/purview/loaders/sqlite.rb

@@ -0,0 +1,85 @@
+module Purview
+  module Loaders
+    class SQLite < Base
+      private
+
+      def dialect_type
+        Purview::Dialects::SQLite
+      end
+
+      def id_in_sql(temporary_table_name)
+        'SELECT %s FROM %s' % [
+          table.id_column.name,
+          temporary_table_name,
+        ]
+      end
+
+      def in_window_sql(window)
+        '%s BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def not_in_window_sql(window)
+        '%s NOT BETWEEN %s AND %s' % [
+          table.updated_timestamp_column.name,
+          quoted(window.min),
+          quoted(window.max),
+        ]
+      end
+
+      def table_delete_sql(window, temporary_table_name)
+        'DELETE FROM %s WHERE %s AND %s NOT IN (%s)' % [
+          table.name,
+          in_window_sql(window),
+          table.id_column.name,
+          id_in_sql(temporary_table_name),
+        ]
+      end
+
+      def table_insert_sql(window, temporary_table_name)
+        'INSERT INTO %s (%s) SELECT %s FROM %s t1 WHERE NOT EXISTS (SELECT 1 FROM %s t2 WHERE t1.%s = t2.%s)' % [
+          table.name,
+          table.column_names.join(', '),
+          table.column_names.join(', '),
+          temporary_table_name,
+          table.name,
+          table.id_column.name,
+          table.id_column.name,
+        ]
+      end
+
+      def table_update_sql(window, temporary_table_name)
+        'UPDATE %s t1 JOIN %s t2 ON t1.%s = t2.%s SET %s' % [
+          table.name,
+          temporary_table_name,
+          table.id_column.name,
+          table.id_column.name,
+          table.column_names.map { |column_name| "t1.#{column_name} = t2.#{column_name}" }.join(', '),
+        ]
+      end
+
+      def temporary_table_insert_sql(temporary_table_name, rows)
+        'INSERT INTO %s (%s) VALUES %s' % [
+          temporary_table_name,
+          table.column_names.join(', '),
+          rows.map { |row| "(#{row_values(row)})" }.join(', ')
+        ]
+      end
+
+      def temporary_table_opts
+        super.merge(:create_indices => false)
+      end
+
+      def temporary_table_verify_sql(temporary_table_name, rows, window)
+        'SELECT COUNT(1) %s FROM %s WHERE %s' % [
+          count_column_name,
+          temporary_table_name,
+          not_in_window_sql(window),
+        ]
+      end
+    end
+  end
+end

data/lib/purview/pullers.rb

@@ -4,4 +4,5 @@ require 'purview/pullers/base_sql'
 require 'purview/pullers/mssql'
 require 'purview/pullers/mysql'
 require 'purview/pullers/postgresql'
+require 'purview/pullers/sqlite'
 require 'purview/pullers/uri'

data/lib/purview/pullers/sqlite.rb

@@ -0,0 +1,15 @@
+module Purview
+  module Pullers
+    class SQLite < BaseSQL
+      private
+
+      def connection_type
+        Purview::Connections::SQLite
+      end
+
+      def dialect_type
+        Purview::Dialects::SQLite
+      end
+    end
+  end
+end

data/lib/purview/raw_connections.rb

@@ -4,14 +4,17 @@ require 'purview/raw_connections/jdbc/base'
 require 'purview/raw_connections/jdbc/jtds'
 require 'purview/raw_connections/jdbc/mysql'
 require 'purview/raw_connections/jdbc/postgres'
+require 'purview/raw_connections/jdbc/sqlite3'
 
 require 'purview/raw_connections/mysql2'
 require 'purview/raw_connections/pg'
+require 'purview/raw_connections/sqlite3'
 require 'purview/raw_connections/tiny_tds'
 
 if \
   !defined?(Purview::RawConnections::MSSQL) &&
   !defined?(Purview::RawConnections::MySQL) &&
-  !defined?(Purview::RawConnections::PostgreSQL)
-  raise 'Could not initialize raw-connections; please install and require one or more of the following gems: `jdbc-jtds`, `jdbc-mysql`, `jdbc-postgres`, `mysql2`, `pg` and/or `tiny_tds`'
+  !defined?(Purview::RawConnections::PostgreSQL) &&
+  !defined?(Purview::RawConnections::SQLite)
+  raise 'Could not initialize raw-connections; please install and require one or more of the following gems: `jdbc-jtds`, `jdbc-mysql`, `jdbc-postgres`, `jdbc-sqlite3`, `mysql2`, `pg`, `sqlite3` and/or `tiny_tds`'
 end

data/lib/purview/raw_connections/jdbc/mysql.rb

@@ -1,4 +1,4 @@
-safe_require('jdbc-mysql')
+safe_require('jdbc/mysql')
 
 if defined?(Jdbc::MySQL)
   Jdbc::MySQL.load_driver