cranium 0.5 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63bb4574d9b494202c0aff700e4263c1b14753b0b973b9dcb52148ed24a7a861
-  data.tar.gz: ee3238eee62a1c6ee3cd9d3c69ef7ab19a42133996b5fa6334d1a56a9f2f5ff0
+  metadata.gz: 4715c818c4f8f98f7d1a489979c3cf5ac1a2b50114052c6d619526fc8d0a4ca0
+  data.tar.gz: 6c619a8f8570cae9ef6c51002db8cf2ea6785983b70e0ace6e28b14f37e72330
 SHA512:
-  metadata.gz: 2351552d7be2849c5698df99b977c3c56d54ef376d351125bdb51526b6b826d2f4afecc79292479d0d8308af7edd086fd47213479e693c18ecfe9bc31d40549a
-  data.tar.gz: a091064e7f4fa3a56a741b4704012ab1e8a5ced7b6dcad0b6826d226dfd06c5520dddfc1a8e8eabbf04dcd7585d7f1bb97801b539f67a3fdb7fbdd063713be4d
+  metadata.gz: e12f463f677b70d7765120658c0efcb00da40c92ef9dd321eb4e6804106cf22f1da85505dfcfd1c5baf8fb46f67f5aa3784cccd4bc7214c6ab767f4d46cd5fa9
+  data.tar.gz: '080472dd1a1c70907496fdf0978231b82f6b6c8da0baf6c6b0e8488398387ef75a2b2ce95731818dd2c712eff316823ac168fc4063202ad887d662304f4a30a2'

data/.env

@@ -1 +1,3 @@
-DATABASE_HOST=127.0.0.1
+GREENPLUM_HOST=127.0.0.1
+GPFDIST_HOST=127.0.0.1
+

data/.gitignore

@@ -1,3 +1,4 @@
+*.aes
 *.gem
 *.rbc
 .bundle
@@ -5,6 +6,7 @@
 .yardoc
 .vagrant
 .idea
+.ruby-version
 Gemfile.lock
 InstalledFiles
 _yardoc

data/Dockerfile

@@ -0,0 +1,22 @@
+FROM ruby:2.7.3-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+                      build-essential \
+                      libpq-dev \
+                      postgresql-client \
+                      git-core
+
+RUN mkdir /app
+WORKDIR /app
+
+RUN gem install bundler
+
+COPY Gemfile .
+COPY cranium.gemspec .
+
+ARG https_proxy
+ARG http_proxy
+RUN bundle install -j 5
+COPY . .
+RUN mkdir /tmp/custdata/

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Zoltan Ormandi
+Copyright (c) 2013-2020 Emarsys
 
 MIT License
 

data/bin/run_tests.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+sleep 30
+bundle exec rake

data/codeship-services.yml

@@ -0,0 +1,24 @@
+cranium:
+  build:
+    image: cranium
+    dockerfile: Dockerfile
+  cached: true
+
+greenplum:
+  image: kevinmtrowbridge/gpdb-docker
+  cached: true
+  volumes:
+    - gpfdist-init:/home/gpadmin/gpfdist-data
+    - ./docker/:/docker-entrypoint-initdb.d
+    - ./db:/db
+  command: sh /docker-entrypoint-initdb.d/initialize_greenplum.sh
+
+build-app:
+  image: cranium
+  depends_on:
+    - greenplum
+  environment:
+    GREENPLUM_HOST: "greenplum"
+    GPFDIST_HOST: "127.0.0.1"
+  volumes:
+    - gpfdist-init:/app/tmp/custdata

data/codeship-steps.yml

@@ -0,0 +1,3 @@
+- name: Run tests
+  service: build-app
+  command: sh bin/run_tests.sh

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.5'
+  spec.version = '0.8.1'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}
@@ -13,13 +13,13 @@ Gem::Specification.new do |spec|
   spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'pg', '~> 0'
+  spec.add_runtime_dependency 'pg', '>= 0'
   spec.add_runtime_dependency 'progressbar', '~> 0'
   spec.add_runtime_dependency 'sequel', '>= 4', '< 6'
   spec.add_runtime_dependency 'slop', '~> 3'
 
-  spec.add_development_dependency 'bundler', '~> 1'
-  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'
   spec.add_development_dependency 'ruby-prof', '~> 0'
   spec.add_development_dependency 'cucumber', '~> 1'

data/docker/create_databases.sql

@@ -0,0 +1,8 @@
+CREATE RESOURCE QUEUE smart_insight WITH (ACTIVE_STATEMENTS=10, PRIORITY=MEDIUM);
+
+CREATE ROLE cranium WITH RESOURCE QUEUE smart_insight CREATEEXTTABLE LOGIN PASSWORD 'cranium';
+COMMENT ON ROLE cranium IS 'Cranium test user';
+
+CREATE DATABASE cranium WITH OWNER=cranium;
+
+CREATE ROLE database_administrator WITH SUPERUSER LOGIN PASSWORD 'emarsys';

data/docker/initialize_greenplum.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "127.0.0.1 $(cat ~/orig_hostname)" >> /etc/hosts
+
+service sshd start
+
+su - gpadmin -c /usr/local/bin/run.sh
+su - gpadmin -c 'cat /docker-entrypoint-initdb.d/create_databases.sql | psql -e'
+su - gpadmin -c 'gpfdist -d /home/gpadmin/gpfdist-data -p 8123'

data/docker-compose.yml

@@ -1,9 +1,10 @@
-greenplum:
-  image: kevinmtrowbridge/gpdb-docker
-  ports:
-    - 5432:5432
-    - 8123:8123
-  volumes:
-    - ./tmp/custdata:/home/gpadmin/gpfdist-data
-    - ./db:/db
-  command: sh /db/start.sh
+services:
+  greenplum:
+    image: kevinmtrowbridge/gpdb-docker
+    ports:
+      - 5432:5432
+      - 8123:8123
+    volumes:
+      - ./tmp/custdata:/home/gpadmin/gpfdist-data
+      - ./db:/db
+    command: sh /db/start.sh

data/features/import/delete_from_table_based_on_csv.feature

@@ -0,0 +1,39 @@
+Feature: Delete rows from table provided by CSV file
+
+  Scenario: Successful delete
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | source_id   | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | source_id (i) |
+      | 1             |
+      | 2             |
+      | 3             |
+      | 4             |
+      | 5             |
+    And a "deleted_contacts_extract.csv" data file containing:
+    """
+    source_id
+    3
+    4
+    """
+    And the following definition:
+    """
+    source :deleted_contacts_extract do
+      field :source_id, String
+    end
+
+    import :deleted_contacts_extract do
+      into :dim_contact
+      put :source_id
+
+      delete_on :source_id
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_contact" table should contain:
+      | source_id   |
+      | 1           |
+      | 2           |
+      | 5           |

data/features/restore_database_connection.feature

@@ -16,7 +16,7 @@ Feature: Sequel database connections are fault tolerant
     require 'sequel'
 
     def terminate_connections
-      connection = Sequel.connect "postgres://database_administrator:emarsys@#{ENV['DATABASE_HOST'] || '192.168.56.43'}:5432/cranium", loggers: Cranium.configuration.loggers
+      connection = Sequel.connect "postgres://database_administrator:emarsys@#{ENV['GREENPLUM_HOST'] || '192.168.56.43'}:5432/cranium", loggers: Cranium.configuration.loggers
       connection.run("SELECT pg_terminate_backend(procpid) FROM pg_stat_activity WHERE procpid <> pg_backend_pid() AND datname = 'cranium'")
     end
 

data/features/step_definitions/database_table_steps.rb

@@ -3,13 +3,13 @@ Given(/^a database table called "([^"]*)" with the following fields:$/) do |tabl
 end
 
 
-Given (/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
+Given(/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
   database_table(table_name).clear
   step %Q(the following new rows in the "#{table_name}" database table:), rows
 end
 
 
-Given (/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
+Given(/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
   database_table(table_name).insert rows.data
 end
 

data/features/support/env.rb

@@ -5,8 +5,8 @@ require_relative "../../lib/cranium"
 FileUtils.mkdir_p("log") unless Dir.exists?("log")
 
 Cranium.configure do |config|
-  config.greenplum_connection_string = "postgres://cranium:cranium@#{ENV['DATABASE_HOST'] || '192.168.56.43'}:5432/cranium"
-  config.gpfdist_url = "#{ENV['DATABASE_HOST'] || '192.168.56.43'}:8123"
+  config.greenplum_connection_string = "postgres://cranium:cranium@#{ENV['GREENPLUM_HOST'] || '192.168.56.43'}:5432/cranium"
+  config.gpfdist_url = "#{ ENV['GPFDIST_HOST'] || '192.168.56.43'}:8123"
   config.gpfdist_home_directory = "tmp/custdata"
   config.upload_directory = "cranium_build"
   config.loggers << Logger.new("log/cucumber.log")

data/lib/cranium/data_importer.rb

@@ -17,14 +17,16 @@ class Cranium::DataImporter
   private
 
   def importer_for_definition(import_definition)
-    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, import_definition.truncate_insert].count(true) > 1
-      raise StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, !import_definition.delete_on.empty?, import_definition.truncate_insert].count(true) > 1
+      raise StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
     end
 
     if !import_definition.merge_fields.empty?
       Cranium::ImportStrategy::Merge.new(import_definition)
     elsif !import_definition.delete_insert_on.empty?
       Cranium::ImportStrategy::DeleteInsert.new(import_definition)
+    elsif !import_definition.delete_on.empty?
+      Cranium::ImportStrategy::Delete.new(import_definition)
     elsif import_definition.truncate_insert
       Cranium::ImportStrategy::TruncateInsert.new(import_definition)
     else

data/lib/cranium/database.rb

@@ -13,7 +13,9 @@ module Cranium::Database
 
   def self.[](name)
     @connections ||= {}
-    @connections[name] ||= setup_connection(@definitions[name].connect_to)
+    @connections[name] ||= setup_connection(@definitions[name].connect_to,
+                                            @definitions[name].retry_count,
+                                            @definitions[name].retry_delay)
   end
 
 
@@ -28,15 +30,19 @@ module Cranium::Database
   private
 
 
-  def self.setup_connection(connection_string)
-    connection = if Cranium.configuration.log_queries
-                   Sequel.connect(connection_string, loggers: Cranium.configuration.loggers)
-                 else
-                   Sequel.connect(connection_string)
-                 end
-    connection.extension :connection_validator
-    connection.pool.connection_validation_timeout = -1
-    return connection
+  def self.setup_connection(connection_details, retry_count = 0, retry_delay = 0)
+    (retry_count + 1).times do |try_count|
+      connection = if Cranium.configuration.log_queries
+                     Sequel.connect(connection_details, loggers: Cranium.configuration.loggers)
+                   else
+                     Sequel.connect(connection_details)
+                   end
+      connection.extension :connection_validator
+      connection.pool.connection_validation_timeout = -1
+      break connection
+    rescue Sequel::DatabaseConnectionError
+      (try_count == retry_count) ? raise : sleep(retry_delay)
+    end
   end
 
 end

data/lib/cranium/dsl/database_definition.rb

@@ -7,11 +7,15 @@ class Cranium::DSL::DatabaseDefinition
   attr_reader :name
 
   define_attribute :connect_to
+  define_attribute :retry_count
+  define_attribute :retry_delay
 
 
 
   def initialize(name)
     @name = name
+    @retry_count = 0
+    @retry_delay = 0
   end
 
 

data/lib/cranium/dsl/import_definition.rb

@@ -10,8 +10,10 @@ class Cranium::DSL::ImportDefinition
   attr_reader :merge_fields
 
   define_attribute :into
+  define_attribute :error_threshold
   define_boolean_attribute :truncate_insert
   define_array_attribute :delete_insert_on
+  define_array_attribute :delete_on
 
 
   def initialize(name)

data/lib/cranium/external_table.rb

@@ -1,46 +1,45 @@
 class Cranium::ExternalTable
 
-  def initialize(source, db_connection)
-    @source, @connection = source, db_connection
+  def initialize(source, db_connection, error_threshold: nil)
+    @source = source
+    @connection = db_connection
+    @error_threshold = error_threshold
   end
 
-
-
   def create
-    @connection.run <<-sql
-      CREATE EXTERNAL TABLE "#{name}" (
-          #{field_definitions}
-      )
-      LOCATION (#{external_location})
-      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
-      ENCODING 'UTF8'
-    sql
+    @connection.run external_table_sql
   end
 
-
-
   def destroy
     @connection.run %Q[DROP EXTERNAL TABLE "#{name}"]
   end
 
-
-
   def name
     :"external_#{@source.name}"
   end
 
+  private
 
+  def external_table_sql
+    external_table_sql = <<~sql
+      CREATE EXTERNAL TABLE "#{name}" (
+          #{field_definitions}
+      )
+      LOCATION (#{external_location})
+      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
+      ENCODING 'UTF8'
+    sql
 
-  private
+    external_table_sql << "SEGMENT REJECT LIMIT #{@error_threshold} PERCENT\n" unless @error_threshold.nil?
+    external_table_sql
+  end
 
   def field_definitions
     @source.fields.map do |name, type|
       %Q("#{name}" #{sql_type_for_ruby_type(type)})
-    end.join ",\n          "
+    end.join ",\n    "
   end
 
-
-
   def sql_type_for_ruby_type(type)
     case type.to_s
       when "Integer" then
@@ -58,14 +57,10 @@ class Cranium::ExternalTable
     end
   end
 
-
-
   def quote(text)
     text.gsub "'", "''"
   end
 
-
-
   def external_location
     @source.files.map do |file_name|
       "'gpfdist://#{Cranium.configuration.gpfdist_url}/#{Cranium.configuration.upload_directory}/#{file_name}'"

data/lib/cranium/import_strategy/base.rb

@@ -11,7 +11,7 @@ class Cranium::ImportStrategy::Base
 
 
   def import
-    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection
+    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection, error_threshold: @import_definition.error_threshold
 
     external_table.create
     number_of_items_imported = import_from external_table.name

data/lib/cranium/import_strategy/delete.rb

@@ -0,0 +1,34 @@
+class Cranium::ImportStrategy::Delete < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    @source_table = source_table
+
+    delete_existing_records
+    puts @source_table
+    database[@source_table].count
+  end
+
+
+
+  private
+
+  def delete_existing_records
+    database.
+        from(Sequel.as(target_table, "target"), Sequel.as(@source_table, "source")).
+        where(delete_by_fields.qualify keys_with: :source, values_with: :target).
+        delete
+  end
+
+
+
+  def delete_by_fields
+    Cranium::Sequel::Hash[delete_field_mapping]
+  end
+
+
+
+  def delete_field_mapping
+    import_definition.field_associations.select { |_, target_field| import_definition.delete_on.include? target_field }
+  end
+
+end

data/lib/cranium/import_strategy.rb

@@ -2,6 +2,7 @@ module Cranium::ImportStrategy
 
   autoload :Base, 'cranium/import_strategy/base'
   autoload :DeleteInsert, 'cranium/import_strategy/delete_insert'
+  autoload :Delete, 'cranium/import_strategy/delete'
   autoload :TruncateInsert, 'cranium/import_strategy/truncate_insert'
   autoload :Delta, 'cranium/import_strategy/delta'
   autoload :Merge, 'cranium/import_strategy/merge'

data/repo-info.json

@@ -0,0 +1,7 @@
+{
+  "is_in_production": true,
+  "is_scannable": true,
+  "is_critical": false,
+  "contact": "g-gsuite-smartinsight@emarsys.com",
+  "hosted": null
+}

data/spec/cranium/data_importer_spec.rb

@@ -1,11 +1,12 @@
 require_relative '../spec_helper'
 
 describe Cranium::DataImporter do
+  let(:connection) { double 'a_connection' }
 
   before do
-    connection = double
     allow(Cranium::Database).to receive(:connection).and_return connection
     allow(connection).to receive(:transaction).and_yield
+    allow(Cranium.application).to receive(:apply_hook).with :after_import
   end
 
   let(:importer) { Cranium::DataImporter.new }
@@ -13,12 +14,32 @@ describe Cranium::DataImporter do
 
   describe "#import" do
 
+    context "when called with delete_on strategy" do
+      it "calls Delete strategy" do
+        import_strategy = instance_double Cranium::ImportStrategy::Delete
+        allow(Cranium::ImportStrategy::Delete).to receive(:new).with(definition).and_return import_strategy
+        expect(import_strategy).to receive(:import).and_return 0
+        definition.delete_on :source_id
+
+        importer.import definition
+      end
+    end
+
+    context "when called with both merge and delete_on fields set" do
+      it "should raise an exception" do
+        definition.delete_on :source_id
+        definition.merge_on :another_field
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
+      end
+    end
+
     context "when called with both merge and delete_insert fields set" do
       it "should raise an exception" do
         definition.delete_insert_on :some_field
         definition.merge_on :another_field
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -27,7 +48,7 @@ describe Cranium::DataImporter do
         definition.truncate_insert true
         definition.merge_on :another_field
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -36,7 +57,7 @@ describe Cranium::DataImporter do
         definition.delete_insert_on :some_field
         definition.truncate_insert true
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -46,7 +67,7 @@ describe Cranium::DataImporter do
         definition.merge_on :another_field
         definition.truncate_insert true
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 

data/spec/cranium/database_spec.rb

@@ -81,6 +81,40 @@ describe Cranium::Database do
 
       expect(database[:dwh]).not_to eq database[:dwh2]
     end
+
+    context 'when retry_count is specified' do
+      before do
+        database.register_database :dwh do
+          connect_to "other connection string"
+          retry_count 3
+          retry_delay 15
+        end
+        allow(database).to receive(:sleep)
+      end
+
+      it "should retry connecting to the DB the specified number of times" do
+        call_count = 0
+        allow(Sequel).to receive(:connect) do
+          call_count += 1
+          call_count < 3 ? raise(Sequel::DatabaseConnectionError) : connection
+        end
+
+        expect(database[:dwh]).to eq connection
+      end
+
+      it "should not retry connecting to the DB more than the specified number of times" do
+        allow(Sequel).to receive(:connect).exactly(4).times.and_raise(Sequel::DatabaseConnectionError)
+
+        expect { database[:dwh] }.to raise_error(Sequel::DatabaseConnectionError)
+      end
+
+      it "should wait retry_delay seconds between connection attempts" do
+        allow(Sequel).to receive(:connect).and_raise(Sequel::DatabaseConnectionError)
+        expect(database).to receive(:sleep).with(15).exactly(3).times
+
+        expect { database[:dwh] }.to raise_error(Sequel::DatabaseConnectionError)
+      end
+    end
   end
 
 end

data/spec/cranium/dsl/database_definition_spec.rb

@@ -20,4 +20,38 @@ describe Cranium::DSL::DatabaseDefinition do
     end
   end
 
+
+  describe "#retry_count" do
+    context 'when not set' do
+      it "should return 0 by default" do
+        expect(database.retry_count).to eq(0)
+      end
+    end
+
+    context 'when set' do
+      it "should return the number of retries specified for the database" do
+        database.retry_count 3
+
+        expect(database.retry_count).to eq(3)
+      end
+    end
+  end
+
+
+  describe "#retry_delay" do
+    context 'when not set' do
+      it "should return 0 by default" do
+        expect(database.retry_delay).to eq(0)
+      end
+    end
+
+    context 'when set' do
+      it "should return the number of retries specified for the database" do
+        database.retry_delay 15
+
+        expect(database.retry_delay).to eq(15)
+      end
+    end
+  end
+
 end

data/spec/cranium/dsl/import_definition_spec.rb

@@ -12,6 +12,14 @@ describe Cranium::DSL::ImportDefinition do
     end
   end
 
+  describe "#error_threshold" do
+    it "should set the error threshold to the given percentage" do
+      import.error_threshold 10
+
+      expect(import.error_threshold).to eq 10
+    end
+  end
+
 
   describe "#name" do
     it "should return the name of the import definition" do

data/spec/cranium/external_table_spec.rb

@@ -1,5 +1,6 @@
 require_relative '../spec_helper'
 require 'ostruct'
+require 'date'
 
 describe Cranium::ExternalTable do
 
@@ -19,37 +20,64 @@ describe Cranium::ExternalTable do
       source.escape "'"
     end
   end
-  let(:external_table) { Cranium::ExternalTable.new source, connection }
 
+  subject(:external_table) { Cranium::ExternalTable.new source, connection }
 
   describe "#create" do
-    it "should create an external table from the specified source" do
+    before do
       allow(Cranium).to receive_messages configuration: OpenStruct.new(
-        gpfdist_url: "gpfdist-url",
-        gpfdist_home_directory: "/gpfdist-home",
-        upload_directory: "upload-dir"
+          gpfdist_url: "gpfdist-url",
+          gpfdist_home_directory: "/gpfdist-home",
+          upload_directory: "upload-dir"
       )
 
       allow(source).to receive_messages files: %w(test_products_a.csv test_products_b.csv)
+    end
 
-      expect(connection).to receive(:run).with(<<-sql
-      CREATE EXTERNAL TABLE "external_products" (
-          "text_field" TEXT,
-          "integer_field" INTEGER,
-          "numeric_field" NUMERIC,
-          "date_field" DATE,
-          "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
-          "boolean_field1" BOOLEAN,
-          "boolean_field2" BOOLEAN
-      )
-      LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
-      FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
-      ENCODING 'UTF8'
+    it "should create an external table from the specified source" do
+      expect(connection).to receive(:run).with(<<~sql
+        CREATE EXTERNAL TABLE "external_products" (
+            "text_field" TEXT,
+            "integer_field" INTEGER,
+            "numeric_field" NUMERIC,
+            "date_field" DATE,
+            "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+            "boolean_field1" BOOLEAN,
+            "boolean_field2" BOOLEAN
+        )
+        LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+        FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+        ENCODING 'UTF8'
       sql
       )
 
       external_table.create
     end
+
+    context "with error_threshold argument" do
+      subject(:external_table) { Cranium::ExternalTable.new source, connection, error_threshold: 10 }
+
+      it "should create an external table from the specified source" do
+        expect(connection).to receive(:run).with(<<~sql
+          CREATE EXTERNAL TABLE "external_products" (
+              "text_field" TEXT,
+              "integer_field" INTEGER,
+              "numeric_field" NUMERIC,
+              "date_field" DATE,
+              "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+              "boolean_field1" BOOLEAN,
+              "boolean_field2" BOOLEAN
+          )
+          LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+          FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+          ENCODING 'UTF8'
+          SEGMENT REJECT LIMIT 10 PERCENT
+        sql
+        )
+
+        external_table.create
+      end
+    end
   end
 
 
@@ -67,5 +95,4 @@ describe Cranium::ExternalTable do
       expect(external_table.name).to eq(:external_products)
     end
   end
-
 end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: 0.8.1
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-03 00:00:00.000000000 Z
+date: 2021-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -76,30 +76,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -162,24 +162,30 @@ email:
 - smart-insight-dev@emarsys.com
 executables:
 - cranium
+- run_tests.sh
 extensions: []
 extra_rdoc_files: []
 files:
 - ".env"
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - Vagrantfile
 - bin/cranium
+- bin/run_tests.sh
+- codeship-services.yml
+- codeship-steps.yml
 - config/cucumber.yml
 - cranium.gemspec
 - db/setup.sql
 - db/start.sh
 - docker-compose.yml
+- docker/create_databases.sql
+- docker/initialize_greenplum.sh
 - examples/config.rb
 - examples/deduplication.rb
 - examples/import_csv_with_field_lookup_inserting_new_dimension_keys.rb
@@ -188,6 +194,7 @@ files:
 - features/archive.feature
 - features/extract/incremental_extract.feature
 - features/extract/simple_extract.feature
+- features/import/delete_from_table_based_on_csv.feature
 - features/import/import_csv_to_database_as_delta.feature
 - features/import/import_csv_to_database_with_delete_insert_merging.feature
 - features/import/import_csv_to_database_with_truncate_insert.feature
@@ -247,6 +254,7 @@ files:
 - lib/cranium/file_utils.rb
 - lib/cranium/import_strategy.rb
 - lib/cranium/import_strategy/base.rb
+- lib/cranium/import_strategy/delete.rb
 - lib/cranium/import_strategy/delete_insert.rb
 - lib/cranium/import_strategy/delta.rb
 - lib/cranium/import_strategy/merge.rb
@@ -271,6 +279,7 @@ files:
 - lib/cranium/transformation/sequence.rb
 - lib/cranium/transformation_record.rb
 - rake/test.rake
+- repo-info.json
 - spec/cranium/application_spec.rb
 - spec/cranium/archiver_spec.rb
 - spec/cranium/command_line_options_spec.rb
@@ -318,8 +327,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.2.27
 signing_key: 
 specification_version: 4
 summary: Pure Ruby ETL framework
@@ -327,6 +335,7 @@ test_files:
 - features/archive.feature
 - features/extract/incremental_extract.feature
 - features/extract/simple_extract.feature
+- features/import/delete_from_table_based_on_csv.feature
 - features/import/import_csv_to_database_as_delta.feature
 - features/import/import_csv_to_database_with_delete_insert_merging.feature
 - features/import/import_csv_to_database_with_truncate_insert.feature

data/.ruby-version

@@ -1 +0,0 @@
-2.5.1