cranium 0.4.2 → 0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5b878ec72b1afbd62b6d0c576653c4d5e5b1a0ee
-  data.tar.gz: 41b765f1e0cf25648cf30cee548555b7f0ffaaaf
+SHA256:
+  metadata.gz: 3d6f7888fe46d32c156a48c081e57ddf8942d483b6a42c1b69faa4e0f276a128
+  data.tar.gz: 9f49d912ca7cf12a8d3e90e1e6d00565099f4bf0fe4cee90e29f439c345a86df
 SHA512:
-  metadata.gz: b52aa636e49d51d48fc86397330527fe31c40214a055ef7c405befe1e5b53fe255064259de4857a57e0b8374ae111ce5e66a3d677878b010502f5d8faadc9f3d
-  data.tar.gz: 98d0e5a309f127c50e7910c5510908dfb5e71a6f4f347f804c0118bb21a680bf45fb64b059b30866febb81e827dac8f65ea95a2830ea3826927d47e54f773c68
+  metadata.gz: fb4ed093460aaf4eddde16e0ed1236e34d232ef128047683111459a836c7c544a4c5fe3ad5d05abb6609927affaf286c88f8269695153f21c1513b4c802445ea
+  data.tar.gz: 65730273e55b87a3ba2b35f303f21613af3e7cf9328a8f7612cee60c37c54e606f2d50a3891597275ea084ce4405bc445c3d7277d2f5cb3d31b7be1ae805a740

data/.env

@@ -0,0 +1 @@
+DATABASE_HOST=127.0.0.1

data/.gitignore

@@ -5,6 +5,7 @@
 .yardoc
 .vagrant
 .idea
+.ruby-version
 Gemfile.lock
 InstalledFiles
 _yardoc

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--require spec_helper
+--order random
+--fail-fast

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Zoltan Ormandi
+Copyright (c) 2013-2020 Emarsys
 
 MIT License
 

data/README.md

@@ -18,21 +18,9 @@ Or install it yourself as:
 
 ## Development
 
-start up the db
-
-    docker-compose create && docker-compose start
-    
-find out what's the ip is (in case you're using native docker)
-
-    docker-compose ps 
-    
-(if using docker-machine use the machine's ip)
-setup the DATABASE_HOST enviroment variable to this IP (192.168.64.4 in my case)
-    
-    export DATABASE_HOST=192.168.64.4
-    
-Now, your ready to run the integration tests :)
+start up the db:
 
+    $ docker-compose up -d
 
 ## Contributing
 

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.4.2'
+  spec.version = '0.8'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'pg', '~> 0'
   spec.add_runtime_dependency 'progressbar', '~> 0'
-  spec.add_runtime_dependency 'sequel', '~> 4'
+  spec.add_runtime_dependency 'sequel', '>= 4', '< 6'
   spec.add_runtime_dependency 'slop', '~> 3'
 
   spec.add_development_dependency 'bundler', '~> 1'
@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3'
   spec.add_development_dependency 'ruby-prof', '~> 0'
   spec.add_development_dependency 'cucumber', '~> 1'
+  spec.add_development_dependency 'dotenv', '~> 2.5'
 end

data/features/archive.feature

@@ -2,6 +2,7 @@ Feature: Archive source files
 
   Scenario:
     Given no "/tmp/cranium_archive" directory
+    And no "/tmp/cranium_storage" directory
     And a "products_1.csv" data file containing:
     """
     """
@@ -39,6 +40,8 @@ Feature: Archive source files
     end
 
     archive :products, :contacts
+
+    move :purchases, to: "/tmp/cranium_storage"
     """
     When I execute the definition
     Then the process should exit successfully
@@ -47,3 +50,6 @@ Feature: Archive source files
       | .*contacts.csv   |
       | .*products_1.csv |
       | .*products_2.csv |
+    And the "/tmp/cranium_storage" directory should contain the following files:
+      | filename      |
+      | purchases.csv |

data/features/import/delete_from_table_based_on_csv.feature

@@ -0,0 +1,39 @@
+Feature: Delete rows from table provided by CSV file
+
+  Scenario: Successful delete
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | source_id   | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | source_id (i) |
+      | 1             |
+      | 2             |
+      | 3             |
+      | 4             |
+      | 5             |
+    And a "deleted_contacts_extract.csv" data file containing:
+    """
+    source_id
+    3
+    4
+    """
+    And the following definition:
+    """
+    source :deleted_contacts_extract do
+      field :source_id, String
+    end
+
+    import :deleted_contacts_extract do
+      into :dim_contact
+      put :source_id
+
+      delete_on :source_id
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_contact" table should contain:
+      | source_id   |
+      | 1           |
+      | 2           |
+      | 5           |

data/features/step_definitions/database_table_steps.rb

@@ -3,13 +3,13 @@ Given(/^a database table called "([^"]*)" with the following fields:$/) do |tabl
 end
 
 
-Given (/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
+Given(/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
   database_table(table_name).clear
   step %Q(the following new rows in the "#{table_name}" database table:), rows
 end
 
 
-Given (/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
+Given(/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
   database_table(table_name).insert rows.data
 end
 

data/features/support/env.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'dotenv/load'
 require_relative "../../lib/cranium"
 
 FileUtils.mkdir_p("log") unless Dir.exists?("log")

data/lib/cranium/archiver.rb

@@ -1,36 +1,37 @@
 require 'fileutils'
 
 module Cranium::Archiver
-
-  def self.archive(*files)
-    create_archive_directory
-    archive_files files
-  end
-
-
-
-  def self.remove(*files)
-    files.each do |file_name|
-      FileUtils.rm File.join(Cranium.configuration.upload_path, file_name)
+  class << self
+    def archive(*files)
+      create_directory(Cranium.configuration.archive_directory)
+      archive_datetime = Time.now.strftime("%Y-%m-%d_%Hh%Mm%Ss")
+      move_files_from_upload_directory(files, Cranium.configuration.archive_directory, prefix: "#{archive_datetime}_")
     end
-  end
-
 
+    def remove(*files)
+      files.each do |file_name|
+        FileUtils.rm File.join(Cranium.configuration.upload_path, file_name)
+      end
+    end
 
-  private
-
-  def self.create_archive_directory
-    FileUtils.mkpath Cranium.configuration.archive_directory unless Dir.exists? Cranium.configuration.archive_directory
-  end
+    def move(*files, target_directory:)
+      create_directory(target_directory)
+      move_files_from_upload_directory(files, target_directory)
+    end
 
+    private
 
+    def create_directory(path)
+      FileUtils.mkdir_p(path)
+    end
 
-  def self.archive_files(files)
-    archive_datetime = Time.now.strftime("%Y-%m-%d_%Hh%Mm%Ss")
-    files.each do |file_name|
-      FileUtils.mv File.join(Cranium.configuration.upload_path, file_name),
-                   File.join(Cranium.configuration.archive_directory, "#{archive_datetime}_#{file_name}")
+    def move_files_from_upload_directory(files, target_directory, prefix: "")
+      files.each do |file_name|
+        FileUtils.mv(
+          File.join(Cranium.configuration.upload_path, file_name),
+          File.join(target_directory, "#{prefix}#{file_name}")
+        )
+      end
     end
   end
-
 end

data/lib/cranium/data_importer.rb

@@ -17,14 +17,16 @@ class Cranium::DataImporter
   private
 
   def importer_for_definition(import_definition)
-    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, import_definition.truncate_insert].count(true) > 1
-      raise StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, !import_definition.delete_on.empty?, import_definition.truncate_insert].count(true) > 1
+      raise StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
     end
 
     if !import_definition.merge_fields.empty?
       Cranium::ImportStrategy::Merge.new(import_definition)
     elsif !import_definition.delete_insert_on.empty?
       Cranium::ImportStrategy::DeleteInsert.new(import_definition)
+    elsif !import_definition.delete_on.empty?
+      Cranium::ImportStrategy::Delete.new(import_definition)
     elsif import_definition.truncate_insert
       Cranium::ImportStrategy::TruncateInsert.new(import_definition)
     else

data/lib/cranium/database.rb

@@ -13,7 +13,9 @@ module Cranium::Database
 
   def self.[](name)
     @connections ||= {}
-    @connections[name] ||= setup_connection(@definitions[name].connect_to)
+    @connections[name] ||= setup_connection(@definitions[name].connect_to,
+                                            @definitions[name].retry_count,
+                                            @definitions[name].retry_delay)
   end
 
 
@@ -28,15 +30,19 @@ module Cranium::Database
   private
 
 
-  def self.setup_connection(connection_string)
-    connection = if Cranium.configuration.log_queries
-                   Sequel.connect(connection_string, loggers: Cranium.configuration.loggers)
-                 else
-                   Sequel.connect(connection_string)
-                 end
-    connection.extension :connection_validator
-    connection.pool.connection_validation_timeout = -1
-    return connection
+  def self.setup_connection(connection_details, retry_count = 0, retry_delay = 0)
+    (retry_count + 1).times do |try_count|
+      connection = if Cranium.configuration.log_queries
+                     Sequel.connect(connection_details, loggers: Cranium.configuration.loggers)
+                   else
+                     Sequel.connect(connection_details)
+                   end
+      connection.extension :connection_validator
+      connection.pool.connection_validation_timeout = -1
+      break connection
+    rescue Sequel::DatabaseConnectionError
+      (try_count == retry_count) ? raise : sleep(retry_delay)
+    end
   end
 
 end

data/lib/cranium/dsl.rb

@@ -87,6 +87,14 @@ module Cranium::DSL
 
 
 
+  def move(*sources, to: "")
+    sources.each do |source_name|
+      Cranium::Archiver.move *Cranium.application.sources[source_name].files, target_directory: to
+    end
+  end
+
+
+
   def sequence(name)
     Cranium::Transformation::Sequence.new name
   end

data/lib/cranium/dsl/database_definition.rb

@@ -7,11 +7,15 @@ class Cranium::DSL::DatabaseDefinition
   attr_reader :name
 
   define_attribute :connect_to
+  define_attribute :retry_count
+  define_attribute :retry_delay
 
 
 
   def initialize(name)
     @name = name
+    @retry_count = 0
+    @retry_delay = 0
   end
 
 

data/lib/cranium/dsl/import_definition.rb

@@ -10,8 +10,10 @@ class Cranium::DSL::ImportDefinition
   attr_reader :merge_fields
 
   define_attribute :into
+  define_attribute :error_threshold
   define_boolean_attribute :truncate_insert
   define_array_attribute :delete_insert_on
+  define_array_attribute :delete_on
 
 
   def initialize(name)

data/lib/cranium/external_table.rb

@@ -1,46 +1,45 @@
 class Cranium::ExternalTable
 
-  def initialize(source, db_connection)
-    @source, @connection = source, db_connection
+  def initialize(source, db_connection, error_threshold: nil)
+    @source = source
+    @connection = db_connection
+    @error_threshold = error_threshold
   end
 
-
-
   def create
-    @connection.run <<-sql
-      CREATE EXTERNAL TABLE "#{name}" (
-          #{field_definitions}
-      )
-      LOCATION (#{external_location})
-      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
-      ENCODING 'UTF8'
-    sql
+    @connection.run external_table_sql
   end
 
-
-
   def destroy
     @connection.run %Q[DROP EXTERNAL TABLE "#{name}"]
   end
 
-
-
   def name
     :"external_#{@source.name}"
   end
 
+  private
 
+  def external_table_sql
+    external_table_sql = <<~sql
+      CREATE EXTERNAL TABLE "#{name}" (
+          #{field_definitions}
+      )
+      LOCATION (#{external_location})
+      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
+      ENCODING 'UTF8'
+    sql
 
-  private
+    external_table_sql << "SEGMENT REJECT LIMIT #{@error_threshold} PERCENT\n" unless @error_threshold.nil?
+    external_table_sql
+  end
 
   def field_definitions
     @source.fields.map do |name, type|
       %Q("#{name}" #{sql_type_for_ruby_type(type)})
-    end.join ",\n          "
+    end.join ",\n    "
   end
 
-
-
   def sql_type_for_ruby_type(type)
     case type.to_s
       when "Integer" then
@@ -58,14 +57,10 @@ class Cranium::ExternalTable
     end
   end
 
-
-
   def quote(text)
     text.gsub "'", "''"
   end
 
-
-
   def external_location
     @source.files.map do |file_name|
       "'gpfdist://#{Cranium.configuration.gpfdist_url}/#{Cranium.configuration.upload_directory}/#{file_name}'"

data/lib/cranium/import_strategy.rb

@@ -2,6 +2,7 @@ module Cranium::ImportStrategy
 
   autoload :Base, 'cranium/import_strategy/base'
   autoload :DeleteInsert, 'cranium/import_strategy/delete_insert'
+  autoload :Delete, 'cranium/import_strategy/delete'
   autoload :TruncateInsert, 'cranium/import_strategy/truncate_insert'
   autoload :Delta, 'cranium/import_strategy/delta'
   autoload :Merge, 'cranium/import_strategy/merge'

data/lib/cranium/import_strategy/base.rb

@@ -11,7 +11,7 @@ class Cranium::ImportStrategy::Base
 
 
   def import
-    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection
+    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection, error_threshold: @import_definition.error_threshold
 
     external_table.create
     number_of_items_imported = import_from external_table.name

data/lib/cranium/import_strategy/delete.rb

@@ -0,0 +1,34 @@
+class Cranium::ImportStrategy::Delete < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    @source_table = source_table
+
+    delete_existing_records
+    puts @source_table
+    database[@source_table].count
+  end
+
+
+
+  private
+
+  def delete_existing_records
+    database.
+        from(Sequel.as(target_table, "target"), Sequel.as(@source_table, "source")).
+        where(delete_by_fields.qualify keys_with: :source, values_with: :target).
+        delete
+  end
+
+
+
+  def delete_by_fields
+    Cranium::Sequel::Hash[delete_field_mapping]
+  end
+
+
+
+  def delete_field_mapping
+    import_definition.field_associations.select { |_, target_field| import_definition.delete_on.include? target_field }
+  end
+
+end

data/lib/cranium/test_framework/world.rb

@@ -22,8 +22,9 @@ class Cranium::TestFramework::World
 
 
   def save_definition(definition)
-    config = <<-config_string
+    config = <<~config_string
       require 'logger'
+      require 'date'
 
       Cranium.configure do |config|
         config.greenplum_connection_string = "#{Cranium.configuration.greenplum_connection_string}"
@@ -32,6 +33,7 @@ class Cranium::TestFramework::World
         config.upload_directory = "#{Cranium.configuration.upload_directory}"
         config.loggers << Logger.new("log/application.log")
       end
+
     config_string
 
     upload_directory.save_file DEFINITION_FILE, config + definition

data/spec/cranium/archiver_spec.rb

@@ -1,44 +1,74 @@
-require_relative '../spec_helper'
+RSpec.describe Cranium::Archiver do
+  subject(:archiver) { described_class }
 
-describe Cranium::Archiver do
-
-  before(:each) do
-    allow(Cranium).to receive_messages(configuration: Cranium::Configuration.new.tap do |config|
-      config.gpfdist_home_directory = "gpfdist_home"
-      config.upload_directory = "upload_dir"
-      config.archive_directory = "path/to/archive"
-    end)
+  let(:configuration) do
+    Cranium::Configuration.new.tap do |config|
+      config.gpfdist_home_directory = "tmp"
+      config.upload_directory = "upload_directory"
+      config.archive_directory = "tmp/archive_directory"
+    end
   end
+  let(:file1) { "file.txt" }
+  let(:file2) { "another_file.txt" }
 
+  before do
+    allow(Cranium).to receive_messages(configuration: configuration)
+
+    FileUtils.mkdir_p(configuration.upload_path)
+    FileUtils.touch(File.join(configuration.upload_path, file1))
+    FileUtils.touch(File.join(configuration.upload_path, file2))
+  end
 
   describe ".archive" do
-    it "should create the archive directory if it doesn't exist" do
-      allow(Dir).to receive(:exists?).with("path/to/archive").and_return(false)
+    context "when archive directory does not exist" do
+      before { FileUtils.rm_rf configuration.archive_directory }
 
-      expect(FileUtils).to receive(:mkpath).with "path/to/archive"
+      it "creates the archive directory" do
+        archiver.archive file1, file2
 
-      Cranium::Archiver.archive
+        expect(File.exists?(configuration.archive_directory)).to eq true
+      end
     end
 
-    it "should move files to the archive directory" do
-      allow(Dir).to receive(:exists?).with("path/to/archive").and_return(true)
-      allow(Time).to receive(:now).and_return Time.new(2000, 1, 1, 1, 2, 3)
+    context "when there are some file in the upload directory" do
+      it "moves files to the archive directory" do
+        archiver.archive file1, file2
 
-      expect(FileUtils).to receive(:mv).with "gpfdist_home/upload_dir/file.txt", "path/to/archive/2000-01-01_01h02m03s_file.txt"
-      expect(FileUtils).to receive(:mv).with "gpfdist_home/upload_dir/another_file.txt", "path/to/archive/2000-01-01_01h02m03s_another_file.txt"
-
-      Cranium::Archiver.archive "file.txt", "another_file.txt"
+        expect(File.exist?(File.join(configuration.upload_path, file1))).to eq false
+        expect(File.exist?(File.join(configuration.upload_path, file2))).to eq false
+        expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file1}")))).to eq true
+        expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file2}")))).to eq true
+      end
     end
   end
 
-
   describe ".remove" do
-    it "should remove files from the upload directory" do
-      expect(FileUtils).to receive(:rm).with "gpfdist_home/upload_dir/file.txt"
-      expect(FileUtils).to receive(:rm).with "gpfdist_home/upload_dir/another_file.txt"
+    before { FileUtils.mkdir_p configuration.archive_directory }
+
+    it "removes files from the upload directory" do
+      archiver.remove file1, file2
 
-      Cranium::Archiver.remove "file.txt", "another_file.txt"
+      expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file1}")))).to eq true
+      expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file2}")))).to eq true
     end
   end
 
+  describe ".move" do
+    let(:target_directory) { "tmp/target_directory" }
+
+    it "creates given directory if it does not exist" do
+      archiver.move(file1, file2, target_directory: target_directory)
+
+      expect(File.exists?(target_directory)).to eq true
+    end
+
+    it "moves files from upload directory into a given directory" do
+      archiver.move(file1, file2, target_directory: target_directory)
+
+      expect(File.exist?(File.join(configuration.upload_path, file1))).to eq false
+      expect(File.exist?(File.join(configuration.upload_path, file2))).to eq false
+      expect(File.exist?(File.join(target_directory, file1))).to eq true
+      expect(File.exist?(File.join(target_directory, file2))).to eq true
+    end
+  end
 end

data/spec/cranium/data_importer_spec.rb

@@ -1,11 +1,12 @@
 require_relative '../spec_helper'
 
 describe Cranium::DataImporter do
+  let(:connection) { double 'a_connection' }
 
   before do
-    connection = double
     allow(Cranium::Database).to receive(:connection).and_return connection
     allow(connection).to receive(:transaction).and_yield
+    allow(Cranium.application).to receive(:apply_hook).with :after_import
   end
 
   let(:importer) { Cranium::DataImporter.new }
@@ -13,12 +14,32 @@ describe Cranium::DataImporter do
 
   describe "#import" do
 
+    context "when called with delete_on strategy" do
+      it "calls Delete strategy" do
+        import_strategy = instance_double Cranium::ImportStrategy::Delete
+        allow(Cranium::ImportStrategy::Delete).to receive(:new).with(definition).and_return import_strategy
+        expect(import_strategy).to receive(:import).and_return 0
+        definition.delete_on :source_id
+
+        importer.import definition
+      end
+    end
+
+    context "when called with both merge and delete_on fields set" do
+      it "should raise an exception" do
+        definition.delete_on :source_id
+        definition.merge_on :another_field
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
+      end
+    end
+
     context "when called with both merge and delete_insert fields set" do
       it "should raise an exception" do
         definition.delete_insert_on :some_field
         definition.merge_on :another_field
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -27,7 +48,7 @@ describe Cranium::DataImporter do
         definition.truncate_insert true
         definition.merge_on :another_field
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -36,7 +57,7 @@ describe Cranium::DataImporter do
         definition.delete_insert_on :some_field
         definition.truncate_insert true
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -46,7 +67,7 @@ describe Cranium::DataImporter do
         definition.merge_on :another_field
         definition.truncate_insert true
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 

data/spec/cranium/database_spec.rb

@@ -81,6 +81,40 @@ describe Cranium::Database do
 
       expect(database[:dwh]).not_to eq database[:dwh2]
     end
+
+    context 'when retry_count is specified' do
+      before do
+        database.register_database :dwh do
+          connect_to "other connection string"
+          retry_count 3
+          retry_delay 15
+        end
+        allow(database).to receive(:sleep)
+      end
+
+      it "should retry connecting to the DB the specified number of times" do
+        call_count = 0
+        allow(Sequel).to receive(:connect) do
+          call_count += 1
+          call_count < 3 ? raise(Sequel::DatabaseConnectionError) : connection
+        end
+
+        expect(database[:dwh]).to eq connection
+      end
+
+      it "should not retry connecting to the DB more than the specified number of times" do
+        allow(Sequel).to receive(:connect).exactly(4).times.and_raise(Sequel::DatabaseConnectionError)
+
+        expect { database[:dwh] }.to raise_error(Sequel::DatabaseConnectionError)
+      end
+
+      it "should wait retry_delay seconds between connection attempts" do
+        allow(Sequel).to receive(:connect).and_raise(Sequel::DatabaseConnectionError)
+        expect(database).to receive(:sleep).with(15).exactly(3).times
+
+        expect { database[:dwh] }.to raise_error(Sequel::DatabaseConnectionError)
+      end
+    end
   end
 
 end

data/spec/cranium/dsl/database_definition_spec.rb

@@ -20,4 +20,38 @@ describe Cranium::DSL::DatabaseDefinition do
     end
   end
 
+
+  describe "#retry_count" do
+    context 'when not set' do
+      it "should return 0 by default" do
+        expect(database.retry_count).to eq(0)
+      end
+    end
+
+    context 'when set' do
+      it "should return the number of retries specified for the database" do
+        database.retry_count 3
+
+        expect(database.retry_count).to eq(3)
+      end
+    end
+  end
+
+
+  describe "#retry_delay" do
+    context 'when not set' do
+      it "should return 0 by default" do
+        expect(database.retry_delay).to eq(0)
+      end
+    end
+
+    context 'when set' do
+      it "should return the number of retries specified for the database" do
+        database.retry_delay 15
+
+        expect(database.retry_delay).to eq(15)
+      end
+    end
+  end
+
 end

data/spec/cranium/dsl/import_definition_spec.rb

@@ -12,6 +12,14 @@ describe Cranium::DSL::ImportDefinition do
     end
   end
 
+  describe "#error_threshold" do
+    it "should set the error threshold to the given percentage" do
+      import.error_threshold 10
+
+      expect(import.error_threshold).to eq 10
+    end
+  end
+
 
   describe "#name" do
     it "should return the name of the import definition" do

data/spec/cranium/dsl/source_definition_spec.rb

@@ -38,11 +38,11 @@ describe Cranium::DSL::SourceDefinition do
 
     it "should return the fields and types that were set" do
       source.field :field1, String
-      source.field :field2, Fixnum
+      source.field :field2, Integer
 
       expect(source.fields).to eq({
         field1: String,
-        field2: Fixnum
+        field2: Integer
       })
     end
   end

data/spec/cranium/dsl_spec.rb

@@ -97,6 +97,22 @@ describe Cranium::DSL do
   end
 
 
+  describe "#move" do
+    let(:target_directory) { "/tmp/target" }
+
+    it "should move files for the specified sources" do
+      allow(Cranium.application).to receive_messages sources: {first_source: double(files: ["file1", "file2"]),
+                                                               second_source: double(files: ["file3"]),
+                                                               third_source: double(files: ["file4"])}
+
+      expect(Cranium::Archiver).to receive(:move).with "file1", "file2", target_directory: target_directory
+      expect(Cranium::Archiver).to receive(:move).with "file3", target_directory: target_directory
+
+      dsl_object.move :first_source, :second_source, to: target_directory
+    end
+  end
+
+
   describe "#sequence" do
     it "should return a sequence with the specified name" do
       result = dsl_object.sequence "test_sequence"

data/spec/cranium/external_table_spec.rb

@@ -1,5 +1,6 @@
 require_relative '../spec_helper'
 require 'ostruct'
+require 'date'
 
 describe Cranium::ExternalTable do
 
@@ -19,37 +20,64 @@ describe Cranium::ExternalTable do
       source.escape "'"
     end
   end
-  let(:external_table) { Cranium::ExternalTable.new source, connection }
 
+  subject(:external_table) { Cranium::ExternalTable.new source, connection }
 
   describe "#create" do
-    it "should create an external table from the specified source" do
+    before do
       allow(Cranium).to receive_messages configuration: OpenStruct.new(
-        gpfdist_url: "gpfdist-url",
-        gpfdist_home_directory: "/gpfdist-home",
-        upload_directory: "upload-dir"
+          gpfdist_url: "gpfdist-url",
+          gpfdist_home_directory: "/gpfdist-home",
+          upload_directory: "upload-dir"
       )
 
       allow(source).to receive_messages files: %w(test_products_a.csv test_products_b.csv)
+    end
 
-      expect(connection).to receive(:run).with(<<-sql
-      CREATE EXTERNAL TABLE "external_products" (
-          "text_field" TEXT,
-          "integer_field" INTEGER,
-          "numeric_field" NUMERIC,
-          "date_field" DATE,
-          "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
-          "boolean_field1" BOOLEAN,
-          "boolean_field2" BOOLEAN
-      )
-      LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
-      FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
-      ENCODING 'UTF8'
+    it "should create an external table from the specified source" do
+      expect(connection).to receive(:run).with(<<~sql
+        CREATE EXTERNAL TABLE "external_products" (
+            "text_field" TEXT,
+            "integer_field" INTEGER,
+            "numeric_field" NUMERIC,
+            "date_field" DATE,
+            "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+            "boolean_field1" BOOLEAN,
+            "boolean_field2" BOOLEAN
+        )
+        LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+        FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+        ENCODING 'UTF8'
       sql
       )
 
       external_table.create
     end
+
+    context "with error_threshold argument" do
+      subject(:external_table) { Cranium::ExternalTable.new source, connection, error_threshold: 10 }
+
+      it "should create an external table from the specified source" do
+        expect(connection).to receive(:run).with(<<~sql
+          CREATE EXTERNAL TABLE "external_products" (
+              "text_field" TEXT,
+              "integer_field" INTEGER,
+              "numeric_field" NUMERIC,
+              "date_field" DATE,
+              "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+              "boolean_field1" BOOLEAN,
+              "boolean_field2" BOOLEAN
+          )
+          LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+          FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+          ENCODING 'UTF8'
+          SEGMENT REJECT LIMIT 10 PERCENT
+        sql
+        )
+
+        external_table.create
+      end
+    end
   end
 
 
@@ -67,5 +95,4 @@ describe Cranium::ExternalTable do
       expect(external_table.name).to eq(:external_products)
     end
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: '0.8'
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-01 00:00:00.000000000 Z
+date: 2020-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -42,16 +42,22 @@ dependencies:
   name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +142,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 description: Provides Extract, Transform and Load functionality for loading data from
   CSV files to a Greenplum database.
 email:
@@ -145,8 +165,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env"
 - ".gitignore"
-- ".ruby-version"
+- ".rspec"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -166,6 +187,7 @@ files:
 - features/archive.feature
 - features/extract/incremental_extract.feature
 - features/extract/simple_extract.feature
+- features/import/delete_from_table_based_on_csv.feature
 - features/import/import_csv_to_database_as_delta.feature
 - features/import/import_csv_to_database_with_delete_insert_merging.feature
 - features/import/import_csv_to_database_with_truncate_insert.feature
@@ -225,6 +247,7 @@ files:
 - lib/cranium/file_utils.rb
 - lib/cranium/import_strategy.rb
 - lib/cranium/import_strategy/base.rb
+- lib/cranium/import_strategy/delete.rb
 - lib/cranium/import_strategy/delete_insert.rb
 - lib/cranium/import_strategy/delta.rb
 - lib/cranium/import_strategy/merge.rb
@@ -296,8 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Pure Ruby ETL framework
@@ -305,6 +327,7 @@ test_files:
 - features/archive.feature
 - features/extract/incremental_extract.feature
 - features/extract/simple_extract.feature
+- features/import/delete_from_table_based_on_csv.feature
 - features/import/import_csv_to_database_as_delta.feature
 - features/import/import_csv_to_database_with_delete_insert_merging.feature
 - features/import/import_csv_to_database_with_truncate_insert.feature

data/.ruby-version

@@ -1 +0,0 @@
-2.3.0