cranium 0.7 → 0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b61de113471ffdd1d3b83dccd8076faee6951352456022b1eda76680cc6d52c4
-  data.tar.gz: 10e4a762604b5d985e8675cde783e3165dc4d94da41fe017c05e6b4a8c514b83
+  metadata.gz: 3d6f7888fe46d32c156a48c081e57ddf8942d483b6a42c1b69faa4e0f276a128
+  data.tar.gz: 9f49d912ca7cf12a8d3e90e1e6d00565099f4bf0fe4cee90e29f439c345a86df
 SHA512:
-  metadata.gz: 2d1ea26710cf8585f6c081a0f7c5da88213209685ed73fd8deacacef03e17a4423ba47f1974987e3528fbae50e2ce15db1eca521dbf1e30c73ebf4639cbe53bc
-  data.tar.gz: 3b799dc7963a54cd38b7d46b8313a3a76e6350894f6f6162dcd56387a884e36c601573e94deae2ad4d7e5ac31194e36eb3669437adfec45edf13e7585fb8fedf
+  metadata.gz: fb4ed093460aaf4eddde16e0ed1236e34d232ef128047683111459a836c7c544a4c5fe3ad5d05abb6609927affaf286c88f8269695153f21c1513b4c802445ea
+  data.tar.gz: 65730273e55b87a3ba2b35f303f21613af3e7cf9328a8f7612cee60c37c54e606f2d50a3891597275ea084ce4405bc445c3d7277d2f5cb3d31b7be1ae805a740

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Zoltan Ormandi
+Copyright (c) 2013-2020 Emarsys
 
 MIT License
 

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.7'
+  spec.version = '0.8'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}

data/features/import/delete_from_table_based_on_csv.feature

@@ -0,0 +1,39 @@
+Feature: Delete rows from table provided by CSV file
+
+  Scenario: Successful delete
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | source_id   | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | source_id (i) |
+      | 1             |
+      | 2             |
+      | 3             |
+      | 4             |
+      | 5             |
+    And a "deleted_contacts_extract.csv" data file containing:
+    """
+    source_id
+    3
+    4
+    """
+    And the following definition:
+    """
+    source :deleted_contacts_extract do
+      field :source_id, String
+    end
+
+    import :deleted_contacts_extract do
+      into :dim_contact
+      put :source_id
+
+      delete_on :source_id
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_contact" table should contain:
+      | source_id   |
+      | 1           |
+      | 2           |
+      | 5           |

data/lib/cranium/data_importer.rb

@@ -17,14 +17,16 @@ class Cranium::DataImporter
   private
 
   def importer_for_definition(import_definition)
-    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, import_definition.truncate_insert].count(true) > 1
-      raise StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+    if [!import_definition.merge_fields.empty?, !import_definition.delete_insert_on.empty?, !import_definition.delete_on.empty?, import_definition.truncate_insert].count(true) > 1
+      raise StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
     end
 
     if !import_definition.merge_fields.empty?
       Cranium::ImportStrategy::Merge.new(import_definition)
     elsif !import_definition.delete_insert_on.empty?
       Cranium::ImportStrategy::DeleteInsert.new(import_definition)
+    elsif !import_definition.delete_on.empty?
+      Cranium::ImportStrategy::Delete.new(import_definition)
     elsif import_definition.truncate_insert
       Cranium::ImportStrategy::TruncateInsert.new(import_definition)
     else

data/lib/cranium/dsl/import_definition.rb

@@ -13,6 +13,7 @@ class Cranium::DSL::ImportDefinition
   define_attribute :error_threshold
   define_boolean_attribute :truncate_insert
   define_array_attribute :delete_insert_on
+  define_array_attribute :delete_on
 
 
   def initialize(name)

data/lib/cranium/import_strategy.rb

@@ -2,6 +2,7 @@ module Cranium::ImportStrategy
 
   autoload :Base, 'cranium/import_strategy/base'
   autoload :DeleteInsert, 'cranium/import_strategy/delete_insert'
+  autoload :Delete, 'cranium/import_strategy/delete'
   autoload :TruncateInsert, 'cranium/import_strategy/truncate_insert'
   autoload :Delta, 'cranium/import_strategy/delta'
   autoload :Merge, 'cranium/import_strategy/merge'

data/lib/cranium/import_strategy/delete.rb

@@ -0,0 +1,34 @@
+class Cranium::ImportStrategy::Delete < Cranium::ImportStrategy::Base
+
+  def import_from(source_table)
+    @source_table = source_table
+
+    delete_existing_records
+    puts @source_table
+    database[@source_table].count
+  end
+
+
+
+  private
+
+  def delete_existing_records
+    database.
+        from(Sequel.as(target_table, "target"), Sequel.as(@source_table, "source")).
+        where(delete_by_fields.qualify keys_with: :source, values_with: :target).
+        delete
+  end
+
+
+
+  def delete_by_fields
+    Cranium::Sequel::Hash[delete_field_mapping]
+  end
+
+
+
+  def delete_field_mapping
+    import_definition.field_associations.select { |_, target_field| import_definition.delete_on.include? target_field }
+  end
+
+end

data/spec/cranium/data_importer_spec.rb

@@ -1,11 +1,12 @@
 require_relative '../spec_helper'
 
 describe Cranium::DataImporter do
+  let(:connection) { double 'a_connection' }
 
   before do
-    connection = double
     allow(Cranium::Database).to receive(:connection).and_return connection
     allow(connection).to receive(:transaction).and_yield
+    allow(Cranium.application).to receive(:apply_hook).with :after_import
   end
 
   let(:importer) { Cranium::DataImporter.new }
@@ -13,12 +14,32 @@ describe Cranium::DataImporter do
 
   describe "#import" do
 
+    context "when called with delete_on strategy" do
+      it "calls Delete strategy" do
+        import_strategy = instance_double Cranium::ImportStrategy::Delete
+        allow(Cranium::ImportStrategy::Delete).to receive(:new).with(definition).and_return import_strategy
+        expect(import_strategy).to receive(:import).and_return 0
+        definition.delete_on :source_id
+
+        importer.import definition
+      end
+    end
+
+    context "when called with both merge and delete_on fields set" do
+      it "should raise an exception" do
+        definition.delete_on :source_id
+        definition.merge_on :another_field
+
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
+      end
+    end
+
     context "when called with both merge and delete_insert fields set" do
       it "should raise an exception" do
         definition.delete_insert_on :some_field
         definition.merge_on :another_field
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -27,7 +48,7 @@ describe Cranium::DataImporter do
         definition.truncate_insert true
         definition.merge_on :another_field
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -36,7 +57,7 @@ describe Cranium::DataImporter do
         definition.delete_insert_on :some_field
         definition.truncate_insert true
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 
@@ -46,7 +67,7 @@ describe Cranium::DataImporter do
         definition.merge_on :another_field
         definition.truncate_insert true
 
-        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on and truncate_insert settings"
+        expect { importer.import(definition) }.to raise_error StandardError, "Import should not combine merge_on, delete_insert_on, delete_on and truncate_insert settings"
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: '0.7'
+  version: '0.8'
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-15 00:00:00.000000000 Z
+date: 2020-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -187,6 +187,7 @@ files:
 - features/archive.feature
 - features/extract/incremental_extract.feature
 - features/extract/simple_extract.feature
+- features/import/delete_from_table_based_on_csv.feature
 - features/import/import_csv_to_database_as_delta.feature
 - features/import/import_csv_to_database_with_delete_insert_merging.feature
 - features/import/import_csv_to_database_with_truncate_insert.feature
@@ -246,6 +247,7 @@ files:
 - lib/cranium/file_utils.rb
 - lib/cranium/import_strategy.rb
 - lib/cranium/import_strategy/base.rb
+- lib/cranium/import_strategy/delete.rb
 - lib/cranium/import_strategy/delete_insert.rb
 - lib/cranium/import_strategy/delta.rb
 - lib/cranium/import_strategy/merge.rb
@@ -317,7 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Pure Ruby ETL framework
@@ -325,6 +327,7 @@ test_files:
 - features/archive.feature
 - features/extract/incremental_extract.feature
 - features/extract/simple_extract.feature
+- features/import/delete_from_table_based_on_csv.feature
 - features/import/import_csv_to_database_as_delta.feature
 - features/import/import_csv_to_database_with_delete_insert_merging.feature
 - features/import/import_csv_to_database_with_truncate_insert.feature