cranium 0.5 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63bb4574d9b494202c0aff700e4263c1b14753b0b973b9dcb52148ed24a7a861
-  data.tar.gz: ee3238eee62a1c6ee3cd9d3c69ef7ab19a42133996b5fa6334d1a56a9f2f5ff0
+  metadata.gz: cf60640829ad3e3977a453a18b4a91c01538ecfe79fb8759b3d978348e10cb8c
+  data.tar.gz: e297e5c85dd7f0c6296aa1e9ce01dc112d43b08e7087e34f266ff856a0e41ac9
 SHA512:
-  metadata.gz: 2351552d7be2849c5698df99b977c3c56d54ef376d351125bdb51526b6b826d2f4afecc79292479d0d8308af7edd086fd47213479e693c18ecfe9bc31d40549a
-  data.tar.gz: a091064e7f4fa3a56a741b4704012ab1e8a5ced7b6dcad0b6826d226dfd06c5520dddfc1a8e8eabbf04dcd7585d7f1bb97801b539f67a3fdb7fbdd063713be4d
+  metadata.gz: c47d42ecea886eb75e21b471c9e7b5cdc810d020174e43a76b5554cd3f4f6a71d2517bde5e041a544167ec461d74e2c0945459ac7b3223bf6793ef9fe4d0192d
+  data.tar.gz: 9dbc134d24bd752b0adfe664d2a0187fb07a8c3e8d0a3d956b5949a7135dbe1c147e158bd6c5f54eca9580f0f424fd56d68b88a032e8352c7709c6ade6c673da

data/.gitignore

@@ -5,6 +5,7 @@
 .yardoc
 .vagrant
 .idea
+.ruby-version
 Gemfile.lock
 InstalledFiles
 _yardoc

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.5'
+  spec.version = '0.6.1'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}

data/features/step_definitions/database_table_steps.rb

@@ -3,13 +3,13 @@ Given(/^a database table called "([^"]*)" with the following fields:$/) do |tabl
 end
 
 
-Given (/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
+Given(/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
   database_table(table_name).clear
   step %Q(the following new rows in the "#{table_name}" database table:), rows
 end
 
 
-Given (/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
+Given(/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
   database_table(table_name).insert rows.data
 end
 

data/lib/cranium/dsl/import_definition.rb

@@ -10,6 +10,7 @@ class Cranium::DSL::ImportDefinition
   attr_reader :merge_fields
 
   define_attribute :into
+  define_attribute :error_threshold
   define_boolean_attribute :truncate_insert
   define_array_attribute :delete_insert_on
 

data/lib/cranium/external_table.rb

@@ -1,46 +1,45 @@
 class Cranium::ExternalTable
 
-  def initialize(source, db_connection)
-    @source, @connection = source, db_connection
+  def initialize(source, db_connection, error_threshold: nil)
+    @source = source
+    @connection = db_connection
+    @error_threshold = error_threshold
   end
 
-
-
   def create
-    @connection.run <<-sql
-      CREATE EXTERNAL TABLE "#{name}" (
-          #{field_definitions}
-      )
-      LOCATION (#{external_location})
-      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
-      ENCODING 'UTF8'
-    sql
+    @connection.run external_table_sql
   end
 
-
-
   def destroy
     @connection.run %Q[DROP EXTERNAL TABLE "#{name}"]
   end
 
-
-
   def name
     :"external_#{@source.name}"
   end
 
+  private
 
+  def external_table_sql
+    external_table_sql = <<~sql
+      CREATE EXTERNAL TABLE "#{name}" (
+          #{field_definitions}
+      )
+      LOCATION (#{external_location})
+      FORMAT 'CSV' (DELIMITER '#{quote @source.delimiter}' ESCAPE '#{quote @source.escape}' QUOTE '#{quote @source.quote}' HEADER)
+      ENCODING 'UTF8'
+    sql
 
-  private
+    external_table_sql << "SEGMENT REJECT LIMIT #{@error_threshold} PERCENT\n" unless @error_threshold.nil?
+    external_table_sql
+  end
 
   def field_definitions
     @source.fields.map do |name, type|
       %Q("#{name}" #{sql_type_for_ruby_type(type)})
-    end.join ",\n          "
+    end.join ",\n    "
   end
 
-
-
   def sql_type_for_ruby_type(type)
     case type.to_s
       when "Integer" then
@@ -58,14 +57,10 @@ class Cranium::ExternalTable
     end
   end
 
-
-
   def quote(text)
     text.gsub "'", "''"
   end
 
-
-
   def external_location
     @source.files.map do |file_name|
       "'gpfdist://#{Cranium.configuration.gpfdist_url}/#{Cranium.configuration.upload_directory}/#{file_name}'"

data/lib/cranium/import_strategy/base.rb

@@ -11,7 +11,7 @@ class Cranium::ImportStrategy::Base
 
 
   def import
-    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection
+    external_table = Cranium::ExternalTable.new Cranium.application.sources[import_definition.name], Cranium::Database.connection, error_threshold: @import_definition.error_threshold
 
     external_table.create
     number_of_items_imported = import_from external_table.name

data/spec/cranium/dsl/import_definition_spec.rb

@@ -12,6 +12,14 @@ describe Cranium::DSL::ImportDefinition do
     end
   end
 
+  describe "#error_threshold" do
+    it "should set the error threshold to the given percentage" do
+      import.error_threshold 10
+
+      expect(import.error_threshold).to eq 10
+    end
+  end
+
 
   describe "#name" do
     it "should return the name of the import definition" do

data/spec/cranium/external_table_spec.rb

@@ -1,5 +1,6 @@
 require_relative '../spec_helper'
 require 'ostruct'
+require 'date'
 
 describe Cranium::ExternalTable do
 
@@ -19,37 +20,64 @@ describe Cranium::ExternalTable do
       source.escape "'"
     end
   end
-  let(:external_table) { Cranium::ExternalTable.new source, connection }
 
+  subject(:external_table) { Cranium::ExternalTable.new source, connection }
 
   describe "#create" do
-    it "should create an external table from the specified source" do
+    before do
       allow(Cranium).to receive_messages configuration: OpenStruct.new(
-        gpfdist_url: "gpfdist-url",
-        gpfdist_home_directory: "/gpfdist-home",
-        upload_directory: "upload-dir"
+          gpfdist_url: "gpfdist-url",
+          gpfdist_home_directory: "/gpfdist-home",
+          upload_directory: "upload-dir"
       )
 
       allow(source).to receive_messages files: %w(test_products_a.csv test_products_b.csv)
+    end
 
-      expect(connection).to receive(:run).with(<<-sql
-      CREATE EXTERNAL TABLE "external_products" (
-          "text_field" TEXT,
-          "integer_field" INTEGER,
-          "numeric_field" NUMERIC,
-          "date_field" DATE,
-          "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
-          "boolean_field1" BOOLEAN,
-          "boolean_field2" BOOLEAN
-      )
-      LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
-      FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
-      ENCODING 'UTF8'
+    it "should create an external table from the specified source" do
+      expect(connection).to receive(:run).with(<<~sql
+        CREATE EXTERNAL TABLE "external_products" (
+            "text_field" TEXT,
+            "integer_field" INTEGER,
+            "numeric_field" NUMERIC,
+            "date_field" DATE,
+            "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+            "boolean_field1" BOOLEAN,
+            "boolean_field2" BOOLEAN
+        )
+        LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+        FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+        ENCODING 'UTF8'
       sql
       )
 
       external_table.create
     end
+
+    context "with error_threshold argument" do
+      subject(:external_table) { Cranium::ExternalTable.new source, connection, error_threshold: 10 }
+
+      it "should create an external table from the specified source" do
+        expect(connection).to receive(:run).with(<<~sql
+          CREATE EXTERNAL TABLE "external_products" (
+              "text_field" TEXT,
+              "integer_field" INTEGER,
+              "numeric_field" NUMERIC,
+              "date_field" DATE,
+              "timestamp_field" TIMESTAMP WITHOUT TIME ZONE,
+              "boolean_field1" BOOLEAN,
+              "boolean_field2" BOOLEAN
+          )
+          LOCATION ('gpfdist://gpfdist-url/upload-dir/test_products_a.csv', 'gpfdist://gpfdist-url/upload-dir/test_products_b.csv')
+          FORMAT 'CSV' (DELIMITER ';' ESCAPE '''' QUOTE '"' HEADER)
+          ENCODING 'UTF8'
+          SEGMENT REJECT LIMIT 10 PERCENT
+        sql
+        )
+
+        external_table.create
+      end
+    end
   end
 
 
@@ -67,5 +95,4 @@ describe Cranium::ExternalTable do
       expect(external_table.name).to eq(:external_products)
     end
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: 0.6.1
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-03 00:00:00.000000000 Z
+date: 2019-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -168,7 +168,6 @@ files:
 - ".env"
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -318,8 +317,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Pure Ruby ETL framework

data/.ruby-version

@@ -1 +0,0 @@
-2.5.1