cranium 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ded6526a7ab3b7875f8561df207318f256eda9ee
-  data.tar.gz: 13d3b0f04ed8d2a69f2ff21250ab48abf226368d
+  metadata.gz: 95d417150867a455805bdc4b21372050ee81f8cc
+  data.tar.gz: 6a1ee615b84a1a6eefd83e4590d947e408fb1244
 SHA512:
-  metadata.gz: 4e7e5574b8a0f51ebcd8bd2b3fcfdf6c9d2b8ece9fbc361022cc90bf7183fb00f9bc4c2a5eca3ca6c390763d429c0bf5b358a8fc58368efb49ce3f4965c0d795
-  data.tar.gz: 4f7a839e74dc68656d2b1754ca20ffa936241b1954f38fc841928fec471ea065ca83e00d2c5069bb313d00eaed4080a8c78e4cf350f410bc9bafa964fc2c6778
+  metadata.gz: 6ee2e4a162601873d76cc96b1d88b02b4e6348019f04a899d68ba6c1723a9d1f678f51314bdc748e30357f7d04d8026330c1d428761c7c65506d46483bc45e3e
+  data.tar.gz: 114072465d737d6d8fd18ea884d7ed3c1f5b26bfc4dbdf3c2da2e6c6fb0994d5922b75502de2c0cbe75bb60042d3a1c90f788258403a580098175e9e7f108c23

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.2.1'
+  spec.version = '0.3.0'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}

data/features/import/import_csv_with_always_inserting_new_dimension_keys.feature

@@ -62,7 +62,7 @@ Feature: Import a CSV file into the database with new dimension values always in
 
 
   Scenario: Example use case for the insert
-    If purchases made by a predefined contact identifier (NA in this case) do not look for it insert .
+    If purchases made by a predefined contact identifier (NA in this case) do not look for it insert.
     Otherwise use lookup to find or create that contact
 
     Given a database table called "dim_contact" with the following fields:
@@ -135,3 +135,64 @@ Feature: Import a CSV file into the database with new dimension values always in
       | 11              | NA      | Unknown contact NA |
       | 12              | NA      | Unknown contact NA |
       | 13              | 2       | Unknown contact 2  |
+
+
+  Scenario: Successful import with a large number of users (contacts) to insert
+    Given a database table called "dim_contact" with the following fields:
+      | field_name   | field_type |
+      | contact_key  | SERIAL     |
+      | user_id      | TEXT       |
+      | name         | TEXT       |
+      | load_id      | NUMERIC    |
+      | load_date    | DATE       |
+      | is_generated | BOOLEAN    |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key (i) | user_id | name |
+      | 10              | 1       | Alma |
+    And the current value in sequence "dim_contact_contact_key_seq" is 10
+    And a database table called "fct_purchases" with the following fields:
+      | field_name  | field_type |
+      | contact_key | INTEGER    |
+      | amount      | TEXT       |
+    And a 500_000 lines long "purchases.csv" data file containing rows like:
+    """
+    user_id,amount
+    NA,100
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key, Integer
+      field :amount, String
+    end
+
+    now = Time.now
+    transform :purchases => :transformed_purchases do |record|
+      record[:contact_key] = insert :contact_key,
+                                    table: :dim_contact,
+                                    record: {
+                                      load_id: 1,
+                                      load_date: now,
+                                      contact_key: next_value_in_sequence("dim_contact_contact_key_seq"),
+                                      user_id: record[:user_id],
+                                      name: "Unknown contact #{record[:user_id]}",
+                                      is_generated: true
+                                    }
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key
+      put :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain 500_000 purchases
+    And the "dim_contact" table should contain 500_001 contacts

data/features/step_definitions/database_table_steps.rb

@@ -38,3 +38,8 @@ Then(/^the "([^"]*)" table should contain:$/) do |table_name, data|
 
   expect(database_table(table_name).content(data.fields)).to match_array expected_data
 end
+
+
+Then(/^the "([^"]*)" table should contain ([\d_]+) .+$/) do |table_name, count|
+  expect(database_table(table_name).count).to eq count.to_i
+end

data/features/step_definitions/file_steps.rb

@@ -13,6 +13,13 @@ Given /^an? "([^"]*)" data file containing:$/ do |file_name, content|
 end
 
 
+Given /^an? ([\d_]+) lines long "([^"]*)" data file containing rows like:$/ do |lines_count, file_name, content|
+  lines = content.split("\n")
+
+  upload_directory.save_file file_name, "#{lines.first}\n" + "#{lines.last}\n" * lines_count.to_i
+end
+
+
 Given /^the "([^"]*)" file is deleted$/ do |file_name|
   upload_directory.delete_file file_name
 end

data/lib/cranium/dimension_manager.rb

@@ -36,7 +36,7 @@ class Cranium::DimensionManager
 
 
   def flush
-    db.multi_insert(@rows) unless @rows.empty?
+    db.multi_insert(@rows, slice: INSERT_BATCH_SIZE) unless @rows.empty?
     @rows = []
   end
 
@@ -44,6 +44,10 @@ class Cranium::DimensionManager
 
   private
 
+  INSERT_BATCH_SIZE = 100_000.freeze
+
+
+
   def to_multi_key_cache(table_data)
     Hash[table_data.map { |row| [row[0..-2], row.last] }]
   end

data/lib/cranium/test_framework/database_table.rb

@@ -14,6 +14,12 @@ class Cranium::TestFramework::DatabaseTable < Cranium::TestFramework::DatabaseEn
 
 
 
+  def count
+    connection[entity_name].count
+  end
+
+
+
   def content(fields = ["*".to_sym])
     connection[entity_name].select(*fields).all
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-24 00:00:00.000000000 Z
+date: 2016-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg