cranium 0.2.0

data/features/import/import_csv_to_database_with_update_merging.feature

@@ -0,0 +1,46 @@
+Feature: Import a CSV file into the database with merging
+
+  The merge_on property can be used to specify an id field that is used to detect duplicates while importing.
+  Duplicates are updated and new items are added.
+
+  Scenario: Successful import with merged items
+    Given a database table called "dim_product" with the following fields:
+      | field_name  | field_type |
+      | item        | TEXT       |
+      | title       | TEXT       |
+      | description | TEXT       |
+    And only the following rows in the "dim_product" database table:
+      | item    | title                | description                  |
+      | JNI-123 | Just a product name  | Very interesting description |
+      | CDI-234 | Another product name | Yet another cool description |
+    And a "products.csv" data file containing:
+    """
+    id,name,description
+    JNI-123,Just a product name,"Very interesting description, updated"
+    CDI-234,Updated product name,Yet another cool description
+    KLM-987,Inserted product name,This is the best product
+    """
+    And the following definition:
+    """
+    source :products do
+      field :id, String
+      field :name, String
+      field :description, String
+    end
+
+    import :products do
+      into :dim_product
+      put :id => :item
+      put :name => :title
+      put :description => :description
+
+      merge_on :id => :item
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_product" table should contain:
+      | item    | title                 | description                           |
+      | JNI-123 | Just a product name   | Very interesting description, updated |
+      | CDI-234 | Updated product name  | Yet another cool description          |
+      | KLM-987 | Inserted product name | This is the best product              |

data/features/import/import_csv_with_always_inserting_new_dimension_keys.feature

@@ -0,0 +1,137 @@
+Feature: Import a CSV file into the database with new dimension values always inserted
+
+  Scenario: Successful import
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | contact_key | SERIAL     |
+      | user_id     | TEXT       |
+      | name        | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key (i) | user_id | name |
+      | 10              | 1       | Alma |
+    And the current value in sequence "dim_contact_contact_key_seq" is 10
+    And a database table called "fct_purchases" with the following fields:
+      | field_name  | field_type |
+      | contact_key | INTEGER    |
+      | amount      | TEXT       |
+    And a "purchases.csv" data file containing:
+    """
+    user_id,amount
+    1,100
+    NA,200
+    NA,300
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key, Integer
+      field :amount, String
+    end
+
+    transform :purchases => :transformed_purchases do |record|
+      record[:contact_key] = insert :contact_key,
+                                    table: :dim_contact,
+                                    record: {contact_key: next_value_in_sequence("dim_contact_contact_key_seq"), user_id: record[:user_id], name: "Unknown contact #{record[:user_id]}"}
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key
+      put :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain:
+      | contact_key (i) | amount |
+      | 11              | 100    |
+      | 12              | 200    |
+      | 13              | 300    |
+    And the "dim_contact" table should contain:
+      | contact_key (i) | user_id | name               |
+      | 10              | 1       | Alma               |
+      | 11              | 1       | Unknown contact 1  |
+      | 12              | NA      | Unknown contact NA |
+      | 13              | NA      | Unknown contact NA |
+
+
+  Scenario: Example use case for the insert
+    If purchases made by a predefined contact identifier (NA in this case) do not look for it insert .
+    Otherwise use lookup to find or create that contact
+
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | contact_key | SERIAL     |
+      | user_id     | TEXT       |
+      | name        | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key (i) | user_id | name |
+      | 10              | 1       | Alma |
+    And the current value in sequence "dim_contact_contact_key_seq" is 10
+    And a database table called "fct_purchases" with the following fields:
+      | field_name  | field_type |
+      | contact_key | INTEGER    |
+      | amount      | TEXT       |
+    And a "purchases.csv" data file containing:
+    """
+    user_id,amount
+    1,100
+    NA,200
+    NA,300
+    2,400
+    2,500
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key, Integer
+      field :amount, String
+    end
+
+    transform :purchases => :transformed_purchases do |record|
+      if record[:user_id] == 'NA'
+        record[:contact_key] = insert :contact_key,
+                                      table: :dim_contact,
+                                      record: {contact_key: next_value_in_sequence("dim_contact_contact_key_seq"), user_id: record[:user_id], name: "Unknown contact #{record[:user_id]}"}
+      else
+        record[:contact_key] = lookup :contact_key,
+                                      from_table: :dim_contact,
+                                      match_column: :user_id,
+                                      to_value: record[:user_id],
+                                      if_not_found_then_insert: {contact_key: next_value_in_sequence("dim_contact_contact_key_seq"), name: "Unknown contact #{record[:user_id]}"}
+      end
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key
+      put :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain:
+      | contact_key (i) | amount |
+      | 10              | 100    |
+      | 11              | 200    |
+      | 12              | 300    |
+      | 13              | 400    |
+      | 13              | 500    |
+    And the "dim_contact" table should contain:
+      | contact_key (i) | user_id | name               |
+      | 10              | 1       | Alma               |
+      | 11              | NA      | Unknown contact NA |
+      | 12              | NA      | Unknown contact NA |
+      | 13              | 2       | Unknown contact 2  |

data/features/import/import_csv_with_field_lookup_inserting_new_dimension_keys.feature

@@ -0,0 +1,62 @@
+Feature: Import a CSV file into the database with new dimension values inserted when not found during lookup
+
+  Scenario: Successful import
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | contact_key | SERIAL     |
+      | user_id     | TEXT       |
+      | name        | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key (i) | user_id | name |
+      | 10              | 1       | Alma |
+    And the current value in sequence "dim_contact_contact_key_seq" is 10
+    And a database table called "fct_purchases" with the following fields:
+      | field_name  | field_type |
+      | contact_key | INTEGER    |
+      | amount      | TEXT       |
+    And a "purchases.csv" data file containing:
+    """
+    user_id,amount
+    1,100
+    2,200
+    2,300
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key, Integer
+      field :amount, String
+    end
+
+    transform :purchases => :transformed_purchases do |record|
+      record[:contact_key] = lookup :contact_key,
+                                    from_table: :dim_contact,
+                                    match_column: :user_id,
+                                    to_value: record[:user_id],
+                                    if_not_found_then_insert: { contact_key: next_value_in_sequence("dim_contact_contact_key_seq"),
+                                                                name: "Unknown contact #{record[:user_id]}" }
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key
+      put :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain:
+      | contact_key (i) | amount |
+      | 10              | 100    |
+      | 11              | 200    |
+      | 11              | 300    |
+    And the "dim_contact" table should contain:
+      | contact_key (i) | user_id | name              |
+      | 10              | 1       | Alma              |
+      | 11              | 2       | Unknown contact 2 |

data/features/import/import_csv_with_field_lookup_transformation.feature

@@ -0,0 +1,125 @@
+Feature: Import a CSV file into the database with IDs looked up from the database
+
+  Scenario: Successful import
+    Given a database table called "dim_contact" with the following fields:
+      | field_name  | field_type |
+      | contact_key | SERIAL     |
+      | user_id     | TEXT       |
+      | name        | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key (i) | user_id | name  |
+      | 10              | 1       | Alma  |
+      | 20              | 2       | Korte |
+    And a database table called "fct_purchases" with the following fields:
+      | field_name  | field_type |
+      | contact_key | INTEGER    |
+      | amount      | TEXT       |
+    And a "purchases.csv" data file containing:
+    """
+    user_id,amount
+    1,100
+    2,200
+    3,300
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key, Integer
+      field :amount, String
+    end
+
+    transform :purchases => :transformed_purchases do |record|
+      record[:contact_key] = lookup :contact_key,
+                                    from_table: :dim_contact,
+                                    match_column: :user_id,
+                                    to_value: record[:user_id],
+                                    if_not_found_then: -1
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key => :contact_key
+      put :amount => :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain:
+      | contact_key (i) | amount |
+      | 10              | 100    |
+      | 20              | 200    |
+      | -1              | 300    |
+
+
+  Scenario: Multiple fields looked up by one key
+    Given a database table called "dim_contact" with the following fields:
+      | field_name    | field_type |
+      | contact_key_1 | INTEGER    |
+      | contact_key_2 | INTEGER    |
+      | user_id       | TEXT       |
+      | name          | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key_1 (i) | contact_key_2 (i) | user_id | name  |
+      | 10                | 100               | 1       | Alma  |
+      | 20                | 200               | 2       | Korte |
+    And a database table called "fct_purchases" with the following fields:
+      | field_name    | field_type |
+      | contact_key_1 | INTEGER    |
+      | contact_key_2 | INTEGER    |
+      | amount        | TEXT       |
+    And a "purchases.csv" data file containing:
+    """
+    user_id,amount
+    1,100
+    2,200
+    3,300
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key_1, Integer
+      field :contact_key_2, Integer
+      field :amount, String
+    end
+
+    transform :purchases => :transformed_purchases do |record|
+      record[:contact_key_1] = lookup :contact_key_1,
+                                    from_table: :dim_contact,
+                                    match_column: :user_id,
+                                    to_value: record[:user_id],
+                                    if_not_found_then: -1
+
+      record[:contact_key_2] = lookup :contact_key_2,
+                                    from_table: :dim_contact,
+                                    match_column: :user_id,
+                                    to_value: record[:user_id],
+                                    if_not_found_then: -2
+
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key_1 => :contact_key_1
+      put :contact_key_2 => :contact_key_2
+      put :amount => :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain:
+      | contact_key_1 (i) | contact_key_2 (i) | amount |
+      | 10                | 100               | 100    |
+      | 20                | 200               | 200    |
+      | -1                | -2                | 300    |

data/features/import/import_csv_with_transformation.feature

@@ -0,0 +1,55 @@
+Feature: Import a CSV file into the database with a split transformation
+
+  Scenario: Successful import
+    Given a database table called "dim_product" with the following fields:
+      | field_name | field_type |
+      | item       | TEXT       |
+      | title      | TEXT       |
+      | category1  | TEXT       |
+      | category2  | TEXT       |
+      | category3  | TEXT       |
+    And a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory > Ultra-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff | 3dim > 2dim > 1dim
+    """
+    And the following definition:
+    """
+    source :products do
+      encoding "UTF-8"
+      delimiter ','
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    source :transformed_products do
+      field :id, String
+      field :name, String
+      field :main_category, String
+      field :sub_category, String
+      field :department, String
+    end
+
+    transform :products => :transformed_products do |record|
+      record.split_field :category, into: [:category], by: "|"
+      record.split_field :category, into: [:main_category, :sub_category, :department], by: ">"
+      output record
+    end
+
+    import :transformed_products do
+      into :dim_product
+      put :id => :item
+      put :name => :title
+      put :main_category => :category1
+      put :sub_category => :category2
+      put :department => :category3
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_product" table should contain:
+      | item    | title                | category1     | category2   | category3       |
+      | JNI-123 | Just a product name  | Main category | Subcategory | Sub-subcategory |
+      | CDI-234 | Another product name | Smart Insight | Cool stuff  | Cool stuff      |

data/features/import/import_multiple_csv_files_without_transformations.feature

@@ -0,0 +1,44 @@
+Feature: Import multiple CSV files into the database without any transformations
+
+  Scenario: Successful import
+    Given a database table called "dim_product" with the following fields:
+      | field_name | field_type |
+      | item       | TEXT       |
+      | title      | TEXT       |
+      | category   | TEXT       |
+    And a "products1.csv" data file containing:
+    """
+    id,name,category
+    PROD-1,product name 1,Main category > Subcategory > Sub-subcategory
+    PROD-2,product name 2,Main category > Subcategory > Sub-subcategory
+    """
+    And a "products2.csv" data file containing:
+    """
+    id,name,category
+    PROD-3,product name 3,Main category > Subcategory > Sub-subcategory
+    PROD-4,product name 4,Main category > Subcategory > Sub-subcategory
+    """
+    And the following definition:
+    """
+    source :products do
+      file "products*.csv"
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    import :products do
+      into :dim_product
+      put :id => :item
+      put :name => :title
+      put :category => :category
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_product" table should contain:
+      | item   | title          | category                                      |
+      | PROD-1 | product name 1 | Main category > Subcategory > Sub-subcategory |
+      | PROD-2 | product name 2 | Main category > Subcategory > Sub-subcategory |
+      | PROD-3 | product name 3 | Main category > Subcategory > Sub-subcategory |
+      | PROD-4 | product name 4 | Main category > Subcategory > Sub-subcategory |

data/features/import/import_with_load_id_from_sequence.feature

@@ -0,0 +1,53 @@
+Feature: Import data and assign a load id (audit information) from a sequence to all records
+
+  Scenario: Successful import
+    Given a database table called "dim_product" with the following fields:
+      | field_name | field_type |
+      | load_id    | INTEGER    |
+      | item       | TEXT       |
+      | title      | TEXT       |
+    And a sequence called "some_sequence" starting from 33
+    And a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory > Ultra-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff | 3dim > 2dim > 1dim
+    """
+    And the following definition:
+    """
+    LOAD_ID = sequence("some_sequence").next_value
+
+    source :products do
+      encoding "UTF-8"
+      delimiter ','
+
+      field :id, String
+      field :name, String
+    end
+
+    source :transformed_products do
+      field :load_id, Integer
+
+      field :id, String
+      field :name, String
+    end
+
+    transform :products => :transformed_products do |record|
+      record[:load_id] = LOAD_ID
+      output record
+    end
+
+    import :transformed_products do
+      into :dim_product
+
+      put :load_id
+      put :id => :item
+      put :name => :title
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_product" table should contain:
+      | load_id (i) | item    | title                |
+      | 34          | JNI-123 | Just a product name  |
+      | 34          | CDI-234 | Another product name |

data/features/import/import_with_lookup_from_multiple_fields.feature

@@ -0,0 +1,64 @@
+Feature: Import a CSV file into the database with IDs looked up from multiple columns of the database
+
+  Scenario: Successful import
+    Given a database table called "dim_contact" with the following fields:
+      | field_name     | field_type |
+      | contact_key    | SERIAL     |
+      | user_id_part_1 | TEXT       |
+      | user_id_part_2 | TEXT       |
+      | name           | TEXT       |
+    And only the following rows in the "dim_contact" database table:
+      | contact_key (i) | user_id_part_1 | user_id_part_2 | name   |
+      | 11              | 1              | 1              | Alma   |
+      | 12              | 1              | 2              | Korte  |
+      | 21              | 2              | 1              | Szilva |
+      | 22              | 2              | 2              | Barack |
+    And a database table called "fct_purchases" with the following fields:
+      | field_name  | field_type |
+      | contact_key | INTEGER    |
+      | amount      | TEXT       |
+    And a "purchases.csv" data file containing:
+    """
+    user_id_1,user_id_2,amount
+    1,1,100
+    1,2,200
+    2,1,300
+    2,2,400
+    3,1,500
+    """
+    And the following definition:
+    """
+    source :purchases do
+      field :user_id_1, String
+      field :user_id_2, String
+      field :amount, String
+    end
+
+    source :transformed_purchases do
+      field :contact_key, Integer
+      field :amount, String
+    end
+
+    transform :purchases => :transformed_purchases do |record|
+      record[:contact_key] = lookup :contact_key,
+                                    from_table: :dim_contact,
+                                    match: { :user_id_part_1 => record[:user_id_1], :user_id_part_2 => record[:user_id_2] },
+                                    if_not_found_then: -1
+      output record
+    end
+
+    import :transformed_purchases do
+      into :fct_purchases
+      put :contact_key => :contact_key
+      put :amount => :amount
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "fct_purchases" table should contain:
+      | contact_key (i) | amount |
+      | 11              | 100    |
+      | 12              | 200    |
+      | 21              | 300    |
+      | 22              | 400    |
+      | -1              | 500    |

data/features/read.feature

@@ -0,0 +1,56 @@
+Feature: Remove source files
+
+  Scenario:
+    Given a "forms.csv" data file containing:
+    """
+    id,name
+    1,Landing form
+    2,Other form
+    """
+    And a "contacts_extract.csv" data file containing:
+    """
+    id,created,form_id
+    1,2001-01-01,1
+    2,2002-02-02,2
+    3,2003-03-03,1
+    """
+    And the following definition:
+    """
+    source :forms do
+      field :id, Integer
+      field :name, String
+    end
+
+    source :contacts_extract do
+      field :id, Integer
+      field :created, String
+      field :form_id, Integer
+    end
+
+    source :contacts do
+      field :id, Integer
+      field :created, String
+      field :form, String
+    end
+
+    form_mapping = {}
+
+    read :forms do |record|
+      form_mapping[record[:id]] = record[:name]
+    end
+
+    transform :contacts_extract => :contacts do |record|
+      record[:form] = form_mapping[record[:form_id]]
+      output record
+    end
+
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "contacts.csv" data file in the upload directory containing:
+    """
+    id,created,form
+    1,2001-01-01,Landing form
+    2,2002-02-02,Other form
+    3,2003-03-03,Landing form
+    """

data/features/remove.feature

@@ -0,0 +1,44 @@
+Feature: Remove source files
+
+  Scenario:
+    Given a "contacts_extract_1.csv" data file containing:
+    """
+    """
+    And a "contacts_extract_2.csv" data file containing:
+    """
+    """
+    And a "clicks_extract_1.csv" data file containing:
+    """
+    """
+    And a "products.csv" data file containing:
+    """
+    """
+    And the following definition:
+    """
+    source :contacts_extract do
+      file "contacts_extract_*.csv"
+    end
+
+    source :clicks_extract do
+      file "clicks_extract_*.csv"
+    end
+
+    source :products do
+      file "products.csv"
+    end
+
+    source :products_transformed do end
+
+    transform :products => :products_transformed do |record|
+      output record
+    end
+
+    remove :contacts_extract, :clicks_extract
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the upload directory should contain the following files:
+      | filename                 |
+      | definition.rb            |
+      | products.csv             |
+      | products_transformed.csv |