cranium 0.2.0

data/features/restore_database_connection.feature

@@ -0,0 +1,55 @@
+Feature: Sequel database connections are fault tolerant
+
+  Scenario:
+    Given a database table called "dim_product" with the following fields:
+      | field_name | field_type |
+      | item       | TEXT       |
+      | title      | TEXT       |
+    And a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory > Ultra-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff | 3dim > 2dim > 1dim
+    """
+    And the following definition:
+    """
+    require 'sequel'
+
+    def terminate_connections
+      connection_string = ENV['GREENPLUM_AS_ADMIN_URL'] || "postgres://database_administrator:emarsys@192.168.56.43:5432/cranium"
+      connection = Sequel.connect connection_string, loggers: Cranium.configuration.loggers
+      connection.run("SELECT pg_terminate_backend(procpid) FROM pg_stat_activity WHERE procpid <> pg_backend_pid() AND datname = 'cranium'")
+    end
+
+    source :products do
+      encoding "UTF-8"
+      delimiter ','
+
+      field :id, String
+      field :name, String
+    end
+
+    source :transformed_products do
+      field :id, String
+      field :name, String
+    end
+
+    transform :products => :transformed_products do |record|
+      output record
+    end
+
+    terminate_connections
+
+    import :transformed_products do
+      into :dim_product
+
+      put :id => :item
+      put :name => :title
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And the "dim_product" table should contain:
+      | item    | title                |
+      | JNI-123 | Just a product name  |
+      | CDI-234 | Another product name |

data/features/step_definitions/database_table_steps.rb

@@ -0,0 +1,40 @@
+Given(/^a database table called "([^"]*)" with the following fields:$/) do |table_name, fields|
+  database_table(table_name).create(fields.data)
+end
+
+
+Given (/^only the following rows in the "([^"]*)" database table:$/) do |table_name, rows|
+  database_table(table_name).clear
+  step %Q(the following new rows in the "#{table_name}" database table:), rows
+end
+
+
+Given (/^the following new rows in the "([^"]*)" database table:$/) do |table_name, rows|
+  database_table(table_name).insert rows.data
+end
+
+
+Given(/^the current value in sequence "([^"]*)" is (\d+)$/) do |sequence_name, current_value|
+  Cranium::Database.connection.run "SELECT setval('#{sequence_name}', #{current_value})"
+end
+
+
+Given(/^a sequence called "([^"]*)" starting from (\d+)$/) do |sequence_name, start_value|
+  database_sequence(sequence_name).create
+
+  step %Q[the current value in sequence "#{sequence_name}" is #{start_value}]
+end
+
+
+Then(/^the "([^"]*)" table should contain:$/) do |table_name, data|
+  expected_data, hashes = [], data.data
+  hashes.map do |hash|
+    new_row = {}
+    hash.each_key do |key|
+      new_row[key.to_sym] = hash[key]
+    end
+    expected_data << new_row
+  end
+
+  expect(database_table(table_name).content(data.fields)).to match_array expected_data
+end

data/features/step_definitions/definition_steps.rb

@@ -0,0 +1,3 @@
+Given(/^the following definition:$/) do |definition|
+  save_definition definition
+end

data/features/step_definitions/execution_steps.rb

@@ -0,0 +1,23 @@
+Given /^the definition is executed(?: again)?$/ do
+  step "I execute the definition"
+end
+
+
+When /^I execute the definition(?: again)?$/ do
+  execute_definition
+end
+
+
+Then /^the process should exit successfully$/ do
+  expect(result_code).to eq(0), "Expected script exit code to be 0, but received #{result_code}\n\n#{script_output}\n"
+end
+
+
+Then /^the process should exit with an error$/ do
+  expect(result_code).to eq(1), "Expected script exit code to be 1, but received #{result_code}\n\n#{script_output}\n"
+end
+
+
+Then /^the error message should contain:$/ do |message|
+  expect(error_output).to include message
+end

data/features/step_definitions/file_steps.rb

@@ -0,0 +1,39 @@
+Given /^no "([^"]*)" directory/ do |dir_path|
+  upload_directory.remove_directory dir_path
+end
+
+
+Given /^an empty "([^"]*)" data file$/ do |file_name|
+  step %Q(a "#{file_name}" data file containing:), ""
+end
+
+
+Given /^an? "([^"]*)" data file containing:$/ do |file_name, content|
+  upload_directory.save_file file_name, content
+end
+
+
+Given /^the "([^"]*)" file is deleted$/ do |file_name|
+  upload_directory.delete_file file_name
+end
+
+
+Then /^there should be a "([^"]*)" data file in the upload directory containing:$/ do |file_name, content|
+  expect(upload_directory.file_exists?(file_name)).to be_truthy, "expected file '#{file_name}' to exist"
+  expect(upload_directory.read_file(file_name).chomp).to eq content
+end
+
+
+Then /^the "([^"]*)" directory should contain the following files:$/ do |directory_path, files|
+  expect(Dir.exists?(directory_path)).to be_truthy, "expected directory '#{directory_path}' to exist"
+  files_in_dir = Dir["#{directory_path}/*"].map { |file_name| File.basename file_name }.sort
+  expect(files_in_dir.count).to eq files.data.count
+  0.upto files.data.count-1 do |index|
+    expect(files_in_dir[index]).to match Regexp.new(files.data[index][:filename])
+  end
+end
+
+
+Then /^the upload directory should contain the following files:$/ do |files|
+  step %Q(the "#{Cranium.configuration.upload_path}" directory should contain the following files:), files
+end

data/features/support/class_extensions.rb

@@ -0,0 +1,24 @@
+module Cucumber::Ast
+
+  module MultilineArgument
+    class << self
+
+      alias_method :from_old, :from
+
+
+
+      def from(argument)
+        original_result = from_old(argument)
+        if original_result.is_a? Cucumber::Ast::Table
+          Cranium::TestFramework::CucumberTable.from_ast_table(original_result).with_patterns(
+            "NULL" => nil
+          )
+        else
+          original_result
+        end
+      end
+
+    end
+  end
+
+end

data/features/support/env.rb

@@ -0,0 +1,27 @@
+require 'fileutils'
+require_relative "../../lib/cranium"
+
+FileUtils.mkdir_p("log") unless Dir.exists?("log")
+
+Cranium.configure do |config|
+  config.greenplum_connection_string = ENV['GREENPLUM_URL'] || "postgres://cranium:cranium@192.168.56.43:5432/cranium"
+  config.gpfdist_url = ENV['GPFDIST_URL'] || "192.168.56.43:8123"
+  config.gpfdist_home_directory = "tmp/custdata"
+  config.upload_directory = "cranium_build"
+  config.loggers << Logger.new("log/cucumber.log")
+end
+
+
+Before do
+  FileUtils.rm_rf Cranium.configuration.upload_path
+  FileUtils.mkdir_p Cranium.configuration.upload_path
+end
+
+After do
+  Cranium::TestFramework::DatabaseTable.cleanup
+  Cranium::TestFramework::DatabaseSequence.cleanup
+end
+
+World do
+  Cranium::TestFramework::World.new Cranium.configuration.upload_path, Cranium::Database.connection
+end

data/features/support/randomize.rb

@@ -0,0 +1,22 @@
+cucumber_seed = ENV['CUCUMBER_SEED'] ? ENV['CUCUMBER_SEED'].to_i : srand % 0xFFFF
+cucumber_dry_run = nil
+
+
+AfterConfiguration do |cucumber_config|
+  original_files = cucumber_config.feature_files
+  cucumber_dry_run = cucumber_config.dry_run?
+
+  config_eigenclass = class << cucumber_config;
+    self
+  end
+  config_eigenclass.send :undef_method, :feature_files
+  config_eigenclass.send(:define_method, :feature_files) do
+    Kernel.srand cucumber_seed
+    original_files.sort_by { Kernel.rand original_files.count }
+  end
+end
+
+
+at_exit do
+  puts("Cucumber randomized with seed #{cucumber_seed.inspect}") unless cucumber_dry_run
+end

data/features/support/stop_on_first_error.rb

@@ -0,0 +1,5 @@
+unless ENV["STOP_ON_FIRST_ERROR"] == "no"
+  After do |scenario|
+    Cucumber.wants_to_quit = true if scenario.failed?
+  end
+end

data/features/transform/deduplication.feature

@@ -0,0 +1,37 @@
+Feature: Deduplicate data in CSV file
+
+  Scenario: Singe file transformation
+    Given a "sales_items.csv" data file containing:
+    """
+    order_id,item,item_name
+    1,Item1,Item name 1
+    2,Item1,Item name 1
+    3,Item2,Item name 2
+    4,Item2,Item name 2
+    5,Item3,Item name 3
+    """
+    And the following definition:
+    """
+    source :sales_items do
+      file "sales_items.csv"
+      field :order_id, String
+      field :item, String
+      field :item_name, String
+    end
+
+    source :products do
+      field :item, String
+      field :item_name, String
+    end
+
+    deduplicate :sales_items, into: :products, by: [:item]
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "products.csv" data file in the upload directory containing:
+    """
+    item,item_name
+    Item1,Item name 1
+    Item2,Item name 2
+    Item3,Item name 3
+    """

data/features/transform/empty_transformation.feature

@@ -0,0 +1,72 @@
+Feature: Empty transformation
+
+  Scenario: Empty transformation between the same structures from the default CSV format simply copies the file
+    Given a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff > Scripts
+    """
+    And the following definition:
+    """
+    source :products do
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    source :products_copy do
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    transform :products => :products_copy do |record|
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "products_copy.csv" data file in the upload directory containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff > Scripts
+    """
+
+
+  Scenario: Empty transformation between the same structures but from a custom CSV format converts quotes and delimiters to the default format
+    Given a "products.csv" data file containing:
+    """
+    'id';'name';'category'
+    'JNI-123';'Just a product name';'Main category > Subcategory > Sub-subcategory'
+    'CDI-234';'Another 12" product name';'Smart Insight > Cool stuff > Scripts'
+    """
+    And the following definition:
+    """
+    source :products do
+      delimiter ';'
+      quote "'"
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    source :products_converted do
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    transform :products => :products_converted do |record|
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "products_converted.csv" data file in the upload directory containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory
+    CDI-234,"Another 12"" product name",Smart Insight > Cool stuff > Scripts
+    """

data/features/transform/join.feature

@@ -0,0 +1,180 @@
+Feature: Joining CSV files
+
+  Scenario: Singe file transformation
+    Given an "orders.csv" data file containing:
+    """
+    id,order_date,customer_id,total_price
+    order_1,2011-01-01,customer_1,100
+    order_2,2011-02-02,customer_1,200
+    order_3,2011-03-03,customer_2,300
+    """
+    Given an "order_items.csv" data file containing:
+    """
+    order_id,item_id,item_name,item_category,quantity,sales_amount,comment
+    order_1,item_1,first item,clothing,1,5,some useful comment
+    order_1,item_2,second item,communication,2,6,not so useful comment
+    order_2,item_2,second item,communication,5,12,very misleading comment
+    """
+    And the following definition:
+    """
+    source :orders_file do
+      file "orders.csv"
+
+      field :id, String
+      field :order_date, Date
+      field :customer_id, String
+      field :total_price, Integer
+    end
+
+    source :order_items_file do
+      file "order_items.csv"
+
+      field :order_id, String
+      field :item_id, String
+      field :item_name, String
+      field :item_category, String
+      field :quantity, Integer
+      field :sales_amount, Integer
+      field :comment, String
+    end
+
+    source :sales_items do
+      field :order_id, String
+      field :order_date, String
+      field :new_field, String
+      field :customer_id, String
+      field :item_id, String
+      field :item_name, String
+      field :item_category, String
+      field :quantity, String
+      field :sales_amount, String
+    end
+
+    join :orders_file, with: :order_items_file, into: :sales_items, match_on: { :order_id => :id }
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "sales_items.csv" data file in the upload directory containing:
+    """
+    order_id,order_date,new_field,customer_id,item_id,item_name,item_category,quantity,sales_amount
+    order_1,2011-01-01,,customer_1,item_1,first item,clothing,1,5
+    order_1,2011-01-01,,customer_1,item_2,second item,communication,2,6
+    order_2,2011-02-02,,customer_1,item_2,second item,communication,5,12
+    """
+
+
+  Scenario: File transformation with left join
+    Given an "orders.csv" data file containing:
+    """
+    id,order_date,customer_id,total_price
+    order_1,2011-01-01,customer_1,100
+    order_2,2011-02-02,customer_1,200
+    order_3,2011-03-03,customer_2,300
+    """
+    Given an "order_items.csv" data file containing:
+    """
+    order_id,item_id,item_name,item_category,quantity,sales_amount,comment
+    order_1,item_1,first item,clothing,1,5,some useful comment
+    order_1,item_2,second item,communication,2,6,not so useful comment
+    order_2,item_2,second item,communication,5,12,very misleading comment
+    """
+    And the following definition:
+    """
+    source :orders_file do
+      file "orders.csv"
+
+      field :id, String
+      field :order_date, Date
+      field :customer_id, String
+      field :total_price, Integer
+    end
+
+    source :order_items_file do
+      file "order_items.csv"
+
+      field :order_id, String
+      field :item_id, String
+      field :item_name, String
+      field :item_category, String
+      field :quantity, Integer
+      field :sales_amount, Integer
+      field :comment, String
+    end
+
+    source :sales_items do
+      field :id, String
+      field :item_id, String
+      field :item_name, String
+      field :order_date, String
+      field :customer_id, String
+      field :item_category, String
+      field :quantity, String
+      field :sales_amount, String
+    end
+
+    join :orders_file, with: :order_items_file, into: :sales_items, match_on: { :order_id => :id }, type: :left
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "sales_items.csv" data file in the upload directory containing:
+    """
+    id,item_id,item_name,order_date,customer_id,item_category,quantity,sales_amount
+    order_1,item_1,first item,2011-01-01,customer_1,clothing,1,5
+    order_1,item_2,second item,2011-01-01,customer_1,communication,2,6
+    order_2,item_2,second item,2011-02-02,customer_1,communication,5,12
+    order_3,,,2011-03-03,customer_2,,,
+    """
+
+
+  Scenario: Close file after join
+    Given an "orders.csv" data file containing:
+    """
+    id,order_date
+    order_1,2011-01-01
+    """
+    Given an "order_items.csv" data file containing:
+    """
+    order_id,item_id
+    order_1,item_1
+    """
+    And the following definition:
+    """
+    source :orders_file do
+      file "orders.csv"
+
+      field :id, String
+      field :order_date, Date
+    end
+
+    source :order_items_file do
+      file "order_items.csv"
+
+      field :order_id, String
+      field :item_id, String
+    end
+
+    source :sales_items do
+      field :id, String
+      field :item_id, String
+      field :order_date, String
+    end
+
+    source :sales_items_transformed do
+      field :id, String
+      field :item_id, String
+      field :order_date, String
+    end
+
+    join :orders_file, with: :order_items_file, into: :sales_items, match_on: { :order_id => :id }, type: :left
+
+    transform :sales_items => :sales_items_transformed do |record|
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "sales_items_transformed.csv" data file in the upload directory containing:
+    """
+    id,item_id,order_date
+    order_1,item_1,2011-01-01
+    """

data/features/transform/join_multiple_files_into_one_output_file.feature

@@ -0,0 +1,46 @@
+Feature: Join multiple files into one output file
+
+  Scenario: Successful transformation
+    Given a "products1.csv" data file containing:
+    """
+    id,name,category
+    PROD-1,product name 1,Main category > Subcategory > Sub-subcategory
+    PROD-2,product name 2,Main category > Subcategory > Sub-subcategory
+    """
+    And a "products2.csv" data file containing:
+    """
+    id,name,category
+    PROD-3,product name 3,Main category > Subcategory > Sub-subcategory
+    PROD-4,product name 4,Main category > Subcategory > Sub-subcategory
+    """
+    And the following definition:
+    """
+    source :products do
+      file "products*.csv"
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    source :transformed_products do
+      field :item, String
+      field :title, String
+      field :category, String
+    end
+
+    transform :products => :transformed_products do |record|
+      record[:item] = record[:id]
+      record[:title] = record[:name]
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "transformed_products.csv" data file in the upload directory containing:
+    """
+    item,title,category
+    PROD-1,product name 1,Main category > Subcategory > Sub-subcategory
+    PROD-2,product name 2,Main category > Subcategory > Sub-subcategory
+    PROD-3,product name 3,Main category > Subcategory > Sub-subcategory
+    PROD-4,product name 4,Main category > Subcategory > Sub-subcategory
+    """

data/features/transform/output_rows.feature

@@ -0,0 +1,70 @@
+Feature: Output rows to file
+
+  Background:
+    Given a "products.csv" data file containing:
+    """
+    id,name
+    1,Product 1
+    2, Product 2
+    """
+
+  Scenario: Output Hash instead of record
+    Given the following definition:
+    """
+    source :products do
+      field :id, String
+      field :name, String
+    end
+
+    source :products_copy do
+      field :id, String
+      field :name, String
+    end
+
+    transform :products => :products_copy do |record|
+      output name: record[:name],
+             id: record[:id]
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "products_copy.csv" data file in the upload directory containing:
+    """
+    id,name
+    1,Product 1
+    2,Product 2
+    """
+
+
+  Scenario: Output multiple records for each input row
+    Given the following definition:
+    """
+    source :products do
+      field :id, String
+      field :name, String
+    end
+
+    source :products_doubled do
+      field :id, String
+      field :name, String
+      field :counter, Integer
+    end
+
+    transform :products => :products_doubled do |record|
+      record[:counter] = 1
+      output record
+      record[:counter] = 2
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "products_doubled.csv" data file in the upload directory containing:
+    """
+    id,name,counter
+    1,Product 1,1
+    1,Product 1,2
+    2,Product 2,1
+    2,Product 2,2
+    """
+

data/features/transform/projection.feature

@@ -0,0 +1,34 @@
+Feature: Projection
+
+  Scenario: Empty transformation projects down if the source structure is a superset of the target structure
+    Given a "products.csv" data file containing:
+    """
+    id,name,category
+    JNI-123,Just a product name,Main category > Subcategory > Sub-subcategory
+    CDI-234,Another product name,Smart Insight > Cool stuff > Scripts
+    """
+    And the following definition:
+    """
+    source :products do
+      field :id, String
+      field :name, String
+      field :category, String
+    end
+
+    source :products_projected do
+      field :id, String
+      field :category, String
+    end
+
+    transform :products => :products_projected do |record|
+      output record
+    end
+    """
+    When I execute the definition
+    Then the process should exit successfully
+    And there should be a "products_projected.csv" data file in the upload directory containing:
+    """
+    id,category
+    JNI-123,Main category > Subcategory > Sub-subcategory
+    CDI-234,Smart Insight > Cool stuff > Scripts
+    """