cranium 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5b878ec72b1afbd62b6d0c576653c4d5e5b1a0ee
-  data.tar.gz: 41b765f1e0cf25648cf30cee548555b7f0ffaaaf
+SHA256:
+  metadata.gz: 786956ff56304307aecdf21cd490af6c1bb1c116a87beff9ce7c8afb3935b329
+  data.tar.gz: 0aef7e4d5a467c7551f997a6f374448bf9cc2616afe1f20b929c0f6edc30e574
 SHA512:
-  metadata.gz: b52aa636e49d51d48fc86397330527fe31c40214a055ef7c405befe1e5b53fe255064259de4857a57e0b8374ae111ce5e66a3d677878b010502f5d8faadc9f3d
-  data.tar.gz: 98d0e5a309f127c50e7910c5510908dfb5e71a6f4f347f804c0118bb21a680bf45fb64b059b30866febb81e827dac8f65ea95a2830ea3826927d47e54f773c68
+  metadata.gz: 2175a36fc29efb510af186af5791973781856720dd065308a7d891cdd6e005174f74ecd500e2c5dc81b14d15bca512ad7fb3fd5fe334640328b3d305b48bb10e
+  data.tar.gz: 994c5aa959c1c32417814d68acf26b226949305a92297cdc20b90ef3be77efa7cfa77ec029c60843f5a1b223918df0c5508afa67d3843a0a537da065dcb89f3f

data/.env

@@ -0,0 +1 @@
+DATABASE_HOST=127.0.0.1

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--require spec_helper
+--order random
+--fail-fast

data/.ruby-version

@@ -1 +1 @@
-2.3.0
+2.5.1

data/README.md

@@ -18,21 +18,9 @@ Or install it yourself as:
 
 ## Development
 
-start up the db
-
-    docker-compose create && docker-compose start
-    
-find out what's the ip is (in case you're using native docker)
-
-    docker-compose ps 
-    
-(if using docker-machine use the machine's ip)
-setup the DATABASE_HOST enviroment variable to this IP (192.168.64.4 in my case)
-    
-    export DATABASE_HOST=192.168.64.4
-    
-Now, your ready to run the integration tests :)
+start up the db:
 
+    $ docker-compose up -d
 
 ## Contributing
 

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.4.2'
+  spec.version = '0.4.3'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}
@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3'
   spec.add_development_dependency 'ruby-prof', '~> 0'
   spec.add_development_dependency 'cucumber', '~> 1'
+  spec.add_development_dependency 'dotenv', '~> 2.5'
 end

data/features/archive.feature

@@ -2,6 +2,7 @@ Feature: Archive source files
 
   Scenario:
     Given no "/tmp/cranium_archive" directory
+    And no "/tmp/cranium_storage" directory
     And a "products_1.csv" data file containing:
     """
     """
@@ -39,6 +40,8 @@ Feature: Archive source files
     end
 
     archive :products, :contacts
+
+    move :purchases, to: "/tmp/cranium_storage"
     """
     When I execute the definition
     Then the process should exit successfully
@@ -47,3 +50,6 @@ Feature: Archive source files
       | .*contacts.csv   |
       | .*products_1.csv |
       | .*products_2.csv |
+    And the "/tmp/cranium_storage" directory should contain the following files:
+      | filename      |
+      | purchases.csv |

data/features/support/env.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'dotenv/load'
 require_relative "../../lib/cranium"
 
 FileUtils.mkdir_p("log") unless Dir.exists?("log")

data/lib/cranium/archiver.rb

@@ -1,36 +1,37 @@
 require 'fileutils'
 
 module Cranium::Archiver
-
-  def self.archive(*files)
-    create_archive_directory
-    archive_files files
-  end
-
-
-
-  def self.remove(*files)
-    files.each do |file_name|
-      FileUtils.rm File.join(Cranium.configuration.upload_path, file_name)
+  class << self
+    def archive(*files)
+      create_directory(Cranium.configuration.archive_directory)
+      archive_datetime = Time.now.strftime("%Y-%m-%d_%Hh%Mm%Ss")
+      move_files_from_upload_directory(files, Cranium.configuration.archive_directory, prefix: "#{archive_datetime}_")
     end
-  end
-
 
+    def remove(*files)
+      files.each do |file_name|
+        FileUtils.rm File.join(Cranium.configuration.upload_path, file_name)
+      end
+    end
 
-  private
-
-  def self.create_archive_directory
-    FileUtils.mkpath Cranium.configuration.archive_directory unless Dir.exists? Cranium.configuration.archive_directory
-  end
+    def move(*files, target_directory:)
+      create_directory(target_directory)
+      move_files_from_upload_directory(files, target_directory)
+    end
 
+    private
 
+    def create_directory(path)
+      FileUtils.mkdir_p(path)
+    end
 
-  def self.archive_files(files)
-    archive_datetime = Time.now.strftime("%Y-%m-%d_%Hh%Mm%Ss")
-    files.each do |file_name|
-      FileUtils.mv File.join(Cranium.configuration.upload_path, file_name),
-                   File.join(Cranium.configuration.archive_directory, "#{archive_datetime}_#{file_name}")
+    def move_files_from_upload_directory(files, target_directory, prefix: "")
+      files.each do |file_name|
+        FileUtils.mv(
+          File.join(Cranium.configuration.upload_path, file_name),
+          File.join(target_directory, "#{prefix}#{file_name}")
+        )
+      end
     end
   end
-
 end

data/lib/cranium/dsl.rb

@@ -87,6 +87,14 @@ module Cranium::DSL
 
 
 
+  def move(*sources, to: "")
+    sources.each do |source_name|
+      Cranium::Archiver.move *Cranium.application.sources[source_name].files, target_directory: to
+    end
+  end
+
+
+
   def sequence(name)
     Cranium::Transformation::Sequence.new name
   end

data/lib/cranium/test_framework/world.rb

@@ -22,8 +22,9 @@ class Cranium::TestFramework::World
 
 
   def save_definition(definition)
-    config = <<-config_string
+    config = <<~config_string
       require 'logger'
+      require 'date'
 
       Cranium.configure do |config|
         config.greenplum_connection_string = "#{Cranium.configuration.greenplum_connection_string}"
@@ -32,6 +33,7 @@ class Cranium::TestFramework::World
         config.upload_directory = "#{Cranium.configuration.upload_directory}"
         config.loggers << Logger.new("log/application.log")
       end
+
     config_string
 
     upload_directory.save_file DEFINITION_FILE, config + definition

data/spec/cranium/archiver_spec.rb

@@ -1,44 +1,72 @@
-require_relative '../spec_helper'
+RSpec.describe Cranium::Archiver do
+  subject(:archiver) { described_class }
 
-describe Cranium::Archiver do
-
-  before(:each) do
-    allow(Cranium).to receive_messages(configuration: Cranium::Configuration.new.tap do |config|
-      config.gpfdist_home_directory = "gpfdist_home"
-      config.upload_directory = "upload_dir"
-      config.archive_directory = "path/to/archive"
-    end)
+  let(:configuration) do
+    Cranium::Configuration.new.tap do |config|
+      config.gpfdist_home_directory = "tmp"
+      config.upload_directory = "upload_directory"
+      config.archive_directory = "tmp/archive_directory"
+    end
   end
+  let(:file1) { "file.txt" }
+  let(:file2) { "another_file.txt" }
 
+  before do
+    allow(Cranium).to receive_messages(configuration: configuration)
+
+    FileUtils.mkdir_p(configuration.upload_path)
+    FileUtils.touch(File.join(configuration.upload_path, file1))
+    FileUtils.touch(File.join(configuration.upload_path, file2))
+  end
 
   describe ".archive" do
-    it "should create the archive directory if it doesn't exist" do
-      allow(Dir).to receive(:exists?).with("path/to/archive").and_return(false)
+    before { FileUtils.rm_rf configuration.archive_directory }
 
-      expect(FileUtils).to receive(:mkpath).with "path/to/archive"
+    context "when archive directory does not exist" do
+      it "creates the archive directory" do
+        archiver.archive file1, file2
 
-      Cranium::Archiver.archive
+        expect(File.exists?(configuration.archive_directory)).to eq true
+      end
     end
 
-    it "should move files to the archive directory" do
-      allow(Dir).to receive(:exists?).with("path/to/archive").and_return(true)
-      allow(Time).to receive(:now).and_return Time.new(2000, 1, 1, 1, 2, 3)
+    context "when there are some file in the upload directory" do
+      it "moves files to the archive directory" do
+        archiver.archive file1, file2
 
-      expect(FileUtils).to receive(:mv).with "gpfdist_home/upload_dir/file.txt", "path/to/archive/2000-01-01_01h02m03s_file.txt"
-      expect(FileUtils).to receive(:mv).with "gpfdist_home/upload_dir/another_file.txt", "path/to/archive/2000-01-01_01h02m03s_another_file.txt"
-
-      Cranium::Archiver.archive "file.txt", "another_file.txt"
+        expect(File.exist?(File.join(configuration.upload_path, file1))).to eq false
+        expect(File.exist?(File.join(configuration.upload_path, file2))).to eq false
+        expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file1}")))).to eq true
+        expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file2}")))).to eq true
+      end
     end
   end
 
-
   describe ".remove" do
-    it "should remove files from the upload directory" do
-      expect(FileUtils).to receive(:rm).with "gpfdist_home/upload_dir/file.txt"
-      expect(FileUtils).to receive(:rm).with "gpfdist_home/upload_dir/another_file.txt"
+    it "removes files from the upload directory" do
+      archiver.remove file1, file2
 
-      Cranium::Archiver.remove "file.txt", "another_file.txt"
+      expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file1}")))).to eq true
+      expect(File.exist?(File.join(configuration.archive_directory, Dir.glob("*#{file2}")))).to eq true
     end
   end
 
+  describe ".move" do
+    let(:target_directory) { "tmp/target_directory" }
+
+    it "creates given directory if it does not exist" do
+      archiver.move(file1, file2, target_directory: target_directory)
+
+      expect(File.exists?(target_directory)).to eq true
+    end
+
+    it "moves files from upload directory into a given directory" do
+      archiver.move(file1, file2, target_directory: target_directory)
+
+      expect(File.exist?(File.join(configuration.upload_path, file1))).to eq false
+      expect(File.exist?(File.join(configuration.upload_path, file2))).to eq false
+      expect(File.exist?(File.join(target_directory, file1))).to eq true
+      expect(File.exist?(File.join(target_directory, file2))).to eq true
+    end
+  end
 end

data/spec/cranium/dsl_spec.rb

@@ -97,6 +97,22 @@ describe Cranium::DSL do
   end
 
 
+  describe "#move" do
+    let(:target_directory) { "/tmp/target" }
+
+    it "should move files for the specified sources" do
+      allow(Cranium.application).to receive_messages sources: {first_source: double(files: ["file1", "file2"]),
+                                                               second_source: double(files: ["file3"]),
+                                                               third_source: double(files: ["file4"])}
+
+      expect(Cranium::Archiver).to receive(:move).with "file1", "file2", target_directory: target_directory
+      expect(Cranium::Archiver).to receive(:move).with "file3", target_directory: target_directory
+
+      dsl_object.move :first_source, :second_source, to: target_directory
+    end
+  end
+
+
   describe "#sequence" do
     it "should return a sequence with the specified name" do
       result = dsl_object.sequence "test_sequence"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-01 00:00:00.000000000 Z
+date: 2018-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 description: Provides Extract, Transform and Load functionality for loading data from
   CSV files to a Greenplum database.
 email:
@@ -145,7 +159,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env"
 - ".gitignore"
+- ".rspec"
 - ".ruby-version"
 - Gemfile
 - LICENSE.txt
@@ -297,7 +313,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Pure Ruby ETL framework