cranium 0.8 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d6f7888fe46d32c156a48c081e57ddf8942d483b6a42c1b69faa4e0f276a128
-  data.tar.gz: 9f49d912ca7cf12a8d3e90e1e6d00565099f4bf0fe4cee90e29f439c345a86df
+  metadata.gz: 4715c818c4f8f98f7d1a489979c3cf5ac1a2b50114052c6d619526fc8d0a4ca0
+  data.tar.gz: 6c619a8f8570cae9ef6c51002db8cf2ea6785983b70e0ace6e28b14f37e72330
 SHA512:
-  metadata.gz: fb4ed093460aaf4eddde16e0ed1236e34d232ef128047683111459a836c7c544a4c5fe3ad5d05abb6609927affaf286c88f8269695153f21c1513b4c802445ea
-  data.tar.gz: 65730273e55b87a3ba2b35f303f21613af3e7cf9328a8f7612cee60c37c54e606f2d50a3891597275ea084ce4405bc445c3d7277d2f5cb3d31b7be1ae805a740
+  metadata.gz: e12f463f677b70d7765120658c0efcb00da40c92ef9dd321eb4e6804106cf22f1da85505dfcfd1c5baf8fb46f67f5aa3784cccd4bc7214c6ab767f4d46cd5fa9
+  data.tar.gz: '080472dd1a1c70907496fdf0978231b82f6b6c8da0baf6c6b0e8488398387ef75a2b2ce95731818dd2c712eff316823ac168fc4063202ad887d662304f4a30a2'

data/.env

@@ -1 +1,3 @@
-DATABASE_HOST=127.0.0.1
+GREENPLUM_HOST=127.0.0.1
+GPFDIST_HOST=127.0.0.1
+

data/.gitignore

@@ -1,3 +1,4 @@
+*.aes
 *.gem
 *.rbc
 .bundle

data/Dockerfile

@@ -0,0 +1,22 @@
+FROM ruby:2.7.3-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+                      build-essential \
+                      libpq-dev \
+                      postgresql-client \
+                      git-core
+
+RUN mkdir /app
+WORKDIR /app
+
+RUN gem install bundler
+
+COPY Gemfile .
+COPY cranium.gemspec .
+
+ARG https_proxy
+ARG http_proxy
+RUN bundle install -j 5
+COPY . .
+RUN mkdir /tmp/custdata/

data/bin/run_tests.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+sleep 30
+bundle exec rake

data/codeship-services.yml

@@ -0,0 +1,24 @@
+cranium:
+  build:
+    image: cranium
+    dockerfile: Dockerfile
+  cached: true
+
+greenplum:
+  image: kevinmtrowbridge/gpdb-docker
+  cached: true
+  volumes:
+    - gpfdist-init:/home/gpadmin/gpfdist-data
+    - ./docker/:/docker-entrypoint-initdb.d
+    - ./db:/db
+  command: sh /docker-entrypoint-initdb.d/initialize_greenplum.sh
+
+build-app:
+  image: cranium
+  depends_on:
+    - greenplum
+  environment:
+    GREENPLUM_HOST: "greenplum"
+    GPFDIST_HOST: "127.0.0.1"
+  volumes:
+    - gpfdist-init:/app/tmp/custdata

data/codeship-steps.yml

@@ -0,0 +1,3 @@
+- name: Run tests
+  service: build-app
+  command: sh bin/run_tests.sh

data/cranium.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name = 'cranium'
-  spec.version = '0.8'
+  spec.version = '0.8.1'
   spec.authors = ['Emarsys Technologies']
   spec.email = ['smart-insight-dev@emarsys.com']
   spec.description = %q{Provides Extract, Transform and Load functionality for loading data from CSV files to a Greenplum database.}
@@ -13,13 +13,13 @@ Gem::Specification.new do |spec|
   spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'pg', '~> 0'
+  spec.add_runtime_dependency 'pg', '>= 0'
   spec.add_runtime_dependency 'progressbar', '~> 0'
   spec.add_runtime_dependency 'sequel', '>= 4', '< 6'
   spec.add_runtime_dependency 'slop', '~> 3'
 
-  spec.add_development_dependency 'bundler', '~> 1'
-  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'
   spec.add_development_dependency 'ruby-prof', '~> 0'
   spec.add_development_dependency 'cucumber', '~> 1'

data/docker/create_databases.sql

@@ -0,0 +1,8 @@
+CREATE RESOURCE QUEUE smart_insight WITH (ACTIVE_STATEMENTS=10, PRIORITY=MEDIUM);
+
+CREATE ROLE cranium WITH RESOURCE QUEUE smart_insight CREATEEXTTABLE LOGIN PASSWORD 'cranium';
+COMMENT ON ROLE cranium IS 'Cranium test user';
+
+CREATE DATABASE cranium WITH OWNER=cranium;
+
+CREATE ROLE database_administrator WITH SUPERUSER LOGIN PASSWORD 'emarsys';

data/docker/initialize_greenplum.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "127.0.0.1 $(cat ~/orig_hostname)" >> /etc/hosts
+
+service sshd start
+
+su - gpadmin -c /usr/local/bin/run.sh
+su - gpadmin -c 'cat /docker-entrypoint-initdb.d/create_databases.sql | psql -e'
+su - gpadmin -c 'gpfdist -d /home/gpadmin/gpfdist-data -p 8123'

data/docker-compose.yml

@@ -1,9 +1,10 @@
-greenplum:
-  image: kevinmtrowbridge/gpdb-docker
-  ports:
-    - 5432:5432
-    - 8123:8123
-  volumes:
-    - ./tmp/custdata:/home/gpadmin/gpfdist-data
-    - ./db:/db
-  command: sh /db/start.sh
+services:
+  greenplum:
+    image: kevinmtrowbridge/gpdb-docker
+    ports:
+      - 5432:5432
+      - 8123:8123
+    volumes:
+      - ./tmp/custdata:/home/gpadmin/gpfdist-data
+      - ./db:/db
+    command: sh /db/start.sh

data/features/restore_database_connection.feature

@@ -16,7 +16,7 @@ Feature: Sequel database connections are fault tolerant
     require 'sequel'
 
     def terminate_connections
-      connection = Sequel.connect "postgres://database_administrator:emarsys@#{ENV['DATABASE_HOST'] || '192.168.56.43'}:5432/cranium", loggers: Cranium.configuration.loggers
+      connection = Sequel.connect "postgres://database_administrator:emarsys@#{ENV['GREENPLUM_HOST'] || '192.168.56.43'}:5432/cranium", loggers: Cranium.configuration.loggers
       connection.run("SELECT pg_terminate_backend(procpid) FROM pg_stat_activity WHERE procpid <> pg_backend_pid() AND datname = 'cranium'")
     end
 

data/features/support/env.rb

@@ -5,8 +5,8 @@ require_relative "../../lib/cranium"
 FileUtils.mkdir_p("log") unless Dir.exists?("log")
 
 Cranium.configure do |config|
-  config.greenplum_connection_string = "postgres://cranium:cranium@#{ENV['DATABASE_HOST'] || '192.168.56.43'}:5432/cranium"
-  config.gpfdist_url = "#{ENV['DATABASE_HOST'] || '192.168.56.43'}:8123"
+  config.greenplum_connection_string = "postgres://cranium:cranium@#{ENV['GREENPLUM_HOST'] || '192.168.56.43'}:5432/cranium"
+  config.gpfdist_url = "#{ ENV['GPFDIST_HOST'] || '192.168.56.43'}:8123"
   config.gpfdist_home_directory = "tmp/custdata"
   config.upload_directory = "cranium_build"
   config.loggers << Logger.new("log/cucumber.log")

data/repo-info.json

@@ -0,0 +1,7 @@
+{
+  "is_in_production": true,
+  "is_scannable": true,
+  "is_critical": false,
+  "contact": "g-gsuite-smartinsight@emarsys.com",
+  "hosted": null
+}

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: cranium
 version: !ruby/object:Gem::Version
-  version: '0.8'
+  version: 0.8.1
 platform: ruby
 authors:
 - Emarsys Technologies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-19 00:00:00.000000000 Z
+date: 2021-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -76,30 +76,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -162,23 +162,30 @@ email:
 - smart-insight-dev@emarsys.com
 executables:
 - cranium
+- run_tests.sh
 extensions: []
 extra_rdoc_files: []
 files:
 - ".env"
 - ".gitignore"
 - ".rspec"
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - Vagrantfile
 - bin/cranium
+- bin/run_tests.sh
+- codeship-services.yml
+- codeship-steps.yml
 - config/cucumber.yml
 - cranium.gemspec
 - db/setup.sql
 - db/start.sh
 - docker-compose.yml
+- docker/create_databases.sql
+- docker/initialize_greenplum.sh
 - examples/config.rb
 - examples/deduplication.rb
 - examples/import_csv_with_field_lookup_inserting_new_dimension_keys.rb
@@ -272,6 +279,7 @@ files:
 - lib/cranium/transformation/sequence.rb
 - lib/cranium/transformation_record.rb
 - rake/test.rake
+- repo-info.json
 - spec/cranium/application_spec.rb
 - spec/cranium/archiver_spec.rb
 - spec/cranium/command_line_options_spec.rb
@@ -319,7 +327,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.27
 signing_key: 
 specification_version: 4
 summary: Pure Ruby ETL framework