remi 0.2.21 → 0.2.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 638497273fdec6aeef3e98492848440506d50833
-  data.tar.gz: bcdd635e21fb67975b05b5d0c2255efa70ff0f7a
+  metadata.gz: 6beaa48e9e3760dd0670541bd880e10367b51334
+  data.tar.gz: 7cdc114c661452d3be31e3e5670117739b1f1ab7
 SHA512:
-  metadata.gz: 360b172dabe9b0bbb39b743bddee9ed7ea089644706a98c50859e92300d0b25b5582e934276771f839b99ee6896c723f52ce08c21c490bc6b4406dc6bc329b9b
-  data.tar.gz: 3e7c8e18d29be91314c894d7a6548cb3fa6e7d08af53793f12e390eeaa885cf4f5e82b2ed9539b0b4f9c25e0db23fe95af641ffa276fbd5833a332e061c3800e
+  metadata.gz: b6386ab9a5b07c5086b462169b5e517a09009b38fdd7c1aabcca2be85be817e5ffa21f0a7dc2a727f40d667af2f034bc1037d04999561859cbefe12129663fff
+  data.tar.gz: 67b62fc317e4c2fd8b013b309d04a3ccaf644ac3b286447f98673fde84f9a301461207f4a72cb07417d4206bb7491507d801814bc07d976ceb900949ab672c25

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    remi (0.2.21)
+    remi (0.2.22)
       activesupport (~> 4.2)
       bond (~> 0.5)
       cucumber (~> 2.1)

data/features/step_definitions/remi_step.rb

@@ -483,7 +483,19 @@ Then /^the target field contains a unique value matching the pattern \/(.*)\/$/
   end
 end
 
+Then /^the source field '([^']+)' is truncated to (\d+) characters and loaded into the target field '([^']+)'$/ do |source_field, character_limit, target_field|
+  step "the target field '#{target_field}'"
+  step "the source field '#{source_field}'"
+
+  source_name, source_field_name = @brt.sources.parse_full_field(source_field)
+  target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
 
+  truncated_source = @brt.sources[source_name].fields[source_field_name].value.slice(0, character_limit.to_i)
+  @brt.run_transforms
+  Array(target_names).each do |target_name|
+    expect(@brt.targets[target_name].fields[target_field_name].value).to eq truncated_source
+  end
+end
 
 ### Field presence
 

data/features/transforms/truncate.feature

@@ -12,8 +12,10 @@ Feature: Test the truncate transformer.
 
     And the source field 'My Field' is set to the value "something"
     Then the target field 'Truncated Field' is set to the value "somet"
+    Then the source field 'My Field' is truncated to 5 characters and loaded into the target field 'Truncated Field'
 
     And the job parameter 'truncate_len' is "7"
 
     And the source field 'My Field' is set to the value "something"
     Then the target field 'Truncated Field' is set to the value "somethi"
+    Then the source field 'My Field' is truncated to 7 characters and loaded into the target field 'Truncated Field'

data/lib/remi.rb

@@ -58,5 +58,6 @@ require 'remi/data_target/data_frame'
 require 'remi/data_target/salesforce'
 require 'remi/data_target/csv_file'
 require 'remi/data_target/sftp_file'
+require 'remi/data_target/postgres'
 
 require 'remi/transform'

data/lib/remi/data_source/postgres.rb

@@ -14,21 +14,21 @@ module Remi
 
       def extract
         @logger.info "Executing query #{@query}"
-        @raw_result = pg_conn.exec @query
+        @raw_result = connection.exec @query
       end
 
       def raw_result
         @raw_result ||= extract
       end
 
-      def pg_conn
-        @pg_conn ||= PG.connect(
+      def connection
+        @connection ||= PG.connect(
           host:     @credentials[:host] || 'localhost',
           port:     @credentials[:port] || 5432,
           dbname:   @credentials[:dbname],
           user:     @credentials[:user] || `whoami`.chomp,
           password: @credentials[:password],
-          sslmode:  @credentials[:sslmode] || 'require'
+          sslmode:  @credentials[:sslmode] || 'allow'
         )
       end
 

data/lib/remi/data_target/postgres.rb

@@ -0,0 +1,74 @@
+module Remi
+  module DataTarget
+    class Postgres
+      include DataTarget
+
+      def initialize(credentials:, table_name:, fields:, logger: Remi::Settings.logger)
+        @credentials = credentials
+        @table_name = table_name
+        @fields = fields
+        @logger = logger
+      end
+
+      def load
+        return true if @loaded || df.size == 0
+
+        @logger.info "Performing postgres load to table #{@table_name}"
+        create_target_table
+        load_target_table
+
+        @loaded = true
+      end
+
+
+      def connection
+        @connection ||= PG.connect(
+          host:     @credentials[:host] || 'localhost',
+          port:     @credentials[:port] || 5432,
+          dbname:   @credentials[:dbname],
+          user:     @credentials[:user] || `whoami`.chomp,
+          password: @credentials[:password],
+          sslmode:  @credentials[:sslmode] || 'allow'
+        )
+      end
+
+
+      def fields_with_type_ddl
+        @fields.map { |k,v| "#{k} #{v[:type]}" }.join(', ')
+      end
+
+      def create_target_table
+        connection.exec <<-EOT
+          CREATE TEMPORARY TABLE #{@table_name} (
+            #{fields_with_type_ddl}
+          )
+        EOT
+      end
+
+      def load_target_table
+        connection.copy_data "COPY #{@table_name} (#{@fields.keys.join(', ')}) FROM STDIN" do
+          df.each(:row) do |row|
+            row_str = @fields.keys.map do |field|
+              field = row[field]
+              case
+              when field.respond_to?(:strftime)
+                field.strftime('%Y-%m-%d %H:%M:%S')
+              when field.respond_to?(:map)
+                field.to_json.gsub("\t", '\t')
+              when field.blank? && !field.nil?
+                ''
+              when field.nil?
+                '\N'
+              else
+                field.to_s.gsub("\t", '\t')
+              end
+            end.join("\t")
+
+            connection.put_copy_data row_str + "\n"
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.21'
+  VERSION = '0.2.22'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.21
+  version: 0.2.22
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-21 00:00:00.000000000 Z
+date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daru
@@ -266,6 +266,7 @@ files:
 - lib/remi/data_target.rb
 - lib/remi/data_target/csv_file.rb
 - lib/remi/data_target/data_frame.rb
+- lib/remi/data_target/postgres.rb
 - lib/remi/data_target/salesforce.rb
 - lib/remi/data_target/sftp_file.rb
 - lib/remi/extractor/sftp_file.rb