remi 0.2.21 → 0.2.22

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 638497273fdec6aeef3e98492848440506d50833
4
- data.tar.gz: bcdd635e21fb67975b05b5d0c2255efa70ff0f7a
3
+ metadata.gz: 6beaa48e9e3760dd0670541bd880e10367b51334
4
+ data.tar.gz: 7cdc114c661452d3be31e3e5670117739b1f1ab7
5
5
  SHA512:
6
- metadata.gz: 360b172dabe9b0bbb39b743bddee9ed7ea089644706a98c50859e92300d0b25b5582e934276771f839b99ee6896c723f52ce08c21c490bc6b4406dc6bc329b9b
7
- data.tar.gz: 3e7c8e18d29be91314c894d7a6548cb3fa6e7d08af53793f12e390eeaa885cf4f5e82b2ed9539b0b4f9c25e0db23fe95af641ffa276fbd5833a332e061c3800e
6
+ metadata.gz: b6386ab9a5b07c5086b462169b5e517a09009b38fdd7c1aabcca2be85be817e5ffa21f0a7dc2a727f40d667af2f034bc1037d04999561859cbefe12129663fff
7
+ data.tar.gz: 67b62fc317e4c2fd8b013b309d04a3ccaf644ac3b286447f98673fde84f9a301461207f4a72cb07417d4206bb7491507d801814bc07d976ceb900949ab672c25
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- remi (0.2.21)
4
+ remi (0.2.22)
5
5
  activesupport (~> 4.2)
6
6
  bond (~> 0.5)
7
7
  cucumber (~> 2.1)
@@ -483,7 +483,19 @@ Then /^the target field contains a unique value matching the pattern \/(.*)\/$/
483
483
  end
484
484
  end
485
485
 
486
+ Then /^the source field '([^']+)' is truncated to (\d+) characters and loaded into the target field '([^']+)'$/ do |source_field, character_limit, target_field|
487
+ step "the target field '#{target_field}'"
488
+ step "the source field '#{source_field}'"
489
+
490
+ source_name, source_field_name = @brt.sources.parse_full_field(source_field)
491
+ target_names, target_field_name = @brt.targets.parse_full_field(target_field, multi: true)
486
492
 
493
+ truncated_source = @brt.sources[source_name].fields[source_field_name].value.slice(0, character_limit.to_i)
494
+ @brt.run_transforms
495
+ Array(target_names).each do |target_name|
496
+ expect(@brt.targets[target_name].fields[target_field_name].value).to eq truncated_source
497
+ end
498
+ end
487
499
 
488
500
  ### Field presence
489
501
 
@@ -12,8 +12,10 @@ Feature: Test the truncate transformer.
12
12
 
13
13
  And the source field 'My Field' is set to the value "something"
14
14
  Then the target field 'Truncated Field' is set to the value "somet"
15
+ Then the source field 'My Field' is truncated to 5 characters and loaded into the target field 'Truncated Field'
15
16
 
16
17
  And the job parameter 'truncate_len' is "7"
17
18
 
18
19
  And the source field 'My Field' is set to the value "something"
19
20
  Then the target field 'Truncated Field' is set to the value "somethi"
21
+ Then the source field 'My Field' is truncated to 7 characters and loaded into the target field 'Truncated Field'
data/lib/remi.rb CHANGED
@@ -58,5 +58,6 @@ require 'remi/data_target/data_frame'
58
58
  require 'remi/data_target/salesforce'
59
59
  require 'remi/data_target/csv_file'
60
60
  require 'remi/data_target/sftp_file'
61
+ require 'remi/data_target/postgres'
61
62
 
62
63
  require 'remi/transform'
@@ -14,21 +14,21 @@ module Remi
14
14
 
15
15
  def extract
16
16
  @logger.info "Executing query #{@query}"
17
- @raw_result = pg_conn.exec @query
17
+ @raw_result = connection.exec @query
18
18
  end
19
19
 
20
20
  def raw_result
21
21
  @raw_result ||= extract
22
22
  end
23
23
 
24
- def pg_conn
25
- @pg_conn ||= PG.connect(
24
+ def connection
25
+ @connection ||= PG.connect(
26
26
  host: @credentials[:host] || 'localhost',
27
27
  port: @credentials[:port] || 5432,
28
28
  dbname: @credentials[:dbname],
29
29
  user: @credentials[:user] || `whoami`.chomp,
30
30
  password: @credentials[:password],
31
- sslmode: @credentials[:sslmode] || 'require'
31
+ sslmode: @credentials[:sslmode] || 'allow'
32
32
  )
33
33
  end
34
34
 
@@ -0,0 +1,74 @@
1
+ module Remi
2
+ module DataTarget
3
+ class Postgres
4
+ include DataTarget
5
+
6
+ def initialize(credentials:, table_name:, fields:, logger: Remi::Settings.logger)
7
+ @credentials = credentials
8
+ @table_name = table_name
9
+ @fields = fields
10
+ @logger = logger
11
+ end
12
+
13
+ def load
14
+ return true if @loaded || df.size == 0
15
+
16
+ @logger.info "Performing postgres load to table #{@table_name}"
17
+ create_target_table
18
+ load_target_table
19
+
20
+ @loaded = true
21
+ end
22
+
23
+
24
+ def connection
25
+ @connection ||= PG.connect(
26
+ host: @credentials[:host] || 'localhost',
27
+ port: @credentials[:port] || 5432,
28
+ dbname: @credentials[:dbname],
29
+ user: @credentials[:user] || `whoami`.chomp,
30
+ password: @credentials[:password],
31
+ sslmode: @credentials[:sslmode] || 'allow'
32
+ )
33
+ end
34
+
35
+
36
+ def fields_with_type_ddl
37
+ @fields.map { |k,v| "#{k} #{v[:type]}" }.join(', ')
38
+ end
39
+
40
+ def create_target_table
41
+ connection.exec <<-EOT
42
+ CREATE TEMPORARY TABLE #{@table_name} (
43
+ #{fields_with_type_ddl}
44
+ )
45
+ EOT
46
+ end
47
+
48
+ def load_target_table
49
+ connection.copy_data "COPY #{@table_name} (#{@fields.keys.join(', ')}) FROM STDIN" do
50
+ df.each(:row) do |row|
51
+ row_str = @fields.keys.map do |field|
52
+ field = row[field]
53
+ case
54
+ when field.respond_to?(:strftime)
55
+ field.strftime('%Y-%m-%d %H:%M:%S')
56
+ when field.respond_to?(:map)
57
+ field.to_json.gsub("\t", '\t')
58
+ when field.blank? && !field.nil?
59
+ ''
60
+ when field.nil?
61
+ '\N'
62
+ else
63
+ field.to_s.gsub("\t", '\t')
64
+ end
65
+ end.join("\t")
66
+
67
+ connection.put_copy_data row_str + "\n"
68
+ end
69
+ end
70
+ end
71
+
72
+ end
73
+ end
74
+ end
data/lib/remi/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Remi
2
- VERSION = '0.2.21'
2
+ VERSION = '0.2.22'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: remi
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.21
4
+ version: 0.2.22
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sterling Paramore
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-21 00:00:00.000000000 Z
11
+ date: 2016-03-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: daru
@@ -266,6 +266,7 @@ files:
266
266
  - lib/remi/data_target.rb
267
267
  - lib/remi/data_target/csv_file.rb
268
268
  - lib/remi/data_target/data_frame.rb
269
+ - lib/remi/data_target/postgres.rb
269
270
  - lib/remi/data_target/salesforce.rb
270
271
  - lib/remi/data_target/sftp_file.rb
271
272
  - lib/remi/extractor/sftp_file.rb