arc-furnace 0.1.41 → 0.1.42
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +1 -0
- data/Gemfile.lock +6 -6
- data/lib/arc-furnace/merge.rb +23 -0
- data/lib/arc-furnace/multi_excel_source.rb +7 -7
- data/lib/arc-furnace/nodes.rb +1 -0
- data/lib/arc-furnace/pipeline.rb +11 -3
- data/lib/arc-furnace/version.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 55f7e36acf42d49de0f226a1350f0a955fcfd41f4de577f5bb50c4804559def3
|
4
|
+
data.tar.gz: 2ceeea42ca17ecb2d770354547fface75fc5b2fc81caac5296bbc989f57ac1d1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 57a3687b1a806ba5da7a138de0af10f9a940396a87d2d73f08d276661f9ae8b5c6fe356b46d20390e12364a5ac9df50e833d8d8a1233403672b9b930a9ad8ea7
|
7
|
+
data.tar.gz: 132fc204cd143d6fcf29c60eb49d0cf101b884ccfbe3d55941f02294c4c2b83e31bba924b90a7558e6e2a3743778623582c21f0f56196649735e2837b217c9cc
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
arc-furnace (0.1.
|
4
|
+
arc-furnace (0.1.42)
|
5
5
|
activesupport
|
6
6
|
axlsx
|
7
7
|
eigenclass
|
@@ -30,11 +30,11 @@ GEM
|
|
30
30
|
concurrent-ruby (~> 1.0)
|
31
31
|
ice_nine (0.11.2)
|
32
32
|
method_source (0.9.0)
|
33
|
-
mini_portile2 (2.
|
33
|
+
mini_portile2 (2.4.0)
|
34
34
|
minitest (5.11.3)
|
35
35
|
msgpack (1.2.4)
|
36
|
-
nokogiri (1.
|
37
|
-
mini_portile2 (~> 2.
|
36
|
+
nokogiri (1.9.1)
|
37
|
+
mini_portile2 (~> 2.4.0)
|
38
38
|
pry (0.11.3)
|
39
39
|
coderay (~> 1.1.0)
|
40
40
|
method_source (~> 0.9.0)
|
@@ -55,7 +55,7 @@ GEM
|
|
55
55
|
diff-lcs (>= 1.2.0, < 2.0)
|
56
56
|
rspec-support (~> 3.7.0)
|
57
57
|
rspec-support (3.7.1)
|
58
|
-
rubyzip (1.2.
|
58
|
+
rubyzip (1.2.2)
|
59
59
|
thread_safe (0.3.6)
|
60
60
|
tzinfo (1.2.5)
|
61
61
|
thread_safe (~> 0.1)
|
@@ -73,4 +73,4 @@ DEPENDENCIES
|
|
73
73
|
rspec
|
74
74
|
|
75
75
|
BUNDLED WITH
|
76
|
-
1.
|
76
|
+
1.17.2
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'arc-furnace/source'
|
2
|
+
|
3
|
+
module ArcFurnace
|
4
|
+
class Merge < Source
|
5
|
+
|
6
|
+
private_attr_reader :sources
|
7
|
+
|
8
|
+
def initialize(sources:)
|
9
|
+
@sources = sources
|
10
|
+
end
|
11
|
+
|
12
|
+
def advance
|
13
|
+
sources.first.advance
|
14
|
+
end
|
15
|
+
|
16
|
+
def value
|
17
|
+
value = sources.map(&:value).compact
|
18
|
+
return unless value.present?
|
19
|
+
value.map(&:deep_dup).reduce({}, :merge)
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -6,7 +6,7 @@ module ArcFurnace
|
|
6
6
|
|
7
7
|
private_attr_reader :enumerator, :header_row
|
8
8
|
attr_reader :value, :excel, :sheets_info_array
|
9
|
-
|
9
|
+
|
10
10
|
# Sheets is in the format of:
|
11
11
|
# [
|
12
12
|
# { filename: 'foo.xlsx', sheet: 'sheet name' },
|
@@ -33,12 +33,12 @@ module ArcFurnace
|
|
33
33
|
|
34
34
|
def advance_in_current_file
|
35
35
|
@value =
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
36
|
+
begin
|
37
|
+
enumerator.next if enumerator
|
38
|
+
rescue StopIteration
|
39
|
+
@enumerator = nil
|
40
|
+
nil
|
41
|
+
end
|
42
42
|
value
|
43
43
|
end
|
44
44
|
|
data/lib/arc-furnace/nodes.rb
CHANGED
@@ -15,6 +15,7 @@ require 'arc-furnace/hash_source'
|
|
15
15
|
require 'arc-furnace/array_source'
|
16
16
|
require 'arc-furnace/hash'
|
17
17
|
require 'arc-furnace/inner_join'
|
18
|
+
require 'arc-furnace/merge'
|
18
19
|
require 'arc-furnace/merging_hash'
|
19
20
|
require 'arc-furnace/multi_csv_source'
|
20
21
|
require 'arc-furnace/node'
|
data/lib/arc-furnace/pipeline.rb
CHANGED
@@ -17,7 +17,7 @@ module ArcFurnace
|
|
17
17
|
# Define the sink for this transformation. Only a single sink may be
|
18
18
|
# specified per transformation. The sink is delivered a hash per row or
|
19
19
|
# entity, and feeds them from the graph of nodes above it.
|
20
|
-
def self.sink(type
|
20
|
+
def self.sink(type:, source:, params:)
|
21
21
|
if sink_node
|
22
22
|
raise 'Sink already defined!'
|
23
23
|
end
|
@@ -47,7 +47,7 @@ module ArcFurnace
|
|
47
47
|
define_intermediate(node_id, type: type, params: params)
|
48
48
|
end
|
49
49
|
|
50
|
-
# Define an outer join
|
50
|
+
# Define an outer join node where rows from the source are kept
|
51
51
|
# even if an associated entity is not found in the hash for the join key
|
52
52
|
def self.outer_join(node_id, type: ArcFurnace::OuterJoin, params:)
|
53
53
|
define_intermediate(node_id, type: type, params: params)
|
@@ -65,6 +65,12 @@ module ArcFurnace
|
|
65
65
|
define_intermediate(node_id, type: type, params: params)
|
66
66
|
end
|
67
67
|
|
68
|
+
# Define a merge node where rows from multiple source nodes are merged
|
69
|
+
# into a single row
|
70
|
+
def self.merge(node_id, type: ArcFurnace::Merge, params:)
|
71
|
+
define_intermediate(node_id, type: type, params: params)
|
72
|
+
end
|
73
|
+
|
68
74
|
# Define a node that unfolds rows. By default you get a BlockUnfold
|
69
75
|
# (and when this metaprogramming method is passed a block) that will be passed
|
70
76
|
# a hash for each row. The result of the block becomes the set of rows for the next
|
@@ -173,7 +179,9 @@ module ArcFurnace
|
|
173
179
|
def resolve_parameters(node_id, params_to_resolve)
|
174
180
|
params_to_resolve.each_with_object({}) do |(key, value), result|
|
175
181
|
result[key] =
|
176
|
-
if
|
182
|
+
if key == :sources
|
183
|
+
value.map { |_value| resolve_parameter(node_id, _value) }
|
184
|
+
elsif value.is_a?(Symbol)
|
177
185
|
# Allow resolution of intermediates
|
178
186
|
resolve_parameter(node_id, value)
|
179
187
|
elsif value.nil?
|
data/lib/arc-furnace/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arc-furnace
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.42
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Spangenberger
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2020-03-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -193,6 +193,7 @@ files:
|
|
193
193
|
- lib/arc-furnace/hash_source.rb
|
194
194
|
- lib/arc-furnace/inner_join.rb
|
195
195
|
- lib/arc-furnace/logging_error_handler.rb
|
196
|
+
- lib/arc-furnace/merge.rb
|
196
197
|
- lib/arc-furnace/merging_hash.rb
|
197
198
|
- lib/arc-furnace/multi_csv_source.rb
|
198
199
|
- lib/arc-furnace/multi_excel_source.rb
|
@@ -231,8 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
231
232
|
- !ruby/object:Gem::Version
|
232
233
|
version: '0'
|
233
234
|
requirements: []
|
234
|
-
|
235
|
-
rubygems_version: 2.4.5.1
|
235
|
+
rubygems_version: 3.0.1
|
236
236
|
signing_key:
|
237
237
|
specification_version: 4
|
238
238
|
summary: Melds and transforms data from multiple sources into a single stream
|