beetle_etl 2.0.3 → 2.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/lib/beetle_etl.rb +1 -1
- data/lib/beetle_etl/import.rb +3 -3
- data/lib/beetle_etl/step_runner/async_step_runner.rb +1 -11
- data/lib/beetle_etl/step_runner/{abstract_step_runner.rb → step_runner.rb} +22 -3
- data/lib/beetle_etl/steps/table_diff.rb +6 -5
- data/lib/beetle_etl/version.rb +1 -1
- metadata +4 -5
- data/lib/beetle_etl/step_runner/sequential_step_runner.rb +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 13376260ac59a025f892af7ab4c47346ef7bd399
|
4
|
+
data.tar.gz: e5e0bcad031552006cc3e650725b245e121acc25
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e06aa20a649cd4caf061dc9f847df5446c5d09b1f60b86ecc9c450cb1de3670deff56564e9636e86865748e6b2fcf1697832d6464fd6f5d07d16ad6af987d225
|
7
|
+
data.tar.gz: 68e3004d46baf5d6f1d9eaf9e88f79ff6617e00ce013bb0b7faad89e9b175bd7827f02f6cd82a45f64acf9a1e8e08bcb3f40bd8fe3c5811c5d52256a3df5fa5e
|
data/.travis.yml
CHANGED
data/lib/beetle_etl.rb
CHANGED
@@ -22,7 +22,7 @@ module BeetleETL
|
|
22
22
|
require 'beetle_etl/steps/load'
|
23
23
|
require 'beetle_etl/steps/drop_stage'
|
24
24
|
|
25
|
-
require 'beetle_etl/step_runner/
|
25
|
+
require 'beetle_etl/step_runner/step_runner'
|
26
26
|
require 'beetle_etl/step_runner/async_step_runner'
|
27
27
|
|
28
28
|
require 'beetle_etl/import'
|
data/lib/beetle_etl/import.rb
CHANGED
@@ -27,7 +27,7 @@ module BeetleETL
|
|
27
27
|
CreateStage.new(@config, t.table_name, t.relations, t.column_names)
|
28
28
|
}
|
29
29
|
|
30
|
-
@report.deep_merge
|
30
|
+
@report.deep_merge StepRunner.new(@config, steps).run
|
31
31
|
end
|
32
32
|
|
33
33
|
def run_transform
|
@@ -48,7 +48,7 @@ module BeetleETL
|
|
48
48
|
}
|
49
49
|
|
50
50
|
result = @config.database.transaction do
|
51
|
-
|
51
|
+
StepRunner.new(@config, steps).run
|
52
52
|
end
|
53
53
|
|
54
54
|
@report.deep_merge result
|
@@ -59,7 +59,7 @@ module BeetleETL
|
|
59
59
|
DropStage.new(@config, t.table_name)
|
60
60
|
}
|
61
61
|
|
62
|
-
@report.deep_merge
|
62
|
+
@report.deep_merge StepRunner.new(@config, steps).run
|
63
63
|
end
|
64
64
|
|
65
65
|
end
|
@@ -1,16 +1,10 @@
|
|
1
|
-
require_relative './abstract_step_runner'
|
2
|
-
require_relative './dependency_resolver'
|
3
|
-
|
4
1
|
module BeetleETL
|
5
|
-
class AsyncStepRunner <
|
2
|
+
class AsyncStepRunner < StepRunner
|
6
3
|
|
7
4
|
def initialize(config, steps)
|
8
5
|
super(config, steps)
|
9
6
|
|
10
|
-
@dependency_resolver = DependencyResolver.new(steps)
|
11
|
-
|
12
7
|
@queue = Queue.new
|
13
|
-
@completed = Set.new
|
14
8
|
@started = Set.new
|
15
9
|
end
|
16
10
|
|
@@ -45,9 +39,5 @@ module BeetleETL
|
|
45
39
|
resolvables.reject { |r| @started.include? r.name }
|
46
40
|
end
|
47
41
|
|
48
|
-
def all_steps_complete?
|
49
|
-
@steps.map(&:name).to_set == @completed.to_set
|
50
|
-
end
|
51
|
-
|
52
42
|
end
|
53
43
|
end
|
@@ -1,15 +1,26 @@
|
|
1
|
-
|
1
|
+
require_relative './dependency_resolver'
|
2
2
|
|
3
3
|
module BeetleETL
|
4
|
-
class
|
4
|
+
class StepRunner
|
5
5
|
|
6
6
|
def initialize(config, steps)
|
7
7
|
@config = config
|
8
8
|
@steps = steps
|
9
|
+
@dependency_resolver = DependencyResolver.new(steps)
|
10
|
+
@completed = Set.new
|
9
11
|
end
|
10
12
|
|
11
13
|
def run
|
12
|
-
|
14
|
+
results = {}
|
15
|
+
|
16
|
+
until all_steps_complete?
|
17
|
+
runnables.each do |step|
|
18
|
+
add_result!(results, run_step(step))
|
19
|
+
@completed.add(step.name)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
results
|
13
24
|
end
|
14
25
|
|
15
26
|
private
|
@@ -35,6 +46,10 @@ module BeetleETL
|
|
35
46
|
raise e
|
36
47
|
end
|
37
48
|
|
49
|
+
def runnables
|
50
|
+
@dependency_resolver.resolvables(@completed)
|
51
|
+
end
|
52
|
+
|
38
53
|
def add_result!(results, step_data)
|
39
54
|
table_name = step_data[:table_name]
|
40
55
|
step_name = step_data[:step_name]
|
@@ -43,5 +58,9 @@ module BeetleETL
|
|
43
58
|
results[table_name][step_name] = step_data.slice(:started_at, :finished_at)
|
44
59
|
end
|
45
60
|
|
61
|
+
def all_steps_complete?
|
62
|
+
@steps.map(&:name).to_set == @completed.to_set
|
63
|
+
end
|
64
|
+
|
46
65
|
end
|
47
66
|
end
|
@@ -18,16 +18,17 @@ module BeetleETL
|
|
18
18
|
|
19
19
|
def transition_create
|
20
20
|
database.execute <<-SQL
|
21
|
-
UPDATE "#{target_schema}"."#{stage_table_name}"
|
21
|
+
UPDATE "#{target_schema}"."#{stage_table_name}" stage_update
|
22
22
|
SET
|
23
23
|
transition = 'CREATE',
|
24
24
|
id = NEXTVAL('#{target_schema}.#{table_name}_id_seq')
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
WHERE target.external_id = stage.external_id
|
25
|
+
FROM "#{target_schema}"."#{stage_table_name}" stage
|
26
|
+
LEFT JOIN "#{target_schema}"."#{table_name}" target ON (
|
27
|
+
target.external_id = stage.external_id
|
29
28
|
AND target.external_source = '#{external_source}'
|
30
29
|
)
|
30
|
+
WHERE stage_update.external_id = stage.external_id
|
31
|
+
AND target.external_id IS NULL
|
31
32
|
SQL
|
32
33
|
end
|
33
34
|
|
data/lib/beetle_etl/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle_etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luciano Maiwald
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-09-
|
11
|
+
date: 2017-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -132,10 +132,9 @@ files:
|
|
132
132
|
- lib/beetle_etl/import.rb
|
133
133
|
- lib/beetle_etl/naming.rb
|
134
134
|
- lib/beetle_etl/reporter.rb
|
135
|
-
- lib/beetle_etl/step_runner/abstract_step_runner.rb
|
136
135
|
- lib/beetle_etl/step_runner/async_step_runner.rb
|
137
136
|
- lib/beetle_etl/step_runner/dependency_resolver.rb
|
138
|
-
- lib/beetle_etl/step_runner/
|
137
|
+
- lib/beetle_etl/step_runner/step_runner.rb
|
139
138
|
- lib/beetle_etl/steps/create_stage.rb
|
140
139
|
- lib/beetle_etl/steps/drop_stage.rb
|
141
140
|
- lib/beetle_etl/steps/load.rb
|
@@ -191,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
190
|
version: '0'
|
192
191
|
requirements: []
|
193
192
|
rubyforge_project:
|
194
|
-
rubygems_version: 2.
|
193
|
+
rubygems_version: 2.6.13
|
195
194
|
signing_key:
|
196
195
|
specification_version: 4
|
197
196
|
summary: BeetleETL helps you with your recurring ETL imports.
|