beetle_etl 2.0.4 → 2.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +8 -9
- data/LICENSE.txt +1 -1
- data/beetle_etl.gemspec +1 -3
- data/lib/beetle_etl/import.rb +8 -6
- data/lib/beetle_etl/step_runner/step_runner.rb +4 -1
- data/lib/beetle_etl/version.rb +1 -1
- data/spec/feature/feature_spec.rb +3 -6
- data/spec/steps/load_spec.rb +7 -10
- data/spec/steps/table_diff_spec.rb +16 -17
- metadata +4 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1edda8a1e337f1aa6f17031c95d6eb8b1672c2a1
|
4
|
+
data.tar.gz: bd24f5983ecafd0e08b0573e50e690fdd90ca989
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7318b99966f73b114196f5d0f371220557caf54870c74426e1468178ae63afc4f318a5292922d86977e9d8b028fd12e15d3cf6409c5695e7c7775878cc3e4894
|
7
|
+
data.tar.gz: 5db98f072ccf66e5a373f443d2ce4e9a5134592abfd196b75173413e3c84cc08babf106af5da6c97806481cd9d54217d7cf6b4a80943491bfd23312a1a75ce79
|
data/.travis.yml
CHANGED
@@ -1,14 +1,13 @@
|
|
1
1
|
language: ruby
|
2
2
|
rvm:
|
3
|
-
- 2.0
|
4
|
-
- 2.2
|
5
|
-
- 2.3
|
6
|
-
- 2.4
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
-
|
11
|
-
- gem update bundler
|
3
|
+
- 2.0
|
4
|
+
- 2.2
|
5
|
+
- 2.3
|
6
|
+
- 2.4
|
7
|
+
- 2.6
|
8
|
+
|
9
|
+
services:
|
10
|
+
- postgresql
|
12
11
|
|
13
12
|
before_script:
|
14
13
|
- psql -c 'create database travis_ci_test;' -U postgres
|
data/LICENSE.txt
CHANGED
data/beetle_etl.gemspec
CHANGED
@@ -18,10 +18,8 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
-
spec.add_runtime_dependency 'activesupport', '>= 4.0.0'
|
22
|
-
|
23
21
|
spec.add_development_dependency 'sequel', '>= 4.0.0'
|
24
|
-
spec.add_development_dependency 'bundler', '
|
22
|
+
spec.add_development_dependency 'bundler', '>= 1.11'
|
25
23
|
spec.add_development_dependency 'rspec', '>= 3.0.0'
|
26
24
|
spec.add_development_dependency 'timecop', '>= 0.7.0'
|
27
25
|
spec.add_development_dependency 'pg', '>= 0.18.0'
|
data/lib/beetle_etl/import.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'active_support/core_ext/hash/deep_merge'
|
2
|
-
|
3
1
|
module BeetleETL
|
4
2
|
class Import
|
5
3
|
|
@@ -27,7 +25,7 @@ module BeetleETL
|
|
27
25
|
CreateStage.new(@config, t.table_name, t.relations, t.column_names)
|
28
26
|
}
|
29
27
|
|
30
|
-
|
28
|
+
merge_report! StepRunner.new(@config, steps).run
|
31
29
|
end
|
32
30
|
|
33
31
|
def run_transform
|
@@ -39,7 +37,7 @@ module BeetleETL
|
|
39
37
|
]
|
40
38
|
}
|
41
39
|
|
42
|
-
|
40
|
+
merge_report! AsyncStepRunner.new(@config, steps).run
|
43
41
|
end
|
44
42
|
|
45
43
|
def run_load
|
@@ -51,7 +49,7 @@ module BeetleETL
|
|
51
49
|
StepRunner.new(@config, steps).run
|
52
50
|
end
|
53
51
|
|
54
|
-
|
52
|
+
merge_report! result
|
55
53
|
end
|
56
54
|
|
57
55
|
def run_cleanup
|
@@ -59,7 +57,11 @@ module BeetleETL
|
|
59
57
|
DropStage.new(@config, t.table_name)
|
60
58
|
}
|
61
59
|
|
62
|
-
|
60
|
+
merge_report! StepRunner.new(@config, steps).run
|
61
|
+
end
|
62
|
+
|
63
|
+
def merge_report!(new_report)
|
64
|
+
@report.merge!(new_report) { |key, oldval, newval| oldval.merge(newval) }
|
63
65
|
end
|
64
66
|
|
65
67
|
end
|
@@ -55,7 +55,10 @@ module BeetleETL
|
|
55
55
|
step_name = step_data[:step_name]
|
56
56
|
|
57
57
|
results[table_name] ||= {}
|
58
|
-
results[table_name][step_name] =
|
58
|
+
results[table_name][step_name] = {
|
59
|
+
started_at: step_data[:started_at],
|
60
|
+
finished_at: step_data[:finished_at]
|
61
|
+
}
|
59
62
|
end
|
60
63
|
|
61
64
|
def all_steps_complete?
|
data/lib/beetle_etl/version.rb
CHANGED
@@ -4,18 +4,15 @@ require 'timecop'
|
|
4
4
|
require_relative 'example_schema'
|
5
5
|
require 'yaml'
|
6
6
|
|
7
|
-
require 'active_support/core_ext/date/calculations'
|
8
|
-
require 'active_support/core_ext/numeric/time'
|
9
|
-
|
10
7
|
Thread::abort_on_exception = true
|
11
8
|
|
12
9
|
describe BeetleETL do
|
13
10
|
|
14
11
|
include ExampleSchema
|
15
12
|
|
16
|
-
let!(:time1) { Time.new(2014 , 7 , 17 , 16 , 12)
|
17
|
-
let!(:time2) { Time.new(2015 , 2 , 8 , 22 , 18)
|
18
|
-
let!(:time3) { Time.new(2015 , 11 , 3 , 12 , 17)
|
13
|
+
let!(:time1) { Time.new(2014 , 7 , 17 , 16 , 12) }
|
14
|
+
let!(:time2) { Time.new(2015 , 2 , 8 , 22 , 18) }
|
15
|
+
let!(:time3) { Time.new(2015 , 11 , 3 , 12 , 17) }
|
19
16
|
|
20
17
|
before :each do
|
21
18
|
create_tables
|
data/spec/steps/load_spec.rb
CHANGED
@@ -1,15 +1,12 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
require 'active_support/core_ext/date/calculations'
|
4
|
-
require 'active_support/core_ext/numeric/time'
|
5
|
-
|
6
3
|
module BeetleETL
|
7
4
|
describe Load do
|
8
5
|
|
9
6
|
let(:external_source) { 'my_source' }
|
10
7
|
|
11
|
-
let(:
|
12
|
-
let(:yesterday) {
|
8
|
+
let(:today) { Time.new(2020, 3, 24) }
|
9
|
+
let(:yesterday) { Time.new(2020, 3, 23) }
|
13
10
|
|
14
11
|
let(:config) do
|
15
12
|
Configuration.new.tap do |c|
|
@@ -21,7 +18,7 @@ module BeetleETL
|
|
21
18
|
subject { Load.new(config, :example_table, []) }
|
22
19
|
|
23
20
|
before do
|
24
|
-
allow(subject).to receive(:now) {
|
21
|
+
allow(subject).to receive(:now) { today }
|
25
22
|
|
26
23
|
test_database.create_schema(:stage)
|
27
24
|
test_database.create_table(subject.stage_table_name.to_sym) do
|
@@ -86,7 +83,7 @@ module BeetleETL
|
|
86
83
|
|
87
84
|
expect(:example_table).to have_values(
|
88
85
|
[ :id , :external_id , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload ] ,
|
89
|
-
[ 3 , 'external_id' , external_source , 22 ,
|
86
|
+
[ 3 , 'external_id' , external_source , 22 , today , today , nil , 'content' ] ,
|
90
87
|
)
|
91
88
|
end
|
92
89
|
end
|
@@ -107,7 +104,7 @@ module BeetleETL
|
|
107
104
|
|
108
105
|
expect(:example_table).to have_values(
|
109
106
|
[ :id , :external_id , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload ] ,
|
110
|
-
[ 1 , 'external_id' , external_source , 33 , yesterday ,
|
107
|
+
[ 1 , 'external_id' , external_source , 33 , yesterday , today , nil , 'updated content' ] ,
|
111
108
|
)
|
112
109
|
end
|
113
110
|
|
@@ -126,7 +123,7 @@ module BeetleETL
|
|
126
123
|
|
127
124
|
expect(:example_table).to have_values(
|
128
125
|
[ :id , :external_id , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload ] ,
|
129
|
-
[ 1 , 'external_id' , external_source , 33 , yesterday ,
|
126
|
+
[ 1 , 'external_id' , external_source , 33 , yesterday , today , nil , 'updated content' ] ,
|
130
127
|
)
|
131
128
|
end
|
132
129
|
end
|
@@ -147,7 +144,7 @@ module BeetleETL
|
|
147
144
|
|
148
145
|
expect(:example_table).to have_values(
|
149
146
|
[ :id , :external_id , :external_source , :foo_id , :created_at , :updated_at , :deleted_at , :payload ] ,
|
150
|
-
[ 1 , 'external_id' , external_source , 22 , yesterday ,
|
147
|
+
[ 1 , 'external_id' , external_source , 22 , yesterday , today , today , 'content' ] ,
|
151
148
|
)
|
152
149
|
end
|
153
150
|
end
|
@@ -1,8 +1,5 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
require 'active_support/core_ext/date/calculations'
|
4
|
-
require 'active_support/core_ext/numeric/time'
|
5
|
-
|
6
3
|
module BeetleETL
|
7
4
|
describe TableDiff do
|
8
5
|
|
@@ -14,6 +11,8 @@ module BeetleETL
|
|
14
11
|
end
|
15
12
|
end
|
16
13
|
|
14
|
+
let(:time_in_past) { Time.new(2020, 3, 23) }
|
15
|
+
|
17
16
|
subject { TableDiff.new(config, :example_table) }
|
18
17
|
|
19
18
|
before do
|
@@ -60,9 +59,9 @@ module BeetleETL
|
|
60
59
|
it 'assigns CREATE to new records' do
|
61
60
|
|
62
61
|
insert_into(:example_table).values(
|
63
|
-
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at
|
64
|
-
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil
|
65
|
-
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 ,
|
62
|
+
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at ] ,
|
63
|
+
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil ] ,
|
64
|
+
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 , time_in_past ] ,
|
66
65
|
)
|
67
66
|
|
68
67
|
test_database.run "SELECT setval('public.example_table_id_seq', 99)"
|
@@ -88,10 +87,10 @@ module BeetleETL
|
|
88
87
|
except externald_*_id columns and columns not contained in the stage table' do
|
89
88
|
|
90
89
|
insert_into(:example_table).values(
|
91
|
-
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at
|
92
|
-
[ 1 , 'existing_1' , external_source , 'existing content' , 'ignored content' , 1 , nil
|
93
|
-
[ 2 , 'existing_2' , external_source , 'existing content' , 'ignored content' , 2 , nil
|
94
|
-
[ 3 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 3 ,
|
90
|
+
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at ] ,
|
91
|
+
[ 1 , 'existing_1' , external_source , 'existing content' , 'ignored content' , 1 , nil ] ,
|
92
|
+
[ 2 , 'existing_2' , external_source , 'existing content' , 'ignored content' , 2 , nil ] ,
|
93
|
+
[ 3 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 3 , time_in_past ] ,
|
95
94
|
)
|
96
95
|
|
97
96
|
insert_into(subject.stage_table_name.to_sym).values(
|
@@ -115,9 +114,9 @@ module BeetleETL
|
|
115
114
|
describe 'transition_delete' do
|
116
115
|
it 'creates records with DELETE that no loger exist in the stage table for the given run' do
|
117
116
|
insert_into(:example_table).values(
|
118
|
-
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at
|
119
|
-
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil
|
120
|
-
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 ,
|
117
|
+
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at ] ,
|
118
|
+
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil ] ,
|
119
|
+
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 , time_in_past ] ,
|
121
120
|
)
|
122
121
|
|
123
122
|
subject.transition_delete
|
@@ -132,9 +131,9 @@ module BeetleETL
|
|
132
131
|
describe 'transition_reinstate' do
|
133
132
|
it 'assigns REINSTATE to previously deleted records' do
|
134
133
|
insert_into(:example_table).values(
|
135
|
-
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at
|
136
|
-
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil
|
137
|
-
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 ,
|
134
|
+
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at ] ,
|
135
|
+
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil ] ,
|
136
|
+
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 , time_in_past ] ,
|
138
137
|
)
|
139
138
|
|
140
139
|
insert_into(subject.stage_table_name.to_sym).values(
|
@@ -159,7 +158,7 @@ module BeetleETL
|
|
159
158
|
insert_into(:example_table).values(
|
160
159
|
[ :id , :external_id , :external_source , :payload , :ignored_attribute , :foo_id , :deleted_at ] ,
|
161
160
|
[ 1 , 'existing' , external_source , 'existing content' , 'ignored content' , 1 , nil ] ,
|
162
|
-
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 ,
|
161
|
+
[ 2 , 'deleted' , external_source , 'deleted content' , 'ignored content' , 2 , time_in_past ] ,
|
163
162
|
)
|
164
163
|
|
165
164
|
insert_into(subject.stage_table_name.to_sym).values(
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: beetle_etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luciano Maiwald
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: activesupport
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 4.0.0
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ">="
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: 4.0.0
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: sequel
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -42,14 +28,14 @@ dependencies:
|
|
42
28
|
name: bundler
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
44
30
|
requirements:
|
45
|
-
- - "
|
31
|
+
- - ">="
|
46
32
|
- !ruby/object:Gem::Version
|
47
33
|
version: '1.11'
|
48
34
|
type: :development
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|
52
|
-
- - "
|
38
|
+
- - ">="
|
53
39
|
- !ruby/object:Gem::Version
|
54
40
|
version: '1.11'
|
55
41
|
- !ruby/object:Gem::Dependency
|