ductr-postgres 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +95 -55
- data/README.md +3 -3
- data/Rakefile +3 -1
- data/ductr-postgres.gemspec +8 -36
- data/lib/ductr/postgres/adapter.rb +1 -4
- data/lib/ductr/postgres/basic_lookup.rb +1 -15
- data/lib/ductr/postgres/basic_source.rb +19 -0
- data/lib/ductr/postgres/buffered_destination.rb +1 -10
- data/lib/ductr/postgres/buffered_lookup.rb +1 -12
- data/lib/ductr/postgres/buffered_upsert_destination.rb +2 -32
- data/lib/ductr/postgres/match_lookup.rb +1 -59
- data/lib/ductr/postgres/polling_trigger.rb +1 -26
- data/lib/ductr/postgres/version.rb +1 -1
- data/lib/ductr/postgres.rb +1 -2
- metadata +79 -57
- data/.vscode/settings.json +0 -22
- data/lib/ductr/postgres/polling_handler.rb +0 -40
- data/lib/ductr/postgres/streamed_source.rb +0 -30
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b4bcb117297ac764d6502a16c53b8d19628b112c22cbe617e8e5caf2a8f173d
|
4
|
+
data.tar.gz: ebf1858ea1d46b63b10db3dfc8246c98e8dcff86fb776193e8062927a37a463b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b73627f397c2407fae4295204ca5d5a1de413ad06249cb812593f1a4ca796dae5a6563764c815d2e63d99f6b2740236836205d5780e461abd2db2d19b1ac650
|
7
|
+
data.tar.gz: 8a2587f3ed3db58bafc84e9b0a3b26b569068d18a7048e945024d93a4544cf6e57ed23a963618cf6efa22abb0521ffaab9aeb0e5ff37af79c7ba929bcb18ac4a
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
ductr-postgres (0.1
|
5
|
-
ductr (
|
6
|
-
pg (~> 1.
|
7
|
-
sequel (~> 5.
|
8
|
-
sequel_pg (~> 1.
|
4
|
+
ductr-postgres (0.2.1)
|
5
|
+
ductr (>= 0.2)
|
6
|
+
pg (~> 1.5)
|
7
|
+
sequel (~> 5.91)
|
8
|
+
sequel_pg (~> 1.17)
|
9
9
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
@@ -19,14 +19,19 @@ GEM
|
|
19
19
|
minitest (>= 5.1)
|
20
20
|
tzinfo (~> 2.0)
|
21
21
|
annotable (0.1.2)
|
22
|
-
ast (2.4.
|
22
|
+
ast (2.4.3)
|
23
|
+
bigdecimal (3.1.9)
|
23
24
|
colorize (0.8.1)
|
24
|
-
commander (
|
25
|
-
highline (~>
|
25
|
+
commander (5.0.0)
|
26
|
+
highline (~> 3.0.0)
|
26
27
|
concurrent-ruby (1.2.2)
|
27
|
-
|
28
|
+
date (3.4.1)
|
29
|
+
debug (1.10.0)
|
30
|
+
irb (~> 1.10)
|
31
|
+
reline (>= 0.3.8)
|
32
|
+
diff-lcs (1.6.1)
|
28
33
|
docile (1.4.0)
|
29
|
-
ductr (0.
|
34
|
+
ductr (0.2.2)
|
30
35
|
activejob (~> 7.0)
|
31
36
|
annotable (~> 0.1)
|
32
37
|
colorize (~> 0.8)
|
@@ -41,61 +46,89 @@ GEM
|
|
41
46
|
raabro (~> 1.4)
|
42
47
|
globalid (1.1.0)
|
43
48
|
activesupport (>= 5.0)
|
44
|
-
highline (
|
49
|
+
highline (3.0.1)
|
45
50
|
i18n (1.12.0)
|
46
51
|
concurrent-ruby (~> 1.0)
|
47
|
-
|
52
|
+
io-console (0.8.0)
|
53
|
+
irb (1.15.2)
|
54
|
+
pp (>= 0.6.0)
|
55
|
+
rdoc (>= 4.0.0)
|
56
|
+
reline (>= 0.4.2)
|
57
|
+
json (2.11.3)
|
48
58
|
kiba (4.0.0)
|
59
|
+
language_server-protocol (3.17.0.4)
|
60
|
+
lint_roller (1.1.0)
|
61
|
+
logger (1.7.0)
|
49
62
|
minitest (5.18.0)
|
50
|
-
parallel (1.
|
51
|
-
parlour (
|
52
|
-
commander (~>
|
63
|
+
parallel (1.27.0)
|
64
|
+
parlour (9.1.0)
|
65
|
+
commander (~> 5.0)
|
53
66
|
parser
|
54
67
|
rainbow (~> 3.0)
|
55
68
|
sorbet-runtime (>= 0.5)
|
56
|
-
parser (3.
|
69
|
+
parser (3.3.8.0)
|
57
70
|
ast (~> 2.4.1)
|
58
|
-
|
71
|
+
racc
|
72
|
+
pg (1.5.3)
|
73
|
+
pp (0.6.2)
|
74
|
+
prettyprint
|
75
|
+
prettyprint (0.2.0)
|
76
|
+
prism (1.4.0)
|
77
|
+
psych (5.2.3)
|
78
|
+
date
|
79
|
+
stringio
|
59
80
|
raabro (1.4.0)
|
81
|
+
racc (1.8.1)
|
60
82
|
rainbow (3.1.1)
|
61
83
|
rake (13.0.6)
|
84
|
+
rbs (3.9.2)
|
85
|
+
logger
|
86
|
+
rdoc (6.13.1)
|
87
|
+
psych (>= 4.0.0)
|
62
88
|
redcarpet (3.6.0)
|
63
|
-
regexp_parser (2.
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
rspec-
|
68
|
-
rspec-
|
69
|
-
|
70
|
-
|
71
|
-
|
89
|
+
regexp_parser (2.10.0)
|
90
|
+
reline (0.6.1)
|
91
|
+
io-console (~> 0.5)
|
92
|
+
rspec (3.13.0)
|
93
|
+
rspec-core (~> 3.13.0)
|
94
|
+
rspec-expectations (~> 3.13.0)
|
95
|
+
rspec-mocks (~> 3.13.0)
|
96
|
+
rspec-core (3.13.3)
|
97
|
+
rspec-support (~> 3.13.0)
|
98
|
+
rspec-expectations (3.13.4)
|
72
99
|
diff-lcs (>= 1.2.0, < 2.0)
|
73
|
-
rspec-support (~> 3.
|
74
|
-
rspec-mocks (3.
|
100
|
+
rspec-support (~> 3.13.0)
|
101
|
+
rspec-mocks (3.13.3)
|
75
102
|
diff-lcs (>= 1.2.0, < 2.0)
|
76
|
-
rspec-support (~> 3.
|
77
|
-
rspec-support (3.
|
78
|
-
rubocop (1.
|
103
|
+
rspec-support (~> 3.13.0)
|
104
|
+
rspec-support (3.13.3)
|
105
|
+
rubocop (1.75.4)
|
79
106
|
json (~> 2.3)
|
107
|
+
language_server-protocol (~> 3.17.0.2)
|
108
|
+
lint_roller (~> 1.1.0)
|
80
109
|
parallel (~> 1.10)
|
81
|
-
parser (>= 3.
|
110
|
+
parser (>= 3.3.0.2)
|
82
111
|
rainbow (>= 2.2.2, < 4.0)
|
83
|
-
regexp_parser (>=
|
84
|
-
|
85
|
-
rubocop-ast (>= 1.26.0, < 2.0)
|
112
|
+
regexp_parser (>= 2.9.3, < 3.0)
|
113
|
+
rubocop-ast (>= 1.44.0, < 2.0)
|
86
114
|
ruby-progressbar (~> 1.7)
|
87
|
-
unicode-display_width (>= 2.4.0, <
|
88
|
-
rubocop-ast (1.
|
89
|
-
parser (>= 3.
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
rubocop (~> 1.
|
94
|
-
|
115
|
+
unicode-display_width (>= 2.4.0, < 4.0)
|
116
|
+
rubocop-ast (1.44.1)
|
117
|
+
parser (>= 3.3.7.2)
|
118
|
+
prism (~> 1.4)
|
119
|
+
rubocop-rspec (3.6.0)
|
120
|
+
lint_roller (~> 1.1)
|
121
|
+
rubocop (~> 1.72, >= 1.72.1)
|
122
|
+
ruby-lsp (0.23.16)
|
123
|
+
language_server-protocol (~> 3.17.0)
|
124
|
+
prism (>= 1.2, < 2.0)
|
125
|
+
rbs (>= 3, < 4)
|
126
|
+
sorbet-runtime (>= 0.5.10782)
|
95
127
|
ruby-progressbar (1.13.0)
|
96
128
|
rufus-scheduler (3.8.2)
|
97
129
|
fugit (~> 1.1, >= 1.1.6)
|
98
|
-
sequel (5.
|
130
|
+
sequel (5.91.0)
|
131
|
+
bigdecimal
|
99
132
|
sequel_pg (1.17.1)
|
100
133
|
pg (>= 0.18.0, != 1.2.0)
|
101
134
|
sequel (>= 4.38.0)
|
@@ -105,16 +138,21 @@ GEM
|
|
105
138
|
simplecov_json_formatter (~> 0.1)
|
106
139
|
simplecov-html (0.12.3)
|
107
140
|
simplecov_json_formatter (0.1.4)
|
108
|
-
sorbet-runtime (0.5.
|
109
|
-
sord (
|
110
|
-
commander (~>
|
111
|
-
parlour (~>
|
141
|
+
sorbet-runtime (0.5.12048)
|
142
|
+
sord (7.0.0)
|
143
|
+
commander (~> 5.0)
|
144
|
+
parlour (~> 9.1)
|
145
|
+
parser
|
146
|
+
rbs (~> 3.0)
|
112
147
|
sorbet-runtime
|
113
148
|
yard
|
149
|
+
stringio (3.1.7)
|
114
150
|
thor (1.2.1)
|
115
151
|
tzinfo (2.0.6)
|
116
152
|
concurrent-ruby (~> 1.0)
|
117
|
-
unicode-display_width (
|
153
|
+
unicode-display_width (3.1.4)
|
154
|
+
unicode-emoji (~> 4.0, >= 4.0.4)
|
155
|
+
unicode-emoji (4.0.4)
|
118
156
|
webrick (1.7.0)
|
119
157
|
yard (0.9.28)
|
120
158
|
webrick (~> 1.7.0)
|
@@ -124,14 +162,16 @@ PLATFORMS
|
|
124
162
|
x86_64-linux
|
125
163
|
|
126
164
|
DEPENDENCIES
|
165
|
+
debug (~> 1.10)
|
127
166
|
ductr-postgres!
|
128
167
|
rake (~> 13.0)
|
129
|
-
redcarpet (~> 3.
|
130
|
-
rspec (~> 3.
|
131
|
-
rubocop (~> 1.
|
132
|
-
rubocop-rspec (~>
|
133
|
-
|
134
|
-
|
168
|
+
redcarpet (~> 3.6)
|
169
|
+
rspec (~> 3.13)
|
170
|
+
rubocop (~> 1.75)
|
171
|
+
rubocop-rspec (~> 3.6)
|
172
|
+
ruby-lsp (~> 0.23)
|
173
|
+
simplecov (~> 0.22)
|
174
|
+
sord (~> 7.0)
|
135
175
|
yard (~> 0.9)
|
136
176
|
|
137
177
|
BUNDLED WITH
|
data/README.md
CHANGED
@@ -42,9 +42,9 @@ See [the Sequel PostgreSQL specific options list](https://sequel.jeremyevans.net
|
|
42
42
|
The configured adapter can now be used in Ductr jobs e.g.:
|
43
43
|
|
44
44
|
```ruby
|
45
|
-
source :some_postgres_database, :
|
46
|
-
def select_some_stuff(db
|
47
|
-
db[:items].
|
45
|
+
source :some_postgres_database, :basic
|
46
|
+
def select_some_stuff(db)
|
47
|
+
db[:items].where(name: %w[hello world])
|
48
48
|
end
|
49
49
|
```
|
50
50
|
|
data/Rakefile
CHANGED
@@ -22,12 +22,14 @@ task :sord do
|
|
22
22
|
break_params: 4,
|
23
23
|
replace_errors_with_untyped: false,
|
24
24
|
replace_unresolved_with_untyped: false,
|
25
|
+
hide_private: false,
|
25
26
|
exclude_messages: nil,
|
26
27
|
include_messages: nil,
|
27
28
|
keep_original_comments: false,
|
28
29
|
skip_constants: false,
|
29
30
|
use_original_initialize_return: false,
|
30
|
-
exclude_untyped: false
|
31
|
+
exclude_untyped: false,
|
32
|
+
tags: []
|
31
33
|
}
|
32
34
|
|
33
35
|
plugin = Sord::ParlourPlugin.new(options)
|
data/ductr-postgres.gemspec
CHANGED
@@ -1,48 +1,20 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "../mono_repo"
|
3
4
|
require_relative "lib/ductr/postgres/version"
|
4
5
|
|
5
6
|
Gem::Specification.new do |spec|
|
6
7
|
spec.name = "ductr-postgres"
|
7
8
|
spec.version = Ductr::Postgres::VERSION
|
8
|
-
spec.authors = ["Mathieu MOREL"]
|
9
|
-
spec.email = ["mathieu@lamanufacture.dev"]
|
10
|
-
spec.licenses = ["LGPL-3.0-or-later"]
|
11
|
-
|
12
9
|
spec.summary = "PostgreSQL adapter for Ductr using the `sequel` gem"
|
13
10
|
spec.description = "Allows ductr to interact with PostgreSQL DMBS."
|
14
|
-
spec.homepage = "https://github.com/ductr-io/ductr"
|
15
|
-
spec.required_ruby_version = ">= 3.1.0"
|
16
|
-
|
17
|
-
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
18
|
-
spec.metadata["rubygems_mfa_required"] = "true"
|
19
|
-
|
20
|
-
spec.metadata["homepage_uri"] = spec.homepage
|
21
|
-
spec.metadata["source_code_uri"] = spec.homepage
|
22
|
-
spec.metadata["changelog_uri"] = "#{spec.homepage}/releases"
|
23
|
-
|
24
|
-
# Specify which files should be added to the gem when it is released.
|
25
|
-
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
26
|
-
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
27
|
-
`git ls-files -z`.split("\x0").reject do |f|
|
28
|
-
(f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
|
29
|
-
end
|
30
|
-
end
|
31
|
-
spec.bindir = "exe"
|
32
|
-
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
33
|
-
spec.require_paths = ["lib"]
|
34
11
|
|
35
|
-
spec
|
36
|
-
spec
|
37
|
-
spec
|
38
|
-
spec.add_dependency "sequel_pg", "~> 1.8"
|
12
|
+
MonoRepo.add_metadata(spec)
|
13
|
+
MonoRepo.add_files(spec, __dir__, __FILE__)
|
14
|
+
MonoRepo.add_development_dependencies(spec)
|
39
15
|
|
40
|
-
spec.
|
41
|
-
spec.
|
42
|
-
spec.
|
43
|
-
spec.
|
44
|
-
spec.add_development_dependency "rubocop-rspec", "~> 2.11"
|
45
|
-
spec.add_development_dependency "simplecov", "~> 0.21"
|
46
|
-
spec.add_development_dependency "sord", "~> 4.0"
|
47
|
-
spec.add_development_dependency "yard", "~> 0.9"
|
16
|
+
spec.add_dependency(*MonoRepo::DUCTR)
|
17
|
+
spec.add_dependency "pg", "~> 1.5"
|
18
|
+
spec.add_dependency(*MonoRepo::SEQUEL)
|
19
|
+
spec.add_dependency "sequel_pg", "~> 1.17"
|
48
20
|
end
|
@@ -22,12 +22,9 @@ module Ductr
|
|
22
22
|
# @see https://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html#label-postgres
|
23
23
|
# PostgreSQL specific options
|
24
24
|
#
|
25
|
-
class Adapter < Ductr::Adapter
|
25
|
+
class Adapter < Ductr::SequelBase::Adapter
|
26
26
|
Ductr.adapter_registry.add(self, as: :postgres)
|
27
27
|
|
28
|
-
# @return [Sequel::Database, nil] The database connection instance
|
29
|
-
attr_reader :db
|
30
|
-
|
31
28
|
#
|
32
29
|
# Opens the database connection with the adapter's configuration.
|
33
30
|
#
|
@@ -16,22 +16,8 @@ module Ductr
|
|
16
16
|
#
|
17
17
|
# If the lookup returns a falsy value, nothing won't be merged with the current row.
|
18
18
|
#
|
19
|
-
class BasicLookup < Ductr::
|
19
|
+
class BasicLookup < Ductr::SequelBase::BasicLookup
|
20
20
|
Adapter.lookup_registry.add(self, as: :basic)
|
21
|
-
|
22
|
-
#
|
23
|
-
# Calls the job's method to merge its result with the current row.
|
24
|
-
#
|
25
|
-
# @param [Hash<Symbol, Object>] row The current row, preferably a Hash
|
26
|
-
#
|
27
|
-
# @return [Hash<Symbol, Object>] The row merged with looked up row or the untouched row if nothing was found
|
28
|
-
#
|
29
|
-
def process(row)
|
30
|
-
matching_row = call_method(adapter.db, row).first
|
31
|
-
return row unless matching_row
|
32
|
-
|
33
|
-
row.merge matching_row
|
34
|
-
end
|
35
21
|
end
|
36
22
|
end
|
37
23
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ductr
|
4
|
+
module Postgres
|
5
|
+
#
|
6
|
+
# A source control that yields rows usnig the PostgreSQL streaming feature, registered as `:basic`:
|
7
|
+
#
|
8
|
+
# source :some_postgres_database, :basic
|
9
|
+
# def select_some_stuff(db)
|
10
|
+
# db[:items].limit(42)
|
11
|
+
# end
|
12
|
+
#
|
13
|
+
# You can select a large number of rows, without worrying about pagination handling or memory usage.
|
14
|
+
#
|
15
|
+
class BasicSource < Ductr::SequelBase::BasicSource
|
16
|
+
Adapter.source_registry.add(self, as: :basic)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -13,17 +13,8 @@ module Ductr
|
|
13
13
|
#
|
14
14
|
# @see more Ductr::ETL::BufferedDestination
|
15
15
|
#
|
16
|
-
class BufferedDestination < Ductr::
|
16
|
+
class BufferedDestination < Ductr::SequelBase::BufferedDestination
|
17
17
|
Adapter.destination_registry.add(self, as: :buffered)
|
18
|
-
|
19
|
-
#
|
20
|
-
# Open the database if needed and call the job's method to run the query.
|
21
|
-
#
|
22
|
-
# @return [void]
|
23
|
-
#
|
24
|
-
def on_flush
|
25
|
-
call_method(adapter.db, buffer)
|
26
|
-
end
|
27
18
|
end
|
28
19
|
end
|
29
20
|
end
|
@@ -19,19 +19,8 @@ module Ductr
|
|
19
19
|
# end
|
20
20
|
# end
|
21
21
|
#
|
22
|
-
class BufferedLookup < Ductr::
|
22
|
+
class BufferedLookup < Ductr::SequelBase::BufferedLookup
|
23
23
|
Adapter.lookup_registry.add(self, as: :buffered)
|
24
|
-
|
25
|
-
#
|
26
|
-
# Opens the database if needed, calls the job's method and pass the each block to it.
|
27
|
-
#
|
28
|
-
# @yield The each block
|
29
|
-
#
|
30
|
-
# @return [void]
|
31
|
-
#
|
32
|
-
def on_flush(&)
|
33
|
-
call_method(adapter.db, buffer, &)
|
34
|
-
end
|
35
24
|
end
|
36
25
|
end
|
37
26
|
end
|
@@ -7,44 +7,14 @@ module Ductr
|
|
7
7
|
# Accept the `:buffer_size` option, default value is 10 000:
|
8
8
|
#
|
9
9
|
# destination :some_postgres_database, :buffered_upsert, buffer_size: 42
|
10
|
-
# def my_destination(
|
10
|
+
# def my_destination(db, excluded, buffer)
|
11
11
|
# db[:items].insert_conflict(target: :id, update: excluded).multi_insert(buffer)
|
12
12
|
# end
|
13
13
|
#
|
14
14
|
# @see more Ductr::ETL::BufferedDestination
|
15
15
|
#
|
16
|
-
class BufferedUpsertDestination < Ductr::
|
16
|
+
class BufferedUpsertDestination < Ductr::SequelBase::BufferedUpsertDestination
|
17
17
|
Adapter.destination_registry.add(self, as: :buffered_upsert)
|
18
|
-
|
19
|
-
#
|
20
|
-
# Open the database if needed and call the job's method to run the query.
|
21
|
-
#
|
22
|
-
# @return [void]
|
23
|
-
#
|
24
|
-
def on_flush
|
25
|
-
call_method(adapter.db, excluded, buffer)
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
#
|
31
|
-
# Generate the excluded keys hash e.g.
|
32
|
-
#
|
33
|
-
# ```ruby
|
34
|
-
# {a: Sequel[:excluded][:a]}
|
35
|
-
# ```
|
36
|
-
#
|
37
|
-
# @return [Hash<Symbol, Sequel::SQL::QualifiedIdentifier>] The excluded keys hash
|
38
|
-
#
|
39
|
-
def excluded
|
40
|
-
keys = buffer.first.keys
|
41
|
-
|
42
|
-
excluded_keys = keys.map do |key|
|
43
|
-
Sequel[:excluded][key]
|
44
|
-
end
|
45
|
-
|
46
|
-
keys.zip(excluded_keys).to_h
|
47
|
-
end
|
48
18
|
end
|
49
19
|
end
|
50
20
|
end
|
@@ -18,66 +18,8 @@ module Ductr
|
|
18
18
|
# db[:items_bis].where(item: ids)
|
19
19
|
# end
|
20
20
|
#
|
21
|
-
class MatchLookup < Ductr::
|
21
|
+
class MatchLookup < Ductr::SequelBase::MatchLookup
|
22
22
|
Adapter.lookup_registry.add(self, as: :match)
|
23
|
-
|
24
|
-
#
|
25
|
-
# The looked up row key to match.
|
26
|
-
#
|
27
|
-
# @return [Symbol] The column name
|
28
|
-
#
|
29
|
-
def from_key
|
30
|
-
@options[:merge].first
|
31
|
-
end
|
32
|
-
|
33
|
-
#
|
34
|
-
# The buffer row key to match.
|
35
|
-
#
|
36
|
-
# @return [Symbol] The column name
|
37
|
-
#
|
38
|
-
def to_key
|
39
|
-
@options[:merge].last
|
40
|
-
end
|
41
|
-
|
42
|
-
#
|
43
|
-
# Opens the database if needed, calls the job's method and merges
|
44
|
-
# the looked up rows with corresponding buffer rows.
|
45
|
-
#
|
46
|
-
# @yield [row] The each block
|
47
|
-
# @yieldparam [Hash<Symbol, Object>] row The merged row
|
48
|
-
#
|
49
|
-
# @return [void]
|
50
|
-
#
|
51
|
-
def on_flush(&)
|
52
|
-
call_method(adapter.db, buffer_keys).each do |row|
|
53
|
-
match = buffer_find(row)
|
54
|
-
next yield(row) unless match
|
55
|
-
|
56
|
-
yield(row.merge match)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
private
|
61
|
-
|
62
|
-
#
|
63
|
-
# Find the corresponding row into the buffer.
|
64
|
-
#
|
65
|
-
# @param [Hash<Symbol, Object>] row The looked up row
|
66
|
-
#
|
67
|
-
# @return [Hash<Symbol, Object>, nil] the matching row if exists
|
68
|
-
#
|
69
|
-
def buffer_find(row)
|
70
|
-
buffer.find { |r| r[from_key] == row[to_key] }
|
71
|
-
end
|
72
|
-
|
73
|
-
#
|
74
|
-
# Maps the buffer keys into an array.
|
75
|
-
#
|
76
|
-
# @return [Array<Integer, String>] The keys array
|
77
|
-
#
|
78
|
-
def buffer_keys
|
79
|
-
buffer.map { |row| row[from_key] }
|
80
|
-
end
|
81
23
|
end
|
82
24
|
end
|
83
25
|
end
|
@@ -14,33 +14,8 @@ module Ductr
|
|
14
14
|
# MyJob.perform_later
|
15
15
|
# end
|
16
16
|
#
|
17
|
-
class PollingTrigger < Ductr::
|
17
|
+
class PollingTrigger < Ductr::SequelBase::PollingTrigger
|
18
18
|
Adapter.trigger_registry.add(self, as: :polling)
|
19
|
-
|
20
|
-
#
|
21
|
-
# Closes the connection if the scheduler is stopped.
|
22
|
-
#
|
23
|
-
# @return [void]
|
24
|
-
#
|
25
|
-
def stop
|
26
|
-
super
|
27
|
-
adapter.close!
|
28
|
-
end
|
29
|
-
|
30
|
-
private
|
31
|
-
|
32
|
-
#
|
33
|
-
# Returns a callable object, allowing rufus-scheduler to call it.
|
34
|
-
#
|
35
|
-
# @param [Ductr::Scheduler] scheduler The scheduler instance
|
36
|
-
# @param [Method] method The scheduler's method
|
37
|
-
# @param [Hash] ** The option passed to the trigger annotation
|
38
|
-
#
|
39
|
-
# @return [#call] A callable object
|
40
|
-
#
|
41
|
-
def callable(method, **)
|
42
|
-
PollingHandler.new(method, adapter)
|
43
|
-
end
|
44
19
|
end
|
45
20
|
end
|
46
21
|
end
|
data/lib/ductr/postgres.rb
CHANGED
@@ -14,8 +14,7 @@ module Ductr
|
|
14
14
|
# To get details about the database connection handling, checkout the {Ductr::Postgres::Adapter} class.
|
15
15
|
#
|
16
16
|
# ### Sources
|
17
|
-
# - {Ductr::Postgres::BasicSource} Yields rows one by one.
|
18
|
-
# - {Ductr::Postgres::PaginatedSource} Allows to select a big number of rows by relying on pagination.
|
17
|
+
# - {Ductr::Postgres::BasicSource} Yields rows one by one using postgres streaming feature.
|
19
18
|
#
|
20
19
|
# ### Lookups
|
21
20
|
# - {Ductr::Postgres::BasicLookup} Executes one query per row and merge the looked up row with the current row.
|
metadata
CHANGED
@@ -1,193 +1,219 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ductr-postgres
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
- Mathieu
|
8
|
-
autorequire:
|
7
|
+
- Mathieu Morel
|
9
8
|
bindir: exe
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
13
|
+
name: debug
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
16
|
- - "~>"
|
18
17
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
20
|
-
type: :
|
18
|
+
version: '1.10'
|
19
|
+
type: :development
|
21
20
|
prerelease: false
|
22
21
|
version_requirements: !ruby/object:Gem::Requirement
|
23
22
|
requirements:
|
24
23
|
- - "~>"
|
25
24
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
25
|
+
version: '1.10'
|
27
26
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
27
|
+
name: rake
|
29
28
|
requirement: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
30
|
- - "~>"
|
32
31
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
34
|
-
type: :
|
32
|
+
version: '13.0'
|
33
|
+
type: :development
|
35
34
|
prerelease: false
|
36
35
|
version_requirements: !ruby/object:Gem::Requirement
|
37
36
|
requirements:
|
38
37
|
- - "~>"
|
39
38
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
39
|
+
version: '13.0'
|
41
40
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
41
|
+
name: redcarpet
|
43
42
|
requirement: !ruby/object:Gem::Requirement
|
44
43
|
requirements:
|
45
44
|
- - "~>"
|
46
45
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
46
|
+
version: '3.6'
|
47
|
+
type: :development
|
49
48
|
prerelease: false
|
50
49
|
version_requirements: !ruby/object:Gem::Requirement
|
51
50
|
requirements:
|
52
51
|
- - "~>"
|
53
52
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
53
|
+
version: '3.6'
|
55
54
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
55
|
+
name: rspec
|
57
56
|
requirement: !ruby/object:Gem::Requirement
|
58
57
|
requirements:
|
59
58
|
- - "~>"
|
60
59
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
62
|
-
type: :
|
60
|
+
version: '3.13'
|
61
|
+
type: :development
|
63
62
|
prerelease: false
|
64
63
|
version_requirements: !ruby/object:Gem::Requirement
|
65
64
|
requirements:
|
66
65
|
- - "~>"
|
67
66
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
67
|
+
version: '3.13'
|
69
68
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
69
|
+
name: rubocop
|
71
70
|
requirement: !ruby/object:Gem::Requirement
|
72
71
|
requirements:
|
73
72
|
- - "~>"
|
74
73
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
74
|
+
version: '1.75'
|
76
75
|
type: :development
|
77
76
|
prerelease: false
|
78
77
|
version_requirements: !ruby/object:Gem::Requirement
|
79
78
|
requirements:
|
80
79
|
- - "~>"
|
81
80
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
81
|
+
version: '1.75'
|
83
82
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
83
|
+
name: rubocop-rspec
|
85
84
|
requirement: !ruby/object:Gem::Requirement
|
86
85
|
requirements:
|
87
86
|
- - "~>"
|
88
87
|
- !ruby/object:Gem::Version
|
89
|
-
version: '3.
|
88
|
+
version: '3.6'
|
90
89
|
type: :development
|
91
90
|
prerelease: false
|
92
91
|
version_requirements: !ruby/object:Gem::Requirement
|
93
92
|
requirements:
|
94
93
|
- - "~>"
|
95
94
|
- !ruby/object:Gem::Version
|
96
|
-
version: '3.
|
95
|
+
version: '3.6'
|
97
96
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
97
|
+
name: ruby-lsp
|
99
98
|
requirement: !ruby/object:Gem::Requirement
|
100
99
|
requirements:
|
101
100
|
- - "~>"
|
102
101
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
102
|
+
version: '0.23'
|
104
103
|
type: :development
|
105
104
|
prerelease: false
|
106
105
|
version_requirements: !ruby/object:Gem::Requirement
|
107
106
|
requirements:
|
108
107
|
- - "~>"
|
109
108
|
- !ruby/object:Gem::Version
|
110
|
-
version: '
|
109
|
+
version: '0.23'
|
111
110
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
111
|
+
name: simplecov
|
113
112
|
requirement: !ruby/object:Gem::Requirement
|
114
113
|
requirements:
|
115
114
|
- - "~>"
|
116
115
|
- !ruby/object:Gem::Version
|
117
|
-
version: '
|
116
|
+
version: '0.22'
|
118
117
|
type: :development
|
119
118
|
prerelease: false
|
120
119
|
version_requirements: !ruby/object:Gem::Requirement
|
121
120
|
requirements:
|
122
121
|
- - "~>"
|
123
122
|
- !ruby/object:Gem::Version
|
124
|
-
version: '
|
123
|
+
version: '0.22'
|
125
124
|
- !ruby/object:Gem::Dependency
|
126
|
-
name:
|
125
|
+
name: sord
|
127
126
|
requirement: !ruby/object:Gem::Requirement
|
128
127
|
requirements:
|
129
128
|
- - "~>"
|
130
129
|
- !ruby/object:Gem::Version
|
131
|
-
version: '
|
130
|
+
version: '7.0'
|
132
131
|
type: :development
|
133
132
|
prerelease: false
|
134
133
|
version_requirements: !ruby/object:Gem::Requirement
|
135
134
|
requirements:
|
136
135
|
- - "~>"
|
137
136
|
- !ruby/object:Gem::Version
|
138
|
-
version: '
|
137
|
+
version: '7.0'
|
139
138
|
- !ruby/object:Gem::Dependency
|
140
|
-
name:
|
139
|
+
name: yard
|
141
140
|
requirement: !ruby/object:Gem::Requirement
|
142
141
|
requirements:
|
143
142
|
- - "~>"
|
144
143
|
- !ruby/object:Gem::Version
|
145
|
-
version: '0.
|
144
|
+
version: '0.9'
|
146
145
|
type: :development
|
147
146
|
prerelease: false
|
148
147
|
version_requirements: !ruby/object:Gem::Requirement
|
149
148
|
requirements:
|
150
149
|
- - "~>"
|
151
150
|
- !ruby/object:Gem::Version
|
152
|
-
version: '0.
|
151
|
+
version: '0.9'
|
152
|
+
- !ruby/object:Gem::Dependency
|
153
|
+
name: ductr
|
154
|
+
requirement: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0.2'
|
159
|
+
type: :runtime
|
160
|
+
prerelease: false
|
161
|
+
version_requirements: !ruby/object:Gem::Requirement
|
162
|
+
requirements:
|
163
|
+
- - ">="
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0.2'
|
166
|
+
- !ruby/object:Gem::Dependency
|
167
|
+
name: pg
|
168
|
+
requirement: !ruby/object:Gem::Requirement
|
169
|
+
requirements:
|
170
|
+
- - "~>"
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '1.5'
|
173
|
+
type: :runtime
|
174
|
+
prerelease: false
|
175
|
+
version_requirements: !ruby/object:Gem::Requirement
|
176
|
+
requirements:
|
177
|
+
- - "~>"
|
178
|
+
- !ruby/object:Gem::Version
|
179
|
+
version: '1.5'
|
153
180
|
- !ruby/object:Gem::Dependency
|
154
|
-
name:
|
181
|
+
name: sequel
|
155
182
|
requirement: !ruby/object:Gem::Requirement
|
156
183
|
requirements:
|
157
184
|
- - "~>"
|
158
185
|
- !ruby/object:Gem::Version
|
159
|
-
version: '
|
160
|
-
type: :
|
186
|
+
version: '5.91'
|
187
|
+
type: :runtime
|
161
188
|
prerelease: false
|
162
189
|
version_requirements: !ruby/object:Gem::Requirement
|
163
190
|
requirements:
|
164
191
|
- - "~>"
|
165
192
|
- !ruby/object:Gem::Version
|
166
|
-
version: '
|
193
|
+
version: '5.91'
|
167
194
|
- !ruby/object:Gem::Dependency
|
168
|
-
name:
|
195
|
+
name: sequel_pg
|
169
196
|
requirement: !ruby/object:Gem::Requirement
|
170
197
|
requirements:
|
171
198
|
- - "~>"
|
172
199
|
- !ruby/object:Gem::Version
|
173
|
-
version: '
|
174
|
-
type: :
|
200
|
+
version: '1.17'
|
201
|
+
type: :runtime
|
175
202
|
prerelease: false
|
176
203
|
version_requirements: !ruby/object:Gem::Requirement
|
177
204
|
requirements:
|
178
205
|
- - "~>"
|
179
206
|
- !ruby/object:Gem::Version
|
180
|
-
version: '
|
207
|
+
version: '1.17'
|
181
208
|
description: Allows ductr to interact with PostgreSQL DMBS.
|
182
209
|
email:
|
183
|
-
-
|
210
|
+
- matmorel@users.noreply.github.com
|
184
211
|
executables: []
|
185
212
|
extensions: []
|
186
213
|
extra_rdoc_files: []
|
187
214
|
files:
|
188
215
|
- ".rspec"
|
189
216
|
- ".rubocop.yml"
|
190
|
-
- ".vscode/settings.json"
|
191
217
|
- ".yardopts"
|
192
218
|
- CODE_OF_CONDUCT.md
|
193
219
|
- COPYING
|
@@ -200,25 +226,22 @@ files:
|
|
200
226
|
- lib/ductr/postgres.rb
|
201
227
|
- lib/ductr/postgres/adapter.rb
|
202
228
|
- lib/ductr/postgres/basic_lookup.rb
|
229
|
+
- lib/ductr/postgres/basic_source.rb
|
203
230
|
- lib/ductr/postgres/buffered_destination.rb
|
204
231
|
- lib/ductr/postgres/buffered_lookup.rb
|
205
232
|
- lib/ductr/postgres/buffered_upsert_destination.rb
|
206
233
|
- lib/ductr/postgres/match_lookup.rb
|
207
|
-
- lib/ductr/postgres/polling_handler.rb
|
208
234
|
- lib/ductr/postgres/polling_trigger.rb
|
209
|
-
- lib/ductr/postgres/streamed_source.rb
|
210
235
|
- lib/ductr/postgres/version.rb
|
211
236
|
- sig/ductr/postgres.rbs
|
212
|
-
homepage: https://github.com/
|
237
|
+
homepage: https://github.com/matmorel/ductr
|
213
238
|
licenses:
|
214
239
|
- LGPL-3.0-or-later
|
215
240
|
metadata:
|
241
|
+
source_code_uri: https://github.com/matmorel/ductr
|
242
|
+
changelog_uri: https://github.com/matmorel/ductr/releases
|
216
243
|
allowed_push_host: https://rubygems.org
|
217
244
|
rubygems_mfa_required: 'true'
|
218
|
-
homepage_uri: https://github.com/ductr-io/ductr
|
219
|
-
source_code_uri: https://github.com/ductr-io/ductr
|
220
|
-
changelog_uri: https://github.com/ductr-io/ductr/releases
|
221
|
-
post_install_message:
|
222
245
|
rdoc_options: []
|
223
246
|
require_paths:
|
224
247
|
- lib
|
@@ -233,8 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
233
256
|
- !ruby/object:Gem::Version
|
234
257
|
version: '0'
|
235
258
|
requirements: []
|
236
|
-
rubygems_version: 3.
|
237
|
-
signing_key:
|
259
|
+
rubygems_version: 3.7.1
|
238
260
|
specification_version: 4
|
239
261
|
summary: PostgreSQL adapter for Ductr using the `sequel` gem
|
240
262
|
test_files: []
|
data/.vscode/settings.json
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"cSpell.words": [
|
3
|
-
"bindir",
|
4
|
-
"circleci",
|
5
|
-
"DMBS",
|
6
|
-
"ductr",
|
7
|
-
"inflector",
|
8
|
-
"kiba",
|
9
|
-
"Mathieu",
|
10
|
-
"nodoc",
|
11
|
-
"progressbar",
|
12
|
-
"redcarpet",
|
13
|
-
"rexml",
|
14
|
-
"rubocop",
|
15
|
-
"simplecov",
|
16
|
-
"sord",
|
17
|
-
"postgres",
|
18
|
-
"webrick",
|
19
|
-
"yieldparam",
|
20
|
-
"zeitwerk"
|
21
|
-
]
|
22
|
-
}
|
@@ -1,40 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Ductr
|
4
|
-
module Postgres
|
5
|
-
#
|
6
|
-
# The rufus-scheduler handler class.
|
7
|
-
# @see https://github.com/jmettraux/rufus-scheduler#scheduling-handler-instances
|
8
|
-
# For further information
|
9
|
-
#
|
10
|
-
class PollingHandler
|
11
|
-
#
|
12
|
-
# Creates the handler based on the given scheduler, its method name and the trigger's adapter instance.
|
13
|
-
#
|
14
|
-
# @param [Method] method The scheduler's method
|
15
|
-
# @param [Ductr::Adapter] adapter The trigger's adapter
|
16
|
-
#
|
17
|
-
def initialize(method, adapter)
|
18
|
-
@method = method
|
19
|
-
@adapter = adapter
|
20
|
-
@last_triggering_key = nil
|
21
|
-
end
|
22
|
-
|
23
|
-
#
|
24
|
-
# The callable method used by the trigger, actually calls the scheduler's method.
|
25
|
-
#
|
26
|
-
# @return [void]
|
27
|
-
#
|
28
|
-
def call
|
29
|
-
@adapter.open do |db|
|
30
|
-
@method.call(db) do |triggering_key|
|
31
|
-
return false if triggering_key == @last_triggering_key
|
32
|
-
|
33
|
-
@last_triggering_key = triggering_key
|
34
|
-
true
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Ductr
|
4
|
-
module Postgres
|
5
|
-
#
|
6
|
-
# A source control that yields rows usnig the PostgreSQL streaming feature, registered as `:streamed`:
|
7
|
-
#
|
8
|
-
# source :some_postgres_database, :streamed
|
9
|
-
# def select_some_stuff(db)
|
10
|
-
# db[:items].limit(42)
|
11
|
-
# end
|
12
|
-
#
|
13
|
-
# You can select a large number of rows, without worrying about pagination handling or memory usage.
|
14
|
-
#
|
15
|
-
class StreamedSource < Ductr::ETL::Source
|
16
|
-
Adapter.source_registry.add(self, as: :streamed)
|
17
|
-
|
18
|
-
#
|
19
|
-
# Opens the database, calls the job's method and iterate over the query results.
|
20
|
-
#
|
21
|
-
# @yield The each block
|
22
|
-
#
|
23
|
-
# @return [void]
|
24
|
-
#
|
25
|
-
def each(&)
|
26
|
-
call_method(adapter.db).each(&)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|