ductr 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/.rubocop.yml +14 -0
  4. data/.vscode/settings.json +18 -0
  5. data/COPYING +674 -0
  6. data/COPYING.LESSER +165 -0
  7. data/Gemfile +6 -0
  8. data/Gemfile.lock +121 -0
  9. data/README.md +37 -0
  10. data/Rakefile +37 -0
  11. data/bin/console +15 -0
  12. data/bin/setup +8 -0
  13. data/ductr.gemspec +50 -0
  14. data/exe/ductr +24 -0
  15. data/lib/ductr/adapter.rb +94 -0
  16. data/lib/ductr/cli/default.rb +25 -0
  17. data/lib/ductr/cli/main.rb +60 -0
  18. data/lib/ductr/cli/new_project_generator.rb +72 -0
  19. data/lib/ductr/cli/templates/project/bin_ductr.rb +7 -0
  20. data/lib/ductr/cli/templates/project/config_app.rb +5 -0
  21. data/lib/ductr/cli/templates/project/config_development.yml +8 -0
  22. data/lib/ductr/cli/templates/project/config_environment_development.rb +18 -0
  23. data/lib/ductr/cli/templates/project/gemfile.rb +6 -0
  24. data/lib/ductr/cli/templates/project/rubocop.yml +14 -0
  25. data/lib/ductr/cli/templates/project/tool-versions +1 -0
  26. data/lib/ductr/configuration.rb +145 -0
  27. data/lib/ductr/etl/controls/buffered_destination.rb +65 -0
  28. data/lib/ductr/etl/controls/buffered_transform.rb +76 -0
  29. data/lib/ductr/etl/controls/control.rb +46 -0
  30. data/lib/ductr/etl/controls/destination.rb +28 -0
  31. data/lib/ductr/etl/controls/paginated_source.rb +47 -0
  32. data/lib/ductr/etl/controls/source.rb +21 -0
  33. data/lib/ductr/etl/controls/transform.rb +28 -0
  34. data/lib/ductr/etl/fiber_control.rb +136 -0
  35. data/lib/ductr/etl/fiber_runner.rb +68 -0
  36. data/lib/ductr/etl/kiba_runner.rb +26 -0
  37. data/lib/ductr/etl/parser.rb +115 -0
  38. data/lib/ductr/etl/runner.rb +37 -0
  39. data/lib/ductr/etl_job.rb +161 -0
  40. data/lib/ductr/job.rb +58 -0
  41. data/lib/ductr/job_etl_runner.rb +37 -0
  42. data/lib/ductr/job_status.rb +56 -0
  43. data/lib/ductr/kiba_job.rb +130 -0
  44. data/lib/ductr/log/formatters/color_formatter.rb +48 -0
  45. data/lib/ductr/log/logger.rb +169 -0
  46. data/lib/ductr/log/outputs/file_output.rb +30 -0
  47. data/lib/ductr/log/outputs/standard_output.rb +39 -0
  48. data/lib/ductr/pipeline.rb +133 -0
  49. data/lib/ductr/pipeline_runner.rb +95 -0
  50. data/lib/ductr/pipeline_step.rb +92 -0
  51. data/lib/ductr/registry.rb +55 -0
  52. data/lib/ductr/rufus_trigger.rb +106 -0
  53. data/lib/ductr/scheduler.rb +117 -0
  54. data/lib/ductr/store/job_serializer.rb +59 -0
  55. data/lib/ductr/store/job_store.rb +59 -0
  56. data/lib/ductr/store/pipeline_serializer.rb +106 -0
  57. data/lib/ductr/store/pipeline_store.rb +48 -0
  58. data/lib/ductr/store.rb +81 -0
  59. data/lib/ductr/trigger.rb +49 -0
  60. data/lib/ductr/version.rb +6 -0
  61. data/lib/ductr.rb +143 -0
  62. data/sig/ductr.rbs +1107 -0
  63. metadata +292 -0
metadata ADDED
@@ -0,0 +1,292 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ductr
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Mathieu Morel
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2023-03-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: debug
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.6'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '13.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '13.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rubocop
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.21'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.21'
69
+ - !ruby/object:Gem::Dependency
70
+ name: sord
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '4.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '4.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: yard
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '0.9'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '0.9'
97
+ - !ruby/object:Gem::Dependency
98
+ name: activejob
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '7.0'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '7.0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: annotable
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.1'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.1'
125
+ - !ruby/object:Gem::Dependency
126
+ name: colorize
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '0.8'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '0.8'
139
+ - !ruby/object:Gem::Dependency
140
+ name: kiba
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '4.0'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '4.0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: rufus-scheduler
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '3.8'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '3.8'
167
+ - !ruby/object:Gem::Dependency
168
+ name: thor
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '1.2'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '1.2'
181
+ - !ruby/object:Gem::Dependency
182
+ name: zeitwerk
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '2.6'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - "~>"
193
+ - !ruby/object:Gem::Version
194
+ version: '2.6'
195
+ description: A data pipeline and ETL framework for ruby.
196
+ email:
197
+ - mathieu@lamanufacture.dev
198
+ executables:
199
+ - ductr
200
+ extensions: []
201
+ extra_rdoc_files: []
202
+ files:
203
+ - ".rspec"
204
+ - ".rubocop.yml"
205
+ - ".vscode/settings.json"
206
+ - COPYING
207
+ - COPYING.LESSER
208
+ - Gemfile
209
+ - Gemfile.lock
210
+ - README.md
211
+ - Rakefile
212
+ - bin/console
213
+ - bin/setup
214
+ - ductr.gemspec
215
+ - exe/ductr
216
+ - lib/ductr.rb
217
+ - lib/ductr/adapter.rb
218
+ - lib/ductr/cli/default.rb
219
+ - lib/ductr/cli/main.rb
220
+ - lib/ductr/cli/new_project_generator.rb
221
+ - lib/ductr/cli/templates/project/bin_ductr.rb
222
+ - lib/ductr/cli/templates/project/config_app.rb
223
+ - lib/ductr/cli/templates/project/config_development.yml
224
+ - lib/ductr/cli/templates/project/config_environment_development.rb
225
+ - lib/ductr/cli/templates/project/gemfile.rb
226
+ - lib/ductr/cli/templates/project/rubocop.yml
227
+ - lib/ductr/cli/templates/project/tool-versions
228
+ - lib/ductr/configuration.rb
229
+ - lib/ductr/etl/controls/buffered_destination.rb
230
+ - lib/ductr/etl/controls/buffered_transform.rb
231
+ - lib/ductr/etl/controls/control.rb
232
+ - lib/ductr/etl/controls/destination.rb
233
+ - lib/ductr/etl/controls/paginated_source.rb
234
+ - lib/ductr/etl/controls/source.rb
235
+ - lib/ductr/etl/controls/transform.rb
236
+ - lib/ductr/etl/fiber_control.rb
237
+ - lib/ductr/etl/fiber_runner.rb
238
+ - lib/ductr/etl/kiba_runner.rb
239
+ - lib/ductr/etl/parser.rb
240
+ - lib/ductr/etl/runner.rb
241
+ - lib/ductr/etl_job.rb
242
+ - lib/ductr/job.rb
243
+ - lib/ductr/job_etl_runner.rb
244
+ - lib/ductr/job_status.rb
245
+ - lib/ductr/kiba_job.rb
246
+ - lib/ductr/log/formatters/color_formatter.rb
247
+ - lib/ductr/log/logger.rb
248
+ - lib/ductr/log/outputs/file_output.rb
249
+ - lib/ductr/log/outputs/standard_output.rb
250
+ - lib/ductr/pipeline.rb
251
+ - lib/ductr/pipeline_runner.rb
252
+ - lib/ductr/pipeline_step.rb
253
+ - lib/ductr/registry.rb
254
+ - lib/ductr/rufus_trigger.rb
255
+ - lib/ductr/scheduler.rb
256
+ - lib/ductr/store.rb
257
+ - lib/ductr/store/job_serializer.rb
258
+ - lib/ductr/store/job_store.rb
259
+ - lib/ductr/store/pipeline_serializer.rb
260
+ - lib/ductr/store/pipeline_store.rb
261
+ - lib/ductr/trigger.rb
262
+ - lib/ductr/version.rb
263
+ - sig/ductr.rbs
264
+ homepage: https://github.com/ductr-io/ductr
265
+ licenses:
266
+ - LGPL-3.0-or-later
267
+ metadata:
268
+ allowed_push_host: https://rubygems.org
269
+ rubygems_mfa_required: 'true'
270
+ homepage_uri: https://github.com/ductr-io/ductr
271
+ source_code_uri: https://github.com/ductr-io/ductr
272
+ changelog_uri: https://github.com/ductr-io/ductr/releases
273
+ post_install_message:
274
+ rdoc_options: []
275
+ require_paths:
276
+ - lib
277
+ required_ruby_version: !ruby/object:Gem::Requirement
278
+ requirements:
279
+ - - ">="
280
+ - !ruby/object:Gem::Version
281
+ version: 3.1.0
282
+ required_rubygems_version: !ruby/object:Gem::Requirement
283
+ requirements:
284
+ - - ">="
285
+ - !ruby/object:Gem::Version
286
+ version: '0'
287
+ requirements: []
288
+ rubygems_version: 3.3.26
289
+ signing_key:
290
+ specification_version: 4
291
+ summary: Data pipeline and ETL framework.
292
+ test_files: []