chronicle-etl 0.5.5 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +15 -25
  3. data/.rubocop.yml +2 -44
  4. data/Gemfile +2 -2
  5. data/Guardfile +3 -3
  6. data/README.md +75 -68
  7. data/Rakefile +2 -2
  8. data/bin/console +4 -5
  9. data/chronicle-etl.gemspec +51 -49
  10. data/exe/chronicle-etl +1 -1
  11. data/lib/chronicle/etl/authorizer.rb +3 -4
  12. data/lib/chronicle/etl/cli/authorizations.rb +8 -6
  13. data/lib/chronicle/etl/cli/connectors.rb +7 -7
  14. data/lib/chronicle/etl/cli/jobs.rb +130 -53
  15. data/lib/chronicle/etl/cli/main.rb +29 -29
  16. data/lib/chronicle/etl/cli/plugins.rb +14 -15
  17. data/lib/chronicle/etl/cli/secrets.rb +14 -12
  18. data/lib/chronicle/etl/cli/subcommand_base.rb +5 -3
  19. data/lib/chronicle/etl/config.rb +18 -8
  20. data/lib/chronicle/etl/configurable.rb +20 -9
  21. data/lib/chronicle/etl/exceptions.rb +3 -3
  22. data/lib/chronicle/etl/extraction.rb +12 -2
  23. data/lib/chronicle/etl/extractors/csv_extractor.rb +9 -0
  24. data/lib/chronicle/etl/extractors/extractor.rb +15 -2
  25. data/lib/chronicle/etl/extractors/file_extractor.rb +5 -3
  26. data/lib/chronicle/etl/extractors/helpers/input_reader.rb +2 -2
  27. data/lib/chronicle/etl/extractors/json_extractor.rb +14 -4
  28. data/lib/chronicle/etl/extractors/stdin_extractor.rb +3 -0
  29. data/lib/chronicle/etl/job.rb +35 -17
  30. data/lib/chronicle/etl/job_definition.rb +38 -26
  31. data/lib/chronicle/etl/job_log.rb +14 -16
  32. data/lib/chronicle/etl/job_logger.rb +4 -4
  33. data/lib/chronicle/etl/loaders/csv_loader.rb +17 -4
  34. data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb +4 -0
  35. data/lib/chronicle/etl/loaders/json_loader.rb +30 -10
  36. data/lib/chronicle/etl/loaders/loader.rb +0 -17
  37. data/lib/chronicle/etl/loaders/rest_loader.rb +7 -7
  38. data/lib/chronicle/etl/loaders/table_loader.rb +37 -12
  39. data/lib/chronicle/etl/logger.rb +2 -2
  40. data/lib/chronicle/etl/oauth_authorizer.rb +8 -8
  41. data/lib/chronicle/etl/record.rb +15 -0
  42. data/lib/chronicle/etl/registry/connector_registration.rb +15 -23
  43. data/lib/chronicle/etl/registry/connectors.rb +93 -36
  44. data/lib/chronicle/etl/registry/plugin_registration.rb +1 -1
  45. data/lib/chronicle/etl/registry/plugins.rb +27 -19
  46. data/lib/chronicle/etl/runner.rb +158 -128
  47. data/lib/chronicle/etl/secrets.rb +4 -4
  48. data/lib/chronicle/etl/transformers/buffer_transformer.rb +29 -0
  49. data/lib/chronicle/etl/transformers/chronicle_transformer.rb +32 -0
  50. data/lib/chronicle/etl/transformers/chronobase_transformer.rb +100 -0
  51. data/lib/chronicle/etl/transformers/fields_limit_transformer.rb +23 -0
  52. data/lib/chronicle/etl/transformers/filter_fields_transformer.rb +60 -0
  53. data/lib/chronicle/etl/transformers/filter_transformer.rb +30 -0
  54. data/lib/chronicle/etl/transformers/format_transformer.rb +32 -0
  55. data/lib/chronicle/etl/transformers/merge_meta_transformer.rb +19 -0
  56. data/lib/chronicle/etl/transformers/multiply_transformer.rb +21 -0
  57. data/lib/chronicle/etl/transformers/null_transformer.rb +5 -7
  58. data/lib/chronicle/etl/transformers/sampler_transformer.rb +21 -0
  59. data/lib/chronicle/etl/transformers/sort_transformer.rb +31 -0
  60. data/lib/chronicle/etl/transformers/transformer.rb +63 -41
  61. data/lib/chronicle/etl/utils/binary_attachments.rb +1 -1
  62. data/lib/chronicle/etl/utils/progress_bar.rb +2 -3
  63. data/lib/chronicle/etl/version.rb +1 -1
  64. data/lib/chronicle/etl.rb +6 -8
  65. metadata +49 -47
  66. data/lib/chronicle/etl/models/activity.rb +0 -15
  67. data/lib/chronicle/etl/models/attachment.rb +0 -14
  68. data/lib/chronicle/etl/models/base.rb +0 -122
  69. data/lib/chronicle/etl/models/entity.rb +0 -29
  70. data/lib/chronicle/etl/models/raw.rb +0 -26
  71. data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +0 -31
  72. data/lib/chronicle/etl/serializers/raw_serializer.rb +0 -10
  73. data/lib/chronicle/etl/serializers/serializer.rb +0 -28
  74. data/lib/chronicle/etl/transformers/image_file_transformer.rb +0 -247
  75. data/lib/chronicle/etl/utils/hash_utilities.rb +0 -19
  76. data/lib/chronicle/etl/utils/text_recognition.rb +0 -15
@@ -7,7 +7,7 @@ module Chronicle
7
7
 
8
8
  # Macro for setting provider on an Authorizer
9
9
  def provider(provider_name)
10
- @provider_name = provider_name
10
+ @provider_name = provider_name.to_sym
11
11
  end
12
12
 
13
13
  # From all loaded Authorizers, return the first one that matches
@@ -16,15 +16,14 @@ module Chronicle
16
16
  # @todo Have a proper identifier system for authorizers
17
17
  # (to have more than one per plugin)
18
18
  def find_by_provider(provider)
19
- ObjectSpace.each_object(::Class).select {|klass| klass < self }.find do |authorizer|
19
+ ObjectSpace.each_object(::Class).select { |klass| klass < self }.find do |authorizer|
20
20
  authorizer.provider_name == provider
21
21
  end
22
22
  end
23
23
  end
24
24
 
25
25
  # Construct a new authorizer
26
- def initialize(args)
27
- end
26
+ def initialize(args); end
28
27
 
29
28
  # Main entry-point for authorization flows. Implemented by subclass
30
29
  def authorize!
@@ -2,7 +2,6 @@
2
2
 
3
3
  require 'sinatra'
4
4
  require 'launchy'
5
- require 'pp'
6
5
 
7
6
  module Chronicle
8
7
  module ETL
@@ -12,11 +11,14 @@ module Chronicle
12
11
  default_task 'new'
13
12
  namespace :authorizations
14
13
 
15
- desc "authorize", "Authorize with a third-party provider"
14
+ desc 'authorize', 'Authorize with a third-party provider'
16
15
  option :port, desc: 'Port to run authorization server on', type: :numeric, default: 4567
17
- option :credentials, desc: 'Secrets namespace for where to read credentials from (default: PROVIDER)', type: :string, banner: 'NAMESPACE'
18
- option :secrets, desc: 'Secrets namespace for where authorization should be saved to (default: PROVIDER)', type: :string, banner: 'NAMESPACE'
19
- option :print, desc: 'Show authorization results (instead of just saving secrets)', type: :boolean, default: false
16
+ option :credentials, desc: 'Secrets namespace for where to read credentials from (default: PROVIDER)',
17
+ type: :string, banner: 'NAMESPACE'
18
+ option :secrets, desc: 'Secrets namespace for where authorization should be saved to (default: PROVIDER)',
19
+ type: :string, banner: 'NAMESPACE'
20
+ option :print, desc: 'Show authorization results (instead of just saving secrets)', type: :boolean,
21
+ default: false
20
22
  def new(provider)
21
23
  authorizer_klass = find_authorizer_klass(provider)
22
24
  credentials = load_credentials(provider: provider, credentials_source: options[:credentials])
@@ -30,7 +32,7 @@ module Chronicle
30
32
 
31
33
  cli_exit(message: "Authorization saved to '#{secrets_namespace}' secrets")
32
34
  rescue StandardError => e
33
- cli_fail(message: "Authorization not successful.\n" + e.message, exception: e)
35
+ cli_fail(message: "Authorization not successful.\n#{e.message}", exception: e)
34
36
  end
35
37
 
36
38
  private
@@ -10,7 +10,7 @@ module Chronicle
10
10
  default_task 'list'
11
11
  namespace :connectors
12
12
 
13
- desc "list", "Lists available connectors"
13
+ desc 'list', 'Lists available connectors'
14
14
  # Display all available connectors that chronicle-etl has access to
15
15
  def list
16
16
  connector_info = Chronicle::ETL::Registry::Connectors.connectors.map do |connector_registration|
@@ -18,7 +18,7 @@ module Chronicle
18
18
  identifier: connector_registration.identifier,
19
19
  phase: connector_registration.phase,
20
20
  description: connector_registration.descriptive_phrase,
21
- provider: connector_registration.provider,
21
+ source: connector_registration.source,
22
22
  core: connector_registration.built_in? ? '✓' : '',
23
23
  class: connector_registration.klass_name
24
24
  }
@@ -36,10 +36,10 @@ module Chronicle
36
36
  puts table.render(indent: 0, padding: [0, 2])
37
37
  end
38
38
 
39
- desc "show PHASE IDENTIFIER", "Show information about a connector"
39
+ desc 'show PHASE IDENTIFIER', 'Show information about a connector'
40
40
  def show(phase, identifier)
41
- unless ['extractor', 'transformer', 'loader'].include?(phase)
42
- cli_fail(message: "Phase argument must be one of: [extractor, transformer, loader]")
41
+ unless %w[extractor transformer loader].include?(phase)
42
+ cli_fail(message: 'Phase argument must be one of: [extractor, transformer, loader]')
43
43
  end
44
44
 
45
45
  begin
@@ -51,9 +51,9 @@ module Chronicle
51
51
  puts connector.klass.to_s.bold
52
52
  puts " #{connector.descriptive_phrase}"
53
53
  puts
54
- puts "Settings:"
54
+ puts 'Settings:'
55
55
 
56
- headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
56
+ headers = %w[name default required].map { |h| h.to_s.upcase.bold }
57
57
 
58
58
  settings = connector.klass.settings.map do |name, setting|
59
59
  [
@@ -1,4 +1,5 @@
1
- require 'pp'
1
+ # frozen_string_literal: true
2
+
2
3
  require 'tty-prompt'
3
4
 
4
5
  module Chronicle
@@ -6,43 +7,67 @@ module Chronicle
6
7
  module CLI
7
8
  # CLI commands for working with ETL jobs
8
9
  class Jobs < SubcommandBase
9
- default_task "start"
10
+ default_task 'start'
10
11
  namespace :jobs
11
12
 
12
- class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'NAME'
13
+ class_option :extractor, aliases: '-e', desc: 'Extractor class. Default: stdin', banner: 'NAME'
13
14
  class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
14
- class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'NAME'
15
- class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
15
+ class_option :transformer,
16
+ aliases: '-t',
17
+ desc: 'Transformer identifier. Default: null',
18
+ banner: 'NAME',
19
+ type: 'array',
20
+ repeatable: true
16
21
  class_option :loader, aliases: '-l', desc: 'Loader class. Default: table', banner: 'NAME'
17
22
  class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
18
23
 
19
24
  # This is an array to deal with shell globbing
20
- class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
21
- class_option :since, desc: "Load records SINCE this date (or fuzzy time duration)", banner: 'DATE'
22
- class_option :until, desc: "Load records UNTIL this date (or fuzzy time duration)", banner: 'DATE'
23
- class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
25
+ class_option :input,
26
+ aliases: '-i',
27
+ desc: 'Input filename or directory',
28
+ default: [],
29
+ type: 'array',
30
+ banner: 'FILENAME'
31
+ class_option :since, desc: 'Load records SINCE this date (or fuzzy time duration)', banner: 'DATE'
32
+ class_option :until, desc: 'Load records UNTIL this date (or fuzzy time duration)', banner: 'DATE'
33
+ class_option :limit, desc: 'Only extract the first LIMIT records', banner: 'N'
34
+
35
+ class_option :schema,
36
+ desc: 'Which Schema to transform',
37
+ banner: 'SCHEMA_NAME',
38
+ type: 'string',
39
+ enum: %w[chronicle activitystream schemaorg chronobase]
40
+ class_option :format,
41
+ desc: 'How to serialize results',
42
+ banner: 'SCHEMA_NAME',
43
+ type: 'string',
44
+ enum: %w[jsonapi jsonld]
24
45
 
25
46
  class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
26
47
  class_option :fields, desc: 'Output only these fields', type: 'array', banner: 'field1 field2 ...'
48
+ class_option :'fields-limit', desc: 'Output first N fields', type: :numeric
49
+ class_option :filter, desc: 'Filter records', type: 'array', banner: 'field=value'
27
50
  class_option :header_row, desc: 'Output the header row of tabular output', type: 'boolean'
28
51
 
29
52
  # Thor doesn't like `run` as a command name
30
53
  map run: :start
31
- desc "run", "Start a job"
54
+ desc 'run', 'Start a job'
32
55
  option :dry_run, desc: 'Only run the extraction and transform steps, not the loading', type: :boolean
33
56
  long_desc <<-LONG_DESC
34
57
  This will run an ETL job. Each job needs three parts:
35
58
 
36
59
  1. #{'Extractor'.underline}: pulls data from an external source. By default, this is stdout. Other common options including pulling data from an API or reading JSON from a file.
37
60
 
38
- 2. #{'Transformer'.underline}: transforms data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
61
+ 2. #{'Transformers'.underline}: transform data into a new format. If none is specified, we use the `null` transformer which does nothing to the data.
39
62
 
40
63
  3. #{'Loader'.underline}: takes that transformed data and loads it externally. This can be an API, flat files, (or by default), stdout. With the --dry-run option, this step won't be run.
41
64
 
42
65
  If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
43
- LONG_DESC
66
+ LONG_DESC
44
67
  # Run an ETL job
45
- def start(name = nil)
68
+ def start(*args)
69
+ name = args.first
70
+
46
71
  # If someone runs `$ chronicle-etl` with no arguments, show help menu.
47
72
  # TODO: decide if we should check that there's nothing in stdin pipe
48
73
  # in case user wants to actually run this sort of job stdin->null->stdout
@@ -52,7 +77,7 @@ LONG_DESC
52
77
  cli_exit
53
78
  end
54
79
 
55
- cli_fail(message: "Job '#{name}' does not exist") if name && !Chronicle::ETL::Config.exists?("jobs", name)
80
+ cli_fail(message: "Job '#{name}' does not exist") if name && !Chronicle::ETL::Config.exists?('jobs', name)
56
81
 
57
82
  job_definition = build_job_definition(name, options)
58
83
 
@@ -66,7 +91,7 @@ LONG_DESC
66
91
 
67
92
  run_job(job_definition)
68
93
  rescue Chronicle::ETL::JobDefinitionError => e
69
- message = ""
94
+ message = ''
70
95
  job_definition.errors.each_pair do |category, errors|
71
96
  message << "Problem with #{category}:\n - #{errors.map(&:to_s).join("\n - ")}"
72
97
  end
@@ -74,14 +99,14 @@ LONG_DESC
74
99
  end
75
100
 
76
101
  option :'skip-confirmation', aliases: '-y', type: :boolean
77
- desc "save", "Save a job"
102
+ desc 'save', 'Save a job'
78
103
  # Create an ETL job
79
104
  def save(name)
80
105
  write_config = true
81
106
  job_definition = build_job_definition(name, options)
82
107
  job_definition.validate!
83
108
 
84
- if Chronicle::ETL::Config.exists?("jobs", name) && !options[:'skip-confirmation']
109
+ if Chronicle::ETL::Config.exists?('jobs', name) && !options[:'skip-confirmation']
85
110
  prompt = TTY::Prompt.new
86
111
  write_config = false
87
112
  message = "Job '#{name}' exists already. Ovewrite it?"
@@ -92,34 +117,50 @@ LONG_DESC
92
117
  end
93
118
 
94
119
  if write_config
95
- Chronicle::ETL::Config.write("jobs", name, job_definition.definition)
120
+ Chronicle::ETL::Config.write('jobs', name, job_definition.definition)
96
121
  cli_exit(message: "Job saved. Run it with `$ chronicle-etl jobs:run #{name}`")
97
122
  else
98
123
  cli_fail(message: "\nJob not saved")
99
124
  end
100
125
  rescue Chronicle::ETL::JobDefinitionError => e
101
- cli_fail(message: "Job definition error", exception: e)
126
+ cli_fail(message: 'Job definition error', exception: e)
102
127
  end
103
128
 
104
- desc "show", "Show details about a job"
129
+ desc 'show', 'Show details about a job'
105
130
  # Show an ETL job
106
131
  def show(name = nil)
107
- cli_fail(message: "Job '#{name}' does not exist") if name && !Chronicle::ETL::Config.exists?("jobs", name)
132
+ cli_fail(message: "Job '#{name}' does not exist") if name && !Chronicle::ETL::Config.exists?('jobs', name)
108
133
 
109
134
  job_definition = build_job_definition(name, options)
110
135
  job_definition.validate!
111
136
  puts Chronicle::ETL::Job.new(job_definition)
112
137
  rescue Chronicle::ETL::JobDefinitionError => e
113
- cli_fail(message: "Job definition error", exception: e)
138
+ cli_fail(message: 'Job definition error', exception: e)
139
+ end
140
+
141
+ desc 'edit', 'Edit a job in default editor ($EDITOR)'
142
+ def edit(name = nil)
143
+ cli_fail(message: "Job '#{name}' does not exist") if name && !Chronicle::ETL::Config.exists?('jobs', name)
144
+
145
+ filename = Chronicle::ETL::Config.path('jobs', name)
146
+ system "${VISUAL:-${EDITOR:-vi}} \"#{filename}\""
147
+
148
+ definition = Chronicle::ETL::JobDefinition.new
149
+ definition.add_config(load_job_config(name))
150
+ definition.validate!
151
+
152
+ cli_exit(message: "Job '#{name}' saved")
153
+ rescue Chronicle::ETL::JobDefinitionError => e
154
+ cli_fail(message: 'Job definition error', exception: e)
114
155
  end
115
156
 
116
- desc "list", "List all available jobs"
157
+ desc 'list', 'List all available jobs'
117
158
  # List available ETL jobs
118
159
  def list
119
160
  jobs = Chronicle::ETL::Config.available_jobs
120
161
 
121
162
  job_details = jobs.map do |job|
122
- r = Chronicle::ETL::Config.load("jobs", job)
163
+ r = Chronicle::ETL::Config.load('jobs', job)
123
164
 
124
165
  extractor = r[:extractor][:name] if r[:extractor]
125
166
  transformer = r[:transformer][:name] if r[:transformer]
@@ -128,9 +169,9 @@ LONG_DESC
128
169
  [job, extractor, transformer, loader]
129
170
  end
130
171
 
131
- headers = ['name', 'extractor', 'transformer', 'loader'].map { |h| h.upcase.bold }
172
+ headers = %w[name extractor transformer loader].map { |h| h.upcase.bold }
132
173
 
133
- puts "Available jobs:"
174
+ puts 'Available jobs:'
134
175
  table = TTY::Table.new(headers, job_details)
135
176
  puts table.render(indent: 0, padding: [0, 2])
136
177
  rescue Chronicle::ETL::ConfigError => e
@@ -148,18 +189,19 @@ LONG_DESC
148
189
  runner = Chronicle::ETL::Runner.new(job)
149
190
  runner.run!
150
191
  rescue RunnerError => e
151
- cli_fail(message: "#{e.message}", exception: e)
192
+ cli_fail(message: e.message.to_s, exception: e)
152
193
  end
153
194
 
154
195
  # TODO: probably could merge this with something in cli/plugin
155
196
  def install_missing_plugins(missing_plugins)
156
197
  prompt = TTY::Prompt.new
157
198
  message = "Plugin#{'s' if missing_plugins.count > 1} specified by job not installed.\n"
158
- message += "Do you want to install "
159
- message += missing_plugins.map { |name| "chronicle-#{name}".bold}.join(", ")
160
- message += " and start the job?"
199
+ message += 'Do you want to install '
200
+ message += missing_plugins.map { |name| "chronicle-#{name}".bold }
201
+ .join(', ')
202
+ message += ' and start the job?'
161
203
  will_install = prompt.yes?(message)
162
- cli_fail(message: "Must install #{missing_plugins.join(", ")} plugin to run job") unless will_install
204
+ cli_fail(message: "Must install #{missing_plugins.join(', ')} plugin to run job") unless will_install
163
205
 
164
206
  Chronicle::ETL::CLI::Plugins.new.install(*missing_plugins)
165
207
  end
@@ -172,43 +214,78 @@ LONG_DESC
172
214
  definition
173
215
  end
174
216
 
175
- def load_job_config name
217
+ def load_job_config(name)
176
218
  Chronicle::ETL::Config.read_job(name)
177
219
  end
178
220
 
179
221
  # Takes flag options and turns them into a runner config
180
222
  # TODO: this needs a lot of refactoring
181
- def process_flag_options options
182
- extractor_options = options[:'extractor-opts'].transform_keys(&:to_sym).merge({
183
- input: (options[:input] if options[:input].any?),
184
- since: options[:since],
185
- until: options[:until],
186
- limit: options[:limit]
187
- }.compact)
188
-
189
- transformer_options = options[:'transformer-opts'].transform_keys(&:to_sym)
190
-
191
- loader_options = options[:'loader-opts'].transform_keys(&:to_sym).merge({
192
- output: options[:output],
193
- header_row: options[:header_row],
194
- fields: options[:fields]
195
- }.compact)
196
-
197
- {
223
+ def process_flag_options(options)
224
+ extractor_options = options[:'extractor-opts'].transform_keys(&:to_sym).merge(
225
+ {
226
+ input: (options[:input] if options[:input].any?),
227
+ since: options[:since],
228
+ until: options[:until],
229
+ limit: options[:limit]
230
+ }.compact
231
+ )
232
+
233
+ loader_options = options[:'loader-opts'].transform_keys(&:to_sym).merge(
234
+ {
235
+ output: options[:output],
236
+ header_row: options[:header_row]
237
+ }.compact
238
+ )
239
+
240
+ processed_options = {
198
241
  dry_run: options[:dry_run],
199
242
  extractor: {
200
243
  name: options[:extractor],
201
244
  options: extractor_options
202
245
  }.compact,
203
- transformer: {
204
- name: options[:transformer],
205
- options: transformer_options
206
- }.compact,
207
246
  loader: {
208
247
  name: options[:loader],
209
248
  options: loader_options
210
249
  }.compact
211
250
  }
251
+
252
+ add_transformer(processed_options, 'chronicle') if options[:schema]
253
+ add_transformer(processed_options, options[:schema]) if options[:schema] && options[:schema] != 'chronicle'
254
+ add_transformers_from_option(processed_options, options[:transformer]) if options[:transformer]&.any?
255
+ if options[:filter]
256
+ add_transformer(processed_options, :filter, { filters: options[:filter].to_h do |f|
257
+ f.split('=')
258
+ end })
259
+ end
260
+ add_transformer(processed_options, :format, { format: options[:format] }) if options[:format]
261
+ add_transformer(processed_options, :filter_fields, { fields: options[:fields] }) if options[:fields]
262
+ if options[:'fields-limit']
263
+ add_transformer(processed_options, :fields_limit,
264
+ { limit: options[:'fields-limit'] })
265
+ end
266
+
267
+ processed_options
268
+ end
269
+
270
+ def add_transformer(processed_options, name, options = {})
271
+ processed_options[:transformers] ||= []
272
+ processed_options[:transformers] << { name:, options: }
273
+ end
274
+
275
+ def add_transformers_from_option(processed_options, transformer_option)
276
+ processed_options[:transformers] ||= []
277
+ processed_options[:transformers] += transformer_option.map do |transformer_args|
278
+ transformer_name, *transformer_options = transformer_args
279
+ transformer_options = transformer_options.filter { |opt| opt.include?('=') }
280
+
281
+ {
282
+ name: transformer_name,
283
+ options: transformer_options.to_h do |opt|
284
+ key, value = opt.split('=')
285
+ [key.to_sym, value]
286
+ end
287
+ }
288
+ end
212
289
  end
213
290
  end
214
291
  end
@@ -13,7 +13,7 @@ module Chronicle
13
13
  class_option :silent, desc: 'Silence all output', type: :boolean
14
14
  class_option :'no-color', desc: 'Disable colour output', type: :boolean
15
15
 
16
- default_task "jobs"
16
+ default_task 'jobs'
17
17
 
18
18
  desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
19
19
  subcommand 'connectors', Connectors
@@ -45,49 +45,49 @@ module Chronicle
45
45
  true
46
46
  end
47
47
 
48
- desc "version", "Show version"
49
- map %w(--version -v) => :version
48
+ desc 'version', 'Show version'
49
+ map %w[--version -v] => :version
50
50
  def version
51
51
  shell.say "chronicle-etl #{Chronicle::ETL::VERSION}"
52
52
  end
53
53
 
54
54
  # Displays help options for chronicle-etl
55
- def help(meth = nil, subcommand = false)
55
+ def help(meth = nil, _subcommand = false)
56
56
  if meth && !respond_to?(meth)
57
57
  klass, task = ::Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
58
- klass.start(['-h', task].compact, shell: shell)
58
+ klass.start(['-h', task].compact, shell:)
59
59
  else
60
- shell.say "ABOUT:".bold
60
+ shell.say 'ABOUT:'.bold
61
61
  shell.say " #{'chronicle-etl'.italic} is a toolkit for extracting and working with your digital"
62
- shell.say " history. 📜"
62
+ shell.say ' history. 📜'
63
63
  shell.say
64
64
  shell.say " A job #{'extracts'.underline} personal data from a source, #{'transforms'.underline} it (Chronicle"
65
65
  shell.say " Schema or preserves raw data), and then #{'loads'.underline} it to a destination. Use"
66
- shell.say " built-in extractors (json, csv, stdin) and loaders (csv, json, table,"
67
- shell.say " rest) or use plugins to connect to third-party services."
66
+ shell.say ' built-in extractors (json, csv, stdin) and loaders (csv, json, table,'
67
+ shell.say ' rest) or use plugins to connect to third-party services.'
68
68
  shell.say
69
- shell.say " Plugins: https://github.com/chronicle-app/chronicle-etl#currently-available"
69
+ shell.say ' Plugins: https://github.com/chronicle-app/chronicle-etl#currently-available'
70
70
  shell.say
71
- shell.say "USAGE:".bold
72
- shell.say " # Basic job usage:".italic.light_black
73
- shell.say " $ chronicle-etl --extractor NAME --transformer NAME --loader NAME"
71
+ shell.say 'USAGE:'.bold
72
+ shell.say ' # Basic job usage:'.italic.light_black
73
+ shell.say ' $ chronicle-etl --extractor NAME --transformer NAME --loader NAME'
74
74
  shell.say
75
- shell.say " # Read test.csv and display it to stdout as a table:".italic.light_black
76
- shell.say " $ chronicle-etl --extractor csv --input data.csv --loader table"
75
+ shell.say ' # Read test.csv and display it to stdout as a table:'.italic.light_black
76
+ shell.say ' $ chronicle-etl --extractor csv --input data.csv --loader table'
77
77
  shell.say
78
- shell.say " # Show available plugins:".italic.light_black
79
- shell.say " $ chronicle-etl plugins:list"
78
+ shell.say ' # Show available plugins:'.italic.light_black
79
+ shell.say ' $ chronicle-etl plugins:list'
80
80
  shell.say
81
- shell.say " # Save an access token as a secret and use it in a job:".italic.light_black
82
- shell.say " $ chronicle-etl secrets:set pinboard access_token username:foo123"
83
- shell.say " $ chronicle-etl secrets:list"
84
- shell.say " $ chronicle-etl -e pinboard --since 1mo"
81
+ shell.say ' # Save an access token as a secret and use it in a job:'.italic.light_black
82
+ shell.say ' $ chronicle-etl secrets:set pinboard access_token username:foo123'
83
+ shell.say ' $ chronicle-etl secrets:list'
84
+ shell.say ' $ chronicle-etl -e pinboard --since 1mo'
85
85
  shell.say
86
- shell.say " # Show full job options:".italic.light_black
87
- shell.say " $ chronicle-etl jobs help run"
86
+ shell.say ' # Show full job options:'.italic.light_black
87
+ shell.say ' $ chronicle-etl jobs help run'
88
88
  shell.say
89
- shell.say "FULL DOCUMENTATION:".bold
90
- shell.say " https://github.com/chronicle-app/chronicle-etl".blue
89
+ shell.say 'FULL DOCUMENTATION:'.bold
90
+ shell.say ' https://github.com/chronicle-app/chronicle-etl'.blue
91
91
  shell.say
92
92
 
93
93
  list = []
@@ -95,17 +95,17 @@ module Chronicle
95
95
  list += thor_class.printable_tasks(false)
96
96
  end
97
97
  list.sort! { |a, b| a[0] <=> b[0] }
98
- list.unshift ["help", "# This help menu"]
98
+ list.unshift ['help', '# This help menu']
99
99
 
100
100
  shell.say
101
101
  shell.say 'ALL COMMANDS:'.bold
102
102
  shell.print_table(list, indent: 2, truncate: true)
103
103
  shell.say
104
- shell.say "VERSION:".bold
104
+ shell.say 'VERSION:'.bold
105
105
  shell.say " #{Chronicle::ETL::VERSION}"
106
106
  shell.say
107
- shell.say " Display current version:".italic.light_black
108
- shell.say " $ chronicle-etl --version"
107
+ shell.say ' Display current version:'.italic.light_black
108
+ shell.say ' $ chronicle-etl --version'
109
109
  end
110
110
  end
111
111
 
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "tty-prompt"
4
- require "tty-spinner"
3
+ require 'tty-prompt'
4
+ require 'tty-spinner'
5
5
 
6
6
  module Chronicle
7
7
  module ETL
@@ -11,63 +11,62 @@ module Chronicle
11
11
  default_task 'list'
12
12
  namespace :plugins
13
13
 
14
- desc "install", "Install a plugin"
14
+ desc 'install', 'Install a plugin'
15
15
  def install(*plugins)
16
- cli_fail(message: "Please specify a plugin to install") unless plugins.any?
16
+ cli_fail(message: 'Please specify a plugin to install') unless plugins.any?
17
17
 
18
18
  installed, not_installed = plugins.partition do |plugin|
19
19
  Chronicle::ETL::Registry::Plugins.installed?(plugin)
20
20
  end
21
21
 
22
- puts "Already installed: #{installed.join(", ")}" if installed.any?
22
+ puts "Already installed: #{installed.join(', ')}" if installed.any?
23
23
  cli_exit unless not_installed.any?
24
24
 
25
- spinner = TTY::Spinner.new("[:spinner] Installing #{not_installed.join(", ")}...", format: :dots_2)
25
+ spinner = TTY::Spinner.new("[:spinner] Installing #{not_installed.join(', ')}...", format: :dots_2)
26
26
  spinner.auto_spin
27
27
 
28
28
  not_installed.each do |plugin|
29
29
  spinner.update(title: "Installing #{plugin}")
30
30
  Chronicle::ETL::Registry::Plugins.install(plugin)
31
-
32
31
  rescue Chronicle::ETL::PluginError => e
33
- spinner.error("Error".red)
32
+ spinner.error('Error'.red)
34
33
  cli_fail(message: "Plugin '#{plugin}' could not be installed", exception: e)
35
34
  end
36
35
 
37
36
  spinner.success("(#{'successful'.green})")
38
37
  end
39
38
 
40
- desc "uninstall", "Unintall a plugin"
39
+ desc 'uninstall', 'Unintall a plugin'
41
40
  def uninstall(name)
42
41
  spinner = TTY::Spinner.new("[:spinner] Uninstalling plugin #{name}...", format: :dots_2)
43
42
  spinner.auto_spin
44
43
  Chronicle::ETL::Registry::Plugins.uninstall(name)
45
44
  spinner.success("(#{'successful'.green})")
46
45
  rescue Chronicle::ETL::PluginError => e
47
- spinner.error("Error".red)
46
+ spinner.error('Error'.red)
48
47
  cli_fail(message: "Plugin '#{name}' could not be uninstalled (was it installed?)", exception: e)
49
48
  end
50
49
 
51
- desc "list", "Lists available plugins"
50
+ desc 'list', 'Lists available plugins'
52
51
  # Display all available plugins that chronicle-etl has access to
53
52
  def list
54
53
  values = Chronicle::ETL::Registry::Plugins.all
55
54
  .map do |plugin|
56
55
  [
57
- plugin.name,
56
+ plugin.name,
58
57
  plugin.description,
59
58
  plugin.installed ? '✓' : '',
60
59
  plugin.version
61
60
  ]
62
61
  end
63
62
 
64
- headers = ['name', 'description', 'installed', 'version'].map{ |h| h.to_s.upcase.bold }
63
+ headers = %w[name description installed version].map { |h| h.to_s.upcase.bold }
65
64
  table = TTY::Table.new(headers, values)
66
- puts "Available plugins:"
65
+ puts 'Available plugins:'
67
66
  puts table.render(
68
67
  indent: 2,
69
68
  padding: [0, 0],
70
- alignments: [:left, :left, :center, :left]
69
+ alignments: %i[left left center left]
71
70
  )
72
71
  end
73
72
  end