chronicle-etl 0.5.5 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +15 -25
- data/.rubocop.yml +2 -44
- data/Gemfile +2 -2
- data/Guardfile +3 -3
- data/README.md +75 -68
- data/Rakefile +2 -2
- data/bin/console +4 -5
- data/chronicle-etl.gemspec +51 -49
- data/exe/chronicle-etl +1 -1
- data/lib/chronicle/etl/authorizer.rb +3 -4
- data/lib/chronicle/etl/cli/authorizations.rb +8 -6
- data/lib/chronicle/etl/cli/connectors.rb +7 -7
- data/lib/chronicle/etl/cli/jobs.rb +130 -53
- data/lib/chronicle/etl/cli/main.rb +29 -29
- data/lib/chronicle/etl/cli/plugins.rb +14 -15
- data/lib/chronicle/etl/cli/secrets.rb +14 -12
- data/lib/chronicle/etl/cli/subcommand_base.rb +5 -3
- data/lib/chronicle/etl/config.rb +18 -8
- data/lib/chronicle/etl/configurable.rb +20 -9
- data/lib/chronicle/etl/exceptions.rb +3 -3
- data/lib/chronicle/etl/extraction.rb +12 -2
- data/lib/chronicle/etl/extractors/csv_extractor.rb +9 -0
- data/lib/chronicle/etl/extractors/extractor.rb +15 -2
- data/lib/chronicle/etl/extractors/file_extractor.rb +5 -3
- data/lib/chronicle/etl/extractors/helpers/input_reader.rb +2 -2
- data/lib/chronicle/etl/extractors/json_extractor.rb +14 -4
- data/lib/chronicle/etl/extractors/stdin_extractor.rb +3 -0
- data/lib/chronicle/etl/job.rb +35 -17
- data/lib/chronicle/etl/job_definition.rb +38 -26
- data/lib/chronicle/etl/job_log.rb +14 -16
- data/lib/chronicle/etl/job_logger.rb +4 -4
- data/lib/chronicle/etl/loaders/csv_loader.rb +17 -4
- data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb +4 -0
- data/lib/chronicle/etl/loaders/json_loader.rb +30 -10
- data/lib/chronicle/etl/loaders/loader.rb +0 -17
- data/lib/chronicle/etl/loaders/rest_loader.rb +7 -7
- data/lib/chronicle/etl/loaders/table_loader.rb +37 -12
- data/lib/chronicle/etl/logger.rb +2 -2
- data/lib/chronicle/etl/oauth_authorizer.rb +8 -8
- data/lib/chronicle/etl/record.rb +15 -0
- data/lib/chronicle/etl/registry/connector_registration.rb +15 -23
- data/lib/chronicle/etl/registry/connectors.rb +93 -36
- data/lib/chronicle/etl/registry/plugin_registration.rb +1 -1
- data/lib/chronicle/etl/registry/plugins.rb +27 -19
- data/lib/chronicle/etl/runner.rb +158 -128
- data/lib/chronicle/etl/secrets.rb +4 -4
- data/lib/chronicle/etl/transformers/buffer_transformer.rb +29 -0
- data/lib/chronicle/etl/transformers/chronicle_transformer.rb +32 -0
- data/lib/chronicle/etl/transformers/chronobase_transformer.rb +100 -0
- data/lib/chronicle/etl/transformers/fields_limit_transformer.rb +23 -0
- data/lib/chronicle/etl/transformers/filter_fields_transformer.rb +60 -0
- data/lib/chronicle/etl/transformers/filter_transformer.rb +30 -0
- data/lib/chronicle/etl/transformers/format_transformer.rb +32 -0
- data/lib/chronicle/etl/transformers/merge_meta_transformer.rb +19 -0
- data/lib/chronicle/etl/transformers/multiply_transformer.rb +21 -0
- data/lib/chronicle/etl/transformers/null_transformer.rb +5 -7
- data/lib/chronicle/etl/transformers/sampler_transformer.rb +21 -0
- data/lib/chronicle/etl/transformers/sort_transformer.rb +31 -0
- data/lib/chronicle/etl/transformers/transformer.rb +63 -41
- data/lib/chronicle/etl/utils/binary_attachments.rb +1 -1
- data/lib/chronicle/etl/utils/progress_bar.rb +2 -3
- data/lib/chronicle/etl/version.rb +1 -1
- data/lib/chronicle/etl.rb +6 -8
- metadata +49 -47
- data/lib/chronicle/etl/models/activity.rb +0 -15
- data/lib/chronicle/etl/models/attachment.rb +0 -14
- data/lib/chronicle/etl/models/base.rb +0 -122
- data/lib/chronicle/etl/models/entity.rb +0 -29
- data/lib/chronicle/etl/models/raw.rb +0 -26
- data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +0 -31
- data/lib/chronicle/etl/serializers/raw_serializer.rb +0 -10
- data/lib/chronicle/etl/serializers/serializer.rb +0 -28
- data/lib/chronicle/etl/transformers/image_file_transformer.rb +0 -247
- data/lib/chronicle/etl/utils/hash_utilities.rb +0 -19
- data/lib/chronicle/etl/utils/text_recognition.rb +0 -15
@@ -9,13 +9,13 @@ module Chronicle
|
|
9
9
|
extend Forwardable
|
10
10
|
|
11
11
|
attr_accessor :job,
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
12
|
+
:job_id,
|
13
|
+
:last_id,
|
14
|
+
:highest_timestamp,
|
15
|
+
:num_records_processed,
|
16
|
+
:started_at,
|
17
|
+
:finished_at,
|
18
|
+
:success
|
19
19
|
|
20
20
|
def_delegators :@job, :save_log?
|
21
21
|
|
@@ -28,11 +28,11 @@ module Chronicle
|
|
28
28
|
|
29
29
|
# Log the result of a single transformation in a job
|
30
30
|
# @param transformer [Chronicle::ETL::Tranformer] The transformer that ran
|
31
|
-
def log_transformation(
|
32
|
-
@last_id = transformer.id if transformer.id
|
31
|
+
def log_transformation(_transformer)
|
32
|
+
# @last_id = transformer.id if transformer.id
|
33
33
|
|
34
34
|
# Save the highest timestamp that we've encountered so far
|
35
|
-
@highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
|
35
|
+
# @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
|
36
36
|
|
37
37
|
# TODO: a transformer might yield nil. We might also want certain transformers to explode
|
38
38
|
# records into multiple new ones. Therefore, this this variable will need more subtle behaviour
|
@@ -54,13 +54,13 @@ module Chronicle
|
|
54
54
|
@finished_at = Time.now
|
55
55
|
end
|
56
56
|
|
57
|
-
def job=
|
57
|
+
def job=(job)
|
58
58
|
@job = job
|
59
59
|
@job_id = job.id
|
60
60
|
end
|
61
61
|
|
62
62
|
def duration
|
63
|
-
return unless @finished_at
|
63
|
+
return unless @finished_at && @started_at
|
64
64
|
|
65
65
|
@finished_at - @started_at
|
66
66
|
end
|
@@ -78,14 +78,12 @@ module Chronicle
|
|
78
78
|
}
|
79
79
|
end
|
80
80
|
|
81
|
-
private
|
82
|
-
|
83
81
|
# Create a new JobLog and set its instance variables from a serialized hash
|
84
|
-
def self.build_from_serialized
|
82
|
+
def self.build_from_serialized(attrs)
|
85
83
|
attrs.delete(:id)
|
86
84
|
new do |job_log|
|
87
85
|
attrs.each do |key, value|
|
88
|
-
setter = "#{key
|
86
|
+
setter = :"#{key}="
|
89
87
|
job_log.send(setter, value)
|
90
88
|
end
|
91
89
|
end
|
@@ -12,7 +12,7 @@ module Chronicle
|
|
12
12
|
attr_accessor :job_log
|
13
13
|
|
14
14
|
# For a given `job_id`, return the last successful log
|
15
|
-
def self.load_latest(
|
15
|
+
def self.load_latest(_job_id)
|
16
16
|
with_db_connection do |db|
|
17
17
|
attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
|
18
18
|
JobLog.build_from_serialized(attrs) if attrs
|
@@ -28,11 +28,11 @@ module Chronicle
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def self.db_exists?
|
31
|
-
File.
|
31
|
+
File.exist?(db_filename)
|
32
32
|
end
|
33
33
|
|
34
34
|
def self.schema_exists?(db)
|
35
|
-
|
35
|
+
db.tables.include? :job_logs
|
36
36
|
end
|
37
37
|
|
38
38
|
def self.db_filename
|
@@ -44,7 +44,7 @@ module Chronicle
|
|
44
44
|
FileUtils.mkdir_p(File.dirname(db_filename))
|
45
45
|
end
|
46
46
|
|
47
|
-
def self.initialize_schema
|
47
|
+
def self.initialize_schema(db)
|
48
48
|
db.create_table :job_logs do
|
49
49
|
primary_key :id
|
50
50
|
String :job_id, null: false
|
@@ -1,4 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'csv'
|
4
|
+
require 'chronicle/utils/hash_utils'
|
2
5
|
|
3
6
|
module Chronicle
|
4
7
|
module ETL
|
@@ -6,6 +9,7 @@ module Chronicle
|
|
6
9
|
include Chronicle::ETL::Loaders::Helpers::StdoutHelper
|
7
10
|
|
8
11
|
register_connector do |r|
|
12
|
+
r.identifier = :csv
|
9
13
|
r.description = 'CSV'
|
10
14
|
end
|
11
15
|
|
@@ -18,13 +22,14 @@ module Chronicle
|
|
18
22
|
end
|
19
23
|
|
20
24
|
def load(record)
|
21
|
-
records << record
|
25
|
+
records << record
|
22
26
|
end
|
23
27
|
|
24
28
|
def finish
|
25
29
|
return unless records.any?
|
26
30
|
|
27
|
-
headers =
|
31
|
+
# headers = filtered_headers(records)
|
32
|
+
headers = gather_headers(records)
|
28
33
|
|
29
34
|
csv_options = {}
|
30
35
|
if @config.headers
|
@@ -34,8 +39,7 @@ module Chronicle
|
|
34
39
|
|
35
40
|
csv_output = CSV.generate(**csv_options) do |csv|
|
36
41
|
records.each do |record|
|
37
|
-
csv << record
|
38
|
-
.transform_keys(&:to_sym)
|
42
|
+
csv << Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
|
39
43
|
.values_at(*headers)
|
40
44
|
.map { |value| force_utf8(value) }
|
41
45
|
end
|
@@ -48,6 +52,15 @@ module Chronicle
|
|
48
52
|
File.write(@config.output, csv_output)
|
49
53
|
end
|
50
54
|
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def gather_headers(records)
|
59
|
+
records_flattened = records.map do |record|
|
60
|
+
Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
|
61
|
+
end
|
62
|
+
records_flattened.flat_map(&:keys).uniq
|
63
|
+
end
|
51
64
|
end
|
52
65
|
end
|
53
66
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'tempfile'
|
2
4
|
|
3
5
|
module Chronicle
|
@@ -5,6 +7,8 @@ module Chronicle
|
|
5
7
|
module Loaders
|
6
8
|
module Helpers
|
7
9
|
module StdoutHelper
|
10
|
+
# TODO: have option to immediately output to stdout
|
11
|
+
|
8
12
|
# TODO: let users use "stdout" as an option for the `output` setting
|
9
13
|
# Assume we're using stdout if no output is specified
|
10
14
|
def output_to_stdout?
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'tempfile'
|
2
4
|
|
3
5
|
module Chronicle
|
@@ -6,10 +8,10 @@ module Chronicle
|
|
6
8
|
include Chronicle::ETL::Loaders::Helpers::StdoutHelper
|
7
9
|
|
8
10
|
register_connector do |r|
|
11
|
+
r.identifier = :json
|
9
12
|
r.description = 'json'
|
10
13
|
end
|
11
14
|
|
12
|
-
setting :serializer
|
13
15
|
setting :output
|
14
16
|
|
15
17
|
# If true, one JSON record per line. If false, output a single json
|
@@ -26,23 +28,24 @@ module Chronicle
|
|
26
28
|
if output_to_stdout?
|
27
29
|
create_stdout_temp_file
|
28
30
|
else
|
29
|
-
File.open(@config.output,
|
31
|
+
File.open(@config.output, 'w+')
|
30
32
|
end
|
31
33
|
|
32
34
|
@output_file.puts("[\n") unless @config.line_separated
|
33
35
|
end
|
34
36
|
|
35
37
|
def load(record)
|
36
|
-
serialized =
|
38
|
+
serialized = record.to_h
|
37
39
|
|
38
40
|
# When dealing with raw data, we can get improperly encoded strings
|
39
41
|
# (eg from sqlite database columns). We force conversion to UTF-8
|
40
42
|
# before converting into JSON
|
41
|
-
encoded = serialized.transform_values do |value|
|
42
|
-
|
43
|
+
# encoded = serialized.transform_values do |value|
|
44
|
+
# next value unless value.is_a?(String)
|
43
45
|
|
44
|
-
|
45
|
-
end
|
46
|
+
# force_utf8(value)
|
47
|
+
# end
|
48
|
+
encoded = deeply_force_utf8(serialized)
|
46
49
|
|
47
50
|
line = encoded.to_json
|
48
51
|
# For line-separated output, we just put json + newline
|
@@ -57,6 +60,8 @@ module Chronicle
|
|
57
60
|
@output_file.write(line)
|
58
61
|
|
59
62
|
@first_line = false
|
63
|
+
# rescue StandardError => e
|
64
|
+
# binding.pry
|
60
65
|
end
|
61
66
|
|
62
67
|
def finish
|
@@ -70,9 +75,24 @@ module Chronicle
|
|
70
75
|
|
71
76
|
private
|
72
77
|
|
73
|
-
# TODO:
|
74
|
-
def
|
75
|
-
|
78
|
+
# TODO: Move this to a helper module
|
79
|
+
def deeply_force_utf8(hash)
|
80
|
+
# FIXME: probably shouldn't happen but it does
|
81
|
+
return hash.map { |x| force_utf8(x) } if hash.is_a?(Array)
|
82
|
+
return force_utf8(hash) unless hash.is_a?(Hash)
|
83
|
+
|
84
|
+
hash.transform_values do |value|
|
85
|
+
case value
|
86
|
+
when String
|
87
|
+
force_utf8(value)
|
88
|
+
when Hash
|
89
|
+
deeply_force_utf8(value)
|
90
|
+
when Array
|
91
|
+
value.map { |v| deeply_force_utf8(v) }
|
92
|
+
else
|
93
|
+
value
|
94
|
+
end
|
95
|
+
end
|
76
96
|
end
|
77
97
|
end
|
78
98
|
end
|
@@ -32,23 +32,6 @@ module Chronicle
|
|
32
32
|
|
33
33
|
# Called once there are no more records to process
|
34
34
|
def finish; end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
def build_headers(records)
|
39
|
-
headers =
|
40
|
-
if @config.fields && @config.fields.any?
|
41
|
-
Set[*@config.fields]
|
42
|
-
else
|
43
|
-
# use all the keys of the flattened record hash
|
44
|
-
Set[*records.map(&:keys).flatten.map(&:to_s).uniq]
|
45
|
-
end
|
46
|
-
|
47
|
-
headers = headers.delete_if { |header| header.end_with?(*@config.fields_exclude) }
|
48
|
-
headers = headers.first(@config.fields_limit) if @config.fields_limit
|
49
|
-
|
50
|
-
headers.to_a.map(&:to_sym)
|
51
|
-
end
|
52
35
|
end
|
53
36
|
end
|
54
37
|
end
|
@@ -1,11 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'net/http'
|
2
4
|
require 'uri'
|
3
5
|
require 'json'
|
6
|
+
require 'chronicle/serialization'
|
4
7
|
|
5
8
|
module Chronicle
|
6
9
|
module ETL
|
7
10
|
class RestLoader < Chronicle::ETL::Loader
|
8
11
|
register_connector do |r|
|
12
|
+
r.identifier = :rest
|
9
13
|
r.description = 'a REST endpoint'
|
10
14
|
end
|
11
15
|
|
@@ -13,16 +17,12 @@ module Chronicle
|
|
13
17
|
setting :endpoint, required: true
|
14
18
|
setting :access_token
|
15
19
|
|
16
|
-
def load(
|
17
|
-
payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
|
18
|
-
# have the outer data key that json-api expects
|
19
|
-
payload = { data: payload } unless payload[:data]
|
20
|
-
|
20
|
+
def load(payload)
|
21
21
|
uri = URI.parse("#{@config.hostname}#{@config.endpoint}")
|
22
22
|
|
23
23
|
header = {
|
24
|
-
|
25
|
-
|
24
|
+
'Authorization' => "Bearer #{@config.access_token}",
|
25
|
+
'Content-Type': 'application/json'
|
26
26
|
}
|
27
27
|
use_ssl = uri.scheme == 'https'
|
28
28
|
|
@@ -1,49 +1,74 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'tty/table'
|
4
|
+
require 'chronicle/utils/hash_utils'
|
2
5
|
require 'active_support/core_ext/string/filters'
|
3
6
|
require 'active_support/core_ext/hash/reverse_merge'
|
4
7
|
|
5
8
|
module Chronicle
|
6
9
|
module ETL
|
7
10
|
class TableLoader < Chronicle::ETL::Loader
|
11
|
+
|
8
12
|
register_connector do |r|
|
13
|
+
r.identifier = :table
|
9
14
|
r.description = 'an ASCII table'
|
10
15
|
end
|
11
16
|
|
12
17
|
setting :truncate_values_at, default: 40
|
13
18
|
setting :table_renderer, default: :basic
|
14
|
-
setting :fields_exclude, default: ['
|
19
|
+
setting :fields_exclude, default: ['type']
|
15
20
|
setting :header_row, default: true
|
16
21
|
|
17
22
|
def load(record)
|
18
|
-
records << record
|
23
|
+
records << record
|
19
24
|
end
|
20
25
|
|
21
26
|
def finish
|
22
27
|
return if records.empty?
|
23
28
|
|
24
|
-
headers =
|
29
|
+
headers = gather_headers(records)
|
25
30
|
rows = build_rows(records, headers)
|
26
31
|
|
32
|
+
render_table(headers, rows)
|
33
|
+
end
|
34
|
+
|
35
|
+
def records
|
36
|
+
@records ||= []
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def render_table(headers, rows)
|
27
42
|
@table = TTY::Table.new(header: (headers if @config.header_row), rows: rows)
|
28
43
|
puts @table.render(
|
29
44
|
@config.table_renderer.to_sym,
|
30
45
|
padding: [0, 2, 0, 0]
|
31
46
|
)
|
47
|
+
rescue TTY::Table::ResizeError
|
48
|
+
# The library throws this error before trying to render the table
|
49
|
+
# vertically. These options seem to work.
|
50
|
+
puts @table.render(
|
51
|
+
@config.table_renderer.to_sym,
|
52
|
+
padding: [0, 2, 0, 0],
|
53
|
+
width: 10_000,
|
54
|
+
resize: false
|
55
|
+
)
|
32
56
|
end
|
33
57
|
|
34
|
-
def records
|
35
|
-
|
58
|
+
def gather_headers(records)
|
59
|
+
records_flattened = records.map do |record|
|
60
|
+
Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
|
61
|
+
end
|
62
|
+
records_flattened.flat_map(&:keys).uniq
|
36
63
|
end
|
37
64
|
|
38
|
-
private
|
39
|
-
|
40
65
|
def build_rows(records, headers)
|
41
66
|
records.map do |record|
|
42
|
-
values =
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
67
|
+
values = Chronicle::Utils::HashUtils.flatten_hash(record.to_h)
|
68
|
+
.values_at(*headers)
|
69
|
+
.map { |value| force_utf8(value.to_s) }
|
70
|
+
|
71
|
+
values = values.map { |value| value.truncate(@config.truncate_values_at) } if @config.truncate_values_at
|
47
72
|
|
48
73
|
values
|
49
74
|
end
|
data/lib/chronicle/etl/logger.rb
CHANGED
@@ -14,13 +14,13 @@ module Chronicle
|
|
14
14
|
|
15
15
|
@log_level = INFO
|
16
16
|
|
17
|
-
def output
|
17
|
+
def output(message, level)
|
18
18
|
return unless level >= @log_level
|
19
19
|
|
20
20
|
if @ui_element
|
21
21
|
@ui_element.log(message)
|
22
22
|
else
|
23
|
-
|
23
|
+
warn(message)
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
@@ -49,14 +49,14 @@ module Chronicle
|
|
49
49
|
def authorize!
|
50
50
|
associate_oauth_credentials
|
51
51
|
@server = load_server
|
52
|
-
spinner = TTY::Spinner.new(
|
52
|
+
spinner = TTY::Spinner.new(':spinner :title', format: :dots_2)
|
53
53
|
spinner.auto_spin
|
54
|
-
spinner.update(title: "Starting temporary authorization server on port #{@port}"
|
54
|
+
spinner.update(title: "Starting temporary authorization server on port #{@port}"'')
|
55
55
|
|
56
56
|
server_thread = start_authorization_server(port: @port)
|
57
57
|
start_oauth_flow
|
58
58
|
|
59
|
-
spinner.update(title:
|
59
|
+
spinner.update(title: 'Waiting for authorization to complete in your browser')
|
60
60
|
sleep 0.1 while authorization_pending?(server_thread)
|
61
61
|
|
62
62
|
@server.quit!
|
@@ -85,7 +85,7 @@ module Chronicle
|
|
85
85
|
def load_server
|
86
86
|
# Load at runtime so that we can set omniauth strategies based on
|
87
87
|
# which chronicle plugin has been loaded.
|
88
|
-
require_relative '
|
88
|
+
require_relative 'authorization_server'
|
89
89
|
Chronicle::ETL::AuthorizationServer
|
90
90
|
end
|
91
91
|
|
@@ -97,7 +97,7 @@ module Chronicle
|
|
97
97
|
|
98
98
|
Thread.new do
|
99
99
|
@server.run!({ port: @port }) do |s|
|
100
|
-
s.silent = true if s.
|
100
|
+
s.silent = true if defined?(::Thin::Server) && s.instance_of?(::Thin::Server)
|
101
101
|
end
|
102
102
|
end
|
103
103
|
end
|
@@ -117,7 +117,7 @@ module Chronicle
|
|
117
117
|
AccessLog: [],
|
118
118
|
# TODO: make this windows friendly
|
119
119
|
# https://github.com/winton/stasis/commit/77da36f43285fda129300e382f18dfaff48571b0
|
120
|
-
Logger: WEBrick::Log
|
120
|
+
Logger: WEBrick::Log.new('/dev/null')
|
121
121
|
}
|
122
122
|
)
|
123
123
|
rescue LoadError
|
@@ -127,8 +127,8 @@ module Chronicle
|
|
127
127
|
def extract_secrets(authorization:, pluck_values:)
|
128
128
|
return authorization unless pluck_values&.any?
|
129
129
|
|
130
|
-
pluck_values.
|
131
|
-
|
130
|
+
pluck_values.transform_values do |identifiers|
|
131
|
+
authorization.dig(*identifiers)
|
132
132
|
end
|
133
133
|
end
|
134
134
|
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# TODO: move this into chronicle-core after figuring out what to do about data vs properties
|
4
|
+
module Chronicle
|
5
|
+
module ETL
|
6
|
+
class Record
|
7
|
+
attr_accessor :data, :extraction
|
8
|
+
|
9
|
+
def initialize(data: {}, extraction: nil)
|
10
|
+
@data = data
|
11
|
+
@extraction = extraction
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -1,15 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Chronicle
|
2
4
|
module ETL
|
3
5
|
module Registry
|
4
|
-
# Records details about a connector such as its provider and a description
|
6
|
+
# Records details about a connector such as its source provider and a description
|
5
7
|
class ConnectorRegistration
|
6
|
-
|
7
|
-
attr_accessor :identifier, :provider, :klass, :description
|
8
|
+
attr_accessor :klass, :identifier, :source, :strategy, :type, :description, :from_schema, :to_schema
|
8
9
|
|
10
|
+
# Create a new connector registration
|
9
11
|
def initialize(klass)
|
10
12
|
@klass = klass
|
11
13
|
end
|
12
14
|
|
15
|
+
# The ETL phase of this connector
|
13
16
|
def phase
|
14
17
|
if klass.ancestors.include? Chronicle::ETL::Extractor
|
15
18
|
:extractor
|
@@ -24,6 +27,7 @@ module Chronicle
|
|
24
27
|
"#{phase}-#{identifier}"
|
25
28
|
end
|
26
29
|
|
30
|
+
# Whether this connector is built-in to Chronicle
|
27
31
|
def built_in?
|
28
32
|
@klass.to_s.include? 'Chronicle::ETL'
|
29
33
|
end
|
@@ -32,32 +36,20 @@ module Chronicle
|
|
32
36
|
@klass.to_s
|
33
37
|
end
|
34
38
|
|
35
|
-
def identifier
|
36
|
-
@identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
|
37
|
-
end
|
38
|
-
|
39
|
-
def description
|
40
|
-
@description || @klass.to_s.split('::').last
|
41
|
-
end
|
42
|
-
|
43
|
-
def provider
|
44
|
-
@provider || (built_in? ? 'chronicle' : '')
|
45
|
-
end
|
46
|
-
|
47
39
|
# TODO: allow overriding here. Maybe through self-registration process
|
48
40
|
def plugin
|
49
|
-
@
|
41
|
+
@source
|
50
42
|
end
|
51
43
|
|
52
44
|
def descriptive_phrase
|
53
45
|
prefix = case phase
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
46
|
+
when :extractor
|
47
|
+
'Extracts from'
|
48
|
+
when :transformer
|
49
|
+
'Transforms'
|
50
|
+
when :loader
|
51
|
+
'Loads to'
|
52
|
+
end
|
61
53
|
|
62
54
|
"#{prefix} #{description}"
|
63
55
|
end
|