chronicle-etl 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +15 -25
- data/.rubocop.yml +2 -44
- data/Gemfile +2 -2
- data/Guardfile +3 -3
- data/README.md +98 -73
- data/Rakefile +2 -2
- data/bin/console +4 -5
- data/chronicle-etl.gemspec +50 -45
- data/exe/chronicle-etl +1 -1
- data/lib/chronicle/etl/authorizer.rb +3 -4
- data/lib/chronicle/etl/cli/authorizations.rb +10 -8
- data/lib/chronicle/etl/cli/connectors.rb +9 -9
- data/lib/chronicle/etl/cli/jobs.rb +130 -53
- data/lib/chronicle/etl/cli/main.rb +29 -29
- data/lib/chronicle/etl/cli/plugins.rb +29 -26
- data/lib/chronicle/etl/cli/secrets.rb +14 -12
- data/lib/chronicle/etl/cli/subcommand_base.rb +5 -3
- data/lib/chronicle/etl/config.rb +20 -7
- data/lib/chronicle/etl/configurable.rb +24 -9
- data/lib/chronicle/etl/exceptions.rb +3 -3
- data/lib/chronicle/etl/extraction.rb +12 -2
- data/lib/chronicle/etl/extractors/csv_extractor.rb +9 -0
- data/lib/chronicle/etl/extractors/extractor.rb +15 -2
- data/lib/chronicle/etl/extractors/file_extractor.rb +5 -3
- data/lib/chronicle/etl/extractors/helpers/input_reader.rb +2 -2
- data/lib/chronicle/etl/extractors/json_extractor.rb +14 -4
- data/lib/chronicle/etl/extractors/stdin_extractor.rb +3 -0
- data/lib/chronicle/etl/job.rb +35 -17
- data/lib/chronicle/etl/job_definition.rb +39 -27
- data/lib/chronicle/etl/job_log.rb +14 -16
- data/lib/chronicle/etl/job_logger.rb +4 -4
- data/lib/chronicle/etl/loaders/csv_loader.rb +17 -4
- data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb +4 -0
- data/lib/chronicle/etl/loaders/json_loader.rb +30 -10
- data/lib/chronicle/etl/loaders/loader.rb +0 -17
- data/lib/chronicle/etl/loaders/rest_loader.rb +7 -7
- data/lib/chronicle/etl/loaders/table_loader.rb +37 -12
- data/lib/chronicle/etl/logger.rb +3 -3
- data/lib/chronicle/etl/oauth_authorizer.rb +8 -10
- data/lib/chronicle/etl/record.rb +15 -0
- data/lib/chronicle/etl/registry/connector_registration.rb +15 -23
- data/lib/chronicle/etl/registry/connectors.rb +117 -0
- data/lib/chronicle/etl/registry/plugin_registration.rb +19 -0
- data/lib/chronicle/etl/registry/plugins.rb +171 -0
- data/lib/chronicle/etl/registry/registry.rb +3 -52
- data/lib/chronicle/etl/registry/self_registering.rb +1 -1
- data/lib/chronicle/etl/runner.rb +158 -128
- data/lib/chronicle/etl/secrets.rb +5 -5
- data/lib/chronicle/etl/transformers/buffer_transformer.rb +29 -0
- data/lib/chronicle/etl/transformers/chronicle_transformer.rb +32 -0
- data/lib/chronicle/etl/transformers/chronobase_transformer.rb +100 -0
- data/lib/chronicle/etl/transformers/fields_limit_transformer.rb +23 -0
- data/lib/chronicle/etl/transformers/filter_fields_transformer.rb +60 -0
- data/lib/chronicle/etl/transformers/filter_transformer.rb +30 -0
- data/lib/chronicle/etl/transformers/format_transformer.rb +32 -0
- data/lib/chronicle/etl/transformers/merge_meta_transformer.rb +19 -0
- data/lib/chronicle/etl/transformers/multiply_transformer.rb +21 -0
- data/lib/chronicle/etl/transformers/null_transformer.rb +5 -7
- data/lib/chronicle/etl/transformers/sampler_transformer.rb +21 -0
- data/lib/chronicle/etl/transformers/sort_transformer.rb +31 -0
- data/lib/chronicle/etl/transformers/transformer.rb +63 -41
- data/lib/chronicle/etl/utils/binary_attachments.rb +1 -1
- data/lib/chronicle/etl/utils/progress_bar.rb +2 -3
- data/lib/chronicle/etl/version.rb +1 -1
- data/lib/chronicle/etl.rb +6 -8
- metadata +91 -45
- data/lib/chronicle/etl/models/activity.rb +0 -15
- data/lib/chronicle/etl/models/attachment.rb +0 -14
- data/lib/chronicle/etl/models/base.rb +0 -122
- data/lib/chronicle/etl/models/entity.rb +0 -29
- data/lib/chronicle/etl/models/raw.rb +0 -26
- data/lib/chronicle/etl/registry/plugin_registry.rb +0 -95
- data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +0 -31
- data/lib/chronicle/etl/serializers/raw_serializer.rb +0 -10
- data/lib/chronicle/etl/serializers/serializer.rb +0 -28
- data/lib/chronicle/etl/transformers/image_file_transformer.rb +0 -247
- data/lib/chronicle/etl/utils/hash_utilities.rb +0 -19
- data/lib/chronicle/etl/utils/text_recognition.rb +0 -15
@@ -1,18 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Chronicle
|
2
4
|
module ETL
|
3
5
|
class NullTransformer < Chronicle::ETL::Transformer
|
4
6
|
register_connector do |r|
|
5
|
-
r.identifier =
|
7
|
+
r.identifier = :null
|
6
8
|
r.description = 'in no way'
|
7
9
|
end
|
8
10
|
|
9
|
-
def transform
|
10
|
-
|
11
|
+
def transform(record)
|
12
|
+
yield record.data
|
11
13
|
end
|
12
|
-
|
13
|
-
def timestamp; end
|
14
|
-
|
15
|
-
def id; end
|
16
14
|
end
|
17
15
|
end
|
18
16
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Chronicle
|
4
|
+
module ETL
|
5
|
+
class SamplerTransformer < Chronicle::ETL::Transformer
|
6
|
+
register_connector do |r|
|
7
|
+
r.identifier = :sampler
|
8
|
+
r.description = 'by taking a sample'
|
9
|
+
end
|
10
|
+
|
11
|
+
setting :percent, default: 10, type: :numeric
|
12
|
+
|
13
|
+
# return the result, `percent` percentage of the time. otherwise nil
|
14
|
+
def transform(record)
|
15
|
+
return unless rand(100) < @config.percent
|
16
|
+
|
17
|
+
record.data
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Chronicle
|
4
|
+
module ETL
|
5
|
+
class SortTransformer < Chronicle::ETL::Transformer
|
6
|
+
register_connector do |r|
|
7
|
+
r.identifier = :sort
|
8
|
+
r.description = 'sorts records by a given field'
|
9
|
+
end
|
10
|
+
|
11
|
+
setting :key, required: true, default: 'id'
|
12
|
+
setting :direction, required: false, default: 'desc'
|
13
|
+
|
14
|
+
def transform(record)
|
15
|
+
stash_record(record)
|
16
|
+
end
|
17
|
+
|
18
|
+
def finish
|
19
|
+
return unless @stashed_records&.any?
|
20
|
+
|
21
|
+
sorted = @stashed_records.sort_by do |record|
|
22
|
+
value = record.data[@config.key]
|
23
|
+
value.nil? ? [1] : [0, value]
|
24
|
+
end
|
25
|
+
|
26
|
+
sorted.reverse! if @config.direction == 'desc'
|
27
|
+
sorted
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Chronicle
|
2
4
|
module ETL
|
3
5
|
# Abstract class representing an Transformer for an ETL job
|
@@ -5,65 +7,85 @@ module Chronicle
|
|
5
7
|
extend Chronicle::ETL::Registry::SelfRegistering
|
6
8
|
include Chronicle::ETL::Configurable
|
7
9
|
|
10
|
+
attr_reader :stashed_records
|
11
|
+
|
8
12
|
# Construct a new instance of this transformer. Options are passed in from a Runner
|
9
13
|
# == Parameters:
|
10
14
|
# options::
|
11
15
|
# Options for configuring this Transformer
|
12
|
-
def initialize(
|
13
|
-
unless extraction.is_a?(Chronicle::ETL::Extraction)
|
14
|
-
raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
|
15
|
-
end
|
16
|
-
|
17
|
-
@extraction = extraction
|
16
|
+
def initialize(options = {})
|
18
17
|
apply_options(options)
|
19
18
|
end
|
20
19
|
|
21
|
-
#
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
20
|
+
# Called once for each extracted record. Can return 0 or more transformed records.
|
21
|
+
def call(record, &block)
|
22
|
+
raise ArgumentError, 'Input must be a Chronicle::ETL::Record' unless record.is_a?(Record)
|
23
|
+
|
24
|
+
yielded = false
|
25
|
+
|
26
|
+
transformed_data = transform(record) do |data|
|
27
|
+
new_record = update_data(record, data)
|
28
|
+
block.call(new_record)
|
29
|
+
|
30
|
+
yielded = true
|
31
|
+
end
|
32
|
+
|
33
|
+
return if yielded
|
34
|
+
|
35
|
+
# Handle transformers that don't yield anything and return
|
36
|
+
# transformed data directly. Skip nil values.
|
37
|
+
[transformed_data].flatten.compact.each do |data|
|
38
|
+
new_record = update_data(record, data)
|
39
|
+
block.call(new_record)
|
40
|
+
end
|
32
41
|
end
|
33
42
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
43
|
+
def call_finish(&block)
|
44
|
+
remaining_records = finish
|
45
|
+
return if remaining_records.nil?
|
46
|
+
|
47
|
+
remaining_records.each do |record|
|
48
|
+
block.call(record)
|
49
|
+
end
|
39
50
|
end
|
40
51
|
|
41
|
-
|
42
|
-
|
43
|
-
def friendly_identifier
|
44
|
-
id
|
52
|
+
def transform(_record)
|
53
|
+
raise NotImplementedError, 'You must implement the transform method'
|
45
54
|
end
|
46
55
|
|
47
|
-
|
48
|
-
|
49
|
-
unknown = "???"
|
50
|
-
timestamp&.iso8601 || unknown
|
51
|
-
rescue TransformationError, NotImplementedError
|
52
|
-
unknown
|
53
|
-
end
|
56
|
+
# Called once after runner has processed all records
|
57
|
+
def finish; end
|
54
58
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
59
|
+
protected
|
60
|
+
|
61
|
+
def stash_record(record)
|
62
|
+
@stashed_records ||= []
|
63
|
+
@stashed_records << record
|
64
|
+
nil
|
65
|
+
end
|
66
|
+
|
67
|
+
def flush_stashed_records
|
68
|
+
@stashed_records.tap(&:clear)
|
69
|
+
end
|
61
70
|
|
62
|
-
|
71
|
+
def update_data(record, new_data)
|
72
|
+
new_record = record.clone
|
73
|
+
new_record.data = new_data
|
74
|
+
new_record
|
63
75
|
end
|
64
76
|
end
|
65
77
|
end
|
66
78
|
end
|
67
79
|
|
68
80
|
require_relative 'null_transformer'
|
69
|
-
require_relative '
|
81
|
+
require_relative 'sampler_transformer'
|
82
|
+
require_relative 'buffer_transformer'
|
83
|
+
require_relative 'multiply_transformer'
|
84
|
+
require_relative 'sort_transformer'
|
85
|
+
require_relative 'chronicle_transformer'
|
86
|
+
require_relative 'format_transformer'
|
87
|
+
require_relative 'filter_fields_transformer'
|
88
|
+
require_relative 'fields_limit_transformer'
|
89
|
+
require_relative 'merge_meta_transformer'
|
90
|
+
require_relative 'filter_transformer'
|
91
|
+
require_relative 'chronobase_transformer'
|
@@ -7,7 +7,7 @@ module Chronicle
|
|
7
7
|
# Utility methods for dealing with binary files
|
8
8
|
module BinaryAttachments
|
9
9
|
def self.filename_to_base64(filename:, mimetype: nil)
|
10
|
-
mimetype
|
10
|
+
mimetype ||= guess_mimetype(filename: filename)
|
11
11
|
|
12
12
|
"data:#{mimetype};base64,#{Base64.strict_encode64(File.read(filename))}"
|
13
13
|
end
|
@@ -4,7 +4,6 @@ require 'colorize'
|
|
4
4
|
module Chronicle
|
5
5
|
module ETL
|
6
6
|
module Utils
|
7
|
-
|
8
7
|
class ProgressBar
|
9
8
|
FORMAT_WITH_TOTAL = [
|
10
9
|
':bar ',
|
@@ -37,7 +36,7 @@ module Chronicle
|
|
37
36
|
'/s) '.light_black
|
38
37
|
].join.freeze
|
39
38
|
|
40
|
-
def initialize(title: 'Loading'
|
39
|
+
def initialize(total:, title: 'Loading')
|
41
40
|
opts = {
|
42
41
|
clear: true,
|
43
42
|
complete: '▓'.light_blue,
|
@@ -64,7 +63,7 @@ module Chronicle
|
|
64
63
|
end
|
65
64
|
|
66
65
|
def log(message)
|
67
|
-
message.split("\n").each do |
|
66
|
+
message.split("\n").each do |_line|
|
68
67
|
@pbar.log message
|
69
68
|
end
|
70
69
|
end
|
data/lib/chronicle/etl.rb
CHANGED
@@ -1,25 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'chronicle/schema'
|
4
|
+
require 'chronicle/models/base'
|
5
|
+
|
1
6
|
require_relative 'etl/registry/registry'
|
2
7
|
require_relative 'etl/authorizer'
|
3
8
|
require_relative 'etl/config'
|
4
9
|
require_relative 'etl/configurable'
|
5
10
|
require_relative 'etl/exceptions'
|
6
11
|
require_relative 'etl/extraction'
|
12
|
+
require_relative 'etl/record'
|
7
13
|
require_relative 'etl/job_definition'
|
8
14
|
require_relative 'etl/job_log'
|
9
15
|
require_relative 'etl/job_logger'
|
10
16
|
require_relative 'etl/job'
|
11
17
|
require_relative 'etl/logger'
|
12
|
-
require_relative 'etl/models/activity'
|
13
|
-
require_relative 'etl/models/attachment'
|
14
|
-
require_relative 'etl/models/base'
|
15
|
-
require_relative 'etl/models/raw'
|
16
|
-
require_relative 'etl/models/entity'
|
17
18
|
require_relative 'etl/runner'
|
18
19
|
require_relative 'etl/secrets'
|
19
|
-
require_relative 'etl/serializers/serializer'
|
20
20
|
require_relative 'etl/utils/binary_attachments'
|
21
|
-
require_relative 'etl/utils/hash_utilities'
|
22
|
-
require_relative 'etl/utils/text_recognition'
|
23
21
|
require_relative 'etl/utils/progress_bar'
|
24
22
|
require_relative 'etl/version'
|
25
23
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chronicle-etl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Louis
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-05-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -39,75 +39,75 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.10.6
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: chronicle-core
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: '0.3'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: '0.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: colorize
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 0.8.1
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 0.8.1
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: gems
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 1
|
75
|
+
version: '1'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: 1
|
82
|
+
version: '1'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: launchy
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
98
|
+
name: marcel
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
103
|
+
version: 1.0.2
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
110
|
+
version: 1.0.2
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: omniauth
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -240,14 +240,14 @@ dependencies:
|
|
240
240
|
requirements:
|
241
241
|
- - "~>"
|
242
242
|
- !ruby/object:Gem::Version
|
243
|
-
version: '0.
|
243
|
+
version: '0.12'
|
244
244
|
type: :runtime
|
245
245
|
prerelease: false
|
246
246
|
version_requirements: !ruby/object:Gem::Requirement
|
247
247
|
requirements:
|
248
248
|
- - "~>"
|
249
249
|
- !ruby/object:Gem::Version
|
250
|
-
version: '0.
|
250
|
+
version: '0.12'
|
251
251
|
- !ruby/object:Gem::Dependency
|
252
252
|
name: xdg
|
253
253
|
requirement: !ruby/object:Gem::Requirement
|
@@ -280,16 +280,16 @@ dependencies:
|
|
280
280
|
name: fakefs
|
281
281
|
requirement: !ruby/object:Gem::Requirement
|
282
282
|
requirements:
|
283
|
-
- - "
|
283
|
+
- - "~>"
|
284
284
|
- !ruby/object:Gem::Version
|
285
|
-
version: '
|
285
|
+
version: '1.4'
|
286
286
|
type: :development
|
287
287
|
prerelease: false
|
288
288
|
version_requirements: !ruby/object:Gem::Requirement
|
289
289
|
requirements:
|
290
|
-
- - "
|
290
|
+
- - "~>"
|
291
291
|
- !ruby/object:Gem::Version
|
292
|
-
version: '
|
292
|
+
version: '1.4'
|
293
293
|
- !ruby/object:Gem::Dependency
|
294
294
|
name: guard-rspec
|
295
295
|
requirement: !ruby/object:Gem::Requirement
|
@@ -352,14 +352,14 @@ dependencies:
|
|
352
352
|
requirements:
|
353
353
|
- - "~>"
|
354
354
|
- !ruby/object:Gem::Version
|
355
|
-
version: 1.
|
355
|
+
version: '1.57'
|
356
356
|
type: :development
|
357
357
|
prerelease: false
|
358
358
|
version_requirements: !ruby/object:Gem::Requirement
|
359
359
|
requirements:
|
360
360
|
- - "~>"
|
361
361
|
- !ruby/object:Gem::Version
|
362
|
-
version: 1.
|
362
|
+
version: '1.57'
|
363
363
|
- !ruby/object:Gem::Dependency
|
364
364
|
name: simplecov
|
365
365
|
requirement: !ruby/object:Gem::Requirement
|
@@ -374,6 +374,48 @@ dependencies:
|
|
374
374
|
- - "~>"
|
375
375
|
- !ruby/object:Gem::Version
|
376
376
|
version: '0.21'
|
377
|
+
- !ruby/object:Gem::Dependency
|
378
|
+
name: vcr
|
379
|
+
requirement: !ruby/object:Gem::Requirement
|
380
|
+
requirements:
|
381
|
+
- - "~>"
|
382
|
+
- !ruby/object:Gem::Version
|
383
|
+
version: '6.1'
|
384
|
+
type: :development
|
385
|
+
prerelease: false
|
386
|
+
version_requirements: !ruby/object:Gem::Requirement
|
387
|
+
requirements:
|
388
|
+
- - "~>"
|
389
|
+
- !ruby/object:Gem::Version
|
390
|
+
version: '6.1'
|
391
|
+
- !ruby/object:Gem::Dependency
|
392
|
+
name: webmock
|
393
|
+
requirement: !ruby/object:Gem::Requirement
|
394
|
+
requirements:
|
395
|
+
- - "~>"
|
396
|
+
- !ruby/object:Gem::Version
|
397
|
+
version: '3'
|
398
|
+
type: :development
|
399
|
+
prerelease: false
|
400
|
+
version_requirements: !ruby/object:Gem::Requirement
|
401
|
+
requirements:
|
402
|
+
- - "~>"
|
403
|
+
- !ruby/object:Gem::Version
|
404
|
+
version: '3'
|
405
|
+
- !ruby/object:Gem::Dependency
|
406
|
+
name: webrick
|
407
|
+
requirement: !ruby/object:Gem::Requirement
|
408
|
+
requirements:
|
409
|
+
- - "~>"
|
410
|
+
- !ruby/object:Gem::Version
|
411
|
+
version: '1.7'
|
412
|
+
type: :development
|
413
|
+
prerelease: false
|
414
|
+
version_requirements: !ruby/object:Gem::Requirement
|
415
|
+
requirements:
|
416
|
+
- - "~>"
|
417
|
+
- !ruby/object:Gem::Version
|
418
|
+
version: '1.7'
|
377
419
|
- !ruby/object:Gem::Dependency
|
378
420
|
name: yard
|
379
421
|
requirement: !ruby/object:Gem::Requirement
|
@@ -447,28 +489,31 @@ files:
|
|
447
489
|
- lib/chronicle/etl/loaders/rest_loader.rb
|
448
490
|
- lib/chronicle/etl/loaders/table_loader.rb
|
449
491
|
- lib/chronicle/etl/logger.rb
|
450
|
-
- lib/chronicle/etl/models/activity.rb
|
451
|
-
- lib/chronicle/etl/models/attachment.rb
|
452
|
-
- lib/chronicle/etl/models/base.rb
|
453
|
-
- lib/chronicle/etl/models/entity.rb
|
454
|
-
- lib/chronicle/etl/models/raw.rb
|
455
492
|
- lib/chronicle/etl/oauth_authorizer.rb
|
493
|
+
- lib/chronicle/etl/record.rb
|
456
494
|
- lib/chronicle/etl/registry/connector_registration.rb
|
457
|
-
- lib/chronicle/etl/registry/
|
495
|
+
- lib/chronicle/etl/registry/connectors.rb
|
496
|
+
- lib/chronicle/etl/registry/plugin_registration.rb
|
497
|
+
- lib/chronicle/etl/registry/plugins.rb
|
458
498
|
- lib/chronicle/etl/registry/registry.rb
|
459
499
|
- lib/chronicle/etl/registry/self_registering.rb
|
460
500
|
- lib/chronicle/etl/runner.rb
|
461
501
|
- lib/chronicle/etl/secrets.rb
|
462
|
-
- lib/chronicle/etl/
|
463
|
-
- lib/chronicle/etl/
|
464
|
-
- lib/chronicle/etl/
|
465
|
-
- lib/chronicle/etl/transformers/
|
502
|
+
- lib/chronicle/etl/transformers/buffer_transformer.rb
|
503
|
+
- lib/chronicle/etl/transformers/chronicle_transformer.rb
|
504
|
+
- lib/chronicle/etl/transformers/chronobase_transformer.rb
|
505
|
+
- lib/chronicle/etl/transformers/fields_limit_transformer.rb
|
506
|
+
- lib/chronicle/etl/transformers/filter_fields_transformer.rb
|
507
|
+
- lib/chronicle/etl/transformers/filter_transformer.rb
|
508
|
+
- lib/chronicle/etl/transformers/format_transformer.rb
|
509
|
+
- lib/chronicle/etl/transformers/merge_meta_transformer.rb
|
510
|
+
- lib/chronicle/etl/transformers/multiply_transformer.rb
|
466
511
|
- lib/chronicle/etl/transformers/null_transformer.rb
|
512
|
+
- lib/chronicle/etl/transformers/sampler_transformer.rb
|
513
|
+
- lib/chronicle/etl/transformers/sort_transformer.rb
|
467
514
|
- lib/chronicle/etl/transformers/transformer.rb
|
468
515
|
- lib/chronicle/etl/utils/binary_attachments.rb
|
469
|
-
- lib/chronicle/etl/utils/hash_utilities.rb
|
470
516
|
- lib/chronicle/etl/utils/progress_bar.rb
|
471
|
-
- lib/chronicle/etl/utils/text_recognition.rb
|
472
517
|
- lib/chronicle/etl/version.rb
|
473
518
|
homepage: https://github.com/chronicle-app
|
474
519
|
licenses:
|
@@ -478,6 +523,7 @@ metadata:
|
|
478
523
|
homepage_uri: https://github.com/chronicle-app
|
479
524
|
source_code_uri: https://github.com/chronicle-app/chronicle-etl
|
480
525
|
changelog_uri: https://github.com/chronicle-app/chronicle-etl/releases
|
526
|
+
rubygems_mfa_required: 'true'
|
481
527
|
post_install_message:
|
482
528
|
rdoc_options: []
|
483
529
|
require_paths:
|
@@ -486,14 +532,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
486
532
|
requirements:
|
487
533
|
- - ">="
|
488
534
|
- !ruby/object:Gem::Version
|
489
|
-
version: '
|
535
|
+
version: '3.1'
|
490
536
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
491
537
|
requirements:
|
492
538
|
- - ">="
|
493
539
|
- !ruby/object:Gem::Version
|
494
540
|
version: '0'
|
495
541
|
requirements: []
|
496
|
-
rubygems_version: 3.
|
542
|
+
rubygems_version: 3.4.10
|
497
543
|
signing_key:
|
498
544
|
specification_version: 4
|
499
545
|
summary: ETL tool for personal data
|
@@ -1,15 +0,0 @@
|
|
1
|
-
require 'chronicle/etl/models/base'
|
2
|
-
|
3
|
-
module Chronicle
|
4
|
-
module ETL
|
5
|
-
module Models
|
6
|
-
class Activity < Chronicle::ETL::Models::Base
|
7
|
-
TYPE = 'activities'.freeze
|
8
|
-
ATTRIBUTES = [:verb, :start_at, :end_at].freeze
|
9
|
-
ASSOCIATIONS = [:involved, :actor].freeze
|
10
|
-
|
11
|
-
attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
|
12
|
-
end
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
@@ -1,14 +0,0 @@
|
|
1
|
-
require 'chronicle/etl/models/base'
|
2
|
-
|
3
|
-
module Chronicle
|
4
|
-
module ETL
|
5
|
-
module Models
|
6
|
-
class Attachment < Chronicle::ETL::Models::Base
|
7
|
-
TYPE = 'attachments'.freeze
|
8
|
-
ATTRIBUTES = [:url_original, :data].freeze
|
9
|
-
|
10
|
-
attr_accessor(*ATTRIBUTES)
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
14
|
-
end
|