chronicle-etl 0.3.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +35 -0
  3. data/.rubocop.yml +31 -1
  4. data/Guardfile +7 -0
  5. data/README.md +157 -82
  6. data/Rakefile +4 -2
  7. data/chronicle-etl.gemspec +11 -3
  8. data/exe/chronicle-etl +1 -1
  9. data/lib/chronicle/etl/cli/connectors.rb +34 -5
  10. data/lib/chronicle/etl/cli/jobs.rb +90 -24
  11. data/lib/chronicle/etl/cli/main.rb +41 -19
  12. data/lib/chronicle/etl/cli/plugins.rb +62 -0
  13. data/lib/chronicle/etl/cli/subcommand_base.rb +2 -2
  14. data/lib/chronicle/etl/cli.rb +9 -0
  15. data/lib/chronicle/etl/config.rb +7 -4
  16. data/lib/chronicle/etl/configurable.rb +163 -0
  17. data/lib/chronicle/etl/exceptions.rb +29 -1
  18. data/lib/chronicle/etl/extractors/csv_extractor.rb +24 -23
  19. data/lib/chronicle/etl/extractors/extractor.rb +16 -15
  20. data/lib/chronicle/etl/extractors/file_extractor.rb +34 -11
  21. data/lib/chronicle/etl/extractors/helpers/input_reader.rb +76 -0
  22. data/lib/chronicle/etl/extractors/json_extractor.rb +19 -18
  23. data/lib/chronicle/etl/job.rb +8 -2
  24. data/lib/chronicle/etl/job_definition.rb +20 -5
  25. data/lib/chronicle/etl/loaders/csv_loader.rb +36 -9
  26. data/lib/chronicle/etl/loaders/helpers/encoding_helper.rb +18 -0
  27. data/lib/chronicle/etl/loaders/json_loader.rb +44 -0
  28. data/lib/chronicle/etl/loaders/loader.rb +28 -2
  29. data/lib/chronicle/etl/loaders/rest_loader.rb +5 -5
  30. data/lib/chronicle/etl/loaders/table_loader.rb +18 -37
  31. data/lib/chronicle/etl/logger.rb +6 -2
  32. data/lib/chronicle/etl/models/base.rb +3 -0
  33. data/lib/chronicle/etl/models/entity.rb +8 -2
  34. data/lib/chronicle/etl/models/raw.rb +26 -0
  35. data/lib/chronicle/etl/registry/connector_registration.rb +6 -0
  36. data/lib/chronicle/etl/registry/plugin_registry.rb +70 -0
  37. data/lib/chronicle/etl/registry/registry.rb +27 -14
  38. data/lib/chronicle/etl/runner.rb +35 -17
  39. data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +6 -0
  40. data/lib/chronicle/etl/serializers/raw_serializer.rb +10 -0
  41. data/lib/chronicle/etl/serializers/serializer.rb +2 -1
  42. data/lib/chronicle/etl/transformers/image_file_transformer.rb +22 -28
  43. data/lib/chronicle/etl/transformers/null_transformer.rb +1 -1
  44. data/lib/chronicle/etl/transformers/transformer.rb +3 -2
  45. data/lib/chronicle/etl/version.rb +1 -1
  46. data/lib/chronicle/etl.rb +12 -4
  47. metadata +123 -18
  48. data/.ruby-version +0 -1
  49. data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb +0 -104
  50. data/lib/chronicle/etl/loaders/stdout_loader.rb +0 -14
  51. data/lib/chronicle/etl/models/generic.rb +0 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b74c4a7782c1ab31173e628b3e5ccb8743fe21f29d6f48d739b0e3cc2dfda22e
4
- data.tar.gz: 7ea44638b08f6da12c0a5386f3d852600f50336ce0bb57347114804770f75691
3
+ metadata.gz: f041e90fc6019ecbc2424e8e75e7f21fa5ba715c2cdbde73d61c298e9ca82e07
4
+ data.tar.gz: 2feda7669f8fefc7ad80fe7918530a86ad2ced431f75e70ad42653c871b67d90
5
5
  SHA512:
6
- metadata.gz: efb23677c731a54b0382c3095dc9bb5f98a97365c1daf031bbc8c20335e7bd146b76b3a50486971e48192e7540bc0ae1b09f232590a590257203ae3560396767
7
- data.tar.gz: cba40b71a7e8c0b17a286ecd3db3724bff290fdd79b3fdf55ab89967f6af14228911c0e6928a949b8dd899acd6ad396b8a21fb03162a8561247c97b1200bac29
6
+ metadata.gz: 4a40c72dcb6514037c6e53214dc0af3bfba20c272b959c3e83496658b4b8dc3f841d7399aa215a5fcc5c5cd494278223666b8b881f645ed2c61b667351ccde94
7
+ data.tar.gz: 5b5450b76a8c03d7cb8405888b2e059390c1585a66524dd7403b458c828a1936422e03a9654ec6d270b634bf6bfe17d6dafda54e05aca8a8732207366d03ffd2
@@ -0,0 +1,35 @@
1
+ # This workflow uses actions that are not certified by GitHub.
2
+ # They are provided by a third-party and are governed by
3
+ # separate terms of service, privacy policy, and support
4
+ # documentation.
5
+ # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
6
+ # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
7
+
8
+ name: Ruby
9
+
10
+ on:
11
+ push:
12
+ branches: [ main ]
13
+ pull_request:
14
+ branches: [ main ]
15
+
16
+ jobs:
17
+ test:
18
+
19
+ runs-on: ubuntu-latest
20
+ strategy:
21
+ matrix:
22
+ ruby-version: ['2.7', '3.0']
23
+
24
+ steps:
25
+ - uses: actions/checkout@v2
26
+ - name: Set up Ruby
27
+ # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
28
+ # change this to (see https://github.com/ruby/setup-ruby#versioning):
29
+ # uses: ruby/setup-ruby@v1
30
+ uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
31
+ with:
32
+ ruby-version: ${{ matrix.ruby-version }}
33
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
34
+ - name: Run tests
35
+ run: bundle exec rake
data/.rubocop.yml CHANGED
@@ -1,11 +1,41 @@
1
1
  AllCops:
2
2
  EnabledByDefault: true
3
+ TargetRubyVersion: 2.7
4
+
5
+ Style/FrozenStringLiteralComment:
6
+ SafeAutoCorrect: true
3
7
 
4
8
  Style/StringLiterals:
5
9
  Enabled: false
6
10
 
11
+ Layout/MultilineAssignmentLayout:
12
+ Enabled: false
13
+
14
+ Layout/RedundantLineBreak:
15
+ Enabled: false
16
+
7
17
  Style/MethodCallWithArgsParentheses:
8
18
  Enabled: false
9
19
 
20
+ Style/MethodCalledOnDoEndBlock:
21
+ Exclude:
22
+ - 'spec/**/*'
23
+
24
+ Style/OpenStructUse:
25
+ Enabled: false
26
+
27
+ Style/Copyright:
28
+ Enabled: false
29
+
30
+ Style/MissingElse:
31
+ Enabled: false
32
+
33
+ Style/SymbolArray:
34
+ EnforcedStyle: brackets
35
+
36
+ Style/WordArray:
37
+ EnforcedStyle: brackets
38
+
10
39
  Lint/ConstantResolution:
11
- Enabled: false
40
+ Enabled: false
41
+
data/Guardfile ADDED
@@ -0,0 +1,7 @@
1
+ guard :rspec, cmd: "bundle exec rspec" do
2
+ require "guard/rspec/dsl"
3
+
4
+ watch(%r{^spec/.+_spec\.rb$})
5
+ watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
6
+ watch('spec/spec_helper.rb') { "spec" }
7
+ end
data/README.md CHANGED
@@ -1,125 +1,200 @@
1
- # Chronicle::ETL
1
+ ## A CLI toolkit for extracting and working with your digital history
2
2
 
3
- [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
3
+ ![chronicle-etl-banner](https://user-images.githubusercontent.com/6291/157330518-0f934c9a-9ec4-43d9-9cc2-12f156d09b37.png)
4
4
 
5
- Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
5
+ [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl) [![Ruby](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml/badge.svg)](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml)
6
6
 
7
- This tool is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex) and the dozens of existing importers are being migrated to Chronicle.
7
+ Are you trying to archive your digital history or incorporate it into your own projects? You’ve probably discovered how frustrating it is to get machine-readable access to your own data. While [building a memex](https://hyfen.net/memex/), I learned first-hand what great efforts must be made before you can begin using the data in interesting ways.
8
8
 
9
- ## Installation
9
+ If you don’t want to spend all your time writing scrapers, reverse-engineering APIs, or parsing takeout data, this project is for you! (*If you do enjoy these things, please see the [open issues](https://github.com/chronicle-app/chronicle-etl/issues).*)
10
10
 
11
- ```bash
12
- $ gem install chronicle-etl
11
+ **`chronicle-etl` is a CLI tool that gives you a unified interface for accessing your personal data.** It uses the ETL pattern to *extract* it from a source (e.g. your local browser history, a directory of images, goodreads.com reading history), *transform* it (into a given schema), and *load* it to a source (e.g. a CSV file, JSON, external API).
12
+
13
+ ## What does `chronicle-etl` give you?
14
+ * **CLI tool for working with personal data**. You can monitor progress of exports, manipulate the output, set up recurring jobs, manage credentials, and more.
15
+ * **Plugins for many third-party providers**. A plugin system allows you to access data from third-party providers and hook it into the shared CLI infrastructure.
16
+ * **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
17
+
18
+ ## Installation
19
+ ```sh
20
+ # Install chronicle-etl
21
+ gem install chronicle-etl
13
22
  ```
14
23
 
15
- ## Usage
24
+ After installation, the `chronicle-etl` command will be available in your shell. Homebrew support [is coming soon](https://github.com/chronicle-app/chronicle-etl/issues/13).
16
25
 
17
- After installing the gem, `chronicle-etl` is available to run in your shell.
26
+ ## Basic usage and running jobs
18
27
 
19
- ```bash
20
- # read test.csv and display it as a table
21
- $ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
28
+ ```sh
29
+ # Display help
30
+ $ chronicle-etl help
22
31
 
23
- # Display help for the jobs:run command
24
- $ chronicle-etl jobs help run
32
+ # Basic job usage
33
+ $ chronicle-etl --extractor NAME --transformer NAME --loader NAME
34
+
35
+ # Read test.csv and display it to stdout as a table
36
+ $ chronicle-etl --extractor csv --input ./data.csv --loader table
25
37
  ```
26
38
 
27
- ## Connectors
39
+ ### Common options
40
+ ```sh
41
+ Options:
42
+ -j, [--name=NAME] # Job configuration name
43
+ -e, [--extractor=EXTRACTOR-NAME] # Extractor class. Default: stdin
44
+ [--extractor-opts=key:value] # Extractor options
45
+ -t, [--transformer=TRANFORMER-NAME] # Transformer class. Default: null
46
+ [--transformer-opts=key:value] # Transformer options
47
+ -l, [--loader=LOADER-NAME] # Loader class. Default: stdout
48
+ [--loader-opts=key:value] # Loader options
49
+ -i, [--input=FILENAME] # Input filename or directory
50
+ [--since=DATE] # Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options
51
+ [--until=DATE] # Load records UNTIL this date
52
+ [--limit=N] # Only extract the first LIMIT records
53
+ -o, [--output=OUTPUT] # Output filename
54
+ [--fields=field1 field2 ...] # Output only these fields
55
+ [--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
56
+ # Default: info
57
+ -v, [--verbose], [--no-verbose] # Set log level to verbose
58
+ [--silent], [--no-silent] # Silence all output
59
+ ```
28
60
 
61
+ ## Connectors
29
62
  Connectors are available to read, process, and load data from different formats or external services.
30
63
 
31
- ```bash
64
+ ```sh
32
65
  # List all available connectors
33
66
  $ chronicle-etl connectors:list
34
-
35
- # Install a connector
36
- $ chronicle-etl connectors:install imessage
37
67
  ```
38
68
 
39
- Built in connectors:
40
-
41
- ### Extractors
42
- - `stdin` - (default) Load records from line-separated stdin
43
- - `csv`
44
- - `file` - load from a single file or directory (with a glob pattern)
45
-
46
- ### Transformers
47
- - `null` - (default) Don't do anything
48
-
49
- ### Loaders
50
- - `stdout` - (default) output records to stdout serialized as JSON
51
- - `csv` - Load records to a csv file
52
- - `rest` - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
53
- - `table` - Output an ascii table of records. Useful for debugging.
69
+ ### Built-in Connectors
70
+ `chronicle-etl` comes with several built-in connectors for common formats and sources.
54
71
 
55
- ### Provider-specific importers
72
+ #### Extractors
73
+ - [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records from CSV files or stdin
74
+ - [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/json_extractor.rb) - Load JSON (either [line-separated objects](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON) or one object)
75
+ - [`file`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/file_extractor.rb) - load from a single file or directory (with a glob pattern)
56
76
 
57
- In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
77
+ #### Transformers
78
+ - [`null`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/transformers/null_transformer.rb) - (default) Don’t do anything and pass on raw extraction data
58
79
 
59
- - [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files
60
- - [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`
61
- - [imessage](https://github.com/chronicle-app/chronicle-imessage). Extract iMessage messages from a local macOS installation
80
+ #### Loaders
81
+ - [`table`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/table_loader.rb) - (default) Output an ascii table of records. Useful for exploring data.
82
+ - [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records to CSV
83
+ - [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/json_loader.rb) - Load records serialized as JSON
84
+ - [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
62
85
 
63
- To install any of these, run `gem install chronicle-PROVIDER`.
86
+ ### Plugins
87
+ Plugins provide access to data from third-party platforms, services, or formats.
64
88
 
65
- If you don't want to use the available rubygem importers, `chronicle-etl` can use `stdin` as an Extractor source (newline separated records). You can also use `stdout` as a loader — transformed records will be outputted separated by newlines.
66
-
67
- I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
68
-
69
- ## Full commands
89
+ ```bash
90
+ # Install a plugin
91
+ $ chronicle-etl plugins:install NAME
70
92
 
71
- ```
72
- $ chronicle-etl help
73
-
74
- ALL COMMANDS
75
- help # This help menu
76
- connectors help [COMMAND] # Describe subcommands or one specific subcommand
77
- connectors:install NAME # Installs connector NAME
78
- connectors:list # Lists available connectors
79
- jobs help [COMMAND] # Describe subcommands or one specific subcommand
80
- jobs:create # Create a job
81
- jobs:list # List all available jobs
82
- jobs:run # Start a job
83
- jobs:show # Show details about a job
84
- ```
93
+ # Install the imessage plugin
94
+ $ chronicle-etl plugins:install imessage
85
95
 
86
- ### Running a job
96
+ # List installed plugins
97
+ $ chronicle-etl plugins:list
87
98
 
99
+ # Uninstall a plugin
100
+ $ chronicle-etl plugins:uninstall NAME
88
101
  ```
89
- Usage:
90
- chronicle-etl jobs:run
91
102
 
92
- Options:
93
- [--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
94
- # Default: info
95
- -v, [--verbose], [--no-verbose] # Set log level to verbose
96
- [--dry-run], [--no-dry-run] # Only run the extraction and transform steps, not the loading
97
- -e, [--extractor=extractor-name] # Extractor class. Default: stdin
98
- [--extractor-opts=key:value] # Extractor options
99
- -t, [--transformer=transformer-name] # Transformer class. Default: null
100
- [--transformer-opts=key:value] # Transformer options
101
- -l, [--loader=loader-name] # Loader class. Default: stdout
102
- [--loader-opts=key:value] # Loader options
103
- -j, [--name=NAME] # Job configuration name
104
-
105
-
106
- Runs an ETL job
103
+ A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and they’re being ported over to the Chronicle system. This table shows what’s available now and what’s coming. Rows are sorted in very rough order of priority.
104
+
105
+ If you want to work together on a connector, please [get in touch](#get-in-touch)!
106
+
107
+ | Name | Description | Availability |
108
+ |-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
109
+ | [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments | Available |
110
+ | [shell](https://github.com/chronicle-app/chronicle-shell) | Shell command history | Available (zsh support pending) |
111
+ | [email](https://github.com/chronicle-app/chronicle-email) | Emails and attachments from IMAP or .mbox files | Available (imap support pending) |
112
+ | [pinboard](https://github.com/chronicle-app/chronicle-email) | Bookmarks and tags | Available |
113
+ | [safari](https://github.com/chronicle-app/chronicle-safari) | Browser history from local sqlite db | Available |
114
+ | github | Github user and repo activity | In progress |
115
+ | chrome | Browser history from local sqlite db | Needs porting |
116
+ | whatsapp | Messaging history (via individual chat exports) or reverse-engineered local desktop install | Unstarted |
117
+ | anki | Studying and card creation history | Needs porting |
118
+ | facebook | Messaging and history posting via data export files | Needs porting |
119
+ | twitter | History via API or export data files | Needs porting |
120
+ | foursquare | Location history via API | Needs porting |
121
+ | goodreads | Reading history via export csv (RIP goodreads API) | Needs porting |
122
+ | lastfm | Listening history via API | Needs porting |
123
+ | images | Process image files | Needs porting |
124
+ | arc | Location history from synced icloud backup files | Needs porting |
125
+ | firefox | Browser history from local sqlite db | Needs porting |
126
+ | fitbit | Personal analytics via API | Needs porting |
127
+ | git | Commit history on a repo | Needs porting |
128
+ | google-calendar | Calendar events via API | Needs porting |
129
+ | instagram | Posting and messaging history via export data | Needs porting |
130
+ | shazam | Song tags via reverse-engineered API | Needs porting |
131
+ | slack | Messaging history via API | Need rethinking |
132
+ | strava | Activity history via API | Needs porting |
133
+ | things | Task activity via local sqlite db | Needs porting |
134
+ | bear | Note taking activity via local sqlite db | Needs porting |
135
+ | youtube | Video activity via takeout data and API | Needs porting |
136
+
137
+ ### Writing your own connector
138
+
139
+ Additional connectors are packaged as separate ruby gems. You can view the [iMessage plugin](https://github.com/chronicle-app/chronicle-imessage) for an example.
140
+
141
+ If you want to load a custom connector without creating a gem, you can help by [completing this issue](https://github.com/chronicle-app/chronicle-etl/issues/23).
142
+
143
+ If you want to work together on a connector, please [get in touch](#get-in-touch)!
144
+
145
+ #### Sample custom Extractor class
146
+ ```ruby
147
+ module Chronicle
148
+ module FooService
149
+ class FooExtractor < Chronicle::ETL::Extractor
150
+ register_connector do |r|
151
+ r.identifier = 'foo'
152
+ r.description = 'From foo.com'
153
+ end
154
+
155
+ setting :access_token, required: true
156
+
157
+ def prepare
158
+ @records = # load from somewhere
159
+ end
160
+
161
+ def extract
162
+ @records.each do |record|
163
+ yield Chronicle::ETL::Extraction.new(data: row.to_h)
164
+ end
165
+ end
166
+ end
167
+ end
168
+ end
107
169
  ```
108
170
 
109
171
  ## Development
110
-
111
172
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
112
173
 
113
174
  To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
114
175
 
115
- ## Contributing
176
+ ### Additional development commands
177
+ ```bash
178
+ # run tests
179
+ bundle exec rake spec
180
+
181
+ # generate docs
182
+ bundle exec rake yard
183
+
184
+ # use Guard to run specs automatically
185
+ bundle exec guard
186
+ ```
116
187
 
188
+ ## Get in touch
189
+ - [@hyfen](https://twitter.com/hyfen) on Twitter
190
+ - [@hyfen](https://github.com/hyfen) on Github
191
+ - Email: andrew@hyfen.net
192
+
193
+ ## Contributing
117
194
  Bug reports and pull requests are welcome on GitHub at https://github.com/chronicle-app/chronicle-etl. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
118
195
 
119
196
  ## License
120
-
121
197
  The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
122
198
 
123
199
  ## Code of Conduct
124
-
125
- Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
200
+ Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/main/CODE_OF_CONDUCT.md).
data/Rakefile CHANGED
@@ -1,6 +1,8 @@
1
1
  require "bundler/gem_tasks"
2
2
  require "rspec/core/rake_task"
3
-
4
3
  RSpec::Core::RakeTask.new(:spec)
5
4
 
6
- task :default => :spec
5
+ require 'yard'
6
+ YARD::Rake::YardocTask.new
7
+
8
+ task default: :spec
@@ -35,22 +35,30 @@ Gem::Specification.new do |spec|
35
35
  spec.bindir = "exe"
36
36
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
37
37
  spec.require_paths = ["lib"]
38
+ spec.required_ruby_version = ">= 2.7"
38
39
 
39
- spec.add_dependency "activesupport"
40
+ spec.add_dependency "activesupport", "~> 7.0"
40
41
  spec.add_dependency "chronic_duration", "~> 0.10.6"
41
42
  spec.add_dependency "colorize", "~> 0.8.1"
42
43
  spec.add_dependency "marcel", "~> 1.0.2"
43
44
  spec.add_dependency "mini_exiftool", "~> 2.10"
44
45
  spec.add_dependency "nokogiri", "~> 1.13"
45
- spec.add_dependency "runcom", "~> 6.2"
46
+ spec.add_dependency "runcom", ">= 6.0"
46
47
  spec.add_dependency "sequel", "~> 5.35"
47
48
  spec.add_dependency "sqlite3", "~> 1.4"
48
- spec.add_dependency "thor", "~> 0.20"
49
+ spec.add_dependency "thor", "~> 1.2"
50
+ spec.add_dependency "thor-hollaback", "~> 0.2"
49
51
  spec.add_dependency "tty-progressbar", "~> 0.17"
52
+ spec.add_dependency "tty-spinner"
50
53
  spec.add_dependency "tty-table", "~> 0.11"
54
+ spec.add_dependency "tty-prompt", "~> 0.23"
51
55
 
52
56
  spec.add_development_dependency "bundler", "~> 2.1"
53
57
  spec.add_development_dependency "pry-byebug", "~> 3.9"
54
58
  spec.add_development_dependency "rake", "~> 13.0"
55
59
  spec.add_development_dependency "rspec", "~> 3.9"
60
+ spec.add_development_dependency "simplecov", "~> 0.21"
61
+ spec.add_development_dependency "guard-rspec", "~> 4.7.3"
62
+ spec.add_development_dependency "yard", "~> 0.9.7"
63
+ spec.add_development_dependency "rubocop", "~> 1.25.1"
56
64
  end
data/exe/chronicle-etl CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require "chronicle/etl/cli/main"
3
+ require "chronicle/etl/cli"
4
4
 
5
5
  Chronicle::ETL::CLI::Main.start(ARGV)
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Chronicle
2
4
  module ETL
3
5
  module CLI
@@ -6,11 +8,6 @@ module Chronicle
6
8
  default_task 'list'
7
9
  namespace :connectors
8
10
 
9
- desc "install NAME", "Installs connector NAME"
10
- def install(name)
11
- Chronicle::ETL::Registry.install_connector(name)
12
- end
13
-
14
11
  desc "list", "Lists available connectors"
15
12
  # Display all available connectors that chronicle-etl has access to
16
13
  def list
@@ -38,6 +35,38 @@ module Chronicle
38
35
  table = TTY::Table.new(headers, connector_info.map(&:values))
39
36
  puts table.render(indent: 0, padding: [0, 2])
40
37
  end
38
+
39
+ desc "show PHASE IDENTIFIER", "Show information about a connector"
40
+ def show(phase, identifier)
41
+ unless ['extractor', 'transformer', 'loader'].include?(phase)
42
+ Chronicle::ETL::Logger.fatal("Phase argument must be one of: [extractor, transformer, loader]")
43
+ exit 1
44
+ end
45
+
46
+ begin
47
+ connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
48
+ rescue Chronicle::ETL::ConnectorNotAvailableError, Chronicle::ETL::PluginError
49
+ Chronicle::ETL::Logger.fatal("Could not find #{phase} #{identifier}")
50
+ exit 1
51
+ end
52
+
53
+ puts connector.klass.to_s.bold
54
+ puts " #{connector.descriptive_phrase}"
55
+ puts
56
+ puts "Settings:"
57
+
58
+ headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
59
+
60
+ settings = connector.klass.settings.map do |name, setting|
61
+ [
62
+ name,
63
+ setting.default,
64
+ setting.required ? 'yes' : 'no'
65
+ ]
66
+ end
67
+ table = TTY::Table.new(headers, settings)
68
+ puts table.render(indent: 0, padding: [0, 2])
69
+ end
41
70
  end
42
71
  end
43
72
  end