chronicle-etl 0.3.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +35 -0
  3. data/.rubocop.yml +31 -1
  4. data/Guardfile +7 -0
  5. data/README.md +157 -82
  6. data/Rakefile +4 -2
  7. data/chronicle-etl.gemspec +11 -3
  8. data/exe/chronicle-etl +1 -1
  9. data/lib/chronicle/etl/cli/connectors.rb +34 -5
  10. data/lib/chronicle/etl/cli/jobs.rb +90 -24
  11. data/lib/chronicle/etl/cli/main.rb +41 -19
  12. data/lib/chronicle/etl/cli/plugins.rb +62 -0
  13. data/lib/chronicle/etl/cli/subcommand_base.rb +2 -2
  14. data/lib/chronicle/etl/cli.rb +9 -0
  15. data/lib/chronicle/etl/config.rb +7 -4
  16. data/lib/chronicle/etl/configurable.rb +163 -0
  17. data/lib/chronicle/etl/exceptions.rb +29 -1
  18. data/lib/chronicle/etl/extractors/csv_extractor.rb +24 -23
  19. data/lib/chronicle/etl/extractors/extractor.rb +16 -15
  20. data/lib/chronicle/etl/extractors/file_extractor.rb +34 -11
  21. data/lib/chronicle/etl/extractors/helpers/input_reader.rb +76 -0
  22. data/lib/chronicle/etl/extractors/json_extractor.rb +19 -18
  23. data/lib/chronicle/etl/job.rb +8 -2
  24. data/lib/chronicle/etl/job_definition.rb +20 -5
  25. data/lib/chronicle/etl/loaders/csv_loader.rb +36 -9
  26. data/lib/chronicle/etl/loaders/helpers/encoding_helper.rb +18 -0
  27. data/lib/chronicle/etl/loaders/json_loader.rb +44 -0
  28. data/lib/chronicle/etl/loaders/loader.rb +28 -2
  29. data/lib/chronicle/etl/loaders/rest_loader.rb +5 -5
  30. data/lib/chronicle/etl/loaders/table_loader.rb +18 -37
  31. data/lib/chronicle/etl/logger.rb +6 -2
  32. data/lib/chronicle/etl/models/base.rb +3 -0
  33. data/lib/chronicle/etl/models/entity.rb +8 -2
  34. data/lib/chronicle/etl/models/raw.rb +26 -0
  35. data/lib/chronicle/etl/registry/connector_registration.rb +6 -0
  36. data/lib/chronicle/etl/registry/plugin_registry.rb +70 -0
  37. data/lib/chronicle/etl/registry/registry.rb +27 -14
  38. data/lib/chronicle/etl/runner.rb +35 -17
  39. data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +6 -0
  40. data/lib/chronicle/etl/serializers/raw_serializer.rb +10 -0
  41. data/lib/chronicle/etl/serializers/serializer.rb +2 -1
  42. data/lib/chronicle/etl/transformers/image_file_transformer.rb +22 -28
  43. data/lib/chronicle/etl/transformers/null_transformer.rb +1 -1
  44. data/lib/chronicle/etl/transformers/transformer.rb +3 -2
  45. data/lib/chronicle/etl/version.rb +1 -1
  46. data/lib/chronicle/etl.rb +12 -4
  47. metadata +123 -18
  48. data/.ruby-version +0 -1
  49. data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb +0 -104
  50. data/lib/chronicle/etl/loaders/stdout_loader.rb +0 -14
  51. data/lib/chronicle/etl/models/generic.rb +0 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b74c4a7782c1ab31173e628b3e5ccb8743fe21f29d6f48d739b0e3cc2dfda22e
4
- data.tar.gz: 7ea44638b08f6da12c0a5386f3d852600f50336ce0bb57347114804770f75691
3
+ metadata.gz: f041e90fc6019ecbc2424e8e75e7f21fa5ba715c2cdbde73d61c298e9ca82e07
4
+ data.tar.gz: 2feda7669f8fefc7ad80fe7918530a86ad2ced431f75e70ad42653c871b67d90
5
5
  SHA512:
6
- metadata.gz: efb23677c731a54b0382c3095dc9bb5f98a97365c1daf031bbc8c20335e7bd146b76b3a50486971e48192e7540bc0ae1b09f232590a590257203ae3560396767
7
- data.tar.gz: cba40b71a7e8c0b17a286ecd3db3724bff290fdd79b3fdf55ab89967f6af14228911c0e6928a949b8dd899acd6ad396b8a21fb03162a8561247c97b1200bac29
6
+ metadata.gz: 4a40c72dcb6514037c6e53214dc0af3bfba20c272b959c3e83496658b4b8dc3f841d7399aa215a5fcc5c5cd494278223666b8b881f645ed2c61b667351ccde94
7
+ data.tar.gz: 5b5450b76a8c03d7cb8405888b2e059390c1585a66524dd7403b458c828a1936422e03a9654ec6d270b634bf6bfe17d6dafda54e05aca8a8732207366d03ffd2
data/.github/workflows/ruby.yml ADDED
@@ -0,0 +1,35 @@
1
+ # This workflow uses actions that are not certified by GitHub.
2
+ # They are provided by a third-party and are governed by
3
+ # separate terms of service, privacy policy, and support
4
+ # documentation.
5
+ # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
6
+ # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
7
+
8
+ name: Ruby
9
+
10
+ on:
11
+ push:
12
+ branches: [ main ]
13
+ pull_request:
14
+ branches: [ main ]
15
+
16
+ jobs:
17
+ test:
18
+
19
+ runs-on: ubuntu-latest
20
+ strategy:
21
+ matrix:
22
+ ruby-version: ['2.7', '3.0']
23
+
24
+ steps:
25
+ - uses: actions/checkout@v2
26
+ - name: Set up Ruby
27
+ # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
28
+ # change this to (see https://github.com/ruby/setup-ruby#versioning):
29
+ # uses: ruby/setup-ruby@v1
30
+ uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
31
+ with:
32
+ ruby-version: ${{ matrix.ruby-version }}
33
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
34
+ - name: Run tests
35
+ run: bundle exec rake
data/.rubocop.yml CHANGED
@@ -1,11 +1,41 @@
1
1
  AllCops:
2
2
  EnabledByDefault: true
3
+ TargetRubyVersion: 2.7
4
+
5
+ Style/FrozenStringLiteralComment:
6
+ SafeAutoCorrect: true
3
7
 
4
8
  Style/StringLiterals:
5
9
  Enabled: false
6
10
 
11
+ Layout/MultilineAssignmentLayout:
12
+ Enabled: false
13
+
14
+ Layout/RedundantLineBreak:
15
+ Enabled: false
16
+
7
17
  Style/MethodCallWithArgsParentheses:
8
18
  Enabled: false
9
19
 
20
+ Style/MethodCalledOnDoEndBlock:
21
+ Exclude:
22
+ - 'spec/**/*'
23
+
24
+ Style/OpenStructUse:
25
+ Enabled: false
26
+
27
+ Style/Copyright:
28
+ Enabled: false
29
+
30
+ Style/MissingElse:
31
+ Enabled: false
32
+
33
+ Style/SymbolArray:
34
+ EnforcedStyle: brackets
35
+
36
+ Style/WordArray:
37
+ EnforcedStyle: brackets
38
+
10
39
  Lint/ConstantResolution:
11
- Enabled: false
40
+ Enabled: false
41
+
data/Guardfile ADDED
@@ -0,0 +1,7 @@
1
+ guard :rspec, cmd: "bundle exec rspec" do
2
+ require "guard/rspec/dsl"
3
+
4
+ watch(%r{^spec/.+_spec\.rb$})
5
+ watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
6
+ watch('spec/spec_helper.rb') { "spec" }
7
+ end
data/README.md CHANGED
@@ -1,125 +1,200 @@
1
- # Chronicle::ETL
1
+ ## A CLI toolkit for extracting and working with your digital history
2
2
 
3
- [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
3
+ ![chronicle-etl-banner](https://user-images.githubusercontent.com/6291/157330518-0f934c9a-9ec4-43d9-9cc2-12f156d09b37.png)
4
4
 
5
- Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
5
+ [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl) [![Ruby](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml/badge.svg)](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml)
6
6
 
7
- This tool is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex) and the dozens of existing importers are being migrated to Chronicle.
7
+ Are you trying to archive your digital history or incorporate it into your own projects? You’ve probably discovered how frustrating it is to get machine-readable access to your own data. While [building a memex](https://hyfen.net/memex/), I learned first-hand what great efforts must be made before you can begin using the data in interesting ways.
8
8
 
9
- ## Installation
9
+ If you don’t want to spend all your time writing scrapers, reverse-engineering APIs, or parsing takeout data, this project is for you! (*If you do enjoy these things, please see the [open issues](https://github.com/chronicle-app/chronicle-etl/issues).*)
10
10
 
11
- ```bash
12
- $ gem install chronicle-etl
11
+ **`chronicle-etl` is a CLI tool that gives you a unified interface for accessing your personal data.** It uses the ETL pattern to *extract* it from a source (e.g. your local browser history, a directory of images, goodreads.com reading history), *transform* it (into a given schema), and *load* it to a source (e.g. a CSV file, JSON, external API).
12
+
13
+ ## What does `chronicle-etl` give you?
14
+ * **CLI tool for working with personal data**. You can monitor progress of exports, manipulate the output, set up recurring jobs, manage credentials, and more.
15
+ * **Plugins for many third-party providers**. A plugin system allows you to access data from third-party providers and hook it into the shared CLI infrastructure.
16
+ * **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
17
+
18
+ ## Installation
19
+ ```sh
20
+ # Install chronicle-etl
21
+ gem install chronicle-etl
13
22
  ```
14
23
 
15
- ## Usage
24
+ After installation, the `chronicle-etl` command will be available in your shell. Homebrew support [is coming soon](https://github.com/chronicle-app/chronicle-etl/issues/13).
16
25
 
17
- After installing the gem, `chronicle-etl` is available to run in your shell.
26
+ ## Basic usage and running jobs
18
27
 
19
- ```bash
20
- # read test.csv and display it as a table
21
- $ chronicle-etl jobs:run --extractor csv --extractor-opts filename:test.csv --loader table
28
+ ```sh
29
+ # Display help
30
+ $ chronicle-etl help
22
31
 
23
- # Display help for the jobs:run command
24
- $ chronicle-etl jobs help run
32
+ # Basic job usage
33
+ $ chronicle-etl --extractor NAME --transformer NAME --loader NAME
34
+
35
+ # Read test.csv and display it to stdout as a table
36
+ $ chronicle-etl --extractor csv --input ./data.csv --loader table
25
37
  ```
26
38
 
27
- ## Connectors
39
+ ### Common options
40
+ ```sh
41
+ Options:
42
+ -j, [--name=NAME] # Job configuration name
43
+ -e, [--extractor=EXTRACTOR-NAME] # Extractor class. Default: stdin
44
+ [--extractor-opts=key:value] # Extractor options
45
+ -t, [--transformer=TRANFORMER-NAME] # Transformer class. Default: null
46
+ [--transformer-opts=key:value] # Transformer options
47
+ -l, [--loader=LOADER-NAME] # Loader class. Default: stdout
48
+ [--loader-opts=key:value] # Loader options
49
+ -i, [--input=FILENAME] # Input filename or directory
50
+ [--since=DATE] # Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options
51
+ [--until=DATE] # Load records UNTIL this date
52
+ [--limit=N] # Only extract the first LIMIT records
53
+ -o, [--output=OUTPUT] # Output filename
54
+ [--fields=field1 field2 ...] # Output only these fields
55
+ [--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
56
+ # Default: info
57
+ -v, [--verbose], [--no-verbose] # Set log level to verbose
58
+ [--silent], [--no-silent] # Silence all output
59
+ ```
28
60
 
61
+ ## Connectors
29
62
  Connectors are available to read, process, and load data from different formats or external services.
30
63
 
31
- ```bash
64
+ ```sh
32
65
  # List all available connectors
33
66
  $ chronicle-etl connectors:list
34
-
35
- # Install a connector
36
- $ chronicle-etl connectors:install imessage
37
67
  ```
38
68
 
39
- Built in connectors:
40
-
41
- ### Extractors
42
- - `stdin` - (default) Load records from line-separated stdin
43
- - `csv`
44
- - `file` - load from a single file or directory (with a glob pattern)
45
-
46
- ### Transformers
47
- - `null` - (default) Don't do anything
48
-
49
- ### Loaders
50
- - `stdout` - (default) output records to stdout serialized as JSON
51
- - `csv` - Load records to a csv file
52
- - `rest` - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
53
- - `table` - Output an ascii table of records. Useful for debugging.
69
+ ### Built-in Connectors
70
+ `chronicle-etl` comes with several built-in connectors for common formats and sources.
54
71
 
55
- ### Provider-specific importers
72
+ #### Extractors
73
+ - [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records from CSV files or stdin
74
+ - [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/json_extractor.rb) - Load JSON (either [line-separated objects](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON) or one object)
75
+ - [`file`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/file_extractor.rb) - load from a single file or directory (with a glob pattern)
56
76
 
57
- In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
77
+ #### Transformers
78
+ - [`null`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/transformers/null_transformer.rb) - (default) Don’t do anything and pass on raw extraction data
58
79
 
59
- - [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files
60
- - [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`
61
- - [imessage](https://github.com/chronicle-app/chronicle-imessage). Extract iMessage messages from a local macOS installation
80
+ #### Loaders
81
+ - [`table`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/table_loader.rb) - (default) Output an ascii table of records. Useful for exploring data.
82
+ - [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records to CSV
83
+ - [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/json_loader.rb) - Load records serialized as JSON
84
+ - [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
62
85
 
63
- To install any of these, run `gem install chronicle-PROVIDER`.
86
+ ### Plugins
87
+ Plugins provide access to data from third-party platforms, services, or formats.
64
88
 
65
- If you don't want to use the available rubygem importers, `chronicle-etl` can use `stdin` as an Extractor source (newline separated records). You can also use `stdout` as a loader — transformed records will be outputted separated by newlines.
66
-
67
- I'll be open-sourcing more importers. Please [contact me](mailto:andrew@hyfen.net) to chat about what will be available!
68
-
69
- ## Full commands
89
+ ```bash
90
+ # Install a plugin
91
+ $ chronicle-etl plugins:install NAME
70
92
 
71
- ```
72
- $ chronicle-etl help
73
-
74
- ALL COMMANDS
75
- help # This help menu
76
- connectors help [COMMAND] # Describe subcommands or one specific subcommand
77
- connectors:install NAME # Installs connector NAME
78
- connectors:list # Lists available connectors
79
- jobs help [COMMAND] # Describe subcommands or one specific subcommand
80
- jobs:create # Create a job
81
- jobs:list # List all available jobs
82
- jobs:run # Start a job
83
- jobs:show # Show details about a job
84
- ```
93
+ # Install the imessage plugin
94
+ $ chronicle-etl plugins:install imessage
85
95
 
86
- ### Running a job
96
+ # List installed plugins
97
+ $ chronicle-etl plugins:list
87
98
 
99
+ # Uninstall a plugin
100
+ $ chronicle-etl plugins:uninstall NAME
88
101
  ```
89
- Usage:
90
- chronicle-etl jobs:run
91
102
 
92
- Options:
93
- [--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
94
- # Default: info
95
- -v, [--verbose], [--no-verbose] # Set log level to verbose
96
- [--dry-run], [--no-dry-run] # Only run the extraction and transform steps, not the loading
97
- -e, [--extractor=extractor-name] # Extractor class. Default: stdin
98
- [--extractor-opts=key:value] # Extractor options
99
- -t, [--transformer=transformer-name] # Transformer class. Default: null
100
- [--transformer-opts=key:value] # Transformer options
101
- -l, [--loader=loader-name] # Loader class. Default: stdout
102
- [--loader-opts=key:value] # Loader options
103
- -j, [--name=NAME] # Job configuration name
104
-
105
-
106
- Runs an ETL job
103
+ A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and they’re being ported over to the Chronicle system. This table shows what’s available now and what’s coming. Rows are sorted in very rough order of priority.
104
+
105
+ If you want to work together on a connector, please [get in touch](#get-in-touch)!
106
+
107
+ | Name | Description | Availability |
108
+ |-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
109
+ | [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments | Available |
110
+ | [shell](https://github.com/chronicle-app/chronicle-shell) | Shell command history | Available (zsh support pending) |
111
+ | [email](https://github.com/chronicle-app/chronicle-email) | Emails and attachments from IMAP or .mbox files | Available (imap support pending) |
112
+ | [pinboard](https://github.com/chronicle-app/chronicle-email) | Bookmarks and tags | Available |
113
+ | [safari](https://github.com/chronicle-app/chronicle-safari) | Browser history from local sqlite db | Available |
114
+ | github | Github user and repo activity | In progress |
115
+ | chrome | Browser history from local sqlite db | Needs porting |
116
+ | whatsapp | Messaging history (via individual chat exports) or reverse-engineered local desktop install | Unstarted |
117
+ | anki | Studying and card creation history | Needs porting |
118
+ | facebook | Messaging and history posting via data export files | Needs porting |
119
+ | twitter | History via API or export data files | Needs porting |
120
+ | foursquare | Location history via API | Needs porting |
121
+ | goodreads | Reading history via export csv (RIP goodreads API) | Needs porting |
122
+ | lastfm | Listening history via API | Needs porting |
123
+ | images | Process image files | Needs porting |
124
+ | arc | Location history from synced icloud backup files | Needs porting |
125
+ | firefox | Browser history from local sqlite db | Needs porting |
126
+ | fitbit | Personal analytics via API | Needs porting |
127
+ | git | Commit history on a repo | Needs porting |
128
+ | google-calendar | Calendar events via API | Needs porting |
129
+ | instagram | Posting and messaging history via export data | Needs porting |
130
+ | shazam | Song tags via reverse-engineered API | Needs porting |
131
+ | slack | Messaging history via API | Need rethinking |
132
+ | strava | Activity history via API | Needs porting |
133
+ | things | Task activity via local sqlite db | Needs porting |
134
+ | bear | Note taking activity via local sqlite db | Needs porting |
135
+ | youtube | Video activity via takeout data and API | Needs porting |
136
+
137
+ ### Writing your own connector
138
+
139
+ Additional connectors are packaged as separate ruby gems. You can view the [iMessage plugin](https://github.com/chronicle-app/chronicle-imessage) for an example.
140
+
141
+ If you want to load a custom connector without creating a gem, you can help by [completing this issue](https://github.com/chronicle-app/chronicle-etl/issues/23).
142
+
143
+ If you want to work together on a connector, please [get in touch](#get-in-touch)!
144
+
145
+ #### Sample custom Extractor class
146
+ ```ruby
147
+ module Chronicle
148
+ module FooService
149
+ class FooExtractor < Chronicle::ETL::Extractor
150
+ register_connector do |r|
151
+ r.identifier = 'foo'
152
+ r.description = 'From foo.com'
153
+ end
154
+
155
+ setting :access_token, required: true
156
+
157
+ def prepare
158
+ @records = # load from somewhere
159
+ end
160
+
161
+ def extract
162
+ @records.each do |record|
163
+ yield Chronicle::ETL::Extraction.new(data: row.to_h)
164
+ end
165
+ end
166
+ end
167
+ end
168
+ end
107
169
  ```
108
170
 
109
171
  ## Development
110
-
111
172
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
112
173
 
113
174
  To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
114
175
 
115
- ## Contributing
176
+ ### Additional development commands
177
+ ```bash
178
+ # run tests
179
+ bundle exec rake spec
180
+
181
+ # generate docs
182
+ bundle exec rake yard
183
+
184
+ # use Guard to run specs automatically
185
+ bundle exec guard
186
+ ```
116
187
 
188
+ ## Get in touch
189
+ - [@hyfen](https://twitter.com/hyfen) on Twitter
190
+ - [@hyfen](https://github.com/hyfen) on Github
191
+ - Email: andrew@hyfen.net
192
+
193
+ ## Contributing
117
194
  Bug reports and pull requests are welcome on GitHub at https://github.com/chronicle-app/chronicle-etl. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
118
195
 
119
196
  ## License
120
-
121
197
  The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
122
198
 
123
199
  ## Code of Conduct
124
-
125
- Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
200
+ Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/main/CODE_OF_CONDUCT.md).
data/Rakefile CHANGED
@@ -1,6 +1,8 @@
1
1
  require "bundler/gem_tasks"
2
2
  require "rspec/core/rake_task"
3
-
4
3
  RSpec::Core::RakeTask.new(:spec)
5
4
 
6
- task :default => :spec
5
+ require 'yard'
6
+ YARD::Rake::YardocTask.new
7
+
8
+ task default: :spec
data/chronicle-etl.gemspec CHANGED
@@ -35,22 +35,30 @@ Gem::Specification.new do |spec|
35
35
  spec.bindir = "exe"
36
36
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
37
37
  spec.require_paths = ["lib"]
38
+ spec.required_ruby_version = ">= 2.7"
38
39
 
39
- spec.add_dependency "activesupport"
40
+ spec.add_dependency "activesupport", "~> 7.0"
40
41
  spec.add_dependency "chronic_duration", "~> 0.10.6"
41
42
  spec.add_dependency "colorize", "~> 0.8.1"
42
43
  spec.add_dependency "marcel", "~> 1.0.2"
43
44
  spec.add_dependency "mini_exiftool", "~> 2.10"
44
45
  spec.add_dependency "nokogiri", "~> 1.13"
45
- spec.add_dependency "runcom", "~> 6.2"
46
+ spec.add_dependency "runcom", ">= 6.0"
46
47
  spec.add_dependency "sequel", "~> 5.35"
47
48
  spec.add_dependency "sqlite3", "~> 1.4"
48
- spec.add_dependency "thor", "~> 0.20"
49
+ spec.add_dependency "thor", "~> 1.2"
50
+ spec.add_dependency "thor-hollaback", "~> 0.2"
49
51
  spec.add_dependency "tty-progressbar", "~> 0.17"
52
+ spec.add_dependency "tty-spinner"
50
53
  spec.add_dependency "tty-table", "~> 0.11"
54
+ spec.add_dependency "tty-prompt", "~> 0.23"
51
55
 
52
56
  spec.add_development_dependency "bundler", "~> 2.1"
53
57
  spec.add_development_dependency "pry-byebug", "~> 3.9"
54
58
  spec.add_development_dependency "rake", "~> 13.0"
55
59
  spec.add_development_dependency "rspec", "~> 3.9"
60
+ spec.add_development_dependency "simplecov", "~> 0.21"
61
+ spec.add_development_dependency "guard-rspec", "~> 4.7.3"
62
+ spec.add_development_dependency "yard", "~> 0.9.7"
63
+ spec.add_development_dependency "rubocop", "~> 1.25.1"
56
64
  end
data/exe/chronicle-etl CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require "chronicle/etl/cli/main"
3
+ require "chronicle/etl/cli"
4
4
 
5
5
  Chronicle::ETL::CLI::Main.start(ARGV)
data/lib/chronicle/etl/cli/connectors.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Chronicle
2
4
  module ETL
3
5
  module CLI
@@ -6,11 +8,6 @@ module Chronicle
6
8
  default_task 'list'
7
9
  namespace :connectors
8
10
 
9
- desc "install NAME", "Installs connector NAME"
10
- def install(name)
11
- Chronicle::ETL::Registry.install_connector(name)
12
- end
13
-
14
11
  desc "list", "Lists available connectors"
15
12
  # Display all available connectors that chronicle-etl has access to
16
13
  def list
@@ -38,6 +35,38 @@ module Chronicle
38
35
  table = TTY::Table.new(headers, connector_info.map(&:values))
39
36
  puts table.render(indent: 0, padding: [0, 2])
40
37
  end
38
+
39
+ desc "show PHASE IDENTIFIER", "Show information about a connector"
40
+ def show(phase, identifier)
41
+ unless ['extractor', 'transformer', 'loader'].include?(phase)
42
+ Chronicle::ETL::Logger.fatal("Phase argument must be one of: [extractor, transformer, loader]")
43
+ exit 1
44
+ end
45
+
46
+ begin
47
+ connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
48
+ rescue Chronicle::ETL::ConnectorNotAvailableError, Chronicle::ETL::PluginError
49
+ Chronicle::ETL::Logger.fatal("Could not find #{phase} #{identifier}")
50
+ exit 1
51
+ end
52
+
53
+ puts connector.klass.to_s.bold
54
+ puts " #{connector.descriptive_phrase}"
55
+ puts
56
+ puts "Settings:"
57
+
58
+ headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
59
+
60
+ settings = connector.klass.settings.map do |name, setting|
61
+ [
62
+ name,
63
+ setting.default,
64
+ setting.required ? 'yes' : 'no'
65
+ ]
66
+ end
67
+ table = TTY::Table.new(headers, settings)
68
+ puts table.render(indent: 0, padding: [0, 2])
69
+ end
41
70
  end
42
71
  end
43
72
  end