chronicle-etl 0.3.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +35 -0
- data/.rubocop.yml +31 -1
- data/Guardfile +7 -0
- data/README.md +157 -82
- data/Rakefile +4 -2
- data/chronicle-etl.gemspec +11 -3
- data/exe/chronicle-etl +1 -1
- data/lib/chronicle/etl/cli/connectors.rb +34 -5
- data/lib/chronicle/etl/cli/jobs.rb +90 -24
- data/lib/chronicle/etl/cli/main.rb +41 -19
- data/lib/chronicle/etl/cli/plugins.rb +62 -0
- data/lib/chronicle/etl/cli/subcommand_base.rb +2 -2
- data/lib/chronicle/etl/cli.rb +9 -0
- data/lib/chronicle/etl/config.rb +7 -4
- data/lib/chronicle/etl/configurable.rb +163 -0
- data/lib/chronicle/etl/exceptions.rb +29 -1
- data/lib/chronicle/etl/extractors/csv_extractor.rb +24 -23
- data/lib/chronicle/etl/extractors/extractor.rb +16 -15
- data/lib/chronicle/etl/extractors/file_extractor.rb +34 -11
- data/lib/chronicle/etl/extractors/helpers/input_reader.rb +76 -0
- data/lib/chronicle/etl/extractors/json_extractor.rb +19 -18
- data/lib/chronicle/etl/job.rb +8 -2
- data/lib/chronicle/etl/job_definition.rb +20 -5
- data/lib/chronicle/etl/loaders/csv_loader.rb +36 -9
- data/lib/chronicle/etl/loaders/helpers/encoding_helper.rb +18 -0
- data/lib/chronicle/etl/loaders/json_loader.rb +44 -0
- data/lib/chronicle/etl/loaders/loader.rb +28 -2
- data/lib/chronicle/etl/loaders/rest_loader.rb +5 -5
- data/lib/chronicle/etl/loaders/table_loader.rb +18 -37
- data/lib/chronicle/etl/logger.rb +6 -2
- data/lib/chronicle/etl/models/base.rb +3 -0
- data/lib/chronicle/etl/models/entity.rb +8 -2
- data/lib/chronicle/etl/models/raw.rb +26 -0
- data/lib/chronicle/etl/registry/connector_registration.rb +6 -0
- data/lib/chronicle/etl/registry/plugin_registry.rb +70 -0
- data/lib/chronicle/etl/registry/registry.rb +27 -14
- data/lib/chronicle/etl/runner.rb +35 -17
- data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +6 -0
- data/lib/chronicle/etl/serializers/raw_serializer.rb +10 -0
- data/lib/chronicle/etl/serializers/serializer.rb +2 -1
- data/lib/chronicle/etl/transformers/image_file_transformer.rb +22 -28
- data/lib/chronicle/etl/transformers/null_transformer.rb +1 -1
- data/lib/chronicle/etl/transformers/transformer.rb +3 -2
- data/lib/chronicle/etl/version.rb +1 -1
- data/lib/chronicle/etl.rb +12 -4
- metadata +123 -18
- data/.ruby-version +0 -1
- data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb +0 -104
- data/lib/chronicle/etl/loaders/stdout_loader.rb +0 -14
- data/lib/chronicle/etl/models/generic.rb +0 -23
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f041e90fc6019ecbc2424e8e75e7f21fa5ba715c2cdbde73d61c298e9ca82e07
|
|
4
|
+
data.tar.gz: 2feda7669f8fefc7ad80fe7918530a86ad2ced431f75e70ad42653c871b67d90
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4a40c72dcb6514037c6e53214dc0af3bfba20c272b959c3e83496658b4b8dc3f841d7399aa215a5fcc5c5cd494278223666b8b881f645ed2c61b667351ccde94
|
|
7
|
+
data.tar.gz: 5b5450b76a8c03d7cb8405888b2e059390c1585a66524dd7403b458c828a1936422e03a9654ec6d270b634bf6bfe17d6dafda54e05aca8a8732207366d03ffd2
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
2
|
+
# They are provided by a third-party and are governed by
|
|
3
|
+
# separate terms of service, privacy policy, and support
|
|
4
|
+
# documentation.
|
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
|
7
|
+
|
|
8
|
+
name: Ruby
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [ main ]
|
|
13
|
+
pull_request:
|
|
14
|
+
branches: [ main ]
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
strategy:
|
|
21
|
+
matrix:
|
|
22
|
+
ruby-version: ['2.7', '3.0']
|
|
23
|
+
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v2
|
|
26
|
+
- name: Set up Ruby
|
|
27
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
|
28
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
|
29
|
+
# uses: ruby/setup-ruby@v1
|
|
30
|
+
uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
|
|
31
|
+
with:
|
|
32
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
33
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
34
|
+
- name: Run tests
|
|
35
|
+
run: bundle exec rake
|
data/.rubocop.yml
CHANGED
|
@@ -1,11 +1,41 @@
|
|
|
1
1
|
AllCops:
|
|
2
2
|
EnabledByDefault: true
|
|
3
|
+
TargetRubyVersion: 2.7
|
|
4
|
+
|
|
5
|
+
Style/FrozenStringLiteralComment:
|
|
6
|
+
SafeAutoCorrect: true
|
|
3
7
|
|
|
4
8
|
Style/StringLiterals:
|
|
5
9
|
Enabled: false
|
|
6
10
|
|
|
11
|
+
Layout/MultilineAssignmentLayout:
|
|
12
|
+
Enabled: false
|
|
13
|
+
|
|
14
|
+
Layout/RedundantLineBreak:
|
|
15
|
+
Enabled: false
|
|
16
|
+
|
|
7
17
|
Style/MethodCallWithArgsParentheses:
|
|
8
18
|
Enabled: false
|
|
9
19
|
|
|
20
|
+
Style/MethodCalledOnDoEndBlock:
|
|
21
|
+
Exclude:
|
|
22
|
+
- 'spec/**/*'
|
|
23
|
+
|
|
24
|
+
Style/OpenStructUse:
|
|
25
|
+
Enabled: false
|
|
26
|
+
|
|
27
|
+
Style/Copyright:
|
|
28
|
+
Enabled: false
|
|
29
|
+
|
|
30
|
+
Style/MissingElse:
|
|
31
|
+
Enabled: false
|
|
32
|
+
|
|
33
|
+
Style/SymbolArray:
|
|
34
|
+
EnforcedStyle: brackets
|
|
35
|
+
|
|
36
|
+
Style/WordArray:
|
|
37
|
+
EnforcedStyle: brackets
|
|
38
|
+
|
|
10
39
|
Lint/ConstantResolution:
|
|
11
|
-
Enabled: false
|
|
40
|
+
Enabled: false
|
|
41
|
+
|
data/Guardfile
ADDED
data/README.md
CHANGED
|
@@ -1,125 +1,200 @@
|
|
|
1
|
-
|
|
1
|
+
## A CLI toolkit for extracting and working with your digital history
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[](https://badge.fury.io/rb/chronicle-etl) [](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml)
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Are you trying to archive your digital history or incorporate it into your own projects? You’ve probably discovered how frustrating it is to get machine-readable access to your own data. While [building a memex](https://hyfen.net/memex/), I learned first-hand what great efforts must be made before you can begin using the data in interesting ways.
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
If you don’t want to spend all your time writing scrapers, reverse-engineering APIs, or parsing takeout data, this project is for you! (*If you do enjoy these things, please see the [open issues](https://github.com/chronicle-app/chronicle-etl/issues).*)
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
**`chronicle-etl` is a CLI tool that gives you a unified interface for accessing your personal data.** It uses the ETL pattern to *extract* it from a source (e.g. your local browser history, a directory of images, goodreads.com reading history), *transform* it (into a given schema), and *load* it to a source (e.g. a CSV file, JSON, external API).
|
|
12
|
+
|
|
13
|
+
## What does `chronicle-etl` give you?
|
|
14
|
+
* **CLI tool for working with personal data**. You can monitor progress of exports, manipulate the output, set up recurring jobs, manage credentials, and more.
|
|
15
|
+
* **Plugins for many third-party providers**. A plugin system allows you to access data from third-party providers and hook it into the shared CLI infrastructure.
|
|
16
|
+
* **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
```sh
|
|
20
|
+
# Install chronicle-etl
|
|
21
|
+
gem install chronicle-etl
|
|
13
22
|
```
|
|
14
23
|
|
|
15
|
-
|
|
24
|
+
After installation, the `chronicle-etl` command will be available in your shell. Homebrew support [is coming soon](https://github.com/chronicle-app/chronicle-etl/issues/13).
|
|
16
25
|
|
|
17
|
-
|
|
26
|
+
## Basic usage and running jobs
|
|
18
27
|
|
|
19
|
-
```
|
|
20
|
-
#
|
|
21
|
-
$ chronicle-etl
|
|
28
|
+
```sh
|
|
29
|
+
# Display help
|
|
30
|
+
$ chronicle-etl help
|
|
22
31
|
|
|
23
|
-
#
|
|
24
|
-
$ chronicle-etl
|
|
32
|
+
# Basic job usage
|
|
33
|
+
$ chronicle-etl --extractor NAME --transformer NAME --loader NAME
|
|
34
|
+
|
|
35
|
+
# Read test.csv and display it to stdout as a table
|
|
36
|
+
$ chronicle-etl --extractor csv --input ./data.csv --loader table
|
|
25
37
|
```
|
|
26
38
|
|
|
27
|
-
|
|
39
|
+
### Common options
|
|
40
|
+
```sh
|
|
41
|
+
Options:
|
|
42
|
+
-j, [--name=NAME] # Job configuration name
|
|
43
|
+
-e, [--extractor=EXTRACTOR-NAME] # Extractor class. Default: stdin
|
|
44
|
+
[--extractor-opts=key:value] # Extractor options
|
|
45
|
+
-t, [--transformer=TRANFORMER-NAME] # Transformer class. Default: null
|
|
46
|
+
[--transformer-opts=key:value] # Transformer options
|
|
47
|
+
-l, [--loader=LOADER-NAME] # Loader class. Default: stdout
|
|
48
|
+
[--loader-opts=key:value] # Loader options
|
|
49
|
+
-i, [--input=FILENAME] # Input filename or directory
|
|
50
|
+
[--since=DATE] # Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options
|
|
51
|
+
[--until=DATE] # Load records UNTIL this date
|
|
52
|
+
[--limit=N] # Only extract the first LIMIT records
|
|
53
|
+
-o, [--output=OUTPUT] # Output filename
|
|
54
|
+
[--fields=field1 field2 ...] # Output only these fields
|
|
55
|
+
[--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
|
|
56
|
+
# Default: info
|
|
57
|
+
-v, [--verbose], [--no-verbose] # Set log level to verbose
|
|
58
|
+
[--silent], [--no-silent] # Silence all output
|
|
59
|
+
```
|
|
28
60
|
|
|
61
|
+
## Connectors
|
|
29
62
|
Connectors are available to read, process, and load data from different formats or external services.
|
|
30
63
|
|
|
31
|
-
```
|
|
64
|
+
```sh
|
|
32
65
|
# List all available connectors
|
|
33
66
|
$ chronicle-etl connectors:list
|
|
34
|
-
|
|
35
|
-
# Install a connector
|
|
36
|
-
$ chronicle-etl connectors:install imessage
|
|
37
67
|
```
|
|
38
68
|
|
|
39
|
-
Built
|
|
40
|
-
|
|
41
|
-
### Extractors
|
|
42
|
-
- `stdin` - (default) Load records from line-separated stdin
|
|
43
|
-
- `csv`
|
|
44
|
-
- `file` - load from a single file or directory (with a glob pattern)
|
|
45
|
-
|
|
46
|
-
### Transformers
|
|
47
|
-
- `null` - (default) Don't do anything
|
|
48
|
-
|
|
49
|
-
### Loaders
|
|
50
|
-
- `stdout` - (default) output records to stdout serialized as JSON
|
|
51
|
-
- `csv` - Load records to a csv file
|
|
52
|
-
- `rest` - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
|
|
53
|
-
- `table` - Output an ascii table of records. Useful for debugging.
|
|
69
|
+
### Built-in Connectors
|
|
70
|
+
`chronicle-etl` comes with several built-in connectors for common formats and sources.
|
|
54
71
|
|
|
55
|
-
|
|
72
|
+
#### Extractors
|
|
73
|
+
- [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records from CSV files or stdin
|
|
74
|
+
- [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/json_extractor.rb) - Load JSON (either [line-separated objects](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON) or one object)
|
|
75
|
+
- [`file`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/file_extractor.rb) - load from a single file or directory (with a glob pattern)
|
|
56
76
|
|
|
57
|
-
|
|
77
|
+
#### Transformers
|
|
78
|
+
- [`null`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/transformers/null_transformer.rb) - (default) Don’t do anything and pass on raw extraction data
|
|
58
79
|
|
|
59
|
-
|
|
60
|
-
- [
|
|
61
|
-
- [
|
|
80
|
+
#### Loaders
|
|
81
|
+
- [`table`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/table_loader.rb) - (default) Output an ascii table of records. Useful for exploring data.
|
|
82
|
+
- [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records to CSV
|
|
83
|
+
- [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/json_loader.rb) - Load records serialized as JSON
|
|
84
|
+
- [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
|
|
62
85
|
|
|
63
|
-
|
|
86
|
+
### Plugins
|
|
87
|
+
Plugins provide access to data from third-party platforms, services, or formats.
|
|
64
88
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
## Full commands
|
|
89
|
+
```bash
|
|
90
|
+
# Install a plugin
|
|
91
|
+
$ chronicle-etl plugins:install NAME
|
|
70
92
|
|
|
71
|
-
|
|
72
|
-
$ chronicle-etl
|
|
73
|
-
|
|
74
|
-
ALL COMMANDS
|
|
75
|
-
help # This help menu
|
|
76
|
-
connectors help [COMMAND] # Describe subcommands or one specific subcommand
|
|
77
|
-
connectors:install NAME # Installs connector NAME
|
|
78
|
-
connectors:list # Lists available connectors
|
|
79
|
-
jobs help [COMMAND] # Describe subcommands or one specific subcommand
|
|
80
|
-
jobs:create # Create a job
|
|
81
|
-
jobs:list # List all available jobs
|
|
82
|
-
jobs:run # Start a job
|
|
83
|
-
jobs:show # Show details about a job
|
|
84
|
-
```
|
|
93
|
+
# Install the imessage plugin
|
|
94
|
+
$ chronicle-etl plugins:install imessage
|
|
85
95
|
|
|
86
|
-
|
|
96
|
+
# List installed plugins
|
|
97
|
+
$ chronicle-etl plugins:list
|
|
87
98
|
|
|
99
|
+
# Uninstall a plugin
|
|
100
|
+
$ chronicle-etl plugins:uninstall NAME
|
|
88
101
|
```
|
|
89
|
-
Usage:
|
|
90
|
-
chronicle-etl jobs:run
|
|
91
102
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
103
|
+
A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and they’re being ported over to the Chronicle system. This table shows what’s available now and what’s coming. Rows are sorted in very rough order of priority.
|
|
104
|
+
|
|
105
|
+
If you want to work together on a connector, please [get in touch](#get-in-touch)!
|
|
106
|
+
|
|
107
|
+
| Name | Description | Availability |
|
|
108
|
+
|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
|
|
109
|
+
| [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments | Available |
|
|
110
|
+
| [shell](https://github.com/chronicle-app/chronicle-shell) | Shell command history | Available (zsh support pending) |
|
|
111
|
+
| [email](https://github.com/chronicle-app/chronicle-email) | Emails and attachments from IMAP or .mbox files | Available (imap support pending) |
|
|
112
|
+
| [pinboard](https://github.com/chronicle-app/chronicle-email) | Bookmarks and tags | Available |
|
|
113
|
+
| [safari](https://github.com/chronicle-app/chronicle-safari) | Browser history from local sqlite db | Available |
|
|
114
|
+
| github | Github user and repo activity | In progress |
|
|
115
|
+
| chrome | Browser history from local sqlite db | Needs porting |
|
|
116
|
+
| whatsapp | Messaging history (via individual chat exports) or reverse-engineered local desktop install | Unstarted |
|
|
117
|
+
| anki | Studying and card creation history | Needs porting |
|
|
118
|
+
| facebook | Messaging and history posting via data export files | Needs porting |
|
|
119
|
+
| twitter | History via API or export data files | Needs porting |
|
|
120
|
+
| foursquare | Location history via API | Needs porting |
|
|
121
|
+
| goodreads | Reading history via export csv (RIP goodreads API) | Needs porting |
|
|
122
|
+
| lastfm | Listening history via API | Needs porting |
|
|
123
|
+
| images | Process image files | Needs porting |
|
|
124
|
+
| arc | Location history from synced icloud backup files | Needs porting |
|
|
125
|
+
| firefox | Browser history from local sqlite db | Needs porting |
|
|
126
|
+
| fitbit | Personal analytics via API | Needs porting |
|
|
127
|
+
| git | Commit history on a repo | Needs porting |
|
|
128
|
+
| google-calendar | Calendar events via API | Needs porting |
|
|
129
|
+
| instagram | Posting and messaging history via export data | Needs porting |
|
|
130
|
+
| shazam | Song tags via reverse-engineered API | Needs porting |
|
|
131
|
+
| slack | Messaging history via API | Need rethinking |
|
|
132
|
+
| strava | Activity history via API | Needs porting |
|
|
133
|
+
| things | Task activity via local sqlite db | Needs porting |
|
|
134
|
+
| bear | Note taking activity via local sqlite db | Needs porting |
|
|
135
|
+
| youtube | Video activity via takeout data and API | Needs porting |
|
|
136
|
+
|
|
137
|
+
### Writing your own connector
|
|
138
|
+
|
|
139
|
+
Additional connectors are packaged as separate ruby gems. You can view the [iMessage plugin](https://github.com/chronicle-app/chronicle-imessage) for an example.
|
|
140
|
+
|
|
141
|
+
If you want to load a custom connector without creating a gem, you can help by [completing this issue](https://github.com/chronicle-app/chronicle-etl/issues/23).
|
|
142
|
+
|
|
143
|
+
If you want to work together on a connector, please [get in touch](#get-in-touch)!
|
|
144
|
+
|
|
145
|
+
#### Sample custom Extractor class
|
|
146
|
+
```ruby
|
|
147
|
+
module Chronicle
|
|
148
|
+
module FooService
|
|
149
|
+
class FooExtractor < Chronicle::ETL::Extractor
|
|
150
|
+
register_connector do |r|
|
|
151
|
+
r.identifier = 'foo'
|
|
152
|
+
r.description = 'From foo.com'
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
setting :access_token, required: true
|
|
156
|
+
|
|
157
|
+
def prepare
|
|
158
|
+
@records = # load from somewhere
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def extract
|
|
162
|
+
@records.each do |record|
|
|
163
|
+
yield Chronicle::ETL::Extraction.new(data: row.to_h)
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
107
169
|
```
|
|
108
170
|
|
|
109
171
|
## Development
|
|
110
|
-
|
|
111
172
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
112
173
|
|
|
113
174
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
114
175
|
|
|
115
|
-
|
|
176
|
+
### Additional development commands
|
|
177
|
+
```bash
|
|
178
|
+
# run tests
|
|
179
|
+
bundle exec rake spec
|
|
180
|
+
|
|
181
|
+
# generate docs
|
|
182
|
+
bundle exec rake yard
|
|
183
|
+
|
|
184
|
+
# use Guard to run specs automatically
|
|
185
|
+
bundle exec guard
|
|
186
|
+
```
|
|
116
187
|
|
|
188
|
+
## Get in touch
|
|
189
|
+
- [@hyfen](https://twitter.com/hyfen) on Twitter
|
|
190
|
+
- [@hyfen](https://github.com/hyfen) on Github
|
|
191
|
+
- Email: andrew@hyfen.net
|
|
192
|
+
|
|
193
|
+
## Contributing
|
|
117
194
|
Bug reports and pull requests are welcome on GitHub at https://github.com/chronicle-app/chronicle-etl. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
|
118
195
|
|
|
119
196
|
## License
|
|
120
|
-
|
|
121
197
|
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
|
122
198
|
|
|
123
199
|
## Code of Conduct
|
|
124
|
-
|
|
125
|
-
Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
|
|
200
|
+
Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/main/CODE_OF_CONDUCT.md).
|
data/Rakefile
CHANGED
data/chronicle-etl.gemspec
CHANGED
|
@@ -35,22 +35,30 @@ Gem::Specification.new do |spec|
|
|
|
35
35
|
spec.bindir = "exe"
|
|
36
36
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
37
37
|
spec.require_paths = ["lib"]
|
|
38
|
+
spec.required_ruby_version = ">= 2.7"
|
|
38
39
|
|
|
39
|
-
spec.add_dependency "activesupport"
|
|
40
|
+
spec.add_dependency "activesupport", "~> 7.0"
|
|
40
41
|
spec.add_dependency "chronic_duration", "~> 0.10.6"
|
|
41
42
|
spec.add_dependency "colorize", "~> 0.8.1"
|
|
42
43
|
spec.add_dependency "marcel", "~> 1.0.2"
|
|
43
44
|
spec.add_dependency "mini_exiftool", "~> 2.10"
|
|
44
45
|
spec.add_dependency "nokogiri", "~> 1.13"
|
|
45
|
-
spec.add_dependency "runcom", "
|
|
46
|
+
spec.add_dependency "runcom", ">= 6.0"
|
|
46
47
|
spec.add_dependency "sequel", "~> 5.35"
|
|
47
48
|
spec.add_dependency "sqlite3", "~> 1.4"
|
|
48
|
-
spec.add_dependency "thor", "~>
|
|
49
|
+
spec.add_dependency "thor", "~> 1.2"
|
|
50
|
+
spec.add_dependency "thor-hollaback", "~> 0.2"
|
|
49
51
|
spec.add_dependency "tty-progressbar", "~> 0.17"
|
|
52
|
+
spec.add_dependency "tty-spinner"
|
|
50
53
|
spec.add_dependency "tty-table", "~> 0.11"
|
|
54
|
+
spec.add_dependency "tty-prompt", "~> 0.23"
|
|
51
55
|
|
|
52
56
|
spec.add_development_dependency "bundler", "~> 2.1"
|
|
53
57
|
spec.add_development_dependency "pry-byebug", "~> 3.9"
|
|
54
58
|
spec.add_development_dependency "rake", "~> 13.0"
|
|
55
59
|
spec.add_development_dependency "rspec", "~> 3.9"
|
|
60
|
+
spec.add_development_dependency "simplecov", "~> 0.21"
|
|
61
|
+
spec.add_development_dependency "guard-rspec", "~> 4.7.3"
|
|
62
|
+
spec.add_development_dependency "yard", "~> 0.9.7"
|
|
63
|
+
spec.add_development_dependency "rubocop", "~> 1.25.1"
|
|
56
64
|
end
|
data/exe/chronicle-etl
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Chronicle
|
|
2
4
|
module ETL
|
|
3
5
|
module CLI
|
|
@@ -6,11 +8,6 @@ module Chronicle
|
|
|
6
8
|
default_task 'list'
|
|
7
9
|
namespace :connectors
|
|
8
10
|
|
|
9
|
-
desc "install NAME", "Installs connector NAME"
|
|
10
|
-
def install(name)
|
|
11
|
-
Chronicle::ETL::Registry.install_connector(name)
|
|
12
|
-
end
|
|
13
|
-
|
|
14
11
|
desc "list", "Lists available connectors"
|
|
15
12
|
# Display all available connectors that chronicle-etl has access to
|
|
16
13
|
def list
|
|
@@ -38,6 +35,38 @@ module Chronicle
|
|
|
38
35
|
table = TTY::Table.new(headers, connector_info.map(&:values))
|
|
39
36
|
puts table.render(indent: 0, padding: [0, 2])
|
|
40
37
|
end
|
|
38
|
+
|
|
39
|
+
desc "show PHASE IDENTIFIER", "Show information about a connector"
|
|
40
|
+
def show(phase, identifier)
|
|
41
|
+
unless ['extractor', 'transformer', 'loader'].include?(phase)
|
|
42
|
+
Chronicle::ETL::Logger.fatal("Phase argument must be one of: [extractor, transformer, loader]")
|
|
43
|
+
exit 1
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
begin
|
|
47
|
+
connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
|
|
48
|
+
rescue Chronicle::ETL::ConnectorNotAvailableError, Chronicle::ETL::PluginError
|
|
49
|
+
Chronicle::ETL::Logger.fatal("Could not find #{phase} #{identifier}")
|
|
50
|
+
exit 1
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
puts connector.klass.to_s.bold
|
|
54
|
+
puts " #{connector.descriptive_phrase}"
|
|
55
|
+
puts
|
|
56
|
+
puts "Settings:"
|
|
57
|
+
|
|
58
|
+
headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
|
|
59
|
+
|
|
60
|
+
settings = connector.klass.settings.map do |name, setting|
|
|
61
|
+
[
|
|
62
|
+
name,
|
|
63
|
+
setting.default,
|
|
64
|
+
setting.required ? 'yes' : 'no'
|
|
65
|
+
]
|
|
66
|
+
end
|
|
67
|
+
table = TTY::Table.new(headers, settings)
|
|
68
|
+
puts table.render(indent: 0, padding: [0, 2])
|
|
69
|
+
end
|
|
41
70
|
end
|
|
42
71
|
end
|
|
43
72
|
end
|