chronicle-etl 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +35 -0
- data/.rubocop.yml +28 -1
- data/Guardfile +7 -0
- data/README.md +149 -85
- data/Rakefile +4 -2
- data/chronicle-etl.gemspec +10 -5
- data/exe/chronicle-etl +1 -1
- data/lib/chronicle/etl/cli/connectors.rb +34 -0
- data/lib/chronicle/etl/cli/jobs.rb +44 -12
- data/lib/chronicle/etl/cli/main.rb +13 -19
- data/lib/chronicle/etl/cli/subcommand_base.rb +2 -2
- data/lib/chronicle/etl/cli.rb +7 -0
- data/lib/chronicle/etl/configurable.rb +158 -0
- data/lib/chronicle/etl/exceptions.rb +7 -1
- data/lib/chronicle/etl/extractors/csv_extractor.rb +24 -23
- data/lib/chronicle/etl/extractors/extractor.rb +23 -19
- data/lib/chronicle/etl/extractors/file_extractor.rb +34 -11
- data/lib/chronicle/etl/extractors/helpers/input_reader.rb +76 -0
- data/lib/chronicle/etl/extractors/json_extractor.rb +19 -18
- data/lib/chronicle/etl/job.rb +1 -1
- data/lib/chronicle/etl/job_definition.rb +1 -1
- data/lib/chronicle/etl/loaders/csv_loader.rb +1 -1
- data/lib/chronicle/etl/loaders/json_loader.rb +44 -0
- data/lib/chronicle/etl/loaders/loader.rb +5 -2
- data/lib/chronicle/etl/loaders/rest_loader.rb +5 -5
- data/lib/chronicle/etl/loaders/table_loader.rb +21 -24
- data/lib/chronicle/etl/logger.rb +1 -0
- data/lib/chronicle/etl/models/base.rb +3 -0
- data/lib/chronicle/etl/models/entity.rb +8 -2
- data/lib/chronicle/etl/models/raw.rb +26 -0
- data/lib/chronicle/etl/registry/connector_registration.rb +1 -0
- data/lib/chronicle/etl/runner.rb +6 -4
- data/lib/chronicle/etl/serializers/jsonapi_serializer.rb +6 -0
- data/lib/chronicle/etl/serializers/raw_serializer.rb +10 -0
- data/lib/chronicle/etl/serializers/serializer.rb +2 -1
- data/lib/chronicle/etl/transformers/image_file_transformer.rb +22 -28
- data/lib/chronicle/etl/transformers/null_transformer.rb +1 -1
- data/lib/chronicle/etl/transformers/transformer.rb +3 -2
- data/lib/chronicle/etl/version.rb +1 -1
- data/lib/chronicle/etl.rb +12 -4
- metadata +80 -19
- data/.ruby-version +0 -1
- data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb +0 -104
- data/lib/chronicle/etl/loaders/stdout_loader.rb +0 -14
- data/lib/chronicle/etl/models/generic.rb +0 -23
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8a267de435b41b579e36128b7392729ef499eb37f05fabaead7811f089938ddb
|
|
4
|
+
data.tar.gz: d4af2f62f3f5de926bdfbb0e3d6dbe2c952ec286c07317af4dca8d98f665d6da
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c78080cce008340f0b2795be46da2b5eb6562b2bffd97728150960343870f2bea4699e4efa07905710dd0e2eba7aaa1e803d8c0f727196f5d9d655b28a04f02e
|
|
7
|
+
data.tar.gz: cae3a3ffb6527f5c0b3ff89c75dc98d9cd66157ee6230c9db797f4683f90e2146daadf291108e55d3090d0120d3c9e25135cb21c4e9078bcaf4d1edf2172c930
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
2
|
+
# They are provided by a third-party and are governed by
|
|
3
|
+
# separate terms of service, privacy policy, and support
|
|
4
|
+
# documentation.
|
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
|
7
|
+
|
|
8
|
+
name: Ruby
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [ main ]
|
|
13
|
+
pull_request:
|
|
14
|
+
branches: [ main ]
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
|
|
19
|
+
runs-on: ubuntu-latest
|
|
20
|
+
strategy:
|
|
21
|
+
matrix:
|
|
22
|
+
ruby-version: ['2.7', '3.0']
|
|
23
|
+
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v2
|
|
26
|
+
- name: Set up Ruby
|
|
27
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
|
28
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
|
29
|
+
# uses: ruby/setup-ruby@v1
|
|
30
|
+
uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
|
|
31
|
+
with:
|
|
32
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
33
|
+
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
|
|
34
|
+
- name: Run tests
|
|
35
|
+
run: bundle exec rake
|
data/.rubocop.yml
CHANGED
|
@@ -1,11 +1,38 @@
|
|
|
1
1
|
AllCops:
|
|
2
2
|
EnabledByDefault: true
|
|
3
|
+
TargetRubyVersion: 2.7
|
|
4
|
+
|
|
5
|
+
Style/FrozenStringLiteralComment:
|
|
6
|
+
SafeAutoCorrect: true
|
|
3
7
|
|
|
4
8
|
Style/StringLiterals:
|
|
5
9
|
Enabled: false
|
|
6
10
|
|
|
11
|
+
Layout/MultilineAssignmentLayout:
|
|
12
|
+
Enabled: false
|
|
13
|
+
|
|
14
|
+
Layout/RedundantLineBreak:
|
|
15
|
+
Enabled: false
|
|
16
|
+
|
|
7
17
|
Style/MethodCallWithArgsParentheses:
|
|
8
18
|
Enabled: false
|
|
9
19
|
|
|
20
|
+
Style/MethodCalledOnDoEndBlock:
|
|
21
|
+
Exclude:
|
|
22
|
+
- 'spec/**/*'
|
|
23
|
+
|
|
24
|
+
Style/OpenStructUse:
|
|
25
|
+
Enabled: false
|
|
26
|
+
|
|
27
|
+
Style/Copyright:
|
|
28
|
+
Enabled: false
|
|
29
|
+
|
|
30
|
+
Style/SymbolArray:
|
|
31
|
+
EnforcedStyle: brackets
|
|
32
|
+
|
|
33
|
+
Style/WordArray:
|
|
34
|
+
EnforcedStyle: brackets
|
|
35
|
+
|
|
10
36
|
Lint/ConstantResolution:
|
|
11
|
-
Enabled: false
|
|
37
|
+
Enabled: false
|
|
38
|
+
|
data/Guardfile
ADDED
data/README.md
CHANGED
|
@@ -1,125 +1,189 @@
|
|
|
1
|
-
|
|
1
|
+
## A CLI toolkit for extracting and working with your digital history
|
|
2
2
|
|
|
3
|
-
[](https://badge.fury.io/rb/chronicle-etl)
|
|
3
|
+
[](https://badge.fury.io/rb/chronicle-etl) [](https://github.com/chronicle-app/chronicle-etl/actions/workflows/ruby.yml)
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Are you trying to archive your digital history or incorporate it into your own projects? You’ve probably discovered how frustrating it is to get machine-readable access to your own data. While [building a memex](https://hyfen.net/memex/), I learned first-hand what great efforts must be made before you can begin using the data in interesting ways.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
If you don’t want to spend all your time writing scrapers, reverse-engineering APIs, or parsing takeout data, this project is for you! (*If you do enjoy these things, please see the [open issues](https://github.com/chronicle-app/chronicle-etl/issues).*)
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
`chronicle-etl` is a CLI tool that gives you the ability to easily access your personal data. It uses the ETL pattern to **extract** it from a source (e.g. your local browser history, a directory of images, goodreads.com reading history), **transform** it (into a given schema), and **load** it to a source (e.g. a CSV file, JSON, external API).
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
## What does `chronicle-etl` give you?
|
|
12
|
+
* **CLI tool for working with personal data**. You can monitor progress of exports, manipulate the output, set up recurring jobs, manage credentials, and more.
|
|
13
|
+
* **Plugins for many third-party providers**. A plugin system allows you to access data from third-party providers and hook it into the shared CLI infrastructure.
|
|
14
|
+
* **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
```sh
|
|
18
|
+
# Install chronicle-etl
|
|
19
|
+
gem install chronicle-etl
|
|
13
20
|
```
|
|
14
21
|
|
|
15
|
-
|
|
22
|
+
After installation, the `chronicle-etl` command will be available in your shell. Homebrew support [is coming soon](https://github.com/chronicle-app/chronicle-etl/issues/13).
|
|
16
23
|
|
|
17
|
-
|
|
24
|
+
## Basic usage and running jobs
|
|
18
25
|
|
|
19
|
-
```
|
|
20
|
-
#
|
|
21
|
-
$ chronicle-etl
|
|
26
|
+
```sh
|
|
27
|
+
# Display help
|
|
28
|
+
$ chronicle-etl help
|
|
22
29
|
|
|
23
|
-
#
|
|
24
|
-
$ chronicle-etl
|
|
30
|
+
# Basic job usage
|
|
31
|
+
$ chronicle-etl --extractor NAME --transformer NAME --loader NAME
|
|
32
|
+
|
|
33
|
+
# Read test.csv and display it to stdout as a table
|
|
34
|
+
$ chronicle-etl --extractor csv --input ./data.csv --loader table
|
|
25
35
|
```
|
|
26
36
|
|
|
27
|
-
|
|
37
|
+
### Common options
|
|
38
|
+
```sh
|
|
39
|
+
Options:
|
|
40
|
+
-j, [--name=NAME] # Job configuration name
|
|
41
|
+
-e, [--extractor=EXTRACTOR-NAME] # Extractor class. Default: stdin
|
|
42
|
+
[--extractor-opts=key:value] # Extractor options
|
|
43
|
+
-t, [--transformer=TRANFORMER-NAME] # Transformer class. Default: null
|
|
44
|
+
[--transformer-opts=key:value] # Transformer options
|
|
45
|
+
-l, [--loader=LOADER-NAME] # Loader class. Default: stdout
|
|
46
|
+
[--loader-opts=key:value] # Loader options
|
|
47
|
+
-i, [--input=FILENAME] # Input filename or directory
|
|
48
|
+
[--since=DATE] # Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options
|
|
49
|
+
[--until=DATE] # Load records UNTIL this date
|
|
50
|
+
[--limit=N] # Only extract the first LIMIT records
|
|
51
|
+
-o, [--output=OUTPUT] # Output filename
|
|
52
|
+
[--fields=field1 field2 ...] # Output only these fields
|
|
53
|
+
[--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
|
|
54
|
+
# Default: info
|
|
55
|
+
-v, [--verbose], [--no-verbose] # Set log level to verbose
|
|
56
|
+
[--silent], [--no-silent] # Silence all output
|
|
57
|
+
```
|
|
28
58
|
|
|
59
|
+
## Connectors
|
|
29
60
|
Connectors are available to read, process, and load data from different formats or external services.
|
|
30
61
|
|
|
31
|
-
```
|
|
62
|
+
```sh
|
|
32
63
|
# List all available connectors
|
|
33
64
|
$ chronicle-etl connectors:list
|
|
34
|
-
|
|
35
|
-
# Install a connector
|
|
36
|
-
$ chronicle-etl connectors:install imessage
|
|
37
65
|
```
|
|
38
66
|
|
|
39
|
-
Built
|
|
40
|
-
|
|
41
|
-
### Extractors
|
|
42
|
-
- `stdin` - (default) Load records from line-separated stdin
|
|
43
|
-
- `csv`
|
|
44
|
-
- `file` - load from a single file or directory (with a glob pattern)
|
|
45
|
-
|
|
46
|
-
### Transformers
|
|
47
|
-
- `null` - (default) Don't do anything
|
|
48
|
-
|
|
49
|
-
### Loaders
|
|
50
|
-
- `stdout` - (default) output records to stdout serialized as JSON
|
|
51
|
-
- `csv` - Load records to a csv file
|
|
52
|
-
- `rest` - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
|
|
53
|
-
- `table` - Output an ascii table of records. Useful for debugging.
|
|
54
|
-
|
|
55
|
-
### Provider-specific importers
|
|
56
|
-
|
|
57
|
-
In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
|
|
67
|
+
### Built-in Connectors
|
|
68
|
+
`chronicle-etl` comes with several built-in connectors for common formats and sources.
|
|
58
69
|
|
|
59
|
-
|
|
60
|
-
- [
|
|
61
|
-
- [
|
|
70
|
+
#### Extractors
|
|
71
|
+
- [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records from CSV files or stdin
|
|
72
|
+
- [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/json_extractor.rb) - Load JSON (either [line-separated objects](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON) or one object)
|
|
73
|
+
- [`file`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/file_extractor.rb) - load from a single file or directory (with a glob pattern)
|
|
62
74
|
|
|
63
|
-
|
|
75
|
+
#### Transformers
|
|
76
|
+
- [`null`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/transformers/null_transformer.rb) - (default) Don’t do anything and pass on raw extraction data
|
|
64
77
|
|
|
65
|
-
|
|
78
|
+
#### Loaders
|
|
79
|
+
- [`table`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/table_loader.rb) - (default) Output an ascii table of records. Useful for exploring data.
|
|
80
|
+
- [`csv`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/extractors/csv_extractor.rb) - Load records to CSV
|
|
81
|
+
- [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/json_loader.rb) - Load records serialized as JSON
|
|
82
|
+
- [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
|
|
66
83
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
## Full commands
|
|
70
|
-
|
|
71
|
-
```
|
|
72
|
-
$ chronicle-etl help
|
|
73
|
-
|
|
74
|
-
ALL COMMANDS
|
|
75
|
-
help # This help menu
|
|
76
|
-
connectors help [COMMAND] # Describe subcommands or one specific subcommand
|
|
77
|
-
connectors:install NAME # Installs connector NAME
|
|
78
|
-
connectors:list # Lists available connectors
|
|
79
|
-
jobs help [COMMAND] # Describe subcommands or one specific subcommand
|
|
80
|
-
jobs:create # Create a job
|
|
81
|
-
jobs:list # List all available jobs
|
|
82
|
-
jobs:run # Start a job
|
|
83
|
-
jobs:show # Show details about a job
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
### Running a job
|
|
84
|
+
### Plugins
|
|
85
|
+
Plugins provide access to data from third-party platforms, services, or formats.
|
|
87
86
|
|
|
87
|
+
```bash
|
|
88
|
+
# Install a plugin
|
|
89
|
+
$ chronicle-etl connectors:install NAME
|
|
88
90
|
```
|
|
89
|
-
Usage:
|
|
90
|
-
chronicle-etl jobs:run
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
92
|
+
A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and they’re being ported over to the Chronicle system. This table shows what’s available now and what’s coming. Rows are sorted in very rough order of priority.
|
|
93
|
+
|
|
94
|
+
If you want to work together on a connector, please [get in touch](#get-in-touch)!
|
|
95
|
+
|
|
96
|
+
| Name | Description | Availability |
|
|
97
|
+
|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
|
|
98
|
+
| [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments | Available |
|
|
99
|
+
| [shell](https://github.com/chronicle-app/chronicle-shell) | Shell command history | Available (zsh support pending) |
|
|
100
|
+
| [email](https://github.com/chronicle-app/chronicle-email) | Emails and attachments from IMAP or .mbox files | Available (imap support pending) |
|
|
101
|
+
| [pinboard](https://github.com/chronicle-app/chronicle-email) | Bookmarks and tags | Available |
|
|
102
|
+
| github | Github user and repo activity | In progress |
|
|
103
|
+
| safari | Browser history from local sqlite db | Needs porting |
|
|
104
|
+
| chrome | Browser history from local sqlite db | Needs porting |
|
|
105
|
+
| whatsapp | Messaging history (via individual chat exports) or reverse-engineered local desktop install | Unstarted |
|
|
106
|
+
| anki | Studying and card creation history | Needs porting |
|
|
107
|
+
| facebook | Messaging and history posting via data export files | Needs porting |
|
|
108
|
+
| twitter | History via API or export data files | Needs porting |
|
|
109
|
+
| foursquare | Location history via API | Needs porting |
|
|
110
|
+
| goodreads | Reading history via export csv (RIP goodreads API) | Needs porting |
|
|
111
|
+
| lastfm | Listening history via API | Needs porting |
|
|
112
|
+
| images | Process image files | Needs porting |
|
|
113
|
+
| arc | Location history from synced icloud backup files | Needs porting |
|
|
114
|
+
| firefox | Browser history from local sqlite db | Needs porting |
|
|
115
|
+
| fitbit | Personal analytics via API | Needs porting |
|
|
116
|
+
| git | Commit history on a repo | Needs porting |
|
|
117
|
+
| google-calendar | Calendar events via API | Needs porting |
|
|
118
|
+
| instagram | Posting and messaging history via export data | Needs porting |
|
|
119
|
+
| shazam | Song tags via reverse-engineered API | Needs porting |
|
|
120
|
+
| slack | Messaging history via API | Need rethinking |
|
|
121
|
+
| strava | Activity history via API | Needs porting |
|
|
122
|
+
| things | Task activity via local sqlite db | Needs porting |
|
|
123
|
+
| bear | Note taking activity via local sqlite db | Needs porting |
|
|
124
|
+
| youtube | Video activity via takeout data and API | Needs porting |
|
|
125
|
+
|
|
126
|
+
### Writing your own connector
|
|
127
|
+
|
|
128
|
+
Additional connectors are packaged as separate ruby gems. You can view the [iMessage plugin](https://github.com/chronicle-app/chronicle-imessage) for an example.
|
|
129
|
+
|
|
130
|
+
If you want to load a custom connector without creating a gem, you can help by [completing this issue](https://github.com/chronicle-app/chronicle-etl/issues/23).
|
|
131
|
+
|
|
132
|
+
If you want to work together on a connector, please [get in touch](#get-in-touch)!
|
|
133
|
+
|
|
134
|
+
#### Sample custom Extractor class
|
|
135
|
+
```ruby
|
|
136
|
+
module Chronicle
|
|
137
|
+
module FooService
|
|
138
|
+
class FooExtractor < Chronicle::ETL::Extractor
|
|
139
|
+
register_connector do |r|
|
|
140
|
+
r.identifier = 'foo'
|
|
141
|
+
r.description = 'From foo.com'
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
setting :access_token, required: true
|
|
145
|
+
|
|
146
|
+
def prepare
|
|
147
|
+
@records = # load from somewhere
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def extract
|
|
151
|
+
@records.each do |record|
|
|
152
|
+
yield Chronicle::ETL::Extraction.new(data: row.to_h)
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
107
158
|
```
|
|
108
159
|
|
|
109
160
|
## Development
|
|
110
|
-
|
|
111
161
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
112
162
|
|
|
113
163
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
114
164
|
|
|
115
|
-
|
|
165
|
+
### Additional development commands
|
|
166
|
+
```bash
|
|
167
|
+
# run tests
|
|
168
|
+
bundle exec rake spec
|
|
169
|
+
|
|
170
|
+
# generate docs
|
|
171
|
+
bundle exec rake yard
|
|
172
|
+
|
|
173
|
+
# use Guard to run specs automatically
|
|
174
|
+
bundle exec guard
|
|
175
|
+
```
|
|
116
176
|
|
|
177
|
+
## Get in touch
|
|
178
|
+
- [@hyfen](https://twitter.com/hyfen) on Twitter
|
|
179
|
+
- [@hyfen](https://github.com/hyfen) on Github
|
|
180
|
+
- Email: andrew@hyfen.net
|
|
181
|
+
|
|
182
|
+
## Contributing
|
|
117
183
|
Bug reports and pull requests are welcome on GitHub at https://github.com/chronicle-app/chronicle-etl. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
|
118
184
|
|
|
119
185
|
## License
|
|
120
|
-
|
|
121
186
|
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
|
122
187
|
|
|
123
188
|
## Code of Conduct
|
|
124
|
-
|
|
125
|
-
Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
|
|
189
|
+
Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-etl/blob/master/CODE_OF_CONDUCT.md).
|
data/Rakefile
CHANGED
data/chronicle-etl.gemspec
CHANGED
|
@@ -17,11 +17,11 @@ Gem::Specification.new do |spec|
|
|
|
17
17
|
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
|
18
18
|
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
|
19
19
|
if spec.respond_to?(:metadata)
|
|
20
|
-
|
|
20
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
|
21
21
|
|
|
22
22
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
23
23
|
spec.metadata["source_code_uri"] = "https://github.com/chronicle-app/chronicle-etl"
|
|
24
|
-
spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-etl/
|
|
24
|
+
spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-etl/releases"
|
|
25
25
|
else
|
|
26
26
|
raise "RubyGems 2.0 or newer is required to protect against " \
|
|
27
27
|
"public gem pushes."
|
|
@@ -35,17 +35,18 @@ Gem::Specification.new do |spec|
|
|
|
35
35
|
spec.bindir = "exe"
|
|
36
36
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
37
37
|
spec.require_paths = ["lib"]
|
|
38
|
+
spec.required_ruby_version = ">= 2.7"
|
|
38
39
|
|
|
39
|
-
spec.add_dependency "activesupport"
|
|
40
|
+
spec.add_dependency "activesupport", "~> 7.0"
|
|
40
41
|
spec.add_dependency "chronic_duration", "~> 0.10.6"
|
|
41
42
|
spec.add_dependency "colorize", "~> 0.8.1"
|
|
42
43
|
spec.add_dependency "marcel", "~> 1.0.2"
|
|
43
44
|
spec.add_dependency "mini_exiftool", "~> 2.10"
|
|
44
45
|
spec.add_dependency "nokogiri", "~> 1.13"
|
|
45
|
-
spec.add_dependency "runcom", "
|
|
46
|
+
spec.add_dependency "runcom", ">= 6.0"
|
|
46
47
|
spec.add_dependency "sequel", "~> 5.35"
|
|
47
48
|
spec.add_dependency "sqlite3", "~> 1.4"
|
|
48
|
-
spec.add_dependency "thor", "~>
|
|
49
|
+
spec.add_dependency "thor", "~> 1.2"
|
|
49
50
|
spec.add_dependency "tty-progressbar", "~> 0.17"
|
|
50
51
|
spec.add_dependency "tty-table", "~> 0.11"
|
|
51
52
|
|
|
@@ -53,4 +54,8 @@ Gem::Specification.new do |spec|
|
|
|
53
54
|
spec.add_development_dependency "pry-byebug", "~> 3.9"
|
|
54
55
|
spec.add_development_dependency "rake", "~> 13.0"
|
|
55
56
|
spec.add_development_dependency "rspec", "~> 3.9"
|
|
57
|
+
spec.add_development_dependency "simplecov", "~> 0.21"
|
|
58
|
+
spec.add_development_dependency "guard-rspec", "~> 4.7.3"
|
|
59
|
+
spec.add_development_dependency "yard", "~> 0.9.7"
|
|
60
|
+
spec.add_development_dependency "rubocop", "~> 1.25.1"
|
|
56
61
|
end
|
data/exe/chronicle-etl
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Chronicle
|
|
2
4
|
module ETL
|
|
3
5
|
module CLI
|
|
@@ -38,6 +40,38 @@ module Chronicle
|
|
|
38
40
|
table = TTY::Table.new(headers, connector_info.map(&:values))
|
|
39
41
|
puts table.render(indent: 0, padding: [0, 2])
|
|
40
42
|
end
|
|
43
|
+
|
|
44
|
+
desc "show PHASE IDENTIFIER", "Show information about a connector"
|
|
45
|
+
def show(phase, identifier)
|
|
46
|
+
unless ['extractor', 'transformer', 'loader'].include?(phase)
|
|
47
|
+
puts "phase argument must be one of: [extractor, transformer, loader]"
|
|
48
|
+
return
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
begin
|
|
52
|
+
connector = Chronicle::ETL::Registry.find_by_phase_and_identifier(phase.to_sym, identifier)
|
|
53
|
+
rescue Chronicle::ETL::ConnectorNotAvailableError
|
|
54
|
+
puts "Could not find #{phase} #{identifier}"
|
|
55
|
+
return
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
puts connector.klass.to_s.bold
|
|
59
|
+
puts " #{connector.descriptive_phrase}"
|
|
60
|
+
puts
|
|
61
|
+
puts "OPTIONS"
|
|
62
|
+
|
|
63
|
+
headers = ['name', 'default', 'required'].map{ |h| h.to_s.upcase.bold }
|
|
64
|
+
|
|
65
|
+
settings = connector.klass.settings.map do |name, setting|
|
|
66
|
+
[
|
|
67
|
+
name,
|
|
68
|
+
setting.default,
|
|
69
|
+
setting.required ? 'yes' : 'no'
|
|
70
|
+
]
|
|
71
|
+
end
|
|
72
|
+
table = TTY::Table.new(headers, settings)
|
|
73
|
+
puts table.render(indent: 0, padding: [0, 2])
|
|
74
|
+
end
|
|
41
75
|
end
|
|
42
76
|
end
|
|
43
77
|
end
|
|
@@ -1,21 +1,37 @@
|
|
|
1
1
|
require 'pp'
|
|
2
|
+
|
|
2
3
|
module Chronicle
|
|
3
4
|
module ETL
|
|
4
5
|
module CLI
|
|
5
6
|
# CLI commands for working with ETL jobs
|
|
6
7
|
class Jobs < SubcommandBase
|
|
7
8
|
default_task "start"
|
|
8
|
-
namespace :jobs
|
|
9
|
+
namespace :jobs
|
|
10
|
+
|
|
11
|
+
class_option :name, aliases: '-j', desc: 'Job configuration name'
|
|
9
12
|
|
|
10
|
-
class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: '
|
|
13
|
+
class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'NAME'
|
|
11
14
|
class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
|
|
12
|
-
class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: '
|
|
15
|
+
class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'NAME'
|
|
13
16
|
class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
|
|
14
|
-
class_option :loader, aliases: '-l', desc: 'Loader class. Default:
|
|
17
|
+
class_option :loader, aliases: '-l', desc: 'Loader class. Default: table', banner: 'NAME'
|
|
15
18
|
class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
|
|
16
|
-
class_option :name, aliases: '-j', desc: 'Job configuration name'
|
|
17
19
|
|
|
18
|
-
|
|
20
|
+
# This is an array to deal with shell globbing
|
|
21
|
+
class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
|
|
22
|
+
class_option :since, desc: "Load records SINCE this date", banner: 'DATE'
|
|
23
|
+
class_option :until, desc: "Load records UNTIL this date", banner: 'DATE'
|
|
24
|
+
class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
|
|
25
|
+
|
|
26
|
+
class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
|
|
27
|
+
class_option :fields, desc: 'Output only these fields', type: 'array', banner: 'field1 field2 ...'
|
|
28
|
+
|
|
29
|
+
class_option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
|
|
30
|
+
class_option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
|
|
31
|
+
class_option :silent, desc: 'Silence all output', type: :boolean
|
|
32
|
+
|
|
33
|
+
# Thor doesn't like `run` as a command name
|
|
34
|
+
map run: :start
|
|
19
35
|
desc "run", "Start a job"
|
|
20
36
|
option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
|
|
21
37
|
option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
|
|
@@ -69,7 +85,7 @@ LONG_DESC
|
|
|
69
85
|
[job, extractor, transformer, loader]
|
|
70
86
|
end
|
|
71
87
|
|
|
72
|
-
headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
|
|
88
|
+
headers = ['name', 'extractor', 'transformer', 'loader'].map { |h| h.upcase.bold }
|
|
73
89
|
|
|
74
90
|
table = TTY::Table.new(headers, job_details)
|
|
75
91
|
puts table.render(indent: 0, padding: [0, 2])
|
|
@@ -78,7 +94,9 @@ LONG_DESC
|
|
|
78
94
|
private
|
|
79
95
|
|
|
80
96
|
def setup_log_level
|
|
81
|
-
if options[:
|
|
97
|
+
if options[:silent]
|
|
98
|
+
Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::SILENT
|
|
99
|
+
elsif options[:verbose]
|
|
82
100
|
Chronicle::ETL::Logger.log_level = Chronicle::ETL::Logger::DEBUG
|
|
83
101
|
elsif options[:log_level]
|
|
84
102
|
level = Chronicle::ETL::Logger.const_get(options[:log_level].upcase)
|
|
@@ -90,7 +108,7 @@ LONG_DESC
|
|
|
90
108
|
def build_job_definition(options)
|
|
91
109
|
definition = Chronicle::ETL::JobDefinition.new
|
|
92
110
|
definition.add_config(load_job_config(options[:name]))
|
|
93
|
-
definition.add_config(process_flag_options(options))
|
|
111
|
+
definition.add_config(process_flag_options(options).transform_keys(&:to_sym))
|
|
94
112
|
definition
|
|
95
113
|
end
|
|
96
114
|
|
|
@@ -100,19 +118,33 @@ LONG_DESC
|
|
|
100
118
|
|
|
101
119
|
# Takes flag options and turns them into a runner config
|
|
102
120
|
def process_flag_options options
|
|
121
|
+
extractor_options = options[:'extractor-opts'].merge({
|
|
122
|
+
input: (options[:input] if options[:input].any?),
|
|
123
|
+
since: options[:since],
|
|
124
|
+
until: options[:until],
|
|
125
|
+
limit: options[:limit],
|
|
126
|
+
}.compact)
|
|
127
|
+
|
|
128
|
+
transformer_options = options[:'transformer-opts']
|
|
129
|
+
|
|
130
|
+
loader_options = options[:'loader-opts'].merge({
|
|
131
|
+
output: options[:output],
|
|
132
|
+
fields: options[:fields]
|
|
133
|
+
}.compact)
|
|
134
|
+
|
|
103
135
|
{
|
|
104
136
|
dry_run: options[:dry_run],
|
|
105
137
|
extractor: {
|
|
106
138
|
name: options[:extractor],
|
|
107
|
-
options:
|
|
139
|
+
options: extractor_options
|
|
108
140
|
}.compact,
|
|
109
141
|
transformer: {
|
|
110
142
|
name: options[:transformer],
|
|
111
|
-
options:
|
|
143
|
+
options: transformer_options
|
|
112
144
|
}.compact,
|
|
113
145
|
loader: {
|
|
114
146
|
name: options[:loader],
|
|
115
|
-
options:
|
|
147
|
+
options: loader_options
|
|
116
148
|
}.compact
|
|
117
149
|
}
|
|
118
150
|
end
|