chronicle-etl 0.4.4 → 0.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/README.md +126 -54
- data/chronicle-etl.gemspec +5 -4
- data/lib/chronicle/etl/cli/connectors.rb +2 -2
- data/lib/chronicle/etl/cli/jobs.rb +48 -23
- data/lib/chronicle/etl/cli/main.rb +3 -0
- data/lib/chronicle/etl/cli/plugins.rb +12 -2
- data/lib/chronicle/etl/cli/secrets.rb +69 -0
- data/lib/chronicle/etl/cli.rb +1 -0
- data/lib/chronicle/etl/config.rb +43 -25
- data/lib/chronicle/etl/configurable.rb +14 -5
- data/lib/chronicle/etl/exceptions.rb +3 -0
- data/lib/chronicle/etl/job_definition.rb +28 -2
- data/lib/chronicle/etl/job_logger.rb +4 -3
- data/lib/chronicle/etl/loaders/csv_loader.rb +10 -13
- data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb +36 -0
- data/lib/chronicle/etl/loaders/json_loader.rb +43 -8
- data/lib/chronicle/etl/loaders/loader.rb +1 -0
- data/lib/chronicle/etl/registry/plugin_registry.rb +15 -6
- data/lib/chronicle/etl/registry/registry.rb +10 -14
- data/lib/chronicle/etl/runner.rb +1 -1
- data/lib/chronicle/etl/secrets.rb +55 -0
- data/lib/chronicle/etl/version.rb +1 -1
- data/lib/chronicle/etl.rb +1 -0
- metadata +58 -41
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b8faa084cfe4a9f080ee5494c69b268b78bfa8f3502354e740264e6941f13daf
|
|
4
|
+
data.tar.gz: 1bf4f2751c71cadedc78a2fe3ed5b09bf86cd601a909e2fa2db0a0de8cc2c21d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ff10779b663a3321b779fb03e07249856174d96fb96e405ae906a47441c288d6a245c852525801ba250cce1125cf05c523ef4ec75fdfb4335cef9003091437ed
|
|
7
|
+
data.tar.gz: 509f6f92e95341d212c54b6b000bc54e8ba03898497191a3e5d3b14db7bff3ed625d0fee403888fbb6103c1edc14de66b215d9aa84ddb68cefcf51c0e6c74138
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
|
@@ -16,12 +16,21 @@ If you don’t want to spend all your time writing scrapers, reverse-engineering
|
|
|
16
16
|
* **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
|
|
17
17
|
|
|
18
18
|
## Installation
|
|
19
|
+
|
|
20
|
+
Using homebrew:
|
|
21
|
+
```sh
|
|
22
|
+
$ brew install chronicle-app/etl/chronicle-etl
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
Using rubygems:
|
|
19
26
|
```sh
|
|
20
|
-
|
|
21
|
-
gem install chronicle-etl
|
|
27
|
+
$ gem install chronicle-etl
|
|
22
28
|
```
|
|
23
29
|
|
|
24
|
-
|
|
30
|
+
Confirm it installed successfully:
|
|
31
|
+
```sh
|
|
32
|
+
$ chronicle-etl --version
|
|
33
|
+
```
|
|
25
34
|
|
|
26
35
|
## Basic usage and running jobs
|
|
27
36
|
|
|
@@ -34,28 +43,60 @@ $ chronicle-etl --extractor NAME --transformer NAME --loader NAME
|
|
|
34
43
|
|
|
35
44
|
# Read test.csv and display it to stdout as a table
|
|
36
45
|
$ chronicle-etl --extractor csv --input ./data.csv --loader table
|
|
46
|
+
|
|
47
|
+
# Retrieve shell commands run in the last 5 hours
|
|
48
|
+
$ chronicle-etl -e shell --since 5h
|
|
49
|
+
|
|
50
|
+
# Get email senders from an .mbox email archive file
|
|
51
|
+
$ chronicle-etl --extractor email:mbox -i sample-email-archive.mbox -t email --fields actor.slug
|
|
52
|
+
|
|
53
|
+
# Save an access token as a secret and use it in a job
|
|
54
|
+
$ chronicle-etl secrets:set pinboard access_token username:foo123
|
|
55
|
+
$ chronicle-etl secrets:list # Verify that's it's available
|
|
56
|
+
$ chronicle-etl -e pinboard --since 1mo # Used automatically based on plugin name
|
|
37
57
|
```
|
|
38
58
|
|
|
39
59
|
### Common options
|
|
40
60
|
```sh
|
|
41
61
|
Options:
|
|
42
|
-
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
[--
|
|
51
|
-
[--
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
[--
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
62
|
+
-e, [--extractor=NAME] # Extractor class. Default: stdin
|
|
63
|
+
[--extractor-opts=key:value] # Extractor options
|
|
64
|
+
-t, [--transformer=NAME] # Transformer class. Default: null
|
|
65
|
+
[--transformer-opts=key:value] # Transformer options
|
|
66
|
+
-l, [--loader=NAME] # Loader class. Default: table
|
|
67
|
+
[--loader-opts=key:value] # Loader options
|
|
68
|
+
-i, [--input=FILENAME] # Input filename or directory
|
|
69
|
+
[--since=DATE] # Load records SINCE this date (or fuzzy time duration)
|
|
70
|
+
[--until=DATE] # Load records UNTIL this date (or fuzzy time duration)
|
|
71
|
+
[--limit=N] # Only extract the first LIMIT records
|
|
72
|
+
-o, [--output=OUTPUT] # Output filename
|
|
73
|
+
[--fields=field1 field2 ...] # Output only these fields
|
|
74
|
+
[--header-row], [--no-header-row] # Output the header row of tabular output
|
|
75
|
+
|
|
76
|
+
[--log-level=LOG_LEVEL] # Log level (debug, info, warn, error, fatal)
|
|
77
|
+
# Default: info
|
|
78
|
+
-v, [--verbose], [--no-verbose] # Set log level to verbose
|
|
79
|
+
[--silent], [--no-silent] # Silence all output
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Saving jobs
|
|
83
|
+
|
|
84
|
+
You can save details about a job to a local config file (saved by default in `~/.config/chronicle/etl/jobs/job_name.yml`) to save yourself the trouble of setting the CLI flags for each run.
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
# Save a job named 'sample' to ~/.config/chronicle/etl/jobs/sample.yml
|
|
88
|
+
$ chronicle-etl jobs:save sample --extractor pinboard --since 10d
|
|
89
|
+
|
|
90
|
+
# Show details about the job
|
|
91
|
+
$ chronicle-etl jobs:show sample
|
|
92
|
+
|
|
93
|
+
# Run the job
|
|
94
|
+
$ chronicle-etl jobs:run sample
|
|
95
|
+
# Or more simply:
|
|
96
|
+
$ chronicle-etl sample
|
|
97
|
+
|
|
98
|
+
# Show all saved jobs
|
|
99
|
+
$ chronicle-etl jobs:list
|
|
59
100
|
```
|
|
60
101
|
|
|
61
102
|
## Connectors
|
|
@@ -83,58 +124,51 @@ $ chronicle-etl connectors:list
|
|
|
83
124
|
- [`json`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/json_loader.rb) - Load records serialized as JSON
|
|
84
125
|
- [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
|
|
85
126
|
|
|
86
|
-
|
|
87
|
-
Plugins provide access to data from third-party platforms, services, or formats.
|
|
127
|
+
## Chronicle Plugins
|
|
128
|
+
Plugins provide access to data from third-party platforms, services, or formats. Plugins are packaged as separate rubygems and can be installed through the CLI (which installs the Gems under the hood).
|
|
129
|
+
|
|
130
|
+
### Plugin usage
|
|
88
131
|
|
|
89
132
|
```bash
|
|
90
133
|
# Install a plugin
|
|
91
134
|
$ chronicle-etl plugins:install NAME
|
|
92
135
|
|
|
93
|
-
# Install the imessage plugin
|
|
94
|
-
$ chronicle-etl plugins:install imessage
|
|
95
|
-
|
|
96
136
|
# List installed plugins
|
|
97
137
|
$ chronicle-etl plugins:list
|
|
98
138
|
|
|
139
|
+
# Use a plugin
|
|
140
|
+
$ chronicle-etl plugins:install shell
|
|
141
|
+
$ chronicle-etl --extractor shell:history --limit 10
|
|
142
|
+
|
|
99
143
|
# Uninstall a plugin
|
|
100
144
|
$ chronicle-etl plugins:uninstall NAME
|
|
101
145
|
```
|
|
102
146
|
|
|
103
|
-
|
|
147
|
+
### Status
|
|
148
|
+
|
|
149
|
+
A few dozen importers exist [in my Memex project](https://hyfen.net/memex/) and I'm porting them over to the Chronicle system. The [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1/views/1) lets you keep track what's available and what's coming soon.
|
|
104
150
|
|
|
105
|
-
If you want to work together on a
|
|
151
|
+
If you don't see a plugin for a third-party provider or data source that you're interested in using with `chronicle-etl`, [please open an issue](https://github.com/chronicle-app/chronicle-etl/issues/new). If you want to work together on a plugin, please [get in touch](#get-in-touch)!
|
|
152
|
+
|
|
153
|
+
#### Currently available
|
|
106
154
|
|
|
107
155
|
| Name | Description | Availability |
|
|
108
156
|
|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
|
|
109
157
|
| [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments | Available |
|
|
110
|
-
| [shell](https://github.com/chronicle-app/chronicle-shell) | Shell command history | Available (zsh support
|
|
111
|
-
| [email](https://github.com/chronicle-app/chronicle-email) | Emails and attachments from IMAP or .mbox files | Available (
|
|
158
|
+
| [shell](https://github.com/chronicle-app/chronicle-shell) | Shell command history | Available (still needs zsh support) |
|
|
159
|
+
| [email](https://github.com/chronicle-app/chronicle-email) | Emails and attachments from IMAP or .mbox files | Available (still needs IMAP support) |
|
|
112
160
|
| [pinboard](https://github.com/chronicle-app/chronicle-email) | Bookmarks and tags | Available |
|
|
113
161
|
| [safari](https://github.com/chronicle-app/chronicle-safari) | Browser history from local sqlite db | Available |
|
|
114
|
-
| github
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
| arc | Location history from synced icloud backup files | Needs porting |
|
|
125
|
-
| firefox | Browser history from local sqlite db | Needs porting |
|
|
126
|
-
| fitbit | Personal analytics via API | Needs porting |
|
|
127
|
-
| git | Commit history on a repo | Needs porting |
|
|
128
|
-
| google-calendar | Calendar events via API | Needs porting |
|
|
129
|
-
| instagram | Posting and messaging history via export data | Needs porting |
|
|
130
|
-
| shazam | Song tags via reverse-engineered API | Needs porting |
|
|
131
|
-
| slack | Messaging history via API | Need rethinking |
|
|
132
|
-
| strava | Activity history via API | Needs porting |
|
|
133
|
-
| things | Task activity via local sqlite db | Needs porting |
|
|
134
|
-
| bear | Note taking activity via local sqlite db | Needs porting |
|
|
135
|
-
| youtube | Video activity via takeout data and API | Needs porting |
|
|
136
|
-
|
|
137
|
-
### Writing your own connector
|
|
162
|
+
| [github](https://github.com/chronicle-app/chronicle-github) | Github activity stream | Available |
|
|
163
|
+
|
|
164
|
+
#### Coming soon
|
|
165
|
+
|
|
166
|
+
In summary, the following **are coming soon**:
|
|
167
|
+
anki, arc, bear, chrome, facebook, firefox, fitbit, foursquare, git, github, goodreads, google-calendar, images, instagram, lastfm, shazam, slack, strava, things, twitter, whatsapp, youtube.
|
|
168
|
+
|
|
169
|
+
Please check the [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1/views/1) for details.
|
|
170
|
+
|
|
171
|
+
### Writing your own plugin
|
|
138
172
|
|
|
139
173
|
Additional connectors are packaged as separate ruby gems. You can view the [iMessage plugin](https://github.com/chronicle-app/chronicle-imessage) for an example.
|
|
140
174
|
|
|
@@ -149,7 +183,7 @@ module Chronicle
|
|
|
149
183
|
class FooExtractor < Chronicle::ETL::Extractor
|
|
150
184
|
register_connector do |r|
|
|
151
185
|
r.identifier = 'foo'
|
|
152
|
-
r.description = '
|
|
186
|
+
r.description = 'from foo.com'
|
|
153
187
|
end
|
|
154
188
|
|
|
155
189
|
setting :access_token, required: true
|
|
@@ -168,6 +202,44 @@ module Chronicle
|
|
|
168
202
|
end
|
|
169
203
|
```
|
|
170
204
|
|
|
205
|
+
## Secrets Management
|
|
206
|
+
|
|
207
|
+
If your job needs secrets such as access tokens or passwords, `chronicle-etl` has a built-in secret management system.
|
|
208
|
+
|
|
209
|
+
Secrets are organized in namespaces. Typically, you use one namespace per plugin (`pinboard` secrets for the `pinboard` plugin). When you run a job that uses the `pinboard` plugin extractor, for example, the secrets from that namespace will automatically be included in the extractor's options. To override which secrets get included, you can use do it in the connector options with `secrets: ALT-NAMESPACE`.
|
|
210
|
+
|
|
211
|
+
Under the hood, secrets are stored in `~/.config/chronicle/etl/secrets/NAMESPACE.yml` with 0600 permissions on each file.
|
|
212
|
+
|
|
213
|
+
### Using the secret manager
|
|
214
|
+
|
|
215
|
+
```sh
|
|
216
|
+
# Save a secret under the 'pinboard' namespace
|
|
217
|
+
$ chronicle-etl secrets:set pinboard access_token username:foo123
|
|
218
|
+
|
|
219
|
+
# Set a secret using stdin
|
|
220
|
+
$ echo -n "username:foo123" | chronicle-etl secrets:set pinboard access_token
|
|
221
|
+
|
|
222
|
+
# List available secretes
|
|
223
|
+
$ chronicle-etl secrets:list
|
|
224
|
+
|
|
225
|
+
# Use 'pinboard' secrets in the pinboard extractor's options (happens automatically)
|
|
226
|
+
$ chronicle-etl -e pinboard --since 1mo
|
|
227
|
+
|
|
228
|
+
# Use a custom secrets namespace
|
|
229
|
+
$ chronicle-etl secrets:set pinboard-alt access_token different-username:foo123
|
|
230
|
+
$ chronicle-etl -e pinboard --extractor-opts secrets:pinboard-alt --since 1mo
|
|
231
|
+
|
|
232
|
+
# Remove a secret
|
|
233
|
+
$ chronicle-etl secrets:unset pinboard access_token
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
## Roadmap
|
|
237
|
+
|
|
238
|
+
- Keep tackling **new plugins**. See: [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1)
|
|
239
|
+
- Add support for **incremental extractions** #37
|
|
240
|
+
- **Improve stdin extractor and shell command transformer** (#5) so that users can easily integrate their own scripts/tools into jobs
|
|
241
|
+
- **Add documentation for Chronicle Schema**. It's found throughout this project but never explained.
|
|
242
|
+
|
|
171
243
|
## Development
|
|
172
244
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
173
245
|
|
data/chronicle-etl.gemspec
CHANGED
|
@@ -43,22 +43,23 @@ Gem::Specification.new do |spec|
|
|
|
43
43
|
spec.add_dependency "marcel", "~> 1.0.2"
|
|
44
44
|
spec.add_dependency "mini_exiftool", "~> 2.10"
|
|
45
45
|
spec.add_dependency "nokogiri", "~> 1.13"
|
|
46
|
-
spec.add_dependency "runcom", ">= 6.0"
|
|
47
46
|
spec.add_dependency "sequel", "~> 5.35"
|
|
48
47
|
spec.add_dependency "sqlite3", "~> 1.4"
|
|
49
48
|
spec.add_dependency "thor", "~> 1.2"
|
|
50
49
|
spec.add_dependency "thor-hollaback", "~> 0.2"
|
|
51
50
|
spec.add_dependency "tty-progressbar", "~> 0.17"
|
|
51
|
+
spec.add_dependency "tty-prompt", "~> 0.23"
|
|
52
52
|
spec.add_dependency "tty-spinner"
|
|
53
53
|
spec.add_dependency "tty-table", "~> 0.11"
|
|
54
|
-
spec.add_dependency "
|
|
54
|
+
spec.add_dependency "xdg", ">= 4.0"
|
|
55
55
|
|
|
56
56
|
spec.add_development_dependency "bundler", "~> 2.1"
|
|
57
|
+
spec.add_development_dependency "guard-rspec", "~> 4.7.3"
|
|
58
|
+
spec.add_development_dependency "fakefs"
|
|
57
59
|
spec.add_development_dependency "pry-byebug", "~> 3.9"
|
|
58
60
|
spec.add_development_dependency "rake", "~> 13.0"
|
|
59
61
|
spec.add_development_dependency "rspec", "~> 3.9"
|
|
62
|
+
spec.add_development_dependency "rubocop", "~> 1.25.1"
|
|
60
63
|
spec.add_development_dependency "simplecov", "~> 0.21"
|
|
61
|
-
spec.add_development_dependency "guard-rspec", "~> 4.7.3"
|
|
62
64
|
spec.add_development_dependency "yard", "~> 0.9.7"
|
|
63
|
-
spec.add_development_dependency "rubocop", "~> 1.25.1"
|
|
64
65
|
end
|
|
@@ -4,6 +4,8 @@ module Chronicle
|
|
|
4
4
|
module ETL
|
|
5
5
|
module CLI
|
|
6
6
|
# CLI commands for working with ETL connectors
|
|
7
|
+
#
|
|
8
|
+
# @todo make this work with new plugin system (i.e. no loading of all plugins)
|
|
7
9
|
class Connectors < SubcommandBase
|
|
8
10
|
default_task 'list'
|
|
9
11
|
namespace :connectors
|
|
@@ -11,8 +13,6 @@ module Chronicle
|
|
|
11
13
|
desc "list", "Lists available connectors"
|
|
12
14
|
# Display all available connectors that chronicle-etl has access to
|
|
13
15
|
def list
|
|
14
|
-
Chronicle::ETL::Registry.load_all!
|
|
15
|
-
|
|
16
16
|
connector_info = Chronicle::ETL::Registry.connectors.map do |connector_registration|
|
|
17
17
|
{
|
|
18
18
|
identifier: connector_registration.identifier,
|
|
@@ -9,8 +9,6 @@ module Chronicle
|
|
|
9
9
|
default_task "start"
|
|
10
10
|
namespace :jobs
|
|
11
11
|
|
|
12
|
-
class_option :name, aliases: '-j', desc: 'Job configuration name'
|
|
13
|
-
|
|
14
12
|
class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'NAME'
|
|
15
13
|
class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
|
|
16
14
|
class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'NAME'
|
|
@@ -20,8 +18,8 @@ module Chronicle
|
|
|
20
18
|
|
|
21
19
|
# This is an array to deal with shell globbing
|
|
22
20
|
class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
|
|
23
|
-
class_option :since, desc: "Load records SINCE this date", banner: 'DATE'
|
|
24
|
-
class_option :until, desc: "Load records UNTIL this date", banner: 'DATE'
|
|
21
|
+
class_option :since, desc: "Load records SINCE this date (or fuzzy time duration)", banner: 'DATE'
|
|
22
|
+
class_option :until, desc: "Load records UNTIL this date (or fuzzy time duration)", banner: 'DATE'
|
|
25
23
|
class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
|
|
26
24
|
|
|
27
25
|
class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
|
|
@@ -44,12 +42,12 @@ module Chronicle
|
|
|
44
42
|
If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
|
|
45
43
|
LONG_DESC
|
|
46
44
|
# Run an ETL job
|
|
47
|
-
def start
|
|
48
|
-
job_definition = build_job_definition(options)
|
|
45
|
+
def start(name = nil)
|
|
46
|
+
job_definition = build_job_definition(name, options)
|
|
49
47
|
|
|
50
48
|
if job_definition.plugins_missing?
|
|
51
49
|
missing_plugins = job_definition.errors[:plugins]
|
|
52
|
-
.select { |error| error.is_a?(Chronicle::ETL::
|
|
50
|
+
.select { |error| error.is_a?(Chronicle::ETL::PluginNotInstalledError) }
|
|
53
51
|
.map(&:name)
|
|
54
52
|
.uniq
|
|
55
53
|
install_missing_plugins(missing_plugins)
|
|
@@ -57,25 +55,46 @@ LONG_DESC
|
|
|
57
55
|
|
|
58
56
|
run_job(job_definition)
|
|
59
57
|
rescue Chronicle::ETL::JobDefinitionError => e
|
|
60
|
-
|
|
58
|
+
message = ""
|
|
59
|
+
job_definition.errors.each_pair do |category, errors|
|
|
60
|
+
message << "Problem with #{category}:\n - #{errors.map(&:to_s).join("\n - ")}"
|
|
61
|
+
end
|
|
62
|
+
cli_fail(message: "Error running job.\n#{message}", exception: e)
|
|
61
63
|
end
|
|
62
64
|
|
|
63
|
-
|
|
65
|
+
option :'skip-confirmation', aliases: '-y', type: :boolean
|
|
66
|
+
desc "save", "Save a job"
|
|
64
67
|
# Create an ETL job
|
|
65
|
-
def
|
|
66
|
-
|
|
68
|
+
def save(name)
|
|
69
|
+
write_config = true
|
|
70
|
+
job_definition = build_job_definition(name, options)
|
|
67
71
|
job_definition.validate!
|
|
68
72
|
|
|
69
|
-
|
|
70
|
-
|
|
73
|
+
if Chronicle::ETL::Config.exists?("jobs", name) && !options[:'skip-confirmation']
|
|
74
|
+
prompt = TTY::Prompt.new
|
|
75
|
+
write_config = false
|
|
76
|
+
message = "Job '#{name}' exists already. Ovewrite it?"
|
|
77
|
+
begin
|
|
78
|
+
write_config = prompt.yes?(message)
|
|
79
|
+
rescue TTY::Reader::InputInterrupt
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
if write_config
|
|
84
|
+
Chronicle::ETL::Config.write("jobs", name, job_definition.definition)
|
|
85
|
+
cli_exit(message: "Job saved. Run it with `$chronicle-etl jobs:run #{name}`")
|
|
86
|
+
else
|
|
87
|
+
cli_fail(message: "\nJob not saved")
|
|
88
|
+
end
|
|
89
|
+
|
|
71
90
|
rescue Chronicle::ETL::JobDefinitionError => e
|
|
72
91
|
cli_fail(message: "Job definition error", exception: e)
|
|
73
92
|
end
|
|
74
93
|
|
|
75
94
|
desc "show", "Show details about a job"
|
|
76
95
|
# Show an ETL job
|
|
77
|
-
def show
|
|
78
|
-
job_definition = build_job_definition(options)
|
|
96
|
+
def show(name = nil)
|
|
97
|
+
job_definition = build_job_definition(name, options)
|
|
79
98
|
job_definition.validate!
|
|
80
99
|
puts Chronicle::ETL::Job.new(job_definition)
|
|
81
100
|
rescue Chronicle::ETL::JobDefinitionError => e
|
|
@@ -88,7 +107,7 @@ LONG_DESC
|
|
|
88
107
|
jobs = Chronicle::ETL::Config.available_jobs
|
|
89
108
|
|
|
90
109
|
job_details = jobs.map do |job|
|
|
91
|
-
r = Chronicle::ETL::Config.load("
|
|
110
|
+
r = Chronicle::ETL::Config.load("jobs", job)
|
|
92
111
|
|
|
93
112
|
extractor = r[:extractor][:name] if r[:extractor]
|
|
94
113
|
transformer = r[:transformer][:name] if r[:transformer]
|
|
@@ -109,6 +128,11 @@ LONG_DESC
|
|
|
109
128
|
private
|
|
110
129
|
|
|
111
130
|
def run_job(job_definition)
|
|
131
|
+
# FIXME: have to validate here so next method can work. This is clumsy
|
|
132
|
+
job_definition.validate!
|
|
133
|
+
# FIXME: clumsy to make CLI responsible for setting secrets here. Think about a better way to do this
|
|
134
|
+
job_definition.apply_default_secrets
|
|
135
|
+
|
|
112
136
|
job = Chronicle::ETL::Job.new(job_definition)
|
|
113
137
|
runner = Chronicle::ETL::Runner.new(job)
|
|
114
138
|
runner.run!
|
|
@@ -128,29 +152,30 @@ LONG_DESC
|
|
|
128
152
|
end
|
|
129
153
|
|
|
130
154
|
# Create job definition by reading config file and then overwriting with flag options
|
|
131
|
-
def build_job_definition(options)
|
|
155
|
+
def build_job_definition(name, options)
|
|
132
156
|
definition = Chronicle::ETL::JobDefinition.new
|
|
133
|
-
definition.add_config(load_job_config(
|
|
157
|
+
definition.add_config(load_job_config(name))
|
|
134
158
|
definition.add_config(process_flag_options(options).transform_keys(&:to_sym))
|
|
135
159
|
definition
|
|
136
160
|
end
|
|
137
161
|
|
|
138
162
|
def load_job_config name
|
|
139
|
-
Chronicle::ETL::Config.
|
|
163
|
+
Chronicle::ETL::Config.read_job(name)
|
|
140
164
|
end
|
|
141
165
|
|
|
142
166
|
# Takes flag options and turns them into a runner config
|
|
167
|
+
# TODO: this needs a lot of refactoring
|
|
143
168
|
def process_flag_options options
|
|
144
|
-
extractor_options = options[:'extractor-opts'].merge({
|
|
169
|
+
extractor_options = options[:'extractor-opts'].transform_keys(&:to_sym).merge({
|
|
145
170
|
input: (options[:input] if options[:input].any?),
|
|
146
171
|
since: options[:since],
|
|
147
172
|
until: options[:until],
|
|
148
|
-
limit: options[:limit]
|
|
173
|
+
limit: options[:limit]
|
|
149
174
|
}.compact)
|
|
150
175
|
|
|
151
|
-
transformer_options = options[:'transformer-opts']
|
|
176
|
+
transformer_options = options[:'transformer-opts'].transform_keys(&:to_sym)
|
|
152
177
|
|
|
153
|
-
loader_options = options[:'loader-opts'].merge({
|
|
178
|
+
loader_options = options[:'loader-opts'].transform_keys(&:to_sym).merge({
|
|
154
179
|
output: options[:output],
|
|
155
180
|
header_row: options[:header_row],
|
|
156
181
|
fields: options[:fields]
|
|
@@ -24,6 +24,9 @@ module Chronicle
|
|
|
24
24
|
desc 'plugins:COMMAND', 'Configure plugins', hide: true
|
|
25
25
|
subcommand 'plugins', Plugins
|
|
26
26
|
|
|
27
|
+
desc 'secrets:COMMAND', 'Manage secrets', hide: true
|
|
28
|
+
subcommand 'secrets', Secrets
|
|
29
|
+
|
|
27
30
|
# Entrypoint for the CLI
|
|
28
31
|
def self.start(given_args = ARGV, config = {})
|
|
29
32
|
# take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
|
|
@@ -15,15 +15,25 @@ module Chronicle
|
|
|
15
15
|
def install(*plugins)
|
|
16
16
|
cli_fail(message: "Please specify a plugin to install") unless plugins.any?
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
installed, not_installed = plugins.partition do |plugin|
|
|
19
|
+
Chronicle::ETL::Registry::PluginRegistry.installed?(plugin)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
puts "Already installed: #{installed.join(", ")}" if installed.any?
|
|
23
|
+
cli_exit unless not_installed.any?
|
|
24
|
+
|
|
25
|
+
spinner = TTY::Spinner.new("[:spinner] Installing #{not_installed.join(", ")}...", format: :dots_2)
|
|
19
26
|
spinner.auto_spin
|
|
20
|
-
|
|
27
|
+
|
|
28
|
+
not_installed.each do |plugin|
|
|
21
29
|
spinner.update(title: "Installing #{plugin}")
|
|
22
30
|
Chronicle::ETL::Registry::PluginRegistry.install(plugin)
|
|
31
|
+
|
|
23
32
|
rescue Chronicle::ETL::PluginError => e
|
|
24
33
|
spinner.error("Error".red)
|
|
25
34
|
cli_fail(message: "Plugin '#{plugin}' could not be installed", exception: e)
|
|
26
35
|
end
|
|
36
|
+
|
|
27
37
|
spinner.success("(#{'successful'.green})")
|
|
28
38
|
end
|
|
29
39
|
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tty-prompt"
|
|
4
|
+
|
|
5
|
+
module Chronicle
|
|
6
|
+
module ETL
|
|
7
|
+
module CLI
|
|
8
|
+
# CLI commands for working with ETL plugins
|
|
9
|
+
class Secrets < SubcommandBase
|
|
10
|
+
default_task 'list'
|
|
11
|
+
namespace :secrets
|
|
12
|
+
|
|
13
|
+
desc "set NAMESPACE KEY [VALUE]", "Add a secret. VALUE can be set as argument or from stdin"
|
|
14
|
+
def set(namespace, key, value=nil)
|
|
15
|
+
validate_namespace(namespace)
|
|
16
|
+
|
|
17
|
+
if value
|
|
18
|
+
# came as argument
|
|
19
|
+
elsif $stdin.respond_to?(:stat) && $stdin.stat.pipe?
|
|
20
|
+
value = $stdin.read
|
|
21
|
+
else
|
|
22
|
+
prompt = TTY::Prompt.new
|
|
23
|
+
value = prompt.mask("Please enter #{key} for #{namespace}:")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
Chronicle::ETL::Secrets.set(namespace, key, value.strip)
|
|
27
|
+
cli_exit(message: "Secret set")
|
|
28
|
+
rescue TTY::Reader::InputInterrupt
|
|
29
|
+
cli_fail(message: "\nSecret not set")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
desc "unset NAMESPACE KEY", "Remove a secret"
|
|
33
|
+
def unset(namespace, key)
|
|
34
|
+
validate_namespace(namespace)
|
|
35
|
+
|
|
36
|
+
Chronicle::ETL::Secrets.unset(namespace, key)
|
|
37
|
+
cli_exit(message: "Secret unset")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
desc "list", "List available secrets"
|
|
41
|
+
def list(namespace=nil)
|
|
42
|
+
all_secrets = Chronicle::ETL::Secrets.all(namespace)
|
|
43
|
+
cli_exit(message: "No secrets are stored") unless all_secrets.any?
|
|
44
|
+
|
|
45
|
+
rows = []
|
|
46
|
+
all_secrets.each do |namespace, secrets|
|
|
47
|
+
rows += secrets.map do |key, value|
|
|
48
|
+
# hidden_value = (value[0..5] + ("*" * [0, [value.length - 5, 30].min].max)).truncate(30)
|
|
49
|
+
truncated_value = value.truncate(30)
|
|
50
|
+
[namespace, key, truncated_value]
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
headers = ['namespace', 'key', 'value'].map { |h| h.upcase.bold }
|
|
55
|
+
|
|
56
|
+
puts "Available secrets:"
|
|
57
|
+
table = TTY::Table.new(headers, rows)
|
|
58
|
+
puts table.render(indent: 0, padding: [0, 2])
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def validate_namespace(namespace)
|
|
64
|
+
cli_fail(message: "'#{namespace}' is not a valid namespace") unless Chronicle::ETL::Secrets.valid_namespace_name?(namespace)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|