chronicle-etl 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8faa084cfe4a9f080ee5494c69b268b78bfa8f3502354e740264e6941f13daf
-  data.tar.gz: 1bf4f2751c71cadedc78a2fe3ed5b09bf86cd601a909e2fa2db0a0de8cc2c21d
+  metadata.gz: d0f305f15f4eda7a5851dfff2155da2c12ee010d4619346a13551f298d5b7991
+  data.tar.gz: d44f82b2bd06521740ad2b0e58cad0db840884fc5616858ef857d78fccb2b5dd
 SHA512:
-  metadata.gz: ff10779b663a3321b779fb03e07249856174d96fb96e405ae906a47441c288d6a245c852525801ba250cce1125cf05c523ef4ec75fdfb4335cef9003091437ed
-  data.tar.gz: 509f6f92e95341d212c54b6b000bc54e8ba03898497191a3e5d3b14db7bff3ed625d0fee403888fbb6103c1edc14de66b215d9aa84ddb68cefcf51c0e6c74138
+  metadata.gz: 99214409831e2799dffe2e3b096e9406222cf571d4e49fe71d3e3c645ad635e73c3aa42cb6af6569431064ce2750dc38ba051122a826b9feb6b21724ebd31db8
+  data.tar.gz: 77a30ecb069906b0e992adbcb1b7470642bcda65b8767dd8ba0145d63e834cb83e2a0a7bc5212edcf8c5028c637b2edbd8476b559f8e28765267516308b160eb

data/README.md

@@ -8,19 +8,26 @@ Are you trying to archive your digital history or incorporate it into your own p
 
 If you don’t want to spend all your time writing scrapers, reverse-engineering APIs, or parsing takeout data, this project is for you! (*If you do enjoy these things, please see the [open issues](https://github.com/chronicle-app/chronicle-etl/issues).*)
 
-**`chronicle-etl` is a CLI tool that gives you a unified interface for accessing your personal data.** It uses the ETL pattern to *extract* it from a source (e.g. your local browser history, a directory of images, goodreads.com reading history), *transform* it (into a given schema), and *load* it to a source (e.g. a CSV file, JSON, external API).
+**`chronicle-etl` is a CLI tool that gives you a unified interface for accessing your personal data.** It uses the ETL pattern to *extract* it from a source (e.g. your local browser history, a directory of images, goodreads.com reading history), *transform* it (into a given schema), and *load* it to a destination (e.g. a CSV file, JSON, external API).
 
 ## What does `chronicle-etl` give you?
 * **CLI tool for working with personal data**. You can monitor progress of exports, manipulate the output, set up recurring jobs, manage credentials, and more.
 * **Plugins for many third-party providers**. A plugin system allows you to access data from third-party providers and hook it into the shared CLI infrastructure.
 * **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
 
+## Chronicle-ETL in action
+
+![demo](https://user-images.githubusercontent.com/6291/161410839-b5ce931a-2353-4585-b530-929f46e3f960.svg)
+
+### Longer screencast
+
+[![asciicast](https://asciinema.org/a/483455.svg)](https://asciinema.org/a/483455)
+
 ## Installation
 
 Using homebrew:
 ```sh
 $ brew install chronicle-app/etl/chronicle-etl
-
 ```
 Using rubygems:
 ```sh
@@ -42,7 +49,7 @@ $ chronicle-etl help
 $ chronicle-etl --extractor NAME --transformer NAME --loader NAME
 
 # Read test.csv and display it to stdout as a table 
-$ chronicle-etl --extractor csv --input ./data.csv --loader table
+$ chronicle-etl --extractor csv --input data.csv --loader table
 
 # Retrieve shell commands run in the last 5 hours
 $ chronicle-etl -e shell --since 5h
@@ -154,12 +161,12 @@ If you don't see a plugin for a third-party provider or data source that you're
 
 | Name                                                            | Description                                                                                 | Availability                     |
 |-----------------------------------------------------------------|---------------------------------------------------------------------------------------------|----------------------------------|
+| [email](https://github.com/chronicle-app/chronicle-email)       | Emails and attachments from IMAP or .mbox files                                             | Available |
+| [github](https://github.com/chronicle-app/chronicle-github)     | Github activity stream                                                                      | Available                        |
 | [imessage](https://github.com/chronicle-app/chronicle-imessage) | iMessage messages and attachments                                                           | Available                        |
-| [shell](https://github.com/chronicle-app/chronicle-shell)       | Shell command history                                                                       | Available (still needs zsh support)  |
-| [email](https://github.com/chronicle-app/chronicle-email)       | Emails and attachments from IMAP or .mbox files                                             | Available (still needs IMAP support) |
 | [pinboard](https://github.com/chronicle-app/chronicle-email)    | Bookmarks and tags                                                                          | Available                        |
 | [safari](https://github.com/chronicle-app/chronicle-safari)     | Browser history from local sqlite db                                                        | Available                        |
-| [github](https://github.com/chronicle-app/chronicle-github)     | Github activity stream                                                                      | Available                        |
+| [shell](https://github.com/chronicle-app/chronicle-shell)       | Shell command history                                                                       | Available (still needs zsh support)  |
 
 #### Coming soon
 

data/lib/chronicle/etl/cli/cli_base.rb

@@ -6,6 +6,7 @@ module Chronicle
         no_commands do
           # Shorthand for cli_exit(status: :failure)
           def cli_fail(message: nil, exception: nil)
+            message += "\nRe-run the command with --verbose to see details." if Chronicle::ETL::Logger.log_level > Chronicle::ETL::Logger::DEBUG
             cli_exit(status: :failure, message: message, exception: exception)
           end
 

data/lib/chronicle/etl/cli/jobs.rb

@@ -43,6 +43,15 @@ module Chronicle
 LONG_DESC
         # Run an ETL job
         def start(name = nil)
+          # If someone runs `$ chronicle-etl` with no arguments, show help menu.
+          # TODO: decide if we should check that there's nothing in stdin pipe
+          # in case user wants to actually run this sort of job stdin->null->stdout
+          if name.nil? && options[:extractor].nil?
+            m = Chronicle::ETL::CLI::Main.new
+            m.help
+            cli_exit
+          end
+
           job_definition = build_job_definition(name, options)
 
           if job_definition.plugins_missing?
@@ -82,11 +91,10 @@ LONG_DESC
 
           if write_config
             Chronicle::ETL::Config.write("jobs", name, job_definition.definition)
-            cli_exit(message: "Job saved. Run it with `$chronicle-etl jobs:run #{name}`")
+            cli_exit(message: "Job saved. Run it with `$ chronicle-etl jobs:run #{name}`")
           else
             cli_fail(message: "\nJob not saved")
           end
-
         rescue Chronicle::ETL::JobDefinitionError => e
           cli_fail(message: "Job definition error", exception: e)
         end
@@ -136,6 +144,8 @@ LONG_DESC
           job = Chronicle::ETL::Job.new(job_definition)
           runner = Chronicle::ETL::Runner.new(job)
           runner.run!
+        rescue RunnerError => e
+          cli_fail(message: "#{e.message}", exception: e)
         end
 
         # TODO: probably could merge this with something in cli/plugin

data/lib/chronicle/etl/cli/main.rb

@@ -54,24 +54,40 @@ module Chronicle
             klass, task = ::Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
             klass.start(['-h', task].compact, shell: shell)
           else
-            shell.say "ABOUT".bold
-            shell.say "  #{'chronicle-etl'.italic} is a utility tool for #{'extracting'.underline}, #{'transforming'.underline}, and #{'loading'.underline} personal data."
+            shell.say "ABOUT:".bold
+            shell.say "  #{'chronicle-etl'.italic} is a toolkit for extracting and working with your digital"
+            shell.say "  history. 📜"
             shell.say
-            shell.say "USAGE".bold
-            shell.say "  $ chronicle-etl COMMAND"
+            shell.say "  A job #{'extracts'.underline} personal data from a source, #{'transforms'.underline} it (Chronicle"
+            shell.say "  Schema or preserves raw data), and then #{'loads'.underline} it to a destination. Use"
+            shell.say "  built-in extractors (json, csv, stdin) and loaders (csv, json, table,"
+            shell.say "  rest) or use plugins to connect to third-party services."
             shell.say
-            shell.say "EXAMPLES".bold
-            shell.say "  Show available connectors:".italic.light_black
-            shell.say "  $ chronicle-etl connectors:list"
+            shell.say "  Plugins: https://github.com/chronicle-app/chronicle-etl#currently-available"
             shell.say
-            shell.say "  Run a simple job:".italic.light_black
-            shell.say "  $ chronicle-etl jobs:run --extractor stdin --transformer null --loader stdout"
+            shell.say "USAGE:".bold
+            shell.say "  # Basic job usage:".italic.light_black
+            shell.say "  $ chronicle-etl --extractor NAME --transformer NAME --loader NAME"
             shell.say
-            shell.say "  Show full job options:".italic.light_black
+            shell.say "  # Read test.csv and display it to stdout as a table:".italic.light_black
+            shell.say "  $ chronicle-etl --extractor csv --input data.csv --loader table"
+            shell.say
+            shell.say "  # Show available plugins:".italic.light_black
+            shell.say "  $ chronicle-etl plugins:list"
+            shell.say
+            shell.say "  # Save an access token as a secret and use it in a job:".italic.light_black
+            shell.say "  $ chronicle-etl secrets:set pinboard access_token username:foo123"
+            shell.say "  $ chronicle-etl secrets:list"
+            shell.say "  $ chronicle-etl -e pinboard --since 1mo"
+            shell.say
+            shell.say "  # Show full job options:".italic.light_black
             shell.say "  $ chronicle-etl jobs help run"
+            shell.say
+            shell.say "FULL DOCUMENTATION:".bold
+            shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
+            shell.say
 
             list = []
-
             ::Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
               list += thor_class.printable_tasks(false)
             end
@@ -79,25 +95,18 @@ module Chronicle
             list.unshift ["help", "# This help menu"]
 
             shell.say
-            shell.say 'ALL COMMANDS'.bold
+            shell.say 'ALL COMMANDS:'.bold
             shell.print_table(list, indent: 2, truncate: true)
             shell.say
-            shell.say "VERSION".bold
+            shell.say "VERSION:".bold
             shell.say "  #{Chronicle::ETL::VERSION}"
             shell.say
             shell.say "  Display current version:".italic.light_black
             shell.say "  $ chronicle-etl --version"
-            shell.say
-            shell.say "FULL DOCUMENTATION".bold
-            shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
-            shell.say
           end
         end
 
         no_commands do
-          def testb
-            puts "hi"
-          end
           def set_color_output
             String.disable_colorization true if options[:'no-color'] || ENV['NO_COLOR']
           end

data/lib/chronicle/etl/cli/plugins.rb

@@ -61,7 +61,7 @@ module Chronicle
             }
           end
 
-          headers = ['name', 'description', 'latest version'].map{ |h| h.to_s.upcase.bold }
+          headers = ['name', 'description', 'version'].map{ |h| h.to_s.upcase.bold }
           table = TTY::Table.new(headers, info.map(&:values))
           puts "Installed plugins:"
           puts table.render(indent: 2, padding: [0, 0])

data/lib/chronicle/etl/exceptions.rb

@@ -6,6 +6,9 @@ module Chronicle
 
     class ConfigError < Error; end
 
+    class RunnerError < Error; end
+    class RunInterruptedError < RunnerError; end
+
     class RunnerTypeError < Error; end
 
     class JobDefinitionError < Error

data/lib/chronicle/etl/models/base.rb

@@ -9,7 +9,7 @@ module Chronicle
       # @todo Experiment with just mixing in ActiveModel instead of this
       #   this reimplementation
       class Base
-        ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
+        ATTRIBUTES = [:provider, :provider_id, :provider_namespace, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze
 
         attr_accessor(:id, :dedupe_on, *ATTRIBUTES)

data/lib/chronicle/etl/models/entity.rb

@@ -10,7 +10,7 @@ module Chronicle
         # TODO: This desperately needs a validation system
         ASSOCIATIONS = [
           :involvements, # inverse of activity's `involved`
-
+          :analogous,
           :attachments,
           :abouts,
           :aboutables, # inverse of above

data/lib/chronicle/etl/runner.rb

@@ -1,5 +1,6 @@
 require 'colorize'
 require 'chronic_duration'
+require "tty-spinner"
 
 class Chronicle::ETL::Runner
   def initialize(job)
@@ -8,30 +9,55 @@ class Chronicle::ETL::Runner
   end
 
   def run!
+    begin_job
     validate_job
     instantiate_connectors
     prepare_job
     prepare_ui
     run_extraction
+  rescue Chronicle::ETL::ExtractionError => e
+    @job_logger&.error
+    raise(Chronicle::ETL::RunnerError, "Extraction failed. #{e.message}")
+  rescue Interrupt
+    @job_logger&.error
+    raise(Chronicle::ETL::RunInterruptedError, "Job interrupted.")
+  rescue StandardError => e
+    # Just throwing this in here until we have better exception handling in
+    # loaders, etc
+    @job_logger&.error
+    raise(Chronicle::ETL::RunnerError, "Error running job. #{e.message}")
+  ensure
     finish_job
   end
 
   private
 
+  def begin_job
+    Chronicle::ETL::Logger.info(tty_log_job_initialize)
+    @initialization_spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
+  end
+
   def validate_job
+    @initialization_spinner.update(title: "Validating job")
     @job.job_definition.validate!
   end
 
   def instantiate_connectors
+    @initialization_spinner.update(title: "Initializing connectors")
     @extractor = @job.instantiate_extractor
     @loader = @job.instantiate_loader
   end
 
   def prepare_job
-    Chronicle::ETL::Logger.info(tty_log_job_start)
+    @initialization_spinner.update(title: "Preparing job")
     @job_logger.start
     @loader.start
+
+    @initialization_spinner.update(title: "Preparing extraction")
+    @initialization_spinner.auto_spin
     @extractor.prepare
+    @initialization_spinner.success("(#{'successful'.green})")
+    Chronicle::ETL::Logger.info("\n")
   end
 
   def prepare_ui
@@ -40,34 +66,34 @@ class Chronicle::ETL::Runner
     Chronicle::ETL::Logger.attach_to_progress_bar(@progress_bar)
   end
 
-  # TODO: refactor this further
   def run_extraction
     @extractor.extract do |extraction|
-      unless extraction.is_a?(Chronicle::ETL::Extraction)
-        raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
-      end
-
-      transformer = @job.instantiate_transformer(extraction)
-      record = transformer.transform
-
-      Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
-      @job_logger.log_transformation(transformer)
-
-      @loader.load(record) unless @job.dry_run?
-    rescue Chronicle::ETL::TransformationError => e
-      Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
-    ensure
+      process_extraction(extraction)
       @progress_bar.increment
     end
 
     @progress_bar.finish
+
+    # This is typically a slow method (writing to stdout, writing a big file, etc)
+    # TODO: consider adding a spinner?
     @loader.finish
     @job_logger.finish
-  rescue Interrupt
-    Chronicle::ETL::Logger.error("\n#{'Job interrupted'.red}")
-    @job_logger.error
-  rescue StandardError => e
-    raise e
+  end
+
+  def process_extraction(extraction)
+    # For each extraction from our extractor, we create a new tarnsformer
+    transformer = @job.instantiate_transformer(extraction)
+
+    # And then transform that record, logging it if we're in debug log level
+    record = transformer.transform
+    Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
+    @job_logger.log_transformation(transformer)
+
+    # Then send the results to the loader
+    @loader.load(record) unless @job.dry_run?
+  rescue Chronicle::ETL::TransformationError => e
+    # TODO: have an option to cancel job if we encounter an error
+    Chronicle::ETL::Logger.error(tty_log_transformation_failure(e, transformer))
   end
 
   def finish_job
@@ -77,7 +103,7 @@ class Chronicle::ETL::Runner
     Chronicle::ETL::Logger.info(tty_log_completion)
   end
 
-  def tty_log_job_start
+  def tty_log_job_initialize
     output = "Beginning job "
     output += "'#{@job.name}'".bold if @job.name
     output
@@ -95,8 +121,9 @@ class Chronicle::ETL::Runner
 
   def tty_log_completion
     status = @job_logger.success ? 'Success' : 'Failed'
-    output = "\nCompleted job "
-    output += "'#{@job.name}'".bold if @job.name
+    job_completion = @job_logger.success ? 'Completed' : 'Partially completed'
+    output = "\n#{job_completion} job"
+    output += " '#{@job.name}'".bold if @job.name
     output += " in #{ChronicDuration.output(@job_logger.duration)}" if @job_logger.duration
     output += "\n  Status:\t".light_black + status
     output += "\n  Completed:\t".light_black + "#{@job_logger.job_log.num_records_processed}"

data/lib/chronicle/etl/transformers/transformer.rb

@@ -10,6 +10,10 @@ module Chronicle
       # options::
       #   Options for configuring this Transformer
       def initialize(extraction, options = {})
+        unless extraction.is_a?(Chronicle::ETL::Extraction)
+          raise Chronicle::ETL::RunnerTypeError, "Extracted should be a Chronicle::ETL::Extraction"
+        end
+
         @extraction = extraction
         apply_options(options)
       end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.5.2"
+    VERSION = "0.5.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-30 00:00:00.000000000 Z
+date: 2022-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport