chronicle-etl 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e9f565004fce3539bdcfec9b98f64e33d8d4f1dde532f573adf54bfb61eadfa
-  data.tar.gz: 81da054f627ae084d45b4fe53480d5ef26fadf9cfd3ad4eb53d908741d8214e4
+  metadata.gz: b8faa084cfe4a9f080ee5494c69b268b78bfa8f3502354e740264e6941f13daf
+  data.tar.gz: 1bf4f2751c71cadedc78a2fe3ed5b09bf86cd601a909e2fa2db0a0de8cc2c21d
 SHA512:
-  metadata.gz: ffe35b9c55a442610b9832b7f918f16dff0ccad3245248f76c52afbd88c43d671e8724f72a8d181a13e3cafffae6bc97f5963f65eb5c2a0ddb7080d9b6f1d145
-  data.tar.gz: c063698040b906f99e57cc3f9e829ff8b8ca1d4e5314a3a4179dc965c17ba70da04155d01b0b1fadf573e6a110a69862e9651435bdcdb1160bfe84cb8a55a10a
+  metadata.gz: ff10779b663a3321b779fb03e07249856174d96fb96e405ae906a47441c288d6a245c852525801ba250cce1125cf05c523ef4ec75fdfb4335cef9003091437ed
+  data.tar.gz: 509f6f92e95341d212c54b6b000bc54e8ba03898497191a3e5d3b14db7bff3ed625d0fee403888fbb6103c1edc14de66b215d9aa84ddb68cefcf51c0e6c74138

data/README.md

@@ -16,12 +16,21 @@ If you don’t want to spend all your time writing scrapers, reverse-engineering
 * **A common, opinionated schema**: You can normalize different datasets into a single schema so that, for example, all your iMessages and emails are stored in a common schema. Don’t want to use the schema? `chronicle-etl` always allows you to fall back on working with the raw extraction data.
 
 ## Installation
+
+Using homebrew:
+```sh
+$ brew install chronicle-app/etl/chronicle-etl
+
+```
+Using rubygems:
 ```sh
-# Install chronicle-etl
-gem install chronicle-etl
+$ gem install chronicle-etl
 ```
 
-After installation, the `chronicle-etl` command will be available in your shell. Homebrew support [is coming soon](https://github.com/chronicle-app/chronicle-etl/issues/13).
+Confirm it installed successfully:
+```sh
+$ chronicle-etl --version
+```
 
 ## Basic usage and running jobs
 
@@ -50,7 +59,6 @@ $ chronicle-etl -e pinboard --since 1mo # Used automatically based on plugin nam
 ### Common options
 ```sh
 Options:
-  -j, [--name=NAME]                      # Job configuration name
   -e, [--extractor=NAME]                 # Extractor class. Default: stdin
       [--extractor-opts=key:value]       # Extractor options
   -t, [--transformer=NAME]               # Transformer class. Default: null
@@ -71,6 +79,26 @@ Options:
       [--silent], [--no-silent]          # Silence all output
 ```
 
+### Saving jobs
+
+You can save details about a job to a local config file (saved by default in `~/.config/chronicle/etl/jobs/job_name.yml`) to save yourself the trouble of setting the CLI flags for each run.
+
+```sh
+# Save a job named 'sample' to ~/.config/chronicle/etl/jobs/sample.yml
+$ chronicle-etl jobs:save sample --extractor pinboard --since 10d
+
+# Show details about the job
+$ chronicle-etl jobs:show sample
+
+# Run the job
+$ chronicle-etl jobs:run sample
+# Or more simply:
+$ chronicle-etl sample
+
+# Show all saved jobs
+$ chronicle-etl jobs:list
+```
+
 ## Connectors
 Connectors are available to read, process, and load data from different formats or external services.
 
@@ -97,7 +125,7 @@ $ chronicle-etl connectors:list
 - [`rest`](https://github.com/chronicle-app/chronicle-etl/blob/main/lib/chronicle/etl/loaders/rest_loader.rb) - Serialize records with [JSONAPI](https://jsonapi.org/) and send to a REST API
 
 ## Chronicle Plugins
-Plugins provide access to data from third-party platforms, services, or formats. Plugins are packaged as separate rubygems and can be installed through `$ gem install` or through the CLI itself.
+Plugins provide access to data from third-party platforms, services, or formats. Plugins are packaged as separate rubygems and can be installed through the CLI (which installs the Gems under the hood).
 
 ### Plugin usage
 
@@ -131,6 +159,7 @@ If you don't see a plugin for a third-party provider or data source that you're
 | [email](https://github.com/chronicle-app/chronicle-email)       | Emails and attachments from IMAP or .mbox files                                             | Available (still needs IMAP support) |
 | [pinboard](https://github.com/chronicle-app/chronicle-email)    | Bookmarks and tags                                                                          | Available                        |
 | [safari](https://github.com/chronicle-app/chronicle-safari)     | Browser history from local sqlite db                                                        | Available                        |
+| [github](https://github.com/chronicle-app/chronicle-github)     | Github activity stream                                                                      | Available                        |
 
 #### Coming soon
 
@@ -206,7 +235,6 @@ $ chronicle-etl secrets:unset pinboard access_token
 
 ## Roadmap
 
-- Add **homebrew formula** for easier installation. #13
 - Keep tackling **new plugins**. See: [Chronicle Plugin Tracker](https://github.com/orgs/chronicle-app/projects/1)
 - Add support for **incremental extractions** #37
 - **Improve stdin extractor and shell command transformer** (#5) so that users can easily integrate their own scripts/tools into jobs

data/lib/chronicle/etl/cli/jobs.rb

@@ -9,8 +9,6 @@ module Chronicle
         default_task "start"
         namespace :jobs
 
-        class_option :name, aliases: '-j', desc: 'Job configuration name'
-
         class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'NAME'
         class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
         class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'NAME'
@@ -44,8 +42,8 @@ module Chronicle
             If you do not want to use the command line flags, you can also configure a job with a .yml config file. You can either specify the path to this file or use the filename and place the file in ~/.config/chronicle/etl/jobs/NAME.yml and call it with `--job NAME`
 LONG_DESC
         # Run an ETL job
-        def start
-          job_definition = build_job_definition(options)
+        def start(name = nil)
+          job_definition = build_job_definition(name, options)
 
           if job_definition.plugins_missing?
             missing_plugins = job_definition.errors[:plugins]
@@ -64,21 +62,39 @@ LONG_DESC
           cli_fail(message: "Error running job.\n#{message}", exception: e)
         end
 
-        desc "create", "Create a job"
+        option :'skip-confirmation', aliases: '-y', type: :boolean
+        desc "save", "Save a job"
         # Create an ETL job
-        def create
-          job_definition = build_job_definition(options)
+        def save(name)
+          write_config = true
+          job_definition = build_job_definition(name, options)
           job_definition.validate!
 
-          Chronicle::ETL::Config.write("jobs", options[:name], job_definition.definition)
+          if Chronicle::ETL::Config.exists?("jobs", name) && !options[:'skip-confirmation']
+            prompt = TTY::Prompt.new
+            write_config = false
+            message = "Job '#{name}' exists already. Ovewrite it?"
+            begin
+              write_config = prompt.yes?(message)
+            rescue TTY::Reader::InputInterrupt
+            end
+          end
+
+          if write_config
+            Chronicle::ETL::Config.write("jobs", name, job_definition.definition)
+            cli_exit(message: "Job saved. Run it with `$chronicle-etl jobs:run #{name}`")
+          else
+            cli_fail(message: "\nJob not saved")
+          end
+
         rescue Chronicle::ETL::JobDefinitionError => e
           cli_fail(message: "Job definition error", exception: e)
         end
 
         desc "show", "Show details about a job"
         # Show an ETL job
-        def show
-          job_definition = build_job_definition(options)
+        def show(name = nil)
+          job_definition = build_job_definition(name, options)
           job_definition.validate!
           puts Chronicle::ETL::Job.new(job_definition)
         rescue Chronicle::ETL::JobDefinitionError => e
@@ -136,9 +152,9 @@ LONG_DESC
         end
 
         # Create job definition by reading config file and then overwriting with flag options
-        def build_job_definition(options)
+        def build_job_definition(name, options)
           definition = Chronicle::ETL::JobDefinition.new
-          definition.add_config(load_job_config(options[:name]))
+          definition.add_config(load_job_config(name))
           definition.add_config(process_flag_options(options).transform_keys(&:to_sym))
           definition
         end

data/lib/chronicle/etl/config.rb

@@ -28,6 +28,12 @@ module Chronicle
         end
       end
 
+      def exists?(type, identifier)
+        base = config_pathname_for_type(type)
+        path = base.join("#{identifier}.yml")
+        return path.exist?
+      end
+
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
       def available_jobs
         Dir.glob(File.join(config_pathname_for_type("jobs"), "*.yml")).map do |filename|

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -3,11 +3,13 @@ require 'csv'
 module Chronicle
   module ETL
     class CSVLoader < Chronicle::ETL::Loader
+      include Chronicle::ETL::Loaders::Helpers::StdoutHelper
+
       register_connector do |r|
         r.description = 'CSV'
       end
 
-      setting :output, default: $stdout
+      setting :output
       setting :headers, default: true
       setting :header_row, default: true
 
@@ -30,16 +32,7 @@ module Chronicle
           csv_options[:headers] = headers
         end
 
-        if @config.output.is_a?(IO)
-          # This might seem like a duplication of the default value ($stdout)
-          # but it's because rspec overwrites $stdout (in helper #capture) to
-          # capture output.
-          io = $stdout.dup
-        else
-          io = File.open(@config.output, "w+")
-        end
-
-        output = CSV.generate(**csv_options) do |csv|
+        csv_output = CSV.generate(**csv_options) do |csv|
           records.each do |record|
             csv << record
               .transform_keys(&:to_sym)
@@ -48,8 +41,12 @@ module Chronicle
           end
         end
 
-        io.write(output)
-        io.close
+        # TODO: just write to io directly
+        if output_to_stdout?
+          write_to_stdout(csv_output)
+        else
+          File.write(@config.output, csv_output)
+        end
       end
     end
   end

data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb

@@ -0,0 +1,36 @@
+require 'tempfile'
+
+module Chronicle
+  module ETL
+    module Loaders
+      module Helpers
+        module StdoutHelper
+          # TODO: let users use "stdout" as an option for the `output` setting
+          # Assume we're using stdout if no output is specified
+          def output_to_stdout?
+            !@config.output
+          end
+
+          def create_stdout_temp_file
+            file = Tempfile.new('chronicle-stdout')
+            file.unlink
+            file
+          end
+
+          def write_to_stdout_from_temp_file(file)
+            file.rewind
+            write_to_stdout(file.read)
+          end
+
+          def write_to_stdout(output)
+            # We .dup because rspec overwrites $stdout (in helper #capture) to
+            # capture output.
+            stdout = $stdout.dup
+            stdout.write(output)
+            stdout.flush
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -1,22 +1,35 @@
+require 'tempfile'
+
 module Chronicle
   module ETL
     class JSONLoader < Chronicle::ETL::Loader
+      include Chronicle::ETL::Loaders::Helpers::StdoutHelper
+
       register_connector do |r|
         r.description = 'json'
       end
 
       setting :serializer
-      setting :output, default: $stdout
+      setting :output
+
+      # If true, one JSON record per line. If false, output a single json
+      # object with an array of records
+      setting :line_separated, default: true, type: :boolean
+
+      def initialize(*args)
+        super
+        @first_line = true
+      end
 
       def start
-        if @config.output.is_a?(IO)
-          # This might seem like a duplication of the default value ($stdout)
-          # but it's because rspec overwrites $stdout (in helper #capture) to
-          # capture output.
-          @output = $stdout.dup
-        else
-          @output = File.open(@config.output, "w+")
-        end
+        @output_file =
+          if output_to_stdout?
+            create_stdout_temp_file
+          else
+            File.open(@config.output, "w+")
+          end
+
+        @output_file.puts("[\n") unless @config.line_separated
       end
 
       def load(record)
@@ -30,15 +43,34 @@ module Chronicle
 
           force_utf8(value)
         end
-        @output.puts encoded.to_json
+
+        line = encoded.to_json
+        # For line-separated output, we just put json + newline
+        if @config.line_separated
+          line = "#{line}\n"
+        # Otherwise, we add a comma and newline and then add record to the
+        # array we created in #start (unless it's the first line).
+        else
+          line = ",\n#{line}" unless @first_line
+        end
+
+        @output_file.write(line)
+
+        @first_line = false
       end
 
       def finish
-        @output.close if @output.is_a?(IO)
+        # Close the array unless we're doing line-separated JSON
+        @output_file.puts("\n]") unless @config.line_separated
+
+        write_to_stdout_from_temp_file(@output_file) if output_to_stdout?
+
+        @output_file.close
       end
 
       private
 
+      # TODO: implement this
       def serializer
         @config.serializer || Chronicle::ETL::RawSerializer
       end

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,4 +1,5 @@
 require_relative 'helpers/encoding_helper'
+require_relative 'helpers/stdout_helper'
 
 module Chronicle
   module ETL

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.5.1"
+    VERSION = "0.5.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-25 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -396,6 +396,7 @@ files:
 - lib/chronicle/etl/job_logger.rb
 - lib/chronicle/etl/loaders/csv_loader.rb
 - lib/chronicle/etl/loaders/helpers/encoding_helper.rb
+- lib/chronicle/etl/loaders/helpers/stdout_helper.rb
 - lib/chronicle/etl/loaders/json_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
 - lib/chronicle/etl/loaders/rest_loader.rb