chronicle-etl 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1c08bc4f71c807525090abbf1701be19ab72cce08a99cc3bbec9b0db7150a02
-  data.tar.gz: 172a5d7e7ba7a9424ef7b5ab4da2b8c44defdb4e0a34c833248ff1b63f40407e
+  metadata.gz: 12a38a335c05b1626b9f259318956480df3f96e447cc2b1a25b8a9c23d591e49
+  data.tar.gz: d8ed027154403e68e5684213b0d0f58218a23dc2f667a882dcd3b2e8ab0c69b7
 SHA512:
-  metadata.gz: 0f671c00928b15f9c0f6fa159ac106ff9c4f65a8bd16048e5d0cab82d680945317f7680e7796e98c665bb5cc757e0657f1a36d773d89e3e1587d9eebc12abdd8
-  data.tar.gz: 449d1368e0054f39006c7903218300b9b97ca839d6eff43b6b7bd659e5146d443a31c53325c4769ae7a56db9d42417020ccde17362ae024c01aca2ed63029044
+  metadata.gz: 396863ed665137905cfa9fe51ee925776e0a0f616721658a889b9b587dda83b9cd1e0fa2a483b08fc65ec70797f07facec082c1c88403aa8d61e1ce4ae791779
+  data.tar.gz: 705d626f45c816494949d6bc5c4f83cc4a8cd1c527aef72911bb90000e2151f758889c6080e9ac489f235df4169fbc158a7713dbf76d5e7ba5fdaf2a6ad51567

data/Gemfile.lock

@@ -1,8 +1,10 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.2.2)
+    chronicle-etl (0.2.3)
       colorize (~> 0.8.1)
+      deep_merge (~> 1.2)
+      sequel (~> 5.35)
       thor (~> 0.20)
       tty-progressbar (~> 0.17)
       tty-table (~> 0.11)
@@ -13,6 +15,7 @@ GEM
     byebug (11.1.3)
     coderay (1.1.3)
     colorize (0.8.1)
+    deep_merge (1.2.1)
     diff-lcs (1.4.4)
     equatable (0.6.1)
     method_source (1.0.0)
@@ -45,6 +48,8 @@ GEM
     runcom (6.2.0)
       refinements (~> 7.4)
       xdg (~> 4.0)
+    sequel (5.36.0)
+    sqlite3 (1.4.2)
     strings (0.1.8)
       strings-ansi (~> 0.1)
       unicode-display_width (~> 1.5)
@@ -80,6 +85,7 @@ DEPENDENCIES
   redcarpet (~> 3.5)
   rspec (~> 3.9)
   runcom (~> 6.2)
+  sqlite3 (~> 1.4)
 
 BUNDLED WITH
    2.1.4

data/chronicle-etl.gemspec

@@ -40,6 +40,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency "colorize", "~> 0.8.1"
   spec.add_dependency "tty-table", "~> 0.11"
   spec.add_dependency "tty-progressbar", "~> 0.17"
+  spec.add_dependency 'sequel', '~> 5.35'
+  spec.add_dependency 'deep_merge', '~> 1.2'
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0"
@@ -47,4 +49,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug", "~> 3.9"
   spec.add_development_dependency 'runcom', '~> 6.2'
   spec.add_development_dependency 'redcarpet', '~> 3.5'
+  spec.add_development_dependency 'sqlite3', '~> 1.4'
 end

data/lib/chronicle/etl.rb

@@ -1,9 +1,13 @@
 require_relative 'etl/catalog'
 require_relative 'etl/config'
+require_relative 'etl/job_definition'
 require_relative 'etl/exceptions'
 require_relative 'etl/extractors/extractor'
+require_relative 'etl/job_log'
+require_relative 'etl/job_logger'
+require_relative 'etl/job'
 require_relative 'etl/loaders/loader'
 require_relative 'etl/runner'
 require_relative 'etl/transformers/transformer'
 require_relative 'etl/utils/progress_bar'
-require_relative 'etl/version'
+require_relative 'etl/version'

data/lib/chronicle/etl/catalog.rb

@@ -3,6 +3,7 @@ module Chronicle
     # Utility methods to catalogue which Extractor, Transformer, and
     # Loader connector classes are available to chronicle-etl
     module Catalog
+      PHASES = [:extractor, :transformer, :loader]
       PLUGINS = ['email', 'bash']
       BUILTIN = {
         extractor: ['stdin', 'json', 'csv', 'file'],
@@ -43,6 +44,11 @@ module Chronicle
         end
       end
 
+      # Take a phase (e, t, or l) and an identifier and return the right class
+      def self.phase_and_identifier_to_klass(phase, identifier)
+        Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
+      end
+
       # For a given connector identifier, return the class (either builtin, or from a 
       # external chronicle gem)
       def self.identifier_to_klass(identifier:, phase:)

data/lib/chronicle/etl/cli/jobs.rb

@@ -1,5 +1,4 @@
 require 'pp'
-
 module Chronicle
   module ETL
     module CLI
@@ -14,7 +13,7 @@ module Chronicle
         class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
         class_option :loader, aliases: '-l', desc: 'Loader class (available: stdout, csv, table)', default: 'stdout', banner: 'loader-name'
         class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-        class_option :job, aliases: '-j', desc: 'Job configuration name (or filename)'
+        class_option :name, aliases: '-j', desc: 'Job configuration name'
 
         map run: :start  # Thor doesn't like `run` as a command name
         desc "run", "Start a job"
@@ -31,30 +30,38 @@ module Chronicle
 LONG_DESC
         # Run an ETL job
         def start
-          runner_options = build_runner_options(options)
-          runner = Chronicle::ETL::Runner.new(runner_options)
+          job_definition = build_job_definition(options)
+          job = Chronicle::ETL::Job.new(job_definition)
+          runner = Chronicle::ETL::Runner.new(job)
           runner.run!
+        rescue Chronicle::ETL::ProviderNotAvailableError => e
+          warn(e.message.red)
+          warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
+          exit(false)
+        rescue Chronicle::ETL::ConnectorNotAvailableError => e
+          warn(e.message.red)
+          exit(false)
         end
 
         desc "create", "Create a job"
         # Create an ETL job
         def create
-          runner_options = build_runner_options(options)
-          path = File.join('chronicle', 'etl', 'jobs', options[:job])
-          Chronicle::ETL::Config.write(path, runner_options)
+          job_definition = build_job_definition(options)
+          path = File.join('chronicle', 'etl', 'jobs', options[:name])
+          Chronicle::ETL::Config.write(path, job_definition)
         end
 
         desc "show", "Show details about a job"
         # Show an ETL job
         def show
-          runner_options = build_runner_options(options)
-          pp runner_options
+          job_config = build_job_definition(options)
+          pp job_config
         end
 
         desc "list", "List all available jobs"
         # List available ETL jobs
         def list
-          jobs = Chronicle::ETL::Config.jobs
+          jobs = Chronicle::ETL::Config.available_jobs
 
           job_details = jobs.map do |job|
             r = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
@@ -74,17 +81,16 @@ LONG_DESC
 
         private
 
-        # Create runner options by reading config file and then overwriting with flag options
-        def build_runner_options options
-          flag_options = process_flag_options(options)
-          job_options = load_job(options[:job])
-          flag_options.merge(job_options)
+        # Create job definition by reading config file and then overwriting with flag options
+        def build_job_definition(options)
+          definition = Chronicle::ETL::JobDefinition.new
+          definition.add_config(process_flag_options(options))
+          definition.add_config(load_job_config(options[:name]))
+          definition
         end
 
-        def load_job job
-          yml_config = Chronicle::ETL::Config.load("chronicle/etl/jobs/#{job}.yml")
-          # FIXME: use better trick to depely symbolize keys
-          JSON.parse(yml_config.to_json, symbolize_names: true)
+        def load_job_config name
+          Chronicle::ETL::Config.load_job_from_config(name)
         end
 
         # Takes flag options and turns them into a runner config

data/lib/chronicle/etl/config.rb

@@ -4,15 +4,17 @@ module Chronicle
   module ETL
     # Utility methods to read, write, and access config files
     module Config
+      module_function
+
       # Loads a yml config file
-      def self.load(path)
+      def load(path)
         config = Runcom::Config.new(path)
         # FIXME: hack to deeply symbolize keys
         JSON.parse(config.to_h.to_json, symbolize_names: true)
       end
 
       # Writes a hash as a yml config file
-      def self.write(path, data)
+      def write(path, data)
         config = Runcom::Config.new(path)
         filename = config.all[0].to_s + '.yml'
         File.open(filename, 'w') do |f|
@@ -21,12 +23,31 @@ module Chronicle
       end
 
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
-      def self.jobs
+      def available_jobs
         job_directory = Runcom::Config.new('chronicle/etl/jobs').current
         Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
+
+      # Returns all available credentials available in ~/.config/chronilce/etl/credenetials/*.yml
+      def available_credentials
+        job_directory = Runcom::Config.new('chronicle/etl/credentials').current
+        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+          File.basename(filename, ".*")
+        end
+      end
+
+      # Load a job definition from job config directory
+      def load_job_from_config(job_name)
+        definition = self.load("chronicle/etl/jobs/#{job_name}.yml")
+        definition[:name] = job_name
+        definition
+      end
+
+      def load_credentials(name)
+        config = self.load("chronicle/etl/credentials/#{name}.yml")
+      end
     end
   end
 end

data/lib/chronicle/etl/extractors/extractor.rb

@@ -12,6 +12,7 @@ module Chronicle
       #   Options for configuring this Extractor
       def initialize(options = {})
         @options = options.transform_keys!(&:to_sym)
+        handle_continuation
       end
 
       # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
@@ -22,6 +23,12 @@ module Chronicle
       # An optional method to calculate how many records there are to extract. Used primarily for
       # building the progress bar
       def results_count; end
+
+      private
+
+      def handle_continuation
+        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation] && @options[:continuation].highest_timestamp
+      end
     end
   end
 end

data/lib/chronicle/etl/job.rb

@@ -0,0 +1,62 @@
+module Chronicle
+  module ETL
+    class Job
+      attr_accessor :name,
+                    :extractor_klass,
+                    :extractor_options,
+                    :transformer_klass,
+                    :transformer_options,
+                    :loader_klass,
+                    :loader_options
+
+      # TODO: build a proper id system
+      alias id name
+
+      def initialize(definition)
+        definition = definition.definition # FIXME
+        @name = definition[:name]
+        @extractor_klass = load_klass(:extractor, definition[:extractor][:name])
+        @extractor_options = definition[:extractor][:options] || {}
+
+        @transformer_klass = load_klass(:transformer, definition[:transformer][:name])
+        @transformer_options = definition[:transformer][:options] || {}
+
+        @loader_klass = load_klass(:loader, definition[:loader][:name])
+        @loader_options = definition[:loader][:options] || {}
+
+        set_continuation
+        yield self if block_given?
+      end
+
+      def instantiate_extractor
+        instantiate_klass(:extractor)
+      end
+
+      def instantiate_transformer data
+        instantiate_klass(:transformer, data)
+      end
+
+      def instantiate_loader
+        instantiate_klass(:loader)
+      end
+
+      private
+
+      def instantiate_klass(phase, *args)
+        options = self.send("#{phase.to_s}_options")
+        args = args.unshift(options)
+        klass = self.send("#{phase.to_s}_klass")
+        klass.new(*args)
+      end
+
+      def load_klass phase, identifier
+        Chronicle::ETL::Catalog.phase_and_identifier_to_klass(phase, identifier)
+      end
+
+      def set_continuation
+        continuation = Chronicle::ETL::JobLogger.load_latest(@job_id)
+        @extractor_options[:continuation] = continuation
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_definition.rb

@@ -0,0 +1,51 @@
+require 'deep_merge'
+
+module Chronicle
+  module ETL
+    class JobDefinition
+      SKELETON_DEFINITION = {
+        extractor: {
+          name: nil,
+          options: {}
+        },
+        transformer: {
+          name: nil,
+          options: {}
+        },
+        loader: {
+          name: nil,
+          options: {}
+        }
+      }.freeze
+
+      attr_accessor :definition
+
+      def initialize()
+        @definition = SKELETON_DEFINITION
+      end
+
+      # Add config hash to this definition
+      def add_config(config = {})
+        @definition = config.deep_merge(@definition)
+        load_credentials
+        validate
+      end
+
+      private
+
+      def load_credentials
+        Chronicle::ETL::Catalog::PHASES.each do |phase|
+          credentials_name = @definition[phase][:options][:credentials]
+          if credentials_name
+            credentials = Chronicle::ETL::Config.load_credentials(credentials_name)
+            @definition[phase][:options].deep_merge(credentials)
+          end
+        end
+      end
+
+      def validate
+        return true   # TODO
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_log.rb

@@ -0,0 +1,79 @@
+require 'pry'
+
+module Chronicle
+  module ETL
+    # A record of what happened in the running of a job. We're interested in
+    # tracking when it ran, if it was successful, and what the latest record
+    # we found is (to use as a cursor for the next time)
+    class JobLog
+      attr_accessor :job,
+                    :job_id,
+                    :last_id,
+                    :highest_timestamp,
+                    :num_records_processed,
+                    :started_at,
+                    :finished_at,
+                    :success
+
+      # Create a new JobLog for a given Job
+      def initialize
+        @num_records_processed = 0
+        @success = false
+        yield self if block_given?
+      end
+
+      # Log the result of a single transformation in a job
+      # @param transformer [Chronicle::ETL::Tranformer] The transformer that ran
+      def log_transformation(transformer)
+        @last_id = transformer.id if transformer.id
+
+        # Save the highest timestamp that we've encountered so far
+        @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
+
+        # TODO: a transformer might yield nil. We might also want certain transformers to explode
+        # records into multiple new ones. Therefore, this this variable will need more subtle behaviour
+        @num_records_processed += 1
+      end
+
+      # Indicate that a job has started
+      def start
+        @started_at = Time.now
+      end
+
+      # Indicate that a job has finished
+      def finish
+        @finished_at = Time.now
+        @success = true
+      end
+
+      def job= job
+        @job = job
+        @job_id = job.id
+      end
+
+      # Take a JobLog's instance variables and turn them into a hash representation
+      def serialize
+        {
+          job_id: @job_id,
+          last_id: @last_id,
+          highest_timestamp: @highest_timestamp,
+          num_records_processed: @num_records_processed,
+          started_at: @started_at,
+          finished_at: @finished_at,
+          success: @success
+        }
+      end
+
+      # Create a new JobLog and set its instance variables from a serialized hash
+      def self.build_from_serialized attrs
+        attrs.delete(:id)
+        new do |job_log|
+          attrs.each do |key, value|
+            setter = "#{key.to_s}=".to_sym
+            job_log.send(setter, value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_logger.rb

@@ -0,0 +1,76 @@
+require 'sequel'
+require 'forwardable'
+
+require 'pry'
+
+module Chronicle
+  module ETL
+    # Saves JobLogs to db and loads previous ones
+    class JobLogger
+      extend Forwardable
+
+      def_delegators :@job_log, :start, :finish, :log_transformation
+
+      # Create a new JobLogger
+      def initialize(job)
+        @job_log = JobLog.new do |job_log|
+          job_log.job = job
+        end
+      end
+
+      # Save this JobLogger's JobLog to db
+      def save
+        JobLogger.with_db_connection do |db|
+          dataset = db[:job_logs]
+          dataset.insert(@job_log.serialize)
+        end
+      end
+
+      # For a given `job_id`, return the last successful log
+      def self.load_latest(job_id)
+        with_db_connection do |db|
+          attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
+          JobLog.build_from_serialized(attrs) if attrs
+        end
+      end
+
+      def self.with_db_connection
+        initialize_db unless db_exists?
+        Sequel.connect("sqlite://#{db_filename}") do |db|
+          initialize_schema(db) unless schema_exists?(db)
+          yield db
+        end
+      end
+
+      def self.db_exists?
+        File.exists?(db_filename)
+      end
+
+      def self.schema_exists?(db)
+        return db.tables.include? :job_logs
+      end
+
+      def self.db_filename
+        data = Runcom::Data.new "chronicle/etl/job_log.db"
+        filename = data.all[0].to_s
+      end
+
+      def self.initialize_db
+        FileUtils.mkdir_p(File.dirname(db_filename))
+      end
+
+      def self.initialize_schema db
+        db.create_table :job_logs do
+          primary_key :id
+          String :job_id, null: false
+          String :last_id
+          Time :highest_timestamp
+          Integer :num_records_processed
+          boolean :success, default: false
+          Time :started_at
+          Time :finished_at
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -1,46 +1,32 @@
 require 'colorize'
 
 class Chronicle::ETL::Runner
-  def initialize(options = {})
-    @options = options
+  def initialize(job)
+    @job = job
+    @job_logger = Chronicle::ETL::JobLogger.new(@job)
   end
 
   def run!
-    extractor = instantiate_klass(:extractor)
-    loader = instantiate_klass(:loader)
+    extractor = @job.instantiate_extractor
+    loader = @job.instantiate_loader
+
+    @job_logger.start
+    loader.start
 
     total = extractor.results_count
     progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
 
-    loader.start
-
     extractor.extract do |data, metadata|
-      transformer = instantiate_klass(:transformer, data)
+      transformer = @job.instantiate_transformer(data)
       transformed_data = transformer.transform
-
+      @job_logger.log_transformation(transformer)
       loader.load(transformed_data)
       progress_bar.increment
     end
 
     progress_bar.finish
     loader.finish
-  end
-
-  private
-
-  def instantiate_klass(phase, *args)
-    klass = load_etl_class(phase, @options[phase][:name])
-    klass.new(@options[phase][:options], *args)
-  end
-
-  def load_etl_class(phase, identifier)
-    Chronicle::ETL::Catalog.identifier_to_klass(phase: phase, identifier: identifier)
-  rescue Chronicle::ETL::ProviderNotAvailableError => e
-    warn(e.message.red)
-    warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{e.provider}`")
-    exit(false)
-  rescue Chronicle::ETL::ConnectorNotAvailableError => e
-    warn(e.message.red)
-    exit(false)
+    @job_logger.finish
+    @job_logger.save
   end
 end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.2.2"
+    VERSION = "0.2.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-29 00:00:00.000000000 Z
+date: 2020-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.35'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.35'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +178,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -165,7 +207,6 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - ".yardopts"
-- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -188,6 +229,10 @@ files:
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb
 - lib/chronicle/etl/extractors/stdin_extractor.rb
+- lib/chronicle/etl/job.rb
+- lib/chronicle/etl/job_definition.rb
+- lib/chronicle/etl/job_log.rb
+- lib/chronicle/etl/job_logger.rb
 - lib/chronicle/etl/loaders/csv_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
 - lib/chronicle/etl/loaders/rest_loader.rb

data/CHANGELOG.md

@@ -1,23 +0,0 @@
-# Changelog
-
-This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.4] - 2020-08-18
-### Updated
-- Better display of available ETL classes
-- Updated documentation
-
-## [0.1.3] - 2020-08-13
-### Added
-- Ability to list all available ETL classes
-- Refactored E, T, L module and class structure
-- Better progress bar
-
-## [0.1.2] - 2020-08-02
-### Added
-- This changelog
-- Ability to use extractors, transformers, and loaders from other gems
-
-## [0.1.0] - 2020-08-01
-### Added
-- Basic job runner and ETL classes