chronicle-etl 0.1.3 → 0.2.3

data/lib/chronicle/etl/job_logger.rb

@@ -0,0 +1,76 @@
+require 'sequel'
+require 'forwardable'
+
+require 'pry'
+
+module Chronicle
+  module ETL
+    # Saves JobLogs to db and loads previous ones
+    class JobLogger
+      extend Forwardable
+
+      def_delegators :@job_log, :start, :finish, :log_transformation
+
+      # Create a new JobLogger
+      def initialize(job)
+        @job_log = JobLog.new do |job_log|
+          job_log.job = job
+        end
+      end
+
+      # Save this JobLogger's JobLog to db
+      def save
+        JobLogger.with_db_connection do |db|
+          dataset = db[:job_logs]
+          dataset.insert(@job_log.serialize)
+        end
+      end
+
+      # For a given `job_id`, return the last successful log
+      def self.load_latest(job_id)
+        with_db_connection do |db|
+          attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
+          JobLog.build_from_serialized(attrs) if attrs
+        end
+      end
+
+      def self.with_db_connection
+        initialize_db unless db_exists?
+        Sequel.connect("sqlite://#{db_filename}") do |db|
+          initialize_schema(db) unless schema_exists?(db)
+          yield db
+        end
+      end
+
+      def self.db_exists?
+        File.exists?(db_filename)
+      end
+
+      def self.schema_exists?(db)
+        return db.tables.include? :job_logs
+      end
+
+      def self.db_filename
+        data = Runcom::Data.new "chronicle/etl/job_log.db"
+        filename = data.all[0].to_s
+      end
+
+      def self.initialize_db
+        FileUtils.mkdir_p(File.dirname(db_filename))
+      end
+
+      def self.initialize_schema db
+        db.create_table :job_logs do
+          primary_key :id
+          String :job_id, null: false
+          String :last_id
+          Time :highest_timestamp
+          Integer :num_records_processed
+          boolean :success, default: false
+          Time :started_at
+          Time :finished_at
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -1,8 +1,8 @@
 require 'csv'
 
 module Chronicle
-  module Etl
-    class CsvLoader < Chronicle::Etl::Loader
+  module ETL
+    class CsvLoader < Chronicle::ETL::Loader
       def initialize(options={})
         super(options)
         @rows = []

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,25 +1,32 @@
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing a Loader for an ETL job
     class Loader
-      extend Chronicle::Etl::Catalog
-      
+      extend Chronicle::ETL::Catalog
+
+      # Construct a new instance of this loader. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Loader
       def initialize(options = {})
         @options = options
       end
 
+      # Called once before processing records
       def start; end
 
-      def first_load result; end
-
+      # Load a single record
       def load
         raise NotImplementedError
       end
 
+      # Called once there are no more records to process
       def finish; end
     end
   end
 end
 
 require_relative 'csv_loader'
+require_relative 'rest_loader'
 require_relative 'stdout_loader'
-require_relative 'table_loader'
+require_relative 'table_loader'

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -0,0 +1,30 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Chronicle
+  module ETL
+    class RestLoader < Chronicle::ETL::Loader
+      def initialize(options={})
+        super(options)
+      end
+
+      def load(result)
+        uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
+
+        header = {
+          "Authorization" => "Bearer #{@options[:access_token]}",
+          "Content-Type": 'application/json'
+        }
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        request = Net::HTTP::Post.new(uri.request_uri, header)
+
+        obj = {data: result} unless result[:data]
+        request.body = obj.to_json
+
+        response = http.request(request)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,6 +1,6 @@
 module Chronicle
-  module Etl
-    class StdoutLoader < Chronicle::Etl::Loader
+  module ETL
+    class StdoutLoader < Chronicle::ETL::Loader
       def load(result)
         puts result.inspect
       end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -1,24 +1,20 @@
 require 'tty/table'
 
 module Chronicle
-  module Etl
-    class TableLoader < Chronicle::Etl::Loader
+  module ETL
+    class TableLoader < Chronicle::ETL::Loader
       def initialize(options)
         super(options)
       end
 
-      # defer creating table until we get first result and can determine headers
-      def first_load(result)
-        headers = result.keys
-        @table = TTY::Table.new(header: headers)
-      end
-
       def load(result)
-        @table << result
+        @table ||= TTY::Table.new(header: result.keys)
+        values = result.values.map{|x| x.to_s[0..30]}
+        @table << values
       end
 
       def finish
-        puts @table.render(:ascii)
+        puts @table.render(:ascii, padding: [0, 1])
       end
     end
   end

data/lib/chronicle/etl/runner.rb

@@ -1,64 +1,32 @@
-class Chronicle::Etl::Runner
-  BUILTIN = {
-    extractor: ['stdin', 'json', 'csv', 'file'],
-    transformer: ['null'],
-    loader: ['stdout', 'csv', 'table']
-  }.freeze
+require 'colorize'
 
-  def initialize(options)
-    @options = options
-
-    instantiate_etl_classes
+class Chronicle::ETL::Runner
+  def initialize(job)
+    @job = job
+    @job_logger = Chronicle::ETL::JobLogger.new(@job)
   end
 
   def run!
-    progress_bar = Chronicle::Etl::Utils::ProgressBar.new(title: "Running job", total: @extractor.results_count)
-    count = 0
+    extractor = @job.instantiate_extractor
+    loader = @job.instantiate_loader
 
-    @loader.start
+    @job_logger.start
+    loader.start
 
-    @extractor.extract do |data, metadata|
-      transformed_data = @transformer.transform(data)
-      
-      @loader.first_load(transformed_data) if count == 0
-      @loader.load(transformed_data)
+    total = extractor.results_count
+    progress_bar = Chronicle::ETL::Utils::ProgressBar.new(title: 'Running job', total: total)
 
+    extractor.extract do |data, metadata|
+      transformer = @job.instantiate_transformer(data)
+      transformed_data = transformer.transform
+      @job_logger.log_transformation(transformer)
+      loader.load(transformed_data)
       progress_bar.increment
-      count += 1
-    # rescue StandardError => e
-    #   require 'pry'
-    #   binding.pry
-    #   progress_bar.log "Error processing; #{e.inspect}"
     end
 
     progress_bar.finish
-    @loader.finish
-  end
-
-  private
-
-  def instantiate_etl_classes
-    @extractor = load_etl_class(:extractor, @options[:extractor][:name]).new(@options[:extractor][:options])
-    @transformer = load_etl_class(:transformer, @options[:transformer][:name]).new(@options[:transformer][:options])
-    @loader = load_etl_class(:loader, @options[:loader][:name]).new(@options[:loader][:options])
-  end
-
-  def load_etl_class(phase, name)
-    if BUILTIN[phase].include? name
-      klass_name = "Chronicle::Etl::#{name.capitalize}#{phase.to_s.capitalize}"
-    else
-      # TODO: come up with syntax for specifying a particular extractor in a provider library
-      # provider, extractor = name.split(":")
-      provider = name
-      begin
-        require "chronicle/#{provider}"
-      rescue LoadError => e
-        warn("Error loading #{phase} '#{provider}'")
-        warn("  Perhaps you haven't installed it yet: `$ gem install chronicle-#{provider}`")
-        exit(false)
-      end
-      klass_name = "Chronicle::#{name.capitalize}::ChronicleTransformer"
-    end
-    Object.const_get(klass_name)
+    loader.finish
+    @job_logger.finish
+    @job_logger.save
   end
 end

data/lib/chronicle/etl/transformers/json_transformer.rb

@@ -1,8 +1,8 @@
 require 'json'
 
 module Chronicle
-  module Etl
-    class JsonTransformer < Chronicle::Etl::Transformer
+  module ETL
+    class JsonTransformer < Chronicle::ETL::Transformer
       def transform data
         return JSON.parse(data)
       end

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -1,8 +1,8 @@
 module Chronicle
-  module Etl
-    class NullTransformer < Chronicle::Etl::Transformer
-      def transform data
-        return data
+  module ETL
+    class NullTransformer < Chronicle::ETL::Transformer
+      def transform
+        return @data
       end
     end
 

data/lib/chronicle/etl/transformers/transformer.rb

@@ -1,15 +1,32 @@
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing an Transformer for an ETL job
     class Transformer
-      extend Chronicle::Etl::Catalog
+      extend Chronicle::ETL::Catalog
 
-      def initialize(options = {})
+      # Construct a new instance of this transformer. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Transformer
+      def initialize(options = {}, data)
         @options = options
+        @data = data
       end
 
-      def transform data
+      # The main entrypoint for transforming a record. Called by a Runner on each extracted record
+      def transform
         raise NotImplementedError
       end
+
+      # The domain or provider-specific id of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def id; end
+
+      # The domain or provider-specific timestamp of the record this transformer is working on.
+      # Used for building a cursor so an extractor doesn't have to start from the beginning of a
+      # data source from the beginning.
+      def timestamp; end
     end
   end
 end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -2,7 +2,7 @@ require 'tty/progressbar'
 require 'colorize'
 
 module Chronicle
-  module Etl
+  module ETL
     module Utils
 
       class ProgressBar

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
-  module Etl
-    VERSION = "0.1.3"
+  module ETL
+    VERSION = "0.2.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.35'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.35'
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +150,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: runcom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 description: Chronicle-ETL allows you to extract personal data from a variety of services,
   transformer it, and load it.
 email:
@@ -133,9 +203,10 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
-- CHANGELOG.md
+- ".yardopts"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -148,13 +219,23 @@ files:
 - exe/chronicle-etl
 - lib/chronicle/etl.rb
 - lib/chronicle/etl/catalog.rb
-- lib/chronicle/etl/cli.rb
+- lib/chronicle/etl/cli/connectors.rb
+- lib/chronicle/etl/cli/jobs.rb
+- lib/chronicle/etl/cli/main.rb
+- lib/chronicle/etl/cli/subcommand_base.rb
+- lib/chronicle/etl/config.rb
+- lib/chronicle/etl/exceptions.rb
 - lib/chronicle/etl/extractors/csv_extractor.rb
 - lib/chronicle/etl/extractors/extractor.rb
 - lib/chronicle/etl/extractors/file_extractor.rb
 - lib/chronicle/etl/extractors/stdin_extractor.rb
+- lib/chronicle/etl/job.rb
+- lib/chronicle/etl/job_definition.rb
+- lib/chronicle/etl/job_log.rb
+- lib/chronicle/etl/job_logger.rb
 - lib/chronicle/etl/loaders/csv_loader.rb
 - lib/chronicle/etl/loaders/loader.rb
+- lib/chronicle/etl/loaders/rest_loader.rb
 - lib/chronicle/etl/loaders/stdout_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
 - lib/chronicle/etl/runner.rb