chronicle-etl 0.1.4 → 0.2.4

data/lib/chronicle/etl/cli/main.rb

@@ -0,0 +1,83 @@
+require 'thor'
+require 'chronicle/etl'
+require 'colorize'
+
+require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/connectors'
+require 'chronicle/etl/cli/jobs'
+
+module Chronicle
+  module ETL
+    module CLI
+      # Main entrypoint for CLI app
+      class Main < Thor
+        class_option "verbose", type: :boolean, default: false
+        default_task "jobs"
+
+        desc 'connectors:COMMAND', 'Connectors available for ETL jobs', hide: true
+        subcommand 'connectors', Connectors
+
+        desc 'jobs:COMMAND', 'Configure and run jobs', hide: true
+        subcommand 'jobs', Jobs
+
+        # Entrypoint for the CLI
+        def self.start(given_args = ARGV, config = {})
+          if given_args.none?
+            abort "No command entered or job specified. To see commands, run `chronicle-etl help`".red
+          end
+
+          # take a subcommand:command and splits them so Thor knows how to hand off to the subcommand class
+          if given_args.any? && given_args[0].include?(':')
+            commands = given_args.shift.split(':')
+            given_args = given_args.unshift(commands).flatten
+          end
+
+          super(given_args, config)
+        end
+
+        # Displays help options for chronicle-etl
+        def help(meth = nil, subcommand = false)
+          if meth && !respond_to?(meth)
+            klass, task = Thor::Util.find_class_and_task_by_namespace("#{meth}:#{meth}")
+            klass.start(['-h', task].compact, shell: shell)
+          else
+            shell.say "ABOUT".bold
+            shell.say "  #{'chronicle-etl'.italic} is a utility tool for #{'extracting'.underline}, #{'transforming'.underline}, and #{'loading'.underline} personal data."
+            shell.say
+            shell.say "USAGE".bold
+            shell.say "  $ chronicle-etl COMMAND"
+            shell.say
+            shell.say "EXAMPLES".bold
+            shell.say "  Show available connectors:".italic.light_black
+            shell.say "  $ chronicle-etl connectors:list"
+            shell.say
+            shell.say "  Run a simple job:".italic.light_black
+            shell.say "  $ chronicle-etl jobs:start --extractor stdin --transformer null --loader stdout"
+            shell.say
+            shell.say "  Show full job options:".italic.light_black
+            shell.say "  $ chronicle-etl jobs help start"
+
+            list = []
+
+            Thor::Util.thor_classes_in(Chronicle::ETL::CLI).each do |thor_class|
+              list += thor_class.printable_tasks(false)
+            end
+            list.sort! { |a, b| a[0] <=> b[0] }
+            list.unshift ["help", "# This help menu"]
+
+            shell.say
+            shell.say 'ALL COMMANDS'.bold
+            shell.print_table(list, indent: 2, truncate: true)
+            shell.say
+            shell.say "VERSION".bold
+            shell.say "  #{Chronicle::ETL::VERSION}"
+            shell.say
+            shell.say "FULL DOCUMENTATION".bold
+            shell.say "  https://github.com/chronicle-app/chronicle-etl".blue
+            shell.say
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/cli/subcommand_base.rb

@@ -0,0 +1,37 @@
+module Chronicle
+  module ETL
+    module CLI
+      # Base class for CLI subcommands. Overrides Thor methods so we can use command:subcommand syntax
+      class SubcommandBase < Thor
+        # Print usage instructions for a subcommand
+        def self.help(shell, subcommand = false)
+          list = printable_commands(true, subcommand)
+          Thor::Util.thor_classes_in(self).each do |klass|
+            list += klass.printable_commands(false)
+          end
+          list.sort! { |a, b| a[0] <=> b[0] }
+
+          shell.say "COMMANDS".bold
+          shell.print_table(list, indent: 2, truncate: true)
+          shell.say
+          class_options_help(shell)
+        end
+
+        # Show docs with command:subcommand pattern.
+        # For `help` command, don't use colon
+        def self.banner(command, namespace = nil, subcommand = false)
+          if command.name == 'help'
+            "#{subcommand_prefix} #{command.usage}"
+          else
+            "#{subcommand_prefix}:#{command.usage}"
+          end
+        end
+
+        # Use subcommand classname to derive display name for subcommand
+        def self.subcommand_prefix
+          self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/config.rb

@@ -0,0 +1,53 @@
+require 'runcom'
+
+module Chronicle
+  module ETL
+    # Utility methods to read, write, and access config files
+    module Config
+      module_function
+
+      # Loads a yml config file
+      def load(path)
+        config = Runcom::Config.new(path)
+        # FIXME: hack to deeply symbolize keys
+        JSON.parse(config.to_h.to_json, symbolize_names: true)
+      end
+
+      # Writes a hash as a yml config file
+      def write(path, data)
+        config = Runcom::Config.new(path)
+        filename = config.all[0].to_s + '.yml'
+        File.open(filename, 'w') do |f|
+          f << data.to_yaml
+        end
+      end
+
+      # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
+      def available_jobs
+        job_directory = Runcom::Config.new('chronicle/etl/jobs').current
+        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+          File.basename(filename, ".*")
+        end
+      end
+
+      # Returns all available credentials available in ~/.config/chronilce/etl/credenetials/*.yml
+      def available_credentials
+        job_directory = Runcom::Config.new('chronicle/etl/credentials').current
+        Dir.glob(File.join(job_directory, "*.yml")).map do |filename|
+          File.basename(filename, ".*")
+        end
+      end
+
+      # Load a job definition from job config directory
+      def load_job_from_config(job_name)
+        definition = self.load("chronicle/etl/jobs/#{job_name}.yml")
+        definition[:name] = job_name
+        definition
+      end
+
+      def load_credentials(name)
+        config = self.load("chronicle/etl/credentials/#{name}.yml")
+      end
+    end
+  end
+end

data/lib/chronicle/etl/exceptions.rb

@@ -0,0 +1,19 @@
+module Chronicle
+  module ETL
+    class Error < StandardError; end;
+
+    class InvalidTransformedRecordError < Error; end
+
+    class ConnectorNotAvailableError < Error
+      def initialize(message, provider: nil, name: nil)
+        super(message)
+        @provider = provider
+        @name = name
+      end
+      attr_reader :name, :provider
+    end
+
+    class ProviderNotAvailableError < ConnectorNotAvailableError; end
+    class ProviderConnectorNotAvailableError < ConnectorNotAvailableError; end
+  end
+end

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -1,5 +1,5 @@
 require 'csv'
-class Chronicle::Etl::CsvExtractor < Chronicle::Etl::Extractor
+class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
   DEFAULT_OPTIONS = {
     headers: true,
     filename: $stdin
@@ -28,8 +28,7 @@ class Chronicle::Etl::CsvExtractor < Chronicle::Etl::Extractor
 
     csv_options = {
       headers: headers,
-      header_converters: :symbol,
-      converters: [:all]
+      converters: :all
     }
 
     stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]

data/lib/chronicle/etl/extractors/extractor.rb

@@ -1,21 +1,37 @@
 require 'chronicle/etl'
 
 module Chronicle
-  module Etl
+  module ETL
+    # Abstract class representing an Extractor for an ETL job
     class Extractor
-      extend Chronicle::Etl::Catalog
-
-      ETL_PHASE = :extractor
+      extend Chronicle::ETL::Catalog
 
+      # Construct a new instance of this extractor. Options are passed in from a Runner
+      # == Paramters:
+      # options::
+      #   Options for configuring this Extractor
       def initialize(options = {})
-        @options = options.transform_keys!(&:to_sym) 
+        @options = options.transform_keys!(&:to_sym)
+        handle_continuation
       end
 
+      # Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
       def extract
         raise NotImplementedError
       end
 
+      # An optional method to calculate how many records there are to extract. Used primarily for
+      # building the progress bar
       def results_count; end
+
+      private
+
+      def handle_continuation
+        return unless @options[:continuation]
+
+        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation].highest_timestamp
+        @options[:load_after_id] = @options[:continuation].last_id if @options[:continuation].last_id
+      end
     end
   end
 end

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -1,8 +1,8 @@
 require 'pathname'
 
 module Chronicle
-  module Etl
-    class FileExtractor < Chronicle::Etl::Extractor
+  module ETL
+    class FileExtractor < Chronicle::ETL::Extractor
       def extract
         if file?
           extract_file do |data, metadata|

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -1,6 +1,6 @@
 module Chronicle
-  module Etl
-    class StdinExtractor < Chronicle::Etl::Extractor
+  module ETL
+    class StdinExtractor < Chronicle::ETL::Extractor
       def extract
         $stdin.read.each_line do |line|
           yield line

data/lib/chronicle/etl/job.rb

@@ -0,0 +1,71 @@
+module Chronicle
+  module ETL
+    class Job
+      attr_accessor :name,
+                    :extractor_klass,
+                    :extractor_options,
+                    :transformer_klass,
+                    :transformer_options,
+                    :loader_klass,
+                    :loader_options
+
+      # TODO: build a proper id system
+      alias id name
+
+      def initialize(definition)
+        definition = definition.definition # FIXME
+        @name = definition[:name]
+        @extractor_klass = load_klass(:extractor, definition[:extractor][:name])
+        @extractor_options = definition[:extractor][:options] || {}
+
+        @transformer_klass = load_klass(:transformer, definition[:transformer][:name])
+        @transformer_options = definition[:transformer][:options] || {}
+
+        @loader_klass = load_klass(:loader, definition[:loader][:name])
+        @loader_options = definition[:loader][:options] || {}
+
+        set_continuation if load_continuation?
+        yield self if block_given?
+      end
+
+      def instantiate_extractor
+        instantiate_klass(:extractor)
+      end
+
+      def instantiate_transformer(data)
+        instantiate_klass(:transformer, data)
+      end
+
+      def instantiate_loader
+        instantiate_klass(:loader)
+      end
+
+      def save_log?
+        # TODO: this needs more nuance
+        return !id.nil?
+      end
+
+      private
+
+      def instantiate_klass(phase, *args)
+        options = self.send("#{phase.to_s}_options")
+        args = args.unshift(options)
+        klass = self.send("#{phase.to_s}_klass")
+        klass.new(*args)
+      end
+
+      def load_klass phase, identifier
+        Chronicle::ETL::Catalog.phase_and_identifier_to_klass(phase, identifier)
+      end
+
+      def set_continuation
+        continuation = Chronicle::ETL::JobLogger.load_latest(@job_id)
+        @extractor_options[:continuation] = continuation
+      end
+
+      def load_continuation?
+        save_log?
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_definition.rb

@@ -0,0 +1,51 @@
+require 'deep_merge'
+
+module Chronicle
+  module ETL
+    class JobDefinition
+      SKELETON_DEFINITION = {
+        extractor: {
+          name: nil,
+          options: {}
+        },
+        transformer: {
+          name: nil,
+          options: {}
+        },
+        loader: {
+          name: nil,
+          options: {}
+        }
+      }.freeze
+
+      attr_accessor :definition
+
+      def initialize()
+        @definition = SKELETON_DEFINITION
+      end
+
+      # Add config hash to this definition
+      def add_config(config = {})
+        @definition = config.deep_merge(@definition)
+        load_credentials
+        validate
+      end
+
+      private
+
+      def load_credentials
+        Chronicle::ETL::Catalog::PHASES.each do |phase|
+          credentials_name = @definition[phase][:options][:credentials]
+          if credentials_name
+            credentials = Chronicle::ETL::Config.load_credentials(credentials_name)
+            @definition[phase][:options].deep_merge(credentials)
+          end
+        end
+      end
+
+      def validate
+        return true   # TODO
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_log.rb

@@ -0,0 +1,85 @@
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    # A record of what happened in the running of a job. We're interested in
+    # tracking when it ran, if it was successful, and what the latest record
+    # we found is (to use as a cursor for the next time)
+    class JobLog
+      extend Forwardable
+
+      attr_accessor :job,
+                    :job_id,
+                    :last_id,
+                    :highest_timestamp,
+                    :num_records_processed,
+                    :started_at,
+                    :finished_at,
+                    :success
+
+      def_delegators :@job, :save_log?
+
+      # Create a new JobLog for a given Job
+      def initialize
+        @num_records_processed = 0
+        @success = false
+        yield self if block_given?
+      end
+
+      # Log the result of a single transformation in a job
+      # @param transformer [Chronicle::ETL::Tranformer] The transformer that ran
+      def log_transformation(transformer)
+        @last_id = transformer.id if transformer.id
+
+        # Save the highest timestamp that we've encountered so far
+        @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
+
+        # TODO: a transformer might yield nil. We might also want certain transformers to explode
+        # records into multiple new ones. Therefore, this this variable will need more subtle behaviour
+        @num_records_processed += 1
+      end
+
+      # Indicate that a job has started
+      def start
+        @started_at = Time.now
+      end
+
+      # Indicate that a job has finished
+      def finish
+        @finished_at = Time.now
+        @success = true
+      end
+
+      def job= job
+        @job = job
+        @job_id = job.id
+      end
+
+      # Take a JobLog's instance variables and turn them into a hash representation
+      def serialize
+        {
+          job_id: @job_id,
+          last_id: @last_id,
+          highest_timestamp: @highest_timestamp,
+          num_records_processed: @num_records_processed,
+          started_at: @started_at,
+          finished_at: @finished_at,
+          success: @success
+        }
+      end
+
+      private
+
+      # Create a new JobLog and set its instance variables from a serialized hash
+      def self.build_from_serialized attrs
+        attrs.delete(:id)
+        new do |job_log|
+          attrs.each do |key, value|
+            setter = "#{key.to_s}=".to_sym
+            job_log.send(setter, value)
+          end
+        end
+      end
+    end
+  end
+end