chronicle-etl 0.2.2 → 0.3.1

data/lib/chronicle/etl/extractors/file_extractor.rb

@@ -3,49 +3,31 @@ require 'pathname'
 module Chronicle
   module ETL
     class FileExtractor < Chronicle::ETL::Extractor
-      def extract
-        if file?
-          extract_file do |data, metadata|
-            yield(data, metadata)
-          end
-        elsif directory?
-          extract_from_directory do |data, metadata|
-            yield(data, metadata)
-          end
-        end
-      end
+      include Extractors::Helpers::FilesystemReader
 
-      def results_count
-        if file?
-          return 1
-        else
-          search_pattern = File.join(@options[:filename], '**/*.eml')
-          Dir.glob(search_pattern).count
-        end
+      register_connector do |r|
+        r.description = 'file or directory of files'
       end
 
-      private
-
-      def extract_from_directory
-        search_pattern = File.join(@options[:filename], '**/*.eml')
-        filenames = Dir.glob(search_pattern)
+      def extract
         filenames.each do |filename|
-          file = File.open(filename)
-          yield(file.read, {filename: file})
+          yield Chronicle::ETL::Extraction.new(data: filename)
         end
       end
 
-      def extract_file
-        file = File.open(@options[:filename])
-        yield(file.read, {filename: @options[:filename]})
+      def results_count
+        filenames.count
       end
 
-      def directory?
-        Pathname.new(@options[:filename]).directory?
-      end
+      private
 
-      def file?
-        Pathname.new(@options[:filename]).file?
+      def filenames
+        @filenames ||= filenames_in_directory(
+          path: @options[:filename],
+          dir_glob_pattern: @options[:dir_glob_pattern],
+          load_since: @options[:load_since],
+          load_until: @options[:load_until]
+        )
       end
     end
   end

data/lib/chronicle/etl/extractors/helpers/filesystem_reader.rb

@@ -0,0 +1,104 @@
+require 'pathname'
+
+module Chronicle
+  module ETL
+    module Extractors
+      module Helpers
+        module FilesystemReader
+
+          def filenames_in_directory(...)
+            filenames = gather_files(...)
+            if block_given?
+              filenames.each do |filename|
+                yield filename
+              end
+            else
+              filenames
+            end
+          end
+
+          def read_from_filesystem(filename:, yield_each_line: true, dir_glob_pattern: '**/*')
+            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
+              if yield_each_line
+                file.each_line do |line|
+                  yield line
+                end
+              else
+                yield file.read
+              end
+            end
+          end
+
+          def open_from_filesystem(filename:, dir_glob_pattern: '**/*')
+            open_files(filename: filename, dir_glob_pattern: dir_glob_pattern) do |file|
+              yield file
+            end
+          end
+
+          def results_count
+            raise NotImplementedError
+            # if file?
+            #   return 1
+            # else
+            #   search_pattern = File.join(@options[:filename], '**/*')
+            #   Dir.glob(search_pattern).count
+            # end
+          end
+
+          private
+
+          def gather_files(path:, dir_glob_pattern: '**/*', load_since: nil, load_until: nil, smaller_than: nil, larger_than: nil, sort: :mtime)
+            search_pattern = File.join(path, '**', dir_glob_pattern)
+            files = Dir.glob(search_pattern)
+
+            files = files.keep_if {|f| (File.mtime(f) > load_since)} if load_since
+            files = files.keep_if {|f| (File.mtime(f) < load_until)} if load_until
+
+            # pass in file sizes in bytes
+            files = files.keep_if {|f| (File.size(f) < smaller_than)} if smaller_than
+            files = files.keep_if {|f| (File.size(f) > larger_than)} if larger_than
+
+            # TODO: incorporate sort argument
+            files.sort_by{ |f| File.mtime(f) }
+          end
+
+          def select_files_in_directory(path:, dir_glob_pattern: '**/*')
+            raise IOError.new("#{path} is not a directory.") unless directory?(path)
+
+            search_pattern = File.join(path, dir_glob_pattern)
+            Dir.glob(search_pattern).each do |filename|
+              yield(filename)
+            end
+          end
+
+          def open_files(filename:, dir_glob_pattern:)
+            if stdin?(filename)
+              yield $stdin
+            elsif directory?(filename)
+              search_pattern = File.join(filename, dir_glob_pattern)
+              filenames = Dir.glob(search_pattern)
+              filenames.each do |filename|
+                file = File.open(filename)
+                yield(file)
+              end
+            elsif file?(filename)
+              yield File.open(filename)
+            end
+          end
+
+          def stdin?(filename)
+            filename == $stdin
+          end
+
+          def directory?(filename)
+            Pathname.new(filename).directory?
+          end
+
+          def file?(filename)
+            Pathname.new(filename).file?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/json_extractor.rb

@@ -0,0 +1,45 @@
+module Chronicle
+  module ETL
+    class JsonExtractor < Chronicle::ETL::Extractor
+      include Extractors::Helpers::FilesystemReader
+
+      register_connector do |r|
+        r.description = 'input as JSON'
+      end
+
+      DEFAULT_OPTIONS = {
+        filename: $stdin,
+
+        # We're expecting line-separated json objects
+        jsonl: true
+      }.freeze
+
+      def initialize(options = {})
+        super(DEFAULT_OPTIONS.merge(options))
+      end
+
+      def extract
+        load_input do |input|
+          parsed_data = parse_data(input)
+          yield Chronicle::ETL::Extraction.new(data: parsed_data) if parsed_data
+        end
+      end
+
+      def results_count
+      end
+
+      private
+
+      def parse_data data
+        JSON.parse(data)
+      rescue JSON::ParserError => e
+      end
+
+      def load_input
+        read_from_filesystem(filename: @options[:filename]) do |data|
+          yield data
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/extractors/stdin_extractor.rb

@@ -1,9 +1,14 @@
 module Chronicle
   module ETL
     class StdinExtractor < Chronicle::ETL::Extractor
+      register_connector do |r|
+        r.description = 'stdin'
+      end
+
       def extract
         $stdin.read.each_line do |line|
-          yield line
+          data = { line: line.strip }
+          yield Chronicle::ETL::Extraction.new(data: data)
         end
       end
     end

data/lib/chronicle/etl/job.rb

@@ -0,0 +1,72 @@
+require 'forwardable'
+module Chronicle
+  module ETL
+    class Job
+      extend Forwardable
+
+      def_delegators :@job_definition, :dry_run?
+
+      attr_accessor :name,
+                    :extractor_klass,
+                    :extractor_options,
+                    :transformer_klass,
+                    :transformer_options,
+                    :loader_klass,
+                    :loader_options
+
+      # TODO: build a proper id system
+      alias id name
+
+      def initialize(job_definition)
+        @job_definition = job_definition
+        @name = @job_definition.definition[:name]
+        @extractor_options = @job_definition.extractor_options
+        @transformer_options = @job_definition.transformer_options
+        @loader_options = @job_definition.loader_options
+
+        set_continuation if use_continuation?
+        yield self if block_given?
+      end
+
+      def instantiate_extractor
+        @extractor_klass = @job_definition.extractor_klass
+        @extractor_klass.new(@extractor_options)
+      end
+
+      def instantiate_transformer(extraction)
+        @transformer_klass = @job_definition.transformer_klass
+        @transformer_klass.new(@transformer_options, extraction)
+      end
+
+      def instantiate_loader
+        @loader_klass = @job_definition.loader_klass
+        @loader_klass.new(@loader_options)
+      end
+
+      def save_log?
+        # TODO: this needs more nuance
+        return !id.nil?
+      end
+
+      def to_s
+        output = "Job"
+        output += " '#{name}'".bold if name
+        output += "\n"
+        output += "  → Extracting from #{@job_definition.extractor_klass.description}\n"
+        output += "  → Transforming #{@job_definition.transformer_klass.description}\n"
+        output += "  → Loading to #{@job_definition.loader_klass.description}\n"
+      end
+
+      private
+
+      def set_continuation
+        continuation = Chronicle::ETL::JobLogger.load_latest(@id)
+        @extractor_options[:continuation] = continuation
+      end
+
+      def use_continuation?
+        @job_definition.incremental?
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_definition.rb

@@ -0,0 +1,89 @@
+require 'active_support/core_ext/hash/deep_merge'
+
+module Chronicle
+  module ETL
+    class JobDefinition
+      SKELETON_DEFINITION = {
+        incremental: false,
+        extractor: {
+          name: 'stdin',
+          options: {}
+        },
+        transformer: {
+          name: 'null',
+          options: {}
+        },
+        loader: {
+          name: 'stdout',
+          options: {}
+        }
+      }.freeze
+
+      attr_accessor :definition
+
+      def initialize()
+        @definition = SKELETON_DEFINITION
+      end
+
+      # Add config hash to this definition
+      def add_config(config = {})
+        @definition = @definition.deep_merge(config)
+        load_credentials
+        validate
+      end
+
+      # Is this job continuing from a previous run?
+      def incremental?
+        @definition[:incremental]
+      end
+
+      def dry_run?
+        @definition[:dry_run]
+      end
+
+      def extractor_klass
+        load_klass(:extractor, @definition[:extractor][:name])
+      end
+
+      def transformer_klass
+        load_klass(:transformer, @definition[:transformer][:name])
+      end
+
+      def loader_klass
+        load_klass(:loader, @definition[:loader][:name])
+      end
+
+      def extractor_options
+        @definition[:extractor][:options]
+      end
+
+      def transformer_options
+        @definition[:transformer][:options]
+      end
+
+      def loader_options
+        @definition[:loader][:options]
+      end
+
+      private
+
+      def load_klass(phase, identifier)
+        Chronicle::ETL::Registry.find_by_phase_and_identifier(phase, identifier).klass
+      end
+
+      def load_credentials
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          credentials_name = @definition[phase].dig(:options, :credentials)
+          if credentials_name
+            credentials = Chronicle::ETL::Config.load_credentials(credentials_name)
+            @definition[phase][:options].deep_merge(credentials)
+          end
+        end
+      end
+
+      def validate
+        return true   # TODO
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_log.rb

@@ -0,0 +1,95 @@
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    # A record of what happened in the running of a job. We're interested in
+    # tracking when it ran, if it was successful, and what the latest record
+    # we found is (to use as a cursor for the next time)
+    class JobLog
+      extend Forwardable
+
+      attr_accessor :job,
+                    :job_id,
+                    :last_id,
+                    :highest_timestamp,
+                    :num_records_processed,
+                    :started_at,
+                    :finished_at,
+                    :success
+
+      def_delegators :@job, :save_log?
+
+      # Create a new JobLog for a given Job
+      def initialize
+        @num_records_processed = 0
+        @success = false
+        yield self if block_given?
+      end
+
+      # Log the result of a single transformation in a job
+      # @param transformer [Chronicle::ETL::Tranformer] The transformer that ran
+      def log_transformation(transformer)
+        @last_id = transformer.id if transformer.id
+
+        # Save the highest timestamp that we've encountered so far
+        @highest_timestamp = [transformer.timestamp, @highest_timestamp].compact.max if transformer.timestamp
+
+        # TODO: a transformer might yield nil. We might also want certain transformers to explode
+        # records into multiple new ones. Therefore, this this variable will need more subtle behaviour
+        @num_records_processed += 1
+      end
+
+      # Indicate that a job has started
+      def start
+        @started_at = Time.now
+      end
+
+      # Indicate that a job has finished
+      def finish
+        @finished_at = Time.now
+        @success = true
+      end
+
+      def error
+        @finished_at = Time.now
+      end
+
+      def job= job
+        @job = job
+        @job_id = job.id
+      end
+
+      def duration
+        return unless @finished_at
+
+        @finished_at - @started_at
+      end
+
+      # Take a JobLog's instance variables and turn them into a hash representation
+      def serialize
+        {
+          job_id: @job_id,
+          last_id: @last_id,
+          highest_timestamp: @highest_timestamp,
+          num_records_processed: @num_records_processed,
+          started_at: @started_at,
+          finished_at: @finished_at,
+          success: @success
+        }
+      end
+
+      private
+
+      # Create a new JobLog and set its instance variables from a serialized hash
+      def self.build_from_serialized attrs
+        attrs.delete(:id)
+        new do |job_log|
+          attrs.each do |key, value|
+            setter = "#{key.to_s}=".to_sym
+            job_log.send(setter, value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/job_logger.rb

@@ -0,0 +1,81 @@
+require 'sequel'
+require 'forwardable'
+
+module Chronicle
+  module ETL
+    # Saves JobLogs to db and loads previous ones
+    class JobLogger
+      extend Forwardable
+
+      def_delegators :@job_log, :start, :finish, :error, :log_transformation, :duration, :success
+      attr_accessor :job_log
+
+      # For a given `job_id`, return the last successful log
+      def self.load_latest(job_id)
+        with_db_connection do |db|
+          attrs = db[:job_logs].reverse_order(:finished_at).where(success: true).first
+          JobLog.build_from_serialized(attrs) if attrs
+        end
+      end
+
+      def self.with_db_connection
+        initialize_db unless db_exists?
+        Sequel.connect("sqlite://#{db_filename}") do |db|
+          initialize_schema(db) unless schema_exists?(db)
+          yield db
+        end
+      end
+
+      def self.db_exists?
+        File.exists?(db_filename)
+      end
+
+      def self.schema_exists?(db)
+        return db.tables.include? :job_logs
+      end
+
+      def self.db_filename
+        data = Runcom::Data.new "chronicle/etl/job_log.db"
+        filename = data.all[0].to_s
+      end
+
+      def self.initialize_db
+        FileUtils.mkdir_p(File.dirname(db_filename))
+      end
+
+      def self.initialize_schema db
+        db.create_table :job_logs do
+          primary_key :id
+          String :job_id, null: false
+          String :last_id
+          Time :highest_timestamp
+          Integer :num_records_processed
+          boolean :success, default: false
+          Time :started_at
+          Time :finished_at
+        end
+      end
+
+      # Create a new JobLogger
+      def initialize(job)
+        @job_log = JobLog.new do |job_log|
+          job_log.job = job
+        end
+      end
+
+      # Save this JobLogger's JobLog to db
+      def save
+        return unless @job_log.save_log?
+
+        JobLogger.with_db_connection do |db|
+          dataset = db[:job_logs]
+          dataset.insert(@job_log.serialize)
+        end
+      end
+
+      def summarize
+        @job_log.inspect
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -3,17 +3,17 @@ require 'csv'
 module Chronicle
   module ETL
     class CsvLoader < Chronicle::ETL::Loader
+      register_connector do |r|
+        r.description = 'CSV'
+      end
+
       def initialize(options={})
         super(options)
         @rows = []
       end
 
-      def load(result)
-        if (result.is_a? Hash)
-          @rows << result.values
-        else
-          @rows << result
-        end
+      def load(record)
+        @rows << record.to_h_flattened.values
       end
 
       def finish

data/lib/chronicle/etl/loaders/loader.rb

@@ -2,10 +2,10 @@ module Chronicle
   module ETL
     # Abstract class representing a Loader for an ETL job
     class Loader
-      extend Chronicle::ETL::Catalog
+      extend Chronicle::ETL::Registry::SelfRegistering
 
       # Construct a new instance of this loader. Options are passed in from a Runner
-      # == Paramters:
+      # == Parameters:
       # options::
       #   Options for configuring this Loader
       def initialize(options = {})

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -5,25 +5,32 @@ require 'json'
 module Chronicle
   module ETL
     class RestLoader < Chronicle::ETL::Loader
-      def initialize(options={})
+      register_connector do |r|
+        r.description = 'a REST endpoint'
+      end
+
+      def initialize( options={} )
         super(options)
       end
 
-      def load(result)
+      def load(record)
+        payload = Chronicle::ETL::JSONAPISerializer.serialize(record)
+        # have the outer data key that json-api expects
+        payload = { data: payload } unless payload[:data]
+
         uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
 
         header = {
           "Authorization" => "Bearer #{@options[:access_token]}",
           "Content-Type": 'application/json'
         }
+        use_ssl = uri.scheme == 'https'
 
-        http = Net::HTTP.new(uri.host, uri.port)
-        request = Net::HTTP::Post.new(uri.request_uri, header)
-
-        obj = {data: result} unless result[:data]
-        request.body = obj.to_json
-
-        response = http.request(request)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
+          request = Net::HTTP::Post.new(uri.request_uri, header)
+          request.body = payload.to_json
+          http.request(request)
+        end
       end
     end
   end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,9 +1,14 @@
 module Chronicle
   module ETL
     class StdoutLoader < Chronicle::ETL::Loader
-      def load(result)
-        puts result.inspect
+      register_connector do |r|
+        r.description = 'stdout'
+      end
+
+      def load(record)
+        serializer = Chronicle::ETL::JSONAPISerializer.new(record)
+        puts serializer.serializable_hash.to_json
       end
     end
   end
-end
+end