chronicle-etl 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12a38a335c05b1626b9f259318956480df3f96e447cc2b1a25b8a9c23d591e49
-  data.tar.gz: d8ed027154403e68e5684213b0d0f58218a23dc2f667a882dcd3b2e8ab0c69b7
+  metadata.gz: 7a02a2377d0e8d4135f3b931bc73641eac28058d736d9c1dba0a97107c1d4c0e
+  data.tar.gz: 810d5bff80e852fa08ef9824ed6b313aa309bb69e84228bc1fbb7595069e043b
 SHA512:
-  metadata.gz: 396863ed665137905cfa9fe51ee925776e0a0f616721658a889b9b587dda83b9cd1e0fa2a483b08fc65ec70797f07facec082c1c88403aa8d61e1ce4ae791779
-  data.tar.gz: 705d626f45c816494949d6bc5c4f83cc4a8cd1c527aef72911bb90000e2151f758889c6080e9ac489f235df4169fbc158a7713dbf76d5e7ba5fdaf2a6ad51567
+  metadata.gz: 0d5fbea3c63349bb3f566e6137755f6cc8a4060d0e401abf5a0e7d8b44a4c4278089c10ffb8bb9cf2d783a238449140e5e54d90f3ad158aa362c6335eedca5aa
+  data.tar.gz: bf6fa83b1d5e55760e62d3cc090bf09bb69a7c761ae4a9358fb4d82192c7efc7500b6db361f39adac3581982862654aa4603a78dfbb3aed53b51d01137ffd736

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    chronicle-etl (0.2.3)
+    chronicle-etl (0.2.4)
       colorize (~> 0.8.1)
       deep_merge (~> 1.2)
       sequel (~> 5.35)

data/README.md

@@ -2,9 +2,9 @@
 
 [![Gem Version](https://badge.fury.io/rb/chronicle-etl.svg)](https://badge.fury.io/rb/chronicle-etl)
 
-Chronicle ETL is a utility tool for archiving and processing personal data. You can extract it from a variety of source, transform it, and load it to different APIs or file formats.
+Chronicle ETL is a utility that helps you archive and processes personal data. You can *extract* it from a variety of sources, *transform* it, and *load* it to an external API, file, or stdout.
 
-This project is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex).
+This tool is an adaptation of Andrew Louis's experimental [Memex project](https://hyfen.net/memex) and the dozens of existing importers are being migrated to Chronicle.
 
 ## Installation
 
@@ -52,7 +52,7 @@ Built in connectors:
 
 In addition to the built-in importers, importers for third-party platforms are available. They are packaged as individual Ruby gems.
 
-- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` files. Transformers for chronicle schema
+- [email](https://github.com/chronicle-app/chronicle-email). Extractors for `mbox` and other email files. Transformers for chronicle schema
 - [bash](https://github.com/chronicle-app/chronicle-bash). Extract bash history from `~/.bash_history`. Transform it for chronicle schema
 
 To install any of these, run `gem install chronicle-PROVIDER`. 

data/lib/chronicle/etl.rb

@@ -1,13 +1,19 @@
 require_relative 'etl/catalog'
 require_relative 'etl/config'
-require_relative 'etl/job_definition'
 require_relative 'etl/exceptions'
 require_relative 'etl/extractors/extractor'
+require_relative 'etl/job_definition'
 require_relative 'etl/job_log'
 require_relative 'etl/job_logger'
 require_relative 'etl/job'
 require_relative 'etl/loaders/loader'
+require_relative 'etl/models/activity'
+require_relative 'etl/models/base'
+require_relative 'etl/models/entity'
+require_relative 'etl/models/generic'
 require_relative 'etl/runner'
 require_relative 'etl/transformers/transformer'
+require_relative 'etl/utils/hash_utilities'
+require_relative 'etl/utils/jsonapi'
 require_relative 'etl/utils/progress_bar'
-require_relative 'etl/version'
+require_relative 'etl/version'

data/lib/chronicle/etl/exceptions.rb

@@ -2,6 +2,8 @@ module Chronicle
   module ETL
     class Error < StandardError; end;
 
+    class InvalidTransformedRecordError < Error; end
+
     class ConnectorNotAvailableError < Error
       def initialize(message, provider: nil, name: nil)
         super(message)

data/lib/chronicle/etl/extractors/csv_extractor.rb

@@ -28,8 +28,7 @@ class Chronicle::ETL::CsvExtractor < Chronicle::ETL::Extractor
 
     csv_options = {
       headers: headers,
-      header_converters: :symbol,
-      converters: [:all]
+      converters: :all
     }
 
     stream = read_from_file? ? File.open(@options[:filename]) : @options[:filename]

data/lib/chronicle/etl/extractors/extractor.rb

@@ -27,7 +27,10 @@ module Chronicle
       private
 
       def handle_continuation
-        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation] && @options[:continuation].highest_timestamp
+        return unless @options[:continuation]
+
+        @options[:load_since] = @options[:continuation].highest_timestamp if @options[:continuation].highest_timestamp
+        @options[:load_after_id] = @options[:continuation].last_id if @options[:continuation].last_id
       end
     end
   end

data/lib/chronicle/etl/job.rb

@@ -24,7 +24,7 @@ module Chronicle
         @loader_klass = load_klass(:loader, definition[:loader][:name])
         @loader_options = definition[:loader][:options] || {}
 
-        set_continuation
+        set_continuation if load_continuation?
         yield self if block_given?
       end
 
@@ -32,7 +32,7 @@ module Chronicle
         instantiate_klass(:extractor)
       end
 
-      def instantiate_transformer data
+      def instantiate_transformer(data)
         instantiate_klass(:transformer, data)
       end
 
@@ -40,6 +40,11 @@ module Chronicle
         instantiate_klass(:loader)
       end
 
+      def save_log?
+        # TODO: this needs more nuance
+        return !id.nil?
+      end
+
       private
 
       def instantiate_klass(phase, *args)
@@ -57,6 +62,10 @@ module Chronicle
         continuation = Chronicle::ETL::JobLogger.load_latest(@job_id)
         @extractor_options[:continuation] = continuation
       end
+
+      def load_continuation?
+        save_log?
+      end
     end
   end
 end

data/lib/chronicle/etl/job_log.rb

@@ -1,4 +1,4 @@
-require 'pry'
+require 'forwardable'
 
 module Chronicle
   module ETL
@@ -6,6 +6,8 @@ module Chronicle
     # tracking when it ran, if it was successful, and what the latest record
     # we found is (to use as a cursor for the next time)
     class JobLog
+      extend Forwardable
+
       attr_accessor :job,
                     :job_id,
                     :last_id,
@@ -15,6 +17,8 @@ module Chronicle
                     :finished_at,
                     :success
 
+      def_delegators :@job, :save_log?
+
       # Create a new JobLog for a given Job
       def initialize
         @num_records_processed = 0
@@ -64,6 +68,8 @@ module Chronicle
         }
       end
 
+      private
+
       # Create a new JobLog and set its instance variables from a serialized hash
       def self.build_from_serialized attrs
         attrs.delete(:id)

data/lib/chronicle/etl/job_logger.rb

@@ -20,6 +20,8 @@ module Chronicle
 
       # Save this JobLogger's JobLog to db
       def save
+        return unless @job_log.save_log?
+
         JobLogger.with_db_connection do |db|
           dataset = db[:job_logs]
           dataset.insert(@job_log.serialize)

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -8,12 +8,8 @@ module Chronicle
         @rows = []
       end
 
-      def load(result)
-        if (result.is_a? Hash)
-          @rows << result.values
-        else
-          @rows << result
-        end
+      def load(record)
+        @rows << record.to_h_flattened.values
       end
 
       def finish

data/lib/chronicle/etl/loaders/loader.rb

@@ -5,7 +5,7 @@ module Chronicle
       extend Chronicle::ETL::Catalog
 
       # Construct a new instance of this loader. Options are passed in from a Runner
-      # == Paramters:
+      # == Parameters:
       # options::
       #   Options for configuring this Loader
       def initialize(options = {})

data/lib/chronicle/etl/loaders/rest_loader.rb

@@ -5,25 +5,28 @@ require 'json'
 module Chronicle
   module ETL
     class RestLoader < Chronicle::ETL::Loader
-      def initialize(options={})
+      def initialize( options={} )
         super(options)
       end
 
-      def load(result)
+      def load(record)
+        payload = Chronicle::ETL::Utils::JSONAPI.serialize(record)
+        # have the outer data key that json-api expects
+        payload = { data: payload } unless payload[:data]
+
         uri = URI.parse("#{@options[:hostname]}#{@options[:endpoint]}")
 
         header = {
           "Authorization" => "Bearer #{@options[:access_token]}",
           "Content-Type": 'application/json'
         }
+        use_ssl = uri.scheme == 'https'
 
-        http = Net::HTTP.new(uri.host, uri.port)
-        request = Net::HTTP::Post.new(uri.request_uri, header)
-
-        obj = {data: result} unless result[:data]
-        request.body = obj.to_json
-
-        response = http.request(request)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
+          request = Net::HTTP::Post.new(uri.request_uri, header)
+          request.body = payload.to_json
+          http.request(request)
+        end
       end
     end
   end

data/lib/chronicle/etl/loaders/stdout_loader.rb

@@ -1,9 +1,9 @@
 module Chronicle
   module ETL
     class StdoutLoader < Chronicle::ETL::Loader
-      def load(result)
-        puts result.inspect
+      def load(record)
+        puts record.to_h
       end
     end
   end
-end
+end

data/lib/chronicle/etl/loaders/table_loader.rb

@@ -7,14 +7,15 @@ module Chronicle
         super(options)
       end
 
-      def load(result)
-        @table ||= TTY::Table.new(header: result.keys)
-        values = result.values.map{|x| x.to_s[0..30]}
+      def load(record)
+        record_hash = record.to_h_flattened
+        @table ||= TTY::Table.new(header: record_hash.keys)
+        values = record_hash.values.map{|x| x.to_s[0..30]}
         @table << values
       end
 
       def finish
-        puts @table.render(:ascii, padding: [0, 1])
+        puts @table.render(:ascii, padding: [0, 1]) if @table
       end
     end
   end

data/lib/chronicle/etl/models/activity.rb

@@ -0,0 +1,15 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Activity < Chronicle::ETL::Models::Base
+        TYPE = 'activities'.freeze
+        ATTRIBUTES = [:verb, :start_at, :end_at].freeze
+        ASSOCIATIONS = [:involved, :actor].freeze
+
+        attr_accessor(*ATTRIBUTES, *ASSOCIATIONS)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/base.rb

@@ -0,0 +1,103 @@
+require 'digest'
+
+module Chronicle
+  module ETL
+    module Models
+      # Represents a record that's been transformed by a Transformer and
+      # ready to be loaded. Loosely based on ActiveModel.
+      class Base
+        ATTRIBUTES = [:provider, :provider_id, :lat, :lng].freeze
+        ASSOCIATIONS = [].freeze
+
+        attr_accessor(:id, :dedupe_on, *ATTRIBUTES)
+
+        def initialize(attributes = {})
+          assign_attributes(attributes) if attributes
+          @dedupe_on = []
+        end
+
+        # A unique identifier for this model is formed from a type
+        # and either an id or lids.
+        def identifier_hash
+          {
+            type: self.class::TYPE,
+            id: @id,
+            lids: lids
+          }.compact
+        end
+
+        # Array of local ids that uniquely identify this record
+        def lids
+          @dedupe_on.map do |fields|
+            generate_lid(fields)
+          end.compact.uniq
+        end
+
+        # For a given set of fields of this model, generate a
+        # unique local id by hashing the field values
+        def generate_lid fields
+          values = fields.sort.map do |field|
+            instance_variable = "@#{field.to_s}"
+            self.instance_variable_get(instance_variable)
+          end
+
+          return if values.any? { |e| e.nil? }
+
+          Digest::SHA256.hexdigest(values.join(","))
+        end
+
+        # Set of attribute names that this model has is Base's shared
+        # attributes combined with the child class's
+        def attribute_list
+          (ATTRIBUTES + self.class::ATTRIBUTES).uniq
+        end
+
+        # All of this record's attributes
+        def attributes
+          attributes = {}
+          attribute_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            attributes[attribute] = self.instance_variable_get(instance_variable)
+          end
+          attributes.compact
+        end
+
+        # All of this record's associations
+        def associations
+          association_list = ASSOCIATIONS + self.class::ASSOCIATIONS
+          attributes = {}
+          association_list.each do |attribute|
+            instance_variable = "@#{attribute.to_s}"
+            association = self.instance_variable_get(instance_variable)
+            attributes[attribute] = association if association
+          end
+          attributes.compact
+        end
+
+        def associations_hash
+          Hash[associations.map do |k, v|
+            [k, v.to_h]
+          end]
+        end
+
+        # FIXME: move this to a Utils module
+        def to_h_flattened
+          Chronicle::ETL::Utils::HashUtilities.flatten_hash(to_h)
+        end
+
+        def to_h
+          identifier_hash.merge(attributes).merge(associations_hash)
+        end
+
+        private
+
+        def assign_attributes attributes
+          attributes.each do |k, v|
+            setter = :"#{k}="
+            public_send(setter, v) if respond_to? setter
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/entity.rb

@@ -0,0 +1,15 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Entity < Chronicle::ETL::Models::Base
+        TYPE = 'entities'.freeze
+        ATTRIBUTES = [:title, :body, :represents, :slug].freeze
+        ASSOCIATIONS = [].freeze  # TODO: add these to reflect Chronicle Schema
+
+        attr_accessor(*ATTRIBUTES)
+      end
+    end
+  end
+end

data/lib/chronicle/etl/models/generic.rb

@@ -0,0 +1,23 @@
+require 'chronicle/etl/models/base'
+
+module Chronicle
+  module ETL
+    module Models
+      class Generic < Chronicle::ETL::Models::Base
+        TYPE = 'generic'
+
+        attr_accessor :properties
+
+        def initialize(properties = {})
+          @properties = properties
+          super
+        end
+
+        # Generic models have arbitrary attributes stored in @properties
+        def attributes
+          @properties.transform_keys(&:to_sym)
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/runner.rb

@@ -18,9 +18,14 @@ class Chronicle::ETL::Runner
 
     extractor.extract do |data, metadata|
       transformer = @job.instantiate_transformer(data)
-      transformed_data = transformer.transform
+      record = transformer.transform
+
+      unless record.is_a?(Chronicle::ETL::Models::Base)
+        raise Chronicle::ETL::InvalidTransformedRecordError, "Transformed data is not a type of Chronicle::ETL::Models"
+      end
+
       @job_logger.log_transformation(transformer)
-      loader.load(transformed_data)
+      loader.load(record)
       progress_bar.increment
     end
 

data/lib/chronicle/etl/transformers/null_transformer.rb

@@ -2,9 +2,8 @@ module Chronicle
   module ETL
     class NullTransformer < Chronicle::ETL::Transformer
       def transform
-        return @data
+        Chronicle::ETL::Models::Generic.new(@data)
       end
     end
-
   end
-end
+end

data/lib/chronicle/etl/transformers/transformer.rb

@@ -11,12 +11,12 @@ module Chronicle
       def initialize(options = {}, data)
         @options = options
         @data = data
+        @record = Chronicle::ETL::Models::Activity.new
       end
 
-      # The main entrypoint for transforming a record. Called by a Runner on each extracted record
-      def transform
-        raise NotImplementedError
-      end
+      # @abstract Subclass is expected to implement #transform
+      # @!method transform
+      #   The main entrypoint for transforming a record. Called by a Runner on each extracted record
 
       # The domain or provider-specific id of the record this transformer is working on.
       # Used for building a cursor so an extractor doesn't have to start from the beginning of a
@@ -31,5 +31,4 @@ module Chronicle
   end
 end
 
-require_relative 'json_transformer'
 require_relative 'null_transformer'

data/lib/chronicle/etl/utils/hash_utilities.rb

@@ -0,0 +1,19 @@
+module Chronicle
+  module ETL
+    module Utils
+      module HashUtilities
+        def self.flatten_hash(hash)
+          hash.each_with_object({}) do |(k, v), h|
+            if v.is_a? Hash
+              flatten_hash(v).map do |h_k, h_v|
+                h["#{k}.#{h_k}".to_sym] = h_v
+              end
+            else 
+              h[k] = v
+            end
+           end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/utils/jsonapi.rb

@@ -0,0 +1,28 @@
+module Chronicle
+  module ETL
+    module Utils
+      module JSONAPI
+        # For a given Chronicle::ETL::Model, serialize it as jsonapi
+        def self.serialize(record)
+          return unless record.is_a? Chronicle::ETL::Models::Base
+
+          obj = record.identifier_hash
+          obj[:attributes] = record.attributes
+
+          relationships = Hash[record.associations.map do |k, v|
+            if v.is_a?(Array)
+              data = { data: v.map{ |association| serialize(association) } }
+            else
+              data = { data: serialize(v) }
+            end
+
+            [k, data]
+          end]
+
+          obj[:relationships] = relationships if relationships.any?
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/utils/progress_bar.rb

@@ -64,7 +64,7 @@ module Chronicle
         end
 
         def log(message)
-          @pbar.log message.inspect
+          @pbar.log message
         end
 
         def finish

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.2.3"
+    VERSION = "0.2.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-etl
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-09-04 00:00:00.000000000 Z
+date: 2020-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -238,10 +238,15 @@ files:
 - lib/chronicle/etl/loaders/rest_loader.rb
 - lib/chronicle/etl/loaders/stdout_loader.rb
 - lib/chronicle/etl/loaders/table_loader.rb
+- lib/chronicle/etl/models/activity.rb
+- lib/chronicle/etl/models/base.rb
+- lib/chronicle/etl/models/entity.rb
+- lib/chronicle/etl/models/generic.rb
 - lib/chronicle/etl/runner.rb
-- lib/chronicle/etl/transformers/json_transformer.rb
 - lib/chronicle/etl/transformers/null_transformer.rb
 - lib/chronicle/etl/transformers/transformer.rb
+- lib/chronicle/etl/utils/hash_utilities.rb
+- lib/chronicle/etl/utils/jsonapi.rb
 - lib/chronicle/etl/utils/progress_bar.rb
 - lib/chronicle/etl/version.rb
 homepage: https://github.com/chronicle-app

data/lib/chronicle/etl/transformers/json_transformer.rb

@@ -1,11 +0,0 @@
-require 'json'
-
-module Chronicle
-  module ETL
-    class JsonTransformer < Chronicle::ETL::Transformer
-      def transform data
-        return JSON.parse(data)
-      end
-    end
-  end
-end