chronicle-etl 0.4.4 → 0.5.2

data/lib/chronicle/etl/config.rb

@@ -1,55 +1,73 @@
-require 'runcom'
+require 'fileutils'
+require 'yaml'
 
 module Chronicle
   module ETL
     # Utility methods to read, write, and access config files
     module Config
-      module_function
+      extend self
 
-      # Loads a yml config file
-      def load(path)
-        config = Runcom::Config.new(path)
-        # FIXME: hack to deeply symbolize keys
-        JSON.parse(config.to_h.to_json, symbolize_names: true)
+      attr_accessor :xdg_environment
+
+      def load(type, identifier)
+        base = config_pathname_for_type(type)
+        path = base.join("#{identifier}.yml")
+        return {} unless path.exist?
+
+        YAML.safe_load(File.read(path), symbolize_names: true, permitted_classes: [Symbol, Date, Time])
       end
 
       # Writes a hash as a yml config file
-      def write(path, data)
-        config = Runcom::Config.new(path)
-        filename = config.all[0].to_s + '.yml'
-        File.open(filename, 'w') do |f|
-          f << data.to_yaml
+      def write(type, identifier, data)
+        base = config_pathname_for_type(type)
+        path = base.join("#{identifier}.yml")
+        FileUtils.mkdir_p(File.dirname(path))
+        File.open(path, 'w', 0o600) do |f|
+          # Ruby likes to add --- separators when writing yaml files
+          f << data.to_yaml.gsub(/^-+\n/, '')
         end
       end
 
+      def exists?(type, identifier)
+        base = config_pathname_for_type(type)
+        path = base.join("#{identifier}.yml")
+        return path.exist?
+      end
+
       # Returns all jobs available in ~/.config/chronicle/etl/jobs/*.yml
       def available_jobs
-        Dir.glob(File.join(config_directory("jobs"), "*.yml")).map do |filename|
+        Dir.glob(File.join(config_pathname_for_type("jobs"), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
 
-      # Returns all available credentials available in ~/.config/chronicle/etl/credentials/*.yml
-      def available_credentials
-        Dir.glob(File.join(config_directory("credentials"), "*.yml")).map do |filename|
+      def available_configs(type)
+        Dir.glob(File.join(config_pathname_for_type(type), "*.yml")).map do |filename|
           File.basename(filename, ".*")
         end
       end
 
       # Load a job definition from job config directory
-      def load_job_from_config(job_name)
-        definition = self.load("chronicle/etl/jobs/#{job_name}.yml")
-        definition[:name] = job_name
-        definition
+      def read_job(job_name)
+        load('jobs', job_name)
       end
 
-      def load_credentials(name)
-        config = self.load("chronicle/etl/credentials/#{name}.yml")
+      def config_pathname
+        base = Pathname.new(xdg_config.config_home)
+        base.join('chronicle', 'etl')
       end
 
-      def config_directory(type)
-        path = "chronicle/etl/#{type}"
-        Runcom::Config.new(path).current || raise(Chronicle::ETL::ConfigError, "Could not access config directory (#{path})")
+      def config_pathname_for_type(type)
+        config_pathname.join(type)
+      end
+
+      def xdg_config
+        # Only used for overriding ENV['HOME'] for XDG-related specs
+        if @xdg_environment
+          XDG::Environment.new(environment: @xdg_environment)
+        else
+          XDG::Environment.new
+        end
       end
     end
   end

data/lib/chronicle/etl/configurable.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "ostruct"
+require "chronic_duration"
 
 module Chronicle
   module ETL
@@ -57,7 +58,9 @@ module Chronicle
 
           options.each do |name, value|
             setting = self.class.all_settings[name]
-            raise(Chronicle::ETL::ConnectorConfigurationError, "Unrecognized setting: #{name}") unless setting
+
+            # Do nothing with a given option if it's not a connector setting
+            next unless setting
 
             @config[name] = coerced_value(setting, value)
           end
@@ -83,6 +86,8 @@ module Chronicle
 
         def coerced_value(setting, value)
           setting.type ? __send__("coerce_#{setting.type}", value) : value
+        rescue StandardError
+          raise(Chronicle::ETL::ConnectorConfigurationError, "Could not coerce #{value} into a #{setting.type}")
         end
 
         def coerce_string(value)
@@ -103,11 +108,15 @@ module Chronicle
         end
 
         def coerce_time(value)
-          # TODO: handle durations like '3h'
-          if value.is_a?(String)
-            Time.parse(value)
+          return value unless value.is_a?(String)
+
+          # Hacky check for duration strings like "60m"
+          if value.match(/[a-z]+/)
+            ChronicDuration.raise_exceptions = true
+            duration_ago = ChronicDuration.parse(value)
+            Time.now - duration_ago
           else
-            value
+            Time.parse(value)
           end
         end
       end

data/lib/chronicle/etl/exceptions.rb

@@ -2,6 +2,8 @@ module Chronicle
   module ETL
     class Error < StandardError; end
 
+    class SecretsError < Error; end
+
     class ConfigError < Error; end
 
     class RunnerTypeError < Error; end
@@ -23,6 +25,7 @@ module Chronicle
       end
     end
 
+    class PluginNotInstalledError < PluginError; end
     class PluginConflictError < PluginError; end
     class PluginNotAvailableError < PluginError; end
     class PluginLoadError < PluginError; end

data/lib/chronicle/etl/job_definition.rb

@@ -45,8 +45,10 @@ module Chronicle
       def plugins_missing?
         validate
 
-        @errors[:plugins] || []
-          .filter { |e| e.instance_of?(Chronicle::ETL::PluginLoadError) }
+        return false unless @errors[:plugins]&.any?
+
+        @errors[:plugins]
+          .filter { |e| e.instance_of?(Chronicle::ETL::PluginNotInstalledError) }
           .any?
       end
 
@@ -62,6 +64,30 @@ module Chronicle
         load_credentials
       end
 
+      # For each connector in this job, mix in secrets into the options
+      def apply_default_secrets
+        Chronicle::ETL::Registry::PHASES.each do |phase|
+          # If the option have a `secrets` key, we look up those secrets and
+          # mix them in. If not, use the connector's plugin name and look up 
+          # secrets with the same namespace
+          if @definition[phase][:options][:secrets]
+            namespace = @definition[phase][:options][:secrets]
+          else
+            # We don't want to do this lookup for built-in connectors
+            next if __send__("#{phase}_klass".to_sym).connector_registration.built_in?
+
+            # infer plugin name from connector name and use it for secrets
+            # namesepace
+            namespace = @definition[phase][:name].split(":").first
+          end
+
+          # Reverse merge secrets into connector's options (we want to preserve
+          # options that came from job file or CLI options)
+          secrets = Chronicle::ETL::Secrets.read(namespace)
+          @definition[phase][:options] = secrets.merge(@definition[phase][:options])
+        end
+      end
+
       # Is this job continuing from a previous run?
       def incremental?
         @definition[:incremental]

data/lib/chronicle/etl/job_logger.rb

@@ -1,5 +1,6 @@
-require 'sequel'
 require 'forwardable'
+require 'sequel'
+require 'xdg'
 
 module Chronicle
   module ETL
@@ -35,8 +36,8 @@ module Chronicle
       end
 
       def self.db_filename
-        data = Runcom::Data.new "chronicle/etl/job_log.db"
-        filename = data.all[0].to_s
+        base = Pathname.new(XDG::Data.new.home)
+        base.join('job_log.db')
       end
 
       def self.initialize_db

data/lib/chronicle/etl/loaders/csv_loader.rb

@@ -3,11 +3,13 @@ require 'csv'
 module Chronicle
   module ETL
     class CSVLoader < Chronicle::ETL::Loader
+      include Chronicle::ETL::Loaders::Helpers::StdoutHelper
+
       register_connector do |r|
         r.description = 'CSV'
       end
 
-      setting :output, default: $stdout
+      setting :output
       setting :headers, default: true
       setting :header_row, default: true
 
@@ -30,16 +32,7 @@ module Chronicle
           csv_options[:headers] = headers
         end
 
-        if @config.output.is_a?(IO)
-          # This might seem like a duplication of the default value ($stdout)
-          # but it's because rspec overwrites $stdout (in helper #capture) to
-          # capture output.
-          io = $stdout.dup
-        else
-          io = File.open(@config.output, "w+")
-        end
-
-        output = CSV.generate(**csv_options) do |csv|
+        csv_output = CSV.generate(**csv_options) do |csv|
           records.each do |record|
             csv << record
               .transform_keys(&:to_sym)
@@ -48,8 +41,12 @@ module Chronicle
           end
         end
 
-        io.write(output)
-        io.close
+        # TODO: just write to io directly
+        if output_to_stdout?
+          write_to_stdout(csv_output)
+        else
+          File.write(@config.output, csv_output)
+        end
       end
     end
   end

data/lib/chronicle/etl/loaders/helpers/stdout_helper.rb

@@ -0,0 +1,36 @@
+require 'tempfile'
+
+module Chronicle
+  module ETL
+    module Loaders
+      module Helpers
+        module StdoutHelper
+          # TODO: let users use "stdout" as an option for the `output` setting
+          # Assume we're using stdout if no output is specified
+          def output_to_stdout?
+            !@config.output
+          end
+
+          def create_stdout_temp_file
+            file = Tempfile.new('chronicle-stdout')
+            file.unlink
+            file
+          end
+
+          def write_to_stdout_from_temp_file(file)
+            file.rewind
+            write_to_stdout(file.read)
+          end
+
+          def write_to_stdout(output)
+            # We .dup because rspec overwrites $stdout (in helper #capture) to
+            # capture output.
+            stdout = $stdout.dup
+            stdout.write(output)
+            stdout.flush
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/loaders/json_loader.rb

@@ -1,19 +1,35 @@
+require 'tempfile'
+
 module Chronicle
   module ETL
     class JSONLoader < Chronicle::ETL::Loader
+      include Chronicle::ETL::Loaders::Helpers::StdoutHelper
+
       register_connector do |r|
         r.description = 'json'
       end
 
       setting :serializer
-      setting :output, default: $stdout
+      setting :output
+
+      # If true, one JSON record per line. If false, output a single json
+      # object with an array of records
+      setting :line_separated, default: true, type: :boolean
+
+      def initialize(*args)
+        super
+        @first_line = true
+      end
 
       def start
-        if @config.output == $stdout
-          @output = @config.output
-        else
-          @output = File.open(@config.output, "w")
-        end
+        @output_file =
+          if output_to_stdout?
+            create_stdout_temp_file
+          else
+            File.open(@config.output, "w+")
+          end
+
+        @output_file.puts("[\n") unless @config.line_separated
       end
 
       def load(record)
@@ -27,15 +43,34 @@ module Chronicle
 
           force_utf8(value)
         end
-        @output.puts encoded.to_json
+
+        line = encoded.to_json
+        # For line-separated output, we just put json + newline
+        if @config.line_separated
+          line = "#{line}\n"
+        # Otherwise, we add a comma and newline and then add record to the
+        # array we created in #start (unless it's the first line).
+        else
+          line = ",\n#{line}" unless @first_line
+        end
+
+        @output_file.write(line)
+
+        @first_line = false
       end
 
       def finish
-        @output.close
+        # Close the array unless we're doing line-separated JSON
+        @output_file.puts("\n]") unless @config.line_separated
+
+        write_to_stdout_from_temp_file(@output_file) if output_to_stdout?
+
+        @output_file.close
       end
 
       private
 
+      # TODO: implement this
       def serializer
         @config.serializer || Chronicle::ETL::RawSerializer
       end

data/lib/chronicle/etl/loaders/loader.rb

@@ -1,4 +1,5 @@
 require_relative 'helpers/encoding_helper'
+require_relative 'helpers/stdout_helper'
 
 module Chronicle
   module ETL

data/lib/chronicle/etl/registry/plugin_registry.rb

@@ -13,8 +13,8 @@ module Chronicle
       module PluginRegistry
         # Does this plugin exist?
         def self.exists?(name)
-          # TODO: implement this. Could query rubygems.org or have a
-          # hardcoded approved list
+          # TODO: implement this. Could query rubygems.org or use a hardcoded
+          # list somewhere
           true
         end
 
@@ -31,6 +31,12 @@ module Chronicle
             .values
         end
 
+        # Check whether a given plugin is installed
+        def self.installed?(name)
+          gem_name = "chronicle-#{name}"
+          all_installed.map(&:name).include?(gem_name)
+        end
+
         # Activate a plugin with given name by `require`ing it
         def self.activate(name)
           # By default, activates the latest available version of a gem
@@ -39,14 +45,17 @@ module Chronicle
         rescue Gem::ConflictError => e
           # TODO: figure out if there's more we can do here
           raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
-        rescue LoadError => e
-          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded" if exists?(name)
-
-          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist"
+        rescue StandardError, LoadError => e
+          # StandardError to catch random non-loading problems that might occur
+          # when requiring the plugin (eg class macro invoked the wrong way)
+          # TODO: decide if this should be separated
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded"
         end
 
         # Install a plugin to local gems
         def self.install(name)
+          return if installed?(name)
+
           gem_name = "chronicle-#{name}"
           raise(Chronicle::ETL::PluginNotAvailableError.new(gem_name), "Plugin #{name} doesn't exist") unless exists?(gem_name)
 

data/lib/chronicle/etl/registry/registry.rb

@@ -9,18 +9,7 @@ module Chronicle
       class << self
         attr_accessor :connectors
 
-        def load_all!
-          load_connectors_from_gems
-        end
-
-        def load_connectors_from_gems
-          Gem::Specification.filter{|s| s.name.match(/^chronicle/) }.each do |gem|
-            require_str = gem.name.gsub('chronicle-', 'chronicle/')
-            require require_str rescue LoadError
-          end
-        end
-
-        def register connector
+        def register(connector)
           connectors << connector
         end
 
@@ -28,9 +17,14 @@ module Chronicle
           @connectors ||= []
         end
 
-        def find_by_phase_and_identifier(phase, identifier)
-          # Simple case: built in connector
+        # Find connector from amongst those currently loaded
+        def find_by_phase_and_identifier_local(phase, identifier)
           connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
+        end
+
+        # Find connector and load relevant plugin to find it if necessary
+        def find_by_phase_and_identifier(phase, identifier)
+          connector = find_by_phase_and_identifier_local(phase, identifier)
           return connector if connector
 
           # if not available in built-in connectors, try to activate a
@@ -44,6 +38,8 @@ module Chronicle
             plugin = identifier
           end
 
+          raise(Chronicle::ETL::PluginNotInstalledError.new(plugin)) unless PluginRegistry.installed?(plugin)
+
           PluginRegistry.activate(plugin)
 
           candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }

data/lib/chronicle/etl/runner.rb

@@ -50,7 +50,7 @@ class Chronicle::ETL::Runner
       transformer = @job.instantiate_transformer(extraction)
       record = transformer.transform
 
-      Chronicle::ETL::Logger.info(tty_log_transformation(transformer))
+      Chronicle::ETL::Logger.debug(tty_log_transformation(transformer))
       @job_logger.log_transformation(transformer)
 
       @loader.load(record) unless @job.dry_run?

data/lib/chronicle/etl/secrets.rb

@@ -0,0 +1,55 @@
+module Chronicle
+  module ETL
+    # Secret management module
+    module Secrets
+      module_function
+
+      # Save a setting to a namespaced config file
+      def set(namespace, key, value)
+        config = read(namespace)
+        config[key.to_sym] = value
+        write(namespace, config)
+      end
+
+      # Remove a setting from a namespaced config file
+      def unset(namespace, key)
+        config = read(namespace)
+        config.delete(key.to_sym)
+        write(namespace, config)
+      end
+
+      # Retrieve all secrets from all namespaces
+      def all(namespace = nil)
+        namespaces = namespace.nil? ? available_secrets : [namespace]
+        namespaces
+          .to_h { |namespace| [namespace.to_sym, read(namespace)] }
+          .delete_if { |_, v| v.empty? }
+      end
+
+      # Return whether a namespace name is valid (lowercase alphanumeric and -)
+      def valid_namespace_name?(namespace)
+        namespace.match(/^[a-z0-9\-]+$/)
+      end
+
+      # Read secrets from a config file
+      def read(namespace)
+        definition = Chronicle::ETL::Config.load("secrets", namespace)
+        definition[:secrets] || {}
+      end
+
+      # Write secrets to a config file
+      def write(namespace, secrets)
+        data = {
+          secrets: (secrets || {}).transform_keys(&:to_s),
+          chronicle_etl_version: Chronicle::ETL::VERSION
+        }.transform_keys(&:to_s) # Should I implement deeply_transform_keys...?
+        Chronicle::ETL::Config.write("secrets", namespace, data)
+      end
+
+      # Which config files are available in ~/.config/chronicle/etl/secrets
+      def available_secrets
+        Chronicle::ETL::Config.available_configs('secrets')
+      end
+    end
+  end
+end

data/lib/chronicle/etl/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module ETL
-    VERSION = "0.4.4"
+    VERSION = "0.5.2"
   end
 end

data/lib/chronicle/etl.rb

@@ -14,6 +14,7 @@ require_relative 'etl/models/base'
 require_relative 'etl/models/raw'
 require_relative 'etl/models/entity'
 require_relative 'etl/runner'
+require_relative 'etl/secrets'
 require_relative 'etl/serializers/serializer'
 require_relative 'etl/utils/binary_attachments'
 require_relative 'etl/utils/hash_utilities'