chronicle-etl 0.5.2 → 0.5.5

data/lib/chronicle/etl/cli/secrets.rb

@@ -46,7 +46,7 @@ module Chronicle
           all_secrets.each do |namespace, secrets|
             rows += secrets.map do |key, value|
               # hidden_value = (value[0..5] + ("*" * [0, [value.length - 5, 30].min].max)).truncate(30)
-              truncated_value = value.truncate(30)
+              truncated_value = value&.truncate(30)
               [namespace, key, truncated_value]
             end
           end

data/lib/chronicle/etl/cli.rb

@@ -4,6 +4,7 @@ require 'chronicle/etl'
 
 require 'chronicle/etl/cli/cli_base'
 require 'chronicle/etl/cli/subcommand_base'
+require 'chronicle/etl/cli/authorizations'
 require 'chronicle/etl/cli/connectors'
 require 'chronicle/etl/cli/jobs'
 require 'chronicle/etl/cli/plugins'

data/lib/chronicle/etl/config.rb

@@ -1,3 +1,4 @@
+require "active_support/core_ext/hash/keys"
 require 'fileutils'
 require 'yaml'
 
@@ -21,6 +22,8 @@ module Chronicle
       def write(type, identifier, data)
         base = config_pathname_for_type(type)
         path = base.join("#{identifier}.yml")
+
+        data.deep_stringify_keys!
         FileUtils.mkdir_p(File.dirname(path))
         File.open(path, 'w', 0o600) do |f|
           # Ruby likes to add --- separators when writing yaml files

data/lib/chronicle/etl/configurable.rb

@@ -108,6 +108,10 @@ module Chronicle
         end
 
         def coerce_time(value)
+          # parsing yml files might result in us getting Date objects
+          # we convert to DateTime first to to ensure UTC
+          return value.to_datetime.to_time if value.is_a?(Date)
+
           return value unless value.is_a?(String)
 
           # Hacky check for duration strings like "60m"

data/lib/chronicle/etl/exceptions.rb

@@ -4,8 +4,13 @@ module Chronicle
 
     class SecretsError < Error; end
 
+    class AuthorizationError < Error; end
+
     class ConfigError < Error; end
 
+    class RunnerError < Error; end
+    class RunInterruptedError < RunnerError; end
+
     class RunnerTypeError < Error; end
 
     class JobDefinitionError < Error

data/lib/chronicle/etl/job_definition.rb

@@ -34,7 +34,7 @@ module Chronicle
       def validate
         @errors = {}
 
-        Chronicle::ETL::Registry::PHASES.each do |phase|
+        Chronicle::ETL::Registry::Connectors::PHASES.each do |phase|
           __send__("#{phase}_klass".to_sym)
         rescue Chronicle::ETL::PluginError => e
           @errors[:plugins] ||= []
@@ -66,7 +66,7 @@ module Chronicle
 
       # For each connector in this job, mix in secrets into the options
       def apply_default_secrets
-        Chronicle::ETL::Registry::PHASES.each do |phase|
+        Chronicle::ETL::Registry::Connectors::PHASES.each do |phase|
           # If the option have a `secrets` key, we look up those secrets and
           # mix them in. If not, use the connector's plugin name and look up 
           # secrets with the same namespace
@@ -124,11 +124,11 @@ module Chronicle
       private
 
       def load_klass(phase, identifier)
-        Chronicle::ETL::Registry.find_by_phase_and_identifier(phase, identifier).klass
+        Chronicle::ETL::Registry::Connectors.find_by_phase_and_identifier(phase, identifier).klass
       end
 
       def load_credentials
-        Chronicle::ETL::Registry::PHASES.each do |phase|
+        Chronicle::ETL::Registry::Connectors::PHASES.each do |phase|
           credentials_name = @definition[phase].dig(:options, :credentials)
           if credentials_name
             credentials = Chronicle::ETL::Config.load_credentials(credentials_name)

data/lib/chronicle/etl/logger.rb

@@ -17,8 +17,8 @@ module Chronicle
       def output message, level
         return unless level >= @log_level
 
-        if @progress_bar
-          @progress_bar.log(message)
+        if @ui_element
+          @ui_element.log(message)
         else
           $stderr.puts(message)
         end
@@ -40,12 +40,12 @@ module Chronicle
         output(message, DEBUG)
       end
 
-      def attach_to_progress_bar(progress_bar)
-        @progress_bar = progress_bar
+      def attach_to_ui(ui_element)
+        @ui_element = ui_element
       end
 
-      def detach_from_progress_bar
-        @progress_bar = nil
+      def detach_from_ui
+        @ui_element = nil
       end
     end
   end

data/lib/chronicle/etl/models/base.rb

@@ -9,7 +9,7 @@ module Chronicle
       # @todo Experiment with just mixing in ActiveModel instead of this
       #   this reimplementation
       class Base
-        ATTRIBUTES = [:provider, :provider_id, :lat, :lng, :metadata].freeze
+        ATTRIBUTES = [:provider, :provider_id, :provider_namespace, :lat, :lng, :metadata].freeze
         ASSOCIATIONS = [].freeze
 
         attr_accessor(:id, :dedupe_on, *ATTRIBUTES)

data/lib/chronicle/etl/models/entity.rb

@@ -10,12 +10,14 @@ module Chronicle
         # TODO: This desperately needs a validation system
         ASSOCIATIONS = [
           :involvements, # inverse of activity's `involved`
-
+          :analogous,
           :attachments,
           :abouts,
           :aboutables, # inverse of above
           :depicts,
           :consumers,
+          :creators,
+          :creations,
           :contains,
           :containers # inverse of above
         ].freeze  # TODO: add these to reflect Chronicle Schema

data/lib/chronicle/etl/oauth_authorizer.rb

@@ -0,0 +1,140 @@
+require 'omniauth'
+require 'tty-spinner'
+
+module Chronicle
+  module ETL
+    # An authorization strategy that uses oauth2 (and omniauth under the hood)
+    class OauthAuthorizer < Authorizer
+      class << self
+        attr_reader :strategy, :provider_name, :authorization_to_secret_map
+        attr_accessor :client_id, :client_secret
+
+        # Macro for specifying which omniauth strategy to use
+        def omniauth_strategy(strategy)
+          @strategy = strategy
+        end
+
+        # Macro for specifying which omniauth scopes to request
+        def scope(value)
+          options[:scope] = value
+        end
+
+        # Macro for specifying hash of returned authorization to secrets hash
+        def pluck_secrets(map)
+          @authorization_to_secret_map = map
+        end
+
+        # # Macro for specifying options to pass to omniauth
+        def options
+          @options ||= {}
+        end
+
+        # Returns all subclasses of OauthAuthorizer
+        # (Used by AuthorizationServer to build omniauth providers)
+        def all
+          ObjectSpace.each_object(::Class).select { |klass| klass < self }
+        end
+      end
+
+      attr_reader :authorization
+
+      # Create a new instance of OauthAuthorizer
+      def initialize(port:, credentials: {})
+        @port = port
+        @credentials = credentials
+        super
+      end
+
+      # Start up an authorization server and handle the oauth flow
+      def authorize!
+        associate_oauth_credentials
+        @server = load_server
+        spinner = TTY::Spinner.new(":spinner :title", format: :dots_2)
+        spinner.auto_spin
+        spinner.update(title: "Starting temporary authorization server on port #{@port}""")
+
+        server_thread = start_authorization_server(port: @port)
+        start_oauth_flow
+
+        spinner.update(title: "Waiting for authorization to complete in your browser")
+        sleep 0.1 while authorization_pending?(server_thread)
+
+        @server.quit!
+        server_thread.join
+        spinner.success("(#{'successful'.green})")
+
+        # TODO: properly handle failed authorizations
+        raise Chronicle::ETL::AuthorizationError unless @server.latest_authorization
+
+        @authorization = @server.latest_authorization
+
+        extract_secrets(authorization: @authorization, pluck_values: self.class.authorization_to_secret_map)
+      end
+
+      private
+
+      def authorization_pending?(server_thread)
+        server_thread.status && !@server.latest_authorization
+      end
+
+      def associate_oauth_credentials
+        self.class.client_id = @credentials[:client_id]
+        self.class.client_secret = @credentials[:client_secret]
+      end
+
+      def load_server
+        # Load at runtime so that we can set omniauth strategies based on
+        # which chronicle plugin has been loaded.
+        require_relative './authorization_server'
+        Chronicle::ETL::AuthorizationServer
+      end
+
+      def start_authorization_server(port:)
+        @server.settings.port = port
+        suppress_webrick_logging(@server)
+        Thread.abort_on_exception = true
+        Thread.report_on_exception = false
+
+        Thread.new do
+          @server.run!({ port: @port }) do |s|
+            s.silent = true if s.class.to_s == "Thin::Server"
+          end
+        end
+      end
+
+      def start_oauth_flow
+        url = "http://localhost:#{@port}/auth/#{omniauth_strategy}"
+        Launchy.open(url)
+      rescue Launchy::CommandNotFoundError
+        Chronicle::ETL::Logger.info("Please open #{url} in a browser to continue")
+      end
+
+      def suppress_webrick_logging(server)
+        require 'webrick'
+        server.set(
+          :server_settings,
+          {
+            AccessLog: [],
+            # TODO: make this windows friendly
+            # https://github.com/winton/stasis/commit/77da36f43285fda129300e382f18dfaff48571b0
+            Logger: WEBrick::Log::new("/dev/null")
+          }
+        )
+      rescue LoadError
+        # no worries if we're not using WEBrick
+      end
+
+      def extract_secrets(authorization:, pluck_values:)
+        return authorization unless pluck_values&.any?
+
+        pluck_values.each_with_object({}) do |(key, identifiers), secrets|
+          secrets[key] = authorization.dig(*identifiers)
+        end
+      end
+
+      def omniauth_strategy
+        self.class.strategy
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/connectors.rb

@@ -0,0 +1,60 @@
+require 'rubygems'
+
+module Chronicle
+  module ETL
+    module Registry
+      # A singleton class that acts as a registry of connector classes available for ETL jobs
+      module Connectors
+        PHASES = [:extractor, :transformer, :loader].freeze
+        public_constant :PHASES
+
+        class << self
+          attr_accessor :connectors
+
+          def register(connector)
+            connectors << connector
+          end
+
+          def connectors
+            @connectors ||= []
+          end
+
+          # Find connector from amongst those currently loaded
+          def find_by_phase_and_identifier_local(phase, identifier)
+            connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
+          end
+
+          # Find connector and load relevant plugin to find it if necessary
+          def find_by_phase_and_identifier(phase, identifier)
+            connector = find_by_phase_and_identifier_local(phase, identifier)
+            return connector if connector
+
+            # if not available in built-in connectors, try to activate a
+            # relevant plugin and try again
+            if identifier.include?(":")
+              plugin, name = identifier.split(":")
+            else
+              # This case handles the case where the identifier is a 
+              # shorthand (ie `imessage`) because there's only one default
+              # connector.
+              plugin = identifier
+            end
+
+            raise(Chronicle::ETL::PluginNotInstalledError.new(plugin)) unless Chronicle::ETL::Registry::Plugins.installed?(plugin)
+
+            Chronicle::ETL::Registry::Plugins.activate(plugin)
+
+            candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }
+            # if no name given, just use first connector with right phase/plugin
+            # TODO: set up a property for connectors to specify that they're the
+            # default connector for the plugin
+            candidates = candidates.select { |c| c.identifier == name } if name
+            connector = candidates.first
+
+            connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/plugin_registration.rb

@@ -0,0 +1,19 @@
+module Chronicle
+  module ETL
+    module Registry
+      class PluginRegistration
+        attr_accessor :name, :description, :gem, :version, :installed, :gemspec
+
+        def initialize(name=nil)
+          @installed = false
+          @name = name
+          yield self if block_given?
+        end
+
+        def installed?
+          @installed || false
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/plugins.rb

@@ -0,0 +1,163 @@
+require 'rubygems'
+require 'rubygems/command'
+require 'rubygems/commands/install_command'
+require 'rubygems/uninstaller'
+require 'gems'
+require 'active_support/core_ext/hash/deep_merge'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Responsible for managing plugins available to chronicle-etl
+      #
+      # @todo Better validation for whether a gem is actually a plugin
+      # @todo Add ways to load a plugin that don't require a gem on rubygems.org
+      module Plugins
+        KNOWN_PLUGINS = [
+          'email',
+          'foursquare',
+          'github',
+          'imessage',
+          'pinboard',
+          'safari',
+          'shell',
+          'spotify',
+          'zulip'
+        ].freeze
+        public_constant :KNOWN_PLUGINS
+
+        # Start of a system for having non-gem plugins. Right now, we just
+        # make registry aware of existence of name of non-gem plugin
+        def self.register_standalone(name:)
+          plugin = Chronicle::ETL::Registry::PluginRegistration.new do |p|
+            p.name = name
+            p.installed = true
+          end
+
+          installed_standalone << plugin
+        end
+
+        # Plugins either installed as gems or manually loaded/registered
+        def self.installed
+          installed_standalone + installed_as_gem
+        end
+
+        # Check whether a given plugin is installed
+        def self.installed?(name)
+          installed.map(&:name).include?(name)
+        end
+
+        # List of plugins installed as standalone
+        def self.installed_standalone
+          @standalones ||= []
+        end
+
+        # List of plugins installed as gems
+        def self.installed_as_gem
+          installed_gemspecs_latest.map do |gem|
+            Chronicle::ETL::Registry::PluginRegistration.new do |p|
+              p.name = gem.name.sub("chronicle-", "")
+              p.gem = gem.name
+              p.description = gem.description
+              p.version = gem.version.to_s
+              p.installed = true
+            end
+          end
+        end
+
+        # List of all plugins available to chronicle-etl
+        def self.available
+          available_as_gem
+        end
+
+        # List of plugins available through rubygems
+        # TODO: make this concurrent
+        def self.available_as_gem
+          KNOWN_PLUGINS.map do |name|
+            info = gem_info(name)
+            Chronicle::ETL::Registry::PluginRegistration.new do |p|
+              p.name = name
+              p.gem = info['name']
+              p.version = info['version']
+              p.description = info['info']
+            end
+          end
+        end
+
+        # Load info about a gem plugin from rubygems API
+        def self.gem_info(name)
+          gem_name = "chronicle-#{name}"
+          Gems.info(gem_name)
+        end
+
+        # Union of installed gems (latest version) + available gems
+        def self.all
+          (installed + available)
+            .group_by(&:name)
+            .transform_values { |plugin| plugin.find(&:installed) || plugin.first }
+            .values
+        end
+
+        # Does a plugin with a given name exist?
+        def self.exists?(name)
+          KNOWN_PLUGINS.include?(name)
+        end
+
+        # All versions of all plugins currently installed
+        def self.installed_gemspecs
+          # TODO: add check for chronicle-etl dependency
+          Gem::Specification.filter { |s| s.name.match(/^chronicle-/) && s.name != "chronicle-etl" }
+        end
+
+        # Latest version of each installed plugin
+        def self.installed_gemspecs_latest
+          installed_gemspecs.group_by(&:name)
+            .transform_values { |versions| versions.sort_by(&:version).reverse.first }
+            .values
+        end
+
+        # Activate a plugin with given name by `require`ing it
+        def self.activate(name)
+          # By default, activates the latest available version of a gem
+          # so don't have to run Kernel#gem separately
+          require "chronicle/#{name}"
+        rescue Gem::ConflictError => e
+          # TODO: figure out if there's more we can do here
+          raise Chronicle::ETL::PluginConflictError.new(name), "Plugin '#{name}' couldn't be loaded. #{e.message}"
+        rescue StandardError, LoadError => e
+          # StandardError to catch random non-loading problems that might occur
+          # when requiring the plugin (eg class macro invoked the wrong way)
+          # TODO: decide if this should be separated
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{name}' couldn't be loaded"
+        end
+
+        # Install a plugin to local gems
+        def self.install(name)
+          return if installed?(name)
+          raise(Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist") unless exists?(name)
+
+          gem_name = "chronicle-#{name}"
+
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          Gem.install(gem_name)
+
+          activate(name)
+        rescue Gem::UnsatisfiableDependencyError
+          # TODO: we need to catch a lot more than this here
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} could not be installed."
+        end
+
+        # Uninstall a plugin
+        def self.uninstall(name)
+          gem_name = "chronicle-#{name}"
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          uninstaller = Gem::Uninstaller.new(gem_name)
+          uninstaller.uninstall
+        rescue Gem::InstallError
+          # TODO: strengthen this exception handling
+          raise(Chronicle::ETL::PluginError.new(name), "Plugin #{name} wasn't uninstalled")
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -1,61 +1,12 @@
-require 'rubygems'
-
 module Chronicle
   module ETL
-    # A singleton class that acts as a registry of connector classes available for ETL jobs
     module Registry
-      PHASES = [:extractor, :transformer, :loader]
-
-      class << self
-        attr_accessor :connectors
-
-        def register(connector)
-          connectors << connector
-        end
-
-        def connectors
-          @connectors ||= []
-        end
-
-        # Find connector from amongst those currently loaded
-        def find_by_phase_and_identifier_local(phase, identifier)
-          connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
-        end
-
-        # Find connector and load relevant plugin to find it if necessary
-        def find_by_phase_and_identifier(phase, identifier)
-          connector = find_by_phase_and_identifier_local(phase, identifier)
-          return connector if connector
-
-          # if not available in built-in connectors, try to activate a
-          # relevant plugin and try again
-          if identifier.include?(":")
-            plugin, name = identifier.split(":")
-          else
-            # This case handles the case where the identifier is a 
-            # shorthand (ie `imessage`) because there's only one default
-            # connector.
-            plugin = identifier
-          end
-
-          raise(Chronicle::ETL::PluginNotInstalledError.new(plugin)) unless PluginRegistry.installed?(plugin)
-
-          PluginRegistry.activate(plugin)
-
-          candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }
-          # if no name given, just use first connector with right phase/plugin
-          # TODO: set up a property for connectors to specify that they're the
-          # default connector for the plugin
-          candidates = candidates.select { |c| c.identifier == name } if name
-          connector = candidates.first
-
-          connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
-        end
-      end
     end
   end
 end
 
 require_relative 'self_registering'
 require_relative 'connector_registration'
-require_relative 'plugin_registry'
+require_relative 'connectors'
+require_relative 'plugin_registration'
+require_relative 'plugins'

data/lib/chronicle/etl/registry/self_registering.rb

@@ -17,7 +17,7 @@ module Chronicle
         def register_connector
           @connector_registration ||= ::Chronicle::ETL::Registry::ConnectorRegistration.new(self)
           yield @connector_registration if block_given?
-          ::Chronicle::ETL::Registry.register(@connector_registration)
+          ::Chronicle::ETL::Registry::Connectors.register(@connector_registration)
         end
       end
     end