chronicle-etl 0.5.4 → 0.6.1

data/lib/chronicle/etl/registry/connector_registration.rb

@@ -1,15 +1,18 @@
+# frozen_string_literal: true
+
 module Chronicle
   module ETL
     module Registry
-      # Records details about a connector such as its provider and a description
+      # Records details about a connector such as its source provider and a description
       class ConnectorRegistration
-        # FIXME: refactor custom accessor methods later in file
-        attr_accessor :identifier, :provider, :klass, :description
+        attr_accessor :klass, :identifier, :source, :strategy, :type, :description, :from_schema, :to_schema
 
+        # Create a new connector registration
         def initialize(klass)
           @klass = klass
         end
 
+        # The ETL phase of this connector
         def phase
           if klass.ancestors.include? Chronicle::ETL::Extractor
             :extractor
@@ -24,6 +27,7 @@ module Chronicle
           "#{phase}-#{identifier}"
         end
 
+        # Whether this connector is built-in to Chronicle
         def built_in?
           @klass.to_s.include? 'Chronicle::ETL'
         end
@@ -32,32 +36,20 @@ module Chronicle
           @klass.to_s
         end
 
-        def identifier
-          @identifier || @klass.to_s.split('::').last.gsub!(/(Extractor$|Loader$|Transformer$)/, '').downcase
-        end
-
-        def description
-          @description || @klass.to_s.split('::').last
-        end
-
-        def provider
-          @provider || (built_in? ? 'chronicle' : '')
-        end
-
         # TODO: allow overriding here. Maybe through self-registration process
         def plugin
-          @provider
+          @source
         end
 
         def descriptive_phrase
           prefix = case phase
-          when :extractor
-            "Extracts from"
-          when :transformer
-            "Transforms"
-          when :loader
-            "Loads to"
-          end
+                   when :extractor
+                     'Extracts from'
+                   when :transformer
+                     'Transforms'
+                   when :loader
+                     'Loads to'
+                   end
 
           "#{prefix} #{description}"
         end

data/lib/chronicle/etl/registry/connectors.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+
+module Chronicle
+  module ETL
+    module Registry
+      # A singleton class that acts as a registry of connector classes available for ETL jobs
+      module Connectors
+        PHASES = %i[extractor transformer loader].freeze
+        public_constant :PHASES
+
+        class << self
+          attr_accessor :connectors
+        end
+
+        def self.register(connector)
+          connectors << connector
+        end
+
+        def self.connectors
+          @connectors ||= []
+        end
+
+        def self.ancestor_for_phase(phase)
+          case phase
+          when :extractor
+            Chronicle::ETL::Extractor
+          when :transformer
+            Chronicle::ETL::Transformer
+          when :loader
+            Chronicle::ETL::Loader
+          end
+        end
+
+        def self.find_converter_for_source(source:, type: nil, strategy: nil, target: nil)
+          # FIXME: we're assuming extractor plugin has been loaded already
+          # This may not be the case if the schema converter is running
+          # off a json dump off extraction data.
+          # plugin = source_klass.connector_registration.source
+          # type = source_klass.connector_registration.type
+          # strategy = source_klass.connector_registration.strategy
+
+          connectors.find do |c|
+            c.phase == :transformer &&
+              c.source == source &&
+              (type.nil? || c.type == type) &&
+              (strategy.nil? || c.strategy == strategy || c.strategy.nil?) &&
+              (target.nil? || c.to_schema == target)
+          end
+        end
+
+        # Find connector from amongst those currently loaded
+        def self.find_by_phase_and_identifier_built_in(phase, identifier)
+          connectors.find { |c| c.phase == phase.to_sym && c.identifier == identifier.to_sym }
+        end
+
+        # Find connector and load relevant plugin to find it if necessary
+        def self.find_by_phase_and_identifier(phase, identifier)
+          connector = find_by_phase_and_identifier_built_in(phase, identifier)
+          return connector if connector
+
+          # determine if we need to try to load a local file. if it has a dot in the identifier, we treat it as a file
+          return find_by_phase_and_identifier_local(phase, identifier) if identifier.to_s.include?('.')
+
+          # Example identifier: lastfm:listens:api
+          plugin, type, strategy = identifier.split(':')
+            .map { |part| part.gsub('-', '_') }
+            .map(&:to_sym)
+
+          plugin_identifier = plugin.to_s.gsub('_', '-')
+
+          unless Chronicle::ETL::Registry::Plugins.installed?(plugin_identifier)
+            raise Chronicle::ETL::PluginNotInstalledError, plugin_identifier
+          end
+
+          Chronicle::ETL::Registry::Plugins.activate(plugin_identifier)
+
+          # find most specific connector that matches the identifier
+          connector = connectors.find do |c|
+            c.plugin == plugin && (type.nil? || c.type == type) && (strategy.nil? || c.strategy == strategy)
+          end
+
+          connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
+        end
+
+        # Load a plugin from local file system
+        def self.find_by_phase_and_identifier_local(phase, identifier)
+          script = File.read(identifier)
+          raise ConnectorNotAvailableError, "Connector '#{identifier}' not found" if script.nil?
+
+          # load the file by evaluating the contents
+          eval(script, TOPLEVEL_BINDING, __FILE__, __LINE__) # rubocop:disable Security/Eval
+
+          # read the file and look for all class definitions in the ruby script.
+          class_names = script.scan(/class (\w+)/).flatten
+
+          class_names.each do |class_name|
+            klass = Object.const_get(class_name)
+
+            next unless klass.ancestors.include?(ancestor_for_phase(phase))
+
+            registration = ::Chronicle::ETL::Registry::ConnectorRegistration.new(klass)
+
+            klass.connector_registration = registration
+            return registration
+            # return klass
+          rescue NameError
+            # ignore
+          end
+
+          raise ConnectorNotAvailableError, "Connector '#{identifier}' not found"
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/plugin_registration.rb

@@ -0,0 +1,19 @@
+module Chronicle
+  module ETL
+    module Registry
+      class PluginRegistration
+        attr_accessor :name, :description, :gem, :version, :installed, :gemspec
+
+        def initialize(name = nil)
+          @installed = false
+          @name = name
+          yield self if block_given?
+        end
+
+        def installed?
+          @installed || false
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/plugins.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'rubygems/command'
+require 'rubygems/commands/install_command'
+require 'rubygems/uninstaller'
+require 'gems'
+require 'active_support/core_ext/hash/deep_merge'
+
+module Chronicle
+  module ETL
+    module Registry
+      # Responsible for managing plugins available to chronicle-etl
+      #
+      # @todo Better validation for whether a gem is actually a plugin
+      # @todo Add ways to load a plugin that don't require a gem on rubygems.org
+      module Plugins
+        KNOWN_PLUGINS = %w[
+          apple-podcasts
+          email
+          foursquare
+          github
+          imessage
+          pinboard
+          safari
+          shell
+          spotify
+          zulip
+        ].freeze
+        public_constant :KNOWN_PLUGINS
+
+        # Start of a system for having non-gem plugins. Right now, we just
+        # make registry aware of existence of name of non-gem plugin
+        def self.register_standalone(name:)
+          plugin = Chronicle::ETL::Registry::PluginRegistration.new do |p|
+            p.name = name.to_sym
+            p.installed = true
+          end
+
+          installed_standalone << plugin
+        end
+
+        # Plugins either installed as gems or manually loaded/registered
+        def self.installed
+          installed_standalone + installed_as_gem
+        end
+
+        # Check whether a given plugin is installed
+        def self.installed?(name)
+          installed.map(&:name).include?(name.to_sym)
+        end
+
+        # List of plugins installed as standalone
+        def self.installed_standalone
+          @installed_standalone ||= []
+        end
+
+        # List of plugins installed as gems
+        def self.installed_as_gem
+          installed_gemspecs_latest.map do |gem|
+            Chronicle::ETL::Registry::PluginRegistration.new do |p|
+              p.name = gem.name.sub('chronicle-', '').to_sym
+              p.gem = gem.name
+              p.description = gem.description
+              p.version = gem.version.to_s
+              p.installed = true
+            end
+          end
+        end
+
+        # List of all plugins available to chronicle-etl
+        def self.available
+          available_as_gem
+        end
+
+        # List of plugins available through rubygems
+        # TODO: make this concurrent
+        def self.available_as_gem
+          KNOWN_PLUGINS.map do |name|
+            info = gem_info(name)
+            Chronicle::ETL::Registry::PluginRegistration.new do |p|
+              p.name = name
+              p.gem = info['name']
+              p.version = info['version']
+              p.description = info['info']
+            end
+          end
+        end
+
+        # Load info about a gem plugin from rubygems API
+        def self.gem_info(name)
+          gem_name = "chronicle-#{name}"
+          Gems.info(gem_name)
+        end
+
+        # Union of installed gems (latest version) + available gems
+        def self.all
+          (installed + available)
+            .group_by(&:name)
+            .transform_values { |plugin| plugin.find(&:installed) || plugin.first }
+            .values
+        end
+
+        # Does a plugin with a given name exist?
+        def self.exists?(name)
+          KNOWN_PLUGINS.include?(name)
+        end
+
+        # All versions of all plugins currently installed
+        def self.installed_gemspecs
+          # TODO: add check for chronicle-etl dependency
+          Gem::Specification.filter do |s|
+            s.name.match(/^chronicle-/) && s.name != 'chronicle-etl' && s.name != 'chronicle-core'
+          end
+        end
+
+        # Latest version of each installed plugin
+        def self.installed_gemspecs_latest
+          installed_gemspecs.group_by(&:name)
+            .transform_values { |versions| versions.sort_by(&:version).reverse.first }
+            .values
+        end
+
+        # Activate a plugin with given name by `require`ing it
+        def self.activate(name)
+          # By default, activates the latest available version of a gem
+          # so don't have to run Kernel#gem separately
+
+          plugin_require_name = name.to_s.gsub('-', '_')
+          require "chronicle/#{plugin_require_name}"
+        rescue Gem::ConflictError => e
+          # TODO: figure out if there's more we can do here
+          raise Chronicle::ETL::PluginConflictError.new(name),
+            "Plugin '#{plugin_require_name}' couldn't be loaded. #{e.message}"
+        rescue StandardError, LoadError
+          # StandardError to catch random non-loading problems that might occur
+          # when requiring the plugin (eg class macro invoked the wrong way)
+          # TODO: decide if this should be separated
+          raise Chronicle::ETL::PluginLoadError.new(name), "Plugin '#{plugin_require_name}' couldn't be loaded"
+        end
+
+        # Install a plugin to local gems
+        def self.install(name)
+          return if installed?(name)
+          raise(Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} doesn't exist") unless exists?(name)
+
+          gem_name = "chronicle-#{name}"
+
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          Gem.install(gem_name)
+
+          activate(name)
+        rescue Gem::UnsatisfiableDependencyError
+          # TODO: we need to catch a lot more than this here
+          raise Chronicle::ETL::PluginNotAvailableError.new(name), "Plugin #{name} could not be installed."
+        end
+
+        # Uninstall a plugin
+        def self.uninstall(name)
+          gem_name = "chronicle-#{name}"
+          Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
+          uninstaller = Gem::Uninstaller.new(gem_name)
+          uninstaller.uninstall
+        rescue Gem::InstallError
+          # TODO: strengthen this exception handling
+          raise(Chronicle::ETL::PluginError.new(name), "Plugin #{name} wasn't uninstalled")
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/etl/registry/registry.rb

@@ -1,61 +1,12 @@
-require 'rubygems'
-
 module Chronicle
   module ETL
-    # A singleton class that acts as a registry of connector classes available for ETL jobs
     module Registry
-      PHASES = [:extractor, :transformer, :loader]
-
-      class << self
-        attr_accessor :connectors
-
-        def register(connector)
-          connectors << connector
-        end
-
-        def connectors
-          @connectors ||= []
-        end
-
-        # Find connector from amongst those currently loaded
-        def find_by_phase_and_identifier_local(phase, identifier)
-          connector = connectors.find { |c| c.phase == phase && c.identifier == identifier }
-        end
-
-        # Find connector and load relevant plugin to find it if necessary
-        def find_by_phase_and_identifier(phase, identifier)
-          connector = find_by_phase_and_identifier_local(phase, identifier)
-          return connector if connector
-
-          # if not available in built-in connectors, try to activate a
-          # relevant plugin and try again
-          if identifier.include?(":")
-            plugin, name = identifier.split(":")
-          else
-            # This case handles the case where the identifier is a 
-            # shorthand (ie `imessage`) because there's only one default
-            # connector.
-            plugin = identifier
-          end
-
-          raise(Chronicle::ETL::PluginNotInstalledError.new(plugin)) unless PluginRegistry.installed?(plugin)
-
-          PluginRegistry.activate(plugin)
-
-          candidates = connectors.select { |c| c.phase == phase && c.plugin == plugin }
-          # if no name given, just use first connector with right phase/plugin
-          # TODO: set up a property for connectors to specify that they're the
-          # default connector for the plugin
-          candidates = candidates.select { |c| c.identifier == name } if name
-          connector = candidates.first
-
-          connector || raise(ConnectorNotAvailableError, "Connector '#{identifier}' not found")
-        end
-      end
     end
   end
 end
 
 require_relative 'self_registering'
 require_relative 'connector_registration'
-require_relative 'plugin_registry'
+require_relative 'connectors'
+require_relative 'plugin_registration'
+require_relative 'plugins'

data/lib/chronicle/etl/registry/self_registering.rb

@@ -17,7 +17,7 @@ module Chronicle
         def register_connector
           @connector_registration ||= ::Chronicle::ETL::Registry::ConnectorRegistration.new(self)
           yield @connector_registration if block_given?
-          ::Chronicle::ETL::Registry.register(@connector_registration)
+          ::Chronicle::ETL::Registry::Connectors.register(@connector_registration)
         end
       end
     end