connectors_service 8.5.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c395f06754b76baa88beae0fbd14d858c0eecdef41c5042e53ce14bb0296f70c
+  data.tar.gz: 85145ec2a3f589a0316663427b292f80a21fa0a87329895d69a3081deecaa237
+SHA512:
+  metadata.gz: 0c5946fccce23ac02c552c5000759e280c335575236db6836bebbb6318589a02ee67df3753a8e988b286172a6b90e905cb02f22da69dc441c7430d6860bb990c
+  data.tar.gz: 4ee0b23db692918aa26507e141f2e0f7afc6eb70c42c717c60a3d0420cd5da161ccbda167805ecf75e88cf86c78dbec0107aa0399aa5132e692e9496ca1722cf

data/LICENSE

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

data/NOTICE.txt

@@ -0,0 +1,2 @@
+connectors
+Copyright 2022 Elasticsearch B.V.

data/bin/connectors_service

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'connectors_service'
+ConnectorsService.run!

data/bin/list_connectors

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'list_connectors'
+ListConnectors.run!

data/config/connectors.yml

@@ -0,0 +1,25 @@
+# general metadata
+version: 8.5.0.1
+repository: https://github.com/elastic/connectors-ruby.git
+revision: 5392882cd3d8c03eafc68b2b1bf3c42c92ff3559
+elasticsearch:
+  cloud_id: CHANGEME
+  hosts: http://localhost:9200
+  api_key: CHANGEME
+  retry_on_failure: 3
+  request_timeout: 120
+  disable_warnings: true
+  trace: false
+  log: false
+thread_pool:
+  min_threads: 0
+  max_threads: 5
+  max_queue: 100
+log_level: info
+ecs_logging: true
+poll_interval: 3
+termination_timeout: 60
+heartbeat_interval: 1800
+native_mode: true
+connector_id: CHANGEME
+service_type: CHANGEME

data/lib/app/app.rb

@@ -0,0 +1,25 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+$LOAD_PATH << '../'
+
+require 'app/dispatcher'
+require 'app/config'
+require 'app/preflight_check'
+require 'utility/environment'
+require 'utility/logger'
+
+module App
+  Utility::Environment.set_execution_environment(App::Config) do
+    App::PreflightCheck.run!
+    App::Dispatcher.start!
+  rescue App::PreflightCheck::CheckFailure => e
+    Utility::Logger.error("Preflight check failed: #{e.message}")
+    exit(-1)
+  end
+end

data/lib/app/config.rb

@@ -0,0 +1,132 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'config'
+require_relative '../utility/logger'
+
+# We look for places in this order:
+# - CONNECTORS_CONFIG environment variable
+# - here: /../../config/connectors.yml
+CONFIG_FILE = ENV['CONNECTORS_CONFIG'] || File.join(__dir__, '../..', 'config', 'connectors.yml')
+
+puts "Parsing #{CONFIG_FILE} configuration file."
+
+::Config.setup do |config|
+  config.evaluate_erb_in_yaml = false
+  config.use_env = true
+  config.env_prefix = ''
+
+  config.schema do
+    required(:version).value(:string)
+    required(:repository).value(:string)
+    required(:revision).value(:string)
+
+    required(:elasticsearch).hash do
+      optional(:cloud_id).value(:string)
+      optional(:hosts).value(:string)
+      optional(:api_key).value(:string)
+      optional(:retry_on_failure).value(:integer)
+      optional(:request_timeout).value(:integer)
+      optional(:disable_warnings).value(:bool?)
+      optional(:trace).value(:bool?)
+      optional(:log).value(:bool?)
+    end
+
+    optional(:thread_pool).hash do
+      optional(:min_threads).value(:integer, gteq?: 0)
+      optional(:max_threads).value(:integer, gteq?: 0)
+      optional(:max_queue).value(:integer, gteq?: 0)
+    end
+
+    required(:native_mode).value(:bool?)
+    optional(:connector_id).value(:string)
+    optional(:service_type).value(:string)
+    required(:log_level).value(:string)
+    required(:ecs_logging).value(:bool?)
+
+    optional(:poll_interval).value(:integer)
+    optional(:termination_timeout).value(:integer)
+    optional(:heartbeat_interval).value(:integer)
+  end
+end
+
+::Config.load_and_set_settings(CONFIG_FILE)
+
+module App
+  DEFAULT_PASSWORD = 'changeme'
+
+  # If it's on cloud (i.e. EnvVar ENT_SEARCH_CONFIG_PATH is set), elasticsearch config in ent-search will be used.
+  def self.ent_search_es_config
+    ent_search_config_path = ENV['ENT_SEARCH_CONFIG_PATH']
+    unless ent_search_config_path
+      Utility::Logger.info('ENT_SEARCH_CONFIG_PATH is not found, use connector service config.')
+      return nil
+    end
+
+    Utility::Logger.info("Found ENT_SEARCH_CONFIG_PATH, loading ent-search config from #{ent_search_config_path}")
+    ent_search_config = begin
+      YAML.load_file(ent_search_config_path)
+    rescue StandardError => e
+      Utility::Logger.error("Failed to load ent-search config #{ent_search_config_path}: #{e.message}")
+      return nil
+    end
+
+    unless ent_search_config.is_a?(Hash)
+      Utility::Logger.error("Invalid ent-search config: #{ent_search_config.inspect}")
+      return nil
+    end
+
+    host = ent_search_config['elasticsearch.host'] || ent_search_config.dig('elasticsearch', 'host')
+    username = ent_search_config['elasticsearch.username'] || ent_search_config.dig('elasticsearch', 'username')
+    password = ent_search_config['elasticsearch.password'] || ent_search_config.dig('elasticsearch', 'password')
+
+    missing_fields = []
+    missing_fields << 'elasticsearch.host' unless host
+    missing_fields << 'elasticsearch.username' unless username
+    missing_fields << 'elasticsearch.password' unless password
+    if missing_fields.any?
+      Utility::Logger.error("Incomplete elasticsearch config, missing #{missing_fields.join(', ')}")
+      return nil
+    end
+
+    uri = begin
+      URI.parse(host)
+    rescue URI::InvalidURIError => e
+      Utility::Logger.error("Failed to parse elasticsearch host #{host}: #{e.message}")
+      return nil
+    end
+
+    unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+      Utility::Logger.error("Invalid elasticsearch host #{host}, it must be a http or https URI.")
+      return nil
+    end
+
+    {
+      :hosts => [
+        {
+          scheme: uri.scheme,
+          user: username,
+          password: password,
+          host: uri.host,
+          port: uri.port
+        }
+      ]
+    }
+  end
+
+  Config = ::Settings.tap do |config|
+    if ent_search_config = ent_search_es_config
+      Utility::Logger.error('Overriding elasticsearch config with ent-search config')
+      original_es_config = config[:elasticsearch].to_h
+      original_es_config.delete(:cloud_id)
+      original_es_config.delete(:hosts)
+      original_es_config.delete(:api_key)
+      config[:elasticsearch] = original_es_config.merge(ent_search_config)
+    end
+  end
+end

data/lib/app/console_app.rb

@@ -0,0 +1,278 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+$LOAD_PATH << '../'
+
+require 'app/config'
+require 'app/menu'
+require 'app/preflight_check'
+require 'app/worker'
+require 'connectors/registry'
+require 'utility'
+require 'core'
+
+module App
+  module ConsoleApp
+    extend self
+
+    INDEX_NAME_REGEXP = /[a-zA-Z]+[\d_\-a-zA-Z]*/
+
+    @commands = [
+      { :command => :sync_now, :hint => 'start one-time synchronization NOW' },
+      { :command => :register, :hint => 'register connector with Elasticsearch' },
+      { :command => :scheduling_on, :hint => 'enable connector scheduling' },
+      { :command => :scheduling_off, :hint => 'disable connector scheduling' },
+      { :command => :set_configurable_field, :hint => 'update the values of configurable fields' },
+      { :command => :read_configurable_fields, :hint => 'read the stored values of configurable fields' },
+      { :command => :status, :hint => 'check the status of a third-party service' },
+      { :command => :exit, :hint => 'end the program' }
+    ]
+
+    def connector_id
+      App::Config[:connector_id]
+    end
+
+    def update_connector_id(connector_id)
+      App::Config[:connector_id] = connector_id
+    end
+
+    def start_sync_now
+      return unless connector_registered?
+      puts 'Initiating synchronization NOW...'
+      Core::ElasticConnectorActions.force_sync(connector_id)
+      puts "Successfully synced for connector #{connector_id}"
+
+      config_settings = Core::ConnectorSettings.fetch_by_id(connector_id)
+      Core::ElasticConnectorActions.ensure_content_index_exists(config_settings[:index_name])
+      Core::SyncJobRunner.new(config_settings, App::Config[:service_type]).execute
+    end
+
+    def show_status
+      return unless connector_registered?
+      connector = current_connector
+
+      puts 'Checking status...'
+      puts connector.source_status
+      puts
+    end
+
+    def register_connector
+      if connector_id.present?
+        puts "You already have registered a connector with ID: #{connector_id}. Registering a new connector will overwrite the existing one."
+        puts 'Are you sure you want to continue? (y/N)'
+        return false unless gets.chomp.strip.casecmp('y').zero?
+      end
+      puts 'Please enter index name for data ingestion. Use only letters, underscored and dashes.'
+      index_name = gets.chomp.strip
+      unless INDEX_NAME_REGEXP.match?(index_name)
+        puts "Index name #{index_name} contains symbols that aren't allowed!"
+        return false
+      end
+      puts 'Do you want to use ICU Analysis Plugin? (y/N)'
+      use_analysis_icu = gets.chomp.strip.casecmp('y').zero?
+      language_code = select_analyzer
+      # create the connector
+      created_id = create_connector(index_name, use_analysis_icu, language_code)
+      update_connector_id(created_id)
+      true
+    end
+
+    def validate_cronline(cronline)
+      !!Fugit::Cron.parse(Utility::Cron.quartz_to_crontab(cronline))
+    end
+
+    def enable_scheduling
+      return unless connector_registered?
+
+      previous_schedule = Core::ConnectorSettings.fetch_by_id(connector_id)&.scheduling_settings&.fetch(:interval, nil)
+      if previous_schedule.present?
+        puts "Please enter a valid crontab expression for scheduling. Previous schedule was: #{previous_schedule}."
+      else
+        puts 'Please enter a valid crontab expression for scheduling.'
+      end
+      cron_expression = gets.chomp.strip.downcase
+      unless validate_cronline(cron_expression)
+        puts "Quartz Cron expression #{cron_expression} isn't valid!"
+        return
+      end
+      Core::ElasticConnectorActions.enable_connector_scheduling(connector_id, cron_expression)
+      puts "Enabled scheduling for connector #{connector_id} with cron expression #{cron_expression}"
+    end
+
+    def disable_scheduling
+      return unless connector_registered?
+      puts "Are you sure you want to disable scheduling for connector #{connector_id}? (y/n)"
+      return unless gets.chomp.strip.casecmp('y').zero?
+      Core::ElasticConnectorActions.disable_connector_scheduling(connector_id)
+      puts "Disabled scheduling for connector #{connector_id}"
+    end
+
+    def connector_registered?(warn_if_not: true)
+      result = connector_id.present?
+      if warn_if_not && !result
+        'You have no connector ID yet. Register a new connector before continuing.'
+      end
+      result
+    end
+
+    def create_connector(index_name, use_analysis_icu = false, language_code = :en)
+      id = Core::ElasticConnectorActions.create_connector(index_name, App::Config[:service_type])
+      Core::ElasticConnectorActions.ensure_content_index_exists(index_name, use_analysis_icu, language_code)
+      puts "Successfully registered connector #{index_name} with ID #{id}"
+
+      connector_settings = Core::ConnectorSettings.fetch_by_id(id)
+
+      connector_settings.id
+    end
+
+    def read_command
+      menu = App::Menu.new('Please select the command:', @commands)
+      menu.select_command
+    rescue Interrupt
+      exit_normally
+    end
+
+    def select_analyzer
+      analyzers = App::Menu.new('Please select a language analyzer', supported_analyzers)
+      analyzers.select_command
+    rescue Interrupt
+      exit_normally
+    end
+
+    def supported_analyzers
+      @supported_analyzers ||= YAML.safe_load(
+        File.read(Utility::Elasticsearch::Index::TextAnalysisSettings::LANGUAGE_DATA_FILE_PATH),
+        symbolize_names: true
+      ).map do |language_code, data|
+        { :command => language_code, :hint => data[:name] }
+      end
+      puts @supported_analyzers
+      @supported_analyzers
+    end
+
+    def wait_for_keypress(message = nil)
+      if message.present?
+        puts message
+      end
+      puts 'Press any key to continue...'
+      gets
+    end
+
+    def current_connector
+      connector_settings = Core::ConnectorSettings.fetch_by_id(App::Config[:connector_id])
+      service_type = App::Config[:service_type]
+      if service_type.present?
+        return registry.connector(service_type, connector_settings.configuration)
+      end
+      puts 'You have not set connector service type in settings. Please do so before continuing.'
+      nil
+    end
+
+    def exit_normally(message = 'Kthxbye!... ¯\_(ツ)_/¯')
+      puts(message)
+      exit(true)
+    end
+
+    def registry
+      @registry = Connectors::REGISTRY
+    end
+
+    puts 'Hello Connectors 3.0!'
+    sleep(1)
+
+    def set_configurable_field
+      return unless connector_registered?
+
+      connector = current_connector
+      connector_class = connector.class
+      current_values = Core::ConnectorSettings.fetch_by_id(connector_id)&.configuration
+      return unless connector.present?
+
+      puts 'Provided configurable fields:'
+      configurable_fields = connector_class.configurable_fields
+      fields = configurable_fields.each_key.map do |key|
+        field = configurable_fields[key].with_indifferent_access
+        current_value = current_values&.fetch(key, nil)
+        { :command => key, :hint => "#{field[:label]} (current value: #{current_value}, default: #{field[:value]})" }
+      end
+
+      menu = App::Menu.new('Please select the configurable field:', fields)
+      field_name = menu.select_command
+      field_label = configurable_fields.dig(field_name, :label)
+
+      puts 'Please enter the new value:'
+      new_value = gets.chomp.strip
+      Core::ElasticConnectorActions.set_configurable_field(connector_id, field_name, field_label, new_value)
+      Utility::Logger.debug("Successfully updated field #{field_name} for connector #{connector_id} to #{new_value}")
+    end
+
+    def read_configurable_fields
+      return unless connector_registered?
+
+      connector = current_connector
+      connector_class = connector.class
+
+      current_values = Core::ConnectorSettings.fetch_by_id(connector_id)&.configuration
+      return unless connector.present?
+
+      puts 'Persisted values of configurable fields:'
+      connector_class.configurable_fields.each_key.each do |key|
+        field = connector_class.configurable_fields[key].with_indifferent_access
+        current_value = current_values&.fetch(key, nil)
+        puts "* #{field[:label]} - current value: #{current_value}, default: #{field[:value]}"
+      end
+    end
+
+    Utility::Environment.set_execution_environment(App::Config) do
+      App::PreflightCheck.run!
+      loop do
+        command = read_command
+        case command
+        when :sync_now
+          start_sync_now
+          wait_for_keypress('Synchronization finished!')
+        when :status
+          show_status
+          wait_for_keypress('Status checked!')
+        when :register
+          if register_connector
+            wait_for_keypress('Please store connector ID in config file and restart the program.')
+          else
+            wait_for_keypress('Registration canceled!')
+          end
+        when :scheduling_on
+          enable_scheduling
+          wait_for_keypress('Scheduling enabled! Start synchronization to see it in action.')
+        when :scheduling_off
+          disable_scheduling
+          wait_for_keypress('Scheduling disabled! Starting synchronization will have no effect now.')
+        when :set_configurable_field
+          set_configurable_field
+          wait_for_keypress('Configurable field is updated!')
+        when :read_configurable_fields
+          read_configurable_fields
+          wait_for_keypress
+        when :exit
+          exit_normally
+        else
+          exit_normally('Sorry, this command is not yet implemented')
+        end
+      end
+    end
+  rescue App::PreflightCheck::CheckFailure => e
+    Utility::Logger.error("Preflight check failed: #{e.message}")
+    exit(-1)
+  rescue SystemExit
+    puts 'Exiting.'
+  rescue Interrupt
+    exit_normally
+  rescue StandardError => e
+    Utility::ExceptionTracking.log_exception(e)
+    exit(false)
+  end
+end