connectors_utility 8.4.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 824ceaf9dec38db287b8ec5739847b5dc08b3c1ea140bce575076bceb8b6caf6
+  data.tar.gz: b287e744ebe57f49162437744649e0e2d304c3e0a2ecfbb1bdeb082e0463f9f8
+SHA512:
+  metadata.gz: 856965cb7aca1080b86e241e688495c18228dd75d9ee1dc895aed16b3ab86f5a8694c6715bd0e88699803665590c3d4c18d302e3b593ee4fba4a85ccc921c778
+  data.tar.gz: 32398b3b4c4771665dcddd9f493f7ca1155757828cfcb9ddb52ca9f2f2e8eca09d4f8f1a84d6aee108d9af18da2dc5dd2eb1829009bc1bce1896a87b28fb7474

data/LICENSE

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

data/NOTICE.txt

@@ -0,0 +1,2 @@
+connectors
+Copyright 2022 Elasticsearch B.V.

data/lib/connectors_utility.rb

@@ -0,0 +1,10 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require_relative 'utility/elasticsearch/index/text_analysis_settings'
+require_relative 'utility/elasticsearch/index/mappings'

data/lib/utility/elasticsearch/index/language_data.yml

@@ -0,0 +1,111 @@
+---
+da:
+  name: Danish
+  stemmer: danish
+  stop_words: _danish_
+de:
+  name: German
+  stemmer: light_german
+  stop_words: _german_
+en:
+  name: English
+  stemmer: light_english
+  stop_words: _english_
+es:
+  name: Spanish
+  stemmer: light_spanish
+  stop_words: _spanish_
+fr:
+  name: French
+  stemmer: light_french
+  stop_words: _french_
+  custom_filter_definitions:
+    fr-elision:
+      type: elision
+      articles:
+      - l
+      - m
+      - t
+      - qu
+      - n
+      - s
+      - j
+      - d
+      - c
+      - jusqu
+      - quoiqu
+      - lorsqu
+      - puisqu
+      articles_case: true
+  prepended_filters:
+  - fr-elision
+it:
+  name: Italian
+  stemmer: light_italian
+  stop_words: _italian_
+  custom_filter_definitions:
+    it-elision:
+      type: elision
+      articles:
+      - c
+      - l
+      - all
+      - dall
+      - dell
+      - nell
+      - sull
+      - coll
+      - pell
+      - gl
+      - agl
+      - dagl
+      - degl
+      - negl
+      - sugl
+      - un
+      - m
+      - t
+      - s
+      - v
+      - d
+      articles_case: true
+  prepended_filters:
+  - it-elision
+ja:
+  name: Japanese
+  stemmer: light_english
+  stop_words: _english_
+  postpended_filters:
+  - cjk_bigram
+ko:
+  name: Korean
+  stemmer: light_english
+  stop_words: _english_
+  postpended_filters:
+  - cjk_bigram
+nl:
+  name: Dutch
+  stemmer: dutch
+  stop_words: _dutch_
+pt:
+  name: Portuguese
+  stemmer: light_portuguese
+  stop_words: _portuguese_
+pt-br:
+  name: Portuguese (Brazil)
+  stemmer: brazilian
+  stop_words: _brazilian_
+ru:
+  name: Russian
+  stemmer: russian
+  stop_words: _russian_
+th:
+  name: Thai
+  stemmer: light_english
+  stop_words: _thai_
+zh:
+  name: Chinese
+  stemmer: light_english
+  stop_words: _english_
+  postpended_filters:
+  - cjk_bigram

data/lib/utility/elasticsearch/index/mappings.rb

@@ -0,0 +1,78 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+module Utility
+  module Elasticsearch
+    module Index
+      module Mappings
+        ENUM_IGNORE_ABOVE = 2048
+
+        WORKPLACE_SEARCH_SUBEXTRACTION_STAMP_FIELD_MAPPINGS = {
+          _subextracted_as_of: {
+            type: 'date'
+          },
+          _subextracted_version: {
+            type: 'keyword'
+          }
+        }.freeze
+
+        def self.default_text_fields_mappings(connectors_index:)
+          {
+            dynamic: true,
+            dynamic_templates: [
+              {
+                data: {
+                  match_mapping_type: 'string',
+                  mapping: {
+                    type: 'text',
+                    analyzer: 'iq_text_base',
+                    index_options: 'freqs',
+                    fields: {
+                      'stem': {
+                        type: 'text',
+                        analyzer: 'iq_text_stem'
+                      },
+                      'prefix' => {
+                        type: 'text',
+                        analyzer: 'i_prefix',
+                        search_analyzer: 'q_prefix',
+                        index_options: 'docs'
+                      },
+                      'delimiter' => {
+                        type: 'text',
+                        analyzer: 'iq_text_delimiter',
+                        index_options: 'freqs'
+                      },
+                      'joined': {
+                        type: 'text',
+                        analyzer: 'i_text_bigram',
+                        search_analyzer: 'q_text_bigram',
+                        index_options: 'freqs'
+                      },
+                      'enum': {
+                        type: 'keyword',
+                        ignore_above: ENUM_IGNORE_ABOVE
+                      }
+                    }
+                  }
+                }
+              }
+            ],
+            properties: {
+              id: {
+                type: 'keyword'
+              }
+            }.tap do |properties|
+              properties.merge!(WORKPLACE_SEARCH_SUBEXTRACTION_STAMP_FIELD_MAPPINGS) if connectors_index
+            end
+          }
+        end
+      end
+    end
+  end
+end

data/lib/utility/elasticsearch/index/text_analysis_settings.rb

@@ -0,0 +1,226 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Utility
+  module Elasticsearch
+    module Index
+      class TextAnalysisSettings
+        class UnsupportedLanguageCode < StandardError; end
+
+        DEFAULT_LANGUAGE = :en
+        FRONT_NGRAM_MAX_GRAM = 12
+        LANGUAGE_DATA_FILE_PATH = File.join(File.dirname(__FILE__), 'language_data.yml')
+
+        GENERIC_FILTERS = {
+          front_ngram: {
+            type: 'edge_ngram',
+            min_gram: 1,
+            max_gram: FRONT_NGRAM_MAX_GRAM
+          },
+          delimiter: {
+            type: 'word_delimiter_graph',
+            generate_word_parts: true,
+            generate_number_parts: true,
+            catenate_words: true,
+            catenate_numbers: true,
+            catenate_all: true,
+            preserve_original: false,
+            split_on_case_change: true,
+            split_on_numerics: true,
+            stem_english_possessive: true
+          },
+          bigram_joiner: {
+            type: 'shingle',
+            token_separator: '',
+            max_shingle_size: 2,
+            output_unigrams: false
+          },
+          bigram_joiner_unigrams: {
+            type: 'shingle',
+            token_separator: '',
+            max_shingle_size: 2,
+            output_unigrams: true
+          },
+          bigram_max_size: {
+            type: 'length',
+            min: 0,
+            max: 16
+          }
+        }.freeze
+
+        NON_ICU_ANALYSIS_SETTINGS = {
+          tokenizer_name: 'standard', folding_filters: %w(cjk_width lowercase asciifolding)
+        }.freeze
+
+        ICU_ANALYSIS_SETTINGS = {
+          tokenizer_name: 'icu_tokenizer', folding_filters: %w(icu_folding)
+        }.freeze
+
+        def initialize(language_code: nil, analysis_icu: false)
+          @language_code = (language_code || DEFAULT_LANGUAGE).to_sym
+
+          raise UnsupportedLanguageCode, "Language '#{language_code}' is not supported" unless language_data[@language_code]
+
+          @analysis_icu = analysis_icu
+          @analysis_settings = icu_settings(analysis_icu)
+        end
+
+        def to_h
+          {
+            analysis: {
+              analyzer: analyzer_definitions,
+              filter: filter_definitions
+            },
+            index: {
+              similarity: {
+                default: {
+                  type: 'BM25'
+                }
+              }
+            }
+          }
+        end
+
+        private
+
+        attr_reader :language_code, :analysis_settings
+
+        def icu_settings(analysis_settings)
+          return ICU_ANALYSIS_SETTINGS if analysis_settings
+
+          NON_ICU_ANALYSIS_SETTINGS
+        end
+
+        def stemmer_name
+          language_data[language_code][:stemmer]
+        end
+
+        def stop_words_name_or_list
+          language_data[language_code][:stop_words]
+        end
+
+        def custom_filter_definitions
+          language_data[language_code][:custom_filter_definitions] || {}
+        end
+
+        def prepended_filters
+          language_data[language_code][:prepended_filters] || []
+        end
+
+        def postpended_filters
+          language_data[language_code][:postpended_filters] || []
+        end
+
+        def stem_filter_name
+          "#{language_code}-stem-filter".to_sym
+        end
+
+        def stop_words_filter_name
+          "#{language_code}-stop-words-filter".to_sym
+        end
+
+        def filter_definitions
+          definitions = GENERIC_FILTERS.dup
+
+          definitions[stem_filter_name] = {
+            type: 'stemmer',
+            name: stemmer_name
+          }
+
+          definitions[stop_words_filter_name] = {
+            type: 'stop',
+            stopwords: stop_words_name_or_list
+          }
+
+          definitions.merge(custom_filter_definitions)
+        end
+
+        def analyzer_definitions
+          definitions = {}
+
+          definitions[:i_prefix] = {
+            tokenizer: analysis_settings[:tokenizer_name],
+            filter: [
+              *analysis_settings[:folding_filters],
+              'front_ngram'
+            ]
+          }
+
+          definitions[:q_prefix] = {
+            tokenizer: analysis_settings[:tokenizer_name],
+            filter: [
+              *analysis_settings[:folding_filters]
+            ]
+          }
+
+          definitions[:iq_text_base] = {
+            tokenizer: analysis_settings[:tokenizer_name],
+            filter: [
+              *analysis_settings[:folding_filters],
+              stop_words_filter_name
+            ]
+          }
+
+          definitions[:iq_text_stem] = {
+            tokenizer: analysis_settings[:tokenizer_name],
+            filter: [
+              *prepended_filters,
+              *analysis_settings[:folding_filters],
+              stop_words_filter_name,
+              stem_filter_name,
+              *postpended_filters
+            ]
+          }
+
+          definitions[:iq_text_delimiter] = {
+            tokenizer: 'whitespace',
+            filter: [
+              *prepended_filters,
+              'delimiter',
+              *analysis_settings[:folding_filters],
+              stop_words_filter_name,
+              stem_filter_name,
+              *postpended_filters
+            ]
+          }
+
+          definitions[:i_text_bigram] = {
+            tokenizer: analysis_settings[:tokenizer_name],
+            filter: [
+              *analysis_settings[:folding_filters],
+              stem_filter_name,
+              'bigram_joiner',
+              'bigram_max_size'
+            ]
+          }
+
+          definitions[:q_text_bigram] = {
+            tokenizer: analysis_settings[:tokenizer_name],
+            filter: [
+              *analysis_settings[:folding_filters],
+              stem_filter_name,
+              'bigram_joiner_unigrams',
+              'bigram_max_size'
+            ]
+          }
+
+          definitions
+        end
+
+        def language_data
+          @language_data ||= YAML.safe_load(
+            File.read(LANGUAGE_DATA_FILE_PATH),
+            symbolize_names: true
+          )
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: connectors_utility
+version: !ruby/object:Gem::Version
+  version: 8.4.0.0
+platform: ruby
+authors:
+- Elastic
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-07-14 00:00:00.000000000 Z
+dependencies: []
+description: ''
+email: ent-search-dev@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- NOTICE.txt
+- lib/connectors_utility.rb
+- lib/utility/elasticsearch/index/language_data.yml
+- lib/utility/elasticsearch/index/mappings.rb
+- lib/utility/elasticsearch/index/text_analysis_settings.rb
+homepage: https://github.com/elastic/connectors-ruby
+licenses:
+- Elastic-2.0
+metadata:
+  revision: c9283d0e12a3ae8253225becbefef02d0c6153c8
+  repository: git@github.com:elastic/connectors.git
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3.1
+signing_key: 
+specification_version: 4
+summary: Gem containing shared Connector Services libraries
+test_files: []