elasticgraph-opensearch 0.18.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: af83310eb66d78a17317a02ba1c62e052dead3d24a3ea7882d3593eb650a88a2
+  data.tar.gz: f80f785ba656811a3f1464b70621cd35bbaf5dcd77172f58c3b6b5e59ca6b5b3
+SHA512:
+  metadata.gz: 4d0c0bd2a60b02c9dce6ac6da9ef5a60a7441631491a877749b7f7780eef899972bc7b0698f277a3a804ad9e234ec589cc4402533a99d3f2633142cf6805ed0d
+  data.tar.gz: 554632704faac8a60529f59a87718af041b3f890ce833c660f5ad169e5173cce6f92995bf4fb629651ed274c2423733f5bae1286417af23498358a2b4ff848c7

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# ElasticGraph::OpenSearch
+
+Wraps the official OpenSearch client for use by ElasticGraph.

data/elasticgraph-opensearch.gemspec

@@ -0,0 +1,20 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :datastore_adapter) do |spec, eg_version|
+  spec.summary = "Wraps the OpenSearch client for use by ElasticGraph."
+
+  spec.add_dependency "elasticgraph-support", eg_version
+  spec.add_dependency "faraday", "~> 2.10"
+  spec.add_dependency "faraday-retry", "~> 2.2"
+  spec.add_dependency "opensearch-ruby", "~> 3.1"
+
+  spec.add_development_dependency "httpx", ">= 1.2.6", "< 2.0"
+end

data/lib/elastic_graph/opensearch/client.rb

@@ -0,0 +1,218 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "elastic_graph/error"
+require "elastic_graph/support/faraday_middleware/msearch_using_get_instead_of_post"
+require "elastic_graph/support/faraday_middleware/support_timeouts"
+require "elastic_graph/support/hash_util"
+require "faraday"
+require "faraday/retry"
+require "opensearch"
+
+module ElasticGraph
+  module OpenSearch
+    class Client
+      # @dynamic cluster_name
+      attr_reader :cluster_name
+
+      def initialize(cluster_name, url:, faraday_adapter: nil, retry_on_failure: 3, logger: nil)
+        @cluster_name = cluster_name
+
+        @raw_client = ::OpenSearch::Client.new(
+          adapter: faraday_adapter,
+          url: url,
+          retry_on_failure: retry_on_failure,
+          # We use `logger` for both the tracer and logger to log everything we can. While the trace and log output do overlap, one is
+          # not a strict superset of the other (for example, warnings go to `logger`, while full request bodies go to `tracer`).
+          logger: logger,
+          tracer: logger
+        ) do |faraday|
+          faraday.use Support::FaradayMiddleware::MSearchUsingGetInsteadOfPost
+          faraday.use Support::FaradayMiddleware::SupportTimeouts
+
+          # Note: this overrides the default retry exceptions, which includes `Faraday::TimeoutError`.
+          # That's important because we do NOT want a retry on timeout -- a timeout indicates a slow,
+          # expensive query, and is transformed to a `RequestExceededDeadlineError` by `SupportTimeouts`,
+          # anyway.
+          #
+          # In addition, it's worth noting that the retry middleware ONLY retries known idempotent HTTP
+          # methods (e.g. get/put/delete/head/options). POST requests will not be retried. We could
+          # configure it to make it retry POSTs but we'd need to do an analysis of all ElasticGraph requests to
+          # make sure all POST requests are truly idempotent, and at least for now, it's sufficient to skip
+          # any POST requests we make.
+          faraday.request :retry,
+            exceptions: [::Faraday::ConnectionFailed, ::Faraday::RetriableResponse],
+            max: retry_on_failure,
+            retry_statuses: [500, 502, 503] # Internal Server Error, Bad Gateway, Service Unavailable
+
+          yield faraday if block_given?
+        end
+
+        # Here we call `app` on each Faraday connection as a way to force it to resolve
+        # all configured middlewares and adapters. If it cannot load a required dependency
+        # (e.g. `httpx`), it'll fail fast with a clear error.
+        #
+        # Without this, we would instead get an error when the client was used to make
+        # a request for the first time, which isn't as ideal.
+        @raw_client.transport.transport.connections.each { |c| c.connection.app }
+      end
+
+      # Cluster APIs
+
+      def get_cluster_health
+        transform_errors { |c| c.cluster.health }
+      end
+
+      def get_node_os_stats
+        transform_errors { |c| c.nodes.stats(metric: "os") }
+      end
+
+      def get_flat_cluster_settings
+        transform_errors { |c| c.cluster.get_settings(flat_settings: true) }
+      end
+
+      # We only support persistent settings here because the Elasticsearch docs recommend against using transient settings:
+      # https://www.elastic.co/guide/en/elasticsearch/reference/8.13/cluster-update-settings.html
+      #
+      # > We no longer recommend using transient cluster settings. Use persistent cluster settings instead. If a cluster becomes unstable,
+      # > transient settings can clear unexpectedly, resulting in a potentially undesired cluster configuration.
+      #
+      # The OpenSearch documentation doesn't specifically mention this, but the same principle applies.
+      def put_persistent_cluster_settings(settings)
+        transform_errors { |c| c.cluster.put_settings(body: {persistent: settings}) }
+      end
+
+      # Script APIs
+
+      # Gets the script with the given ID. Returns `nil` if the script does not exist.
+      def get_script(id:)
+        transform_errors { |c| c.get_script(id: id) }
+      rescue ::OpenSearch::Transport::Transport::Errors::NotFound
+        nil
+      end
+
+      def put_script(id:, body:, context:)
+        transform_errors { |c| c.put_script(id: id, body: body, context: context) }
+      end
+
+      def delete_script(id:)
+        transform_errors { |c| c.delete_script(id: id) }
+      rescue ::OpenSearch::Transport::Transport::Errors::NotFound
+        # it's ok if it's already not there.
+      end
+
+      # Index Template APIs
+
+      def get_index_template(index_template_name)
+        transform_errors do |client|
+          client.indices.get_index_template(name: index_template_name)
+            .fetch("index_templates").to_h do |entry|
+              index_template = entry.fetch("index_template")
+
+              # OpenSearch ignores  `flat_settings` on the `/_index_template` API (but _only_ returns flattened settings from the index
+              # API). Here we flatten the settings to align with the flattened form ElasticGraph expects and uses everywhere.
+              flattened_settings = Support::HashUtil.flatten_and_stringify_keys(index_template.fetch("template").fetch("settings"))
+
+              index_template = index_template.merge({
+                "template" => index_template.fetch("template").merge({
+                  "settings" => flattened_settings
+                })
+              })
+
+              [entry.fetch("name"), index_template]
+            end.dig(index_template_name) || {}
+        end
+      rescue ::OpenSearch::Transport::Transport::Errors::NotFound
+        {}
+      end
+
+      def put_index_template(name:, body:)
+        transform_errors { |c| c.indices.put_index_template(name: name, body: body) }
+      end
+
+      def delete_index_template(index_template_name)
+        transform_errors { |c| c.indices.delete_index_template(name: [index_template_name], ignore: [404]) }
+      end
+
+      # Index APIs
+
+      def get_index(index_name)
+        transform_errors do |client|
+          client.indices.get(
+            index: index_name,
+            ignore_unavailable: true,
+            flat_settings: true
+          )[index_name] || {}
+        end
+      end
+
+      def list_indices_matching(index_expression)
+        transform_errors do |client|
+          client
+            .cat
+            .indices(index: index_expression, format: "json", h: ["index"])
+            .map { |index_hash| index_hash.fetch("index") }
+        end
+      end
+
+      def create_index(index:, body:)
+        transform_errors { |c| c.indices.create(index: index, body: body) }
+      end
+
+      def put_index_mapping(index:, body:)
+        transform_errors { |c| c.indices.put_mapping(index: index, body: body) }
+      end
+
+      def put_index_settings(index:, body:)
+        transform_errors { |c| c.indices.put_settings(index: index, body: body) }
+      end
+
+      def delete_indices(*index_names)
+        # `allow_no_indices: true` is needed when we attempt to delete a non-existing index to avoid errors. For rollover indices,
+        # when we delete the actual indices, we will always perform a wildcard deletion, and `allow_no_indices: true` is needed.
+        #
+        # Note that the Elasticsearch API documentation[^1] says that `allow_no_indices` defaults to `true` but a Elasticsearch Ruby
+        # client code comment[^2] says it defaults to `false`. Regardless, we don't want to rely on the default behavior that could change.
+        #
+        # [^1]: https://www.elastic.co/guide/en/elasticsearch/reference/8.12/indices-delete-index.html#delete-index-api-query-params
+        # [^2]: https://github.com/elastic/elasticsearch-ruby/blob/8.12/elasticsearch-api/lib/elasticsearch/api/actions/indices/delete.rb#L31
+        transform_errors do |client|
+          client.indices.delete(index: index_names, ignore_unavailable: true, allow_no_indices: true)
+        end
+      end
+
+      # Document APIs
+
+      def msearch(body:, headers: nil)
+        transform_errors { |c| c.msearch(body: body, headers: headers) }
+      end
+
+      def bulk(body:, refresh: false)
+        transform_errors { |c| c.bulk(body: body, filter_path: DATASTORE_BULK_FILTER_PATH, refresh: refresh) }
+      end
+
+      # Synchronously deletes all documents in the cluster. Intended for tests to give ourselves a clean slate.
+      # Supports an `index` argument so the caller can limit the deletion to a specific "scope" (e.g. a set of indices with a common prefix).
+      #
+      # Overrides `scroll` to `10s` to avoid getting a "Trying to create too many scroll contexts" error, as discussed here:
+      # https://discuss.elastic.co/t/too-many-scroll-contexts-with-update-by-query-and-or-delete-by-query/282325/1
+      def delete_all_documents(index: "_all")
+        transform_errors { |c| c.delete_by_query(index: index, body: {query: {match_all: _ = {}}}, refresh: true, scroll: "10s") }
+      end
+
+      private
+
+      def transform_errors
+        yield @raw_client
+      rescue ::OpenSearch::Transport::Transport::Errors::BadRequest => ex
+        raise BadDatastoreRequest, ex.message
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,290 @@
+--- !ruby/object:Gem::Specification
+name: elasticgraph-opensearch
+version: !ruby/object:Gem::Version
+  version: 0.18.0.0
+platform: ruby
+authors:
+- Myron Marston
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubocop-factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: coderay
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: flatware-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: super_diff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: opensearch-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: httpx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: 
+email:
+- myron@squareup.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- elasticgraph-opensearch.gemspec
+- lib/elastic_graph/opensearch/client.rb
+homepage: 
+licenses:
+- MIT
+metadata:
+  gem_category: datastore_adapter
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key: 
+specification_version: 4
+summary: Wraps the OpenSearch client for use by ElasticGraph.
+test_files: []