elasticgraph-datastore_core 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49631f3a48d49c98045429ad98a4e848b2f9bc74a76bc47d4e2fd790da0eca68
-  data.tar.gz: a04ec3ed650e6f3e0afbd016edf41b07e2b2dd30d75d090386d3a952eac305ea
+  metadata.gz: c82cda2b331f4a1eb7111a446031627802fe44b0460cb3ebb15ede1db975ac53
+  data.tar.gz: 65797d36a520d041e8095ba734ddf6a6c19ad39c2caabdacc3765c840bbc162b
 SHA512:
-  metadata.gz: 40153b2348f715ab6770a2300c1584889d11201a4616d39d0f2c8d92c83fb105a180e84cf15540b12f6dee92fe249a16ebfbba805af2f81d527b577ddb6f6a9a
-  data.tar.gz: 0eb31f6054a9529447f5177d1be60f56ac151aa4ccb0d43682b6dffdd76f5223bbf1c89dfb25ee233e93fbbbc3c03bcffe02cfa8c0a1e493cd26af832b70a84a
+  metadata.gz: b78b03e874a0177ecaf152a9cea09bb5ac791ba5fb1aafb5d8228a9c5b588df744235ade212eb1ee705f578434387855042c64b67ba910bdd20ff629cf547a42
+  data.tar.gz: 9c88f94fe2acd6c20503b8f67037968219393dd2bb41677debf788f1b6cb8e27f53c17b6fbb60ef67c2e57817b7901ab551046af7bab1c61dd29dfff1cb645ed

data/lib/elastic_graph/datastore_core/config.rb

@@ -6,7 +6,7 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/datastore_core/configuration/client_faraday_adapter"
+require "elastic_graph/support/config"
 require "elastic_graph/datastore_core/configuration/cluster_definition"
 require "elastic_graph/datastore_core/configuration/index_definition"
 require "elastic_graph/errors"
@@ -14,45 +14,273 @@ require "elastic_graph/errors"
 module ElasticGraph
   class DatastoreCore
     # Defines the configuration related to datastores.
-    class Config < ::Data.define(
-      # Configuration of the faraday adapter to use with the datastore client.
-      :client_faraday_adapter,
-      # Map of datastore cluster definitions, keyed by cluster name. The names will be referenced within
-      # `index_definitions` by `query_cluster` and `index_into_clusters` to identify
-      # datastore clusters. Each definition has a `url` and `settings`. `settings` contains datastore
-      # settings in the flattened name form, e.g. `"cluster.max_shards_per_node": 2000`.
-      :clusters,
-      # Map of index definition names to `IndexDefinition` objects containing customizations
-      # for the named index definitions for this environment.
-      :index_definitions,
-      # Determines if we log requests/responses to/from the datastore.
-      # Defaults to `false`.
-      :log_traffic,
-      # Passed down to the datastore client, controls the number of times ElasticGraph attempts a call against
-      # the datastore before failing. Retrying a handful of times is generally advantageous, since some sporadic
-      # failures are expected during the course of operation, and better to retry than fail the entire call.
-      # Defaults to 3.
-      :max_client_retries
-    )
-      # Helper method to build an instance from parsed YAML config.
-      def self.from_parsed_yaml(parsed_yaml)
-        parsed_yaml = parsed_yaml.fetch("datastore")
-        extra_keys = parsed_yaml.keys - EXPECTED_KEYS
-
-        unless extra_keys.empty?
-          raise Errors::ConfigError, "Unknown `datastore` config settings: #{extra_keys.join(", ")}"
-        end
-
-        new(
-          client_faraday_adapter: Configuration::ClientFaradayAdapter.from_parsed_yaml(parsed_yaml),
-          clusters: Configuration::ClusterDefinition.definitions_by_name_hash_from(parsed_yaml.fetch("clusters")),
-          index_definitions: Configuration::IndexDefinition.definitions_by_name_hash_from(parsed_yaml.fetch("index_definitions")),
-          log_traffic: parsed_yaml.fetch("log_traffic", false),
-          max_client_retries: parsed_yaml.fetch("max_client_retries", 3)
+    class Config < Support::Config.define(:client_faraday_adapter, :clusters, :index_definitions, :log_traffic, :max_client_retries)
+      all_json_schema_types = ["array", "string", "number", "boolean", "object", "null"]
+
+      json_schema at: "datastore",
+        optional: false,
+        description: "Configuration for datastore connections and index definitions used by all parts of ElasticGraph.",
+        properties: {
+          client_faraday_adapter: {
+            type: "object",
+            description: "Configuration of the faraday adapter to use with the datastore client.",
+            properties: {
+              name: {
+                type: ["string", "null"],
+                minLength: 1,
+                description: "The faraday adapter to use with the datastore client, such as `httpx` or `typhoeus`.",
+                examples: ["net_http", "httpx", "typhoeus", nil],
+                default: nil
+              },
+              require: {
+                type: ["string", "null"],
+                minLength: 1,
+                description: "A Ruby library to require which provides the named adapter (optional).",
+                examples: ["httpx/adapters/faraday"],
+                default: nil
+              }
+            },
+            default: {"name" => nil, "require" => nil},
+            examples: [
+              {"name" => "net_http"},
+              {"name" => "httpx", "require" => "httpx/adapters/faraday"}
+            ]
+          },
+
+          clusters: {
+            type: "object",
+            description: "Map of datastore cluster definitions, keyed by cluster name. The names will be referenced within " \
+              "`index_definitions` by `query_cluster` and `index_into_clusters` to identify datastore clusters.",
+            patternProperties: {
+              /.+/.source => {
+                type: "object",
+                description: "Configuration for a specific datastore cluster.",
+                examples: [{
+                  url: "http://localhost:9200",
+                  backend: "elasticsearch",
+                  settings: {"cluster.max_shards_per_node" => 2000}
+                }],
+                properties: {
+                  url: {
+                    type: "string",
+                    minLength: 1,
+                    description: "The URL of the datastore cluster.",
+                    examples: ["http://localhost:9200", "https://my-cluster.example.com:9200"]
+                  },
+                  backend: {
+                    enum: ["elasticsearch", "opensearch"],
+                    description: "Determines whether `elasticgraph-elasticsearch` or `elasticgraph-opensearch` is used for the datastore client.",
+                    examples: ["elasticsearch", "opensearch"]
+                  },
+                  settings: {
+                    type: "object",
+                    description: "Datastore settings in flattened (i.e. dot-separated) name form.",
+                    patternProperties: {/.+/.source => {"type" => all_json_schema_types}},
+                    examples: [{"cluster.max_shards_per_node" => 2000}],
+                    default: {} # : untyped
+                  }
+                },
+                required: ["url", "backend"]
+              }
+            },
+            examples: [{
+              "main" => {
+                url: "http://localhost:9200",
+                backend: "elasticsearch",
+                settings: {"cluster.max_shards_per_node" => 2000}
+              }
+            }]
+          },
+
+          index_definitions: {
+            type: "object",
+            description: "Map of index definition names to `IndexDefinition` objects containing customizations for the named index definitions for this environment.",
+            patternProperties: {
+              /.+/.source => {
+                type: "object",
+                description: "Configuration for a specific index definition.",
+                examples: [example_index_def = {
+                  "query_cluster" => "main",
+                  "index_into_clusters" => ["main"],
+                  "ignore_routing_values" => ["ABC1234567"], # : untyped
+                  "setting_overrides" => {
+                    "number_of_shards" => 256
+                  },
+                  "setting_overrides_by_timestamp" => {
+                    "2022-01-01T00:00:00Z" => {
+                      "number_of_shards" => 64
+                    },
+                    "2023-01-01T00:00:00Z" => {
+                      "number_of_shards" => 96
+                    },
+                    "2024-01-01T00:00:00Z" => {
+                      "number_of_shards" => 128
+                    }
+                  },
+                  "custom_timestamp_ranges" => [
+                    {
+                      "index_name_suffix" => "before_2022",
+                      "lt" => "2022-01-01T00:00:00Z",
+                      "setting_overrides" => {"number_of_shards" => 32}
+                    },
+                    {
+                      "index_name_suffix" => "after_2026",
+                      "gte" => "2027-01-01T00:00:00Z",
+                      "setting_overrides" => {"number_of_shards" => 32}
+                    }
+                  ]
+                }],
+                properties: {
+                  query_cluster: {
+                    type: "string",
+                    description: "Named search cluster to be used for queries on this index. The value must match be a key in the `clusters` map.",
+                    examples: ["main", "search_cluster"]
+                  },
+                  index_into_clusters: {
+                    type: "array",
+                    items: {type: "string", minLength: 1},
+                    description: "Named search clusters to index data into. The values must match keys in the `clusters` map.",
+                    examples: [["main"], ["cluster1", "cluster2"]]
+                  },
+                  ignore_routing_values: {
+                    type: "array",
+                    items: {"type" => "string"},
+                    description: "Shard routing values for which the data should be spread across all shards instead of concentrating it " \
+                      "on a single shard. This is intended to be used when a handful of known routing value contain such a large portion " \
+                      "of the dataset that it extremely lopsided shards would result. Spreading the data across all shards may perform " \
+                      "better.",
+                    default: [], # : untyped
+                    examples: [
+                      [], # : untyped
+                      ["mega_tenant1", "mega_tenant2"]
+                    ]
+                  },
+                  setting_overrides: {
+                    type: "object",
+                    description: "Overrides for index (or index template) settings. The settings specified here will override any settings " \
+                      "specified on the Ruby schema definition. This is commonly used to configure a different `number_of_shards` in each " \
+                      "environment. An `index.` prefix will be added to the names of all settings before submitting them to the datastore.",
+                    patternProperties: {/.+/.source => {"type" => all_json_schema_types}},
+                    default: {}, # :untyped
+                    examples: [{"number_of_shards" => 5}]
+                  },
+                  setting_overrides_by_timestamp: {
+                    type: "object",
+                    description: "Overrides for index template settings for specific dates, allowing variation of settings for different " \
+                      "rollover indices. This is commonly used to configure a different `number_of_shards` for each year or month when " \
+                      "using yearly or monthly rollover.",
+                    propertyNames: {type: "string", format: "date-time"},
+                    additionalProperties: {type: "object", patternProperties: {/.+/.source => {"type" => all_json_schema_types}}},
+                    default: {}, # : untyped
+                    examples: [{"2025-01-01T00:00:00Z" => {"number_of_shards" => 10}}]
+                  },
+                  custom_timestamp_ranges: {
+                    type: "array",
+                    description: "Array of custom timestamp ranges that allow different index settings for specific time periods.",
+                    items: {
+                      type: "object",
+                      properties: {
+                        index_name_suffix: {
+                          type: "string",
+                          description: "Suffix to append to the index name for this custom range.",
+                          examples: ["before_2020", "after_2027"]
+                        },
+                        setting_overrides: {
+                          type: "object",
+                          description: "Setting overrides for this custom timestamp range.",
+                          patternProperties: {/.+/.source => {"type" => all_json_schema_types}},
+                          examples: [{"number_of_shards" => 17}, {"number_of_replicas" => 2}]
+                        },
+                        lt: {
+                          type: ["string", "null"],
+                          format: "date-time",
+                          description: "Less than timestamp boundary (ISO 8601 format).",
+                          examples: ["2015-01-01T00:00:00Z", "2020-12-31T23:59:59Z"],
+                          default: nil
+                        },
+                        lte: {
+                          type: ["string", "null"],
+                          format: "date-time",
+                          description: "Less than or equal timestamp boundary (ISO 8601 format).",
+                          examples: ["2015-01-01T00:00:00Z", "2020-12-31T23:59:59Z"],
+                          default: nil
+                        },
+                        gt: {
+                          type: ["string", "null"],
+                          format: "date-time",
+                          description: "Greater than timestamp boundary (ISO 8601 format).",
+                          examples: ["2015-01-01T00:00:00Z", "2020-01-01T00:00:00Z"],
+                          default: nil
+                        },
+                        gte: {
+                          type: ["string", "null"],
+                          format: "date-time",
+                          description: "Greater than or equal timestamp boundary (ISO 8601 format).",
+                          examples: ["2015-01-01T00:00:00Z", "2020-01-01T00:00:00Z"],
+                          default: nil
+                        }
+                      },
+                      required: ["index_name_suffix", "setting_overrides"],
+                      anyOf: [
+                        {required: ["lt"]},
+                        {required: ["lte"]},
+                        {required: ["gt"]},
+                        {required: ["gte"]}
+                      ]
+                    },
+                    default: [], # : untyped
+                    examples: [[{
+                      "index_name_suffix" => "before_2015",
+                      "lt" => "2015-01-01T00:00:00Z",
+                      "setting_overrides" => {"number_of_shards" => 17}
+                    }]]
+                  }
+                },
+                required: ["query_cluster", "index_into_clusters"]
+              }
+            },
+            examples: [{"widgets" => example_index_def}]
+          },
+
+          log_traffic: {
+            type: "boolean",
+            description: "Determines if we log requests/responses to/from the datastore.",
+            default: false,
+            examples: [false, true]
+          },
+
+          max_client_retries: {
+            type: "integer",
+            description: "Passed down to the datastore client. Controls the number of times ElasticGraph attempts a call against the " \
+              "datastore before failing. Retrying a handful of times is generally advantageous, since some sporadic failures are expected " \
+              "during the course of operation, and better to retry than fail the entire call.",
+            default: 3,
+            minimum: 0,
+            examples: [3, 5, 10]
+          }
+        },
+        required: ["clusters", "index_definitions"]
+
+      private
+
+      def convert_values(clusters:, index_definitions:, client_faraday_adapter:, **values)
+        clusters = Configuration::ClusterDefinition.definitions_by_name_hash_from(clusters)
+        index_definitions = Configuration::IndexDefinition.definitions_by_name_hash_from(index_definitions)
+        client_faraday_adapter = Configuration::ClientFaradayAdapter.new(
+          name: client_faraday_adapter.fetch("name")&.to_sym,
+          require: client_faraday_adapter.fetch("require")
         )
+
+        values.merge({
+          client_faraday_adapter: client_faraday_adapter,
+          clusters: clusters,
+          index_definitions: index_definitions
+        })
       end
+    end
 
-      EXPECTED_KEYS = members.map(&:to_s)
+    module Configuration
+      ClientFaradayAdapter = ::Data.define(:name, :require)
     end
   end
 end

data/lib/elastic_graph/datastore_core/configuration/cluster_definition.rb

@@ -12,25 +12,15 @@ module ElasticGraph
   class DatastoreCore
     module Configuration
       class ClusterDefinition < ::Data.define(:url, :backend_client_class, :settings)
-        def self.from_hash(hash)
-          extra_keys = hash.keys - EXPECTED_KEYS
-
-          unless extra_keys.empty?
-            raise Errors::ConfigError, "Unknown `datastore.clusters` config settings: #{extra_keys.join(", ")}"
-          end
+        BACKEND_CLIENT_CLASSES = {
+          "elasticsearch" => "ElasticGraph::Elasticsearch::Client",
+          "opensearch" => "ElasticGraph::OpenSearch::Client"
+        }
 
-          backend_name = hash["backend"]
-          backend_client_class =
-            case backend_name
-            when "elasticsearch"
-              require "elastic_graph/elasticsearch/client"
-              Elasticsearch::Client
-            when "opensearch"
-              require "elastic_graph/opensearch/client"
-              OpenSearch::Client
-            else
-              raise Errors::ConfigError, "Unknown `datastore.clusters` backend: `#{backend_name}`. Valid backends are `elasticsearch` and `opensearch`."
-            end
+        def self.from_hash(hash)
+          backend_name = hash.fetch("backend")
+          require "elastic_graph/#{backend_name}/client"
+          backend_client_class = ::Object.const_get(BACKEND_CLIENT_CLASSES.fetch(backend_name))
 
           new(
             url: hash.fetch("url"),
@@ -44,8 +34,6 @@ module ElasticGraph
             from_hash(cluster_def_hash)
           end
         end
-
-        EXPECTED_KEYS = members.map(&:to_s) - ["backend_client_class"] + ["backend"]
       end
     end
   end

data/lib/elastic_graph/datastore_core/configuration/index_definition.rb

@@ -85,7 +85,7 @@ module ElasticGraph
 
           def self.ranges_from(range_hashes)
             range_hashes.map do |range_hash|
-              __skip__ = from(**range_hash.transform_keys(&:to_sym))
+              __skip__ = from(**range_hash.compact.transform_keys(&:to_sym))
             end
           end
 

data/lib/elastic_graph/datastore_core.rb

@@ -22,7 +22,7 @@ module ElasticGraph
 
     def self.from_parsed_yaml(parsed_yaml, &client_customization_block)
       new(
-        config: DatastoreCore::Config.from_parsed_yaml(parsed_yaml),
+        config: DatastoreCore::Config.from_parsed_yaml!(parsed_yaml),
         logger: Support::Logger.from_parsed_yaml(parsed_yaml),
         schema_artifacts: SchemaArtifacts.from_parsed_yaml(parsed_yaml),
         client_customization_block: client_customization_block

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-datastore_core
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Myron Marston
@@ -17,84 +17,84 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-admin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-opensearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-schema_definition
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 email:
 - myron@squareup.com
 executables: []
@@ -105,7 +105,6 @@ files:
 - README.md
 - lib/elastic_graph/datastore_core.rb
 - lib/elastic_graph/datastore_core/config.rb
-- lib/elastic_graph/datastore_core/configuration/client_faraday_adapter.rb
 - lib/elastic_graph/datastore_core/configuration/cluster_definition.rb
 - lib/elastic_graph/datastore_core/configuration/index_definition.rb
 - lib/elastic_graph/datastore_core/index_config_normalizer.rb
@@ -119,10 +118,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/block/elasticgraph/issues
-  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0
-  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0/
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.1
+  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.1/
   homepage_uri: https://block.github.io/elasticgraph/
-  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0/elasticgraph-datastore_core
+  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.1/elasticgraph-datastore_core
   gem_category: core
 rdoc_options: []
 require_paths:

data/lib/elastic_graph/datastore_core/configuration/client_faraday_adapter.rb

@@ -1,38 +0,0 @@
-# Copyright 2024 - 2025 Block, Inc.
-#
-# Use of this source code is governed by an MIT-style
-# license that can be found in the LICENSE file or at
-# https://opensource.org/licenses/MIT.
-#
-# frozen_string_literal: true
-
-module ElasticGraph
-  class DatastoreCore
-    module Configuration
-      class ClientFaradayAdapter < ::Data.define(
-        # The faraday adapter to use with the datastore client, such as `httpx` or `typhoeus`.
-        # For more info, see:
-        # https://github.com/elastic/elasticsearch-ruby/commit/a7bbdbf2a96168c1b33dca46ee160d2d4d75ada0
-        :name,
-        # A Ruby library to require which provides the named adapter (optional).
-        :require
-      )
-        def self.from_parsed_yaml(parsed_yaml)
-          parsed_yaml = parsed_yaml.fetch("client_faraday_adapter") || {}
-          extra_keys = parsed_yaml.keys - EXPECTED_KEYS
-
-          unless extra_keys.empty?
-            raise Errors::ConfigError, "Unknown `datastore.client_faraday_adapter` config settings: #{extra_keys.join(", ")}"
-          end
-
-          new(
-            name: parsed_yaml["name"]&.to_sym,
-            require: parsed_yaml["require"]
-          )
-        end
-
-        EXPECTED_KEYS = members.map(&:to_s)
-      end
-    end
-  end
-end