elasticgraph-local 0.17.1.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 67889904d465a3b0ab2d61c61cd65f23bd9f50a3681655b51cafba0341863231
+  data.tar.gz: 17b51c009ae9fdc26b86960ca21a1f3369210804f1d86276ea4396de4ce0dd80
+SHA512:
+  metadata.gz: 4f2515eab25e53d9a3ad562d7a47d30b81f961de84b1df3896042be6386092dfbd1e2304389e1549da582d607f803cb68355e09ca2ff9149eff684d8366df1bc
+  data.tar.gz: 7251dc4f6bf5075d9c185098dae862a5be5e7df972c89c4e85d5fc1a7cec207a57078174a27be7c9fabf777d78238556e0025e55f6b9b4c445922eb7775aedd2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,80 @@
+# ElasticGraph::Local
+
+Provides support for developing and running ElasticGraph applications locally.
+These locally running ElasticGraph applications use 100% fake generated data
+so as not to require a publisher of real data to be implemented.
+
+## Installation
+
+Add `elasticgraph-local` to a new project `Gemfile`:
+
+```ruby
+source "https://rubygems.org"
+
+group :development do
+  gem "factory_bot"
+  gem "faker"
+  gem "elasticgraph-local"
+end
+```
+
+As shown above, you can also pull in any gems that will help you
+generate fake data. We tend to use [factory_bot](https://github.com/thoughtbot/factory_bot)
+and [faker](https://github.com/faker-ruby/faker). `elasticgraph-local` should be defined
+in the `development` group (you don't want to include it in any staging or production
+deployment).
+
+Next, install the `elasticgraph-local` rake tasks in your `Rakefile`, with code like:
+
+``` ruby
+require 'elastic_graph/local/rake_tasks'
+
+ElasticGraph::Local::RakeTasks.new(
+  local_config_yaml: "config/settings/development.yaml",
+  path_to_schema: "config/schema.rb"
+) do |tasks|
+  tasks.define_fake_data_batch_for :widgets do |batch|
+    # Use faker/factory_bot etc here to generate fake data
+    # and add it to the `batch` array.
+    # You'll probably want to put that logic in another file
+    # and load it from here.
+  end
+end
+```
+
+## Usage
+
+Everything you need is provided by rake tasks. Run the following to see what they are:
+
+```bash
+$ bundle exec rake -T
+```
+
+At a high level, this provides tasks that help you to:
+
+1. Boot Elasticsearch/OpenSearch (+ their corresponding dashboards) locally using the `opensearch:*`/`elasticsearch:*` tasks.
+2. Generate and validate ElasticGraph schema artifacts using the `schema_artifacts:*` tasks.
+3. Configure your locally running Elasticsearch/OpenSearch using the `clusters:configure:perform` task.
+4. Index fake data into Elasticsearch/OpenSearch (either running locally or on AWS) using the `index_fake_data:*` tasks.
+5. Boot the ElasticGraph GraphQL endpoint and GraphiQL in-browser UI using the `boot_graphiql` task.
+
+If you just want to boot ElasticGraph locally without worrying about any of the details, run:
+
+```
+$ bundle exec rake boot_locally
+```
+
+That sequences each of the other tasks so that, with a single command, you can go from nothing to a
+locally running ElasticGraph instance with data that you can query from your browser.
+
+### Managing Elasticsearch/Opensearch
+
+The `opensearch:`/`elasticsearch:` tasks will boot the desired Elasticsearch or OpenSearch version using docker
+along with the corresponding dashboards (Kibana for Elasticsearch, OpenSearch Dashboards for OpenSearch). You can
+use either the `:boot` or `:daemon` tasks:
+
+* The `:boot` task will keep Elasticsearch/Opensearch in the foreground, allowing you to see the logs.
+* The `:daemon` task runs Elasticsearch/Opensearch as a background daemon task. Notably, it waits to return
+  until Elasticsearch/Opensearch are ready to receive traffic.
+
+If you use a `:daemon` task, you can later use the corresponding `:halt` task to stop the daemon.

data/elasticgraph-local.gemspec

@@ -0,0 +1,25 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :local) do |spec, eg_version|
+  spec.summary = "Provides support for developing and running ElasticGraph applications locally."
+
+  spec.add_dependency "elasticgraph-admin", eg_version
+  spec.add_dependency "elasticgraph-graphql", eg_version
+  spec.add_dependency "elasticgraph-indexer", eg_version
+  spec.add_dependency "elasticgraph-rack", eg_version
+  spec.add_dependency "elasticgraph-schema_definition", eg_version
+  spec.add_dependency "rackup", "~> 2.1"
+  spec.add_dependency "rake", "~> 13.2"
+
+  spec.add_development_dependency "elasticgraph-elasticsearch", eg_version
+  spec.add_development_dependency "elasticgraph-opensearch", eg_version
+  spec.add_development_dependency "httpx", ">= 1.2.6", "< 2.0"
+end

data/lib/elastic_graph/local/config.ru

@@ -0,0 +1,15 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+# This `config.ru` file is used by the `rake boot_graphiql` task.
+
+require "elastic_graph/graphql"
+require "elastic_graph/rack/graphiql"
+
+graphql = ElasticGraph::GraphQL.from_yaml_file(ENV.fetch("ELASTICGRAPH_YAML_FILE"))
+run ElasticGraph::Rack::GraphiQL.new(graphql)

data/lib/elastic_graph/local/docker_runner.rb

@@ -0,0 +1,117 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "timeout"
+
+module ElasticGraph
+  module Local
+    # @private
+    class DockerRunner
+      def initialize(variant, port:, ui_port:, version:, env:, ready_log_line:, daemon_timeout:, output:)
+        @variant = variant
+        @port = port
+        @ui_port = ui_port
+        @version = version
+        @env = env
+        @ready_log_line = ready_log_line
+        @daemon_timeout = daemon_timeout
+        @output = output
+      end
+
+      def boot
+        # :nocov: -- this is actually covered via a call from `boot_as_daemon` but it happens in a forked process so simplecov doesn't see it.
+        halt
+
+        prepare_docker_compose_run "up" do |command|
+          exec(command) # we use `exec` so that our process is replaced with that one.
+        end
+        # :nocov:
+      end
+
+      def halt
+        prepare_docker_compose_run "down --volumes" do |command|
+          system(command)
+        end
+      end
+
+      def boot_as_daemon(halt_command:)
+        with_pipe do |read_io, write_io|
+          fork do
+            # :nocov: -- simplecov can't track coverage that happens in another process
+            read_io.close
+            Process.daemon
+            pid = Process.pid
+            $stdout.reopen(write_io)
+            $stderr.reopen(write_io)
+            puts pid
+            boot
+            write_io.close
+            # :nocov:
+          end
+
+          # The `Process.daemon` call in the subprocess changes the pid so we have to capture it this way instead of using
+          # the return value of `fork`.
+          pid = read_io.gets.to_i
+
+          @output.puts "Booting #{@variant}; monitoring logs for readiness..."
+
+          ::Timeout.timeout(
+            @daemon_timeout,
+            ::Timeout::Error,
+            <<~EOS
+              Timed out after #{@daemon_timeout} seconds. The expected "ready" log line[1] was not found in the logs.
+
+              [1] #{@ready_log_line.inspect}
+            EOS
+          ) do
+            loop do
+              sleep 0.01
+              line = read_io.gets
+              @output.puts line
+              break if @ready_log_line.match?(line.to_s)
+            end
+          end
+
+          @output.puts
+          @output.puts
+          @output.puts <<~EOS
+            Success! #{@variant} #{@version} (pid: #{pid}) has been booted for the #{@env} environment on port #{@port}.
+            It will continue to run in the background as a daemon. To halt it, run:
+
+            #{halt_command}
+          EOS
+        end
+      end
+
+      private
+
+      def prepare_docker_compose_run(*commands)
+        name = "#{@env}-#{@version.tr(".", "_")}"
+
+        full_command = commands.map do |command|
+          "VERSION=#{@version} PORT=#{@port} UI_PORT=#{@ui_port} ENV=#{@env} docker-compose --project-name #{name} #{command}"
+        end.join(" && ")
+
+        ::Dir.chdir(::File.join(__dir__.to_s, @variant.to_s)) do
+          yield full_command
+        end
+      end
+
+      def with_pipe
+        read_io, write_io = ::IO.pipe
+
+        begin
+          yield read_io, write_io
+        ensure
+          read_io.close
+          write_io.close
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/local/elasticsearch/Dockerfile

@@ -0,0 +1,3 @@
+ARG VERSION
+FROM elasticsearch:${VERSION}
+RUN bin/elasticsearch-plugin install mapper-size

data/lib/elastic_graph/local/elasticsearch/UI-Dockerfile

@@ -0,0 +1,2 @@
+ARG VERSION
+FROM kibana:${VERSION}

data/lib/elastic_graph/local/elasticsearch/docker-compose.yaml

@@ -0,0 +1,74 @@
+---
+networks:
+  default:
+    name: elastic
+    external: false
+services:
+  elasticsearch:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        VERSION: ${VERSION}
+    container_name: elasticsearch-${VERSION}-${ENV}
+    environment:
+    # Note: we use `discovery.type=single-node` to ensure that the Elasticsearch node does not
+    # try to join a cluster (or let another node join it). This prevents problems when you
+    # have multiple projects using elasticgraph-local at the same time. You do not want
+    # their Elasticsearch nodes to try to join into a single cluster.
+    - discovery.type=single-node
+    # Note: we use `xpack.security.enabled=false` to silence an annoying warning Elasticsearch 7.13 has
+    # started spewing (as in hundreds of times!) as we run our test suite:
+    #
+    # > warning: 299 Elasticsearch-7.13.0-5ca8591c6fcdb1260ce95b08a8e023559635c6f3 "Elasticsearch built-in
+    # > security features are not enabled. Without authentication, your cluster could be accessible to anyone.
+    # > See https://www.elastic.co/guide/en/elasticsearch/reference/7.13/security-minimal-setup.html to enable
+    # > security."
+    #
+    # Since this is only used in local dev/test environments where the added security would make things harder
+    # (we'd have to setup credentials in our tests), it's simpler/better just to explicitly disable the security,
+    # which silences the warning.
+    - xpack.security.enabled=false
+    # We disable `xpack.ml` because it's not compatible with the `darwin-aarch64` distribution we use on M1 Macs.
+    # Without that flag, we get this error:
+    #
+    # > [2022-01-20T10:06:54,582][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [myron-macbookpro.local] uncaught exception in thread [main]
+    # > org.elasticsearch.bootstrap.StartupException: ElasticsearchException[Failure running machine learning native code. This could be due to running
+    # > on an unsupported OS or distribution, missing OS libraries, or a problem with the temp directory. To bypass this problem by running Elasticsearch
+    # > without machine learning functionality set [xpack.ml.enabled: false].]
+    #
+    # See also this github issue: https://github.com/elastic/elasticsearch/pull/68068
+    - xpack.ml.enabled=false
+    # We don't want Elasticsearch to block writes when the disk allocation passes a threshold for our local/test
+    # Elasticsearch we run using this docker setup.
+    # https://stackoverflow.com/a/75962819
+    #
+    # Without this, I frequently get `FORBIDDEN/10/cluster create-index blocked (api)` errors when running tests.
+    - cluster.routing.allocation.disk.threshold_enabled=false
+    # Necessary on Elasticsearch 8 since our test suites indiscriminately deletes all documents
+    # between tests to sandbox the state of each test. Without this setting, we get errors like:
+    #
+    # > illegal_argument_exception: Wildcard expressions or all indices are not allowed
+    - action.destructive_requires_name=false
+    - ES_JAVA_OPTS=-Xms4g -Xmx4g
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+    - elasticsearch:/usr/share/elasticsearch/data
+    ports:
+    - ${PORT:-9200}:9200
+  kibana:
+    build:
+      context: .
+      dockerfile: UI-Dockerfile
+      args:
+        VERSION: ${VERSION}
+    container_name: kibana-${VERSION}-${ENV}
+    environment:
+    - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    ports:
+    - ${UI_PORT:-5601}:5601
+volumes:
+  elasticsearch:

data/lib/elastic_graph/local/indexing_coordinator.rb

@@ -0,0 +1,58 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/indexer/test_support/converters"
+
+module ElasticGraph
+  module Local
+    # Responsible for coordinating the generation and indexing of fake data batches.
+    # Designed to be pluggable with different publishing strategies.
+    #
+    # @private
+    class IndexingCoordinator
+      PARALLELISM = 8
+
+      def initialize(fake_data_batch_generator, output: $stdout, &publish_batch)
+        @fake_data_batch_generator = fake_data_batch_generator
+        @publish_batch = publish_batch
+        @output = output
+      end
+
+      def index_fake_data(num_batches)
+        batch_queue = ::Thread::Queue.new
+
+        publishing_threads = Array.new(PARALLELISM) { new_publishing_thread(batch_queue) }
+
+        num_batches.times do
+          batch = [] # : ::Array[::Hash[::String, untyped]]
+          @fake_data_batch_generator.call(batch)
+          @output.puts "Generated batch of #{batch.size} documents..."
+          batch_queue << batch
+        end
+
+        publishing_threads.map { batch_queue << :done }
+        publishing_threads.each(&:join)
+
+        @output.puts "...done."
+      end
+
+      private
+
+      def new_publishing_thread(batch_queue)
+        ::Thread.new do
+          loop do
+            batch = batch_queue.pop
+            break if batch == :done
+            @publish_batch.call(ElasticGraph::Indexer::TestSupport::Converters.upsert_events_for_records(batch))
+            @output.puts "Published batch of #{batch.size} documents..."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastic_graph/local/local_indexer.rb

@@ -0,0 +1,28 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/indexer"
+require "elastic_graph/local/indexing_coordinator"
+
+module ElasticGraph
+  module Local
+    # @private
+    class LocalIndexer
+      def initialize(local_config_yaml, fake_data_batch_generator, output:)
+        @local_indexer = ElasticGraph::Indexer.from_yaml_file(local_config_yaml)
+        @indexing_coordinator = IndexingCoordinator.new(fake_data_batch_generator, output: output) do |batch|
+          @local_indexer.processor.process(batch)
+        end
+      end
+
+      def index_fake_data(num_batches)
+        @indexing_coordinator.index_fake_data(num_batches)
+      end
+    end
+  end
+end

data/lib/elastic_graph/local/opensearch/Dockerfile

@@ -0,0 +1,4 @@
+ARG VERSION
+FROM opensearchproject/opensearch:${VERSION}
+RUN /usr/share/opensearch/bin/opensearch-plugin remove opensearch-security
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch mapper-size

data/lib/elastic_graph/local/opensearch/UI-Dockerfile

@@ -0,0 +1,2 @@
+ARG VERSION
+FROM opensearchproject/opensearch-dashboards:${VERSION}

data/lib/elastic_graph/local/opensearch/docker-compose.yaml

@@ -0,0 +1,50 @@
+---
+networks:
+  default:
+    name: opensearch
+    external: false
+services:
+  opensearch:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        VERSION: ${VERSION}
+    container_name: opensearch-${VERSION}-${ENV}
+    environment:
+    # Note: we use `discovery.type=single-node` to ensure that the OpenSearch node does not
+    # try to join a cluster (or let another node join it). This prevents problems when you
+    # have multiple projects using elasticgraph-local at the same time. You do not want
+    # their OpenSearch nodes to try to join into a single cluster.
+    - discovery.type=single-node
+    # recommended by https://opensearch.org/downloads.html#minimal
+    - bootstrap.memory_lock=true
+    # We don't want OpenSearch to block writes when the disk allocation passes a threshold for our local/test
+    # OpenSearch we run using this docker setup.
+    # https://stackoverflow.com/a/75962819
+    #
+    # Without this, I frequently get `FORBIDDEN/10/cluster create-index blocked (api)` errors when running tests.
+    - cluster.routing.allocation.disk.threshold_enabled=false
+    - OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+    - opensearch:/usr/share/opensearch/data
+    ports:
+    - ${PORT}:9200
+  dashboards:
+    build:
+      context: .
+      dockerfile: UI-Dockerfile
+      args:
+        VERSION: ${VERSION}
+    container_name: dashboards-${VERSION}-${ENV}
+    environment:
+    - OPENSEARCH_HOSTS=http://opensearch:9200
+    - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true
+    ports:
+    - ${UI_PORT}:5601
+volumes:
+  opensearch: