elasticgraph-indexer_lambda 0.18.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0bfc417f9a3f74f5dea10bc89aa7b015c52a2423b9bae24f83c23072fcac94e0
+  data.tar.gz: 45845f3166226ff66f38dc0242be414d37d4db6d75254d55d6013097e41dcb2b
+SHA512:
+  metadata.gz: 0fd527fd93ef7f9728a9f172c1544c6bd55f6f62d85319af211f07f5b0911496a1ea42df622a319d56b6690cc516ab76724acd03f8cc56963fbbdb839a24b179
+  data.tar.gz: c64da253570961c96c351d4eb0ed9e55d523d21ed5908a11ec0c6c59315381852def1c1a776afcab742141bac326b01c708156838f12f134dec59d63b6858d54

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# ElasticGraph::IndexerLambda
+
+Adapts elasticgraph-indexer to run in an AWS Lambda.
+
+## SQS Message Payload Format
+
+We use [JSON Lines](http://jsonlines.org/) to encode our indexing events. It is just individual JSON objects
+delimited by a newline control character(not the `\n` string sequence), such as:
+
+```jsonl
+{"op": "upsert", "__typename": "Payment", "id": "123", "version": "1", "record": {...} }
+{"op": "upsert", "__typename": "Payment", "id": "123", "version": "2", record: {...} }
+{"op": "delete", "__typename": "Payment", "id": "123", "version": "3"}
+```
+
+However, due to SQS message size limit, we have to batch our events carefully so each batch is below the size limit.
+This makes payload encoding a bit more complicated on the publisher side because each message has a size limit.
+The following code snippet respects the max message size limit and sends JSON Lines payloads with proper size:
+
+```ruby
+def partition_into_acceptably_sized_chunks(batch, max_size_per_chunk)
+  chunk_size = 0
+  batch
+    .map { |item| JSON.generate(item) }
+    .slice_before do |json|
+      chunk_size += (json.bytesize + 1)
+      (chunk_size > max_size_per_chunk).tap { |chunk_done| chunk_size = 0 if chunk_done }
+     end
+    .map { |chunk| chunk.join("\n") }
+end
+```

data/elasticgraph-indexer_lambda.gemspec

@@ -0,0 +1,24 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../gemspec_helper"
+
+ElasticGraphGemspecHelper.define_elasticgraph_gem(gemspec_file: __FILE__, category: :lambda) do |spec, eg_version|
+  spec.summary = "Provides an AWS Lambda interface for an elasticgraph API"
+
+  spec.add_dependency "elasticgraph-indexer", eg_version
+  spec.add_dependency "elasticgraph-lambda_support", eg_version
+  spec.add_dependency "aws-sdk-s3", "~> 1.146"
+
+  # aws-sdk-s3 requires an XML library be available. On Ruby < 3 it'll use rexml from the standard library but on Ruby 3.0+
+  # we have to add an explicit dependency. It supports ox, oga, libxml, nokogiri or rexml, and of those, ox seems to be the
+  # best choice: it leads benchmarks, is well-maintained, has no dependencies, and is MIT-licensed.
+  spec.add_dependency "ox", "~> 2.14"
+
+  spec.add_development_dependency "httpx", ">= 1.2.6", "< 2.0"
+end

data/lib/elastic_graph/indexer_lambda/lambda_function.rb

@@ -0,0 +1,38 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+# :nocov: -- exercised by a test that loads this file in a sub-process, so not seen by SimpleCov.
+require "elastic_graph/lambda_support/lambda_function"
+
+module ElasticGraph
+  module IndexerLambda
+    class LambdaFunction
+      prepend LambdaSupport::LambdaFunction
+
+      def initialize
+        require "elastic_graph/indexer_lambda"
+        require "elastic_graph/indexer_lambda/sqs_processor"
+
+        indexer = ElasticGraph::IndexerLambda.indexer_from_env
+        @sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new(
+          indexer.processor,
+          logger: indexer.logger,
+          report_batch_item_failures: ENV["REPORT_BATCH_ITEM_FAILURES"] == "true"
+        )
+      end
+
+      def handle_request(event:, context:)
+        @sqs_processor.process(event)
+      end
+    end
+  end
+end
+
+# This is the constant used by the AWS Lambda function.
+ProcessEventStreamEvent = ElasticGraph::IndexerLambda::LambdaFunction.new
+# :nocov:

data/lib/elastic_graph/indexer_lambda/sqs_processor.rb

@@ -0,0 +1,142 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/error"
+require "elastic_graph/indexer/indexing_failures_error"
+require "json"
+
+module ElasticGraph
+  module IndexerLambda
+    # Responsible for handling lambda event payloads from an SQS lambda trigger.
+    class SqsProcessor
+      def initialize(indexer_processor, report_batch_item_failures:, logger:, s3_client: nil)
+        @indexer_processor = indexer_processor
+        @report_batch_item_failures = report_batch_item_failures
+        @logger = logger
+        @s3_client = s3_client
+      end
+
+      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the database.
+      def process(lambda_event, refresh_indices: false)
+        events = events_from(lambda_event)
+        failures = @indexer_processor.process_returning_failures(events, refresh_indices: refresh_indices)
+
+        if failures.any?
+          failures_error = Indexer::IndexingFailuresError.for(failures: failures, events: events)
+          raise failures_error unless @report_batch_item_failures
+          @logger.error(failures_error.message)
+        end
+
+        format_response(failures)
+      end
+
+      private
+
+      # Given a lambda event payload, returns an array of raw operations in JSON format.
+      #
+      # The SQS payload is wrapped in the following format already:
+      # See https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#example-standard-queue-message-event for more details
+      # {
+      #   Records: {
+      #     [
+      #        { body: <JSON Lines encoded JSON where each line is a JSON object> },
+      #        { body: <JSON Lines encoded JSON where each line is a JSON object> },
+      #        ...
+      #      ]
+      #   }
+      # }
+      #
+      # Each entry in "Records" is a SQS entry.  Since lambda handles some batching
+      # for you (with some limits), you can get multiple.
+      #
+      # We also want to do our own batching in order to cram more into a given payload
+      # and issue fewer SQS entries and Lambda invocations when possible. As such, we
+      # encoded multiple JSON with JSON Lines (http://jsonlines.org/) in record body.
+      # Each JSON Lines object under 'body' should be of the form:
+      #
+      #  {op: 'upsert', __typename: 'Payment', id: "123", version: "1", record: {...} } \n
+      #  {op: 'upsert', __typename: 'Payment', id: "123", version: "2", record: {...} } \n
+      #   ...
+      # Note: "\n" at the end of each line is a single byte newline control character, instead of a string sequence
+      def events_from(lambda_event)
+        lambda_event.fetch("Records").flat_map do |record|
+          sqs_metadata = extract_sqs_metadata(record)
+
+          parse_jsonl(record.fetch("body")).map do |event|
+            ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
+          end
+        end
+      end
+
+      S3_OFFLOADING_INDICATOR = '["software.amazon.payloadoffloading.PayloadS3Pointer"'
+
+      def parse_jsonl(jsonl_string)
+        if jsonl_string.start_with?(S3_OFFLOADING_INDICATOR)
+          jsonl_string = get_payload_from_s3(jsonl_string)
+        end
+        jsonl_string.split("\n").map { |event| JSON.parse(event) }
+      end
+
+      def extract_sqs_metadata(record)
+        sqs_timestamps = {
+          "processing_first_attempted_at" => millis_to_iso8601(record.dig("attributes", "ApproximateFirstReceiveTimestamp")),
+          "sqs_received_at" => millis_to_iso8601(record.dig("attributes", "SentTimestamp"))
+        }.compact
+
+        {
+          "latency_timestamps" => (sqs_timestamps unless sqs_timestamps.empty?),
+          "message_id" => record["messageId"]
+        }.compact
+      end
+
+      def millis_to_iso8601(millis)
+        return unless millis
+        seconds, millis = millis.to_i.divmod(1000)
+        Time.at(seconds, millis, :millisecond).getutc.iso8601(3)
+      end
+
+      def get_payload_from_s3(json_string)
+        s3_pointer = JSON.parse(json_string)[1]
+        bucket_name = s3_pointer.fetch("s3BucketName")
+        object_key = s3_pointer.fetch("s3Key")
+
+        begin
+          s3_client.get_object(bucket: bucket_name, key: object_key).body.read
+        rescue Aws::S3::Errors::ServiceError => e
+          raise S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
+        end
+      end
+
+      # The s3 client is being lazily initialized, as it's slow to import/init and it will only be used
+      # in rare scenarios where large messages need offloaded from SQS -> S3.
+      # See: (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-s3-messages.html)
+      def s3_client
+        @s3_client ||= begin
+          require "aws-sdk-s3"
+          Aws::S3::Client.new
+        end
+      end
+
+      # Formats the response, including any failures, based on
+      # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting
+      def format_response(failures)
+        failure_ids = failures.map do |failure| # $ {"itemIdentifier" => String}
+          {"itemIdentifier" => failure.event["message_id"]}
+        end
+
+        if failure_ids.any? { |f| f.fetch("itemIdentifier").nil? }
+          # If we are not able to identify one or more failed events, then we must raise an exception instead of
+          # returning `batchItemFailures`. Otherwise, the unidentified failed events will not get retried.
+          raise MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
+        end
+
+        {"batchItemFailures" => failure_ids}
+      end
+    end
+  end
+end

data/lib/elastic_graph/indexer_lambda.rb

@@ -0,0 +1,19 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/indexer"
+require "elastic_graph/lambda_support"
+
+module ElasticGraph
+  module IndexerLambda
+    # Builds an `ElasticGraph::Indexer` instance from our lambda ENV vars.
+    def self.indexer_from_env
+      LambdaSupport.build_from_env(Indexer)
+    end
+  end
+end

metadata

@@ -0,0 +1,306 @@
+--- !ruby/object:Gem::Specification
+name: elasticgraph-indexer_lambda
+version: !ruby/object:Gem::Version
+  version: 0.18.0.0
+platform: ruby
+authors:
+- Myron Marston
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubocop-factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.26'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: coderay
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: flatware-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: super_diff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws_lambda_ric
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-indexer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: elasticgraph-lambda_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.0.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.146'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.146'
+- !ruby/object:Gem::Dependency
+  name: ox
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: httpx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: 
+email:
+- myron@squareup.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- elasticgraph-indexer_lambda.gemspec
+- lib/elastic_graph/indexer_lambda.rb
+- lib/elastic_graph/indexer_lambda/lambda_function.rb
+- lib/elastic_graph/indexer_lambda/sqs_processor.rb
+homepage: 
+licenses:
+- MIT
+metadata:
+  gem_category: lambda
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key: 
+specification_version: 4
+summary: Provides an AWS Lambda interface for an elasticgraph API
+test_files: []