elasticgraph-indexer_lambda 1.0.0.rc1 → 1.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60f11ef75eaa4c8accb6cf8c4625da887c978c7621a86e119b9b3b2e8ee77737
-  data.tar.gz: 8a8f6f9bd7a11cf066236369b0ee8f18be6162d5e0ddea93c69fe097abe3e070
+  metadata.gz: 7e35fa2355ab8a732034a795a4b15a9f5f8efc3e0a358652fcfe65615fef57d1
+  data.tar.gz: a7f30ebf536d235ead8d0dd62139510bf8579d766c3eb24383c3de2cb7a31b9a
 SHA512:
-  metadata.gz: 31aa96ae29655393453a6e8e0e3abfb675138e0e71be6a45c34bd32a2a0c2ff62f2c044770eaec4334f1599ce17ffb34ef72d741a84577df611e66878958a2c4
-  data.tar.gz: 45f3e218a156769156d70456bafb55446de78190d98f4e6d486d29ef13213e0fbc870ba1e7dd484c4951db4ac955a47b921216d7fb7a6309e727a944c8bc4e15
+  metadata.gz: f0b24d1a734d396d41ce5a9284a80db6f597e1a74b61d32c98013c71e8308204e10d656422b90704f4e6ff20ab6b5875ce9986a69ac34fad284afeae7a426d83
+  data.tar.gz: 52853ea123e634c1e2df1527e4d4da826d67bf884a61eeb8dfddfdd1e80ff6ada1f4fd7fb30d4bc4263b7628986f9137f92e86c178871bb35681e53449a77a9f

data/README.md

@@ -2,6 +2,31 @@
 
 Adapts elasticgraph-indexer to run in an AWS Lambda.
 
+## Dependency Diagram
+
+```mermaid
+graph LR;
+    classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px;
+    classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000;
+    classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9;
+    elasticgraph-indexer_lambda["elasticgraph-indexer_lambda"];
+    class elasticgraph-indexer_lambda targetGemStyle;
+    elasticgraph-indexer["elasticgraph-indexer"];
+    elasticgraph-indexer_lambda --> elasticgraph-indexer;
+    class elasticgraph-indexer otherEgGemStyle;
+    elasticgraph-lambda_support["elasticgraph-lambda_support"];
+    elasticgraph-indexer_lambda --> elasticgraph-lambda_support;
+    class elasticgraph-lambda_support otherEgGemStyle;
+    aws-sdk-s3["aws-sdk-s3"];
+    elasticgraph-indexer_lambda --> aws-sdk-s3;
+    class aws-sdk-s3 externalGemStyle;
+    ox["ox"];
+    elasticgraph-indexer_lambda --> ox;
+    class ox externalGemStyle;
+    click aws-sdk-s3 href "https://rubygems.org/gems/aws-sdk-s3" "Open on RubyGems.org" _blank;
+    click ox href "https://rubygems.org/gems/ox" "Open on RubyGems.org" _blank;
+```
+
 ## SQS Message Payload Format
 
 We use [JSON Lines](http://jsonlines.org/) to encode our indexing events. It is just individual JSON objects

data/lib/elastic_graph/indexer_lambda/lambda_function.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/lambda_support/lambda_function"
+require "json"
 
 module ElasticGraph
   module IndexerLambda
@@ -14,12 +15,21 @@ module ElasticGraph
     class LambdaFunction
       prepend LambdaSupport::LambdaFunction
 
+      # @dynamic sqs_processor
+      attr_reader :sqs_processor
+
       def initialize
         require "elastic_graph/indexer_lambda"
         require "elastic_graph/indexer_lambda/sqs_processor"
 
         indexer = ElasticGraph::IndexerLambda.indexer_from_env
-        @sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new(indexer.processor, logger: indexer.logger)
+        ignore_sqs_latency_timestamps_from_arns = ::JSON.parse(ENV.fetch("IGNORE_SQS_LATENCY_TIMESTAMPS_FROM_ARNS", "[]")).to_set
+
+        @sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new(
+          indexer.processor,
+          ignore_sqs_latency_timestamps_from_arns: ignore_sqs_latency_timestamps_from_arns,
+          logger: indexer.logger
+        )
       end
 
       def handle_request(event:, context:)

data/lib/elastic_graph/indexer_lambda/sqs_processor.rb

@@ -16,10 +16,14 @@ module ElasticGraph
     #
     # @private
     class SqsProcessor
-      def initialize(indexer_processor, logger:, s3_client: nil)
+      # @dynamic ignore_sqs_latency_timestamps_from_arns
+      attr_reader :ignore_sqs_latency_timestamps_from_arns
+
+      def initialize(indexer_processor, logger:, ignore_sqs_latency_timestamps_from_arns:, s3_client: nil)
         @indexer_processor = indexer_processor
         @logger = logger
         @s3_client = s3_client
+        @ignore_sqs_latency_timestamps_from_arns = ignore_sqs_latency_timestamps_from_arns
       end
 
       # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore.
@@ -67,9 +71,15 @@ module ElasticGraph
         sqs_received_at_by_message_id = {} # : Hash[String, String]
         lambda_event.fetch("Records").flat_map do |record|
           sqs_metadata = extract_sqs_metadata(record)
+
           if (message_id = sqs_metadata.fetch("message_id", nil))
             sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at")
           end
+
+          if @ignore_sqs_latency_timestamps_from_arns.include?(record.fetch("eventSourceARN"))
+            sqs_metadata = sqs_metadata.except("latency_timestamps")
+          end
+
           parse_jsonl(record.fetch("body")).map do |event|
             ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer_lambda
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc1
+  version: 1.0.0.rc2
 platform: ruby
 authors:
 - Myron Marston
@@ -17,42 +17,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0.rc1
+        version: 1.0.0.rc2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0.rc1
+        version: 1.0.0.rc2
 - !ruby/object:Gem::Dependency
   name: elasticgraph-lambda_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0.rc1
+        version: 1.0.0.rc2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0.rc1
+        version: 1.0.0.rc2
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.184'
+        version: '1.192'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.184'
+        version: '1.192'
 - !ruby/object:Gem::Dependency
   name: ox
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ dependencies:
         version: '2.14'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.14.22
+        version: 2.14.23
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -72,7 +72,7 @@ dependencies:
         version: '2.14'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.14.22
+        version: 2.14.23
 email:
 - myron@squareup.com
 executables: []
@@ -89,10 +89,10 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/block/elasticgraph/issues
-  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.rc1
-  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.rc1/
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.rc2
+  documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.rc2/
   homepage_uri: https://block.github.io/elasticgraph/
-  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.rc1/elasticgraph-indexer_lambda
+  source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.rc2/elasticgraph-indexer_lambda
   gem_category: lambda
 rdoc_options: []
 require_paths: