elasticgraph-indexer_lambda 0.18.0.4 → 0.18.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbd6851f8d0d3b534a29e9966af449fe830e702388add5e59007268718884643
-  data.tar.gz: f6b8751a60b161d79e8cc59ab959287037cfd7702cc7dac69b584b08c72e8ca0
+  metadata.gz: c40fc6f5e803a2355e9c4b5623b7aef3bc98e11c29953c0c31468e4a1c768993
+  data.tar.gz: de7c637b901f9e251884056975bd7631f219a6de7dcc2ca17c87e51d0b13ad34
 SHA512:
-  metadata.gz: f6d2360305069114468626b67e889d21eaa8a45c269446e995e9eb3fa1c836e32cd54a5bc3717c9c52e13f70155dbe27d045549e68382108440de680e800b67e
-  data.tar.gz: d534b6e2c0c6c59c5f90ba92b4f79d3c1014a51e3f0074a14c932140e650f19801d49dfc08cfca867e55075d3ae1bce0089f3d3bf4e11388b7efec9805b4bedb
+  metadata.gz: 57861df48c60a47828f11c0536b6deadf6639012a93f2bc28a72761e882617e713cde6f66cd2726e996379637ef04a345deb3841d1808ca0c3f0c12fca6bda03
+  data.tar.gz: 16463e9a14fbc544983f8dced31f95dca6f132ff0b73dad5f083ac266c9a874870e2aab61374a36571ff7e04250fe80fe913746fb4c0da7596f547faedf31996

data/lib/elastic_graph/indexer_lambda/sqs_processor.rb

@@ -6,7 +6,7 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/error"
+require "elastic_graph/errors"
 require "elastic_graph/indexer/indexing_failures_error"
 require "json"
 
@@ -23,7 +23,7 @@ module ElasticGraph
         @s3_client = s3_client
       end
 
-      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the database.
+      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore.
       def process(lambda_event, refresh_indices: false)
         events = events_from(lambda_event)
         failures = @indexer_processor.process_returning_failures(events, refresh_indices: refresh_indices)
@@ -66,12 +66,20 @@ module ElasticGraph
       #   ...
       # Note: "\n" at the end of each line is a single byte newline control character, instead of a string sequence
       def events_from(lambda_event)
+        sqs_received_at_by_message_id = {} # : Hash[String, String]
         lambda_event.fetch("Records").flat_map do |record|
           sqs_metadata = extract_sqs_metadata(record)
-
+          if (message_id = sqs_metadata.fetch("message_id", nil))
+            sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at")
+          end
           parse_jsonl(record.fetch("body")).map do |event|
             ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
           end
+        end.tap do
+          @logger.info({
+            "message_type" => "ReceivedSqsMessages",
+            "sqs_received_at_by_message_id" => sqs_received_at_by_message_id
+          })
         end
       end
 
@@ -110,7 +118,7 @@ module ElasticGraph
         begin
           s3_client.get_object(bucket: bucket_name, key: object_key).body.read
         rescue Aws::S3::Errors::ServiceError => e
-          raise S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
+          raise Errors::S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
         end
       end
 
@@ -134,7 +142,7 @@ module ElasticGraph
         if failure_ids.any? { |f| f.fetch("itemIdentifier").nil? }
           # If we are not able to identify one or more failed events, then we must raise an exception instead of
           # returning `batchItemFailures`. Otherwise, the unidentified failed events will not get retried.
-          raise MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
+          raise Errors::MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
         end
 
         {"batchItemFailures" => failure_ids}

metadata

@@ -1,14 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer_lambda
 version: !ruby/object:Gem::Version
-  version: 0.18.0.4
+  version: 0.18.0.5
 platform: ruby
 authors:
 - Myron Marston
+- Ben VandenBos
+- Square Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-06 00:00:00.000000000 Z
+date: 2024-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop-factory_bot
@@ -190,28 +192,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.18.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.18.0.5
 - !ruby/object:Gem::Dependency
   name: elasticgraph-lambda_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.18.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.18.0.5
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement