elasticgraph-indexer_lambda 0.18.0.3 → 0.18.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83afbe4a7ecedfc0b8ac3e866c18fda2588c7888eabf648da5a9a857402f4e8e
-  data.tar.gz: bae2fd009be9d42274d38cf666fb776bf9cfcaceb53c8f191b300199c735dd23
+  metadata.gz: c40fc6f5e803a2355e9c4b5623b7aef3bc98e11c29953c0c31468e4a1c768993
+  data.tar.gz: de7c637b901f9e251884056975bd7631f219a6de7dcc2ca17c87e51d0b13ad34
 SHA512:
-  metadata.gz: f3fce975d914479aea7711d54ee7c1679fceb63cd15cfe1ca63dd45be2a3d295ea707fe2d07278ebae7832df521c1d038fb7a0c5642c7178888f393340944cd7
-  data.tar.gz: '08ba363c62ab2bd573ac6a9b9656f09e20c6c46bd5a44c4bb2af0810d1cd2317fdf1e1e0bb20f92c297a1c15fee6b1f566b556a307584937eef08d76006b4e4d'
+  metadata.gz: 57861df48c60a47828f11c0536b6deadf6639012a93f2bc28a72761e882617e713cde6f66cd2726e996379637ef04a345deb3841d1808ca0c3f0c12fca6bda03
+  data.tar.gz: 16463e9a14fbc544983f8dced31f95dca6f132ff0b73dad5f083ac266c9a874870e2aab61374a36571ff7e04250fe80fe913746fb4c0da7596f547faedf31996

data/lib/elastic_graph/indexer_lambda/sqs_processor.rb

@@ -6,7 +6,7 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/error"
+require "elastic_graph/errors"
 require "elastic_graph/indexer/indexing_failures_error"
 require "json"
 
@@ -23,7 +23,7 @@ module ElasticGraph
         @s3_client = s3_client
       end
 
-      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the database.
+      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore.
       def process(lambda_event, refresh_indices: false)
         events = events_from(lambda_event)
         failures = @indexer_processor.process_returning_failures(events, refresh_indices: refresh_indices)
@@ -66,12 +66,20 @@ module ElasticGraph
       #   ...
       # Note: "\n" at the end of each line is a single byte newline control character, instead of a string sequence
       def events_from(lambda_event)
+        sqs_received_at_by_message_id = {} # : Hash[String, String]
         lambda_event.fetch("Records").flat_map do |record|
           sqs_metadata = extract_sqs_metadata(record)
-
+          if (message_id = sqs_metadata.fetch("message_id", nil))
+            sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at")
+          end
           parse_jsonl(record.fetch("body")).map do |event|
             ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
           end
+        end.tap do
+          @logger.info({
+            "message_type" => "ReceivedSqsMessages",
+            "sqs_received_at_by_message_id" => sqs_received_at_by_message_id
+          })
         end
       end
 
@@ -110,7 +118,7 @@ module ElasticGraph
         begin
           s3_client.get_object(bucket: bucket_name, key: object_key).body.read
         rescue Aws::S3::Errors::ServiceError => e
-          raise S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
+          raise Errors::S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
         end
       end
 
@@ -134,7 +142,7 @@ module ElasticGraph
         if failure_ids.any? { |f| f.fetch("itemIdentifier").nil? }
           # If we are not able to identify one or more failed events, then we must raise an exception instead of
           # returning `batchItemFailures`. Otherwise, the unidentified failed events will not get retried.
-          raise MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
+          raise Errors::MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
         end
 
         {"batchItemFailures" => failure_ids}

metadata

@@ -1,14 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer_lambda
 version: !ruby/object:Gem::Version
-  version: 0.18.0.3
+  version: 0.18.0.5
 platform: ruby
 authors:
 - Myron Marston
+- Ben VandenBos
+- Square Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-05 00:00:00.000000000 Z
+date: 2024-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop-factory_bot
@@ -190,28 +192,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.3
+        version: 0.18.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.3
+        version: 0.18.0.5
 - !ruby/object:Gem::Dependency
   name: elasticgraph-lambda_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.3
+        version: 0.18.0.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.3
+        version: 0.18.0.5
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement