elasticgraph-indexer_lambda 0.18.0.4 → 0.19.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbd6851f8d0d3b534a29e9966af449fe830e702388add5e59007268718884643
-  data.tar.gz: f6b8751a60b161d79e8cc59ab959287037cfd7702cc7dac69b584b08c72e8ca0
+  metadata.gz: 3bfc9565971f681207f41974cda4f44cf821992c8409b15cc5beac389ca1a434
+  data.tar.gz: 5039dcacf6cf702d4b889d6518a54932d9a45a52e6ec151d42db43bef912372a
 SHA512:
-  metadata.gz: f6d2360305069114468626b67e889d21eaa8a45c269446e995e9eb3fa1c836e32cd54a5bc3717c9c52e13f70155dbe27d045549e68382108440de680e800b67e
-  data.tar.gz: d534b6e2c0c6c59c5f90ba92b4f79d3c1014a51e3f0074a14c932140e650f19801d49dfc08cfca867e55075d3ae1bce0089f3d3bf4e11388b7efec9805b4bedb
+  metadata.gz: 9acd5b9b9fa5690f48d404ac02ce67514d063a701ab826b5b32986876f5adce7f8ed957edfa740134357e9da4630787ef584501727318670ba74c5c0ffeb9cb9
+  data.tar.gz: 77bf6c267ae7f27cccaa24e8bda41112a63548a7f396840cb0932669b3de63a26b9a820b48b1108738ad77f8131d446b53aa17a193798c3dbe1f094900ab5b16

data/lib/elastic_graph/indexer_lambda/sqs_processor.rb

@@ -6,7 +6,7 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/error"
+require "elastic_graph/errors"
 require "elastic_graph/indexer/indexing_failures_error"
 require "json"
 
@@ -23,7 +23,7 @@ module ElasticGraph
         @s3_client = s3_client
       end
 
-      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the database.
+      # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore.
       def process(lambda_event, refresh_indices: false)
         events = events_from(lambda_event)
         failures = @indexer_processor.process_returning_failures(events, refresh_indices: refresh_indices)
@@ -66,12 +66,20 @@ module ElasticGraph
       #   ...
       # Note: "\n" at the end of each line is a single byte newline control character, instead of a string sequence
       def events_from(lambda_event)
+        sqs_received_at_by_message_id = {} # : Hash[String, String]
         lambda_event.fetch("Records").flat_map do |record|
           sqs_metadata = extract_sqs_metadata(record)
-
+          if (message_id = sqs_metadata.fetch("message_id", nil))
+            sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at")
+          end
           parse_jsonl(record.fetch("body")).map do |event|
             ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
           end
+        end.tap do
+          @logger.info({
+            "message_type" => "ReceivedSqsMessages",
+            "sqs_received_at_by_message_id" => sqs_received_at_by_message_id
+          })
         end
       end
 
@@ -110,7 +118,7 @@ module ElasticGraph
         begin
           s3_client.get_object(bucket: bucket_name, key: object_key).body.read
         rescue Aws::S3::Errors::ServiceError => e
-          raise S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
+          raise Errors::S3OperationFailedError, "Error reading large message from S3. bucket: `#{bucket_name}` key: `#{object_key}` message: `#{e.message}`"
         end
       end
 
@@ -134,7 +142,7 @@ module ElasticGraph
         if failure_ids.any? { |f| f.fetch("itemIdentifier").nil? }
           # If we are not able to identify one or more failed events, then we must raise an exception instead of
           # returning `batchItemFailures`. Otherwise, the unidentified failed events will not get retried.
-          raise MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
+          raise Errors::MessageIdsMissingError, "Unexpected: some failures did not have a `message_id`, so we are raising an exception instead of returning `batchItemFailures`."
         end
 
         {"batchItemFailures" => failure_ids}

metadata

@@ -1,14 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: elasticgraph-indexer_lambda
 version: !ruby/object:Gem::Version
-  version: 0.18.0.4
+  version: 0.19.0.0.rc1
 platform: ruby
 authors:
 - Myron Marston
+- Ben VandenBos
+- Block Engineering
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-06 00:00:00.000000000 Z
+date: 2024-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubocop-factory_bot
@@ -44,42 +46,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.40.0
+        version: 1.41.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.40.0
+        version: 1.41.0
 - !ruby/object:Gem::Dependency
   name: steep
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: coderay
   requirement: !ruby/object:Gem::Requirement
@@ -134,14 +136,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -190,28 +192,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.19.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.19.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: elasticgraph-lambda_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.19.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.18.0.4
+        version: 0.19.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
@@ -267,10 +269,15 @@ files:
 - lib/elastic_graph/indexer_lambda.rb
 - lib/elastic_graph/indexer_lambda/lambda_function.rb
 - lib/elastic_graph/indexer_lambda/sqs_processor.rb
-homepage: 
+homepage: https://block.github.io/elasticgraph/
 licenses:
 - MIT
 metadata:
+  bug_tracker_uri: https://github.com/block/elasticgraph/issues
+  changelog_uri: https://github.com/block/elasticgraph/releases/tag/v0.19.0.0.rc1
+  documentation_uri: https://block.github.io/elasticgraph/docs/main/
+  homepage_uri: https://block.github.io/elasticgraph/
+  source_code_uri: https://github.com/block/elasticgraph/tree/v0.19.0.0.rc1/elasticgraph-indexer_lambda
   gem_category: lambda
 post_install_message: 
 rdoc_options: []