elasticgraph-indexer_lambda 1.0.0.rc1 → 1.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 60f11ef75eaa4c8accb6cf8c4625da887c978c7621a86e119b9b3b2e8ee77737
4
- data.tar.gz: 8a8f6f9bd7a11cf066236369b0ee8f18be6162d5e0ddea93c69fe097abe3e070
3
+ metadata.gz: 7e35fa2355ab8a732034a795a4b15a9f5f8efc3e0a358652fcfe65615fef57d1
4
+ data.tar.gz: a7f30ebf536d235ead8d0dd62139510bf8579d766c3eb24383c3de2cb7a31b9a
5
5
  SHA512:
6
- metadata.gz: 31aa96ae29655393453a6e8e0e3abfb675138e0e71be6a45c34bd32a2a0c2ff62f2c044770eaec4334f1599ce17ffb34ef72d741a84577df611e66878958a2c4
7
- data.tar.gz: 45f3e218a156769156d70456bafb55446de78190d98f4e6d486d29ef13213e0fbc870ba1e7dd484c4951db4ac955a47b921216d7fb7a6309e727a944c8bc4e15
6
+ metadata.gz: f0b24d1a734d396d41ce5a9284a80db6f597e1a74b61d32c98013c71e8308204e10d656422b90704f4e6ff20ab6b5875ce9986a69ac34fad284afeae7a426d83
7
+ data.tar.gz: 52853ea123e634c1e2df1527e4d4da826d67bf884a61eeb8dfddfdd1e80ff6ada1f4fd7fb30d4bc4263b7628986f9137f92e86c178871bb35681e53449a77a9f
data/README.md CHANGED
@@ -2,6 +2,31 @@
2
2
 
3
3
  Adapts elasticgraph-indexer to run in an AWS Lambda.
4
4
 
5
+ ## Dependency Diagram
6
+
7
+ ```mermaid
8
+ graph LR;
9
+ classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px;
10
+ classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000;
11
+ classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9;
12
+ elasticgraph-indexer_lambda["elasticgraph-indexer_lambda"];
13
+ class elasticgraph-indexer_lambda targetGemStyle;
14
+ elasticgraph-indexer["elasticgraph-indexer"];
15
+ elasticgraph-indexer_lambda --> elasticgraph-indexer;
16
+ class elasticgraph-indexer otherEgGemStyle;
17
+ elasticgraph-lambda_support["elasticgraph-lambda_support"];
18
+ elasticgraph-indexer_lambda --> elasticgraph-lambda_support;
19
+ class elasticgraph-lambda_support otherEgGemStyle;
20
+ aws-sdk-s3["aws-sdk-s3"];
21
+ elasticgraph-indexer_lambda --> aws-sdk-s3;
22
+ class aws-sdk-s3 externalGemStyle;
23
+ ox["ox"];
24
+ elasticgraph-indexer_lambda --> ox;
25
+ class ox externalGemStyle;
26
+ click aws-sdk-s3 href "https://rubygems.org/gems/aws-sdk-s3" "Open on RubyGems.org" _blank;
27
+ click ox href "https://rubygems.org/gems/ox" "Open on RubyGems.org" _blank;
28
+ ```
29
+
5
30
  ## SQS Message Payload Format
6
31
 
7
32
  We use [JSON Lines](http://jsonlines.org/) to encode our indexing events. It is just individual JSON objects
@@ -7,6 +7,7 @@
7
7
  # frozen_string_literal: true
8
8
 
9
9
  require "elastic_graph/lambda_support/lambda_function"
10
+ require "json"
10
11
 
11
12
  module ElasticGraph
12
13
  module IndexerLambda
@@ -14,12 +15,21 @@ module ElasticGraph
14
15
  class LambdaFunction
15
16
  prepend LambdaSupport::LambdaFunction
16
17
 
18
+ # @dynamic sqs_processor
19
+ attr_reader :sqs_processor
20
+
17
21
  def initialize
18
22
  require "elastic_graph/indexer_lambda"
19
23
  require "elastic_graph/indexer_lambda/sqs_processor"
20
24
 
21
25
  indexer = ElasticGraph::IndexerLambda.indexer_from_env
22
- @sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new(indexer.processor, logger: indexer.logger)
26
+ ignore_sqs_latency_timestamps_from_arns = ::JSON.parse(ENV.fetch("IGNORE_SQS_LATENCY_TIMESTAMPS_FROM_ARNS", "[]")).to_set
27
+
28
+ @sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new(
29
+ indexer.processor,
30
+ ignore_sqs_latency_timestamps_from_arns: ignore_sqs_latency_timestamps_from_arns,
31
+ logger: indexer.logger
32
+ )
23
33
  end
24
34
 
25
35
  def handle_request(event:, context:)
@@ -16,10 +16,14 @@ module ElasticGraph
16
16
  #
17
17
  # @private
18
18
  class SqsProcessor
19
- def initialize(indexer_processor, logger:, s3_client: nil)
19
+ # @dynamic ignore_sqs_latency_timestamps_from_arns
20
+ attr_reader :ignore_sqs_latency_timestamps_from_arns
21
+
22
+ def initialize(indexer_processor, logger:, ignore_sqs_latency_timestamps_from_arns:, s3_client: nil)
20
23
  @indexer_processor = indexer_processor
21
24
  @logger = logger
22
25
  @s3_client = s3_client
26
+ @ignore_sqs_latency_timestamps_from_arns = ignore_sqs_latency_timestamps_from_arns
23
27
  end
24
28
 
25
29
  # Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore.
@@ -67,9 +71,15 @@ module ElasticGraph
67
71
  sqs_received_at_by_message_id = {} # : Hash[String, String]
68
72
  lambda_event.fetch("Records").flat_map do |record|
69
73
  sqs_metadata = extract_sqs_metadata(record)
74
+
70
75
  if (message_id = sqs_metadata.fetch("message_id", nil))
71
76
  sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at")
72
77
  end
78
+
79
+ if @ignore_sqs_latency_timestamps_from_arns.include?(record.fetch("eventSourceARN"))
80
+ sqs_metadata = sqs_metadata.except("latency_timestamps")
81
+ end
82
+
73
83
  parse_jsonl(record.fetch("body")).map do |event|
74
84
  ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
75
85
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticgraph-indexer_lambda
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0.rc1
4
+ version: 1.0.0.rc2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Myron Marston
@@ -17,42 +17,42 @@ dependencies:
17
17
  requirements:
18
18
  - - '='
19
19
  - !ruby/object:Gem::Version
20
- version: 1.0.0.rc1
20
+ version: 1.0.0.rc2
21
21
  type: :runtime
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - '='
26
26
  - !ruby/object:Gem::Version
27
- version: 1.0.0.rc1
27
+ version: 1.0.0.rc2
28
28
  - !ruby/object:Gem::Dependency
29
29
  name: elasticgraph-lambda_support
30
30
  requirement: !ruby/object:Gem::Requirement
31
31
  requirements:
32
32
  - - '='
33
33
  - !ruby/object:Gem::Version
34
- version: 1.0.0.rc1
34
+ version: 1.0.0.rc2
35
35
  type: :runtime
36
36
  prerelease: false
37
37
  version_requirements: !ruby/object:Gem::Requirement
38
38
  requirements:
39
39
  - - '='
40
40
  - !ruby/object:Gem::Version
41
- version: 1.0.0.rc1
41
+ version: 1.0.0.rc2
42
42
  - !ruby/object:Gem::Dependency
43
43
  name: aws-sdk-s3
44
44
  requirement: !ruby/object:Gem::Requirement
45
45
  requirements:
46
46
  - - "~>"
47
47
  - !ruby/object:Gem::Version
48
- version: '1.184'
48
+ version: '1.192'
49
49
  type: :runtime
50
50
  prerelease: false
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
53
  - - "~>"
54
54
  - !ruby/object:Gem::Version
55
- version: '1.184'
55
+ version: '1.192'
56
56
  - !ruby/object:Gem::Dependency
57
57
  name: ox
58
58
  requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ dependencies:
62
62
  version: '2.14'
63
63
  - - ">="
64
64
  - !ruby/object:Gem::Version
65
- version: 2.14.22
65
+ version: 2.14.23
66
66
  type: :runtime
67
67
  prerelease: false
68
68
  version_requirements: !ruby/object:Gem::Requirement
@@ -72,7 +72,7 @@ dependencies:
72
72
  version: '2.14'
73
73
  - - ">="
74
74
  - !ruby/object:Gem::Version
75
- version: 2.14.22
75
+ version: 2.14.23
76
76
  email:
77
77
  - myron@squareup.com
78
78
  executables: []
@@ -89,10 +89,10 @@ licenses:
89
89
  - MIT
90
90
  metadata:
91
91
  bug_tracker_uri: https://github.com/block/elasticgraph/issues
92
- changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.rc1
93
- documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.rc1/
92
+ changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.rc2
93
+ documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.rc2/
94
94
  homepage_uri: https://block.github.io/elasticgraph/
95
- source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.rc1/elasticgraph-indexer_lambda
95
+ source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.rc2/elasticgraph-indexer_lambda
96
96
  gem_category: lambda
97
97
  rdoc_options: []
98
98
  require_paths: