elasticgraph-indexer_lambda 1.0.0.rc1 → 1.0.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -0
- data/lib/elastic_graph/indexer_lambda/lambda_function.rb +11 -1
- data/lib/elastic_graph/indexer_lambda/sqs_processor.rb +11 -1
- metadata +12 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7e35fa2355ab8a732034a795a4b15a9f5f8efc3e0a358652fcfe65615fef57d1
|
4
|
+
data.tar.gz: a7f30ebf536d235ead8d0dd62139510bf8579d766c3eb24383c3de2cb7a31b9a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0b24d1a734d396d41ce5a9284a80db6f597e1a74b61d32c98013c71e8308204e10d656422b90704f4e6ff20ab6b5875ce9986a69ac34fad284afeae7a426d83
|
7
|
+
data.tar.gz: 52853ea123e634c1e2df1527e4d4da826d67bf884a61eeb8dfddfdd1e80ff6ada1f4fd7fb30d4bc4263b7628986f9137f92e86c178871bb35681e53449a77a9f
|
data/README.md
CHANGED
@@ -2,6 +2,31 @@
|
|
2
2
|
|
3
3
|
Adapts elasticgraph-indexer to run in an AWS Lambda.
|
4
4
|
|
5
|
+
## Dependency Diagram
|
6
|
+
|
7
|
+
```mermaid
|
8
|
+
graph LR;
|
9
|
+
classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px;
|
10
|
+
classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000;
|
11
|
+
classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9;
|
12
|
+
elasticgraph-indexer_lambda["elasticgraph-indexer_lambda"];
|
13
|
+
class elasticgraph-indexer_lambda targetGemStyle;
|
14
|
+
elasticgraph-indexer["elasticgraph-indexer"];
|
15
|
+
elasticgraph-indexer_lambda --> elasticgraph-indexer;
|
16
|
+
class elasticgraph-indexer otherEgGemStyle;
|
17
|
+
elasticgraph-lambda_support["elasticgraph-lambda_support"];
|
18
|
+
elasticgraph-indexer_lambda --> elasticgraph-lambda_support;
|
19
|
+
class elasticgraph-lambda_support otherEgGemStyle;
|
20
|
+
aws-sdk-s3["aws-sdk-s3"];
|
21
|
+
elasticgraph-indexer_lambda --> aws-sdk-s3;
|
22
|
+
class aws-sdk-s3 externalGemStyle;
|
23
|
+
ox["ox"];
|
24
|
+
elasticgraph-indexer_lambda --> ox;
|
25
|
+
class ox externalGemStyle;
|
26
|
+
click aws-sdk-s3 href "https://rubygems.org/gems/aws-sdk-s3" "Open on RubyGems.org" _blank;
|
27
|
+
click ox href "https://rubygems.org/gems/ox" "Open on RubyGems.org" _blank;
|
28
|
+
```
|
29
|
+
|
5
30
|
## SQS Message Payload Format
|
6
31
|
|
7
32
|
We use [JSON Lines](http://jsonlines.org/) to encode our indexing events. It is just individual JSON objects
|
@@ -7,6 +7,7 @@
|
|
7
7
|
# frozen_string_literal: true
|
8
8
|
|
9
9
|
require "elastic_graph/lambda_support/lambda_function"
|
10
|
+
require "json"
|
10
11
|
|
11
12
|
module ElasticGraph
|
12
13
|
module IndexerLambda
|
@@ -14,12 +15,21 @@ module ElasticGraph
|
|
14
15
|
class LambdaFunction
|
15
16
|
prepend LambdaSupport::LambdaFunction
|
16
17
|
|
18
|
+
# @dynamic sqs_processor
|
19
|
+
attr_reader :sqs_processor
|
20
|
+
|
17
21
|
def initialize
|
18
22
|
require "elastic_graph/indexer_lambda"
|
19
23
|
require "elastic_graph/indexer_lambda/sqs_processor"
|
20
24
|
|
21
25
|
indexer = ElasticGraph::IndexerLambda.indexer_from_env
|
22
|
-
|
26
|
+
ignore_sqs_latency_timestamps_from_arns = ::JSON.parse(ENV.fetch("IGNORE_SQS_LATENCY_TIMESTAMPS_FROM_ARNS", "[]")).to_set
|
27
|
+
|
28
|
+
@sqs_processor = ElasticGraph::IndexerLambda::SqsProcessor.new(
|
29
|
+
indexer.processor,
|
30
|
+
ignore_sqs_latency_timestamps_from_arns: ignore_sqs_latency_timestamps_from_arns,
|
31
|
+
logger: indexer.logger
|
32
|
+
)
|
23
33
|
end
|
24
34
|
|
25
35
|
def handle_request(event:, context:)
|
@@ -16,10 +16,14 @@ module ElasticGraph
|
|
16
16
|
#
|
17
17
|
# @private
|
18
18
|
class SqsProcessor
|
19
|
-
|
19
|
+
# @dynamic ignore_sqs_latency_timestamps_from_arns
|
20
|
+
attr_reader :ignore_sqs_latency_timestamps_from_arns
|
21
|
+
|
22
|
+
def initialize(indexer_processor, logger:, ignore_sqs_latency_timestamps_from_arns:, s3_client: nil)
|
20
23
|
@indexer_processor = indexer_processor
|
21
24
|
@logger = logger
|
22
25
|
@s3_client = s3_client
|
26
|
+
@ignore_sqs_latency_timestamps_from_arns = ignore_sqs_latency_timestamps_from_arns
|
23
27
|
end
|
24
28
|
|
25
29
|
# Processes the ElasticGraph events in the given `lambda_event`, indexing the data in the datastore.
|
@@ -67,9 +71,15 @@ module ElasticGraph
|
|
67
71
|
sqs_received_at_by_message_id = {} # : Hash[String, String]
|
68
72
|
lambda_event.fetch("Records").flat_map do |record|
|
69
73
|
sqs_metadata = extract_sqs_metadata(record)
|
74
|
+
|
70
75
|
if (message_id = sqs_metadata.fetch("message_id", nil))
|
71
76
|
sqs_received_at_by_message_id[message_id] = sqs_metadata.dig("latency_timestamps", "sqs_received_at")
|
72
77
|
end
|
78
|
+
|
79
|
+
if @ignore_sqs_latency_timestamps_from_arns.include?(record.fetch("eventSourceARN"))
|
80
|
+
sqs_metadata = sqs_metadata.except("latency_timestamps")
|
81
|
+
end
|
82
|
+
|
73
83
|
parse_jsonl(record.fetch("body")).map do |event|
|
74
84
|
ElasticGraph::Support::HashUtil.deep_merge(event, sqs_metadata)
|
75
85
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticgraph-indexer_lambda
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.rc2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Myron Marston
|
@@ -17,42 +17,42 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - '='
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: 1.0.0.
|
20
|
+
version: 1.0.0.rc2
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - '='
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: 1.0.0.
|
27
|
+
version: 1.0.0.rc2
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: elasticgraph-lambda_support
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - '='
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 1.0.0.
|
34
|
+
version: 1.0.0.rc2
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - '='
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 1.0.0.
|
41
|
+
version: 1.0.0.rc2
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: aws-sdk-s3
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - "~>"
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: '1.
|
48
|
+
version: '1.192'
|
49
49
|
type: :runtime
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: '1.
|
55
|
+
version: '1.192'
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
57
|
name: ox
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -62,7 +62,7 @@ dependencies:
|
|
62
62
|
version: '2.14'
|
63
63
|
- - ">="
|
64
64
|
- !ruby/object:Gem::Version
|
65
|
-
version: 2.14.
|
65
|
+
version: 2.14.23
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
68
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -72,7 +72,7 @@ dependencies:
|
|
72
72
|
version: '2.14'
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 2.14.
|
75
|
+
version: 2.14.23
|
76
76
|
email:
|
77
77
|
- myron@squareup.com
|
78
78
|
executables: []
|
@@ -89,10 +89,10 @@ licenses:
|
|
89
89
|
- MIT
|
90
90
|
metadata:
|
91
91
|
bug_tracker_uri: https://github.com/block/elasticgraph/issues
|
92
|
-
changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.
|
93
|
-
documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.
|
92
|
+
changelog_uri: https://github.com/block/elasticgraph/releases/tag/v1.0.0.rc2
|
93
|
+
documentation_uri: https://block.github.io/elasticgraph/api-docs/v1.0.0.rc2/
|
94
94
|
homepage_uri: https://block.github.io/elasticgraph/
|
95
|
-
source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.
|
95
|
+
source_code_uri: https://github.com/block/elasticgraph/tree/v1.0.0.rc2/elasticgraph-indexer_lambda
|
96
96
|
gem_category: lambda
|
97
97
|
rdoc_options: []
|
98
98
|
require_paths:
|