@laserdata/laser-sdk 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -2
- package/dist/agent/reliable-consumer.d.ts.map +1 -1
- package/dist/agent/reliable-consumer.js +9 -6
- package/dist/agent/reliable-consumer.js.map +1 -1
- package/dist/bridges/a2a.d.ts +1 -1
- package/dist/bridges/a2a.js +1 -1
- package/dist/client/capabilities.d.ts +27 -2
- package/dist/client/capabilities.d.ts.map +1 -1
- package/dist/client/capabilities.js +33 -4
- package/dist/client/capabilities.js.map +1 -1
- package/dist/client/errors.d.ts +36 -1
- package/dist/client/errors.d.ts.map +1 -1
- package/dist/client/errors.js +54 -0
- package/dist/client/errors.js.map +1 -1
- package/dist/client/internals.d.ts +1 -0
- package/dist/client/internals.d.ts.map +1 -1
- package/dist/client/internals.js +1 -0
- package/dist/client/internals.js.map +1 -1
- package/dist/client/laser.d.ts +1 -0
- package/dist/client/laser.d.ts.map +1 -1
- package/dist/client/laser.js +41 -7
- package/dist/client/laser.js.map +1 -1
- package/dist/client/managed.d.ts +1 -0
- package/dist/client/managed.d.ts.map +1 -1
- package/dist/client/managed.js +15 -2
- package/dist/client/managed.js.map +1 -1
- package/dist/iggy/apache-iggy.d.ts +63 -2
- package/dist/iggy/apache-iggy.d.ts.map +1 -1
- package/dist/iggy/apache-iggy.js +343 -14
- package/dist/iggy/apache-iggy.js.map +1 -1
- package/dist/index.d.ts +10 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/managed/filters.d.ts +287 -0
- package/dist/managed/filters.d.ts.map +1 -0
- package/dist/managed/filters.js +1923 -0
- package/dist/managed/filters.js.map +1 -0
- package/dist/stream/consumer-group.d.ts +168 -0
- package/dist/stream/consumer-group.d.ts.map +1 -0
- package/dist/stream/consumer-group.js +394 -0
- package/dist/stream/consumer-group.js.map +1 -0
- package/dist/stream/consumer.d.ts +46 -3
- package/dist/stream/consumer.d.ts.map +1 -1
- package/dist/stream/consumer.js +233 -36
- package/dist/stream/consumer.js.map +1 -1
- package/dist/stream/producer-statistics.d.ts +22 -0
- package/dist/stream/producer-statistics.d.ts.map +1 -0
- package/dist/stream/producer-statistics.js +214 -0
- package/dist/stream/producer-statistics.js.map +1 -0
- package/dist/stream/producer.d.ts +2 -0
- package/dist/stream/producer.d.ts.map +1 -1
- package/dist/stream/producer.js +16 -0
- package/dist/stream/producer.js.map +1 -1
- package/dist/stream/stream.d.ts +3 -1
- package/dist/stream/stream.d.ts.map +1 -1
- package/dist/stream/stream.js +4 -2
- package/dist/stream/stream.js.map +1 -1
- package/dist/stream/topic.d.ts +20 -2
- package/dist/stream/topic.d.ts.map +1 -1
- package/dist/stream/topic.js +33 -4
- package/dist/stream/topic.js.map +1 -1
- package/dist/wire/authz.d.ts +1 -1
- package/dist/wire/authz.d.ts.map +1 -1
- package/dist/wire/authz.js +12 -3
- package/dist/wire/authz.js.map +1 -1
- package/dist/wire/codes.d.ts +17 -0
- package/dist/wire/codes.d.ts.map +1 -1
- package/dist/wire/codes.js +17 -0
- package/dist/wire/codes.js.map +1 -1
- package/dist/wire/commands.d.ts +12 -1
- package/dist/wire/commands.d.ts.map +1 -1
- package/dist/wire/commands.js +13 -1
- package/dist/wire/commands.js.map +1 -1
- package/dist/wire/control.d.ts +4 -0
- package/dist/wire/control.d.ts.map +1 -1
- package/dist/wire/control.js +5 -0
- package/dist/wire/control.js.map +1 -1
- package/dist/wire/filter-codecs.d.ts +16 -0
- package/dist/wire/filter-codecs.d.ts.map +1 -0
- package/dist/wire/filter-codecs.js +519 -0
- package/dist/wire/filter-codecs.js.map +1 -0
- package/dist/wire/filter-eval.d.ts +75 -0
- package/dist/wire/filter-eval.d.ts.map +1 -0
- package/dist/wire/filter-eval.js +899 -0
- package/dist/wire/filter-eval.js.map +1 -0
- package/dist/wire/filter.d.ts +835 -0
- package/dist/wire/filter.d.ts.map +1 -0
- package/dist/wire/filter.js +3037 -0
- package/dist/wire/filter.js.map +1 -0
- package/dist/wire/hello.d.ts +19 -0
- package/dist/wire/hello.d.ts.map +1 -1
- package/dist/wire/hello.js +30 -3
- package/dist/wire/hello.js.map +1 -1
- package/dist/wire/http.d.ts +23 -0
- package/dist/wire/http.d.ts.map +1 -1
- package/dist/wire/http.js +56 -47
- package/dist/wire/http.js.map +1 -1
- package/dist/wire/limits.d.ts +27 -0
- package/dist/wire/limits.d.ts.map +1 -1
- package/dist/wire/limits.js +27 -0
- package/dist/wire/limits.js.map +1 -1
- package/dist/wire/result.d.ts +4 -0
- package/dist/wire/result.d.ts.map +1 -1
- package/dist/wire/result.js +50 -0
- package/dist/wire/result.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,1923 @@
|
|
|
1
|
+
import { resultCodeIsRetryable } from "../wire/result.js";
|
|
2
|
+
import { jsonCodec } from "../stream/codecs.js";
|
|
3
|
+
import { decodeBrowseReply, encodeGetSchema } from "../wire/browse.js";
|
|
4
|
+
import { AGDX_GET_SCHEMA_CODE, QUERY_OP_VERSION } from "../wire/codes.js";
|
|
5
|
+
import { AmbiguousMutationError, ConfigError, FilterExecutionError, FilterStopError, InvalidError, ProtocolError, QueryExecutionError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
|
|
6
|
+
import { executeManaged, requireManagedCommand } from "../client/managed.js";
|
|
7
|
+
import { decodePolledBody, serverErrorCode } from "../iggy/apache-iggy.js";
|
|
8
|
+
import { mintUlidValue } from "../runtime/ulid.js";
|
|
9
|
+
import { decodeOne, encodeNamed } from "../wire/cbor.js";
|
|
10
|
+
import { AGDX_FILTERED_ACK_CODE, AGDX_FILTERED_POLL_CODE, FILTER_OP_VERSION } from "../wire/codes.js";
|
|
11
|
+
import { FilterMutateCommand, FilterOperationCommand, FilterPreviewCommand, FilterTestCommand, GetFilterBindingCommand, ListFilterBindingsCommand, ListFilterRevisionsCommand, ListFiltersCommand } from "../wire/commands.js";
|
|
12
|
+
import { CompiledFilter, usesRegex } from "../wire/filter-eval.js";
|
|
13
|
+
import { decodeFilterReply, encodeFilteredAck, encodeFilteredPollRequest, nextFilteredStart, validateCatalogPage, validateFilterGroupRef, validateFilterMutationRequest, validateFilterPreviewRequest, validateFilterTestRequest, validateFilteredPollRequest } from "../wire/filter.js";
|
|
14
|
+
import { MAX_FILTER_CATALOG_PAGE, MAX_FILTER_PARSE_DEPTH, MAX_FILTER_PREVIEW_EXAMINED, MAX_FILTERED_PAGE_BYTES } from "../wire/limits.js";
|
|
15
|
+
const DEFAULT_COUNT = 100;
|
|
16
|
+
const DEFAULT_MAX_REPLY_BYTES = 1024 * 1024;
|
|
17
|
+
const DEFAULT_IDLE_INTERVAL_MS = 250;
|
|
18
|
+
const DEFAULT_PAGE_SIZE = 50;
|
|
19
|
+
const DEFAULT_PREVIEW_RECORDS = 20;
|
|
20
|
+
const DEFAULT_MAX_UNACKED_PAGES = 1024;
|
|
21
|
+
const MAX_NODE_CONNECTIONS = 16;
|
|
22
|
+
const ASSIGNMENT_REFRESH_MS = 2_000;
|
|
23
|
+
const MUTATION_ATTEMPTS = 3;
|
|
24
|
+
const MUTATION_RETRY_BACKOFF_MS = 200;
|
|
25
|
+
const OUTCOME_POLL_INTERVAL_MS = 100;
|
|
26
|
+
/** How long the typed catalog verbs wait for a mutation's authoritative outcome. */
|
|
27
|
+
export const DEFAULT_OUTCOME_WAIT_MS = 30_000;
|
|
28
|
+
const GET_STREAM_CODE = 200;
|
|
29
|
+
const GET_TOPIC_CODE = 300;
|
|
30
|
+
const GET_CONSUMER_GROUP_CODE = 600;
|
|
31
|
+
const ATTACH_CONSUMER_SESSION_CODE = 14;
|
|
32
|
+
const SYNC_CONSUMER_GROUP_CODE = 606;
|
|
33
|
+
const GET_CONSUMER_OFFSET_ROUTING_CODE = 123;
|
|
34
|
+
const GET_POLL_ROUTING_CODE = 103;
|
|
35
|
+
const CONSUMER_SESSION_BYTES = 32;
|
|
36
|
+
// A consumer session is a u128 client id and a u64 session number, then the u64 metadata watermark.
|
|
37
|
+
const SESSION_IDENTITY_BYTES = 24;
|
|
38
|
+
const POLLING_NEXT = 5;
|
|
39
|
+
const CONSUMER_GROUP_PARTITION_NOT_OWNED = 5009;
|
|
40
|
+
// Wider than any server bound, so a record the server could decode always
|
|
41
|
+
// decodes here too and a mismatch is a real disagreement.
|
|
42
|
+
const GUARD_LIMITS = {
|
|
43
|
+
maxPayloadBytes: MAX_FILTERED_PAGE_BYTES,
|
|
44
|
+
maxDepth: MAX_FILTER_PARSE_DEPTH
|
|
45
|
+
};
|
|
46
|
+
const UTF8 = new TextEncoder();
|
|
47
|
+
/**
|
|
48
|
+
* Marks a page or record with the reader that read it, so an acknowledgment
|
|
49
|
+
* can tell its own records from another reader's. A spread copy keeps it.
|
|
50
|
+
*/
|
|
51
|
+
export const READER_TAG = Symbol("laser.filters.reader");
|
|
52
|
+
/**
|
|
53
|
+
* The catalog and native filter commands behind a consumer group's filter
|
|
54
|
+
* handle, reached through `ConsumerGroup.filter()`. Native commands need a
|
|
55
|
+
* server that serves consumer filters. The catalog also needs a managed
|
|
56
|
+
* plane. Either missing throws `UnsupportedError`.
|
|
57
|
+
*/
|
|
58
|
+
export class Filters {
|
|
59
|
+
transport;
|
|
60
|
+
capabilities;
|
|
61
|
+
constructor(transport, capabilities) {
|
|
62
|
+
this.transport = transport;
|
|
63
|
+
this.capabilities = capabilities;
|
|
64
|
+
}
|
|
65
|
+
/** Throws `UnsupportedError` unless the group filter catalog is served. */
|
|
66
|
+
async requireCatalog() {
|
|
67
|
+
const capabilities = await this.capabilities();
|
|
68
|
+
requireManagedCommand(capabilities, FilterMutateCommand);
|
|
69
|
+
}
|
|
70
|
+
/** Evaluate a filter against one supplied record and explain the verdict. */
|
|
71
|
+
async test(filter, payload, headers = []) {
|
|
72
|
+
const request = {
|
|
73
|
+
v: FILTER_OP_VERSION,
|
|
74
|
+
filter,
|
|
75
|
+
payload: typeof payload === "string" ? UTF8.encode(payload) : payload,
|
|
76
|
+
headers
|
|
77
|
+
};
|
|
78
|
+
validateFilterTestRequest(request);
|
|
79
|
+
const outcome = await this.native(FilterTestCommand, request);
|
|
80
|
+
if (outcome.kind === "tested")
|
|
81
|
+
return outcome.result;
|
|
82
|
+
throw unexpected("test");
|
|
83
|
+
}
|
|
84
|
+
/** Preview a filter over stored records of one partition. It joins no group and stores no offset. */
|
|
85
|
+
async preview(stream, topic, partitionId, filter, options = {}) {
|
|
86
|
+
const request = {
|
|
87
|
+
v: FILTER_OP_VERSION,
|
|
88
|
+
source: { stream, topic },
|
|
89
|
+
partitionId,
|
|
90
|
+
filter,
|
|
91
|
+
fromOffset: options.fromOffset ?? 0n,
|
|
92
|
+
maxExamined: options.maxExamined ?? MAX_FILTER_PREVIEW_EXAMINED,
|
|
93
|
+
maxRecords: options.maxRecords ?? DEFAULT_PREVIEW_RECORDS,
|
|
94
|
+
explain: options.explain ?? false
|
|
95
|
+
};
|
|
96
|
+
validateFilterPreviewRequest(request);
|
|
97
|
+
const outcome = await this.native(FilterPreviewCommand, request);
|
|
98
|
+
if (outcome.kind === "preview")
|
|
99
|
+
return outcome.preview;
|
|
100
|
+
throw unexpected("preview");
|
|
101
|
+
}
|
|
102
|
+
/** One page of a filter's revisions, newest first. */
|
|
103
|
+
async revisions(filterId, options = {}) {
|
|
104
|
+
const request = {
|
|
105
|
+
v: FILTER_OP_VERSION,
|
|
106
|
+
filterId,
|
|
107
|
+
page: options.page ?? 0,
|
|
108
|
+
pageSize: options.pageSize ?? DEFAULT_PAGE_SIZE
|
|
109
|
+
};
|
|
110
|
+
validateCatalogPage(request.page, request.pageSize);
|
|
111
|
+
const outcome = await this.catalog(ListFilterRevisionsCommand, request);
|
|
112
|
+
if (outcome.kind === "revisions")
|
|
113
|
+
return outcome.page;
|
|
114
|
+
throw unexpected("revisions");
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* One page of saved filters whose name contains `nameContains`, newest first.
|
|
118
|
+
*
|
|
119
|
+
* @internal
|
|
120
|
+
*/
|
|
121
|
+
async list(nameContains, options = {}) {
|
|
122
|
+
const request = {
|
|
123
|
+
v: FILTER_OP_VERSION,
|
|
124
|
+
nameContains,
|
|
125
|
+
page: options.page ?? 0,
|
|
126
|
+
pageSize: options.pageSize ?? DEFAULT_PAGE_SIZE
|
|
127
|
+
};
|
|
128
|
+
validateCatalogPage(request.page, request.pageSize);
|
|
129
|
+
const outcome = await this.catalog(ListFiltersCommand, request);
|
|
130
|
+
if (outcome.kind === "filters")
|
|
131
|
+
return outcome.page;
|
|
132
|
+
throw unexpected("filters");
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Drop a saved filter. The catalog releases every group bound to it.
|
|
136
|
+
*
|
|
137
|
+
* @internal
|
|
138
|
+
*/
|
|
139
|
+
async dropFilter(filterId) {
|
|
140
|
+
const applied = await this.apply({ kind: "drop", filterId });
|
|
141
|
+
if (applied.result.kind === "dropped")
|
|
142
|
+
return applied.catalogPosition;
|
|
143
|
+
throw unexpected("drop");
|
|
144
|
+
}
|
|
145
|
+
/** The binding of one consumer group. Throws with reason `not_found` when it is unbound. */
|
|
146
|
+
async binding(group) {
|
|
147
|
+
validateFilterGroupRef(group);
|
|
148
|
+
const outcome = await this.catalog(GetFilterBindingCommand, { v: FILTER_OP_VERSION, group });
|
|
149
|
+
if (outcome.kind === "binding")
|
|
150
|
+
return outcome.binding;
|
|
151
|
+
throw unexpected("binding");
|
|
152
|
+
}
|
|
153
|
+
/** One page of bindings, optionally narrowed to one filter, one stream, or one stream and topic. */
|
|
154
|
+
async bindings(options = {}) {
|
|
155
|
+
const request = {
|
|
156
|
+
v: FILTER_OP_VERSION,
|
|
157
|
+
...(options.filterId !== undefined ? { filterId: options.filterId } : {}),
|
|
158
|
+
...(options.stream !== undefined ? { stream: options.stream } : {}),
|
|
159
|
+
...(options.topic !== undefined ? { topic: options.topic } : {}),
|
|
160
|
+
page: options.page ?? 0,
|
|
161
|
+
pageSize: options.pageSize ?? DEFAULT_PAGE_SIZE
|
|
162
|
+
};
|
|
163
|
+
validateCatalogPage(request.page, request.pageSize);
|
|
164
|
+
const outcome = await this.catalog(ListFilterBindingsCommand, request);
|
|
165
|
+
if (outcome.kind === "bindings")
|
|
166
|
+
return outcome.page;
|
|
167
|
+
throw unexpected("bindings");
|
|
168
|
+
}
|
|
169
|
+
/** The recorded outcome of a mutation. Throws with reason `not_found` until the catalog applies it. */
|
|
170
|
+
async operation(operationId) {
|
|
171
|
+
const outcome = await this.catalog(FilterOperationCommand, {
|
|
172
|
+
v: FILTER_OP_VERSION,
|
|
173
|
+
operationId
|
|
174
|
+
});
|
|
175
|
+
if (outcome.kind === "mutation")
|
|
176
|
+
return checkedMutationOutcome(operationId, outcome.outcome);
|
|
177
|
+
throw unexpected("operation");
|
|
178
|
+
}
|
|
179
|
+
/** Add a revision. `expectedRevision` must still be the latest one. */
|
|
180
|
+
async revise(filterId, expectedRevision, filter) {
|
|
181
|
+
const applied = await this.apply({ kind: "revise", filterId, expectedRevision, filter });
|
|
182
|
+
if (applied.result.kind === "revised")
|
|
183
|
+
return applied.result.revision;
|
|
184
|
+
throw unexpected("revise");
|
|
185
|
+
}
|
|
186
|
+
/** Pause or resume a revision without changing its executable content or digest. */
|
|
187
|
+
async setRevisionEnabled(filterId, revision, enabled) {
|
|
188
|
+
await this.apply({ kind: "set_revision_enabled", filterId, revision, enabled });
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Give `group` its policy in one catalog transaction: a definition saved as
|
|
192
|
+
* the group's own filter, or one of its existing revisions. A repeat with
|
|
193
|
+
* the same digest keeps the binding, another digest conflicts. The returned
|
|
194
|
+
* position is where the fold applied it, carried by the group's reads so
|
|
195
|
+
* they never see this configuration as absent.
|
|
196
|
+
*/
|
|
197
|
+
async configureGroup(group, policy, operationId = mintUlidValue(), expectedIdentity) {
|
|
198
|
+
const identity = expectedIdentity ?? (await nativeGroup(this.transport, group, group.group)).identity;
|
|
199
|
+
const applied = await this.apply({ kind: "configure_group", group, policy, expectedIdentity: identity }, operationId);
|
|
200
|
+
if (applied.result.kind === "bound")
|
|
201
|
+
return bound(applied.result.binding, applied);
|
|
202
|
+
throw unexpected("configure_group");
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Release this exact saved binding, including a group incarnation that was
|
|
206
|
+
* deleted and recreated under the same name.
|
|
207
|
+
*/
|
|
208
|
+
async releaseGroup(binding) {
|
|
209
|
+
const applied = await this.apply({
|
|
210
|
+
kind: "unbind",
|
|
211
|
+
group: binding.group,
|
|
212
|
+
expectedDigest: binding.digest,
|
|
213
|
+
expectedIdentity: binding.identity
|
|
214
|
+
});
|
|
215
|
+
if (applied.result.kind === "unbound")
|
|
216
|
+
return bound(applied.result.binding, applied);
|
|
217
|
+
throw unexpected("unbind");
|
|
218
|
+
}
|
|
219
|
+
/** Send one mutation under a caller-chosen `operationId`. The outcome may be pending. */
|
|
220
|
+
async mutate(operationId, mutation) {
|
|
221
|
+
const request = { v: FILTER_OP_VERSION, operationId, mutation };
|
|
222
|
+
validateFilterMutationRequest(request);
|
|
223
|
+
const outcome = await this.catalog(FilterMutateCommand, request);
|
|
224
|
+
if (outcome.kind === "mutation")
|
|
225
|
+
return checkedMutationOutcome(operationId, outcome.outcome);
|
|
226
|
+
throw unexpected("mutate");
|
|
227
|
+
}
|
|
228
|
+
// Submit a mutation and wait for its authoritative outcome. A lost reply is
|
|
229
|
+
// retried under the same id, which the catalog answers with the first
|
|
230
|
+
// attempt's outcome. A rejection throws `FilterExecutionError`. An outcome
|
|
231
|
+
// still pending after `DEFAULT_OUTCOME_WAIT_MS` throws
|
|
232
|
+
// `AmbiguousMutationError` naming the operation id.
|
|
233
|
+
async apply(mutation, operationId = mintUlidValue()) {
|
|
234
|
+
const submitted = await this.submit(operationId, mutation);
|
|
235
|
+
const outcome = submitted.status.kind === "pending"
|
|
236
|
+
? await this.waitForOutcome(operationId, DEFAULT_OUTCOME_WAIT_MS)
|
|
237
|
+
: submitted;
|
|
238
|
+
return {
|
|
239
|
+
result: settled(operationId, outcome.status),
|
|
240
|
+
...(outcome.catalogPosition !== undefined ? { catalogPosition: outcome.catalogPosition } : {})
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
async waitForOutcome(operationId, timeoutMs) {
|
|
244
|
+
const deadline = Date.now() + timeoutMs;
|
|
245
|
+
for (;;) {
|
|
246
|
+
try {
|
|
247
|
+
const outcome = await this.operation(operationId);
|
|
248
|
+
if (outcome.status.kind !== "pending")
|
|
249
|
+
return outcome;
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
if (!isNotYetApplied(error))
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
if (Date.now() >= deadline) {
|
|
256
|
+
throw new AmbiguousMutationError(`filter mutation ${operationId.toString()} has no outcome yet`);
|
|
257
|
+
}
|
|
258
|
+
await sleep(OUTCOME_POLL_INTERVAL_MS);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
async submit(operationId, mutation) {
|
|
262
|
+
for (let attempt = 1;; attempt += 1) {
|
|
263
|
+
try {
|
|
264
|
+
return await this.mutate(operationId, mutation);
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
if (!isTransient(error))
|
|
268
|
+
throw error;
|
|
269
|
+
if (attempt >= MUTATION_ATTEMPTS) {
|
|
270
|
+
throw new AmbiguousMutationError(`filter mutation ${operationId.toString()}: outcome is unknown`, { cause: error });
|
|
271
|
+
}
|
|
272
|
+
await sleep(MUTATION_RETRY_BACKOFF_MS);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
async native(command, request) {
|
|
277
|
+
const reply = await executeManaged(this.transport, await this.capabilities(), command, request);
|
|
278
|
+
if (reply.kind === "ok")
|
|
279
|
+
return reply.outcome;
|
|
280
|
+
throw filterError(reply.error);
|
|
281
|
+
}
|
|
282
|
+
async catalog(command, request) {
|
|
283
|
+
const reply = await executeManaged(this.transport, await this.capabilities(), command, request);
|
|
284
|
+
if (reply.kind === "ok")
|
|
285
|
+
return reply.outcome;
|
|
286
|
+
throw filterError(reply.error);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
function checkedMutationOutcome(operationId, outcome) {
|
|
290
|
+
if (outcome.v !== FILTER_OP_VERSION ||
|
|
291
|
+
outcome.operationId !== operationId ||
|
|
292
|
+
(outcome.catalogPosition?.operationId !== undefined &&
|
|
293
|
+
outcome.catalogPosition.operationId !== operationId)) {
|
|
294
|
+
throw new ProtocolError("filter mutation outcome does not match the requested operation");
|
|
295
|
+
}
|
|
296
|
+
return outcome;
|
|
297
|
+
}
|
|
298
|
+
function bound(binding, applied) {
|
|
299
|
+
return {
|
|
300
|
+
binding,
|
|
301
|
+
...(applied.catalogPosition !== undefined ? { catalogPosition: applied.catalogPosition } : {})
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
/** Builds a {@link FilteredReader} over one consumer group. Start from `ConsumerGroup.reader()`. */
|
|
305
|
+
export class FilteredReaderBuilder {
|
|
306
|
+
transport;
|
|
307
|
+
capabilities;
|
|
308
|
+
filters;
|
|
309
|
+
source;
|
|
310
|
+
consumer;
|
|
311
|
+
filter;
|
|
312
|
+
minCatalogPosition;
|
|
313
|
+
startAt = { kind: "next" };
|
|
314
|
+
pageCount = DEFAULT_COUNT;
|
|
315
|
+
examinedBudget;
|
|
316
|
+
replyBytes = DEFAULT_MAX_REPLY_BYTES;
|
|
317
|
+
mode = "primary";
|
|
318
|
+
guarded = false;
|
|
319
|
+
idleMs = DEFAULT_IDLE_INTERVAL_MS;
|
|
320
|
+
maxUnacked = DEFAULT_MAX_UNACKED_PAGES;
|
|
321
|
+
selectedPartitions = new Set();
|
|
322
|
+
constructor(transport, capabilities, filters, source, consumer, filter, minCatalogPosition) {
|
|
323
|
+
this.transport = transport;
|
|
324
|
+
this.capabilities = capabilities;
|
|
325
|
+
this.filters = filters;
|
|
326
|
+
this.source = source;
|
|
327
|
+
this.consumer = consumer;
|
|
328
|
+
this.filter = filter;
|
|
329
|
+
this.minCatalogPosition = minCatalogPosition;
|
|
330
|
+
}
|
|
331
|
+
/** @internal */
|
|
332
|
+
static create(transport, capabilities, filters, source, consumer, filter, minCatalogPosition) {
|
|
333
|
+
return new FilteredReaderBuilder(transport, capabilities, filters, source, consumer, filter, minCatalogPosition);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Where each partition read at the build starts. Defaults to after the
|
|
337
|
+
* stored offset. A partition the group hands this reader later always resumes
|
|
338
|
+
* after the group's stored offset, so the previous owner's unacknowledged
|
|
339
|
+
* backlog is read, not skipped.
|
|
340
|
+
*/
|
|
341
|
+
start(start) {
|
|
342
|
+
this.startAt = start;
|
|
343
|
+
return this;
|
|
344
|
+
}
|
|
345
|
+
/** Read this partition only while the group assigns it to this member. Repeat for several. */
|
|
346
|
+
partition(partitionId) {
|
|
347
|
+
if (!Number.isSafeInteger(partitionId) || partitionId < 0 || partitionId > 0xffff_ffff) {
|
|
348
|
+
throw new InvalidError("partition must be an unsigned 32-bit integer");
|
|
349
|
+
}
|
|
350
|
+
this.selectedPartitions.add(partitionId);
|
|
351
|
+
return this;
|
|
352
|
+
}
|
|
353
|
+
/** Most matching records per page. */
|
|
354
|
+
count(count) {
|
|
355
|
+
this.pageCount = count;
|
|
356
|
+
return this;
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Most source records one page examines, independent of `count`. Without it
|
|
360
|
+
* the server's own budget bounds the scan. A page that examines its budget
|
|
361
|
+
* without a match returns empty and the reader continues from where it
|
|
362
|
+
* stopped.
|
|
363
|
+
*/
|
|
364
|
+
maxExamined(records) {
|
|
365
|
+
this.examinedBudget = records;
|
|
366
|
+
return this;
|
|
367
|
+
}
|
|
368
|
+
/** Most record bytes per page. The server may lower it. */
|
|
369
|
+
maxReplyBytes(bytes) {
|
|
370
|
+
this.replyBytes = bytes;
|
|
371
|
+
return this;
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* `primary` (the default) reads from each partition primary and can
|
|
375
|
+
* acknowledge, and needs a Laser connected from a connection string.
|
|
376
|
+
* `local` reads any replica, cannot acknowledge, and keeps no pending pages.
|
|
377
|
+
*/
|
|
378
|
+
readMode(mode) {
|
|
379
|
+
this.mode = mode;
|
|
380
|
+
return this;
|
|
381
|
+
}
|
|
382
|
+
/** Re-evaluate returned records and fail on disagreement. Regex predicates cannot use this guard. */
|
|
383
|
+
localGuard(enabled) {
|
|
384
|
+
this.guarded = enabled;
|
|
385
|
+
return this;
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* How long `nextPage` waits after a round that found nothing new, and how
|
|
389
|
+
* long a failing or blocked partition waits before it is read again.
|
|
390
|
+
*/
|
|
391
|
+
idleInterval(milliseconds) {
|
|
392
|
+
if (!Number.isFinite(milliseconds) || milliseconds < 0)
|
|
393
|
+
throw new InvalidError("the idle interval must be finite and non-negative");
|
|
394
|
+
this.idleMs = milliseconds;
|
|
395
|
+
return this;
|
|
396
|
+
}
|
|
397
|
+
/** Most outstanding record-bearing pages per partition. Defaults to 1024. */
|
|
398
|
+
maxUnackedPages(pages) {
|
|
399
|
+
if (!Number.isSafeInteger(pages) || pages < 1)
|
|
400
|
+
throw new InvalidError("maxUnackedPages must be a positive integer");
|
|
401
|
+
this.maxUnacked = pages;
|
|
402
|
+
return this;
|
|
403
|
+
}
|
|
404
|
+
async build() {
|
|
405
|
+
const capabilities = await this.capabilities();
|
|
406
|
+
if (this.filter.kind === "group" && !capabilities.filters.groupPolicyReads) {
|
|
407
|
+
throw new UnsupportedError("this server serves consumer filters but not group-aware reads, upgrade it before reading groups through the Laser SDK");
|
|
408
|
+
}
|
|
409
|
+
if (this.filter.kind !== "group" && !capabilities.filters.native) {
|
|
410
|
+
throw new UnsupportedError("consumer filters are not served by this server");
|
|
411
|
+
}
|
|
412
|
+
if (this.mode === "primary" && this.transport.connectsNodes !== true) {
|
|
413
|
+
throw new ConfigError("primary group reads open data connections, so they need a Laser connected from a connection string. Read in local mode for an injected client");
|
|
414
|
+
}
|
|
415
|
+
const consumer = this.consumer;
|
|
416
|
+
const request = {
|
|
417
|
+
v: FILTER_OP_VERSION,
|
|
418
|
+
source: this.source,
|
|
419
|
+
partitionId: 0,
|
|
420
|
+
consumer,
|
|
421
|
+
filter: this.filter,
|
|
422
|
+
start: this.startAt,
|
|
423
|
+
count: this.pageCount,
|
|
424
|
+
maxReplyBytes: this.replyBytes,
|
|
425
|
+
readMode: this.mode,
|
|
426
|
+
...(this.examinedBudget !== undefined ? { maxExamined: this.examinedBudget } : {}),
|
|
427
|
+
...(this.minCatalogPosition !== undefined
|
|
428
|
+
? { minCatalogPosition: this.minCatalogPosition }
|
|
429
|
+
: {})
|
|
430
|
+
};
|
|
431
|
+
validateFilteredPollRequest(request);
|
|
432
|
+
const pinnedSource = consumer.kind === "group_id"
|
|
433
|
+
? await sourceIncarnation(this.transport, this.source)
|
|
434
|
+
: undefined;
|
|
435
|
+
// A group reader owns the connection that holds its membership, so two
|
|
436
|
+
// readers are two members. An injected client has only the shared one.
|
|
437
|
+
const coordinator = this.transport.connectsNodes === true && this.transport.openCoordinator !== undefined
|
|
438
|
+
? await this.transport.openCoordinator()
|
|
439
|
+
: sharedCoordinator(this.transport);
|
|
440
|
+
let membership;
|
|
441
|
+
try {
|
|
442
|
+
membership = await Membership.join(coordinator, this.source, consumer.kind === "group_id" ? Number(consumer.id) : consumer.name, this.transport, pinnedSource);
|
|
443
|
+
}
|
|
444
|
+
catch (error) {
|
|
445
|
+
await coordinator.close();
|
|
446
|
+
throw error;
|
|
447
|
+
}
|
|
448
|
+
return FilteredReader.create({
|
|
449
|
+
transport: this.transport,
|
|
450
|
+
...(pinnedSource === undefined ? {} : { pinnedSource }),
|
|
451
|
+
filters: this.filters,
|
|
452
|
+
coordinator,
|
|
453
|
+
request,
|
|
454
|
+
start: this.startAt,
|
|
455
|
+
idleIntervalMs: this.idleMs,
|
|
456
|
+
maxUnackedPages: this.maxUnacked,
|
|
457
|
+
guard: undefined,
|
|
458
|
+
guardEnabled: this.guarded,
|
|
459
|
+
membership,
|
|
460
|
+
selectedPartitions: new Set(this.selectedPartitions),
|
|
461
|
+
partitionIds: membership.partitions.filter((partition) => this.selectedPartitions.size === 0 || this.selectedPartitions.has(partition))
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Reads the records a consumer group's policy selects, with their original
|
|
467
|
+
* offsets, and stores progress through fenced acknowledgments. Pages are read
|
|
468
|
+
* ahead of acknowledgments, and a page's safe offset is stored only after it
|
|
469
|
+
* and every earlier page of the partition are fully handled. Pages that
|
|
470
|
+
* matched nothing are stored by the reader itself. A partition that fails,
|
|
471
|
+
* faults, or blocks waits one idle interval before it is read again, while
|
|
472
|
+
* the others keep reading. A purge or a new group binding restarts the
|
|
473
|
+
* partition from its stored offset once, and the error that reports it
|
|
474
|
+
* reaches the caller. Drive one reader from one task at a time.
|
|
475
|
+
*/
|
|
476
|
+
export class FilteredReader {
|
|
477
|
+
settings;
|
|
478
|
+
progress = new Map();
|
|
479
|
+
revoked = new Set();
|
|
480
|
+
routes;
|
|
481
|
+
buffered = [];
|
|
482
|
+
guard;
|
|
483
|
+
cursor = 0;
|
|
484
|
+
examined = 0;
|
|
485
|
+
sequence = 0;
|
|
486
|
+
owner = {};
|
|
487
|
+
retiredOwners = new WeakSet();
|
|
488
|
+
closed = false;
|
|
489
|
+
deferredError;
|
|
490
|
+
pinnedSource;
|
|
491
|
+
// Partitions whose last read lost its route, for an independent reader.
|
|
492
|
+
lostRoutes = new Set();
|
|
493
|
+
constructor(settings) {
|
|
494
|
+
this.settings = settings;
|
|
495
|
+
this.routes = new Routes(settings.transport, settings.coordinator);
|
|
496
|
+
this.guard = settings.guard;
|
|
497
|
+
this.pinnedSource = settings.pinnedSource;
|
|
498
|
+
for (const partitionId of settings.partitionIds) {
|
|
499
|
+
this.progress.set(partitionId, new PartitionProgress(settings.start));
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
/** @internal */
|
|
503
|
+
static create(settings) {
|
|
504
|
+
return new FilteredReader(settings);
|
|
505
|
+
}
|
|
506
|
+
/** The next page with at least one match. Waits while nothing is new, up to `timeoutMs` when given. */
|
|
507
|
+
async nextPage(options = {}) {
|
|
508
|
+
const deadline = options.timeoutMs === undefined ? undefined : Date.now() + options.timeoutMs;
|
|
509
|
+
for (;;) {
|
|
510
|
+
const round = await this.round();
|
|
511
|
+
if (round.kind === "page")
|
|
512
|
+
return round.page;
|
|
513
|
+
if (deadline !== undefined && Date.now() >= deadline) {
|
|
514
|
+
throw new TimeoutError("a filtered page with a match");
|
|
515
|
+
}
|
|
516
|
+
if (round.kind === "idle")
|
|
517
|
+
await sleep(this.settings.idleIntervalMs);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* Read each partition in turn until a page matches or every partition has
|
|
522
|
+
* nothing new, without the idle wait. `undefined` when nothing is new.
|
|
523
|
+
*/
|
|
524
|
+
async tryNextPage() {
|
|
525
|
+
for (;;) {
|
|
526
|
+
const round = await this.round();
|
|
527
|
+
if (round.kind === "page")
|
|
528
|
+
return round.page;
|
|
529
|
+
if (round.kind === "idle")
|
|
530
|
+
return undefined;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
/** One bounded round without an idle wait. The flag means unmatched work remains. */
|
|
534
|
+
async readRound() {
|
|
535
|
+
const round = await this.round();
|
|
536
|
+
return [round.kind === "page" ? round.page : undefined, round.kind === "busy"];
|
|
537
|
+
}
|
|
538
|
+
/** Source records examined in the last bounded round, including empty pages. */
|
|
539
|
+
examinedInRound() {
|
|
540
|
+
return this.examined;
|
|
541
|
+
}
|
|
542
|
+
/** The next matching record, one at a time over `nextPage`. */
|
|
543
|
+
async nextRecord(options = {}) {
|
|
544
|
+
for (;;) {
|
|
545
|
+
this.requireOpen();
|
|
546
|
+
const record = this.buffered.shift();
|
|
547
|
+
if (record !== undefined)
|
|
548
|
+
return record;
|
|
549
|
+
const page = await this.nextPage(options);
|
|
550
|
+
this.requireOpen();
|
|
551
|
+
this.buffered.push(...page.records);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
/** Mark one record handled, and store the progress this completes. */
|
|
555
|
+
async ack(record) {
|
|
556
|
+
this.requireAcknowledgments();
|
|
557
|
+
const sequence = this.requireSequence(record);
|
|
558
|
+
this.partitionProgress(record.partitionId).completeRecord(sequence, record.offset);
|
|
559
|
+
await this.flush(record.partitionId);
|
|
560
|
+
}
|
|
561
|
+
/** Mark this partition's preceding records through `record` handled. Later records stay pending. */
|
|
562
|
+
async ackThrough(record) {
|
|
563
|
+
this.requireAcknowledgments();
|
|
564
|
+
const sequence = this.requireSequence(record);
|
|
565
|
+
if (!this.partitionProgress(record.partitionId).completeThrough(sequence, record.offset)) {
|
|
566
|
+
throw new InvalidError("the record is past the page acknowledgment boundary");
|
|
567
|
+
}
|
|
568
|
+
await this.flush(record.partitionId);
|
|
569
|
+
}
|
|
570
|
+
/** Mark every record of `page` handled, and store the progress this completes. */
|
|
571
|
+
async ackPage(page) {
|
|
572
|
+
this.requireAcknowledgments();
|
|
573
|
+
const sequence = this.requireSequence(page);
|
|
574
|
+
this.partitionProgress(page.partitionId).completePage(sequence);
|
|
575
|
+
await this.flush(page.partitionId);
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* Mark this partition's records through `record` handled without storing
|
|
579
|
+
* yet. The normal consumer delivers in order, so every delivery completes
|
|
580
|
+
* the prefix through it, and it stores that prefix on its own cadence
|
|
581
|
+
* through `flushCompleted`.
|
|
582
|
+
*
|
|
583
|
+
* @internal
|
|
584
|
+
*/
|
|
585
|
+
handled(record) {
|
|
586
|
+
const tag = record[READER_TAG];
|
|
587
|
+
if (tag?.owner !== this.owner)
|
|
588
|
+
return;
|
|
589
|
+
this.progress.get(record.partitionId)?.completeThrough(tag.sequence, record.offset);
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Store every completed prefix now.
|
|
593
|
+
*
|
|
594
|
+
* @internal
|
|
595
|
+
*/
|
|
596
|
+
async flushCompleted() {
|
|
597
|
+
if (this.settings.request.readMode !== "primary")
|
|
598
|
+
return;
|
|
599
|
+
let failure;
|
|
600
|
+
for (const partitionId of [...this.progress.keys()]) {
|
|
601
|
+
try {
|
|
602
|
+
await this.flush(partitionId);
|
|
603
|
+
}
|
|
604
|
+
catch (error) {
|
|
605
|
+
failure ??= error;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
if (failure !== undefined) {
|
|
609
|
+
throw failure instanceof Error
|
|
610
|
+
? failure
|
|
611
|
+
: new TransportError("storing group progress failed", true, { cause: failure });
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
/** The partitions this reader reads now. */
|
|
615
|
+
partitions() {
|
|
616
|
+
return [...this.progress.keys()].filter((partitionId) => !this.revoked.has(partitionId));
|
|
617
|
+
}
|
|
618
|
+
/** How long this reader waits, in milliseconds, when nothing is new. */
|
|
619
|
+
idleIntervalMs() {
|
|
620
|
+
return this.settings.idleIntervalMs;
|
|
621
|
+
}
|
|
622
|
+
/**
|
|
623
|
+
* Whether `value` was read by this reader in its current membership, so it
|
|
624
|
+
* can still be acknowledged. A rejoin retires every earlier page.
|
|
625
|
+
*/
|
|
626
|
+
owns(value) {
|
|
627
|
+
const tag = value[READER_TAG];
|
|
628
|
+
if (tag?.owner !== this.owner)
|
|
629
|
+
return false;
|
|
630
|
+
return this.progress.get(value.partitionId)?.knows(tag.sequence) ?? false;
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Data connections this reader opened to partition primaries. A healthy
|
|
634
|
+
* reader opens one per node and keeps it: the count grows only when a node
|
|
635
|
+
* stops being primary, a connection fails, or the group session ends.
|
|
636
|
+
*/
|
|
637
|
+
dataConnectionsOpened() {
|
|
638
|
+
return this.routes.opened;
|
|
639
|
+
}
|
|
640
|
+
/**
|
|
641
|
+
* Store completed progress, leave the group, and close the data
|
|
642
|
+
* connections. A round still running stops at its next step and opens
|
|
643
|
+
* nothing new.
|
|
644
|
+
*/
|
|
645
|
+
async close() {
|
|
646
|
+
if (this.closed)
|
|
647
|
+
return;
|
|
648
|
+
this.closed = true;
|
|
649
|
+
this.buffered.length = 0;
|
|
650
|
+
let failure;
|
|
651
|
+
if (this.settings.request.readMode === "primary") {
|
|
652
|
+
for (const partitionId of [...this.progress.keys()]) {
|
|
653
|
+
try {
|
|
654
|
+
await this.flush(partitionId);
|
|
655
|
+
}
|
|
656
|
+
catch (error) {
|
|
657
|
+
failure ??= error;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
try {
|
|
662
|
+
await this.settings.membership?.leave();
|
|
663
|
+
}
|
|
664
|
+
catch (error) {
|
|
665
|
+
failure ??= error;
|
|
666
|
+
}
|
|
667
|
+
await this.routes.close();
|
|
668
|
+
await this.settings.coordinator.close();
|
|
669
|
+
if (failure !== undefined) {
|
|
670
|
+
throw failure instanceof Error
|
|
671
|
+
? failure
|
|
672
|
+
: new TransportError("closing the filtered reader failed", false, { cause: failure });
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
async [Symbol.asyncDispose]() {
|
|
676
|
+
await this.close();
|
|
677
|
+
}
|
|
678
|
+
async *[Symbol.asyncIterator]() {
|
|
679
|
+
for (;;)
|
|
680
|
+
yield await this.nextRecord();
|
|
681
|
+
}
|
|
682
|
+
// One read of every partition that is not waiting, in turn from where the
|
|
683
|
+
// last round stopped. A failing partition is deferred and the rest still
|
|
684
|
+
// read, so one partition never starves the others, and its error reaches
|
|
685
|
+
// the caller at most once per idle interval.
|
|
686
|
+
async round() {
|
|
687
|
+
this.requireOpen();
|
|
688
|
+
this.examined = 0;
|
|
689
|
+
if (this.deferredError !== undefined) {
|
|
690
|
+
const error = this.deferredError;
|
|
691
|
+
this.deferredError = undefined;
|
|
692
|
+
throw error instanceof Error
|
|
693
|
+
? error
|
|
694
|
+
: new TransportError("a filtered partition failed", true, { cause: error });
|
|
695
|
+
}
|
|
696
|
+
if (this.settings.membership?.isDue() === true)
|
|
697
|
+
await this.refreshMembership();
|
|
698
|
+
const readable = this.partitions();
|
|
699
|
+
if (readable.length === 0)
|
|
700
|
+
return { kind: "idle" };
|
|
701
|
+
const now = performance.now();
|
|
702
|
+
let busy = false;
|
|
703
|
+
let failure;
|
|
704
|
+
const saturated = [];
|
|
705
|
+
for (let step = 0; step < readable.length; step += 1) {
|
|
706
|
+
this.requireOpen();
|
|
707
|
+
const index = (this.cursor + step) % readable.length;
|
|
708
|
+
const partitionId = readable[index] ?? 0;
|
|
709
|
+
if (this.progress.get(partitionId)?.waiting(now) === true)
|
|
710
|
+
continue;
|
|
711
|
+
try {
|
|
712
|
+
const read = await this.read(partitionId);
|
|
713
|
+
if (read.kind === "page") {
|
|
714
|
+
this.cursor = index + 1;
|
|
715
|
+
this.deferredError = failure;
|
|
716
|
+
return read;
|
|
717
|
+
}
|
|
718
|
+
if (read.kind === "empty")
|
|
719
|
+
busy ||= read.more;
|
|
720
|
+
if (read.kind === "saturated")
|
|
721
|
+
saturated.push(partitionId);
|
|
722
|
+
}
|
|
723
|
+
catch (error) {
|
|
724
|
+
if (this.closed)
|
|
725
|
+
throw error;
|
|
726
|
+
this.progress.get(partitionId)?.defer(now + this.settings.idleIntervalMs);
|
|
727
|
+
failure ??= error;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
if (failure !== undefined) {
|
|
731
|
+
throw failure instanceof Error
|
|
732
|
+
? failure
|
|
733
|
+
: new TransportError("a filtered read failed", true, { cause: failure });
|
|
734
|
+
}
|
|
735
|
+
if (saturated.length > 0 && !busy) {
|
|
736
|
+
throw new InvalidError(`partitions ${saturated.join(", ")} each have ${String(this.settings.maxUnackedPages ?? DEFAULT_MAX_UNACKED_PAGES)} unacknowledged pages. Acknowledge before reading more`);
|
|
737
|
+
}
|
|
738
|
+
return { kind: busy ? "busy" : "idle" };
|
|
739
|
+
}
|
|
740
|
+
async read(partitionId) {
|
|
741
|
+
const progress = this.partitionProgress(partitionId);
|
|
742
|
+
if (progress.inFlight >= (this.settings.maxUnackedPages ?? DEFAULT_MAX_UNACKED_PAGES))
|
|
743
|
+
return { kind: "saturated" };
|
|
744
|
+
const request = {
|
|
745
|
+
...this.settings.request,
|
|
746
|
+
partitionId,
|
|
747
|
+
start: progress.start
|
|
748
|
+
};
|
|
749
|
+
let outcome;
|
|
750
|
+
try {
|
|
751
|
+
outcome = await this.exchange(partitionId, AGDX_FILTERED_POLL_CODE, encodeFilteredPollRequest(request));
|
|
752
|
+
this.lostRoutes.delete(partitionId);
|
|
753
|
+
}
|
|
754
|
+
catch (error) {
|
|
755
|
+
if (isRouteLost(error)) {
|
|
756
|
+
// The round ends empty and the next one routes again, so the other
|
|
757
|
+
// partitions of this round never wait. A group follows its refreshed
|
|
758
|
+
// assignment. A reader whose route is lost twice in a row surfaces the refusal.
|
|
759
|
+
if (this.lostRoutes.has(partitionId))
|
|
760
|
+
throw error;
|
|
761
|
+
this.lostRoutes.add(partitionId);
|
|
762
|
+
this.settings.membership?.expire();
|
|
763
|
+
return { kind: "empty", more: false };
|
|
764
|
+
}
|
|
765
|
+
this.restartAfter(partitionId, error);
|
|
766
|
+
// The group's policy changed under this read. The partition restarted
|
|
767
|
+
// from its stored offset and is read again at once under the policy the
|
|
768
|
+
// group holds now. Only an explicit acknowledgment of a record read
|
|
769
|
+
// under the old policy reports the change.
|
|
770
|
+
if (error instanceof FilterExecutionError && error.reason === "conflict") {
|
|
771
|
+
return { kind: "empty", more: true };
|
|
772
|
+
}
|
|
773
|
+
throw error;
|
|
774
|
+
}
|
|
775
|
+
this.requireOpen();
|
|
776
|
+
if (outcome.kind !== "page")
|
|
777
|
+
throw unexpected("filtered poll");
|
|
778
|
+
const page = outcome.page;
|
|
779
|
+
this.examined += page.examined;
|
|
780
|
+
if (page.records.byteLength > request.maxReplyBytes + 16)
|
|
781
|
+
throw new ProtocolError("filtered page exceeds the requested record byte limit");
|
|
782
|
+
if (page.records.byteLength < 16)
|
|
783
|
+
throw new ProtocolError("filtered record body is truncated");
|
|
784
|
+
const recordHead = new DataView(page.records.buffer, page.records.byteOffset, page.records.byteLength);
|
|
785
|
+
if (recordHead.getUint32(12, true) !== page.matched || page.matched > request.count)
|
|
786
|
+
throw new ProtocolError("filtered record body has invalid counts");
|
|
787
|
+
const messages = decodePolledBody(page.records);
|
|
788
|
+
validatePage(request, page, messages);
|
|
789
|
+
if (request.consumer.kind === "group_id") {
|
|
790
|
+
const pinned = this.pinnedSource;
|
|
791
|
+
const current = page.generation;
|
|
792
|
+
if (pinned !== undefined &&
|
|
793
|
+
(pinned.streamId !== current.streamId ||
|
|
794
|
+
pinned.streamCreatedAtMicros !== current.streamCreatedAtMicros ||
|
|
795
|
+
pinned.topicId !== current.topicId ||
|
|
796
|
+
pinned.topicCreatedAtMicros !== current.topicCreatedAtMicros))
|
|
797
|
+
throw filterError({
|
|
798
|
+
code: { kind: "known", name: "NotFound" },
|
|
799
|
+
reason: "not_found",
|
|
800
|
+
message: "the numeric consumer group belongs to a replaced stream or topic, build a new reader"
|
|
801
|
+
});
|
|
802
|
+
this.pinnedSource ??= current;
|
|
803
|
+
}
|
|
804
|
+
const unevaluated = new Set(page.unevaluated);
|
|
805
|
+
// An unfiltered page ran no filter, so there is nothing to check.
|
|
806
|
+
if (page.policy.mode === "filtered") {
|
|
807
|
+
if (this.settings.guardEnabled && this.guard === undefined) {
|
|
808
|
+
this.guard = await compileGuard(this.settings.transport, await boundDefinition(this.settings.filters, request.source, page));
|
|
809
|
+
}
|
|
810
|
+
if (this.guard !== undefined) {
|
|
811
|
+
try {
|
|
812
|
+
checkGuard(this.guard, page, messages, unevaluated);
|
|
813
|
+
}
|
|
814
|
+
catch (error) {
|
|
815
|
+
this.restartAfter(partitionId, error);
|
|
816
|
+
throw error;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
this.sequence += 1;
|
|
821
|
+
const sequence = this.sequence;
|
|
822
|
+
const primary = request.readMode === "primary";
|
|
823
|
+
// A local page cannot be acknowledged, so its records are not pending
|
|
824
|
+
// work and never saturate the partition.
|
|
825
|
+
progress.record(sequence, page, primary ? messages.map((message) => message.offset) : []);
|
|
826
|
+
const blocked = progress.blocked;
|
|
827
|
+
if (blocked !== undefined)
|
|
828
|
+
progress.defer(performance.now() + this.settings.idleIntervalMs);
|
|
829
|
+
if (messages.length === 0) {
|
|
830
|
+
if (primary) {
|
|
831
|
+
try {
|
|
832
|
+
await this.flush(partitionId);
|
|
833
|
+
}
|
|
834
|
+
catch {
|
|
835
|
+
// Kept unstored: the next acknowledgment of this partition sends it again.
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
if (blocked !== undefined) {
|
|
839
|
+
throw new FilterStopError(blocked.stop, partitionId, blocked.offset, blocked.reason);
|
|
840
|
+
}
|
|
841
|
+
return { kind: "empty", more: page.stop === "filled" || page.stop === "budget" };
|
|
842
|
+
}
|
|
843
|
+
const tag = { owner: this.owner, sequence };
|
|
844
|
+
const records = messages.map((message) => ({
|
|
845
|
+
json: () => jsonCodec((value) => value).decode(message.payload),
|
|
846
|
+
partitionId,
|
|
847
|
+
offset: message.offset,
|
|
848
|
+
frontier: page.frontier,
|
|
849
|
+
evaluated: page.policy.mode === "filtered" && !unevaluated.has(message.offset),
|
|
850
|
+
payload: message.payload,
|
|
851
|
+
headers: message.headers,
|
|
852
|
+
headersMalformed: message.headersMalformed !== undefined,
|
|
853
|
+
...(message.timestampMicros !== undefined
|
|
854
|
+
? { timestampMicros: message.timestampMicros }
|
|
855
|
+
: {}),
|
|
856
|
+
[READER_TAG]: tag
|
|
857
|
+
}));
|
|
858
|
+
const matched = {
|
|
859
|
+
partitionId,
|
|
860
|
+
records,
|
|
861
|
+
policy: page.policy,
|
|
862
|
+
generation: page.generation,
|
|
863
|
+
stop: page.stop,
|
|
864
|
+
examined: page.examined,
|
|
865
|
+
frontier: page.frontier,
|
|
866
|
+
...(page.safeAckOffset !== undefined ? { safeAckOffset: page.safeAckOffset } : {}),
|
|
867
|
+
[READER_TAG]: tag
|
|
868
|
+
};
|
|
869
|
+
return { kind: "page", page: matched };
|
|
870
|
+
}
|
|
871
|
+
// A purge, a recreated group, or a new group binding ends the history or
|
|
872
|
+
// the policy the partition's continuation was issued for. The partition
|
|
873
|
+
// starts over from its stored offset, the guard reloads a rebound filter,
|
|
874
|
+
// and the error reaches the caller once.
|
|
875
|
+
restartAfter(partitionId, error) {
|
|
876
|
+
if (!(error instanceof FilterExecutionError) ||
|
|
877
|
+
(error.reason !== "source_changed" && error.reason !== "conflict")) {
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
this.progress.get(partitionId)?.restart({ kind: "next" });
|
|
881
|
+
for (let index = this.buffered.length - 1; index >= 0; index -= 1) {
|
|
882
|
+
if (this.buffered[index]?.partitionId === partitionId)
|
|
883
|
+
this.buffered.splice(index, 1);
|
|
884
|
+
}
|
|
885
|
+
if (error.reason === "source_changed")
|
|
886
|
+
this.settings.membership?.expire();
|
|
887
|
+
if (this.settings.guardEnabled)
|
|
888
|
+
this.guard = undefined;
|
|
889
|
+
}
|
|
890
|
+
// Store the completed prefix of one partition. A lost reply keeps the target
|
|
891
|
+
// unstored and the next store sends it again. Stores are monotone.
|
|
892
|
+
async flush(partitionId) {
|
|
893
|
+
const target = this.partitionProgress(partitionId).unstored;
|
|
894
|
+
if (target === undefined) {
|
|
895
|
+
this.retire(partitionId);
|
|
896
|
+
return;
|
|
897
|
+
}
|
|
898
|
+
const request = this.settings.request;
|
|
899
|
+
const ack = {
|
|
900
|
+
...(target.groupId !== undefined ? { groupId: target.groupId } : {}),
|
|
901
|
+
v: FILTER_OP_VERSION,
|
|
902
|
+
source: request.source,
|
|
903
|
+
partitionId,
|
|
904
|
+
consumer: request.consumer,
|
|
905
|
+
generation: target.generation,
|
|
906
|
+
...(target.digest === undefined ? {} : { digest: target.digest }),
|
|
907
|
+
offset: target.offset,
|
|
908
|
+
mode: target.mode,
|
|
909
|
+
policyGeneration: target.policyGeneration
|
|
910
|
+
};
|
|
911
|
+
const encoded = encodeFilteredAck(ack);
|
|
912
|
+
for (let rerouted = false;; rerouted = true) {
|
|
913
|
+
try {
|
|
914
|
+
const outcome = await this.exchange(partitionId, AGDX_FILTERED_ACK_CODE, encoded);
|
|
915
|
+
if (outcome.kind !== "acknowledged")
|
|
916
|
+
throw unexpected("filtered acknowledgment");
|
|
917
|
+
if (outcome.receipt.partitionId !== partitionId ||
|
|
918
|
+
outcome.receipt.offset !== target.offset ||
|
|
919
|
+
!sameGeneration(outcome.receipt.generation, target.generation)) {
|
|
920
|
+
throw new ProtocolError("filtered acknowledgment does not match its request");
|
|
921
|
+
}
|
|
922
|
+
this.progress.get(partitionId)?.stored(target);
|
|
923
|
+
const revoked = this.revoked.has(partitionId);
|
|
924
|
+
this.retire(partitionId);
|
|
925
|
+
if (revoked)
|
|
926
|
+
await this.routes.release(partitionId);
|
|
927
|
+
return;
|
|
928
|
+
}
|
|
929
|
+
catch (error) {
|
|
930
|
+
this.restartAfter(partitionId, error);
|
|
931
|
+
if (rerouted || !isRouteLost(error))
|
|
932
|
+
throw error;
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
async exchange(partitionId, code, request) {
|
|
937
|
+
const payload = encodeNamed(request);
|
|
938
|
+
let reply;
|
|
939
|
+
const read = this.settings.request;
|
|
940
|
+
if (read.readMode === "local") {
|
|
941
|
+
reply = await this.settings.coordinator.send(code, payload);
|
|
942
|
+
}
|
|
943
|
+
else {
|
|
944
|
+
const connection = await this.routes.connection(read.source, read.consumer, partitionId, code === AGDX_FILTERED_ACK_CODE);
|
|
945
|
+
try {
|
|
946
|
+
reply = await connection.send(code, payload);
|
|
947
|
+
}
|
|
948
|
+
catch (cause) {
|
|
949
|
+
await this.routes.invalidate(partitionId);
|
|
950
|
+
throw new TransportError(`filtered command ${String(code)} failed`, true, { cause });
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
const decoded = decodeFilterReply(decodeOne(reply, "filter reply"), "filter reply");
|
|
954
|
+
if (decoded.kind === "ok")
|
|
955
|
+
return decoded.outcome;
|
|
956
|
+
if (decoded.error.reason === "not_primary" || decoded.error.reason === "membership_stale") {
|
|
957
|
+
await this.routes.invalidate(partitionId);
|
|
958
|
+
}
|
|
959
|
+
throw filterError(decoded.error);
|
|
960
|
+
}
|
|
961
|
+
// Follow the group's current assignment. The whole change is applied to the
|
|
962
|
+
// reader before anything awaits. A partition that left the assignment stops
|
|
963
|
+
// being read but keeps its in-flight pages, so records already handed out
|
|
964
|
+
// can still be acknowledged while the server's offset fence allows it.
|
|
965
|
+
async refreshMembership() {
|
|
966
|
+
const membership = this.settings.membership;
|
|
967
|
+
if (membership === undefined || !(await membership.sync()))
|
|
968
|
+
return;
|
|
969
|
+
this.requireOpen();
|
|
970
|
+
const selection = this.settings.selectedPartitions;
|
|
971
|
+
const assigned = new Set(membership.partitions.filter((partition) => selection === undefined || selection.size === 0 || selection.has(partition)));
|
|
972
|
+
const rejoined = membership.takeRejoined();
|
|
973
|
+
const stale = rejoined ? this.routes.detach() : [];
|
|
974
|
+
if (rejoined) {
|
|
975
|
+
// A rejoin can land in a recreated group with another binding.
|
|
976
|
+
if (this.settings.guardEnabled)
|
|
977
|
+
this.guard = undefined;
|
|
978
|
+
this.retiredOwners.add(this.owner);
|
|
979
|
+
this.owner = {};
|
|
980
|
+
this.progress.clear();
|
|
981
|
+
this.revoked.clear();
|
|
982
|
+
this.buffered.length = 0;
|
|
983
|
+
}
|
|
984
|
+
const left = [...this.progress.keys()].filter((partitionId) => !assigned.has(partitionId));
|
|
985
|
+
for (const partitionId of left) {
|
|
986
|
+
this.revoked.add(partitionId);
|
|
987
|
+
this.retire(partitionId);
|
|
988
|
+
}
|
|
989
|
+
// A partition gained on a rebalance resumes after the group's stored
|
|
990
|
+
// offset, which holds what the previous owner acknowledged. One that
|
|
991
|
+
// comes back before its revoked entry retired starts over the same way,
|
|
992
|
+
// so it never replays another member's work from an old continuation.
|
|
993
|
+
for (const partitionId of assigned) {
|
|
994
|
+
if (this.revoked.delete(partitionId) || !this.progress.has(partitionId)) {
|
|
995
|
+
for (let index = this.buffered.length - 1; index >= 0; index--) {
|
|
996
|
+
if (this.buffered[index]?.partitionId === partitionId)
|
|
997
|
+
this.buffered.splice(index, 1);
|
|
998
|
+
}
|
|
999
|
+
this.progress.set(partitionId, new PartitionProgress({ kind: "next" }));
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
await Promise.all(stale.map((connection) => connection.close()));
|
|
1003
|
+
for (const partitionId of left)
|
|
1004
|
+
await this.routes.release(partitionId);
|
|
1005
|
+
}
|
|
1006
|
+
// Forget a revoked partition once nothing of it is in flight.
|
|
1007
|
+
retire(partitionId) {
|
|
1008
|
+
const progress = this.progress.get(partitionId);
|
|
1009
|
+
if (this.revoked.has(partitionId) &&
|
|
1010
|
+
progress?.inFlight === 0 &&
|
|
1011
|
+
progress.unstored === undefined) {
|
|
1012
|
+
this.progress.delete(partitionId);
|
|
1013
|
+
this.revoked.delete(partitionId);
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
partitionProgress(partitionId) {
|
|
1017
|
+
const progress = this.progress.get(partitionId);
|
|
1018
|
+
if (progress === undefined) {
|
|
1019
|
+
throw filterError({
|
|
1020
|
+
code: { kind: "known", name: "Stale" },
|
|
1021
|
+
reason: "membership_stale",
|
|
1022
|
+
message: `partition ${String(partitionId)} is no longer read by this reader. Its unacknowledged records are read again`
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
return progress;
|
|
1026
|
+
}
|
|
1027
|
+
// The owner is checked before the partition, so a foreign record reads as
|
|
1028
|
+
// foreign, and a record of this reader's earlier membership reads as retired.
|
|
1029
|
+
requireSequence(value) {
|
|
1030
|
+
const sequence = this.sequenceOf(value);
|
|
1031
|
+
if (!this.partitionProgress(value.partitionId).knows(sequence))
|
|
1032
|
+
throw filterError({
|
|
1033
|
+
code: { kind: "known", name: "Stale" },
|
|
1034
|
+
reason: "membership_stale",
|
|
1035
|
+
message: "the acknowledgment belongs to a retired partition read"
|
|
1036
|
+
});
|
|
1037
|
+
return sequence;
|
|
1038
|
+
}
|
|
1039
|
+
sequenceOf(value) {
|
|
1040
|
+
const tag = value[READER_TAG];
|
|
1041
|
+
if (tag?.owner === this.owner)
|
|
1042
|
+
return tag.sequence;
|
|
1043
|
+
if (tag !== undefined && this.retiredOwners.has(tag.owner)) {
|
|
1044
|
+
throw new InvalidError("the page or record was read before this reader rejoined its group, so it can no longer be acknowledged");
|
|
1045
|
+
}
|
|
1046
|
+
throw new InvalidError("the page or record belongs to another filtered reader");
|
|
1047
|
+
}
|
|
1048
|
+
requireOpen() {
|
|
1049
|
+
if (this.closed)
|
|
1050
|
+
throw new ConfigError("the filtered reader is closed");
|
|
1051
|
+
}
|
|
1052
|
+
requireAcknowledgments() {
|
|
1053
|
+
this.requireOpen();
|
|
1054
|
+
if (this.settings.request.readMode === "local") {
|
|
1055
|
+
throw new InvalidError("a local read mode reader cannot acknowledge: only the partition primary proves the history a page came from");
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
/** One partition's read position and acknowledgment state. */
|
|
1060
|
+
class PartitionProgress {
|
|
1061
|
+
original;
|
|
1062
|
+
current;
|
|
1063
|
+
pages = [];
|
|
1064
|
+
pendingStore;
|
|
1065
|
+
stoppedAt;
|
|
1066
|
+
retryAt;
|
|
1067
|
+
firstSequence;
|
|
1068
|
+
settledSequence = 0;
|
|
1069
|
+
constructor(original) {
|
|
1070
|
+
this.original = original;
|
|
1071
|
+
this.current = original;
|
|
1072
|
+
}
|
|
1073
|
+
get start() {
|
|
1074
|
+
return this.current;
|
|
1075
|
+
}
|
|
1076
|
+
get blocked() {
|
|
1077
|
+
return this.stoppedAt;
|
|
1078
|
+
}
|
|
1079
|
+
get inFlight() {
|
|
1080
|
+
return this.pages.length;
|
|
1081
|
+
}
|
|
1082
|
+
get unstored() {
|
|
1083
|
+
return this.pendingStore;
|
|
1084
|
+
}
|
|
1085
|
+
/**
|
|
1086
|
+
* Start over at `start` after the source history or the executed filter
|
|
1087
|
+
* changed. Every page in flight is dropped, so its records are delivered
|
|
1088
|
+
* again from the stored offset.
|
|
1089
|
+
*/
|
|
1090
|
+
restart(start) {
|
|
1091
|
+
this.original = start;
|
|
1092
|
+
this.current = start;
|
|
1093
|
+
this.pages = [];
|
|
1094
|
+
this.pendingStore = undefined;
|
|
1095
|
+
this.stoppedAt = undefined;
|
|
1096
|
+
this.retryAt = undefined;
|
|
1097
|
+
this.firstSequence = undefined;
|
|
1098
|
+
this.settledSequence = 0;
|
|
1099
|
+
}
|
|
1100
|
+
knows(sequence) {
|
|
1101
|
+
return (this.pages.some((page) => page.sequence === sequence) ||
|
|
1102
|
+
(this.firstSequence !== undefined &&
|
|
1103
|
+
sequence >= this.firstSequence &&
|
|
1104
|
+
sequence <= this.settledSequence));
|
|
1105
|
+
}
|
|
1106
|
+
/** Whether the partition waits out a delay after an error or a block. */
|
|
1107
|
+
waiting(now) {
|
|
1108
|
+
return this.retryAt !== undefined && now < this.retryAt;
|
|
1109
|
+
}
|
|
1110
|
+
defer(until) {
|
|
1111
|
+
this.retryAt = until;
|
|
1112
|
+
}
|
|
1113
|
+
/**
|
|
1114
|
+
* Track a page and its returned `offsets`, and continue after it. A blocked
|
|
1115
|
+
* partition is read again after its delay, so a block ends once the record
|
|
1116
|
+
* ages out of the partition or the policy stops faulting.
|
|
1117
|
+
*/
|
|
1118
|
+
record(sequence, page, offsets) {
|
|
1119
|
+
this.retryAt = undefined;
|
|
1120
|
+
this.firstSequence ??= sequence;
|
|
1121
|
+
this.current = nextFilteredStart(page, this.original);
|
|
1122
|
+
this.stoppedAt =
|
|
1123
|
+
page.fault === undefined
|
|
1124
|
+
? undefined
|
|
1125
|
+
: page.stop === "fault"
|
|
1126
|
+
? { stop: "fault", offset: page.fault.offset, reason: page.fault.reason }
|
|
1127
|
+
: page.stop === "oversized_record"
|
|
1128
|
+
? { stop: "oversized_record", offset: page.fault.offset }
|
|
1129
|
+
: undefined;
|
|
1130
|
+
const target = page.safeAckOffset === undefined
|
|
1131
|
+
? undefined
|
|
1132
|
+
: {
|
|
1133
|
+
offset: page.safeAckOffset,
|
|
1134
|
+
generation: page.generation,
|
|
1135
|
+
...(page.policy.digest === undefined ? {} : { digest: page.policy.digest }),
|
|
1136
|
+
mode: page.policy.mode,
|
|
1137
|
+
policyGeneration: page.policy.policyGeneration,
|
|
1138
|
+
...(page.policy.groupId !== undefined ? { groupId: page.policy.groupId } : {})
|
|
1139
|
+
};
|
|
1140
|
+
const newest = this.pages.at(-1);
|
|
1141
|
+
if (offsets.length === 0 && newest !== undefined) {
|
|
1142
|
+
if (target !== undefined)
|
|
1143
|
+
newest.target = target;
|
|
1144
|
+
}
|
|
1145
|
+
else {
|
|
1146
|
+
this.pages.push({ sequence, target, outstanding: new Set(offsets) });
|
|
1147
|
+
}
|
|
1148
|
+
this.settle();
|
|
1149
|
+
}
|
|
1150
|
+
completeRecord(sequence, offset) {
|
|
1151
|
+
this.pages.find((page) => page.sequence === sequence)?.outstanding.delete(offset);
|
|
1152
|
+
this.settle();
|
|
1153
|
+
}
|
|
1154
|
+
completeThrough(sequence, offset) {
|
|
1155
|
+
const target = this.pages.find((page) => page.sequence === sequence)?.target;
|
|
1156
|
+
if (target === undefined)
|
|
1157
|
+
return true;
|
|
1158
|
+
if (offset > target.offset)
|
|
1159
|
+
return false;
|
|
1160
|
+
for (const page of this.pages) {
|
|
1161
|
+
if (page.sequence < sequence)
|
|
1162
|
+
page.outstanding.clear();
|
|
1163
|
+
else if (page.sequence === sequence) {
|
|
1164
|
+
for (const pending of page.outstanding)
|
|
1165
|
+
if (pending <= offset)
|
|
1166
|
+
page.outstanding.delete(pending);
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
this.settle();
|
|
1170
|
+
if (this.pendingStore === undefined || this.pendingStore.offset < offset) {
|
|
1171
|
+
this.pendingStore = { ...target, offset };
|
|
1172
|
+
}
|
|
1173
|
+
return true;
|
|
1174
|
+
}
|
|
1175
|
+
completePage(sequence) {
|
|
1176
|
+
this.pages.find((page) => page.sequence === sequence)?.outstanding.clear();
|
|
1177
|
+
this.settle();
|
|
1178
|
+
}
|
|
1179
|
+
stored(target) {
|
|
1180
|
+
if (this.pendingStore === target)
|
|
1181
|
+
this.pendingStore = undefined;
|
|
1182
|
+
}
|
|
1183
|
+
// Retire the completed prefix. Its newest safe offset supersedes any older
|
|
1184
|
+
// unstored one, because a stored offset covers everything before it.
|
|
1185
|
+
settle() {
|
|
1186
|
+
while (this.pages[0]?.outstanding.size === 0) {
|
|
1187
|
+
const page = this.pages.shift();
|
|
1188
|
+
if (page !== undefined)
|
|
1189
|
+
this.settledSequence = page.sequence;
|
|
1190
|
+
if (page?.target !== undefined)
|
|
1191
|
+
this.pendingStore = page.target;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* Data connections to partition primaries, one per node endpoint, each
|
|
1197
|
+
* attached to the coordinator's consumer session. The coordinator keeps the
|
|
1198
|
+
* session, the user, and any group membership.
|
|
1199
|
+
*/
|
|
1200
|
+
class Routes {
|
|
1201
|
+
transport;
|
|
1202
|
+
coordinator;
|
|
1203
|
+
connections = new Map();
|
|
1204
|
+
// One open per endpoint at a time, so two routes to one node share it.
|
|
1205
|
+
opening = new Map();
|
|
1206
|
+
partitions = new Map();
|
|
1207
|
+
closed = false;
|
|
1208
|
+
/** Data connections these routes opened. */
|
|
1209
|
+
opened = 0;
|
|
1210
|
+
constructor(transport, coordinator) {
|
|
1211
|
+
this.transport = transport;
|
|
1212
|
+
this.coordinator = coordinator;
|
|
1213
|
+
}
|
|
1214
|
+
async connection(source, consumer, partitionId, acknowledgment) {
|
|
1215
|
+
const routed = this.partitions.get(partitionId);
|
|
1216
|
+
const known = routed === undefined ? undefined : this.connections.get(routed);
|
|
1217
|
+
if (known !== undefined)
|
|
1218
|
+
return known.connection;
|
|
1219
|
+
const open = this.transport.openNodeConnection;
|
|
1220
|
+
if (open === undefined) {
|
|
1221
|
+
throw new ConfigError("primary filtered reads need a transport that can open node connections");
|
|
1222
|
+
}
|
|
1223
|
+
const route = decodePollRouting(await this.coordinator.send(acknowledgment ? GET_CONSUMER_OFFSET_ROUTING_CODE : GET_POLL_ROUTING_CODE, acknowledgment
|
|
1224
|
+
? encodeOffsetRouting(source, consumer, partitionId)
|
|
1225
|
+
: encodePollRouting(source, consumer, partitionId)));
|
|
1226
|
+
this.requireOpen();
|
|
1227
|
+
const endpoint = `${route.ip}:${String(route.tcpPort)}`;
|
|
1228
|
+
const existing = this.connections.get(endpoint);
|
|
1229
|
+
const reuse = existing === undefined ? undefined : sessionReuse(existing.session, route.session);
|
|
1230
|
+
if (existing !== undefined && reuse === "raise") {
|
|
1231
|
+
try {
|
|
1232
|
+
await existing.connection.send(ATTACH_CONSUMER_SESSION_CODE, route.session);
|
|
1233
|
+
existing.session = route.session;
|
|
1234
|
+
}
|
|
1235
|
+
catch {
|
|
1236
|
+
await this.dropEndpoint(endpoint);
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
else if (reuse === "replace") {
|
|
1240
|
+
await this.dropEndpoint(endpoint);
|
|
1241
|
+
}
|
|
1242
|
+
if (!this.connections.has(endpoint)) {
|
|
1243
|
+
if (route.tcpPort === 0) {
|
|
1244
|
+
throw new UnsupportedError(`partition primary ${route.name} serves no TCP endpoint`);
|
|
1245
|
+
}
|
|
1246
|
+
let pending = this.opening.get(endpoint);
|
|
1247
|
+
if (pending === undefined) {
|
|
1248
|
+
if (this.connections.size >= MAX_NODE_CONNECTIONS) {
|
|
1249
|
+
const idle = [...this.connections.keys()].find((candidate) => ![...this.partitions.values()].includes(candidate));
|
|
1250
|
+
const oldest = idle ?? this.connections.keys().next().value;
|
|
1251
|
+
if (oldest !== undefined)
|
|
1252
|
+
await this.dropEndpoint(oldest);
|
|
1253
|
+
}
|
|
1254
|
+
pending = this.opening.get(endpoint);
|
|
1255
|
+
if (pending === undefined && !this.connections.has(endpoint)) {
|
|
1256
|
+
pending = attach(open.bind(this.transport), route).then((connection) => {
|
|
1257
|
+
this.opened += 1;
|
|
1258
|
+
return connection;
|
|
1259
|
+
});
|
|
1260
|
+
this.opening.set(endpoint, pending);
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
if (pending !== undefined) {
|
|
1264
|
+
let connection;
|
|
1265
|
+
try {
|
|
1266
|
+
connection = await pending;
|
|
1267
|
+
}
|
|
1268
|
+
finally {
|
|
1269
|
+
if (this.opening.get(endpoint) === pending)
|
|
1270
|
+
this.opening.delete(endpoint);
|
|
1271
|
+
}
|
|
1272
|
+
if (this.closed) {
|
|
1273
|
+
await connection.close();
|
|
1274
|
+
this.requireOpen();
|
|
1275
|
+
}
|
|
1276
|
+
if (!this.connections.has(endpoint))
|
|
1277
|
+
this.connections.set(endpoint, { connection, session: route.session });
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
this.partitions.set(partitionId, endpoint);
|
|
1281
|
+
const attached = this.connections.get(endpoint);
|
|
1282
|
+
if (attached === undefined)
|
|
1283
|
+
throw new ProtocolError("the partition route vanished");
|
|
1284
|
+
return attached.connection;
|
|
1285
|
+
}
|
|
1286
|
+
/**
|
|
1287
|
+
* Stop routing `partitionId`, for a partition this reader no longer reads.
|
|
1288
|
+
* The node connection stays open for the other partitions it serves and
|
|
1289
|
+
* closes only when none is left.
|
|
1290
|
+
*/
|
|
1291
|
+
async release(partitionId) {
|
|
1292
|
+
const endpoint = this.partitions.get(partitionId);
|
|
1293
|
+
if (endpoint === undefined)
|
|
1294
|
+
return;
|
|
1295
|
+
this.partitions.delete(partitionId);
|
|
1296
|
+
if (![...this.partitions.values()].includes(endpoint))
|
|
1297
|
+
await this.dropEndpoint(endpoint);
|
|
1298
|
+
}
|
|
1299
|
+
/**
|
|
1300
|
+
* Route `partitionId` again on its next use and drop the connection it
|
|
1301
|
+
* used, because that node no longer serves it as primary or the session
|
|
1302
|
+
* behind the attachment has ended.
|
|
1303
|
+
*/
|
|
1304
|
+
async invalidate(partitionId) {
|
|
1305
|
+
const endpoint = this.partitions.get(partitionId);
|
|
1306
|
+
this.partitions.delete(partitionId);
|
|
1307
|
+
if (endpoint !== undefined)
|
|
1308
|
+
await this.dropEndpoint(endpoint);
|
|
1309
|
+
}
|
|
1310
|
+
/** Forget every route and hand back the connections, which the caller closes. */
|
|
1311
|
+
detach() {
|
|
1312
|
+
this.partitions.clear();
|
|
1313
|
+
const connections = [...this.connections.values()].map((entry) => entry.connection);
|
|
1314
|
+
this.connections.clear();
|
|
1315
|
+
return connections;
|
|
1316
|
+
}
|
|
1317
|
+
/** Close every connection. No connection opens after this. */
|
|
1318
|
+
async close() {
|
|
1319
|
+
this.closed = true;
|
|
1320
|
+
await Promise.all(this.detach().map((connection) => connection.close()));
|
|
1321
|
+
}
|
|
1322
|
+
// The routing table forgets the endpoint before the close awaits.
|
|
1323
|
+
async dropEndpoint(endpoint) {
|
|
1324
|
+
for (const [partitionId, routed] of this.partitions) {
|
|
1325
|
+
if (routed === endpoint)
|
|
1326
|
+
this.partitions.delete(partitionId);
|
|
1327
|
+
}
|
|
1328
|
+
const entry = this.connections.get(endpoint);
|
|
1329
|
+
this.connections.delete(endpoint);
|
|
1330
|
+
await entry?.connection.close();
|
|
1331
|
+
}
|
|
1332
|
+
requireOpen() {
|
|
1333
|
+
if (this.closed)
|
|
1334
|
+
throw new ConfigError("the filtered reader is closed");
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
async function attach(open, route) {
|
|
1338
|
+
const connection = await open(route.ip, route.tcpPort);
|
|
1339
|
+
try {
|
|
1340
|
+
await connection.send(ATTACH_CONSUMER_SESSION_CODE, route.session);
|
|
1341
|
+
}
|
|
1342
|
+
catch (cause) {
|
|
1343
|
+
await connection.close();
|
|
1344
|
+
throw new TransportError("attaching the consumer session failed", true, { cause });
|
|
1345
|
+
}
|
|
1346
|
+
return connection;
|
|
1347
|
+
}
|
|
1348
|
+
// The shared transport as a coordinator, for a reader that has no connection
|
|
1349
|
+
// of its own: an injected client.
|
|
1350
|
+
function sharedCoordinator(transport) {
|
|
1351
|
+
return {
|
|
1352
|
+
send: (code, payload) => transport.sendManaged(code, payload),
|
|
1353
|
+
joinConsumerGroup: (streamId, topicId, name) => transport.joinExistingConsumerGroup !== undefined
|
|
1354
|
+
? transport.joinExistingConsumerGroup(streamId, topicId, name)
|
|
1355
|
+
: typeof name === "string"
|
|
1356
|
+
? transport.joinConsumerGroup(streamId, topicId, name)
|
|
1357
|
+
: Promise.reject(new ConfigError("this transport does not support numeric group membership")),
|
|
1358
|
+
leaveConsumerGroup: (streamId, topicId, name) => transport.leaveConsumerGroup(streamId, topicId, name),
|
|
1359
|
+
close: () => Promise.resolve()
|
|
1360
|
+
};
|
|
1361
|
+
}
|
|
1362
|
+
/** This reader's membership of a consumer group, held by its coordinator connection. */
|
|
1363
|
+
class Membership {
|
|
1364
|
+
coordinator;
|
|
1365
|
+
source;
|
|
1366
|
+
group;
|
|
1367
|
+
transport;
|
|
1368
|
+
pinnedSource;
|
|
1369
|
+
sourceReplaced = false;
|
|
1370
|
+
generation = 0n;
|
|
1371
|
+
assigned = [];
|
|
1372
|
+
syncedAt;
|
|
1373
|
+
rejoined = false;
|
|
1374
|
+
request;
|
|
1375
|
+
constructor(coordinator, source, group, transport, pinnedSource) {
|
|
1376
|
+
this.coordinator = coordinator;
|
|
1377
|
+
this.source = source;
|
|
1378
|
+
this.group = group;
|
|
1379
|
+
this.transport = transport;
|
|
1380
|
+
this.pinnedSource = pinnedSource;
|
|
1381
|
+
this.request = concat([identifier(source.stream), identifier(source.topic), identifier(group)]);
|
|
1382
|
+
}
|
|
1383
|
+
static async join(coordinator, source, group, transport, pinnedSource) {
|
|
1384
|
+
const membership = new Membership(coordinator, source, group, transport, pinnedSource);
|
|
1385
|
+
await coordinator.joinConsumerGroup(source.stream, source.topic, group);
|
|
1386
|
+
await membership.sync();
|
|
1387
|
+
return membership;
|
|
1388
|
+
}
|
|
1389
|
+
get partitions() {
|
|
1390
|
+
return this.assigned;
|
|
1391
|
+
}
|
|
1392
|
+
isDue() {
|
|
1393
|
+
return this.syncedAt === undefined || performance.now() - this.syncedAt >= ASSIGNMENT_REFRESH_MS;
|
|
1394
|
+
}
|
|
1395
|
+
expire() {
|
|
1396
|
+
this.syncedAt = undefined;
|
|
1397
|
+
}
|
|
1398
|
+
/** Read the current assignment. `true` when it changed. */
|
|
1399
|
+
async sync() {
|
|
1400
|
+
let reply = await this.coordinator.send(SYNC_CONSUMER_GROUP_CODE, this.request);
|
|
1401
|
+
if (reply.byteLength === 0) {
|
|
1402
|
+
if (this.pinnedSource !== undefined) {
|
|
1403
|
+
const current = await sourceIncarnation(this.transport, this.source);
|
|
1404
|
+
if (current.streamId !== this.pinnedSource.streamId ||
|
|
1405
|
+
current.streamCreatedAtMicros !== this.pinnedSource.streamCreatedAtMicros ||
|
|
1406
|
+
current.topicId !== this.pinnedSource.topicId ||
|
|
1407
|
+
current.topicCreatedAtMicros !== this.pinnedSource.topicCreatedAtMicros) {
|
|
1408
|
+
this.sourceReplaced = true;
|
|
1409
|
+
await this.coordinator
|
|
1410
|
+
.leaveConsumerGroup(this.source.stream, this.source.topic, this.group)
|
|
1411
|
+
.catch(() => undefined);
|
|
1412
|
+
throw filterError({
|
|
1413
|
+
code: { kind: "known", name: "NotFound" },
|
|
1414
|
+
reason: "not_found",
|
|
1415
|
+
message: "the numeric consumer group belongs to a replaced stream or topic"
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
await this.coordinator.joinConsumerGroup(this.source.stream, this.source.topic, this.group);
|
|
1420
|
+
this.rejoined = true;
|
|
1421
|
+
reply = await this.coordinator.send(SYNC_CONSUMER_GROUP_CODE, this.request);
|
|
1422
|
+
if (reply.byteLength === 0)
|
|
1423
|
+
throw filterError({
|
|
1424
|
+
code: { kind: "known", name: "Stale" },
|
|
1425
|
+
reason: "membership_stale",
|
|
1426
|
+
message: "the consumer group has not accepted this reader's rejoin"
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1429
|
+
this.syncedAt = performance.now();
|
|
1430
|
+
const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
1431
|
+
if (reply.byteLength < 12)
|
|
1432
|
+
throw new ProtocolError("the group assignment is truncated");
|
|
1433
|
+
const generation = view.getBigUint64(0, true);
|
|
1434
|
+
const count = view.getUint32(8, true);
|
|
1435
|
+
if (count > (reply.byteLength - 12) / 4 || reply.byteLength !== 12 + count * 4)
|
|
1436
|
+
throw new ProtocolError("the group assignment length is invalid");
|
|
1437
|
+
const partitions = Array.from({ length: count }, (_, index) => view.getUint32(12 + index * 4, true));
|
|
1438
|
+
const changed = this.rejoined ||
|
|
1439
|
+
generation !== this.generation ||
|
|
1440
|
+
partitions.length !== this.assigned.length ||
|
|
1441
|
+
partitions.some((partitionId, index) => partitionId !== this.assigned[index]);
|
|
1442
|
+
this.generation = generation;
|
|
1443
|
+
this.assigned = partitions;
|
|
1444
|
+
return changed;
|
|
1445
|
+
}
|
|
1446
|
+
takeRejoined() {
|
|
1447
|
+
const rejoined = this.rejoined;
|
|
1448
|
+
this.rejoined = false;
|
|
1449
|
+
return rejoined;
|
|
1450
|
+
}
|
|
1451
|
+
async leave() {
|
|
1452
|
+
if (this.sourceReplaced)
|
|
1453
|
+
return;
|
|
1454
|
+
await this.coordinator.leaveConsumerGroup(this.source.stream, this.source.topic, this.group);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
async function sourceIncarnation(transport, source) {
|
|
1458
|
+
// Native metadata responses begin with id:u32 and created_at:u64. Reading
|
|
1459
|
+
// the timestamp directly preserves microseconds that the Node SDK's Date loses.
|
|
1460
|
+
const read = (bytes) => {
|
|
1461
|
+
if (bytes.length < 12)
|
|
1462
|
+
throw new ProtocolError("source metadata is absent or truncated");
|
|
1463
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
1464
|
+
return { id: view.getUint32(0, true), created: view.getBigUint64(4, true) };
|
|
1465
|
+
};
|
|
1466
|
+
const stream = read(await transport.sendManaged(GET_STREAM_CODE, identifier(source.stream)));
|
|
1467
|
+
const topic = read(await transport.sendManaged(GET_TOPIC_CODE, concat([identifier(stream.id), identifier(source.topic)])));
|
|
1468
|
+
const current = read(await transport.sendManaged(GET_STREAM_CODE, identifier(stream.id)));
|
|
1469
|
+
if (current.created !== stream.created)
|
|
1470
|
+
throw new ProtocolError("source stream changed while building the reader");
|
|
1471
|
+
return {
|
|
1472
|
+
streamId: stream.id,
|
|
1473
|
+
streamCreatedAtMicros: stream.created,
|
|
1474
|
+
topicId: topic.id,
|
|
1475
|
+
topicCreatedAtMicros: topic.created
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
/**
|
|
1479
|
+
* Read a consumer group's native id and name, and the stream and topic
|
|
1480
|
+
* incarnations that scope it. Throws with reason `not_found` when the group
|
|
1481
|
+
* does not exist.
|
|
1482
|
+
*/
|
|
1483
|
+
export async function nativeGroup(transport, source, group) {
|
|
1484
|
+
const incarnation = await sourceIncarnation(transport, source);
|
|
1485
|
+
const reply = await transport.sendManaged(GET_CONSUMER_GROUP_CODE, concat([identifier(source.stream), identifier(source.topic), identifier(group)]));
|
|
1486
|
+
// The reply begins with id:u32, partitions:u32, members:u32, then the name
|
|
1487
|
+
// as one length byte and its bytes. A missing group answers an empty body.
|
|
1488
|
+
if (reply.byteLength === 0)
|
|
1489
|
+
throw filterError({
|
|
1490
|
+
code: { kind: "known", name: "NotFound" },
|
|
1491
|
+
reason: "not_found",
|
|
1492
|
+
message: `consumer group ${String(group)} does not exist`
|
|
1493
|
+
});
|
|
1494
|
+
if (reply.byteLength < 13)
|
|
1495
|
+
throw new ProtocolError("consumer group metadata is truncated");
|
|
1496
|
+
const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
1497
|
+
const id = view.getUint32(0, true);
|
|
1498
|
+
if (typeof group === "number" && id !== group)
|
|
1499
|
+
throw new ProtocolError("consumer group metadata identifies another group");
|
|
1500
|
+
const length = view.getUint8(12);
|
|
1501
|
+
if (reply.byteLength < 13 + length)
|
|
1502
|
+
throw new ProtocolError("consumer group metadata is truncated");
|
|
1503
|
+
return {
|
|
1504
|
+
id,
|
|
1505
|
+
name: new TextDecoder().decode(reply.subarray(13, 13 + length)),
|
|
1506
|
+
identity: { ...incarnation, groupId: BigInt(id) }
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
function identifier(name) {
|
|
1510
|
+
if (typeof name === "number") {
|
|
1511
|
+
const encoded = new Uint8Array(6);
|
|
1512
|
+
encoded.set([1, 4]);
|
|
1513
|
+
new DataView(encoded.buffer).setUint32(2, name, true);
|
|
1514
|
+
return encoded;
|
|
1515
|
+
}
|
|
1516
|
+
const bytes = UTF8.encode(name);
|
|
1517
|
+
return concat([Uint8Array.of(2, bytes.byteLength), bytes]);
|
|
1518
|
+
}
|
|
1519
|
+
// The stored-offset lookup shape, which routes an acknowledgment to the
|
|
1520
|
+
// partition primary that stores the offset.
|
|
1521
|
+
function encodeOffsetRouting(source, consumer, partitionId) {
|
|
1522
|
+
const partition = new Uint8Array(5);
|
|
1523
|
+
const view = new DataView(partition.buffer);
|
|
1524
|
+
view.setUint8(0, 1);
|
|
1525
|
+
view.setUint32(1, partitionId, true);
|
|
1526
|
+
return concat([
|
|
1527
|
+
Uint8Array.of(consumer.kind !== "consumer" ? 2 : 1),
|
|
1528
|
+
identifier(consumer.kind === "group_id" ? Number(consumer.id) : consumer.name),
|
|
1529
|
+
identifier(source.stream),
|
|
1530
|
+
identifier(source.topic),
|
|
1531
|
+
partition
|
|
1532
|
+
]);
|
|
1533
|
+
}
|
|
1534
|
+
// A committing poll shape, which the coordinator requires before it answers
|
|
1535
|
+
// with a route. It routes and reads nothing.
|
|
1536
|
+
function encodePollRouting(source, consumer, partitionId) {
|
|
1537
|
+
const tail = new Uint8Array(1 + 4 + 1 + 8 + 4 + 1);
|
|
1538
|
+
const view = new DataView(tail.buffer);
|
|
1539
|
+
view.setUint8(0, 1);
|
|
1540
|
+
view.setUint32(1, partitionId, true);
|
|
1541
|
+
view.setUint8(5, POLLING_NEXT);
|
|
1542
|
+
view.setBigUint64(6, 0n, true);
|
|
1543
|
+
view.setUint32(14, 1, true);
|
|
1544
|
+
view.setUint8(18, 1);
|
|
1545
|
+
return concat([
|
|
1546
|
+
Uint8Array.of(consumer.kind !== "consumer" ? 2 : 1),
|
|
1547
|
+
identifier(consumer.kind === "group_id" ? Number(consumer.id) : consumer.name),
|
|
1548
|
+
identifier(source.stream),
|
|
1549
|
+
identifier(source.topic),
|
|
1550
|
+
tail
|
|
1551
|
+
]);
|
|
1552
|
+
}
|
|
1553
|
+
function decodePollRouting(reply) {
|
|
1554
|
+
const truncated = () => {
|
|
1555
|
+
throw new ProtocolError("the partition route is truncated");
|
|
1556
|
+
};
|
|
1557
|
+
const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
1558
|
+
const decoder = new TextDecoder();
|
|
1559
|
+
let position = CONSUMER_SESSION_BYTES;
|
|
1560
|
+
const text = () => {
|
|
1561
|
+
if (position + 4 > reply.byteLength)
|
|
1562
|
+
truncated();
|
|
1563
|
+
const length = view.getUint32(position, true);
|
|
1564
|
+
position += 4;
|
|
1565
|
+
if (position + length > reply.byteLength)
|
|
1566
|
+
truncated();
|
|
1567
|
+
const value = decoder.decode(reply.subarray(position, position + length));
|
|
1568
|
+
position += length;
|
|
1569
|
+
return value;
|
|
1570
|
+
};
|
|
1571
|
+
if (reply.byteLength < CONSUMER_SESSION_BYTES)
|
|
1572
|
+
truncated();
|
|
1573
|
+
const name = text();
|
|
1574
|
+
const ip = text();
|
|
1575
|
+
if (reply.byteLength < position + 10)
|
|
1576
|
+
truncated();
|
|
1577
|
+
return {
|
|
1578
|
+
session: reply.slice(0, CONSUMER_SESSION_BYTES),
|
|
1579
|
+
name,
|
|
1580
|
+
ip,
|
|
1581
|
+
tcpPort: view.getUint16(position, true)
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
function concat(parts) {
|
|
1585
|
+
const total = parts.reduce((sum, part) => sum + part.byteLength, 0);
|
|
1586
|
+
const out = new Uint8Array(total);
|
|
1587
|
+
let offset = 0;
|
|
1588
|
+
for (const part of parts) {
|
|
1589
|
+
out.set(part, offset);
|
|
1590
|
+
offset += part.byteLength;
|
|
1591
|
+
}
|
|
1592
|
+
return out;
|
|
1593
|
+
}
|
|
1594
|
+
/**
|
|
1595
|
+
* What a fresh route means for the data connection already open to its node.
|
|
1596
|
+
*
|
|
1597
|
+
* A connection stays open for the life of its session. The first 24 bytes
|
|
1598
|
+
* name the session, its client id and number. The metadata watermark after
|
|
1599
|
+
* them grows with every write the coordinator commits, and that alone never
|
|
1600
|
+
* justifies a new connection: the attachment is sent again on the open one,
|
|
1601
|
+
* which only raises its read-your-writes floor.
|
|
1602
|
+
*/
|
|
1603
|
+
export function sessionReuse(open, routed) {
|
|
1604
|
+
const identity = (bytes) => bytes.subarray(0, SESSION_IDENTITY_BYTES);
|
|
1605
|
+
if (!sameBytes(identity(open), identity(routed)))
|
|
1606
|
+
return "replace";
|
|
1607
|
+
const watermark = (bytes) => new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getBigUint64(SESSION_IDENTITY_BYTES, true);
|
|
1608
|
+
return watermark(routed) > watermark(open) ? "raise" : "keep";
|
|
1609
|
+
}
|
|
1610
|
+
function sameBytes(left, right) {
|
|
1611
|
+
return left.byteLength === right.byteLength && left.every((byte, index) => byte === right[index]);
|
|
1612
|
+
}
|
|
1613
|
+
// Re-evaluate every returned record with the shared evaluator before the
|
|
1614
|
+
// caller sees it.
|
|
1615
|
+
function checkGuard(guard, page, messages, unevaluated) {
|
|
1616
|
+
if (page.policy.digest === undefined || !sameBytes(page.policy.digest, guard.digest)) {
|
|
1617
|
+
throw filterError({
|
|
1618
|
+
code: { kind: "known", name: "Conflict" },
|
|
1619
|
+
reason: "conflict",
|
|
1620
|
+
message: "the server ran another filter than the one the local guard checks"
|
|
1621
|
+
});
|
|
1622
|
+
}
|
|
1623
|
+
const filter = guard.filter;
|
|
1624
|
+
if (unevaluated.size > 0 &&
|
|
1625
|
+
filter.faultPolicy !== "pass" &&
|
|
1626
|
+
filter.foreignPolicy !== "pass" &&
|
|
1627
|
+
filter.mismatchPolicy !== "pass") {
|
|
1628
|
+
throw filterError({
|
|
1629
|
+
code: { kind: "known", name: "Backend" },
|
|
1630
|
+
reason: "backend",
|
|
1631
|
+
message: "the server returned unevaluated records without a pass policy"
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
const limits = page.evaluationLimits ?? GUARD_LIMITS;
|
|
1635
|
+
if (limits.maxPayloadBytes <= 0 ||
|
|
1636
|
+
limits.maxPayloadBytes > MAX_FILTERED_PAGE_BYTES ||
|
|
1637
|
+
limits.maxDepth <= 0 ||
|
|
1638
|
+
limits.maxDepth > MAX_FILTER_PARSE_DEPTH) {
|
|
1639
|
+
throw filterError({
|
|
1640
|
+
code: { kind: "known", name: "Backend" },
|
|
1641
|
+
reason: "backend",
|
|
1642
|
+
message: "the server returned invalid decoder limits"
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
if (unevaluated.size > 0 &&
|
|
1646
|
+
filter.faultPolicy === "pass" &&
|
|
1647
|
+
page.evaluationLimits === undefined) {
|
|
1648
|
+
throw new UnsupportedError("local verification of pass-through decode faults requires the server's evaluation limits");
|
|
1649
|
+
}
|
|
1650
|
+
for (const message of messages) {
|
|
1651
|
+
// The server decodes only the headers the filter needs, so a broken part
|
|
1652
|
+
// it never reads does not fault the record there either.
|
|
1653
|
+
const outcome = headersFault(guard, message.headersMalformed)
|
|
1654
|
+
? { verdict: "fault", fault: "malformed" }
|
|
1655
|
+
: guard.outcomeOf({
|
|
1656
|
+
payload: message.payload,
|
|
1657
|
+
headers: guard.needsHeaders ? filterHeaders(message.headers) : []
|
|
1658
|
+
}, limits);
|
|
1659
|
+
const agrees = unevaluated.has(message.offset)
|
|
1660
|
+
? outcome.verdict === "fault" &&
|
|
1661
|
+
outcome.fault !== undefined &&
|
|
1662
|
+
guard.policyFor(outcome.fault) === "pass"
|
|
1663
|
+
: outcome.verdict === "selected";
|
|
1664
|
+
if (!agrees) {
|
|
1665
|
+
throw filterError({
|
|
1666
|
+
code: { kind: "known", name: "Backend" },
|
|
1667
|
+
reason: "backend",
|
|
1668
|
+
message: `the server's evaluation of offset ${message.offset.toString()} on partition ${String(page.partitionId)} disagrees with the local guard`
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
// Whether the part of the header block that did not decode is one the
|
|
1674
|
+
// server reads for this filter: every entry for header predicates, only the
|
|
1675
|
+
// structure and `agdx.ct` for a payload filter, nothing otherwise.
|
|
1676
|
+
function headersFault(guard, fault) {
|
|
1677
|
+
if (fault === undefined)
|
|
1678
|
+
return false;
|
|
1679
|
+
if (guard.readsHeaders)
|
|
1680
|
+
return true;
|
|
1681
|
+
return guard.needsHeaders && fault !== "entry";
|
|
1682
|
+
}
|
|
1683
|
+
// The typed user headers, mapped exactly as the server maps them.
|
|
1684
|
+
function filterHeaders(headers) {
|
|
1685
|
+
return [...headers].map(([key, header]) => {
|
|
1686
|
+
switch (header.kind) {
|
|
1687
|
+
case "bool":
|
|
1688
|
+
return { key, value: { kind: "bool", value: header.value } };
|
|
1689
|
+
case "string":
|
|
1690
|
+
return { key, value: { kind: "string", value: header.value } };
|
|
1691
|
+
case "int8":
|
|
1692
|
+
case "int16":
|
|
1693
|
+
case "int32":
|
|
1694
|
+
return { key, value: { kind: "int", value: BigInt(header.value) } };
|
|
1695
|
+
case "int64":
|
|
1696
|
+
return { key, value: { kind: "int", value: header.value } };
|
|
1697
|
+
case "uint8":
|
|
1698
|
+
case "uint16":
|
|
1699
|
+
case "uint32":
|
|
1700
|
+
return { key, value: { kind: "uint", value: BigInt(header.value) } };
|
|
1701
|
+
case "uint64":
|
|
1702
|
+
return { key, value: { kind: "uint", value: header.value } };
|
|
1703
|
+
case "float":
|
|
1704
|
+
case "double":
|
|
1705
|
+
return { key, value: { kind: "float", value: header.value } };
|
|
1706
|
+
case "raw":
|
|
1707
|
+
case "int128":
|
|
1708
|
+
case "uint128":
|
|
1709
|
+
return { key, value: { kind: "raw", value: header.value } };
|
|
1710
|
+
}
|
|
1711
|
+
});
|
|
1712
|
+
}
|
|
1713
|
+
async function compileGuard(transport, filter) {
|
|
1714
|
+
if (usesRegex(filter.expr)) {
|
|
1715
|
+
throw new ConfigError("the TypeScript local guard cannot verify regex predicates, disable localGuard or use Rust or Python");
|
|
1716
|
+
}
|
|
1717
|
+
const schemas = [];
|
|
1718
|
+
for (const id of filter.schemaRefs) {
|
|
1719
|
+
const bytes = await transport.sendManaged(AGDX_GET_SCHEMA_CODE, encodeNamed(encodeGetSchema({ v: QUERY_OP_VERSION, id })));
|
|
1720
|
+
const reply = decodeBrowseReply(decodeOne(bytes, "writer schema"), "writer schema");
|
|
1721
|
+
if (reply.kind === "err")
|
|
1722
|
+
throw new QueryExecutionError("writer schema lookup failed", reply.error);
|
|
1723
|
+
if (reply.kind !== "ok" ||
|
|
1724
|
+
reply.outcome.kind !== "schema" ||
|
|
1725
|
+
reply.outcome.schema?.schema.id !== id) {
|
|
1726
|
+
throw new ProtocolError(`writer schema ${String(id)} was not returned`);
|
|
1727
|
+
}
|
|
1728
|
+
schemas.push(reply.outcome.schema.schema);
|
|
1729
|
+
}
|
|
1730
|
+
const compiled = CompiledFilter.compile(filter, schemas);
|
|
1731
|
+
return compiled;
|
|
1732
|
+
}
|
|
1733
|
+
// The full definition a reader executes, for the local guard.
|
|
1734
|
+
async function boundDefinition(filters, requested, page) {
|
|
1735
|
+
const source = page.generation;
|
|
1736
|
+
for (let index = 0;; index += 1) {
|
|
1737
|
+
const bindings = await filters.bindings({
|
|
1738
|
+
stream: requested.stream,
|
|
1739
|
+
topic: requested.topic,
|
|
1740
|
+
page: index,
|
|
1741
|
+
pageSize: MAX_FILTER_CATALOG_PAGE
|
|
1742
|
+
});
|
|
1743
|
+
const binding = bindings.items.find(({ identity }) => identity.groupId === page.policy.groupId &&
|
|
1744
|
+
identity.streamId === source.streamId &&
|
|
1745
|
+
identity.streamCreatedAtMicros === source.streamCreatedAtMicros &&
|
|
1746
|
+
identity.topicId === source.topicId &&
|
|
1747
|
+
identity.topicCreatedAtMicros === source.topicCreatedAtMicros);
|
|
1748
|
+
if (binding !== undefined) {
|
|
1749
|
+
if (page.policy.digest === undefined || !sameBytes(binding.digest, page.policy.digest))
|
|
1750
|
+
throw new ProtocolError("served policy differs from the group's catalog binding");
|
|
1751
|
+
return definition(filters, {
|
|
1752
|
+
kind: "revision",
|
|
1753
|
+
filterId: binding.filterId,
|
|
1754
|
+
revision: binding.revision
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
if ((index + 1) * MAX_FILTER_CATALOG_PAGE >= bindings.total)
|
|
1758
|
+
throw filterError({
|
|
1759
|
+
code: { kind: "known", name: "NotFound" },
|
|
1760
|
+
reason: "not_found",
|
|
1761
|
+
message: "no readable catalog binding matches this group identity"
|
|
1762
|
+
});
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
async function definition(filters, filter) {
|
|
1766
|
+
if (filter.kind === "inline")
|
|
1767
|
+
return filter.filter;
|
|
1768
|
+
if (filter.kind === "bound" || filter.kind === "group")
|
|
1769
|
+
throw new ConfigError("resolve a group guard from the served page");
|
|
1770
|
+
const { filterId, revision } = filter;
|
|
1771
|
+
for (let page = 0;; page += 1) {
|
|
1772
|
+
const revisions = await filters.revisions(filterId, { page, pageSize: MAX_FILTER_CATALOG_PAGE });
|
|
1773
|
+
const found = revisions.items.find((info) => info.revision === revision);
|
|
1774
|
+
if (found !== undefined)
|
|
1775
|
+
return found.filter;
|
|
1776
|
+
if ((page + 1) * MAX_FILTER_CATALOG_PAGE >= revisions.total) {
|
|
1777
|
+
throw filterError({
|
|
1778
|
+
code: { kind: "known", name: "NotFound" },
|
|
1779
|
+
reason: "not_found",
|
|
1780
|
+
message: `filter ${String(filterId)} has no revision ${String(revision)}`
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
function settled(operationId, status) {
|
|
1786
|
+
if (status.kind === "applied")
|
|
1787
|
+
return status.result;
|
|
1788
|
+
if (status.kind === "rejected")
|
|
1789
|
+
throw filterError(status.error);
|
|
1790
|
+
throw new AmbiguousMutationError(`filter mutation ${operationId.toString()} has no outcome yet`);
|
|
1791
|
+
}
|
|
1792
|
+
function filterError(error) {
|
|
1793
|
+
if (error.reason === "unsupported")
|
|
1794
|
+
return new UnsupportedError(error.message);
|
|
1795
|
+
return new FilterExecutionError(`${error.reason}: ${error.message}`, error);
|
|
1796
|
+
}
|
|
1797
|
+
// The data connection no longer reaches the partition primary for this
|
|
1798
|
+
// consumer: the primary moved, the attached session ended, or the group
|
|
1799
|
+
// assignment changed, which the coordinator reports as a partition this member
|
|
1800
|
+
// no longer owns. Reads are side-effect free, so the next one routes again.
|
|
1801
|
+
function isRouteLost(error) {
|
|
1802
|
+
if (error instanceof FilterExecutionError) {
|
|
1803
|
+
return error.reason === "not_primary" || error.reason === "membership_stale";
|
|
1804
|
+
}
|
|
1805
|
+
return serverErrorCode(error) === CONSUMER_GROUP_PARTITION_NOT_OWNED;
|
|
1806
|
+
}
|
|
1807
|
+
function isNotYetApplied(error) {
|
|
1808
|
+
return ((error instanceof FilterExecutionError &&
|
|
1809
|
+
error.detail.code.kind === "known" &&
|
|
1810
|
+
error.detail.code.name === "NotFound") ||
|
|
1811
|
+
(error instanceof FilterExecutionError && resultCodeIsRetryable(error.detail.code)) ||
|
|
1812
|
+
(error instanceof TransportError && error.retryable));
|
|
1813
|
+
}
|
|
1814
|
+
function isTransient(error) {
|
|
1815
|
+
return ((error instanceof TransportError && error.retryable) ||
|
|
1816
|
+
(error instanceof FilterExecutionError && resultCodeIsRetryable(error.detail.code)));
|
|
1817
|
+
}
|
|
1818
|
+
function unexpected(verb) {
|
|
1819
|
+
return new ProtocolError(`filters ${verb}: unexpected reply variant`);
|
|
1820
|
+
}
|
|
1821
|
+
function sleep(milliseconds) {
|
|
1822
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
1823
|
+
}
|
|
1824
|
+
function sameGeneration(left, right) {
|
|
1825
|
+
return (left.streamId === right.streamId &&
|
|
1826
|
+
left.streamCreatedAtMicros === right.streamCreatedAtMicros &&
|
|
1827
|
+
left.topicId === right.topicId &&
|
|
1828
|
+
left.topicCreatedAtMicros === right.topicCreatedAtMicros &&
|
|
1829
|
+
left.partitionId === right.partitionId &&
|
|
1830
|
+
left.partitionCreatedRevision === right.partitionCreatedRevision &&
|
|
1831
|
+
left.purgeGeneration === right.purgeGeneration);
|
|
1832
|
+
}
|
|
1833
|
+
function validatePage(request, page, messages) {
|
|
1834
|
+
const invalid = () => {
|
|
1835
|
+
throw new ProtocolError("filtered page does not match its request or scan boundary");
|
|
1836
|
+
};
|
|
1837
|
+
if (request.consumer.kind === "group_id" && page.policy.groupId !== request.consumer.id)
|
|
1838
|
+
invalid();
|
|
1839
|
+
if (page.v !== FILTER_OP_VERSION ||
|
|
1840
|
+
page.partitionId !== request.partitionId ||
|
|
1841
|
+
page.generation.partitionId !== request.partitionId ||
|
|
1842
|
+
page.readMode !== request.readMode ||
|
|
1843
|
+
page.matched !== messages.length ||
|
|
1844
|
+
page.matched > request.count ||
|
|
1845
|
+
page.matched > page.examined ||
|
|
1846
|
+
(request.maxExamined !== undefined && page.examined > request.maxExamined) ||
|
|
1847
|
+
(page.policy.mode === "filtered") !== (page.policy.digest !== undefined) ||
|
|
1848
|
+
(page.policy.digest !== undefined && page.policy.digest.length !== 32) ||
|
|
1849
|
+
(page.policy.mode === "unfiltered" && page.policy.groupId === undefined) ||
|
|
1850
|
+
(page.policy.mode === "unfiltered" && page.matched !== page.examined) ||
|
|
1851
|
+
(page.policy.mode === "unfiltered" &&
|
|
1852
|
+
(page.policy.filterId !== undefined || page.policy.revision !== undefined)) ||
|
|
1853
|
+
// A strict group read runs a binding and never accepts an unfiltered page.
|
|
1854
|
+
(page.policy.mode === "unfiltered" && request.filter.kind !== "group") ||
|
|
1855
|
+
(request.consumer.kind !== "consumer") !== (page.policy.groupId !== undefined))
|
|
1856
|
+
invalid();
|
|
1857
|
+
const body = new DataView(page.records.buffer, page.records.byteOffset, page.records.byteLength);
|
|
1858
|
+
if (body.getUint32(0, true) !== page.partitionId || body.getBigUint64(4, true) !== page.frontier)
|
|
1859
|
+
invalid();
|
|
1860
|
+
if (request.filter.kind === "revision" &&
|
|
1861
|
+
(page.policy.filterId !== request.filter.filterId ||
|
|
1862
|
+
page.policy.revision !== request.filter.revision))
|
|
1863
|
+
invalid();
|
|
1864
|
+
let start;
|
|
1865
|
+
if (request.start.kind === "continue") {
|
|
1866
|
+
const cursor = request.start.continuation;
|
|
1867
|
+
if (!sameGeneration(cursor.generation, page.generation) ||
|
|
1868
|
+
cursor.groupId !== page.policy.groupId ||
|
|
1869
|
+
cursor.mode !== page.policy.mode ||
|
|
1870
|
+
cursor.policyGeneration !== page.policy.policyGeneration ||
|
|
1871
|
+
(cursor.digest === undefined) !== (page.policy.digest === undefined) ||
|
|
1872
|
+
(cursor.digest !== undefined &&
|
|
1873
|
+
page.policy.digest !== undefined &&
|
|
1874
|
+
!sameBytes(cursor.digest, page.policy.digest)) ||
|
|
1875
|
+
cursor.readMode !== request.readMode)
|
|
1876
|
+
invalid();
|
|
1877
|
+
start = cursor.nextScanOffset;
|
|
1878
|
+
}
|
|
1879
|
+
else if (request.start.kind === "offset")
|
|
1880
|
+
start = request.start.offset;
|
|
1881
|
+
else if (request.start.kind === "first")
|
|
1882
|
+
start = 0n;
|
|
1883
|
+
// The last offset the page examined. A primary page offers it as its safe
|
|
1884
|
+
// acknowledgment offset, a local page offers none.
|
|
1885
|
+
const scannedTo = page.nextScanOffset === undefined || page.nextScanOffset === 0n
|
|
1886
|
+
? undefined
|
|
1887
|
+
: page.nextScanOffset - 1n;
|
|
1888
|
+
if (page.examined === 0) {
|
|
1889
|
+
if (page.safeAckOffset !== undefined || page.nextScanOffset !== start)
|
|
1890
|
+
invalid();
|
|
1891
|
+
}
|
|
1892
|
+
else if (scannedTo === undefined ||
|
|
1893
|
+
(request.readMode === "primary"
|
|
1894
|
+
? page.safeAckOffset !== scannedTo
|
|
1895
|
+
: page.safeAckOffset !== undefined))
|
|
1896
|
+
invalid();
|
|
1897
|
+
if (scannedTo !== undefined && scannedTo > page.frontier)
|
|
1898
|
+
invalid();
|
|
1899
|
+
if (start !== undefined && page.nextScanOffset !== undefined && page.nextScanOffset < start)
|
|
1900
|
+
invalid();
|
|
1901
|
+
let previous;
|
|
1902
|
+
const offsets = new Set();
|
|
1903
|
+
for (const message of messages) {
|
|
1904
|
+
const offset = message.offset;
|
|
1905
|
+
if (message.partitionId !== request.partitionId ||
|
|
1906
|
+
(previous !== undefined && offset <= previous) ||
|
|
1907
|
+
(start !== undefined && offset < start) ||
|
|
1908
|
+
scannedTo === undefined ||
|
|
1909
|
+
offset > scannedTo ||
|
|
1910
|
+
offset > page.frontier)
|
|
1911
|
+
invalid();
|
|
1912
|
+
previous = offset;
|
|
1913
|
+
offsets.add(offset);
|
|
1914
|
+
}
|
|
1915
|
+
if (new Set(page.unevaluated).size !== page.unevaluated.length ||
|
|
1916
|
+
page.unevaluated.some((offset) => !offsets.has(offset)))
|
|
1917
|
+
invalid();
|
|
1918
|
+
if ((page.stop === "fault" || page.stop === "oversized_record") !== (page.fault !== undefined))
|
|
1919
|
+
invalid();
|
|
1920
|
+
if (page.fault !== undefined && scannedTo !== undefined && scannedTo >= page.fault.offset)
|
|
1921
|
+
invalid();
|
|
1922
|
+
}
|
|
1923
|
+
//# sourceMappingURL=filters.js.map
|