@laserdata/laser-sdk 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +49 -3
  2. package/dist/agent/reliable-consumer.d.ts.map +1 -1
  3. package/dist/agent/reliable-consumer.js +5 -1
  4. package/dist/agent/reliable-consumer.js.map +1 -1
  5. package/dist/bridges/a2a.d.ts +1 -1
  6. package/dist/bridges/a2a.js +1 -1
  7. package/dist/client/capabilities.d.ts +15 -2
  8. package/dist/client/capabilities.d.ts.map +1 -1
  9. package/dist/client/capabilities.js +22 -1
  10. package/dist/client/capabilities.js.map +1 -1
  11. package/dist/client/connect-options.d.ts +5 -0
  12. package/dist/client/connect-options.d.ts.map +1 -0
  13. package/dist/client/connect-options.js +18 -0
  14. package/dist/client/connect-options.js.map +1 -0
  15. package/dist/client/errors.d.ts +23 -1
  16. package/dist/client/errors.d.ts.map +1 -1
  17. package/dist/client/errors.js +34 -0
  18. package/dist/client/errors.js.map +1 -1
  19. package/dist/client/laser.d.ts +17 -1
  20. package/dist/client/laser.d.ts.map +1 -1
  21. package/dist/client/laser.js +96 -26
  22. package/dist/client/laser.js.map +1 -1
  23. package/dist/client/managed.d.ts +1 -0
  24. package/dist/client/managed.d.ts.map +1 -1
  25. package/dist/client/managed.js +15 -2
  26. package/dist/client/managed.js.map +1 -1
  27. package/dist/iggy/apache-iggy.d.ts +69 -4
  28. package/dist/iggy/apache-iggy.d.ts.map +1 -1
  29. package/dist/iggy/apache-iggy.js +416 -23
  30. package/dist/iggy/apache-iggy.js.map +1 -1
  31. package/dist/index.d.ts +8 -2
  32. package/dist/index.d.ts.map +1 -1
  33. package/dist/index.js +4 -1
  34. package/dist/index.js.map +1 -1
  35. package/dist/managed/filters.d.ts +358 -0
  36. package/dist/managed/filters.d.ts.map +1 -0
  37. package/dist/managed/filters.js +1913 -0
  38. package/dist/managed/filters.js.map +1 -0
  39. package/dist/stream/consumer.d.ts +11 -1
  40. package/dist/stream/consumer.d.ts.map +1 -1
  41. package/dist/stream/consumer.js +82 -9
  42. package/dist/stream/consumer.js.map +1 -1
  43. package/dist/stream/stream.d.ts +4 -1
  44. package/dist/stream/stream.d.ts.map +1 -1
  45. package/dist/stream/stream.js +11 -1
  46. package/dist/stream/stream.js.map +1 -1
  47. package/dist/stream/topic.d.ts +2 -0
  48. package/dist/stream/topic.d.ts.map +1 -1
  49. package/dist/stream/topic.js +6 -0
  50. package/dist/stream/topic.js.map +1 -1
  51. package/dist/wire/authz.d.ts +1 -1
  52. package/dist/wire/authz.d.ts.map +1 -1
  53. package/dist/wire/authz.js +12 -3
  54. package/dist/wire/authz.js.map +1 -1
  55. package/dist/wire/codes.d.ts +17 -0
  56. package/dist/wire/codes.d.ts.map +1 -1
  57. package/dist/wire/codes.js +17 -0
  58. package/dist/wire/codes.js.map +1 -1
  59. package/dist/wire/commands.d.ts +12 -1
  60. package/dist/wire/commands.d.ts.map +1 -1
  61. package/dist/wire/commands.js +13 -1
  62. package/dist/wire/commands.js.map +1 -1
  63. package/dist/wire/control.d.ts +4 -0
  64. package/dist/wire/control.d.ts.map +1 -1
  65. package/dist/wire/control.js +5 -0
  66. package/dist/wire/control.js.map +1 -1
  67. package/dist/wire/filter-codecs.d.ts +16 -0
  68. package/dist/wire/filter-codecs.d.ts.map +1 -0
  69. package/dist/wire/filter-codecs.js +519 -0
  70. package/dist/wire/filter-codecs.js.map +1 -0
  71. package/dist/wire/filter-eval.d.ts +75 -0
  72. package/dist/wire/filter-eval.d.ts.map +1 -0
  73. package/dist/wire/filter-eval.js +899 -0
  74. package/dist/wire/filter-eval.js.map +1 -0
  75. package/dist/wire/filter.d.ts +772 -0
  76. package/dist/wire/filter.d.ts.map +1 -0
  77. package/dist/wire/filter.js +2869 -0
  78. package/dist/wire/filter.js.map +1 -0
  79. package/dist/wire/hello.d.ts +18 -0
  80. package/dist/wire/hello.d.ts.map +1 -1
  81. package/dist/wire/hello.js +29 -3
  82. package/dist/wire/hello.js.map +1 -1
  83. package/dist/wire/http.d.ts +22 -0
  84. package/dist/wire/http.d.ts.map +1 -1
  85. package/dist/wire/http.js +54 -47
  86. package/dist/wire/http.js.map +1 -1
  87. package/dist/wire/limits.d.ts +25 -0
  88. package/dist/wire/limits.d.ts.map +1 -1
  89. package/dist/wire/limits.js +25 -0
  90. package/dist/wire/limits.js.map +1 -1
  91. package/dist/wire/result.d.ts +4 -0
  92. package/dist/wire/result.d.ts.map +1 -1
  93. package/dist/wire/result.js +50 -0
  94. package/dist/wire/result.js.map +1 -1
  95. package/package.json +6 -6
@@ -0,0 +1,1913 @@
1
+ import { requireCapability } from "../client/capabilities.js";
2
+ import { resultCodeIsRetryable } from "../wire/result.js";
3
+ import { jsonCodec } from "../stream/codecs.js";
4
+ import { decodeBrowseReply, encodeGetSchema } from "../wire/browse.js";
5
+ import { AGDX_GET_SCHEMA_CODE, QUERY_OP_VERSION } from "../wire/codes.js";
6
+ import { AmbiguousMutationError, ConfigError, FilterExecutionError, FilterStopError, InvalidError, ProtocolError, QueryExecutionError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
7
+ import { executeManaged, requireManagedCommand } from "../client/managed.js";
8
+ import { decodePolledBody, serverErrorCode } from "../iggy/apache-iggy.js";
9
+ import { mintUlidValue } from "../runtime/ulid.js";
10
+ import { decodeOne, encodeNamed } from "../wire/cbor.js";
11
+ import { AGDX_FILTERED_ACK_CODE, AGDX_FILTERED_POLL_CODE, FILTER_OP_VERSION } from "../wire/codes.js";
12
+ import { FilterMutateCommand, FilterOperationCommand, FilterPreviewCommand, FilterTestCommand, FilterValidateCommand, GetFilterBindingCommand, GetFilterCommand, ListFilterBindingsCommand, ListFilterRevisionsCommand, ListFiltersCommand } from "../wire/commands.js";
13
+ import { CompiledFilter, usesRegex } from "../wire/filter-eval.js";
14
+ import { consumerFilterDigest, decodeFilterReply, encodeFilteredAck, encodeFilteredPollRequest, nextFilteredStart, validateCatalogPage, validateConsumerFilter, validateFilterGroupRef, validateFilterMutationRequest, validateFilterPreviewRequest, validateFilterTestRequest, validateFilteredPollRequest } from "../wire/filter.js";
15
+ import { filterAnnounceEvaluates } from "../wire/hello.js";
16
+ import { MAX_FILTER_CATALOG_PAGE, MAX_FILTER_PARSE_DEPTH, MAX_FILTER_PREVIEW_EXAMINED, MAX_FILTERED_PAGE_BYTES } from "../wire/limits.js";
17
+ const DEFAULT_COUNT = 100;
18
+ const DEFAULT_MAX_REPLY_BYTES = 1024 * 1024;
19
+ const DEFAULT_IDLE_INTERVAL_MS = 250;
20
+ const DEFAULT_PAGE_SIZE = 50;
21
+ const DEFAULT_PREVIEW_RECORDS = 20;
22
+ const DEFAULT_MAX_UNACKED_PAGES = 1024;
23
+ const MAX_NODE_CONNECTIONS = 16;
24
+ const ASSIGNMENT_REFRESH_MS = 2_000;
25
+ const MUTATION_ATTEMPTS = 3;
26
+ const MUTATION_RETRY_BACKOFF_MS = 200;
27
+ const OUTCOME_POLL_INTERVAL_MS = 100;
28
+ /** How long the typed catalog verbs wait for a mutation's authoritative outcome. */
29
+ export const DEFAULT_OUTCOME_WAIT_MS = 30_000;
30
+ const GET_STREAM_CODE = 200;
31
+ const GET_TOPIC_CODE = 300;
32
+ const ATTACH_CONSUMER_SESSION_CODE = 14;
33
+ const SYNC_CONSUMER_GROUP_CODE = 606;
34
+ const GET_CONSUMER_OFFSET_ROUTING_CODE = 123;
35
+ const GET_POLL_ROUTING_CODE = 103;
36
+ const CONSUMER_SESSION_BYTES = 32;
37
+ // A consumer session is a u128 client id and a u64 session number, then the u64 metadata watermark.
38
+ const SESSION_IDENTITY_BYTES = 24;
39
+ const POLLING_NEXT = 5;
40
+ const CONSUMER_GROUP_PARTITION_NOT_OWNED = 5009;
41
+ // Wider than any server bound, so a record the server could decode always
42
+ // decodes here too and a mismatch is a real disagreement.
43
+ const GUARD_LIMITS = {
44
+ maxPayloadBytes: MAX_FILTERED_PAGE_BYTES,
45
+ maxDepth: MAX_FILTER_PARSE_DEPTH
46
+ };
47
+ const UTF8 = new TextEncoder();
48
+ /**
49
+ * Marks a page or record with the reader that read it, so an acknowledgment
50
+ * can tell its own records from another reader's. A spread copy keeps it.
51
+ */
52
+ export const READER_TAG = Symbol("laser.filters.reader");
53
+ /**
54
+ * Server-side consumer filters: filtered readers, previews, sample tests, and
55
+ * the saved-filter catalog. Build it with `Laser.filters()`. Native reads need
56
+ * a server that serves consumer filters. The catalog also needs a managed
57
+ * plane. Either missing throws `UnsupportedError`.
58
+ */
59
+ export class Filters {
60
+ transport;
61
+ capabilities;
62
+ constructor(transport, capabilities) {
63
+ this.transport = transport;
64
+ this.capabilities = capabilities;
65
+ }
66
+ /** A filtered reader over `stream` / `topic`. */
67
+ reader(stream, topic) {
68
+ return new FilteredReaderBuilder(this.transport, this.capabilities, this, { stream, topic });
69
+ }
70
+ /** Validate and compile `filter` on the server without running it. */
71
+ async validate(filter) {
72
+ validateConsumerFilter(filter);
73
+ const outcome = await this.native(FilterValidateCommand, filter);
74
+ if (outcome.kind === "validated")
75
+ return outcome.validation;
76
+ throw unexpected("validate");
77
+ }
78
+ /** Evaluate a filter against one supplied record and explain the verdict. */
79
+ async test(filter, payload, headers = []) {
80
+ const request = {
81
+ v: FILTER_OP_VERSION,
82
+ filter,
83
+ payload: typeof payload === "string" ? UTF8.encode(payload) : payload,
84
+ headers
85
+ };
86
+ validateFilterTestRequest(request);
87
+ const outcome = await this.native(FilterTestCommand, request);
88
+ if (outcome.kind === "tested")
89
+ return outcome.result;
90
+ throw unexpected("test");
91
+ }
92
+ /** Preview a filter over stored records of one partition. It joins no group and stores no offset. */
93
+ async preview(stream, topic, partitionId, filter, options = {}) {
94
+ const request = {
95
+ v: FILTER_OP_VERSION,
96
+ source: { stream, topic },
97
+ partitionId,
98
+ filter,
99
+ fromOffset: options.fromOffset ?? 0n,
100
+ maxExamined: options.maxExamined ?? MAX_FILTER_PREVIEW_EXAMINED,
101
+ maxRecords: options.maxRecords ?? DEFAULT_PREVIEW_RECORDS,
102
+ explain: options.explain ?? false
103
+ };
104
+ validateFilterPreviewRequest(request);
105
+ const outcome = await this.native(FilterPreviewCommand, request);
106
+ if (outcome.kind === "preview")
107
+ return outcome.preview;
108
+ throw unexpected("preview");
109
+ }
110
+ /** One saved filter with its latest revision and bindings. */
111
+ async get(filterId) {
112
+ const outcome = await this.catalog(GetFilterCommand, { v: FILTER_OP_VERSION, filterId });
113
+ if (outcome.kind === "filter")
114
+ return outcome.detail;
115
+ throw unexpected("get");
116
+ }
117
+ /** One page of saved filters, newest first. */
118
+ async list(options = {}) {
119
+ const request = {
120
+ v: FILTER_OP_VERSION,
121
+ ...(options.nameContains !== undefined ? { nameContains: options.nameContains } : {}),
122
+ ...(options.state !== undefined ? { state: options.state } : {}),
123
+ ...(options.beforeId !== undefined ? { beforeId: options.beforeId } : {}),
124
+ page: options.page ?? 0,
125
+ pageSize: options.pageSize ?? DEFAULT_PAGE_SIZE
126
+ };
127
+ validateCatalogPage(request.page, request.pageSize);
128
+ const outcome = await this.catalog(ListFiltersCommand, request);
129
+ if (outcome.kind === "filters")
130
+ return outcome.page;
131
+ throw unexpected("list");
132
+ }
133
+ /** One page of a filter's revisions, newest first. */
134
+ async revisions(filterId, options = {}) {
135
+ const request = {
136
+ v: FILTER_OP_VERSION,
137
+ filterId,
138
+ page: options.page ?? 0,
139
+ pageSize: options.pageSize ?? DEFAULT_PAGE_SIZE
140
+ };
141
+ validateCatalogPage(request.page, request.pageSize);
142
+ const outcome = await this.catalog(ListFilterRevisionsCommand, request);
143
+ if (outcome.kind === "revisions")
144
+ return outcome.page;
145
+ throw unexpected("revisions");
146
+ }
147
+ /** The binding of one consumer group. Throws with reason `not_found` when it is unbound. */
148
+ async binding(group) {
149
+ validateFilterGroupRef(group);
150
+ const outcome = await this.catalog(GetFilterBindingCommand, { v: FILTER_OP_VERSION, group });
151
+ if (outcome.kind === "binding")
152
+ return outcome.binding;
153
+ throw unexpected("binding");
154
+ }
155
+ /** One page of bindings, optionally narrowed to one filter, one stream, or one stream and topic. */
156
+ async bindings(options = {}) {
157
+ const request = {
158
+ v: FILTER_OP_VERSION,
159
+ ...(options.filterId !== undefined ? { filterId: options.filterId } : {}),
160
+ ...(options.stream !== undefined ? { stream: options.stream } : {}),
161
+ ...(options.topic !== undefined ? { topic: options.topic } : {}),
162
+ page: options.page ?? 0,
163
+ pageSize: options.pageSize ?? DEFAULT_PAGE_SIZE
164
+ };
165
+ validateCatalogPage(request.page, request.pageSize);
166
+ const outcome = await this.catalog(ListFilterBindingsCommand, request);
167
+ if (outcome.kind === "bindings")
168
+ return outcome.page;
169
+ throw unexpected("bindings");
170
+ }
171
+ /** The recorded outcome of a mutation. Throws with reason `not_found` until the catalog applies it. */
172
+ async operation(operationId) {
173
+ const outcome = await this.catalog(FilterOperationCommand, {
174
+ v: FILTER_OP_VERSION,
175
+ operationId
176
+ });
177
+ if (outcome.kind === "mutation")
178
+ return outcome.outcome;
179
+ throw unexpected("operation");
180
+ }
181
+ /** Save a new filter as revision 1. */
182
+ async register(name, filter, options = {}) {
183
+ const result = await this.apply({
184
+ kind: "register",
185
+ name,
186
+ description: options.description ?? "",
187
+ filter
188
+ });
189
+ if (result.kind === "registered")
190
+ return result.revision;
191
+ throw unexpected("register");
192
+ }
193
+ /** Add a revision. `expectedRevision` must still be the latest one. */
194
+ async revise(filterId, expectedRevision, filter) {
195
+ const result = await this.apply({ kind: "revise", filterId, expectedRevision, filter });
196
+ if (result.kind === "revised")
197
+ return result.revision;
198
+ throw unexpected("revise");
199
+ }
200
+ /** Replace a filter's description. */
201
+ async describe(filterId, description) {
202
+ await this.apply({ kind: "describe", filterId, description });
203
+ }
204
+ /** Hide a filter from new bindings. Existing bindings keep executing. */
205
+ async archive(filterId) {
206
+ await this.apply({ kind: "archive", filterId });
207
+ }
208
+ /** Delete a filter. Throws with reason `conflict` while a consumer group is bound to it. */
209
+ async delete(filterId) {
210
+ await this.apply({ kind: "drop", filterId });
211
+ }
212
+ /** Pin an existing consumer group to one revision. */
213
+ async bind(group, filterId, revision) {
214
+ const result = await this.apply({ kind: "bind", group, filterId, revision });
215
+ if (result.kind === "bound")
216
+ return result.binding;
217
+ throw unexpected("bind");
218
+ }
219
+ /** Release a consumer group while it is still bound to `expectedDigest`. */
220
+ async unbind(group, expectedDigest) {
221
+ const result = await this.apply({ kind: "unbind", group, expectedDigest });
222
+ if (result.kind === "unbound")
223
+ return result.binding;
224
+ throw unexpected("unbind");
225
+ }
226
+ /** Release this exact saved binding, even after its group name was reused. */
227
+ async unbindBinding(binding) {
228
+ const result = await this.apply({
229
+ kind: "unbind",
230
+ group: binding.group,
231
+ expectedDigest: binding.digest,
232
+ expectedIdentity: binding.identity
233
+ });
234
+ if (result.kind === "unbound")
235
+ return result.binding;
236
+ throw unexpected("unbind");
237
+ }
238
+ /**
239
+ * Submit a mutation under a new operation id and wait for its authoritative
240
+ * outcome. A lost reply is retried under the same id, which the catalog
241
+ * answers with the first attempt's outcome. A rejection throws
242
+ * `FilterExecutionError`. An outcome still pending after
243
+ * {@link DEFAULT_OUTCOME_WAIT_MS} throws `AmbiguousMutationError` naming the
244
+ * operation id.
245
+ */
246
+ async apply(mutation) {
247
+ return this.applyAs(mintUlidValue(), mutation);
248
+ }
249
+ /**
250
+ * {@link Filters.apply} under a caller-chosen `operationId`, so a caller that
251
+ * records the id first can resume the same mutation after a crash.
252
+ */
253
+ async applyAs(operationId, mutation) {
254
+ const outcome = await this.submit(operationId, mutation);
255
+ if (outcome.status.kind === "pending") {
256
+ return this.waitForOutcome(operationId, DEFAULT_OUTCOME_WAIT_MS);
257
+ }
258
+ return settled(operationId, outcome.status);
259
+ }
260
+ /** Create the group if absent, then bind it. A failed bind can leave an unbound group. */
261
+ async createConsumerGroup(group, filterId, revision) {
262
+ const capabilities = await this.capabilities();
263
+ requireCapability(capabilities, "filters");
264
+ requireManagedCommand(capabilities, FilterMutateCommand);
265
+ if (this.transport.ensureConsumerGroup === undefined)
266
+ throw new ConfigError("this transport cannot create consumer groups");
267
+ await this.transport.ensureConsumerGroup(group.stream, group.topic, group.group);
268
+ return this.bind(group, filterId, revision);
269
+ }
270
+ /** Pause or resume a revision without changing its executable content or digest. */
271
+ async setRevisionEnabled(filterId, revision, enabled) {
272
+ await this.apply({ kind: "set_revision_enabled", filterId, revision, enabled });
273
+ }
274
+ /** Send one mutation under a caller-chosen `operationId`. The outcome may be pending. */
275
+ async mutate(operationId, mutation) {
276
+ const request = { v: FILTER_OP_VERSION, operationId, mutation };
277
+ validateFilterMutationRequest(request);
278
+ const outcome = await this.catalog(FilterMutateCommand, request);
279
+ if (outcome.kind === "mutation")
280
+ return outcome.outcome;
281
+ throw unexpected("mutate");
282
+ }
283
+ /** Wait for the outcome of `operationId` for at most `timeoutMs`. */
284
+ async waitForOutcome(operationId, timeoutMs) {
285
+ const deadline = Date.now() + timeoutMs;
286
+ for (;;) {
287
+ try {
288
+ const outcome = await this.operation(operationId);
289
+ if (outcome.status.kind !== "pending")
290
+ return settled(operationId, outcome.status);
291
+ }
292
+ catch (error) {
293
+ if (!isNotYetApplied(error))
294
+ throw error;
295
+ }
296
+ if (Date.now() >= deadline) {
297
+ throw new AmbiguousMutationError(`filter mutation ${operationId.toString()} has no outcome yet`);
298
+ }
299
+ await sleep(OUTCOME_POLL_INTERVAL_MS);
300
+ }
301
+ }
302
+ async submit(operationId, mutation) {
303
+ for (let attempt = 1;; attempt += 1) {
304
+ try {
305
+ return await this.mutate(operationId, mutation);
306
+ }
307
+ catch (error) {
308
+ if (!isTransient(error))
309
+ throw error;
310
+ if (attempt >= MUTATION_ATTEMPTS) {
311
+ throw new AmbiguousMutationError(`filter mutation ${operationId.toString()}: outcome is unknown`, { cause: error });
312
+ }
313
+ await sleep(MUTATION_RETRY_BACKOFF_MS);
314
+ }
315
+ }
316
+ }
317
+ async native(command, request) {
318
+ const reply = await executeManaged(this.transport, await this.capabilities(), command, request);
319
+ if (reply.kind === "ok")
320
+ return reply.outcome;
321
+ throw filterError(reply.error);
322
+ }
323
+ async catalog(command, request) {
324
+ const reply = await executeManaged(this.transport, await this.capabilities(), command, request);
325
+ if (reply.kind === "ok")
326
+ return reply.outcome;
327
+ throw filterError(reply.error);
328
+ }
329
+ }
330
+ /** Builds a {@link FilteredReader}. Start from `Filters.reader`. */
331
+ export class FilteredReaderBuilder {
332
+ transport;
333
+ capabilities;
334
+ filters;
335
+ source;
336
+ consumerName;
337
+ partitionIds = [];
338
+ filterRef;
339
+ startAt = { kind: "next" };
340
+ pageCount = DEFAULT_COUNT;
341
+ replyBytes = DEFAULT_MAX_REPLY_BYTES;
342
+ mode = "primary";
343
+ guarded = false;
344
+ idleMs = DEFAULT_IDLE_INTERVAL_MS;
345
+ maxUnacked = DEFAULT_MAX_UNACKED_PAGES;
346
+ constructor(transport, capabilities, filters, source) {
347
+ this.transport = transport;
348
+ this.capabilities = capabilities;
349
+ this.filters = filters;
350
+ this.source = source;
351
+ }
352
+ /** Read as the independent consumer `name`. */
353
+ consumer(name) {
354
+ this.consumerName = { kind: "consumer", name };
355
+ return this;
356
+ }
357
+ /**
358
+ * Read as a member of the existing consumer group `name`. A reader built
359
+ * from a connection string joins over its own connection, so every reader
360
+ * is its own member.
361
+ */
362
+ group(name) {
363
+ this.consumerName = { kind: "group", name };
364
+ return this;
365
+ }
366
+ /** Read the existing group by its native numeric identity. */
367
+ groupId(id) {
368
+ if (typeof id === "number" && !Number.isSafeInteger(id))
369
+ throw new InvalidError("consumer group id must be an integer");
370
+ const value = BigInt(id);
371
+ if (value < 0n || value > 0xffffffffn)
372
+ throw new InvalidError("consumer group id exceeds 32 bits");
373
+ this.consumerName = { kind: "group_id", id: value };
374
+ return this;
375
+ }
376
+ /**
377
+ * Read only `partitionId`. Repeat for more. An independent consumer reads
378
+ * every partition when none is named, including partitions added later.
379
+ */
380
+ partition(partitionId) {
381
+ this.partitionIds.push(partitionId);
382
+ return this;
383
+ }
384
+ /** Which filter runs. A group defaults to its bound revision. */
385
+ filter(filter) {
386
+ this.filterRef = filter;
387
+ return this;
388
+ }
389
+ /** Run `filter` from the request. Needs no catalog. */
390
+ inline(filter) {
391
+ return this.filter({ kind: "inline", filter });
392
+ }
393
+ /** Run a saved revision. */
394
+ revision(filterId, revision) {
395
+ return this.filter({ kind: "revision", filterId, revision });
396
+ }
397
+ /**
398
+ * Where each partition read at the build starts. Defaults to after the
399
+ * stored offset. A partition a group hands this reader later always resumes
400
+ * after the group's stored offset, so the previous owner's unacknowledged
401
+ * backlog is read, not skipped.
402
+ */
403
+ start(start) {
404
+ this.startAt = start;
405
+ return this;
406
+ }
407
+ /** Most matching records per page. */
408
+ count(count) {
409
+ this.pageCount = count;
410
+ return this;
411
+ }
412
+ /** Most record bytes per page. The server may lower it. */
413
+ maxReplyBytes(bytes) {
414
+ this.replyBytes = bytes;
415
+ return this;
416
+ }
417
+ /**
418
+ * `primary` (the default) reads from each partition primary and can
419
+ * acknowledge, and needs a Laser connected from a connection string.
420
+ * `local` reads any replica, cannot acknowledge, and keeps no pending pages.
421
+ */
422
+ readMode(mode) {
423
+ this.mode = mode;
424
+ return this;
425
+ }
426
+ /** Re-evaluate returned records and fail on disagreement. Regex predicates cannot use this guard. */
427
+ localGuard(enabled) {
428
+ this.guarded = enabled;
429
+ return this;
430
+ }
431
+ /**
432
+ * How long `nextPage` waits after a round that found nothing new, and how
433
+ * long a failing or blocked partition waits before it is read again.
434
+ */
435
+ idleInterval(milliseconds) {
436
+ if (!Number.isFinite(milliseconds) || milliseconds < 0)
437
+ throw new InvalidError("the idle interval must be finite and non-negative");
438
+ this.idleMs = milliseconds;
439
+ return this;
440
+ }
441
+ /** Most outstanding record-bearing pages per partition. Defaults to 1024. */
442
+ maxUnackedPages(pages) {
443
+ if (!Number.isSafeInteger(pages) || pages < 1)
444
+ throw new InvalidError("maxUnackedPages must be a positive integer");
445
+ this.maxUnacked = pages;
446
+ return this;
447
+ }
448
+ async build() {
449
+ const capabilities = await this.capabilities();
450
+ if (!capabilities.filters.native) {
451
+ throw new UnsupportedError("consumer filters are not served by this server");
452
+ }
453
+ const evaluation = capabilities.filters.evaluation;
454
+ if (this.filterRef?.kind === "inline" &&
455
+ evaluation !== undefined &&
456
+ !filterAnnounceEvaluates(evaluation, this.filterRef.filter.evaluatorVersion, this.filterRef.filter.codec)) {
457
+ throw new UnsupportedError(`this server does not evaluate ${this.filterRef.filter.codec} filters of evaluator version ${String(this.filterRef.filter.evaluatorVersion)}`);
458
+ }
459
+ if (this.mode === "primary" && this.transport.connectsNodes !== true) {
460
+ throw new ConfigError("primary filtered reads open data connections, so they need a Laser connected from a connection string. Read in local mode for an injected client");
461
+ }
462
+ const consumer = this.consumerName;
463
+ if (consumer === undefined) {
464
+ throw new ConfigError("a filtered reader needs .consumer(name), .group(name), or .groupId(id)");
465
+ }
466
+ const filter = this.filterRef ?? (consumer.kind !== "consumer" ? { kind: "bound" } : undefined);
467
+ if (filter === undefined) {
468
+ throw new ConfigError("an independent filtered reader needs .inline(..), .revision(..), or .filter(..)");
469
+ }
470
+ const request = {
471
+ v: FILTER_OP_VERSION,
472
+ source: this.source,
473
+ partitionId: 0,
474
+ consumer,
475
+ filter,
476
+ start: this.startAt,
477
+ count: this.pageCount,
478
+ maxReplyBytes: this.replyBytes,
479
+ readMode: this.mode
480
+ };
481
+ validateFilteredPollRequest(request);
482
+ const guard = this.guarded && filter.kind !== "bound"
483
+ ? await compileGuard(this.transport, await definition(this.filters, filter))
484
+ : undefined;
485
+ if (consumer.kind !== "consumer" && this.partitionIds.length > 0) {
486
+ throw new ConfigError("a group reader reads the partitions the group assigns it");
487
+ }
488
+ const pinnedSource = consumer.kind === "group_id"
489
+ ? await sourceIncarnation(this.transport, this.source)
490
+ : undefined;
491
+ const coordinator = consumer.kind !== "consumer" &&
492
+ this.transport.connectsNodes === true &&
493
+ this.transport.openCoordinator !== undefined
494
+ ? await this.transport.openCoordinator()
495
+ : sharedCoordinator(this.transport);
496
+ let membership;
497
+ let partitionIds;
498
+ let discovers = false;
499
+ try {
500
+ if (consumer.kind !== "consumer") {
501
+ membership = await Membership.join(coordinator, this.source, consumer.kind === "group_id" ? Number(consumer.id) : consumer.name, this.transport, pinnedSource);
502
+ partitionIds = membership.partitions;
503
+ }
504
+ else if (this.partitionIds.length > 0) {
505
+ partitionIds = this.partitionIds;
506
+ }
507
+ else {
508
+ partitionIds = await topicPartitions(this.transport, this.source);
509
+ discovers = true;
510
+ }
511
+ }
512
+ catch (error) {
513
+ await coordinator.close();
514
+ throw error;
515
+ }
516
+ return new FilteredReader({
517
+ transport: this.transport,
518
+ ...(pinnedSource === undefined ? {} : { pinnedSource }),
519
+ filters: this.filters,
520
+ coordinator,
521
+ request,
522
+ inlineDigest: filter.kind === "inline" ? consumerFilterDigest(filter.filter) : undefined,
523
+ start: this.startAt,
524
+ idleIntervalMs: this.idleMs,
525
+ maxUnackedPages: this.maxUnacked,
526
+ guard,
527
+ guardEnabled: this.guarded,
528
+ membership,
529
+ discovers,
530
+ partitionIds
531
+ });
532
+ }
533
+ }
534
+ /**
535
+ * Reads only the records a consumer filter selects, with their original
536
+ * offsets, and stores progress through fenced acknowledgments. Pages are read
537
+ * ahead of acknowledgments, and a page's safe offset is stored only after it
538
+ * and every earlier page of the partition are fully handled. Pages that
539
+ * matched nothing are stored by the reader itself. A partition that fails,
540
+ * faults, or blocks waits one idle interval before it is read again, while
541
+ * the others keep reading. A purge or a new group binding restarts the
542
+ * partition from its stored offset once, and the error that reports it
543
+ * reaches the caller. Drive one reader from one task at a time.
544
+ */
545
+ export class FilteredReader {
546
+ settings;
547
+ progress = new Map();
548
+ revoked = new Set();
549
+ routes;
550
+ buffered = [];
551
+ guard;
552
+ cursor = 0;
553
+ examined = 0;
554
+ sequence = 0;
555
+ owner = {};
556
+ retiredOwners = new WeakSet();
557
+ listedAt;
558
+ closed = false;
559
+ deferredError;
560
+ pinnedSource;
561
+ // Partitions whose last read lost its route, for an independent reader.
562
+ lostRoutes = new Set();
563
+ constructor(settings) {
564
+ this.settings = settings;
565
+ this.routes = new Routes(settings.transport, settings.coordinator);
566
+ this.guard = settings.guard;
567
+ this.pinnedSource = settings.pinnedSource;
568
+ this.listedAt = settings.discovers ? performance.now() : undefined;
569
+ for (const partitionId of settings.partitionIds) {
570
+ this.progress.set(partitionId, new PartitionProgress(settings.start));
571
+ }
572
+ }
573
+ /** The next page with at least one match. Waits while nothing is new, up to `timeoutMs` when given. */
574
+ async nextPage(options = {}) {
575
+ const deadline = options.timeoutMs === undefined ? undefined : Date.now() + options.timeoutMs;
576
+ for (;;) {
577
+ const round = await this.round();
578
+ if (round.kind === "page")
579
+ return round.page;
580
+ if (deadline !== undefined && Date.now() >= deadline) {
581
+ throw new TimeoutError("a filtered page with a match");
582
+ }
583
+ if (round.kind === "idle")
584
+ await sleep(this.settings.idleIntervalMs);
585
+ }
586
+ }
587
+ /**
588
+ * Read each partition in turn until a page matches or every partition has
589
+ * nothing new, without the idle wait. `undefined` when nothing is new.
590
+ */
591
+ async tryNextPage() {
592
+ for (;;) {
593
+ const round = await this.round();
594
+ if (round.kind === "page")
595
+ return round.page;
596
+ if (round.kind === "idle")
597
+ return undefined;
598
+ }
599
+ }
600
+ /** One bounded round without an idle wait. The flag means unmatched work remains. */
601
+ async readRound() {
602
+ const round = await this.round();
603
+ return [round.kind === "page" ? round.page : undefined, round.kind === "busy"];
604
+ }
605
+ /** Source records examined in the last bounded round, including empty pages. */
606
+ examinedInRound() {
607
+ return this.examined;
608
+ }
609
+ /** The next matching record, one at a time over `nextPage`. */
610
+ async nextRecord(options = {}) {
611
+ for (;;) {
612
+ this.requireOpen();
613
+ const record = this.buffered.shift();
614
+ if (record !== undefined)
615
+ return record;
616
+ const page = await this.nextPage(options);
617
+ this.requireOpen();
618
+ this.buffered.push(...page.records);
619
+ }
620
+ }
621
+ /** Mark one record handled, and store the progress this completes. */
622
+ async ack(record) {
623
+ this.requireAcknowledgments();
624
+ const sequence = this.requireSequence(record);
625
+ this.partitionProgress(record.partitionId).completeRecord(sequence, record.offset);
626
+ await this.flush(record.partitionId);
627
+ }
628
+ /** Mark this partition's preceding records through `record` handled. Later records stay pending. */
629
+ async ackThrough(record) {
630
+ this.requireAcknowledgments();
631
+ const sequence = this.requireSequence(record);
632
+ if (!this.partitionProgress(record.partitionId).completeThrough(sequence, record.offset)) {
633
+ throw new InvalidError("the record is past the page acknowledgment boundary");
634
+ }
635
+ await this.flush(record.partitionId);
636
+ }
637
+ /** Mark every record of `page` handled, and store the progress this completes. */
638
+ async ackPage(page) {
639
+ this.requireAcknowledgments();
640
+ const sequence = this.requireSequence(page);
641
+ this.partitionProgress(page.partitionId).completePage(sequence);
642
+ await this.flush(page.partitionId);
643
+ }
644
+ /** The partitions this reader reads now. */
645
+ partitions() {
646
+ return [...this.progress.keys()].filter((partitionId) => !this.revoked.has(partitionId));
647
+ }
648
+ /** How long this reader waits, in milliseconds, when nothing is new. */
649
+ idleIntervalMs() {
650
+ return this.settings.idleIntervalMs;
651
+ }
652
+ /**
653
+ * Whether `value` was read by this reader in its current membership, so it
654
+ * can still be acknowledged. A rejoin retires every earlier page.
655
+ */
656
+ owns(value) {
657
+ const tag = value[READER_TAG];
658
+ if (tag?.owner !== this.owner)
659
+ return false;
660
+ return this.progress.get(value.partitionId)?.knows(tag.sequence) ?? false;
661
+ }
662
+ /**
663
+ * Data connections this reader opened to partition primaries. A healthy
664
+ * reader opens one per node and keeps it: the count grows only when a node
665
+ * stops being primary, a connection fails, or the group session ends.
666
+ */
667
+ dataConnectionsOpened() {
668
+ return this.routes.opened;
669
+ }
670
+ /**
671
+ * Store completed progress, leave the group, and close the data
672
+ * connections. A round still running stops at its next step and opens
673
+ * nothing new.
674
+ */
675
+ async close() {
676
+ if (this.closed)
677
+ return;
678
+ this.closed = true;
679
+ this.buffered.length = 0;
680
+ let failure;
681
+ if (this.settings.request.readMode === "primary") {
682
+ for (const partitionId of [...this.progress.keys()]) {
683
+ try {
684
+ await this.flush(partitionId);
685
+ }
686
+ catch (error) {
687
+ failure ??= error;
688
+ }
689
+ }
690
+ }
691
+ try {
692
+ await this.settings.membership?.leave();
693
+ }
694
+ catch (error) {
695
+ failure ??= error;
696
+ }
697
+ await this.routes.close();
698
+ await this.settings.coordinator.close();
699
+ if (failure !== undefined) {
700
+ throw failure instanceof Error
701
+ ? failure
702
+ : new TransportError("closing the filtered reader failed", false, { cause: failure });
703
+ }
704
+ }
705
+ async [Symbol.asyncDispose]() {
706
+ await this.close();
707
+ }
708
+ async *[Symbol.asyncIterator]() {
709
+ for (;;)
710
+ yield await this.nextRecord();
711
+ }
712
+ // One read of every partition that is not waiting, in turn from where the
713
+ // last round stopped. A failing partition is deferred and the rest still
714
+ // read, so one partition never starves the others, and its error reaches
715
+ // the caller at most once per idle interval.
716
+ async round() {
717
+ this.requireOpen();
718
+ this.examined = 0;
719
+ if (this.deferredError !== undefined) {
720
+ const error = this.deferredError;
721
+ this.deferredError = undefined;
722
+ throw error instanceof Error
723
+ ? error
724
+ : new TransportError("a filtered partition failed", true, { cause: error });
725
+ }
726
+ if (this.settings.membership?.isDue() === true)
727
+ await this.refreshMembership();
728
+ if (this.listedAt !== undefined && performance.now() - this.listedAt >= ASSIGNMENT_REFRESH_MS) {
729
+ await this.refreshPartitions();
730
+ }
731
+ const readable = this.partitions();
732
+ if (readable.length === 0)
733
+ return { kind: "idle" };
734
+ const now = performance.now();
735
+ let busy = false;
736
+ let failure;
737
+ const saturated = [];
738
+ for (let step = 0; step < readable.length; step += 1) {
739
+ this.requireOpen();
740
+ const index = (this.cursor + step) % readable.length;
741
+ const partitionId = readable[index] ?? 0;
742
+ if (this.progress.get(partitionId)?.waiting(now) === true)
743
+ continue;
744
+ try {
745
+ const read = await this.read(partitionId);
746
+ if (read.kind === "page") {
747
+ this.cursor = index + 1;
748
+ this.deferredError = failure;
749
+ return read;
750
+ }
751
+ if (read.kind === "empty")
752
+ busy ||= read.more;
753
+ if (read.kind === "saturated")
754
+ saturated.push(partitionId);
755
+ }
756
+ catch (error) {
757
+ if (this.closed)
758
+ throw error;
759
+ this.progress.get(partitionId)?.defer(now + this.settings.idleIntervalMs);
760
+ failure ??= error;
761
+ }
762
+ }
763
+ if (failure !== undefined) {
764
+ throw failure instanceof Error
765
+ ? failure
766
+ : new TransportError("a filtered read failed", true, { cause: failure });
767
+ }
768
+ if (saturated.length > 0 && !busy) {
769
+ throw new InvalidError(`partitions ${saturated.join(", ")} each have ${String(this.settings.maxUnackedPages ?? DEFAULT_MAX_UNACKED_PAGES)} unacknowledged pages. Acknowledge before reading more`);
770
+ }
771
+ return { kind: busy ? "busy" : "idle" };
772
+ }
773
+ async read(partitionId) {
774
+ const progress = this.partitionProgress(partitionId);
775
+ if (progress.inFlight >= (this.settings.maxUnackedPages ?? DEFAULT_MAX_UNACKED_PAGES))
776
+ return { kind: "saturated" };
777
+ const request = {
778
+ ...this.settings.request,
779
+ partitionId,
780
+ start: progress.start
781
+ };
782
+ let outcome;
783
+ try {
784
+ outcome = await this.exchange(partitionId, AGDX_FILTERED_POLL_CODE, encodeFilteredPollRequest(request));
785
+ this.lostRoutes.delete(partitionId);
786
+ }
787
+ catch (error) {
788
+ if (isRouteLost(error)) {
789
+ // The round ends empty and the next one routes again, so the other
790
+ // partitions of this round never wait. A group follows its refreshed
791
+ // assignment. A reader whose route is lost twice in a row surfaces the refusal.
792
+ if (this.lostRoutes.has(partitionId))
793
+ throw error;
794
+ this.lostRoutes.add(partitionId);
795
+ this.settings.membership?.expire();
796
+ return { kind: "empty", more: false };
797
+ }
798
+ this.restartAfter(partitionId, error);
799
+ throw error;
800
+ }
801
+ this.requireOpen();
802
+ if (outcome.kind !== "page")
803
+ throw unexpected("filtered poll");
804
+ const page = outcome.page;
805
+ this.examined += page.examined;
806
+ if (page.records.byteLength > request.maxReplyBytes + 16)
807
+ throw new ProtocolError("filtered page exceeds the requested record byte limit");
808
+ if (page.records.byteLength < 16)
809
+ throw new ProtocolError("filtered record body is truncated");
810
+ const recordHead = new DataView(page.records.buffer, page.records.byteOffset, page.records.byteLength);
811
+ if (recordHead.getUint32(12, true) !== page.matched || page.matched > request.count)
812
+ throw new ProtocolError("filtered record body has invalid counts");
813
+ const messages = decodePolledBody(page.records);
814
+ validatePage(request, this.settings.inlineDigest, page, messages);
815
+ if (request.consumer.kind === "group_id") {
816
+ const pinned = this.pinnedSource;
817
+ const current = page.generation;
818
+ if (pinned !== undefined &&
819
+ (pinned.streamId !== current.streamId ||
820
+ pinned.streamCreatedAtMicros !== current.streamCreatedAtMicros ||
821
+ pinned.topicId !== current.topicId ||
822
+ pinned.topicCreatedAtMicros !== current.topicCreatedAtMicros))
823
+ throw filterError({
824
+ code: { kind: "known", name: "NotFound" },
825
+ reason: "not_found",
826
+ message: "the numeric consumer group belongs to a replaced stream or topic, build a new reader"
827
+ });
828
+ this.pinnedSource ??= current;
829
+ }
830
+ const unevaluated = new Set(page.unevaluated);
831
+ if (this.settings.guardEnabled && this.guard === undefined) {
832
+ this.guard = await compileGuard(this.settings.transport, await boundDefinition(this.settings.filters, request.source, page));
833
+ }
834
+ if (this.guard !== undefined) {
835
+ try {
836
+ checkGuard(this.guard, page, messages, unevaluated);
837
+ }
838
+ catch (error) {
839
+ this.restartAfter(partitionId, error);
840
+ throw error;
841
+ }
842
+ }
843
+ this.sequence += 1;
844
+ const sequence = this.sequence;
845
+ const primary = request.readMode === "primary";
846
+ // A local page cannot be acknowledged, so its records are not pending
847
+ // work and never saturate the partition.
848
+ progress.record(sequence, page, primary ? messages.map((message) => message.offset) : []);
849
+ const blocked = progress.blocked;
850
+ if (blocked !== undefined)
851
+ progress.defer(performance.now() + this.settings.idleIntervalMs);
852
+ if (messages.length === 0) {
853
+ if (primary) {
854
+ try {
855
+ await this.flush(partitionId);
856
+ }
857
+ catch {
858
+ // Kept unstored: the next acknowledgment of this partition sends it again.
859
+ }
860
+ }
861
+ if (blocked !== undefined) {
862
+ throw new FilterStopError(blocked.stop, partitionId, blocked.offset, blocked.reason);
863
+ }
864
+ return { kind: "empty", more: page.stop === "filled" || page.stop === "budget" };
865
+ }
866
+ const tag = { owner: this.owner, sequence };
867
+ const records = messages.map((message) => ({
868
+ json: () => jsonCodec((value) => value).decode(message.payload),
869
+ partitionId,
870
+ offset: message.offset,
871
+ frontier: page.frontier,
872
+ evaluated: !unevaluated.has(message.offset),
873
+ payload: message.payload,
874
+ headers: message.headers,
875
+ headersMalformed: message.headersMalformed !== undefined,
876
+ ...(message.timestampMicros !== undefined
877
+ ? { timestampMicros: message.timestampMicros }
878
+ : {}),
879
+ [READER_TAG]: tag
880
+ }));
881
+ const matched = {
882
+ partitionId,
883
+ records,
884
+ policy: page.policy,
885
+ generation: page.generation,
886
+ stop: page.stop,
887
+ examined: page.examined,
888
+ frontier: page.frontier,
889
+ ...(page.safeAckOffset !== undefined ? { safeAckOffset: page.safeAckOffset } : {}),
890
+ [READER_TAG]: tag
891
+ };
892
+ return { kind: "page", page: matched };
893
+ }
894
+ // A purge, a recreated group, or a new group binding ends the history or
895
+ // the policy the partition's continuation was issued for. The partition
896
+ // starts over from its stored offset, the guard reloads a rebound filter,
897
+ // and the error reaches the caller once.
898
+ restartAfter(partitionId, error) {
899
+ if (!(error instanceof FilterExecutionError) ||
900
+ (error.reason !== "source_changed" && error.reason !== "conflict")) {
901
+ return;
902
+ }
903
+ this.progress.get(partitionId)?.restart({ kind: "next" });
904
+ for (let index = this.buffered.length - 1; index >= 0; index -= 1) {
905
+ if (this.buffered[index]?.partitionId === partitionId)
906
+ this.buffered.splice(index, 1);
907
+ }
908
+ if (error.reason === "source_changed")
909
+ this.settings.membership?.expire();
910
+ const request = this.settings.request;
911
+ if (request.filter.kind === "bound")
912
+ this.guard = undefined;
913
+ }
914
+ // Store the completed prefix of one partition. A lost reply keeps the target
915
+ // unstored and the next store sends it again. Stores are monotone.
916
+ async flush(partitionId) {
917
+ const target = this.partitionProgress(partitionId).unstored;
918
+ if (target === undefined) {
919
+ this.retire(partitionId);
920
+ return;
921
+ }
922
+ const request = this.settings.request;
923
+ const ack = {
924
+ ...(target.groupId !== undefined ? { groupId: target.groupId } : {}),
925
+ v: FILTER_OP_VERSION,
926
+ source: request.source,
927
+ partitionId,
928
+ consumer: request.consumer,
929
+ generation: target.generation,
930
+ digest: target.digest,
931
+ offset: target.offset
932
+ };
933
+ const encoded = encodeFilteredAck(ack);
934
+ for (let rerouted = false;; rerouted = true) {
935
+ try {
936
+ const outcome = await this.exchange(partitionId, AGDX_FILTERED_ACK_CODE, encoded);
937
+ if (outcome.kind !== "acknowledged")
938
+ throw unexpected("filtered acknowledgment");
939
+ if (outcome.receipt.partitionId !== partitionId ||
940
+ outcome.receipt.offset !== target.offset ||
941
+ !sameGeneration(outcome.receipt.generation, target.generation)) {
942
+ throw new ProtocolError("filtered acknowledgment does not match its request");
943
+ }
944
+ this.progress.get(partitionId)?.stored(target);
945
+ const revoked = this.revoked.has(partitionId);
946
+ this.retire(partitionId);
947
+ if (revoked)
948
+ await this.routes.release(partitionId);
949
+ return;
950
+ }
951
+ catch (error) {
952
+ if (rerouted || !isRouteLost(error))
953
+ throw error;
954
+ }
955
+ }
956
+ }
957
+ async exchange(partitionId, code, request) {
958
+ const payload = encodeNamed(request);
959
+ let reply;
960
+ const read = this.settings.request;
961
+ if (read.readMode === "local") {
962
+ reply = await this.settings.coordinator.send(code, payload);
963
+ }
964
+ else {
965
+ const connection = await this.routes.connection(read.source, read.consumer, partitionId, code === AGDX_FILTERED_ACK_CODE);
966
+ try {
967
+ reply = await connection.send(code, payload);
968
+ }
969
+ catch (cause) {
970
+ await this.routes.invalidate(partitionId);
971
+ throw new TransportError(`filtered command ${String(code)} failed`, true, { cause });
972
+ }
973
+ }
974
+ const decoded = decodeFilterReply(decodeOne(reply, "filter reply"), "filter reply");
975
+ if (decoded.kind === "ok")
976
+ return decoded.outcome;
977
+ if (decoded.error.reason === "not_primary" || decoded.error.reason === "membership_stale") {
978
+ await this.routes.invalidate(partitionId);
979
+ }
980
+ throw filterError(decoded.error);
981
+ }
982
+ // Follow the group's current assignment. The whole change is applied to the
983
+ // reader before anything awaits. A partition that left the assignment stops
984
+ // being read but keeps its in-flight pages, so records already handed out
985
+ // can still be acknowledged while the server's offset fence allows it.
986
+ async refreshMembership() {
987
+ const membership = this.settings.membership;
988
+ if (membership === undefined || !(await membership.sync()))
989
+ return;
990
+ this.requireOpen();
991
+ const assigned = new Set(membership.partitions);
992
+ const rejoined = membership.takeRejoined();
993
+ const stale = rejoined ? this.routes.detach() : [];
994
+ if (rejoined) {
995
+ if (this.settings.request.filter.kind === "bound")
996
+ this.guard = undefined;
997
+ this.retiredOwners.add(this.owner);
998
+ this.owner = {};
999
+ this.progress.clear();
1000
+ this.revoked.clear();
1001
+ this.buffered.length = 0;
1002
+ }
1003
+ const left = [...this.progress.keys()].filter((partitionId) => !assigned.has(partitionId));
1004
+ for (const partitionId of left) {
1005
+ this.revoked.add(partitionId);
1006
+ this.retire(partitionId);
1007
+ }
1008
+ // A partition gained on a rebalance resumes after the group's stored
1009
+ // offset, which holds what the previous owner acknowledged. One that
1010
+ // comes back before its revoked entry retired starts over the same way,
1011
+ // so it never replays another member's work from an old continuation.
1012
+ for (const partitionId of assigned) {
1013
+ if (this.revoked.delete(partitionId) || !this.progress.has(partitionId)) {
1014
+ for (let index = this.buffered.length - 1; index >= 0; index--) {
1015
+ if (this.buffered[index]?.partitionId === partitionId)
1016
+ this.buffered.splice(index, 1);
1017
+ }
1018
+ this.progress.set(partitionId, new PartitionProgress({ kind: "next" }));
1019
+ }
1020
+ }
1021
+ await Promise.all(stale.map((connection) => connection.close()));
1022
+ for (const partitionId of left)
1023
+ await this.routes.release(partitionId);
1024
+ }
1025
+ // An independent reader of every partition picks up partitions added to
1026
+ // the topic since the build, resuming their stored progress.
1027
+ async refreshPartitions() {
1028
+ const listed = await topicPartitions(this.settings.transport, this.settings.request.source);
1029
+ this.listedAt = performance.now();
1030
+ for (const partitionId of listed) {
1031
+ if (!this.progress.has(partitionId)) {
1032
+ this.progress.set(partitionId, new PartitionProgress({ kind: "next" }));
1033
+ }
1034
+ }
1035
+ }
1036
+ // Forget a revoked partition once nothing of it is in flight.
1037
+ retire(partitionId) {
1038
+ const progress = this.progress.get(partitionId);
1039
+ if (this.revoked.has(partitionId) &&
1040
+ progress?.inFlight === 0 &&
1041
+ progress.unstored === undefined) {
1042
+ this.progress.delete(partitionId);
1043
+ this.revoked.delete(partitionId);
1044
+ }
1045
+ }
1046
+ partitionProgress(partitionId) {
1047
+ const progress = this.progress.get(partitionId);
1048
+ if (progress === undefined) {
1049
+ throw filterError({
1050
+ code: { kind: "known", name: "Stale" },
1051
+ reason: "membership_stale",
1052
+ message: `partition ${String(partitionId)} is no longer read by this reader. Its unacknowledged records are read again`
1053
+ });
1054
+ }
1055
+ return progress;
1056
+ }
1057
+ // The owner is checked before the partition, so a foreign record reads as
1058
+ // foreign, and a record of this reader's earlier membership reads as retired.
1059
+ requireSequence(value) {
1060
+ const sequence = this.sequenceOf(value);
1061
+ if (!this.partitionProgress(value.partitionId).knows(sequence))
1062
+ throw filterError({
1063
+ code: { kind: "known", name: "Stale" },
1064
+ reason: "membership_stale",
1065
+ message: "the acknowledgment belongs to a retired partition read"
1066
+ });
1067
+ return sequence;
1068
+ }
1069
+ sequenceOf(value) {
1070
+ const tag = value[READER_TAG];
1071
+ if (tag?.owner === this.owner)
1072
+ return tag.sequence;
1073
+ if (tag !== undefined && this.retiredOwners.has(tag.owner)) {
1074
+ throw new InvalidError("the page or record was read before this reader rejoined its group, so it can no longer be acknowledged");
1075
+ }
1076
+ throw new InvalidError("the page or record belongs to another filtered reader");
1077
+ }
1078
+ requireOpen() {
1079
+ if (this.closed)
1080
+ throw new ConfigError("the filtered reader is closed");
1081
+ }
1082
+ requireAcknowledgments() {
1083
+ this.requireOpen();
1084
+ if (this.settings.request.readMode === "local") {
1085
+ throw new InvalidError("a local read mode reader cannot acknowledge: only the partition primary proves the history a page came from");
1086
+ }
1087
+ }
1088
+ }
1089
+ /** One partition's read position and acknowledgment state. */
1090
+ class PartitionProgress {
1091
+ original;
1092
+ current;
1093
+ pages = [];
1094
+ pendingStore;
1095
+ stoppedAt;
1096
+ retryAt;
1097
+ firstSequence;
1098
+ settledSequence = 0;
1099
+ constructor(original) {
1100
+ this.original = original;
1101
+ this.current = original;
1102
+ }
1103
+ get start() {
1104
+ return this.current;
1105
+ }
1106
+ get blocked() {
1107
+ return this.stoppedAt;
1108
+ }
1109
+ get inFlight() {
1110
+ return this.pages.length;
1111
+ }
1112
+ get unstored() {
1113
+ return this.pendingStore;
1114
+ }
1115
+ /**
1116
+ * Start over at `start` after the source history or the executed filter
1117
+ * changed. Every page in flight is dropped, so its records are delivered
1118
+ * again from the stored offset.
1119
+ */
1120
+ restart(start) {
1121
+ this.original = start;
1122
+ this.current = start;
1123
+ this.pages = [];
1124
+ this.pendingStore = undefined;
1125
+ this.stoppedAt = undefined;
1126
+ this.retryAt = undefined;
1127
+ this.firstSequence = undefined;
1128
+ this.settledSequence = 0;
1129
+ }
1130
+ knows(sequence) {
1131
+ return (this.pages.some((page) => page.sequence === sequence) ||
1132
+ (this.firstSequence !== undefined &&
1133
+ sequence >= this.firstSequence &&
1134
+ sequence <= this.settledSequence));
1135
+ }
1136
+ /** Whether the partition waits out a delay after an error or a block. */
1137
+ waiting(now) {
1138
+ return this.retryAt !== undefined && now < this.retryAt;
1139
+ }
1140
+ defer(until) {
1141
+ this.retryAt = until;
1142
+ }
1143
+ /**
1144
+ * Track a page and its returned `offsets`, and continue after it. A blocked
1145
+ * partition is read again after its delay, so a block ends once the record
1146
+ * ages out of the partition or the policy stops faulting.
1147
+ */
1148
+ record(sequence, page, offsets) {
1149
+ this.retryAt = undefined;
1150
+ this.firstSequence ??= sequence;
1151
+ this.current = nextFilteredStart(page, this.original);
1152
+ this.stoppedAt =
1153
+ page.fault === undefined
1154
+ ? undefined
1155
+ : page.stop === "fault"
1156
+ ? { stop: "fault", offset: page.fault.offset, reason: page.fault.reason }
1157
+ : page.stop === "oversized_record"
1158
+ ? { stop: "oversized_record", offset: page.fault.offset }
1159
+ : undefined;
1160
+ const target = page.safeAckOffset === undefined
1161
+ ? undefined
1162
+ : {
1163
+ offset: page.safeAckOffset,
1164
+ generation: page.generation,
1165
+ digest: page.policy.digest,
1166
+ ...(page.policy.groupId !== undefined ? { groupId: page.policy.groupId } : {})
1167
+ };
1168
+ const newest = this.pages.at(-1);
1169
+ if (offsets.length === 0 && newest !== undefined) {
1170
+ if (target !== undefined)
1171
+ newest.target = target;
1172
+ }
1173
+ else {
1174
+ this.pages.push({ sequence, target, outstanding: new Set(offsets) });
1175
+ }
1176
+ this.settle();
1177
+ }
1178
+ completeRecord(sequence, offset) {
1179
+ this.pages.find((page) => page.sequence === sequence)?.outstanding.delete(offset);
1180
+ this.settle();
1181
+ }
1182
+ completeThrough(sequence, offset) {
1183
+ const target = this.pages.find((page) => page.sequence === sequence)?.target;
1184
+ if (target === undefined)
1185
+ return true;
1186
+ if (offset > target.offset)
1187
+ return false;
1188
+ for (const page of this.pages) {
1189
+ if (page.sequence < sequence)
1190
+ page.outstanding.clear();
1191
+ else if (page.sequence === sequence) {
1192
+ for (const pending of page.outstanding)
1193
+ if (pending <= offset)
1194
+ page.outstanding.delete(pending);
1195
+ }
1196
+ }
1197
+ this.settle();
1198
+ if (this.pendingStore === undefined || this.pendingStore.offset < offset) {
1199
+ this.pendingStore = { ...target, offset };
1200
+ }
1201
+ return true;
1202
+ }
1203
+ completePage(sequence) {
1204
+ this.pages.find((page) => page.sequence === sequence)?.outstanding.clear();
1205
+ this.settle();
1206
+ }
1207
+ stored(target) {
1208
+ if (this.pendingStore === target)
1209
+ this.pendingStore = undefined;
1210
+ }
1211
+ // Retire the completed prefix. Its newest safe offset supersedes any older
1212
+ // unstored one, because a stored offset covers everything before it.
1213
+ settle() {
1214
+ while (this.pages[0]?.outstanding.size === 0) {
1215
+ const page = this.pages.shift();
1216
+ if (page !== undefined)
1217
+ this.settledSequence = page.sequence;
1218
+ if (page?.target !== undefined)
1219
+ this.pendingStore = page.target;
1220
+ }
1221
+ }
1222
+ }
1223
+ /**
1224
+ * Data connections to partition primaries, one per node endpoint, each
1225
+ * attached to the coordinator's consumer session. The coordinator keeps the
1226
+ * session, the user, and any group membership.
1227
+ */
1228
+ class Routes {
1229
+ transport;
1230
+ coordinator;
1231
+ connections = new Map();
1232
+ // One open per endpoint at a time, so two routes to one node share it.
1233
+ opening = new Map();
1234
+ partitions = new Map();
1235
+ closed = false;
1236
+ /** Data connections these routes opened. */
1237
+ opened = 0;
1238
+ constructor(transport, coordinator) {
1239
+ this.transport = transport;
1240
+ this.coordinator = coordinator;
1241
+ }
1242
+ async connection(source, consumer, partitionId, acknowledgment) {
1243
+ const routed = this.partitions.get(partitionId);
1244
+ const known = routed === undefined ? undefined : this.connections.get(routed);
1245
+ if (known !== undefined)
1246
+ return known.connection;
1247
+ const open = this.transport.openNodeConnection;
1248
+ if (open === undefined) {
1249
+ throw new ConfigError("primary filtered reads need a transport that can open node connections");
1250
+ }
1251
+ const route = decodePollRouting(await this.coordinator.send(acknowledgment ? GET_CONSUMER_OFFSET_ROUTING_CODE : GET_POLL_ROUTING_CODE, acknowledgment
1252
+ ? encodeOffsetRouting(source, consumer, partitionId)
1253
+ : encodePollRouting(source, consumer, partitionId)));
1254
+ this.requireOpen();
1255
+ const endpoint = `${route.ip}:${String(route.tcpPort)}`;
1256
+ const existing = this.connections.get(endpoint);
1257
+ const reuse = existing === undefined ? undefined : sessionReuse(existing.session, route.session);
1258
+ if (existing !== undefined && reuse === "raise") {
1259
+ try {
1260
+ await existing.connection.send(ATTACH_CONSUMER_SESSION_CODE, route.session);
1261
+ existing.session = route.session;
1262
+ }
1263
+ catch {
1264
+ await this.dropEndpoint(endpoint);
1265
+ }
1266
+ }
1267
+ else if (reuse === "replace") {
1268
+ await this.dropEndpoint(endpoint);
1269
+ }
1270
+ if (!this.connections.has(endpoint)) {
1271
+ if (route.tcpPort === 0) {
1272
+ throw new UnsupportedError(`partition primary ${route.name} serves no TCP endpoint`);
1273
+ }
1274
+ let pending = this.opening.get(endpoint);
1275
+ if (pending === undefined) {
1276
+ if (this.connections.size >= MAX_NODE_CONNECTIONS) {
1277
+ const idle = [...this.connections.keys()].find((candidate) => ![...this.partitions.values()].includes(candidate));
1278
+ const oldest = idle ?? this.connections.keys().next().value;
1279
+ if (oldest !== undefined)
1280
+ await this.dropEndpoint(oldest);
1281
+ }
1282
+ pending = this.opening.get(endpoint);
1283
+ if (pending === undefined && !this.connections.has(endpoint)) {
1284
+ pending = attach(open.bind(this.transport), route).then((connection) => {
1285
+ this.opened += 1;
1286
+ return connection;
1287
+ });
1288
+ this.opening.set(endpoint, pending);
1289
+ }
1290
+ }
1291
+ if (pending !== undefined) {
1292
+ let connection;
1293
+ try {
1294
+ connection = await pending;
1295
+ }
1296
+ finally {
1297
+ if (this.opening.get(endpoint) === pending)
1298
+ this.opening.delete(endpoint);
1299
+ }
1300
+ if (this.closed) {
1301
+ await connection.close();
1302
+ this.requireOpen();
1303
+ }
1304
+ if (!this.connections.has(endpoint))
1305
+ this.connections.set(endpoint, { connection, session: route.session });
1306
+ }
1307
+ }
1308
+ this.partitions.set(partitionId, endpoint);
1309
+ const attached = this.connections.get(endpoint);
1310
+ if (attached === undefined)
1311
+ throw new ProtocolError("the partition route vanished");
1312
+ return attached.connection;
1313
+ }
1314
+ /**
1315
+ * Stop routing `partitionId`, for a partition this reader no longer reads.
1316
+ * The node connection stays open for the other partitions it serves and
1317
+ * closes only when none is left.
1318
+ */
1319
+ async release(partitionId) {
1320
+ const endpoint = this.partitions.get(partitionId);
1321
+ if (endpoint === undefined)
1322
+ return;
1323
+ this.partitions.delete(partitionId);
1324
+ if (![...this.partitions.values()].includes(endpoint))
1325
+ await this.dropEndpoint(endpoint);
1326
+ }
1327
+ /**
1328
+ * Route `partitionId` again on its next use and drop the connection it
1329
+ * used, because that node no longer serves it as primary or the session
1330
+ * behind the attachment has ended.
1331
+ */
1332
+ async invalidate(partitionId) {
1333
+ const endpoint = this.partitions.get(partitionId);
1334
+ this.partitions.delete(partitionId);
1335
+ if (endpoint !== undefined)
1336
+ await this.dropEndpoint(endpoint);
1337
+ }
1338
+ /** Forget every route and hand back the connections, which the caller closes. */
1339
+ detach() {
1340
+ this.partitions.clear();
1341
+ const connections = [...this.connections.values()].map((entry) => entry.connection);
1342
+ this.connections.clear();
1343
+ return connections;
1344
+ }
1345
+ /** Close every connection. No connection opens after this. */
1346
+ async close() {
1347
+ this.closed = true;
1348
+ await Promise.all(this.detach().map((connection) => connection.close()));
1349
+ }
1350
+ // The routing table forgets the endpoint before the close awaits.
1351
+ async dropEndpoint(endpoint) {
1352
+ for (const [partitionId, routed] of this.partitions) {
1353
+ if (routed === endpoint)
1354
+ this.partitions.delete(partitionId);
1355
+ }
1356
+ const entry = this.connections.get(endpoint);
1357
+ this.connections.delete(endpoint);
1358
+ await entry?.connection.close();
1359
+ }
1360
+ requireOpen() {
1361
+ if (this.closed)
1362
+ throw new ConfigError("the filtered reader is closed");
1363
+ }
1364
+ }
1365
+ async function attach(open, route) {
1366
+ const connection = await open(route.ip, route.tcpPort);
1367
+ try {
1368
+ await connection.send(ATTACH_CONSUMER_SESSION_CODE, route.session);
1369
+ }
1370
+ catch (cause) {
1371
+ await connection.close();
1372
+ throw new TransportError("attaching the consumer session failed", true, { cause });
1373
+ }
1374
+ return connection;
1375
+ }
1376
+ // The shared transport as a coordinator, for a reader that has no connection
1377
+ // of its own: an injected client, or an independent reader.
1378
+ function sharedCoordinator(transport) {
1379
+ return {
1380
+ send: (code, payload) => transport.sendManaged(code, payload),
1381
+ joinConsumerGroup: (streamId, topicId, name) => transport.joinExistingConsumerGroup !== undefined
1382
+ ? transport.joinExistingConsumerGroup(streamId, topicId, name)
1383
+ : typeof name === "string"
1384
+ ? transport.joinConsumerGroup(streamId, topicId, name)
1385
+ : Promise.reject(new ConfigError("this transport does not support numeric group membership")),
1386
+ leaveConsumerGroup: (streamId, topicId, name) => transport.leaveConsumerGroup(streamId, topicId, name),
1387
+ close: () => Promise.resolve()
1388
+ };
1389
+ }
1390
+ async function topicPartitions(transport, source) {
1391
+ const count = await transport.getTopicPartitionCount(source.stream, source.topic);
1392
+ return Array.from({ length: count }, (_, index) => index);
1393
+ }
1394
+ /** This reader's membership of a consumer group, held by its coordinator connection. */
1395
+ class Membership {
1396
+ coordinator;
1397
+ source;
1398
+ group;
1399
+ transport;
1400
+ pinnedSource;
1401
+ sourceReplaced = false;
1402
+ generation = 0n;
1403
+ assigned = [];
1404
+ syncedAt;
1405
+ rejoined = false;
1406
+ request;
1407
+ constructor(coordinator, source, group, transport, pinnedSource) {
1408
+ this.coordinator = coordinator;
1409
+ this.source = source;
1410
+ this.group = group;
1411
+ this.transport = transport;
1412
+ this.pinnedSource = pinnedSource;
1413
+ this.request = concat([identifier(source.stream), identifier(source.topic), identifier(group)]);
1414
+ }
1415
+ static async join(coordinator, source, group, transport, pinnedSource) {
1416
+ const membership = new Membership(coordinator, source, group, transport, pinnedSource);
1417
+ await coordinator.joinConsumerGroup(source.stream, source.topic, group);
1418
+ await membership.sync();
1419
+ return membership;
1420
+ }
1421
+ get partitions() {
1422
+ return this.assigned;
1423
+ }
1424
+ isDue() {
1425
+ return this.syncedAt === undefined || performance.now() - this.syncedAt >= ASSIGNMENT_REFRESH_MS;
1426
+ }
1427
+ expire() {
1428
+ this.syncedAt = undefined;
1429
+ }
1430
+ /** Read the current assignment. `true` when it changed. */
1431
+ async sync() {
1432
+ let reply = await this.coordinator.send(SYNC_CONSUMER_GROUP_CODE, this.request);
1433
+ if (reply.byteLength === 0) {
1434
+ if (this.pinnedSource !== undefined) {
1435
+ const current = await sourceIncarnation(this.transport, this.source);
1436
+ if (current.streamId !== this.pinnedSource.streamId ||
1437
+ current.streamCreatedAtMicros !== this.pinnedSource.streamCreatedAtMicros ||
1438
+ current.topicId !== this.pinnedSource.topicId ||
1439
+ current.topicCreatedAtMicros !== this.pinnedSource.topicCreatedAtMicros) {
1440
+ this.sourceReplaced = true;
1441
+ await this.coordinator
1442
+ .leaveConsumerGroup(this.source.stream, this.source.topic, this.group)
1443
+ .catch(() => undefined);
1444
+ throw filterError({
1445
+ code: { kind: "known", name: "NotFound" },
1446
+ reason: "not_found",
1447
+ message: "the numeric consumer group belongs to a replaced stream or topic"
1448
+ });
1449
+ }
1450
+ }
1451
+ await this.coordinator.joinConsumerGroup(this.source.stream, this.source.topic, this.group);
1452
+ this.rejoined = true;
1453
+ reply = await this.coordinator.send(SYNC_CONSUMER_GROUP_CODE, this.request);
1454
+ if (reply.byteLength === 0)
1455
+ throw filterError({
1456
+ code: { kind: "known", name: "Stale" },
1457
+ reason: "membership_stale",
1458
+ message: "the consumer group has not accepted this reader's rejoin"
1459
+ });
1460
+ }
1461
+ this.syncedAt = performance.now();
1462
+ const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
1463
+ if (reply.byteLength < 12)
1464
+ throw new ProtocolError("the group assignment is truncated");
1465
+ const generation = view.getBigUint64(0, true);
1466
+ const count = view.getUint32(8, true);
1467
+ if (count > (reply.byteLength - 12) / 4 || reply.byteLength !== 12 + count * 4)
1468
+ throw new ProtocolError("the group assignment length is invalid");
1469
+ const partitions = Array.from({ length: count }, (_, index) => view.getUint32(12 + index * 4, true));
1470
+ const changed = this.rejoined ||
1471
+ generation !== this.generation ||
1472
+ partitions.length !== this.assigned.length ||
1473
+ partitions.some((partitionId, index) => partitionId !== this.assigned[index]);
1474
+ this.generation = generation;
1475
+ this.assigned = partitions;
1476
+ return changed;
1477
+ }
1478
+ takeRejoined() {
1479
+ const rejoined = this.rejoined;
1480
+ this.rejoined = false;
1481
+ return rejoined;
1482
+ }
1483
+ async leave() {
1484
+ if (this.sourceReplaced)
1485
+ return;
1486
+ await this.coordinator.leaveConsumerGroup(this.source.stream, this.source.topic, this.group);
1487
+ }
1488
+ }
1489
+ async function sourceIncarnation(transport, source) {
1490
+ // Native metadata responses begin with id:u32 and created_at:u64. Reading
1491
+ // the timestamp directly preserves microseconds that the Node SDK's Date loses.
1492
+ const read = (bytes) => {
1493
+ if (bytes.length < 12)
1494
+ throw new ProtocolError("source metadata is absent or truncated");
1495
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
1496
+ return { id: view.getUint32(0, true), created: view.getBigUint64(4, true) };
1497
+ };
1498
+ const stream = read(await transport.sendManaged(GET_STREAM_CODE, identifier(source.stream)));
1499
+ const topic = read(await transport.sendManaged(GET_TOPIC_CODE, concat([identifier(stream.id), identifier(source.topic)])));
1500
+ const current = read(await transport.sendManaged(GET_STREAM_CODE, identifier(stream.id)));
1501
+ if (current.created !== stream.created)
1502
+ throw new ProtocolError("source stream changed while building the reader");
1503
+ return {
1504
+ streamId: stream.id,
1505
+ streamCreatedAtMicros: stream.created,
1506
+ topicId: topic.id,
1507
+ topicCreatedAtMicros: topic.created
1508
+ };
1509
+ }
1510
+ function identifier(name) {
1511
+ if (typeof name === "number") {
1512
+ const encoded = new Uint8Array(6);
1513
+ encoded.set([1, 4]);
1514
+ new DataView(encoded.buffer).setUint32(2, name, true);
1515
+ return encoded;
1516
+ }
1517
+ const bytes = UTF8.encode(name);
1518
+ return concat([Uint8Array.of(2, bytes.byteLength), bytes]);
1519
+ }
1520
+ // The stored-offset lookup shape, which routes an acknowledgment to the
1521
+ // partition primary that stores the offset.
1522
+ function encodeOffsetRouting(source, consumer, partitionId) {
1523
+ const partition = new Uint8Array(5);
1524
+ const view = new DataView(partition.buffer);
1525
+ view.setUint8(0, 1);
1526
+ view.setUint32(1, partitionId, true);
1527
+ return concat([
1528
+ Uint8Array.of(consumer.kind !== "consumer" ? 2 : 1),
1529
+ identifier(consumer.kind === "group_id" ? Number(consumer.id) : consumer.name),
1530
+ identifier(source.stream),
1531
+ identifier(source.topic),
1532
+ partition
1533
+ ]);
1534
+ }
1535
+ // A committing poll shape, which the coordinator requires before it answers
1536
+ // with a route. It routes and reads nothing.
1537
+ function encodePollRouting(source, consumer, partitionId) {
1538
+ const tail = new Uint8Array(1 + 4 + 1 + 8 + 4 + 1);
1539
+ const view = new DataView(tail.buffer);
1540
+ view.setUint8(0, 1);
1541
+ view.setUint32(1, partitionId, true);
1542
+ view.setUint8(5, POLLING_NEXT);
1543
+ view.setBigUint64(6, 0n, true);
1544
+ view.setUint32(14, 1, true);
1545
+ view.setUint8(18, 1);
1546
+ return concat([
1547
+ Uint8Array.of(consumer.kind !== "consumer" ? 2 : 1),
1548
+ identifier(consumer.kind === "group_id" ? Number(consumer.id) : consumer.name),
1549
+ identifier(source.stream),
1550
+ identifier(source.topic),
1551
+ tail
1552
+ ]);
1553
+ }
1554
+ function decodePollRouting(reply) {
1555
+ const truncated = () => {
1556
+ throw new ProtocolError("the partition route is truncated");
1557
+ };
1558
+ const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
1559
+ const decoder = new TextDecoder();
1560
+ let position = CONSUMER_SESSION_BYTES;
1561
+ const text = () => {
1562
+ if (position + 4 > reply.byteLength)
1563
+ truncated();
1564
+ const length = view.getUint32(position, true);
1565
+ position += 4;
1566
+ if (position + length > reply.byteLength)
1567
+ truncated();
1568
+ const value = decoder.decode(reply.subarray(position, position + length));
1569
+ position += length;
1570
+ return value;
1571
+ };
1572
+ if (reply.byteLength < CONSUMER_SESSION_BYTES)
1573
+ truncated();
1574
+ const name = text();
1575
+ const ip = text();
1576
+ if (reply.byteLength < position + 10)
1577
+ truncated();
1578
+ return {
1579
+ session: reply.slice(0, CONSUMER_SESSION_BYTES),
1580
+ name,
1581
+ ip,
1582
+ tcpPort: view.getUint16(position, true)
1583
+ };
1584
+ }
1585
+ function concat(parts) {
1586
+ const total = parts.reduce((sum, part) => sum + part.byteLength, 0);
1587
+ const out = new Uint8Array(total);
1588
+ let offset = 0;
1589
+ for (const part of parts) {
1590
+ out.set(part, offset);
1591
+ offset += part.byteLength;
1592
+ }
1593
+ return out;
1594
+ }
1595
+ /**
1596
+ * What a fresh route means for the data connection already open to its node.
1597
+ *
1598
+ * A connection stays open for the life of its session. The first 24 bytes
1599
+ * name the session, its client id and number. The metadata watermark after
1600
+ * them grows with every write the coordinator commits, and that alone never
1601
+ * justifies a new connection: the attachment is sent again on the open one,
1602
+ * which only raises its read-your-writes floor.
1603
+ */
1604
+ export function sessionReuse(open, routed) {
1605
+ const identity = (bytes) => bytes.subarray(0, SESSION_IDENTITY_BYTES);
1606
+ if (!sameBytes(identity(open), identity(routed)))
1607
+ return "replace";
1608
+ const watermark = (bytes) => new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getBigUint64(SESSION_IDENTITY_BYTES, true);
1609
+ return watermark(routed) > watermark(open) ? "raise" : "keep";
1610
+ }
1611
+ function sameBytes(left, right) {
1612
+ return left.byteLength === right.byteLength && left.every((byte, index) => byte === right[index]);
1613
+ }
1614
+ // Re-evaluate every returned record with the shared evaluator before the
1615
+ // caller sees it.
1616
+ function checkGuard(guard, page, messages, unevaluated) {
1617
+ if (!sameBytes(page.policy.digest, guard.digest)) {
1618
+ throw filterError({
1619
+ code: { kind: "known", name: "Conflict" },
1620
+ reason: "conflict",
1621
+ message: "the server ran another filter than the one the local guard checks"
1622
+ });
1623
+ }
1624
+ const filter = guard.filter;
1625
+ if (unevaluated.size > 0 &&
1626
+ filter.faultPolicy !== "pass" &&
1627
+ filter.foreignPolicy !== "pass" &&
1628
+ filter.mismatchPolicy !== "pass") {
1629
+ throw filterError({
1630
+ code: { kind: "known", name: "Backend" },
1631
+ reason: "backend",
1632
+ message: "the server returned unevaluated records without a pass policy"
1633
+ });
1634
+ }
1635
+ const limits = page.evaluationLimits ?? GUARD_LIMITS;
1636
+ if (limits.maxPayloadBytes <= 0 ||
1637
+ limits.maxPayloadBytes > MAX_FILTERED_PAGE_BYTES ||
1638
+ limits.maxDepth <= 0 ||
1639
+ limits.maxDepth > MAX_FILTER_PARSE_DEPTH) {
1640
+ throw filterError({
1641
+ code: { kind: "known", name: "Backend" },
1642
+ reason: "backend",
1643
+ message: "the server returned invalid decoder limits"
1644
+ });
1645
+ }
1646
+ if (unevaluated.size > 0 &&
1647
+ filter.faultPolicy === "pass" &&
1648
+ page.evaluationLimits === undefined) {
1649
+ throw new UnsupportedError("local verification of pass-through decode faults requires the server's evaluation limits");
1650
+ }
1651
+ for (const message of messages) {
1652
+ // The server decodes only the headers the filter needs, so a broken part
1653
+ // it never reads does not fault the record there either.
1654
+ const outcome = headersFault(guard, message.headersMalformed)
1655
+ ? { verdict: "fault", fault: "malformed" }
1656
+ : guard.outcomeOf({
1657
+ payload: message.payload,
1658
+ headers: guard.needsHeaders ? filterHeaders(message.headers) : []
1659
+ }, limits);
1660
+ const agrees = unevaluated.has(message.offset)
1661
+ ? outcome.verdict === "fault" &&
1662
+ outcome.fault !== undefined &&
1663
+ guard.policyFor(outcome.fault) === "pass"
1664
+ : outcome.verdict === "selected";
1665
+ if (!agrees) {
1666
+ throw filterError({
1667
+ code: { kind: "known", name: "Backend" },
1668
+ reason: "backend",
1669
+ message: `the server's evaluation of offset ${message.offset.toString()} on partition ${String(page.partitionId)} disagrees with the local guard`
1670
+ });
1671
+ }
1672
+ }
1673
+ }
1674
+ // Whether the part of the header block that did not decode is one the
1675
+ // server reads for this filter: every entry for header predicates, only the
1676
+ // structure and `agdx.ct` for a payload filter, nothing otherwise.
1677
+ function headersFault(guard, fault) {
1678
+ if (fault === undefined)
1679
+ return false;
1680
+ if (guard.readsHeaders)
1681
+ return true;
1682
+ return guard.needsHeaders && fault !== "entry";
1683
+ }
1684
+ // The typed user headers, mapped exactly as the server maps them.
1685
+ function filterHeaders(headers) {
1686
+ return [...headers].map(([key, header]) => {
1687
+ switch (header.kind) {
1688
+ case "bool":
1689
+ return { key, value: { kind: "bool", value: header.value } };
1690
+ case "string":
1691
+ return { key, value: { kind: "string", value: header.value } };
1692
+ case "int8":
1693
+ case "int16":
1694
+ case "int32":
1695
+ return { key, value: { kind: "int", value: BigInt(header.value) } };
1696
+ case "int64":
1697
+ return { key, value: { kind: "int", value: header.value } };
1698
+ case "uint8":
1699
+ case "uint16":
1700
+ case "uint32":
1701
+ return { key, value: { kind: "uint", value: BigInt(header.value) } };
1702
+ case "uint64":
1703
+ return { key, value: { kind: "uint", value: header.value } };
1704
+ case "float":
1705
+ case "double":
1706
+ return { key, value: { kind: "float", value: header.value } };
1707
+ case "raw":
1708
+ case "int128":
1709
+ case "uint128":
1710
+ return { key, value: { kind: "raw", value: header.value } };
1711
+ }
1712
+ });
1713
+ }
1714
+ async function compileGuard(transport, filter) {
1715
+ if (usesRegex(filter.expr)) {
1716
+ throw new ConfigError("the TypeScript local guard cannot verify regex predicates, disable localGuard or use Rust or Python");
1717
+ }
1718
+ const schemas = [];
1719
+ for (const id of filter.schemaRefs) {
1720
+ const bytes = await transport.sendManaged(AGDX_GET_SCHEMA_CODE, encodeNamed(encodeGetSchema({ v: QUERY_OP_VERSION, id })));
1721
+ const reply = decodeBrowseReply(decodeOne(bytes, "writer schema"), "writer schema");
1722
+ if (reply.kind === "err")
1723
+ throw new QueryExecutionError("writer schema lookup failed", reply.error);
1724
+ if (reply.kind !== "ok" ||
1725
+ reply.outcome.kind !== "schema" ||
1726
+ reply.outcome.schema?.schema.id !== id) {
1727
+ throw new ProtocolError(`writer schema ${String(id)} was not returned`);
1728
+ }
1729
+ schemas.push(reply.outcome.schema.schema);
1730
+ }
1731
+ const compiled = CompiledFilter.compile(filter, schemas);
1732
+ return compiled;
1733
+ }
1734
+ // The full definition a reader executes, for the local guard.
1735
+ async function boundDefinition(filters, requested, page) {
1736
+ const source = page.generation;
1737
+ for (let index = 0;; index += 1) {
1738
+ const bindings = await filters.bindings({
1739
+ stream: requested.stream,
1740
+ topic: requested.topic,
1741
+ page: index,
1742
+ pageSize: MAX_FILTER_CATALOG_PAGE
1743
+ });
1744
+ const binding = bindings.items.find(({ identity }) => identity.groupId === page.policy.groupId &&
1745
+ identity.streamId === source.streamId &&
1746
+ identity.streamCreatedAtMicros === source.streamCreatedAtMicros &&
1747
+ identity.topicId === source.topicId &&
1748
+ identity.topicCreatedAtMicros === source.topicCreatedAtMicros);
1749
+ if (binding !== undefined) {
1750
+ if (!sameBytes(binding.digest, page.policy.digest))
1751
+ throw new ProtocolError("served policy differs from the group's catalog binding");
1752
+ return definition(filters, {
1753
+ kind: "revision",
1754
+ filterId: binding.filterId,
1755
+ revision: binding.revision
1756
+ });
1757
+ }
1758
+ if ((index + 1) * MAX_FILTER_CATALOG_PAGE >= bindings.total)
1759
+ throw filterError({
1760
+ code: { kind: "known", name: "NotFound" },
1761
+ reason: "not_found",
1762
+ message: "no readable catalog binding matches this group identity"
1763
+ });
1764
+ }
1765
+ }
1766
+ async function definition(filters, filter) {
1767
+ if (filter.kind === "inline")
1768
+ return filter.filter;
1769
+ if (filter.kind === "bound")
1770
+ throw new ConfigError("resolve a bound guard from the served page");
1771
+ const { filterId, revision } = filter;
1772
+ for (let page = 0;; page += 1) {
1773
+ const revisions = await filters.revisions(filterId, { page, pageSize: MAX_FILTER_CATALOG_PAGE });
1774
+ const found = revisions.items.find((info) => info.revision === revision);
1775
+ if (found !== undefined)
1776
+ return found.filter;
1777
+ if ((page + 1) * MAX_FILTER_CATALOG_PAGE >= revisions.total) {
1778
+ throw filterError({
1779
+ code: { kind: "known", name: "NotFound" },
1780
+ reason: "not_found",
1781
+ message: `filter ${String(filterId)} has no revision ${String(revision)}`
1782
+ });
1783
+ }
1784
+ }
1785
+ }
1786
+ function settled(operationId, status) {
1787
+ if (status.kind === "applied")
1788
+ return status.result;
1789
+ if (status.kind === "rejected")
1790
+ throw filterError(status.error);
1791
+ throw new AmbiguousMutationError(`filter mutation ${operationId.toString()} has no outcome yet`);
1792
+ }
1793
+ function filterError(error) {
1794
+ if (error.reason === "unsupported")
1795
+ return new UnsupportedError(error.message);
1796
+ return new FilterExecutionError(`${error.reason}: ${error.message}`, error);
1797
+ }
1798
+ // The data connection no longer reaches the partition primary for this
1799
+ // consumer: the primary moved, the attached session ended, or the group
1800
+ // assignment changed, which the coordinator reports as a partition this member
1801
+ // no longer owns. Reads are side-effect free, so the next one routes again.
1802
+ function isRouteLost(error) {
1803
+ if (error instanceof FilterExecutionError) {
1804
+ return error.reason === "not_primary" || error.reason === "membership_stale";
1805
+ }
1806
+ return serverErrorCode(error) === CONSUMER_GROUP_PARTITION_NOT_OWNED;
1807
+ }
1808
+ function isNotYetApplied(error) {
1809
+ return ((error instanceof FilterExecutionError &&
1810
+ error.detail.code.kind === "known" &&
1811
+ error.detail.code.name === "NotFound") ||
1812
+ (error instanceof FilterExecutionError && resultCodeIsRetryable(error.detail.code)) ||
1813
+ (error instanceof TransportError && error.retryable));
1814
+ }
1815
+ function isTransient(error) {
1816
+ return ((error instanceof TransportError && error.retryable) ||
1817
+ (error instanceof FilterExecutionError && resultCodeIsRetryable(error.detail.code)));
1818
+ }
1819
+ function unexpected(verb) {
1820
+ return new ProtocolError(`filters ${verb}: unexpected reply variant`);
1821
+ }
1822
+ function sleep(milliseconds) {
1823
+ return new Promise((resolve) => setTimeout(resolve, milliseconds));
1824
+ }
1825
+ function sameGeneration(left, right) {
1826
+ return (left.streamId === right.streamId &&
1827
+ left.streamCreatedAtMicros === right.streamCreatedAtMicros &&
1828
+ left.topicId === right.topicId &&
1829
+ left.topicCreatedAtMicros === right.topicCreatedAtMicros &&
1830
+ left.partitionId === right.partitionId &&
1831
+ left.partitionCreatedRevision === right.partitionCreatedRevision &&
1832
+ left.purgeGeneration === right.purgeGeneration);
1833
+ }
1834
+ function validatePage(request, inlineDigest, page, messages) {
1835
+ const invalid = () => {
1836
+ throw new ProtocolError("filtered page does not match its request or scan boundary");
1837
+ };
1838
+ if (request.consumer.kind === "group_id" && page.policy.groupId !== request.consumer.id)
1839
+ invalid();
1840
+ if (page.v !== FILTER_OP_VERSION ||
1841
+ page.partitionId !== request.partitionId ||
1842
+ page.generation.partitionId !== request.partitionId ||
1843
+ page.readMode !== request.readMode ||
1844
+ page.matched !== messages.length ||
1845
+ page.matched > request.count ||
1846
+ page.matched > page.examined ||
1847
+ page.policy.digest.length !== 32 ||
1848
+ (request.consumer.kind !== "consumer") !== (page.policy.groupId !== undefined))
1849
+ invalid();
1850
+ const body = new DataView(page.records.buffer, page.records.byteOffset, page.records.byteLength);
1851
+ if (body.getUint32(0, true) !== page.partitionId || body.getBigUint64(4, true) !== page.frontier)
1852
+ invalid();
1853
+ if (inlineDigest !== undefined && !sameBytes(inlineDigest, page.policy.digest))
1854
+ invalid();
1855
+ if (request.filter.kind === "revision" &&
1856
+ (page.policy.filterId !== request.filter.filterId ||
1857
+ page.policy.revision !== request.filter.revision))
1858
+ invalid();
1859
+ let start;
1860
+ if (request.start.kind === "continue") {
1861
+ const cursor = request.start.continuation;
1862
+ if (!sameGeneration(cursor.generation, page.generation) ||
1863
+ cursor.groupId !== page.policy.groupId ||
1864
+ !sameBytes(cursor.digest, page.policy.digest) ||
1865
+ cursor.readMode !== request.readMode)
1866
+ invalid();
1867
+ start = cursor.nextScanOffset;
1868
+ }
1869
+ else if (request.start.kind === "offset")
1870
+ start = request.start.offset;
1871
+ else if (request.start.kind === "first")
1872
+ start = 0n;
1873
+ // The last offset the page examined. A primary page offers it as its safe
1874
+ // acknowledgment offset, a local page offers none.
1875
+ const scannedTo = page.nextScanOffset === undefined || page.nextScanOffset === 0n
1876
+ ? undefined
1877
+ : page.nextScanOffset - 1n;
1878
+ if (page.examined === 0) {
1879
+ if (page.safeAckOffset !== undefined || page.nextScanOffset !== start)
1880
+ invalid();
1881
+ }
1882
+ else if (scannedTo === undefined ||
1883
+ (request.readMode === "primary"
1884
+ ? page.safeAckOffset !== scannedTo
1885
+ : page.safeAckOffset !== undefined))
1886
+ invalid();
1887
+ if (scannedTo !== undefined && scannedTo > page.frontier)
1888
+ invalid();
1889
+ if (start !== undefined && page.nextScanOffset !== undefined && page.nextScanOffset < start)
1890
+ invalid();
1891
+ let previous;
1892
+ const offsets = new Set();
1893
+ for (const message of messages) {
1894
+ const offset = message.offset;
1895
+ if (message.partitionId !== request.partitionId ||
1896
+ (previous !== undefined && offset <= previous) ||
1897
+ (start !== undefined && offset < start) ||
1898
+ scannedTo === undefined ||
1899
+ offset > scannedTo ||
1900
+ offset > page.frontier)
1901
+ invalid();
1902
+ previous = offset;
1903
+ offsets.add(offset);
1904
+ }
1905
+ if (new Set(page.unevaluated).size !== page.unevaluated.length ||
1906
+ page.unevaluated.some((offset) => !offsets.has(offset)))
1907
+ invalid();
1908
+ if ((page.stop === "fault" || page.stop === "oversized_record") !== (page.fault !== undefined))
1909
+ invalid();
1910
+ if (page.fault !== undefined && scannedTo !== undefined && scannedTo >= page.fault.offset)
1911
+ invalid();
1912
+ }
1913
+ //# sourceMappingURL=filters.js.map