@laserdata/laser-sdk 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -79
- package/dist/agent/agdx.d.ts +8 -3
- package/dist/agent/agdx.d.ts.map +1 -1
- package/dist/agent/agdx.js +39 -22
- package/dist/agent/agdx.js.map +1 -1
- package/dist/agent/builder.d.ts +25 -15
- package/dist/agent/builder.d.ts.map +1 -1
- package/dist/agent/builder.js +103 -59
- package/dist/agent/builder.js.map +1 -1
- package/dist/agent/context.d.ts +3 -6
- package/dist/agent/context.d.ts.map +1 -1
- package/dist/agent/context.js +17 -9
- package/dist/agent/context.js.map +1 -1
- package/dist/agent/contract.d.ts +2 -2
- package/dist/agent/contract.d.ts.map +1 -1
- package/dist/agent/contract.js +10 -6
- package/dist/agent/contract.js.map +1 -1
- package/dist/agent/memory-handler.d.ts +25 -0
- package/dist/agent/memory-handler.d.ts.map +1 -0
- package/dist/agent/memory-handler.js +43 -0
- package/dist/agent/memory-handler.js.map +1 -0
- package/dist/agent/registry.d.ts +3 -10
- package/dist/agent/registry.d.ts.map +1 -1
- package/dist/agent/registry.js +22 -19
- package/dist/agent/registry.js.map +1 -1
- package/dist/agent/reliable-consumer.d.ts +19 -9
- package/dist/agent/reliable-consumer.d.ts.map +1 -1
- package/dist/agent/reliable-consumer.js +193 -54
- package/dist/agent/reliable-consumer.js.map +1 -1
- package/dist/agent/replies.d.ts +6 -0
- package/dist/agent/replies.d.ts.map +1 -1
- package/dist/agent/replies.js +55 -4
- package/dist/agent/replies.js.map +1 -1
- package/dist/agent/router.d.ts +1 -1
- package/dist/agent/router.d.ts.map +1 -1
- package/dist/agent/router.js +5 -19
- package/dist/agent/router.js.map +1 -1
- package/dist/agent/scope.d.ts +7 -3
- package/dist/agent/scope.d.ts.map +1 -1
- package/dist/agent/scope.js +11 -1
- package/dist/agent/scope.js.map +1 -1
- package/dist/agent/timer.d.ts +6 -0
- package/dist/agent/timer.d.ts.map +1 -0
- package/dist/agent/timer.js +21 -0
- package/dist/agent/timer.js.map +1 -0
- package/dist/agent/workflow.d.ts +16 -19
- package/dist/agent/workflow.d.ts.map +1 -1
- package/dist/agent/workflow.js +17 -6
- package/dist/agent/workflow.js.map +1 -1
- package/dist/blob.d.ts +2 -3
- package/dist/blob.d.ts.map +1 -1
- package/dist/blob.js.map +1 -1
- package/dist/bridges/a2a.d.ts +34 -23
- package/dist/bridges/a2a.d.ts.map +1 -1
- package/dist/bridges/a2a.js +29 -19
- package/dist/bridges/a2a.js.map +1 -1
- package/dist/bridges/hops.d.ts +1 -1
- package/dist/bridges/hops.d.ts.map +1 -1
- package/dist/bridges/hops.js +1 -1
- package/dist/bridges/hops.js.map +1 -1
- package/dist/bridges/mcp.d.ts +27 -7
- package/dist/bridges/mcp.d.ts.map +1 -1
- package/dist/bridges/mcp.js +24 -9
- package/dist/bridges/mcp.js.map +1 -1
- package/dist/client/bytes.d.ts +5 -0
- package/dist/client/bytes.d.ts.map +1 -1
- package/dist/client/bytes.js +15 -0
- package/dist/client/bytes.js.map +1 -1
- package/dist/client/capabilities.d.ts +25 -13
- package/dist/client/capabilities.d.ts.map +1 -1
- package/dist/client/capabilities.js +145 -123
- package/dist/client/capabilities.js.map +1 -1
- package/dist/client/error-classify.d.ts +57 -0
- package/dist/client/error-classify.d.ts.map +1 -0
- package/dist/client/error-classify.js +347 -0
- package/dist/client/error-classify.js.map +1 -0
- package/dist/client/errors.d.ts +113 -40
- package/dist/client/errors.d.ts.map +1 -1
- package/dist/client/errors.js +191 -53
- package/dist/client/errors.js.map +1 -1
- package/dist/client/internals.d.ts +1 -0
- package/dist/client/internals.d.ts.map +1 -1
- package/dist/client/internals.js +1 -0
- package/dist/client/internals.js.map +1 -1
- package/dist/client/laser.d.ts +62 -46
- package/dist/client/laser.d.ts.map +1 -1
- package/dist/client/laser.js +206 -120
- package/dist/client/laser.js.map +1 -1
- package/dist/client/managed.d.ts +3 -0
- package/dist/client/managed.d.ts.map +1 -1
- package/dist/client/managed.js +70 -3
- package/dist/client/managed.js.map +1 -1
- package/dist/context-scope.d.ts +25 -9
- package/dist/context-scope.d.ts.map +1 -1
- package/dist/context-scope.js +36 -9
- package/dist/context-scope.js.map +1 -1
- package/dist/context.d.ts +33 -16
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +174 -95
- package/dist/context.js.map +1 -1
- package/dist/conversation-state.d.ts +1 -1
- package/dist/conversation-state.d.ts.map +1 -1
- package/dist/conversation-state.js +35 -16
- package/dist/conversation-state.js.map +1 -1
- package/dist/govern.d.ts +24 -5
- package/dist/govern.d.ts.map +1 -1
- package/dist/govern.js +116 -28
- package/dist/govern.js.map +1 -1
- package/dist/iggy/apache-iggy.d.ts +59 -56
- package/dist/iggy/apache-iggy.d.ts.map +1 -1
- package/dist/iggy/apache-iggy.js +300 -137
- package/dist/iggy/apache-iggy.js.map +1 -1
- package/dist/index.d.ts +97 -65
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -30
- package/dist/index.js.map +1 -1
- package/dist/intent.d.ts +16 -2
- package/dist/intent.d.ts.map +1 -1
- package/dist/intent.js +50 -18
- package/dist/intent.js.map +1 -1
- package/dist/managed/coordination.d.ts +82 -0
- package/dist/managed/coordination.d.ts.map +1 -0
- package/dist/managed/coordination.js +367 -0
- package/dist/managed/coordination.js.map +1 -0
- package/dist/managed/destinations.d.ts +4 -3
- package/dist/managed/destinations.d.ts.map +1 -1
- package/dist/managed/destinations.js +29 -13
- package/dist/managed/destinations.js.map +1 -1
- package/dist/managed/filters.d.ts +5 -5
- package/dist/managed/filters.d.ts.map +1 -1
- package/dist/managed/filters.js +39 -22
- package/dist/managed/filters.js.map +1 -1
- package/dist/managed/forks.d.ts +8 -11
- package/dist/managed/forks.d.ts.map +1 -1
- package/dist/managed/forks.js +33 -13
- package/dist/managed/forks.js.map +1 -1
- package/dist/managed/graph.d.ts +1 -4
- package/dist/managed/graph.d.ts.map +1 -1
- package/dist/managed/graph.js +5 -3
- package/dist/managed/graph.js.map +1 -1
- package/dist/managed/kv.d.ts +26 -14
- package/dist/managed/kv.d.ts.map +1 -1
- package/dist/managed/kv.js +88 -61
- package/dist/managed/kv.js.map +1 -1
- package/dist/managed/projections.d.ts +5 -8
- package/dist/managed/projections.d.ts.map +1 -1
- package/dist/managed/projections.js +22 -2
- package/dist/managed/projections.js.map +1 -1
- package/dist/managed/query.d.ts +20 -6
- package/dist/managed/query.d.ts.map +1 -1
- package/dist/managed/query.js +97 -24
- package/dist/managed/query.js.map +1 -1
- package/dist/managed/runs.d.ts +2 -4
- package/dist/managed/runs.d.ts.map +1 -1
- package/dist/managed/runs.js +9 -1
- package/dist/managed/runs.js.map +1 -1
- package/dist/managed/watch.d.ts +2 -4
- package/dist/managed/watch.d.ts.map +1 -1
- package/dist/managed/watch.js +9 -1
- package/dist/managed/watch.js.map +1 -1
- package/dist/memory/handle.d.ts +75 -22
- package/dist/memory/handle.d.ts.map +1 -1
- package/dist/memory/handle.js +232 -58
- package/dist/memory/handle.js.map +1 -1
- package/dist/memory/log-memory.d.ts +24 -12
- package/dist/memory/log-memory.d.ts.map +1 -1
- package/dist/memory/log-memory.js +214 -100
- package/dist/memory/log-memory.js.map +1 -1
- package/dist/memory/topic.d.ts +2 -2
- package/dist/memory/topic.d.ts.map +1 -1
- package/dist/memory/topic.js +7 -5
- package/dist/memory/topic.js.map +1 -1
- package/dist/memory/types.d.ts +41 -5
- package/dist/memory/types.d.ts.map +1 -1
- package/dist/memory/types.js +44 -18
- package/dist/memory/types.js.map +1 -1
- package/dist/memory/vector-memory.d.ts +8 -6
- package/dist/memory/vector-memory.d.ts.map +1 -1
- package/dist/memory/vector-memory.js +65 -34
- package/dist/memory/vector-memory.js.map +1 -1
- package/dist/memory.d.ts +6 -5
- package/dist/memory.d.ts.map +1 -1
- package/dist/memory.js +4 -4
- package/dist/memory.js.map +1 -1
- package/dist/provenance/agent-topic.d.ts +3 -1
- package/dist/provenance/agent-topic.d.ts.map +1 -1
- package/dist/provenance/agent-topic.js +3 -1
- package/dist/provenance/agent-topic.js.map +1 -1
- package/dist/provenance/provenance.d.ts +3 -3
- package/dist/provenance/provenance.d.ts.map +1 -1
- package/dist/provenance/provenance.js +30 -18
- package/dist/provenance/provenance.js.map +1 -1
- package/dist/runtime/clock.d.ts +3 -0
- package/dist/runtime/clock.d.ts.map +1 -1
- package/dist/runtime/clock.js +16 -3
- package/dist/runtime/clock.js.map +1 -1
- package/dist/schema-codecs.d.ts +7 -3
- package/dist/schema-codecs.d.ts.map +1 -1
- package/dist/schema-codecs.js +80 -12
- package/dist/schema-codecs.js.map +1 -1
- package/dist/session.d.ts +27 -18
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +95 -37
- package/dist/session.js.map +1 -1
- package/dist/signing.d.ts +11 -5
- package/dist/signing.d.ts.map +1 -1
- package/dist/signing.js +64 -21
- package/dist/signing.js.map +1 -1
- package/dist/snapshot.d.ts +10 -2
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +62 -28
- package/dist/snapshot.js.map +1 -1
- package/dist/state-store.d.ts +1 -1
- package/dist/state-store.d.ts.map +1 -1
- package/dist/state-store.js.map +1 -1
- package/dist/stream/batching.d.ts +81 -0
- package/dist/stream/batching.d.ts.map +1 -0
- package/dist/stream/batching.js +210 -0
- package/dist/stream/batching.js.map +1 -0
- package/dist/stream/bson.d.ts +3 -0
- package/dist/stream/bson.d.ts.map +1 -0
- package/dist/stream/bson.js +245 -0
- package/dist/stream/bson.js.map +1 -0
- package/dist/stream/codecs.d.ts +58 -6
- package/dist/stream/codecs.d.ts.map +1 -1
- package/dist/stream/codecs.js +143 -59
- package/dist/stream/codecs.js.map +1 -1
- package/dist/stream/consumer-group.d.ts +13 -10
- package/dist/stream/consumer-group.d.ts.map +1 -1
- package/dist/stream/consumer-group.js +91 -31
- package/dist/stream/consumer-group.js.map +1 -1
- package/dist/stream/consumer-start.d.ts +17 -0
- package/dist/stream/consumer-start.d.ts.map +1 -0
- package/dist/stream/consumer-start.js +2 -0
- package/dist/stream/consumer-start.js.map +1 -0
- package/dist/stream/consumer.d.ts +166 -24
- package/dist/stream/consumer.d.ts.map +1 -1
- package/dist/stream/consumer.js +451 -68
- package/dist/stream/consumer.js.map +1 -1
- package/dist/stream/cursor.d.ts +5 -12
- package/dist/stream/cursor.d.ts.map +1 -1
- package/dist/stream/cursor.js +34 -8
- package/dist/stream/cursor.js.map +1 -1
- package/dist/stream/header-value.d.ts +61 -15
- package/dist/stream/header-value.d.ts.map +1 -1
- package/dist/stream/header-value.js.map +1 -1
- package/dist/stream/message.d.ts +17 -0
- package/dist/stream/message.d.ts.map +1 -0
- package/dist/stream/message.js +9 -0
- package/dist/stream/message.js.map +1 -0
- package/dist/stream/producer.d.ts +127 -11
- package/dist/stream/producer.d.ts.map +1 -1
- package/dist/stream/producer.js +401 -36
- package/dist/stream/producer.js.map +1 -1
- package/dist/stream/publish.d.ts +6 -15
- package/dist/stream/publish.d.ts.map +1 -1
- package/dist/stream/publish.js +18 -28
- package/dist/stream/publish.js.map +1 -1
- package/dist/stream/record.d.ts +4 -6
- package/dist/stream/record.d.ts.map +1 -1
- package/dist/stream/record.js +4 -3
- package/dist/stream/record.js.map +1 -1
- package/dist/stream/routing.d.ts +12 -0
- package/dist/stream/routing.d.ts.map +1 -1
- package/dist/stream/routing.js +6 -1
- package/dist/stream/routing.js.map +1 -1
- package/dist/stream/stream.d.ts +1 -4
- package/dist/stream/stream.d.ts.map +1 -1
- package/dist/stream/stream.js +5 -1
- package/dist/stream/stream.js.map +1 -1
- package/dist/stream/topic.d.ts +22 -23
- package/dist/stream/topic.d.ts.map +1 -1
- package/dist/stream/topic.js +90 -30
- package/dist/stream/topic.js.map +1 -1
- package/dist/stream/typed-topic.d.ts +9 -13
- package/dist/stream/typed-topic.d.ts.map +1 -1
- package/dist/stream/typed-topic.js +25 -10
- package/dist/stream/typed-topic.js.map +1 -1
- package/dist/swarm.d.ts +0 -1
- package/dist/swarm.d.ts.map +1 -1
- package/dist/swarm.js +1 -0
- package/dist/swarm.js.map +1 -1
- package/dist/testing.d.ts +3 -3
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +3 -3
- package/dist/testing.js.map +1 -1
- package/dist/types/ids.d.ts +17 -2
- package/dist/types/ids.d.ts.map +1 -1
- package/dist/types/ids.js +34 -14
- package/dist/types/ids.js.map +1 -1
- package/dist/wire/agent.d.ts +22 -0
- package/dist/wire/agent.d.ts.map +1 -1
- package/dist/wire/agent.js +80 -25
- package/dist/wire/agent.js.map +1 -1
- package/dist/wire/authz.d.ts +35 -1
- package/dist/wire/authz.d.ts.map +1 -1
- package/dist/wire/authz.js +45 -15
- package/dist/wire/authz.js.map +1 -1
- package/dist/wire/cbor.d.ts.map +1 -1
- package/dist/wire/cbor.js +13 -2
- package/dist/wire/cbor.js.map +1 -1
- package/dist/wire/checkpoint.d.ts +17 -12
- package/dist/wire/checkpoint.d.ts.map +1 -1
- package/dist/wire/checkpoint.js +28 -3
- package/dist/wire/checkpoint.js.map +1 -1
- package/dist/wire/control.d.ts +82 -0
- package/dist/wire/control.d.ts.map +1 -1
- package/dist/wire/control.js +219 -0
- package/dist/wire/control.js.map +1 -1
- package/dist/wire/filter-eval.d.ts +41 -7
- package/dist/wire/filter-eval.d.ts.map +1 -1
- package/dist/wire/filter-eval.js +26 -28
- package/dist/wire/filter-eval.js.map +1 -1
- package/dist/wire/filter.d.ts +51 -17
- package/dist/wire/filter.d.ts.map +1 -1
- package/dist/wire/filter.js +99 -10
- package/dist/wire/filter.js.map +1 -1
- package/dist/wire/fork.d.ts +14 -1
- package/dist/wire/fork.d.ts.map +1 -1
- package/dist/wire/fork.js +19 -8
- package/dist/wire/fork.js.map +1 -1
- package/dist/wire/graph.d.ts +3 -0
- package/dist/wire/graph.d.ts.map +1 -1
- package/dist/wire/graph.js +18 -4
- package/dist/wire/graph.js.map +1 -1
- package/dist/wire/hello.d.ts +4 -1
- package/dist/wire/hello.d.ts.map +1 -1
- package/dist/wire/hello.js +13 -3
- package/dist/wire/hello.js.map +1 -1
- package/dist/wire/index.d.ts +4 -1
- package/dist/wire/index.d.ts.map +1 -1
- package/dist/wire/index.js +4 -1
- package/dist/wire/index.js.map +1 -1
- package/dist/wire/kv.d.ts +37 -2
- package/dist/wire/kv.d.ts.map +1 -1
- package/dist/wire/kv.js +36 -18
- package/dist/wire/kv.js.map +1 -1
- package/dist/wire/query.d.ts +6 -0
- package/dist/wire/query.d.ts.map +1 -1
- package/dist/wire/query.js +38 -12
- package/dist/wire/query.js.map +1 -1
- package/dist/wire/schema.d.ts +29 -6
- package/dist/wire/schema.d.ts.map +1 -1
- package/dist/wire/schema.js +61 -26
- package/dist/wire/schema.js.map +1 -1
- package/dist/wire/snapshot.js +1 -1
- package/dist/wire/snapshot.js.map +1 -1
- package/dist/wire/value.d.ts +10 -2
- package/dist/wire/value.d.ts.map +1 -1
- package/dist/wire/value.js +29 -3
- package/dist/wire/value.js.map +1 -1
- package/package.json +2 -3
- package/dist/stream/polling-strategy.d.ts +0 -14
- package/dist/stream/polling-strategy.d.ts.map +0 -1
- package/dist/stream/polling-strategy.js +0 -2
- package/dist/stream/polling-strategy.js.map +0 -1
package/dist/iggy/apache-iggy.js
CHANGED
|
@@ -1,21 +1,34 @@
|
|
|
1
|
+
import { JOIN_GROUP } from "apache-iggy/dist/wire/consumer-group/join-group.command.js";
|
|
2
|
+
import { SYNC_GROUP } from "apache-iggy/dist/wire/consumer-group/sync-group.command.js";
|
|
1
3
|
import { connectOptions } from "../client/connect-options.js";
|
|
2
4
|
import { publishOptions, publishWithin } from "../client/publish-options.js";
|
|
3
|
-
import { Consumer, DeserializeError, HeaderKeyFactory, HeaderValue as IggyHeaderValueFactory, Partitioning, PollingStrategy as IggyPollingStrategy,
|
|
5
|
+
import { Consumer, DeserializeError, HeaderKeyFactory, HeaderValue as IggyHeaderValueFactory, Partitioning, PollingStrategy as IggyPollingStrategy, SimpleClient, getRawClient } from "apache-iggy";
|
|
4
6
|
import { readFileSync } from "node:fs";
|
|
5
7
|
import { isIP } from "node:net";
|
|
6
|
-
import { AmbiguousMutationError, ConfigError, ProtocolError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
|
|
8
|
+
import { AmbiguousMutationError, ConfigError, InvalidError, ProtocolError, PublishFailedError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
|
|
7
9
|
import { LASERDATA_ROOT_CA } from "../client/laserdata-ca.js";
|
|
8
10
|
import { Mutex } from "../runtime/mutex.js";
|
|
9
11
|
import { mintUlidValue } from "../runtime/ulid.js";
|
|
10
12
|
import { encodeNamed } from "../wire/cbor.js";
|
|
11
13
|
import { isIdempotentManagedRequest } from "../wire/codes.js";
|
|
12
14
|
import { MANAGED_REQUEST_VERSION, encodeManagedRequestEnvelope } from "../wire/mutation.js";
|
|
15
|
+
/** The `maxTopicSize` value that lifts the topic size limit (u64 max). */
|
|
16
|
+
export const UNLIMITED_TOPIC_SIZE = 18446744073709551615n;
|
|
17
|
+
/** The message expiry that keeps records until retention removes them (u64 max). */
|
|
18
|
+
export const NEVER_EXPIRE = 18446744073709551615n;
|
|
19
|
+
const TOPIC_NAME_ALREADY_EXISTS = 2013;
|
|
20
|
+
const UNAUTHENTICATED = 40;
|
|
21
|
+
const POLL_MESSAGES_CODE = 100;
|
|
22
|
+
const PUBLISH_BATCH_LENGTH = 1_000;
|
|
23
|
+
const NUMERIC_IDENTIFIER = 1;
|
|
24
|
+
const STRING_IDENTIFIER = 2;
|
|
25
|
+
// Server replies a later attempt can clear: a write the cluster did not
|
|
26
|
+
// commit or admit, and a partition the node could not read or find yet.
|
|
27
|
+
const TRANSIENT_PUBLISH_CODES = new Set([57, 58, 3004, 3007, 10002]);
|
|
13
28
|
const DEFAULT_RECONNECT_INTERVAL_MS = 1_000;
|
|
14
29
|
const ACCEPT_STAGE = "Iggy server to accept the connection";
|
|
15
30
|
const LOGIN_STAGE = "Iggy login reply";
|
|
16
31
|
const VSR_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
17
|
-
const TRANSIENT_NOT_COMMITTED = 57;
|
|
18
|
-
const TRANSIENT_NOT_ACCEPTED = 58;
|
|
19
32
|
const POLLED_HEAD_BYTES = 16;
|
|
20
33
|
const BATCH_HEADER_BYTES = 256;
|
|
21
34
|
const FRAME_HEADER_BYTES = 48;
|
|
@@ -25,7 +38,18 @@ const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
|
|
|
25
38
|
export function toNodeBuffer(bytes) {
|
|
26
39
|
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
27
40
|
}
|
|
41
|
+
/** The records with an id each: a record without one gets a fresh id. */
|
|
42
|
+
export function withMessageIds(records) {
|
|
43
|
+
return records.map((record) => record.id === undefined || record.id === 0n ? { ...record, id: mintUlidValue() } : record);
|
|
44
|
+
}
|
|
28
45
|
const SYNC_CONSUMER_GROUP_CODE = 606;
|
|
46
|
+
const GROUP_MEMBER_NOT_FOUND = 5006;
|
|
47
|
+
const GROUP_PARTITION_NOT_OWNED = 5009;
|
|
48
|
+
const RESYNC_REQUIRED_PARTITION = 0xffff_ffff;
|
|
49
|
+
const GROUP_ASSIGNMENT_REFRESH_MS = 5_000;
|
|
50
|
+
const GROUP_POLL_MAX_ATTEMPTS = 2;
|
|
51
|
+
const TRANSIENT_NOT_COMMITTED = 57;
|
|
52
|
+
const GROUP_POLL_STATES = new WeakMap();
|
|
29
53
|
function toIggyConsumer(target) {
|
|
30
54
|
return target.kind === "single"
|
|
31
55
|
? target.name === undefined
|
|
@@ -33,6 +57,52 @@ function toIggyConsumer(target) {
|
|
|
33
57
|
: { kind: 1, id: target.name }
|
|
34
58
|
: Consumer.Group(target.name);
|
|
35
59
|
}
|
|
60
|
+
// The poll command body: consumer, stream, topic, partition, strategy,
|
|
61
|
+
// count, and the auto-commit flag, laid out as the Apache Iggy client lays it.
|
|
62
|
+
function encodePollRequest(streamId, topicId, target, partitionId, strategy, count, autoCommit) {
|
|
63
|
+
const consumer = target.kind === "group" ? encodeIdentifier(target.name) : encodeIdentifier(target.name ?? 0);
|
|
64
|
+
const parts = [
|
|
65
|
+
Uint8Array.of(target.kind === "group" ? 2 : 1),
|
|
66
|
+
consumer,
|
|
67
|
+
encodeIdentifier(streamId),
|
|
68
|
+
encodeIdentifier(topicId)
|
|
69
|
+
];
|
|
70
|
+
const tail = new Uint8Array(19);
|
|
71
|
+
const view = new DataView(tail.buffer);
|
|
72
|
+
view.setUint8(0, 1);
|
|
73
|
+
view.setUint32(1, partitionId, true);
|
|
74
|
+
const polling = toIggyPollingStrategy(strategy);
|
|
75
|
+
view.setUint8(5, polling.kind);
|
|
76
|
+
view.setBigUint64(6, polling.value, true);
|
|
77
|
+
view.setUint32(14, count, true);
|
|
78
|
+
view.setUint8(18, autoCommit ? 1 : 0);
|
|
79
|
+
parts.push(tail);
|
|
80
|
+
const request = new Uint8Array(parts.reduce((size, part) => size + part.byteLength, 0));
|
|
81
|
+
let offset = 0;
|
|
82
|
+
for (const part of parts) {
|
|
83
|
+
request.set(part, offset);
|
|
84
|
+
offset += part.byteLength;
|
|
85
|
+
}
|
|
86
|
+
return request;
|
|
87
|
+
}
|
|
88
|
+
function encodeIdentifier(value) {
|
|
89
|
+
if (typeof value === "number") {
|
|
90
|
+
const numeric = new Uint8Array(6);
|
|
91
|
+
const view = new DataView(numeric.buffer);
|
|
92
|
+
view.setUint8(0, NUMERIC_IDENTIFIER);
|
|
93
|
+
view.setUint8(1, 4);
|
|
94
|
+
view.setUint32(2, value, true);
|
|
95
|
+
return numeric;
|
|
96
|
+
}
|
|
97
|
+
const name = new TextEncoder().encode(value);
|
|
98
|
+
if (name.byteLength === 0 || name.byteLength > 255)
|
|
99
|
+
throw new ConfigError("an Iggy identifier must contain 1 to 255 UTF-8 bytes");
|
|
100
|
+
const encoded = new Uint8Array(2 + name.byteLength);
|
|
101
|
+
encoded[0] = STRING_IDENTIFIER;
|
|
102
|
+
encoded[1] = name.byteLength;
|
|
103
|
+
encoded.set(name, 2);
|
|
104
|
+
return encoded;
|
|
105
|
+
}
|
|
36
106
|
function toIggyOffsetConsumer(target) {
|
|
37
107
|
return target.kind === "group"
|
|
38
108
|
? Consumer.Group(target.name)
|
|
@@ -100,7 +170,7 @@ function toIggyPollingStrategy(strategy) {
|
|
|
100
170
|
return IggyPollingStrategy.Next;
|
|
101
171
|
case "offset":
|
|
102
172
|
return IggyPollingStrategy.Offset(strategy.value);
|
|
103
|
-
case "
|
|
173
|
+
case "timestampMicros":
|
|
104
174
|
return IggyPollingStrategy.Timestamp(strategy.value);
|
|
105
175
|
}
|
|
106
176
|
}
|
|
@@ -138,70 +208,16 @@ function toIggyHeaderValue(value) {
|
|
|
138
208
|
return IggyHeaderValueFactory.Double(value.value);
|
|
139
209
|
}
|
|
140
210
|
}
|
|
141
|
-
const HEADER_KIND_BY_NUMBER = {
|
|
142
|
-
1: "raw",
|
|
143
|
-
2: "string",
|
|
144
|
-
3: "bool",
|
|
145
|
-
4: "int8",
|
|
146
|
-
5: "int16",
|
|
147
|
-
6: "int32",
|
|
148
|
-
7: "int64",
|
|
149
|
-
8: "int128",
|
|
150
|
-
9: "uint8",
|
|
151
|
-
10: "uint16",
|
|
152
|
-
11: "uint32",
|
|
153
|
-
12: "uint64",
|
|
154
|
-
13: "uint128",
|
|
155
|
-
14: "float",
|
|
156
|
-
15: "double"
|
|
157
|
-
};
|
|
158
|
-
function fromParsedHeaderValue(kind, value) {
|
|
159
|
-
const tag = HEADER_KIND_BY_NUMBER[kind];
|
|
160
|
-
if (tag === undefined)
|
|
161
|
-
return undefined;
|
|
162
|
-
switch (tag) {
|
|
163
|
-
case "raw":
|
|
164
|
-
case "int128":
|
|
165
|
-
case "uint128": {
|
|
166
|
-
if (!(value instanceof Buffer))
|
|
167
|
-
return undefined;
|
|
168
|
-
return { kind: tag, value: new Uint8Array(value) };
|
|
169
|
-
}
|
|
170
|
-
case "string":
|
|
171
|
-
return typeof value === "string" ? { kind: tag, value } : undefined;
|
|
172
|
-
case "bool":
|
|
173
|
-
return typeof value === "boolean" ? { kind: tag, value } : undefined;
|
|
174
|
-
case "int64":
|
|
175
|
-
case "uint64":
|
|
176
|
-
return typeof value === "bigint" ? { kind: tag, value } : undefined;
|
|
177
|
-
case "int8":
|
|
178
|
-
case "int16":
|
|
179
|
-
case "int32":
|
|
180
|
-
case "uint8":
|
|
181
|
-
case "uint16":
|
|
182
|
-
case "uint32":
|
|
183
|
-
case "float":
|
|
184
|
-
case "double":
|
|
185
|
-
return typeof value === "number" ? { kind: tag, value } : undefined;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
function parsedHeadersToMap(entries) {
|
|
189
|
-
const map = new Map();
|
|
190
|
-
for (const entry of entries) {
|
|
191
|
-
if (typeof entry.key.value !== "string")
|
|
192
|
-
continue;
|
|
193
|
-
const value = fromParsedHeaderValue(entry.value.kind, entry.value.value);
|
|
194
|
-
if (value !== undefined)
|
|
195
|
-
map.set(entry.key.value, value);
|
|
196
|
-
}
|
|
197
|
-
return map;
|
|
198
|
-
}
|
|
199
211
|
/** The Iggy error code a server reply carried, through any transport wrapping. */
|
|
200
212
|
export function serverErrorCode(error) {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
213
|
+
let current = error;
|
|
214
|
+
for (let depth = 0; depth < 8; depth += 1) {
|
|
215
|
+
if (typeof current !== "object" || current === null)
|
|
216
|
+
return undefined;
|
|
217
|
+
const code = current.errorCode;
|
|
218
|
+
if (typeof code === "number")
|
|
219
|
+
return code;
|
|
220
|
+
current = "cause" in current ? current.cause : undefined;
|
|
205
221
|
}
|
|
206
222
|
return undefined;
|
|
207
223
|
}
|
|
@@ -216,6 +232,7 @@ export function decodePolledBody(body) {
|
|
|
216
232
|
throw malformed("head");
|
|
217
233
|
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
218
234
|
const partitionId = view.getUint32(0, true);
|
|
235
|
+
const currentOffset = view.getBigUint64(4, true);
|
|
219
236
|
const messages = [];
|
|
220
237
|
let position = POLLED_HEAD_BYTES;
|
|
221
238
|
while (position < body.byteLength) {
|
|
@@ -223,6 +240,7 @@ export function decodePolledBody(body) {
|
|
|
223
240
|
throw malformed("batch header");
|
|
224
241
|
const baseOffset = view.getBigUint64(position + 8, true);
|
|
225
242
|
const baseTimestamp = view.getBigUint64(position + 16, true);
|
|
243
|
+
const originTimestamp = view.getBigUint64(position + 24, true);
|
|
226
244
|
const batchLength = view.getBigUint64(position + 32, true);
|
|
227
245
|
if (batchLength < BigInt(BATCH_HEADER_BYTES) ||
|
|
228
246
|
batchLength > BigInt(body.byteLength - position))
|
|
@@ -248,6 +266,11 @@ export function decodePolledBody(body) {
|
|
|
248
266
|
partitionId,
|
|
249
267
|
offset: baseOffset + BigInt(offsetDelta),
|
|
250
268
|
timestampMicros: baseTimestamp,
|
|
269
|
+
messageId: view.getBigUint64(position + 8, true) | (view.getBigUint64(position + 16, true) << 64n),
|
|
270
|
+
checksum: view.getBigUint64(position, true),
|
|
271
|
+
originTimestampMicros: originTimestamp + BigInt(view.getUint32(position + 28, true)),
|
|
272
|
+
currentOffset,
|
|
273
|
+
...(block.byteLength > 0 ? { userHeaders: block } : {}),
|
|
251
274
|
...(typeof headers === "string"
|
|
252
275
|
? {
|
|
253
276
|
headers: typeof contentType === "string" ? new Map() : contentType,
|
|
@@ -557,7 +580,7 @@ function pinnedClient(raw, loginDeadline) {
|
|
|
557
580
|
const data = raw;
|
|
558
581
|
const queue = data._queueCommand;
|
|
559
582
|
if (typeof queue !== "function")
|
|
560
|
-
throw new UnsupportedError("this Apache Iggy SDK does not expose the partition data request queue required by filtered Primary reads");
|
|
583
|
+
throw new UnsupportedError("this Apache Iggy SDK does not expose the partition data request queue required by filtered Primary reads", { surface: "filters" });
|
|
561
584
|
const facade = {
|
|
562
585
|
sendCommand(code, payload, options) {
|
|
563
586
|
return queue.call(data, code, payload, options?.handleResponse ?? true, options?.last ?? true, false, options?.deadline ??
|
|
@@ -805,7 +828,10 @@ export class ApacheIggyTransport {
|
|
|
805
828
|
payload
|
|
806
829
|
}))
|
|
807
830
|
: payload;
|
|
808
|
-
|
|
831
|
+
return this.sendManagedPreframed(code, request, options);
|
|
832
|
+
}
|
|
833
|
+
async sendManagedPreframed(code, payload, options) {
|
|
834
|
+
const buffer = toNodeBuffer(payload);
|
|
809
835
|
const reply = await this.execute((client) => client.sendBinaryRequest(code, buffer), `managed command ${String(code)} failed`, options?.retryAfterReconnect ?? true);
|
|
810
836
|
return new Uint8Array(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
811
837
|
}
|
|
@@ -836,23 +862,45 @@ export class ApacheIggyTransport {
|
|
|
836
862
|
async ensureConsumerGroup(streamId, topicId, name) {
|
|
837
863
|
await this.execute((client) => client.group.ensure(streamId, topicId, name), `failed to ensure consumer group \`${name}\``);
|
|
838
864
|
}
|
|
865
|
+
/** Creates the topic when it is absent, keeping records until retention
|
|
866
|
+
* removes them like Rust `ensure_topic`. An existing topic is left as it is. */
|
|
839
867
|
async ensureTopic(streamId, topicId, partitions) {
|
|
840
|
-
|
|
841
|
-
|
|
868
|
+
await this.createTopicIfAbsent(streamId, topicId, partitions, {
|
|
869
|
+
messageExpiryMicros: NEVER_EXPIRE
|
|
870
|
+
});
|
|
842
871
|
}
|
|
872
|
+
/** `ensureTopic` with a message expiry for a topic it creates. An existing
|
|
873
|
+
* topic keeps its own expiry. */
|
|
843
874
|
async ensureTopicWithExpiry(streamId, topicId, partitions, messageExpiryMicros) {
|
|
875
|
+
await this.createTopicIfAbsent(streamId, topicId, partitions, { messageExpiryMicros });
|
|
876
|
+
}
|
|
877
|
+
async createTopicIfAbsent(streamId, topicId, partitions, settings) {
|
|
844
878
|
const partitionCount = await this.execute(async (client) => {
|
|
845
|
-
const
|
|
846
|
-
if (
|
|
847
|
-
return
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
879
|
+
const existing = await client.topic.get({ streamId, topicId });
|
|
880
|
+
if (existing !== null)
|
|
881
|
+
return existing.partitionsCount;
|
|
882
|
+
try {
|
|
883
|
+
const created = await client.topic.create({
|
|
884
|
+
streamId,
|
|
885
|
+
name: topicId,
|
|
886
|
+
partitionCount: partitions,
|
|
887
|
+
compressionAlgorithm: 1,
|
|
888
|
+
...(settings.messageExpiryMicros === undefined
|
|
889
|
+
? {}
|
|
890
|
+
: { messageExpiry: settings.messageExpiryMicros }),
|
|
891
|
+
...(settings.maxTopicSize === undefined ? {} : { maxTopicSize: settings.maxTopicSize })
|
|
892
|
+
});
|
|
893
|
+
return created.partitionsCount;
|
|
894
|
+
}
|
|
895
|
+
catch (error) {
|
|
896
|
+
if (serverErrorCode(error) !== TOPIC_NAME_ALREADY_EXISTS)
|
|
897
|
+
throw error;
|
|
898
|
+
const raced = await client.topic.get({ streamId, topicId });
|
|
899
|
+
if (raced === null)
|
|
900
|
+
throw error;
|
|
901
|
+
return raced.partitionsCount;
|
|
902
|
+
}
|
|
903
|
+
}, `failed to create topic \`${topicId}\` on stream \`${streamId}\``);
|
|
856
904
|
this.partitionCounts.set(this.topicKey(streamId, topicId), partitionCount);
|
|
857
905
|
}
|
|
858
906
|
async findTopicPartitionCount(streamId, topicId) {
|
|
@@ -915,9 +963,13 @@ export class ApacheIggyTransport {
|
|
|
915
963
|
}
|
|
916
964
|
catch (cause) {
|
|
917
965
|
const response = serverResponseError(cause);
|
|
918
|
-
const code =
|
|
919
|
-
const transient = code
|
|
966
|
+
const code = serverErrorCode(cause);
|
|
967
|
+
const transient = code !== undefined && TRANSIENT_PUBLISH_CODES.has(code);
|
|
968
|
+
// An expired session on a connection that logged in once means the
|
|
969
|
+
// socket was re-dialed underneath it. A reconnect logs in again.
|
|
970
|
+
const reauthenticate = code === UNAUTHENTICATED && this.connection !== undefined;
|
|
920
971
|
const retryable = transient ||
|
|
972
|
+
reauthenticate ||
|
|
921
973
|
(response === undefined &&
|
|
922
974
|
!(cause instanceof DeserializeError) &&
|
|
923
975
|
!(cause instanceof ConfigError) &&
|
|
@@ -926,7 +978,7 @@ export class ApacheIggyTransport {
|
|
|
926
978
|
// the current one. A concurrent publish may already have replaced it,
|
|
927
979
|
// and destroying that fresh connection would starve every publisher for
|
|
928
980
|
// as long as failures keep arriving.
|
|
929
|
-
if (response === undefined && retryable && this.client === used) {
|
|
981
|
+
if ((response === undefined || reauthenticate) && retryable && this.client === used) {
|
|
930
982
|
this.disconnected.add(used);
|
|
931
983
|
// Destroy before retry so a timed-out queued write cannot run on the old socket.
|
|
932
984
|
await used.destroy().catch(() => undefined);
|
|
@@ -944,28 +996,12 @@ export class ApacheIggyTransport {
|
|
|
944
996
|
throw new TransportError("publish attempts exhausted", true);
|
|
945
997
|
}
|
|
946
998
|
async sendMessages(streamId, topicId, payloads, routing) {
|
|
947
|
-
|
|
948
|
-
const messages = payloads.map((payload) => ({
|
|
949
|
-
id: mintUlidValue(),
|
|
950
|
-
payload: toNodeBuffer(payload)
|
|
951
|
-
}));
|
|
952
|
-
return this.publish(async (client) => {
|
|
953
|
-
partitionId ??= await this.resolvePartition(streamId, topicId, routing, client);
|
|
954
|
-
if (this.disconnected.has(client))
|
|
955
|
-
throw new TransportError("publish connection was retired", true);
|
|
956
|
-
return client.message.send({
|
|
957
|
-
streamId,
|
|
958
|
-
topicId,
|
|
959
|
-
messages,
|
|
960
|
-
partition: Partitioning.PartitionId(partitionId)
|
|
961
|
-
});
|
|
962
|
-
});
|
|
999
|
+
return this.sendRecords(streamId, topicId, payloads.map((payload) => ({ payload, headers: new Map() })), routing);
|
|
963
1000
|
}
|
|
964
1001
|
async sendMessageWithHeaders(streamId, topicId, payload, headers, partitionKey, partitionId) {
|
|
965
1002
|
return this.sendMessagesWithHeaders(streamId, topicId, [{ payload, headers }], partitionKey, partitionId);
|
|
966
1003
|
}
|
|
967
1004
|
async sendMessagesWithHeaders(streamId, topicId, messages, partitionKey, partitionId, options) {
|
|
968
|
-
let resolvedPartition;
|
|
969
1005
|
const routing = partitionId !== undefined
|
|
970
1006
|
? { kind: "partition", partition: partitionId }
|
|
971
1007
|
: partitionKey !== undefined
|
|
@@ -976,43 +1012,84 @@ export class ApacheIggyTransport {
|
|
|
976
1012
|
: partitionKey
|
|
977
1013
|
}
|
|
978
1014
|
: { kind: "balanced" };
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1015
|
+
return this.sendRecords(streamId, topicId, messages, routing, options);
|
|
1016
|
+
}
|
|
1017
|
+
// Sends the records in consecutive requests of at most `batchLength`, each
|
|
1018
|
+
// with bounded retries, like the Rust `send_batch_on`. A failure reports the
|
|
1019
|
+
// confirmed requests and every record from the failed request on, with the
|
|
1020
|
+
// ids its attempts used. An empty batch sends nothing.
|
|
1021
|
+
async sendRecords(streamId, topicId, messages, routing, options) {
|
|
1022
|
+
if (messages.length === 0)
|
|
1023
|
+
return { confirmations: [] };
|
|
1024
|
+
if (routing.kind === "key" && (routing.key.byteLength === 0 || routing.key.byteLength > 255))
|
|
1025
|
+
throw new InvalidError("a partition key must hold 1 to 255 bytes");
|
|
1026
|
+
const records = withMessageIds(messages);
|
|
1027
|
+
const batchLength = options?.batchLength ?? PUBLISH_BATCH_LENGTH;
|
|
1028
|
+
const confirmations = [];
|
|
1029
|
+
let resolvedPartition;
|
|
1030
|
+
for (let start = 0; start < records.length; start += batchLength) {
|
|
1031
|
+
const prepared = records.slice(start, start + batchLength).map((record) => ({
|
|
1032
|
+
id: record.id ?? mintUlidValue(),
|
|
1033
|
+
payload: toNodeBuffer(record.payload),
|
|
1034
|
+
headers: [...record.headers].map(([key, value]) => ({
|
|
1035
|
+
key: HeaderKeyFactory.String(key),
|
|
1036
|
+
value: toIggyHeaderValue(value)
|
|
1037
|
+
}))
|
|
1038
|
+
}));
|
|
1039
|
+
try {
|
|
1040
|
+
const response = await this.publish(async (client) => {
|
|
1041
|
+
resolvedPartition ??= await this.resolvePartition(streamId, topicId, routing, client);
|
|
1042
|
+
if (this.disconnected.has(client))
|
|
1043
|
+
throw new TransportError("publish connection was retired", true);
|
|
1044
|
+
return client.message.send({
|
|
1045
|
+
streamId,
|
|
1046
|
+
topicId,
|
|
1047
|
+
messages: prepared,
|
|
1048
|
+
partition: Partitioning.PartitionId(resolvedPartition)
|
|
1049
|
+
});
|
|
1050
|
+
}, options);
|
|
1051
|
+
confirmations.push(...response.confirmations);
|
|
1052
|
+
}
|
|
1053
|
+
catch (cause) {
|
|
1054
|
+
throw new PublishFailedError(streamId, topicId, confirmations, records.slice(start), cause);
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
return { confirmations };
|
|
998
1058
|
}
|
|
999
1059
|
async pollMessages(streamId, topicId, target, strategy, count, autoCommit) {
|
|
1000
|
-
const
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1060
|
+
const partitionId = target.partitionId;
|
|
1061
|
+
if (partitionId === undefined) {
|
|
1062
|
+
if (target.kind !== "group")
|
|
1063
|
+
throw new InvalidError("a single consumer needs a partition");
|
|
1064
|
+
return this.execute(async (client) => {
|
|
1065
|
+
const raw = await client.clientProvider();
|
|
1066
|
+
const release = raw.hold?.();
|
|
1067
|
+
try {
|
|
1068
|
+
const state = groupPollState(raw);
|
|
1069
|
+
for (;;) {
|
|
1070
|
+
const generation = state.sessionGeneration;
|
|
1071
|
+
try {
|
|
1072
|
+
return await pollGroupRecords(raw, state, streamId, topicId, target, strategy, count, autoCommit);
|
|
1073
|
+
}
|
|
1074
|
+
catch (error) {
|
|
1075
|
+
if (state.sessionGeneration === generation ||
|
|
1076
|
+
serverErrorCode(error) === TRANSIENT_NOT_COMMITTED)
|
|
1077
|
+
throw error;
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
finally {
|
|
1082
|
+
release?.();
|
|
1083
|
+
}
|
|
1084
|
+
}, `failed to poll topic \`${topicId}\``);
|
|
1085
|
+
}
|
|
1086
|
+
const request = toNodeBuffer(encodePollRequest(streamId, topicId, target, partitionId, strategy, count, autoCommit));
|
|
1087
|
+
const body = await this.execute((client) => client.sendBinaryRequest(POLL_MESSAGES_CODE, request), `failed to poll topic \`${topicId}\``);
|
|
1088
|
+
// The stored records exactly: microsecond times, message ids, and a
|
|
1089
|
+
// header block that does not decode marked rather than failing the poll.
|
|
1090
|
+
return body.byteLength === 0
|
|
1091
|
+
? []
|
|
1092
|
+
: decodePolledBody(new Uint8Array(body.buffer, body.byteOffset, body.byteLength));
|
|
1016
1093
|
}
|
|
1017
1094
|
async storeOffset(streamId, topicId, target, partitionId, offset) {
|
|
1018
1095
|
await this.execute((client) => client.offset.store({
|
|
@@ -1023,6 +1100,14 @@ export class ApacheIggyTransport {
|
|
|
1023
1100
|
offset
|
|
1024
1101
|
}), `failed to store offset for topic \`${topicId}\``);
|
|
1025
1102
|
}
|
|
1103
|
+
async deleteOffset(streamId, topicId, target, partitionId) {
|
|
1104
|
+
await this.execute((client) => client.offset.delete({
|
|
1105
|
+
streamId,
|
|
1106
|
+
topicId,
|
|
1107
|
+
consumer: toIggyConsumer(target),
|
|
1108
|
+
partitionId
|
|
1109
|
+
}), `failed to delete offset for topic \`${topicId}\``);
|
|
1110
|
+
}
|
|
1026
1111
|
async getConsumerOffset(streamId, topicId, target, partitionId) {
|
|
1027
1112
|
const offset = await this.execute((client) => client.offset.get({
|
|
1028
1113
|
streamId,
|
|
@@ -1172,4 +1257,82 @@ export class ApacheIggyTransport {
|
|
|
1172
1257
|
return cursor % partitionCount;
|
|
1173
1258
|
}
|
|
1174
1259
|
}
|
|
1260
|
+
function groupPollState(client) {
|
|
1261
|
+
const current = GROUP_POLL_STATES.get(client);
|
|
1262
|
+
if (current !== undefined)
|
|
1263
|
+
return current;
|
|
1264
|
+
const state = { cursors: new Map(), sessionGeneration: 0 };
|
|
1265
|
+
GROUP_POLL_STATES.set(client, state);
|
|
1266
|
+
client.on("sessionReset", () => {
|
|
1267
|
+
state.cursors.clear();
|
|
1268
|
+
state.sessionGeneration += 1;
|
|
1269
|
+
});
|
|
1270
|
+
client.on("heartbeat", () => {
|
|
1271
|
+
for (const cursor of state.cursors.values())
|
|
1272
|
+
cursor.synchronizedAt = 0;
|
|
1273
|
+
});
|
|
1274
|
+
return state;
|
|
1275
|
+
}
|
|
1276
|
+
async function pollGroupRecords(client, state, streamId, topicId, target, strategy, count, autoCommit) {
|
|
1277
|
+
const key = `${streamId}\0${topicId}\0${target.name}`;
|
|
1278
|
+
for (let attempt = 0; attempt < GROUP_POLL_MAX_ATTEMPTS; attempt += 1) {
|
|
1279
|
+
let cursor = state.cursors.get(key);
|
|
1280
|
+
const age = cursor === undefined ? -1 : Date.now() - cursor.synchronizedAt;
|
|
1281
|
+
if (cursor === undefined || age < 0 || age >= GROUP_ASSIGNMENT_REFRESH_MS) {
|
|
1282
|
+
const group = { streamId, topicId, groupId: target.name };
|
|
1283
|
+
let assignment;
|
|
1284
|
+
try {
|
|
1285
|
+
assignment = SYNC_GROUP.deserialize(await client.sendCommand(SYNC_GROUP.code, SYNC_GROUP.serialize(group)));
|
|
1286
|
+
}
|
|
1287
|
+
catch (error) {
|
|
1288
|
+
if (serverErrorCode(error) !== GROUP_MEMBER_NOT_FOUND)
|
|
1289
|
+
throw error;
|
|
1290
|
+
assignment = null;
|
|
1291
|
+
}
|
|
1292
|
+
if (assignment === null) {
|
|
1293
|
+
await client.sendCommand(JOIN_GROUP.code, JOIN_GROUP.serialize(group));
|
|
1294
|
+
assignment = SYNC_GROUP.deserialize(await client.sendCommand(SYNC_GROUP.code, SYNC_GROUP.serialize(group)));
|
|
1295
|
+
}
|
|
1296
|
+
if (assignment === null)
|
|
1297
|
+
throw new ProtocolError("the consumer group assignment is absent");
|
|
1298
|
+
cursor = {
|
|
1299
|
+
generation: assignment.generation,
|
|
1300
|
+
partitions: assignment.partitions,
|
|
1301
|
+
position: cursor?.generation === assignment.generation &&
|
|
1302
|
+
cursor.position < assignment.partitions.length
|
|
1303
|
+
? cursor.position
|
|
1304
|
+
: 0,
|
|
1305
|
+
synchronizedAt: Date.now()
|
|
1306
|
+
};
|
|
1307
|
+
state.cursors.set(key, cursor);
|
|
1308
|
+
}
|
|
1309
|
+
const partition = cursor.partitions[cursor.position];
|
|
1310
|
+
if (partition === undefined)
|
|
1311
|
+
return [];
|
|
1312
|
+
cursor.position = (cursor.position + 1) % cursor.partitions.length;
|
|
1313
|
+
let body;
|
|
1314
|
+
try {
|
|
1315
|
+
const response = await client.sendCommand(POLL_MESSAGES_CODE, toNodeBuffer(encodePollRequest(streamId, topicId, target, partition, strategy, count, autoCommit)));
|
|
1316
|
+
body = new Uint8Array(response.data.buffer, response.data.byteOffset, response.data.byteLength);
|
|
1317
|
+
}
|
|
1318
|
+
catch (error) {
|
|
1319
|
+
const code = serverErrorCode(error);
|
|
1320
|
+
if (code !== GROUP_MEMBER_NOT_FOUND && code !== GROUP_PARTITION_NOT_OWNED)
|
|
1321
|
+
throw error;
|
|
1322
|
+
state.cursors.delete(key);
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
if (body.byteLength === 0)
|
|
1326
|
+
return [];
|
|
1327
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
1328
|
+
if (body.byteLength >= POLLED_HEAD_BYTES &&
|
|
1329
|
+
view.getUint32(0, true) === RESYNC_REQUIRED_PARTITION &&
|
|
1330
|
+
view.getUint32(12, true) === 0) {
|
|
1331
|
+
state.cursors.delete(key);
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
return decodePolledBody(body);
|
|
1335
|
+
}
|
|
1336
|
+
return [];
|
|
1337
|
+
}
|
|
1175
1338
|
//# sourceMappingURL=apache-iggy.js.map
|