@laserdata/laser-sdk 0.4.1 → 0.5.0

package/dist/iggy/apache-iggy.js

@@ -1,9 +1,9 @@
 import { connectOptions } from "../client/connect-options.js";
 import { publishOptions, publishWithin } from "../client/publish-options.js";
-import { Consumer, DeserializeError, HeaderKeyFactory, HeaderValue as IggyHeaderValueFactory, Partitioning, PollingStrategy as IggyPollingStrategy, SimpleClient, getRawClient } from "apache-iggy";
+import { Consumer, DeserializeError, HeaderKeyFactory, HeaderValue as IggyHeaderValueFactory, Partitioning, PollingStrategy as IggyPollingStrategy, ResponseError, SimpleClient, getRawClient } from "apache-iggy";
 import { readFileSync } from "node:fs";
 import { isIP } from "node:net";
-import { AmbiguousMutationError, ConfigError, TimeoutError, TransportError } from "../client/errors.js";
+import { AmbiguousMutationError, ConfigError, ProtocolError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
 import { LASERDATA_ROOT_CA } from "../client/laserdata-ca.js";
 import { Mutex } from "../runtime/mutex.js";
 import { mintUlidValue } from "../runtime/ulid.js";
@@ -16,9 +16,16 @@ const LOGIN_STAGE = "Iggy login reply";
 const VSR_HEARTBEAT_INTERVAL_MS = 5_000;
 const TRANSIENT_NOT_COMMITTED = 57;
 const TRANSIENT_NOT_ACCEPTED = 58;
+const POLLED_HEAD_BYTES = 16;
+const BATCH_HEADER_BYTES = 256;
+const FRAME_HEADER_BYTES = 48;
+const STRING_HEADER_KIND = 2;
+const CONTENT_TYPE_HEADER = "agdx.ct";
+const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
 export function toNodeBuffer(bytes) {
     return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
 }
+const SYNC_CONSUMER_GROUP_CODE = 606;
 function toIggyConsumer(target) {
     return target.kind === "single"
         ? target.name === undefined
@@ -189,6 +196,162 @@ function parsedHeadersToMap(entries) {
     }
     return map;
 }
+/** The Iggy error code a server reply carried, through any transport wrapping. */
+export function serverErrorCode(error) {
+    for (let current = error; current !== undefined;) {
+        if (current instanceof ResponseError)
+            return current.errorCode;
+        current = current instanceof Error ? current.cause : undefined;
+    }
+    return undefined;
+}
+/**
+ * The records of a standard polled-messages body, decoded exactly as stored:
+ * original offsets, per-record microsecond timestamps, payloads, and typed
+ * user headers.
+ */
+export function decodePolledBody(body) {
+    const malformed = (what) => new ProtocolError(`the polled record body has a malformed ${what}`);
+    if (body.byteLength < POLLED_HEAD_BYTES)
+        throw malformed("head");
+    const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
+    const partitionId = view.getUint32(0, true);
+    const messages = [];
+    let position = POLLED_HEAD_BYTES;
+    while (position < body.byteLength) {
+        if (position + BATCH_HEADER_BYTES > body.byteLength)
+            throw malformed("batch header");
+        const baseOffset = view.getBigUint64(position + 8, true);
+        const baseTimestamp = view.getBigUint64(position + 16, true);
+        const batchLength = view.getBigUint64(position + 32, true);
+        if (batchLength < BigInt(BATCH_HEADER_BYTES) ||
+            batchLength > BigInt(body.byteLength - position))
+            throw malformed("batch length");
+        const batchEnd = position + Number(batchLength);
+        position += BATCH_HEADER_BYTES;
+        while (position < batchEnd) {
+            if (position + FRAME_HEADER_BYTES > batchEnd)
+                throw malformed("message frame");
+            const offsetDelta = view.getUint32(position + 24, true);
+            const headersLength = view.getUint32(position + 32, true);
+            const payloadLength = view.getUint32(position + 36, true);
+            const payloadStart = position + FRAME_HEADER_BYTES;
+            const headersStart = payloadStart + payloadLength;
+            const frameEnd = headersStart + headersLength;
+            if (frameEnd > batchEnd)
+                throw malformed("message frame");
+            const block = body.subarray(headersStart, frameEnd);
+            const headers = decodeUserHeaders(block);
+            const contentType = headers === "entry" ? decodeUserHeaders(block, true) : headers;
+            messages.push({
+                payload: body.subarray(payloadStart, headersStart),
+                partitionId,
+                offset: baseOffset + BigInt(offsetDelta),
+                timestampMicros: baseTimestamp,
+                ...(typeof headers === "string"
+                    ? {
+                        headers: typeof contentType === "string" ? new Map() : contentType,
+                        headersMalformed: typeof contentType === "string" ? contentType : headers
+                    }
+                    : { headers })
+            });
+            position = frameEnd;
+        }
+    }
+    return messages;
+}
+// The typed user headers as the server reads them for a filter: string keys
+// only, a kind this build does not know kept raw, a value that does not fit
+// its kind faulting the record, and the last of a repeated key kept. A structurally broken
+// block names what did not decode, which the server treats as a malformed
+// record when its filter needs that part.
+function decodeUserHeaders(block, contentTypeOnly = false) {
+    const headers = new Map();
+    const view = new DataView(block.buffer, block.byteOffset, block.byteLength);
+    let position = 0;
+    while (position < block.byteLength) {
+        if (position + 5 > block.byteLength)
+            return "structure";
+        const keyKind = view.getUint8(position);
+        const keyStart = position + 5;
+        const keyEnd = keyStart + view.getUint32(position + 1, true);
+        if (keyKind === 0 ||
+            keyEnd === keyStart ||
+            keyEnd - keyStart > 255 ||
+            keyEnd + 5 > block.byteLength)
+            return "structure";
+        const valueKind = view.getUint8(keyEnd);
+        const valueStart = keyEnd + 5;
+        const valueEnd = valueStart + view.getUint32(keyEnd + 1, true);
+        if (valueKind === 0 ||
+            valueEnd === valueStart ||
+            valueEnd - valueStart > 255 ||
+            valueEnd > block.byteLength)
+            return "structure";
+        position = valueEnd;
+        if (keyKind !== STRING_HEADER_KIND)
+            continue;
+        if (contentTypeOnly &&
+            (keyEnd - keyStart !== CONTENT_TYPE_HEADER.length ||
+                CONTENT_TYPE_HEADER.split("").some((character, index) => block[keyStart + index] !== character.charCodeAt(0))))
+            continue;
+        let key;
+        try {
+            key = STRICT_UTF8.decode(block.subarray(keyStart, keyEnd));
+        }
+        catch {
+            return "entry";
+        }
+        const value = headerValueOf(valueKind, block.subarray(valueStart, valueEnd));
+        if (value === undefined)
+            return key === CONTENT_TYPE_HEADER ? "content_type" : "entry";
+        headers.set(key, value);
+    }
+    return headers;
+}
+function headerValueOf(kind, bytes) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const sized = (size) => bytes.byteLength === size;
+    switch (kind) {
+        case STRING_HEADER_KIND:
+            try {
+                return { kind: "string", value: STRICT_UTF8.decode(bytes) };
+            }
+            catch {
+                return undefined;
+            }
+        case 3:
+            return sized(1) && (bytes[0] === 0 || bytes[0] === 1)
+                ? { kind: "bool", value: bytes[0] === 1 }
+                : undefined;
+        case 4:
+            return sized(1) ? { kind: "int8", value: view.getInt8(0) } : undefined;
+        case 5:
+            return sized(2) ? { kind: "int16", value: view.getInt16(0, true) } : undefined;
+        case 6:
+            return sized(4) ? { kind: "int32", value: view.getInt32(0, true) } : undefined;
+        case 7:
+            return sized(8) ? { kind: "int64", value: view.getBigInt64(0, true) } : undefined;
+        case 8:
+            return sized(16) ? { kind: "int128", value: bytes.slice() } : undefined;
+        case 9:
+            return sized(1) ? { kind: "uint8", value: view.getUint8(0) } : undefined;
+        case 10:
+            return sized(2) ? { kind: "uint16", value: view.getUint16(0, true) } : undefined;
+        case 11:
+            return sized(4) ? { kind: "uint32", value: view.getUint32(0, true) } : undefined;
+        case 12:
+            return sized(8) ? { kind: "uint64", value: view.getBigUint64(0, true) } : undefined;
+        case 13:
+            return sized(16) ? { kind: "uint128", value: bytes.slice() } : undefined;
+        case 14:
+            return sized(4) ? { kind: "float", value: view.getFloat32(0, true) } : undefined;
+        case 15:
+            return sized(8) ? { kind: "double", value: view.getFloat64(0, true) } : undefined;
+        default:
+            return { kind: "raw", value: bytes.slice() };
+    }
+}
 function laserDataHost(host) {
     const normalized = host.toLowerCase();
     return (normalized === "laserdata.cloud" ||
@@ -295,7 +458,7 @@ export function watchConnectionLoss(connection, onLost) {
         onLost();
     });
 }
-async function connectSimpleClient(parsed, deadline) {
+async function connectSimpleClient(parsed, deadline, pinned = false) {
     const config = parsed.tls
         ? {
             heartbeatInterval: VSR_HEARTBEAT_INTERVAL_MS,
@@ -303,7 +466,11 @@ async function connectSimpleClient(parsed, deadline) {
             options: {
                 port: parsed.port,
                 host: parsed.host,
-                ...(isIP(parsed.host) === 0 ? { servername: parsed.host } : {}),
+                ...(parsed.servername !== undefined
+                    ? { servername: parsed.servername }
+                    : isIP(parsed.host) === 0
+                        ? { servername: parsed.host }
+                        : {}),
                 ...(parsed.ca !== undefined ? { ca: parsed.ca } : {})
             },
             credentials: parsed.credentials,
@@ -325,7 +492,7 @@ async function connectSimpleClient(parsed, deadline) {
     }
     try {
         const connection = raw.connection;
-        const client = new SimpleClient(raw);
+        const client = pinned ? pinnedClient(raw, deadline) : new SimpleClient(raw);
         // The client emits `connect` only once the transport, TLS included, is up, so it
         // separates a server that never accepts the socket from one that never answers login.
         let accepted = false;
@@ -338,7 +505,18 @@ async function connectSimpleClient(parsed, deadline) {
                 reject(cause instanceof Error ? cause : new Error(String(cause)));
             };
             connection.once("error", failed);
-            client.client.getMe().then(() => {
+            const login = async () => {
+                if (pinned) {
+                    const connection = raw.connection;
+                    await connection.connect(true);
+                    if ("token" in parsed.credentials)
+                        await client.session.loginWithToken(parsed.credentials);
+                    else
+                        await client.session.login(parsed.credentials);
+                }
+                await client.client.getMe();
+            };
+            login().then(() => {
                 connection.off("error", failed);
                 resolve();
             }, reject);
@@ -366,13 +544,44 @@ async function connectSimpleClient(parsed, deadline) {
     }
     catch (cause) {
         raw.destroy();
-        if (cause instanceof TimeoutError)
+        if (cause instanceof TimeoutError || cause instanceof UnsupportedError)
             throw cause;
         throw new TransportError(`failed to connect to ${parsed.host}:${String(parsed.port)}`, true, {
             cause
         });
     }
 }
+// Use the same request queue the Apache Iggy SDK uses for its native partition
+// data connections. This bypasses coordinator login settlement on this connection.
+function pinnedClient(raw, loginDeadline) {
+    const data = raw;
+    const queue = data._queueCommand;
+    if (typeof queue !== "function")
+        throw new UnsupportedError("this Apache Iggy SDK does not expose the partition data request queue required by filtered Primary reads");
+    const facade = {
+        sendCommand(code, payload, options) {
+            return queue.call(data, code, payload, options?.handleResponse ?? true, options?.last ?? true, false, options?.deadline ??
+                (raw.isAuthenticated
+                    ? Date.now() + connectOptions().timeoutMs
+                    : (loginDeadline ?? Date.now() + connectOptions().timeoutMs)));
+        },
+        get isAuthenticated() {
+            return raw.isAuthenticated;
+        },
+        authenticate: (credentials) => raw.authenticate(credentials),
+        destroy: () => {
+            raw.destroy();
+        },
+        on: (event, callback) => {
+            raw.on(event, callback);
+        },
+        once: (event, callback) => {
+            raw.once(event, callback);
+        },
+        getReadStream: () => raw.getReadStream()
+    };
+    return new SimpleClient(facade);
+}
 function serverResponseError(error) {
     let current = error;
     for (let depth = 0; depth < 8; depth += 1) {
@@ -428,6 +637,8 @@ export class ApacheIggyTransport {
     reconnectLock = new Mutex();
     publishLane = new Mutex();
     disconnected = new WeakSet();
+    // Groups this transport rejoins after a reconnect. Setup helpers allow
+    // recreation, while joining an existing group keeps `create: false`.
     consumerGroups = new Map();
     partitionCounts = new Map();
     balancedCursors = new Map();
@@ -441,6 +652,9 @@ export class ApacheIggyTransport {
     get iggyClient() {
         return this.client;
     }
+    get connectsNodes() {
+        return this.connection !== undefined;
+    }
     static async connect(connectionString, options, deadline = Date.now() + connectOptions().timeoutMs) {
         const config = publishOptions(options);
         const parsed = parseConnectionString(connectionString);
@@ -469,7 +683,12 @@ export class ApacheIggyTransport {
         const stale = this.client;
         if (this.disconnected.has(stale)) {
             await this.reconnect(stale);
-            return operation(this.client);
+            try {
+                return await operation(this.client);
+            }
+            catch (cause) {
+                throw new TransportError(message, serverResponseError(cause) === undefined, { cause });
+            }
         }
         try {
             return await operation(stale);
@@ -523,8 +742,21 @@ export class ApacheIggyTransport {
                         throw new TimeoutError("Iggy publish reconnect");
                     const connected = await connectSimpleClient(this.connection, deadline);
                     try {
-                        for (const group of this.consumerGroups.values()) {
-                            const join = connected.client.group.ensureAndJoin(group.streamId, group.topicId, group.name);
+                        for (const [key, group] of [...this.consumerGroups]) {
+                            const join = group.create && typeof group.name === "string"
+                                ? connected.client.group.ensureAndJoin(group.streamId, group.topicId, group.name)
+                                : connected.client.group
+                                    .join({
+                                    streamId: group.streamId,
+                                    topicId: group.topicId,
+                                    groupId: group.name
+                                })
+                                    .then(() => undefined, 
+                                // A deleted group is not recreated: its member finds out
+                                // on its next assignment read.
+                                () => {
+                                    this.consumerGroups.delete(key);
+                                });
                             await (deadline === undefined
                                 ? join
                                 : publishWithin(join, Math.max(1, deadline - Date.now())));
@@ -601,6 +833,9 @@ export class ApacheIggyTransport {
         }
         return deleted;
     }
+    async ensureConsumerGroup(streamId, topicId, name) {
+        await this.execute((client) => client.group.ensure(streamId, topicId, name), `failed to ensure consumer group \`${name}\``);
+    }
     async ensureTopic(streamId, topicId, partitions) {
         const topic = await this.execute((client) => client.topic.ensure(streamId, topicId, partitions), `failed to ensure topic \`${topicId}\` on stream \`${streamId}\``);
         this.partitionCounts.set(this.topicKey(streamId, topicId), topic.partitionsCount);
@@ -765,7 +1000,7 @@ export class ApacheIggyTransport {
         const reply = await this.execute((client) => client.message.poll({
             streamId,
             topicId,
-            partitionId: target.kind === "single" ? target.partitionId : null,
+            partitionId: target.partitionId ?? null,
             consumer: toIggyConsumer(target),
             pollingStrategy: toIggyPollingStrategy(strategy),
             count,
@@ -801,11 +1036,105 @@ export class ApacheIggyTransport {
     }
     async joinConsumerGroup(streamId, topicId, name) {
         await this.execute((client) => client.group.ensureAndJoin(streamId, topicId, name), `failed to join consumer group \`${name}\``);
-        this.consumerGroups.set(`${streamId}\0${topicId}\0${name}`, { streamId, topicId, name });
+        this.consumerGroups.set(`${streamId}\0${topicId}\0${name}`, {
+            streamId,
+            topicId,
+            name,
+            create: true
+        });
+    }
+    async syncConsumerGroup(streamId, topicId, name) {
+        const identifiers = [streamId, topicId, name].map((value) => new TextEncoder().encode(value));
+        if (identifiers.some((value) => value.byteLength === 0 || value.byteLength > 255))
+            throw new ConfigError("consumer group identifiers must contain 1 to 255 UTF-8 bytes");
+        const payload = new Uint8Array(identifiers.reduce((size, value) => size + 2 + value.byteLength, 0));
+        let offset = 0;
+        for (const value of identifiers) {
+            payload.set([2, value.byteLength], offset);
+            payload.set(value, offset + 2);
+            offset += value.byteLength + 2;
+        }
+        let reply = await this.sendManaged(SYNC_CONSUMER_GROUP_CODE, payload, {
+            retryAfterReconnect: true
+        });
+        let rejoined = false;
+        if (reply.byteLength === 0) {
+            await this.joinConsumerGroup(streamId, topicId, name);
+            rejoined = true;
+            reply = await this.sendManaged(SYNC_CONSUMER_GROUP_CODE, payload, {
+                retryAfterReconnect: true
+            });
+            if (reply.byteLength === 0)
+                return undefined;
+        }
+        if (reply.byteLength < 12)
+            throw new ProtocolError("the consumer group assignment is truncated");
+        const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
+        const count = view.getUint32(8, true);
+        if (reply.byteLength !== 12 + count * 4)
+            throw new ProtocolError("the consumer group assignment length is invalid");
+        return {
+            generation: view.getBigUint64(0, true),
+            rejoined,
+            partitions: Array.from({ length: count }, (_, index) => view.getUint32(12 + index * 4, true))
+        };
     }
     async leaveConsumerGroup(streamId, topicId, name) {
-        await this.execute((client) => client.group.leave({ streamId, topicId, groupId: name }), `failed to leave consumer group \`${name}\``);
-        this.consumerGroups.delete(`${streamId}\0${topicId}\0${name}`);
+        await this.execute((client) => client.group.leave({ streamId, topicId, groupId: name }), `failed to leave consumer group \`${String(name)}\``);
+        this.consumerGroups.delete(`${streamId}\0${topicId}\0${String(name)}`);
+    }
+    async joinExistingConsumerGroup(streamId, topicId, name) {
+        await this.execute((client) => client.group.join({ streamId, topicId, groupId: name }), `failed to join consumer group \`${String(name)}\``);
+        this.consumerGroups.set(`${streamId}\0${topicId}\0${String(name)}`, {
+            streamId,
+            topicId,
+            name,
+            create: false
+        });
+    }
+    async openNodeConnection(ip, port) {
+        if (this.connection === undefined) {
+            throw new ConfigError("a node connection needs a Laser connected from a connection string, not an injected client");
+        }
+        const unspecified = ip.length === 0 || ip === "0.0.0.0" || ip === "::" || ip === "[::]";
+        const host = unspecified ? this.connection.host : ip;
+        const connected = await connectSimpleClient({
+            ...this.connection,
+            host,
+            port,
+            ...(isIP(this.connection.host) === 0 ? { servername: this.connection.host } : {})
+        }, Date.now() + connectOptions().timeoutMs, true);
+        const client = connected.client;
+        return {
+            async send(code, payload) {
+                const reply = await client.sendBinaryRequest(code, toNodeBuffer(payload));
+                return new Uint8Array(reply.buffer, reply.byteOffset, reply.byteLength);
+            },
+            async close() {
+                await client.destroy().catch(() => undefined);
+            }
+        };
+    }
+    async openCoordinator() {
+        if (this.connection === undefined) {
+            throw new ConfigError("a coordinator connection needs a Laser connected from a connection string, not an injected client");
+        }
+        const connected = await connectSimpleClient(this.connection, Date.now() + connectOptions().timeoutMs);
+        const coordinator = new ApacheIggyTransport(connected.client, this.connection, "owned", this.publishConfig);
+        coordinator.watch(connected);
+        return {
+            async send(code, payload) {
+                const reply = await coordinator.execute((client) => client.sendBinaryRequest(code, toNodeBuffer(payload)), `coordinator command ${String(code)} failed`);
+                return new Uint8Array(reply.buffer, reply.byteOffset, reply.byteLength);
+            },
+            async joinConsumerGroup(streamId, topicId, name) {
+                await coordinator.execute((client) => client.group.join({ streamId, topicId, groupId: name }), `failed to join consumer group \`${String(name)}\``);
+            },
+            async leaveConsumerGroup(streamId, topicId, name) {
+                await coordinator.execute((client) => client.group.leave({ streamId, topicId, groupId: name }), `failed to leave consumer group \`${String(name)}\``);
+            },
+            close: () => coordinator.close()
+        };
     }
     async close() {
         this.closed = true;