apache-iggy 0.10.1-edge.2 → 0.10.1-edge.3

package/README.md

@@ -22,7 +22,7 @@ note: previous works on node.js http client has been moved to [iggy-node-http-cl
 npm i --save apache-iggy@edge
 ```
 
-Use matching SDK and server builds from the coordinated VSR release. Older builds are incompatible. This release requires fresh server storage (`IGGY-RESOURCE-OPTIONS-1`). Existing incompatible data is refused unchanged. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples).
+Use matching SDK and server builds from the coordinated VSR release. Older builds are incompatible. This release requires fresh server storage (`IGGY-METADATA-CHANGE-LOG-1`). Existing incompatible data is refused unchanged. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples).
 
 ## basic usage
 
@@ -362,6 +362,69 @@ await client.topic.update({
 
 Reads return every setting in `options` (keys the client set) and `derivedOptions` (keys the server filled). `u64` values stay `bigint`, so values above 2^53 keep their precision. `system.describeOptions({ scope: OptionsScope.Stream })` lists the keys a server accepts per scope, with `updatable`, `resettable`, `secret`, `readOnly` and `critical`.
 
+### metadata changes
+
+`system.getMetadataChanges` reads the node's change log: one record per public result of a committed metadata operation, such as a stream created, a topic renamed, a user's permissions changed or a session closed. A mirror keeps a cursor, asks for the records after it, and stores the returned `next` cursor. The call needs root or the global `manage_servers` permission.
+
+```ts
+import { setTimeout as sleep } from "node:timers/promises";
+import {
+  Client, MetadataChangeStart, MetadataChangeStatus,
+  type MetadataChangeGroup, type MetadataCurrentPageToken,
+} from "apache-iggy";
+
+const follow = async (client: Client, groups: MetadataChangeGroup[]) => {
+  let start: MetadataChangeStart = MetadataChangeStart.Earliest;
+  for (;;) {
+    const page = await client.system.getMetadataChanges({ start, groups });
+    switch (page.status) {
+      case MetadataChangeStatus.Ok:
+        for (const record of page.records)
+          console.log(record.op, record.kind, record.actorUserId, record.body);
+        if (page.next) start = MetadataChangeStart.After(page.next);
+        if (!page.hasMore) await sleep(1000);
+        break;
+      case MetadataChangeStatus.Behind:
+        await sleep(100);
+        break;
+      default:
+        start = await rebuild(client, groups);
+    }
+  }
+};
+
+/** Pages the current state, then returns the position to follow from. */
+const rebuild = async (client: Client, groups: MetadataChangeGroup[]) => {
+  let pageToken: MetadataCurrentPageToken | undefined;
+  for (;;) {
+    const page = await client.system.getMetadataChanges({
+      start: MetadataChangeStart.Current(pageToken), groups,
+    });
+    if (page.status === MetadataChangeStatus.Behind) {
+      await sleep(100);
+      continue;
+    }
+    if (page.status !== MetadataChangeStatus.Ok) {
+      pageToken = undefined;
+      continue;
+    }
+    for (const item of page.items)
+      console.log(item.group, item.body);
+    if (page.currentPageToken) pageToken = page.currentPageToken;
+    else if (page.next) return MetadataChangeStart.After(page.next);
+    else pageToken = undefined;
+  }
+};
+```
+
+`expired` means the log no longer holds the history after the cursor, and `reset` means the cursor belongs to another history, for example after the data directory was rebuilt. Either way the mirror rebuilds from current state pages and then follows the changes after the anchor they return.
+
+Current pages need the parent groups of every selected group, for example `topics` with `streams`. Keep a rebuilt state private until a change page answers `ok` with `hasMore` false and a `headOp` at or above the last current page's.
+
+Records narrow by `kind` and items by `group`, so `record.kind === "stream_created"` types `record.body` as a stream. A group or kind newer than this build arrives as `"unknown"` with the raw body and the wire codes in `groupCode` and `kindCode`, while a record of a newer kind keeps its known `group`, so a mirror that cannot apply it can stop without advancing its cursor. Ops, timestamps and 128-bit ids are `bigint`.
+
+A query that breaks the request rules, such as a limit outside 1 to 1000 or a time bound with a latest or current start, is refused with error 3 before anything is sent. `MetadataChangeCursor` and `MetadataCurrentPageToken` print and parse as the `mc1_` and `cs1_` text forms the HTTP API and the CLI use, and `JSON.stringify` writes them as that text.
+
 ## use sources
 
 Cluster polling uses command 15 (`BindSession`), route discovery 105 and poll
@@ -398,7 +461,7 @@ npm run test:unit
 
 e2e test expect an iggy-server at tcp://127.0.0.1:8090
 
-The resource options tests also expect the server to run with encryption enabled and a config key (`IGGY_ENCRYPTION_ENABLED=true` and `IGGY_ENCRYPTION_KEY` set to the base64 of 32 bytes), so a stream created without a key reports a server-generated one.
+The resource options tests also expect the server to run with encryption enabled and a config key (`IGGY_ENCRYPTION_ENABLED=true` and `IGGY_ENCRYPTION_KEY` set to the base64 of 32 bytes), so a stream created without a key reports a server-generated one. The keyed metadata changes case registered from the same file needs the config key too.
 
 ```bash
 npm run test:e2e

package/dist/client/client.socket.js

@@ -67,8 +67,8 @@ const TOPIC_DISCOVERY_RESETS = new Set([
 ]);
 const READ_COMMANDS = new Set([
     COMMAND_CODE.Ping, COMMAND_CODE.GetStats, COMMAND_CODE.GetClusterMetadata,
-    COMMAND_CODE.DescribeOptions, COMMAND_CODE.GetMe, COMMAND_CODE.GetClient,
-    COMMAND_CODE.GetClients, COMMAND_CODE.GetUser, COMMAND_CODE.GetUsers,
+    COMMAND_CODE.DescribeOptions, COMMAND_CODE.GetMetadataChanges, COMMAND_CODE.GetMe,
+    COMMAND_CODE.GetClient, COMMAND_CODE.GetClients, COMMAND_CODE.GetUser, COMMAND_CODE.GetUsers,
     COMMAND_CODE.GetAccessTokens, COMMAND_CODE.GetPartitionRoute, COMMAND_CODE.GetOffset,
     COMMAND_CODE.GetStream, COMMAND_CODE.GetStreams, COMMAND_CODE.GetTopic,
     COMMAND_CODE.GetTopics, COMMAND_CODE.GetGroup, COMMAND_CODE.GetGroups, COMMAND_CODE.SyncGroup

package/dist/client/client.socket.test.js

@@ -29,6 +29,7 @@ import { UPDATE_USER } from '../wire/user/update-user.command.js';
 import { UPDATE_TOPIC } from '../wire/topic/update-topic.command.js';
 import { UPDATE_STREAM } from '../wire/stream/update-stream.command.js';
 import { OptionsScope } from '../wire/system/describe-options.command.js';
+import { MetadataChangeStart, getMetadataChanges } from '../wire/system/get-metadata-changes.command.js';
 import { CHANGE_PASSWORD } from '../wire/user/change-password.command.js';
 import { POLL_MESSAGES, pollMessages } from '../wire/message/poll-messages.command.js';
 import { SEND_MESSAGES, sendMessages } from '../wire/message/send-messages.command.js';
@@ -1251,6 +1252,35 @@ describe('VSR client socket', () => {
             await server.close();
         }
     });
+    it('resends a metadata changes read after an uncommitted response', async () => {
+        // A behind answer before op 1: status 1, phase 0, no log id, zero ops, no cursor.
+        const behind = Buffer.concat([Buffer.from([1, 0, 0]), Buffer.alloc(24), Buffer.alloc(2 + 4 + 4 + 2)]);
+        let attempts = 0;
+        const server = await startVsrServer((frame, socket) => {
+            const operation = frame.readUInt8(REQUEST_OFFSET.operation);
+            if (operation !== Operation.NonReplicated ||
+                frame.readUInt32LE(REQUEST_OFFSET.reserved) !== COMMAND_CODE.GetMetadataChanges)
+                return singleNodeHandler(server.port)(frame, socket);
+            attempts += 1;
+            socket.write(attempts === 1
+                ? replyFrame(frame, operation, Buffer.alloc(0), 57)
+                : replyFrame(frame, operation, behind));
+        });
+        const client = new CommandResponseStream(vsrConfig(server.port));
+        try {
+            const changes = await getMetadataChanges(async () => client)({ start: MetadataChangeStart.Latest });
+            assert.equal(attempts, 2);
+            assert.equal(changes.status, 'behind');
+            assert.equal(changes.next, null);
+            const sent = server.frames.filter((frame) => frame.readUInt32LE(REQUEST_OFFSET.reserved) === COMMAND_CODE.GetMetadataChanges);
+            assert.equal(sent.length, 2);
+            assert.deepEqual(sent[0], sent[1]);
+        }
+        finally {
+            client.destroy();
+            await server.close();
+        }
+    });
     it('routes eviction out of band without desynchronizing replies', async () => {
         const server = await startVsrServer((frame, socket) => {
             if (frame.readUInt32LE(REQUEST_OFFSET.reserved) === 60_003) {

package/dist/e2e/metadata-changes.e2e.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=metadata-changes.e2e.d.ts.map

package/dist/e2e/metadata-changes.e2e.js

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// The keyed stream case needs a server key, so resource-options.e2e.ts and
+// tls.system.e2e.ts register it, and the shipped-default lane skips both.
+import { Client } from '../client/client.js';
+import { getIggyAddress } from '../tcp.sm.utils.js';
+import { describeMetadataChanges } from './metadata-changes.suite.js';
+const [host, port] = getIggyAddress();
+describeMetadataChanges('e2e -> metadata changes', 'e2e-mc', (credentials) => new Client({
+    transport: 'TCP', options: { host, port }, credentials
+}));
+//# sourceMappingURL=metadata-changes.e2e.js.map

package/dist/e2e/metadata-changes.suite.d.ts

@@ -0,0 +1,17 @@
+import type { Client } from '../client/client.js';
+type Credentials = {
+    username: string;
+    password: string;
+};
+/**
+ * Registers the metadata changes cases for one transport. `connect` builds a
+ * client of that transport signed in with the given credentials.
+ */
+export declare const describeMetadataChanges: (label: string, prefix: string, connect: (credentials: Credentials) => Client, skip?: string | false) => Promise<void>;
+/**
+ * Registers the keyed stream case, which needs a server with a config key:
+ * a client key is refused without one.
+ */
+export declare const describeKeyedMetadataChanges: (label: string, prefix: string, connect: (credentials: Credentials) => Client, skip?: string | false) => Promise<void>;
+export {};
+//# sourceMappingURL=metadata-changes.suite.d.ts.map

package/dist/e2e/metadata-changes.suite.js

@@ -0,0 +1,328 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import { randomBytes } from 'node:crypto';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { after, describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { ResponseError } from '../wire/error.utils.js';
+import { EncryptionKeySource, MAX_METADATA_CHANGES_LIMIT, MetadataChangeCursor, MetadataChangeKind, MetadataChangeStart, MetadataChangeStatus, StreamEncryptionKey, UserStatus, encryptionKeyFingerprint } from '../wire/index.js';
+const NEVER_RETURNED = ['encryption_key', 'encryption_key_wrapped', 'critical_keys'];
+const FINGERPRINT = /^[0-9a-f]{16}$/;
+const INVALID_COMMAND = 3;
+const UNAUTHORIZED = 41;
+// A two phase delete commits its record after the call returns.
+const FOLLOW_BUDGET_MS = 15_000;
+const FOLLOW_INTERVAL_MS = 100;
+const latestCursor = async (c, groups) => {
+    const changes = await c.system.getMetadataChanges({ start: MetadataChangeStart.Latest, groups });
+    assert.equal(changes.status, MetadataChangeStatus.Ok);
+    assert.deepEqual(changes.records, []);
+    assert.ok(changes.next);
+    assert.equal(changes.next.logId, changes.logId);
+    return changes.next;
+};
+/** The stream a stream, topic or partitions record names, null for any other record. */
+const recordStream = (record) => {
+    switch (record.kind) {
+        case 'stream_created':
+        case 'stream_updated':
+        case 'stream_deleted':
+        case 'topic_created':
+        case 'topic_updated':
+        case 'topic_deleted':
+        case 'partitions_changed':
+            return { streamId: record.body.streamId, streamCreatedOp: record.body.streamCreatedOp };
+        default:
+            return null;
+    }
+};
+/**
+ * Follows the change log after `cursor` until `want` records name the stream
+ * the first record `introduces` accepts. Other e2e files run concurrently and
+ * stream ids are reused, so the stream key identifies the stream, not its id.
+ */
+const followStream = async (c, cursor, groups, introduces, want) => {
+    const records = [];
+    let identity;
+    const deadline = Date.now() + FOLLOW_BUDGET_MS;
+    for (;;) {
+        const changes = await c.system.getMetadataChanges({
+            start: MetadataChangeStart.After(cursor), groups, limit: MAX_METADATA_CHANGES_LIMIT
+        });
+        assert.equal(changes.status, MetadataChangeStatus.Ok);
+        assert.equal(changes.phase, 'changes');
+        assert.deepEqual(changes.items, []);
+        assert.equal(changes.currentPageToken, null);
+        assert.ok(changes.next);
+        cursor = changes.next;
+        for (const record of changes.records) {
+            assert.ok(groups.includes(record.group), `${record.group} was not requested`);
+            const stream = recordStream(record);
+            if (stream === null)
+                continue;
+            if (identity === undefined && introduces(record))
+                identity = stream;
+            if (identity !== undefined && stream.streamId === identity.streamId &&
+                stream.streamCreatedOp === identity.streamCreatedOp)
+                records.push(record);
+        }
+        if (changes.hasMore)
+            continue;
+        if (records.length >= want || Date.now() > deadline)
+            break;
+        await sleep(FOLLOW_INTERVAL_MS);
+    }
+    assert.equal(records.length, want);
+    return records;
+};
+/** Accepts the stream_created record of a stream by its id and name. */
+const createdStream = (streamId, name) => (record) => record.kind === 'stream_created' && record.body.streamId === streamId && record.body.name === name;
+/** Pages the streams state one item at a time, returning the names by id. */
+const currentStreams = async (c) => {
+    const streams = new Map();
+    let pageToken;
+    for (;;) {
+        const changes = await c.system.getMetadataChanges({
+            start: MetadataChangeStart.Current(pageToken), groups: ['streams'], limit: 1
+        });
+        assert.equal(changes.status, MetadataChangeStatus.Ok);
+        assert.equal(changes.phase, 'current');
+        assert.deepEqual(changes.records, []);
+        assert.ok(changes.items.length <= 1);
+        for (const item of changes.items) {
+            assert.equal(item.group, 'streams');
+            if (item.group === 'streams')
+                streams.set(item.body.streamId, item.body.name);
+        }
+        if (changes.currentPageToken === null) {
+            assert.ok(changes.next, 'the last page returns the anchor');
+            return streams;
+        }
+        assert.equal(changes.hasMore, true);
+        pageToken = changes.currentPageToken;
+    }
+};
+const isRefusedLocally = (code) => (error) => error instanceof ResponseError && error.errorCode === code && !error.fromServer;
+/**
+ * Registers the metadata changes cases for one transport. `connect` builds a
+ * client of that transport signed in with the given credentials.
+ */
+export const describeMetadataChanges = (label, prefix, connect, skip = false) => describe(label, { skip }, async () => {
+    const c = connect({ username: 'iggy', password: 'iggy' });
+    it('given latest cursor when creating renaming and deleting a stream should follow its records', async () => {
+        const cursor = await latestCursor(c, ['streams']);
+        const created = await c.stream.create({ name: `${prefix}-follow` });
+        assert.ok(await c.stream.update({ streamId: created.id, name: `${prefix}-follow-renamed` }));
+        assert.ok(await c.stream.delete({ streamId: created.id }));
+        const records = await followStream(c, cursor, ['streams'], createdStream(created.id, created.name), 3);
+        assert.deepEqual(records.map((record) => record.kind), [
+            MetadataChangeKind.StreamCreated, MetadataChangeKind.StreamUpdated, MetadataChangeKind.StreamDeleted
+        ]);
+        for (const record of records) {
+            assert.equal(record.actorUserId, 0);
+            assert.equal(record.group, 'streams');
+            assert.ok(record.checksum > 0n);
+            assert.ok(record.timestamp > 0n);
+        }
+        const [createdRecord, updatedRecord, deletedRecord] = records;
+        assert.ok(createdRecord.kind === 'stream_created' && updatedRecord.kind === 'stream_updated' &&
+            deletedRecord.kind === 'stream_deleted');
+        assert.equal(createdRecord.body.name, `${prefix}-follow`);
+        assert.equal(updatedRecord.body.name, `${prefix}-follow-renamed`);
+        assert.equal(deletedRecord.body.name, `${prefix}-follow-renamed`);
+        assert.equal(createdRecord.body.streamCreatedOp, createdRecord.op);
+        assert.equal(createdRecord.body.options.unreadable, false);
+        assert.ok(createdRecord.op < updatedRecord.op && updatedRecord.op < deletedRecord.op);
+    });
+    it('given topics group when creating a stream and topic should return only the topic record', async () => {
+        const cursor = await latestCursor(c, ['topics']);
+        const stream = await c.stream.create({ name: `${prefix}-filter` });
+        try {
+            const topic = await c.topic.create({ streamId: stream.id, name: 'events', partitionCount: 2 });
+            const introduces = (record) => record.kind === 'topic_created' && record.body.streamId === stream.id && record.body.topicId === topic.id;
+            const records = await followStream(c, cursor, ['topics'], introduces, 1);
+            const [record] = records;
+            assert.equal(record.kind, MetadataChangeKind.TopicCreated);
+            assert.ok(record.kind === 'topic_created');
+            assert.equal(record.body.topicId, topic.id);
+            assert.equal(record.body.name, 'events');
+            assert.equal(record.body.partitionsCount, 2);
+            assert.equal(record.body.topicCreatedOp, record.op);
+        }
+        finally {
+            await c.stream.delete({ streamId: stream.id });
+        }
+    });
+    it('given limit of one when following three streams should page one record at a time', async () => {
+        let cursor = await latestCursor(c, ['streams']);
+        const created = new Map();
+        try {
+            for (const index of [1, 2, 3]) {
+                const stream = await c.stream.create({ name: `${prefix}-page-${index}` });
+                created.set(stream.id, stream.name);
+            }
+            const seen = new Map();
+            const deadline = Date.now() + FOLLOW_BUDGET_MS;
+            while (seen.size < created.size) {
+                assert.ok(Date.now() < deadline, `paged ${[...seen.values()]} of ${[...created.values()]}`);
+                const changes = await c.system.getMetadataChanges({
+                    start: MetadataChangeStart.After(cursor), groups: ['streams'], limit: 1
+                });
+                assert.equal(changes.status, MetadataChangeStatus.Ok);
+                assert.ok(changes.records.length <= 1);
+                assert.ok(changes.next);
+                for (const record of changes.records) {
+                    if (record.kind === 'stream_created' && created.get(record.body.streamId) === record.body.name)
+                        seen.set(record.body.streamId, record.body.name);
+                    assert.equal(record.op, changes.next.op);
+                    assert.equal(record.ordinal, changes.next.ordinal);
+                }
+                assert.ok(changes.next.op >= cursor.op);
+                cursor = changes.next;
+                if (!changes.hasMore && seen.size < created.size)
+                    await sleep(FOLLOW_INTERVAL_MS);
+            }
+            assert.deepEqual(seen, created);
+        }
+        finally {
+            for (const streamId of created.keys())
+                await c.stream.delete({ streamId });
+        }
+    });
+    it('given streams when paging current state one item at a time should list the streams the list returns', async () => {
+        const created = [
+            await c.stream.create({ name: `${prefix}-current-1` }),
+            await c.stream.create({ name: `${prefix}-current-2` })
+        ];
+        try {
+            const before = await c.stream.list();
+            const projected = await currentStreams(c);
+            const afterwards = await c.stream.list();
+            for (const stream of created)
+                assert.equal(projected.get(stream.id), stream.name);
+            // Other e2e files change streams meanwhile: a stream listed unchanged
+            // before and after the scan must be in it.
+            for (const stream of before)
+                if (afterwards.some((later) => later.id === stream.id && later.name === stream.name))
+                    assert.equal(projected.get(stream.id), stream.name, `stream ${stream.id}`);
+        }
+        finally {
+            for (const stream of created)
+                await c.stream.delete({ streamId: stream.id });
+        }
+    });
+    it('given queries breaking the request rules when reading should refuse before sending', async () => {
+        const created = [
+            await c.stream.create({ name: `${prefix}-token-1` }),
+            await c.stream.create({ name: `${prefix}-token-2` })
+        ];
+        try {
+            const first = await c.system.getMetadataChanges({
+                start: MetadataChangeStart.Current(), groups: ['streams'], limit: 1
+            });
+            assert.ok(first.currentPageToken);
+            const refused = [
+                { limit: 0 },
+                { limit: MAX_METADATA_CHANGES_LIMIT + 1 },
+                { since: 2n, until: 1n },
+                { start: MetadataChangeStart.Latest, since: 1n },
+                { start: MetadataChangeStart.Current(), until: 1n },
+                { start: MetadataChangeStart.Current(), groups: ['topics'] },
+                { start: MetadataChangeStart.Current(first.currentPageToken), groups: ['streams', 'users'] }
+            ];
+            for (const query of refused)
+                await assert.rejects(c.system.getMetadataChanges(query), isRefusedLocally(INVALID_COMMAND));
+            assert.throws(() => MetadataChangeCursor.parse('mc1_x'), isRefusedLocally(INVALID_COMMAND));
+            const latest = await c.system.getMetadataChanges({ start: MetadataChangeStart.Latest });
+            assert.equal(latest.status, MetadataChangeStatus.Ok, 'the connection stays usable');
+        }
+        finally {
+            for (const stream of created)
+                await c.stream.delete({ streamId: stream.id });
+        }
+    });
+    it('given user without manage servers when reading changes should be refused as unauthorized', async () => {
+        const password = 'changes-secret';
+        const user = await c.user.create({
+            username: `${prefix}-reader`, password, status: UserStatus.Active,
+            permissions: {
+                global: {
+                    ManageServers: false, ReadServers: true, ManageUsers: false, ReadUsers: true,
+                    ManageStreams: false, ReadStreams: true, ManageTopics: false, ReadTopics: true,
+                    PollMessages: false, SendMessages: false
+                },
+                streams: []
+            }
+        });
+        const reader = connect({ username: `${prefix}-reader`, password });
+        try {
+            await assert.rejects(reader.system.getMetadataChanges({}), (error) => error instanceof ResponseError && error.errorCode === UNAUTHORIZED && error.fromServer);
+        }
+        finally {
+            await reader.destroy();
+            await c.user.delete({ userId: user.id });
+        }
+    });
+    after(() => {
+        c.destroy();
+    });
+});
+/**
+ * Registers the keyed stream case, which needs a server with a config key:
+ * a client key is refused without one.
+ */
+export const describeKeyedMetadataChanges = (label, prefix, connect, skip = false) => describe(label, { skip }, async () => {
+    const c = connect({ username: 'iggy', password: 'iggy' });
+    it('given keyed stream when following its creation should expose fingerprints and no key', async () => {
+        const cursor = await latestCursor(c, ['streams']);
+        const key = randomBytes(32);
+        const created = await c.stream.create({
+            name: `${prefix}-keyed`, encryptionKey: StreamEncryptionKey.from(key)
+        });
+        try {
+            const [record] = await followStream(c, cursor, ['streams'], createdStream(created.id, created.name), 1);
+            assert.equal(record.kind, MetadataChangeKind.StreamCreated);
+            assert.ok(record.kind === 'stream_created');
+            const { options } = record.body;
+            assert.equal(options.unreadable, false);
+            assert.ok(!options.unreadable);
+            const text = (name) => options.derivedOptions[name] ?? options.options[name];
+            assert.equal(text('encryption_key_source'), EncryptionKeySource.Stream);
+            assert.equal(text('encryption_key_fingerprint'), encryptionKeyFingerprint(key));
+            const keks = String(text('encryption_kek_fingerprint')).split(',');
+            assert.deepEqual(keks, created.encryptionKekFingerprints);
+            for (const kek of keks)
+                assert.match(kek, FINGERPRINT);
+            for (const block of [options.options, options.derivedOptions]) {
+                for (const name of NEVER_RETURNED)
+                    assert.equal(name in block, false, `${name} was returned`);
+                for (const value of Object.values(block)) {
+                    assert.ok(!(Buffer.isBuffer(value) && value.includes(key)), 'an option holds the key');
+                    for (const encoded of [key.toString('base64'), key.toString('base64url'), key.toString('hex')])
+                        assert.ok(!String(value).includes(encoded), 'an option holds the encoded key');
+                }
+            }
+        }
+        finally {
+            await c.stream.delete({ streamId: created.id });
+        }
+    });
+    after(() => {
+        c.destroy();
+    });
+});
+//# sourceMappingURL=metadata-changes.suite.js.map

package/dist/e2e/resource-options.e2e.js

@@ -20,9 +20,13 @@
 import { Client } from '../client/client.js';
 import { getIggyAddress } from '../tcp.sm.utils.js';
 import { EncryptionKeySource } from '../wire/index.js';
+import { describeKeyedMetadataChanges } from './metadata-changes.suite.js';
 import { describeResourceOptions } from './resource-options.suite.js';
 const [host, port] = getIggyAddress();
 describeResourceOptions('e2e -> resource options', 'e2e-ro', (credentials) => new Client({
     transport: 'TCP', options: { host, port }, credentials
 }), EncryptionKeySource.Server);
+describeKeyedMetadataChanges('e2e -> keyed metadata changes', 'e2e-mc', (credentials) => new Client({
+    transport: 'TCP', options: { host, port }, credentials
+}));
 //# sourceMappingURL=resource-options.e2e.js.map

package/dist/e2e/tls.system.e2e.js

@@ -39,6 +39,7 @@ import assert from 'node:assert/strict';
 import { Client } from '../client/client.js';
 import { Partitioning, Consumer, EncryptionKeySource, PollingStrategy } from '../wire/index.js';
 import { getIggyAddress } from '../tcp.sm.utils.js';
+import { describeKeyedMetadataChanges, describeMetadataChanges } from './metadata-changes.suite.js';
 import { describeResourceOptions } from './resource-options.suite.js';
 const tlsEnabled = process.env.IGGY_TCP_TLS_ENABLED === 'true';
 // Path to the CA certificate. Override with E2E_ROOT_CA_CERT env var,
@@ -63,6 +64,8 @@ const getTlsClient = (credentials = { username: 'iggy', password: 'iggy' }) => {
     });
 };
 describeResourceOptions('e2e -> tls resource options', 'tls-e2e-ro', getTlsClient, EncryptionKeySource.None, !tlsEnabled && 'IGGY_TCP_TLS_ENABLED is not set');
+describeMetadataChanges('e2e -> tls metadata changes', 'tls-e2e-mc', getTlsClient, !tlsEnabled && 'IGGY_TCP_TLS_ENABLED is not set');
+describeKeyedMetadataChanges('e2e -> tls keyed metadata changes', 'tls-e2e-mc', getTlsClient, !tlsEnabled && 'IGGY_TCP_TLS_ENABLED is not set');
 describe('e2e -> tls', { skip: !tlsEnabled && 'IGGY_TCP_TLS_ENABLED is not set' }, async () => {
     const c = getTlsClient();
     const credentials = { username: 'iggy', password: 'iggy' };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export { type Id, PollingStrategy, Consumer, Partitioning, HeaderValue, HeaderKeyFactory, type SendMessagesConfirmation, type SendMessagesResponse, CompressionAlgorithm, Durability, PartitionResizePolicy, EncryptionKeySource, OptionsScope, StreamOptionKey, TopicOptionKey, UserOptionKey, UserStatus, StreamEncryptionKey, encryptionKeyFingerprint, type OptionEntry, type OptionSpec, type ParsedOptions, type StreamOptions, type TopicOptions, type UserOptions, } from "./wire/index.js";
+export { type Id, PollingStrategy, Consumer, Partitioning, HeaderValue, HeaderKeyFactory, type SendMessagesConfirmation, type SendMessagesResponse, CompressionAlgorithm, Durability, PartitionResizePolicy, EncryptionKeySource, OptionsScope, StreamOptionKey, TopicOptionKey, UserOptionKey, UserStatus, StreamEncryptionKey, encryptionKeyFingerprint, type OptionEntry, type OptionSpec, type ParsedOptions, type StreamOptions, type TopicOptions, type UserOptions, MetadataChangeGroup, MetadataChangeKind, MetadataChangeStatus, MetadataChangePhase, MetadataMemberLeftReason, MetadataTokenDeletedReason, MetadataSessionClosedReason, MetadataPermissionsUpdatedReason, MetadataChangeStart, MetadataChangeCursor, MetadataCurrentPageToken, DEFAULT_METADATA_CHANGES_LIMIT, MAX_METADATA_CHANGES_LIMIT, METADATA_CHANGE_CURSOR_PREFIX, METADATA_CURRENT_PAGE_TOKEN_PREFIX, metadataChangeGroupName, metadataChangeKindName, metadataChangeKindGroup, type GetMetadataChanges, type MetadataChanges, type MetadataChange, type MetadataChangeRecord, type MetadataCurrentItem, type MetadataOptions, type MetadataStreamKey, type MetadataTopicKey, type MetadataUserKey, type MetadataStreamChange, type MetadataStreamDeleted, type MetadataTopicChange, type MetadataTopicItem, type MetadataTopicDeleted, type MetadataPartitionsChanged, type MetadataPartitionsItem, type MetadataConsumerGroupCreated, type MetadataConsumerGroupDeleted, type MetadataConsumerGroupItem, type MetadataMemberJoined, type MetadataMemberLeft, type MetadataPartitionHandedOver, type MetadataMemberItem, type MetadataUserChange, type MetadataUserIdentity, type MetadataUserPermissionsUpdated, type MetadataTokenChange, type MetadataTokenDeleted, type MetadataSession, type MetadataSessionClosed, } from "./wire/index.js";
 export * from "./client/index.js";
 export * from "./stream/index.js";
 export { DeserializeError, ResponseError } from './wire/error.utils.js';

package/dist/index.js

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-export { PollingStrategy, Consumer, Partitioning, HeaderValue, HeaderKeyFactory, CompressionAlgorithm, Durability, PartitionResizePolicy, EncryptionKeySource, OptionsScope, StreamOptionKey, TopicOptionKey, UserOptionKey, UserStatus, StreamEncryptionKey, encryptionKeyFingerprint, } from "./wire/index.js";
+export { PollingStrategy, Consumer, Partitioning, HeaderValue, HeaderKeyFactory, CompressionAlgorithm, Durability, PartitionResizePolicy, EncryptionKeySource, OptionsScope, StreamOptionKey, TopicOptionKey, UserOptionKey, UserStatus, StreamEncryptionKey, encryptionKeyFingerprint, MetadataChangeGroup, MetadataChangeKind, MetadataChangeStatus, MetadataChangePhase, MetadataMemberLeftReason, MetadataTokenDeletedReason, MetadataSessionClosedReason, MetadataPermissionsUpdatedReason, MetadataChangeStart, MetadataChangeCursor, MetadataCurrentPageToken, DEFAULT_METADATA_CHANGES_LIMIT, MAX_METADATA_CHANGES_LIMIT, METADATA_CHANGE_CURSOR_PREFIX, METADATA_CURRENT_PAGE_TOKEN_PREFIX, metadataChangeGroupName, metadataChangeKindName, metadataChangeKindGroup, } from "./wire/index.js";
 export * from "./client/index.js";
 export * from "./stream/index.js";
 export { DeserializeError, ResponseError } from './wire/error.utils.js';

package/dist/wire/command-set.d.ts

@@ -81,6 +81,7 @@ declare const systemAPI: (c: ClientProvider) => {
     getStats: (arg: void) => Promise<import("./system/get-stats.command.js").Stats>;
     snapshot: (arg: void | import("./system/snapshot.command.js").SnapshotOptions) => Promise<Buffer<ArrayBufferLike>>;
     describeOptions: (arg: import("./system/describe-options.command.js").DescribeOptions) => Promise<import("./system/describe-options.command.js").OptionSpec[]>;
+    getMetadataChanges: (arg: import("./system/get-metadata-changes.command.js").GetMetadataChanges) => Promise<import("./system/get-metadata-changes.command.js").MetadataChanges>;
 };
 type SystemAPI = ReturnType<typeof systemAPI>;
 declare const clusterAPI: (c: ClientProvider) => {

package/dist/wire/command-set.js

@@ -50,6 +50,7 @@ import { createPartition } from './partition/create-partition.command.js';
 import { deletePartition } from './partition/delete-partition.command.js';
 import { deleteSegments } from './segment/delete-segments.command.js';
 import { describeOptions } from './system/describe-options.command.js';
+import { getMetadataChanges } from './system/get-metadata-changes.command.js';
 import { getStats } from './system/get-stats.command.js';
 import { snapshot } from './system/snapshot.command.js';
 import { ping } from './system/ping.command.js';
@@ -134,7 +135,8 @@ const systemAPI = (c) => ({
     ping: ping(c),
     getStats: getStats(c),
     snapshot: snapshot(c),
-    describeOptions: describeOptions(c)
+    describeOptions: describeOptions(c),
+    getMetadataChanges: getMetadataChanges(c)
 });
 const clusterAPI = (c) => ({
     getClusterMetadata: getClusterMetadata(c)

package/dist/wire/command.code.d.ts

@@ -6,6 +6,7 @@ export declare const COMMAND_CODE: {
     DescribeOptions: number;
     BindSession: number;
     UpdateResource: number;
+    GetMetadataChanges: number;
     GetMe: number;
     GetClient: number;
     GetClients: number;

package/dist/wire/command.code.js

@@ -23,6 +23,7 @@ export const COMMAND_CODE = {
     DescribeOptions: 13,
     BindSession: 15,
     UpdateResource: 16,
+    GetMetadataChanges: 17,
     GetMe: 20,
     GetClient: 21,
     GetClients: 22,