apache-iggy 0.10.0-edge.6 → 0.10.0-edge.7

package/dist/wire/message/send-messages.command.d.ts

@@ -28,9 +28,8 @@ export type SendMessagesConfirmation = {
      *
      * Delivery is at-least-once, so an earlier retry of the same batch may
      * already have committed at a lower offset: this never identifies a batch
-     * uniquely. A batch is confirmed once it is committed in memory, not once it
-     * is fsynced, so a crash-restart can stamp a later batch with an offset a
-     * client has already recorded.
+     * uniquely. Confirmation follows VSR quorum commit. Persisted message
+     * durability also requires recoverable stable-storage copies on the quorum.
      */
     baseOffset: bigint;
 };

package/dist/wire/options.utils.test.js

@@ -28,22 +28,25 @@ const prefixed = (block) => {
  *
  * Rust pins the identical bytes in `core/binary_protocol/src/primitives/options.rs`,
  * as do the Go and Java SDKs. Round-tripping through this SDK's own decoder proves
- * nothing about interoperability; these bytes are the contract.
+ * nothing about interoperability. This insertion-order vector covers Bool, Uint64
+ * and String values, independently of Rust's sorted map order.
  */
 const GOLDEN_OPTIONS_BLOCK = Buffer.from([
-    2, 13, 0, 0, 0,
-    ...Buffer.from('enforce_fsync'),
+    2, 20, 0, 0, 0,
+    ...Buffer.from('preallocate_segments'),
     3, 1, 0, 0, 0, 1,
     2, 12, 0, 0, 0,
     ...Buffer.from('segment_size'),
     12, 8, 0, 0, 0,
-    0, 0, 0, 64, 0, 0, 0, 0
+    0, 0, 0, 64, 0, 0, 0, 0,
+    2, 10, 0, 0, 0, 100, 117, 114, 97, 98, 105, 108, 105, 116, 121, 2, 9, 0, 0, 0, 112, 101, 114, 115, 105, 115, 116, 101, 100
 ]);
 describe('serializeOptions', () => {
     it('encodes the cross-SDK golden vector byte for byte', () => {
         const encoded = serializeOptions([
-            { key: 'enforce_fsync', value: HeaderValue.Bool(true) },
-            { key: 'segment_size', value: HeaderValue.Uint64(1073741824n) }
+            { key: 'preallocate_segments', value: HeaderValue.Bool(true) },
+            { key: 'segment_size', value: HeaderValue.Uint64(1073741824n) },
+            { key: 'durability', value: HeaderValue.String('persisted') }
         ]);
         assert.deepEqual(encoded, GOLDEN_OPTIONS_BLOCK);
     });

package/dist/wire/topic/create-topic.command.d.ts

@@ -1,7 +1,7 @@
 import type { CommandResponse } from '../../client/client.type.js';
 import { type Id } from '../identifier.utils.js';
 import { type OptionEntry } from '../options.utils.js';
-import { type Topic, type CompressionAlgorithm as CompressionAlgorithmT } from './topic.utils.js';
+import { Durability, type Topic, type CompressionAlgorithm as CompressionAlgorithmT } from './topic.utils.js';
 /**
  * Parameters for the create topic command.
  */
@@ -28,8 +28,9 @@ export type CreateTopic = {
     maxTopicSize?: bigint;
     /** Segment size in bytes: 512-byte multiple between 1 MiB and 1 GiB */
     segmentSize?: bigint;
-    /** Fsync every write instead of leaving it to the page cache */
-    enforceFsync?: boolean;
+    /** Message completion policy. Defaults to replicated, independently of offsets. */
+    durability?: Durability;
+    consumerOffsetDurability?: Durability;
     /** Message count that triggers a save (must be non-zero) */
     messagesRequiredToSave?: number;
     /** Accumulated message bytes that trigger a save */
@@ -52,7 +53,7 @@ export type CreateTopic = {
  */
 export declare const CREATE_TOPIC: {
     code: number;
-    serialize: ({ streamId, name, partitionCount, compressionAlgorithm, messageExpiry, maxTopicSize, segmentSize, enforceFsync, messagesRequiredToSave, sizeOfMessagesRequiredToSave, preallocateSegments, options: extraOptions }: CreateTopic) => Buffer<ArrayBuffer>;
+    serialize: ({ streamId, name, partitionCount, compressionAlgorithm, messageExpiry, maxTopicSize, segmentSize, durability, consumerOffsetDurability, messagesRequiredToSave, sizeOfMessagesRequiredToSave, preallocateSegments, options: extraOptions }: CreateTopic) => Buffer<ArrayBuffer>;
     deserialize: (r: CommandResponse) => Topic;
 };
 /**

package/dist/wire/topic/create-topic.command.js

@@ -19,7 +19,7 @@ import { wrapCommand } from '../command.utils.js';
 import { COMMAND_CODE } from '../command.code.js';
 import { dedupeOptions, serializeOptions } from '../options.utils.js';
 import { HeaderValue } from '../message/header.utils.js';
-import { isValidCompressionAlgorithm, CompressionAlgorithm, compressionAlgorithmName, deserializeTopic } from './topic.utils.js';
+import { isValidCompressionAlgorithm, CompressionAlgorithm, Durability, compressionAlgorithmName, deserializeTopic } from './topic.utils.js';
 /**
  * Create topic command definition.
  * Creates a new topic within a stream.
@@ -28,7 +28,7 @@ import { isValidCompressionAlgorithm, CompressionAlgorithm, compressionAlgorithm
  */
 export const CREATE_TOPIC = {
     code: COMMAND_CODE.CreateTopic,
-    serialize: ({ streamId, name, partitionCount, compressionAlgorithm = CompressionAlgorithm.None, messageExpiry = 0n, maxTopicSize = 0n, segmentSize, enforceFsync, messagesRequiredToSave, sizeOfMessagesRequiredToSave, preallocateSegments, options: extraOptions = [] }) => {
+    serialize: ({ streamId, name, partitionCount, compressionAlgorithm = CompressionAlgorithm.None, messageExpiry = 0n, maxTopicSize = 0n, segmentSize, durability = Durability.Replicated, consumerOffsetDurability = Durability.Replicated, messagesRequiredToSave, sizeOfMessagesRequiredToSave, preallocateSegments, options: extraOptions = [] }) => {
         // Topic ID is now auto-assigned by the server, not sent in the protocol
         const streamIdentifier = serializeIdentifier(streamId);
         const bName = Buffer.from(name);
@@ -60,10 +60,19 @@ export const CREATE_TOPIC = {
             options.push({
                 key: 'segment_size', value: HeaderValue.Uint64(segmentSize)
             });
-        if (enforceFsync !== undefined)
-            options.push({
-                key: 'enforce_fsync', value: HeaderValue.Bool(enforceFsync)
-            });
+        for (const [key, policy] of [
+            ['durability', durability],
+            ['consumer_offset_durability', consumerOffsetDurability]
+        ]) {
+            if (policy !== Durability.Replicated && policy !== Durability.Persisted)
+                throw new Error(`Invalid ${key}: ${policy}`);
+            const expected = HeaderValue.String(policy);
+            for (const entry of extraOptions) {
+                if (entry.key === key && (entry.value.kind !== expected.kind || entry.value.value !== expected.value))
+                    throw new Error(`Conflicting ${key}`);
+            }
+            options.push({ key, value: expected });
+        }
         if (messagesRequiredToSave !== undefined)
             options.push({
                 key: 'messages_required_to_save',

package/dist/wire/topic/create-topic.command.test.js

@@ -16,6 +16,7 @@
 // under the License.
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
+import { Durability } from './topic.utils.js';
 import { CREATE_TOPIC } from './create-topic.command.js';
 import { deserializeOptions } from '../options.utils.js';
 import { HeaderValue } from '../message/header.utils.js';
@@ -32,17 +33,19 @@ describe('CreateTopic', () => {
         // TLV field: [kind:u8][len:u32_le][bytes]
         const tlvSize = (bytes) => 1 + 4 + bytes;
         const identifierSize = 1 + 1 + 4; // numeric stream id
+        const defaultPolicySize = tlvSize('durability'.length) + tlvSize('replicated'.length)
+            + tlvSize('consumer_offset_durability'.length) + tlvSize('replicated'.length);
         const fixedSize = identifierSize + 4 + 1; // + partitions_count + name_len
         it('serialize name and default options into buffer', () => {
-            // Server-default sentinels are omitted, leaving an empty options block.
-            assert.deepEqual(CREATE_TOPIC.serialize(t1).length, fixedSize + t1.name.length);
+            // Durability defaults are explicit, while the other sentinels are omitted.
+            assert.deepEqual(CREATE_TOPIC.serialize(t1).length, fixedSize + t1.name.length + defaultPolicySize);
         });
         it('serialize partitionCount as a fixed u32 before the name', () => {
             const t = { ...t1, partitionCount: 7 };
             const b = CREATE_TOPIC.serialize(t);
             assert.equal(b.readUInt32LE(identifierSize), 7);
             assert.equal(b.readUInt8(identifierSize + 4), t.name.length);
-            assert.equal(b.subarray(fixedSize).toString(), t.name);
+            assert.equal(b.subarray(fixedSize, fixedSize + t.name.length).toString(), t.name);
         });
         it('serialize non-default options into buffer', () => {
             const t = {
@@ -51,7 +54,7 @@ describe('CreateTopic', () => {
                 messageExpiry: 42n,
                 maxTopicSize: 1024n
             };
-            assert.deepEqual(CREATE_TOPIC.serialize(t).length, fixedSize + t1.name.length
+            assert.deepEqual(CREATE_TOPIC.serialize(t).length, fixedSize + t1.name.length + defaultPolicySize
                 + tlvSize('compression_algorithm'.length) + tlvSize('gzip'.length)
                 + tlvSize('message_expiry'.length) + tlvSize(8)
                 + tlvSize('max_topic_size'.length) + tlvSize(8));
@@ -60,14 +63,14 @@ describe('CreateTopic', () => {
             const t = {
                 ...t1,
                 segmentSize: 1048576n,
-                enforceFsync: true,
+                durability: Durability.Persisted,
                 messagesRequiredToSave: 1000,
                 sizeOfMessagesRequiredToSave: 4096n,
                 preallocateSegments: false
             };
-            assert.deepEqual(CREATE_TOPIC.serialize(t).length, fixedSize + t1.name.length
+            assert.deepEqual(CREATE_TOPIC.serialize(t).length, fixedSize + t1.name.length + defaultPolicySize
                 + tlvSize('segment_size'.length) + tlvSize(8)
-                + tlvSize('enforce_fsync'.length) + tlvSize(1)
+                + 'persisted'.length - 'replicated'.length
                 + tlvSize('messages_required_to_save'.length) + tlvSize(4)
                 + tlvSize('size_of_messages_required_to_save'.length) + tlvSize(8)
                 + tlvSize('preallocate_segments'.length) + tlvSize(1));
@@ -77,7 +80,7 @@ describe('CreateTopic', () => {
                 ...t1,
                 maxTopicSize: 4096n,
                 options: [
-                    { key: 'enforce_fsync', value: HeaderValue.Bool(true) },
+                    { key: 'preallocate_segments', value: HeaderValue.Bool(true) },
                     // The typed field covers this key, so the caller's entry is dropped:
                     // a duplicate key makes the server refuse the whole block.
                     { key: 'max_topic_size', value: HeaderValue.String('1 GiB') }
@@ -86,10 +89,27 @@ describe('CreateTopic', () => {
             const b = CREATE_TOPIC.serialize(t);
             // The create payload runs its options block to the end, unprefixed.
             const options = deserializeOptions(b, fixedSize + t.name.length);
-            assert.deepEqual(Object.keys(options).sort(), ['enforce_fsync', 'max_topic_size']);
-            assert.equal(options.enforce_fsync, true);
+            assert.deepEqual(Object.keys(options).sort(), ['consumer_offset_durability', 'durability', 'max_topic_size', 'preallocate_segments']);
+            assert.equal(options.preallocate_segments, true);
             assert.equal(options.max_topic_size, 4096n);
         });
+        it('keeps each omitted durability policy replicated', () => {
+            for (const selected of [
+                { durability: Durability.Persisted },
+                { consumerOffsetDurability: Durability.Persisted }
+            ]) {
+                const input = { ...t1, ...selected };
+                const encoded = CREATE_TOPIC.serialize(input);
+                const options = deserializeOptions(encoded, fixedSize + input.name.length);
+                assert.equal(options.durability, selected.durability ?? 'replicated');
+                assert.equal(options.consumer_offset_durability, selected.consumerOffsetDurability ?? 'replicated');
+            }
+        });
+        it('rejects a conflicting raw durability instead of weakening it', () => {
+            assert.throws(() => CREATE_TOPIC.serialize({ ...t1, options: [
+                    { key: 'durability', value: HeaderValue.String('persisted') }
+                ] }));
+        });
         it('throw on name < 1', () => {
             const t = { ...t1, name: '' };
             assert.throws(() => CREATE_TOPIC.serialize(t));

package/dist/wire/topic/index.d.ts

@@ -5,5 +5,5 @@ export * from './get-topics.command.js';
 export * from './purge-topic.command.js';
 export * from './update-topic.command.js';
 export * from './ensure-topic.virtual.command.js';
-export { CompressionAlgorithm } from './topic.utils.js';
+export { CompressionAlgorithm, Durability } from './topic.utils.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/wire/topic/index.js

@@ -21,5 +21,5 @@ export * from './get-topics.command.js';
 export * from './purge-topic.command.js';
 export * from './update-topic.command.js';
 export * from './ensure-topic.virtual.command.js';
-export { CompressionAlgorithm } from './topic.utils.js';
+export { CompressionAlgorithm, Durability } from './topic.utils.js';
 //# sourceMappingURL=index.js.map

package/dist/wire/topic/topic.utils.d.ts

@@ -138,5 +138,10 @@ export declare const deserializeTopic: (p: Buffer, pos?: number) => TopicSeriali
  * @returns Array of deserialized topics
  */
 export declare const deserializeTopics: (p: Buffer, pos?: number) => Topic[];
+export declare const Durability: {
+    readonly Replicated: "replicated";
+    readonly Persisted: "persisted";
+};
+export type Durability = typeof Durability[keyof typeof Durability];
 export {};
 //# sourceMappingURL=topic.utils.d.ts.map

package/dist/wire/topic/topic.utils.js

@@ -146,4 +146,8 @@ export const deserializeTopics = (p, pos = 0) => {
     }
     return topics;
 };
+export const Durability = {
+    Replicated: 'replicated',
+    Persisted: 'persisted'
+};
 //# sourceMappingURL=topic.utils.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "apache-iggy",
   "type": "module",
-  "version": "0.10.0-edge.6",
+  "version": "0.10.0-edge.7",
   "description": "Official Apache Iggy NodeJS SDK",
   "keywords": [
     "iggy",