apache-iggy 0.10.0-edge.5 → 0.10.0-edge.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/wire/error.code.js +1 -0
- package/dist/wire/error.code.test.js +3 -0
- package/dist/wire/message/message.utils.d.ts +14 -0
- package/dist/wire/message/message.utils.js +34 -6
- package/dist/wire/message/message.utils.test.d.ts +2 -0
- package/dist/wire/message/message.utils.test.js +107 -0
- package/dist/wire/number.utils.d.ts +10 -5
- package/dist/wire/number.utils.js +16 -8
- package/dist/wire/number.utils.test.d.ts +2 -0
- package/dist/wire/number.utils.test.js +55 -0
- package/dist/wire/vsr/header.d.ts +4 -2
- package/dist/wire/vsr/header.js +4 -2
- package/dist/wire/vsr/index.js +5 -4
- package/dist/wire/vsr/operation.d.ts +0 -2
- package/dist/wire/vsr/operation.js +0 -3
- package/dist/wire/vsr/operation.test.js +1 -3
- package/dist/wire/vsr/session.d.ts +3 -3
- package/dist/wire/vsr/session.js +3 -3
- package/dist/wire/vsr/vsr.test.js +15 -4
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -161,7 +161,7 @@ npm run build
|
|
|
161
161
|
### test
|
|
162
162
|
|
|
163
163
|
note: use env var `IGGY_TCP_ADDRESS="host:port"` to set the server
|
|
164
|
-
address for bdd
|
|
164
|
+
address for e2e tests. bdd tests need more variables, see below.
|
|
165
165
|
|
|
166
166
|
#### unit tests
|
|
167
167
|
|
|
@@ -179,15 +179,27 @@ npm run test:e2e
|
|
|
179
179
|
|
|
180
180
|
#### bdd tests
|
|
181
181
|
|
|
182
|
-
bdd
|
|
182
|
+
the bdd suite has no defaults and fails when `IGGY_TCP_ADDRESS`,
|
|
183
|
+
`IGGY_ROOT_USERNAME` or `IGGY_ROOT_PASSWORD` is missing. from the repository
|
|
184
|
+
root run
|
|
183
185
|
|
|
184
186
|
```bash
|
|
185
|
-
|
|
187
|
+
./scripts/run-bdd-tests.sh node
|
|
186
188
|
```
|
|
187
189
|
|
|
190
|
+
the script starts the server and sets every variable, so none of them have to be
|
|
191
|
+
exported by hand. to iterate against a server you started yourself, see
|
|
192
|
+
[src/bdd/README.md](./src/bdd/README.md).
|
|
193
|
+
|
|
188
194
|
#### run all test
|
|
189
195
|
|
|
190
|
-
`npm run test` runs unit, bdd and e2e tests suite
|
|
196
|
+
`npm run test` runs unit, bdd and e2e tests suite against an iggy-server at
|
|
197
|
+
tcp://127.0.0.1:8090, started with the same root credentials
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
IGGY_TCP_ADDRESS=127.0.0.1:8090 IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \
|
|
201
|
+
npm run test
|
|
202
|
+
```
|
|
191
203
|
|
|
192
204
|
### lint
|
|
193
205
|
|
package/dist/wire/error.code.js
CHANGED
|
@@ -158,6 +158,7 @@ export const translateErrorCode = (code) => {
|
|
|
158
158
|
case '3021': return "Consumer offset for consumer with ID: {0} was not found.";
|
|
159
159
|
case '3022': return "Failed to resolve consumer with ID: {0}";
|
|
160
160
|
case '3023': return "Cannot open consumer offsets file for path: {0}";
|
|
161
|
+
case '3024': return "Consumer offset limit reached for partition, raise [partition] consumer_offsets_max";
|
|
161
162
|
case '3013': return "Partition id space exhausted for this topic";
|
|
162
163
|
// MESSAGE
|
|
163
164
|
case '4000': return "Segment not found";
|
|
@@ -17,6 +17,9 @@
|
|
|
17
17
|
import assert from 'node:assert/strict';
|
|
18
18
|
import { it } from 'node:test';
|
|
19
19
|
import { translateErrorCode } from './error.code.js';
|
|
20
|
+
it('translates the consumer-offset capacity error', () => {
|
|
21
|
+
assert.equal(translateErrorCode(3024), 'Consumer offset limit reached for partition, raise [partition] consumer_offsets_max');
|
|
22
|
+
});
|
|
20
23
|
it('translates the consumer-group error range', () => {
|
|
21
24
|
assert.equal(translateErrorCode(5000), 'Consumer group with ID: {0} for topic with ID: {1} was not found.');
|
|
22
25
|
assert.equal(translateErrorCode(5001), 'error');
|
|
@@ -45,6 +45,20 @@ export declare const isValidMessageId: (x?: unknown) => x is MessageIdKind;
|
|
|
45
45
|
* @throws Error if the ID format is invalid
|
|
46
46
|
*/
|
|
47
47
|
export declare const serializeMessageId: (id?: unknown) => Buffer<ArrayBufferLike>;
|
|
48
|
+
/**
|
|
49
|
+
* Mints a random 16-byte message ID from the pool, refilling when drained.
|
|
50
|
+
*
|
|
51
|
+
* @returns 16-byte buffer of random bytes owned by the caller
|
|
52
|
+
*/
|
|
53
|
+
export declare const mintMessageId: () => Buffer;
|
|
54
|
+
/**
|
|
55
|
+
* Resolves a message ID to a 16-byte little-endian buffer, minting a random
|
|
56
|
+
* one when the ID is absent or zero.
|
|
57
|
+
*
|
|
58
|
+
* @param id - Optional message ID
|
|
59
|
+
* @returns 16-byte little-endian buffer containing a non-zero ID
|
|
60
|
+
*/
|
|
61
|
+
export declare const resolveMessageId: (id?: MessageIdKind) => Buffer;
|
|
48
62
|
/**
|
|
49
63
|
* Encodes messages into the canonical batch format.
|
|
50
64
|
* Format: [batch header][frames], one frame per message.
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// KIND, either express or implied. See the License for the
|
|
15
15
|
// specific language governing permissions and limitations
|
|
16
16
|
// under the License.
|
|
17
|
-
import {
|
|
17
|
+
import { randomFillSync } from 'node:crypto';
|
|
18
18
|
import { uint32ToBuf, u128ToBuf, uint8ToBuf } from '../number.utils.js';
|
|
19
19
|
import { serializeHeaders } from './header.utils.js';
|
|
20
20
|
import { serializeIdentifier } from '../identifier.utils.js';
|
|
@@ -23,6 +23,8 @@ import { parse as parseUUID } from '../uuid.utils.js';
|
|
|
23
23
|
import { BATCH_HEADER_SIZE, FRAME_HEADER_SIZE, batchChecksum, frameChecksum, serializeBatchHeader, } from './iggy-header.utils.js';
|
|
24
24
|
/** Size of the message ID in bytes (u128) */
|
|
25
25
|
const MESSAGE_ID_SIZE = 16;
|
|
26
|
+
/** Exclusive upper bound for a numeric message ID: it must be < 2^128 */
|
|
27
|
+
const MESSAGE_ID_UPPER_BOUND = 1n << BigInt((MESSAGE_ID_SIZE * 8));
|
|
26
28
|
/** Largest representable frame timestamp delta (u32, microseconds) */
|
|
27
29
|
const MAX_TIMESTAMP_DELTA = 0xffffffffn;
|
|
28
30
|
/**
|
|
@@ -52,6 +54,8 @@ export const serializeMessageId = (id) => {
|
|
|
52
54
|
if (id < 0)
|
|
53
55
|
throw new Error(`invalid message id: '${id}' (numeric id must be >= 0)`);
|
|
54
56
|
const idValue = 'number' === typeof id ? BigInt(id) : id;
|
|
57
|
+
if (idValue >= MESSAGE_ID_UPPER_BOUND)
|
|
58
|
+
throw new Error(`invalid message id: '${id}' (numeric id must be < 2^${MESSAGE_ID_SIZE * 8})`);
|
|
55
59
|
return u128ToBuf(idValue);
|
|
56
60
|
}
|
|
57
61
|
try {
|
|
@@ -62,17 +66,41 @@ export const serializeMessageId = (id) => {
|
|
|
62
66
|
throw new Error(`invalid message id: '${id}' (use uuid string | number | bigint >= 0)`, { cause: err });
|
|
63
67
|
}
|
|
64
68
|
};
|
|
69
|
+
/** Number of ids drawn from the pool per CSPRNG refill */
|
|
70
|
+
const ID_POOL_COUNT = 4096;
|
|
71
|
+
/** Pooled random bytes and a cursor into them, filled lazily on first mint */
|
|
72
|
+
const idPool = Buffer.allocUnsafe(ID_POOL_COUNT * MESSAGE_ID_SIZE);
|
|
73
|
+
let idPoolCursor = idPool.length; // past the end -> refill on first use
|
|
65
74
|
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
75
|
+
* Mints a random 16-byte message ID from the pool, refilling when drained.
|
|
76
|
+
*
|
|
77
|
+
* @returns 16-byte buffer of random bytes owned by the caller
|
|
78
|
+
*/
|
|
79
|
+
export const mintMessageId = () => {
|
|
80
|
+
if (idPoolCursor + MESSAGE_ID_SIZE > idPool.length) {
|
|
81
|
+
randomFillSync(idPool);
|
|
82
|
+
idPoolCursor = 0;
|
|
83
|
+
}
|
|
84
|
+
const id = Buffer.allocUnsafe(MESSAGE_ID_SIZE);
|
|
85
|
+
idPool.copy(id, 0, idPoolCursor, idPoolCursor + MESSAGE_ID_SIZE);
|
|
86
|
+
idPoolCursor += MESSAGE_ID_SIZE;
|
|
87
|
+
return id;
|
|
88
|
+
};
|
|
89
|
+
/**
|
|
90
|
+
* Resolves a message ID to a 16-byte little-endian buffer, minting a random
|
|
91
|
+
* one when the ID is absent or zero.
|
|
68
92
|
*
|
|
69
93
|
* @param id - Optional message ID
|
|
70
94
|
* @returns 16-byte little-endian buffer containing a non-zero ID
|
|
71
95
|
*/
|
|
72
|
-
const resolveMessageId = (id) => {
|
|
96
|
+
export const resolveMessageId = (id) => {
|
|
97
|
+
// An absent or zero id mints a random one.
|
|
98
|
+
if (id === undefined || id === 0 || id === 0n)
|
|
99
|
+
return mintMessageId();
|
|
73
100
|
const bId = serializeMessageId(id);
|
|
74
|
-
|
|
75
|
-
|
|
101
|
+
// A string id can still be the all-zero nil UUID; mint in that case too.
|
|
102
|
+
return 'string' === typeof id && bId.every((byte) => byte === 0)
|
|
103
|
+
? mintMessageId()
|
|
76
104
|
: bId;
|
|
77
105
|
};
|
|
78
106
|
/**
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
import { describe, it } from "node:test";
|
|
18
|
+
import assert from "node:assert/strict";
|
|
19
|
+
import { u128ToBuf } from "../number.utils.js";
|
|
20
|
+
import { isValidMessageId, serializeMessageId, resolveMessageId, mintMessageId, } from "./message.utils.js";
|
|
21
|
+
const MESSAGE_ID_SIZE = 16;
|
|
22
|
+
const MAX_U128 = (1n << 128n) - 1n;
|
|
23
|
+
const NIL_UUID = "00000000-0000-0000-0000-000000000000";
|
|
24
|
+
const isZero = (b) => b.every((byte) => byte === 0);
|
|
25
|
+
describe("isValidMessageId", () => {
|
|
26
|
+
it("accepts undefined, string, number, and bigint", () => {
|
|
27
|
+
assert.ok(isValidMessageId(undefined));
|
|
28
|
+
assert.ok(isValidMessageId("id"));
|
|
29
|
+
assert.ok(isValidMessageId(7));
|
|
30
|
+
assert.ok(isValidMessageId(7n));
|
|
31
|
+
});
|
|
32
|
+
it("rejects other types", () => {
|
|
33
|
+
assert.ok(!isValidMessageId(null));
|
|
34
|
+
assert.ok(!isValidMessageId({}));
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
describe("serializeMessageId", () => {
|
|
38
|
+
it("serializes undefined to a zero u128", () => {
|
|
39
|
+
assert.deepEqual(serializeMessageId(), Buffer.alloc(MESSAGE_ID_SIZE, 0));
|
|
40
|
+
});
|
|
41
|
+
it("serializes a number as little-endian u128", () => {
|
|
42
|
+
assert.deepEqual(serializeMessageId(7), u128ToBuf(7n));
|
|
43
|
+
});
|
|
44
|
+
it("serializes a bigint as little-endian u128", () => {
|
|
45
|
+
assert.deepEqual(serializeMessageId(8n), u128ToBuf(8n));
|
|
46
|
+
});
|
|
47
|
+
it("serializes a UUID string to the same bytes as its numeric value", () => {
|
|
48
|
+
const uuid = "00000000-0000-0000-0000-000000000007";
|
|
49
|
+
assert.deepEqual(serializeMessageId(uuid), u128ToBuf(7n));
|
|
50
|
+
});
|
|
51
|
+
it("accepts the largest u128", () => {
|
|
52
|
+
assert.deepEqual(serializeMessageId(MAX_U128), u128ToBuf(MAX_U128));
|
|
53
|
+
});
|
|
54
|
+
it("rejects a numeric id at or above 2^128", () => {
|
|
55
|
+
assert.throws(() => serializeMessageId(1n << 128n), /2\^128/);
|
|
56
|
+
});
|
|
57
|
+
it("rejects a negative numeric id", () => {
|
|
58
|
+
assert.throws(() => serializeMessageId(-1n), />= 0/);
|
|
59
|
+
});
|
|
60
|
+
it("rejects an unparsable string", () => {
|
|
61
|
+
assert.throws(() => serializeMessageId("not-a-uuid"), /invalid message id/);
|
|
62
|
+
});
|
|
63
|
+
it("rejects an invalid type", () => {
|
|
64
|
+
assert.throws(() => serializeMessageId({}), /invalid message id/);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
describe("resolveMessageId", () => {
|
|
68
|
+
it("mints a non-zero id for undefined, 0, and 0n", () => {
|
|
69
|
+
for (const id of [undefined, 0, 0n]) {
|
|
70
|
+
const b = resolveMessageId(id);
|
|
71
|
+
assert.equal(b.length, MESSAGE_ID_SIZE);
|
|
72
|
+
assert.ok(!isZero(b));
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
it("mints for the all-zero nil UUID string", () => {
|
|
76
|
+
assert.ok(!isZero(resolveMessageId(NIL_UUID)));
|
|
77
|
+
});
|
|
78
|
+
it("passes a provided non-zero id through unchanged", () => {
|
|
79
|
+
assert.deepEqual(resolveMessageId(7n), serializeMessageId(7n));
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
describe("mintMessageId", () => {
|
|
83
|
+
it("returns a non-zero 16-byte buffer", () => {
|
|
84
|
+
const b = mintMessageId();
|
|
85
|
+
assert.equal(b.length, MESSAGE_ID_SIZE);
|
|
86
|
+
assert.ok(!isZero(b));
|
|
87
|
+
});
|
|
88
|
+
it("produces unique, non-zero ids across pool refills", () => {
|
|
89
|
+
const seen = new Set();
|
|
90
|
+
for (let i = 0; i < 10_000; i++) {
|
|
91
|
+
const hex = mintMessageId().toString("hex");
|
|
92
|
+
assert.notEqual(hex, "0".repeat(MESSAGE_ID_SIZE * 2));
|
|
93
|
+
seen.add(hex);
|
|
94
|
+
}
|
|
95
|
+
assert.equal(seen.size, 10_000);
|
|
96
|
+
});
|
|
97
|
+
it("returns owned bytes that survive a later refill", () => {
|
|
98
|
+
const first = mintMessageId().toString("hex");
|
|
99
|
+
const held = mintMessageId();
|
|
100
|
+
const snapshot = held.toString("hex");
|
|
101
|
+
for (let i = 0; i < 10_000; i++)
|
|
102
|
+
mintMessageId();
|
|
103
|
+
assert.equal(held.toString("hex"), snapshot);
|
|
104
|
+
assert.notEqual(snapshot, first);
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
//# sourceMappingURL=message.utils.test.js.map
|
|
@@ -76,13 +76,18 @@ export declare const floatToBuf: (v: number) => Buffer<ArrayBuffer>;
|
|
|
76
76
|
*/
|
|
77
77
|
export declare const doubleToBuf: (v: number) => Buffer<ArrayBuffer>;
|
|
78
78
|
/**
|
|
79
|
-
* Converts a
|
|
79
|
+
* Converts a u128 value to a 16-byte unsigned integer Buffer in little-endian
|
|
80
|
+
* format.
|
|
80
81
|
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
82
|
+
* Writes the two 64-bit halves directly rather than round-tripping through a hex
|
|
83
|
+
* string. `value` must be a non-negative integer below 2^128; anything outside
|
|
84
|
+
* that range throws (a `RangeError` from the underlying write), which is
|
|
85
|
+
* preferable to silently truncating a message id.
|
|
86
|
+
*
|
|
87
|
+
* @param value - u128 value (0 <= value < 2^128)
|
|
88
|
+
* @returns 16-byte buffer containing the value in little-endian byte order
|
|
84
89
|
*/
|
|
85
|
-
export declare function u128ToBuf(
|
|
90
|
+
export declare function u128ToBuf(value: bigint): Buffer;
|
|
86
91
|
/**
|
|
87
92
|
* Converts a 128-bit unsigned integer Buffer in little-endian format to a BigInt.
|
|
88
93
|
*
|
|
@@ -135,17 +135,25 @@ export const doubleToBuf = (v) => {
|
|
|
135
135
|
b.writeDoubleLE(v);
|
|
136
136
|
return b;
|
|
137
137
|
};
|
|
138
|
+
/** Mask selecting the low 64 bits of a u128, for little-endian half-writes. */
|
|
139
|
+
const U64_MASK = 0xffffffffffffffffn;
|
|
138
140
|
/**
|
|
139
|
-
* Converts a
|
|
141
|
+
* Converts a u128 value to a 16-byte unsigned integer Buffer in little-endian
|
|
142
|
+
* format.
|
|
140
143
|
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
+
* Writes the two 64-bit halves directly rather than round-tripping through a hex
|
|
145
|
+
* string. `value` must be a non-negative integer below 2^128; anything outside
|
|
146
|
+
* that range throws (a `RangeError` from the underlying write), which is
|
|
147
|
+
* preferable to silently truncating a message id.
|
|
148
|
+
*
|
|
149
|
+
* @param value - u128 value (0 <= value < 2^128)
|
|
150
|
+
* @returns 16-byte buffer containing the value in little-endian byte order
|
|
144
151
|
*/
|
|
145
|
-
export function u128ToBuf(
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
|
|
152
|
+
export function u128ToBuf(value) {
|
|
153
|
+
const b = Buffer.allocUnsafe(16);
|
|
154
|
+
b.writeBigUInt64LE(value & U64_MASK, 0);
|
|
155
|
+
b.writeBigUInt64LE(value >> 64n, 8);
|
|
156
|
+
return b;
|
|
149
157
|
}
|
|
150
158
|
/**
|
|
151
159
|
* Converts a 128-bit unsigned integer Buffer in little-endian format to a BigInt.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
import { describe, it } from "node:test";
|
|
18
|
+
import assert from "node:assert/strict";
|
|
19
|
+
import { u128ToBuf } from "./number.utils.js";
|
|
20
|
+
const MAX_U128 = (1n << 128n) - 1n;
|
|
21
|
+
const hex = (v) => u128ToBuf(v).toString("hex");
|
|
22
|
+
describe("u128ToBuf", () => {
|
|
23
|
+
it("encodes zero as 16 zero bytes", () => {
|
|
24
|
+
assert.equal(hex(0n), "0".repeat(32));
|
|
25
|
+
});
|
|
26
|
+
it("encodes a small value little-endian", () => {
|
|
27
|
+
assert.equal(hex(7n), "07" + "0".repeat(30));
|
|
28
|
+
});
|
|
29
|
+
it("orders all 16 bytes little-endian", () => {
|
|
30
|
+
// Distinct bytes so a byte-order slip is visible.
|
|
31
|
+
assert.equal(hex(0x0102030405060708090a0b0c0d0e0f10n), "100f0e0d0c0b0a090807060504030201");
|
|
32
|
+
});
|
|
33
|
+
it("spans the 64-bit half boundary", () => {
|
|
34
|
+
assert.equal(hex((1n << 64n) - 1n), "ff".repeat(8) + "00".repeat(8));
|
|
35
|
+
assert.equal(hex(1n << 64n), "00".repeat(8) + "01" + "00".repeat(7));
|
|
36
|
+
});
|
|
37
|
+
it("encodes the largest u128 as all ones", () => {
|
|
38
|
+
assert.equal(hex(MAX_U128), "ff".repeat(16));
|
|
39
|
+
});
|
|
40
|
+
it("round-trips through readBigUInt64LE halves", () => {
|
|
41
|
+
for (const v of [0n, 1n, 42n, 1n << 64n, MAX_U128]) {
|
|
42
|
+
const b = u128ToBuf(v);
|
|
43
|
+
const low = b.readBigUInt64LE(0);
|
|
44
|
+
const high = b.readBigUInt64LE(8);
|
|
45
|
+
assert.equal((high << 64n) | low, v);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
it("throws for a value at or above 2^128", () => {
|
|
49
|
+
assert.throws(() => u128ToBuf(1n << 128n));
|
|
50
|
+
});
|
|
51
|
+
it("throws for a negative value", () => {
|
|
52
|
+
assert.throws(() => u128ToBuf(-1n));
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
//# sourceMappingURL=number.utils.test.js.map
|
|
@@ -86,8 +86,10 @@ export type RequestHeaderFields = {
|
|
|
86
86
|
};
|
|
87
87
|
/**
|
|
88
88
|
* Encodes a 256-byte request header. Only the six fields the server reads
|
|
89
|
-
* are written
|
|
90
|
-
*
|
|
89
|
+
* are written. The checksums stay zero: the frame and body checksums are not
|
|
90
|
+
* read on the client request path, and `request_checksum` treats zero as
|
|
91
|
+
* unstamped, which opts out of the server's payload comparison. Stamping it is
|
|
92
|
+
* optional -- the Rust SDK does for deduped ops, this SDK does not yet.
|
|
91
93
|
*/
|
|
92
94
|
export declare const encodeRequestHeader: (fields: RequestHeaderFields) => Buffer;
|
|
93
95
|
/**
|
package/dist/wire/vsr/header.js
CHANGED
|
@@ -88,8 +88,10 @@ export const EvictionReason = {
|
|
|
88
88
|
const U64_MASK = 0xffffffffffffffffn;
|
|
89
89
|
/**
|
|
90
90
|
* Encodes a 256-byte request header. Only the six fields the server reads
|
|
91
|
-
* are written
|
|
92
|
-
*
|
|
91
|
+
* are written. The checksums stay zero: the frame and body checksums are not
|
|
92
|
+
* read on the client request path, and `request_checksum` treats zero as
|
|
93
|
+
* unstamped, which opts out of the server's payload comparison. Stamping it is
|
|
94
|
+
* optional -- the Rust SDK does for deduped ops, this SDK does not yet.
|
|
93
95
|
*/
|
|
94
96
|
export const encodeRequestHeader = (fields) => {
|
|
95
97
|
const header = Buffer.alloc(HEADER_SIZE);
|
package/dist/wire/vsr/index.js
CHANGED
|
@@ -18,7 +18,7 @@ import { createRequire } from 'node:module';
|
|
|
18
18
|
import { COMMAND_CODE } from '../command.code.js';
|
|
19
19
|
import { responseError } from '../error.utils.js';
|
|
20
20
|
import { HEADER_SIZE, encodeRequestHeader } from './header.js';
|
|
21
|
-
import { Operation,
|
|
21
|
+
import { Operation, operationForCode } from './operation.js';
|
|
22
22
|
import { deserializeLoginRegister, serializeLoginRegister, serializeLoginRegisterWithPat, } from './register.js';
|
|
23
23
|
import { decodeResponse } from './reply.js';
|
|
24
24
|
import { ConsensusSession } from './session.js';
|
|
@@ -61,9 +61,10 @@ export class VsrSession {
|
|
|
61
61
|
else {
|
|
62
62
|
if (this.state.session === null)
|
|
63
63
|
throw responseError(command, UNAUTHENTICATED);
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
// Partition ops consume an id too, even though no partition-plane dedup
|
|
65
|
+
// exists yet: dedup needs each send to carry a distinct number, and the
|
|
66
|
+
// metadata watermark tolerates the gaps.
|
|
67
|
+
request = this.state.nextRequestId();
|
|
67
68
|
session = this.state.session;
|
|
68
69
|
}
|
|
69
70
|
const header = encodeRequestHeader({
|
|
@@ -50,8 +50,6 @@ export declare const isInternal: (operation: number) => boolean;
|
|
|
50
50
|
* `Operation::is_metadata`.
|
|
51
51
|
*/
|
|
52
52
|
export declare const isMetadata: (operation: number) => boolean;
|
|
53
|
-
/** Partition band is a bare range, mirroring `Operation::is_partition`. */
|
|
54
|
-
export declare const isPartition: (operation: number) => boolean;
|
|
55
53
|
/** Whether a reply body leads with a committed result section. */
|
|
56
54
|
export declare const isResultFramed: (operation: number) => boolean;
|
|
57
55
|
/**
|
|
@@ -59,7 +59,6 @@ export const Operation = {
|
|
|
59
59
|
};
|
|
60
60
|
const INTERNAL_START = 64;
|
|
61
61
|
const METADATA_START = 128;
|
|
62
|
-
const PARTITION_START = 160;
|
|
63
62
|
/**
|
|
64
63
|
* Replicated command code to `Operation` mapping, the client half of the
|
|
65
64
|
* dispatch table. Codes absent here are sent as non-replicated so the server
|
|
@@ -115,8 +114,6 @@ export const isMetadata = (operation) => {
|
|
|
115
114
|
return operation >= METADATA_START &&
|
|
116
115
|
operation <= Operation.LeaveConsumerGroup;
|
|
117
116
|
};
|
|
118
|
-
/** Partition band is a bare range, mirroring `Operation::is_partition`. */
|
|
119
|
-
export const isPartition = (operation) => operation >= PARTITION_START;
|
|
120
117
|
/** Whether a reply body leads with a committed result section. */
|
|
121
118
|
export const isResultFramed = (operation) => isMetadata(operation) ||
|
|
122
119
|
operation === Operation.StoreConsumerOffset ||
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
import assert from 'node:assert/strict';
|
|
18
18
|
import { describe, it } from 'node:test';
|
|
19
19
|
import { COMMAND_CODE } from '../command.code.js';
|
|
20
|
-
import { isInternal, isKnownOperation, isMetadata,
|
|
20
|
+
import { isInternal, isKnownOperation, isMetadata, isResultFramed, Operation, operationForCode } from './operation.js';
|
|
21
21
|
const replicated = new Map([
|
|
22
22
|
[COMMAND_CODE.CreateStream, Operation.CreateStream],
|
|
23
23
|
[COMMAND_CODE.UpdateStream, Operation.UpdateStream],
|
|
@@ -72,8 +72,6 @@ describe('VSR operation classification', () => {
|
|
|
72
72
|
assert.equal(isMetadata(Operation.LeaveConsumerGroup), true);
|
|
73
73
|
assert.equal(isMetadata(Operation.DeleteSegments), false);
|
|
74
74
|
assert.equal(isMetadata(150), false);
|
|
75
|
-
assert.equal(isPartition(Operation.SendMessages), true);
|
|
76
|
-
assert.equal(isPartition(159), false);
|
|
77
75
|
assert.equal(isResultFramed(Operation.StoreConsumerOffset), true);
|
|
78
76
|
assert.equal(isResultFramed(Operation.DeleteConsumerOffset), true);
|
|
79
77
|
assert.equal(isResultFramed(Operation.SendMessages), false);
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Each client instance generates an ephemeral random `clientId` (u128).
|
|
5
5
|
* After a Register commits, the server assigns a `session` number (commit op
|
|
6
|
-
* number).
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* number). Every replicated request (metadata and partition) advances a
|
|
7
|
+
* monotonic request watermark; non-replicated requests reuse the current
|
|
8
|
+
* value because they bypass server-side request sequencing.
|
|
9
9
|
*/
|
|
10
10
|
export declare class ConsensusSession {
|
|
11
11
|
private _clientId;
|
package/dist/wire/vsr/session.js
CHANGED
|
@@ -21,9 +21,9 @@ const MAX_U64 = 0xffffffffffffffffn;
|
|
|
21
21
|
*
|
|
22
22
|
* Each client instance generates an ephemeral random `clientId` (u128).
|
|
23
23
|
* After a Register commits, the server assigns a `session` number (commit op
|
|
24
|
-
* number).
|
|
25
|
-
*
|
|
26
|
-
*
|
|
24
|
+
* number). Every replicated request (metadata and partition) advances a
|
|
25
|
+
* monotonic request watermark; non-replicated requests reuse the current
|
|
26
|
+
* value because they bypass server-side request sequencing.
|
|
27
27
|
*/
|
|
28
28
|
export class ConsensusSession {
|
|
29
29
|
_clientId;
|
|
@@ -48,17 +48,28 @@ describe('VSR custom request framing', () => {
|
|
|
48
48
|
const frame = new VsrSession(7n).encode(prepared.command, prepared.payload);
|
|
49
49
|
assert.equal(frame.readUInt8(REQUEST_OFFSET.operation), Operation.Register);
|
|
50
50
|
});
|
|
51
|
-
it('does not advance for non-replicated
|
|
51
|
+
it('does not advance for non-replicated operations', () => {
|
|
52
52
|
const session = new VsrSession(7n);
|
|
53
53
|
session.bind(42n);
|
|
54
54
|
const custom = session.encode(60_001, Buffer.alloc(0));
|
|
55
55
|
assert.equal(custom.readBigUInt64LE(REQUEST_OFFSET.request), 1n);
|
|
56
|
-
const partition = session.encode(COMMAND_CODE.SendMessages, serializeSendMessages(1, 2, [{ payload: 'x' }], Partitioning.PartitionId(3)));
|
|
57
|
-
assert.equal(partition.readUInt8(REQUEST_OFFSET.operation), Operation.SendMessages);
|
|
58
|
-
assert.equal(partition.readBigUInt64LE(REQUEST_OFFSET.request), 1n);
|
|
59
56
|
const metadata = session.encode(COMMAND_CODE.CreateStream, Buffer.alloc(0));
|
|
60
57
|
assert.equal(metadata.readBigUInt64LE(REQUEST_OFFSET.request), 1n);
|
|
61
58
|
assert.equal(metadata.length, HEADER_SIZE);
|
|
62
59
|
});
|
|
60
|
+
it('partition operations consume a distinct id per send', () => {
|
|
61
|
+
// Dedup identity requires each send to carry a distinct number, so
|
|
62
|
+
// partition ops advance the counter exactly like metadata ops and the
|
|
63
|
+
// two planes interleave on one sequence.
|
|
64
|
+
const session = new VsrSession(7n);
|
|
65
|
+
session.bind(42n);
|
|
66
|
+
const sendMessages = () => session.encode(COMMAND_CODE.SendMessages, serializeSendMessages(1, 2, [{ payload: 'x' }], Partitioning.PartitionId(3)));
|
|
67
|
+
const first = sendMessages();
|
|
68
|
+
assert.equal(first.readUInt8(REQUEST_OFFSET.operation), Operation.SendMessages);
|
|
69
|
+
assert.equal(first.readBigUInt64LE(REQUEST_OFFSET.request), 1n);
|
|
70
|
+
assert.equal(sendMessages().readBigUInt64LE(REQUEST_OFFSET.request), 2n);
|
|
71
|
+
const metadata = session.encode(COMMAND_CODE.CreateStream, Buffer.alloc(0));
|
|
72
|
+
assert.equal(metadata.readBigUInt64LE(REQUEST_OFFSET.request), 3n);
|
|
73
|
+
});
|
|
63
74
|
});
|
|
64
75
|
//# sourceMappingURL=vsr.test.js.map
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apache-iggy",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.10.0-edge.
|
|
4
|
+
"version": "0.10.0-edge.6",
|
|
5
5
|
"description": "Official Apache Iggy NodeJS SDK",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"iggy",
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
"@cucumber/cucumber": "13.2.1",
|
|
62
62
|
"@swc-node/register": "1.12.1",
|
|
63
63
|
"@types/debug": "4.1.13",
|
|
64
|
-
"@types/node": "26.
|
|
64
|
+
"@types/node": "26.4.0",
|
|
65
65
|
"c8": "^12.0.0",
|
|
66
66
|
"husky": "9.1.7",
|
|
67
67
|
"typescript": "6.0.3",
|