@s2-dev/streamstore 0.15.13 → 0.16.0

package/src/streams.ts

@@ -0,0 +1,156 @@
+import type { DataToObject, S2RequestOptions } from "./common";
+import { S2Error } from "./error";
+import {
+	type CreateStreamData,
+	createStream,
+	type DeleteStreamData,
+	deleteStream,
+	type GetStreamConfigData,
+	getStreamConfig,
+	type ListStreamsData,
+	listStreams,
+	type ReconfigureStreamData,
+	reconfigureStream,
+} from "./generated";
+import type { Client } from "./generated/client/types.gen";
+
+export interface ListStreamsArgs extends DataToObject<ListStreamsData> {}
+export interface CreateStreamArgs extends DataToObject<CreateStreamData> {}
+export interface GetStreamConfigArgs
+	extends DataToObject<GetStreamConfigData> {}
+export interface DeleteStreamArgs extends DataToObject<DeleteStreamData> {}
+export interface ReconfigureStreamArgs
+	extends DataToObject<ReconfigureStreamData> {}
+
+export class S2Streams {
+	private readonly client: Client;
+	constructor(client: Client) {
+		this.client = client;
+	}
+
+	/**
+	 * List streams in the basin.
+	 *
+	 * @param args.prefix Return streams whose names start with the given prefix
+	 * @param args.start_after Name to start after (for pagination)
+	 * @param args.limit Max results (up to 1000)
+	 */
+	public async list(args?: ListStreamsArgs, options?: S2RequestOptions) {
+		const response = await listStreams({
+			client: this.client,
+			query: args,
+			...options,
+		});
+
+		if (response.error) {
+			throw new S2Error({
+				message: response.error.message,
+				code: response.error.code ?? undefined,
+				status: response.response.status,
+			});
+		}
+
+		return response.data;
+	}
+
+	/**
+	 * Create a stream.
+	 *
+	 * @param args.stream Stream name (1-512 bytes, unique within the basin)
+	 * @param args.config Stream configuration (retention, storage class, timestamping, delete-on-empty)
+	 */
+	public async create(args: CreateStreamArgs, options?: S2RequestOptions) {
+		const response = await createStream({
+			client: this.client,
+			body: args,
+			...options,
+		});
+
+		if (response.error) {
+			throw new S2Error({
+				message: response.error.message,
+				code: response.error.code ?? undefined,
+				status: response.response.status,
+			});
+		}
+
+		return response.data;
+	}
+
+	/**
+	 * Get stream configuration.
+	 *
+	 * @param args.stream Stream name
+	 */
+	public async getConfig(
+		args: GetStreamConfigArgs,
+		options?: S2RequestOptions,
+	) {
+		const response = await getStreamConfig({
+			client: this.client,
+			path: args,
+			...options,
+		});
+
+		if (response.error) {
+			throw new S2Error({
+				message: response.error.message,
+				code: response.error.code ?? undefined,
+				status: response.response.status,
+			});
+		}
+
+		return response.data;
+	}
+
+	/**
+	 * Delete a stream.
+	 *
+	 * @param args.stream Stream name
+	 */
+	public async delete(args: DeleteStreamArgs, options?: S2RequestOptions) {
+		const response = await deleteStream({
+			client: this.client,
+			path: args,
+			...options,
+		});
+
+		if (response.error) {
+			throw new S2Error({
+				message: response.error.message,
+				code: response.error.code ?? undefined,
+				status: response.response.status,
+			});
+		}
+
+		return response.data;
+	}
+
+	/**
+	 * Reconfigure a stream.
+	 *
+	 * @param args.stream Stream name
+	 * @param args.body Configuration fields to change
+	 */
+	public async reconfigure(
+		args: ReconfigureStreamArgs,
+		options?: S2RequestOptions,
+	) {
+		const response = await reconfigureStream({
+			client: this.client,
+			path: args,
+			body: args,
+			...options,
+		});
+
+		if (response.error) {
+			throw new S2Error({
+				message: response.error.message,
+				code: response.error.code ?? undefined,
+				status: response.response.status,
+			});
+		}
+
+		return response.data;
+	}
+}

package/src/tests/appendSession.test.ts

@@ -0,0 +1,149 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { AppendAck } from "../generated";
+import type { AppendRecord } from "../stream";
+import { S2Stream } from "../stream";
+
+// Minimal Client shape to satisfy S2Stream constructor; we won't use it directly
+const fakeClient: any = {};
+
+const makeStream = () => new S2Stream("test-stream", fakeClient);
+
+const makeAck = (n: number): AppendAck => ({
+	start: { seq_num: n - 1, timestamp: 0 },
+	end: { seq_num: n, timestamp: 0 },
+	tail: { seq_num: n, timestamp: 0 },
+});
+
+describe("AppendSession", () => {
+	beforeEach(() => {
+		vi.useFakeTimers();
+	});
+
+	afterEach(() => {
+		vi.useRealTimers();
+		vi.restoreAllMocks();
+	});
+
+	it("serializes submit calls and emits acks in order", async () => {
+		const stream = makeStream();
+		const appendSpy = vi.spyOn(stream, "append");
+
+		// ensure only one in flight at a time by controlling resolution of spy
+		let firstResolved = false;
+		appendSpy.mockImplementationOnce(async (..._args: any[]) => {
+			await vi.advanceTimersByTimeAsync(10);
+			firstResolved = true;
+			return makeAck(1);
+		});
+		appendSpy.mockImplementationOnce(async (..._args: any[]) => {
+			expect(firstResolved).toBe(true);
+			await vi.advanceTimersByTimeAsync(5);
+			return makeAck(2);
+		});
+		// default fallback
+		appendSpy.mockResolvedValue(makeAck(999));
+
+		const session = await stream.appendSession();
+
+		const p1 = session.submit([{ body: "a" }]);
+		const p2 = session.submit([{ body: "b" }]);
+
+		const ack1 = await p1;
+		const ack2 = await p2;
+
+		expect(appendSpy).toHaveBeenCalledTimes(2);
+		expect(ack1.end.seq_num).toBe(1);
+		expect(ack2.end.seq_num).toBe(2);
+	});
+
+	it("acks() stream receives emitted acks and closes on session.close()", async () => {
+		const stream = makeStream();
+		const appendSpy = vi
+			.spyOn(stream, "append")
+			.mockResolvedValueOnce(makeAck(1))
+			.mockResolvedValueOnce(makeAck(2));
+
+		const session = await stream.appendSession();
+		const acks = session.acks();
+
+		const received: AppendAck[] = [];
+		const consumer = (async () => {
+			for await (const ack of acks) {
+				received.push(ack);
+			}
+		})();
+
+		await session.submit([{ body: "a" }]);
+		await session.submit([{ body: "b" }]);
+
+		await session.close();
+		await consumer;
+
+		expect(appendSpy).toHaveBeenCalledTimes(2);
+		expect(received.map((a) => a.end.seq_num)).toEqual([1, 2]);
+	});
+
+	it("close() waits for drain before resolving", async () => {
+		const stream = makeStream();
+		const appendSpy = vi.spyOn(stream, "append");
+
+		appendSpy.mockResolvedValueOnce(makeAck(1));
+		appendSpy.mockResolvedValueOnce(makeAck(2));
+
+		const session = await stream.appendSession();
+
+		const p1 = session.submit([{ body: "x" }]);
+		const p2 = session.submit([{ body: "y" }]);
+
+		await Promise.all([p1, p2]);
+		await session.close();
+
+		await expect(p1).resolves.toBeTruthy();
+		await expect(p2).resolves.toBeTruthy();
+		expect(appendSpy).toHaveBeenCalledTimes(2);
+	});
+
+	it("submit after close() rejects", async () => {
+		const stream = makeStream();
+		vi.spyOn(stream, "append").mockResolvedValue(makeAck(1));
+		const session = await stream.appendSession();
+
+		await session.close();
+
+		await expect(session.submit([{ body: "x" }])).rejects.toMatchObject({
+			message: expect.stringContaining("AppendSession is closed"),
+		});
+	});
+
+	it("error during processing rejects current and queued, clears queue", async () => {
+		const stream = makeStream();
+		const appendSpy = vi.spyOn(stream, "append");
+
+		appendSpy.mockRejectedValueOnce(new Error("boom"));
+
+		const session = await stream.appendSession();
+
+		const p1 = session.submit([{ body: "a" }]);
+		const p2 = session.submit([{ body: "b" }]);
+		// suppress unhandled rejection warnings
+		p1.catch(() => {});
+		p2.catch(() => {});
+
+		await expect(p1).rejects.toBeTruthy();
+		await expect(p2).rejects.toBeTruthy();
+
+		// After error, queue should be empty; new submit should restart processing
+		appendSpy.mockResolvedValueOnce(makeAck(3));
+		const p3 = session.submit([{ body: "c" }]);
+		await expect(p3).resolves.toBeTruthy();
+		expect(appendSpy).toHaveBeenCalledTimes(2); // 1 throw + 1 success
+	});
+
+	it("updates lastSeenPosition after successful append", async () => {
+		const stream = makeStream();
+		vi.spyOn(stream, "append").mockResolvedValue(makeAck(42));
+		const session = await stream.appendSession();
+		await session.submit([{ body: "z" }]);
+		expect(session.lastSeenPosition?.end.seq_num).toBe(42);
+	});
+});

package/src/tests/batcher-session.integration.test.ts

@@ -0,0 +1,80 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { AppendAck } from "../generated";
+import { S2Stream } from "../stream";
+
+const fakeClient: any = {};
+const makeStream = () => new S2Stream("test-stream", fakeClient);
+const makeAck = (n: number): AppendAck => ({
+	start: { seq_num: n - 1, timestamp: 0 },
+	end: { seq_num: n, timestamp: 0 },
+	tail: { seq_num: n, timestamp: 0 },
+});
+
+describe("Batcher + AppendSession integration", () => {
+	beforeEach(() => {
+		vi.useFakeTimers();
+	});
+
+	afterEach(() => {
+		vi.useRealTimers();
+		vi.restoreAllMocks();
+	});
+
+	it("linger-driven batching yields single session submission", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const appendSpy = vi.spyOn(stream, "append").mockResolvedValue(makeAck(1));
+		const batcher = session.makeBatcher({
+			lingerDuration: 10,
+			maxBatchSize: 100,
+		});
+
+		const p1 = batcher.submit({ body: "a" });
+		const p2 = batcher.submit({ body: "b" });
+
+		await vi.advanceTimersByTimeAsync(12);
+		await Promise.all([p1, p2]);
+
+		expect(appendSpy).toHaveBeenCalledTimes(1);
+		expect(appendSpy.mock.calls?.[0]?.[0]).toHaveLength(2);
+	});
+
+	it("batch overflow increments match_seq_num across multiple flushes", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const appendSpy = vi.spyOn(stream, "append");
+		appendSpy.mockResolvedValueOnce(makeAck(1));
+		appendSpy.mockResolvedValueOnce(makeAck(2));
+		const batcher = session.makeBatcher({
+			lingerDuration: 0,
+			maxBatchSize: 2,
+			match_seq_num: 5,
+		});
+
+		const p1 = batcher.submit([{ body: "1" }, { body: "2" }]);
+		batcher.flush();
+		await p1;
+		const p2 = batcher.submit([{ body: "3" }]);
+		batcher.flush();
+		await p2;
+
+		expect(appendSpy).toHaveBeenCalledTimes(2);
+		expect(appendSpy.mock.calls?.[0]?.[1]).toMatchObject({ match_seq_num: 5 });
+		expect(appendSpy.mock.calls?.[1]?.[1]).toMatchObject({ match_seq_num: 7 });
+	});
+
+	it("batcher promise resolves with ack from session/stream", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		vi.spyOn(stream, "append").mockResolvedValue(makeAck(123));
+		const batcher = session.makeBatcher({
+			lingerDuration: 0,
+			maxBatchSize: 10,
+		});
+
+		const ackPromise = batcher.submit({ body: "x" });
+		batcher.flush();
+		const ack = await ackPromise;
+		expect(ack.end.seq_num).toBe(123);
+	});
+});

package/src/tests/batcher.test.ts

@@ -0,0 +1,216 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { AppendAck } from "../generated";
+import type { AppendRecord } from "../stream";
+import { S2Stream } from "../stream";
+
+const fakeClient: any = {};
+const makeStream = () => new S2Stream("test-stream", fakeClient);
+const makeAck = (n: number): AppendAck => ({
+	start: { seq_num: n - 1, timestamp: 0 },
+	end: { seq_num: n, timestamp: 0 },
+	tail: { seq_num: n, timestamp: 0 },
+});
+
+describe("Batcher", () => {
+	beforeEach(() => {
+		vi.useFakeTimers();
+	});
+
+	afterEach(() => {
+		vi.useRealTimers();
+		vi.restoreAllMocks();
+	});
+
+	it("batches submits within linger window into single session.submit", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const submitSpy = vi.spyOn(session, "submit").mockResolvedValue(makeAck(1));
+
+		const batcher = session.makeBatcher({
+			lingerDuration: 20,
+			maxBatchSize: 10,
+		});
+
+		const p1 = batcher.submit({ body: "a" });
+		const p2 = batcher.submit({ body: "b" });
+
+		// nothing flushed yet
+		expect(submitSpy).toHaveBeenCalledTimes(0);
+
+		await vi.advanceTimersByTimeAsync(20);
+
+		// now one flush
+		await Promise.all([p1, p2]);
+		expect(submitSpy).toHaveBeenCalledTimes(1);
+		const firstCall = submitSpy.mock.calls[0] as unknown[];
+		const recordsArg = firstCall?.[0] as AppendRecord[];
+		expect(recordsArg).toHaveLength(2);
+	});
+
+	it("manual flush cancels timer and sends immediately", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const submitSpy = vi
+			.spyOn(session, "submit")
+			.mockResolvedValue(makeAck(99));
+
+		const batcher = session.makeBatcher({
+			lingerDuration: 1000,
+			maxBatchSize: 10,
+		});
+		const p = batcher.submit({ body: "x" });
+		batcher.flush();
+
+		await p;
+		expect(submitSpy).toHaveBeenCalledTimes(1);
+	});
+
+	it("close() flushes remaining and prevents further submit", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		vi.spyOn(session, "submit").mockResolvedValue(makeAck(1));
+		const batcher = session.makeBatcher({
+			lingerDuration: 1000,
+			maxBatchSize: 10,
+		});
+
+		const p = batcher.submit({ body: "x" });
+		await batcher.close();
+		await p;
+
+		await expect(batcher.submit({ body: "y" })).rejects.toMatchObject({
+			message: expect.stringContaining("Batcher is closed"),
+		});
+	});
+
+	it("abort rejects pending with S2Error, clears batch", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		vi.spyOn(session, "submit").mockResolvedValue(makeAck(1));
+		const batcher = session.makeBatcher({
+			lingerDuration: 1000,
+			maxBatchSize: 10,
+		});
+
+		const p = batcher.submit({ body: "x" });
+		const writer = batcher.getWriter?.() ?? batcher;
+		await writer.abort?.("stop");
+
+		await expect(p).rejects.toMatchObject({
+			message: expect.stringContaining("Batcher was aborted: stop"),
+		});
+	});
+
+	it("propagates fencing_token and auto-increments match_seq_num across batches", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const submitSpy = vi.spyOn(session, "submit");
+		submitSpy.mockResolvedValueOnce(makeAck(1));
+		submitSpy.mockResolvedValueOnce(makeAck(2));
+
+		const batcher = session.makeBatcher({
+			lingerDuration: 0,
+			maxBatchSize: 2,
+			fencing_token: "ft",
+			match_seq_num: 10,
+		});
+
+		// First batch: two records
+		const p1 = batcher.submit([{ body: "a" }, { body: "b" }]);
+		batcher.flush();
+		await p1;
+		// Second batch: one record
+		const p2 = batcher.submit([{ body: "c" }]);
+		batcher.flush();
+		await p2;
+
+		expect(submitSpy).toHaveBeenCalledTimes(2);
+		const c0 = submitSpy.mock.calls[0] as unknown[];
+		const c1 = submitSpy.mock.calls[1] as unknown[];
+		expect(c0?.[1]).toMatchObject({
+			fencing_token: "ft",
+			match_seq_num: 10,
+		});
+		expect(c1?.[1]).toMatchObject({
+			fencing_token: "ft",
+			match_seq_num: 12,
+		});
+	});
+
+	it("array submit is atomic: if it won't fit, flush current then enqueue whole array", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const submitSpy = vi.spyOn(session, "submit").mockResolvedValue(makeAck(1));
+
+		const batcher = session.makeBatcher({
+			lingerDuration: 0,
+			maxBatchSize: 3,
+		});
+
+		// Fill current batch with two singles (no auto flush because linger=0)
+		batcher.submit({ body: "a" });
+		batcher.submit({ body: "b" });
+
+		// Now submit an array of 2 (atomic). Since 2+2>3, it flushes current 2,
+		// then enqueues both array records into the next batch
+		const p = batcher.submit([{ body: "c" }, { body: "d" }]);
+
+		// First flush happened immediately for the two singles
+		expect(submitSpy).toHaveBeenCalledTimes(1);
+		expect(
+			(submitSpy.mock.calls[0] as unknown[])?.[0] as AppendRecord[],
+		).toHaveLength(2);
+
+		// Explicitly flush the array batch because linger=0
+		batcher.flush();
+		await p;
+		expect(submitSpy).toHaveBeenCalledTimes(2);
+		expect(
+			(submitSpy.mock.calls[1] as unknown[])?.[0] as AppendRecord[],
+		).toHaveLength(2);
+	});
+
+	it("array submit never splits across batches and resolves with single batch ack", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const submitSpy = vi.spyOn(session, "submit");
+
+		// Two submits will occur: first flush of existing single; second flush of the array
+		submitSpy.mockResolvedValueOnce(makeAck(1));
+		submitSpy.mockResolvedValueOnce(makeAck(2));
+		const batcher = session.makeBatcher({ lingerDuration: 0, maxBatchSize: 2 });
+
+		// Fill current batch to 1
+		batcher.submit({ body: "x" });
+		// Submit 2-record array: should flush current (1), then submit both together (2)
+		const promise = batcher.submit([{ body: "a" }, { body: "b" }]);
+
+		expect(submitSpy).toHaveBeenCalledTimes(1);
+		expect(
+			(submitSpy.mock.calls[0] as unknown[])?.[0] as AppendRecord[],
+		).toHaveLength(1);
+
+		batcher.flush();
+		await promise;
+		expect(submitSpy).toHaveBeenCalledTimes(2);
+		expect(
+			(submitSpy.mock.calls[1] as unknown[])?.[0] as AppendRecord[],
+		).toHaveLength(2);
+	});
+
+	it("array submit rejects if its single batch fails", async () => {
+		const stream = makeStream();
+		const session = await stream.appendSession();
+		const submitSpy = vi.spyOn(session, "submit");
+
+		submitSpy.mockRejectedValueOnce(new Error("batch failed"));
+
+		const batcher = session.makeBatcher({ lingerDuration: 0, maxBatchSize: 2 });
+		const promise = batcher.submit([{ body: "a" }, { body: "b" }]);
+		// suppress unhandled rejection warning
+		promise.catch(() => {});
+
+		batcher.flush();
+		await expect(promise).rejects.toThrow("batch failed");
+	});
+});

package/src/tests/index.test.ts

@@ -0,0 +1,7 @@
+import { describe, expect, it } from "vitest";
+
+describe("S2", () => {
+	it("should succeed", () => {
+		expect(true).toBe(true);
+	});
+});

package/src/utils.ts

@@ -0,0 +1,80 @@
+import type { AppendRecord as AppendRecordType } from "./stream";
+
+type Headers =
+	| Record<string, string>
+	| Array<[string | Uint8Array, string | Uint8Array]>;
+
+export type AppendRecord = AppendRecordType;
+
+/**
+ * Helpers to construct appendable records.
+ *
+ * These helpers mirror the OpenAPI record schema and add convenience builders for S2 command records:
+ * - `make` creates a normal record
+ * - `command` creates a command record with an empty-name header set to the command name
+ * - `fence` is a command record enforcing a fencing token
+ * - `trim` is a command record that encodes a sequence number for trimming
+ */
+export const AppendRecord = {
+	make: (
+		body?: string | Uint8Array,
+		headers?: Headers,
+		timestamp?: AppendRecordType["timestamp"],
+	): AppendRecordType => {
+		return {
+			body,
+			headers,
+			timestamp,
+		};
+	},
+	command: (
+		/** Command name (e.g. "fence" or "trim"). */
+		command: string,
+		body?: string | Uint8Array,
+		additionalHeaders?: Headers,
+		timestamp?: AppendRecordType["timestamp"],
+	): AppendRecordType => {
+		const headers: AppendRecordType["headers"] = (() => {
+			if (!additionalHeaders) {
+				return [["", command]];
+			} else if (Array.isArray(additionalHeaders)) {
+				return [["", command] as [string, string], ...additionalHeaders];
+			} else {
+				return [
+					["", command] as [string, string],
+					...Object.entries(additionalHeaders).map(
+						([key, value]) => [key, value] as [string, string],
+					),
+				];
+			}
+		})();
+		return {
+			body,
+			headers,
+			timestamp,
+		};
+	},
+	fence: (
+		fencing_token: string,
+		additionalHeaders?: Headers,
+		timestamp?: AppendRecordType["timestamp"],
+	) => {
+		return AppendRecord.command(
+			"fence",
+			fencing_token,
+			additionalHeaders,
+			timestamp,
+		);
+	},
+	trim: (
+		seqNum: number | bigint,
+		additionalHeaders?: Headers,
+		timestamp?: AppendRecordType["timestamp"],
+	): AppendRecordType => {
+		// Encode sequence number as 8 big-endian bytes
+		const buffer = new Uint8Array(8);
+		const view = new DataView(buffer.buffer);
+		view.setBigUint64(0, BigInt(seqNum), false); // false = big-endian
+		return AppendRecord.command("trim", buffer, additionalHeaders, timestamp);
+	},
+} as const;

package/tsconfig.build.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["src/**/*"],
+  "exclude": [
+    "src/**/*.test.ts",
+    "**/*.test.ts",
+    "examples",
+    "openapi-ts.config.ts"
+  ]
+}