@streamsy/storage 0.5.0 → 0.7.0

package/README.md

@@ -122,6 +122,10 @@ In a Durable Object, use `DurableObjectStorage.layerProtocol` with
   rejected without being changed. Use a fresh database path.
 - Stream ids are limited to 512 characters, and producer ids to 255. A longer
   id fails with a non-retryable `StorageFault` before anything is written.
+- Projection checkpoint keys have no length limit. MySQL stores 512 key
+  characters, so on MySQL a longer key is stored as its first 447 characters,
+  `#` and the SHA-256 hex of the full key. The other dialects store every key
+  as it is.
 - Bun defaults the client `busyTimeout` to zero and retries storage-owned
   transactions with a bounded yielding policy. Tune `client.busyTimeout`,
   `transactionRetryAttempts`, and `transactionRetryDelayMs` if you need to.

package/dist/bun.d.ts

@@ -1,4 +1,4 @@
-import { r as SqlStorageOptions } from "./storage-kHdg037u.js";
+import { r as SqlStorageOptions } from "./storage-C1XYywlK.js";
 import { Layer } from "effect";
 import { StorageFault, Streamsy } from "@streamsy/core";
 import { Protocol } from "@streamsy/core/protocol";
@@ -12,8 +12,8 @@ interface BunStorageOptions extends SqlStorageOptions {
 /** Options for `layer` and `layerProtocol`. */
 interface Options extends BunStorageOptions, Protocol.ProtocolOptions {}
 /** Official Bun SQLite host layer with one shared SqlClient/Reactivity graph. */
-declare const storageLayer: (options: BunStorageOptions) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").OwnerRecords | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage, StorageFault, never>;
+declare const storageLayer: (options: BunStorageOptions) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").CommitHooks | import("@streamsy/core/internal").OwnerRecords | import("@streamsy/core/internal").ProjectionMarks | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage, StorageFault, never>;
 /** Complete persistent protocol Layer for an HttpRouter application. */
-declare const layerProtocol: (options: Options) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").OwnerRecords | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage | import("@streamsy/core/protocol").StreamsReader | import("@streamsy/core/protocol").StreamsWriter, StorageFault, never>;
+declare const layerProtocol: (options: Options) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").CommitHooks | import("@streamsy/core/internal").OwnerRecords | import("@streamsy/core/internal").ProjectionMarks | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage | import("@streamsy/core/protocol").StreamsReader | import("@streamsy/core/protocol").StreamsWriter, StorageFault, never>;
 declare function layer<const Keys extends ReadonlyArray<Streamsy.Any>>(group: Streamsy.Group<Keys>, options: Options): Layer.Layer<Layer.Success<ReturnType<typeof storageLayer>> | Streamsy.Host<Streamsy.Group<Keys>>, Layer.Error<ReturnType<typeof storageLayer>>>;
 export { bun_storage_d_exports as BunStorage };

package/dist/bun.js

@@ -1,5 +1,5 @@
 import { t as __exportAll } from "./rolldown-runtime-D8jQpDG7.js";
-import { a as storageLayer$1, d as retryWithPolicy, f as transactionRetryPolicy, o as inspectFormat, s as newerVersionFault, u as sharedSqlClientLayer } from "./storage-D3Ubo2kB.js";
+import { a as storageLayer$1, d as retryWithPolicy, f as transactionRetryPolicy, o as inspectFormat, s as newerVersionFault, u as sharedSqlClientLayer } from "./storage-D5lTbsiK.js";
 import { Array, Cause, Effect, FileSystem, Layer, Option, Path, Result, Schema } from "effect";
 import { StorageFault } from "@streamsy/core";
 import { Protocol } from "@streamsy/core/protocol";
@@ -26,7 +26,7 @@ const readFormat = (reader) => {
 const preflight = (filename, driver, retry) => Effect.gen(function* () {
 	const state = yield* retryWithPolicy(driver.read(filename, readFormat), retry, (error) => error.retryable);
 	yield* inspectFormat(state.names);
-	if (state.version > 1) return yield* newerVersionFault(state.version);
+	if (state.version > 2) return yield* newerVersionFault(state.version);
 });
 /**
 * Prepare a SQLite file before the client opens it: create the parent folder,

package/dist/durable-object.d.ts

@@ -1,4 +1,4 @@
-import { r as SqlStorageOptions } from "./storage-kHdg037u.js";
+import { r as SqlStorageOptions } from "./storage-C1XYywlK.js";
 import { Layer } from "effect";
 import { Streamsy } from "@streamsy/core";
 import { Protocol } from "@streamsy/core/protocol";
@@ -16,8 +16,8 @@ declare const DEFAULT_LONG_POLL_TIMEOUT_MS = 25000;
 /** Options for `layer` and `layerProtocol`. */
 interface Options extends DurableObjectStorageOptions, Protocol.ProtocolOptions {}
 /** Official local/host Durable Object SQLite layer with commit-boundary support. */
-declare const storageLayer: (options: DurableObjectStorageOptions) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").OwnerRecords | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage, import("@streamsy/core").StorageFault, never>;
+declare const storageLayer: (options: DurableObjectStorageOptions) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").CommitHooks | import("@streamsy/core/internal").OwnerRecords | import("@streamsy/core/internal").ProjectionMarks | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage, import("@streamsy/core").StorageFault, never>;
 /** Protocol over the official Durable Object SQLite layer. */
-declare const layerProtocol: (options: Options) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").OwnerRecords | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage | import("@streamsy/core/protocol").StreamsReader | import("@streamsy/core/protocol").StreamsWriter, import("@streamsy/core").StorageFault, never>;
+declare const layerProtocol: (options: Options) => Layer.Layer<import("@streamsy/core").CommitBoundary | import("@streamsy/core/internal").CommitHooks | import("@streamsy/core/internal").OwnerRecords | import("@streamsy/core/internal").ProjectionMarks | import("effect/reactivity/Reactivity").Reactivity | import("effect/sql/SqlClient").SqlClient | import("@streamsy/core").Storage | import("@streamsy/core/protocol").StreamsReader | import("@streamsy/core/protocol").StreamsWriter, import("@streamsy/core").StorageFault, never>;
 declare function layer<const Keys extends ReadonlyArray<Streamsy.Any>>(group: Streamsy.Group<Keys>, options: Options): Layer.Layer<Layer.Success<ReturnType<typeof storageLayer>> | Streamsy.Host<Streamsy.Group<Keys>>, Layer.Error<ReturnType<typeof storageLayer>>>;
 export { durable_object_storage_d_exports as DurableObjectStorage };

package/dist/durable-object.js

@@ -1,5 +1,5 @@
 import { t as __exportAll } from "./rolldown-runtime-D8jQpDG7.js";
-import { i as layerWithDurableObjectTransactions, u as sharedSqlClientLayer } from "./storage-D3Ubo2kB.js";
+import { i as layerWithDurableObjectTransactions, u as sharedSqlClientLayer } from "./storage-D5lTbsiK.js";
 import { Layer } from "effect";
 import "@streamsy/core";
 import { Protocol } from "@streamsy/core/protocol";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { a as layerProtocol, c as DEFAULT_TRANSACTION_RETRY_DELAY_MS, i as layer, n as Options, o as clientLayer, s as DEFAULT_TRANSACTION_RETRY_ATTEMPTS, t as DEFAULT_REPAIR_INTERVAL_MS } from "./storage-kHdg037u.js";
+import { a as layerProtocol, c as DEFAULT_TRANSACTION_RETRY_DELAY_MS, i as layer, n as Options, o as clientLayer, s as DEFAULT_TRANSACTION_RETRY_ATTEMPTS, t as DEFAULT_REPAIR_INTERVAL_MS } from "./storage-C1XYywlK.js";
 declare namespace sql_storage_d_exports {
   export { DEFAULT_REPAIR_INTERVAL_MS, DEFAULT_TRANSACTION_RETRY_ATTEMPTS, DEFAULT_TRANSACTION_RETRY_DELAY_MS, Options, clientLayer, layer, layerProtocol };
 }

package/dist/index.js

@@ -1,5 +1,5 @@
 import { t as __exportAll } from "./rolldown-runtime-D8jQpDG7.js";
-import { l as clientLayer, n as layer, r as layerProtocol, t as DEFAULT_REPAIR_INTERVAL_MS } from "./storage-D3Ubo2kB.js";
+import { l as clientLayer, n as layer, r as layerProtocol, t as DEFAULT_REPAIR_INTERVAL_MS } from "./storage-D5lTbsiK.js";
 // src/sql-storage.ts
 var sql_storage_exports = /* @__PURE__ */ __exportAll({
 	DEFAULT_REPAIR_INTERVAL_MS: () => DEFAULT_REPAIR_INTERVAL_MS,

package/dist/{storage-kHdg037u.d.ts → storage-C1XYywlK.d.ts}

@@ -1,8 +1,8 @@
-import { Context, Effect, Layer, Queue, Scope, Stream } from "effect";
+import { Context, Effect, Layer, PubSub, Queue, Scope, Stream } from "effect";
 import { Reactivity } from "effect/reactivity";
 import { SqlClient, SqlError } from "effect/sql";
-import { OwnerRecords } from "@streamsy/core/internal";
-import { CommitBoundary, Storage, StorageFault, Streamsy } from "@streamsy/core";
+import { CommitHooks, OwnerRecords, ProjectionMarks } from "@streamsy/core/internal";
+import { CommitBoundary, Storage, StorageFault, StreamId, Streamsy } from "@streamsy/core";
 import { Protocol } from "@streamsy/core/protocol";
 declare const DEFAULT_TRANSACTION_RETRY_ATTEMPTS = 16;
 declare const DEFAULT_TRANSACTION_RETRY_DELAY_MS = 25;
@@ -27,10 +27,10 @@ interface SqlStorageOptions {
   readonly transactionRetryDelayMs?: number;
 }
 /** Generic SQLite-stream SQL layer. Hosts supply one matching SqlClient/Reactivity graph. */
-declare const storageLayer: (options?: SqlStorageOptions) => Layer.Layer<Storage | CommitBoundary | OwnerRecords, StorageFault, SqlClient.SqlClient | Reactivity.Reactivity>;
+declare const storageLayer: (options?: SqlStorageOptions) => Layer.Layer<Storage | CommitBoundary | OwnerRecords | CommitHooks | ProjectionMarks, StorageFault, SqlClient.SqlClient | Reactivity.Reactivity>;
 /** Options for `layer` and `layerProtocol`. */
 interface Options extends SqlStorageOptions, Protocol.ProtocolOptions {}
 /** Protocol over SQL storage. */
-declare const layerProtocol: (options?: Options) => Layer.Layer<CommitBoundary | OwnerRecords | Storage | import("@streamsy/core/protocol").StreamsReader | import("@streamsy/core/protocol").StreamsWriter, StorageFault, Reactivity.Reactivity | SqlClient.SqlClient>;
+declare const layerProtocol: (options?: Options) => Layer.Layer<CommitBoundary | CommitHooks | OwnerRecords | ProjectionMarks | Storage | import("@streamsy/core/protocol").StreamsReader | import("@streamsy/core/protocol").StreamsWriter, StorageFault, Reactivity.Reactivity | SqlClient.SqlClient>;
 declare function layer<const Keys extends ReadonlyArray<Streamsy.Any>>(group: Streamsy.Group<Keys>, options?: Options): Layer.Layer<Layer.Success<ReturnType<typeof storageLayer>> | Streamsy.Host<Streamsy.Group<Keys>>, Layer.Error<ReturnType<typeof storageLayer>>, Layer.Services<ReturnType<typeof storageLayer>>>;
 export { layerProtocol as a, DEFAULT_TRANSACTION_RETRY_DELAY_MS as c, layer as i, Options as n, clientLayer as o, SqlStorageOptions as r, DEFAULT_TRANSACTION_RETRY_ATTEMPTS as s, DEFAULT_REPAIR_INTERVAL_MS as t };

package/dist/{storage-D3Ubo2kB.js → storage-D5lTbsiK.js}

@@ -1,7 +1,7 @@
-import { Array, Clock, Context, Deferred, Effect, Fiber, Layer, Option, Order, Predicate, Queue, Schedule, Scheduler, Schema, Stream } from "effect";
+import { Array, Clock, Context, Deferred, Effect, Fiber, Layer, Option, Order, Predicate, PubSub, Queue, Schedule, Scheduler, Schema, Stream } from "effect";
 import { Reactivity } from "effect/reactivity";
 import { SqlClient, SqlError } from "effect/sql";
-import { OwnerRecords, attachOwner, ownerOf } from "@streamsy/core/internal";
+import { CommitHooks, MAX_PRODUCER_ID_LENGTH, OwnerRecords, ProjectionMarks, attachOwner, commitHookRegistry, fitKey, ownerOf } from "@streamsy/core/internal";
 import { CommitBoundary, MutationRejected, Mutations, Offset, OperationResults, Storage, StorageFault, StreamId, ZERO_OFFSET } from "@streamsy/core";
 import { Protocol } from "@streamsy/core/protocol";
 const transactionRetryPolicy = (options) => {
@@ -41,7 +41,9 @@ const makeBoundary = (repairIntervalMs, transactionRetry, probe, transactionMaxO
 	const sql = yield* SqlClient.SqlClient;
 	const reactivity = yield* Reactivity.Reactivity;
 	const subscriptions = /* @__PURE__ */ new Set();
+	const feed = yield* Effect.acquireRelease(PubSub.unbounded(), PubSub.shutdown);
 	let closed = false;
+	const hooks = commitHookRegistry();
 	yield* Effect.addFinalizer(() => Effect.gen(function* () {
 		const active = yield* Effect.sync(() => {
 			closed = true;
@@ -52,17 +54,20 @@ const makeBoundary = (repairIntervalMs, transactionRetry, probe, transactionMaxO
 	}));
 	const ownedTransaction = (body) => Effect.suspend(() => {
 		if (probe !== void 0) probe.transactionAttempts += 1;
-		const pending = /* @__PURE__ */ new Set();
+		const pending = {
+			keys: /* @__PURE__ */ new Set(),
+			ids: /* @__PURE__ */ new Set()
+		};
 		let bodyCompleted = false;
-		const transactionBody = body.pipe(Effect.provideService(PendingInvalidations, pending), Effect.tap(() => Effect.sync(() => {
+		const transactionBody = body.pipe(Effect.tap(() => hooks.run(pending.ids)), Effect.provideService(PendingInvalidations, pending), Effect.tap(() => Effect.sync(() => {
 			bodyCompleted = true;
 		})));
 		const transaction = sql.withTransaction(transactionBody).pipe(Effect.catchIf((error) => bodyCompleted && SqlError.isSqlError(error), (error) => Effect.die(error)));
 		return (transactionMaxOpsBeforeYield === void 0 ? transaction : transaction.pipe(Effect.provideService(Scheduler.MaxOpsBeforeYield, transactionMaxOpsBeforeYield))).pipe(Effect.tap(() => {
-			if (pending.size === 0) return Effect.void;
+			if (pending.keys.size === 0) return Effect.void;
 			if (probe !== void 0) probe.invalidations += 1;
-			return reactivity.invalidate([...pending]);
-		}));
+			return reactivity.invalidate([...pending.keys]);
+		}), Effect.tap(() => pending.ids.size === 0 ? Effect.void : PubSub.publish(feed, [...pending.ids])));
 	});
 	const withTransaction = (body) => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (owned) => {
 		if (Option.isSome(owned)) return body;
@@ -71,12 +76,13 @@ const makeBoundary = (repairIntervalMs, transactionRetry, probe, transactionMaxO
 			return ownedTransaction(body);
 		});
 	});
-	const mutation = ({ keys, effect, retryable }) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (pendingBefore) => {
+	const mutation = ({ keys, ids, effect, retryable }) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (pendingBefore) => {
 		if (Option.isSome(ambient) && Option.isNone(pendingBefore)) return Effect.die(/* @__PURE__ */ new Error("Ambient SQL transaction is not owned by CommitBoundary"));
 		const run = Effect.tap(effect, () => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (pending) => {
 			if (Option.isNone(pending)) return Effect.die(/* @__PURE__ */ new Error("CommitBoundary owner context disappeared"));
 			return Effect.sync(() => {
-				for (const key of keys) pending.value.add(key);
+				for (const key of keys) pending.value.keys.add(key);
+				for (const id of ids) pending.value.ids.add(id);
 			});
 		}));
 		return Option.isSome(ambient) ? run : retryWithPolicy(ownedTransaction(run), transactionRetry, (error) => SqlError.isSqlError(error) ? error.isRetryable : retryable(error));
@@ -134,7 +140,9 @@ const makeBoundary = (repairIntervalMs, transactionRetry, probe, transactionMaxO
 	return {
 		withTransaction,
 		mutation,
-		changes
+		changes,
+		committed: PubSub.subscribe(feed),
+		beforeCommit: hooks.beforeCommit
 	};
 });
 const boundaryLayer = (repairIntervalMs, transactionRetry, probe, transactionMaxOpsBeforeYield) => Layer.effect(BoundaryRuntimeService, makeBoundary(repairIntervalMs, transactionRetry, probe, transactionMaxOpsBeforeYield));
@@ -158,7 +166,8 @@ const STORAGE_TABLES = [
 	"streamsy_streams",
 	"streamsy_messages",
 	"streamsy_producers",
-	"streamsy_owner_records"
+	"streamsy_owner_records",
+	"streamsy_projection_pending"
 ];
 const STORAGE_VERSION_TABLE_DDL = {
 	sqlite: `CREATE TABLE ${STORAGE_VERSION_TABLE} (version INTEGER PRIMARY KEY, applied_at_ms INTEGER NOT NULL)`,
@@ -252,6 +261,23 @@ const STORAGE_MIGRATIONS = [{
  owner_key VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PRIMARY KEY, owner_value LONGTEXT NOT NULL
 ) ENGINE=InnoDB`
 	]
+}, {
+	sqlite: [`CREATE TABLE streamsy_projection_pending (
+ projection TEXT NOT NULL, params TEXT NOT NULL, seq INTEGER NOT NULL,
+ attempts INTEGER NOT NULL DEFAULT 0, due_at INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY(projection, params)
+)`, `CREATE INDEX idx_streamsy_projection_pending_due ON streamsy_projection_pending(projection, due_at)`],
+	pg: [`CREATE TABLE streamsy_projection_pending (
+ projection TEXT COLLATE "C" NOT NULL, params TEXT COLLATE "C" NOT NULL, seq BIGINT NOT NULL,
+ attempts INTEGER NOT NULL DEFAULT 0, due_at BIGINT NOT NULL DEFAULT 0,
+ PRIMARY KEY(projection, params)
+)`, `CREATE INDEX idx_streamsy_projection_pending_due ON streamsy_projection_pending(projection, due_at)`],
+	mysql: [`CREATE TABLE IF NOT EXISTS streamsy_projection_pending (
+ projection VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL, params VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL,
+ seq BIGINT NOT NULL, attempts INT NOT NULL DEFAULT 0, due_at BIGINT NOT NULL DEFAULT 0,
+ PRIMARY KEY(projection, params),
+ INDEX idx_streamsy_projection_pending_due (projection, due_at)
+) ENGINE=InnoDB`]
 }];
 // src/migrations.ts
 /** Fixed `pg_advisory_xact_lock` key: "streamsy" in ASCII, read as a 64-bit integer. */
@@ -259,13 +285,13 @@ const PG_MIGRATION_LOCK_KEY = "8319400191411975033";
 /** Named lock that serialises MySQL migrations, whose DDL cannot share one transaction. */
 const MYSQL_MIGRATION_LOCK = STORAGE_VERSION_TABLE;
 const MYSQL_MIGRATION_LOCK_TIMEOUT_SECONDS = 30;
-const fault = (operation, error) => new StorageFault({
+const fault$1 = (operation, error) => new StorageFault({
 	operation,
 	message: error.message,
 	retryable: error.isRetryable,
 	cause: error
 });
-const sqlAttempt = (operation, effect) => effect.pipe(Effect.mapError((error) => fault(operation, error)));
+const sqlAttempt = (operation, effect) => effect.pipe(Effect.mapError((error) => fault$1(operation, error)));
 /** Return whether the database already holds this format, or fail on an older format. */
 const inspectFormat = (tables) => {
 	const ownsFormat = tables.has(STORAGE_VERSION_TABLE);
@@ -277,7 +303,7 @@ const inspectFormat = (tables) => {
 };
 const newerVersionFault = (version) => new StorageFault({
 	operation: "migration.open",
-	message: `Unsupported newer Streamsy storage schema version ${version}; supported version is 1`,
+	message: `Unsupported newer Streamsy storage schema version ${version}; supported version is 2`,
 	retryable: false
 });
 const listTables = (sql, dialect) => sqlAttempt("migration.inspect", {
@@ -290,8 +316,8 @@ const listTables = (sql, dialect) => sqlAttempt("migration.inspect", {
 const migrateVersions = (sql, dialect, hooks) => Effect.gen(function* () {
 	if (!(yield* inspectFormat(yield* listTables(sql, dialect)))) yield* sqlAttempt("migration.create-version-table", sql.unsafe(STORAGE_VERSION_TABLE_DDL[dialect]).pipe(Effect.asVoid));
 	const current = yield* sqlAttempt("migration.read-version", sql`SELECT MAX(version) AS version FROM ${sql(STORAGE_VERSION_TABLE)}`.pipe(Effect.map((rows) => Number(rows[0]?.version ?? 0))));
-	if (current > 1) return yield* newerVersionFault(current);
-	for (let version = current + 1; version <= 1; version++) {
+	if (current > 2) return yield* newerVersionFault(current);
+	for (let version = current + 1; version <= 2; version++) {
 		if (hooks.beforeVersion !== void 0) yield* hooks.beforeVersion(version);
 		const statements = STORAGE_MIGRATIONS[version - 1]?.[dialect];
 		if (statements === void 0) return yield* Effect.die(/* @__PURE__ */ new Error("Missing migration"));
@@ -326,7 +352,86 @@ const migrate = (sql, hooks = {}, transactionRetry = transactionRetryPolicy({}))
 	hooks.onTransactionAttempt?.();
 	if (dialect === "mysql") return migrateMysql(sql, hooks);
 	return sql.withTransaction(dialect === "pg" ? sqlAttempt("migration.lock", sql`SELECT pg_advisory_xact_lock(${sql.literal(PG_MIGRATION_LOCK_KEY)})`).pipe(Effect.andThen(migrateVersions(sql, dialect, hooks))) : migrateVersions(sql, dialect, hooks));
-}), transactionRetry, (error) => SqlError.isSqlError(error) ? error.isRetryable : error.retryable).pipe(Effect.catchIf(SqlError.isSqlError, (error) => Effect.fail(fault("migration.transaction", error)))));
+}), transactionRetry, (error) => SqlError.isSqlError(error) ? error.isRetryable : error.retryable).pipe(Effect.catchIf(SqlError.isSqlError, (error) => Effect.fail(fault$1("migration.transaction", error)))));
+// src/projection-marks.ts
+const TABLE = "streamsy_projection_pending";
+/**
+* Durable Object SQL accepts at most 100 bound parameters in one statement. A
+* marked row binds five values and a params lookup binds one, so both chunk.
+*/
+const MARK_CHUNK = 16;
+const READ_CHUNK = 64;
+const fault = (operation) => (error) => new StorageFault({
+	operation: `marks.${operation}`,
+	message: error.message,
+	retryable: error.isRetryable,
+	cause: error
+});
+const pendingMark = (row) => ({
+	params: row.params,
+	seq: Number(row.seq),
+	attempts: Number(row.attempts),
+	dueAt: Number(row.due_at)
+});
+/**
+* Pending marks in `streamsy_projection_pending`. Statements join the boundary
+* transaction when one is open, so a mark written by a commit hook commits or
+* rolls back with the write that caused it.
+*/
+const sqlProjectionMarks = (sql) => {
+	const table = sql(TABLE);
+	const mark = (projection, params) => Effect.forEach(Array.chunksOf(params, MARK_CHUNK), (chunk) => {
+		const rows = sql.insert(Array.map(chunk, (value) => ({
+			projection,
+			params: value,
+			seq: 1,
+			attempts: 0,
+			due_at: 0
+		})));
+		return sql.onDialectOrElse({
+			mysql: () => sql`INSERT INTO ${table} ${rows} ON DUPLICATE KEY UPDATE seq = seq + 1`,
+			orElse: () => sql`INSERT INTO ${table} ${rows}
+ ON CONFLICT (projection, params) DO UPDATE SET seq = ${table}.seq + 1`
+		});
+	}, { discard: true });
+	const nextDue = (projection) => sql`SELECT MIN(due_at) AS at FROM ${table} WHERE projection = ${projection}`.pipe(Effect.map((rows) => {
+		const at = rows[0]?.at;
+		return at === null || at === void 0 ? Option.none() : Option.some(Number(at));
+	}), Effect.mapError(fault("nextDue")));
+	const due = (projection, now, limit) => sql`SELECT params, seq, attempts, due_at FROM ${table}
+ WHERE projection = ${projection} AND due_at <= ${now}
+ ORDER BY due_at, params LIMIT ${limit}`.pipe(Effect.map(Array.map(pendingMark)), Effect.mapError(fault("due")));
+	const read = (projection, params) => Effect.forEach(Array.chunksOf(params, READ_CHUNK), (chunk) => sql`SELECT params, seq, attempts, due_at FROM ${table}
+ WHERE projection = ${projection} AND ${sql.in("params", chunk)}`).pipe(Effect.map((chunks) => new Map(Array.map(chunks.flat(), (row) => [row.params, pendingMark(row)]))), Effect.mapError(fault("read")));
+	const clear = (projection, params, seq) => sql`DELETE FROM ${table} WHERE projection = ${projection} AND params = ${params} AND seq = ${seq}`.pipe(Effect.asVoid, Effect.mapError(fault("clear")));
+	const remove = (projection, params) => sql`DELETE FROM ${table} WHERE projection = ${projection} AND params = ${params}`.pipe(Effect.asVoid, Effect.mapError(fault("remove")));
+	const defer = (projection, params, attempts, dueAt) => {
+		const row = sql.insert({
+			projection,
+			params,
+			seq: 1,
+			attempts,
+			due_at: dueAt
+		});
+		return sql.onDialectOrElse({
+			mysql: () => sql`INSERT INTO ${table} ${row}
+ ON DUPLICATE KEY UPDATE attempts = VALUES(attempts), due_at = VALUES(due_at)`,
+			orElse: () => sql`INSERT INTO ${table} ${row}
+ ON CONFLICT (projection, params) DO UPDATE SET attempts = excluded.attempts, due_at = excluded.due_at`
+		}).pipe(Effect.asVoid, Effect.mapError(fault("defer")));
+	};
+	const deferred = (projection) => sql`SELECT COUNT(*) AS count FROM ${table} WHERE projection = ${projection} AND attempts > 0`.pipe(Effect.map((rows) => Number(rows[0]?.count ?? 0)), Effect.mapError(fault("deferred")));
+	return ProjectionMarks.of({
+		mark,
+		nextDue,
+		due,
+		read,
+		clear,
+		remove,
+		defer,
+		deferred
+	});
+};
 // src/storage.ts
 const DEFAULT_REPAIR_INTERVAL_MS = 1e3;
 const STORAGE_KEY = "streamsy:storage";
@@ -448,7 +553,7 @@ const tooLong = (operation, label, value, limit) => {
 };
 const oversizedKey = (mutation) => {
 	for (const operation of mutation.operations) {
-		const fault = Predicate.isTagged(operation, "Create") ? tooLong("mutate.stream-id", "Stream id", operation.record.id, 512) : Predicate.isTagged(operation, "Append") && operation.producer !== void 0 ? tooLong("mutate.producer-id", "Producer id", operation.producer.producerId, 255) : Option.none();
+		const fault = Predicate.isTagged(operation, "Create") ? tooLong("mutate.stream-id", "Stream id", operation.record.id, 512) : Predicate.isTagged(operation, "Append") && operation.producer !== void 0 ? tooLong("mutate.producer-id", "Producer id", operation.producer.producerId, MAX_PRODUCER_ID_LENGTH) : Option.none();
 		if (Option.isSome(fault)) return fault;
 	}
 	return Option.none();
@@ -723,6 +828,7 @@ const makeStorage = (sql, boundary) => Storage.of({
 		if (Option.isSome(oversized)) return Effect.fail(oversized.value);
 		return boundary.mutation({
 			keys: [STORAGE_KEY],
+			ids,
 			effect: applyMutation(sql, mutation),
 			retryable: (error) => Predicate.isTagged(error, "StorageFault") && error.retryable
 		}).pipe(Effect.catchIf(SqlError.isSqlError, (error) => Effect.fail(sqlFault("mutate.transaction", error))));
@@ -731,6 +837,7 @@ const makeStorage = (sql, boundary) => Storage.of({
 		keys: [STORAGE_KEY],
 		read: snapshot(sql, id)
 	}),
+	committed: boundary.committed,
 	nextExpiry: attempt("nextExpiry", sql`SELECT expires_at_ms AS at, stream_id FROM streamsy_streams
  WHERE expires_at_ms IS NOT NULL AND soft_deleted = 0
  ORDER BY expires_at_ms, stream_id LIMIT 1`).pipe(Effect.flatMap((rows) => rows[0] === void 0 ? Effect.succeed(Option.none()) : decodeExpiryRow(rows[0]).pipe(Effect.map(({ at, stream_id: streamId }) => Option.some({
@@ -756,6 +863,10 @@ const OWNER_RECORD_CONFLICT = "records.conflict";
 */
 const makeOwnerRecords = (sql) => {
 	const absentKeys = /* @__PURE__ */ new WeakMap();
+	const storedKey = (key) => sql.onDialectOrElse({
+		mysql: () => fitKey(key, 512),
+		orElse: () => Effect.succeed(key)
+	});
 	const select = (key, forUpdate) => attempt("records.read", sql`SELECT owner_value FROM streamsy_owner_records WHERE owner_key = ${key}${sql.literal(forUpdate ? " FOR UPDATE" : "")}`).pipe(Effect.map((rows) => Option.fromUndefinedOr(rows[0]?.owner_value)));
 	const lockFound = (key, found) => Option.isNone(found) ? Effect.succeed(found) : sql.onDialectOrElse({
 		sqlite: () => Effect.succeed(found),
@@ -783,7 +894,8 @@ const makeOwnerRecords = (sql) => {
 		cause: error
 	}) : sqlFault("records.write", error)), Effect.asVoid);
 	return OwnerRecords.of({
-		read: Effect.fn("SqlStorage.records.read")(function* (key) {
+		read: Effect.fn("SqlStorage.records.read")(function* (requested) {
+			const key = yield* storedKey(requested);
 			const transaction = yield* Effect.serviceOption(sql.transactionService);
 			if (Option.isNone(transaction)) return yield* select(key, false);
 			const found = yield* Effect.flatMap(select(key, false), (first) => lockFound(key, first));
@@ -793,7 +905,8 @@ const makeOwnerRecords = (sql) => {
 			else absent.delete(key);
 			return found;
 		}),
-		write: Effect.fn("SqlStorage.records.write")(function* (key, encoded) {
+		write: Effect.fn("SqlStorage.records.write")(function* (requested, encoded) {
+			const key = yield* storedKey(requested);
 			const transaction = yield* Effect.serviceOption(sql.transactionService);
 			const absent = Option.isSome(transaction) ? absentKeys.get(transaction.value) : void 0;
 			if (absent?.has(key) === true) {
@@ -801,7 +914,10 @@ const makeOwnerRecords = (sql) => {
 				absent.delete(key);
 			} else yield* upsert(key, encoded);
 		}),
-		remove: Effect.fn("SqlStorage.records.remove")((key) => execute("records.remove", sql`DELETE FROM streamsy_owner_records WHERE owner_key = ${key}`))
+		remove: Effect.fn("SqlStorage.records.remove")(function* (requested) {
+			const key = yield* storedKey(requested);
+			yield* execute("records.remove", sql`DELETE FROM streamsy_owner_records WHERE owner_key = ${key}`);
+		})
 	});
 };
 const makeLayer = (options, probe, transactionMaxOpsBeforeYield) => {
@@ -813,7 +929,7 @@ const makeLayer = (options, probe, transactionMaxOpsBeforeYield) => {
 		} }, transactionRetry);
 		const boundary = yield* BoundaryRuntimeService;
 		const owner = ownerOf(boundary);
-		return Context.empty().pipe(Context.add(Storage, attachOwner(makeStorage(sql, boundary), owner)), Context.add(CommitBoundary, CommitBoundary.of(boundary)), Context.add(OwnerRecords, attachOwner(makeOwnerRecords(sql), owner)));
+		return Context.empty().pipe(Context.add(Storage, attachOwner(makeStorage(sql, boundary), owner)), Context.add(CommitBoundary, CommitBoundary.of(boundary)), Context.add(OwnerRecords, attachOwner(makeOwnerRecords(sql), owner)), Context.add(CommitHooks, attachOwner(CommitHooks.of({ beforeCommit: boundary.beforeCommit }), owner)), Context.add(ProjectionMarks, attachOwner(sqlProjectionMarks(sql), owner)));
 	})).pipe(Layer.provide(boundaryLayer(options.repairIntervalMs ?? 1e3, transactionRetry, probe, transactionMaxOpsBeforeYield)));
 };
 /** Generic SQLite-stream SQL layer. Hosts supply one matching SqlClient/Reactivity graph. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@streamsy/storage",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Effect SQL storage for Streamsy on SQLite, Postgres and MySQL, with Bun and Durable Object SQLite host layers.",
   "license": "MIT",
   "repository": {
@@ -42,7 +42,7 @@
     "test:dialects": "bun --bun vitest run test/dialects"
   },
   "dependencies": {
-    "@streamsy/core": "^0.5.0"
+    "@streamsy/core": "^0.7.0"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "4.20251228.0",
@@ -52,7 +52,7 @@
     "@effect/sql-sqlite-bun": "4.0.5",
     "@effect/sql-sqlite-do": "4.0.5",
     "@effect/vitest": "4.0.5",
-    "@streamsy/serve": "0.5.0",
+    "@streamsy/serve": "0.7.0",
     "@types/bun": "^1.4.3",
     "effect": "4.0.5",
     "miniflare": "4.20260730.0",