@cmmd-center/forge 0.13.116 → 0.13.118

package/dist/bin.cjs

@@ -66337,7 +66337,7 @@ function normalizeNumberish(value) {
 }
 //#endregion
 //#region package.json
-var version$1 = "0.13.116";
+var version$1 = "0.13.118";
 //#endregion
 //#region src/sentry.ts
 const SERVER_APP_NAME = "forge-server";
@@ -95424,6 +95424,16 @@ var RuntimeReplicationClient = class extends require_Schema$1.Service()("forge/r
 * from the retry cap and registration can outlast the cap.
 */
 const RUNTIME_NOT_REGISTERED_CODE = "RUNTIME_NOT_REGISTERED";
+/**
+* CMMD returned 409 THREAD_STREAM_PAYLOAD_CONFLICT or THREAD_STREAM_HEAD_MISMATCH:
+* its canonical history for a thread generation has diverged from the batch
+* Forge just sent (forge#3620). Exported so `processRuntimeReplicationOutboxItem`
+* can tell when a recorded `fail()` may have deferred the actual repair to a
+* full re-projection instead of finishing it inline -- see
+* `RuntimeReplicationOutbox.ts`'s `fail()`/`repairThreadStreamFromRemoteConflict`
+* for the repair itself.
+*/
+const THREAD_STREAM_REMOTE_CONFLICT_CODES = new Set(["THREAD_STREAM_PAYLOAD_CONFLICT", "THREAD_STREAM_HEAD_MISMATCH"]);
 const retryableCodes = new Set([
 	"RUNTIME_NOT_READY",
 	"RUNTIME_THREAD_NOT_READY",
@@ -300397,7 +300407,7 @@ function resolveBuildCommitFromEnv(env) {
 * environment descriptor down instead of reporting an honest "unknown".
 */
 function readBakedBuildCommit() {
-	return "b758fc443098891193c447a97051962c12fdbee2";
+	return "1de5f431edc763d9146f9be86885d18e191ea9f7";
 }
 async function resolveServerBuildCommit(input) {
 	if (isFullCommitSha(input.baked)) return input.baked;
@@ -314744,6 +314754,33 @@ function applyPendingThreadArchives(input) {
 }
 //#endregion
 //#region src/runtimeReplication/processOutboxItem.ts
+/** The thread id a `thread:<id>` outbox stream key was minted from. */
+function threadIdFromStreamKey(streamKey) {
+	return ThreadId.make(streamKey.slice(7));
+}
+/**
+* Attempts `enqueueThread` gets for one outbox item before this gives up and
+* logs loudly. `dependencies.enqueueThread` is the closure RuntimeReplication.ts
+* builds for `processItem` specifically -- unlike the swallowing variant it
+* hands to the domain-event reactor and the 5-minute heartbeat (which retry
+* on their own cadence regardless), a Cause escaping this one is visible
+* here. Right after `fail()` parks a thread-events row under
+* THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE, this call is the
+* corrective re-projection that clears it; without a retry here, one failed
+* attempt left the row silently blocked until the next heartbeat, a later
+* domain event, or a manual retry (forge#3621 finding 2). A transient
+* failure gets a few immediate retries; one that survives them all is logged
+* loudly instead of disappearing.
+*/
+const ENQUEUE_THREAD_RETRY_ATTEMPTS = 3;
+function enqueueThreadWithBoundedRetry(enqueueThread, threadId, context, attemptsRemaining = ENQUEUE_THREAD_RETRY_ATTEMPTS) {
+	return enqueueThread(threadId).pipe(require_Schema$1.catchCause((cause) => attemptsRemaining > 1 ? enqueueThreadWithBoundedRetry(enqueueThread, threadId, context, attemptsRemaining - 1) : require_Schema$1.logError("Runtime replication thread re-projection failed after retrying; it will run again on the next heartbeat, domain event, or manual retry.", {
+		...context,
+		threadId,
+		cause,
+		attempts: ENQUEUE_THREAD_RETRY_ATTEMPTS
+	})));
+}
 function publicationMetricAttributes(item, outcome, failure) {
 	return {
 		operation: item.operation,
@@ -314781,7 +314818,12 @@ function processRuntimeReplicationOutboxItem(item, dependencies) {
 				failureKind: failure.kind,
 				errorCode: failure.code
 			});
+			return;
 		}
+		if (item.operation === "thread-events" && THREAD_STREAM_REMOTE_CONFLICT_CODES.has(failure.code)) yield* enqueueThreadWithBoundedRetry(dependencies.enqueueThread, threadIdFromStreamKey(item.streamKey), {
+			runtimeId: item.runtimeId,
+			streamKey: item.streamKey
+		});
 	});
 	return require_Schema$1.gen(function* () {
 		const payload = yield* require_Schema$1.sync(() => {
@@ -314822,7 +314864,10 @@ function processRuntimeReplicationOutboxItem(item, dependencies) {
 				return require_Schema$1.all([
 					increment(runtimeReplicationPublicationAttemptsTotal, publicationMetricAttributes(item, "success")),
 					require_Schema$1.update(require_Schema$1.withAttributes(runtimeReplicationPublicationLatency, metricAttributes({ operation: item.operation })), require_Schema$1.millis(publicationLatency)),
-					item.operation === "thread-events" ? dependencies.enqueueThread(ThreadId.make(item.streamKey.slice(7))) : require_Schema$1.void_
+					item.operation === "thread-events" ? enqueueThreadWithBoundedRetry(dependencies.enqueueThread, threadIdFromStreamKey(item.streamKey), {
+						runtimeId: item.runtimeId,
+						streamKey: item.streamKey
+					}) : require_Schema$1.void_
 				]).pipe(require_Schema$1.asVoid);
 			}))
 		}));
@@ -314899,15 +314944,16 @@ const RuntimeReplicationLive = require_Schema$1.effect(RuntimeReplication, requi
 		operation: "inventory",
 		payload
 	}) : require_Schema$1.void_;
-	const enqueueThread = (threadId) => runtimeId ? snapshots.getThreadDetailById(threadId).pipe(require_Schema$1.flatMap((thread) => require_Schema$1.isSome(thread) ? logTurnsMissingModelSelection(threadId, thread.value).pipe(require_Schema$1.andThen(outbox.enqueueThread({
+	const enqueueThreadUnsafe = (threadId) => runtimeId ? snapshots.getThreadDetailById(threadId).pipe(require_Schema$1.flatMap((thread) => require_Schema$1.isSome(thread) ? logTurnsMissingModelSelection(threadId, thread.value).pipe(require_Schema$1.andThen(outbox.enqueueThread({
 		runtimeId,
 		streamKey: `thread:${threadId}`,
 		events: serializeRuntimeThreadEvents(thread.value),
 		historyFinalized: thread.value.archivedAt != null
-	}))) : outbox.remove(runtimeId, `thread:${threadId}`)), require_Schema$1.catchCause((cause) => require_Schema$1.logError("Runtime replication thread projection failed.", {
+	}))) : outbox.remove(runtimeId, `thread:${threadId}`)), require_Schema$1.orDie) : require_Schema$1.void_;
+	const enqueueThread = (threadId) => enqueueThreadUnsafe(threadId).pipe(require_Schema$1.catchCause((cause) => require_Schema$1.logError("Runtime replication thread projection failed.", {
 		cause,
 		threadId
-	}))) : require_Schema$1.void_;
+	})));
 	const projectEvent = (event) => runtimeId ? activeHourMeter.markAwake(runtimeId).pipe(require_Schema$1.andThen(require_Schema$1.all([snapshots.getShellSnapshot(), providerRegistry.getProviders])), require_Schema$1.flatMap(([snapshot, providers]) => {
 		const inventory = serializeRuntimeInventory(snapshot, providers);
 		const threadId = event.aggregateKind === "thread" ? ThreadId.make(event.aggregateId) : null;
@@ -314920,7 +314966,7 @@ const RuntimeReplicationLive = require_Schema$1.effect(RuntimeReplication, requi
 	const processItem = (item) => processRuntimeReplicationOutboxItem(item, {
 		outbox,
 		client,
-		enqueueThread
+		enqueueThread: enqueueThreadUnsafe
 	});
 	const drain = outbox.claim(BATCH_SIZE).pipe(require_Schema$1.flatMap((items) => require_Schema$1.forEach(items, processItem, { concurrency: 1 })), require_Schema$1.catchCause((cause) => require_Schema$1.logError("Runtime replication outbox drain failed.", { cause })));
 	const start = () => {
@@ -315016,7 +315062,14 @@ const RuntimeReplicationLive = require_Schema$1.effect(RuntimeReplication, requi
 			observeOutbox
 		]).pipe(require_Schema$1.asVoid);
 	};
-	return RuntimeReplication.of({ start });
+	const retryReprojectionPending = (streamKeys) => require_Schema$1.forEach(streamKeys, (streamKey) => enqueueThread(threadIdFromStreamKey(streamKey)), {
+		concurrency: 1,
+		discard: true
+	});
+	return RuntimeReplication.of({
+		start,
+		retryReprojectionPending
+	});
 }));
 //#endregion
 //#region src/runtimeReplication/activeHourProtocol.ts
@@ -315315,6 +315368,100 @@ const RUNTIME_REPLICATION_RETRY_EXHAUSTED_CODE = "RUNTIME_REPLICATION_RETRY_EXHA
 * claims it; only `recover()` and the manual retry endpoint do.
 */
 const THREAD_STREAM_PAYLOAD_HASH_MISMATCH_CODE = "THREAD_STREAM_PAYLOAD_HASH_MISMATCH";
+/**
+* CMMD returned 409 THREAD_STREAM_PAYLOAD_CONFLICT or THREAD_STREAM_HEAD_MISMATCH:
+* its canonical history for this generation has diverged from the batch Forge
+* just sent (forge#3620). The common trigger is CMMD's own reclaim job
+* replacing the Sprite (CMMD-Center/CMMD#5865) mid-turn, which wipes this
+* runtime's local `runtime_replication_streams` cursor; Forge then rebuilds a
+* batch from sequence 1 that CMMD, still holding the old generation's
+* acknowledged history, refuses. `fail()` recovers by starting a brand-new
+* repair generation from sequence 1 with an empty previous head -- the one
+* shape CMMD's `validateRuntimeThreadAppend` accepts unconditionally as a
+* reset -- rather than blocking the thread forever.
+*
+* The refused batch's own `events` are never safe to rebuild that repair from
+* directly, regardless of `fromSequence`. A batch is capped at `batchSize`
+* (500, see `buildRuntimeThreadBatch` in protocol.ts) and `isFinalBatch` is
+* computed relative to whatever array it was handed -- so re-running
+* `buildRuntimeThreadBatch` on a refused batch's own, already-truncated
+* `events` array makes an 800-event thread's 500-event suffix look like a
+* complete 500-event history all over again, `isFinalBatch: true` included.
+* CMMD accepts that as this generation's full canonical history on the reset
+* rule and events 501 onward are gone for good (CodeRabbit finding on
+* forge#3621, forge#3620 finding 1). This holds even at `fromSequence === 1`:
+* a fresh generation's very first batch is truncated exactly the same way
+* once the thread has more events than `batchSize`.
+*
+* `fail()` instead always resets the local cursor to a new repair generation
+* and parks the row blocked under
+* `THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE` until
+* `processRuntimeReplicationOutboxItem` re-derives the FULL thread history
+* through the same `enqueueThread` projection path the success path already
+* runs after every publish (see `processOutboxItem.ts`) and overwrites this
+* row with a correctly full batch. That re-projection reads the domain
+* model's complete event list, not a persisted batch's truncated `events`,
+* so it is safe to rebuild from no matter how many events the thread holds.
+*
+* The reset is bounded to one per distinct conflict, keyed on whether the
+* reset itself has made any progress: see `refusedResetIsStuck` below. The
+* re-projection failing to land promptly does not strand the row forever --
+* `processRuntimeReplicationOutboxItem` retries it a bounded number of times
+* before giving up for this cycle, the 5-minute replication heartbeat and any
+* later domain event on this thread call `enqueueThread` again regardless,
+* and the manual `/api/runtime-replication/retry` endpoint calls it directly
+* for rows parked under this code (see `RuntimeReplication.ts`'s
+* `retryReprojectionPending`).
+*/
+const THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE = "THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING";
+/**
+* Whether a refused batch is the exact canonical reset shape (`fromSequence
+* === 1`, empty previous head, deterministic content) under a generation the
+* local cursor already holds with zero acknowledged progress -- i.e. this
+* exact reset was already sent once and CMMD refused it again, which makes
+* it genuinely corrupt local history rather than a race to retry. Applies to
+* any non-base generation (both the `:repair:` suffix this file mints and
+* the `:final:` suffix `runtimeThreadFinalGenerationId` mints for an
+* archived thread's first post-transition batch -- both always start a fresh
+* generation at sequence 1 from an empty head the same way).
+*
+* The base generation is exempt unconditionally: its own first-sequence
+* conflict is the original, always-eligible incident trigger (Forge's local
+* cursor was wiped and it resent from scratch), not a repeat of an
+* already-attempted reset. A repair or final generation that already made
+* progress (`fromSequence > 1` under it) is exempt too -- CMMD acknowledged
+* it at least once, so a later refusal is a new, unrelated conflict and
+* earns its own fresh reset (forge#3620 finding 2); the bound must not key
+* on a generation id merely containing `:repair:`/`:final:`, or a thread
+* repaired once could never recover from a later, unrelated conflict.
+*/
+function refusedResetIsStuck(input) {
+	if (input.refusedGenerationId === input.baseGenerationId) return false;
+	if (input.refusedFromSequence !== 1) return false;
+	return input.cursor === void 0 || input.cursor.generationId !== input.refusedGenerationId || input.cursor.acknowledgedSequence === 0;
+}
+/**
+* The generation id, starting sequence, and raw source events out of a
+* persisted thread batch, read straight off the JSON rather than through
+* `decodeThreadBatch` -- that schema only keeps `sequence`/`payloadHash` per
+* event, and a repair needs the full canonical fields to re-hash from
+* scratch. Extra fields (the old `sequence`/`payloadHash`) are harmless:
+* `buildRuntimeThreadBatch` overwrites both and its hasher reads only the
+* named canonical fields.
+*/
+function parseThreadBatchForRepair(payloadJson) {
+	try {
+		const parsed = JSON.parse(payloadJson);
+		if (typeof parsed.generationId !== "string" || typeof parsed.fromSequence !== "number" || !Array.isArray(parsed.events)) return null;
+		return {
+			generationId: parsed.generationId,
+			fromSequence: parsed.fromSequence,
+			events: parsed.events
+		};
+	} catch {
+		return null;
+	}
+}
 const STARTUP_RECOVERABLE_BLOCKED_CODES = [
 	...CLAIMABLE_BLOCKED_CODES,
 	"RUNTIME_NOT_READY",
@@ -315525,6 +315672,26 @@ const retryBlockedRuntimeReplicationOutbox = require_Schema$1.gen(function* () {
     RETURNING stream_key AS "streamKey"
   `).length;
 }).pipe(require_Schema$1.orDie);
+/**
+* Thread streams parked under THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE,
+* for the manual `/api/runtime-replication/retry` endpoint. This code is deliberately
+* absent from MANUALLY_RECOVERABLE_BLOCKED_CODES: flipping the row back to `pending`
+* the way `retryBlockedRuntimeReplicationOutbox` does for other blocked codes would
+* requeue the same truncated payload `fail()` already refused, undoing the very repair
+* it just started. `http.ts`'s retry handler passes these to `RuntimeReplication`'s
+* `retryReprojectionPending`, which re-runs `enqueueThread` for each -- the same
+* corrective re-projection that already clears this code after every publish and after
+* every conflict (forge#3621 finding 2). No `runtimeId` filter, matching
+* `retryBlockedRuntimeReplicationOutbox` above: this outbox table belongs to one Forge
+* server.
+*/
+const readReprojectionPendingThreadStreamKeys = require_Schema$1.gen(function* () {
+	return (yield* (yield* require_SqlError.SqlClient)`
+    SELECT stream_key AS "streamKey" FROM runtime_replication_outbox
+    WHERE operation = 'thread-events' AND status = 'blocked'
+      AND last_error_code = ${THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE}
+  `).map((row) => row.streamKey);
+}).pipe(require_Schema$1.orDie);
 const RuntimeReplicationOutboxLive = require_Schema$1.effect(RuntimeReplicationOutbox, require_Schema$1.gen(function* () {
 	const sql = yield* require_SqlError.SqlClient;
 	const upsertPayload = (input) => {
@@ -316052,7 +316219,69 @@ const RuntimeReplicationOutboxLive = require_Schema$1.effect(RuntimeReplicationO
 			discard: true
 		});
 	})).pipe(require_Schema$1.orDie);
-	const fail = (item, failure) => {
+	/**
+	* Recovers from a CMMD-reported thread stream conflict (see
+	* THREAD_STREAM_REMOTE_CONFLICT_CODES). Returns `null` when this item is
+	* not eligible (its payload does not decode, or `refusedResetIsStuck`
+	* says the exact reset shape already failed once -- in which case the
+	* caller falls back to blocking) and `false` when the row was superseded
+	* before this could apply, mirroring `fail`'s own supersession signal.
+	* Otherwise replaces the local cursor with a brand-new repair generation
+	* from sequence 1, empty previous head -- the shape CMMD's reset rule
+	* accepts unconditionally -- and parks the row blocked under
+	* `THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE`, deferring the
+	* actual rebuild to the caller's next full re-projection regardless of
+	* `fromSequence` (see the big comment above
+	* `THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING_CODE`): the refused
+	* batch's own `events` are truncated at `batchSize` the same way whether
+	* this was its first batch or its tenth, so rebuilding from them directly
+	* can silently drop everything past the cap.
+	*/
+	const repairThreadStreamFromRemoteConflict = (item, failure) => require_Schema$1.gen(function* () {
+		const parsedBatch = parseThreadBatchForRepair(item.payloadJson);
+		if (!parsedBatch) return null;
+		const baseGenerationId = runtimeThreadGenerationId(item.runtimeId, item.streamKey);
+		const cursor = yield* readThreadCursor({
+			runtimeId: item.runtimeId,
+			streamKey: item.streamKey
+		});
+		if (refusedResetIsStuck({
+			baseGenerationId,
+			refusedGenerationId: parsedBatch.generationId,
+			refusedFromSequence: parsedBatch.fromSequence,
+			cursor
+		})) return null;
+		const now = (/* @__PURE__ */ new Date()).toISOString();
+		const repairGenerationId = `${baseGenerationId}:repair:${node_crypto.randomUUID()}`;
+		const message = `Runtime replication conflict (${failure.code}) refused a thread-events batch; deferring to a full-history re-projection under a new repair generation.`;
+		if (!(yield* sql.withTransaction(require_Schema$1.gen(function* () {
+			if ((yield* sql`
+              UPDATE runtime_replication_outbox SET
+                status = 'blocked', next_attempt_at = '9999-12-31T23:59:59.999Z',
+                last_error_code = ${"THREAD_STREAM_REMOTE_CONFLICT_REPROJECTION_PENDING"},
+                last_error = ${message}, updated_at = ${now}
+              WHERE runtime_id = ${item.runtimeId} AND stream_key = ${item.streamKey}
+                AND payload_hash = ${item.payloadHash}
+              RETURNING runtime_id AS "runtimeId"
+            `).length !== 1) return false;
+			yield* replaceThreadCursor({
+				runtimeId: item.runtimeId,
+				streamKey: item.streamKey,
+				generationId: repairGenerationId,
+				now
+			});
+			return true;
+		})))) return false;
+		yield* require_Schema$1.logWarning("Runtime replication deferred a thread-stream repair to a full re-projection after a remote conflict.", {
+			runtimeId: item.runtimeId,
+			streamKey: item.streamKey,
+			errorCode: failure.code,
+			previousGenerationId: parsedBatch.generationId,
+			repairGenerationId
+		});
+		return true;
+	}).pipe(require_Schema$1.orDie);
+	const failGeneric = (item, failure) => {
 		const attempts = item.attemptCount + 1;
 		const exhausted = failure.kind === "retryable" && attempts >= 8 && failure.code !== "RUNTIME_NOT_REGISTERED";
 		const retryable = failure.kind === "retryable" && !exhausted;
@@ -316067,6 +316296,10 @@ const RuntimeReplicationOutboxLive = require_Schema$1.effect(RuntimeReplicationO
         RETURNING runtime_id AS "runtimeId"
       `.pipe(require_Schema$1.map((rows) => rows.length === 1), require_Schema$1.orDie);
 	};
+	const fail = (item, failure) => {
+		if (item.operation !== "thread-events" || !THREAD_STREAM_REMOTE_CONFLICT_CODES.has(failure.code)) return failGeneric(item, failure);
+		return repairThreadStreamFromRemoteConflict(item, failure).pipe(require_Schema$1.flatMap((repaired) => repaired === null ? failGeneric(item, failure) : require_Schema$1.succeed(repaired)));
+	};
 	const retryBlocked = () => retryBlockedRuntimeReplicationOutbox.pipe(require_Schema$1.provideService(require_SqlError.SqlClient, sql));
 	const recover = () => sql`
         UPDATE runtime_replication_outbox SET status = 'pending',
@@ -316245,10 +316478,13 @@ const statusHandler = require_Schema$1.gen(function* () {
 const retryHandler = require_Schema$1.gen(function* () {
 	const request = yield* require_HttpServer.HttpServerRequest;
 	yield* (yield* ServerAuth).authenticateHttpRequest(request);
+	const replication = yield* require_Schema$1.serviceOption(RuntimeReplication);
 	const requeued = yield* retryBlockedRuntimeReplicationOutbox;
+	const reprojectionPendingStreamKeys = yield* readReprojectionPendingThreadStreamKeys;
+	if (require_Schema$1.isSome(replication)) yield* replication.value.retryReprojectionPending(reprojectionPendingStreamKeys);
 	const status = yield* readRuntimeReplicationOutboxStatus;
 	return require_HttpServer.jsonUnsafe({
-		requeued,
+		requeued: requeued + (require_Schema$1.isSome(replication) ? reprojectionPendingStreamKeys.length : 0),
 		...responseFromStatus(status)
 	});
 }).pipe(require_Schema$1.catchTag("AuthError", respondToAuthError));