@powerhousedao/reactor-attachments 6.2.3-dev.3 → 6.2.3-dev.30

package/dist/index.js

@@ -1,14 +1,15 @@
-import { C as UploadTooLarge, S as SizeMismatch, _ as AttachmentPending, a as RemoteAttachmentUpload, b as InvalidAttachmentRef, c as createFetchUploadTransport, d as createRef, f as parseRef, g as AttachmentNotFound, h as AttachmentAlreadyExists, i as RemoteAttachmentUploadFactory, l as SwitchboardAttachmentTransport, m as parseAttachmentUploadTarget, n as createRemoteAttachmentService, o as RemoteReservationStore, p as parseAttachmentDownloadTarget, r as RemoteAttachmentStore, s as createXhrUploadTransport, t as NullAttachmentTransport, u as AttachmentService, v as AttachmentTransferError, x as ReservationNotFound, y as HashMismatch } from "./null-attachment-transport-CrsUafCi.js";
+import { C as UploadTooLarge, S as SizeMismatch, _ as AttachmentPending, a as RemoteAttachmentUpload, b as InvalidAttachmentRef, c as createFetchUploadTransport, d as createRef, f as parseRef, g as AttachmentNotFound, h as AttachmentAlreadyExists, i as RemoteAttachmentUploadFactory, l as SwitchboardAttachmentTransport, m as parseAttachmentUploadTarget, n as createRemoteAttachmentService, o as RemoteReservationStore, p as parseAttachmentDownloadTarget, r as RemoteAttachmentStore, s as createXhrUploadTransport, t as NullAttachmentTransport, u as AttachmentService, v as AttachmentTransferError, x as ReservationNotFound, y as HashMismatch } from "./null-attachment-transport-D53Rz9lE.js";
 import { dirname, join } from "node:path";
 import { Migrator, sql } from "kysely";
 import { mkdir, rename, rm } from "node:fs/promises";
 import { createReadStream, createWriteStream } from "node:fs";
 import { createHash, randomUUID } from "node:crypto";
 import { Readable } from "node:stream";
+import { finished } from "node:stream/promises";
 import { generatorTypeDefs } from "@powerhousedao/document-engineering/graphql";
 import { constantCase, pascalCase } from "change-case";
 import { Kind, buildASTSchema, getNamedType, isInputObjectType, isListType, isNonNullType, parse, print } from "graphql";
-import { BaseReadModel } from "@powerhousedao/reactor";
+import { BaseReadModel, defaultReadModelIndexingConfig } from "@powerhousedao/reactor";
 import { Buffer as Buffer$1 } from "node:buffer";
 import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -46,6 +47,13 @@ function storageRelativePath(hash) {
 *
 * Returns the number of bytes written.
 */
+async function discardTempFile(writer, tempPath) {
+	writer.destroy();
+	try {
+		await finished(writer, { error: false });
+	} catch {}
+	await rm(tempPath, { force: true });
+}
 async function writeAttachmentBytes(path, data) {
 	await mkdir(dirname(path), { recursive: true });
 	const tempPath = join(dirname(path), `${randomUUID()}.tmp`);
@@ -82,8 +90,7 @@ async function writeAttachmentBytes(path, data) {
 		reader.releaseLock();
 	}
 	if (caughtError) {
-		writer.destroy();
-		await rm(tempPath, { force: true });
+		await discardTempFile(writer, tempPath);
 		throw caughtError;
 	}
 	try {
@@ -96,7 +103,7 @@ async function writeAttachmentBytes(path, data) {
 		});
 		await rename(tempPath, path);
 	} catch (err) {
-		await rm(tempPath, { force: true });
+		await discardTempFile(writer, tempPath);
 		throw err instanceof Error ? err : new Error(String(err));
 	}
 	return bytesWritten;
@@ -255,6 +262,9 @@ function wrapStreamWithCleanup(source, cleanup) {
 	});
 }
 var KyselyAttachmentStore = class {
+	db;
+	transport;
+	basePath;
 	activeReaders = /* @__PURE__ */ new Map();
 	constructor(db, transport, basePath) {
 		this.db = db;
@@ -283,18 +293,10 @@ var KyselyAttachmentStore = class {
 	async has(hash) {
 		return (await this.db.selectFrom("attachment").select("status").where("hash", "=", hash).executeTakeFirst())?.status === "available";
 	}
-	async get(hash, signal) {
+	async get(hash, signal, documentId) {
 		const row = await this.db.selectFrom("attachment").selectAll().where("hash", "=", hash).executeTakeFirst();
 		if (row) {
-			if (row.status === "evicted") {
-				const remote = await this.transport.fetch(hash, signal);
-				if (remote.kind === "data") {
-					await this.put(hash, remote.response.metadata, remote.response.body);
-					return this.get(hash, signal);
-				}
-				if (remote.kind === "pending") throw new AttachmentPending(hash, remote.expiresAtUtc);
-				throw new AttachmentNotFound(hash);
-			}
+			if (row.status === "evicted") return this.fetchRemote(hash, signal, documentId);
 			const now = (/* @__PURE__ */ new Date()).toISOString();
 			await this.db.updateTable("attachment").set({ last_accessed_at_utc: now }).where("hash", "=", hash).execute();
 			const header = rowToHeader$1(row);
@@ -312,13 +314,7 @@ var KyselyAttachmentStore = class {
 			fileName: pending.file_name,
 			sizeBytes: pending.size_bytes
 		});
-		const remote = await this.transport.fetch(hash, signal);
-		if (remote.kind === "data") {
-			await this.put(hash, remote.response.metadata, remote.response.body);
-			return this.get(hash, signal);
-		}
-		if (remote.kind === "pending") throw new AttachmentPending(hash, remote.expiresAtUtc);
-		throw new AttachmentNotFound(hash);
+		return this.fetchRemote(hash, signal, documentId);
 	}
 	async put(hash, metadata, data) {
 		const existing = await this.db.selectFrom("attachment").select(["hash", "status"]).where("hash", "=", hash).executeTakeFirst();
@@ -358,6 +354,16 @@ var KyselyAttachmentStore = class {
 		const result = await this.db.selectFrom("attachment").select(sql`COALESCE(SUM(size_bytes), 0)`.as("total")).where("status", "=", "available").executeTakeFirst();
 		return Number(result?.total ?? 0);
 	}
+	async fetchRemote(hash, signal, documentId) {
+		if (documentId === void 0) throw new AttachmentNotFound(hash);
+		const remote = await this.transport.fetch(hash, documentId, signal);
+		if (remote.kind === "data") {
+			await this.put(hash, remote.response.metadata, remote.response.body);
+			return this.get(hash, signal);
+		}
+		if (remote.kind === "pending") throw new AttachmentPending(hash, remote.expiresAtUtc);
+		throw new AttachmentNotFound(hash);
+	}
 	async findPendingReservation(hash, now) {
 		const row = await this.db.selectFrom("attachment_reservation as r").leftJoin("attachment as a", "a.hash", "r.client_hash").select([
 			"r.mime_type",
@@ -391,7 +397,7 @@ var KyselyAttachmentStore = class {
 };
 //#endregion
 //#region src/storage/kysely/reservation-store.ts
-const DEFAULT_RESERVATION_TTL_MS = 1440 * 60 * 1e3;
+const DEFAULT_RESERVATION_TTL_MS = 864e5;
 function rowToReservation(row) {
 	return {
 		reservationId: row.reservation_id,
@@ -405,6 +411,7 @@ function rowToReservation(row) {
 	};
 }
 var KyselyReservationStore = class {
+	db;
 	ttlMs;
 	constructor(db, ttlMs = DEFAULT_RESERVATION_TTL_MS) {
 		this.db = db;
@@ -594,6 +601,11 @@ function rowToHeader(row) {
 	};
 }
 var DirectAttachmentUpload = class {
+	reservation;
+	db;
+	basePath;
+	reservations;
+	maxBytes;
 	reservationId;
 	ref;
 	expiresAtUtc;
@@ -663,6 +675,10 @@ var DirectAttachmentUpload = class {
 //#endregion
 //#region src/direct/direct-attachment-upload-factory.ts
 var DirectAttachmentUploadFactory = class {
+	db;
+	basePath;
+	reservations;
+	maxBytes;
 	constructor(db, basePath, reservations, maxBytes) {
 		this.db = db;
 		this.basePath = basePath;
@@ -681,6 +697,8 @@ var DirectAttachmentUploadFactory = class {
 * can never accidentally return a direct-provider target.
 */
 var FilesystemAttachmentBackend = class {
+	store;
+	config;
 	kind = "filesystem";
 	constructor(store, config) {
 		this.store = store;
@@ -732,6 +750,8 @@ var S3AttachmentUploadFactory = class {
 //#endregion
 //#region src/attachment-builder.ts
 var AttachmentBuilder = class {
+	db;
+	storagePath;
 	transport = new NullAttachmentTransport();
 	customUploadFactory;
 	maxUploadBytes;
@@ -804,7 +824,7 @@ var AttachmentBuilder = class {
 //#endregion
 //#region src/reference-index/attachment-schema-compiler.ts
 const ATTACHMENT_REF_TYPE = "AttachmentRef";
-const CODEGEN_SCALAR_NAMES = new Set([
+const CODEGEN_SCALAR_NAMES = /* @__PURE__ */ new Set([
 	"Unknown",
 	"DateTime",
 	"Address",
@@ -852,11 +872,12 @@ function buildEffectiveSchema(specification, context) {
 	const stateSchemas = Object.values(specification.state).map((state) => state.schema);
 	const operationSchemas = specification.modules.flatMap((moduleSpecification) => moduleSpecification.operations.flatMap((operation) => operation.schema === null ? [] : [operation.schema]));
 	try {
-		return buildASTSchema(dedupeTypeDefinitions(parse([
+		const document = parse([
 			...scalarSchemas,
 			...stateSchemas,
 			...operationSchemas
-		].filter(Boolean).join("\n\n"))));
+		].filter(Boolean).join("\n\n"));
+		return buildASTSchema(dedupeTypeDefinitions(document));
 	} catch {
 		throw compilationError(context, "the effective GraphQL schema is invalid");
 	}
@@ -994,6 +1015,8 @@ function extractValue(plan, value, path, context, refs, seenRefs, activeObjects)
 	}
 }
 var SchemaCompiledAttachmentExtractor = class {
+	context;
+	rootPlan;
 	constructor(context, rootPlan) {
 		this.context = context;
 		this.rootPlan = rootPlan;
@@ -1056,12 +1079,16 @@ var AttachmentSchemaCompiler = class {
 //#endregion
 //#region src/read-models/attachment-reference/kysely-attachment-reference-store.ts
 var KyselyAttachmentReferenceStore = class {
+	db;
 	constructor(db) {
 		this.db = db;
 	}
 	async hasReference(documentId, ref) {
 		return await this.db.selectFrom("attachment_reference").select("document_id").where("document_id", "=", documentId).where("attachment_ref", "=", ref).executeTakeFirst() !== void 0;
 	}
+	async referencingScopes(documentId, ref) {
+		return (await this.db.selectFrom("attachment_reference").select("scope").distinct().where("document_id", "=", documentId).where("attachment_ref", "=", ref).execute()).map((row) => row.scope);
+	}
 	async addReferences(references) {
 		if (references.length === 0) return;
 		await this.db.insertInto("attachment_reference").values(references.map((reference) => ({
@@ -1153,6 +1180,7 @@ async function getAttachmentReferenceMigrationStatus(db, schema = ATTACHMENT_REF
 //#endregion
 //#region src/read-models/attachment-reference/index-builder.ts
 var AttachmentReferenceIndexBuilder = class {
+	db;
 	constructor(db) {
 		this.db = db;
 	}
@@ -1165,12 +1193,36 @@ var AttachmentReferenceIndexBuilder = class {
 //#endregion
 //#region src/read-models/attachment-reference/attachment-reference-read-model.ts
 const ATTACHMENT_REFERENCE_READ_MODEL_ID = "attachment-reference-read-model";
+/**
+* Indexes attachment references in chunks.
+*
+* Each reference row stands on its own and is written insert-or-do-nothing, so
+* a batch that commits in pieces exposes no partial structure: a reader either
+* sees a reference or does not, and re-indexing the same range writes nothing
+* new. Batches here are the largest in the system, which is what chunking is
+* for.
+*/
 var AttachmentReferenceReadModel = class extends BaseReadModel {
+	documentModelRegistry;
+	schemaCompiler;
+	referenceWriter;
 	indexingQueue = Promise.resolve();
+	checkpointTarget;
+	/**
+	* How far a replay may skip ahead of the cursor: the highest ordinal already
+	* pulled from the index while the cursor was parked.
+	*
+	* NOT a claim that every ordinal below it committed -- the parked ordinal is
+	* precisely the one that did not -- so it is a hint, and {@link replayFrom}
+	* re-probes the gap before honouring it.
+	*/
+	replayedThrough;
+	warnedCheckpoint;
 	constructor(db, operationIndex, writeCache, consistencyTracker, documentModelRegistry, schemaCompiler, referenceWriter) {
 		super(db, operationIndex, writeCache, consistencyTracker, {
 			readModelId: ATTACHMENT_REFERENCE_READ_MODEL_ID,
-			rebuildStateOnInit: false
+			rebuildStateOnInit: false,
+			indexing: defaultReadModelIndexingConfig
 		});
 		this.documentModelRegistry = documentModelRegistry;
 		this.schemaCompiler = schemaCompiler;
@@ -1215,43 +1267,89 @@ var AttachmentReferenceReadModel = class extends BaseReadModel {
 		if (references.length > 0) await this.referenceWriter.addReferences(references);
 	}
 	async indexOperationsInOrdinalOrder(incoming) {
-		const pending = this.sortAndDedupe(incoming.filter(({ context }) => context.ordinal > this.lastOrdinal));
-		if (pending.length === 0) return;
-		const incomingMax = pending[pending.length - 1].context.ordinal;
-		let candidates = pending;
-		if (!this.isContiguousThrough(candidates, incomingMax)) {
+		let candidates = this.sortAndDedupe(incoming);
+		if (candidates.length === 0) return;
+		const incomingMax = candidates[candidates.length - 1].context.ordinal;
+		if (this.contiguousEnd(candidates) < incomingMax) {
 			const replayed = await this.loadThroughOrdinal(incomingMax);
-			candidates = this.sortAndDedupe([...replayed, ...pending]);
+			candidates = this.sortAndDedupe([...replayed, ...candidates]);
 		}
-		const contiguous = [];
-		let expectedOrdinal = this.lastOrdinal + 1;
-		for (const item of candidates) {
-			const ordinal = item.context.ordinal;
-			if (ordinal < expectedOrdinal) continue;
-			if (ordinal > expectedOrdinal || ordinal > incomingMax) break;
-			contiguous.push(item);
-			expectedOrdinal++;
+		const checkpoint = this.contiguousEnd(candidates);
+		if (checkpoint < incomingMax && checkpoint !== this.warnedCheckpoint) {
+			this.warnedCheckpoint = checkpoint;
+			console.warn(`[${this.config.readModelId}] indexed through ordinal ${incomingMax} but parked the cursor at ${checkpoint}: ordinal ${checkpoint + 1} is missing`);
 		}
-		if (expectedOrdinal <= incomingMax) throw new Error(`Attachment reference read model cannot advance past missing ordinal ${expectedOrdinal}`);
 		const previousOrdinal = this.lastOrdinal;
+		this.checkpointTarget = checkpoint;
 		try {
-			await super.indexOperations(contiguous);
+			await super.indexOperations(candidates);
 		} catch (error) {
 			this.lastOrdinal = previousOrdinal;
+			this.replayedThrough = void 0;
 			throw error;
+		} finally {
+			this.checkpointTarget = void 0;
 		}
+		if (checkpoint > previousOrdinal) this.replayedThrough = void 0;
+		else if (checkpoint < incomingMax) this.replayedThrough = Math.max(this.replayedThrough ?? checkpoint, incomingMax);
 	}
-	isContiguousThrough(items, maxOrdinal) {
+	/**
+	* Writes the cursor parked by indexOperationsInOrdinalOrder instead of the
+	* batch maximum, so indexing an operation above a gap never advances past it.
+	*/
+	async saveState(trx, items) {
+		const target = this.checkpointTarget;
+		if (target === void 0) {
+			await super.saveState(trx, items);
+			return;
+		}
+		this.lastOrdinal = target;
+		await trx.updateTable("ViewState").set({
+			lastOrdinal: target,
+			lastOperationTimestamp: /* @__PURE__ */ new Date()
+		}).where("readModelId", "=", this.config.readModelId).execute();
+	}
+	/** Last ordinal of the contiguous run starting at lastOrdinal + 1. */
+	contiguousEnd(items) {
 		let expectedOrdinal = this.lastOrdinal + 1;
 		for (const item of items) {
-			if (item.context.ordinal !== expectedOrdinal) return false;
+			const ordinal = item.context.ordinal;
+			if (ordinal < expectedOrdinal) continue;
+			if (ordinal > expectedOrdinal) break;
 			expectedOrdinal++;
 		}
-		return expectedOrdinal > maxOrdinal;
+		return expectedOrdinal - 1;
+	}
+	/**
+	* Opens a replay at the lowest ordinal still worth reading.
+	*
+	* {@link replayedThrough} is what keeps a permanently held hole -- a
+	* rolled-back insert, which never fills -- from re-reading the whole tail on
+	* every batch. It cannot be trusted on its own: a hole that fills without
+	* being delivered here is visible only in the index, and a mark that is
+	* never questioned would hide that operation for the life of the process.
+	* That happens whenever another writer commits the gap (a second reactor on
+	* the same database), and it leaves the cursor parked below an operation
+	* whose references were never written -- `hasReference` then answers false
+	* for an attachment that is genuinely referenced.
+	*
+	* So a parked batch spends one page probing the gap: the first row above the
+	* cursor is the missing ordinal itself once it commits. Finding it drops the
+	* mark and replays from the cursor; not finding it leaves the mark standing,
+	* which is the cheap path and the common one.
+	*/
+	async replayFrom() {
+		const fromCursor = await this.operationIndex.getSinceOrdinal(this.lastOrdinal);
+		if (this.replayedThrough === void 0) return fromCursor;
+		if (fromCursor.results[0]?.context.ordinal === this.lastOrdinal + 1) {
+			this.replayedThrough = void 0;
+			return fromCursor;
+		}
+		return this.operationIndex.getSinceOrdinal(this.replayedThrough);
 	}
 	async loadThroughOrdinal(maxOrdinal) {
 		const operations = [];
-		let page = await this.operationIndex.getSinceOrdinal(this.lastOrdinal);
+		let page = await this.replayFrom();
 		for (;;) {
 			for (const item of page.results) if (item.context.ordinal <= maxOrdinal) operations.push(item);
 			if (page.results.some(({ context }) => context.ordinal >= maxOrdinal) || !page.next) break;
@@ -1277,7 +1375,7 @@ function required(env, name) {
 	if (value.trim() !== value) throw new Error(`${name} must not have leading or trailing whitespace`);
 	return value;
 }
-const LOOPBACK_HOSTNAMES = new Set([
+const LOOPBACK_HOSTNAMES = /* @__PURE__ */ new Set([
 	"127.0.0.1",
 	"localhost",
 	"[::1]"
@@ -1359,9 +1457,10 @@ function sha256HexToBase64(hash) {
 //#region src/storage/s3/primitives.ts
 const defaultPresigner = (client, command, expiresInSeconds) => getSignedUrl(client, command, {
 	expiresIn: expiresInSeconds,
-	unhoistableHeaders: new Set(["x-amz-checksum-sha256"])
+	unhoistableHeaders: /* @__PURE__ */ new Set(["x-amz-checksum-sha256"])
 });
 var S3AttachmentPrimitives = class {
+	config;
 	client;
 	presign;
 	now;
@@ -1432,7 +1531,7 @@ function createS3AttachmentPrimitives(config, dependencies = {}) {
 //#endregion
 //#region src/storage/s3/backend.ts
 const READINESS_PROBE_HASH = "0".repeat(64);
-const OBJECT_NOT_FOUND_NAMES = new Set([
+const OBJECT_NOT_FOUND_NAMES = /* @__PURE__ */ new Set([
 	"NotFound",
 	"NoSuchKey",
 	"NoSuchObject"
@@ -1450,6 +1549,8 @@ function requireHashFirstReservation(reservation) {
 }
 /** Server-only S3 capability. Callers complete authorization before download. */
 var S3AttachmentBackend = class {
+	db;
+	config;
 	kind = "s3";
 	primitives;
 	now;