@happyvertical/smrt-ingestion 0.55.5 → 0.55.6

package/dist/server.js

@@ -1,300 +1,14 @@
 import "./models.js";
-import { a as providerIdentity, i as validateExtractionRequest, n as emptyExtraction, t as ExtractionFailure } from "./chunks/extraction-providers-C66ADDVe.js";
+import { n as generationSnapshotRead } from "./chunks/execution-internal-DWteDHM2.js";
+import { n as extractAnalysis, r as proposeDocumentSplits, t as createSDKExtractionAdapter } from "./chunks/extraction-bseWZuWN.js";
 import { detectEngine, isEmbeddedDatabase, isTransientDatabaseError, withEmbeddedWriteTransaction } from "@happyvertical/smrt-core";
 import { withTenant } from "@happyvertical/smrt-tenancy";
 import { createHash, randomUUID } from "node:crypto";
 import { Asset } from "@happyvertical/smrt-assets";
 import { SmrtJobCollection } from "@happyvertical/smrt-jobs";
-import { fork } from "node:child_process";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
+import { join } from "node:path";
 import { constants } from "node:fs";
 import { lstat, mkdir, open, readFile, readdir, rename, writeFile } from "node:fs/promises";
-//#region src/extraction.ts
-function createSDKExtractionAdapter(configuration) {
-	try {
-		configuration = structuredClone(configuration);
-	} catch {
-		throw new Error("Serializable SDK configuration required");
-	}
-	const maximum = configuration.maxConcurrentProcesses ?? 1;
-	if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 32) throw new Error("Invalid extraction concurrency");
-	let running = 0;
-	if (configuration.nativeMemoryIsolation !== "host-enforced") throw new Error("Native memory isolation is required");
-	for (const provider of [
-		configuration.pdf,
-		configuration.ocr,
-		configuration.speech,
-		configuration.vision
-	]) if (provider) provider.identity = providerIdentity(provider.identity);
-	if (configuration.pdf?.provider === "auto" || configuration.ocr?.identity.provider === "auto") throw new Error("Pin extraction providers");
-	return { async extract(request) {
-		let latest = emptyExtraction(request);
-		try {
-			validateExtractionRequest(request);
-		} catch (caught) {
-			if (!(caught instanceof ExtractionFailure)) throw caught;
-			latest.errors.push({
-				category: caught.category,
-				location: { kind: "source" }
-			});
-			return latest;
-		}
-		if (request.evidence.mediaType.toLowerCase().startsWith("video/")) {
-			latest.status = "unsupported";
-			latest.errors.push({
-				category: "unsupported_type",
-				location: { kind: "source" }
-			});
-			return latest;
-		}
-		if (request.signal?.aborted) {
-			latest.errors.push({
-				category: "cancelled",
-				location: { kind: "source" }
-			});
-			return latest;
-		}
-		if (running >= maximum) {
-			latest.errors.push({
-				category: "limit",
-				location: { kind: "source" }
-			});
-			return latest;
-		}
-		running++;
-		return new Promise((resolve) => {
-			const child = fork(join(dirname(fileURLToPath(import.meta.url)), "extraction-worker.js"), [], {
-				serialization: "advanced",
-				stdio: [
-					"ignore",
-					"ignore",
-					"ignore",
-					"ipc"
-				],
-				execArgv: [`--max-old-space-size=${request.limits.workerHeapMb}`],
-				detached: process.platform !== "win32"
-			});
-			let done = false;
-			const stop = () => {
-				try {
-					if (process.platform !== "win32" && child.pid) process.kill(-child.pid, "SIGKILL");
-					else child.kill("SIGKILL");
-				} catch {
-					child.kill("SIGKILL");
-				}
-			};
-			const finish = (category) => {
-				if (done) return;
-				done = true;
-				running--;
-				clearTimeout(timer);
-				request.signal?.removeEventListener("abort", abort);
-				stop();
-				if (category) {
-					latest.errors.push({
-						category,
-						location: { kind: "source" }
-					});
-					latest.status = latest.segments.length ? "partial" : "failed";
-					latest.truncated = true;
-					latest.omitted.push({ kind: "source" });
-				}
-				resolve(latest);
-			};
-			const abort = () => finish("cancelled");
-			const timer = setTimeout(() => finish("timeout"), request.limits.timeoutMs);
-			request.signal?.addEventListener("abort", abort, { once: true });
-			child.on("error", () => finish("unavailable"));
-			child.on("exit", () => {
-				if (!done) finish("unavailable");
-			});
-			child.on("message", async (message) => {
-				if (done) return;
-				if (message.kind === "authorize" && Number.isSafeInteger(message.id)) {
-					try {
-						await request.beforeProviderCall?.();
-						if (!done) child.send({
-							kind: "authorized",
-							id: message.id
-						});
-					} catch {
-						finish("cancelled");
-					}
-					return;
-				}
-				if ((message.kind === "progress" || message.kind === "result") && message.result) {
-					if (Buffer.byteLength(JSON.stringify(message.result)) > request.limits.maxOutputBytes + 65536) return finish("limit");
-					latest = message.result;
-					if (message.kind === "result") finish();
-				} else finish("unavailable");
-			});
-			const { signal: _signal, beforeProviderCall: _authorize, ...wireRequest } = request;
-			child.send({
-				request: wireRequest,
-				configuration
-			}, (error) => {
-				if (error) finish("unavailable");
-			});
-			if (request.signal?.aborted) abort();
-		});
-	} };
-}
-function validateAndNormalizeExtractionOutput(result, request) {
-	const numeric = (value) => Number.isFinite(value) && value >= 0;
-	const location = (value) => {
-		if (!value || typeof value !== "object") return false;
-		if (value.kind === "source") return true;
-		if (value.kind === "page") return Number.isSafeInteger(value.page) && value.page > 0;
-		if (value.kind === "pages") return Number.isSafeInteger(value.startPage) && value.startPage > 0 && Number.isSafeInteger(value.endPage) && value.endPage >= value.startPage;
-		return value.kind === "time" && numeric(value.startMs) && numeric(value.endMs) && value.endMs >= value.startMs;
-	};
-	if (!result || ![
-		"complete",
-		"partial",
-		"failed",
-		"unsupported"
-	].includes(result.status) || result.automaticActionEligible !== false || result.configurationRevision !== request.configurationRevision || !result.evidence || Object.entries(request.evidence).some(([key, value]) => result.evidence[key] !== value) || typeof result.truncated !== "boolean" || !Array.isArray(result.capabilities) || result.capabilities.some((capability) => !capability || typeof capability.confidence !== "boolean" || typeof capability.boxes !== "boolean" || ![
-		"source",
-		"page",
-		"time"
-	].includes(capability.location) || !["unknown", "reported"].includes(capability.truncation) || !["unknown", "reported"].includes(capability.usage)) || !result.limits || Object.entries(request.limits).some(([key, value]) => result.limits[key] !== value) || !Array.isArray(result.segments) || !Array.isArray(result.errors) || !Array.isArray(result.omitted) || result.status === "complete" && (result.truncated || result.errors.length > 0 || result.omitted.length > 0) || result.omitted.some((value) => !location(value)) || result.segments.some((segment) => typeof segment.text !== "string" || !location(segment.location) || segment.location.kind === "pages" || segment.provenance?.provider === "unlimited-ocr" && (segment.confidence !== null || segment.boxes !== null) || !segment.provenance || [
-		segment.provenance.provider,
-		segment.provenance.model,
-		segment.provenance.version
-	].some((value) => typeof value !== "string" || !value) || segment.confidence !== null && (!segment.confidence || !numeric(segment.confidence.value) || segment.confidence.value > 1 || segment.confidence.scale !== "0-1" || segment.confidence.calibration !== "unknown") || segment.boxes !== null && (!Array.isArray(segment.boxes) || segment.boxes.some((box) => typeof box.text !== "string" || ![
-		box.x,
-		box.y,
-		box.width,
-		box.height
-	].every(numeric)))) || result.errors.some((error) => ![
-		"unsupported_type",
-		"malformed_output",
-		"limit",
-		"timeout",
-		"unavailable",
-		"authentication",
-		"cancelled",
-		"integrity"
-	].includes(error.category) || !location(error.location)) || !result.usage || Object.values(result.usage).some((value) => !numeric(value))) throw new Error("Invalid extraction output");
-	for (const segment of result.segments) segment.provenance = providerIdentity(segment.provenance);
-	result.capabilities = result.capabilities.map((capability) => ({
-		...providerIdentity(capability),
-		confidence: capability.confidence,
-		boxes: capability.boxes,
-		location: capability.location,
-		truncation: capability.truncation,
-		usage: capability.usage
-	}));
-}
-async function extractAnalysis(service, lease, adapter, options) {
-	const snapshot = await service.getAnalysisInput(lease);
-	const persisted = snapshot.configuration.extraction;
-	if (!persisted || persisted.configurationRevision !== options.configurationRevision || !persisted.limits || Object.keys(options.limits).some((key) => persisted.limits?.[key] !== options.limits[key]) || Object.keys(persisted.limits).length !== Object.keys(options.limits).length) throw new Error("Extraction configuration mismatch");
-	const evidence = snapshot.evidence;
-	const boundOptions = {
-		...options,
-		limits: structuredClone(options.limits)
-	};
-	const results = [];
-	let limited = false;
-	const publication = (reserve = false) => {
-		const allComplete = !limited && results.length === evidence.length && results.every((result) => result.status === "complete");
-		const firstError = limited || reserve ? "limit" : results.flatMap((result) => result.errors)[0]?.category;
-		return {
-			status: reserve ? "needs_attention" : allComplete ? "completed" : results.some((result) => result.segments.length) ? "partial" : "needs_attention",
-			provider: "smrt-ingestion-extraction",
-			model: "none",
-			version: "1",
-			output: {
-				configurationRevision: options.configurationRevision,
-				results,
-				...limited || reserve ? {
-					truncated: true,
-					omittedEvidenceCount: evidence.length - results.length
-				} : {}
-			},
-			usage: {},
-			...firstError ? { error: firstError } : {}
-		};
-	};
-	const fits = (output) => Buffer.byteLength(JSON.stringify(output)) <= snapshot.maxOutputBytes;
-	const minimum = {
-		status: "needs_attention",
-		provider: "smrt-ingestion-extraction",
-		model: "none",
-		version: "1",
-		output: {},
-		usage: {},
-		error: "limit"
-	};
-	if (!fits(minimum)) return service.failAnalysis(lease, "limit");
-	if (!fits(publication(true))) return service.completeAnalysis(lease, minimum);
-	for (const part of evidence) {
-		await service.getAnalysisInput(lease);
-		const bytes = await service.readEvidence(lease.itemId, part.id);
-		await service.getAnalysisInput(lease);
-		const request = {
-			...boundOptions,
-			evidence: part,
-			bytes
-		};
-		const result = await adapter.extract({
-			...request,
-			evidence: { ...part },
-			limits: { ...request.limits },
-			bytes: bytes.slice(),
-			beforeProviderCall: async () => {
-				await service.getAnalysisInput(lease);
-			}
-		});
-		validateAndNormalizeExtractionOutput(result, request);
-		results.push(result);
-		if (!fits(publication(true))) {
-			limited = true;
-			result.status = "partial";
-			result.truncated = true;
-			result.errors.push({
-				category: "limit",
-				location: { kind: "source" }
-			});
-			while (result.segments.length && !fits(publication(true))) {
-				const removed = result.segments.pop();
-				if (removed) result.omitted.push(removed.location);
-			}
-			if (!fits(publication(true))) results.pop();
-		}
-		await service.getAnalysisInput(lease);
-		if (limited || options.signal?.aborted) break;
-	}
-	return service.completeAnalysis(lease, publication());
-}
-function proposeDocumentSplits(result, revision, groups) {
-	if (!Number.isSafeInteger(revision) || revision < 1 || !groups.length) throw new Error("Invalid split revision");
-	const available = new Set(result.segments.flatMap((segment) => segment.location.kind === "page" ? [segment.location.page] : []));
-	const seen = /* @__PURE__ */ new Set();
-	for (const group of groups) {
-		if (!group.length) throw new Error("Empty split");
-		for (const page of group) {
-			if (!Number.isSafeInteger(page) || !available.has(page) || seen.has(page)) throw new Error("Invalid split page");
-			seen.add(page);
-		}
-		if (group.some((page, index) => index > 0 && page <= group[index - 1])) throw new Error("Unordered split");
-	}
-	const proposal = {
-		revision,
-		evidenceId: result.evidence.id,
-		inputHash: result.evidence.contentHash,
-		configurationRevision: result.configurationRevision,
-		groups: groups.map((pages) => ({ pages: [...pages] })),
-		status: "proposed"
-	};
-	return {
-		...proposal,
-		digest: createHash("sha256").update(JSON.stringify(proposal)).digest("hex")
-	};
-}
-//#endregion
 //#region src/sources/common.ts
 var SourceInputError = class extends Error {
 	constructor(category) {
@@ -943,6 +657,26 @@ var IngestionService = class {
 	async applyPlan(planId) {
 		return (await this.execution()).applyPlan(planId);
 	}
+	async proposals() {
+		if (!this.options.proposals) throw new Error("Intake proposals are not configured");
+		const { IngestionProposalService } = await import("./proposals.js");
+		return new IngestionProposalService(this, this.options, this.options.proposals);
+	}
+	async listHandlers(itemId) {
+		return (await this.proposals()).listHandlers(itemId);
+	}
+	async findCandidates(input) {
+		return (await this.proposals()).findCandidates(input);
+	}
+	async prepareGeneration(itemId, attemptId) {
+		return (await this.proposals()).prepareGeneration(itemId, attemptId);
+	}
+	async generateProposals(lease) {
+		return (await this.proposals()).generate(lease);
+	}
+	async previewGeneratedProposals(input) {
+		return (await this.proposals()).preview(input);
+	}
 	now() {
 		return this.options.now?.() ?? /* @__PURE__ */ new Date();
 	}
@@ -1439,6 +1173,102 @@ var IngestionService = class {
 			};
 		});
 	}
+	async completedAnalysisSnapshot(db, itemId, attemptId, current) {
+		await this.lock(db, itemId);
+		const item = await this.item(itemId, "process", db);
+		if (item.cancelled) throw new Error("Analysis unavailable");
+		if (attemptId === void 0) {
+			if (!current) throw new Error("Historical attempt required");
+			const [latest] = await this.rows(db, "intake_analyses", "item_id=? AND revision=?", [itemId, item.analysis_revision]);
+			if (!latest?.current_attempt_id) throw new Error("Analysis unavailable");
+			attemptId = String(latest.current_attempt_id);
+		}
+		const [attempt] = await this.rows(db, "intake_analysis_attempts", "item_id=? AND id=?", [itemId, attemptId]);
+		if (!attempt || !["completed", "partial"].includes(String(attempt.state)) || !attempt.output_digest) throw new Error("Analysis unavailable");
+		const [analysis] = await this.rows(db, "intake_analyses", "item_id=? AND id=?", [itemId, attempt.analysis_id]);
+		if (!analysis || analysis.current_attempt_id !== attemptId || current && Number(item.analysis_revision) !== Number(analysis.revision)) throw new Error("Analysis unavailable");
+		const data = object(analysis.data);
+		const configuration = object(data.configuration);
+		if (!Array.isArray(data.inputs) || !data.inputs.length || digest({
+			inputs: data.inputs,
+			configuration
+		}) !== analysis.input_digest || digest(configuration) !== data.configDigest) throw new Error("Analysis input integrity");
+		const resultData = object(attempt.data);
+		const result = {
+			status: resultData.status,
+			provider: resultData.provider,
+			model: resultData.model,
+			version: resultData.version,
+			output: resultData.output,
+			usage: resultData.usage,
+			...Object.hasOwn(resultData, "error") ? { error: resultData.error } : {},
+			...Object.hasOwn(resultData, "confidence") ? { confidence: resultData.confidence } : {}
+		};
+		if (result.status !== attempt.state || digest(result) !== attempt.output_digest) throw new Error("Analysis output integrity");
+		const evidence = [];
+		for (const input of data.inputs) {
+			const frozen = object(input);
+			const [row] = await this.rows(db, "intake_evidence", "item_id=? AND id=? AND state='durable'", [itemId, frozen.id]);
+			if (!row || row.content_hash !== frozen.hash) throw new Error("Analysis input integrity");
+			evidence.push({
+				id: String(row.id),
+				partId: String(row.part_id),
+				parentEvidenceId: row.parent_evidence_id ? String(row.parent_evidence_id) : null,
+				mediaType: String(row.media_type),
+				contentHash: String(row.content_hash),
+				byteLength: Number(row.byte_length)
+			});
+		}
+		if (new Set(evidence.map((entry) => entry.id)).size !== evidence.length) throw new Error("Analysis input integrity");
+		await this.item(itemId, "process", db);
+		return structuredClone({
+			itemId,
+			attemptId,
+			revision: Number(analysis.revision),
+			inputDigest: String(analysis.input_digest),
+			outputDigest: String(attempt.output_digest),
+			evidenceDigest: digest(data.inputs),
+			configuration,
+			evidence,
+			result
+		});
+	}
+	/** Detached successful current revision; always current-scope/process authorized. */
+	async getCompletedAnalysis(itemId, attemptId) {
+		const snapshot = await this.tx((db) => this.completedAnalysisSnapshot(db, itemId, attemptId, true));
+		await this.point("analysis:snapshot-read");
+		const read = async (db) => {
+			const current = await this.completedAnalysisSnapshot(db, itemId, snapshot.attemptId, true);
+			if (current.inputDigest !== snapshot.inputDigest || current.outputDigest !== snapshot.outputDigest) throw new Error("Analysis changed");
+			return current;
+		};
+		if (snapshot.configuration.stage === "interpret") return (await this.proposals())[generationSnapshotRead](itemId, (work) => this.tx(work), read);
+		return this.tx(read);
+	}
+	/** Historical source is readable only through its frozen pin on a live generation lease. */
+	async getGenerationInput(lease) {
+		const generation = await this.getAnalysisInput(lease);
+		const configuration = generation.configuration;
+		const pin = object(object(configuration.proposals).source);
+		if (configuration.stage !== "interpret" || typeof pin.attemptId !== "string") throw new Error("Invalid generation source");
+		const source = await this.tx((db) => this.completedAnalysisSnapshot(db, lease.itemId, String(pin.attemptId), false));
+		for (const key of [
+			"attemptId",
+			"revision",
+			"inputDigest",
+			"outputDigest",
+			"evidenceDigest"
+		]) if (pin[key] !== source[key]) throw new Error("Generation source changed");
+		if (source.result.provider !== "smrt-ingestion-extraction" || source.configuration.stage === "interpret" || source.revision !== generation.revision - 1 || digest(generation.evidence.map((entry) => ({
+			id: entry.id,
+			hash: entry.contentHash
+		}))) !== source.evidenceDigest) throw new Error("Generation evidence changed");
+		await this.getAnalysisInput(lease);
+		return {
+			generation,
+			source
+		};
+	}
 	/** Reload immutable inputs only through the current live processing lease. */
 	async getAnalysisInput(lease) {
 		return this.tx(async (db) => {