@xopcai/xopc 0.0.388 → 0.0.390

package/dist/src/discussions/media-timeline.js

@@ -0,0 +1,134 @@
+import { spawnProcess } from "../process/run-process.js";
+//#region src/discussions/media-timeline.ts
+const pcmEpoch = (durationMs, baseMs = 0) => ({
+	baseMs,
+	durationMs,
+	verified: true,
+	spans: [{
+		sourceMs: 0,
+		pcmMs: 0,
+		durationMs
+	}]
+});
+/** Packet timestamps expose pauses without decoding the recording into memory. */
+async function probeMediaEpoch(path, durationMs, baseMs, signal) {
+	const result = {
+		baseMs,
+		durationMs,
+		spans: [],
+		verified: false
+	};
+	try {
+		await new Promise((resolve, reject) => {
+			signal.throwIfAborted();
+			const handle = spawnProcess({
+				program: process.env.XOPC_FFPROBE_PATH || "ffprobe",
+				args: [
+					"-v",
+					"error",
+					"-select_streams",
+					"a:0",
+					"-show_packets",
+					"-show_entries",
+					"packet=pts_time,duration_time:packet_side_data=skip_samples,discard_padding",
+					"-of",
+					"compact=p=0",
+					path
+				],
+				maxOutputBytes: 0
+			});
+			const child = handle.child;
+			let pending = "";
+			let pcm = 0;
+			let packets = 0;
+			let lastStart = -Infinity;
+			const abort = () => {
+				handle.terminate();
+				reject(/* @__PURE__ */ new Error("Timing inspection cancelled"));
+			};
+			const timer = setTimeout(abort, 3e4);
+			timer.unref();
+			signal.addEventListener("abort", abort, { once: true });
+			child.stdout?.setEncoding("utf8");
+			child.stdout?.on("data", (text) => {
+				pending += text;
+				const lines = pending.split("\n");
+				pending = lines.pop();
+				if (pending.length > 8192) {
+					abort();
+					return;
+				}
+				for (const line of lines) {
+					if (!line.trim()) continue;
+					const fields = Object.fromEntries(line.split("|").map((part) => part.split("=")));
+					const pts = Number(fields.pts_time) * 1e3;
+					const duration = Number(fields.duration_time) * 1e3;
+					if (++packets > 1e6 || !Number.isFinite(pts) || !Number.isFinite(duration) || duration <= 0 || Number(fields.skip_samples ?? 0) !== 0 || Number(fields.discard_padding ?? 0) !== 0) {
+						abort();
+						return;
+					}
+					const start = Math.max(0, pts);
+					const length = duration - (start - pts);
+					if (length <= 0) continue;
+					const previous = result.spans.at(-1);
+					if (start <= lastStart) {
+						abort();
+						return;
+					}
+					lastStart = start;
+					if (previous && Math.abs(start - previous.sourceMs - previous.durationMs) <= 2) previous.durationMs += length;
+					else result.spans.push({
+						sourceMs: start,
+						pcmMs: pcm,
+						durationMs: length
+					});
+					pcm += length;
+					if (result.spans.length > 1e4) {
+						abort();
+						return;
+					}
+				}
+			});
+			child.once("error", reject);
+			child.once("close", (code) => {
+				clearTimeout(timer);
+				signal.removeEventListener("abort", abort);
+				if (code !== 0 || !result.spans.length || Math.abs(pcm - durationMs) > 100) {
+					reject(/* @__PURE__ */ new Error("Unverified media timing"));
+					return;
+				}
+				result.spans.at(-1).durationMs += durationMs - pcm;
+				result.verified = result.spans.at(-1).durationMs > 0;
+				resolve();
+			});
+		});
+	} catch {
+		result.verified = false;
+	}
+	return result;
+}
+function pointToPcm(epoch, sourceMs) {
+	if (!epoch.verified || !Number.isFinite(sourceMs)) return;
+	const matches = epoch.spans.filter((span) => sourceMs >= span.sourceMs && sourceMs < span.sourceMs + span.durationMs);
+	const span = matches.length === 1 ? matches[0] : void 0;
+	return span ? epoch.baseMs + span.pcmMs + sourceMs - span.sourceMs : void 0;
+}
+function playbackRanges(timeline, start, end) {
+	const total = timeline.epochs.reduce((sum, epoch) => sum + epoch.durationMs, 0);
+	if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start || end > total + 1) return [];
+	if (timeline.primary === "pcm") return [{
+		startedAtMs: start,
+		endedAtMs: end
+	}];
+	if (!timeline.epochs.every((epoch) => epoch.verified)) return [];
+	return timeline.epochs[0].spans.flatMap((span) => {
+		const left = Math.max(start, span.pcmMs);
+		const right = Math.min(end, span.pcmMs + span.durationMs);
+		return right > left ? [{
+			startedAtMs: span.sourceMs + left - span.pcmMs,
+			endedAtMs: span.sourceMs + right - span.pcmMs
+		}] : [];
+	});
+}
+//#endregion
+export { pcmEpoch, playbackRanges, pointToPcm, probeMediaEpoch };

package/dist/src/discussions/organizer-worker.d.ts

@@ -6,10 +6,13 @@ export declare class DiscussionOrganizerWorker {
     private readonly processor;
     private readonly onUpdated?;
     private readonly intervalMs;
+    private readonly requestTimeoutMs;
     private readonly owner;
     private timer?;
     private running;
-    constructor(processor: DiscussionOrganizerProcessor, onUpdated?: (capture: DiscussionCapture) => void, intervalMs?: number);
+    private stopping;
+    private controller?;
+    constructor(processor: DiscussionOrganizerProcessor, onUpdated?: (capture: DiscussionCapture) => void, intervalMs?: number, requestTimeoutMs?: number);
     start(): void;
     stop(): Promise<void>;
     tick(): Promise<void>;

package/dist/src/discussions/organizer-worker.js

@@ -1,6 +1,7 @@
 import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
-import { claimNextDiscussionCapture, getDiscussionCapture, releaseDiscussionWorkLease, renewDiscussionWorkLease, updateDiscussionCapture } from "./repository.js";
+import { claimNextDiscussionCapture, getDiscussionCapture, releaseDiscussionWorkLease, renewDiscussionWorkLease } from "./repository.js";
+import { failOrRetryProcessing, resetProcessingRetry, withProcessingDeadline } from "./processing.js";
 import { randomUUID } from "node:crypto";
 //#region src/discussions/organizer-worker.ts
 init_logger();
@@ -9,47 +10,50 @@ var DiscussionOrganizerWorker = class {
 	owner = randomUUID();
 	timer;
 	running = false;
-	constructor(processor, onUpdated, intervalMs = 1e3) {
+	stopping = false;
+	controller;
+	constructor(processor, onUpdated, intervalMs = 1e3, requestTimeoutMs = 10 * 6e4) {
 		this.processor = processor;
 		this.onUpdated = onUpdated;
 		this.intervalMs = intervalMs;
+		this.requestTimeoutMs = requestTimeoutMs;
 	}
 	start() {
 		if (this.timer) return;
-		this.timer = setInterval(() => void this.tick(), this.intervalMs);
+		this.stopping = false;
+		this.timer = setInterval(() => void this.tick().catch((error) => log.error({ err: error }, "Discussion organizer tick failed")), this.intervalMs);
 		this.timer.unref();
-		this.tick();
+		this.tick().catch((error) => log.error({ err: error }, "Discussion organizer tick failed"));
 	}
 	async stop() {
+		this.stopping = true;
+		this.controller?.abort(/* @__PURE__ */ new Error("Discussion processing stopped"));
 		if (this.timer) clearInterval(this.timer);
 		this.timer = void 0;
 		while (this.running) await new Promise((resolve) => setTimeout(resolve, 10));
 	}
 	async tick() {
-		if (this.running) return;
+		if (this.running || this.stopping) return;
 		this.running = true;
 		try {
 			const capture = claimNextDiscussionCapture(this.owner);
 			if (!capture) return;
 			const controller = new AbortController();
+			this.controller = controller;
 			const renewal = setInterval(() => {
 				if (!renewDiscussionWorkLease(capture.id, this.owner)) controller.abort();
 			}, 3e4);
 			renewal.unref();
 			try {
-				await this.processor.process(capture, this.owner, controller.signal);
+				await withProcessingDeadline((signal) => this.processor.process(capture, this.owner, signal), controller.signal, this.requestTimeoutMs);
+				resetProcessingRetry(capture.id);
 				releaseDiscussionWorkLease(capture.id, this.owner);
 			} catch (error) {
 				const message = error instanceof Error ? error.message : String(error);
 				const ownsLease = renewDiscussionWorkLease(capture.id, this.owner);
 				releaseDiscussionWorkLease(capture.id, this.owner);
 				const current = getDiscussionCapture(capture.id);
-				const updated = ownsLease && current && updateDiscussionCapture(capture.id, {
-					status: "needs_attention",
-					failureStage: "organization",
-					failureCode: "organization_failed",
-					failureMessage: message.slice(0, 1e3)
-				}, ["organizing"]);
+				const updated = ownsLease && !controller.signal.aborted && current ? failOrRetryProcessing(current, "organization", error) : null;
 				if (updated) this.onUpdated?.(updated);
 				log.warn({
 					err: error,
@@ -57,6 +61,7 @@ var DiscussionOrganizerWorker = class {
 				}, `Discussion organization failed: ${message}`);
 			} finally {
 				clearInterval(renewal);
+				this.controller = void 0;
 			}
 		} finally {
 			this.running = false;

package/dist/src/discussions/organizer.js

@@ -2,6 +2,7 @@ import { init_transaction, runSqliteWriteTransaction } from "../storage/sqlite/t
 import { ObjectLinkService } from "../activity/service.js";
 import { completeDiscussionOrganization, createDiscussionOrganization, getDiscussionCapture, getLatestDiscussionOrganization, listDiscussionTranscriptSegments, renewDiscussionWorkLease, updateDiscussionCapture } from "./repository.js";
 import { saveTranscriptRevision } from "./revisions.js";
+import { chapterScope } from "./long-review.js";
 import { analyzeDiscussion } from "./analyzer.js";
 import { applyMeetingEdits } from "./edits.js";
 import { acceptRankedProject, findExactProjectMention } from "./project-inference.js";
@@ -27,6 +28,7 @@ var DiscussionOrganizer = class {
 			transcript,
 			segments: listDiscussionTranscriptSegments(capture.id),
 			discussionId: capture.id,
+			reviewScope: chapterScope(capture.id),
 			template: capture.template,
 			projects: projects.map(({ id, name }) => ({
 				id,

package/dist/src/discussions/processing.d.ts

@@ -0,0 +1,43 @@
+import type { DiscussionCapture, DiscussionProcessingProgress } from './types.js';
+export interface RecoveredTranscriptSegment {
+    text: string;
+    startedAtMs: number;
+    endedAtMs: number;
+    speakerLabel?: string;
+    correctedByUser?: boolean;
+    rawText?: string;
+}
+export interface TranscriptionCheckpoint {
+    segments: RecoveredTranscriptSegment[];
+    language?: string;
+}
+export interface ProcessingRun {
+    discussion_id: string;
+    run_id: string;
+    audio_sha256: string | null;
+    base_transcript_revision: number;
+    force_audio: number;
+    retry_stage: string | null;
+    attempt_count: number;
+    next_attempt_at: number | null;
+    lease_owner: string | null;
+    lease_until: number | null;
+}
+export declare const PROCESSING_MAX_ATTEMPTS = 3;
+export declare function getProcessingRun(id: string): ProcessingRun | undefined;
+/** A new original-audio generation leaves published transcript/summary revisions untouched. */
+export declare function createProcessingRun(capture: DiscussionCapture, forceAudio?: boolean): ProcessingRun;
+export declare function ensureProcessingRun(capture: DiscussionCapture, alignSource?: boolean): ProcessingRun;
+export declare function resetProcessingRetry(id: string): void;
+export declare function processingProgress(id: string): DiscussionProcessingProgress | undefined;
+export declare function claimSealingCapture(owner: string, now: number): DiscussionCapture | null;
+export declare function renewSealingLease(id: string, owner: string): boolean;
+export declare function releaseSealingLease(id: string, owner: string): void;
+export declare function assertProcessingRun(run: ProcessingRun, owner: string, signal: AbortSignal): void;
+export declare function readCheckpoint(runId: string, start: number, end: number): TranscriptionCheckpoint | undefined;
+export declare function saveCheckpoint(runId: string, start: number, end: number, result: TranscriptionCheckpoint): void;
+export declare function isTransientProcessingError(error: unknown, depth?: number): boolean;
+/** Persist bounded retries instead of sleeping inside a worker and blocking other recordings. */
+export declare function failOrRetryProcessing(capture: DiscussionCapture, stage: 'transcript_sealing' | 'organization', error: unknown): DiscussionCapture | null;
+/** Race the signal as well as passing it downstream, so an uncooperative provider cannot hold the worker. */
+export declare function withProcessingDeadline<T>(work: (signal: AbortSignal) => Promise<T>, parent: AbortSignal, timeoutMs: number): Promise<T>;

package/dist/src/discussions/processing.js

@@ -0,0 +1,125 @@
+import { getSqliteDatabase, init_transaction, runSqliteWriteTransaction } from "../storage/sqlite/transaction.js";
+import { getDiscussionCapture, updateDiscussionCapture } from "./repository.js";
+import { randomUUID } from "node:crypto";
+//#region src/discussions/processing.ts
+init_transaction();
+const LEASE_MS = 6e4;
+const PROCESSING_MAX_ATTEMPTS = 3;
+function getProcessingRun(id) {
+	return getSqliteDatabase().prepare("SELECT * FROM discussion_processing WHERE discussion_id=?").get(id);
+}
+/** A new original-audio generation leaves published transcript/summary revisions untouched. */
+function createProcessingRun(capture, forceAudio = false) {
+	return runSqliteWriteTransaction((db) => {
+		db.prepare("DELETE FROM discussion_processing WHERE discussion_id=?").run(capture.id);
+		db.prepare(`INSERT INTO discussion_processing (discussion_id, run_id, audio_sha256, base_transcript_revision, force_audio)
+      VALUES (?, ?, ?, ?, ?)`).run(capture.id, randomUUID(), capture.audioSha256 ?? null, capture.transcriptRevision, forceAudio ? 1 : 0);
+		return getProcessingRun(capture.id);
+	});
+}
+function ensureProcessingRun(capture, alignSource = false) {
+	const run = getProcessingRun(capture.id);
+	return !run || alignSource && (run.audio_sha256 !== (capture.audioSha256 ?? null) || run.base_transcript_revision !== capture.transcriptRevision) ? createProcessingRun(capture, run?.force_audio === 1) : run;
+}
+function resetProcessingRetry(id) {
+	getSqliteDatabase().prepare("UPDATE discussion_processing SET retry_stage=NULL, attempt_count=0, next_attempt_at=NULL WHERE discussion_id=?").run(id);
+}
+function processingProgress(id) {
+	const run = getProcessingRun(id);
+	if (!run) return void 0;
+	const row = getSqliteDatabase().prepare(`SELECT COUNT(*) count, COALESCE(SUM(ended_at_ms-started_at_ms),0) duration
+    FROM discussion_transcription_checkpoints WHERE run_id=?`).get(run.run_id);
+	return {
+		fromAudio: run.force_audio === 1,
+		completedChunks: row.count,
+		transcribedMs: Math.round(row.duration),
+		attemptCount: run.attempt_count,
+		maxAttempts: 3,
+		...run.next_attempt_at !== null ? { nextAttemptAt: run.next_attempt_at } : {}
+	};
+}
+function claimSealingCapture(owner, now) {
+	return runSqliteWriteTransaction((db) => {
+		const row = db.prepare(`SELECT d.id FROM discussion_captures d LEFT JOIN discussion_processing p ON p.discussion_id=d.id
+      WHERE d.status IN ('stopping','sealing') AND (p.next_attempt_at IS NULL OR p.next_attempt_at<=?)
+      AND (p.lease_until IS NULL OR p.lease_until<=?) ORDER BY d.updated_at, d.id LIMIT 1`).get(now, Date.now());
+		if (!row) return null;
+		const capture = getDiscussionCapture(row.id);
+		ensureProcessingRun(capture, true);
+		db.prepare("UPDATE discussion_processing SET lease_owner=?, lease_until=? WHERE discussion_id=?").run(owner, Date.now() + LEASE_MS, row.id);
+		return capture;
+	});
+}
+function renewSealingLease(id, owner) {
+	return getSqliteDatabase().prepare(`UPDATE discussion_processing SET lease_until=? WHERE discussion_id=? AND lease_owner=? AND lease_until>?`).run(Date.now() + LEASE_MS, id, owner, Date.now()).changes === 1;
+}
+function releaseSealingLease(id, owner) {
+	getSqliteDatabase().prepare("UPDATE discussion_processing SET lease_owner=NULL,lease_until=NULL WHERE discussion_id=? AND lease_owner=?").run(id, owner);
+}
+function assertProcessingRun(run, owner, signal) {
+	signal.throwIfAborted();
+	const current = getDiscussionCapture(run.discussion_id);
+	if (!current || current.status !== "sealing" || current.audioDeletedAt || current.audioSha256 !== (run.audio_sha256 ?? void 0) || current.transcriptRevision !== run.base_transcript_revision || getProcessingRun(current.id)?.run_id !== run.run_id || !renewSealingLease(current.id, owner)) throw new Error("Transcription processing lease or source revision changed");
+}
+function readCheckpoint(runId, start, end) {
+	const row = getSqliteDatabase().prepare("SELECT result_json FROM discussion_transcription_checkpoints WHERE run_id=? AND started_at_ms=? AND ended_at_ms=?").get(runId, start, end);
+	return row ? JSON.parse(row.result_json) : void 0;
+}
+function saveCheckpoint(runId, start, end, result) {
+	getSqliteDatabase().prepare(`INSERT INTO discussion_transcription_checkpoints (run_id,started_at_ms,ended_at_ms,result_json,created_at) VALUES (?,?,?,?,?)`).run(runId, start, end, JSON.stringify(result), Date.now());
+}
+function isTransientProcessingError(error, depth = 0) {
+	if (depth > 3) return false;
+	const item = error;
+	const status = item?.status ?? item?.statusCode;
+	if (status !== void 0) return [
+		408,
+		425,
+		429
+	].includes(status) || status >= 500;
+	const text = error instanceof Error ? `${error.name} ${error.message}` : String(error);
+	if (/not configured|no model configured|no api key|invalid.*key|unauthorized|forbidden|authentication|unsupported|invalid.*(audio|time range)|no text|audio.*missing|checksum mismatch|transcript.*missing|ENOENT|lease|revision changed/i.test(text)) return false;
+	if (item?.reasonCode === "not_configured" || item?.reasonCode === "unsupported_format") return false;
+	if (item?.reasonCode === "timeout" || item?.reasonCode === "provider_error") return true;
+	if (/timeout|timed out|rate.?limit|temporar|unavailable|fetch failed|network|connection error|ECONN|ETIMEDOUT|EAI_AGAIN|socket|overload|internal server|server error|try again|invalid discussion organization|invalid evidence|truncated|\b(408|429|5\d\d)\b|SyntaxError/i.test(text)) return true;
+	return item?.cause !== void 0 && item.cause !== error ? isTransientProcessingError(item.cause, depth + 1) : false;
+}
+/** Persist bounded retries instead of sleeping inside a worker and blocking other recordings. */
+function failOrRetryProcessing(capture, stage, error) {
+	return runSqliteWriteTransaction((db) => {
+		const current = getDiscussionCapture(capture.id);
+		if (!current || current.status !== (stage === "organization" ? "organizing" : "sealing")) return null;
+		const run = ensureProcessingRun(current);
+		const attempts = (run.retry_stage === stage ? run.attempt_count : 0) + 1;
+		const retry = attempts < 3 && isTransientProcessingError(error);
+		const next = retry ? Date.now() + 5e3 * 4 ** (attempts - 1) : null;
+		db.prepare("UPDATE discussion_processing SET retry_stage=?,attempt_count=?,next_attempt_at=? WHERE discussion_id=?").run(stage, attempts, next, capture.id);
+		return updateDiscussionCapture(capture.id, {
+			status: retry ? current.status : "needs_attention",
+			failureStage: stage,
+			failureCode: retry ? "processing_retry_scheduled" : `${stage}_failed`,
+			failureMessage: (error instanceof Error ? error.message : String(error)).slice(0, 1e3)
+		}, [current.status]);
+	});
+}
+/** Race the signal as well as passing it downstream, so an uncooperative provider cannot hold the worker. */
+async function withProcessingDeadline(work, parent, timeoutMs) {
+	const timeout = new AbortController();
+	const timer = setTimeout(() => timeout.abort(/* @__PURE__ */ new Error("Processing request timed out")), timeoutMs);
+	timer.unref();
+	const signal = AbortSignal.any([parent, timeout.signal]);
+	let abort;
+	try {
+		signal.throwIfAborted();
+		const aborted = new Promise((_resolve, reject) => {
+			abort = () => reject(signal.reason);
+			signal.addEventListener("abort", abort, { once: true });
+		});
+		return await Promise.race([work(signal), aborted]);
+	} finally {
+		clearTimeout(timer);
+		if (abort) signal.removeEventListener("abort", abort);
+	}
+}
+//#endregion
+export { PROCESSING_MAX_ATTEMPTS, assertProcessingRun, claimSealingCapture, createProcessingRun, ensureProcessingRun, failOrRetryProcessing, getProcessingRun, isTransientProcessingError, processingProgress, readCheckpoint, releaseSealingLease, renewSealingLease, resetProcessingRetry, saveCheckpoint, withProcessingDeadline };