@speclip/pi-talking-head 0.1.1 → 0.1.3

package/src/transcript.ts

@@ -1,13 +1,19 @@
 import type {
 	ArollSegment,
+	FillerCandidate,
 	PauseCandidate,
 	PauseClassification,
+	RepetitionCandidate,
+	SentenceAnalysis,
 	TalkingHeadPolicy,
 	TranscriptAnalysis,
 	TranscriptWord,
 	WordTranscript,
 } from "./contracts.ts";
 
+const HESITATION_FILLERS = new Set(["嗯", "呃", "额", "唔", "呣", "em", "um", "uh"]);
+const CONTEXTUAL_FILLERS = new Set(["啊", "呀", "那个", "这个", "就是", "然后"]);
+
 export const DEFAULT_POLICY: TalkingHeadPolicy = {
 	cutThresholdMs: 500,
 	headPaddingMs: 50,
@@ -40,7 +46,17 @@ function flattenWords(transcript: WordTranscript): TranscriptWord[] {
 		throw new Error("Transcript must contain text and sentences");
 	}
 	const words = transcript.sentences.flatMap((sentence) => {
-		if (!sentence || !Array.isArray(sentence.words)) throw new Error("Transcript sentence is missing words");
+		if (!sentence || !Number.isInteger(sentence.id) || typeof sentence.text !== "string"
+			|| !finiteNonNegative(sentence.beginMs) || !finiteNonNegative(sentence.endMs)
+			|| sentence.endMs <= sentence.beginMs || !Array.isArray(sentence.words) || sentence.words.length === 0) {
+			throw new Error("Transcript contains an invalid sentence timestamp or metadata");
+		}
+		const first = sentence.words[0]!;
+		const last = sentence.words.at(-1)!;
+		if (!finiteNonNegative(first.beginMs) || !finiteNonNegative(last.endMs)
+			|| first.beginMs < sentence.beginMs || last.endMs > sentence.endMs) {
+			throw new Error("Transcript sentence timestamp does not contain its word timestamps");
+		}
 		return sentence.words;
 	});
 	if (words.length === 0) throw new Error("Transcript contains no word timestamps");
@@ -57,14 +73,172 @@ function flattenWords(transcript: WordTranscript): TranscriptWord[] {
 	return words.map((word) => ({ ...word, punctuation: word.punctuation ?? "" }));
 }
 
-function candidatesFrom(words: TranscriptWord[]): PauseCandidate[] {
+function normalizedSpokenText(text: string): string {
+	return text.trim().toLowerCase().replace(/^[\p{P}\p{S}\s]+|[\p{P}\p{S}\s]+$/gu, "");
+}
+
+function fillerCandidates(transcript: WordTranscript, words: TranscriptWord[]): FillerCandidate[] {
+	const candidates: FillerCandidate[] = [];
+	let wordIndex = 0;
+	for (let sentenceIndex = 0; sentenceIndex < transcript.sentences.length; sentenceIndex += 1) {
+		const sentence = transcript.sentences[sentenceIndex]!;
+		for (const _word of sentence.words) {
+			const word = words[wordIndex]!;
+			const normalized = normalizedSpokenText(word.text);
+			const hesitation = HESITATION_FILLERS.has(normalized);
+			const contextual = CONTEXTUAL_FILLERS.has(normalized);
+			if (hesitation || contextual) {
+				candidates.push({
+					id: `filler-${String(candidates.length + 1).padStart(3, "0")}`,
+					wordIndex,
+					sentenceIndex,
+					text: word.text,
+					startMs: word.beginMs,
+					endMs: word.endMs,
+					kind: hesitation ? "hesitation" : "discourse",
+					matchConfidence: hesitation ? "exact" : "contextual",
+					recommendation: "review",
+					contextText: sentence.text,
+					reasons: hesitation
+						? ["The token commonly marks hesitation, but may still carry delivery intent."]
+						: ["The token can be either a filler or meaningful discourse, so context is required."],
+				});
+			}
+			wordIndex += 1;
+		}
+	}
+	return candidates;
+}
+
+function sentenceAnalyses(transcript: WordTranscript, fillers: FillerCandidate[]): SentenceAnalysis[] {
+	let wordStartIndex = 0;
+	const fillersBySentence = new Map<number, FillerCandidate[]>();
+	for (const filler of fillers) {
+		const current = fillersBySentence.get(filler.sentenceIndex) ?? [];
+		current.push(filler);
+		fillersBySentence.set(filler.sentenceIndex, current);
+	}
+	return transcript.sentences.map((sentence, sentenceIndex) => {
+		const wordEndIndex = wordStartIndex + sentence.words.length - 1;
+		const sentenceFillers = fillersBySentence.get(sentenceIndex) ?? [];
+		const deliveryCues: SentenceAnalysis["deliveryCues"] = [];
+		const evidence: string[] = [];
+		const hesitationCount = sentenceFillers.filter((candidate) => candidate.kind === "hesitation").length;
+		if (hesitationCount > 0) {
+			deliveryCues.push("hesitation");
+			evidence.push(`Contains ${hesitationCount} hesitation-lexicon token(s).`);
+		}
+		if (/[？?]/u.test(sentence.text) || sentence.words.some((word) => /[？?]/u.test(word.punctuation ?? ""))) {
+			deliveryCues.push("question");
+			evidence.push("Question punctuation is present in the transcript.");
+		}
+		if (/[！!]/u.test(sentence.text) || sentence.words.some((word) => /[！!]/u.test(word.punctuation ?? ""))) {
+			deliveryCues.push("emphasis");
+			evidence.push("Emphasis punctuation is present in the transcript.");
+		}
+		if (deliveryCues.length === 0) {
+			deliveryCues.push("neutral");
+			evidence.push("No explicit delivery cue was found in transcript text.");
+		}
+		const analysis: SentenceAnalysis = {
+			sentenceIndex,
+			sentenceId: sentence.id,
+			beginMs: sentence.beginMs,
+			endMs: sentence.endMs,
+			text: sentence.text,
+			wordStartIndex,
+			wordEndIndex,
+			deliveryCues,
+			confidence: "low",
+			evidence,
+		};
+		wordStartIndex = wordEndIndex + 1;
+		return analysis;
+	});
+}
+
+function repetitionCandidates(transcript: WordTranscript, words: TranscriptWord[]): RepetitionCandidate[] {
+	const candidates: RepetitionCandidate[] = [];
+	let sentenceWordStart = 0;
+	for (let sentenceIndex = 0; sentenceIndex < transcript.sentences.length; sentenceIndex += 1) {
+		const sentence = transcript.sentences[sentenceIndex]!;
+		for (let offset = 1; offset < sentence.words.length; offset += 1) {
+			const firstWordIndex = sentenceWordStart + offset - 1;
+			const secondWordIndex = sentenceWordStart + offset;
+			const first = words[firstWordIndex]!;
+			const second = words[secondWordIndex]!;
+			const normalized = normalizedSpokenText(first.text);
+			if (!normalized || normalized !== normalizedSpokenText(second.text)) continue;
+			candidates.push({
+				id: `repetition-${String(candidates.length + 1).padStart(3, "0")}`,
+				sentenceIndex,
+				text: first.text,
+				firstWordIndex,
+				secondWordIndex,
+				startMs: first.beginMs,
+				endMs: second.endMs,
+				recommendation: "review",
+				contextText: sentence.text,
+				reasons: ["Two adjacent normalized tokens are identical; review whether this is a false start or intentional emphasis."],
+			});
+		}
+		sentenceWordStart += sentence.words.length;
+	}
+	return candidates;
+}
+
+function wordSentenceIndexes(transcript: WordTranscript): number[] {
+	return transcript.sentences.flatMap((sentence, sentenceIndex) => sentence.words.map(() => sentenceIndex));
+}
+
+function candidatesFrom(
+	words: TranscriptWord[],
+	transcript: WordTranscript,
+	fillers: FillerCandidate[],
+	policy: TalkingHeadPolicy,
+): PauseCandidate[] {
 	const candidates: PauseCandidate[] = [];
+	const sentenceIndexes = wordSentenceIndexes(transcript);
+	const fillersByWordIndex = new Map<number, FillerCandidate[]>();
+	for (const filler of fillers) {
+		fillersByWordIndex.set(filler.wordIndex, [...(fillersByWordIndex.get(filler.wordIndex) ?? []), filler]);
+	}
 	for (let index = 1; index < words.length; index += 1) {
 		const before = words[index - 1];
 		const after = words[index];
 		if (!before || !after) continue;
 		const durationMs = after.beginMs - before.endMs;
 		if (durationMs <= 0) continue;
+		const beforeSentenceIndex = sentenceIndexes[index - 1]!;
+		const afterSentenceIndex = sentenceIndexes[index]!;
+		const adjacentFillers = [
+			...(fillersByWordIndex.get(index - 1) ?? []),
+			...(fillersByWordIndex.get(index) ?? []),
+		];
+		const sentenceBoundary = beforeSentenceIndex !== afterSentenceIndex;
+		const beforeSentence = transcript.sentences[beforeSentenceIndex]!;
+		const expressiveBoundary = /[！!？?…]/u.test(before.punctuation) || /[！!？?…]\s*$/u.test(beforeSentence.text);
+		const reasons: string[] = [];
+		let recommendation: PauseCandidate["recommendation"];
+		if (durationMs < 150) {
+			recommendation = "keep";
+			reasons.push("The gap is shorter than 150ms and removing it risks robotic cadence.");
+		} else if (adjacentFillers.length > 0) {
+			recommendation = "review";
+			reasons.push("The gap touches a possible filler whose meaning must be judged in sentence context.");
+		} else if (expressiveBoundary) {
+			recommendation = "review";
+			reasons.push("The pause follows expressive punctuation and may carry emphasis, emotion, or a question beat.");
+		} else if (sentenceBoundary) {
+			recommendation = "review";
+			reasons.push("The gap is between sentences and may mark a paragraph, topic, or deliberate delivery boundary.");
+		} else if (durationMs >= policy.cutThresholdMs) {
+			recommendation = "cut";
+			reasons.push(`The unprotected word gap meets the ${policy.cutThresholdMs}ms automatic cut threshold.`);
+		} else {
+			recommendation = "review";
+			reasons.push("The gap is noticeable but does not meet the automatic cut threshold.");
+		}
 		candidates.push({
 			id: `pause-${String(candidates.length + 1).padStart(3, "0")}`,
 			startMs: before.endMs,
@@ -73,18 +247,28 @@ function candidatesFrom(words: TranscriptWord[]): PauseCandidate[] {
 			classification: classifyPause(durationMs),
 			beforeText: `${before.text}${before.punctuation}`,
 			afterText: after.text,
+			boundary: beforeSentenceIndex === afterSentenceIndex ? "within-sentence" : "between-sentences",
+			context: {
+				before: transcript.sentences[beforeSentenceIndex]!.text,
+				after: transcript.sentences[afterSentenceIndex]!.text,
+			},
+			adjacentFillerIds: adjacentFillers.map((candidate) => candidate.id),
+			recommendation,
+			reasons,
 		});
 	}
 	return candidates;
 }
 
-function defaultSegments(words: TranscriptWord[], policy: TalkingHeadPolicy): ArollSegment[] {
+function defaultSegments(words: TranscriptWord[], policy: TalkingHeadPolicy, candidates: PauseCandidate[]): ArollSegment[] {
 	const segments: ArollSegment[] = [];
+	const candidateByGap = new Map(candidates.map((candidate) => [`${candidate.startMs}:${candidate.endMs}`, candidate]));
 	let segmentStart = Math.max(0, words[0]!.beginMs - policy.headPaddingMs);
 	for (let index = 1; index < words.length; index += 1) {
 		const before = words[index - 1]!;
 		const after = words[index]!;
-		if (after.beginMs - before.endMs < policy.cutThresholdMs) continue;
+		const candidate = candidateByGap.get(`${before.endMs}:${after.beginMs}`);
+		if (candidate?.recommendation !== "cut") continue;
 		segments.push({
 			id: `a-${String(segments.length + 1).padStart(3, "0")}`,
 			sourceStartMs: segmentStart,
@@ -111,10 +295,18 @@ export function analyzeTranscript(
 	const policy = { ...DEFAULT_POLICY, ...overrides };
 	validatePolicy(policy);
 	const words = flattenWords(transcript);
-	const segments = defaultSegments(words, policy);
+	const fillers = fillerCandidates(transcript, words);
+	const repetitions = repetitionCandidates(transcript, words);
+	const candidates = candidatesFrom(words, transcript, fillers, policy);
+	const segments = defaultSegments(words, policy, candidates);
 	return {
+		schemaVersion: 2,
+		text: transcript.text,
 		words,
-		candidates: candidatesFrom(words),
+		sentences: sentenceAnalyses(transcript, fillers),
+		fillers,
+		repetitions,
+		candidates,
 		segments,
 		outputDurationMs: timelineDuration(segments),
 	};

package/src/workspace.ts

@@ -39,6 +39,17 @@ export async function resolveExistingWorkspaceFile(cwd: string, inputPath: strin
 	return canonical;
 }
 
+export async function resolveExistingWorkspaceDirectory(cwd: string, inputPath: string): Promise<string> {
+	const root = await workspaceRoot(cwd);
+	const lexical = resolve(root, inputPath);
+	if (!isWithin(root, lexical)) throw new Error(`Path is outside the workspace: ${inputPath}`);
+	if ((await lstat(lexical)).isSymbolicLink()) throw new Error(`Unsafe workspace-directory symlink: ${inputPath}`);
+	const canonical = await realpath(lexical);
+	if (!isWithin(root, canonical)) throw new Error(`Path resolves outside the workspace: ${inputPath}`);
+	if (!(await stat(canonical)).isDirectory()) throw new Error(`Path is not a directory: ${inputPath}`);
+	return canonical;
+}
+
 export async function resolveWorkspacePath(cwd: string, inputPath: string): Promise<string> {
 	const root = await workspaceRoot(cwd);
 	const lexical = resolve(root, inputPath);
@@ -55,17 +66,20 @@ export async function workspaceRelativePath(cwd: string, absolutePath: string):
 	return relative(root, absolutePath).split(sep).join("/");
 }
 
-async function sha256File(path: string): Promise<string> {
+async function sha256File(path: string, signal?: AbortSignal): Promise<string> {
+	signal?.throwIfAborted();
 	const hash = createHash("sha256");
-	for await (const chunk of createReadStream(path)) hash.update(chunk);
+	for await (const chunk of createReadStream(path, signal === undefined ? {} : { signal })) hash.update(chunk);
+	signal?.throwIfAborted();
 	return hash.digest("hex");
 }
 
-export async function snapshotFile(cwd: string, inputPath: string): Promise<FileRef> {
+export async function snapshotFile(cwd: string, inputPath: string, signal?: AbortSignal): Promise<FileRef> {
+	signal?.throwIfAborted();
 	const absolute = await resolveExistingWorkspaceFile(cwd, inputPath);
 	return {
 		path: await workspaceRelativePath(cwd, absolute),
 		bytes: (await stat(absolute)).size,
-		sha256: await sha256File(absolute),
+		sha256: await sha256File(absolute, signal),
 	};
 }