dictawhisper 0.0.13 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/bin/dictawhisper.mjs +14 -0
- package/config.example.json +41 -0
- package/dist/ui/_app/immutable/assets/0.C6M3gHo0.css +1 -0
- package/dist/ui/_app/immutable/assets/2.C96vgj_4.css +1 -0
- package/dist/ui/_app/immutable/chunks/BSs9MmST.js +3 -0
- package/dist/ui/_app/immutable/chunks/Bjy-W4x2.js +81 -0
- package/dist/ui/_app/immutable/chunks/BslYtab-.js +1 -0
- package/dist/ui/_app/immutable/chunks/xihTtKlq.js +1 -0
- package/dist/ui/_app/immutable/entry/app.BJlRju4c.js +2 -0
- package/dist/ui/_app/immutable/entry/start.C5XBbXTT.js +1 -0
- package/dist/ui/_app/immutable/nodes/0.D23pkrHr.js +1 -0
- package/dist/ui/_app/immutable/nodes/1.C5T-U4uV.js +1 -0
- package/dist/ui/_app/immutable/nodes/2.59uAq_ko.js +5 -0
- package/dist/ui/_app/version.json +1 -0
- package/dist/ui/favicon.png +0 -0
- package/dist/ui/favicon.svg +3 -0
- package/dist/ui/index.html +38 -0
- package/dist/ui/logo.png +0 -0
- package/package.json +12 -3
- package/scripts/transcribe_faster_whisper.py +428 -0
- package/src/apiRoutes.ts +464 -0
- package/src/classes/Queue.ts +77 -0
- package/src/classes/Watcher.ts +117 -0
- package/src/cli.ts +132 -0
- package/src/config.ts +238 -0
- package/src/dev.ts +63 -0
- package/src/doctor.ts +12 -0
- package/src/lib/alignLib.ts +154 -0
- package/src/lib/audioLib.ts +202 -0
- package/src/lib/audioProbeLib.ts +112 -0
- package/src/lib/cleanupProvenanceLib.ts +80 -0
- package/src/lib/downloadLib.ts +8 -0
- package/src/lib/fileSettleLib.ts +213 -0
- package/src/lib/fsLib.ts +37 -0
- package/src/lib/healthLib.ts +377 -0
- package/src/lib/inboxUrl.ts +123 -0
- package/src/lib/journalEmbedLib.ts +127 -0
- package/src/lib/journalIndexLib.ts +923 -0
- package/src/lib/journalQueryLib.ts +269 -0
- package/src/lib/journalService.ts +287 -0
- package/src/lib/jsonLib.ts +71 -0
- package/src/lib/localberthGet.ts +14 -0
- package/src/lib/ollanetLib.ts +117 -0
- package/src/lib/ollanetReadyLib.ts +7 -0
- package/src/lib/organizationLib.ts +182 -0
- package/src/lib/pathAllowLib.ts +60 -0
- package/src/lib/queueLib.ts +30 -0
- package/src/lib/speechCleanupLib.ts +14 -0
- package/src/lib/tagConsolidateLib.ts +431 -0
- package/src/lib/tailscaleLib.ts +78 -0
- package/src/lib/transcriptionLib.ts +484 -0
- package/src/lib/whisperLib.ts +409 -0
- package/src/mcp.ts +151 -0
- package/src/prompts/cleanTranscription.ts +62 -0
- package/src/prompts/consolidateTags.ts +38 -0
- package/src/prompts/whisperInitial.ts +11 -0
- package/src/server.ts +220 -0
- package/src/socketEvents.ts +72 -0
- package/src/types/transcription.ts +56 -0
|
@@ -0,0 +1,484 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import type { Server as SocketIOServer, Socket } from 'socket.io';
|
|
4
|
+
import z from 'zod';
|
|
5
|
+
import { zodToJsonSchema } from 'zod-to-json-schema';
|
|
6
|
+
import { q } from './queueLib.ts';
|
|
7
|
+
import { audioFileRegex } from './audioLib.ts';
|
|
8
|
+
import { ensurePlaybackCues } from './alignLib.ts';
|
|
9
|
+
import { isSkippedWatchPath, requestWhenSettled } from './fileSettleLib.ts';
|
|
10
|
+
import { Watcher } from '../classes/Watcher.ts';
|
|
11
|
+
import { parseJSON } from './jsonLib.ts';
|
|
12
|
+
import { cleanWithOllanet, describeCleanError } from './ollanetLib.ts';
|
|
13
|
+
import { buildCleanTranscriptionPrompt, CLEAN_PROMPT_VERSION } from '../prompts/cleanTranscription.ts';
|
|
14
|
+
import { collapseSpeechLoops } from './speechCleanupLib.ts';
|
|
15
|
+
import { config } from '../config.ts';
|
|
16
|
+
import { ollanetIsConfigured } from './ollanetReadyLib.ts';
|
|
17
|
+
import type { TranscriptionDocument } from '../types/transcription.ts';
|
|
18
|
+
import { applyCleanupProvenance, dictawhisperVersion } from './cleanupProvenanceLib.ts';
|
|
19
|
+
import { dropSidecar, indexSidecar, notesIndexReload } from './journalService.ts';
|
|
20
|
+
|
|
21
|
+
export const transcriptions: Record<string, any> = {};
|
|
22
|
+
|
|
23
|
+
let liveIo: SocketIOServer | null = null;
|
|
24
|
+
|
|
25
|
+
export function setTranscriptionIo(io: SocketIOServer | null) {
|
|
26
|
+
liveIo = io;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function emitNotesIndex(target: Socket | SocketIOServer | null = null) {
|
|
30
|
+
const dest = target ?? liveIo;
|
|
31
|
+
dest?.emit('notes-index', notesIndexReload());
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function forgetTranscription(jsonFile: string) {
|
|
35
|
+
delete transcriptions[jsonFile];
|
|
36
|
+
dropSidecar(jsonFile);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function relocateTranscription(oldJson: string, newJson: string) {
|
|
40
|
+
if (oldJson === newJson) return;
|
|
41
|
+
if (Object.hasOwn(transcriptions, oldJson)) {
|
|
42
|
+
transcriptions[newJson] = transcriptions[oldJson];
|
|
43
|
+
delete transcriptions[oldJson];
|
|
44
|
+
}
|
|
45
|
+
dropSidecar(oldJson);
|
|
46
|
+
if (fs.existsSync(newJson)) indexSidecar(newJson);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const PREVIEW_LIMIT = 200;
|
|
50
|
+
|
|
51
|
+
export function summarizeTranscription(jsonFile: string, json: any = transcriptions[jsonFile]) {
|
|
52
|
+
const cleaned = String(json?.cleanedTranscription || '').trim();
|
|
53
|
+
const raw = String(json?.text || '').trim();
|
|
54
|
+
const source = cleaned || raw;
|
|
55
|
+
const compact = source.replace(/\s+/g, ' ').trim();
|
|
56
|
+
const preview = json?.audioError && !compact
|
|
57
|
+
? '[unreadable audio]'
|
|
58
|
+
: compact.length > PREVIEW_LIMIT
|
|
59
|
+
? `${compact.slice(0, PREVIEW_LIMIT)}…`
|
|
60
|
+
: compact;
|
|
61
|
+
return {
|
|
62
|
+
jsonFile,
|
|
63
|
+
basename: path.basename(jsonFile),
|
|
64
|
+
transcriptionJson: {
|
|
65
|
+
tags: Array.isArray(json?.tags) ? json.tags : [],
|
|
66
|
+
elapsed: json?.elapsed ?? null,
|
|
67
|
+
cleanupError: json?.cleanupError ?? null,
|
|
68
|
+
cleanupAttempts: json?.cleanupAttempts ?? 0,
|
|
69
|
+
cleanupSkipped: Boolean(json?.cleanupSkipped),
|
|
70
|
+
preview,
|
|
71
|
+
hasCleaned: Boolean(cleaned),
|
|
72
|
+
audioError: json?.audioError ? String(json.audioError) : null,
|
|
73
|
+
starred: Boolean(json?.starred),
|
|
74
|
+
_partial: true,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function listNoteSummaries() {
|
|
80
|
+
return Object.entries(transcriptions).map(([jsonFile, json]) => summarizeTranscription(jsonFile, json));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function readSidecar(transcriptionFile: string): TranscriptionDocument | null {
|
|
84
|
+
try {
|
|
85
|
+
if (!fs.existsSync(transcriptionFile)) return null;
|
|
86
|
+
return JSON.parse(fs.readFileSync(transcriptionFile, 'utf-8')) as TranscriptionDocument;
|
|
87
|
+
} catch (error) {
|
|
88
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
89
|
+
console.error(`[transcription] unreadable sidecar ${transcriptionFile}: ${detail}`);
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function sidecarIsProcessed(json: TranscriptionDocument | null): boolean {
|
|
95
|
+
if (!json) return false;
|
|
96
|
+
if (json.cleanedTranscription || json.audioError || json.cleanupSkipped) return true;
|
|
97
|
+
return typeof json.text === 'string' && !json.text.trim() && Array.isArray(json.segments);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function readTranscription(jsonFile: string) {
|
|
101
|
+
const transcriptionJson = readSidecar(jsonFile);
|
|
102
|
+
if (!transcriptionJson) return null;
|
|
103
|
+
let dirty = false;
|
|
104
|
+
if (ensurePlaybackCues(transcriptionJson)) dirty = true;
|
|
105
|
+
if (dirty) {
|
|
106
|
+
fs.writeFileSync(jsonFile, JSON.stringify(transcriptionJson, null, 2), { encoding: 'utf-8' });
|
|
107
|
+
}
|
|
108
|
+
transcriptions[jsonFile] = transcriptionJson;
|
|
109
|
+
return { jsonFile, transcriptionJson };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function emitTranscription(target: Socket | SocketIOServer | null = null, jsonFile: string, elapsed: string | null = null) {
|
|
113
|
+
if (!fs.existsSync(jsonFile)) {
|
|
114
|
+
console.error(`[emit-transcription-error] JSON file does not exist (yet): ${jsonFile}`);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
const transcriptionJson = readSidecar(jsonFile);
|
|
118
|
+
if (!transcriptionJson) return;
|
|
119
|
+
let dirty = false;
|
|
120
|
+
if (elapsed && !Object.hasOwn(transcriptionJson, 'elapsed')) {
|
|
121
|
+
transcriptionJson.elapsed = elapsed;
|
|
122
|
+
dirty = true;
|
|
123
|
+
}
|
|
124
|
+
if (ensurePlaybackCues(transcriptionJson)) dirty = true;
|
|
125
|
+
if (dirty) {
|
|
126
|
+
fs.writeFileSync(jsonFile, JSON.stringify(transcriptionJson, null, 2), { encoding: 'utf-8' });
|
|
127
|
+
}
|
|
128
|
+
transcriptions[jsonFile] = transcriptionJson;
|
|
129
|
+
indexSidecar(jsonFile);
|
|
130
|
+
const dest = target ?? liveIo;
|
|
131
|
+
dest?.emit('transcription', { jsonFile, transcriptionJson });
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function normalizeTags(tags: unknown): string[] {
|
|
135
|
+
if (!Array.isArray(tags)) return [];
|
|
136
|
+
const seen = new Set<string>();
|
|
137
|
+
const out: string[] = [];
|
|
138
|
+
for (const raw of tags) {
|
|
139
|
+
const tag = String(raw || '').trim().replace(/\s+/g, '-');
|
|
140
|
+
if (!tag) continue;
|
|
141
|
+
const key = tag.toLowerCase();
|
|
142
|
+
if (seen.has(key)) continue;
|
|
143
|
+
seen.add(key);
|
|
144
|
+
out.push(tag);
|
|
145
|
+
}
|
|
146
|
+
return out.slice(0, 40);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function patchTranscription(
|
|
150
|
+
jsonFile: string,
|
|
151
|
+
patch: { tags?: unknown; starred?: boolean },
|
|
152
|
+
) {
|
|
153
|
+
if (!fs.existsSync(jsonFile)) {
|
|
154
|
+
throw new Error('note not found');
|
|
155
|
+
}
|
|
156
|
+
const json = JSON.parse(fs.readFileSync(jsonFile, 'utf-8')) as TranscriptionDocument;
|
|
157
|
+
if (patch.tags !== undefined) {
|
|
158
|
+
json.tags = normalizeTags(patch.tags);
|
|
159
|
+
}
|
|
160
|
+
if (typeof patch.starred === 'boolean') {
|
|
161
|
+
json.starred = patch.starred;
|
|
162
|
+
}
|
|
163
|
+
fs.writeFileSync(jsonFile, JSON.stringify(json, null, 2), { encoding: 'utf-8' });
|
|
164
|
+
transcriptions[jsonFile] = json;
|
|
165
|
+
indexSidecar(jsonFile);
|
|
166
|
+
liveIo?.emit('transcription', { jsonFile, transcriptionJson: json });
|
|
167
|
+
return { jsonFile, transcriptionJson: json };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export function getTranscriptionFilename(file: string): string {
|
|
171
|
+
const parsed = path.parse(file);
|
|
172
|
+
return path.join(parsed.dir, `${parsed.name}.json`);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export function checkTranscription(file: string): {
|
|
176
|
+
isProcessed: boolean;
|
|
177
|
+
transcriptionFile: string;
|
|
178
|
+
transcriptionExists: boolean;
|
|
179
|
+
} {
|
|
180
|
+
const transcriptionFile = getTranscriptionFilename(file);
|
|
181
|
+
const transcriptionExists = fs.existsSync(transcriptionFile);
|
|
182
|
+
if (transcriptionExists) {
|
|
183
|
+
const transcriptionJson = readSidecar(transcriptionFile);
|
|
184
|
+
return {
|
|
185
|
+
isProcessed: sidecarIsProcessed(transcriptionJson),
|
|
186
|
+
transcriptionFile,
|
|
187
|
+
transcriptionExists,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
return { isProcessed: false, transcriptionFile, transcriptionExists };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export async function cleanTranscription(
|
|
194
|
+
file: string,
|
|
195
|
+
callback: (err: Error | null, result?: string) => void,
|
|
196
|
+
options: { reclean?: boolean } = {}
|
|
197
|
+
) {
|
|
198
|
+
const { isProcessed, transcriptionFile, transcriptionExists } = checkTranscription(file);
|
|
199
|
+
if (isProcessed && !options.reclean) {
|
|
200
|
+
try {
|
|
201
|
+
const existing = JSON.parse(fs.readFileSync(transcriptionFile, 'utf-8'));
|
|
202
|
+
if (ensurePlaybackCues(existing)) {
|
|
203
|
+
fs.writeFileSync(transcriptionFile, JSON.stringify(existing, null, 2), { encoding: 'utf-8' });
|
|
204
|
+
}
|
|
205
|
+
} catch {
|
|
206
|
+
// already cleaned; cues are optional
|
|
207
|
+
}
|
|
208
|
+
console.log(`[clean-transcription] Transcription already cleaned: ${transcriptionFile}`);
|
|
209
|
+
callback(null);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (!transcriptionExists) {
|
|
213
|
+
callback(new Error(`No transcription JSON for ${file}`));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
try {
|
|
218
|
+
const transcriptionJson = JSON.parse(fs.readFileSync(transcriptionFile, 'utf-8'));
|
|
219
|
+
if (!String(transcriptionJson.text || '').trim()) {
|
|
220
|
+
console.log(`[clean-transcription] no speech to clean: ${transcriptionFile}`);
|
|
221
|
+
callback(null);
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const { preferredTagsForCleanup } = await import('./tagConsolidateLib.ts');
|
|
226
|
+
const prompt = buildCleanTranscriptionPrompt(
|
|
227
|
+
collapseSpeechLoops(transcriptionJson.text),
|
|
228
|
+
preferredTagsForCleanup()
|
|
229
|
+
);
|
|
230
|
+
console.log(
|
|
231
|
+
`[clean-transcription] Cleaning via ollanet ${config.ollanet.machine} / ${config.ollanet.cleanModel}: ${transcriptionFile}`
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
const responseJSONSchema = zodToJsonSchema(
|
|
235
|
+
z.object({
|
|
236
|
+
cleanedTranscription: z.string().min(10),
|
|
237
|
+
tags: z.array(z.string().min(1)),
|
|
238
|
+
})
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
const { completion, meta, thinking } = await cleanWithOllanet(prompt, responseJSONSchema as Record<string, unknown>);
|
|
242
|
+
const jsonCompletion = await parseJSON(completion);
|
|
243
|
+
if (!jsonCompletion || !jsonCompletion.cleanedTranscription) {
|
|
244
|
+
console.error(`[clean-transcription-error] No cleaned transcription returned for file: ${transcriptionFile}`);
|
|
245
|
+
callback(new Error(`No cleaned transcription returned for ${transcriptionFile}`));
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
applyCleanupProvenance(transcriptionJson, {
|
|
250
|
+
text: jsonCompletion.cleanedTranscription,
|
|
251
|
+
model: String(meta?.model || config.ollanet.cleanModel || '').trim(),
|
|
252
|
+
host: String(meta?.machine || config.ollanet.machine || '').trim(),
|
|
253
|
+
promptVersion: CLEAN_PROMPT_VERSION,
|
|
254
|
+
dictawhisperVersion: dictawhisperVersion(),
|
|
255
|
+
});
|
|
256
|
+
transcriptionJson.tags = jsonCompletion.tags || [];
|
|
257
|
+
if (thinking) transcriptionJson.thinking = thinking;
|
|
258
|
+
if (meta) transcriptionJson.meta = meta;
|
|
259
|
+
delete transcriptionJson.cleanupError;
|
|
260
|
+
delete transcriptionJson.cleanupSkipped;
|
|
261
|
+
ensurePlaybackCues(transcriptionJson);
|
|
262
|
+
fs.writeFileSync(transcriptionFile, JSON.stringify(transcriptionJson, null, 2), { encoding: 'utf-8' });
|
|
263
|
+
console.log(`[clean-transcription] Cleaned transcription file: ${transcriptionFile}`);
|
|
264
|
+
callback(null, transcriptionJson.cleanedTranscription);
|
|
265
|
+
} catch (error: any) {
|
|
266
|
+
const detail = describeCleanError(error);
|
|
267
|
+
console.error(`[clean-transcription-error] ${transcriptionFile}: ${detail}`);
|
|
268
|
+
recordCleanupFailure(transcriptionFile, error);
|
|
269
|
+
callback(error);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function recordCleanupFailure(transcriptionFile: string, error: unknown) {
|
|
274
|
+
try {
|
|
275
|
+
if (!fs.existsSync(transcriptionFile)) return;
|
|
276
|
+
const json = JSON.parse(fs.readFileSync(transcriptionFile, 'utf-8'));
|
|
277
|
+
json.cleanupError = describeCleanError(error);
|
|
278
|
+
json.cleanupAttempts = (Number(json.cleanupAttempts) || 0) + 1;
|
|
279
|
+
fs.writeFileSync(transcriptionFile, JSON.stringify(json, null, 2), { encoding: 'utf-8' });
|
|
280
|
+
} catch {
|
|
281
|
+
// sidecar write is best-effort
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
export function recordAudioFailure(transcriptionFile: string, error: unknown): void {
|
|
286
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
287
|
+
const json = (readSidecar(transcriptionFile) || {}) as TranscriptionDocument;
|
|
288
|
+
json.audioError = message.slice(0, 500);
|
|
289
|
+
json.cleanupSkipped = true;
|
|
290
|
+
fs.writeFileSync(transcriptionFile, JSON.stringify(json, null, 2), { encoding: 'utf-8' });
|
|
291
|
+
transcriptions[transcriptionFile] = json;
|
|
292
|
+
indexSidecar(transcriptionFile);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
export type ProcessOptions = {
|
|
296
|
+
force?: boolean;
|
|
297
|
+
retry?: boolean;
|
|
298
|
+
settleMs?: number;
|
|
299
|
+
/** Put this job ahead of the remaining backlog. */
|
|
300
|
+
front?: boolean;
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
export async function process(file: string, options: ProcessOptions = {}) {
|
|
304
|
+
requestWhenSettled(file, () => enqueueTranscription(file, options), {
|
|
305
|
+
force: options.force,
|
|
306
|
+
retry: options.retry,
|
|
307
|
+
settleMs: options.settleMs,
|
|
308
|
+
label: 'voice-transcribe',
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function enqueueProcessing(file: string, elapsed?: string) {
|
|
313
|
+
const { transcriptionFile } = checkTranscription(file);
|
|
314
|
+
const sidecar = readSidecar(transcriptionFile);
|
|
315
|
+
if (sidecarIsProcessed(sidecar)) {
|
|
316
|
+
emitTranscription(null, transcriptionFile, elapsed ?? null);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
if (!String(sidecar?.text || '').trim()) {
|
|
320
|
+
emitTranscription(null, transcriptionFile, elapsed ?? null);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
if (!ollanetIsConfigured()) {
|
|
324
|
+
if (config.ollanet.required) {
|
|
325
|
+
recordCleanupFailure(transcriptionFile, new Error('ollanet is required but machine/model are not configured'));
|
|
326
|
+
} else {
|
|
327
|
+
console.log(`[process] ollanet not configured; leaving raw transcript ${transcriptionFile}`);
|
|
328
|
+
}
|
|
329
|
+
emitTranscription(null, transcriptionFile, elapsed ?? null);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
if (!q['processing']) {
|
|
333
|
+
emitTranscription(null, transcriptionFile, elapsed ?? null);
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
console.log(`[process] Adding file to processing queue: ${file}`);
|
|
337
|
+
q['processing'].push({ file }, () => {
|
|
338
|
+
emitTranscription(null, transcriptionFile, elapsed ?? null);
|
|
339
|
+
});
|
|
340
|
+
emitTranscription(null, transcriptionFile, elapsed ?? null);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export function skipCleanup(file: string): void {
|
|
344
|
+
const jsonFile = getTranscriptionFilename(file);
|
|
345
|
+
const json = readSidecar(jsonFile) || {};
|
|
346
|
+
json.cleanupSkipped = true;
|
|
347
|
+
json.cleanupError = json.cleanupError || 'skipped';
|
|
348
|
+
fs.writeFileSync(jsonFile, JSON.stringify(json, null, 2), { encoding: 'utf-8' });
|
|
349
|
+
transcriptions[jsonFile] = json;
|
|
350
|
+
emitTranscription(null, jsonFile);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function enqueueTranscription(file: string, options: ProcessOptions = {}) {
|
|
354
|
+
const { isProcessed, transcriptionFile, transcriptionExists } = checkTranscription(file);
|
|
355
|
+
if (!fs.existsSync(file)) {
|
|
356
|
+
console.error(`[process-error] File does not exist: ${file}`);
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
if (transcriptionExists && !options.retry) {
|
|
361
|
+
if (!isProcessed) enqueueProcessing(file);
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
console.log(`[process] Adding file to transcription queue: ${file}`);
|
|
366
|
+
const enqueue = options.front ? q['transcription'].unshift.bind(q['transcription']) : q['transcription'].push.bind(q['transcription']);
|
|
367
|
+
enqueue(
|
|
368
|
+
{ file, transcriptionFile, transcriptionFolder: path.dirname(transcriptionFile) },
|
|
369
|
+
(_err: any, result?: any) => {
|
|
370
|
+
if (_err instanceof Error) return;
|
|
371
|
+
if (_err?.err) return;
|
|
372
|
+
const elapsed = result?.elapsed ?? result?.result?.elapsed ?? _err?.result?.elapsed;
|
|
373
|
+
enqueueProcessing(file, elapsed);
|
|
374
|
+
}
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export function initTranscriptionWatcher(
|
|
379
|
+
watchFolder: null | string = null,
|
|
380
|
+
options: { watchDepth?: number; settleMs?: number } = {}
|
|
381
|
+
) {
|
|
382
|
+
if (!watchFolder) {
|
|
383
|
+
console.error('[watcher-error] No watch folder specified');
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
const watchDepth = options.watchDepth ?? 2;
|
|
387
|
+
new Watcher({
|
|
388
|
+
watchFolder,
|
|
389
|
+
watchDepth,
|
|
390
|
+
ignoreCheck: (filePath) => {
|
|
391
|
+
return (
|
|
392
|
+
isSkippedWatchPath(filePath) ||
|
|
393
|
+
filePath.includes('archive') ||
|
|
394
|
+
filePath.includes('original') ||
|
|
395
|
+
filePath.includes('_clean')
|
|
396
|
+
);
|
|
397
|
+
},
|
|
398
|
+
fileMatchRegex: audioFileRegex,
|
|
399
|
+
addHandler: async (filePath) => {
|
|
400
|
+
const { isProcessed, transcriptionExists } = checkTranscription(filePath);
|
|
401
|
+
if (transcriptionExists && isProcessed) return;
|
|
402
|
+
await process(filePath, { settleMs: options.settleMs });
|
|
403
|
+
},
|
|
404
|
+
changeHandler: (filePath) => {
|
|
405
|
+
const { isProcessed, transcriptionExists } = checkTranscription(filePath);
|
|
406
|
+
if (transcriptionExists && isProcessed) return;
|
|
407
|
+
void process(filePath, { settleMs: options.settleMs });
|
|
408
|
+
},
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
export function loadExistingTranscriptions(roots: string[]) {
|
|
413
|
+
let loaded = 0;
|
|
414
|
+
const walk = (dir: string) => {
|
|
415
|
+
if (!fs.existsSync(dir)) return;
|
|
416
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
417
|
+
const full = path.join(dir, entry.name);
|
|
418
|
+
if (entry.isDirectory()) {
|
|
419
|
+
if (isSkippedWatchPath(full)) continue;
|
|
420
|
+
walk(full);
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
if (!entry.name.toLowerCase().endsWith('.json')) continue;
|
|
424
|
+
if (isSkippedWatchPath(full) || full.includes('_original') || full.includes('_clean')) continue;
|
|
425
|
+
try {
|
|
426
|
+
const json = JSON.parse(fs.readFileSync(full, 'utf-8'));
|
|
427
|
+
if (!json?.text && !json?.cleanedTranscription && !Array.isArray(json?.segments)) continue;
|
|
428
|
+
transcriptions[full] = json;
|
|
429
|
+
loaded += 1;
|
|
430
|
+
} catch {
|
|
431
|
+
// skip unreadable sidecars
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
for (const root of roots) walk(root);
|
|
437
|
+
const indexBytes = Buffer.byteLength(JSON.stringify({ notes: listNoteSummaries() }));
|
|
438
|
+
console.log(
|
|
439
|
+
`[load-transcriptions] loaded ${loaded} sidecar notes from disk (~${(indexBytes / 1024 / 1024).toFixed(1)}MB index)`
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
export function initTranscriptionForSourceFolders(sourceFolders: string[] = []) {
|
|
444
|
+
console.log(
|
|
445
|
+
`[watch-source-folders] skipped separate transcribe watch; voice pipeline owns ${sourceFolders.join(', ')}`
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
export function countStatus(roots: string[]) {
|
|
450
|
+
let pendingAudio = 0;
|
|
451
|
+
let rawOnly = 0;
|
|
452
|
+
let done = 0;
|
|
453
|
+
let unreadable = 0;
|
|
454
|
+
|
|
455
|
+
const walk = (dir: string) => {
|
|
456
|
+
if (!fs.existsSync(dir)) return;
|
|
457
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
458
|
+
const full = path.join(dir, entry.name);
|
|
459
|
+
if (entry.isDirectory()) {
|
|
460
|
+
if (isSkippedWatchPath(full)) continue;
|
|
461
|
+
walk(full);
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
if (!audioFileRegex.test(full)) continue;
|
|
465
|
+
if (full.includes('_original') || full.includes('_clean')) continue;
|
|
466
|
+
const { transcriptionExists, transcriptionFile } = checkTranscription(full);
|
|
467
|
+
if (!transcriptionExists) {
|
|
468
|
+
pendingAudio += 1;
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
try {
|
|
472
|
+
const json = JSON.parse(fs.readFileSync(transcriptionFile, 'utf-8'));
|
|
473
|
+
if (json.audioError) unreadable += 1;
|
|
474
|
+
else if (json.cleanedTranscription) done += 1;
|
|
475
|
+
else rawOnly += 1;
|
|
476
|
+
} catch {
|
|
477
|
+
rawOnly += 1;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
};
|
|
481
|
+
|
|
482
|
+
for (const root of roots) walk(root);
|
|
483
|
+
return { pendingAudio, rawOnly, done, unreadable };
|
|
484
|
+
}
|