dictawhisper 0.0.13 → 0.0.14

package/src/lib/audioLib.ts

@@ -0,0 +1,202 @@
+import { spawn, spawnSync } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { config } from '../config.ts';
+
+export const audioExtensions = ['webm', 'mp3', 'm4a', 'wav', 'ogg'];
+
+export const audioFileRegex = new RegExp(`\\.(${audioExtensions.join('|')})$`, 'i');
+
+const AUDIO_STAMP =
+  /^(?:MTIME_)?(\d{4})-(\d{2})-(\d{2})(?:[_ T](\d{2})[-:](\d{2})[-:](\d{2}))?(Z)?/;
+
+/** Recording time from the basename, or mtime if the name is not dated. */
+export function audioRecencyMs(filePath: string): number {
+  const match = path.basename(filePath).match(AUDIO_STAMP);
+  if (match) {
+    const [, year, month, day, hour, minute, second, z] = match;
+    const y = Number(year);
+    const mo = Number(month) - 1;
+    const d = Number(day);
+    const h = Number(hour || 0);
+    const mi = Number(minute || 0);
+    const s = Number(second || 0);
+    const ms = z ? Date.UTC(y, mo, d, h, mi, s) : new Date(y, mo, d, h, mi, s).getTime();
+    if (!Number.isNaN(ms)) return ms;
+  }
+  try {
+    return fs.statSync(filePath).mtimeMs;
+  } catch {
+    return 0;
+  }
+}
+
+export function compareAudioNewestFirst(a: string, b: string): number {
+  return audioRecencyMs(b) - audioRecencyMs(a);
+}
+
+const AUDIO_TYPES: Record<string, string> = {
+  '.mp3': 'audio/mpeg',
+  '.webm': 'audio/webm',
+  '.m4a': 'audio/mp4',
+  '.wav': 'audio/wav',
+  '.ogg': 'audio/ogg',
+};
+
+export function audioContentType(filePath: string): string {
+  return AUDIO_TYPES[path.extname(filePath).toLowerCase()] || 'application/octet-stream';
+}
+
+/** Working audio next to a sidecar JSON (not `_original` / `_clean`). */
+export function findAudioForSidecar(jsonFile: string): string | null {
+  const parsed = path.parse(jsonFile);
+  const base = path.join(parsed.dir, parsed.name);
+  for (const ext of audioExtensions) {
+    const candidate = `${base}.${ext}`;
+    if (fs.existsSync(candidate)) return candidate;
+  }
+  for (const ext of audioExtensions) {
+    const candidate = `${base}_original.${ext}`;
+    if (fs.existsSync(candidate)) return candidate;
+  }
+  return null;
+}
+
+export function saveAudioFile(dataUrl: string, clipName: string): string {
+  const folder = config.watch.browserDropFolder;
+  fs.mkdirSync(folder, { recursive: true });
+  const safeName = (clipName || 'voice-recording').replace(/[<>:"/\\|?*]/g, '-');
+  const filePath = path.join(folder, `${safeName}.webm`);
+  const data = dataUrl.replace(/data:.*?;base64,/i, '');
+  console.log('[save-audio-file] Saving Audio File', { filePath, dataLength: data.length });
+  fs.writeFileSync(filePath, Buffer.from(data, 'base64'));
+  return filePath;
+}
+
+export function saveUploadedAudio(buffer: Buffer, originalName: string, clipName?: string): string {
+  const folder = config.watch.browserDropFolder;
+  fs.mkdirSync(folder, { recursive: true });
+  const ext = path.extname(originalName || '').toLowerCase() || '.webm';
+  const safeExt = audioExtensions.includes(ext.slice(1)) ? ext : '.webm';
+  const base = (clipName || path.basename(originalName || 'voice-recording', ext) || 'voice-recording').replace(
+    /[<>:"/\\|?*]/g,
+    '-'
+  );
+  const filePath = path.join(folder, `${base}${safeExt}`);
+  fs.writeFileSync(filePath, buffer);
+  console.log('[save-audio-file] saved upload', { filePath, bytes: buffer.length });
+  return filePath;
+}
+
+export type AudioProbe = {
+  ok: boolean;
+  reason: string;
+  durationSec?: number;
+  size: number;
+};
+
+const MIN_AUDIO_BYTES = 512;
+
+/** Cheap ffprobe: reject stubs, empty files, and containers with no duration. */
+export function probeAudioFile(filePath: string): AudioProbe {
+  if (!fs.existsSync(filePath)) return { ok: false, reason: 'missing', size: 0 };
+  const size = fs.statSync(filePath).size;
+  if (size < MIN_AUDIO_BYTES) return { ok: false, reason: `too small (${size} bytes)`, size };
+  const result = spawnSync(
+    'ffprobe',
+    ['-v', 'error', '-show_entries', 'format=duration', '-of', 'json', filePath],
+    { encoding: 'utf8', timeout: 15_000, windowsHide: true },
+  );
+  if (result.status !== 0) {
+    const line = String(result.stderr || result.stdout || 'ffprobe failed')
+      .trim()
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .pop();
+    return { ok: false, reason: line || 'ffprobe failed', size };
+  }
+  try {
+    const duration = Number(JSON.parse(result.stdout || '{}')?.format?.duration);
+    if (!Number.isFinite(duration) || duration <= 0) {
+      return { ok: false, reason: 'no duration', size };
+    }
+    return { ok: true, reason: `${duration.toFixed(1)}s`, durationSec: duration, size };
+  } catch {
+    return { ok: false, reason: 'ffprobe json', size };
+  }
+}
+
+function unlinkIfExists(filePath: string): void {
+  try {
+    if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+  } catch {
+    // leftover cleanup is best-effort
+  }
+}
+
+function runFfmpeg(args: string[]): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn('ffmpeg', args, { windowsHide: true });
+    let stderr = '';
+    child.stderr?.on('data', (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+    child.on('error', reject);
+    child.on('close', (code) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+      reject(new Error(`ffmpeg exited ${code}: ${stderr.slice(-400)}`));
+    });
+  });
+}
+
+/**
+ * Denoise to a working `.mp3`. Keeps `*_original` with the source bytes.
+ * Returns the path Whisper should read (may differ from `file` if the ext changed).
+ */
+export async function cleanAudioFile(file: string, cleanAgain = false): Promise<string> {
+  const parsed = path.parse(file);
+  const base = path.join(parsed.dir, parsed.name.replace(/_original$/i, ''));
+  const workingMp3 = `${base}.mp3`;
+  const originalFile = `${base}_original${parsed.ext}`;
+  const tmpMp3 = path.join(os.tmpdir(), `dicta-clean-${Date.now()}-${path.basename(base)}.mp3`);
+
+  unlinkIfExists(`${base}_clean${parsed.ext}`);
+  unlinkIfExists(`${base}_clean.mp3`);
+
+  if (fs.existsSync(originalFile) && fs.existsSync(workingMp3) && !cleanAgain) {
+    return workingMp3;
+  }
+
+  const source = fs.existsSync(originalFile) && cleanAgain ? originalFile : file;
+  const probe = probeAudioFile(source);
+  if (!probe.ok) {
+    throw new Error(`unreadable audio: ${probe.reason}`);
+  }
+  await runFfmpeg([
+    '-y',
+    '-i',
+    source,
+    '-af',
+    'silenceremove=1:0:-50dB,afftdn=nr=15:nf=-40,highpass=f=200,volume=1.5',
+    '-c:a',
+    'libmp3lame',
+    '-b:a',
+    '128k',
+    tmpMp3,
+  ]);
+
+  if (!fs.existsSync(originalFile)) {
+    fs.copyFileSync(file, originalFile);
+  }
+  fs.copyFileSync(tmpMp3, workingMp3);
+  unlinkIfExists(tmpMp3);
+  if (path.resolve(file) !== path.resolve(workingMp3)) {
+    unlinkIfExists(file);
+  }
+  console.log(`[ffmpeg] cleaned working file ${workingMp3}`);
+  return workingMp3;
+}

package/src/lib/audioProbeLib.ts

@@ -0,0 +1,112 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { audioFileRegex, probeAudioFile } from './audioLib.ts';
+import { isSkippedWatchPath } from './fileSettleLib.ts';
+import { checkTranscription, emitNotesIndex, recordAudioFailure } from './transcriptionLib.ts';
+
+export type ProbeHit = { file: string; reason: string };
+
+export type ProbeReport = {
+  audio: number;
+  pending: number;
+  bad: number;
+  marked: number;
+  files: ProbeHit[];
+};
+
+export type ProbeJob = {
+  running: boolean;
+  apply: boolean;
+  startedAt: string | null;
+  finishedAt: string | null;
+  scanned: number;
+  error: string | null;
+} & ProbeReport;
+
+const idleJob = (): ProbeJob => ({
+  running: false,
+  apply: false,
+  startedAt: null,
+  finishedAt: null,
+  scanned: 0,
+  error: null,
+  audio: 0,
+  pending: 0,
+  bad: 0,
+  marked: 0,
+  files: [],
+});
+
+let job: ProbeJob = idleJob();
+
+function walkAudio(dir: string, out: string[]) {
+  if (!fs.existsSync(dir)) return;
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (isSkippedWatchPath(full)) continue;
+      walkAudio(full, out);
+      continue;
+    }
+    if (!audioFileRegex.test(full)) continue;
+    if (full.includes('_original') || full.includes('_clean')) continue;
+    out.push(full);
+  }
+}
+
+export function listWatchAudio(roots: string[]): string[] {
+  const files: string[] = [];
+  for (const root of roots) walkAudio(root, files);
+  return files;
+}
+
+export function scanPendingAudio(roots: string[], options: { apply?: boolean } = {}): ProbeReport {
+  const apply = Boolean(options.apply);
+  const files = listWatchAudio(roots);
+  const hits: ProbeHit[] = [];
+  let pending = 0;
+  let marked = 0;
+  for (const file of files) {
+    const { isProcessed, transcriptionFile } = checkTranscription(file);
+    if (isProcessed) continue;
+    pending += 1;
+    const probe = probeAudioFile(file);
+    if (probe.ok) continue;
+    hits.push({ file, reason: probe.reason });
+    if (!apply) continue;
+    recordAudioFailure(transcriptionFile, new Error(`unreadable audio: ${probe.reason}`));
+    marked += 1;
+  }
+  if (apply && marked) emitNotesIndex();
+  return { audio: files.length, pending, bad: hits.length, marked, files: hits };
+}
+
+export function getProbeJob(): ProbeJob {
+  return { ...job, files: job.files.slice() };
+}
+
+export function startProbeJob(roots: string[], options: { apply?: boolean } = {}): ProbeJob {
+  if (job.running) return getProbeJob();
+  const apply = Boolean(options.apply);
+  job = { ...idleJob(), running: true, apply, startedAt: new Date().toISOString() };
+  setImmediate(() => {
+    try {
+      const report = scanPendingAudio(roots, { apply });
+      job = {
+        ...job,
+        ...report,
+        running: false,
+        finishedAt: new Date().toISOString(),
+        scanned: report.pending,
+      };
+    } catch (error) {
+      job = {
+        ...job,
+        running: false,
+        finishedAt: new Date().toISOString(),
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  });
+  return getProbeJob();
+}

package/src/lib/cleanupProvenanceLib.ts

@@ -0,0 +1,80 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import type { CleanupRecord, TranscriptionDocument } from '../types/transcription.ts';
+
+export const CLEANUP_HISTORY_CAP = 20;
+
+let cachedVersion = '';
+
+export function dictawhisperVersion(): string {
+  if (cachedVersion) return cachedVersion;
+  try {
+    const pkgPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../package.json');
+    const version = JSON.parse(fs.readFileSync(pkgPath, 'utf8')).version;
+    cachedVersion = typeof version === 'string' && version.trim() ? version.trim() : '0.0.0';
+  } catch {
+    cachedVersion = '0.0.0';
+  }
+  return cachedVersion;
+}
+
+function asRecord(value: unknown): CleanupRecord | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const row = value as CleanupRecord;
+  const text = String(row.text || '').trim();
+  if (!text) return null;
+  const out: CleanupRecord = { text };
+  if (typeof row.createdAt === 'string' && row.createdAt.trim()) out.createdAt = row.createdAt;
+  if (typeof row.model === 'string' && row.model.trim()) out.model = row.model;
+  if (typeof row.host === 'string' && row.host.trim()) out.host = row.host;
+  if (typeof row.promptVersion === 'number' && Number.isFinite(row.promptVersion)) {
+    out.promptVersion = row.promptVersion;
+  }
+  if (typeof row.dictawhisperVersion === 'string' && row.dictawhisperVersion.trim()) {
+    out.dictawhisperVersion = row.dictawhisperVersion;
+  }
+  return out;
+}
+
+/** Current structured record, or a stub from legacy `cleanedTranscription` + `meta`. */
+export function currentCleanupRecord(json: TranscriptionDocument): CleanupRecord | null {
+  const stored = asRecord(json.cleanup);
+  if (stored) return stored;
+  const text = String(json.cleanedTranscription || '').trim();
+  if (!text) return null;
+  const meta = json.meta && typeof json.meta === 'object' ? json.meta : {};
+  const stub: CleanupRecord = { text };
+  if (typeof meta.model === 'string' && meta.model.trim()) stub.model = meta.model;
+  if (typeof meta.machine === 'string' && meta.machine.trim()) stub.host = meta.machine;
+  if (typeof json.promptVersion === 'number' && Number.isFinite(json.promptVersion)) {
+    stub.promptVersion = json.promptVersion;
+  }
+  return stub;
+}
+
+export function applyCleanupProvenance(
+  json: TranscriptionDocument,
+  next: CleanupRecord
+): CleanupRecord {
+  const previous = currentCleanupRecord(json);
+  if (previous) {
+    const history = [previous, ...(Array.isArray(json.cleanupHistory) ? json.cleanupHistory : [])]
+      .map(asRecord)
+      .filter((row): row is CleanupRecord => Boolean(row))
+      .slice(0, CLEANUP_HISTORY_CAP);
+    json.cleanupHistory = history;
+  }
+  const cleanup: CleanupRecord = {
+    text: next.text,
+    createdAt: next.createdAt || new Date().toISOString(),
+    promptVersion: next.promptVersion,
+    dictawhisperVersion: next.dictawhisperVersion || dictawhisperVersion(),
+  };
+  if (next.model) cleanup.model = next.model;
+  if (next.host) cleanup.host = next.host;
+  json.cleanup = cleanup;
+  json.cleanedTranscription = next.text;
+  if (typeof next.promptVersion === 'number') json.promptVersion = next.promptVersion;
+  return cleanup;
+}

package/src/lib/downloadLib.ts

@@ -0,0 +1,8 @@
+import path from 'node:path';
+
+export function contentDisposition(filePath: string, kind: 'inline' | 'attachment' = 'inline'): string {
+  const base = path.basename(filePath).replace(/[\r\n"]/g, '_');
+  const fallback = base.replace(/[^\x20-\x7E]/g, '_');
+  const encoded = encodeURIComponent(base).replace(/[!'()*]/g, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`);
+  return `${kind}; filename="${fallback}"; filename*=UTF-8''${encoded}`;
+}

package/src/lib/fileSettleLib.ts

@@ -0,0 +1,213 @@
+import fs from 'fs';
+import path from 'path';
+import { config } from '../config.ts';
+
+const pendingTimers = new Map<string, ReturnType<typeof setTimeout>>();
+const started = new Set<string>();
+
+const RETRY_SLACK_MS = 2000;
+
+export function getSettleMs(explicit?: number): number {
+  if (typeof explicit === 'number' && Number.isFinite(explicit) && explicit >= 0) return explicit;
+  const msRaw = process.env.VOICE_SETTLE_MS?.trim();
+  if (msRaw) {
+    const parsed = Number(msRaw);
+    if (Number.isFinite(parsed) && parsed >= 0) return parsed;
+  }
+  return config.watch.settleMinutes * 60_000;
+}
+
+export function formatDuration(ms: number): string {
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  const seconds = Math.round(ms / 1000);
+  if (seconds < 90) return `${seconds}s`;
+  const minutes = Math.round(seconds / 60);
+  if (minutes < 90) return `${minutes}m`;
+  const hours = (minutes / 60).toFixed(1);
+  return `${hours}h`;
+}
+
+const SKIP_FOLDERS = new Set(['__inbox']);
+
+/** Syncthing / incomplete-transfer names we should never organize or transcribe. */
+export function isTransientSyncFile(filePath: string): boolean {
+  const name = path.basename(filePath);
+  const lower = filePath.replace(/\\/g, '/').toLowerCase();
+  return (
+    name.includes('sync-conflict') ||
+    name.includes('~syncthing~') ||
+    name.startsWith('.syncthing.') ||
+    name.endsWith('.tmp') ||
+    lower.includes('/.stfolder') ||
+    lower.includes('/.stversions/')
+  );
+}
+
+/** Staging folders that stay on disk but are out of the pipeline for now. */
+export function isSkippedWatchFolder(filePath: string): boolean {
+  return filePath
+    .replace(/\\/g, '/')
+    .split('/')
+    .some((part) => SKIP_FOLDERS.has(part.toLowerCase()));
+}
+
+export function isSkippedWatchPath(filePath: string): boolean {
+  return isTransientSyncFile(filePath) || isSkippedWatchFolder(filePath);
+}
+
+export type FileReadiness = {
+  ready: boolean;
+  reason: string;
+  ageMs: number;
+  waitMs: number;
+  size: number;
+  mtime: Date | null;
+};
+
+export function inspectFileReadiness(filePath: string, settleMs: number = getSettleMs()): FileReadiness {
+  if (isSkippedWatchPath(filePath)) {
+    return {
+      ready: false,
+      reason: 'skipped folder',
+      ageMs: 0,
+      waitMs: 0,
+      size: 0,
+      mtime: null,
+    };
+  }
+  if (!fs.existsSync(filePath)) {
+    return { ready: false, reason: 'missing', ageMs: 0, waitMs: 0, size: 0, mtime: null };
+  }
+
+  const stats = fs.statSync(filePath);
+  if (!stats.isFile()) {
+    return { ready: false, reason: 'not a file', ageMs: 0, waitMs: 0, size: 0, mtime: null };
+  }
+  if (stats.size <= 0) {
+    return {
+      ready: false,
+      reason: 'empty',
+      ageMs: 0,
+      waitMs: Math.max(settleMs, 10_000),
+      size: 0,
+      mtime: stats.mtime,
+    };
+  }
+
+  const ageMs = Date.now() - stats.mtimeMs;
+  if (settleMs > 0 && ageMs < settleMs) {
+    return {
+      ready: false,
+      reason: `mtime only ${formatDuration(ageMs)} old (need ${formatDuration(settleMs)} since last write)`,
+      ageMs,
+      waitMs: settleMs - ageMs + RETRY_SLACK_MS,
+      size: stats.size,
+      mtime: stats.mtime,
+    };
+  }
+
+  return {
+    ready: true,
+    reason: settleMs === 0 ? 'settle disabled' : `mtime stable for ${formatDuration(ageMs)}`,
+    ageMs,
+    waitMs: 0,
+    size: stats.size,
+    mtime: stats.mtime,
+  };
+}
+
+function fileKey(filePath: string): string {
+  return path.normalize(filePath);
+}
+
+export type SettleOptions = {
+  /** Skip the mtime wait. Does not re-run if this file already started this process. */
+  force?: boolean;
+  /** Skip the wait and allow a file that already ran this process to run again. */
+  retry?: boolean;
+  label?: string;
+  settleMs?: number;
+};
+
+/**
+ * Run `action` only after the file has not been written for VOICE_SETTLE_MINUTES
+ * (default 30). Syncthing updates mtime while a phone recording is still syncing,
+ * so this waits until the last write — not until first appearance.
+ *
+ * Pass `{ force: true }` to skip the wait. Pass `{ retry: true }` to skip the wait
+ * and clear the in-process "already started" latch. Reschedules if the file is written again.
+ */
+export function requestWhenSettled(
+  filePath: string,
+  action: () => void | Promise<void>,
+  options: SettleOptions = {}
+): void {
+  const label = options.label ?? 'settle';
+  const key = `${label}:${fileKey(filePath)}`;
+  const settleMs = options.settleMs ?? getSettleMs();
+
+  const clearTimer = () => {
+    const timer = pendingTimers.get(key);
+    if (timer) {
+      clearTimeout(timer);
+      pendingTimers.delete(key);
+    }
+  };
+
+  const run = () => {
+    if (started.has(key)) return;
+    started.add(key);
+    Promise.resolve(action()).catch((error) => {
+      started.delete(key);
+      console.error('[settle] action failed', label, filePath, error);
+    });
+  };
+
+  if (options.retry) {
+    started.delete(key);
+  }
+
+  if (options.force || options.retry) {
+    clearTimer();
+    console.log(
+      `[${label}] ${options.retry ? 'retry' : 'forced'}; skipping settle for ${filePath}`
+    );
+    run();
+    return;
+  }
+
+  if (started.has(key)) return;
+
+  const status = inspectFileReadiness(filePath, settleMs);
+  if (status.ready) {
+    clearTimer();
+    console.log(`[${label}] ready (${status.reason}, ${status.size} bytes): ${filePath}`);
+    run();
+    return;
+  }
+
+  if (status.waitMs <= 0) {
+    console.log(`[${label}] skipping ${filePath}: ${status.reason}`);
+    return;
+  }
+
+  clearTimer();
+  console.log(
+    `[${label}] waiting ${formatDuration(status.waitMs)} for ${filePath} (${status.reason}, ${status.size} bytes)`
+  );
+  pendingTimers.set(
+    key,
+    setTimeout(() => {
+      pendingTimers.delete(key);
+      requestWhenSettled(filePath, action, options);
+    }, status.waitMs)
+  );
+}
+
+export function logSettleConfig(label = 'settle'): void {
+  const settleMs = getSettleMs();
+  console.log(
+    `[${label}] phone/watch files ${settleMs === 0 ? 'immediately (settle disabled)' : `${formatDuration(settleMs)} after last write`}` +
+      ` (watch.settleMinutes; retry via POST /transcribe/force)`
+  );
+}

package/src/lib/fsLib.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+
+export async function moveFile(filePath: string, destPath: string): Promise<void> {
+  console.log(`[fs-move] Moving file from ${filePath} to ${destPath}`);
+  return new Promise((resolve, reject) => {
+    const fileExists = fs.existsSync(destPath);
+    // if the file exist in destPath, reject with an error
+    if (fileExists) {
+      const error = new Error(`Destination file already exists: ${destPath}`);
+      console.error(`[fs-move-error] ${error.message}`);
+      return resolve();
+    }
+    fs.copyFile(filePath, destPath, (err) => {
+      if (err) {
+        console.error('[fs-copy-error] copy failed', filePath, destPath, err);
+        reject(err);
+      } else {
+        fs.unlink(filePath, (unlinkErr) => {
+          if (unlinkErr) {
+            console.error('[fs-delete-error] unlink failed', filePath, unlinkErr);
+            reject(unlinkErr);
+          } else {
+            console.log(`[fs-delete] Moved ${filePath} to ${destPath}`);
+            resolve();
+          }
+        });
+      }
+    });
+  });
+}
+
+export function confirmFolder(folderPath: string): string {
+  if (!fs.existsSync(folderPath)) {
+    fs.mkdirSync(folderPath, { recursive: true });
+  }
+  return folderPath;
+}