dictawhisper 0.0.13 → 0.0.14

package/src/apiRoutes.ts

@@ -0,0 +1,464 @@
+import express from 'express';
+import multer from 'multer';
+import { process, cleanTranscription, countStatus, getTranscriptionFilename, listNoteSummaries, patchTranscription, readTranscription, skipCleanup, emitNotesIndex } from './lib/transcriptionLib.ts';
+import {
+  journalStats,
+  journalTags,
+  journalYears,
+  listInboxNotes,
+  notesIndexPayload,
+  searchJournalIndex,
+} from './lib/journalService.ts';
+import type { SearchMode, SearchSort } from './lib/journalIndexLib.ts';
+import { q } from './lib/queueLib.ts';
+import { config } from './config.ts';
+import { resolveWhisperModel } from './lib/whisperLib.ts';
+import { getHealthReport } from './lib/healthLib.ts';
+import { resolveAllowedPath } from './lib/pathAllowLib.ts';
+import { audioContentType, findAudioForSidecar, saveUploadedAudio } from './lib/audioLib.ts';
+import { contentDisposition } from './lib/downloadLib.ts';
+import { applyConsolidateGroups, buildConsolidatePlan } from './lib/tagConsolidateLib.ts';
+import { resolveHeldFile, type HoldingAction } from './lib/organizationLib.ts';
+import { getProbeJob, startProbeJob } from './lib/audioProbeLib.ts';
+
+const upload = multer({
+  storage: multer.memoryStorage(),
+  limits: { fileSize: 80 * 1024 * 1024 },
+});
+
+function queryList(value: unknown): string[] {
+  if (Array.isArray(value)) return value.flatMap((item) => queryList(item));
+  if (typeof value === 'string' && value.trim()) return [value.trim()];
+  return [];
+}
+
+function queryFlag(value: unknown): boolean {
+  return value === '1' || value === 'true' || value === true;
+}
+
+function requireAllowedFile(req: express.Request, res: express.Response): string | null {
+  const file = typeof req.body?.file === 'string' ? req.body.file.trim() : '';
+  if (!file) {
+    res.status(400).json({ error: 'POST { "file": "<path under a watch root>" }' });
+    return null;
+  }
+  const allowed = resolveAllowedPath(file);
+  if (!allowed.ok) {
+    res.status(403).json({ error: allowed.error });
+    return null;
+  }
+  return allowed.path;
+}
+
+export const apiRoutes = [
+  {
+    path: '/',
+    method: 'GET',
+    handler: (_req: express.Request, res: express.Response) => {
+      res.send('<h1>DictaWhisper</h1><p>Watch, transcribe, clean.</p>');
+    },
+  },
+  {
+    path: '/health',
+    method: 'GET',
+    handler: async (req: express.Request, res: express.Response) => {
+      const fresh = req.query.fresh === '1' || req.query.fresh === 'true';
+      const report = await getHealthReport(config, { mode: 'live', fresh });
+      res.status(report.ok ? 200 : 503).json({
+        ok: report.ok,
+        degraded: report.degraded,
+        host: config.http.host,
+        port: config.http.port,
+        whisper: resolveWhisperModel(),
+        whisperWorker: report.whisper.worker,
+        device: report.whisper.device,
+        ollanet: {
+          ...config.ollanet,
+          reachable: report.ollanet.reachable,
+        },
+        settleMinutes: config.watch.settleMinutes,
+        browserSettleMs: config.watch.browserSettleMs,
+        checks: report.checks,
+        queues: {
+          transcription: q['transcription']
+            ? { length: q['transcription'].length(), running: q['transcription'].running() }
+            : null,
+          processing: q['processing']
+            ? { length: q['processing'].length(), running: q['processing'].running() }
+            : null,
+        },
+      });
+    },
+  },
+  {
+    path: '/audio',
+    method: 'GET',
+    handler: (req: express.Request, res: express.Response) => {
+      const file = typeof req.query.file === 'string' ? req.query.file.trim() : '';
+      if (!file) {
+        res.status(400).json({ error: 'GET /audio?file=<path under a watch root>' });
+        return;
+      }
+      const allowed = resolveAllowedPath(file);
+      if (!allowed.ok) {
+        res.status(403).json({ error: allowed.error });
+        return;
+      }
+      let audioPath = allowed.path;
+      if (audioPath.toLowerCase().endsWith('.json')) {
+        const sibling = findAudioForSidecar(audioPath);
+        if (!sibling) {
+          res.status(404).json({ error: 'no audio file next to that sidecar' });
+          return;
+        }
+        audioPath = sibling;
+      }
+      const audioAllowed = resolveAllowedPath(audioPath);
+      if (!audioAllowed.ok) {
+        res.status(403).json({ error: audioAllowed.error });
+        return;
+      }
+      const asDownload = queryFlag(req.query.download);
+      res.setHeader('Content-Type', audioContentType(audioAllowed.path));
+      res.setHeader(
+        'Content-Disposition',
+        contentDisposition(audioAllowed.path, asDownload ? 'attachment' : 'inline'),
+      );
+      res.sendFile(audioAllowed.relative, { root: audioAllowed.root });
+    },
+  },
+  {
+    path: '/notes/index',
+    method: 'GET',
+    handler: (req: express.Request, res: express.Response) => {
+      try {
+        const year = typeof req.query.year === 'string' ? req.query.year.trim() : '';
+        const month = typeof req.query.month === 'string' ? req.query.month.trim() : '';
+        const unreadable = queryFlag(req.query.unreadable);
+        const starred = queryFlag(req.query.starred);
+        const all = queryFlag(req.query.all);
+        if (year || month || unreadable || starred || all) {
+          res.json({
+            notes: listInboxNotes({
+              year: year || undefined,
+              month: month || undefined,
+              unreadable: unreadable || undefined,
+              starred: starred || undefined,
+              all: all || undefined,
+            }),
+            paged: !all,
+            indexing: journalStats().indexing,
+          });
+          return;
+        }
+        res.json(notesIndexPayload(listNoteSummaries));
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        console.error(`[notes-index] failed: ${detail}`);
+        res.status(500).json({ error: 'failed to build notes index' });
+      }
+    },
+  },
+  {
+    path: '/notes/search',
+    method: 'GET',
+    handler: async (req: express.Request, res: express.Response) => {
+      try {
+        const query = typeof req.query.q === 'string' ? req.query.q : '';
+        const tags = queryList(req.query.tag);
+        const since = typeof req.query.since === 'string' ? req.query.since.trim() : '';
+        const until = typeof req.query.until === 'string' ? req.query.until.trim() : '';
+        const modeRaw = typeof req.query.mode === 'string' ? req.query.mode.trim() : '';
+        const mode = modeRaw === 'lex' || modeRaw === 'semantic' || modeRaw === 'hybrid' ? (modeRaw as SearchMode) : undefined;
+        const sortRaw = typeof req.query.sort === 'string' ? req.query.sort.trim() : '';
+        const sort = sortRaw === 'recent' || sortRaw === 'oldest' || sortRaw === 'relevance' ? (sortRaw as SearchSort) : undefined;
+        const year = typeof req.query.year === 'string' ? req.query.year.trim() : '';
+        const month = typeof req.query.month === 'string' ? req.query.month.trim() : '';
+        const limit = Number(req.query.limit);
+        const hits = await searchJournalIndex({
+          query,
+          tags,
+          since: since || undefined,
+          until: until || undefined,
+          year: year || undefined,
+          month: month || undefined,
+          mode,
+          sort,
+          limit: Number.isFinite(limit) ? limit : undefined,
+          unreadable: queryFlag(req.query.unreadable),
+          starred: queryFlag(req.query.starred),
+        });
+        res.json({ hits, count: hits.length });
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        console.error(`[notes-search] failed: ${detail}`);
+        res.status(500).json({ error: 'search failed' });
+      }
+    },
+  },
+  {
+    path: '/notes/years',
+    method: 'GET',
+    handler: (_req: express.Request, res: express.Response) => {
+      try {
+        res.json({ years: journalYears() });
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        res.status(500).json({ error: detail });
+      }
+    },
+  },
+  {
+    path: '/notes/tags',
+    method: 'GET',
+    handler: (req: express.Request, res: express.Response) => {
+      try {
+        res.json({
+          tags: journalTags({
+            includeSingletons: queryFlag(req.query.includeSingletons),
+            limit: Number(req.query.limit) || undefined,
+          }),
+        });
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        res.status(500).json({ error: detail });
+      }
+    },
+  },
+  {
+    path: '/notes/stats',
+    method: 'GET',
+    handler: (_req: express.Request, res: express.Response) => {
+      try {
+        res.json(journalStats());
+      } catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        res.status(500).json({ error: detail });
+      }
+    },
+  },
+  {
+    path: '/note',
+    method: 'GET',
+    handler: (req: express.Request, res: express.Response) => {
+      const file = typeof req.query.file === 'string' ? req.query.file.trim() : '';
+      if (!file) {
+        res.status(400).json({ error: 'GET /note?file=<sidecar path under a watch root>' });
+        return;
+      }
+      const allowed = resolveAllowedPath(file);
+      if (!allowed.ok) {
+        res.status(403).json({ error: allowed.error });
+        return;
+      }
+      const sidecar = allowed.path.toLowerCase().endsWith('.json')
+        ? allowed.path
+        : getTranscriptionFilename(allowed.path);
+      const sidecarAllowed = resolveAllowedPath(sidecar);
+      if (!sidecarAllowed.ok) {
+        res.status(403).json({ error: sidecarAllowed.error });
+        return;
+      }
+      if (queryFlag(req.query.download)) {
+        res.setHeader('Content-Type', 'application/json; charset=utf-8');
+        res.setHeader('Content-Disposition', contentDisposition(sidecarAllowed.path, 'attachment'));
+        res.sendFile(sidecarAllowed.relative, { root: sidecarAllowed.root });
+        return;
+      }
+      try {
+        const note = readTranscription(sidecarAllowed.path);
+        if (!note) {
+          res.status(404).json({ error: 'note not found' });
+          return;
+        }
+        res.json(note);
+      } catch (error: any) {
+        res.status(500).json({ error: error?.message || 'failed to read note' });
+      }
+    },
+  },
+  {
+    path: '/note',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      const file = requireAllowedFile(req, res);
+      if (!file) return;
+      const jsonFile = file.toLowerCase().endsWith('.json') ? file : getTranscriptionFilename(file);
+      const allowed = resolveAllowedPath(jsonFile);
+      if (!allowed.ok) {
+        res.status(403).json({ error: allowed.error });
+        return;
+      }
+      if (req.body?.tags === undefined && typeof req.body?.starred !== 'boolean') {
+        res.status(400).json({ error: 'POST { "file", "tags"?: string[], "starred"?: boolean }' });
+        return;
+      }
+      try {
+        res.json(
+          patchTranscription(allowed.path, {
+            tags: req.body?.tags,
+            starred: typeof req.body?.starred === 'boolean' ? req.body.starred : undefined,
+          }),
+        );
+      } catch (error: any) {
+        const status = String(error?.message || '').includes('not found') ? 404 : 500;
+        res.status(status).json({ error: error?.message || 'failed to update note' });
+      }
+    },
+  },
+  {
+    path: '/tools/probe',
+    method: 'GET',
+    handler: (_req: express.Request, res: express.Response) => {
+      res.json(getProbeJob());
+    },
+  },
+  {
+    path: '/tools/probe',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      const apply = Boolean(req.body?.apply);
+      const current = getProbeJob();
+      if (current.running) {
+        res.status(409).json(current);
+        return;
+      }
+      const started = startProbeJob([...config.watch.roots, config.watch.browserDropFolder], { apply });
+      res.status(202).json(started);
+    },
+  },
+  {
+    path: '/status',
+    method: 'GET',
+    handler: (_req: express.Request, res: express.Response) => {
+      res.json(countStatus([...config.watch.roots, config.watch.browserDropFolder]));
+    },
+  },
+  {
+    path: '/transcribe/force',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      const file = requireAllowedFile(req, res);
+      if (!file) return;
+      process(file, { retry: true });
+      res.json({ ok: true, file, retried: true });
+    },
+  },
+  {
+    path: '/process/force',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      const file = requireAllowedFile(req, res);
+      if (!file) return;
+      cleanTranscription(file, (err) => {
+        if (err) {
+          res.status(500).json({ ok: false, error: err.message });
+          return;
+        }
+        res.json({ ok: true, file, processed: true });
+      }, { reclean: true });
+    },
+  },
+  {
+    path: '/process/skip',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      const file = requireAllowedFile(req, res);
+      if (!file) return;
+      try {
+        skipCleanup(file);
+        res.json({ ok: true, file, skipped: true });
+      } catch (error: any) {
+        res.status(500).json({ error: error?.message || 'skip failed' });
+      }
+    },
+  },
+  {
+    path: '/audio',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      upload.fields([
+        { name: 'file', maxCount: 1 },
+        { name: 'audio', maxCount: 1 },
+      ])(req, res, (err: unknown) => {
+        if (err) {
+          res.status(400).json({ error: err instanceof Error ? err.message : 'upload failed' });
+          return;
+        }
+        const files = req.files as Record<string, Express.Multer.File[]> | undefined;
+        const uploaded = files?.file?.[0] || files?.audio?.[0];
+        if (!uploaded?.buffer) {
+          res.status(400).json({ error: 'POST multipart field "file" (or "audio")' });
+          return;
+        }
+        const clipName = typeof req.body?.clipName === 'string' ? req.body.clipName.trim() : '';
+        const saved = saveUploadedAudio(uploaded.buffer, uploaded.originalname, clipName || undefined);
+        void process(saved, { force: true });
+        res.json({ ok: true, file: saved });
+      });
+    },
+  },
+  {
+    path: '/holding/resolve',
+    method: 'POST',
+    handler: async (req: express.Request, res: express.Response) => {
+      const file = requireAllowedFile(req, res);
+      if (!file) return;
+      const action = String(req.body?.action || '').trim() as HoldingAction;
+      if (action !== 'overwrite' && action !== 'rename' && action !== 'unfile') {
+        res.status(400).json({ error: 'POST { "file", "action": "overwrite" | "rename" | "unfile" }' });
+        return;
+      }
+      try {
+        const dest = await resolveHeldFile(file, action);
+        void process(dest, { force: true });
+        emitNotesIndex();
+        res.json({ ok: true, file: dest, action });
+      } catch (error: any) {
+        res.status(500).json({ error: error?.message || 'holding resolve failed' });
+      }
+    },
+  },
+  {
+    path: '/tags/consolidate/preview',
+    method: 'POST',
+    handler: async (req: express.Request, res: express.Response) => {
+      try {
+        const useModel = req.body?.useModel !== false;
+        const plan = await buildConsolidatePlan({ useModel });
+        res.json(plan);
+      } catch (error: any) {
+        res.status(500).json({ error: error?.message || 'tag preview failed' });
+      }
+    },
+  },
+  {
+    path: '/tags/consolidate/apply',
+    method: 'POST',
+    handler: (req: express.Request, res: express.Response) => {
+      const groups = Array.isArray(req.body?.groups) ? req.body.groups : null;
+      if (!groups) {
+        res.status(400).json({ error: 'POST { "groups": [{ "keep": "tag", "drop": ["alias"] }] }' });
+        return;
+      }
+      const cleaned: { keep: string; drop: string[] }[] = [];
+      for (const group of groups) {
+        const keep = typeof group?.keep === 'string' ? group.keep.trim() : '';
+        const drop = Array.isArray(group?.drop)
+          ? group.drop.map((tag: unknown) => String(tag || '').trim()).filter(Boolean)
+          : [];
+        if (!keep || !drop.length) continue;
+        cleaned.push({ keep, drop });
+      }
+      if (!cleaned.length) {
+        res.status(400).json({ error: 'no merge groups to apply' });
+        return;
+      }
+      try {
+        res.json(applyConsolidateGroups(cleaned));
+      } catch (error: any) {
+        res.status(500).json({ error: error?.message || 'tag apply failed' });
+      }
+    },
+  },
+];

package/src/classes/Queue.ts

@@ -0,0 +1,77 @@
+import async from 'async';
+
+export type QueueCallback = (err: any, result?: any) => void;
+
+export type QueueConfig = {
+  processor: (task: any, callback: QueueCallback) => void;
+  concurrency: number;
+  active: boolean;
+  onDrain?: () => void;
+  onError?: (err: any, task: any) => void;
+};
+
+export class Queue {
+  private queue: async.QueueObject<any>;
+  private config: QueueConfig;
+  private name: string;
+
+  constructor(name: string, config: QueueConfig) {
+    this.config = config;
+    this.name = name;
+    this.queue = this.init();
+
+  }
+
+  init(): async.QueueObject<any> {
+    const queue = async.queue((task, callback) => {
+      let settled = false;
+      const done: QueueCallback = (err, result) => {
+        if (settled) return;
+        settled = true;
+        (callback as QueueCallback)(err, result);
+      };
+      console.log(`[queue-${this.name}] Processing task (${this.queue.length() + 1} remain in queue):`, task);
+      try {
+        const ret = this.config.processor(task, done);
+        Promise.resolve(ret).catch((err) => done(err));
+      } catch (err) {
+        done(err);
+      }
+    }, this.config.concurrency);
+
+    // Default or custom drain handler
+    queue.drain(this.config.onDrain || (() => {
+      console.log(`[queue] All tasks in ${this.name} queue have been processed.`);
+    }));
+
+    // Default or custom error handler
+    queue.error(this.config.onError || ((err: any, task) => {
+      if (err?.err !== null) {
+        console.error(`[queue-error] Error processing task in ${this.name} queue:`, err, 'Task:', task);
+      } else {
+        console.log(`[queue-success] Successfully processed task in ${this.name} queue in ${err.result.elapsed}:`, task);
+      }
+    }));
+    return queue;
+  }
+
+  push(task: any, callback?: QueueCallback): void {
+    this.queue.push(task, callback as any);
+  }
+
+  unshift(task: any, callback?: QueueCallback): void {
+    this.queue.unshift(task, callback as any);
+  }
+
+  length(): number {
+    return this.queue.length();
+  }
+
+  running(): number {
+    return this.queue.running();
+  }
+
+  idle(): boolean {
+    return this.queue.idle();
+  }
+}

package/src/classes/Watcher.ts

@@ -0,0 +1,117 @@
+import chokidar, { FSWatcher } from 'chokidar';
+import fs from 'fs';
+import { compareAudioNewestFirst } from '../lib/audioLib.ts';
+
+type WatcherConfig = {
+  watchFolder: string,
+  watchDepth: number,
+  logEvents?: boolean,
+  ignoreCheck: (path: string, stats?: fs.Stats) => boolean,
+  fileMatchRegex: RegExp,
+  addHandler: (filePath: string, match: RegExpMatchArray, watchFolder: string) => Promise<void>
+  changeHandler?: (filePath: string, match: RegExpMatchArray, watchFolder: string) => void,
+  unlinkHandler?: (filePath: string) => void,
+  addDirHandler?: (filePath: string) => void,
+  unlinkDirHandler?: (filePath: string) => void,
+  errorHandler?: (error: Error) => void,
+  readyHandler?: () => void
+};
+
+const liveWatchers: Watcher[] = [];
+
+export async function closeAllWatchers(): Promise<void> {
+  const closing = liveWatchers.splice(0, liveWatchers.length);
+  await Promise.all(closing.map((watcher) => watcher.close()));
+}
+
+export class Watcher {
+  private watcher: FSWatcher;
+  private holdingInitial = true;
+  private pendingAdds: string[] = [];
+  logEvents: boolean = false;
+
+  constructor(config: WatcherConfig = {watchFolder: '', watchDepth: 0, logEvents: false, ignoreCheck: () => false, fileMatchRegex: new RegExp(''), addHandler: async () => {}}) {
+    this.watcher = chokidar.watch(config.watchFolder, { persistent: true, ignored: config.ignoreCheck, depth: config.watchDepth });
+    liveWatchers.push(this);
+
+    this.watcher
+      .on('add', async (filePath) => {
+        const match = filePath.match(config.fileMatchRegex);
+        if (!match) return;
+        if (this.logEvents) {
+          console.log(`[watcher] File ${filePath} has been added`);
+        }
+        if (this.holdingInitial) {
+          this.pendingAdds.push(filePath);
+          return;
+        }
+        try {
+          await config.addHandler(filePath, match, config.watchFolder);
+        } catch (error) {
+          console.error(`[watcher] addHandler failed for ${filePath}:`, error);
+        }
+      })
+      .on('change', (filePath) => {
+        const match = filePath.match(config.fileMatchRegex);
+        if (match) {
+          if (this.logEvents) {
+            console.log(`[watcher] File ${filePath} has been changed`);
+          }
+          try {
+            config.changeHandler?.(filePath, match, config.watchFolder);
+          } catch (error) {
+            console.error(`[watcher] changeHandler failed for ${filePath}:`, error);
+          }
+        }
+      })
+      .on('unlink', (filePath) => {
+        if (this.logEvents) {
+          console.log(`[watcher] File ${filePath} has been removed`);
+        }
+        config.unlinkHandler?.(filePath);
+      })
+      .on('addDir', (filePath) => {
+        if (this.logEvents) {
+          console.log(`[watcher] Directory ${filePath} has been added`);
+        }
+        config.addDirHandler?.(filePath);
+      })
+      .on('unlinkDir', (filePath) => {
+        if (this.logEvents) {
+          console.log(`[watcher] Directory ${filePath} has been removed`);
+        }
+        config.unlinkDirHandler?.(filePath);
+      })
+      .on('error', (error: any) => {
+        if (this.logEvents) {
+          console.log(`[watcher-error] Watcher error: ${error}`);
+        }
+        config.errorHandler?.(error);
+      })
+      .on('ready', () => {
+        void this.flushInitialAdds(config);
+      });
+  }
+
+  private async flushInitialAdds(config: WatcherConfig) {
+    this.holdingInitial = false;
+    const files = this.pendingAdds.splice(0).sort(compareAudioNewestFirst);
+    console.log(
+      `[watcher] Initial scan complete of ${config.watchFolder} (${files.length} files, newest first). Ready for changes`,
+    );
+    for (const filePath of files) {
+      const match = filePath.match(config.fileMatchRegex);
+      if (!match) continue;
+      try {
+        await config.addHandler(filePath, match, config.watchFolder);
+      } catch (error) {
+        console.error(`[watcher] addHandler failed for ${filePath}:`, error);
+      }
+    }
+    config.readyHandler?.();
+  }
+
+  close() {
+    return this.watcher.close();
+  }
+}