dictawhisper 0.0.13 → 0.0.14

package/src/lib/journalQueryLib.ts

@@ -0,0 +1,269 @@
+import fs from 'fs';
+import path from 'path';
+import Fuse from 'fuse.js';
+import { isSkippedWatchPath } from './fileSettleLib.ts';
+
+const PREVIEW_LIMIT = 200;
+const FUSE_OPTS = {
+  keys: [
+    { name: 'tags', weight: 2 },
+    { name: 'searchBody', weight: 1.5 },
+    { name: 'basename', weight: 1 },
+    { name: 'searchRaw', weight: 0.6 },
+  ],
+  ignoreLocation: true,
+  threshold: 0.4,
+  includeScore: true,
+};
+
+export type JournalNote = {
+  jsonFile: string;
+  basename: string;
+  tags: string[];
+  preview: string;
+  searchBody: string;
+  searchRaw: string;
+  cleaned: string;
+  raw: string;
+  elapsed: string | null;
+  hasCleaned: boolean;
+  mtimeMs: number;
+  day: string;
+  cleanup: {
+    createdAt?: string;
+    model?: string;
+    host?: string;
+    promptVersion?: number;
+  } | null;
+};
+
+export type JournalHit = {
+  jsonFile: string;
+  basename: string;
+  day: string;
+  tags: string[];
+  preview: string;
+  score: number;
+};
+
+function previewOf(text: string): string {
+  const oneLine = text.replace(/\s+/g, ' ').trim();
+  if (oneLine.length <= PREVIEW_LIMIT) return oneLine;
+  return `${oneLine.slice(0, PREVIEW_LIMIT)}…`;
+}
+
+export function dayOf(jsonFile: string, basename: string, mtimeMs: number): string {
+  const named = basename.match(/^(\d{4}-\d{2}-\d{2})/);
+  if (named) return named[1];
+  const folded = jsonFile.replace(/\\/g, '/').match(/\/(\d{4})\/(\d{2})(?:\/|$)/);
+  if (folded) return `${folded[1]}-${folded[2]}-01`;
+  return new Date(mtimeMs).toISOString().slice(0, 10);
+}
+
+function isSidecarName(name: string): boolean {
+  return name.toLowerCase().endsWith('.json');
+}
+
+function looksLikeNote(json: unknown): json is Record<string, unknown> {
+  if (!json || typeof json !== 'object' || Array.isArray(json)) return false;
+  const row = json as Record<string, unknown>;
+  return Boolean(row.text || row.cleanedTranscription || row.audioError || Array.isArray(row.segments));
+}
+
+export function loadJournalNotes(roots: string[]): JournalNote[] {
+  const notes: JournalNote[] = [];
+  const seen = new Set<string>();
+
+  const walk = (dir: string) => {
+    if (!fs.existsSync(dir)) return;
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const full = path.resolve(path.join(dir, entry.name));
+      if (entry.isDirectory()) {
+        if (isSkippedWatchPath(full)) continue;
+        walk(full);
+        continue;
+      }
+      if (!isSidecarName(entry.name)) continue;
+      if (isSkippedWatchPath(full) || full.includes('_original') || full.includes('_clean')) continue;
+      if (seen.has(full)) continue;
+      try {
+        const json = JSON.parse(fs.readFileSync(full, 'utf8'));
+        if (!looksLikeNote(json)) continue;
+        const cleaned = String(json.cleanedTranscription || '').trim();
+        const raw = String(json.text || '').trim();
+        const source = cleaned || raw;
+        const tags = Array.isArray(json.tags)
+          ? json.tags.map((tag: unknown) => String(tag).trim()).filter(Boolean)
+          : [];
+        let mtimeMs = 0;
+        try {
+          mtimeMs = fs.statSync(full).mtimeMs;
+        } catch {
+          mtimeMs = 0;
+        }
+        const basename = path.basename(full);
+        const rawCleanup = json.cleanup;
+        const cleanupRow =
+          rawCleanup && typeof rawCleanup === 'object' && !Array.isArray(rawCleanup)
+            ? (rawCleanup as Record<string, unknown>)
+            : null;
+        const cleanup = cleanupRow
+          ? {
+              createdAt: typeof cleanupRow.createdAt === 'string' ? cleanupRow.createdAt : undefined,
+              model: typeof cleanupRow.model === 'string' ? cleanupRow.model : undefined,
+              host: typeof cleanupRow.host === 'string' ? cleanupRow.host : undefined,
+              promptVersion:
+                typeof cleanupRow.promptVersion === 'number' ? cleanupRow.promptVersion : undefined,
+            }
+          : null;
+        notes.push({
+          jsonFile: full,
+          basename,
+          tags,
+          preview: previewOf(source),
+          searchBody: source,
+          searchRaw: raw,
+          cleaned,
+          raw,
+          elapsed: json.elapsed != null ? String(json.elapsed) : null,
+          hasCleaned: Boolean(cleaned),
+          mtimeMs,
+          day: dayOf(full, basename, mtimeMs),
+          cleanup,
+        });
+        seen.add(full);
+      } catch {
+        // skip unreadable sidecars
+      }
+    }
+  };
+
+  for (const root of roots) walk(path.resolve(root));
+  return notes;
+}
+
+export function inDayRange(note: JournalNote, since?: string, until?: string): boolean {
+  if (since && note.day < since) return false;
+  if (until && note.day > until) return false;
+  return true;
+}
+
+function hasAllTags(note: JournalNote, tags: string[]): boolean {
+  if (!tags.length) return true;
+  const have = new Set(note.tags.map((tag) => tag.toLowerCase()));
+  return tags.every((tag) => have.has(tag.toLowerCase()));
+}
+
+function toHit(note: JournalNote, score = 0): JournalHit {
+  return {
+    jsonFile: note.jsonFile,
+    basename: note.basename,
+    day: note.day,
+    tags: note.tags,
+    preview: note.preview,
+    score,
+  };
+}
+
+export function searchJournal(
+  notes: JournalNote[],
+  options: { query?: string; tags?: string[]; since?: string; until?: string; limit?: number }
+): JournalHit[] {
+  const tags = (options.tags || []).map((tag) => String(tag).trim()).filter(Boolean);
+  const query = String(options.query || '').trim();
+  const limit = Math.min(50, Math.max(1, options.limit || 10));
+  const pool = notes.filter(
+    (note) => hasAllTags(note, tags) && inDayRange(note, options.since, options.until)
+  );
+  if (!query) {
+    return sortRecent(pool)
+      .slice(0, limit)
+      .map((note) => toHit(note));
+  }
+  const fuse = new Fuse(pool, FUSE_OPTS);
+  return fuse
+    .search(query)
+    .slice(0, limit)
+    .map((result) => toHit(result.item, result.score ?? 0));
+}
+
+export function sortRecent(notes: JournalNote[]): JournalNote[] {
+  return [...notes].sort((a, b) => {
+    if (a.day !== b.day) return b.day.localeCompare(a.day);
+    if (a.basename !== b.basename) return b.basename.localeCompare(a.basename);
+    return b.mtimeMs - a.mtimeMs;
+  });
+}
+
+export function recentJournal(
+  notes: JournalNote[],
+  options: { limit?: number; since?: string; until?: string; tags?: string[] } = {}
+): JournalHit[] {
+  const limit = Math.min(50, Math.max(1, options.limit || 20));
+  const tags = (options.tags || []).map((tag) => String(tag).trim()).filter(Boolean);
+  return sortRecent(
+    notes.filter((note) => hasAllTags(note, tags) && inDayRange(note, options.since, options.until))
+  )
+    .slice(0, limit)
+    .map((note) => toHit(note));
+}
+
+export function listJournalTags(
+  notes: JournalNote[],
+  options: { includeSingletons?: boolean; limit?: number } = {}
+): { tag: string; count: number }[] {
+  const counts = new Map<string, number>();
+  for (const note of notes) {
+    for (const tag of note.tags) {
+      counts.set(tag, (counts.get(tag) || 0) + 1);
+    }
+  }
+  const min = options.includeSingletons ? 1 : 2;
+  const limit = Math.min(500, Math.max(1, options.limit || 200));
+  return [...counts.entries()]
+    .filter(([, count]) => count >= min)
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .slice(0, limit)
+    .map(([tag, count]) => ({ tag, count }));
+}
+
+export function findJournalNote(notes: JournalNote[], file: string): JournalNote | null {
+  const wanted = String(file || '').trim();
+  if (!wanted) return null;
+  const norm = wanted.replace(/\\/g, '/').toLowerCase();
+  const exact = notes.find((note) => note.jsonFile.replace(/\\/g, '/').toLowerCase() === norm);
+  if (exact) return exact;
+  const base = path.basename(wanted).toLowerCase();
+  const matches = notes.filter((note) => note.basename.toLowerCase() === base);
+  if (matches.length === 1) return matches[0];
+  return null;
+}
+
+export function getJournalNote(
+  notes: JournalNote[],
+  file: string,
+  options: { includeRaw?: boolean } = {}
+):
+  | { ok: true; note: Record<string, unknown> }
+  | { ok: false; error: string } {
+  const found = findJournalNote(notes, file);
+  if (!found) return { ok: false, error: 'note not found under configured watch roots' };
+  const note: Record<string, unknown> = {
+    jsonFile: found.jsonFile,
+    basename: found.basename,
+    day: found.day,
+    tags: found.tags,
+    elapsed: found.elapsed,
+    hasCleaned: found.hasCleaned,
+    text: found.cleaned || found.raw,
+    cleanup: found.cleanup,
+  };
+  if (options.includeRaw) note.raw = found.raw;
+  return { ok: true, note };
+}

package/src/lib/journalService.ts

@@ -0,0 +1,287 @@
+import path from 'node:path';
+import { config } from '../config.ts';
+import {
+  getJournalIndex,
+  JournalIndex,
+  openJournalIndex,
+  setJournalIndex,
+  type IndexListOptions,
+  type IndexSearchHit,
+  type SearchMode,
+  type SearchSort,
+  isFilenameQuery,
+} from './journalIndexLib.ts';
+import { createEmbedClient, type EmbedClient } from './journalEmbedLib.ts';
+
+let embedClient: EmbedClient | null = null;
+let embedChain: Promise<void> = Promise.resolve();
+let rebuilding = false;
+
+export function journalIndexPath(): string {
+  return config.journal.index;
+}
+
+export function journalReady(): boolean {
+  return Boolean(getJournalIndex());
+}
+
+export function journalRebuilding(): boolean {
+  return rebuilding;
+}
+
+export function initJournalIndex(): JournalIndex {
+  const existing = getJournalIndex();
+  if (existing) return existing;
+  const index = openJournalIndex(journalIndexPath());
+  setJournalIndex(index);
+  return index;
+}
+
+export function requireJournal(): JournalIndex {
+  return getJournalIndex() || initJournalIndex();
+}
+
+export { getJournalIndex };
+
+export async function startJournalIndex(roots: string[]): Promise<void> {
+  const index = initJournalIndex();
+  const stats = index.stats();
+  if (stats.notes === 0) {
+    console.log('[journal-index] empty; rebuilding FTS from disk');
+    await rebuildJournalIndex(roots, { embed: false });
+    void ensureEmbeddings(index);
+    return;
+  }
+  console.log(
+    `[journal-index] ${stats.notes} notes (${stats.embedded} embedded) at ${stats.path}`,
+  );
+  void ensureEmbeddings(index);
+}
+
+export async function rebuildJournalIndex(
+  roots: string[],
+  options: { embed?: boolean } = {},
+): Promise<{ upserted: number; removed: number; embedded: number }> {
+  const index = requireJournal();
+  rebuilding = true;
+  try {
+    const result = index.rebuildFromRoots(roots);
+    console.log(`[journal-index] rebuilt upserted=${result.upserted} removed=${result.removed}`);
+    let embedded = 0;
+    if (options.embed !== false) {
+      embedded = await ensureEmbeddings(index);
+    }
+    return { ...result, embedded };
+  } finally {
+    rebuilding = false;
+  }
+}
+
+export function indexSidecar(jsonFile: string) {
+  try {
+    const index = getJournalIndex();
+    if (!index) return;
+    const result = index.upsertSidecar(jsonFile);
+    if (result?.changed && !result.skipEmbed) {
+      queueEmbed(index, result.rowid, jsonFile);
+    }
+  } catch (error) {
+    console.warn(`[journal-index] upsert failed: ${error instanceof Error ? error.message : error}`);
+  }
+}
+
+export function dropSidecar(jsonFile: string) {
+  try {
+    getJournalIndex()?.removeSidecar(jsonFile);
+  } catch (error) {
+    console.warn(`[journal-index] remove failed: ${error instanceof Error ? error.message : error}`);
+  }
+}
+
+function queueEmbed(index: JournalIndex, rowid: number, jsonFile: string) {
+  embedChain = embedChain
+    .then(async () => {
+      const client = await getEmbedClient();
+      if (!client) return;
+      const row = index.rowsNeedingEmbed().find((item) => item.rowid === rowid);
+      const text = row?.body;
+      if (!text) return;
+      await embedOne(index, client, rowid, text);
+    })
+    .catch((error) => {
+      console.warn(`[journal-index] embed ${jsonFile}: ${error instanceof Error ? error.message : error}`);
+    });
+}
+
+async function getEmbedClient(): Promise<EmbedClient | null> {
+  if (embedClient) return embedClient;
+  embedClient = await createEmbedClient();
+  if (embedClient) {
+    console.log(`[journal-index] ollama embed model ${embedClient.model} on ${embedClient.host}`);
+  }
+  return embedClient;
+}
+
+async function embedOne(index: JournalIndex, client: EmbedClient, rowid: number, text: string) {
+  const vector = await client.embed(text);
+  index.ensureVec(vector.length, client.model);
+  index.putEmbedding(rowid, vector);
+}
+
+export async function ensureEmbeddings(index: JournalIndex = requireJournal()): Promise<number> {
+  const client = await getEmbedClient();
+  if (!client) {
+    console.log('[journal-index] no ollama embedding model; FTS-only');
+    return 0;
+  }
+  const pending = index.rowsNeedingEmbed();
+  if (!pending.length) {
+    const stats = index.stats();
+    console.log(
+      `[journal-index] embeddings up to date (${stats.embedded} vectors, model ${stats.embedModel || 'none'})`,
+    );
+    return 0;
+  }
+  console.log(`[journal-index] embedding ${pending.length} notes with ${client.model}`);
+  let done = 0;
+  for (const row of pending) {
+    try {
+      await embedOneWithRetry(index, client, row.rowid, row.body, row.json_file);
+      done += 1;
+      if (done % 25 === 0) console.log(`[journal-index] embedded ${done}/${pending.length}`);
+    } catch (error) {
+      console.warn(
+        `[journal-index] embed failed ${row.json_file}: ${error instanceof Error ? error.message : error}`,
+      );
+      if (!isSqliteBusy(error)) break;
+    }
+  }
+  return done;
+}
+
+function isSqliteBusy(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /database is locked|SQLITE_BUSY/i.test(message);
+}
+
+async function embedOneWithRetry(
+  index: JournalIndex,
+  client: EmbedClient,
+  rowid: number,
+  text: string,
+  jsonFile: string,
+) {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= 6; attempt += 1) {
+    try {
+      await embedOne(index, client, rowid, text);
+      return;
+    } catch (error) {
+      lastError = error;
+      if (!isSqliteBusy(error) || attempt === 6) throw error;
+      const waitMs = 200 * attempt;
+      console.warn(
+        `[journal-index] sqlite busy on ${path.basename(jsonFile)}; retry ${attempt}/6 in ${waitMs}ms`,
+      );
+      await new Promise((resolve) => setTimeout(resolve, waitMs));
+    }
+  }
+  throw lastError;
+}
+
+export async function searchJournalIndex(options: {
+  query?: string;
+  tags?: string[];
+  since?: string;
+  until?: string;
+  year?: string;
+  month?: string;
+  mode?: SearchMode;
+  sort?: SearchSort;
+  limit?: number;
+  unreadable?: boolean;
+  starred?: boolean;
+}): Promise<IndexSearchHit[]> {
+  const index = requireJournal();
+  const query = String(options.query || '').trim();
+  const requested =
+    isFilenameQuery(query) ? 'lex' : options.mode || config.journal.search;
+  let queryEmbedding: number[] | null = null;
+  if (requested !== 'lex' && query) {
+    try {
+      const client = await getEmbedClient();
+      if (client) queryEmbedding = await client.embed(String(options.query));
+    } catch (error) {
+      console.warn(`[journal-index] query embed failed: ${error instanceof Error ? error.message : error}`);
+    }
+  }
+  return index.search({
+    ...options,
+    mode: requested,
+    queryEmbedding,
+    synonyms: config.whisper.promptTerms,
+  });
+}
+
+export function notesIndexReload() {
+  return { paged: true, reload: true as const, indexing: rebuilding };
+}
+
+export function listInboxNotes(options: IndexListOptions = {}) {
+  return requireJournal().list(options).map(toInboxSummary);
+}
+
+export function notesIndexPayload(fallback: () => unknown[]) {
+  const index = getJournalIndex();
+  if (index && index.stats().notes > 0) {
+    return { notes: index.list().map(toInboxSummary), paged: true, indexing: rebuilding };
+  }
+  return { notes: fallback(), paged: false, indexing: rebuilding };
+}
+
+export function journalYears() {
+  return requireJournal().years();
+}
+
+export function journalTags(options: { includeSingletons?: boolean; limit?: number } = {}) {
+  return requireJournal().tags(options);
+}
+
+export function journalStats() {
+  const index = requireJournal();
+  return { ...index.stats(), indexing: rebuilding, vec: index.hasVec() };
+}
+
+export function recentJournalIndex(options: {
+  limit?: number;
+  tags?: string[];
+  since?: string;
+  until?: string;
+} = {}) {
+  return requireJournal().recent(options);
+}
+
+export function toInboxSummary(hit: {
+  jsonFile: string;
+  basename: string;
+  day?: string;
+  tags: string[];
+  preview: string;
+  hasCleaned: boolean;
+  audioError: string | null;
+  starred?: boolean;
+}) {
+  return {
+    jsonFile: hit.jsonFile,
+    basename: hit.basename,
+    day: hit.day || '',
+    transcriptionJson: {
+      tags: hit.tags,
+      preview: hit.preview,
+      hasCleaned: hit.hasCleaned,
+      audioError: hit.audioError,
+      starred: Boolean(hit.starred),
+      _partial: true,
+    },
+  };
+}

package/src/lib/jsonLib.ts

@@ -0,0 +1,71 @@
+export async function parseJSON(rawJSON: unknown = '') {
+  if (rawJSON && typeof rawJSON === 'object') {
+    return rawJSON as Record<string, any>;
+  }
+
+  const rawJson = String(rawJSON ?? '')
+    .replaceAll(`\n",`, `",`)
+    .replace(/^\s*```[\w]*\s*\n?([\s\S]*?)\s*```/gm, '$1')
+    .trim();
+
+  try {
+    return JSON.parse(rawJson);
+  } catch (error: any) {
+    const firstObject = extractFirstJsonObject(rawJson);
+    if (firstObject && firstObject !== rawJson) {
+      try {
+        const parsed = JSON.parse(firstObject);
+        console.warn('[parse-json] ignored trailing content after the first JSON object');
+        return parsed;
+      } catch {
+        // fall through
+      }
+    }
+
+    if (String(error.message).includes(`"Expected ',' or '}`) || String(error.message).includes('Unterminated')) {
+      const lastBraceIndex = rawJson.lastIndexOf('}');
+      if (lastBraceIndex !== -1) {
+        return JSON.parse(rawJson.substring(0, lastBraceIndex + 1));
+      }
+    }
+
+    throw new Error(`Error parsing JSON response: ${error.message}`);
+  }
+}
+
+/** First balanced `{...}`, ignoring braces inside strings. */
+function extractFirstJsonObject(text: string): string | null {
+  const start = text.indexOf('{');
+  if (start === -1) return null;
+
+  let depth = 0;
+  let inString = false;
+  let escape = false;
+
+  for (let i = start; i < text.length; i += 1) {
+    const char = text[i];
+    if (inString) {
+      if (escape) {
+        escape = false;
+        continue;
+      }
+      if (char === '\\') {
+        escape = true;
+        continue;
+      }
+      if (char === '"') inString = false;
+      continue;
+    }
+    if (char === '"') {
+      inString = true;
+      continue;
+    }
+    if (char === '{') depth += 1;
+    else if (char === '}') {
+      depth -= 1;
+      if (depth === 0) return text.slice(start, i + 1);
+    }
+  }
+
+  return null;
+}

package/src/lib/localberthGet.ts

@@ -0,0 +1,14 @@
+import { spawnSync } from 'child_process';
+
+/** `localberth get <name>` — undefined if the CLI or lease is missing. */
+export function localberthGet(name: string): number | undefined {
+  const result = spawnSync('localberth', ['get', name], {
+    encoding: 'utf8',
+    timeout: 5000,
+    windowsHide: true,
+    shell: process.platform === 'win32',
+  });
+  if (result.status !== 0) return undefined;
+  const n = Number((result.stdout || '').trim());
+  return Number.isInteger(n) && n > 0 && n <= 65535 ? n : undefined;
+}