hafez 1.3.0 → 1.4.0

package/dist/db.js

@@ -4,14 +4,22 @@ import { readFileSync, statSync, existsSync, unlinkSync } from 'fs';
 import { join, basename } from 'path';
 import { parseFilePath, scanVaultDir, kindDir, INDEXED_KINDS } from './vault.js';
 import { ensureLocalExclude } from './knowledge-index.js';
-import { getNextActions, getBrief, WIKI_LINK_RE, findSection } from './document.js';
-import { parseSessionLog } from './document.js';
+import { WIKI_LINK_RE, documentForFile } from './document.js';
+import { ENTITY_STATUSES, ENTITY_TYPES, CONFIDENCE_LEVELS, VALID_SUBTYPES, DEFAULT_KNOWLEDGE_SUBTYPE } from './contracts.js';
+import { HafezError } from './types.js';
 // Bump this when indexing logic changes to force a full rebuild.
 // v5: wiki link harvesting (mention relations), parent link indexing, relation filter on queries.
 // v6: description column on items.
 // v7: resource column on items (entities only).
+// v8: per-kind structural section binding (documentFor) changes how brief,
+//     next-action, and session-log columns are derived from bodies.
+// v9: Document binds the kind-level section union (no type/subtype
+//     narrowing), changing section boundaries body columns derive from.
+// v10: re-indexing an externally changed file is delete-then-insert instead of
+//     INSERT OR REPLACE. Indexes written by v9 and earlier may hold ghost FTS
+//     rows (REPLACE never fired the delete trigger) and must be rebuilt, not reused.
 // Existing instances will fully rebuild the index on first access after upgrade.
-export const SCHEMA_VERSION = '7';
+export const SCHEMA_VERSION = '10';
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS items (
   slug               TEXT PRIMARY KEY,
@@ -86,13 +94,74 @@ const STOPWORDS = new Set([
 function hashContent(content) {
     return createHash('sha256').update(content).digest('hex');
 }
+// --- Query input validation ---
+//
+// Every enum-ish query opt is checked before it can influence (or fail to
+// influence) SQL. The mappers below only forward values they recognise, so an
+// unchecked typo used to be *dropped*: `--filter actvie` ran unfiltered and
+// exited 0. Validating here covers the CLI and library callers at once.
+const SORT_COL_MAP = {
+    name: 'i.name',
+    created: 'i.created',
+    last_touched: 'i.last_touched',
+    staleness: 'i.staleness_days',
+    reinforcement_count: 'i.reinforcement_count',
+    last_reinforced: 'i.last_reinforced',
+};
+const KNOWLEDGE_ONLY_SORTS = new Set(['reinforcement_count', 'last_reinforced']);
+const SORT_FIELDS = Object.keys(SORT_COL_MAP);
+const SORT_ORDERS = ['asc', 'desc'];
+const QUERY_FILTERS = [...ENTITY_STATUSES, 'all', 'stale', 'capture'];
+function assertEnum(value, valid, label) {
+    if (value === undefined || value === null)
+        return;
+    if (typeof value !== 'string' || !valid.includes(value)) {
+        throw new HafezError('VALIDATION_FAILED', `Invalid ${label}: "${String(value)}". Valid values: ${valid.join(', ')}.`);
+    }
+}
+function assertSortOrder(value) {
+    if (value === undefined || value === null)
+        return;
+    if (typeof value !== 'string' || !SORT_ORDERS.includes(value.toLowerCase())) {
+        throw new HafezError('VALIDATION_FAILED', `Invalid sort_order: "${String(value)}". Valid values: asc, desc.`);
+    }
+}
+function assertCount(value, label) {
+    if (value === undefined || value === null)
+        return;
+    if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+        throw new HafezError('VALIDATION_FAILED', `Invalid ${label}: "${String(value)}". Must be a non-negative integer.`);
+    }
+}
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
+function validateTimeOpts(opts) {
+    const fields = ['since', 'before', 'createdSince', 'createdBefore'];
+    for (const f of fields) {
+        if (opts?.[f] !== undefined && !ISO_DATE.test(opts[f])) {
+            throw new HafezError('VALIDATION_FAILED', `Invalid date format for ${f}: "${opts[f]}". Must be YYYY-MM-DD.`);
+        }
+    }
+}
+function validateListOpts(opts) {
+    assertEnum(opts?.sort_by, SORT_FIELDS, 'sort_by');
+    assertSortOrder(opts?.sort_order);
+    assertCount(opts?.limit, 'limit');
+    assertCount(opts?.offset, 'offset');
+}
 // --- Shared query functions (used by both write and readonly paths) ---
 function sanitizeFtsQuery(raw) {
-    const cleaned = raw.replace(/[^\w\s]/g, ' ');
+    // Unicode-aware: keep letters/digits/underscore in ANY script, strip
+    // punctuation. `[^\w\s]` was ASCII-only, so it erased whole non-Latin
+    // queries (日本語 → '') and the vault's non-Latin notes were unfindable.
+    // For ASCII input this class is identical to the old one.
+    const cleaned = raw.replace(/[^\p{L}\p{N}_\s]/gu, ' ');
     const tokens = cleaned.split(/\s+/).filter(t => t.length > 0);
     const filtered = tokens.filter(t => !STOPWORDS.has(t.toLowerCase()));
+    // All-stopword queries fall back to the raw tokens — which is how bare
+    // AND/OR/NOT reach fts5. Quoting makes every term a literal string, never
+    // an operator, and a trailing `*` still gives prefix matching.
     const terms = filtered.length === 0 ? tokens : filtered;
-    return { fts: terms.map(t => `${t}*`).join(' OR '), terms };
+    return { fts: terms.map(t => `"${t.replace(/"/g, '""')}"*`).join(' OR '), terms };
 }
 function queryItemsFn(db, opts) {
     const conditions = [];
@@ -102,7 +171,11 @@ function queryItemsFn(db, opts) {
         params.kind = opts.kind;
     }
     if (opts.status) {
-        conditions.push('i.status = @status');
+        // Status is an entity attribute — knowledge rows are indexed with a NULL
+        // status. In a kind-unset (unified) query the status filter narrows the
+        // entity side only; applying it to every row is what made `query --kind
+        // all` (filter defaults to active) return entities and nothing else.
+        conditions.push(opts.kind === 'entity' ? 'i.status = @status' : "(i.kind = 'knowledge' OR i.status = @status)");
         params.status = opts.status;
     }
     if (opts.type) {
@@ -175,24 +248,21 @@ function queryItemsFn(db, opts) {
         }
     }
     const where = conditions.length > 0 ? 'WHERE ' + conditions.join(' AND ') : '';
-    const knowledgeOnlySorts = new Set(['reinforcement_count', 'last_reinforced']);
-    const sortColMap = {
-        name: 'i.name',
-        created: 'i.created',
-        last_touched: 'i.last_touched',
-        staleness: 'i.staleness_days',
-        reinforcement_count: 'i.reinforcement_count',
-        last_reinforced: 'i.last_reinforced',
-    };
+    // sort_order / limit / offset are the three opts that reach SQL as text
+    // rather than bound params — check them at the interpolation site, so a
+    // caller that skips the read surface can't hand raw SQL to prepare().
+    assertSortOrder(opts.sort_order);
+    assertCount(opts.limit, 'limit');
+    assertCount(opts.offset, 'offset');
     let sortCol = 'i.last_touched';
     if (opts.sort_by) {
-        const isKnowledgeOnly = knowledgeOnlySorts.has(opts.sort_by);
+        const isKnowledgeOnly = KNOWLEDGE_ONLY_SORTS.has(opts.sort_by);
         const isEntityQuery = opts.kind === 'entity';
         if (isKnowledgeOnly && isEntityQuery) {
             sortCol = 'i.last_touched';
         }
-        else if (sortColMap[opts.sort_by]) {
-            sortCol = sortColMap[opts.sort_by];
+        else if (SORT_COL_MAP[opts.sort_by]) {
+            sortCol = SORT_COL_MAP[opts.sort_by];
         }
     }
     let sortOrder;
@@ -237,6 +307,18 @@ function queryItemsFn(db, opts) {
     }
     return { items: rows, total };
 }
+/** Every indexed document, in the shape index.md renders from. The vault-root
+ * index needs name/status/type/confidence/description/domain and the body (for
+ * the derived summary) — all of it already indexed, so regenerating index.md
+ * after a mutation costs one query instead of a second full-vault parse. */
+function indexEntriesFn(db) {
+    const rows = db.prepare('SELECT slug, kind, name, type, status, confidence, description, body FROM items').all();
+    const getDomains = db.prepare("SELECT target FROM links WHERE source = ? AND relation = 'domain'");
+    return rows.map(row => ({
+        ...row,
+        domain: getDomains.all(row.slug).map(d => d.target),
+    }));
+}
 function searchFn(db, query, kind) {
     const { fts: ftsQuery, terms } = sanitizeFtsQuery(query);
     if (!ftsQuery)
@@ -337,6 +419,17 @@ function getStatsFn(db) {
         recently_created,
     };
 }
+// Internal test seam — NOT public API (createHafez never exposes it). Fires
+// inside syncIfStale after the scan has read the vault and before the new
+// watermark is recorded, so the mid-scan-write race can be injected
+// deterministically instead of raced against the clock.
+// Recorded sync watermarks sit this far behind the scan-start instant: FS
+// timestamps and Date.now() are read from the same kernel clock but round
+// differently, and a write landing in that rounding gap must stay dirty.
+const WATERMARK_SLACK_MS = 10;
+let afterScanHook = null;
+/** @internal */
+export function __setAfterScanHook(fn) { afterScanHook = fn; }
 export function createIndex(vaultPath, opts) {
     const dbPath = join(vaultPath, '.hafez.db');
     const isReadOnly = opts?.readonly ?? false;
@@ -379,7 +472,7 @@ export function createIndex(vaultPath, opts) {
             }
             catch { /* DB may have been temporarily deleted during rebuild */ }
         }
-        return {
+        return withReadSurface({
             rebuild: () => { throw new Error('Index is read-only'); },
             syncIfStale,
             upsertFromFile: () => { throw new Error('Index is read-only'); },
@@ -387,11 +480,12 @@ export function createIndex(vaultPath, opts) {
             queryItems: (qopts) => queryItemsFn(db, qopts),
             search: (q, k) => searchFn(db, q, k),
             getStats: () => getStatsFn(db),
+            indexEntries: () => indexEntriesFn(db),
             close: () => { try {
                 db.close();
             }
             catch { } },
-        };
+        });
     }
     // --- Write-capable path ---
     // The db files are a disposable local cache — keep them out of the user's
@@ -469,8 +563,12 @@ export function createIndex(vaultPath, opts) {
         db.prepare("INSERT INTO meta (key, value) VALUES ('schema_version', ?)").run(SCHEMA_VERSION);
     }
     // Prepared statements
-    const upsertStmt = db.prepare(`
-    INSERT OR REPLACE INTO items
+    // Plain INSERT, never INSERT OR REPLACE: REPLACE's implicit delete does not
+    // fire the AFTER DELETE trigger (recursive_triggers is off), so the replaced
+    // row's terms would linger in the external-content FTS index as ghosts.
+    // Callers delete the row first — see indexFile.
+    const insertStmt = db.prepare(`
+    INSERT INTO items
       (slug, kind, name, type, status, parent, next_action, next_action_count, staleness_days,
        confidence, reinforcement_count, last_reinforced, created, last_touched, body, file_hash, subtype,
        description, resource, brief, session_log_count, last_session_date, last_session_type, last_session_summary)
@@ -507,11 +605,12 @@ export function createIndex(vaultPath, opts) {
             process.stderr.write(`Warning: skipping ${kind}/${slug}.md: missing required field(s): ${missing}\n`);
             return;
         }
-        const bodyActions = getNextActions(body);
+        const doc = documentForFile({ kind });
+        const bodyActions = doc.getNextActions(body);
         const nextAction = bodyActions.length > 0 ? bodyActions[0] : null;
         const nextActionCount = bodyActions.length;
-        const brief = getBrief(body)?.slice(0, 300) ?? null;
-        const logs = parseSessionLog(body);
+        const brief = doc.getBrief(body)?.slice(0, 300) ?? null;
+        const logs = doc.parseSessionLog(body);
         const lastLog = logs[0] ?? null;
         const item = {
             slug,
@@ -530,7 +629,7 @@ export function createIndex(vaultPath, opts) {
             last_touched: fm['last-touched'] ?? null,
             body,
             file_hash: hash,
-            subtype: kind === 'knowledge' ? (fm.subtype ?? 'insight') : null,
+            subtype: kind === 'knowledge' ? (fm.subtype ?? DEFAULT_KNOWLEDGE_SUBTYPE) : null,
             description: fm.description ?? null,
             resource: kind === 'entity' ? (fm.resource ?? null) : null,
             brief,
@@ -539,9 +638,12 @@ export function createIndex(vaultPath, opts) {
             last_session_type: lastLog?.type ?? null,
             last_session_summary: lastLog?.summary?.slice(0, 300) ?? null,
         };
-        // Upsert item and rebuild links
+        // Replace item and rebuild links. Delete-then-insert keeps the FTS index
+        // trigger-consistent for externally changed files (hand edit, git pull),
+        // the same path hafez's own mutations take via upsertFromFile.
+        deleteItemStmt.run(slug);
         deleteLinksStmt.run(slug);
-        upsertStmt.run(item);
+        insertStmt.run(item);
         // Index related links
         if (fm.related && Array.isArray(fm.related)) {
             for (const target of fm.related)
@@ -564,7 +666,7 @@ export function createIndex(vaultPath, opts) {
         // Harvest [[wiki links]] from body (excluding ## Related section)
         // Strip the Related section to avoid double-counting generated links
         let authoredBody = body;
-        const relatedSection = findSection(body, 'Related');
+        const relatedSection = doc.findSection(body, 'Related');
         if (relatedSection) {
             authoredBody = body.slice(0, relatedSection.start) + body.slice(relatedSection.end);
         }
@@ -584,6 +686,8 @@ export function createIndex(vaultPath, opts) {
         }
     }
     const rebuildTransaction = db.transaction(() => {
+        // Watermark the scan by when it STARTED (see syncIfStale).
+        const scanStart = Date.now() - WATERMARK_SLACK_MS;
         db.exec('DELETE FROM items');
         db.exec('DELETE FROM links');
         // Collect all vault slugs for wiki link target validation
@@ -601,7 +705,7 @@ export function createIndex(vaultPath, opts) {
             indexFile(file, 'knowledge', allVaultSlugs);
         }
         db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_sync', ?)")
-            .run(Date.now().toString());
+            .run(scanStart.toString());
     });
     function rebuild() {
         rebuildTransaction();
@@ -614,10 +718,19 @@ export function createIndex(vaultPath, opts) {
         }
         const lastSyncTime = parseInt(lastSync.value, 10);
         let changed = false;
+        // Watermark the scan by when it STARTED, not when it finished. A file
+        // written while the scan is in flight — after its read, before the record —
+        // would otherwise be stamped as already-indexed and skipped by every later
+        // sync, leaving it permanently stale. `>=` covers the same-millisecond
+        // case; a re-read of an unchanged file is a cheap hash compare. The slack
+        // absorbs FS-vs-Date.now clock rounding at the boundary — a file whose
+        // mtime rounds just below the instant the scan started must still read as
+        // dirty; the only cost is re-hashing files touched inside the window.
+        const scanStart = Date.now() - WATERMARK_SLACK_MS;
         const dirs = INDEXED_KINDS.map(kind => ({ dir: kindDir(kind), kind }));
         for (const { dir, kind } of dirs) {
             for (const file of scanVaultDir(join(vaultPath, dir))) {
-                if (statSync(file).mtimeMs > lastSyncTime) {
+                if (statSync(file).mtimeMs >= lastSyncTime) {
                     indexFile(file, kind);
                     changed = true;
                 }
@@ -637,9 +750,10 @@ export function createIndex(vaultPath, opts) {
                 changed = true;
             }
         }
+        afterScanHook?.();
         if (changed) {
             db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_sync', ?)")
-                .run(Date.now().toString());
+                .run(scanStart.toString());
         }
     }
     function upsertFromFile(slug, kind) {
@@ -659,11 +773,173 @@ export function createIndex(vaultPath, opts) {
     const count = db.prepare('SELECT COUNT(*) as c FROM items').get().c;
     if (count === 0)
         rebuild();
-    return {
+    return withReadSurface({
         rebuild, syncIfStale, upsertFromFile, removeItem,
         queryItems: (qopts) => queryItemsFn(db, qopts),
         search: (q, k) => searchFn(db, q, k),
         getStats: () => getStatsFn(db),
+        indexEntries: () => indexEntriesFn(db),
         close: () => db.close(),
+    });
+}
+function withReadSurface(core) {
+    const index = {
+        ...core,
+        queryEntities: (opts) => queryEntitiesFn(index, opts),
+        queryChildren: (parentSlug) => queryEntitiesFn(index, { parent: parentSlug }),
+        queryRelatedTo: (slug, relation) => queryRelatedToFn(index, slug, relation),
+        queryKnowledge: (opts) => queryKnowledgeFn(index, opts),
+        queryUnified: (opts) => queryUnifiedFn(index, opts),
+    };
+    return index;
+}
+function toQueryResult(slug, row) {
+    return {
+        slug,
+        name: row.name,
+        type: row.type,
+        status: row.status,
+        description: row.description ?? null,
+        resource: row.resource ?? null,
+        domain: row.domain ?? [],
+        last_touched: row.last_touched,
+        next_action: row.next_action ?? null,
+        next_action_count: row.next_action_count ?? 0,
+        parent: row.parent ?? null,
+        related: row.related ?? [],
+        staleness_days: row.staleness_days ?? null,
+        tags: row.tags ?? [],
+        brief: row.brief ?? null,
+        session_log_count: row.session_log_count ?? 0,
+        last_session_date: row.last_session_date ?? null,
+        last_session_type: row.last_session_type ?? null,
+        last_session_summary: row.last_session_summary ?? null,
+    };
+}
+function toKnowledgeResult(slug, row) {
+    return {
+        slug,
+        name: row.name,
+        description: row.description ?? null,
+        domain: row.domain ?? [],
+        confidence: row.confidence ?? 'observation',
+        related: row.related ?? [],
+        reinforcement_count: row.reinforcement_count ?? 0,
+        last_reinforced: row.last_reinforced ?? null,
+        last_touched: row.last_touched ?? null,
+    };
+}
+const STATUS_FILTERS = new Set(ENTITY_STATUSES);
+function mapEntityFilterOpts(qopts, opts) {
+    if (opts?.filter && STATUS_FILTERS.has(opts.filter))
+        qopts.status = opts.filter;
+    if (opts?.filter === 'stale')
+        qopts.filter = 'stale';
+    if (opts?.filter === 'capture')
+        qopts.filter = 'capture';
+    if (opts?.type)
+        qopts.type = opts.type;
+    if (opts?.parent)
+        qopts.parent = opts.parent;
+    if (opts?.domain)
+        qopts.domain = opts.domain;
+}
+function mapTimeOpts(qopts, opts) {
+    if (opts?.since)
+        qopts.since = opts.since;
+    if (opts?.before)
+        qopts.before = opts.before;
+    if (opts?.createdSince)
+        qopts.createdSince = opts.createdSince;
+    if (opts?.createdBefore)
+        qopts.createdBefore = opts.createdBefore;
+}
+function mapListOpts(qopts, opts) {
+    if (opts?.tags)
+        qopts.tags = opts.tags;
+    if (opts?.sort_by)
+        qopts.sort_by = opts.sort_by;
+    if (opts?.sort_order)
+        qopts.sort_order = opts.sort_order;
+    if (opts?.limit != null)
+        qopts.limit = opts.limit;
+    if (opts?.offset != null)
+        qopts.offset = opts.offset;
+}
+function mapKnowledgeFilterOpts(qopts, opts) {
+    if (opts?.domain)
+        qopts.domain = opts.domain;
+    if (opts?.confidence)
+        qopts.confidence = opts.confidence;
+    if (opts?.subtype)
+        qopts.subtype = opts.subtype;
+}
+function validateEntityFilterOpts(opts) {
+    assertEnum(opts?.filter, QUERY_FILTERS, 'filter');
+    assertEnum(opts?.type, ENTITY_TYPES, 'type');
+}
+function validateKnowledgeFilterOpts(opts) {
+    assertEnum(opts?.confidence, CONFIDENCE_LEVELS, 'confidence');
+    assertEnum(opts?.subtype, VALID_SUBTYPES, 'subtype');
+}
+function queryEntitiesFn(index, opts) {
+    index.syncIfStale();
+    validateEntityFilterOpts(opts);
+    validateKnowledgeFilterOpts(opts);
+    validateTimeOpts(opts);
+    validateListOpts(opts);
+    const qopts = { kind: 'entity' };
+    mapEntityFilterOpts(qopts, opts);
+    mapTimeOpts(qopts, opts);
+    mapListOpts(qopts, opts);
+    const { items, total } = index.queryItems(qopts);
+    return { items: items.map(i => toQueryResult(i.slug, i)), total };
+}
+function queryRelatedToFn(index, slug, relation) {
+    index.syncIfStale();
+    const { items, total } = index.queryItems({ relatedTo: slug, relation });
+    return {
+        items: items.map(i => i.kind === 'entity'
+            ? { ...toQueryResult(i.slug, i), kind: 'entity' }
+            : { ...toKnowledgeResult(i.slug, i), kind: 'knowledge' }),
+        total,
     };
 }
+function queryKnowledgeFn(index, opts) {
+    index.syncIfStale();
+    validateKnowledgeFilterOpts(opts);
+    validateTimeOpts(opts);
+    validateListOpts(opts);
+    const qopts = { kind: 'knowledge' };
+    mapKnowledgeFilterOpts(qopts, opts);
+    mapTimeOpts(qopts, opts);
+    mapListOpts(qopts, opts);
+    const { items, total } = index.queryItems(qopts);
+    return { items: items.map(i => toKnowledgeResult(i.slug, i)), total };
+}
+function queryUnifiedFn(index, opts) {
+    index.syncIfStale();
+    assertEnum(opts?.kind, ['entity', 'knowledge', 'all'], 'kind');
+    validateEntityFilterOpts(opts);
+    validateKnowledgeFilterOpts(opts);
+    validateTimeOpts(opts);
+    validateListOpts(opts);
+    const qopts = {};
+    if (opts?.kind && opts.kind !== 'all')
+        qopts.kind = opts.kind;
+    // When kind is 'all' or unset, no kind filter — both entities and knowledge are returned
+    mapEntityFilterOpts(qopts, opts);
+    mapKnowledgeFilterOpts(qopts, opts);
+    mapTimeOpts(qopts, opts);
+    mapListOpts(qopts, opts);
+    const result = index.queryItems(qopts);
+    const items = result.items.map(i => {
+        if (i.kind === 'entity') {
+            return { ...toQueryResult(i.slug, i), kind: 'entity' };
+        }
+        else {
+            return { ...toKnowledgeResult(i.slug, i), kind: 'knowledge' };
+        }
+    });
+    return { items, total: result.total };
+}