hafez 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -9
- package/dist/apply.d.ts +30 -0
- package/dist/apply.js +582 -0
- package/dist/batch-ops.d.ts +470 -0
- package/dist/batch-ops.js +446 -0
- package/dist/cli/args.d.ts +19 -0
- package/dist/cli/args.js +53 -0
- package/dist/cli/batch-errors.js +6 -3
- package/dist/cli/commands.d.ts +24 -108
- package/dist/cli/commands.js +297 -313
- package/dist/cli/help-agent.d.ts +0 -5
- package/dist/cli/help-agent.js +44 -5
- package/dist/cli/index.d.ts +41 -0
- package/dist/cli/index.js +172 -172
- package/dist/cli/resolve-vault.js +2 -1
- package/dist/cli/schema-introspect.d.ts +1 -1
- package/dist/cli/schema-introspect.js +6 -22
- package/dist/cli/upgrade.d.ts +42 -0
- package/dist/cli/upgrade.js +305 -0
- package/dist/cli.js +0 -0
- package/dist/contracts.d.ts +55 -6
- package/dist/contracts.js +110 -23
- package/dist/dates.d.ts +10 -0
- package/dist/dates.js +27 -0
- package/dist/db.d.ts +27 -2
- package/dist/db.js +318 -56
- package/dist/digest.js +2 -0
- package/dist/document.d.ts +117 -0
- package/dist/document.js +578 -0
- package/dist/export-okf.js +48 -34
- package/dist/git.d.ts +7 -8
- package/dist/git.js +198 -63
- package/dist/index.d.ts +3 -4
- package/dist/index.js +302 -915
- package/dist/knowledge-index.d.ts +23 -1
- package/dist/knowledge-index.js +71 -13
- package/dist/merge.d.ts +1 -1
- package/dist/merge.js +106 -86
- package/dist/schema.js +4 -4
- package/dist/templates.d.ts +6 -0
- package/dist/templates.js +27 -27
- package/dist/types.d.ts +86 -12
- package/dist/vault.d.ts +51 -6
- package/dist/vault.js +114 -11
- package/package.json +2 -2
- package/skill/SKILL.md +51 -110
- package/dist/parser.d.ts +0 -18
- package/dist/parser.js +0 -68
- package/dist/query.d.ts +0 -24
- package/dist/query.js +0 -167
- package/dist/sections.d.ts +0 -54
- package/dist/sections.js +0 -297
package/dist/db.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
1
|
+
import { type VaultIndexRow } from './knowledge-index.js';
|
|
2
|
+
import type { SearchResult, VaultStats, QueryResult, KnowledgeQueryResult, UnifiedResult, EntityQueryOpts, KnowledgeQueryOpts, UnifiedQueryOpts } from './types.js';
|
|
3
|
+
export declare const SCHEMA_VERSION = "10";
|
|
3
4
|
interface IndexItem {
|
|
4
5
|
slug: string;
|
|
5
6
|
kind: 'entity' | 'knowledge';
|
|
@@ -38,7 +39,29 @@ export interface HafezIndex {
|
|
|
38
39
|
queryItems(opts: QueryOpts): IndexQueryResult;
|
|
39
40
|
search(query: string, kind?: 'entity' | 'knowledge' | 'all'): SearchResult[];
|
|
40
41
|
getStats(): VaultStats;
|
|
42
|
+
/** Every indexed document, in the shape index.md is rendered from. */
|
|
43
|
+
indexEntries(): VaultIndexRow[];
|
|
41
44
|
close(): void;
|
|
45
|
+
queryEntities(opts?: EntityQueryOpts): {
|
|
46
|
+
items: QueryResult[];
|
|
47
|
+
total: number;
|
|
48
|
+
};
|
|
49
|
+
queryChildren(parentSlug: string): {
|
|
50
|
+
items: QueryResult[];
|
|
51
|
+
total: number;
|
|
52
|
+
};
|
|
53
|
+
queryRelatedTo(slug: string, relation?: string): {
|
|
54
|
+
items: UnifiedResult[];
|
|
55
|
+
total: number;
|
|
56
|
+
};
|
|
57
|
+
queryKnowledge(opts?: KnowledgeQueryOpts): {
|
|
58
|
+
items: KnowledgeQueryResult[];
|
|
59
|
+
total: number;
|
|
60
|
+
};
|
|
61
|
+
queryUnified(opts?: UnifiedQueryOpts): {
|
|
62
|
+
items: UnifiedResult[];
|
|
63
|
+
total: number;
|
|
64
|
+
};
|
|
42
65
|
}
|
|
43
66
|
export interface QueryOpts {
|
|
44
67
|
kind?: 'entity' | 'knowledge' | 'all';
|
|
@@ -61,6 +84,8 @@ export interface QueryOpts {
|
|
|
61
84
|
limit?: number;
|
|
62
85
|
offset?: number;
|
|
63
86
|
}
|
|
87
|
+
/** @internal */
|
|
88
|
+
export declare function __setAfterScanHook(fn: (() => void) | null): void;
|
|
64
89
|
export declare function createIndex(vaultPath: string): HafezIndex;
|
|
65
90
|
export declare function createIndex(vaultPath: string, opts: {
|
|
66
91
|
readonly: true;
|
package/dist/db.js
CHANGED
|
@@ -1,17 +1,25 @@
|
|
|
1
1
|
import Database from 'better-sqlite3';
|
|
2
2
|
import { createHash } from 'crypto';
|
|
3
|
-
import {
|
|
3
|
+
import { readFileSync, statSync, existsSync, unlinkSync } from 'fs';
|
|
4
4
|
import { join, basename } from 'path';
|
|
5
|
-
import { parseFilePath } from './vault.js';
|
|
5
|
+
import { parseFilePath, scanVaultDir, kindDir, INDEXED_KINDS } from './vault.js';
|
|
6
6
|
import { ensureLocalExclude } from './knowledge-index.js';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { WIKI_LINK_RE, documentForFile } from './document.js';
|
|
8
|
+
import { ENTITY_STATUSES, ENTITY_TYPES, CONFIDENCE_LEVELS, VALID_SUBTYPES, DEFAULT_KNOWLEDGE_SUBTYPE } from './contracts.js';
|
|
9
|
+
import { HafezError } from './types.js';
|
|
9
10
|
// Bump this when indexing logic changes to force a full rebuild.
|
|
10
11
|
// v5: wiki link harvesting (mention relations), parent link indexing, relation filter on queries.
|
|
11
12
|
// v6: description column on items.
|
|
12
13
|
// v7: resource column on items (entities only).
|
|
14
|
+
// v8: per-kind structural section binding (documentFor) changes how brief,
|
|
15
|
+
// next-action, and session-log columns are derived from bodies.
|
|
16
|
+
// v9: Document binds the kind-level section union (no type/subtype
|
|
17
|
+
// narrowing), changing section boundaries body columns derive from.
|
|
18
|
+
// v10: re-indexing an externally changed file is delete-then-insert instead of
|
|
19
|
+
// INSERT OR REPLACE. Indexes written by v9 and earlier may hold ghost FTS
|
|
20
|
+
// rows (REPLACE never fired the delete trigger) and must be rebuilt, not reused.
|
|
13
21
|
// Existing instances will fully rebuild the index on first access after upgrade.
|
|
14
|
-
export const SCHEMA_VERSION = '
|
|
22
|
+
export const SCHEMA_VERSION = '10';
|
|
15
23
|
const SCHEMA = `
|
|
16
24
|
CREATE TABLE IF NOT EXISTS items (
|
|
17
25
|
slug TEXT PRIMARY KEY,
|
|
@@ -86,23 +94,74 @@ const STOPWORDS = new Set([
|
|
|
86
94
|
function hashContent(content) {
|
|
87
95
|
return createHash('sha256').update(content).digest('hex');
|
|
88
96
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
97
|
+
// --- Query input validation ---
|
|
98
|
+
//
|
|
99
|
+
// Every enum-ish query opt is checked before it can influence (or fail to
|
|
100
|
+
// influence) SQL. The mappers below only forward values they recognise, so an
|
|
101
|
+
// unchecked typo used to be *dropped*: `--filter actvie` ran unfiltered and
|
|
102
|
+
// exited 0. Validating here covers the CLI and library callers at once.
|
|
103
|
+
const SORT_COL_MAP = {
|
|
104
|
+
name: 'i.name',
|
|
105
|
+
created: 'i.created',
|
|
106
|
+
last_touched: 'i.last_touched',
|
|
107
|
+
staleness: 'i.staleness_days',
|
|
108
|
+
reinforcement_count: 'i.reinforcement_count',
|
|
109
|
+
last_reinforced: 'i.last_reinforced',
|
|
110
|
+
};
|
|
111
|
+
const KNOWLEDGE_ONLY_SORTS = new Set(['reinforcement_count', 'last_reinforced']);
|
|
112
|
+
const SORT_FIELDS = Object.keys(SORT_COL_MAP);
|
|
113
|
+
const SORT_ORDERS = ['asc', 'desc'];
|
|
114
|
+
const QUERY_FILTERS = [...ENTITY_STATUSES, 'all', 'stale', 'capture'];
|
|
115
|
+
function assertEnum(value, valid, label) {
|
|
116
|
+
if (value === undefined || value === null)
|
|
117
|
+
return;
|
|
118
|
+
if (typeof value !== 'string' || !valid.includes(value)) {
|
|
119
|
+
throw new HafezError('VALIDATION_FAILED', `Invalid ${label}: "${String(value)}". Valid values: ${valid.join(', ')}.`);
|
|
94
120
|
}
|
|
95
|
-
|
|
96
|
-
|
|
121
|
+
}
|
|
122
|
+
function assertSortOrder(value) {
|
|
123
|
+
if (value === undefined || value === null)
|
|
124
|
+
return;
|
|
125
|
+
if (typeof value !== 'string' || !SORT_ORDERS.includes(value.toLowerCase())) {
|
|
126
|
+
throw new HafezError('VALIDATION_FAILED', `Invalid sort_order: "${String(value)}". Valid values: asc, desc.`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function assertCount(value, label) {
|
|
130
|
+
if (value === undefined || value === null)
|
|
131
|
+
return;
|
|
132
|
+
if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
|
|
133
|
+
throw new HafezError('VALIDATION_FAILED', `Invalid ${label}: "${String(value)}". Must be a non-negative integer.`);
|
|
97
134
|
}
|
|
98
135
|
}
|
|
136
|
+
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
|
|
137
|
+
function validateTimeOpts(opts) {
|
|
138
|
+
const fields = ['since', 'before', 'createdSince', 'createdBefore'];
|
|
139
|
+
for (const f of fields) {
|
|
140
|
+
if (opts?.[f] !== undefined && !ISO_DATE.test(opts[f])) {
|
|
141
|
+
throw new HafezError('VALIDATION_FAILED', `Invalid date format for ${f}: "${opts[f]}". Must be YYYY-MM-DD.`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function validateListOpts(opts) {
|
|
146
|
+
assertEnum(opts?.sort_by, SORT_FIELDS, 'sort_by');
|
|
147
|
+
assertSortOrder(opts?.sort_order);
|
|
148
|
+
assertCount(opts?.limit, 'limit');
|
|
149
|
+
assertCount(opts?.offset, 'offset');
|
|
150
|
+
}
|
|
99
151
|
// --- Shared query functions (used by both write and readonly paths) ---
|
|
100
152
|
function sanitizeFtsQuery(raw) {
|
|
101
|
-
|
|
153
|
+
// Unicode-aware: keep letters/digits/underscore in ANY script, strip
|
|
154
|
+
// punctuation. `[^\w\s]` was ASCII-only, so it erased whole non-Latin
|
|
155
|
+
// queries (日本語 → '') and the vault's non-Latin notes were unfindable.
|
|
156
|
+
// For ASCII input this class is identical to the old one.
|
|
157
|
+
const cleaned = raw.replace(/[^\p{L}\p{N}_\s]/gu, ' ');
|
|
102
158
|
const tokens = cleaned.split(/\s+/).filter(t => t.length > 0);
|
|
103
159
|
const filtered = tokens.filter(t => !STOPWORDS.has(t.toLowerCase()));
|
|
160
|
+
// All-stopword queries fall back to the raw tokens — which is how bare
|
|
161
|
+
// AND/OR/NOT reach fts5. Quoting makes every term a literal string, never
|
|
162
|
+
// an operator, and a trailing `*` still gives prefix matching.
|
|
104
163
|
const terms = filtered.length === 0 ? tokens : filtered;
|
|
105
|
-
return { fts: terms.map(t =>
|
|
164
|
+
return { fts: terms.map(t => `"${t.replace(/"/g, '""')}"*`).join(' OR '), terms };
|
|
106
165
|
}
|
|
107
166
|
function queryItemsFn(db, opts) {
|
|
108
167
|
const conditions = [];
|
|
@@ -112,7 +171,11 @@ function queryItemsFn(db, opts) {
|
|
|
112
171
|
params.kind = opts.kind;
|
|
113
172
|
}
|
|
114
173
|
if (opts.status) {
|
|
115
|
-
|
|
174
|
+
// Status is an entity attribute — knowledge rows are indexed with a NULL
|
|
175
|
+
// status. In a kind-unset (unified) query the status filter narrows the
|
|
176
|
+
// entity side only; applying it to every row is what made `query --kind
|
|
177
|
+
// all` (filter defaults to active) return entities and nothing else.
|
|
178
|
+
conditions.push(opts.kind === 'entity' ? 'i.status = @status' : "(i.kind = 'knowledge' OR i.status = @status)");
|
|
116
179
|
params.status = opts.status;
|
|
117
180
|
}
|
|
118
181
|
if (opts.type) {
|
|
@@ -185,24 +248,21 @@ function queryItemsFn(db, opts) {
|
|
|
185
248
|
}
|
|
186
249
|
}
|
|
187
250
|
const where = conditions.length > 0 ? 'WHERE ' + conditions.join(' AND ') : '';
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
reinforcement_count: 'i.reinforcement_count',
|
|
195
|
-
last_reinforced: 'i.last_reinforced',
|
|
196
|
-
};
|
|
251
|
+
// sort_order / limit / offset are the three opts that reach SQL as text
|
|
252
|
+
// rather than bound params — check them at the interpolation site, so a
|
|
253
|
+
// caller that skips the read surface can't hand raw SQL to prepare().
|
|
254
|
+
assertSortOrder(opts.sort_order);
|
|
255
|
+
assertCount(opts.limit, 'limit');
|
|
256
|
+
assertCount(opts.offset, 'offset');
|
|
197
257
|
let sortCol = 'i.last_touched';
|
|
198
258
|
if (opts.sort_by) {
|
|
199
|
-
const isKnowledgeOnly =
|
|
259
|
+
const isKnowledgeOnly = KNOWLEDGE_ONLY_SORTS.has(opts.sort_by);
|
|
200
260
|
const isEntityQuery = opts.kind === 'entity';
|
|
201
261
|
if (isKnowledgeOnly && isEntityQuery) {
|
|
202
262
|
sortCol = 'i.last_touched';
|
|
203
263
|
}
|
|
204
|
-
else if (
|
|
205
|
-
sortCol =
|
|
264
|
+
else if (SORT_COL_MAP[opts.sort_by]) {
|
|
265
|
+
sortCol = SORT_COL_MAP[opts.sort_by];
|
|
206
266
|
}
|
|
207
267
|
}
|
|
208
268
|
let sortOrder;
|
|
@@ -247,6 +307,18 @@ function queryItemsFn(db, opts) {
|
|
|
247
307
|
}
|
|
248
308
|
return { items: rows, total };
|
|
249
309
|
}
|
|
310
|
+
/** Every indexed document, in the shape index.md renders from. The vault-root
|
|
311
|
+
* index needs name/status/type/confidence/description/domain and the body (for
|
|
312
|
+
* the derived summary) — all of it already indexed, so regenerating index.md
|
|
313
|
+
* after a mutation costs one query instead of a second full-vault parse. */
|
|
314
|
+
function indexEntriesFn(db) {
|
|
315
|
+
const rows = db.prepare('SELECT slug, kind, name, type, status, confidence, description, body FROM items').all();
|
|
316
|
+
const getDomains = db.prepare("SELECT target FROM links WHERE source = ? AND relation = 'domain'");
|
|
317
|
+
return rows.map(row => ({
|
|
318
|
+
...row,
|
|
319
|
+
domain: getDomains.all(row.slug).map(d => d.target),
|
|
320
|
+
}));
|
|
321
|
+
}
|
|
250
322
|
function searchFn(db, query, kind) {
|
|
251
323
|
const { fts: ftsQuery, terms } = sanitizeFtsQuery(query);
|
|
252
324
|
if (!ftsQuery)
|
|
@@ -347,6 +419,17 @@ function getStatsFn(db) {
|
|
|
347
419
|
recently_created,
|
|
348
420
|
};
|
|
349
421
|
}
|
|
422
|
+
// Internal test seam — NOT public API (createHafez never exposes it). Fires
|
|
423
|
+
// inside syncIfStale after the scan has read the vault and before the new
|
|
424
|
+
// watermark is recorded, so the mid-scan-write race can be injected
|
|
425
|
+
// deterministically instead of raced against the clock.
|
|
426
|
+
// Recorded sync watermarks sit this far behind the scan-start instant: FS
|
|
427
|
+
// timestamps and Date.now() are read from the same kernel clock but round
|
|
428
|
+
// differently, and a write landing in that rounding gap must stay dirty.
|
|
429
|
+
const WATERMARK_SLACK_MS = 10;
|
|
430
|
+
let afterScanHook = null;
|
|
431
|
+
/** @internal */
|
|
432
|
+
export function __setAfterScanHook(fn) { afterScanHook = fn; }
|
|
350
433
|
export function createIndex(vaultPath, opts) {
|
|
351
434
|
const dbPath = join(vaultPath, '.hafez.db');
|
|
352
435
|
const isReadOnly = opts?.readonly ?? false;
|
|
@@ -389,7 +472,7 @@ export function createIndex(vaultPath, opts) {
|
|
|
389
472
|
}
|
|
390
473
|
catch { /* DB may have been temporarily deleted during rebuild */ }
|
|
391
474
|
}
|
|
392
|
-
return {
|
|
475
|
+
return withReadSurface({
|
|
393
476
|
rebuild: () => { throw new Error('Index is read-only'); },
|
|
394
477
|
syncIfStale,
|
|
395
478
|
upsertFromFile: () => { throw new Error('Index is read-only'); },
|
|
@@ -397,11 +480,12 @@ export function createIndex(vaultPath, opts) {
|
|
|
397
480
|
queryItems: (qopts) => queryItemsFn(db, qopts),
|
|
398
481
|
search: (q, k) => searchFn(db, q, k),
|
|
399
482
|
getStats: () => getStatsFn(db),
|
|
483
|
+
indexEntries: () => indexEntriesFn(db),
|
|
400
484
|
close: () => { try {
|
|
401
485
|
db.close();
|
|
402
486
|
}
|
|
403
487
|
catch { } },
|
|
404
|
-
};
|
|
488
|
+
});
|
|
405
489
|
}
|
|
406
490
|
// --- Write-capable path ---
|
|
407
491
|
// The db files are a disposable local cache — keep them out of the user's
|
|
@@ -479,8 +563,12 @@ export function createIndex(vaultPath, opts) {
|
|
|
479
563
|
db.prepare("INSERT INTO meta (key, value) VALUES ('schema_version', ?)").run(SCHEMA_VERSION);
|
|
480
564
|
}
|
|
481
565
|
// Prepared statements
|
|
482
|
-
|
|
483
|
-
|
|
566
|
+
// Plain INSERT, never INSERT OR REPLACE: REPLACE's implicit delete does not
|
|
567
|
+
// fire the AFTER DELETE trigger (recursive_triggers is off), so the replaced
|
|
568
|
+
// row's terms would linger in the external-content FTS index as ghosts.
|
|
569
|
+
// Callers delete the row first — see indexFile.
|
|
570
|
+
const insertStmt = db.prepare(`
|
|
571
|
+
INSERT INTO items
|
|
484
572
|
(slug, kind, name, type, status, parent, next_action, next_action_count, staleness_days,
|
|
485
573
|
confidence, reinforcement_count, last_reinforced, created, last_touched, body, file_hash, subtype,
|
|
486
574
|
description, resource, brief, session_log_count, last_session_date, last_session_type, last_session_summary)
|
|
@@ -517,11 +605,12 @@ export function createIndex(vaultPath, opts) {
|
|
|
517
605
|
process.stderr.write(`Warning: skipping ${kind}/${slug}.md: missing required field(s): ${missing}\n`);
|
|
518
606
|
return;
|
|
519
607
|
}
|
|
520
|
-
const
|
|
608
|
+
const doc = documentForFile({ kind });
|
|
609
|
+
const bodyActions = doc.getNextActions(body);
|
|
521
610
|
const nextAction = bodyActions.length > 0 ? bodyActions[0] : null;
|
|
522
611
|
const nextActionCount = bodyActions.length;
|
|
523
|
-
const brief = getBrief(body)?.slice(0, 300) ?? null;
|
|
524
|
-
const logs = parseSessionLog(body);
|
|
612
|
+
const brief = doc.getBrief(body)?.slice(0, 300) ?? null;
|
|
613
|
+
const logs = doc.parseSessionLog(body);
|
|
525
614
|
const lastLog = logs[0] ?? null;
|
|
526
615
|
const item = {
|
|
527
616
|
slug,
|
|
@@ -540,7 +629,7 @@ export function createIndex(vaultPath, opts) {
|
|
|
540
629
|
last_touched: fm['last-touched'] ?? null,
|
|
541
630
|
body,
|
|
542
631
|
file_hash: hash,
|
|
543
|
-
subtype: kind === 'knowledge' ? (fm.subtype ??
|
|
632
|
+
subtype: kind === 'knowledge' ? (fm.subtype ?? DEFAULT_KNOWLEDGE_SUBTYPE) : null,
|
|
544
633
|
description: fm.description ?? null,
|
|
545
634
|
resource: kind === 'entity' ? (fm.resource ?? null) : null,
|
|
546
635
|
brief,
|
|
@@ -549,9 +638,12 @@ export function createIndex(vaultPath, opts) {
|
|
|
549
638
|
last_session_type: lastLog?.type ?? null,
|
|
550
639
|
last_session_summary: lastLog?.summary?.slice(0, 300) ?? null,
|
|
551
640
|
};
|
|
552
|
-
//
|
|
641
|
+
// Replace item and rebuild links. Delete-then-insert keeps the FTS index
|
|
642
|
+
// trigger-consistent for externally changed files (hand edit, git pull),
|
|
643
|
+
// the same path hafez's own mutations take via upsertFromFile.
|
|
644
|
+
deleteItemStmt.run(slug);
|
|
553
645
|
deleteLinksStmt.run(slug);
|
|
554
|
-
|
|
646
|
+
insertStmt.run(item);
|
|
555
647
|
// Index related links
|
|
556
648
|
if (fm.related && Array.isArray(fm.related)) {
|
|
557
649
|
for (const target of fm.related)
|
|
@@ -574,7 +666,7 @@ export function createIndex(vaultPath, opts) {
|
|
|
574
666
|
// Harvest [[wiki links]] from body (excluding ## Related section)
|
|
575
667
|
// Strip the Related section to avoid double-counting generated links
|
|
576
668
|
let authoredBody = body;
|
|
577
|
-
const relatedSection = findSection(body, 'Related');
|
|
669
|
+
const relatedSection = doc.findSection(body, 'Related');
|
|
578
670
|
if (relatedSection) {
|
|
579
671
|
authoredBody = body.slice(0, relatedSection.start) + body.slice(relatedSection.end);
|
|
580
672
|
}
|
|
@@ -583,8 +675,8 @@ export function createIndex(vaultPath, opts) {
|
|
|
583
675
|
while ((wikiMatch = linkRe.exec(authoredBody)) !== null) {
|
|
584
676
|
const targetSlug = wikiMatch[1];
|
|
585
677
|
// Check if target file exists on disk (not in index — target may not be indexed yet)
|
|
586
|
-
const targetEntityPath = join(vaultPath, '
|
|
587
|
-
const targetKnowledgePath = join(vaultPath, 'knowledge', `${targetSlug}.md`);
|
|
678
|
+
const targetEntityPath = join(vaultPath, kindDir('entity'), `${targetSlug}.md`);
|
|
679
|
+
const targetKnowledgePath = join(vaultPath, kindDir('knowledge'), `${targetSlug}.md`);
|
|
588
680
|
const targetExists = knownSlugs
|
|
589
681
|
? knownSlugs.has(targetSlug)
|
|
590
682
|
: existsSync(targetEntityPath) || existsSync(targetKnowledgePath);
|
|
@@ -594,24 +686,26 @@ export function createIndex(vaultPath, opts) {
|
|
|
594
686
|
}
|
|
595
687
|
}
|
|
596
688
|
const rebuildTransaction = db.transaction(() => {
|
|
689
|
+
// Watermark the scan by when it STARTED (see syncIfStale).
|
|
690
|
+
const scanStart = Date.now() - WATERMARK_SLACK_MS;
|
|
597
691
|
db.exec('DELETE FROM items');
|
|
598
692
|
db.exec('DELETE FROM links');
|
|
599
693
|
// Collect all vault slugs for wiki link target validation
|
|
600
694
|
const allVaultSlugs = new Set();
|
|
601
|
-
for (const file of
|
|
695
|
+
for (const file of scanVaultDir(join(vaultPath, kindDir('entity')))) {
|
|
602
696
|
allVaultSlugs.add(basename(file, '.md'));
|
|
603
697
|
}
|
|
604
|
-
for (const file of
|
|
698
|
+
for (const file of scanVaultDir(join(vaultPath, kindDir('knowledge')))) {
|
|
605
699
|
allVaultSlugs.add(basename(file, '.md'));
|
|
606
700
|
}
|
|
607
|
-
for (const file of
|
|
701
|
+
for (const file of scanVaultDir(join(vaultPath, kindDir('entity')))) {
|
|
608
702
|
indexFile(file, 'entity', allVaultSlugs);
|
|
609
703
|
}
|
|
610
|
-
for (const file of
|
|
704
|
+
for (const file of scanVaultDir(join(vaultPath, kindDir('knowledge')))) {
|
|
611
705
|
indexFile(file, 'knowledge', allVaultSlugs);
|
|
612
706
|
}
|
|
613
707
|
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_sync', ?)")
|
|
614
|
-
.run(
|
|
708
|
+
.run(scanStart.toString());
|
|
615
709
|
});
|
|
616
710
|
function rebuild() {
|
|
617
711
|
rebuildTransaction();
|
|
@@ -624,13 +718,19 @@ export function createIndex(vaultPath, opts) {
|
|
|
624
718
|
}
|
|
625
719
|
const lastSyncTime = parseInt(lastSync.value, 10);
|
|
626
720
|
let changed = false;
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
721
|
+
// Watermark the scan by when it STARTED, not when it finished. A file
|
|
722
|
+
// written while the scan is in flight — after its read, before the record —
|
|
723
|
+
// would otherwise be stamped as already-indexed and skipped by every later
|
|
724
|
+
// sync, leaving it permanently stale. `>=` covers the same-millisecond
|
|
725
|
+
// case; a re-read of an unchanged file is a cheap hash compare. The slack
|
|
726
|
+
// absorbs FS-vs-Date.now clock rounding at the boundary — a file whose
|
|
727
|
+
// mtime rounds just below the instant the scan started must still read as
|
|
728
|
+
// dirty; the only cost is re-hashing files touched inside the window.
|
|
729
|
+
const scanStart = Date.now() - WATERMARK_SLACK_MS;
|
|
730
|
+
const dirs = INDEXED_KINDS.map(kind => ({ dir: kindDir(kind), kind }));
|
|
631
731
|
for (const { dir, kind } of dirs) {
|
|
632
|
-
for (const file of
|
|
633
|
-
if (statSync(file).mtimeMs
|
|
732
|
+
for (const file of scanVaultDir(join(vaultPath, dir))) {
|
|
733
|
+
if (statSync(file).mtimeMs >= lastSyncTime) {
|
|
634
734
|
indexFile(file, kind);
|
|
635
735
|
changed = true;
|
|
636
736
|
}
|
|
@@ -640,7 +740,7 @@ export function createIndex(vaultPath, opts) {
|
|
|
640
740
|
const dbSlugs = db.prepare('SELECT slug FROM items').all().map(r => r.slug);
|
|
641
741
|
const fileSlugs = new Set();
|
|
642
742
|
for (const { dir } of dirs) {
|
|
643
|
-
for (const file of
|
|
743
|
+
for (const file of scanVaultDir(join(vaultPath, dir))) {
|
|
644
744
|
fileSlugs.add(basename(file, '.md'));
|
|
645
745
|
}
|
|
646
746
|
}
|
|
@@ -650,14 +750,14 @@ export function createIndex(vaultPath, opts) {
|
|
|
650
750
|
changed = true;
|
|
651
751
|
}
|
|
652
752
|
}
|
|
753
|
+
afterScanHook?.();
|
|
653
754
|
if (changed) {
|
|
654
755
|
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_sync', ?)")
|
|
655
|
-
.run(
|
|
756
|
+
.run(scanStart.toString());
|
|
656
757
|
}
|
|
657
758
|
}
|
|
658
759
|
function upsertFromFile(slug, kind) {
|
|
659
|
-
const
|
|
660
|
-
const filePath = join(vaultPath, dir, `${slug}.md`);
|
|
760
|
+
const filePath = join(vaultPath, kindDir(kind), `${slug}.md`);
|
|
661
761
|
if (existsSync(filePath)) {
|
|
662
762
|
// Force re-index by clearing cached hash
|
|
663
763
|
deleteItemStmt.run(slug);
|
|
@@ -673,11 +773,173 @@ export function createIndex(vaultPath, opts) {
|
|
|
673
773
|
const count = db.prepare('SELECT COUNT(*) as c FROM items').get().c;
|
|
674
774
|
if (count === 0)
|
|
675
775
|
rebuild();
|
|
676
|
-
return {
|
|
776
|
+
return withReadSurface({
|
|
677
777
|
rebuild, syncIfStale, upsertFromFile, removeItem,
|
|
678
778
|
queryItems: (qopts) => queryItemsFn(db, qopts),
|
|
679
779
|
search: (q, k) => searchFn(db, q, k),
|
|
680
780
|
getStats: () => getStatsFn(db),
|
|
781
|
+
indexEntries: () => indexEntriesFn(db),
|
|
681
782
|
close: () => db.close(),
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
function withReadSurface(core) {
|
|
786
|
+
const index = {
|
|
787
|
+
...core,
|
|
788
|
+
queryEntities: (opts) => queryEntitiesFn(index, opts),
|
|
789
|
+
queryChildren: (parentSlug) => queryEntitiesFn(index, { parent: parentSlug }),
|
|
790
|
+
queryRelatedTo: (slug, relation) => queryRelatedToFn(index, slug, relation),
|
|
791
|
+
queryKnowledge: (opts) => queryKnowledgeFn(index, opts),
|
|
792
|
+
queryUnified: (opts) => queryUnifiedFn(index, opts),
|
|
793
|
+
};
|
|
794
|
+
return index;
|
|
795
|
+
}
|
|
796
|
+
function toQueryResult(slug, row) {
|
|
797
|
+
return {
|
|
798
|
+
slug,
|
|
799
|
+
name: row.name,
|
|
800
|
+
type: row.type,
|
|
801
|
+
status: row.status,
|
|
802
|
+
description: row.description ?? null,
|
|
803
|
+
resource: row.resource ?? null,
|
|
804
|
+
domain: row.domain ?? [],
|
|
805
|
+
last_touched: row.last_touched,
|
|
806
|
+
next_action: row.next_action ?? null,
|
|
807
|
+
next_action_count: row.next_action_count ?? 0,
|
|
808
|
+
parent: row.parent ?? null,
|
|
809
|
+
related: row.related ?? [],
|
|
810
|
+
staleness_days: row.staleness_days ?? null,
|
|
811
|
+
tags: row.tags ?? [],
|
|
812
|
+
brief: row.brief ?? null,
|
|
813
|
+
session_log_count: row.session_log_count ?? 0,
|
|
814
|
+
last_session_date: row.last_session_date ?? null,
|
|
815
|
+
last_session_type: row.last_session_type ?? null,
|
|
816
|
+
last_session_summary: row.last_session_summary ?? null,
|
|
817
|
+
};
|
|
818
|
+
}
|
|
819
|
+
function toKnowledgeResult(slug, row) {
|
|
820
|
+
return {
|
|
821
|
+
slug,
|
|
822
|
+
name: row.name,
|
|
823
|
+
description: row.description ?? null,
|
|
824
|
+
domain: row.domain ?? [],
|
|
825
|
+
confidence: row.confidence ?? 'observation',
|
|
826
|
+
related: row.related ?? [],
|
|
827
|
+
reinforcement_count: row.reinforcement_count ?? 0,
|
|
828
|
+
last_reinforced: row.last_reinforced ?? null,
|
|
829
|
+
last_touched: row.last_touched ?? null,
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
const STATUS_FILTERS = new Set(ENTITY_STATUSES);
|
|
833
|
+
function mapEntityFilterOpts(qopts, opts) {
|
|
834
|
+
if (opts?.filter && STATUS_FILTERS.has(opts.filter))
|
|
835
|
+
qopts.status = opts.filter;
|
|
836
|
+
if (opts?.filter === 'stale')
|
|
837
|
+
qopts.filter = 'stale';
|
|
838
|
+
if (opts?.filter === 'capture')
|
|
839
|
+
qopts.filter = 'capture';
|
|
840
|
+
if (opts?.type)
|
|
841
|
+
qopts.type = opts.type;
|
|
842
|
+
if (opts?.parent)
|
|
843
|
+
qopts.parent = opts.parent;
|
|
844
|
+
if (opts?.domain)
|
|
845
|
+
qopts.domain = opts.domain;
|
|
846
|
+
}
|
|
847
|
+
function mapTimeOpts(qopts, opts) {
|
|
848
|
+
if (opts?.since)
|
|
849
|
+
qopts.since = opts.since;
|
|
850
|
+
if (opts?.before)
|
|
851
|
+
qopts.before = opts.before;
|
|
852
|
+
if (opts?.createdSince)
|
|
853
|
+
qopts.createdSince = opts.createdSince;
|
|
854
|
+
if (opts?.createdBefore)
|
|
855
|
+
qopts.createdBefore = opts.createdBefore;
|
|
856
|
+
}
|
|
857
|
+
function mapListOpts(qopts, opts) {
|
|
858
|
+
if (opts?.tags)
|
|
859
|
+
qopts.tags = opts.tags;
|
|
860
|
+
if (opts?.sort_by)
|
|
861
|
+
qopts.sort_by = opts.sort_by;
|
|
862
|
+
if (opts?.sort_order)
|
|
863
|
+
qopts.sort_order = opts.sort_order;
|
|
864
|
+
if (opts?.limit != null)
|
|
865
|
+
qopts.limit = opts.limit;
|
|
866
|
+
if (opts?.offset != null)
|
|
867
|
+
qopts.offset = opts.offset;
|
|
868
|
+
}
|
|
869
|
+
function mapKnowledgeFilterOpts(qopts, opts) {
|
|
870
|
+
if (opts?.domain)
|
|
871
|
+
qopts.domain = opts.domain;
|
|
872
|
+
if (opts?.confidence)
|
|
873
|
+
qopts.confidence = opts.confidence;
|
|
874
|
+
if (opts?.subtype)
|
|
875
|
+
qopts.subtype = opts.subtype;
|
|
876
|
+
}
|
|
877
|
+
function validateEntityFilterOpts(opts) {
|
|
878
|
+
assertEnum(opts?.filter, QUERY_FILTERS, 'filter');
|
|
879
|
+
assertEnum(opts?.type, ENTITY_TYPES, 'type');
|
|
880
|
+
}
|
|
881
|
+
function validateKnowledgeFilterOpts(opts) {
|
|
882
|
+
assertEnum(opts?.confidence, CONFIDENCE_LEVELS, 'confidence');
|
|
883
|
+
assertEnum(opts?.subtype, VALID_SUBTYPES, 'subtype');
|
|
884
|
+
}
|
|
885
|
+
function queryEntitiesFn(index, opts) {
|
|
886
|
+
index.syncIfStale();
|
|
887
|
+
validateEntityFilterOpts(opts);
|
|
888
|
+
validateKnowledgeFilterOpts(opts);
|
|
889
|
+
validateTimeOpts(opts);
|
|
890
|
+
validateListOpts(opts);
|
|
891
|
+
const qopts = { kind: 'entity' };
|
|
892
|
+
mapEntityFilterOpts(qopts, opts);
|
|
893
|
+
mapTimeOpts(qopts, opts);
|
|
894
|
+
mapListOpts(qopts, opts);
|
|
895
|
+
const { items, total } = index.queryItems(qopts);
|
|
896
|
+
return { items: items.map(i => toQueryResult(i.slug, i)), total };
|
|
897
|
+
}
|
|
898
|
+
function queryRelatedToFn(index, slug, relation) {
|
|
899
|
+
index.syncIfStale();
|
|
900
|
+
const { items, total } = index.queryItems({ relatedTo: slug, relation });
|
|
901
|
+
return {
|
|
902
|
+
items: items.map(i => i.kind === 'entity'
|
|
903
|
+
? { ...toQueryResult(i.slug, i), kind: 'entity' }
|
|
904
|
+
: { ...toKnowledgeResult(i.slug, i), kind: 'knowledge' }),
|
|
905
|
+
total,
|
|
682
906
|
};
|
|
683
907
|
}
|
|
908
|
+
function queryKnowledgeFn(index, opts) {
|
|
909
|
+
index.syncIfStale();
|
|
910
|
+
validateKnowledgeFilterOpts(opts);
|
|
911
|
+
validateTimeOpts(opts);
|
|
912
|
+
validateListOpts(opts);
|
|
913
|
+
const qopts = { kind: 'knowledge' };
|
|
914
|
+
mapKnowledgeFilterOpts(qopts, opts);
|
|
915
|
+
mapTimeOpts(qopts, opts);
|
|
916
|
+
mapListOpts(qopts, opts);
|
|
917
|
+
const { items, total } = index.queryItems(qopts);
|
|
918
|
+
return { items: items.map(i => toKnowledgeResult(i.slug, i)), total };
|
|
919
|
+
}
|
|
920
|
+
function queryUnifiedFn(index, opts) {
|
|
921
|
+
index.syncIfStale();
|
|
922
|
+
assertEnum(opts?.kind, ['entity', 'knowledge', 'all'], 'kind');
|
|
923
|
+
validateEntityFilterOpts(opts);
|
|
924
|
+
validateKnowledgeFilterOpts(opts);
|
|
925
|
+
validateTimeOpts(opts);
|
|
926
|
+
validateListOpts(opts);
|
|
927
|
+
const qopts = {};
|
|
928
|
+
if (opts?.kind && opts.kind !== 'all')
|
|
929
|
+
qopts.kind = opts.kind;
|
|
930
|
+
// When kind is 'all' or unset, no kind filter — both entities and knowledge are returned
|
|
931
|
+
mapEntityFilterOpts(qopts, opts);
|
|
932
|
+
mapKnowledgeFilterOpts(qopts, opts);
|
|
933
|
+
mapTimeOpts(qopts, opts);
|
|
934
|
+
mapListOpts(qopts, opts);
|
|
935
|
+
const result = index.queryItems(qopts);
|
|
936
|
+
const items = result.items.map(i => {
|
|
937
|
+
if (i.kind === 'entity') {
|
|
938
|
+
return { ...toQueryResult(i.slug, i), kind: 'entity' };
|
|
939
|
+
}
|
|
940
|
+
else {
|
|
941
|
+
return { ...toKnowledgeResult(i.slug, i), kind: 'knowledge' };
|
|
942
|
+
}
|
|
943
|
+
});
|
|
944
|
+
return { items, total: result.total };
|
|
945
|
+
}
|
package/dist/digest.js
CHANGED
|
@@ -60,6 +60,8 @@ export function digest(input, existingSlugs, now = new Date()) {
|
|
|
60
60
|
// leak into the note body (the schema promises they are omitted).
|
|
61
61
|
const noteBody = buildSessionNoteBody(input, knownTouched);
|
|
62
62
|
const related = knownTouched;
|
|
63
|
+
// Typechecked against the Op Spec's session-create schema so digest output
|
|
64
|
+
// can never silently diverge from what batch validation accepts.
|
|
63
65
|
const noteFields = {
|
|
64
66
|
synthesis: noteBody,
|
|
65
67
|
'session-date': input.session_date,
|