@openparachute/vault 0.7.0-rc.9 → 0.7.2-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/aggregate.test.ts +260 -0
- package/core/src/contract-taxonomy.test.ts +26 -2
- package/core/src/contract-typed-index.test.ts +4 -2
- package/core/src/core.test.ts +737 -2
- package/core/src/cursor-keyset-ms.test.ts +537 -0
- package/core/src/cursor.ts +76 -6
- package/core/src/doctor.ts +23 -1
- package/core/src/hooks.ts +9 -0
- package/core/src/indexed-fields.test.ts +4 -1
- package/core/src/indexed-fields.ts +6 -1
- package/core/src/links.ts +22 -0
- package/core/src/mcp.ts +322 -53
- package/core/src/notes.ts +518 -67
- package/core/src/paths.ts +4 -0
- package/core/src/portable-md.test.ts +161 -0
- package/core/src/portable-md.ts +140 -9
- package/core/src/query-warnings.ts +60 -12
- package/core/src/schema-defaults.ts +7 -1
- package/core/src/schema.ts +128 -2
- package/core/src/seed-packs.ts +55 -15
- package/core/src/store.ts +141 -7
- package/core/src/tag-schemas.ts +20 -7
- package/core/src/txn.test.ts +100 -4
- package/core/src/txn.ts +119 -24
- package/core/src/types.ts +89 -0
- package/core/src/ulid.test.ts +56 -0
- package/core/src/ulid.ts +116 -0
- package/core/src/vault-projection.ts +19 -1
- package/core/src/wikilinks.test.ts +205 -1
- package/core/src/wikilinks.ts +200 -35
- package/package.json +1 -1
- package/src/aggregate-routes.test.ts +230 -0
- package/src/cli.ts +6 -0
- package/src/contract-errors.test.ts +2 -1
- package/src/contract-search.test.ts +17 -0
- package/src/mcp-link-warnings-scope.test.ts +144 -0
- package/src/mcp-query-notes-aggregate-scope.test.ts +183 -0
- package/src/mcp-tools.ts +76 -17
- package/src/mirror-import.ts +5 -0
- package/src/routes.ts +298 -24
- package/src/routing.test.ts +167 -0
- package/src/routing.ts +47 -11
- package/src/tag-integrity-mcp.test.ts +45 -6
- package/src/tag-scope.ts +10 -1
- package/src/vault.test.ts +557 -21
- package/web/ui/dist/assets/{index-B8pGeQam.js → index-CJ6NtIwh.js} +1 -1
- package/web/ui/dist/index.html +1 -1
package/core/src/schema.ts
CHANGED
|
@@ -3,8 +3,19 @@ import { normalizePath } from "./paths.js";
|
|
|
3
3
|
import { rebuildIndexes, listIndexedFields } from "./indexed-fields.js";
|
|
4
4
|
import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
|
|
5
5
|
import { transaction } from "./txn.js";
|
|
6
|
+
import { timestampToMs } from "./cursor.js";
|
|
6
7
|
|
|
7
|
-
export const SCHEMA_VERSION =
|
|
8
|
+
export const SCHEMA_VERSION = 26;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Deterministic last-resort epoch for a note whose `updated_at` AND
|
|
12
|
+
* `created_at` are both unparseable (vault#586). 0 (the Unix epoch) sorts such
|
|
13
|
+
* a row to the very start of the keyset walk — honest "oldest / unknown"
|
|
14
|
+
* placement — and, being a fixed constant, keeps the backfill idempotent (a
|
|
15
|
+
* re-run derives the identical value). NEVER null: a NULL `updated_at_ms`
|
|
16
|
+
* would sort unpredictably and break the `> ?` keyset predicate.
|
|
17
|
+
*/
|
|
18
|
+
const UNPARSEABLE_UPDATED_AT_MS = 0;
|
|
8
19
|
|
|
9
20
|
export const SCHEMA_SQL = `
|
|
10
21
|
-- Notes: the universal record.
|
|
@@ -26,6 +37,13 @@ export const SCHEMA_SQL = `
|
|
|
26
37
|
-- (mcp, surface:NAME, agent:ID, operator/cli, api). Legacy rows stay NULL —
|
|
27
38
|
-- we don't fabricate authors for writes that predate attribution. See
|
|
28
39
|
-- migrateToV23.
|
|
40
|
+
-- updated_at_ms (v26, vault#586) is the integer millisecond-epoch mirror of
|
|
41
|
+
-- updated_at, and the SINGLE source of truth for cursor keyset ordering. The
|
|
42
|
+
-- keyset walk-order, boundary predicate, and watermark all read this one
|
|
43
|
+
-- numeric column, so they can't diverge the way three separate readings of the
|
|
44
|
+
-- TEXT updated_at did on aged/imported vaults with non-canonical timestamps.
|
|
45
|
+
-- Maintained on every write (createNote / updateNote / restore / cascades) via
|
|
46
|
+
-- core/src/cursor.ts timestampToMs; existing rows backfilled by migrateToV26.
|
|
29
47
|
CREATE TABLE IF NOT EXISTS notes (
|
|
30
48
|
id TEXT PRIMARY KEY,
|
|
31
49
|
content TEXT DEFAULT '',
|
|
@@ -37,7 +55,8 @@ CREATE TABLE IF NOT EXISTS notes (
|
|
|
37
55
|
created_by TEXT,
|
|
38
56
|
created_via TEXT,
|
|
39
57
|
last_updated_by TEXT,
|
|
40
|
-
last_updated_via TEXT
|
|
58
|
+
last_updated_via TEXT,
|
|
59
|
+
updated_at_ms INTEGER
|
|
41
60
|
);
|
|
42
61
|
|
|
43
62
|
-- Tags: first-class identity carrying schema, hierarchy, and typed-link
|
|
@@ -529,6 +548,12 @@ export function initSchema(db: Database): void {
|
|
|
529
548
|
// porter stemming, repopulate from every existing note. See vault#551.
|
|
530
549
|
migrateToV25(db);
|
|
531
550
|
|
|
551
|
+
// Migrate v25 → v26: add the integer `notes.updated_at_ms` column — the
|
|
552
|
+
// single source of truth for cursor keyset ordering — plus its
|
|
553
|
+
// (updated_at_ms, id) index, and backfill every existing row from its
|
|
554
|
+
// `updated_at` string with a UTC-correct parse. See vault#586.
|
|
555
|
+
migrateToV26(db);
|
|
556
|
+
|
|
532
557
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
533
558
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
534
559
|
rebuildIndexes(db);
|
|
@@ -1491,6 +1516,107 @@ function migrateToV25(db: Database): void {
|
|
|
1491
1516
|
);
|
|
1492
1517
|
}
|
|
1493
1518
|
|
|
1519
|
+
/**
|
|
1520
|
+
* Migrate v25 → v26: integer `notes.updated_at_ms` as the single source of
|
|
1521
|
+
* truth for cursor keyset ordering (vault#586).
|
|
1522
|
+
*
|
|
1523
|
+
* THE BUG this closes: the `(updated_at, id)` cursor keyset used THREE
|
|
1524
|
+
* inconsistent orderings of `updated_at` that agree only when every timestamp
|
|
1525
|
+
* is canonical `.toISOString()` output — TEXT-lexicographic walk order, a
|
|
1526
|
+
* canonical-ISO boundary string compared as TEXT, and a `Date.parse`-derived
|
|
1527
|
+
* millis watermark that read space-form timestamps in LOCAL time. Import
|
|
1528
|
+
* stores frontmatter timestamps VERBATIM, so aged/imported vaults carry
|
|
1529
|
+
* non-canonical `updated_at` (`2024-11-02 14:30:00` space-form, `+02:00`
|
|
1530
|
+
* offset, no-`Z`) — under which the three orderings diverge and cursor
|
|
1531
|
+
* pagination silently skipped notes, re-delivered rows in a loop, or 400'd the
|
|
1532
|
+
* whole walk. A single integer column makes walk-order, keyset boundary, and
|
|
1533
|
+
* watermark ONE numeric ordering.
|
|
1534
|
+
*
|
|
1535
|
+
* Backfill parse (`timestampToMs`) is UTC-correct — it does NOT use
|
|
1536
|
+
* `Date.parse` for zone-less forms (the live bug was space-form read as LOCAL
|
|
1537
|
+
* time). A genuinely unparseable `updated_at` falls back to the row's
|
|
1538
|
+
* `created_at` ms, then to a stable sentinel ({@link UNPARSEABLE_UPDATED_AT_MS})
|
|
1539
|
+
* — never NULL, never a throw.
|
|
1540
|
+
*
|
|
1541
|
+
* ALL-OR-NOTHING (matches the #565 migrateToV25 pattern): the ALTER + backfill
|
|
1542
|
+
* run inside a SINGLE `transaction`. If the ALTER committed but the backfill
|
|
1543
|
+
* then crashed, a column-presence-only guard would see the column on the next
|
|
1544
|
+
* boot and skip — leaving rows with NULL `updated_at_ms` and a permanently
|
|
1545
|
+
* broken keyset. Wrapping the whole sequence means a rollback removes the
|
|
1546
|
+
* column too, so the next boot re-detects "not migrated" and re-runs cleanly.
|
|
1547
|
+
* The index create lives OUTSIDE the wrapped block (`CREATE INDEX IF NOT
|
|
1548
|
+
* EXISTS`) so a fresh vault — column already present from SCHEMA_SQL — still
|
|
1549
|
+
* gets the index.
|
|
1550
|
+
*
|
|
1551
|
+
* SELF-HEALING (generalist review nit 1): the backfill is NOT gated on column
|
|
1552
|
+
* absence — it runs whenever ANY row has a NULL `updated_at_ms`, via a single
|
|
1553
|
+
* `WHERE updated_at_ms IS NULL` pass that serves BOTH the fresh-ALTER case
|
|
1554
|
+
* (every row NULL right after ADD COLUMN) AND the leak case (a row that slipped
|
|
1555
|
+
* in with NULL while the column already existed). The leak is real: a
|
|
1556
|
+
* schema-v2-era vault upgrades through `migrateFromV2`, whose
|
|
1557
|
+
* `INSERT INTO notes … SELECT` omits `updated_at_ms`, landing rows with NULL
|
|
1558
|
+
* even though SCHEMA_SQL created the column — a column-presence guard would
|
|
1559
|
+
* skip them forever (the exact keyset-invisibility class this migration
|
|
1560
|
+
* closes). Steady state (column present, zero NULLs) does a single cheap
|
|
1561
|
+
* COUNT and no writes.
|
|
1562
|
+
*/
|
|
1563
|
+
function migrateToV26(db: Database): void {
|
|
1564
|
+
if (!hasTable(db, "notes")) return;
|
|
1565
|
+
|
|
1566
|
+
const needsColumn = !hasColumn(db, "notes", "updated_at_ms");
|
|
1567
|
+
// Backfill runs when the column is being added OR when any existing row
|
|
1568
|
+
// carries a NULL keyset key (self-heal). A steady-state boot short-circuits
|
|
1569
|
+
// here on the cheap COUNT and touches nothing.
|
|
1570
|
+
let pending = needsColumn;
|
|
1571
|
+
if (!needsColumn) {
|
|
1572
|
+
const nulls = (
|
|
1573
|
+
db.prepare("SELECT COUNT(*) AS n FROM notes WHERE updated_at_ms IS NULL").get() as { n: number }
|
|
1574
|
+
).n;
|
|
1575
|
+
pending = nulls > 0;
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
if (pending) {
|
|
1579
|
+
let backfilled = 0;
|
|
1580
|
+
let fellBack = 0;
|
|
1581
|
+
transaction(db, () => {
|
|
1582
|
+
if (needsColumn) {
|
|
1583
|
+
db.exec("ALTER TABLE notes ADD COLUMN updated_at_ms INTEGER");
|
|
1584
|
+
}
|
|
1585
|
+
// One `WHERE updated_at_ms IS NULL` pass covers both cases: right after
|
|
1586
|
+
// ALTER every row is NULL; on a self-heal only the leaked rows are.
|
|
1587
|
+
const rows = db.prepare(
|
|
1588
|
+
"SELECT id, created_at, updated_at FROM notes WHERE updated_at_ms IS NULL",
|
|
1589
|
+
).all() as { id: string; created_at: string | null; updated_at: string | null }[];
|
|
1590
|
+
const update = db.prepare("UPDATE notes SET updated_at_ms = ? WHERE id = ?");
|
|
1591
|
+
for (const row of rows) {
|
|
1592
|
+
// `updated_at` is the ordering authority; fall back to `created_at`
|
|
1593
|
+
// (legacy rows can carry NULL updated_at), then the stable sentinel.
|
|
1594
|
+
let ms = timestampToMs(row.updated_at);
|
|
1595
|
+
if (ms === null) {
|
|
1596
|
+
ms = timestampToMs(row.created_at);
|
|
1597
|
+
fellBack++;
|
|
1598
|
+
}
|
|
1599
|
+
if (ms === null) ms = UNPARSEABLE_UPDATED_AT_MS;
|
|
1600
|
+
update.run(ms, row.id);
|
|
1601
|
+
backfilled++;
|
|
1602
|
+
}
|
|
1603
|
+
});
|
|
1604
|
+
if (backfilled > 0) {
|
|
1605
|
+
console.log(
|
|
1606
|
+
`[vault] migrated to schema v26 (vault#586): backfilled updated_at_ms for ${backfilled} note(s)` +
|
|
1607
|
+
(needsColumn ? "" : " (self-heal — NULL keyset keys on an existing column)") +
|
|
1608
|
+
(fellBack > 0 ? ` (${fellBack} fell back to created_at / sentinel for an unparseable updated_at)` : "") +
|
|
1609
|
+
`.`,
|
|
1610
|
+
);
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// Index lives outside the ALTER transaction (idx_tokens_vault_name /
|
|
1615
|
+
// idx_notes_updated precedent): a fresh vault has the column from SCHEMA_SQL
|
|
1616
|
+
// but not yet the index, so this ensures it for both paths. Idempotent.
|
|
1617
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_notes_updated_ms ON notes(updated_at_ms, id)");
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1494
1620
|
function hasTable(db: Database, name: string): boolean {
|
|
1495
1621
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
1496
1622
|
return !!row;
|
package/core/src/seed-packs.ts
CHANGED
|
@@ -50,7 +50,7 @@ import type { Store, TagFieldSchema } from "./types.ts";
|
|
|
50
50
|
* here — a description is vault metadata, not a pack note.
|
|
51
51
|
*/
|
|
52
52
|
export const DEFAULT_VAULT_DESCRIPTION =
|
|
53
|
-
'This is a brand-new personal vault — just its starter guides so far; the person it belongs to is probably new to Parachute. Before helping them set it up, read the "Getting Started" note — it\'s your onboarding brief. Short version: start with a conversation, not a filing system; learn how they want to use this and where their notes live today; do one small real thing first. When the vault has real structure, replace this description (vault-info { description: "..." }) with a current picture of what lives here and how to work it — so every future session, yours or another AI\'s, starts oriented.';
|
|
53
|
+
'This is a brand-new personal vault — just its starter guides so far; the person it belongs to is probably new to Parachute. Before helping them set it up, read the "Getting Started" note — it\'s your onboarding brief. Short version: start with a conversation, not a filing system; learn how they want to use this and where their notes live today; do one small real thing first. When the vault has real structure, replace this description (vault-info { description: "..." } — an admin-scope action) with a current picture of what lives here and how to work it — so every future session, yours or another AI\'s, starts oriented.';
|
|
54
54
|
|
|
55
55
|
/**
|
|
56
56
|
* The `vault-info` description a vault gets when it was **imported/restored** —
|
|
@@ -59,7 +59,7 @@ export const DEFAULT_VAULT_DESCRIPTION =
|
|
|
59
59
|
* text with a current picture.
|
|
60
60
|
*/
|
|
61
61
|
export const IMPORTED_VAULT_DESCRIPTION =
|
|
62
|
-
'This vault was imported — it arrived with its own notes, tags, and history. Orient before you write: vault-info { include_stats: true }, list-tags, read a sample, and read "Getting Started" if present. Your first job is to learn the shape that\'s already here and reflect it back to the person — not to propose new structure cold. When you understand it, replace this description (vault-info { description: "..." }) with a current picture.';
|
|
62
|
+
'This vault was imported — it arrived with its own notes, tags, and history. Orient before you write: vault-info { include_stats: true }, list-tags, read a sample, and read "Getting Started" if present. Your first job is to learn the shape that\'s already here and reflect it back to the person — not to propose new structure cold. When you understand it, replace this description (vault-info { description: "..." } — an admin-scope action) with a current picture.';
|
|
63
63
|
|
|
64
64
|
// ---------------------------------------------------------------------------
|
|
65
65
|
// Pack shape
|
|
@@ -392,16 +392,20 @@ description) so the next one picks up where you left off.
|
|
|
392
392
|
### If this vault already has content
|
|
393
393
|
|
|
394
394
|
Some vaults arrive full — an import, a restore, months of use. Then your first
|
|
395
|
-
job is to **learn it, not seed it**: \`vault-info
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
only
|
|
395
|
+
job is to **learn it, not seed it**: \`vault-info\` — a single call now returns
|
|
396
|
+
a compact \`map\` (every tag's note count, every top-level path's note count,
|
|
397
|
+
total notes) alongside the schema catalog, so you get the shape of a strange
|
|
398
|
+
vault in one round trip; add \`include_stats: true\` only if you also want the
|
|
399
|
+
deeper monthly distribution. Then \`list-tags\` and read a sample
|
|
400
|
+
(\`query-notes { search: "..." }\`), and reflect the shape back to the person
|
|
401
|
+
and ask what's missing or wrong. Propose structure only once you can describe
|
|
402
|
+
what's already there.
|
|
399
403
|
|
|
400
404
|
### Close the loop: describe the vault
|
|
401
405
|
|
|
402
406
|
When the first real structure lands, update the vault description —
|
|
403
|
-
\`vault-info { description: "..." }\` — so every future
|
|
404
|
-
AI's) starts oriented: what this vault is for, its main tags, its conventions.
|
|
407
|
+
\`vault-info { description: "..." }\`, an \`admin\`-scope action — so every future
|
|
408
|
+
session (yours or another AI's) starts oriented: what this vault is for, its main tags, its conventions.
|
|
405
409
|
The default description says "brand-new vault"; once that stops being true,
|
|
406
410
|
replace it. Keep it a few sentences (the projection carries the schema detail,
|
|
407
411
|
and this note carries the richer conventions — see "Adapt this note"). Bringing
|
|
@@ -419,16 +423,44 @@ small welcome web and the \`capture\` tag the Notes surface uses; no other
|
|
|
419
423
|
predefined tags or schema. You and the operator design the structure that fits
|
|
420
424
|
*their* life and work. The vault is the engine; the meaning is yours to bring.
|
|
421
425
|
|
|
422
|
-
Core moves
|
|
423
|
-
|
|
424
|
-
- \`query-notes\`
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
-
|
|
426
|
+
Core moves, grouped by the scope each needs — \`tools/list\` shows exactly the
|
|
427
|
+
ones your token can call (see "What your scope allows" below):
|
|
428
|
+
- **Read:** \`query-notes\` (by id/path, by tag, full-text \`search\`, or graph
|
|
429
|
+
\`near\` a note), \`list-tags\`, \`find-path\` (shortest link path between two
|
|
430
|
+
notes), \`vault-info\` (the live schema projection + a compact \`map\` — tag
|
|
431
|
+
counts, top-level path-bucket counts, total notes — so you orient in one
|
|
432
|
+
call), \`doctor\` (read-only integrity scan).
|
|
433
|
+
- **Write:** \`create-note\` / \`update-note\` / \`delete-note\` — author notes,
|
|
434
|
+
single or batch.
|
|
435
|
+
- **Admin:** \`update-tag\` / \`delete-tag\` / \`rename-tag\` / \`merge-tags\` /
|
|
436
|
+
\`prune-schema\` (the tag vocabulary + schemas), \`vault-info { description }\`
|
|
437
|
+
(the vault's own description), \`manage-token\` (mint scoped child tokens).
|
|
428
438
|
|
|
429
439
|
\`[[wikilinks]]\` in note content auto-link to the note at that path — use them
|
|
430
440
|
freely; they resolve even if the target is created later.
|
|
431
441
|
|
|
442
|
+
## What your scope allows
|
|
443
|
+
|
|
444
|
+
Your connection carries one of three scopes, and it shapes what you can do here
|
|
445
|
+
— by design, so the person can grant exactly as much authority as they mean to:
|
|
446
|
+
|
|
447
|
+
- **\`read\`** — explore and answer: query notes, read tags, traverse the graph,
|
|
448
|
+
run \`doctor\`. You can't change anything.
|
|
449
|
+
- **\`write\`** — everything read can do, plus author content: create, update,
|
|
450
|
+
and delete notes. You can capture and edit freely, but you *can't* reshape the
|
|
451
|
+
vault's structure — renaming/merging/deleting tags, editing schemas, and
|
|
452
|
+
rewriting the vault description all need admin.
|
|
453
|
+
- **\`admin\`** — everything, including that restructuring, plus minting scoped
|
|
454
|
+
child tokens.
|
|
455
|
+
|
|
456
|
+
If a tool this guide mentions fails with "Unknown tool" — or a normally-visible
|
|
457
|
+
tool refuses one argument with "Forbidden" (e.g. \`vault-info { description }\`
|
|
458
|
+
for a non-admin) — that's your scope, not a bug: the action sits above your
|
|
459
|
+
tier. Don't fight it: tell the person plainly
|
|
460
|
+
what you'd do with more access ("I can add notes, but reorganizing your tags
|
|
461
|
+
needs admin — want to grant it?") and let them decide. Most people connect with
|
|
462
|
+
admin, so usually you'll have the whole set.
|
|
463
|
+
|
|
432
464
|
## Tags vs paths vs schemas — the design vocabulary
|
|
433
465
|
|
|
434
466
|
These three axes are the heart of vault design. Use the right one for the job:
|
|
@@ -447,7 +479,7 @@ These three axes are the heart of vault design. Use the right one for the job:
|
|
|
447
479
|
path, tags, or both.
|
|
448
480
|
|
|
449
481
|
- **Schemas = typed metadata fields.** Attach a schema to a tag (via
|
|
450
|
-
\`update-tag\`) to declare typed metadata fields — e.g. \`#meeting\` with a
|
|
482
|
+
\`update-tag\`, an \`admin\`-scope move) to declare typed metadata fields — e.g. \`#meeting\` with a
|
|
451
483
|
\`held_on\` date, \`#person\` with an \`email\`. Each field can **optionally** be
|
|
452
484
|
marked \`indexed: true\` to make it **queryable with operators** (\`query-notes
|
|
453
485
|
{ tag: "meeting", metadata: { held_on: { gte: "2026-01-01" } } }\`); indexing
|
|
@@ -503,6 +535,14 @@ Testers of this vault independently reinvented these — start from them instead
|
|
|
503
535
|
text is escaped, not parsed as FTS5 syntax, so punctuation like "didn't" or
|
|
504
536
|
"18.6" just works. Pass \`search_mode: "advanced"\` only when you actually
|
|
505
537
|
want FTS5 boolean/phrase/prefix syntax.
|
|
538
|
+
- **A front-door "Map" note.** Once a vault grows past a few dozen notes,
|
|
539
|
+
keep one note (path \`Map\`, or whatever fits) as a human-legible index —
|
|
540
|
+
the tags and top-level paths that matter and what each means, with
|
|
541
|
+
wikilinks into anchor notes. It's the "why" companion to \`vault-info\`'s
|
|
542
|
+
auto-computed \`map\` (tag/path-bucket counts, the "what's there"): the
|
|
543
|
+
counts tell a fresh AI the shape of the vault in one call, this note tells
|
|
544
|
+
it what that shape MEANS. Update it when the vault's structure shifts;
|
|
545
|
+
link to it from the vault description so it's easy to find.
|
|
506
546
|
|
|
507
547
|
## Write gotchas
|
|
508
548
|
|
package/core/src/store.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
|
-
import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage } from "./types.js";
|
|
2
|
+
import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage, AggregateRow } from "./types.js";
|
|
3
3
|
import { initSchema } from "./schema.js";
|
|
4
4
|
import * as noteOps from "./notes.js";
|
|
5
5
|
import * as linkOps from "./links.js";
|
|
@@ -10,8 +10,14 @@ import {
|
|
|
10
10
|
reconcileDeclaredIndexes,
|
|
11
11
|
type PrunedField,
|
|
12
12
|
} from "./indexed-fields.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
syncWikilinks,
|
|
15
|
+
resolveUnresolvedWikilinks,
|
|
16
|
+
resolveOrQueueLink,
|
|
17
|
+
clearQueuedLink,
|
|
18
|
+
} from "./wikilinks.js";
|
|
14
19
|
import { pathTitle } from "./paths.js";
|
|
20
|
+
import { timestampToMs } from "./cursor.js";
|
|
15
21
|
import { transaction } from "./txn.js";
|
|
16
22
|
import { HookRegistry } from "./hooks.js";
|
|
17
23
|
import {
|
|
@@ -27,6 +33,7 @@ import {
|
|
|
27
33
|
import {
|
|
28
34
|
loadSchemaConfig,
|
|
29
35
|
validateNote as runValidateNote,
|
|
36
|
+
resolveNoteSchemas,
|
|
30
37
|
type ResolvedSchemas,
|
|
31
38
|
type ValidationStatus,
|
|
32
39
|
} from "./schema-defaults.js";
|
|
@@ -104,6 +111,94 @@ export class BunSqliteStore implements Store {
|
|
|
104
111
|
return runValidateNote(this.getSchemaConfig(), note);
|
|
105
112
|
}
|
|
106
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Auto-link sync for `type: "reference"` schema fields
|
|
116
|
+
* (vault#typed-reference-field — see `docs/design/typed-reference-field.md`
|
|
117
|
+
* for the full design + known gaps).
|
|
118
|
+
*
|
|
119
|
+
* A `reference`-typed field is BOTH an indexed value (handled by the
|
|
120
|
+
* ordinary metadata write — no special casing needed there, see
|
|
121
|
+
* `tag-schemas.ts`'s `VALID_FIELD_TYPES`) AND a graph link. This method
|
|
122
|
+
* maintains the second half: for every field the note's EFFECTIVE schema
|
|
123
|
+
* (its own tags + ancestors, same resolution `validateNoteAgainstSchemas`
|
|
124
|
+
* uses) declares `type: "reference"`, resolve the field's current metadata
|
|
125
|
+
* value to a note (reusing the SAME `resolveOrQueueLink` machinery
|
|
126
|
+
* structured `links` entries use — id, then path/title, with lazy
|
|
127
|
+
* forward-ref queueing on a miss) and maintain a `links` edge from this
|
|
128
|
+
* note to that target, `relationship` = the field name.
|
|
129
|
+
*
|
|
130
|
+
* Called from `createNote`/`updateNote`/`createNotes` — the single
|
|
131
|
+
* chokepoint both MCP and REST funnel through — AFTER the note row itself
|
|
132
|
+
* is written, so `note.tags`/`note.metadata` reflect the final state.
|
|
133
|
+
*
|
|
134
|
+
* `priorMetadata` is the note's metadata BEFORE this write (`undefined` on
|
|
135
|
+
* create). For each reference field, if its value is unchanged from
|
|
136
|
+
* `priorMetadata`, nothing is touched — the existing link (if any) already
|
|
137
|
+
* reflects it, and re-running the resolve/queue machinery on every
|
|
138
|
+
* unrelated write (e.g. a content-only edit) would be wasted work. When
|
|
139
|
+
* the value DID change (set, changed, or removed), every existing link +
|
|
140
|
+
* queued forward-ref under that field's relationship name is cleared
|
|
141
|
+
* first, then re-established from the new value — this makes the field's
|
|
142
|
+
* current value the single source of truth for "this field's link"
|
|
143
|
+
* without needing to track the specific prior target.
|
|
144
|
+
*
|
|
145
|
+
* Known gaps (see the design doc): only scalar (`cardinality: "one"`,
|
|
146
|
+
* the default) reference values are linked — an array value is left as a
|
|
147
|
+
* validated-like-string-per-item... actually an array value simply isn't
|
|
148
|
+
* a string, so no link is created for it (the write still succeeds; a
|
|
149
|
+
* `type_mismatch`/`cardinality_mismatch` warning surfaces via the normal
|
|
150
|
+
* `validation_status` path). A tag gaining a `reference` field declaration
|
|
151
|
+
* does NOT retroactively link notes that already carry a matching value —
|
|
152
|
+
* only writes that actually touch the field (going forward) sync.
|
|
153
|
+
*/
|
|
154
|
+
private syncReferenceFieldLinks(
|
|
155
|
+
note: Note,
|
|
156
|
+
priorMetadata: Record<string, unknown> | undefined,
|
|
157
|
+
): void {
|
|
158
|
+
const resolution = resolveNoteSchemas(this.getSchemaConfig(), { tags: note.tags ?? [] });
|
|
159
|
+
if (resolution.mergedFields.size === 0) return;
|
|
160
|
+
|
|
161
|
+
const metadata = note.metadata ?? {};
|
|
162
|
+
const prior = priorMetadata ?? {};
|
|
163
|
+
|
|
164
|
+
for (const [fieldName, { spec }] of resolution.mergedFields) {
|
|
165
|
+
if (spec.type !== "reference") continue;
|
|
166
|
+
|
|
167
|
+
const nextValue = metadata[fieldName];
|
|
168
|
+
const priorValue = prior[fieldName];
|
|
169
|
+
if (nextValue === priorValue) continue; // unchanged — link (if any) already reflects it
|
|
170
|
+
|
|
171
|
+
// Re-establish this field's link from scratch: drop whatever it
|
|
172
|
+
// previously pointed at (a resolved link, and/or a queued forward-ref)
|
|
173
|
+
// before applying the new value. See this method's doc comment for why
|
|
174
|
+
// "clear then recreate" is safe and simpler than tracking the prior
|
|
175
|
+
// resolved target.
|
|
176
|
+
linkOps.deleteLinksBySourceRelationship(this.db, note.id, fieldName);
|
|
177
|
+
clearQueuedLink(this.db, note.id, fieldName);
|
|
178
|
+
|
|
179
|
+
if (typeof nextValue === "string" && nextValue.trim() !== "") {
|
|
180
|
+
// Resolves now, or queues a lazy forward-ref on a miss — same
|
|
181
|
+
// contract as a structured `links` entry (core/src/mcp.ts,
|
|
182
|
+
// src/routes.ts). A queued forward-ref backfills automatically via
|
|
183
|
+
// `resolveUnresolvedWikilinks` the moment a matching note is
|
|
184
|
+
// created, and surfaces to callers today via the existing
|
|
185
|
+
// `has_broken_links`/`broken_links` query-notes filters (both read
|
|
186
|
+
// the same `unresolved_wikilinks` table this queues into) — see the
|
|
187
|
+
// design doc for the follow-up to also surface an inline
|
|
188
|
+
// `unresolved_link`/`ambiguous_link` warning on the create/update
|
|
189
|
+
// response itself. An `"ambiguous"` outcome (vault#570 — the field's
|
|
190
|
+
// value matched ≥2 notes, e.g. two notes sharing an H1 title) is
|
|
191
|
+
// treated the same as a miss here: no link is created, and nothing
|
|
192
|
+
// is queued (mirrors `resolveOrQueueLink`'s own "don't guess"
|
|
193
|
+
// contract for structured links).
|
|
194
|
+
const outcome = resolveOrQueueLink(this.db, note.id, nextValue, fieldName);
|
|
195
|
+
if (outcome.status === "resolved") {
|
|
196
|
+
linkOps.createLink(this.db, note.id, outcome.note_id, fieldName);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
107
202
|
/**
|
|
108
203
|
* Drop the tag-hierarchy cache if the mutated path is in the `_tags/*`
|
|
109
204
|
* namespace. Called from create/update/delete — old path is passed
|
|
@@ -135,6 +230,11 @@ export class BunSqliteStore implements Store {
|
|
|
135
230
|
resolveUnresolvedWikilinks(this.db, note.path, note.id);
|
|
136
231
|
}
|
|
137
232
|
|
|
233
|
+
// Reference-field auto-link (vault#typed-reference-field) — no prior
|
|
234
|
+
// metadata on a fresh create. Cheap no-op when nothing on the note's
|
|
235
|
+
// effective schema declares `type: "reference"`.
|
|
236
|
+
this.syncReferenceFieldLinks(note, undefined);
|
|
237
|
+
|
|
138
238
|
this.invalidateConfigCachesForPath(note.path);
|
|
139
239
|
this.hooks.dispatch("created", note, this);
|
|
140
240
|
|
|
@@ -171,9 +271,17 @@ export class BunSqliteStore implements Store {
|
|
|
171
271
|
},
|
|
172
272
|
): Promise<Note> {
|
|
173
273
|
let oldPath: string | undefined;
|
|
174
|
-
|
|
274
|
+
// Reference-field auto-link sync (vault#typed-reference-field) needs the
|
|
275
|
+
// PRE-write metadata to detect which reference fields actually changed —
|
|
276
|
+
// read it now, before `noteOps.updateNote` overwrites the row. Only
|
|
277
|
+
// needed when this call touches `metadata` at all (a content/tags/path-
|
|
278
|
+
// only update can't have changed a reference field's value, so this read
|
|
279
|
+
// is skipped on the common non-metadata-touching path).
|
|
280
|
+
let priorMetadataForRefs: Record<string, unknown> | undefined;
|
|
281
|
+
if (updates.path !== undefined || updates.metadata !== undefined) {
|
|
175
282
|
const existing = noteOps.getNote(this.db, id);
|
|
176
|
-
oldPath = existing?.path;
|
|
283
|
+
if (updates.path !== undefined) oldPath = existing?.path;
|
|
284
|
+
if (updates.metadata !== undefined) priorMetadataForRefs = existing?.metadata;
|
|
177
285
|
}
|
|
178
286
|
|
|
179
287
|
const note = noteOps.updateNote(this.db, id, updates);
|
|
@@ -192,6 +300,13 @@ export class BunSqliteStore implements Store {
|
|
|
192
300
|
resolveUnresolvedWikilinks(this.db, note.path, id);
|
|
193
301
|
}
|
|
194
302
|
|
|
303
|
+
// Reference-field auto-link sync (vault#typed-reference-field). Only
|
|
304
|
+
// when this call actually touched `metadata` — see the read above for
|
|
305
|
+
// why a content/tags/path-only update is skipped.
|
|
306
|
+
if (updates.metadata !== undefined) {
|
|
307
|
+
this.syncReferenceFieldLinks(note, priorMetadataForRefs);
|
|
308
|
+
}
|
|
309
|
+
|
|
195
310
|
// Invalidate before the hook dispatch so any handler that re-queries
|
|
196
311
|
// the hierarchy from inside its own logic sees post-write state.
|
|
197
312
|
// `metadata` updates can change the `parents` field on a config note
|
|
@@ -246,9 +361,18 @@ export class BunSqliteStore implements Store {
|
|
|
246
361
|
// the importer write a specific historical timestamp. Skips hooks
|
|
247
362
|
// by design: this isn't a user-edit, it's a state restoration.
|
|
248
363
|
// See vault#308 PR 2.
|
|
364
|
+
//
|
|
365
|
+
// This is THE path by which non-canonical timestamps (space-form,
|
|
366
|
+
// `+02:00` offset, no-`Z`) land in a vault — frontmatter is preserved
|
|
367
|
+
// VERBATIM (byte-identical re-export round-trip), so `updated_at` is NOT
|
|
368
|
+
// canonicalized here. The keyset ordering key `updated_at_ms` (vault#586)
|
|
369
|
+
// is derived UTC-correctly from that verbatim value: `timestampToMs` does
|
|
370
|
+
// NOT read space-form as local time. A genuinely unparseable `updated_at`
|
|
371
|
+
// falls back to `created_at`'s ms, then to 0 — never NULL, never a throw.
|
|
372
|
+
const updatedAtMs = timestampToMs(updatedAt) ?? timestampToMs(createdAt) ?? 0;
|
|
249
373
|
this.db
|
|
250
|
-
.prepare("UPDATE notes SET created_at = ?, updated_at = ? WHERE id = ?")
|
|
251
|
-
.run(createdAt, updatedAt, id);
|
|
374
|
+
.prepare("UPDATE notes SET created_at = ?, updated_at = ?, updated_at_ms = ? WHERE id = ?")
|
|
375
|
+
.run(createdAt, updatedAt, updatedAtMs, id);
|
|
252
376
|
}
|
|
253
377
|
|
|
254
378
|
async deleteNote(id: string): Promise<void> {
|
|
@@ -311,6 +435,13 @@ export class BunSqliteStore implements Store {
|
|
|
311
435
|
return noteOps.queryNotesPaged(this.db, this.expandQueryTags(this.normalizeQueryTags(opts)));
|
|
312
436
|
}
|
|
313
437
|
|
|
438
|
+
async aggregateNotes(opts: QueryOpts): Promise<AggregateRow[]> {
|
|
439
|
+
// Same bare-tag normalization + hierarchy expansion `queryNotes` gets —
|
|
440
|
+
// `opts.tags`/`excludeTags` filter the aggregate's input set the same
|
|
441
|
+
// way they'd filter a normal query's result list.
|
|
442
|
+
return noteOps.aggregateNotes(this.db, this.expandQueryTags(this.normalizeQueryTags(opts)));
|
|
443
|
+
}
|
|
444
|
+
|
|
314
445
|
/**
|
|
315
446
|
* If `tags` are present, attach a parallel `_tagsExpanded` array where
|
|
316
447
|
* each input tag is replaced with `{tag} ∪ descendants(tag)`. The SQL
|
|
@@ -554,6 +685,9 @@ export class BunSqliteStore implements Store {
|
|
|
554
685
|
// would leave the hierarchy cache stale until the next singleton
|
|
555
686
|
// write happened to bust it.
|
|
556
687
|
this.invalidateConfigCachesForPath(note.path);
|
|
688
|
+
// Same reference-field auto-link sync as singleton createNote
|
|
689
|
+
// (vault#typed-reference-field) — no prior metadata on a fresh create.
|
|
690
|
+
this.syncReferenceFieldLinks(note, undefined);
|
|
557
691
|
this.hooks.dispatch("created", note, this);
|
|
558
692
|
}
|
|
559
693
|
return notes;
|
|
@@ -735,7 +869,7 @@ export class BunSqliteStore implements Store {
|
|
|
735
869
|
const mapped = indexedFieldOps.mapFieldType(spec.type);
|
|
736
870
|
if (!mapped) {
|
|
737
871
|
throw new indexedFieldOps.IndexedFieldError(
|
|
738
|
-
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean)`,
|
|
872
|
+
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
|
|
739
873
|
);
|
|
740
874
|
}
|
|
741
875
|
// Throws IndexedFieldError on an invalid identifier (e.g. kebab-case).
|
package/core/src/tag-schemas.ts
CHANGED
|
@@ -255,15 +255,23 @@ export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFi
|
|
|
255
255
|
|
|
256
256
|
/**
|
|
257
257
|
* The full recognized vocabulary for `TagFieldSchema.type` — storage/
|
|
258
|
-
* advisory validation accepts all
|
|
259
|
-
* are INDEXABLE (that narrower subset is `indexed-fields.ts`'s
|
|
260
|
-
* enforced separately via `mapFieldType` for `indexed: true`
|
|
261
|
-
* Matches `defaultMatchesType`'s switch and `schema-defaults.ts`'s
|
|
258
|
+
* advisory validation accepts all seven; only `string`/`integer`/`boolean`/
|
|
259
|
+
* `reference` are INDEXABLE (that narrower subset is `indexed-fields.ts`'s
|
|
260
|
+
* `TYPE_MAP`, enforced separately via `mapFieldType` for `indexed: true`
|
|
261
|
+
* fields). Matches `defaultMatchesType`'s switch and `schema-defaults.ts`'s
|
|
262
262
|
* `SchemaField.type` union — kept in lockstep by hand across the two
|
|
263
263
|
* deliberately-decoupled modules (see `validateFieldDefault`'s doc comment
|
|
264
264
|
* for why they don't cross-import).
|
|
265
|
+
*
|
|
266
|
+
* `reference` (vault#typed-reference-field) is a dual-write field type: the
|
|
267
|
+
* value is stored + validated exactly like `string` (an id/path/title the
|
|
268
|
+
* caller passes), AND the write path (`core/src/store.ts` — see
|
|
269
|
+
* `BunSqliteStore.syncReferenceFieldLinks`) additionally resolves that value
|
|
270
|
+
* to a note and maintains a graph `links` edge from this note to the
|
|
271
|
+
* resolved target, with `relationship` set to the field name. See
|
|
272
|
+
* `docs/design/typed-reference-field.md` for the full design + known gaps.
|
|
265
273
|
*/
|
|
266
|
-
export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object"] as const;
|
|
274
|
+
export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object", "reference"] as const;
|
|
267
275
|
|
|
268
276
|
/**
|
|
269
277
|
* Validate that a field's declared `type` is one of the six recognized
|
|
@@ -320,7 +328,7 @@ export function collectOwnFieldDefaultAndTypeViolations(
|
|
|
320
328
|
return violations;
|
|
321
329
|
}
|
|
322
330
|
|
|
323
|
-
/** Same
|
|
331
|
+
/** Same seven-type vocabulary as `TagFieldSchema.type`'s doc comment. Unknown/unset types pass (nothing to check against). */
|
|
324
332
|
function defaultMatchesType(value: unknown, type: string): boolean {
|
|
325
333
|
switch (type) {
|
|
326
334
|
case "string":
|
|
@@ -335,6 +343,11 @@ function defaultMatchesType(value: unknown, type: string): boolean {
|
|
|
335
343
|
return Array.isArray(value);
|
|
336
344
|
case "object":
|
|
337
345
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
346
|
+
// `reference` stores/validates like `string` — the value is the
|
|
347
|
+
// id/path/title the write path resolves to a target note. See
|
|
348
|
+
// VALID_FIELD_TYPES's doc comment.
|
|
349
|
+
case "reference":
|
|
350
|
+
return typeof value === "string";
|
|
338
351
|
default:
|
|
339
352
|
return true;
|
|
340
353
|
}
|
|
@@ -754,7 +767,7 @@ export function collectTagFieldViolations(
|
|
|
754
767
|
violations.push({
|
|
755
768
|
field: fieldName,
|
|
756
769
|
reason: "unsupported_indexed_type",
|
|
757
|
-
message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean)`,
|
|
770
|
+
message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
|
|
758
771
|
});
|
|
759
772
|
} else {
|
|
760
773
|
try {
|