@openparachute/vault 0.7.1 → 0.7.2-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/aggregate.test.ts +23 -0
- package/core/src/contract-taxonomy.test.ts +22 -0
- package/core/src/core.test.ts +70 -4
- package/core/src/cursor-keyset-ms.test.ts +537 -0
- package/core/src/cursor.ts +76 -6
- package/core/src/hooks.ts +9 -0
- package/core/src/mcp.ts +12 -2
- package/core/src/notes.ts +175 -50
- package/core/src/paths.ts +4 -0
- package/core/src/portable-md.test.ts +161 -0
- package/core/src/portable-md.ts +140 -9
- package/core/src/query-operators.ts +56 -0
- package/core/src/schema.ts +128 -2
- package/core/src/seed-packs.ts +39 -13
- package/core/src/store.ts +20 -2
- package/core/src/tag-hierarchy.ts +13 -0
- package/core/src/txn.test.ts +100 -4
- package/core/src/txn.ts +119 -24
- package/core/src/wikilinks.test.ts +175 -0
- package/core/src/wikilinks.ts +151 -10
- package/package.json +1 -1
- package/src/cli.ts +6 -0
- package/src/mirror-import.ts +5 -0
- package/src/routes.ts +170 -7
- package/src/vault.test.ts +227 -1
package/core/src/mcp.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
|
|
|
2
2
|
import type { Store, Note, QueryOpts } from "./types.js";
|
|
3
3
|
import { transactionAsync } from "./txn.js";
|
|
4
4
|
import * as noteOps from "./notes.js";
|
|
5
|
-
import { filterMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError } from "./notes.js";
|
|
5
|
+
import { filterMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError, validatePath } from "./notes.js";
|
|
6
6
|
import { normalizePath } from "./paths.js";
|
|
7
7
|
import { QueryError } from "./query-operators.js";
|
|
8
8
|
import { TAG_EXPAND_MODES, stripTagHash, suggestSimilarTag, type TagExpandMode } from "./tag-hierarchy.js";
|
|
@@ -1269,6 +1269,10 @@ A note's response carries \`existed\` (true/false) whenever ITS \`if_exists\` wa
|
|
|
1269
1269
|
const extension = item.extension !== undefined
|
|
1270
1270
|
? validateExtension(item.extension)
|
|
1271
1271
|
: undefined;
|
|
1272
|
+
// Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
|
|
1273
|
+
// Throws inside the batch transaction — rolls it back, same as a
|
|
1274
|
+
// path conflict; mcp-http's generic error_type mapping → clean 400.
|
|
1275
|
+
validatePath(item.path);
|
|
1272
1276
|
const effectiveExtension = extension ?? "md";
|
|
1273
1277
|
const ifExists = (item.if_exists as string | undefined) ?? "error";
|
|
1274
1278
|
const upsertMode = ifExists === "ignore" || ifExists === "update" || ifExists === "replace";
|
|
@@ -1719,6 +1723,8 @@ Write-attribution (vault#298): every result carries \`createdBy\`/\`createdVia\`
|
|
|
1719
1723
|
const createExt = item.extension !== undefined
|
|
1720
1724
|
? validateExtension(item.extension)
|
|
1721
1725
|
: undefined;
|
|
1726
|
+
// Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
|
|
1727
|
+
validatePath(explicitPath ?? (idLooksLikePath ? idOrPath : undefined));
|
|
1722
1728
|
const createOpts: Parameters<Store["createNote"]>[1] = {
|
|
1723
1729
|
...(idLooksLikePath ? { path: explicitPath ?? idOrPath } : { id: idOrPath, ...(explicitPath !== undefined ? { path: explicitPath } : {}) }),
|
|
1724
1730
|
...(item.tags && Array.isArray((item.tags as any).add)
|
|
@@ -1895,7 +1901,11 @@ Write-attribution (vault#298): every result carries \`createdBy\`/\`createdVia\`
|
|
|
1895
1901
|
if (item.append !== undefined) updates.append = item.append;
|
|
1896
1902
|
if (item.prepend !== undefined) updates.prepend = item.prepend;
|
|
1897
1903
|
}
|
|
1898
|
-
if (item.path !== undefined)
|
|
1904
|
+
if (item.path !== undefined) {
|
|
1905
|
+
// Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
|
|
1906
|
+
validatePath(item.path);
|
|
1907
|
+
updates.path = item.path;
|
|
1908
|
+
}
|
|
1899
1909
|
if (item.extension !== undefined) {
|
|
1900
1910
|
updates.extension = validateExtension(item.extension);
|
|
1901
1911
|
}
|
package/core/src/notes.ts
CHANGED
|
@@ -14,8 +14,7 @@ import {
|
|
|
14
14
|
computeQueryHash,
|
|
15
15
|
decodeCursor,
|
|
16
16
|
encodeCursor,
|
|
17
|
-
|
|
18
|
-
millisToIso,
|
|
17
|
+
timestampToMs,
|
|
19
18
|
type CursorPayload,
|
|
20
19
|
type QueryHashInputs,
|
|
21
20
|
} from "./cursor.js";
|
|
@@ -115,10 +114,18 @@ export function createNote(
|
|
|
115
114
|
// value. Hook-style writes with `skipUpdatedAt` preserve this; real user
|
|
116
115
|
// edits bump it strictly upward, so `updated_at > created_at` still means
|
|
117
116
|
// "user-touched since creation."
|
|
117
|
+
//
|
|
118
|
+
// `updated_at_ms` (vault#586) is the integer keyset-ordering mirror of
|
|
119
|
+
// `updated_at`. Derived from the SAME `createdAt` value with the UTC-correct
|
|
120
|
+
// `timestampToMs`; a `null` (unparseable caller-supplied `created_at`, e.g.
|
|
121
|
+
// via `createNoteRaw` during import) falls back to wall-clock now so a fresh
|
|
122
|
+
// row never lands with a NULL keyset key. Import's `restoreNoteTimestamps`
|
|
123
|
+
// overwrites both columns from the exported bytes immediately after.
|
|
124
|
+
const updatedAtMs = timestampToMs(createdAt) ?? Date.now();
|
|
118
125
|
try {
|
|
119
126
|
db.prepare(
|
|
120
|
-
`INSERT INTO notes (id, content, path, metadata, created_at, updated_at, extension, created_by, created_via, last_updated_by, last_updated_via) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
121
|
-
).run(id, content, path, metadata, createdAt, createdAt, extension, actor, via, actor, via);
|
|
127
|
+
`INSERT INTO notes (id, content, path, metadata, created_at, updated_at, updated_at_ms, extension, created_by, created_via, last_updated_by, last_updated_via) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
128
|
+
).run(id, content, path, metadata, createdAt, createdAt, updatedAtMs, extension, actor, via, actor, via);
|
|
122
129
|
} catch (err) {
|
|
123
130
|
if (path !== null && isPathUniqueError(err)) {
|
|
124
131
|
throw new PathConflictError(path);
|
|
@@ -466,6 +473,63 @@ export function validateExtension(extension: unknown): string {
|
|
|
466
473
|
return extension;
|
|
467
474
|
}
|
|
468
475
|
|
|
476
|
+
/**
|
|
477
|
+
* Thrown by {@link validatePath} when a caller-supplied note `path` can never
|
|
478
|
+
* round-trip safely (vault#589 / FIX 2). Carries a stable `error_type`
|
|
479
|
+
* (`invalid_path`) so REST's catch and the generic MCP domain-error mapping
|
|
480
|
+
* (`src/mcp-http.ts`) both surface a clean 400 — same shape rule as
|
|
481
|
+
* `ExtensionValidationError`.
|
|
482
|
+
*/
|
|
483
|
+
export class PathValidationError extends Error {
|
|
484
|
+
code = "INVALID_PATH" as const;
|
|
485
|
+
error_type = "invalid_path" as const;
|
|
486
|
+
path: string;
|
|
487
|
+
reason: string;
|
|
488
|
+
|
|
489
|
+
constructor(path: string, reason: string) {
|
|
490
|
+
super(`invalid_path: "${path}" ${reason}`);
|
|
491
|
+
this.name = "PathValidationError";
|
|
492
|
+
this.path = path;
|
|
493
|
+
this.reason = reason;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Reject a caller-supplied note path that can never round-trip safely
|
|
499
|
+
* (vault#589 / FIX 2). Two rules, both enforced at the WRITE surface only
|
|
500
|
+
* (REST + MCP create/update, exactly where `validateExtension` sits):
|
|
501
|
+
*
|
|
502
|
+
* 1. **No NUL byte** — never valid in a filename. A NUL-in-path note keeps
|
|
503
|
+
* its resolved target inside the export root (so it slips the traversal
|
|
504
|
+
* guard), then uncaught-throws `writeFileSync`, aborting the ENTIRE vault
|
|
505
|
+
* export for everyone. Rejecting NUL at write is the primary fix; the
|
|
506
|
+
* export sink's per-file try/catch (portable-md.ts) is the belt for rows
|
|
507
|
+
* that predate this guard.
|
|
508
|
+
* 2. **No `..` path segment** — traversal-shaped. Export guard-skips such a
|
|
509
|
+
* note (silently un-round-trippable) and it has no legitimate use as a
|
|
510
|
+
* vault note path.
|
|
511
|
+
*
|
|
512
|
+
* A `null`/empty-after-normalize path is fine — notes need no path. Reads /
|
|
513
|
+
* queries never call this, so a lookup by a `..`/NUL path degrades to
|
|
514
|
+
* not-found rather than throwing. The Store itself still trusts internal /
|
|
515
|
+
* importer writes (mirrors the `validateExtension` split). Throws
|
|
516
|
+
* `PathValidationError`.
|
|
517
|
+
*/
|
|
518
|
+
export function validatePath(path: unknown): void {
|
|
519
|
+
if (path === null || path === undefined) return;
|
|
520
|
+
if (typeof path !== "string") return; // non-string: not a path; caller's shape check owns it
|
|
521
|
+
// NUL check on the RAW value — `normalizePath` strips NUL, so this must run
|
|
522
|
+
// before normalization to actually reject rather than silently clean.
|
|
523
|
+
if (path.includes("\0")) {
|
|
524
|
+
throw new PathValidationError(path, "contains a NUL byte");
|
|
525
|
+
}
|
|
526
|
+
const normalized = normalizePath(path);
|
|
527
|
+
if (normalized === null) return;
|
|
528
|
+
if (normalized.split("/").some((seg) => seg === "..")) {
|
|
529
|
+
throw new PathValidationError(path, "contains a '..' path segment (path traversal)");
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
469
533
|
/**
|
|
470
534
|
* Match bun:sqlite's UNIQUE-constraint error on the notes path index.
|
|
471
535
|
* Post-vault#328 the unique index is composite `(path, extension)`, so
|
|
@@ -542,7 +606,9 @@ export function updateNote(
|
|
|
542
606
|
// and path has been removed.
|
|
543
607
|
|
|
544
608
|
const sets: string[] = [];
|
|
545
|
-
|
|
609
|
+
// `updated_at_ms` binds as a number (INTEGER column), so the value list
|
|
610
|
+
// carries numbers alongside strings/nulls (vault#586).
|
|
611
|
+
const values: (string | number | null)[] = [];
|
|
546
612
|
|
|
547
613
|
// Hooks and other machine-level writers pass `skipUpdatedAt: true` so
|
|
548
614
|
// their metadata markers don't look like user activity. See issue #44.
|
|
@@ -560,6 +626,12 @@ export function updateNote(
|
|
|
560
626
|
}
|
|
561
627
|
sets.push("updated_at = ?");
|
|
562
628
|
values.push(now);
|
|
629
|
+
// `updated_at_ms` (vault#586) rides the SAME gate as `updated_at` — the
|
|
630
|
+
// integer keyset-ordering mirror must move in lockstep with the string.
|
|
631
|
+
// `now` is canonical `.toISOString()`, so `timestampToMs` never returns
|
|
632
|
+
// null here; the fallback is belt-and-suspenders.
|
|
633
|
+
sets.push("updated_at_ms = ?");
|
|
634
|
+
values.push(timestampToMs(now) ?? Date.now());
|
|
563
635
|
|
|
564
636
|
// Write-attribution (vault#298): the most-recent-write columns ride the
|
|
565
637
|
// SAME gate as `updated_at`. A `skipUpdatedAt` machine write doesn't bump
|
|
@@ -1100,6 +1172,11 @@ function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: str
|
|
|
1100
1172
|
if (field === "created_at") {
|
|
1101
1173
|
column = "n.created_at";
|
|
1102
1174
|
} else if (field === "updated_at") {
|
|
1175
|
+
// NOTE (vault#586 follow-up): this range filter compares the TEXT
|
|
1176
|
+
// `updated_at`, which is inconsistent on non-canonical rows the same way
|
|
1177
|
+
// the cursor keyset was pre-v26. Cursor pagination moved to the integer
|
|
1178
|
+
// `updated_at_ms` column; `date_filter`/`order_by`/export `--since` still
|
|
1179
|
+
// read the string and are tracked separately. Behavior unchanged here.
|
|
1103
1180
|
column = "n.updated_at";
|
|
1104
1181
|
} else {
|
|
1105
1182
|
// Re-uses the same indexed-field gate as `metadata` operator queries
|
|
@@ -1133,11 +1210,19 @@ function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: str
|
|
|
1133
1210
|
return { conditions, params };
|
|
1134
1211
|
}
|
|
1135
1212
|
|
|
1136
|
-
|
|
1213
|
+
/**
|
|
1214
|
+
* `_outUpdatedAtMs` (internal, vault#586) — when supplied, the phase-1 page
|
|
1215
|
+
* query writes each returned id's integer `updated_at_ms` keyset key into it.
|
|
1216
|
+
* Only `queryNotesPaged` passes it, to derive the cursor watermark from the
|
|
1217
|
+
* SAME statement that determined page membership + order (one consistent
|
|
1218
|
+
* snapshot — see the call site). Ordinary callers omit it and are unaffected;
|
|
1219
|
+
* it never touches the returned `Note` shape.
|
|
1220
|
+
*/
|
|
1221
|
+
export function queryNotes(db: Database, opts: QueryOpts, _outUpdatedAtMs?: Map<string, number>): Note[] {
|
|
1137
1222
|
validateLimitOffset(opts);
|
|
1138
1223
|
const { conditions, params } = buildFilterConditions(db, opts);
|
|
1139
1224
|
|
|
1140
|
-
// ---- Cursor predicate (vault#313) ----
|
|
1225
|
+
// ---- Cursor predicate (vault#313; keyset column vault#586) ----
|
|
1141
1226
|
//
|
|
1142
1227
|
// Cursor mode is keyed on PRESENCE of `opts.cursor`, not truthiness
|
|
1143
1228
|
// (vault#550 bootstrap fix). `cursor: ""` is the bootstrap call — "I want
|
|
@@ -1148,24 +1233,38 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1148
1233
|
// Before this fix, `if (opts.cursor)` treated an empty string exactly
|
|
1149
1234
|
// like "no cursor" — the caller's bootstrap intent silently vanished and
|
|
1150
1235
|
// the first page came back in `created_at` order instead of the
|
|
1151
|
-
//
|
|
1236
|
+
// keyset order the SECOND page (a real cursor) would use,
|
|
1152
1237
|
// so naive "did I see this note already" comparisons could skip or
|
|
1153
1238
|
// duplicate rows across the boundary.
|
|
1154
1239
|
//
|
|
1240
|
+
// The keyset orders on the integer `n.updated_at_ms` column (vault#586),
|
|
1241
|
+
// NOT the TEXT `n.updated_at`. That column is the single source of truth
|
|
1242
|
+
// for cursor ordering: walk-order, the boundary predicate below, and the
|
|
1243
|
+
// watermark in `queryNotesPaged` all read the SAME integer, so they can't
|
|
1244
|
+
// diverge the way three separate readings of `updated_at` did on
|
|
1245
|
+
// aged/imported vaults whose timestamps aren't canonical `.toISOString()`
|
|
1246
|
+
// (space-form / offset / no-`Z`) — under which TEXT-lex order, an ISO
|
|
1247
|
+
// boundary string, and a `Date.parse` watermark disagreed and silently
|
|
1248
|
+
// skipped or re-delivered rows.
|
|
1249
|
+
//
|
|
1155
1250
|
// When a REAL (non-empty) cursor is present, decode it, verify its
|
|
1156
1251
|
// query_hash matches the current query, and add a keyset predicate of
|
|
1157
1252
|
// the form:
|
|
1158
1253
|
//
|
|
1159
|
-
// (
|
|
1160
|
-
// OR (
|
|
1254
|
+
// (updated_at_ms > last_updated_at)
|
|
1255
|
+
// OR (updated_at_ms = last_updated_at AND id > last_id)
|
|
1161
1256
|
//
|
|
1162
|
-
// The cursor
|
|
1163
|
-
//
|
|
1164
|
-
//
|
|
1165
|
-
//
|
|
1166
|
-
//
|
|
1167
|
-
//
|
|
1168
|
-
//
|
|
1257
|
+
// The cursor payload's `last_updated_at` is ALREADY a millisecond epoch
|
|
1258
|
+
// (see cursor.ts) — the same units as the column — so it binds directly
|
|
1259
|
+
// with no ISO round-trip. This is also why existing client cursors keep
|
|
1260
|
+
// working unchanged across the upgrade: the encoded watermark was always
|
|
1261
|
+
// ms. The cursor forces ORDER BY n.updated_at_ms ASC, n.id ASC so the
|
|
1262
|
+
// watermark math is sound — paginating by update time while ordering by
|
|
1263
|
+
// created_at would skip rows whose update differs from their creation.
|
|
1264
|
+
// `orderBy` and `sort: "desc"` are mutually exclusive with cursor mode (a
|
|
1265
|
+
// "since last checked" loop wants ascending update order, full stop); we
|
|
1266
|
+
// reject with INVALID_QUERY so callers don't silently get a broken
|
|
1267
|
+
// iteration.
|
|
1169
1268
|
const cursorMode = opts.cursor !== undefined;
|
|
1170
1269
|
let cursorPayload: CursorPayload | null = null;
|
|
1171
1270
|
if (cursorMode) {
|
|
@@ -1190,20 +1289,17 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1190
1289
|
"cursor_query_mismatch",
|
|
1191
1290
|
);
|
|
1192
1291
|
}
|
|
1193
|
-
//
|
|
1194
|
-
//
|
|
1195
|
-
//
|
|
1196
|
-
//
|
|
1197
|
-
//
|
|
1198
|
-
//
|
|
1199
|
-
|
|
1200
|
-
// safe: we round-trip the cursor's millis through `new Date()`'s
|
|
1201
|
-
// canonical ISO so the comparison is apples-to-apples.
|
|
1202
|
-
const cursorIso = millisToIso(cursorPayload.last_updated_at);
|
|
1292
|
+
// The cursor's `last_updated_at` is a millisecond epoch (cursor.ts) —
|
|
1293
|
+
// the SAME units as the integer `n.updated_at_ms` column — so it binds
|
|
1294
|
+
// straight into the keyset predicate with no ISO round-trip. Ordering
|
|
1295
|
+
// and boundary now read one numeric column, so heterogeneous /
|
|
1296
|
+
// non-canonical `updated_at` TEXT (space-form, offset, no-`Z`) can no
|
|
1297
|
+
// longer make the walk and the watermark disagree.
|
|
1298
|
+
const cursorMs = cursorPayload.last_updated_at;
|
|
1203
1299
|
conditions.push(
|
|
1204
|
-
"(n.
|
|
1300
|
+
"(n.updated_at_ms > ? OR (n.updated_at_ms = ? AND n.id > ?))",
|
|
1205
1301
|
);
|
|
1206
|
-
params.push(
|
|
1302
|
+
params.push(cursorMs, cursorMs, cursorPayload.last_id);
|
|
1207
1303
|
}
|
|
1208
1304
|
// else: bootstrap call (`cursor === ""`) — no watermark yet, no
|
|
1209
1305
|
// predicate to add, but the ORDER BY below still switches to the
|
|
@@ -1214,11 +1310,12 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1214
1310
|
const direction = opts.sort === "desc" ? "DESC" : "ASC";
|
|
1215
1311
|
let orderBy: string;
|
|
1216
1312
|
if (cursorMode) {
|
|
1217
|
-
// Cursor mode forces a deterministic keyset order
|
|
1218
|
-
//
|
|
1219
|
-
//
|
|
1220
|
-
//
|
|
1221
|
-
|
|
1313
|
+
// Cursor mode forces a deterministic keyset order on the integer
|
|
1314
|
+
// `updated_at_ms` column (vault#586), matching the boundary predicate
|
|
1315
|
+
// above and the `idx_notes_updated_ms` index. `id` is the tiebreaker —
|
|
1316
|
+
// without it, two notes sharing an `updated_at_ms` would be at the mercy
|
|
1317
|
+
// of SQLite's row order and the next page could miss or duplicate one.
|
|
1318
|
+
orderBy = "n.updated_at_ms ASC, n.id ASC";
|
|
1222
1319
|
} else if (opts.orderBy === "link_count") {
|
|
1223
1320
|
// `link_count` is a pseudo-field — like `created_at`/`updated_at` in the
|
|
1224
1321
|
// dateFilter block above, it bypasses `requireIndexedField` (it's not a
|
|
@@ -1273,15 +1370,26 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1273
1370
|
// Phase 2 fetches full rows for just the page (≤ limit ids) and re-orders
|
|
1274
1371
|
// to the phase-1 order; tags are hydrated in ONE batched query instead of
|
|
1275
1372
|
// one query per returned note.
|
|
1373
|
+
// Phase 1 also selects `n.updated_at_ms` (vault#586) so a paging caller can
|
|
1374
|
+
// read the keyset key from the SAME statement that applied the keyset order
|
|
1375
|
+
// — the watermark then comes from one consistent snapshot, immune to a
|
|
1376
|
+
// cross-process writer bumping a page row's ms between two separate reads.
|
|
1276
1377
|
const idSql = `
|
|
1277
|
-
SELECT n.id FROM notes n
|
|
1378
|
+
SELECT n.id, n.updated_at_ms FROM notes n
|
|
1278
1379
|
${whereClause}
|
|
1279
1380
|
ORDER BY ${orderBy}
|
|
1280
1381
|
LIMIT ? OFFSET ?
|
|
1281
1382
|
`;
|
|
1282
1383
|
params.push(limit, offset);
|
|
1283
1384
|
|
|
1284
|
-
const idRows = db.prepare(idSql).all(...params) as { id: string }[];
|
|
1385
|
+
const idRows = db.prepare(idSql).all(...params) as { id: string; updated_at_ms: number | null }[];
|
|
1386
|
+
if (_outUpdatedAtMs) {
|
|
1387
|
+
for (const r of idRows) {
|
|
1388
|
+
if (r.updated_at_ms !== null && r.updated_at_ms !== undefined) {
|
|
1389
|
+
_outUpdatedAtMs.set(r.id, r.updated_at_ms);
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1285
1393
|
return fetchNotesByIdsOrdered(db, idRows.map((r) => r.id));
|
|
1286
1394
|
}
|
|
1287
1395
|
|
|
@@ -1517,7 +1625,12 @@ function toQueryHashInputs(opts: QueryOpts): QueryHashInputs {
|
|
|
1517
1625
|
* with a non-null `updated_at` greater than the unix epoch).
|
|
1518
1626
|
*/
|
|
1519
1627
|
export function queryNotesPaged(db: Database, opts: QueryOpts): QueryNotesPage {
|
|
1520
|
-
|
|
1628
|
+
// Capture each page row's integer keyset key (vault#586) from the SAME
|
|
1629
|
+
// phase-1 statement that ordered + sliced the page — one consistent
|
|
1630
|
+
// snapshot, so the watermark can't leapfrog rows a cross-process writer
|
|
1631
|
+
// bumps between reads. This replaces a separate post-query round-trip.
|
|
1632
|
+
const updatedAtMsById = new Map<string, number>();
|
|
1633
|
+
const notes = queryNotes(db, opts, updatedAtMsById);
|
|
1521
1634
|
const queryHash = computeQueryHash(toQueryHashInputs(opts));
|
|
1522
1635
|
|
|
1523
1636
|
// Watermark math: pick the larger of (last returned row, prior cursor
|
|
@@ -1536,14 +1649,20 @@ export function queryNotesPaged(db: Database, opts: QueryOpts): QueryNotesPage {
|
|
|
1536
1649
|
lastId = prior.last_id;
|
|
1537
1650
|
}
|
|
1538
1651
|
if (notes.length > 0) {
|
|
1539
|
-
//
|
|
1540
|
-
//
|
|
1541
|
-
//
|
|
1542
|
-
//
|
|
1543
|
-
//
|
|
1652
|
+
// Advance the watermark to the page's MAX (updated_at_ms, id) using the
|
|
1653
|
+
// keyset keys captured in the phase-1 snapshot above (vault#586) — the
|
|
1654
|
+
// SAME integer `updated_at_ms` the walk-order used, from the SAME read.
|
|
1655
|
+
// Using the snapshot, rather than re-deriving ms from each note's
|
|
1656
|
+
// `updatedAt` string or re-reading the column, is what keeps walk-order
|
|
1657
|
+
// and watermark from diverging: a row whose backfilled column and
|
|
1658
|
+
// non-canonical `updated_at` TEXT disagree would otherwise re-parse to a
|
|
1659
|
+
// different ms here and skip or re-deliver at the boundary. No throw — a
|
|
1660
|
+
// NULL key (shouldn't occur post-migrateToV26) reads as 0. When a cursor
|
|
1661
|
+
// is in effect the SQL already returns rows in (updated_at_ms, id) order
|
|
1662
|
+
// so the last row IS the max; the explicit max also covers the
|
|
1663
|
+
// created_at-ordered no-cursor path.
|
|
1544
1664
|
for (const note of notes) {
|
|
1545
|
-
const
|
|
1546
|
-
const ms = isoToMillis(updatedIso);
|
|
1665
|
+
const ms = updatedAtMsById.get(note.id) ?? 0;
|
|
1547
1666
|
if (ms > lastUpdatedAt || (ms === lastUpdatedAt && note.id > lastId)) {
|
|
1548
1667
|
lastUpdatedAt = ms;
|
|
1549
1668
|
lastId = note.id;
|
|
@@ -2041,6 +2160,10 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2041
2160
|
// note_tags FK on `tag_name` has no ON DELETE, so the delete must
|
|
2042
2161
|
// come AFTER the repoint.
|
|
2043
2162
|
const now = new Date().toISOString();
|
|
2163
|
+
// Integer keyset mirror of `now` for the note `updated_at_ms` bumps in the
|
|
2164
|
+
// content/path rewrite passes below (vault#586). `now` is canonical, so
|
|
2165
|
+
// `timestampToMs` never returns null here.
|
|
2166
|
+
const nowMs = timestampToMs(now) ?? Date.now();
|
|
2044
2167
|
const readStmt = db.prepare(
|
|
2045
2168
|
"SELECT description, fields, relationships, parent_names, created_at FROM tags WHERE name = ?",
|
|
2046
2169
|
);
|
|
@@ -2168,12 +2291,13 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2168
2291
|
// `#newtag`) and must bump `updated_at` like any other content
|
|
2169
2292
|
// write, or a cursor/sync-poll loop never sees it. Shares the one
|
|
2170
2293
|
// `now` timestamp for the whole cascade (same convention as the
|
|
2171
|
-
// tag-row rename pass above).
|
|
2172
|
-
|
|
2294
|
+
// tag-row rename pass above). `updated_at_ms` moves with it (vault#586)
|
|
2295
|
+
// so the cursor keyset actually surfaces the rewrite.
|
|
2296
|
+
const updateStmt = db.prepare("UPDATE notes SET content = ?, updated_at = ?, updated_at_ms = ? WHERE id = ?");
|
|
2173
2297
|
for (const row of candidates) {
|
|
2174
2298
|
const next = rewriteNoteBody(row.content, renames);
|
|
2175
2299
|
if (next === row.content) continue;
|
|
2176
|
-
updateStmt.run(next, now, row.id);
|
|
2300
|
+
updateStmt.run(next, now, nowMs, row.id);
|
|
2177
2301
|
notesRewritten++;
|
|
2178
2302
|
}
|
|
2179
2303
|
}
|
|
@@ -2189,12 +2313,13 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2189
2313
|
.prepare(`SELECT id, path FROM notes WHERE path IS NOT NULL AND (${orClauses})`)
|
|
2190
2314
|
.all(...params) as { id: string; path: string }[];
|
|
2191
2315
|
// vault#555 fix 2 — same reasoning as the content rewrite above: a
|
|
2192
|
-
// path rewrite is a real persisted-state change.
|
|
2193
|
-
|
|
2316
|
+
// path rewrite is a real persisted-state change. `updated_at_ms` moves
|
|
2317
|
+
// with `updated_at` (vault#586).
|
|
2318
|
+
const updateStmt = db.prepare("UPDATE notes SET path = ?, updated_at = ?, updated_at_ms = ? WHERE id = ?");
|
|
2194
2319
|
for (const row of candidates) {
|
|
2195
2320
|
const next = rewriteTagConfigPath(row.path, renames);
|
|
2196
2321
|
if (next === row.path) continue;
|
|
2197
|
-
updateStmt.run(next, now, row.id);
|
|
2322
|
+
updateStmt.run(next, now, nowMs, row.id);
|
|
2198
2323
|
pathsRenamed++;
|
|
2199
2324
|
}
|
|
2200
2325
|
}
|
package/core/src/paths.ts
CHANGED
|
@@ -19,6 +19,10 @@ export function normalizePath(path: string | null | undefined): string | null {
|
|
|
19
19
|
if (path === null || path === undefined) return null;
|
|
20
20
|
|
|
21
21
|
let p = path
|
|
22
|
+
.replace(/\0/g, "") // strip NUL bytes — never valid in a path;
|
|
23
|
+
// belt for legacy/imported rows so a stored
|
|
24
|
+
// NUL can't reach the filesystem (writes
|
|
25
|
+
// reject NUL up front via validatePath)
|
|
22
26
|
.trim()
|
|
23
27
|
.replace(/\\/g, "/") // backslash → forward slash
|
|
24
28
|
.replace(/\.md$/i, "") // strip .md extension
|
|
@@ -29,6 +29,7 @@ import { buildVaultProjection } from "./vault-projection.js";
|
|
|
29
29
|
import {
|
|
30
30
|
CaseCollisionError,
|
|
31
31
|
emitYamlDoc,
|
|
32
|
+
exportVault,
|
|
32
33
|
exportVaultToDir,
|
|
33
34
|
importPortableVault,
|
|
34
35
|
noteToPortable,
|
|
@@ -41,7 +42,9 @@ import {
|
|
|
41
42
|
supportsInlineFrontmatter,
|
|
42
43
|
toPortableMarkdown,
|
|
43
44
|
toSidecarYaml,
|
|
45
|
+
type ExportSink,
|
|
44
46
|
type PortableNote,
|
|
47
|
+
type SinkWriteResult,
|
|
45
48
|
} from "./portable-md.js";
|
|
46
49
|
|
|
47
50
|
// ---------------------------------------------------------------------------
|
|
@@ -416,6 +419,86 @@ describe("exportVaultToDir", async () => {
|
|
|
416
419
|
expect(vault).toContain("name: test");
|
|
417
420
|
});
|
|
418
421
|
|
|
422
|
+
// FIX 2(b) (vault#589) — a NUL-in-path note (a pre-fix poisoned row a
|
|
423
|
+
// write-capable token could have planted) must be SKIPPED with a warning +
|
|
424
|
+
// stat, never abort the whole export. Before the sink hardening the
|
|
425
|
+
// uncaught `writeFileSync` throw aborted the entire vault export, so one bad
|
|
426
|
+
// note broke backup/export for everyone.
|
|
427
|
+
//
|
|
428
|
+
// RED-without-fix: drop the try/catch from `FsExportSink.writeText` and this
|
|
429
|
+
// test's `exportVaultToDir` call throws (whole export aborts) instead of
|
|
430
|
+
// returning a partial-with-skip result. Verified manually during development.
|
|
431
|
+
it("hardened export sink: a NUL-in-path note is skipped, not fatal (vault#589)", async () => {
|
|
432
|
+
await store.createNote("clean one", { id: "c1", path: "clean/one" });
|
|
433
|
+
await store.createNote("clean two", { id: "c2", path: "clean/two" });
|
|
434
|
+
await store.createNote("poison", { id: "p1", path: "temp" });
|
|
435
|
+
// Simulate a pre-fix vault: inject a real NUL directly into the stored
|
|
436
|
+
// path, bypassing normalizePath (which now strips NUL). `char(0)` yields a
|
|
437
|
+
// NUL byte in the TEXT column that survives read-back into note.path.
|
|
438
|
+
store.db.prepare("UPDATE notes SET path = 'bad' || char(0) || 'path' WHERE id = ?").run("p1");
|
|
439
|
+
|
|
440
|
+
const outDir = join(tmpBase, "poisoned-out");
|
|
441
|
+
const stats = await exportVaultToDir(store, { outDir, exportedAt: "2026-05-13T00:00:00.000Z" });
|
|
442
|
+
|
|
443
|
+
// The export completes and every OTHER note is written ...
|
|
444
|
+
expect(existsSync(join(outDir, "clean/one.md"))).toBe(true);
|
|
445
|
+
expect(existsSync(join(outDir, "clean/two.md"))).toBe(true);
|
|
446
|
+
expect(stats.notes).toBe(2); // only the two clean notes counted as written
|
|
447
|
+
|
|
448
|
+
// ... while the poisoned note is skipped-with-reason (fs write failure),
|
|
449
|
+
// surfaced as a stat rather than a thrown, aborted export.
|
|
450
|
+
expect(stats.skipped_traversal).toBeGreaterThanOrEqual(1);
|
|
451
|
+
const poisonSkip = stats.skipped_notes.find((s) => s.reason.includes("fs write failed"));
|
|
452
|
+
expect(poisonSkip).toBeTruthy();
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
// BLOCKER (vault#589 review) — the skip-with-warn export belt (FIX 2b) is
|
|
456
|
+
// correct ONLY for per-NOTE writes. The STRUCTURAL writes (the vault.yaml
|
|
457
|
+
// manifest + per-tag schema sidecars) are global: a dir missing
|
|
458
|
+
// `.parachute/vault.yaml` is hard-rejected at restore/import, so a discarded
|
|
459
|
+
// `{ok:false}` there would produce a manifest-less "success" — a silently
|
|
460
|
+
// broken backup. Those must FAIL LOUDLY.
|
|
461
|
+
//
|
|
462
|
+
// RED-without-fix: with the structural writes discarding the sink result,
|
|
463
|
+
// both `exportVault` calls RESOLVE (no throw) and the manifest/schema is
|
|
464
|
+
// silently absent. Verified manually during development.
|
|
465
|
+
it("aborts loudly when the manifest write fails (structural, not skip-with-warn)", async () => {
|
|
466
|
+
await store.createNote("n", { id: "n1", path: "p" });
|
|
467
|
+
// Stub sink that fails ONLY the manifest write; everything else succeeds.
|
|
468
|
+
const sink: ExportSink = {
|
|
469
|
+
caseSensitive: true,
|
|
470
|
+
attachmentsEnabled: false,
|
|
471
|
+
writeText(relPath: string): SinkWriteResult {
|
|
472
|
+
if (relPath === join(SIDECAR_DIR, "vault.yaml")) {
|
|
473
|
+
return { ok: false, reason: "simulated EACCES on .parachute/" };
|
|
474
|
+
}
|
|
475
|
+
return { ok: true };
|
|
476
|
+
},
|
|
477
|
+
copyAttachment(): SinkWriteResult { return { ok: true }; },
|
|
478
|
+
};
|
|
479
|
+
await expect(exportVault(store, sink, { exportedAt: "2026-05-13T00:00:00.000Z" }))
|
|
480
|
+
.rejects.toThrow(/manifest/i);
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
it("aborts loudly when a schema sidecar write fails (structural, not skip-with-warn)", async () => {
|
|
484
|
+
await store.upsertTagSchema("task", { description: "A unit of work" });
|
|
485
|
+
await store.createNote("x", { id: "x1", tags: ["task"] });
|
|
486
|
+
// Fail ONLY the schema write; the manifest (and everything else) succeeds.
|
|
487
|
+
const sink: ExportSink = {
|
|
488
|
+
caseSensitive: true,
|
|
489
|
+
attachmentsEnabled: false,
|
|
490
|
+
writeText(relPath: string): SinkWriteResult {
|
|
491
|
+
if (relPath.startsWith(join(SIDECAR_DIR, "schemas"))) {
|
|
492
|
+
return { ok: false, reason: "simulated disk full" };
|
|
493
|
+
}
|
|
494
|
+
return { ok: true };
|
|
495
|
+
},
|
|
496
|
+
copyAttachment(): SinkWriteResult { return { ok: true }; },
|
|
497
|
+
};
|
|
498
|
+
await expect(exportVault(store, sink, { exportedAt: "2026-05-13T00:00:00.000Z" }))
|
|
499
|
+
.rejects.toThrow(/schema sidecar/i);
|
|
500
|
+
});
|
|
501
|
+
|
|
419
502
|
it("writes per-tag schemas to .parachute/schemas/", async () => {
|
|
420
503
|
await store.upsertTagSchema("task", {
|
|
421
504
|
description: "A unit of work",
|
|
@@ -710,6 +793,84 @@ describe("importPortableVault", async () => {
|
|
|
710
793
|
expect(await targetStore.getNote("k1")).toBeTruthy();
|
|
711
794
|
});
|
|
712
795
|
|
|
796
|
+
// FIX 1 (vault#589) — blow-away must be all-or-nothing. Before the fix, the
|
|
797
|
+
// wipe + replay ran with NO enclosing transaction: a mid-replay throw (here a
|
|
798
|
+
// PathConflictError from two source files declaring the same `path`) left the
|
|
799
|
+
// vault WIPED + partially replayed, originals gone for good. The wrap rolls
|
|
800
|
+
// the whole thing back to the exact pre-import vault.
|
|
801
|
+
//
|
|
802
|
+
// RED-without-fix: temporarily change `importVault` to always call
|
|
803
|
+
// `importVaultReplay` unwrapped (drop the `transactionAsync`) and this test
|
|
804
|
+
// ends with 1 note (`n-first`), all three originals gone — the assertions
|
|
805
|
+
// below fail. Verified manually during development.
|
|
806
|
+
it("--blow-away is atomic — a mid-replay throw rolls back to the original vault (vault#589)", async () => {
|
|
807
|
+
// Target vault holds originals that MUST survive a failed restore.
|
|
808
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
809
|
+
await target.createNote("original one", { id: "orig1", path: "keep/one" });
|
|
810
|
+
await target.createNote("original two", { id: "orig2", path: "keep/two" });
|
|
811
|
+
await target.createNote("original three", { id: "orig3", path: "keep/three" });
|
|
812
|
+
|
|
813
|
+
// Hand-build a poisoned export: two notes declaring the SAME frontmatter
|
|
814
|
+
// `path` → the 2nd createNote throws PathConflictError mid-replay. Files
|
|
815
|
+
// walk in sorted order (first.md before second.md), so first.md lands, then
|
|
816
|
+
// second.md collides.
|
|
817
|
+
const inDir = join(tmpBase, "poison-collide");
|
|
818
|
+
mkdirSync(join(inDir, ".parachute"), { recursive: true });
|
|
819
|
+
writeFileSync(join(inDir, ".parachute", "vault.yaml"), "name: poison\nexport_format_version: 1\n");
|
|
820
|
+
writeFileSync(join(inDir, "first.md"), "---\nid: nfirst\npath: collide\n---\nfirst body\n");
|
|
821
|
+
writeFileSync(join(inDir, "second.md"), "---\nid: nsecond\npath: collide\n---\nsecond body\n");
|
|
822
|
+
|
|
823
|
+
// The blow-away import propagates the collision (rejects) ...
|
|
824
|
+
await expect(importPortableVault(target, { inDir, blowAway: true })).rejects.toThrow();
|
|
825
|
+
|
|
826
|
+
// ... and rolls back: all three ORIGINALS survive, and NONE of the
|
|
827
|
+
// poisoned import's notes landed. The vault is exactly as it was.
|
|
828
|
+
expect(await target.getNote("orig1")).toBeTruthy();
|
|
829
|
+
expect(await target.getNote("orig2")).toBeTruthy();
|
|
830
|
+
expect(await target.getNote("orig3")).toBeTruthy();
|
|
831
|
+
expect(await target.getNote("nfirst")).toBeNull();
|
|
832
|
+
expect(await target.getNote("nsecond")).toBeNull();
|
|
833
|
+
const all = await target.queryNotes({ limit: 100 });
|
|
834
|
+
expect(all.length).toBe(3);
|
|
835
|
+
});
|
|
836
|
+
|
|
837
|
+
// FIX 3 (vault#589) — duplicate-id and blank-id content files must be
|
|
838
|
+
// reported + skipped, not silently clobber (last-wins) or seed a bogus key.
|
|
839
|
+
it("reports duplicate + blank note ids instead of silently clobbering (vault#589)", async () => {
|
|
840
|
+
// Hand-build a tree: two files share id "dup" (distinct content); a third
|
|
841
|
+
// carries a whitespace-only id. Files walk sorted: aaa < bbb < ccc.
|
|
842
|
+
const inDir = join(tmpBase, "dup-ids");
|
|
843
|
+
mkdirSync(join(inDir, ".parachute"), { recursive: true });
|
|
844
|
+
writeFileSync(join(inDir, ".parachute", "vault.yaml"), "name: dup\nexport_format_version: 1\n");
|
|
845
|
+
writeFileSync(join(inDir, "aaa.md"), "---\nid: dup\npath: pa\n---\nfrom aaa\n");
|
|
846
|
+
writeFileSync(join(inDir, "bbb.md"), "---\nid: dup\npath: pb\n---\nfrom bbb\n");
|
|
847
|
+
writeFileSync(join(inDir, "ccc.md"), "---\nid: ' '\npath: pc\n---\nfrom ccc\n");
|
|
848
|
+
|
|
849
|
+
const target = new SqliteStore(new Database(":memory:"));
|
|
850
|
+
const stats = await importPortableVault(target, { inDir });
|
|
851
|
+
|
|
852
|
+
// Exactly ONE note created — the FIRST of the id collision (aaa) — and the
|
|
853
|
+
// collision is NOT folded into "updated".
|
|
854
|
+
expect(stats.notes_created).toBe(1);
|
|
855
|
+
expect(stats.notes_updated).toBe(0);
|
|
856
|
+
const kept = await target.getNote("dup");
|
|
857
|
+
expect(kept).toBeTruthy();
|
|
858
|
+
expect(kept!.content.trimEnd()).toBe("from aaa"); // first-wins, deterministic
|
|
859
|
+
|
|
860
|
+
// Both the duplicate (bbb) and the blank-id (ccc) file are reported as
|
|
861
|
+
// skipped — surfaced in stats, not absorbed into the created/updated count.
|
|
862
|
+
expect(stats.skipped_duplicate_ids.length).toBe(2);
|
|
863
|
+
const dupEntry = stats.skipped_duplicate_ids.find((s) => s.reason.includes("duplicate id"));
|
|
864
|
+
expect(dupEntry).toBeTruthy();
|
|
865
|
+
expect(dupEntry!.path).toBe("pb");
|
|
866
|
+
const blankEntry = stats.skipped_duplicate_ids.find((s) => s.reason.includes("blank"));
|
|
867
|
+
expect(blankEntry).toBeTruthy();
|
|
868
|
+
|
|
869
|
+
// The whitespace-id file clobbered nothing — no bogus key, only "dup" exists.
|
|
870
|
+
const allNotes = await target.queryNotes({ limit: 100 });
|
|
871
|
+
expect(allNotes.length).toBe(1);
|
|
872
|
+
});
|
|
873
|
+
|
|
713
874
|
it("restores tag schemas (description + fields)", async () => {
|
|
714
875
|
await store.upsertTagSchema("task", {
|
|
715
876
|
description: "A unit of work",
|