@openparachute/vault 0.7.1 → 0.7.2-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/core.test.ts +70 -4
- package/core/src/cursor-keyset-ms.test.ts +537 -0
- package/core/src/cursor.ts +76 -6
- package/core/src/hooks.ts +9 -0
- package/core/src/mcp.ts +12 -2
- package/core/src/notes.ts +175 -50
- package/core/src/paths.ts +4 -0
- package/core/src/portable-md.test.ts +161 -0
- package/core/src/portable-md.ts +140 -9
- package/core/src/schema.ts +128 -2
- package/core/src/seed-packs.ts +39 -13
- package/core/src/store.ts +12 -2
- package/core/src/txn.test.ts +100 -4
- package/core/src/txn.ts +119 -24
- package/package.json +1 -1
- package/src/cli.ts +6 -0
- package/src/mirror-import.ts +5 -0
- package/src/routes.ts +27 -2
- package/src/vault.test.ts +88 -0
package/core/src/cursor.ts
CHANGED
|
@@ -287,13 +287,83 @@ export function isoToMillis(iso: string): number {
|
|
|
287
287
|
/**
|
|
288
288
|
* Convert millisecond epoch back to an ISO-8601 timestamp string.
|
|
289
289
|
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
293
|
-
*
|
|
294
|
-
* timestamp vault mints goes through `new Date(...).toISOString()` so the
|
|
295
|
-
* lex-order matches the millis-order.
|
|
290
|
+
* Historically used to translate the cursor's `last_updated_at` into the form
|
|
291
|
+
* SQLite compared (`n.updated_at` TEXT). Post-vault#586 the keyset compares
|
|
292
|
+
* against the integer `notes.updated_at_ms` column directly, so this is no
|
|
293
|
+
* longer on the cursor hot path — kept as a tested utility.
|
|
296
294
|
*/
|
|
297
295
|
export function millisToIso(ms: number): string {
|
|
298
296
|
return new Date(ms).toISOString();
|
|
299
297
|
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Matches an ISO-8601-ish timestamp. The time portion (and each finer field)
|
|
301
|
+
* is optional so a bare `YYYY-MM-DD` or `YYYY-MM-DD HH:MM` still parses. The
|
|
302
|
+
* separator is `T` or a space (aged/imported vaults carry both). Zone is an
|
|
303
|
+
* optional trailing `Z`/`z` or `±HH:MM` / `±HHMM`.
|
|
304
|
+
*/
|
|
305
|
+
const TIMESTAMP_RE =
|
|
306
|
+
/^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2}))?(?:\.(\d+))?\s*(Z|z|[+-]\d{2}:?\d{2})?)?$/;
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* UTC-correct millisecond-epoch parse of a stored timestamp string, or `null`
|
|
310
|
+
* when the value is genuinely unparseable (vault#586).
|
|
311
|
+
*
|
|
312
|
+
* The cursor keyset orders by an integer `notes.updated_at_ms` column; this is
|
|
313
|
+
* the ONE parser that derives that integer from a timestamp string — used by
|
|
314
|
+
* the schema backfill (`migrateToV26`), the write sites that maintain the
|
|
315
|
+
* column, and any watermark fallback. It exists because `Date.parse` is WRONG
|
|
316
|
+
* for the two zone-less shapes aged/imported vaults carry (import stores
|
|
317
|
+
* frontmatter timestamps verbatim, so non-canonical forms are common):
|
|
318
|
+
*
|
|
319
|
+
* - space-separated `YYYY-MM-DD HH:MM:SS` — implementation-defined per spec;
|
|
320
|
+
* V8/Bun parse it as LOCAL time, shifting the epoch by the host's UTC
|
|
321
|
+
* offset. This was the live bug — a watermark computed in local time
|
|
322
|
+
* skipped or re-delivered rows by whole hours.
|
|
323
|
+
* - zone-less ISO `YYYY-MM-DDTHH:MM:SS` — ES2015+ parse date-TIME forms with
|
|
324
|
+
* no offset as LOCAL time too.
|
|
325
|
+
*
|
|
326
|
+
* Both are interpreted here as UTC. An explicit `Z` or `±HH:MM` offset is
|
|
327
|
+
* honored. Fractional seconds are truncated (floored) to millisecond
|
|
328
|
+
* precision. NEVER throws — the caller supplies its own deterministic fallback
|
|
329
|
+
* for a `null` result (e.g. the row's `created_at` ms, else a stable
|
|
330
|
+
* sentinel), so a single pathological timestamp can't 400 an entire walk.
|
|
331
|
+
*/
|
|
332
|
+
export function timestampToMs(value: string | null | undefined): number | null {
|
|
333
|
+
if (typeof value !== "string") return null;
|
|
334
|
+
const s = value.trim();
|
|
335
|
+
if (s.length === 0) return null;
|
|
336
|
+
const m = TIMESTAMP_RE.exec(s);
|
|
337
|
+
if (m) {
|
|
338
|
+
const year = Number(m[1]);
|
|
339
|
+
const month = Number(m[2]) - 1;
|
|
340
|
+
const day = Number(m[3]);
|
|
341
|
+
const hour = m[4] !== undefined ? Number(m[4]) : 0;
|
|
342
|
+
const min = m[5] !== undefined ? Number(m[5]) : 0;
|
|
343
|
+
const sec = m[6] !== undefined ? Number(m[6]) : 0;
|
|
344
|
+
// Fractional seconds → ms: right-pad to 3 digits, take the first 3
|
|
345
|
+
// (millisecond precision; finer digits are floored, not rounded).
|
|
346
|
+
const ms = m[7] !== undefined ? Number((m[7] + "000").slice(0, 3)) : 0;
|
|
347
|
+
let epoch = Date.UTC(year, month, day, hour, min, sec, ms);
|
|
348
|
+
if (!Number.isFinite(epoch)) return null;
|
|
349
|
+
const zone = m[8];
|
|
350
|
+
if (zone && zone !== "Z" && zone !== "z") {
|
|
351
|
+
// `±HH:MM` or `±HHMM` — the wall-clock is stated IN this offset; subtract
|
|
352
|
+
// it to reach UTC. (`14:30+02:00` is `12:30Z`.)
|
|
353
|
+
const sign = zone[0] === "-" ? -1 : 1;
|
|
354
|
+
const digits = zone.slice(1).replace(":", "");
|
|
355
|
+
const offMin = Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2, 4));
|
|
356
|
+
epoch -= sign * offMin * 60_000;
|
|
357
|
+
}
|
|
358
|
+
return epoch;
|
|
359
|
+
}
|
|
360
|
+
// Anything the regex didn't match but that carries an explicit zone marker
|
|
361
|
+
// (exotic offset spellings, RFC-2822) — trust Date.parse ONLY then, since a
|
|
362
|
+
// zone-bearing string doesn't hit the local-time trap. Zone-less non-matches
|
|
363
|
+
// fall through to `null` rather than risk a local-time misread.
|
|
364
|
+
if (/(?:[Zz]|[+-]\d{2}:?\d{2})$/.test(s)) {
|
|
365
|
+
const p = Date.parse(s);
|
|
366
|
+
return Number.isFinite(p) ? p : null;
|
|
367
|
+
}
|
|
368
|
+
return null;
|
|
369
|
+
}
|
package/core/src/hooks.ts
CHANGED
|
@@ -372,6 +372,15 @@ export class HookRegistry {
|
|
|
372
372
|
|
|
373
373
|
// Defer to a microtask so we unwind the caller's stack (and its
|
|
374
374
|
// SQLite transaction, if any) before handlers run.
|
|
375
|
+
//
|
|
376
|
+
// Caveat (vault#589): under an ASYNC transaction that spans awaits (the
|
|
377
|
+
// atomic blow-away import wrapped in `transactionAsync`), the connection
|
|
378
|
+
// stays open across the microtask boundary. A handler that writes to the
|
|
379
|
+
// store BEFORE its first real `await` would `SAVEPOINT`-join that open
|
|
380
|
+
// transaction (rolled back with it). Every handler here is audited safe —
|
|
381
|
+
// it defers real work behind an `await` (semaphore acquire) and does no
|
|
382
|
+
// synchronous store write — but the invariant lives in txn.ts's header
|
|
383
|
+
// ("shared-connection invariant"), not just this comment; keep it true.
|
|
375
384
|
queueMicrotask(() => {
|
|
376
385
|
for (const hook of matches) {
|
|
377
386
|
const task = this.runHandler(hook, event, note, store);
|
package/core/src/mcp.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
|
|
|
2
2
|
import type { Store, Note, QueryOpts } from "./types.js";
|
|
3
3
|
import { transactionAsync } from "./txn.js";
|
|
4
4
|
import * as noteOps from "./notes.js";
|
|
5
|
-
import { filterMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError } from "./notes.js";
|
|
5
|
+
import { filterMetadata, MAX_BATCH_SIZE, validateExtension, ExtensionValidationError, validatePath } from "./notes.js";
|
|
6
6
|
import { normalizePath } from "./paths.js";
|
|
7
7
|
import { QueryError } from "./query-operators.js";
|
|
8
8
|
import { TAG_EXPAND_MODES, stripTagHash, suggestSimilarTag, type TagExpandMode } from "./tag-hierarchy.js";
|
|
@@ -1269,6 +1269,10 @@ A note's response carries \`existed\` (true/false) whenever ITS \`if_exists\` wa
|
|
|
1269
1269
|
const extension = item.extension !== undefined
|
|
1270
1270
|
? validateExtension(item.extension)
|
|
1271
1271
|
: undefined;
|
|
1272
|
+
// Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
|
|
1273
|
+
// Throws inside the batch transaction — rolls it back, same as a
|
|
1274
|
+
// path conflict; mcp-http's generic error_type mapping → clean 400.
|
|
1275
|
+
validatePath(item.path);
|
|
1272
1276
|
const effectiveExtension = extension ?? "md";
|
|
1273
1277
|
const ifExists = (item.if_exists as string | undefined) ?? "error";
|
|
1274
1278
|
const upsertMode = ifExists === "ignore" || ifExists === "update" || ifExists === "replace";
|
|
@@ -1719,6 +1723,8 @@ Write-attribution (vault#298): every result carries \`createdBy\`/\`createdVia\`
|
|
|
1719
1723
|
const createExt = item.extension !== undefined
|
|
1720
1724
|
? validateExtension(item.extension)
|
|
1721
1725
|
: undefined;
|
|
1726
|
+
// Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
|
|
1727
|
+
validatePath(explicitPath ?? (idLooksLikePath ? idOrPath : undefined));
|
|
1722
1728
|
const createOpts: Parameters<Store["createNote"]>[1] = {
|
|
1723
1729
|
...(idLooksLikePath ? { path: explicitPath ?? idOrPath } : { id: idOrPath, ...(explicitPath !== undefined ? { path: explicitPath } : {}) }),
|
|
1724
1730
|
...(item.tags && Array.isArray((item.tags as any).add)
|
|
@@ -1895,7 +1901,11 @@ Write-attribution (vault#298): every result carries \`createdBy\`/\`createdVia\`
|
|
|
1895
1901
|
if (item.append !== undefined) updates.append = item.append;
|
|
1896
1902
|
if (item.prepend !== undefined) updates.prepend = item.prepend;
|
|
1897
1903
|
}
|
|
1898
|
-
if (item.path !== undefined)
|
|
1904
|
+
if (item.path !== undefined) {
|
|
1905
|
+
// Reject NUL / `..` paths at the write surface (vault#589 / FIX 2).
|
|
1906
|
+
validatePath(item.path);
|
|
1907
|
+
updates.path = item.path;
|
|
1908
|
+
}
|
|
1899
1909
|
if (item.extension !== undefined) {
|
|
1900
1910
|
updates.extension = validateExtension(item.extension);
|
|
1901
1911
|
}
|
package/core/src/notes.ts
CHANGED
|
@@ -14,8 +14,7 @@ import {
|
|
|
14
14
|
computeQueryHash,
|
|
15
15
|
decodeCursor,
|
|
16
16
|
encodeCursor,
|
|
17
|
-
|
|
18
|
-
millisToIso,
|
|
17
|
+
timestampToMs,
|
|
19
18
|
type CursorPayload,
|
|
20
19
|
type QueryHashInputs,
|
|
21
20
|
} from "./cursor.js";
|
|
@@ -115,10 +114,18 @@ export function createNote(
|
|
|
115
114
|
// value. Hook-style writes with `skipUpdatedAt` preserve this; real user
|
|
116
115
|
// edits bump it strictly upward, so `updated_at > created_at` still means
|
|
117
116
|
// "user-touched since creation."
|
|
117
|
+
//
|
|
118
|
+
// `updated_at_ms` (vault#586) is the integer keyset-ordering mirror of
|
|
119
|
+
// `updated_at`. Derived from the SAME `createdAt` value with the UTC-correct
|
|
120
|
+
// `timestampToMs`; a `null` (unparseable caller-supplied `created_at`, e.g.
|
|
121
|
+
// via `createNoteRaw` during import) falls back to wall-clock now so a fresh
|
|
122
|
+
// row never lands with a NULL keyset key. Import's `restoreNoteTimestamps`
|
|
123
|
+
// overwrites both columns from the exported bytes immediately after.
|
|
124
|
+
const updatedAtMs = timestampToMs(createdAt) ?? Date.now();
|
|
118
125
|
try {
|
|
119
126
|
db.prepare(
|
|
120
|
-
`INSERT INTO notes (id, content, path, metadata, created_at, updated_at, extension, created_by, created_via, last_updated_by, last_updated_via) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
121
|
-
).run(id, content, path, metadata, createdAt, createdAt, extension, actor, via, actor, via);
|
|
127
|
+
`INSERT INTO notes (id, content, path, metadata, created_at, updated_at, updated_at_ms, extension, created_by, created_via, last_updated_by, last_updated_via) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
128
|
+
).run(id, content, path, metadata, createdAt, createdAt, updatedAtMs, extension, actor, via, actor, via);
|
|
122
129
|
} catch (err) {
|
|
123
130
|
if (path !== null && isPathUniqueError(err)) {
|
|
124
131
|
throw new PathConflictError(path);
|
|
@@ -466,6 +473,63 @@ export function validateExtension(extension: unknown): string {
|
|
|
466
473
|
return extension;
|
|
467
474
|
}
|
|
468
475
|
|
|
476
|
+
/**
|
|
477
|
+
* Thrown by {@link validatePath} when a caller-supplied note `path` can never
|
|
478
|
+
* round-trip safely (vault#589 / FIX 2). Carries a stable `error_type`
|
|
479
|
+
* (`invalid_path`) so REST's catch and the generic MCP domain-error mapping
|
|
480
|
+
* (`src/mcp-http.ts`) both surface a clean 400 — same shape rule as
|
|
481
|
+
* `ExtensionValidationError`.
|
|
482
|
+
*/
|
|
483
|
+
export class PathValidationError extends Error {
|
|
484
|
+
code = "INVALID_PATH" as const;
|
|
485
|
+
error_type = "invalid_path" as const;
|
|
486
|
+
path: string;
|
|
487
|
+
reason: string;
|
|
488
|
+
|
|
489
|
+
constructor(path: string, reason: string) {
|
|
490
|
+
super(`invalid_path: "${path}" ${reason}`);
|
|
491
|
+
this.name = "PathValidationError";
|
|
492
|
+
this.path = path;
|
|
493
|
+
this.reason = reason;
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Reject a caller-supplied note path that can never round-trip safely
|
|
499
|
+
* (vault#589 / FIX 2). Two rules, both enforced at the WRITE surface only
|
|
500
|
+
* (REST + MCP create/update, exactly where `validateExtension` sits):
|
|
501
|
+
*
|
|
502
|
+
* 1. **No NUL byte** — never valid in a filename. A NUL-in-path note keeps
|
|
503
|
+
* its resolved target inside the export root (so it slips the traversal
|
|
504
|
+
* guard), then uncaught-throws `writeFileSync`, aborting the ENTIRE vault
|
|
505
|
+
* export for everyone. Rejecting NUL at write is the primary fix; the
|
|
506
|
+
* export sink's per-file try/catch (portable-md.ts) is the belt for rows
|
|
507
|
+
* that predate this guard.
|
|
508
|
+
* 2. **No `..` path segment** — traversal-shaped. Export guard-skips such a
|
|
509
|
+
* note (silently un-round-trippable) and it has no legitimate use as a
|
|
510
|
+
* vault note path.
|
|
511
|
+
*
|
|
512
|
+
* A `null`/empty-after-normalize path is fine — notes need no path. Reads /
|
|
513
|
+
* queries never call this, so a lookup by a `..`/NUL path degrades to
|
|
514
|
+
* not-found rather than throwing. The Store itself still trusts internal /
|
|
515
|
+
* importer writes (mirrors the `validateExtension` split). Throws
|
|
516
|
+
* `PathValidationError`.
|
|
517
|
+
*/
|
|
518
|
+
export function validatePath(path: unknown): void {
|
|
519
|
+
if (path === null || path === undefined) return;
|
|
520
|
+
if (typeof path !== "string") return; // non-string: not a path; caller's shape check owns it
|
|
521
|
+
// NUL check on the RAW value — `normalizePath` strips NUL, so this must run
|
|
522
|
+
// before normalization to actually reject rather than silently clean.
|
|
523
|
+
if (path.includes("\0")) {
|
|
524
|
+
throw new PathValidationError(path, "contains a NUL byte");
|
|
525
|
+
}
|
|
526
|
+
const normalized = normalizePath(path);
|
|
527
|
+
if (normalized === null) return;
|
|
528
|
+
if (normalized.split("/").some((seg) => seg === "..")) {
|
|
529
|
+
throw new PathValidationError(path, "contains a '..' path segment (path traversal)");
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
469
533
|
/**
|
|
470
534
|
* Match bun:sqlite's UNIQUE-constraint error on the notes path index.
|
|
471
535
|
* Post-vault#328 the unique index is composite `(path, extension)`, so
|
|
@@ -542,7 +606,9 @@ export function updateNote(
|
|
|
542
606
|
// and path has been removed.
|
|
543
607
|
|
|
544
608
|
const sets: string[] = [];
|
|
545
|
-
|
|
609
|
+
// `updated_at_ms` binds as a number (INTEGER column), so the value list
|
|
610
|
+
// carries numbers alongside strings/nulls (vault#586).
|
|
611
|
+
const values: (string | number | null)[] = [];
|
|
546
612
|
|
|
547
613
|
// Hooks and other machine-level writers pass `skipUpdatedAt: true` so
|
|
548
614
|
// their metadata markers don't look like user activity. See issue #44.
|
|
@@ -560,6 +626,12 @@ export function updateNote(
|
|
|
560
626
|
}
|
|
561
627
|
sets.push("updated_at = ?");
|
|
562
628
|
values.push(now);
|
|
629
|
+
// `updated_at_ms` (vault#586) rides the SAME gate as `updated_at` — the
|
|
630
|
+
// integer keyset-ordering mirror must move in lockstep with the string.
|
|
631
|
+
// `now` is canonical `.toISOString()`, so `timestampToMs` never returns
|
|
632
|
+
// null here; the fallback is belt-and-suspenders.
|
|
633
|
+
sets.push("updated_at_ms = ?");
|
|
634
|
+
values.push(timestampToMs(now) ?? Date.now());
|
|
563
635
|
|
|
564
636
|
// Write-attribution (vault#298): the most-recent-write columns ride the
|
|
565
637
|
// SAME gate as `updated_at`. A `skipUpdatedAt` machine write doesn't bump
|
|
@@ -1100,6 +1172,11 @@ function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: str
|
|
|
1100
1172
|
if (field === "created_at") {
|
|
1101
1173
|
column = "n.created_at";
|
|
1102
1174
|
} else if (field === "updated_at") {
|
|
1175
|
+
// NOTE (vault#586 follow-up): this range filter compares the TEXT
|
|
1176
|
+
// `updated_at`, which is inconsistent on non-canonical rows the same way
|
|
1177
|
+
// the cursor keyset was pre-v26. Cursor pagination moved to the integer
|
|
1178
|
+
// `updated_at_ms` column; `date_filter`/`order_by`/export `--since` still
|
|
1179
|
+
// read the string and are tracked separately. Behavior unchanged here.
|
|
1103
1180
|
column = "n.updated_at";
|
|
1104
1181
|
} else {
|
|
1105
1182
|
// Re-uses the same indexed-field gate as `metadata` operator queries
|
|
@@ -1133,11 +1210,19 @@ function buildFilterConditions(db: Database, opts: QueryOpts): { conditions: str
|
|
|
1133
1210
|
return { conditions, params };
|
|
1134
1211
|
}
|
|
1135
1212
|
|
|
1136
|
-
|
|
1213
|
+
/**
|
|
1214
|
+
* `_outUpdatedAtMs` (internal, vault#586) — when supplied, the phase-1 page
|
|
1215
|
+
* query writes each returned id's integer `updated_at_ms` keyset key into it.
|
|
1216
|
+
* Only `queryNotesPaged` passes it, to derive the cursor watermark from the
|
|
1217
|
+
* SAME statement that determined page membership + order (one consistent
|
|
1218
|
+
* snapshot — see the call site). Ordinary callers omit it and are unaffected;
|
|
1219
|
+
* it never touches the returned `Note` shape.
|
|
1220
|
+
*/
|
|
1221
|
+
export function queryNotes(db: Database, opts: QueryOpts, _outUpdatedAtMs?: Map<string, number>): Note[] {
|
|
1137
1222
|
validateLimitOffset(opts);
|
|
1138
1223
|
const { conditions, params } = buildFilterConditions(db, opts);
|
|
1139
1224
|
|
|
1140
|
-
// ---- Cursor predicate (vault#313) ----
|
|
1225
|
+
// ---- Cursor predicate (vault#313; keyset column vault#586) ----
|
|
1141
1226
|
//
|
|
1142
1227
|
// Cursor mode is keyed on PRESENCE of `opts.cursor`, not truthiness
|
|
1143
1228
|
// (vault#550 bootstrap fix). `cursor: ""` is the bootstrap call — "I want
|
|
@@ -1148,24 +1233,38 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1148
1233
|
// Before this fix, `if (opts.cursor)` treated an empty string exactly
|
|
1149
1234
|
// like "no cursor" — the caller's bootstrap intent silently vanished and
|
|
1150
1235
|
// the first page came back in `created_at` order instead of the
|
|
1151
|
-
//
|
|
1236
|
+
// keyset order the SECOND page (a real cursor) would use,
|
|
1152
1237
|
// so naive "did I see this note already" comparisons could skip or
|
|
1153
1238
|
// duplicate rows across the boundary.
|
|
1154
1239
|
//
|
|
1240
|
+
// The keyset orders on the integer `n.updated_at_ms` column (vault#586),
|
|
1241
|
+
// NOT the TEXT `n.updated_at`. That column is the single source of truth
|
|
1242
|
+
// for cursor ordering: walk-order, the boundary predicate below, and the
|
|
1243
|
+
// watermark in `queryNotesPaged` all read the SAME integer, so they can't
|
|
1244
|
+
// diverge the way three separate readings of `updated_at` did on
|
|
1245
|
+
// aged/imported vaults whose timestamps aren't canonical `.toISOString()`
|
|
1246
|
+
// (space-form / offset / no-`Z`) — under which TEXT-lex order, an ISO
|
|
1247
|
+
// boundary string, and a `Date.parse` watermark disagreed and silently
|
|
1248
|
+
// skipped or re-delivered rows.
|
|
1249
|
+
//
|
|
1155
1250
|
// When a REAL (non-empty) cursor is present, decode it, verify its
|
|
1156
1251
|
// query_hash matches the current query, and add a keyset predicate of
|
|
1157
1252
|
// the form:
|
|
1158
1253
|
//
|
|
1159
|
-
// (
|
|
1160
|
-
// OR (
|
|
1254
|
+
// (updated_at_ms > last_updated_at)
|
|
1255
|
+
// OR (updated_at_ms = last_updated_at AND id > last_id)
|
|
1161
1256
|
//
|
|
1162
|
-
// The cursor
|
|
1163
|
-
//
|
|
1164
|
-
//
|
|
1165
|
-
//
|
|
1166
|
-
//
|
|
1167
|
-
//
|
|
1168
|
-
//
|
|
1257
|
+
// The cursor payload's `last_updated_at` is ALREADY a millisecond epoch
|
|
1258
|
+
// (see cursor.ts) — the same units as the column — so it binds directly
|
|
1259
|
+
// with no ISO round-trip. This is also why existing client cursors keep
|
|
1260
|
+
// working unchanged across the upgrade: the encoded watermark was always
|
|
1261
|
+
// ms. The cursor forces ORDER BY n.updated_at_ms ASC, n.id ASC so the
|
|
1262
|
+
// watermark math is sound — paginating by update time while ordering by
|
|
1263
|
+
// created_at would skip rows whose update differs from their creation.
|
|
1264
|
+
// `orderBy` and `sort: "desc"` are mutually exclusive with cursor mode (a
|
|
1265
|
+
// "since last checked" loop wants ascending update order, full stop); we
|
|
1266
|
+
// reject with INVALID_QUERY so callers don't silently get a broken
|
|
1267
|
+
// iteration.
|
|
1169
1268
|
const cursorMode = opts.cursor !== undefined;
|
|
1170
1269
|
let cursorPayload: CursorPayload | null = null;
|
|
1171
1270
|
if (cursorMode) {
|
|
@@ -1190,20 +1289,17 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1190
1289
|
"cursor_query_mismatch",
|
|
1191
1290
|
);
|
|
1192
1291
|
}
|
|
1193
|
-
//
|
|
1194
|
-
//
|
|
1195
|
-
//
|
|
1196
|
-
//
|
|
1197
|
-
//
|
|
1198
|
-
//
|
|
1199
|
-
|
|
1200
|
-
// safe: we round-trip the cursor's millis through `new Date()`'s
|
|
1201
|
-
// canonical ISO so the comparison is apples-to-apples.
|
|
1202
|
-
const cursorIso = millisToIso(cursorPayload.last_updated_at);
|
|
1292
|
+
// The cursor's `last_updated_at` is a millisecond epoch (cursor.ts) —
|
|
1293
|
+
// the SAME units as the integer `n.updated_at_ms` column — so it binds
|
|
1294
|
+
// straight into the keyset predicate with no ISO round-trip. Ordering
|
|
1295
|
+
// and boundary now read one numeric column, so heterogeneous /
|
|
1296
|
+
// non-canonical `updated_at` TEXT (space-form, offset, no-`Z`) can no
|
|
1297
|
+
// longer make the walk and the watermark disagree.
|
|
1298
|
+
const cursorMs = cursorPayload.last_updated_at;
|
|
1203
1299
|
conditions.push(
|
|
1204
|
-
"(n.
|
|
1300
|
+
"(n.updated_at_ms > ? OR (n.updated_at_ms = ? AND n.id > ?))",
|
|
1205
1301
|
);
|
|
1206
|
-
params.push(
|
|
1302
|
+
params.push(cursorMs, cursorMs, cursorPayload.last_id);
|
|
1207
1303
|
}
|
|
1208
1304
|
// else: bootstrap call (`cursor === ""`) — no watermark yet, no
|
|
1209
1305
|
// predicate to add, but the ORDER BY below still switches to the
|
|
@@ -1214,11 +1310,12 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1214
1310
|
const direction = opts.sort === "desc" ? "DESC" : "ASC";
|
|
1215
1311
|
let orderBy: string;
|
|
1216
1312
|
if (cursorMode) {
|
|
1217
|
-
// Cursor mode forces a deterministic keyset order
|
|
1218
|
-
//
|
|
1219
|
-
//
|
|
1220
|
-
//
|
|
1221
|
-
|
|
1313
|
+
// Cursor mode forces a deterministic keyset order on the integer
|
|
1314
|
+
// `updated_at_ms` column (vault#586), matching the boundary predicate
|
|
1315
|
+
// above and the `idx_notes_updated_ms` index. `id` is the tiebreaker —
|
|
1316
|
+
// without it, two notes sharing an `updated_at_ms` would be at the mercy
|
|
1317
|
+
// of SQLite's row order and the next page could miss or duplicate one.
|
|
1318
|
+
orderBy = "n.updated_at_ms ASC, n.id ASC";
|
|
1222
1319
|
} else if (opts.orderBy === "link_count") {
|
|
1223
1320
|
// `link_count` is a pseudo-field — like `created_at`/`updated_at` in the
|
|
1224
1321
|
// dateFilter block above, it bypasses `requireIndexedField` (it's not a
|
|
@@ -1273,15 +1370,26 @@ export function queryNotes(db: Database, opts: QueryOpts): Note[] {
|
|
|
1273
1370
|
// Phase 2 fetches full rows for just the page (≤ limit ids) and re-orders
|
|
1274
1371
|
// to the phase-1 order; tags are hydrated in ONE batched query instead of
|
|
1275
1372
|
// one query per returned note.
|
|
1373
|
+
// Phase 1 also selects `n.updated_at_ms` (vault#586) so a paging caller can
|
|
1374
|
+
// read the keyset key from the SAME statement that applied the keyset order
|
|
1375
|
+
// — the watermark then comes from one consistent snapshot, immune to a
|
|
1376
|
+
// cross-process writer bumping a page row's ms between two separate reads.
|
|
1276
1377
|
const idSql = `
|
|
1277
|
-
SELECT n.id FROM notes n
|
|
1378
|
+
SELECT n.id, n.updated_at_ms FROM notes n
|
|
1278
1379
|
${whereClause}
|
|
1279
1380
|
ORDER BY ${orderBy}
|
|
1280
1381
|
LIMIT ? OFFSET ?
|
|
1281
1382
|
`;
|
|
1282
1383
|
params.push(limit, offset);
|
|
1283
1384
|
|
|
1284
|
-
const idRows = db.prepare(idSql).all(...params) as { id: string }[];
|
|
1385
|
+
const idRows = db.prepare(idSql).all(...params) as { id: string; updated_at_ms: number | null }[];
|
|
1386
|
+
if (_outUpdatedAtMs) {
|
|
1387
|
+
for (const r of idRows) {
|
|
1388
|
+
if (r.updated_at_ms !== null && r.updated_at_ms !== undefined) {
|
|
1389
|
+
_outUpdatedAtMs.set(r.id, r.updated_at_ms);
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1285
1393
|
return fetchNotesByIdsOrdered(db, idRows.map((r) => r.id));
|
|
1286
1394
|
}
|
|
1287
1395
|
|
|
@@ -1517,7 +1625,12 @@ function toQueryHashInputs(opts: QueryOpts): QueryHashInputs {
|
|
|
1517
1625
|
* with a non-null `updated_at` greater than the unix epoch).
|
|
1518
1626
|
*/
|
|
1519
1627
|
export function queryNotesPaged(db: Database, opts: QueryOpts): QueryNotesPage {
|
|
1520
|
-
|
|
1628
|
+
// Capture each page row's integer keyset key (vault#586) from the SAME
|
|
1629
|
+
// phase-1 statement that ordered + sliced the page — one consistent
|
|
1630
|
+
// snapshot, so the watermark can't leapfrog rows a cross-process writer
|
|
1631
|
+
// bumps between reads. This replaces a separate post-query round-trip.
|
|
1632
|
+
const updatedAtMsById = new Map<string, number>();
|
|
1633
|
+
const notes = queryNotes(db, opts, updatedAtMsById);
|
|
1521
1634
|
const queryHash = computeQueryHash(toQueryHashInputs(opts));
|
|
1522
1635
|
|
|
1523
1636
|
// Watermark math: pick the larger of (last returned row, prior cursor
|
|
@@ -1536,14 +1649,20 @@ export function queryNotesPaged(db: Database, opts: QueryOpts): QueryNotesPage {
|
|
|
1536
1649
|
lastId = prior.last_id;
|
|
1537
1650
|
}
|
|
1538
1651
|
if (notes.length > 0) {
|
|
1539
|
-
//
|
|
1540
|
-
//
|
|
1541
|
-
//
|
|
1542
|
-
//
|
|
1543
|
-
//
|
|
1652
|
+
// Advance the watermark to the page's MAX (updated_at_ms, id) using the
|
|
1653
|
+
// keyset keys captured in the phase-1 snapshot above (vault#586) — the
|
|
1654
|
+
// SAME integer `updated_at_ms` the walk-order used, from the SAME read.
|
|
1655
|
+
// Using the snapshot, rather than re-deriving ms from each note's
|
|
1656
|
+
// `updatedAt` string or re-reading the column, is what keeps walk-order
|
|
1657
|
+
// and watermark from diverging: a row whose backfilled column and
|
|
1658
|
+
// non-canonical `updated_at` TEXT disagree would otherwise re-parse to a
|
|
1659
|
+
// different ms here and skip or re-deliver at the boundary. No throw — a
|
|
1660
|
+
// NULL key (shouldn't occur post-migrateToV26) reads as 0. When a cursor
|
|
1661
|
+
// is in effect the SQL already returns rows in (updated_at_ms, id) order
|
|
1662
|
+
// so the last row IS the max; the explicit max also covers the
|
|
1663
|
+
// created_at-ordered no-cursor path.
|
|
1544
1664
|
for (const note of notes) {
|
|
1545
|
-
const
|
|
1546
|
-
const ms = isoToMillis(updatedIso);
|
|
1665
|
+
const ms = updatedAtMsById.get(note.id) ?? 0;
|
|
1547
1666
|
if (ms > lastUpdatedAt || (ms === lastUpdatedAt && note.id > lastId)) {
|
|
1548
1667
|
lastUpdatedAt = ms;
|
|
1549
1668
|
lastId = note.id;
|
|
@@ -2041,6 +2160,10 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2041
2160
|
// note_tags FK on `tag_name` has no ON DELETE, so the delete must
|
|
2042
2161
|
// come AFTER the repoint.
|
|
2043
2162
|
const now = new Date().toISOString();
|
|
2163
|
+
// Integer keyset mirror of `now` for the note `updated_at_ms` bumps in the
|
|
2164
|
+
// content/path rewrite passes below (vault#586). `now` is canonical, so
|
|
2165
|
+
// `timestampToMs` never returns null here.
|
|
2166
|
+
const nowMs = timestampToMs(now) ?? Date.now();
|
|
2044
2167
|
const readStmt = db.prepare(
|
|
2045
2168
|
"SELECT description, fields, relationships, parent_names, created_at FROM tags WHERE name = ?",
|
|
2046
2169
|
);
|
|
@@ -2168,12 +2291,13 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2168
2291
|
// `#newtag`) and must bump `updated_at` like any other content
|
|
2169
2292
|
// write, or a cursor/sync-poll loop never sees it. Shares the one
|
|
2170
2293
|
// `now` timestamp for the whole cascade (same convention as the
|
|
2171
|
-
// tag-row rename pass above).
|
|
2172
|
-
|
|
2294
|
+
// tag-row rename pass above). `updated_at_ms` moves with it (vault#586)
|
|
2295
|
+
// so the cursor keyset actually surfaces the rewrite.
|
|
2296
|
+
const updateStmt = db.prepare("UPDATE notes SET content = ?, updated_at = ?, updated_at_ms = ? WHERE id = ?");
|
|
2173
2297
|
for (const row of candidates) {
|
|
2174
2298
|
const next = rewriteNoteBody(row.content, renames);
|
|
2175
2299
|
if (next === row.content) continue;
|
|
2176
|
-
updateStmt.run(next, now, row.id);
|
|
2300
|
+
updateStmt.run(next, now, nowMs, row.id);
|
|
2177
2301
|
notesRewritten++;
|
|
2178
2302
|
}
|
|
2179
2303
|
}
|
|
@@ -2189,12 +2313,13 @@ export function renameTag(db: Database, oldName: string, newName: string): Renam
|
|
|
2189
2313
|
.prepare(`SELECT id, path FROM notes WHERE path IS NOT NULL AND (${orClauses})`)
|
|
2190
2314
|
.all(...params) as { id: string; path: string }[];
|
|
2191
2315
|
// vault#555 fix 2 — same reasoning as the content rewrite above: a
|
|
2192
|
-
// path rewrite is a real persisted-state change.
|
|
2193
|
-
|
|
2316
|
+
// path rewrite is a real persisted-state change. `updated_at_ms` moves
|
|
2317
|
+
// with `updated_at` (vault#586).
|
|
2318
|
+
const updateStmt = db.prepare("UPDATE notes SET path = ?, updated_at = ?, updated_at_ms = ? WHERE id = ?");
|
|
2194
2319
|
for (const row of candidates) {
|
|
2195
2320
|
const next = rewriteTagConfigPath(row.path, renames);
|
|
2196
2321
|
if (next === row.path) continue;
|
|
2197
|
-
updateStmt.run(next, now, row.id);
|
|
2322
|
+
updateStmt.run(next, now, nowMs, row.id);
|
|
2198
2323
|
pathsRenamed++;
|
|
2199
2324
|
}
|
|
2200
2325
|
}
|
package/core/src/paths.ts
CHANGED
|
@@ -19,6 +19,10 @@ export function normalizePath(path: string | null | undefined): string | null {
|
|
|
19
19
|
if (path === null || path === undefined) return null;
|
|
20
20
|
|
|
21
21
|
let p = path
|
|
22
|
+
.replace(/\0/g, "") // strip NUL bytes — never valid in a path;
|
|
23
|
+
// belt for legacy/imported rows so a stored
|
|
24
|
+
// NUL can't reach the filesystem (writes
|
|
25
|
+
// reject NUL up front via validatePath)
|
|
22
26
|
.trim()
|
|
23
27
|
.replace(/\\/g, "/") // backslash → forward slash
|
|
24
28
|
.replace(/\.md$/i, "") // strip .md extension
|