@openparachute/vault 0.7.1 → 0.7.2-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/aggregate.test.ts +23 -0
- package/core/src/contract-taxonomy.test.ts +22 -0
- package/core/src/core.test.ts +70 -4
- package/core/src/cursor-keyset-ms.test.ts +537 -0
- package/core/src/cursor.ts +76 -6
- package/core/src/hooks.ts +9 -0
- package/core/src/mcp.ts +12 -2
- package/core/src/notes.ts +175 -50
- package/core/src/paths.ts +4 -0
- package/core/src/portable-md.test.ts +161 -0
- package/core/src/portable-md.ts +140 -9
- package/core/src/query-operators.ts +56 -0
- package/core/src/schema.ts +128 -2
- package/core/src/seed-packs.ts +39 -13
- package/core/src/store.ts +20 -2
- package/core/src/tag-hierarchy.ts +13 -0
- package/core/src/txn.test.ts +100 -4
- package/core/src/txn.ts +119 -24
- package/core/src/wikilinks.test.ts +175 -0
- package/core/src/wikilinks.ts +151 -10
- package/package.json +1 -1
- package/src/cli.ts +6 -0
- package/src/mirror-import.ts +5 -0
- package/src/routes.ts +170 -7
- package/src/vault.test.ts +227 -1
package/core/src/portable-md.ts
CHANGED
|
@@ -77,6 +77,7 @@ import { basename, join, relative, extname, dirname, resolve as resolvePath, sep
|
|
|
77
77
|
import type { Store, Note, Link, Attachment } from "./types.js";
|
|
78
78
|
import type { TagRecord } from "./tag-schemas.js";
|
|
79
79
|
import { ParentCycleError } from "./tag-schemas.js";
|
|
80
|
+
import { transactionAsync } from "./txn.js";
|
|
80
81
|
|
|
81
82
|
// ---------------------------------------------------------------------------
|
|
82
83
|
// Format constants
|
|
@@ -786,8 +787,23 @@ export class FsExportSink implements ExportSink {
|
|
|
786
787
|
reason: `path-traversal: resolved write target "${fullResolved}" escapes export root "${this.rootResolved}"`,
|
|
787
788
|
};
|
|
788
789
|
}
|
|
789
|
-
|
|
790
|
-
|
|
790
|
+
// Belt for already-poisoned vaults (vault#589 / FIX 2). A NUL-in-path note
|
|
791
|
+
// resolves WITHIN the export root (passing the traversal guard above) but
|
|
792
|
+
// then makes `writeFileSync` throw ("path contains null byte") — an
|
|
793
|
+
// uncaught throw here would abort the ENTIRE export, so one bad note breaks
|
|
794
|
+
// backup for every note. Any fs failure (NUL, EACCES, ENAMETOOLONG, …)
|
|
795
|
+
// becomes a skip-with-reason the caller records as a `skipped_notes` stat +
|
|
796
|
+
// a warning, exactly like the traversal skip, so the rest of the vault
|
|
797
|
+
// still exports.
|
|
798
|
+
try {
|
|
799
|
+
mkdirSync(dirname(full), { recursive: true });
|
|
800
|
+
writeFileSync(full, content);
|
|
801
|
+
} catch (err) {
|
|
802
|
+
return {
|
|
803
|
+
ok: false,
|
|
804
|
+
reason: `fs write failed for "${relPath}": ${(err as Error)?.message ?? String(err)}`,
|
|
805
|
+
};
|
|
806
|
+
}
|
|
791
807
|
return { ok: true };
|
|
792
808
|
}
|
|
793
809
|
|
|
@@ -812,8 +828,17 @@ export class FsExportSink implements ExportSink {
|
|
|
812
828
|
reason: `path-traversal: dest "${destResolved}" escapes export root "${this.rootResolved}"`,
|
|
813
829
|
};
|
|
814
830
|
}
|
|
815
|
-
|
|
816
|
-
|
|
831
|
+
// Same belt as `writeText` (vault#589 / FIX 2): a NUL/otherwise-unwritable
|
|
832
|
+
// attachment dest must skip-with-reason, not abort the whole export.
|
|
833
|
+
try {
|
|
834
|
+
mkdirSync(dirname(destFull), { recursive: true });
|
|
835
|
+
copyFileSync(srcResolved, destResolved);
|
|
836
|
+
} catch (err) {
|
|
837
|
+
return {
|
|
838
|
+
ok: false,
|
|
839
|
+
reason: `fs copy failed for "${destRelPath}": ${(err as Error)?.message ?? String(err)}`,
|
|
840
|
+
};
|
|
841
|
+
}
|
|
817
842
|
return { ok: true };
|
|
818
843
|
}
|
|
819
844
|
}
|
|
@@ -878,11 +903,29 @@ export async function exportVault(
|
|
|
878
903
|
...(opts.vaultName ? { name: opts.vaultName } : {}),
|
|
879
904
|
...(opts.vaultDescription ? { description: opts.vaultDescription } : {}),
|
|
880
905
|
};
|
|
881
|
-
|
|
906
|
+
// The manifest is STRUCTURAL, not per-note: a portable-md export WITHOUT its
|
|
907
|
+
// `.parachute/vault.yaml` is unusable — restore (`importPortableVault`), CLI
|
|
908
|
+
// autodetect, and mirror-import all hard-reject a dir that lacks it. So the
|
|
909
|
+
// skip-with-warn belt that hardens per-note writes (vault#589 / FIX 2b) is
|
|
910
|
+
// exactly WRONG here: a discarded `{ok:false}` would produce a manifest-less
|
|
911
|
+
// "success" — a silently-broken backup. Fail LOUDLY instead. (Per-note
|
|
912
|
+
// content/attachment writes below stay skip-with-warn — one poisoned note
|
|
913
|
+
// must not abort the whole export.)
|
|
914
|
+
const manifestPath = join(SIDECAR_DIR, "vault.yaml");
|
|
915
|
+
const manifestWrite = sink.writeText(manifestPath, emitYamlDoc(vaultMeta as unknown as Record<string, unknown>));
|
|
916
|
+
if (!manifestWrite.ok) {
|
|
917
|
+
throw new Error(
|
|
918
|
+
`export aborted: failed to write the vault manifest "${manifestPath}": ${manifestWrite.reason}. ` +
|
|
919
|
+
`A portable-md export without its manifest is unusable (restore/import hard-rejects it), so this fails loudly rather than producing a manifest-less "success".`,
|
|
920
|
+
);
|
|
921
|
+
}
|
|
882
922
|
|
|
883
923
|
// 2. Per-tag schemas. Only tags carrying at least one schema-shaped
|
|
884
924
|
// field (description, fields, relationships, parent_names) get a file;
|
|
885
|
-
// tags that are just-a-name don't pollute the sidecar.
|
|
925
|
+
// tags that are just-a-name don't pollute the sidecar. Schemas are
|
|
926
|
+
// structural too — a dropped schema sidecar silently loses a tag's
|
|
927
|
+
// definition from the backup — so a write failure here also fails loudly
|
|
928
|
+
// (and never increments `schemasWritten` on a failed write).
|
|
886
929
|
const tagRecords = await store.listTagRecords();
|
|
887
930
|
let schemasWritten = 0;
|
|
888
931
|
for (const tag of tagRecords) {
|
|
@@ -895,7 +938,14 @@ export async function exportVault(
|
|
|
895
938
|
if (tag.parent_names !== undefined && tag.parent_names.length > 0) {
|
|
896
939
|
doc.parent_names = tag.parent_names;
|
|
897
940
|
}
|
|
898
|
-
|
|
941
|
+
const schemaPath = join(SIDECAR_DIR, "schemas", filename);
|
|
942
|
+
const schemaWrite = sink.writeText(schemaPath, emitYamlDoc(doc));
|
|
943
|
+
if (!schemaWrite.ok) {
|
|
944
|
+
throw new Error(
|
|
945
|
+
`export aborted: failed to write schema sidecar "${schemaPath}" for tag "${tag.tag}": ${schemaWrite.reason}. ` +
|
|
946
|
+
`Tag schemas are structural — a partial export would silently drop a schema definition — so this fails loudly.`,
|
|
947
|
+
);
|
|
948
|
+
}
|
|
899
949
|
schemasWritten++;
|
|
900
950
|
}
|
|
901
951
|
|
|
@@ -1643,6 +1693,17 @@ export interface ImportStats {
|
|
|
1643
1693
|
* imports — the cycle is warned, not fatal. Empty in the common case.
|
|
1644
1694
|
*/
|
|
1645
1695
|
skipped_schema_parents: Array<{ tag: string; parent_names: string[]; reason: string }>;
|
|
1696
|
+
/**
|
|
1697
|
+
* Content files skipped because their frontmatter `id` couldn't be used as
|
|
1698
|
+
* a note key (vault#589 / FIX 3): a DUPLICATE id (a later file shares an id
|
|
1699
|
+
* with one already imported — the first wins, deterministically, since
|
|
1700
|
+
* `listContentFiles()` walks in sorted path order) or a BLANK id
|
|
1701
|
+
* (whitespace-only, e.g. `" "`). Without this the later duplicate silently
|
|
1702
|
+
* clobbered the earlier note (data loss folded into "updated N") and a
|
|
1703
|
+
* blank id seeded a bogus map key. Each entry names the offending id, the
|
|
1704
|
+
* skipped file's path, and why. Empty in the common case.
|
|
1705
|
+
*/
|
|
1706
|
+
skipped_duplicate_ids: Array<{ id: string; path: string | undefined; reason: string }>;
|
|
1646
1707
|
/** Set when the caller passed `blowAway: true`; counts notes removed. */
|
|
1647
1708
|
notes_wiped: number;
|
|
1648
1709
|
/**
|
|
@@ -1815,10 +1876,49 @@ export async function importVault(
|
|
|
1815
1876
|
skipped_attachments: [],
|
|
1816
1877
|
skipped_sidecars: [],
|
|
1817
1878
|
skipped_schema_parents: [],
|
|
1879
|
+
skipped_duplicate_ids: [],
|
|
1818
1880
|
notes_wiped: 0,
|
|
1819
1881
|
indexes_declared: 0,
|
|
1820
1882
|
};
|
|
1821
1883
|
|
|
1884
|
+
// Blow-away is all-or-nothing (vault#589 / FIX 1). A blow-away import wipes
|
|
1885
|
+
// the vault (step 1) then replays every schema/note/link/attachment. If any
|
|
1886
|
+
// replay step throws mid-flight — a `PathConflictError` from two source
|
|
1887
|
+
// files sharing a path, a malformed record, a crash — an UNWRAPPED run would
|
|
1888
|
+
// leave the vault wiped-and-partial with the originals gone for good. Wrap
|
|
1889
|
+
// the wipe + the whole replay in ONE transaction so a failure rolls back to
|
|
1890
|
+
// the exact pre-import vault. The transaction seam is re-entrant (SAVEPOINTs,
|
|
1891
|
+
// vault#589) so the replay's own transactional writes (`upsertTagRecord`,
|
|
1892
|
+
// `deleteTag` cascade, …) compose inside it rather than hitting SQLite's
|
|
1893
|
+
// "cannot start a transaction within a transaction". Additive (non-blow-away)
|
|
1894
|
+
// and dry-run imports are never destructive, so they replay unwrapped —
|
|
1895
|
+
// exactly as before. Hook dispatch stays best-effort/reconciled: no
|
|
1896
|
+
// registered hook does synchronous destructive work (the mirror arms a
|
|
1897
|
+
// debounce; transcription only fires on attachment `created` and kicks an
|
|
1898
|
+
// idempotent worker), so a rollback can't leave a half-applied side effect
|
|
1899
|
+
// the sweep won't reconcile.
|
|
1900
|
+
if (opts.blowAway && !opts.dryRun) {
|
|
1901
|
+
await transactionAsync(store.db, () => importVaultReplay(store, source, opts, stats));
|
|
1902
|
+
} else {
|
|
1903
|
+
await importVaultReplay(store, source, opts, stats);
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
return stats;
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
/**
|
|
1910
|
+
* The replay body of {@link importVault} — wipe (blow-away only) then restore
|
|
1911
|
+
* schemas → notes → links → attachments → wikilinks → indexes, mutating the
|
|
1912
|
+
* shared `stats`. Split out so `importVault` can run it either bare or inside a
|
|
1913
|
+
* single `transactionAsync` (the atomic blow-away path); the restoration order
|
|
1914
|
+
* and every skip/warn policy are unchanged from the pre-split code.
|
|
1915
|
+
*/
|
|
1916
|
+
async function importVaultReplay(
|
|
1917
|
+
store: Store,
|
|
1918
|
+
source: ImportSource,
|
|
1919
|
+
opts: ImportEngineOptions,
|
|
1920
|
+
stats: ImportStats,
|
|
1921
|
+
): Promise<void> {
|
|
1822
1922
|
// 1. Optional wipe. Notes are deleted via the public Store API so hooks
|
|
1823
1923
|
// fire (callers depend on `attachment.deleted` hooks for assets-dir
|
|
1824
1924
|
// cleanup; we don't bypass that on blow-away). Deleted in bounded batches
|
|
@@ -2049,6 +2149,39 @@ export async function importVault(
|
|
|
2049
2149
|
...(links ? { links: links as PortableLink[] } : {}),
|
|
2050
2150
|
...(attachments ? { attachments: attachments as PortableAttachmentRef[] } : {}),
|
|
2051
2151
|
};
|
|
2152
|
+
|
|
2153
|
+
// FIX 3 (vault#589) — guard the `seenNotes` key before it's set:
|
|
2154
|
+
// - A whitespace-only id (" ") slips past the `!id` truthiness guard
|
|
2155
|
+
// above (non-empty string) but is not a real note key. Skip it rather
|
|
2156
|
+
// than seeding a bogus map entry / clobbering a real note.
|
|
2157
|
+
// - A DUPLICATE id means a LATER content file carries an id already
|
|
2158
|
+
// imported from an earlier file. The old `Map.set` silently
|
|
2159
|
+
// last-wins-clobbered the first note (data loss, folded into
|
|
2160
|
+
// "updated N"). Keep the FIRST (deterministic — `listContentFiles()`
|
|
2161
|
+
// returns sorted paths) and skip the later, recording the collision.
|
|
2162
|
+
// Both cases land in `skipped_duplicate_ids` + a per-file `console.warn`
|
|
2163
|
+
// so the operator sees the collision instead of it vanishing.
|
|
2164
|
+
if (id.trim() === "") {
|
|
2165
|
+
stats.skipped_duplicate_ids.push({
|
|
2166
|
+
id,
|
|
2167
|
+
path: portable.path,
|
|
2168
|
+
reason: "blank (whitespace-only) id is not a valid note key",
|
|
2169
|
+
});
|
|
2170
|
+
// eslint-disable-next-line no-console
|
|
2171
|
+
console.warn(`[import] skipped "${relPath}": blank (whitespace-only) \`id\` in frontmatter`);
|
|
2172
|
+
continue;
|
|
2173
|
+
}
|
|
2174
|
+
if (seenNotes.has(id)) {
|
|
2175
|
+
const kept = seenNotes.get(id)!;
|
|
2176
|
+
stats.skipped_duplicate_ids.push({
|
|
2177
|
+
id,
|
|
2178
|
+
path: portable.path,
|
|
2179
|
+
reason: `duplicate id "${id}" — already imported from an earlier file${kept.path ? ` (path="${kept.path}")` : ""}; kept the first, skipped this one`,
|
|
2180
|
+
});
|
|
2181
|
+
// eslint-disable-next-line no-console
|
|
2182
|
+
console.warn(`[import] skipped "${relPath}": duplicate id "${id}" (kept the first file's note; this file's content is NOT imported)`);
|
|
2183
|
+
continue;
|
|
2184
|
+
}
|
|
2052
2185
|
seenNotes.set(id, portable);
|
|
2053
2186
|
|
|
2054
2187
|
if (opts.dryRun) {
|
|
@@ -2232,8 +2365,6 @@ export async function importVault(
|
|
|
2232
2365
|
}
|
|
2233
2366
|
}
|
|
2234
2367
|
}
|
|
2235
|
-
|
|
2236
|
-
return stats;
|
|
2237
2368
|
}
|
|
2238
2369
|
|
|
2239
2370
|
/**
|
|
@@ -101,13 +101,69 @@ function toBinding(field: string, op: string, value: unknown): SQLQueryBindings
|
|
|
101
101
|
);
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Best-effort lookup of a metadata field's declared `type` across every tag
|
|
106
|
+
* schema — used ONLY to sharpen the {@link requireIndexedField} error hint
|
|
107
|
+
* below when the field was never indexed. Scans `tags.fields` directly
|
|
108
|
+
* rather than going through `indexed_fields` (which by construction has no
|
|
109
|
+
* row for a type that can never BE indexed, e.g. `"number"`) or a
|
|
110
|
+
* per-note schema resolution (scoped to one note's tags, not "any tag
|
|
111
|
+
* anywhere"). Only reached on the FIELD_NOT_INDEXED error path — a full
|
|
112
|
+
* table scan here is fine since it runs once per rejected call, never on a
|
|
113
|
+
* hot path. Returns the type from the FIRST tag found declaring `field`
|
|
114
|
+
* (a cross-tag type conflict on the same field name is its own separate
|
|
115
|
+
* validation elsewhere, not this function's job); `undefined` when no tag
|
|
116
|
+
* declares this field name, or its `fields` JSON doesn't parse.
|
|
117
|
+
*/
|
|
118
|
+
function findDeclaredFieldType(db: Database, field: string): string | undefined {
|
|
119
|
+
const rows = db.prepare(
|
|
120
|
+
`SELECT fields FROM tags WHERE fields IS NOT NULL`,
|
|
121
|
+
).all() as { fields: string | null }[];
|
|
122
|
+
for (const row of rows) {
|
|
123
|
+
if (!row.fields) continue;
|
|
124
|
+
try {
|
|
125
|
+
const parsed: unknown = JSON.parse(row.fields);
|
|
126
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) continue;
|
|
127
|
+
const spec = (parsed as Record<string, unknown>)[field];
|
|
128
|
+
if (spec === null || typeof spec !== "object" || Array.isArray(spec)) continue;
|
|
129
|
+
const type = (spec as Record<string, unknown>).type;
|
|
130
|
+
if (typeof type === "string") return type;
|
|
131
|
+
} catch {
|
|
132
|
+
// Malformed `fields` JSON on some other tag — not this lookup's job
|
|
133
|
+
// to validate; skip and keep scanning.
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
|
|
104
139
|
/**
|
|
105
140
|
* Look up `field` in `indexed_fields` or throw a loud error suggesting the
|
|
106
141
|
* caller declare it via `update-tag` with `indexed: true`.
|
|
142
|
+
*
|
|
143
|
+
* That generic advice is IMPOSSIBLE to satisfy for a field whose schema
|
|
144
|
+
* declares `type: "number"` (a float) — only `integer`/`boolean` are
|
|
145
|
+
* indexable numeric shapes (see `indexed-fields.ts`'s `TYPE_MAP`), so
|
|
146
|
+
* `update-tag` with `indexed: true` on a `number` field itself throws
|
|
147
|
+
* "unsupported type ... for indexing". An agent that only sees the generic
|
|
148
|
+
* hint has no escape from that loop (LB — round-4 bug hunt). When the
|
|
149
|
+
* declared type is `"number"`, swap in a hint that names the actual fix:
|
|
150
|
+
* store the value as an integer (e.g. cents instead of dollars) and index
|
|
151
|
+
* THAT field instead.
|
|
107
152
|
*/
|
|
108
153
|
export function requireIndexedField(db: Database, field: string): IndexedField {
|
|
109
154
|
const row = getIndexedField(db, field);
|
|
110
155
|
if (!row) {
|
|
156
|
+
if (findDeclaredFieldType(db, field) === "number") {
|
|
157
|
+
throw new QueryError(
|
|
158
|
+
`metadata field "${field}" is not indexed, and can never be: it's declared type: "number" (a float), and only integer/boolean fields are indexable numeric types. "declare indexed: true" will not work on this field.`,
|
|
159
|
+
"FIELD_NOT_INDEXED",
|
|
160
|
+
{
|
|
161
|
+
error_type: "field_not_indexed",
|
|
162
|
+
field,
|
|
163
|
+
hint: `"${field}" is a float ("number") field — float fields can't be server-side indexed, filtered with operators, used in order_by, or summed via aggregate. Store the value as an integer instead (e.g. cents rather than dollars) and declare indexed: true on THAT field to enable those.`,
|
|
164
|
+
},
|
|
165
|
+
);
|
|
166
|
+
}
|
|
111
167
|
throw new QueryError(
|
|
112
168
|
`metadata field "${field}" is not indexed. To use operator queries or order_by on this field, declare it via update-tag with indexed: true.`,
|
|
113
169
|
"FIELD_NOT_INDEXED",
|
package/core/src/schema.ts
CHANGED
|
@@ -3,8 +3,19 @@ import { normalizePath } from "./paths.js";
|
|
|
3
3
|
import { rebuildIndexes, listIndexedFields } from "./indexed-fields.js";
|
|
4
4
|
import { findMixedTypeIndexedFieldNotes } from "./doctor.js";
|
|
5
5
|
import { transaction } from "./txn.js";
|
|
6
|
+
import { timestampToMs } from "./cursor.js";
|
|
6
7
|
|
|
7
|
-
export const SCHEMA_VERSION =
|
|
8
|
+
export const SCHEMA_VERSION = 26;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Deterministic last-resort epoch for a note whose `updated_at` AND
|
|
12
|
+
* `created_at` are both unparseable (vault#586). 0 (the Unix epoch) sorts such
|
|
13
|
+
* a row to the very start of the keyset walk — honest "oldest / unknown"
|
|
14
|
+
* placement — and, being a fixed constant, keeps the backfill idempotent (a
|
|
15
|
+
* re-run derives the identical value). NEVER null: a NULL `updated_at_ms`
|
|
16
|
+
* would sort unpredictably and break the `> ?` keyset predicate.
|
|
17
|
+
*/
|
|
18
|
+
const UNPARSEABLE_UPDATED_AT_MS = 0;
|
|
8
19
|
|
|
9
20
|
export const SCHEMA_SQL = `
|
|
10
21
|
-- Notes: the universal record.
|
|
@@ -26,6 +37,13 @@ export const SCHEMA_SQL = `
|
|
|
26
37
|
-- (mcp, surface:NAME, agent:ID, operator/cli, api). Legacy rows stay NULL —
|
|
27
38
|
-- we don't fabricate authors for writes that predate attribution. See
|
|
28
39
|
-- migrateToV23.
|
|
40
|
+
-- updated_at_ms (v26, vault#586) is the integer millisecond-epoch mirror of
|
|
41
|
+
-- updated_at, and the SINGLE source of truth for cursor keyset ordering. The
|
|
42
|
+
-- keyset walk-order, boundary predicate, and watermark all read this one
|
|
43
|
+
-- numeric column, so they can't diverge the way three separate readings of the
|
|
44
|
+
-- TEXT updated_at did on aged/imported vaults with non-canonical timestamps.
|
|
45
|
+
-- Maintained on every write (createNote / updateNote / restore / cascades) via
|
|
46
|
+
-- core/src/cursor.ts timestampToMs; existing rows backfilled by migrateToV26.
|
|
29
47
|
CREATE TABLE IF NOT EXISTS notes (
|
|
30
48
|
id TEXT PRIMARY KEY,
|
|
31
49
|
content TEXT DEFAULT '',
|
|
@@ -37,7 +55,8 @@ CREATE TABLE IF NOT EXISTS notes (
|
|
|
37
55
|
created_by TEXT,
|
|
38
56
|
created_via TEXT,
|
|
39
57
|
last_updated_by TEXT,
|
|
40
|
-
last_updated_via TEXT
|
|
58
|
+
last_updated_via TEXT,
|
|
59
|
+
updated_at_ms INTEGER
|
|
41
60
|
);
|
|
42
61
|
|
|
43
62
|
-- Tags: first-class identity carrying schema, hierarchy, and typed-link
|
|
@@ -529,6 +548,12 @@ export function initSchema(db: Database): void {
|
|
|
529
548
|
// porter stemming, repopulate from every existing note. See vault#551.
|
|
530
549
|
migrateToV25(db);
|
|
531
550
|
|
|
551
|
+
// Migrate v25 → v26: add the integer `notes.updated_at_ms` column — the
|
|
552
|
+
// single source of truth for cursor keyset ordering — plus its
|
|
553
|
+
// (updated_at_ms, id) index, and backfill every existing row from its
|
|
554
|
+
// `updated_at` string with a UTC-correct parse. See vault#586.
|
|
555
|
+
migrateToV26(db);
|
|
556
|
+
|
|
532
557
|
// Rebuild any generated columns + indexes declared in indexed_fields.
|
|
533
558
|
// No-op for a fresh vault; idempotent on existing vaults.
|
|
534
559
|
rebuildIndexes(db);
|
|
@@ -1491,6 +1516,107 @@ function migrateToV25(db: Database): void {
|
|
|
1491
1516
|
);
|
|
1492
1517
|
}
|
|
1493
1518
|
|
|
1519
|
+
/**
|
|
1520
|
+
* Migrate v25 → v26: integer `notes.updated_at_ms` as the single source of
|
|
1521
|
+
* truth for cursor keyset ordering (vault#586).
|
|
1522
|
+
*
|
|
1523
|
+
* THE BUG this closes: the `(updated_at, id)` cursor keyset used THREE
|
|
1524
|
+
* inconsistent orderings of `updated_at` that agree only when every timestamp
|
|
1525
|
+
* is canonical `.toISOString()` output — TEXT-lexicographic walk order, a
|
|
1526
|
+
* canonical-ISO boundary string compared as TEXT, and a `Date.parse`-derived
|
|
1527
|
+
* millis watermark that read space-form timestamps in LOCAL time. Import
|
|
1528
|
+
* stores frontmatter timestamps VERBATIM, so aged/imported vaults carry
|
|
1529
|
+
* non-canonical `updated_at` (`2024-11-02 14:30:00` space-form, `+02:00`
|
|
1530
|
+
* offset, no-`Z`) — under which the three orderings diverge and cursor
|
|
1531
|
+
* pagination silently skipped notes, re-delivered rows in a loop, or 400'd the
|
|
1532
|
+
* whole walk. A single integer column makes walk-order, keyset boundary, and
|
|
1533
|
+
* watermark ONE numeric ordering.
|
|
1534
|
+
*
|
|
1535
|
+
* Backfill parse (`timestampToMs`) is UTC-correct — it does NOT use
|
|
1536
|
+
* `Date.parse` for zone-less forms (the live bug was space-form read as LOCAL
|
|
1537
|
+
* time). A genuinely unparseable `updated_at` falls back to the row's
|
|
1538
|
+
* `created_at` ms, then to a stable sentinel ({@link UNPARSEABLE_UPDATED_AT_MS})
|
|
1539
|
+
* — never NULL, never a throw.
|
|
1540
|
+
*
|
|
1541
|
+
* ALL-OR-NOTHING (matches the #565 migrateToV25 pattern): the ALTER + backfill
|
|
1542
|
+
* run inside a SINGLE `transaction`. If the ALTER committed but the backfill
|
|
1543
|
+
* then crashed, a column-presence-only guard would see the column on the next
|
|
1544
|
+
* boot and skip — leaving rows with NULL `updated_at_ms` and a permanently
|
|
1545
|
+
* broken keyset. Wrapping the whole sequence means a rollback removes the
|
|
1546
|
+
* column too, so the next boot re-detects "not migrated" and re-runs cleanly.
|
|
1547
|
+
* The index create lives OUTSIDE the wrapped block (`CREATE INDEX IF NOT
|
|
1548
|
+
* EXISTS`) so a fresh vault — column already present from SCHEMA_SQL — still
|
|
1549
|
+
* gets the index.
|
|
1550
|
+
*
|
|
1551
|
+
* SELF-HEALING (generalist review nit 1): the backfill is NOT gated on column
|
|
1552
|
+
* absence — it runs whenever ANY row has a NULL `updated_at_ms`, via a single
|
|
1553
|
+
* `WHERE updated_at_ms IS NULL` pass that serves BOTH the fresh-ALTER case
|
|
1554
|
+
* (every row NULL right after ADD COLUMN) AND the leak case (a row that slipped
|
|
1555
|
+
* in with NULL while the column already existed). The leak is real: a
|
|
1556
|
+
* schema-v2-era vault upgrades through `migrateFromV2`, whose
|
|
1557
|
+
* `INSERT INTO notes … SELECT` omits `updated_at_ms`, landing rows with NULL
|
|
1558
|
+
* even though SCHEMA_SQL created the column — a column-presence guard would
|
|
1559
|
+
* skip them forever (the exact keyset-invisibility class this migration
|
|
1560
|
+
* closes). Steady state (column present, zero NULLs) does a single cheap
|
|
1561
|
+
* COUNT and no writes.
|
|
1562
|
+
*/
|
|
1563
|
+
function migrateToV26(db: Database): void {
|
|
1564
|
+
if (!hasTable(db, "notes")) return;
|
|
1565
|
+
|
|
1566
|
+
const needsColumn = !hasColumn(db, "notes", "updated_at_ms");
|
|
1567
|
+
// Backfill runs when the column is being added OR when any existing row
|
|
1568
|
+
// carries a NULL keyset key (self-heal). A steady-state boot short-circuits
|
|
1569
|
+
// here on the cheap COUNT and touches nothing.
|
|
1570
|
+
let pending = needsColumn;
|
|
1571
|
+
if (!needsColumn) {
|
|
1572
|
+
const nulls = (
|
|
1573
|
+
db.prepare("SELECT COUNT(*) AS n FROM notes WHERE updated_at_ms IS NULL").get() as { n: number }
|
|
1574
|
+
).n;
|
|
1575
|
+
pending = nulls > 0;
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
if (pending) {
|
|
1579
|
+
let backfilled = 0;
|
|
1580
|
+
let fellBack = 0;
|
|
1581
|
+
transaction(db, () => {
|
|
1582
|
+
if (needsColumn) {
|
|
1583
|
+
db.exec("ALTER TABLE notes ADD COLUMN updated_at_ms INTEGER");
|
|
1584
|
+
}
|
|
1585
|
+
// One `WHERE updated_at_ms IS NULL` pass covers both cases: right after
|
|
1586
|
+
// ALTER every row is NULL; on a self-heal only the leaked rows are.
|
|
1587
|
+
const rows = db.prepare(
|
|
1588
|
+
"SELECT id, created_at, updated_at FROM notes WHERE updated_at_ms IS NULL",
|
|
1589
|
+
).all() as { id: string; created_at: string | null; updated_at: string | null }[];
|
|
1590
|
+
const update = db.prepare("UPDATE notes SET updated_at_ms = ? WHERE id = ?");
|
|
1591
|
+
for (const row of rows) {
|
|
1592
|
+
// `updated_at` is the ordering authority; fall back to `created_at`
|
|
1593
|
+
// (legacy rows can carry NULL updated_at), then the stable sentinel.
|
|
1594
|
+
let ms = timestampToMs(row.updated_at);
|
|
1595
|
+
if (ms === null) {
|
|
1596
|
+
ms = timestampToMs(row.created_at);
|
|
1597
|
+
fellBack++;
|
|
1598
|
+
}
|
|
1599
|
+
if (ms === null) ms = UNPARSEABLE_UPDATED_AT_MS;
|
|
1600
|
+
update.run(ms, row.id);
|
|
1601
|
+
backfilled++;
|
|
1602
|
+
}
|
|
1603
|
+
});
|
|
1604
|
+
if (backfilled > 0) {
|
|
1605
|
+
console.log(
|
|
1606
|
+
`[vault] migrated to schema v26 (vault#586): backfilled updated_at_ms for ${backfilled} note(s)` +
|
|
1607
|
+
(needsColumn ? "" : " (self-heal — NULL keyset keys on an existing column)") +
|
|
1608
|
+
(fellBack > 0 ? ` (${fellBack} fell back to created_at / sentinel for an unparseable updated_at)` : "") +
|
|
1609
|
+
`.`,
|
|
1610
|
+
);
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// Index lives outside the ALTER transaction (idx_tokens_vault_name /
|
|
1615
|
+
// idx_notes_updated precedent): a fresh vault has the column from SCHEMA_SQL
|
|
1616
|
+
// but not yet the index, so this ensures it for both paths. Idempotent.
|
|
1617
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_notes_updated_ms ON notes(updated_at_ms, id)");
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1494
1620
|
function hasTable(db: Database, name: string): boolean {
|
|
1495
1621
|
const row = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(name);
|
|
1496
1622
|
return !!row;
|
package/core/src/seed-packs.ts
CHANGED
|
@@ -50,7 +50,7 @@ import type { Store, TagFieldSchema } from "./types.ts";
|
|
|
50
50
|
* here — a description is vault metadata, not a pack note.
|
|
51
51
|
*/
|
|
52
52
|
export const DEFAULT_VAULT_DESCRIPTION =
|
|
53
|
-
'This is a brand-new personal vault — just its starter guides so far; the person it belongs to is probably new to Parachute. Before helping them set it up, read the "Getting Started" note — it\'s your onboarding brief. Short version: start with a conversation, not a filing system; learn how they want to use this and where their notes live today; do one small real thing first. When the vault has real structure, replace this description (vault-info { description: "..." }) with a current picture of what lives here and how to work it — so every future session, yours or another AI\'s, starts oriented.';
|
|
53
|
+
'This is a brand-new personal vault — just its starter guides so far; the person it belongs to is probably new to Parachute. Before helping them set it up, read the "Getting Started" note — it\'s your onboarding brief. Short version: start with a conversation, not a filing system; learn how they want to use this and where their notes live today; do one small real thing first. When the vault has real structure, replace this description (vault-info { description: "..." } — an admin-scope action) with a current picture of what lives here and how to work it — so every future session, yours or another AI\'s, starts oriented.';
|
|
54
54
|
|
|
55
55
|
/**
|
|
56
56
|
* The `vault-info` description a vault gets when it was **imported/restored** —
|
|
@@ -59,7 +59,7 @@ export const DEFAULT_VAULT_DESCRIPTION =
|
|
|
59
59
|
* text with a current picture.
|
|
60
60
|
*/
|
|
61
61
|
export const IMPORTED_VAULT_DESCRIPTION =
|
|
62
|
-
'This vault was imported — it arrived with its own notes, tags, and history. Orient before you write: vault-info { include_stats: true }, list-tags, read a sample, and read "Getting Started" if present. Your first job is to learn the shape that\'s already here and reflect it back to the person — not to propose new structure cold. When you understand it, replace this description (vault-info { description: "..." }) with a current picture.';
|
|
62
|
+
'This vault was imported — it arrived with its own notes, tags, and history. Orient before you write: vault-info { include_stats: true }, list-tags, read a sample, and read "Getting Started" if present. Your first job is to learn the shape that\'s already here and reflect it back to the person — not to propose new structure cold. When you understand it, replace this description (vault-info { description: "..." } — an admin-scope action) with a current picture.';
|
|
63
63
|
|
|
64
64
|
// ---------------------------------------------------------------------------
|
|
65
65
|
// Pack shape
|
|
@@ -404,8 +404,8 @@ what's already there.
|
|
|
404
404
|
### Close the loop: describe the vault
|
|
405
405
|
|
|
406
406
|
When the first real structure lands, update the vault description —
|
|
407
|
-
\`vault-info { description: "..." }\` — so every future
|
|
408
|
-
AI's) starts oriented: what this vault is for, its main tags, its conventions.
|
|
407
|
+
\`vault-info { description: "..." }\`, an \`admin\`-scope action — so every future
|
|
408
|
+
session (yours or another AI's) starts oriented: what this vault is for, its main tags, its conventions.
|
|
409
409
|
The default description says "brand-new vault"; once that stops being true,
|
|
410
410
|
replace it. Keep it a few sentences (the projection carries the schema detail,
|
|
411
411
|
and this note carries the richer conventions — see "Adapt this note"). Bringing
|
|
@@ -423,18 +423,44 @@ small welcome web and the \`capture\` tag the Notes surface uses; no other
|
|
|
423
423
|
predefined tags or schema. You and the operator design the structure that fits
|
|
424
424
|
*their* life and work. The vault is the engine; the meaning is yours to bring.
|
|
425
425
|
|
|
426
|
-
Core moves
|
|
427
|
-
|
|
428
|
-
- \`query-notes\`
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
-
|
|
432
|
-
|
|
433
|
-
|
|
426
|
+
Core moves, grouped by the scope each needs — \`tools/list\` shows exactly the
|
|
427
|
+
ones your token can call (see "What your scope allows" below):
|
|
428
|
+
- **Read:** \`query-notes\` (by id/path, by tag, full-text \`search\`, or graph
|
|
429
|
+
\`near\` a note), \`list-tags\`, \`find-path\` (shortest link path between two
|
|
430
|
+
notes), \`vault-info\` (the live schema projection + a compact \`map\` — tag
|
|
431
|
+
counts, top-level path-bucket counts, total notes — so you orient in one
|
|
432
|
+
call), \`doctor\` (read-only integrity scan).
|
|
433
|
+
- **Write:** \`create-note\` / \`update-note\` / \`delete-note\` — author notes,
|
|
434
|
+
single or batch.
|
|
435
|
+
- **Admin:** \`update-tag\` / \`delete-tag\` / \`rename-tag\` / \`merge-tags\` /
|
|
436
|
+
\`prune-schema\` (the tag vocabulary + schemas), \`vault-info { description }\`
|
|
437
|
+
(the vault's own description), \`manage-token\` (mint scoped child tokens).
|
|
434
438
|
|
|
435
439
|
\`[[wikilinks]]\` in note content auto-link to the note at that path — use them
|
|
436
440
|
freely; they resolve even if the target is created later.
|
|
437
441
|
|
|
442
|
+
## What your scope allows
|
|
443
|
+
|
|
444
|
+
Your connection carries one of three scopes, and it shapes what you can do here
|
|
445
|
+
— by design, so the person can grant exactly as much authority as they mean to:
|
|
446
|
+
|
|
447
|
+
- **\`read\`** — explore and answer: query notes, read tags, traverse the graph,
|
|
448
|
+
run \`doctor\`. You can't change anything.
|
|
449
|
+
- **\`write\`** — everything read can do, plus author content: create, update,
|
|
450
|
+
and delete notes. You can capture and edit freely, but you *can't* reshape the
|
|
451
|
+
vault's structure — renaming/merging/deleting tags, editing schemas, and
|
|
452
|
+
rewriting the vault description all need admin.
|
|
453
|
+
- **\`admin\`** — everything, including that restructuring, plus minting scoped
|
|
454
|
+
child tokens.
|
|
455
|
+
|
|
456
|
+
If a tool this guide mentions fails with "Unknown tool" — or a normally-visible
|
|
457
|
+
tool refuses one argument with "Forbidden" (e.g. \`vault-info { description }\`
|
|
458
|
+
for a non-admin) — that's your scope, not a bug: the action sits above your
|
|
459
|
+
tier. Don't fight it: tell the person plainly
|
|
460
|
+
what you'd do with more access ("I can add notes, but reorganizing your tags
|
|
461
|
+
needs admin — want to grant it?") and let them decide. Most people connect with
|
|
462
|
+
admin, so usually you'll have the whole set.
|
|
463
|
+
|
|
438
464
|
## Tags vs paths vs schemas — the design vocabulary
|
|
439
465
|
|
|
440
466
|
These three axes are the heart of vault design. Use the right one for the job:
|
|
@@ -453,7 +479,7 @@ These three axes are the heart of vault design. Use the right one for the job:
|
|
|
453
479
|
path, tags, or both.
|
|
454
480
|
|
|
455
481
|
- **Schemas = typed metadata fields.** Attach a schema to a tag (via
|
|
456
|
-
\`update-tag\`) to declare typed metadata fields — e.g. \`#meeting\` with a
|
|
482
|
+
\`update-tag\`, an \`admin\`-scope move) to declare typed metadata fields — e.g. \`#meeting\` with a
|
|
457
483
|
\`held_on\` date, \`#person\` with an \`email\`. Each field can **optionally** be
|
|
458
484
|
marked \`indexed: true\` to make it **queryable with operators** (\`query-notes
|
|
459
485
|
{ tag: "meeting", metadata: { held_on: { gte: "2026-01-01" } } }\`); indexing
|
package/core/src/store.ts
CHANGED
|
@@ -15,8 +15,10 @@ import {
|
|
|
15
15
|
resolveUnresolvedWikilinks,
|
|
16
16
|
resolveOrQueueLink,
|
|
17
17
|
clearQueuedLink,
|
|
18
|
+
requeueInboundWikilinksForDelete,
|
|
18
19
|
} from "./wikilinks.js";
|
|
19
20
|
import { pathTitle } from "./paths.js";
|
|
21
|
+
import { timestampToMs } from "./cursor.js";
|
|
20
22
|
import { transaction } from "./txn.js";
|
|
21
23
|
import { HookRegistry } from "./hooks.js";
|
|
22
24
|
import {
|
|
@@ -360,9 +362,18 @@ export class BunSqliteStore implements Store {
|
|
|
360
362
|
// the importer write a specific historical timestamp. Skips hooks
|
|
361
363
|
// by design: this isn't a user-edit, it's a state restoration.
|
|
362
364
|
// See vault#308 PR 2.
|
|
365
|
+
//
|
|
366
|
+
// This is THE path by which non-canonical timestamps (space-form,
|
|
367
|
+
// `+02:00` offset, no-`Z`) land in a vault — frontmatter is preserved
|
|
368
|
+
// VERBATIM (byte-identical re-export round-trip), so `updated_at` is NOT
|
|
369
|
+
// canonicalized here. The keyset ordering key `updated_at_ms` (vault#586)
|
|
370
|
+
// is derived UTC-correctly from that verbatim value: `timestampToMs` does
|
|
371
|
+
// NOT read space-form as local time. A genuinely unparseable `updated_at`
|
|
372
|
+
// falls back to `created_at`'s ms, then to 0 — never NULL, never a throw.
|
|
373
|
+
const updatedAtMs = timestampToMs(updatedAt) ?? timestampToMs(createdAt) ?? 0;
|
|
363
374
|
this.db
|
|
364
|
-
.prepare("UPDATE notes SET created_at = ?, updated_at = ? WHERE id = ?")
|
|
365
|
-
.run(createdAt, updatedAt, id);
|
|
375
|
+
.prepare("UPDATE notes SET created_at = ?, updated_at = ?, updated_at_ms = ? WHERE id = ?")
|
|
376
|
+
.run(createdAt, updatedAt, updatedAtMs, id);
|
|
366
377
|
}
|
|
367
378
|
|
|
368
379
|
async deleteNote(id: string): Promise<void> {
|
|
@@ -372,6 +383,13 @@ export class BunSqliteStore implements Store {
|
|
|
372
383
|
// by design, hooks subscribing to "deleted" receive a DeletedNoteRef,
|
|
373
384
|
// not a Note.
|
|
374
385
|
const existing = noteOps.getNote(this.db, id);
|
|
386
|
+
// LB6: re-queue inbound wikilink edges BEFORE the FK cascade drops the
|
|
387
|
+
// `links` rows, so recreating a note at this path/title auto-heals the
|
|
388
|
+
// edge instead of leaving every referencing note's `[[link]]` dead until
|
|
389
|
+
// it's individually re-saved. See requeueInboundWikilinksForDelete's doc
|
|
390
|
+
// comment for why this must run pre-delete and what it deliberately
|
|
391
|
+
// excludes (typed `links`, not just wikilinks).
|
|
392
|
+
requeueInboundWikilinksForDelete(this.db, id);
|
|
375
393
|
noteOps.deleteNote(this.db, id);
|
|
376
394
|
if (existing?.path) this.invalidateConfigCachesForPath(existing.path);
|
|
377
395
|
// Dispatch even when `existing` was null — the caller asked for a
|
|
@@ -372,6 +372,19 @@ export function computeExpandedTagCounts(
|
|
|
372
372
|
export function findHierarchyCycles(h: TagHierarchy): string[] {
|
|
373
373
|
const cycles: string[] = [];
|
|
374
374
|
for (const tag of h.childrenOf.keys()) {
|
|
375
|
+
// Direct self-edge (`parent_names` lists the tag itself) — checked
|
|
376
|
+
// separately from the descendant-set gate below because
|
|
377
|
+
// `getTagDescendants` can't see it: its traversal seeds `result` with
|
|
378
|
+
// `{tag}` and then skips re-expanding any child already present in
|
|
379
|
+
// `result` (the visited-set that makes runtime traversal cycle-safe) —
|
|
380
|
+
// so a bare `X -> X` edge never grows `descendants` past size 1, and
|
|
381
|
+
// `descendants.size > 1` alone silently reports it clean. Only
|
|
382
|
+
// reachable via guard-bypassing data (a direct DB write, or an import
|
|
383
|
+
// that skips `upsertTagRecord`'s write-time cycle guard).
|
|
384
|
+
if (h.childrenOf.get(tag)?.has(tag)) {
|
|
385
|
+
cycles.push(tag);
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
375
388
|
const descendants = getTagDescendants(h, tag);
|
|
376
389
|
if (descendants.has(tag) && descendants.size > 1) {
|
|
377
390
|
// tag reaches itself through a non-trivial path
|