@openparachute/vault 0.7.0-rc.9 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/aggregate.test.ts +260 -0
- package/core/src/contract-taxonomy.test.ts +26 -2
- package/core/src/contract-typed-index.test.ts +4 -2
- package/core/src/core.test.ts +669 -0
- package/core/src/doctor.ts +23 -1
- package/core/src/indexed-fields.test.ts +4 -1
- package/core/src/indexed-fields.ts +6 -1
- package/core/src/links.ts +22 -0
- package/core/src/mcp.ts +310 -51
- package/core/src/notes.ts +344 -18
- package/core/src/query-warnings.ts +60 -12
- package/core/src/schema-defaults.ts +7 -1
- package/core/src/seed-packs.ts +19 -5
- package/core/src/store.ts +129 -5
- package/core/src/tag-schemas.ts +20 -7
- package/core/src/types.ts +89 -0
- package/core/src/ulid.test.ts +56 -0
- package/core/src/ulid.ts +116 -0
- package/core/src/vault-projection.ts +19 -1
- package/core/src/wikilinks.test.ts +205 -1
- package/core/src/wikilinks.ts +200 -35
- package/package.json +1 -1
- package/src/aggregate-routes.test.ts +230 -0
- package/src/contract-errors.test.ts +2 -1
- package/src/contract-search.test.ts +17 -0
- package/src/mcp-link-warnings-scope.test.ts +144 -0
- package/src/mcp-query-notes-aggregate-scope.test.ts +183 -0
- package/src/mcp-tools.ts +76 -17
- package/src/routes.ts +272 -23
- package/src/routing.test.ts +167 -0
- package/src/routing.ts +47 -11
- package/src/tag-integrity-mcp.test.ts +45 -6
- package/src/tag-scope.ts +10 -1
- package/src/vault.test.ts +469 -21
- package/web/ui/dist/assets/{index-B8pGeQam.js → index-CJ6NtIwh.js} +1 -1
- package/web/ui/dist/index.html +1 -1
package/core/src/store.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
|
-
import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage } from "./types.js";
|
|
2
|
+
import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage, AggregateRow } from "./types.js";
|
|
3
3
|
import { initSchema } from "./schema.js";
|
|
4
4
|
import * as noteOps from "./notes.js";
|
|
5
5
|
import * as linkOps from "./links.js";
|
|
@@ -10,7 +10,12 @@ import {
|
|
|
10
10
|
reconcileDeclaredIndexes,
|
|
11
11
|
type PrunedField,
|
|
12
12
|
} from "./indexed-fields.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
syncWikilinks,
|
|
15
|
+
resolveUnresolvedWikilinks,
|
|
16
|
+
resolveOrQueueLink,
|
|
17
|
+
clearQueuedLink,
|
|
18
|
+
} from "./wikilinks.js";
|
|
14
19
|
import { pathTitle } from "./paths.js";
|
|
15
20
|
import { transaction } from "./txn.js";
|
|
16
21
|
import { HookRegistry } from "./hooks.js";
|
|
@@ -27,6 +32,7 @@ import {
|
|
|
27
32
|
import {
|
|
28
33
|
loadSchemaConfig,
|
|
29
34
|
validateNote as runValidateNote,
|
|
35
|
+
resolveNoteSchemas,
|
|
30
36
|
type ResolvedSchemas,
|
|
31
37
|
type ValidationStatus,
|
|
32
38
|
} from "./schema-defaults.js";
|
|
@@ -104,6 +110,94 @@ export class BunSqliteStore implements Store {
|
|
|
104
110
|
return runValidateNote(this.getSchemaConfig(), note);
|
|
105
111
|
}
|
|
106
112
|
|
|
113
|
+
/**
|
|
114
|
+
* Auto-link sync for `type: "reference"` schema fields
|
|
115
|
+
* (vault#typed-reference-field — see `docs/design/typed-reference-field.md`
|
|
116
|
+
* for the full design + known gaps).
|
|
117
|
+
*
|
|
118
|
+
* A `reference`-typed field is BOTH an indexed value (handled by the
|
|
119
|
+
* ordinary metadata write — no special casing needed there, see
|
|
120
|
+
* `tag-schemas.ts`'s `VALID_FIELD_TYPES`) AND a graph link. This method
|
|
121
|
+
* maintains the second half: for every field the note's EFFECTIVE schema
|
|
122
|
+
* (its own tags + ancestors, same resolution `validateNoteAgainstSchemas`
|
|
123
|
+
* uses) declares `type: "reference"`, resolve the field's current metadata
|
|
124
|
+
* value to a note (reusing the SAME `resolveOrQueueLink` machinery
|
|
125
|
+
* structured `links` entries use — id, then path/title, with lazy
|
|
126
|
+
* forward-ref queueing on a miss) and maintain a `links` edge from this
|
|
127
|
+
* note to that target, `relationship` = the field name.
|
|
128
|
+
*
|
|
129
|
+
* Called from `createNote`/`updateNote`/`createNotes` — the single
|
|
130
|
+
* chokepoint both MCP and REST funnel through — AFTER the note row itself
|
|
131
|
+
* is written, so `note.tags`/`note.metadata` reflect the final state.
|
|
132
|
+
*
|
|
133
|
+
* `priorMetadata` is the note's metadata BEFORE this write (`undefined` on
|
|
134
|
+
* create). For each reference field, if its value is unchanged from
|
|
135
|
+
* `priorMetadata`, nothing is touched — the existing link (if any) already
|
|
136
|
+
* reflects it, and re-running the resolve/queue machinery on every
|
|
137
|
+
* unrelated write (e.g. a content-only edit) would be wasted work. When
|
|
138
|
+
* the value DID change (set, changed, or removed), every existing link +
|
|
139
|
+
* queued forward-ref under that field's relationship name is cleared
|
|
140
|
+
* first, then re-established from the new value — this makes the field's
|
|
141
|
+
* current value the single source of truth for "this field's link"
|
|
142
|
+
* without needing to track the specific prior target.
|
|
143
|
+
*
|
|
144
|
+
* Known gaps (see the design doc): only scalar (`cardinality: "one"`,
|
|
145
|
+
* the default) reference values are linked — an array value is left as a
|
|
146
|
+
* validated-like-string-per-item... actually an array value simply isn't
|
|
147
|
+
* a string, so no link is created for it (the write still succeeds; a
|
|
148
|
+
* `type_mismatch`/`cardinality_mismatch` warning surfaces via the normal
|
|
149
|
+
* `validation_status` path). A tag gaining a `reference` field declaration
|
|
150
|
+
* does NOT retroactively link notes that already carry a matching value —
|
|
151
|
+
* only writes that actually touch the field (going forward) sync.
|
|
152
|
+
*/
|
|
153
|
+
private syncReferenceFieldLinks(
|
|
154
|
+
note: Note,
|
|
155
|
+
priorMetadata: Record<string, unknown> | undefined,
|
|
156
|
+
): void {
|
|
157
|
+
const resolution = resolveNoteSchemas(this.getSchemaConfig(), { tags: note.tags ?? [] });
|
|
158
|
+
if (resolution.mergedFields.size === 0) return;
|
|
159
|
+
|
|
160
|
+
const metadata = note.metadata ?? {};
|
|
161
|
+
const prior = priorMetadata ?? {};
|
|
162
|
+
|
|
163
|
+
for (const [fieldName, { spec }] of resolution.mergedFields) {
|
|
164
|
+
if (spec.type !== "reference") continue;
|
|
165
|
+
|
|
166
|
+
const nextValue = metadata[fieldName];
|
|
167
|
+
const priorValue = prior[fieldName];
|
|
168
|
+
if (nextValue === priorValue) continue; // unchanged — link (if any) already reflects it
|
|
169
|
+
|
|
170
|
+
// Re-establish this field's link from scratch: drop whatever it
|
|
171
|
+
// previously pointed at (a resolved link, and/or a queued forward-ref)
|
|
172
|
+
// before applying the new value. See this method's doc comment for why
|
|
173
|
+
// "clear then recreate" is safe and simpler than tracking the prior
|
|
174
|
+
// resolved target.
|
|
175
|
+
linkOps.deleteLinksBySourceRelationship(this.db, note.id, fieldName);
|
|
176
|
+
clearQueuedLink(this.db, note.id, fieldName);
|
|
177
|
+
|
|
178
|
+
if (typeof nextValue === "string" && nextValue.trim() !== "") {
|
|
179
|
+
// Resolves now, or queues a lazy forward-ref on a miss — same
|
|
180
|
+
// contract as a structured `links` entry (core/src/mcp.ts,
|
|
181
|
+
// src/routes.ts). A queued forward-ref backfills automatically via
|
|
182
|
+
// `resolveUnresolvedWikilinks` the moment a matching note is
|
|
183
|
+
// created, and surfaces to callers today via the existing
|
|
184
|
+
// `has_broken_links`/`broken_links` query-notes filters (both read
|
|
185
|
+
// the same `unresolved_wikilinks` table this queues into) — see the
|
|
186
|
+
// design doc for the follow-up to also surface an inline
|
|
187
|
+
// `unresolved_link`/`ambiguous_link` warning on the create/update
|
|
188
|
+
// response itself. An `"ambiguous"` outcome (vault#570 — the field's
|
|
189
|
+
// value matched ≥2 notes, e.g. two notes sharing an H1 title) is
|
|
190
|
+
// treated the same as a miss here: no link is created, and nothing
|
|
191
|
+
// is queued (mirrors `resolveOrQueueLink`'s own "don't guess"
|
|
192
|
+
// contract for structured links).
|
|
193
|
+
const outcome = resolveOrQueueLink(this.db, note.id, nextValue, fieldName);
|
|
194
|
+
if (outcome.status === "resolved") {
|
|
195
|
+
linkOps.createLink(this.db, note.id, outcome.note_id, fieldName);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
107
201
|
/**
|
|
108
202
|
* Drop the tag-hierarchy cache if the mutated path is in the `_tags/*`
|
|
109
203
|
* namespace. Called from create/update/delete — old path is passed
|
|
@@ -135,6 +229,11 @@ export class BunSqliteStore implements Store {
|
|
|
135
229
|
resolveUnresolvedWikilinks(this.db, note.path, note.id);
|
|
136
230
|
}
|
|
137
231
|
|
|
232
|
+
// Reference-field auto-link (vault#typed-reference-field) — no prior
|
|
233
|
+
// metadata on a fresh create. Cheap no-op when nothing on the note's
|
|
234
|
+
// effective schema declares `type: "reference"`.
|
|
235
|
+
this.syncReferenceFieldLinks(note, undefined);
|
|
236
|
+
|
|
138
237
|
this.invalidateConfigCachesForPath(note.path);
|
|
139
238
|
this.hooks.dispatch("created", note, this);
|
|
140
239
|
|
|
@@ -171,9 +270,17 @@ export class BunSqliteStore implements Store {
|
|
|
171
270
|
},
|
|
172
271
|
): Promise<Note> {
|
|
173
272
|
let oldPath: string | undefined;
|
|
174
|
-
|
|
273
|
+
// Reference-field auto-link sync (vault#typed-reference-field) needs the
|
|
274
|
+
// PRE-write metadata to detect which reference fields actually changed —
|
|
275
|
+
// read it now, before `noteOps.updateNote` overwrites the row. Only
|
|
276
|
+
// needed when this call touches `metadata` at all (a content/tags/path-
|
|
277
|
+
// only update can't have changed a reference field's value, so this read
|
|
278
|
+
// is skipped on the common non-metadata-touching path).
|
|
279
|
+
let priorMetadataForRefs: Record<string, unknown> | undefined;
|
|
280
|
+
if (updates.path !== undefined || updates.metadata !== undefined) {
|
|
175
281
|
const existing = noteOps.getNote(this.db, id);
|
|
176
|
-
oldPath = existing?.path;
|
|
282
|
+
if (updates.path !== undefined) oldPath = existing?.path;
|
|
283
|
+
if (updates.metadata !== undefined) priorMetadataForRefs = existing?.metadata;
|
|
177
284
|
}
|
|
178
285
|
|
|
179
286
|
const note = noteOps.updateNote(this.db, id, updates);
|
|
@@ -192,6 +299,13 @@ export class BunSqliteStore implements Store {
|
|
|
192
299
|
resolveUnresolvedWikilinks(this.db, note.path, id);
|
|
193
300
|
}
|
|
194
301
|
|
|
302
|
+
// Reference-field auto-link sync (vault#typed-reference-field). Only
|
|
303
|
+
// when this call actually touched `metadata` — see the read above for
|
|
304
|
+
// why a content/tags/path-only update is skipped.
|
|
305
|
+
if (updates.metadata !== undefined) {
|
|
306
|
+
this.syncReferenceFieldLinks(note, priorMetadataForRefs);
|
|
307
|
+
}
|
|
308
|
+
|
|
195
309
|
// Invalidate before the hook dispatch so any handler that re-queries
|
|
196
310
|
// the hierarchy from inside its own logic sees post-write state.
|
|
197
311
|
// `metadata` updates can change the `parents` field on a config note
|
|
@@ -311,6 +425,13 @@ export class BunSqliteStore implements Store {
|
|
|
311
425
|
return noteOps.queryNotesPaged(this.db, this.expandQueryTags(this.normalizeQueryTags(opts)));
|
|
312
426
|
}
|
|
313
427
|
|
|
428
|
+
async aggregateNotes(opts: QueryOpts): Promise<AggregateRow[]> {
|
|
429
|
+
// Same bare-tag normalization + hierarchy expansion `queryNotes` gets —
|
|
430
|
+
// `opts.tags`/`excludeTags` filter the aggregate's input set the same
|
|
431
|
+
// way they'd filter a normal query's result list.
|
|
432
|
+
return noteOps.aggregateNotes(this.db, this.expandQueryTags(this.normalizeQueryTags(opts)));
|
|
433
|
+
}
|
|
434
|
+
|
|
314
435
|
/**
|
|
315
436
|
* If `tags` are present, attach a parallel `_tagsExpanded` array where
|
|
316
437
|
* each input tag is replaced with `{tag} ∪ descendants(tag)`. The SQL
|
|
@@ -554,6 +675,9 @@ export class BunSqliteStore implements Store {
|
|
|
554
675
|
// would leave the hierarchy cache stale until the next singleton
|
|
555
676
|
// write happened to bust it.
|
|
556
677
|
this.invalidateConfigCachesForPath(note.path);
|
|
678
|
+
// Same reference-field auto-link sync as singleton createNote
|
|
679
|
+
// (vault#typed-reference-field) — no prior metadata on a fresh create.
|
|
680
|
+
this.syncReferenceFieldLinks(note, undefined);
|
|
557
681
|
this.hooks.dispatch("created", note, this);
|
|
558
682
|
}
|
|
559
683
|
return notes;
|
|
@@ -735,7 +859,7 @@ export class BunSqliteStore implements Store {
|
|
|
735
859
|
const mapped = indexedFieldOps.mapFieldType(spec.type);
|
|
736
860
|
if (!mapped) {
|
|
737
861
|
throw new indexedFieldOps.IndexedFieldError(
|
|
738
|
-
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean)`,
|
|
862
|
+
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
|
|
739
863
|
);
|
|
740
864
|
}
|
|
741
865
|
// Throws IndexedFieldError on an invalid identifier (e.g. kebab-case).
|
package/core/src/tag-schemas.ts
CHANGED
|
@@ -255,15 +255,23 @@ export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFi
|
|
|
255
255
|
|
|
256
256
|
/**
|
|
257
257
|
* The full recognized vocabulary for `TagFieldSchema.type` — storage/
|
|
258
|
-
* advisory validation accepts all
|
|
259
|
-
* are INDEXABLE (that narrower subset is `indexed-fields.ts`'s
|
|
260
|
-
* enforced separately via `mapFieldType` for `indexed: true`
|
|
261
|
-
* Matches `defaultMatchesType`'s switch and `schema-defaults.ts`'s
|
|
258
|
+
* advisory validation accepts all seven; only `string`/`integer`/`boolean`/
|
|
259
|
+
* `reference` are INDEXABLE (that narrower subset is `indexed-fields.ts`'s
|
|
260
|
+
* `TYPE_MAP`, enforced separately via `mapFieldType` for `indexed: true`
|
|
261
|
+
* fields). Matches `defaultMatchesType`'s switch and `schema-defaults.ts`'s
|
|
262
262
|
* `SchemaField.type` union — kept in lockstep by hand across the two
|
|
263
263
|
* deliberately-decoupled modules (see `validateFieldDefault`'s doc comment
|
|
264
264
|
* for why they don't cross-import).
|
|
265
|
+
*
|
|
266
|
+
* `reference` (vault#typed-reference-field) is a dual-write field type: the
|
|
267
|
+
* value is stored + validated exactly like `string` (an id/path/title the
|
|
268
|
+
* caller passes), AND the write path (`core/src/store.ts` — see
|
|
269
|
+
* `BunSqliteStore.syncReferenceFieldLinks`) additionally resolves that value
|
|
270
|
+
* to a note and maintains a graph `links` edge from this note to the
|
|
271
|
+
* resolved target, with `relationship` set to the field name. See
|
|
272
|
+
* `docs/design/typed-reference-field.md` for the full design + known gaps.
|
|
265
273
|
*/
|
|
266
|
-
export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object"] as const;
|
|
274
|
+
export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object", "reference"] as const;
|
|
267
275
|
|
|
268
276
|
/**
|
|
269
277
|
* Validate that a field's declared `type` is one of the six recognized
|
|
@@ -320,7 +328,7 @@ export function collectOwnFieldDefaultAndTypeViolations(
|
|
|
320
328
|
return violations;
|
|
321
329
|
}
|
|
322
330
|
|
|
323
|
-
/** Same
|
|
331
|
+
/** Same seven-type vocabulary as `TagFieldSchema.type`'s doc comment. Unknown/unset types pass (nothing to check against). */
|
|
324
332
|
function defaultMatchesType(value: unknown, type: string): boolean {
|
|
325
333
|
switch (type) {
|
|
326
334
|
case "string":
|
|
@@ -335,6 +343,11 @@ function defaultMatchesType(value: unknown, type: string): boolean {
|
|
|
335
343
|
return Array.isArray(value);
|
|
336
344
|
case "object":
|
|
337
345
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
346
|
+
// `reference` stores/validates like `string` — the value is the
|
|
347
|
+
// id/path/title the write path resolves to a target note. See
|
|
348
|
+
// VALID_FIELD_TYPES's doc comment.
|
|
349
|
+
case "reference":
|
|
350
|
+
return typeof value === "string";
|
|
338
351
|
default:
|
|
339
352
|
return true;
|
|
340
353
|
}
|
|
@@ -754,7 +767,7 @@ export function collectTagFieldViolations(
|
|
|
754
767
|
violations.push({
|
|
755
768
|
field: fieldName,
|
|
756
769
|
reason: "unsupported_indexed_type",
|
|
757
|
-
message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean)`,
|
|
770
|
+
message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
|
|
758
771
|
});
|
|
759
772
|
} else {
|
|
760
773
|
try {
|
package/core/src/types.ts
CHANGED
|
@@ -118,6 +118,35 @@ export interface VaultStats {
|
|
|
118
118
|
contentBytes: number;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
+
// ---- Vault Map (front-door structural orientation) ----
|
|
122
|
+
|
|
123
|
+
/** One counted bucket in a `VaultMap` — a tag name or a top-level path segment. */
|
|
124
|
+
export interface VaultMapEntry {
|
|
125
|
+
name: string;
|
|
126
|
+
count: number;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Compact structural map of a vault: counts only, no content. Designed so a
|
|
131
|
+
* fresh reader (human or AI) orients in ONE `vault-info` call without also
|
|
132
|
+
* needing `include_stats: true` — see `getVaultMap` (core/src/notes.ts) for
|
|
133
|
+
* the SQL and the scope-aware `tagFilter` contract.
|
|
134
|
+
*/
|
|
135
|
+
export interface VaultMap {
|
|
136
|
+
/** Total notes in scope (all notes when unfiltered). */
|
|
137
|
+
total_notes: number;
|
|
138
|
+
/** Every tag currently carried by at least one in-scope note, with its membership count. Sorted by count desc, then name. */
|
|
139
|
+
tags: VaultMapEntry[];
|
|
140
|
+
/**
|
|
141
|
+
* Every top-level path segment (the text before the first `/`, or the
|
|
142
|
+
* whole path when it has none) among in-scope notes that HAVE a path, with
|
|
143
|
+
* its note count. Sorted by count desc, then name.
|
|
144
|
+
*/
|
|
145
|
+
path_buckets: VaultMapEntry[];
|
|
146
|
+
/** In-scope notes with no `path` set — excluded from `path_buckets` (nothing to bucket). */
|
|
147
|
+
unfiled_notes: number;
|
|
148
|
+
}
|
|
149
|
+
|
|
121
150
|
// ---- Query Options ----
|
|
122
151
|
|
|
123
152
|
export interface QueryOpts {
|
|
@@ -224,6 +253,54 @@ export interface QueryOpts {
|
|
|
224
253
|
* treat the string as opaque.
|
|
225
254
|
*/
|
|
226
255
|
cursor?: string;
|
|
256
|
+
/**
|
|
257
|
+
* Aggregation / rollup mode (top new-feature ask from a UX round). When
|
|
258
|
+
* present, `aggregateNotes` applies every OTHER filter on this `QueryOpts`
|
|
259
|
+
* (tags, metadata, date range, write-attribution, ids, ...) exactly as
|
|
260
|
+
* `queryNotes` would, then groups the matching notes and returns
|
|
261
|
+
* `[{group, value}]` instead of note rows. `cursor` / `orderBy` / `sort` /
|
|
262
|
+
* `limit` / `offset` are ignored in aggregate mode — a rollup returns one
|
|
263
|
+
* row per group, not a paginated note list.
|
|
264
|
+
*/
|
|
265
|
+
aggregate?: AggregateSpec;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Aggregation / rollup spec — `QueryOpts.aggregate`. Mirrored by the
|
|
270
|
+
* `query-notes` MCP tool's `aggregate` param and the REST
|
|
271
|
+
* `?aggregate[group_by]=…&aggregate[op]=…&aggregate[field]=…` params.
|
|
272
|
+
*/
|
|
273
|
+
export interface AggregateSpec {
|
|
274
|
+
/**
|
|
275
|
+
* What to group by: an indexed metadata field name (declared
|
|
276
|
+
* `indexed: true` in a tag schema — same FIELD_NOT_INDEXED contract
|
|
277
|
+
* `metadata` operator queries and `order_by` use), or the special value
|
|
278
|
+
* `"tag"` to group by tag membership. Under `"tag"`, a note carrying N of
|
|
279
|
+
* the tags in the (filtered) result set contributes to N separate groups
|
|
280
|
+
* — this is a membership rollup, not a partition.
|
|
281
|
+
*/
|
|
282
|
+
group_by: string;
|
|
283
|
+
/** `"count"` — number of matching notes per group. `"sum"` — sum of `field` per group. */
|
|
284
|
+
op: "count" | "sum";
|
|
285
|
+
/**
|
|
286
|
+
* Required when `op` is `"sum"`; ignored for `"count"`. Must be an
|
|
287
|
+
* indexed metadata field with SQLite storage type `INTEGER` (i.e. declared
|
|
288
|
+
* `type: "integer"` or `type: "boolean"` — the only indexable numeric
|
|
289
|
+
* shapes; a plain `type: "number"` field is never indexable, see
|
|
290
|
+
* `mapFieldType`, and a `TEXT`-backed field can't be summed).
|
|
291
|
+
*/
|
|
292
|
+
field?: string;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* One rollup row from `aggregateNotes`. `group` is the group_by value (the
|
|
297
|
+
* tag name under `group_by: "tag"`, or the indexed field's stored value) —
|
|
298
|
+
* `null` collects notes where the group_by field is absent/unset (SQL NULL
|
|
299
|
+
* groups together, standard GROUP BY behavior). `value` is the count/sum.
|
|
300
|
+
*/
|
|
301
|
+
export interface AggregateRow {
|
|
302
|
+
group: string | number | boolean | null;
|
|
303
|
+
value: number;
|
|
227
304
|
}
|
|
228
305
|
|
|
229
306
|
/**
|
|
@@ -346,6 +423,18 @@ export interface Store {
|
|
|
346
423
|
* agent loop can persist a single watermark and keep polling.
|
|
347
424
|
*/
|
|
348
425
|
queryNotesPaged(opts: QueryOpts): Promise<QueryNotesPage>;
|
|
426
|
+
/**
|
|
427
|
+
* Aggregation / rollup query. Applies every OTHER filter in `opts` (tags,
|
|
428
|
+
* metadata, date range, write-attribution, `ids`, ...) exactly as
|
|
429
|
+
* `queryNotes` would, then groups by `opts.aggregate.group_by` ("tag" or
|
|
430
|
+
* an indexed metadata field) and returns `[{group, value}]` — one row per
|
|
431
|
+
* group, count or sum per `opts.aggregate.op`. `cursor`/`orderBy`/`sort`/
|
|
432
|
+
* `limit`/`offset` on `opts` are ignored. Throws `QueryError` (code
|
|
433
|
+
* `FIELD_NOT_INDEXED`) when `group_by`/`field` isn't a declared indexed
|
|
434
|
+
* field, or `INVALID_QUERY` for a malformed `aggregate` spec (missing
|
|
435
|
+
* `field` on a `"sum"`, a non-numeric `field`, ...).
|
|
436
|
+
*/
|
|
437
|
+
aggregateNotes(opts: QueryOpts): Promise<AggregateRow[]>;
|
|
349
438
|
/**
|
|
350
439
|
* `mode` (vault#551 — literal-by-default): "literal" (default) escapes +
|
|
351
440
|
* phrase-quotes the query so FTS5 punctuation syntax (hyphen = NOT, an
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the ULID generator (vault#ulid-ids), pinned in isolation
|
|
3
|
+
* from the Store. Integration coverage (new notes get ULID ids, mixed
|
|
4
|
+
* old-format + ULID cursor pagination, old-format id round-trip) lives in
|
|
5
|
+
* core.test.ts under `describe("ULID ids for new notes ...")`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { describe, it, expect } from "bun:test";
|
|
9
|
+
import { generateUlid, ULID_REGEX } from "./ulid.js";
|
|
10
|
+
|
|
11
|
+
describe("generateUlid", () => {
|
|
12
|
+
it("produces a 26-character Crockford base32 string", () => {
|
|
13
|
+
const id = generateUlid();
|
|
14
|
+
expect(id).toHaveLength(26);
|
|
15
|
+
expect(id).toMatch(ULID_REGEX);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("never emits the excluded Crockford letters (I, L, O, U)", () => {
|
|
19
|
+
for (let i = 0; i < 500; i++) {
|
|
20
|
+
const id = generateUlid();
|
|
21
|
+
expect(id).not.toMatch(/[ILOU]/);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("the first 10 chars encode a timestamp close to Date.now()", () => {
|
|
26
|
+
const before = Date.now();
|
|
27
|
+
const id = generateUlid();
|
|
28
|
+
const after = Date.now();
|
|
29
|
+
|
|
30
|
+
const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
31
|
+
let time = 0;
|
|
32
|
+
for (const ch of id.slice(0, 10)) {
|
|
33
|
+
time = time * 32 + CROCKFORD.indexOf(ch);
|
|
34
|
+
}
|
|
35
|
+
expect(time).toBeGreaterThanOrEqual(before);
|
|
36
|
+
expect(time).toBeLessThanOrEqual(after);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it("is monotonically increasing under plain string compare, even across a tight loop", () => {
|
|
40
|
+
const ids = Array.from({ length: 1000 }, () => generateUlid());
|
|
41
|
+
for (let i = 1; i < ids.length; i++) {
|
|
42
|
+
expect(ids[i]! > ids[i - 1]!).toBe(true);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("produces no duplicates across many calls", () => {
|
|
47
|
+
const ids = new Set(Array.from({ length: 5000 }, () => generateUlid()));
|
|
48
|
+
expect(ids.size).toBe(5000);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("two IDs minted back-to-back share the same 10-char timestamp prefix (same ms) or the second sorts strictly after", () => {
|
|
52
|
+
const a = generateUlid();
|
|
53
|
+
const b = generateUlid();
|
|
54
|
+
expect(b > a).toBe(true);
|
|
55
|
+
});
|
|
56
|
+
});
|
package/core/src/ulid.ts
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ULID generator (vault#ulid-ids) — monotonic, lexicographically
|
|
3
|
+
* time-sortable, Crockford base32, opaque, collision-resistant identifiers.
|
|
4
|
+
*
|
|
5
|
+
* Spec: https://github.com/ulid/spec
|
|
6
|
+
*
|
|
7
|
+
* - 48-bit millisecond timestamp, encoded as the first 10 Crockford
|
|
8
|
+
* base32 characters (most-significant digit first).
|
|
9
|
+
* - 80-bit randomness, encoded as the remaining 16 characters.
|
|
10
|
+
* - 26 characters total.
|
|
11
|
+
*
|
|
12
|
+
* Monotonicity: when two IDs are minted within the SAME millisecond, the
|
|
13
|
+
* random component of the second is the first's incremented by 1
|
|
14
|
+
* (odometer-style carry across base32 digits) rather than re-rolled, so
|
|
15
|
+
* lexicographic string order matches generation order even at IDgen rates
|
|
16
|
+
* far above 1kHz. This mirrors the reference `ulid` package's
|
|
17
|
+
* `monotonicFactory`.
|
|
18
|
+
*
|
|
19
|
+
* This is a small, dependency-light, hand-rolled implementation rather than
|
|
20
|
+
* a pull of the `ulid` npm package — the algorithm is short enough (~60
|
|
21
|
+
* lines) that inlining it avoids a new supply-chain dependency for what is,
|
|
22
|
+
* cryptographically, just "48 bits of time + 80 bits of randomness, base32
|
|
23
|
+
* encoded."
|
|
24
|
+
*
|
|
25
|
+
* Randomness source: `crypto.getRandomValues` (Web Crypto — available as a
|
|
26
|
+
* global in Bun), not `Math.random`. This module is runtime application
|
|
27
|
+
* code (invoked from note/attachment creation), not a workflow/sandbox
|
|
28
|
+
* script, so Web Crypto is available; it's also simply better randomness
|
|
29
|
+
* than `Math.random` for anything used as a public, collision-resistant
|
|
30
|
+
* identifier.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
const CROCKFORD_BASE32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
34
|
+
const BASE = CROCKFORD_BASE32.length; // 32
|
|
35
|
+
const TIME_LEN = 10; // 10 chars * 5 bits = 50 bits, enough for the 48-bit timestamp
|
|
36
|
+
const RANDOM_LEN = 16; // 16 chars * 5 bits = 80 bits
|
|
37
|
+
const TIME_MAX = 2 ** 48 - 1;
|
|
38
|
+
|
|
39
|
+
/** Encode a non-negative integer as a fixed-length Crockford base32 string. */
|
|
40
|
+
function encodeTime(time: number, length: number): string {
|
|
41
|
+
let t = time;
|
|
42
|
+
let str = "";
|
|
43
|
+
for (let i = length; i > 0; i--) {
|
|
44
|
+
const mod = t % BASE;
|
|
45
|
+
str = CROCKFORD_BASE32[mod]! + str;
|
|
46
|
+
t = (t - mod) / BASE;
|
|
47
|
+
}
|
|
48
|
+
return str;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Draw `length` cryptographically-random Crockford base32 characters. */
|
|
52
|
+
function randomChars(length: number): string {
|
|
53
|
+
const bytes = new Uint8Array(length);
|
|
54
|
+
crypto.getRandomValues(bytes);
|
|
55
|
+
let str = "";
|
|
56
|
+
for (let i = 0; i < length; i++) {
|
|
57
|
+
// Each byte (0-255) mod 32 introduces a slight bias (256 isn't a
|
|
58
|
+
// multiple of 32... actually 256 = 8*32, so this is exactly uniform).
|
|
59
|
+
str += CROCKFORD_BASE32[bytes[i]! % BASE];
|
|
60
|
+
}
|
|
61
|
+
return str;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Increment a Crockford base32 string by 1, odometer-style: rightmost digit
|
|
66
|
+
* first, carrying leftward on overflow. Used to derive the next monotonic
|
|
67
|
+
* ID within the same millisecond.
|
|
68
|
+
*
|
|
69
|
+
* Throws if every digit is already the max ('Z') — i.e. the full 80-bit
|
|
70
|
+
* random space has been exhausted within a single millisecond. That would
|
|
71
|
+
* require minting 2^80 (~1.2e24) IDs in under a millisecond, several orders
|
|
72
|
+
* of magnitude beyond any realistic write rate; this is a defensive bound,
|
|
73
|
+
* not a real-world concern.
|
|
74
|
+
*/
|
|
75
|
+
function incrementBase32(str: string): string {
|
|
76
|
+
const chars = str.split("");
|
|
77
|
+
for (let i = chars.length - 1; i >= 0; i--) {
|
|
78
|
+
const idx = CROCKFORD_BASE32.indexOf(chars[i]!);
|
|
79
|
+
if (idx < BASE - 1) {
|
|
80
|
+
chars[i] = CROCKFORD_BASE32[idx + 1]!;
|
|
81
|
+
return chars.join("");
|
|
82
|
+
}
|
|
83
|
+
chars[i] = CROCKFORD_BASE32[0]!;
|
|
84
|
+
}
|
|
85
|
+
throw new Error("ULID random component overflow — 2^80 IDs minted within a single millisecond");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Monotonic state, process-local (matches generateId()'s prior idCounter
|
|
89
|
+
// approach — a single vault process is the unit of monotonicity; a cursor's
|
|
90
|
+
// tiebreaker doesn't require cross-process ordering, just a stable total
|
|
91
|
+
// order per id string).
|
|
92
|
+
let lastTime = 0;
|
|
93
|
+
let lastRandom = "";
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Generate a monotonic ULID. See module doc for the algorithm.
|
|
97
|
+
*
|
|
98
|
+
* Exported separately from `generateId()` (in notes.ts) so it's testable in
|
|
99
|
+
* isolation (monotonicity, format, charset) without pulling in the Store.
|
|
100
|
+
*/
|
|
101
|
+
export function generateUlid(): string {
|
|
102
|
+
const now = Math.min(Date.now(), TIME_MAX);
|
|
103
|
+
if (now <= lastTime && lastRandom) {
|
|
104
|
+
// Same (or earlier — clock skew) millisecond as the last mint: keep
|
|
105
|
+
// `lastTime` as-is and bump the random component to preserve strict
|
|
106
|
+
// monotonicity.
|
|
107
|
+
lastRandom = incrementBase32(lastRandom);
|
|
108
|
+
} else {
|
|
109
|
+
lastTime = now;
|
|
110
|
+
lastRandom = randomChars(RANDOM_LEN);
|
|
111
|
+
}
|
|
112
|
+
return encodeTime(lastTime, TIME_LEN) + lastRandom;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** 26-char Crockford base32 (`0-9A-HJKMNP-TV-Z`, case-insensitive by spec — we always emit uppercase). */
|
|
116
|
+
export const ULID_REGEX = /^[0-9A-HJKMNP-TV-Z]{26}$/;
|
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
} from "./tag-schemas.ts";
|
|
33
33
|
import { DEFAULT_TAG_NAME } from "./tag-hierarchy.ts";
|
|
34
34
|
import * as noteOps from "./notes.ts";
|
|
35
|
-
import type { VaultStats } from "./types.ts";
|
|
35
|
+
import type { VaultStats, VaultMap } from "./types.ts";
|
|
36
36
|
import { GETTING_STARTED_PATH } from "./seed-packs.ts";
|
|
37
37
|
|
|
38
38
|
/**
|
|
@@ -103,6 +103,17 @@ export interface VaultProjection {
|
|
|
103
103
|
getting_started?: string;
|
|
104
104
|
/** Included when the caller requests stats; omitted otherwise. */
|
|
105
105
|
stats?: VaultStats;
|
|
106
|
+
/**
|
|
107
|
+
* Compact structural map — total note count, tag counts, top-level
|
|
108
|
+
* path-bucket counts (front-door orientation). ALWAYS present (unlike
|
|
109
|
+
* `stats`): it's three cheap grouped-COUNT queries, so a fresh reader
|
|
110
|
+
* orients in one `vault-info` call without also passing
|
|
111
|
+
* `include_stats: true`. See `noteOps.getVaultMap`. Scope-aware callers
|
|
112
|
+
* (tag-scoped tokens) get a RECOMPUTED map from the server layer — see
|
|
113
|
+
* `applyTagScopeWrappers` in src/mcp-tools.ts — because path-bucket counts
|
|
114
|
+
* can't be reconstructed by post-hoc filtering an unscoped rollup.
|
|
115
|
+
*/
|
|
116
|
+
map: VaultMap;
|
|
106
117
|
}
|
|
107
118
|
|
|
108
119
|
// ---------------------------------------------------------------------------
|
|
@@ -182,6 +193,12 @@ export const QUERY_HINTS: readonly string[] = [
|
|
|
182
193
|
* - `stats`: included when `opts.includeStats === true`. Uses the
|
|
183
194
|
* existing `getVaultStats` shape unchanged — camelCase keys, full
|
|
184
195
|
* monthly distribution.
|
|
196
|
+
*
|
|
197
|
+
* - `map`: ALWAYS included (unlike `stats`) — `getVaultMap`'s cheap
|
|
198
|
+
* total-notes / tag-counts / path-bucket-counts rollup, computed
|
|
199
|
+
* vault-wide (unscoped). See the `VaultProjection.map` doc comment for
|
|
200
|
+
* why a tag-scoped caller's map is recomputed one layer up rather than
|
|
201
|
+
* filtered here.
|
|
185
202
|
*/
|
|
186
203
|
export function buildVaultProjection(
|
|
187
204
|
db: Database,
|
|
@@ -233,6 +250,7 @@ export function buildVaultProjection(
|
|
|
233
250
|
tags,
|
|
234
251
|
indexed_fields,
|
|
235
252
|
query_hints: [...QUERY_HINTS],
|
|
253
|
+
map: noteOps.getVaultMap(db),
|
|
236
254
|
};
|
|
237
255
|
|
|
238
256
|
// A2: point any connected AI at the seeded onboarding guide (a path pointer,
|