@openparachute/vault 0.7.2 → 0.7.3-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/core/src/attachment/policy.test.ts +66 -0
- package/core/src/attachment/policy.ts +131 -0
- package/core/src/attachment/tickets.test.ts +45 -0
- package/core/src/attachment/tickets.ts +117 -0
- package/core/src/attachment-tickets-tool.test.ts +286 -0
- package/core/src/conformance.ts +2 -1
- package/core/src/contract-typed-index.test.ts +4 -3
- package/core/src/core.test.ts +607 -11
- package/core/src/display-title.test.ts +190 -0
- package/core/src/embedding/chunker.test.ts +97 -0
- package/core/src/embedding/chunker.ts +180 -0
- package/core/src/embedding/provider.ts +108 -0
- package/core/src/embedding/staleness.test.ts +87 -0
- package/core/src/embedding/staleness.ts +83 -0
- package/core/src/embedding/vector-codec.test.ts +99 -0
- package/core/src/embedding/vector-codec.ts +60 -0
- package/core/src/embedding/vectors.test.ts +163 -0
- package/core/src/embedding/vectors.ts +135 -0
- package/core/src/expand.ts +11 -3
- package/core/src/indexed-fields.test.ts +9 -3
- package/core/src/indexed-fields.ts +9 -1
- package/core/src/lede.test.ts +96 -0
- package/core/src/mcp-semantic-search.test.ts +160 -0
- package/core/src/mcp.ts +405 -10
- package/core/src/notes.semantic-search.test.ts +131 -0
- package/core/src/notes.ts +319 -7
- package/core/src/query-warnings.ts +40 -0
- package/core/src/schema-defaults.ts +85 -1
- package/core/src/schema-v27-note-vectors.test.ts +178 -0
- package/core/src/schema.ts +81 -1
- package/core/src/search-fts-v25.test.ts +9 -1
- package/core/src/search-query.test.ts +42 -0
- package/core/src/search-query.ts +27 -0
- package/core/src/search-title-boost.test.ts +125 -0
- package/core/src/seed-packs.test.ts +117 -1
- package/core/src/seed-packs.ts +217 -1
- package/core/src/store.semantic-search.test.ts +236 -0
- package/core/src/store.ts +162 -5
- package/core/src/tag-schemas.ts +27 -12
- package/core/src/types.ts +63 -1
- package/core/src/vault-projection.ts +48 -0
- package/package.json +6 -1
- package/src/add-pack.test.ts +32 -0
- package/src/attachment-tickets.test.ts +350 -0
- package/src/attachment-tickets.ts +264 -0
- package/src/auth.ts +8 -2
- package/src/cli.ts +5 -1
- package/src/contract-errors.test.ts +2 -1
- package/src/contract-honest-queries.test.ts +88 -0
- package/src/contract-search.test.ts +41 -0
- package/src/embedding/capability.test.ts +33 -0
- package/src/embedding/capability.ts +34 -0
- package/src/embedding/external-api.test.ts +154 -0
- package/src/embedding/external-api.ts +144 -0
- package/src/embedding/onnx-transformers.test.ts +113 -0
- package/src/embedding/onnx-transformers.ts +141 -0
- package/src/embedding/select.test.ts +99 -0
- package/src/embedding/select.ts +92 -0
- package/src/embedding-worker.test.ts +300 -0
- package/src/embedding-worker.ts +226 -0
- package/src/mcp-http.ts +13 -1
- package/src/mcp-tools.ts +49 -14
- package/src/onboarding-seed.test.ts +64 -0
- package/src/routes.ts +173 -92
- package/src/routing.ts +17 -0
- package/src/scribe-discovery.test.ts +76 -1
- package/src/scribe-discovery.ts +28 -9
- package/src/semantic-search-routes.test.ts +161 -0
- package/src/server.ts +21 -2
- package/src/vault-embeddings-capability.test.ts +82 -0
- package/src/vault-store-embedding-wiring.test.ts +69 -0
- package/src/vault-store.ts +53 -2
- package/src/vault.test.ts +62 -13
package/core/src/store.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
|
-
import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage, AggregateRow } from "./types.js";
|
|
2
|
+
import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage, AggregateRow, SemanticSearchResult } from "./types.js";
|
|
3
3
|
import { initSchema } from "./schema.js";
|
|
4
4
|
import * as noteOps from "./notes.js";
|
|
5
5
|
import * as linkOps from "./links.js";
|
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
loadSchemaConfig,
|
|
39
39
|
validateNote as runValidateNote,
|
|
40
40
|
resolveNoteSchemas,
|
|
41
|
+
normalizeDateFields,
|
|
41
42
|
type ResolvedSchemas,
|
|
42
43
|
type ValidationStatus,
|
|
43
44
|
} from "./schema-defaults.js";
|
|
@@ -47,6 +48,9 @@ import {
|
|
|
47
48
|
} from "./conformance.js";
|
|
48
49
|
import type { SearchMode } from "./search-query.js";
|
|
49
50
|
import { runDoctorScan, type DoctorReport, type DoctorScanOpts } from "./doctor.js";
|
|
51
|
+
import { EmbeddingError, type EmbeddingProvider } from "./embedding/provider.js";
|
|
52
|
+
import { normalize } from "./embedding/vector-codec.js";
|
|
53
|
+
import { QueryError } from "./query-operators.js";
|
|
50
54
|
|
|
51
55
|
/**
|
|
52
56
|
* Normalize a `type: "reference"` field value (scalar OR array — see
|
|
@@ -94,9 +98,38 @@ export class BunSqliteStore implements Store {
|
|
|
94
98
|
private _tagHierarchy: TagHierarchy | null = null;
|
|
95
99
|
private _schemaConfig: ResolvedSchemas | null = null;
|
|
96
100
|
|
|
97
|
-
|
|
101
|
+
/**
|
|
102
|
+
* The active `EmbeddingProvider` (EXPERIMENTAL — semantic search MVP),
|
|
103
|
+
* when one is configured. `undefined` on a vault with no provider —
|
|
104
|
+
* `semanticSearch` reports that honestly (`semantic_unavailable`)
|
|
105
|
+
* rather than silently falling back to keyword search. Resolved ONCE at
|
|
106
|
+
* construction (mirrors `hooks`) — self-host/cloud each build the
|
|
107
|
+
* concrete provider (ONNX/external-API, Workers AI) and pass it in here;
|
|
108
|
+
* core never imports a model library itself (dependency-purity rule).
|
|
109
|
+
*/
|
|
110
|
+
public readonly embeddingProvider?: EmbeddingProvider;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* WHY `embeddingProvider` is absent, when the caller knows a specific
|
|
114
|
+
* reason (currently: the operator set `EMBEDDINGS_ENABLED=false` — see
|
|
115
|
+
* `src/embedding/select.ts`). Core never reads env vars itself
|
|
116
|
+
* (dependency-purity rule — see `embeddingProvider`'s doc comment
|
|
117
|
+
* above), so it can't distinguish "explicitly turned off" from "never
|
|
118
|
+
* configured" on its own; the caller passes this through so
|
|
119
|
+
* `semanticSearch`'s error hint can be honest either way instead of
|
|
120
|
+
* defaulting to generic provider-setup instructions when the operator
|
|
121
|
+
* deliberately opted out.
|
|
122
|
+
*/
|
|
123
|
+
public readonly embeddingDisabledReason?: string;
|
|
124
|
+
|
|
125
|
+
constructor(
|
|
126
|
+
public readonly db: Database,
|
|
127
|
+
opts?: { hooks?: HookRegistry; embeddingProvider?: EmbeddingProvider; embeddingDisabledReason?: string },
|
|
128
|
+
) {
|
|
98
129
|
initSchema(db);
|
|
99
130
|
this.hooks = opts?.hooks ?? new HookRegistry();
|
|
131
|
+
this.embeddingProvider = opts?.embeddingProvider;
|
|
132
|
+
this.embeddingDisabledReason = opts?.embeddingDisabledReason;
|
|
100
133
|
}
|
|
101
134
|
|
|
102
135
|
/**
|
|
@@ -468,6 +501,16 @@ export class BunSqliteStore implements Store {
|
|
|
468
501
|
// ---- Notes ----
|
|
469
502
|
|
|
470
503
|
async createNote(content: string, opts?: { id?: string; path?: string; tags?: string[]; metadata?: Record<string, unknown>; created_at?: string; extension?: string; actor?: string | null; via?: string | null }): Promise<Note> {
|
|
504
|
+
// Normalize `date`-typed field values to canonical UTC ISO form BEFORE
|
|
505
|
+
// the write (vault#date-field-type — mixed-offset TEXT-compare
|
|
506
|
+
// corruption). COPY-ON-WRITE (round 2) — reassign the local `opts`
|
|
507
|
+
// binding from the returned value rather than mutating the caller's
|
|
508
|
+
// object; see `normalizeDateFields`'s doc comment.
|
|
509
|
+
if (opts?.metadata) {
|
|
510
|
+
const normalized = normalizeDateFields(this.getSchemaConfig(), { tags: opts.tags, metadata: opts.metadata });
|
|
511
|
+
if (normalized !== opts.metadata) opts = { ...opts, metadata: normalized };
|
|
512
|
+
}
|
|
513
|
+
|
|
471
514
|
const note = noteOps.createNote(this.db, content, opts);
|
|
472
515
|
|
|
473
516
|
if (content) {
|
|
@@ -516,6 +559,20 @@ export class BunSqliteStore implements Store {
|
|
|
516
559
|
actor?: string | null;
|
|
517
560
|
via?: string | null;
|
|
518
561
|
if_updated_at?: string;
|
|
562
|
+
/**
|
|
563
|
+
* `date`-field normalization override (vault#date-field-type review
|
|
564
|
+
* round 2). By default, `normalizeDateFields` below resolves the note's
|
|
565
|
+
* effective schema against its CURRENT tags in the DB — correct when
|
|
566
|
+
* this call doesn't also change tags. A caller that adds a tag IN THIS
|
|
567
|
+
* SAME logical update (via a separate `store.tagNote` issued right
|
|
568
|
+
* after this call — see mcp.ts's `update-note` handler and the batch
|
|
569
|
+
* upsert "update"/"replace" branch) must pass the PROJECTED final tag
|
|
570
|
+
* set here instead, or a newly-added tag's `type: "date"` field never
|
|
571
|
+
* gets seen (the schema resolution would still be looking at the
|
|
572
|
+
* pre-write tag set) and its offset-bearing value would persist
|
|
573
|
+
* verbatim. Ignored when `updates.metadata` is undefined.
|
|
574
|
+
*/
|
|
575
|
+
tagsForSchemaResolution?: string[];
|
|
519
576
|
},
|
|
520
577
|
): Promise<Note> {
|
|
521
578
|
let oldPath: string | undefined;
|
|
@@ -529,7 +586,19 @@ export class BunSqliteStore implements Store {
|
|
|
529
586
|
if (updates.path !== undefined || updates.metadata !== undefined) {
|
|
530
587
|
const existing = noteOps.getNote(this.db, id);
|
|
531
588
|
if (updates.path !== undefined) oldPath = existing?.path;
|
|
532
|
-
if (updates.metadata !== undefined)
|
|
589
|
+
if (updates.metadata !== undefined) {
|
|
590
|
+
priorMetadataForRefs = existing?.metadata;
|
|
591
|
+
// Normalize `date`-typed field values to canonical UTC ISO form
|
|
592
|
+
// BEFORE the write (vault#date-field-type — mixed-offset TEXT-
|
|
593
|
+
// compare corruption). COPY-ON-WRITE (round 2) — reassign the local
|
|
594
|
+
// `updates` binding from the returned value rather than mutating the
|
|
595
|
+
// caller's object; see `normalizeDateFields`'s doc comment.
|
|
596
|
+
// `tagsForSchemaResolution` (when the caller is ALSO adding a tag in
|
|
597
|
+
// this same logical update) wins over the note's current DB tags.
|
|
598
|
+
const tagsForResolution = updates.tagsForSchemaResolution ?? existing?.tags;
|
|
599
|
+
const normalized = normalizeDateFields(this.getSchemaConfig(), { tags: tagsForResolution, metadata: updates.metadata });
|
|
600
|
+
if (normalized !== updates.metadata) updates = { ...updates, metadata: normalized };
|
|
601
|
+
}
|
|
533
602
|
}
|
|
534
603
|
|
|
535
604
|
const note = noteOps.updateNote(this.db, id, updates);
|
|
@@ -814,6 +883,75 @@ export class BunSqliteStore implements Store {
|
|
|
814
883
|
return noteOps.searchNotes(this.db, query, opts);
|
|
815
884
|
}
|
|
816
885
|
|
|
886
|
+
/**
|
|
887
|
+
* Semantic search (EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). The ONE
|
|
888
|
+
* invocation point for `embeddingProvider`: resolves it, embeds
|
|
889
|
+
* `nearText` into a query vector, normalizes it (stored vectors are
|
|
890
|
+
* ALSO L2-normalized — see `note_vectors`'s schema comment — so the
|
|
891
|
+
* scan's dot product is cosine similarity), and calls
|
|
892
|
+
* `noteOps.semanticSearchNotes` with the SAME tag-hierarchy expansion
|
|
893
|
+
* `queryNotes` applies (so `semantic: true, tag: "manual"` composes with
|
|
894
|
+
* the tag `expand` axis identically to a structured query).
|
|
895
|
+
*
|
|
896
|
+
* Throws a `QueryError` (`error_type: "semantic_unavailable"`) when no
|
|
897
|
+
* provider is configured, the configured one reports itself not ready,
|
|
898
|
+
* OR the embed call itself fails mid-flight (e.g. the bundled ONNX
|
|
899
|
+
* floor's lazy model load fails on this — the FIRST — real `embed()`
|
|
900
|
+
* attempt, after `available()` optimistically reported `ok: true`; see
|
|
901
|
+
* `onnx-transformers.ts`'s "lazy-fail, not crash" doc) — NEVER a silent
|
|
902
|
+
* fallback to keyword search, and never a raw unstructured 500. Callers
|
|
903
|
+
* (MCP/REST) let this propagate uncaught, same as
|
|
904
|
+
* `invalid_search_syntax` above.
|
|
905
|
+
*/
|
|
906
|
+
async semanticSearch(nearText: string, opts?: QueryOpts): Promise<SemanticSearchResult> {
|
|
907
|
+
if (!this.embeddingProvider) {
|
|
908
|
+
// Honest either way: if the caller told us WHY (operator explicitly
|
|
909
|
+
// set EMBEDDINGS_ENABLED=false), say that — not generic provider-setup
|
|
910
|
+
// instructions that would be actively misleading for a deliberate
|
|
911
|
+
// opt-out. See `embeddingDisabledReason`'s doc comment.
|
|
912
|
+
throw new QueryError(
|
|
913
|
+
`semantic search requires an embedding provider — none is configured on this vault`,
|
|
914
|
+
"SEMANTIC_UNAVAILABLE",
|
|
915
|
+
{
|
|
916
|
+
error_type: "semantic_unavailable",
|
|
917
|
+
hint:
|
|
918
|
+
this.embeddingDisabledReason ??
|
|
919
|
+
"configure EMBEDDING_API_URL/EMBEDDING_API_KEY/EMBEDDING_MODEL, or rely on the bundled floor provider (self-host); semantic search is not yet available on this door otherwise",
|
|
920
|
+
},
|
|
921
|
+
);
|
|
922
|
+
}
|
|
923
|
+
const availability = await this.embeddingProvider.available();
|
|
924
|
+
if (!availability.ok) {
|
|
925
|
+
throw new QueryError(
|
|
926
|
+
`semantic search is unavailable: ${availability.reason ?? "embedding provider not ready"}`,
|
|
927
|
+
"SEMANTIC_UNAVAILABLE",
|
|
928
|
+
{ error_type: "semantic_unavailable", hint: availability.reason },
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
let vectors: Float32Array[];
|
|
933
|
+
try {
|
|
934
|
+
({ vectors } = await this.embeddingProvider.embed({ texts: [nearText] }));
|
|
935
|
+
} catch (err) {
|
|
936
|
+
// `available()` passing is only a CHEAP readiness check (never a real
|
|
937
|
+
// network/model round-trip — see EmbeddingProvider.available's doc
|
|
938
|
+
// comment) — it can't catch a provider whose actual first embed call
|
|
939
|
+
// fails (a lazy model load blowing up, a transient upstream error).
|
|
940
|
+
// Map ANY embed()-time failure to the same honest semantic_unavailable
|
|
941
|
+
// shape as the checks above, rather than letting a raw EmbeddingError
|
|
942
|
+
// (or whatever else a provider throws) surface as an unstructured 500.
|
|
943
|
+
const reason = err instanceof EmbeddingError ? err.message : err instanceof Error ? err.message : String(err);
|
|
944
|
+
throw new QueryError(`semantic search is unavailable: ${reason}`, "SEMANTIC_UNAVAILABLE", {
|
|
945
|
+
error_type: "semantic_unavailable",
|
|
946
|
+
hint: reason,
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
const queryVector = normalize(vectors[0]!);
|
|
950
|
+
|
|
951
|
+
const filterOpts = this.expandQueryTags(this.normalizeQueryTags(opts ?? {}));
|
|
952
|
+
return noteOps.semanticSearchNotes(this.db, queryVector, filterOpts, this.embeddingProvider.model);
|
|
953
|
+
}
|
|
954
|
+
|
|
817
955
|
// ---- Tags ----
|
|
818
956
|
|
|
819
957
|
async tagNote(noteId: string, tags: string[]): Promise<void> {
|
|
@@ -933,7 +1071,18 @@ export class BunSqliteStore implements Store {
|
|
|
933
1071
|
// ---- Bulk Operations ----
|
|
934
1072
|
|
|
935
1073
|
async createNotes(inputs: noteOps.BulkNoteInput[]): Promise<Note[]> {
|
|
936
|
-
|
|
1074
|
+
// Same pre-write `date`-field normalization as singleton createNote
|
|
1075
|
+
// (vault#date-field-type) — this bulk path bypasses it otherwise.
|
|
1076
|
+
// COPY-ON-WRITE (round 2) — build a new array with only the items that
|
|
1077
|
+
// actually need a rewrite replaced; never mutate the caller's `inputs`
|
|
1078
|
+
// or its element objects.
|
|
1079
|
+
const schemaConfig = this.getSchemaConfig();
|
|
1080
|
+
const normalizedInputs = inputs.map((input) => {
|
|
1081
|
+
if (!input.metadata) return input;
|
|
1082
|
+
const normalized = normalizeDateFields(schemaConfig, { tags: input.tags, metadata: input.metadata });
|
|
1083
|
+
return normalized !== input.metadata ? { ...input, metadata: normalized } : input;
|
|
1084
|
+
});
|
|
1085
|
+
const notes = noteOps.createNotes(this.db, normalizedInputs);
|
|
937
1086
|
for (const note of notes) {
|
|
938
1087
|
// Bulk path needs the same config-cache invalidation as singleton
|
|
939
1088
|
// createNote — without it, a batch that includes `_tags/*` notes
|
|
@@ -1143,7 +1292,7 @@ export class BunSqliteStore implements Store {
|
|
|
1143
1292
|
const mapped = indexedFieldOps.mapFieldType(spec.type);
|
|
1144
1293
|
if (!mapped) {
|
|
1145
1294
|
throw new indexedFieldOps.IndexedFieldError(
|
|
1146
|
-
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
|
|
1295
|
+
`field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference, date)`,
|
|
1147
1296
|
);
|
|
1148
1297
|
}
|
|
1149
1298
|
// Throws IndexedFieldError on an invalid identifier (e.g. kebab-case).
|
|
@@ -1275,6 +1424,14 @@ export class BunSqliteStore implements Store {
|
|
|
1275
1424
|
* place.)
|
|
1276
1425
|
*/
|
|
1277
1426
|
async createNoteRaw(content: string, opts?: { id?: string; path?: string; tags?: string[]; metadata?: Record<string, unknown>; created_at?: string; extension?: string }): Promise<Note> {
|
|
1427
|
+
// Same pre-write `date`-field normalization as createNote
|
|
1428
|
+
// (vault#date-field-type) — the legacy Obsidian importer (obsidian.ts)
|
|
1429
|
+
// funnels through this path and bypasses createNote's copy otherwise.
|
|
1430
|
+
// COPY-ON-WRITE (round 2) — see normalizeDateFields's doc comment.
|
|
1431
|
+
if (opts?.metadata) {
|
|
1432
|
+
const normalized = normalizeDateFields(this.getSchemaConfig(), { tags: opts.tags, metadata: opts.metadata });
|
|
1433
|
+
if (normalized !== opts.metadata) opts = { ...opts, metadata: normalized };
|
|
1434
|
+
}
|
|
1278
1435
|
return noteOps.createNote(this.db, content, opts);
|
|
1279
1436
|
}
|
|
1280
1437
|
|
package/core/src/tag-schemas.ts
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { Database } from "bun:sqlite";
|
|
19
19
|
import { mapFieldType, validateFieldName } from "./indexed-fields.js";
|
|
20
20
|
import { loadTagHierarchy, findParentCycle } from "./tag-hierarchy.js";
|
|
21
|
+
import { timestampToMs } from "./cursor.js";
|
|
21
22
|
|
|
22
23
|
// ---------------------------------------------------------------------------
|
|
23
24
|
// Types
|
|
@@ -192,7 +193,7 @@ export class InvalidFieldDefaultError extends Error {
|
|
|
192
193
|
|
|
193
194
|
/**
|
|
194
195
|
* Thrown by `upsertTagRecord` (core/src/store.ts's chokepoint) when a
|
|
195
|
-
* field's declared `type` isn't one of the
|
|
196
|
+
* field's declared `type` isn't one of the recognized values (vault#555
|
|
196
197
|
* — `update-tag{fields:{weird:{type:"frobnicator"}}}` used to be accepted
|
|
197
198
|
* and persisted verbatim, no error, for any NON-indexed field: the only
|
|
198
199
|
* existing type check, `mapFieldType`, ran solely on `indexed: true` fields
|
|
@@ -255,13 +256,13 @@ export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFi
|
|
|
255
256
|
|
|
256
257
|
/**
|
|
257
258
|
* The full recognized vocabulary for `TagFieldSchema.type` — storage/
|
|
258
|
-
* advisory validation accepts all
|
|
259
|
-
* `reference` are INDEXABLE (that narrower subset is
|
|
260
|
-
* `TYPE_MAP`, enforced separately via `mapFieldType`
|
|
261
|
-
* fields). Matches `defaultMatchesType`'s switch and
|
|
262
|
-
* `SchemaField.type` union — kept in lockstep by hand
|
|
263
|
-
* deliberately-decoupled modules (see `validateFieldDefault`'s
|
|
264
|
-
* for why they don't cross-import).
|
|
259
|
+
* advisory validation accepts all eight; only `string`/`integer`/`boolean`/
|
|
260
|
+
* `reference`/`date` are INDEXABLE (that narrower subset is
|
|
261
|
+
* `indexed-fields.ts`'s `TYPE_MAP`, enforced separately via `mapFieldType`
|
|
262
|
+
* for `indexed: true` fields). Matches `defaultMatchesType`'s switch and
|
|
263
|
+
* `schema-defaults.ts`'s `SchemaField.type` union — kept in lockstep by hand
|
|
264
|
+
* across the two deliberately-decoupled modules (see `validateFieldDefault`'s
|
|
265
|
+
* doc comment for why they don't cross-import).
|
|
265
266
|
*
|
|
266
267
|
* `reference` (vault#typed-reference-field) is a dual-write field type: the
|
|
267
268
|
* value is stored + validated exactly like `string` (an id/path/title the
|
|
@@ -270,11 +271,18 @@ export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFi
|
|
|
270
271
|
* to a note and maintains a graph `links` edge from this note to the
|
|
271
272
|
* resolved target, with `relationship` set to the field name. See
|
|
272
273
|
* `docs/design/typed-reference-field.md` for the full design + known gaps.
|
|
274
|
+
*
|
|
275
|
+
* `date` stores/validates like `string` (an ISO-8601 date or timestamp) so
|
|
276
|
+
* indexed `date` fields sort correctly under a plain TEXT comparison — see
|
|
277
|
+
* `defaultMatchesType`'s `"date"` case and `schema-defaults.ts`'s
|
|
278
|
+
* `valueMatchesType`, both of which reuse `cursor.ts`'s `timestampToMs`
|
|
279
|
+
* (the SAME UTC-correct parser `date_filter`'s `updated_at` bound uses) so
|
|
280
|
+
* there's exactly one ISO-parsing implementation in the codebase, not two.
|
|
273
281
|
*/
|
|
274
|
-
export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object", "reference"] as const;
|
|
282
|
+
export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object", "reference", "date"] as const;
|
|
275
283
|
|
|
276
284
|
/**
|
|
277
|
-
* Validate that a field's declared `type` is one of the
|
|
285
|
+
* Validate that a field's declared `type` is one of the recognized
|
|
278
286
|
* values (vault#555). Returns `null` when `type` is unset (own-field checks
|
|
279
287
|
* elsewhere already treat an unset type as "nothing to check against") or
|
|
280
288
|
* recognized; otherwise a {@link TagFieldViolation} with `reason:
|
|
@@ -328,7 +336,7 @@ export function collectOwnFieldDefaultAndTypeViolations(
|
|
|
328
336
|
return violations;
|
|
329
337
|
}
|
|
330
338
|
|
|
331
|
-
/** Same
|
|
339
|
+
/** Same eight-type vocabulary as `TagFieldSchema.type`'s doc comment. Unknown/unset types pass (nothing to check against). */
|
|
332
340
|
function defaultMatchesType(value: unknown, type: string): boolean {
|
|
333
341
|
switch (type) {
|
|
334
342
|
case "string":
|
|
@@ -348,6 +356,13 @@ function defaultMatchesType(value: unknown, type: string): boolean {
|
|
|
348
356
|
// VALID_FIELD_TYPES's doc comment.
|
|
349
357
|
case "reference":
|
|
350
358
|
return typeof value === "string";
|
|
359
|
+
// `date` accepts an ISO-8601 date (`YYYY-MM-DD`) or full RFC3339
|
|
360
|
+
// timestamp — the SAME grammar `date_filter`'s `updated_at` bound
|
|
361
|
+
// parses (`timestampToMs`, imported from cursor.ts), not a second,
|
|
362
|
+
// independently-drifting date parser. See VALID_FIELD_TYPES's doc
|
|
363
|
+
// comment.
|
|
364
|
+
case "date":
|
|
365
|
+
return typeof value === "string" && timestampToMs(value) !== null;
|
|
351
366
|
default:
|
|
352
367
|
return true;
|
|
353
368
|
}
|
|
@@ -767,7 +782,7 @@ export function collectTagFieldViolations(
|
|
|
767
782
|
violations.push({
|
|
768
783
|
field: fieldName,
|
|
769
784
|
reason: "unsupported_indexed_type",
|
|
770
|
-
message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
|
|
785
|
+
message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference, date)`,
|
|
771
786
|
});
|
|
772
787
|
} else {
|
|
773
788
|
try {
|
package/core/src/types.ts
CHANGED
|
@@ -31,6 +31,13 @@ export interface Note {
|
|
|
31
31
|
* `core/src/portable-md.ts:supportsInlineFrontmatter`.
|
|
32
32
|
*/
|
|
33
33
|
extension?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Always present on notes read back from the store — NULL/`"{}"`/
|
|
36
|
+
* unparseable metadata all collapse to `{}` rather than an absent key
|
|
37
|
+
* (mirrors `tags`, which is likewise always `[]`). Optional here only so
|
|
38
|
+
* write-input shapes (`createNote`/`updateNote` options) can share this
|
|
39
|
+
* type without forcing every caller to pass an empty object.
|
|
40
|
+
*/
|
|
34
41
|
metadata?: Record<string, unknown>;
|
|
35
42
|
createdAt: string; // ISO-8601
|
|
36
43
|
updatedAt?: string;
|
|
@@ -271,6 +278,43 @@ export interface QueryOpts {
|
|
|
271
278
|
* row per group, not a paginated note list.
|
|
272
279
|
*/
|
|
273
280
|
aggregate?: AggregateSpec;
|
|
281
|
+
/**
|
|
282
|
+
* Semantic-search MVP (EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). Free text
|
|
283
|
+
* to rank notes by MEANING rather than keyword. Requires `semantic:
|
|
284
|
+
* true` and is mutually exclusive with `search` (structurally the same
|
|
285
|
+
* "text in → ranked notes with a score out" shape as `search`, but
|
|
286
|
+
* routed through `note_vectors` cosine similarity instead of FTS5
|
|
287
|
+
* bm25). Every other filter on this `QueryOpts` (tags, metadata, date
|
|
288
|
+
* range, path, ...) narrows the candidate set FIRST, exactly like
|
|
289
|
+
* `search` — see `core/src/notes.ts:semanticSearchNotes`.
|
|
290
|
+
*/
|
|
291
|
+
nearText?: string;
|
|
292
|
+
/**
|
|
293
|
+
* Opt into vector ranking. `true` requires `nearText`; omitting it (or
|
|
294
|
+
* passing `false`) is a plain structured/keyword query, byte-identical
|
|
295
|
+
* to pre-semantic-search behavior. See `nearText`.
|
|
296
|
+
*/
|
|
297
|
+
semantic?: boolean;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Result of a semantic-search scan (EXPERIMENTAL — see
|
|
302
|
+
* `QueryOpts.nearText`/`semantic`). Returned by
|
|
303
|
+
* `core/src/notes.ts:semanticSearchNotes` and `Store.semanticSearch`.
|
|
304
|
+
*/
|
|
305
|
+
export interface SemanticSearchResult {
|
|
306
|
+
/** Notes ranked by cosine similarity (best matching chunk per note), highest first. Each carries `.score`. */
|
|
307
|
+
notes: Note[];
|
|
308
|
+
/**
|
|
309
|
+
* Count of candidate notes (post structured-filter, pre-rank) with NO
|
|
310
|
+
* vector row yet for the active embedding model — the `embeddings_pending`
|
|
311
|
+
* signal. A note with SOME but not all chunks embedded still counts as
|
|
312
|
+
* "has a vector" here (coarse-but-honest: it can be ranked, just not on
|
|
313
|
+
* its full content yet) — see the doc comment on `semanticSearchNotes`.
|
|
314
|
+
*/
|
|
315
|
+
pendingCount: number;
|
|
316
|
+
/** Total candidate notes considered, after structured filters, before ranking/limit. */
|
|
317
|
+
totalCandidates: number;
|
|
274
318
|
}
|
|
275
319
|
|
|
276
320
|
/**
|
|
@@ -352,9 +396,16 @@ export interface NoteIndex {
|
|
|
352
396
|
lastUpdatedBy?: string | null;
|
|
353
397
|
lastUpdatedVia?: string | null;
|
|
354
398
|
tags?: string[];
|
|
399
|
+
/** Always present (`{}` when empty) — see `Note.metadata`. */
|
|
355
400
|
metadata?: Record<string, unknown>;
|
|
356
401
|
byteSize: number;
|
|
357
402
|
preview: string;
|
|
403
|
+
/** Derived from the first non-empty line of content (title axis, ratified
|
|
404
|
+
* 2026-07-17) — NEVER stored, computed fresh at read time by
|
|
405
|
+
* `computeDisplayTitle`. `null` when content has no non-empty line.
|
|
406
|
+
* Surfaces decide how to render a `null` title (e.g. a timestamp/path
|
|
407
|
+
* fallback); core just reports the honest content-derived value. */
|
|
408
|
+
displayTitle: string | null;
|
|
358
409
|
/** Opt-in link degree (see `Note.linkCount`). */
|
|
359
410
|
linkCount?: number;
|
|
360
411
|
/** Full-text search relevance score (see `Note.score`). Carried onto the
|
|
@@ -405,7 +456,7 @@ export interface Store {
|
|
|
405
456
|
*/
|
|
406
457
|
getNoteByPath(path: string, extension?: string): Promise<Note | null>;
|
|
407
458
|
getNotes(ids: string[]): Promise<Note[]>;
|
|
408
|
-
updateNote(id: string, updates: { content?: string; append?: string; prepend?: string; path?: string; extension?: string; metadata?: Record<string, unknown>; created_at?: string; skipUpdatedAt?: boolean; actor?: string | null; via?: string | null; if_updated_at?: string }): Promise<Note>;
|
|
459
|
+
updateNote(id: string, updates: { content?: string; append?: string; prepend?: string; path?: string; extension?: string; metadata?: Record<string, unknown>; created_at?: string; skipUpdatedAt?: boolean; actor?: string | null; via?: string | null; if_updated_at?: string; tagsForSchemaResolution?: string[] }): Promise<Note>;
|
|
409
460
|
/**
|
|
410
461
|
* Set a note's `created_at` and `updated_at` explicitly. Import-only:
|
|
411
462
|
* used by the portable-md round-trip path to restore timestamps from
|
|
@@ -455,6 +506,17 @@ export interface Store {
|
|
|
455
506
|
* `created_at` ordering. See `core/src/search-query.ts`.
|
|
456
507
|
*/
|
|
457
508
|
searchNotes(query: string, opts?: { tags?: string[]; limit?: number; expand?: TagExpandMode; mode?: SearchMode; sort?: "asc" | "desc" }): Promise<Note[]>;
|
|
509
|
+
/**
|
|
510
|
+
* Semantic search (EXPERIMENTAL — see `QueryOpts.nearText`/`semantic`).
|
|
511
|
+
* The one invocation point for the store's `EmbeddingProvider`: embeds
|
|
512
|
+
* `nearText`, ranks notes by cosine similarity over `note_vectors`
|
|
513
|
+
* (best chunk per note), and applies every other filter on `opts`
|
|
514
|
+
* (tags/metadata/dates/...) first, exactly like `queryNotes`. Throws a
|
|
515
|
+
* structured `QueryError` (`error_type: "semantic_unavailable"`) when no
|
|
516
|
+
* provider is configured or the configured one isn't ready — never a
|
|
517
|
+
* silent fallback to keyword search.
|
|
518
|
+
*/
|
|
519
|
+
semanticSearch(nearText: string, opts?: QueryOpts): Promise<SemanticSearchResult>;
|
|
458
520
|
|
|
459
521
|
// Tags
|
|
460
522
|
tagNote(noteId: string, tags: string[]): Promise<void>;
|
|
@@ -272,6 +272,42 @@ function sqliteToUserType(t: string): string {
|
|
|
272
272
|
return t.toLowerCase();
|
|
273
273
|
}
|
|
274
274
|
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
// Attachments orientation block (attachment-tickets design, Wave 0+1)
|
|
277
|
+
// ---------------------------------------------------------------------------
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* The Attachments orientation block, rendered into the connect-time
|
|
281
|
+
* markdown brief (`projectionToMarkdown`) so every agent learns, every
|
|
282
|
+
* session, how to move file bytes into/out of this vault without spending
|
|
283
|
+
* its own context on them (bytes never ride MCP either way).
|
|
284
|
+
*
|
|
285
|
+
* `ticketsEnabled` reflects whether THIS door has wired an
|
|
286
|
+
* `AttachmentTicketProvider` (bun: always, as of this PR; cloud: not yet
|
|
287
|
+
* — its mirror is a separate PR). An unwired door omits BOTH the ticket
|
|
288
|
+
* tools from `tools/list` (see `generateMcpTools`'s `attachmentTickets`
|
|
289
|
+
* opt) AND the ticket-tool sentences here — the brief never dangles a
|
|
290
|
+
* pointer at a tool the agent can't actually call.
|
|
291
|
+
*
|
|
292
|
+
* Kept as a single dense paragraph (not its own multi-line list) to stay
|
|
293
|
+
* inside the connect-time brief's token budget — see
|
|
294
|
+
* `projectionToMarkdown`'s doc comment.
|
|
295
|
+
*/
|
|
296
|
+
export function attachmentsInstructionBlock(opts: { ticketsEnabled: boolean }): string {
|
|
297
|
+
const sentences: string[] = [
|
|
298
|
+
"Notes can carry file attachments (`include_attachments: true` on `query-notes` returns their rows; bytes don't ride MCP).",
|
|
299
|
+
];
|
|
300
|
+
if (opts.ticketsEnabled) {
|
|
301
|
+
sentences.push(
|
|
302
|
+
"To move bytes, call `request-attachment-upload` / `request-attachment-download` — each mints a short-lived, single-use URL (with a ready-to-run `curl_example`) your shell spends directly; no MCP session credential is needed to spend it.",
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
sentences.push(
|
|
306
|
+
"If your runtime holds this vault's own API token, REST works too: upload = `POST {base}/storage/upload` (multipart `file`, ≤100 MB) then `POST {base}/notes/{id}/attachments` `{path, mimeType, transcribe?}`; download = `GET {base}/storage/{path}` with the same `Authorization: Bearer`. Audio attached with `transcribe: true` is transcribed automatically.",
|
|
307
|
+
);
|
|
308
|
+
return sentences.join(" ");
|
|
309
|
+
}
|
|
310
|
+
|
|
275
311
|
// ---------------------------------------------------------------------------
|
|
276
312
|
// Markdown rendering — for getServerInstruction
|
|
277
313
|
// ---------------------------------------------------------------------------
|
|
@@ -302,6 +338,13 @@ export function projectionToMarkdown(args: {
|
|
|
302
338
|
* known and always surfaced. See `chooseHubOrigin` (src/mcp-install.ts).
|
|
303
339
|
*/
|
|
304
340
|
coordinates?: { hubOrigin: string; hubOriginKnown: boolean };
|
|
341
|
+
/**
|
|
342
|
+
* Attachments orientation (see `attachmentsInstructionBlock`). Omitted
|
|
343
|
+
* defaults to `{ ticketsEnabled: false }` — safe-by-default so a caller
|
|
344
|
+
* that hasn't wired ticket support yet (or a test fixture) never
|
|
345
|
+
* accidentally advertises tools it can't back.
|
|
346
|
+
*/
|
|
347
|
+
attachments?: { ticketsEnabled: boolean };
|
|
305
348
|
}): string {
|
|
306
349
|
const { vaultName, description, projection, coordinates } = args;
|
|
307
350
|
const stats = projection.stats;
|
|
@@ -418,6 +461,11 @@ export function projectionToMarkdown(args: {
|
|
|
418
461
|
lines.push("");
|
|
419
462
|
lines.push("If schema or tags change during this session, call `vault-info` to refresh the full projection. Call `list-tags { include_schema: true }` for tag-only details.");
|
|
420
463
|
|
|
464
|
+
lines.push("");
|
|
465
|
+
lines.push("## Attachments");
|
|
466
|
+
lines.push("");
|
|
467
|
+
lines.push(attachmentsInstructionBlock(args.attachments ?? { ticketsEnabled: false }));
|
|
468
|
+
|
|
421
469
|
// Scripting pointer block: the connect-time brief used to dead-end on
|
|
422
470
|
// querying — an agent had no path to "how do I script/automate against this
|
|
423
471
|
// vault." Point at the guide rather than inlining it, to keep this brief
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openparachute/vault",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.3-rc.10",
|
|
4
4
|
"description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
|
|
5
5
|
"module": "src/cli.ts",
|
|
6
6
|
"type": "module",
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
"prepack": "bun run build:spa"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
+
"@huggingface/transformers": "^4.2.0",
|
|
28
29
|
"@modelcontextprotocol/sdk": "^1.12.1",
|
|
29
30
|
"@openparachute/scope-guard": "^0.5.0",
|
|
30
31
|
"jose": "^6.2.2",
|
|
@@ -34,6 +35,10 @@
|
|
|
34
35
|
"devDependencies": {
|
|
35
36
|
"@types/bun": "latest"
|
|
36
37
|
},
|
|
38
|
+
"trustedDependencies": [
|
|
39
|
+
"onnxruntime-node",
|
|
40
|
+
"protobufjs"
|
|
41
|
+
],
|
|
37
42
|
"peerDependencies": {
|
|
38
43
|
"typescript": "^5"
|
|
39
44
|
},
|
package/src/add-pack.test.ts
CHANGED
|
@@ -140,3 +140,35 @@ describe("add-pack surface-starter", () => {
|
|
|
140
140
|
expect(stdout).toContain("~ tag capture (upserted)");
|
|
141
141
|
});
|
|
142
142
|
});
|
|
143
|
+
|
|
144
|
+
describe("add-pack — tag description preservation on re-apply (Aaron-ratified 2026-07-17)", () => {
|
|
145
|
+
test("a hand-edited description survives a re-apply and is reported 'kept', not 'upserted'", () => {
|
|
146
|
+
expect(runCli(["create", "packed", "--json"], { PARACHUTE_HOME: home }).exitCode).toBe(0);
|
|
147
|
+
|
|
148
|
+
// Simulate an operator/AI hand-editing the capture tag's constitution.
|
|
149
|
+
const dbPath = join(home, "vault", "data", "packed", "vault.db");
|
|
150
|
+
const editedDescription = "Our house rules for capture — not the default text.";
|
|
151
|
+
const writeDb = new Database(dbPath);
|
|
152
|
+
writeDb.query("UPDATE tags SET description = ? WHERE name = ?").run(
|
|
153
|
+
editedDescription,
|
|
154
|
+
"capture",
|
|
155
|
+
);
|
|
156
|
+
writeDb.close();
|
|
157
|
+
|
|
158
|
+
const { exitCode, stdout } = runCli(
|
|
159
|
+
["add-pack", "welcome", "--vault", "packed"],
|
|
160
|
+
{ PARACHUTE_HOME: home },
|
|
161
|
+
);
|
|
162
|
+
expect(exitCode).toBe(0);
|
|
163
|
+
expect(stdout).toContain("~ tag capture (kept user description)");
|
|
164
|
+
// guide wasn't edited, so it still reports the old vocabulary.
|
|
165
|
+
expect(stdout).toContain("~ tag guide (upserted)");
|
|
166
|
+
|
|
167
|
+
const readDb = new Database(dbPath, { readonly: true });
|
|
168
|
+
const row = readDb
|
|
169
|
+
.query("SELECT description FROM tags WHERE name = ?")
|
|
170
|
+
.get("capture") as { description: string };
|
|
171
|
+
readDb.close();
|
|
172
|
+
expect(row.description).toBe(editedDescription);
|
|
173
|
+
});
|
|
174
|
+
});
|