@openparachute/vault 0.7.2 → 0.7.3-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +5 -3
  2. package/core/src/attachment/policy.test.ts +66 -0
  3. package/core/src/attachment/policy.ts +131 -0
  4. package/core/src/attachment/tickets.test.ts +45 -0
  5. package/core/src/attachment/tickets.ts +117 -0
  6. package/core/src/attachment-tickets-tool.test.ts +286 -0
  7. package/core/src/conformance.ts +2 -1
  8. package/core/src/contract-typed-index.test.ts +4 -3
  9. package/core/src/core.test.ts +607 -11
  10. package/core/src/display-title.test.ts +190 -0
  11. package/core/src/embedding/chunker.test.ts +97 -0
  12. package/core/src/embedding/chunker.ts +180 -0
  13. package/core/src/embedding/provider.ts +108 -0
  14. package/core/src/embedding/staleness.test.ts +87 -0
  15. package/core/src/embedding/staleness.ts +83 -0
  16. package/core/src/embedding/vector-codec.test.ts +99 -0
  17. package/core/src/embedding/vector-codec.ts +60 -0
  18. package/core/src/embedding/vectors.test.ts +163 -0
  19. package/core/src/embedding/vectors.ts +135 -0
  20. package/core/src/expand.ts +11 -3
  21. package/core/src/indexed-fields.test.ts +9 -3
  22. package/core/src/indexed-fields.ts +9 -1
  23. package/core/src/lede.test.ts +96 -0
  24. package/core/src/mcp-semantic-search.test.ts +160 -0
  25. package/core/src/mcp.ts +405 -10
  26. package/core/src/notes.semantic-search.test.ts +131 -0
  27. package/core/src/notes.ts +319 -7
  28. package/core/src/query-warnings.ts +40 -0
  29. package/core/src/schema-defaults.ts +85 -1
  30. package/core/src/schema-v27-note-vectors.test.ts +178 -0
  31. package/core/src/schema.ts +81 -1
  32. package/core/src/search-fts-v25.test.ts +9 -1
  33. package/core/src/search-query.test.ts +42 -0
  34. package/core/src/search-query.ts +27 -0
  35. package/core/src/search-title-boost.test.ts +125 -0
  36. package/core/src/seed-packs.test.ts +117 -1
  37. package/core/src/seed-packs.ts +217 -1
  38. package/core/src/store.semantic-search.test.ts +236 -0
  39. package/core/src/store.ts +162 -5
  40. package/core/src/tag-schemas.ts +27 -12
  41. package/core/src/types.ts +63 -1
  42. package/core/src/vault-projection.ts +48 -0
  43. package/package.json +6 -1
  44. package/src/add-pack.test.ts +32 -0
  45. package/src/attachment-tickets.test.ts +350 -0
  46. package/src/attachment-tickets.ts +264 -0
  47. package/src/auth.ts +8 -2
  48. package/src/cli.ts +5 -1
  49. package/src/contract-errors.test.ts +2 -1
  50. package/src/contract-honest-queries.test.ts +88 -0
  51. package/src/contract-search.test.ts +41 -0
  52. package/src/embedding/capability.test.ts +33 -0
  53. package/src/embedding/capability.ts +34 -0
  54. package/src/embedding/external-api.test.ts +154 -0
  55. package/src/embedding/external-api.ts +144 -0
  56. package/src/embedding/onnx-transformers.test.ts +113 -0
  57. package/src/embedding/onnx-transformers.ts +141 -0
  58. package/src/embedding/select.test.ts +99 -0
  59. package/src/embedding/select.ts +92 -0
  60. package/src/embedding-worker.test.ts +300 -0
  61. package/src/embedding-worker.ts +226 -0
  62. package/src/mcp-http.ts +13 -1
  63. package/src/mcp-tools.ts +49 -14
  64. package/src/onboarding-seed.test.ts +64 -0
  65. package/src/routes.ts +173 -92
  66. package/src/routing.ts +17 -0
  67. package/src/scribe-discovery.test.ts +76 -1
  68. package/src/scribe-discovery.ts +28 -9
  69. package/src/semantic-search-routes.test.ts +161 -0
  70. package/src/server.ts +21 -2
  71. package/src/vault-embeddings-capability.test.ts +82 -0
  72. package/src/vault-store-embedding-wiring.test.ts +69 -0
  73. package/src/vault-store.ts +53 -2
  74. package/src/vault.test.ts +62 -13
package/core/src/store.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { Database } from "bun:sqlite";
2
- import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage, AggregateRow } from "./types.js";
2
+ import type { Store, Note, Link, Attachment, QueryOpts, QueryNotesPage, AggregateRow, SemanticSearchResult } from "./types.js";
3
3
  import { initSchema } from "./schema.js";
4
4
  import * as noteOps from "./notes.js";
5
5
  import * as linkOps from "./links.js";
@@ -38,6 +38,7 @@ import {
38
38
  loadSchemaConfig,
39
39
  validateNote as runValidateNote,
40
40
  resolveNoteSchemas,
41
+ normalizeDateFields,
41
42
  type ResolvedSchemas,
42
43
  type ValidationStatus,
43
44
  } from "./schema-defaults.js";
@@ -47,6 +48,9 @@ import {
47
48
  } from "./conformance.js";
48
49
  import type { SearchMode } from "./search-query.js";
49
50
  import { runDoctorScan, type DoctorReport, type DoctorScanOpts } from "./doctor.js";
51
+ import { EmbeddingError, type EmbeddingProvider } from "./embedding/provider.js";
52
+ import { normalize } from "./embedding/vector-codec.js";
53
+ import { QueryError } from "./query-operators.js";
50
54
 
51
55
  /**
52
56
  * Normalize a `type: "reference"` field value (scalar OR array — see
@@ -94,9 +98,38 @@ export class BunSqliteStore implements Store {
94
98
  private _tagHierarchy: TagHierarchy | null = null;
95
99
  private _schemaConfig: ResolvedSchemas | null = null;
96
100
 
97
- constructor(public readonly db: Database, opts?: { hooks?: HookRegistry }) {
101
+ /**
102
+ * The active `EmbeddingProvider` (EXPERIMENTAL — semantic search MVP),
103
+ * when one is configured. `undefined` on a vault with no provider —
104
+ * `semanticSearch` reports that honestly (`semantic_unavailable`)
105
+ * rather than silently falling back to keyword search. Resolved ONCE at
106
+ * construction (mirrors `hooks`) — self-host/cloud each build the
107
+ * concrete provider (ONNX/external-API, Workers AI) and pass it in here;
108
+ * core never imports a model library itself (dependency-purity rule).
109
+ */
110
+ public readonly embeddingProvider?: EmbeddingProvider;
111
+
112
+ /**
113
+ * WHY `embeddingProvider` is absent, when the caller knows a specific
114
+ * reason (currently: the operator set `EMBEDDINGS_ENABLED=false` — see
115
+ * `src/embedding/select.ts`). Core never reads env vars itself
116
+ * (dependency-purity rule — see `embeddingProvider`'s doc comment
117
+ * above), so it can't distinguish "explicitly turned off" from "never
118
+ * configured" on its own; the caller passes this through so
119
+ * `semanticSearch`'s error hint can be honest either way instead of
120
+ * defaulting to generic provider-setup instructions when the operator
121
+ * deliberately opted out.
122
+ */
123
+ public readonly embeddingDisabledReason?: string;
124
+
125
+ constructor(
126
+ public readonly db: Database,
127
+ opts?: { hooks?: HookRegistry; embeddingProvider?: EmbeddingProvider; embeddingDisabledReason?: string },
128
+ ) {
98
129
  initSchema(db);
99
130
  this.hooks = opts?.hooks ?? new HookRegistry();
131
+ this.embeddingProvider = opts?.embeddingProvider;
132
+ this.embeddingDisabledReason = opts?.embeddingDisabledReason;
100
133
  }
101
134
 
102
135
  /**
@@ -468,6 +501,16 @@ export class BunSqliteStore implements Store {
468
501
  // ---- Notes ----
469
502
 
470
503
  async createNote(content: string, opts?: { id?: string; path?: string; tags?: string[]; metadata?: Record<string, unknown>; created_at?: string; extension?: string; actor?: string | null; via?: string | null }): Promise<Note> {
504
+ // Normalize `date`-typed field values to canonical UTC ISO form BEFORE
505
+ // the write (vault#date-field-type — mixed-offset TEXT-compare
506
+ // corruption). COPY-ON-WRITE (round 2) — reassign the local `opts`
507
+ // binding from the returned value rather than mutating the caller's
508
+ // object; see `normalizeDateFields`'s doc comment.
509
+ if (opts?.metadata) {
510
+ const normalized = normalizeDateFields(this.getSchemaConfig(), { tags: opts.tags, metadata: opts.metadata });
511
+ if (normalized !== opts.metadata) opts = { ...opts, metadata: normalized };
512
+ }
513
+
471
514
  const note = noteOps.createNote(this.db, content, opts);
472
515
 
473
516
  if (content) {
@@ -516,6 +559,20 @@ export class BunSqliteStore implements Store {
516
559
  actor?: string | null;
517
560
  via?: string | null;
518
561
  if_updated_at?: string;
562
+ /**
563
+ * `date`-field normalization override (vault#date-field-type review
564
+ * round 2). By default, `normalizeDateFields` below resolves the note's
565
+ * effective schema against its CURRENT tags in the DB — correct when
566
+ * this call doesn't also change tags. A caller that adds a tag IN THIS
567
+ * SAME logical update (via a separate `store.tagNote` issued right
568
+ * after this call — see mcp.ts's `update-note` handler and the batch
569
+ * upsert "update"/"replace" branch) must pass the PROJECTED final tag
570
+ * set here instead, or a newly-added tag's `type: "date"` field never
571
+ * gets seen (the schema resolution would still be looking at the
572
+ * pre-write tag set) and its offset-bearing value would persist
573
+ * verbatim. Ignored when `updates.metadata` is undefined.
574
+ */
575
+ tagsForSchemaResolution?: string[];
519
576
  },
520
577
  ): Promise<Note> {
521
578
  let oldPath: string | undefined;
@@ -529,7 +586,19 @@ export class BunSqliteStore implements Store {
529
586
  if (updates.path !== undefined || updates.metadata !== undefined) {
530
587
  const existing = noteOps.getNote(this.db, id);
531
588
  if (updates.path !== undefined) oldPath = existing?.path;
532
- if (updates.metadata !== undefined) priorMetadataForRefs = existing?.metadata;
589
+ if (updates.metadata !== undefined) {
590
+ priorMetadataForRefs = existing?.metadata;
591
+ // Normalize `date`-typed field values to canonical UTC ISO form
592
+ // BEFORE the write (vault#date-field-type — mixed-offset TEXT-
593
+ // compare corruption). COPY-ON-WRITE (round 2) — reassign the local
594
+ // `updates` binding from the returned value rather than mutating the
595
+ // caller's object; see `normalizeDateFields`'s doc comment.
596
+ // `tagsForSchemaResolution` (when the caller is ALSO adding a tag in
597
+ // this same logical update) wins over the note's current DB tags.
598
+ const tagsForResolution = updates.tagsForSchemaResolution ?? existing?.tags;
599
+ const normalized = normalizeDateFields(this.getSchemaConfig(), { tags: tagsForResolution, metadata: updates.metadata });
600
+ if (normalized !== updates.metadata) updates = { ...updates, metadata: normalized };
601
+ }
533
602
  }
534
603
 
535
604
  const note = noteOps.updateNote(this.db, id, updates);
@@ -814,6 +883,75 @@ export class BunSqliteStore implements Store {
814
883
  return noteOps.searchNotes(this.db, query, opts);
815
884
  }
816
885
 
886
+ /**
887
+ * Semantic search (EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). The ONE
888
+ * invocation point for `embeddingProvider`: resolves it, embeds
889
+ * `nearText` into a query vector, normalizes it (stored vectors are
890
+ * ALSO L2-normalized — see `note_vectors`'s schema comment — so the
891
+ * scan's dot product is cosine similarity), and calls
892
+ * `noteOps.semanticSearchNotes` with the SAME tag-hierarchy expansion
893
+ * `queryNotes` applies (so `semantic: true, tag: "manual"` composes with
894
+ * the tag `expand` axis identically to a structured query).
895
+ *
896
+ * Throws a `QueryError` (`error_type: "semantic_unavailable"`) when no
897
+ * provider is configured, the configured one reports itself not ready,
898
+ * OR the embed call itself fails mid-flight (e.g. the bundled ONNX
899
+ * floor's lazy model load fails on this — the FIRST — real `embed()`
900
+ * attempt, after `available()` optimistically reported `ok: true`; see
901
+ * `onnx-transformers.ts`'s "lazy-fail, not crash" doc) — NEVER a silent
902
+ * fallback to keyword search, and never a raw unstructured 500. Callers
903
+ * (MCP/REST) let this propagate uncaught, same as
904
+ * `invalid_search_syntax` above.
905
+ */
906
+ async semanticSearch(nearText: string, opts?: QueryOpts): Promise<SemanticSearchResult> {
907
+ if (!this.embeddingProvider) {
908
+ // Honest either way: if the caller told us WHY (operator explicitly
909
+ // set EMBEDDINGS_ENABLED=false), say that — not generic provider-setup
910
+ // instructions that would be actively misleading for a deliberate
911
+ // opt-out. See `embeddingDisabledReason`'s doc comment.
912
+ throw new QueryError(
913
+ `semantic search requires an embedding provider — none is configured on this vault`,
914
+ "SEMANTIC_UNAVAILABLE",
915
+ {
916
+ error_type: "semantic_unavailable",
917
+ hint:
918
+ this.embeddingDisabledReason ??
919
+ "configure EMBEDDING_API_URL/EMBEDDING_API_KEY/EMBEDDING_MODEL, or rely on the bundled floor provider (self-host); semantic search is not yet available on this door otherwise",
920
+ },
921
+ );
922
+ }
923
+ const availability = await this.embeddingProvider.available();
924
+ if (!availability.ok) {
925
+ throw new QueryError(
926
+ `semantic search is unavailable: ${availability.reason ?? "embedding provider not ready"}`,
927
+ "SEMANTIC_UNAVAILABLE",
928
+ { error_type: "semantic_unavailable", hint: availability.reason },
929
+ );
930
+ }
931
+
932
+ let vectors: Float32Array[];
933
+ try {
934
+ ({ vectors } = await this.embeddingProvider.embed({ texts: [nearText] }));
935
+ } catch (err) {
936
+ // `available()` passing is only a CHEAP readiness check (never a real
937
+ // network/model round-trip — see EmbeddingProvider.available's doc
938
+ // comment) — it can't catch a provider whose actual first embed call
939
+ // fails (a lazy model load blowing up, a transient upstream error).
940
+ // Map ANY embed()-time failure to the same honest semantic_unavailable
941
+ // shape as the checks above, rather than letting a raw EmbeddingError
942
+ // (or whatever else a provider throws) surface as an unstructured 500.
943
+ const reason = err instanceof EmbeddingError ? err.message : err instanceof Error ? err.message : String(err);
944
+ throw new QueryError(`semantic search is unavailable: ${reason}`, "SEMANTIC_UNAVAILABLE", {
945
+ error_type: "semantic_unavailable",
946
+ hint: reason,
947
+ });
948
+ }
949
+ const queryVector = normalize(vectors[0]!);
950
+
951
+ const filterOpts = this.expandQueryTags(this.normalizeQueryTags(opts ?? {}));
952
+ return noteOps.semanticSearchNotes(this.db, queryVector, filterOpts, this.embeddingProvider.model);
953
+ }
954
+
817
955
  // ---- Tags ----
818
956
 
819
957
  async tagNote(noteId: string, tags: string[]): Promise<void> {
@@ -933,7 +1071,18 @@ export class BunSqliteStore implements Store {
933
1071
  // ---- Bulk Operations ----
934
1072
 
935
1073
  async createNotes(inputs: noteOps.BulkNoteInput[]): Promise<Note[]> {
936
- const notes = noteOps.createNotes(this.db, inputs);
1074
+ // Same pre-write `date`-field normalization as singleton createNote
1075
+ // (vault#date-field-type) — this bulk path bypasses it otherwise.
1076
+ // COPY-ON-WRITE (round 2) — build a new array with only the items that
1077
+ // actually need a rewrite replaced; never mutate the caller's `inputs`
1078
+ // or its element objects.
1079
+ const schemaConfig = this.getSchemaConfig();
1080
+ const normalizedInputs = inputs.map((input) => {
1081
+ if (!input.metadata) return input;
1082
+ const normalized = normalizeDateFields(schemaConfig, { tags: input.tags, metadata: input.metadata });
1083
+ return normalized !== input.metadata ? { ...input, metadata: normalized } : input;
1084
+ });
1085
+ const notes = noteOps.createNotes(this.db, normalizedInputs);
937
1086
  for (const note of notes) {
938
1087
  // Bulk path needs the same config-cache invalidation as singleton
939
1088
  // createNote — without it, a batch that includes `_tags/*` notes
@@ -1143,7 +1292,7 @@ export class BunSqliteStore implements Store {
1143
1292
  const mapped = indexedFieldOps.mapFieldType(spec.type);
1144
1293
  if (!mapped) {
1145
1294
  throw new indexedFieldOps.IndexedFieldError(
1146
- `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
1295
+ `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference, date)`,
1147
1296
  );
1148
1297
  }
1149
1298
  // Throws IndexedFieldError on an invalid identifier (e.g. kebab-case).
@@ -1275,6 +1424,14 @@ export class BunSqliteStore implements Store {
1275
1424
  * place.)
1276
1425
  */
1277
1426
  async createNoteRaw(content: string, opts?: { id?: string; path?: string; tags?: string[]; metadata?: Record<string, unknown>; created_at?: string; extension?: string }): Promise<Note> {
1427
+ // Same pre-write `date`-field normalization as createNote
1428
+ // (vault#date-field-type) — the legacy Obsidian importer (obsidian.ts)
1429
+ // funnels through this path and bypasses createNote's copy otherwise.
1430
+ // COPY-ON-WRITE (round 2) — see normalizeDateFields's doc comment.
1431
+ if (opts?.metadata) {
1432
+ const normalized = normalizeDateFields(this.getSchemaConfig(), { tags: opts.tags, metadata: opts.metadata });
1433
+ if (normalized !== opts.metadata) opts = { ...opts, metadata: normalized };
1434
+ }
1278
1435
  return noteOps.createNote(this.db, content, opts);
1279
1436
  }
1280
1437
 
@@ -18,6 +18,7 @@
18
18
  import { Database } from "bun:sqlite";
19
19
  import { mapFieldType, validateFieldName } from "./indexed-fields.js";
20
20
  import { loadTagHierarchy, findParentCycle } from "./tag-hierarchy.js";
21
+ import { timestampToMs } from "./cursor.js";
21
22
 
22
23
  // ---------------------------------------------------------------------------
23
24
  // Types
@@ -192,7 +193,7 @@ export class InvalidFieldDefaultError extends Error {
192
193
 
193
194
  /**
194
195
  * Thrown by `upsertTagRecord` (core/src/store.ts's chokepoint) when a
195
- * field's declared `type` isn't one of the six recognized values (vault#555
196
+ * field's declared `type` isn't one of the recognized values (vault#555
196
197
  * — `update-tag{fields:{weird:{type:"frobnicator"}}}` used to be accepted
197
198
  * and persisted verbatim, no error, for any NON-indexed field: the only
198
199
  * existing type check, `mapFieldType`, ran solely on `indexed: true` fields
@@ -255,13 +256,13 @@ export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFi
255
256
 
256
257
  /**
257
258
  * The full recognized vocabulary for `TagFieldSchema.type` — storage/
258
- * advisory validation accepts all seven; only `string`/`integer`/`boolean`/
259
- * `reference` are INDEXABLE (that narrower subset is `indexed-fields.ts`'s
260
- * `TYPE_MAP`, enforced separately via `mapFieldType` for `indexed: true`
261
- * fields). Matches `defaultMatchesType`'s switch and `schema-defaults.ts`'s
262
- * `SchemaField.type` union — kept in lockstep by hand across the two
263
- * deliberately-decoupled modules (see `validateFieldDefault`'s doc comment
264
- * for why they don't cross-import).
259
+ * advisory validation accepts all eight; only `string`/`integer`/`boolean`/
260
+ * `reference`/`date` are INDEXABLE (that narrower subset is
261
+ * `indexed-fields.ts`'s `TYPE_MAP`, enforced separately via `mapFieldType`
262
+ * for `indexed: true` fields). Matches `defaultMatchesType`'s switch and
263
+ * `schema-defaults.ts`'s `SchemaField.type` union — kept in lockstep by hand
264
+ * across the two deliberately-decoupled modules (see `validateFieldDefault`'s
265
+ * doc comment for why they don't cross-import).
265
266
  *
266
267
  * `reference` (vault#typed-reference-field) is a dual-write field type: the
267
268
  * value is stored + validated exactly like `string` (an id/path/title the
@@ -270,11 +271,18 @@ export function validateFieldDefault(field: string, spec: TagFieldSchema): TagFi
270
271
  * to a note and maintains a graph `links` edge from this note to the
271
272
  * resolved target, with `relationship` set to the field name. See
272
273
  * `docs/design/typed-reference-field.md` for the full design + known gaps.
274
+ *
275
+ * `date` stores/validates like `string` (an ISO-8601 date or timestamp) so
276
+ * indexed `date` fields sort correctly under a plain TEXT comparison — see
277
+ * `defaultMatchesType`'s `"date"` case and `schema-defaults.ts`'s
278
+ * `valueMatchesType`, both of which reuse `cursor.ts`'s `timestampToMs`
279
+ * (the SAME UTC-correct parser `date_filter`'s `updated_at` bound uses) so
280
+ * there's exactly one ISO-parsing implementation in the codebase, not two.
273
281
  */
274
- export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object", "reference"] as const;
282
+ export const VALID_FIELD_TYPES = ["string", "number", "integer", "boolean", "array", "object", "reference", "date"] as const;
275
283
 
276
284
  /**
277
- * Validate that a field's declared `type` is one of the six recognized
285
+ * Validate that a field's declared `type` is one of the recognized
278
286
  * values (vault#555). Returns `null` when `type` is unset (own-field checks
279
287
  * elsewhere already treat an unset type as "nothing to check against") or
280
288
  * recognized; otherwise a {@link TagFieldViolation} with `reason:
@@ -328,7 +336,7 @@ export function collectOwnFieldDefaultAndTypeViolations(
328
336
  return violations;
329
337
  }
330
338
 
331
- /** Same seven-type vocabulary as `TagFieldSchema.type`'s doc comment. Unknown/unset types pass (nothing to check against). */
339
+ /** Same eight-type vocabulary as `TagFieldSchema.type`'s doc comment. Unknown/unset types pass (nothing to check against). */
332
340
  function defaultMatchesType(value: unknown, type: string): boolean {
333
341
  switch (type) {
334
342
  case "string":
@@ -348,6 +356,13 @@ function defaultMatchesType(value: unknown, type: string): boolean {
348
356
  // VALID_FIELD_TYPES's doc comment.
349
357
  case "reference":
350
358
  return typeof value === "string";
359
+ // `date` accepts an ISO-8601 date (`YYYY-MM-DD`) or full RFC3339
360
+ // timestamp — the SAME grammar `date_filter`'s `updated_at` bound
361
+ // parses (`timestampToMs`, imported from cursor.ts), not a second,
362
+ // independently-drifting date parser. See VALID_FIELD_TYPES's doc
363
+ // comment.
364
+ case "date":
365
+ return typeof value === "string" && timestampToMs(value) !== null;
351
366
  default:
352
367
  return true;
353
368
  }
@@ -767,7 +782,7 @@ export function collectTagFieldViolations(
767
782
  violations.push({
768
783
  field: fieldName,
769
784
  reason: "unsupported_indexed_type",
770
- message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference)`,
785
+ message: `field "${fieldName}" has unsupported type "${spec.type}" for indexing (supported: string, integer, boolean, reference, date)`,
771
786
  });
772
787
  } else {
773
788
  try {
package/core/src/types.ts CHANGED
@@ -31,6 +31,13 @@ export interface Note {
31
31
  * `core/src/portable-md.ts:supportsInlineFrontmatter`.
32
32
  */
33
33
  extension?: string;
34
+ /**
35
+ * Always present on notes read back from the store — NULL/`"{}"`/
36
+ * unparseable metadata all collapse to `{}` rather than an absent key
37
+ * (mirrors `tags`, which is likewise always `[]`). Optional here only so
38
+ * write-input shapes (`createNote`/`updateNote` options) can share this
39
+ * type without forcing every caller to pass an empty object.
40
+ */
34
41
  metadata?: Record<string, unknown>;
35
42
  createdAt: string; // ISO-8601
36
43
  updatedAt?: string;
@@ -271,6 +278,43 @@ export interface QueryOpts {
271
278
  * row per group, not a paginated note list.
272
279
  */
273
280
  aggregate?: AggregateSpec;
281
+ /**
282
+ * Semantic-search MVP (EXPERIMENTAL — `SEMANTIC-MVP-PLAN.md`). Free text
283
+ * to rank notes by MEANING rather than keyword. Requires `semantic:
284
+ * true` and is mutually exclusive with `search` (structurally the same
285
+ * "text in → ranked notes with a score out" shape as `search`, but
286
+ * routed through `note_vectors` cosine similarity instead of FTS5
287
+ * bm25). Every other filter on this `QueryOpts` (tags, metadata, date
288
+ * range, path, ...) narrows the candidate set FIRST, exactly like
289
+ * `search` — see `core/src/notes.ts:semanticSearchNotes`.
290
+ */
291
+ nearText?: string;
292
+ /**
293
+ * Opt into vector ranking. `true` requires `nearText`; omitting it (or
294
+ * passing `false`) is a plain structured/keyword query, byte-identical
295
+ * to pre-semantic-search behavior. See `nearText`.
296
+ */
297
+ semantic?: boolean;
298
+ }
299
+
300
+ /**
301
+ * Result of a semantic-search scan (EXPERIMENTAL — see
302
+ * `QueryOpts.nearText`/`semantic`). Returned by
303
+ * `core/src/notes.ts:semanticSearchNotes` and `Store.semanticSearch`.
304
+ */
305
+ export interface SemanticSearchResult {
306
+ /** Notes ranked by cosine similarity (best matching chunk per note), highest first. Each carries `.score`. */
307
+ notes: Note[];
308
+ /**
309
+ * Count of candidate notes (post structured-filter, pre-rank) with NO
310
+ * vector row yet for the active embedding model — the `embeddings_pending`
311
+ * signal. A note with SOME but not all chunks embedded still counts as
312
+ * "has a vector" here (coarse-but-honest: it can be ranked, just not on
313
+ * its full content yet) — see the doc comment on `semanticSearchNotes`.
314
+ */
315
+ pendingCount: number;
316
+ /** Total candidate notes considered, after structured filters, before ranking/limit. */
317
+ totalCandidates: number;
274
318
  }
275
319
 
276
320
  /**
@@ -352,9 +396,16 @@ export interface NoteIndex {
352
396
  lastUpdatedBy?: string | null;
353
397
  lastUpdatedVia?: string | null;
354
398
  tags?: string[];
399
+ /** Always present (`{}` when empty) — see `Note.metadata`. */
355
400
  metadata?: Record<string, unknown>;
356
401
  byteSize: number;
357
402
  preview: string;
403
+ /** Derived from the first non-empty line of content (title axis, ratified
404
+ * 2026-07-17) — NEVER stored, computed fresh at read time by
405
+ * `computeDisplayTitle`. `null` when content has no non-empty line.
406
+ * Surfaces decide how to render a `null` title (e.g. a timestamp/path
407
+ * fallback); core just reports the honest content-derived value. */
408
+ displayTitle: string | null;
358
409
  /** Opt-in link degree (see `Note.linkCount`). */
359
410
  linkCount?: number;
360
411
  /** Full-text search relevance score (see `Note.score`). Carried onto the
@@ -405,7 +456,7 @@ export interface Store {
405
456
  */
406
457
  getNoteByPath(path: string, extension?: string): Promise<Note | null>;
407
458
  getNotes(ids: string[]): Promise<Note[]>;
408
- updateNote(id: string, updates: { content?: string; append?: string; prepend?: string; path?: string; extension?: string; metadata?: Record<string, unknown>; created_at?: string; skipUpdatedAt?: boolean; actor?: string | null; via?: string | null; if_updated_at?: string }): Promise<Note>;
459
+ updateNote(id: string, updates: { content?: string; append?: string; prepend?: string; path?: string; extension?: string; metadata?: Record<string, unknown>; created_at?: string; skipUpdatedAt?: boolean; actor?: string | null; via?: string | null; if_updated_at?: string; tagsForSchemaResolution?: string[] }): Promise<Note>;
409
460
  /**
410
461
  * Set a note's `created_at` and `updated_at` explicitly. Import-only:
411
462
  * used by the portable-md round-trip path to restore timestamps from
@@ -455,6 +506,17 @@ export interface Store {
455
506
  * `created_at` ordering. See `core/src/search-query.ts`.
456
507
  */
457
508
  searchNotes(query: string, opts?: { tags?: string[]; limit?: number; expand?: TagExpandMode; mode?: SearchMode; sort?: "asc" | "desc" }): Promise<Note[]>;
509
+ /**
510
+ * Semantic search (EXPERIMENTAL — see `QueryOpts.nearText`/`semantic`).
511
+ * The one invocation point for the store's `EmbeddingProvider`: embeds
512
+ * `nearText`, ranks notes by cosine similarity over `note_vectors`
513
+ * (best chunk per note), and applies every other filter on `opts`
514
+ * (tags/metadata/dates/...) first, exactly like `queryNotes`. Throws a
515
+ * structured `QueryError` (`error_type: "semantic_unavailable"`) when no
516
+ * provider is configured or the configured one isn't ready — never a
517
+ * silent fallback to keyword search.
518
+ */
519
+ semanticSearch(nearText: string, opts?: QueryOpts): Promise<SemanticSearchResult>;
458
520
 
459
521
  // Tags
460
522
  tagNote(noteId: string, tags: string[]): Promise<void>;
@@ -272,6 +272,42 @@ function sqliteToUserType(t: string): string {
272
272
  return t.toLowerCase();
273
273
  }
274
274
 
275
+ // ---------------------------------------------------------------------------
276
+ // Attachments orientation block (attachment-tickets design, Wave 0+1)
277
+ // ---------------------------------------------------------------------------
278
+
279
+ /**
280
+ * The Attachments orientation block, rendered into the connect-time
281
+ * markdown brief (`projectionToMarkdown`) so every agent learns, every
282
+ * session, how to move file bytes into/out of this vault without spending
283
+ * its own context on them (bytes never ride MCP either way).
284
+ *
285
+ * `ticketsEnabled` reflects whether THIS door has wired an
286
+ * `AttachmentTicketProvider` (bun: always, as of this PR; cloud: not yet
287
+ * — its mirror is a separate PR). An unwired door omits BOTH the ticket
288
+ * tools from `tools/list` (see `generateMcpTools`'s `attachmentTickets`
289
+ * opt) AND the ticket-tool sentences here — the brief never dangles a
290
+ * pointer at a tool the agent can't actually call.
291
+ *
292
+ * Kept as a single dense paragraph (not its own multi-line list) to stay
293
+ * inside the connect-time brief's token budget — see
294
+ * `projectionToMarkdown`'s doc comment.
295
+ */
296
+ export function attachmentsInstructionBlock(opts: { ticketsEnabled: boolean }): string {
297
+ const sentences: string[] = [
298
+ "Notes can carry file attachments (`include_attachments: true` on `query-notes` returns their rows; bytes don't ride MCP).",
299
+ ];
300
+ if (opts.ticketsEnabled) {
301
+ sentences.push(
302
+ "To move bytes, call `request-attachment-upload` / `request-attachment-download` — each mints a short-lived, single-use URL (with a ready-to-run `curl_example`) your shell spends directly; no MCP session credential is needed to spend it.",
303
+ );
304
+ }
305
+ sentences.push(
306
+ "If your runtime holds this vault's own API token, REST works too: upload = `POST {base}/storage/upload` (multipart `file`, ≤100 MB) then `POST {base}/notes/{id}/attachments` `{path, mimeType, transcribe?}`; download = `GET {base}/storage/{path}` with the same `Authorization: Bearer`. Audio attached with `transcribe: true` is transcribed automatically.",
307
+ );
308
+ return sentences.join(" ");
309
+ }
310
+
275
311
  // ---------------------------------------------------------------------------
276
312
  // Markdown rendering — for getServerInstruction
277
313
  // ---------------------------------------------------------------------------
@@ -302,6 +338,13 @@ export function projectionToMarkdown(args: {
302
338
  * known and always surfaced. See `chooseHubOrigin` (src/mcp-install.ts).
303
339
  */
304
340
  coordinates?: { hubOrigin: string; hubOriginKnown: boolean };
341
+ /**
342
+ * Attachments orientation (see `attachmentsInstructionBlock`). Omitted
343
+ * defaults to `{ ticketsEnabled: false }` — safe-by-default so a caller
344
+ * that hasn't wired ticket support yet (or a test fixture) never
345
+ * accidentally advertises tools it can't back.
346
+ */
347
+ attachments?: { ticketsEnabled: boolean };
305
348
  }): string {
306
349
  const { vaultName, description, projection, coordinates } = args;
307
350
  const stats = projection.stats;
@@ -418,6 +461,11 @@ export function projectionToMarkdown(args: {
418
461
  lines.push("");
419
462
  lines.push("If schema or tags change during this session, call `vault-info` to refresh the full projection. Call `list-tags { include_schema: true }` for tag-only details.");
420
463
 
464
+ lines.push("");
465
+ lines.push("## Attachments");
466
+ lines.push("");
467
+ lines.push(attachmentsInstructionBlock(args.attachments ?? { ticketsEnabled: false }));
468
+
421
469
  // Scripting pointer block: the connect-time brief used to dead-end on
422
470
  // querying — an agent had no path to "how do I script/automate against this
423
471
  // vault." Point at the guide rather than inlining it, to keep this brief
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openparachute/vault",
3
- "version": "0.7.2",
3
+ "version": "0.7.3-rc.10",
4
4
  "description": "Agent-native knowledge graph. Notes, tags, links over MCP.",
5
5
  "module": "src/cli.ts",
6
6
  "type": "module",
@@ -25,6 +25,7 @@
25
25
  "prepack": "bun run build:spa"
26
26
  },
27
27
  "dependencies": {
28
+ "@huggingface/transformers": "^4.2.0",
28
29
  "@modelcontextprotocol/sdk": "^1.12.1",
29
30
  "@openparachute/scope-guard": "^0.5.0",
30
31
  "jose": "^6.2.2",
@@ -34,6 +35,10 @@
34
35
  "devDependencies": {
35
36
  "@types/bun": "latest"
36
37
  },
38
+ "trustedDependencies": [
39
+ "onnxruntime-node",
40
+ "protobufjs"
41
+ ],
37
42
  "peerDependencies": {
38
43
  "typescript": "^5"
39
44
  },
@@ -140,3 +140,35 @@ describe("add-pack surface-starter", () => {
140
140
  expect(stdout).toContain("~ tag capture (upserted)");
141
141
  });
142
142
  });
143
+
144
+ describe("add-pack — tag description preservation on re-apply (Aaron-ratified 2026-07-17)", () => {
145
+ test("a hand-edited description survives a re-apply and is reported 'kept', not 'upserted'", () => {
146
+ expect(runCli(["create", "packed", "--json"], { PARACHUTE_HOME: home }).exitCode).toBe(0);
147
+
148
+ // Simulate an operator/AI hand-editing the capture tag's constitution.
149
+ const dbPath = join(home, "vault", "data", "packed", "vault.db");
150
+ const editedDescription = "Our house rules for capture — not the default text.";
151
+ const writeDb = new Database(dbPath);
152
+ writeDb.query("UPDATE tags SET description = ? WHERE name = ?").run(
153
+ editedDescription,
154
+ "capture",
155
+ );
156
+ writeDb.close();
157
+
158
+ const { exitCode, stdout } = runCli(
159
+ ["add-pack", "welcome", "--vault", "packed"],
160
+ { PARACHUTE_HOME: home },
161
+ );
162
+ expect(exitCode).toBe(0);
163
+ expect(stdout).toContain("~ tag capture (kept user description)");
164
+ // guide wasn't edited, so it still reports the old vocabulary.
165
+ expect(stdout).toContain("~ tag guide (upserted)");
166
+
167
+ const readDb = new Database(dbPath, { readonly: true });
168
+ const row = readDb
169
+ .query("SELECT description FROM tags WHERE name = ?")
170
+ .get("capture") as { description: string };
171
+ readDb.close();
172
+ expect(row.description).toBe(editedDescription);
173
+ });
174
+ });