@openparachute/vault 0.7.6 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +16 -16
  2. package/core/src/attachment/policy.test.ts +7 -0
  3. package/core/src/attachment/policy.ts +9 -0
  4. package/core/src/attachment-tickets-tool.test.ts +15 -0
  5. package/core/src/conformance.test.ts +78 -0
  6. package/core/src/conformance.ts +34 -5
  7. package/core/src/connection-pragmas.test.ts +27 -1
  8. package/core/src/core.test.ts +88 -3
  9. package/core/src/cursor.ts +2 -0
  10. package/core/src/do-param-cap.test.ts +167 -0
  11. package/core/src/lede.test.ts +60 -0
  12. package/core/src/mcp-manifest.ts +15 -1
  13. package/core/src/mcp.ts +37 -5
  14. package/core/src/notes.ts +120 -48
  15. package/core/src/query-operators.ts +87 -5
  16. package/core/src/query-warnings.ts +11 -3
  17. package/core/src/schema.ts +32 -0
  18. package/core/src/seed-packs.ts +74 -5
  19. package/core/src/sql-in.ts +32 -2
  20. package/core/src/store.ts +16 -49
  21. package/core/src/test-preload.ts +48 -3
  22. package/core/src/types.ts +13 -1
  23. package/core/src/wikilinks.test.ts +57 -0
  24. package/core/src/wikilinks.ts +63 -29
  25. package/package.json +1 -1
  26. package/src/attachment-tickets.test.ts +62 -0
  27. package/src/attachment-tickets.ts +2 -2
  28. package/src/cli.ts +36 -8
  29. package/src/config.ts +34 -1
  30. package/src/contract-honest-queries.test.ts +33 -1
  31. package/src/contract-search.test.ts +47 -0
  32. package/src/embedding/select.ts +16 -3
  33. package/src/live-match.test.ts +8 -0
  34. package/src/live-match.ts +15 -0
  35. package/src/mcp-http.test.ts +12 -0
  36. package/src/mcp-http.ts +1 -0
  37. package/src/mcp-tools.ts +51 -17
  38. package/src/mirror-routes.test.ts +22 -31
  39. package/src/onboarding-seed.test.ts +68 -0
  40. package/src/routes.ts +88 -25
  41. package/src/routing.test.ts +24 -0
  42. package/src/routing.ts +2 -0
  43. package/src/subscriptions.ts +18 -2
  44. package/src/tag-scope-note-tags.test.ts +476 -0
  45. package/src/tag-scope.ts +73 -5
  46. package/src/test-home-isolation.test.ts +137 -0
  47. package/src/test-support/spawn.ts +12 -0
  48. package/src/transcription/download.test.ts +187 -1
  49. package/src/transcription/download.ts +149 -2
  50. package/src/transcription/install-python.test.ts +23 -2
  51. package/src/transcription/install-python.ts +13 -4
  52. package/src/vault.test.ts +39 -4
  53. package/src/version.test.ts +8 -0
  54. package/src/ws-server.ts +12 -2
@@ -72,6 +72,66 @@ describe("computeLede", () => {
72
72
  });
73
73
  });
74
74
 
75
+ describe("markdown blocks that are not prose (vault#616)", () => {
76
+ it("skips a fenced code block and returns the first prose paragraph after it", () => {
77
+ expect(
78
+ computeLede("# Title\n\n```js\nconst x = 1;\n```\n\nThe real lede."),
79
+ ).toBe("The real lede.");
80
+ });
81
+
82
+ it("skips a fenced block that is not blank-line separated from the title", () => {
83
+ expect(computeLede("# Title\n```\nraw\n```\n\nThe real lede.")).toBe("The real lede.");
84
+ });
85
+
86
+ it("skips a tilde-fenced code block", () => {
87
+ expect(computeLede("# Title\n\n~~~\nraw\n~~~\n\nThe real lede.")).toBe("The real lede.");
88
+ });
89
+
90
+ it("skips a fence whose body contains blank lines", () => {
91
+ expect(
92
+ computeLede("# Title\n\n```\nline one\n\nline two\n```\n\nThe real lede."),
93
+ ).toBe("The real lede.");
94
+ });
95
+
96
+ it("returns null when an unterminated fence runs to end of content", () => {
97
+ expect(computeLede("# Title\n\n```js\nconst x = 1;")).toBeNull();
98
+ });
99
+
100
+ it("returns null when a code fence is all the content after the title", () => {
101
+ expect(computeLede("# Title\n\n```js\nconst x = 1;\n```")).toBeNull();
102
+ });
103
+
104
+ it("skips a heading-only block rather than returning its raw marker", () => {
105
+ expect(computeLede("# Title\n\n## Section\n\nThe real lede.")).toBe("The real lede.");
106
+ });
107
+
108
+ it("skips consecutive headings and fences until it finds prose", () => {
109
+ expect(
110
+ computeLede("# Title\n\n## Section\n\n### Sub\n\n```\ncode\n```\n\nThe real lede."),
111
+ ).toBe("The real lede.");
112
+ });
113
+
114
+ it("returns null when only headings follow the title", () => {
115
+ expect(computeLede("# Title\n\n## Section\n\n### Sub")).toBeNull();
116
+ });
117
+
118
+ it("keeps a paragraph whose FIRST line is a heading-marked line but has prose under it", () => {
119
+ // A heading immediately followed by prose with no blank line: markdown
120
+ // ends the heading at the newline, so the prose is the lede.
121
+ expect(computeLede("# Title\n\n## Section\nThe real lede.")).toBe("The real lede.");
122
+ });
123
+
124
+ it("skips a setext underline / thematic break block under the title", () => {
125
+ expect(computeLede("Title\n---\n\nThe real lede.")).toBe("The real lede.");
126
+ expect(computeLede("# Title\n\n***\n\nThe real lede.")).toBe("The real lede.");
127
+ expect(computeLede("Title\n===\n\nThe real lede.")).toBe("The real lede.");
128
+ });
129
+
130
+ it("does not mistake a prose line for a thematic break", () => {
131
+ expect(computeLede("# Title\n\n--- and then some prose.")).toBe("--- and then some prose.");
132
+ });
133
+ });
134
+
75
135
  describe("length cap", () => {
76
136
  it("truncates to LEDE_MAX_LEN code points", () => {
77
137
  const longParagraph = "a".repeat(1000);
@@ -51,7 +51,7 @@ export const MCP_TOOL_MANIFEST: readonly McpToolManifestEntry[] = [
51
51
  description: `Query notes. Returns notes matching the given filters.
52
52
 
53
53
  - **Single note**: pass \`id\` (accepts note ID, path, e.g., "Projects/README", or — as a last-resort fallback when id/path both miss cleanly and exactly one note matches — its H1 title, e.g. "Weekly Review")
54
- - **Filter**: pass \`tag\`, \`path\`, \`path_prefix\`, \`search\`, \`metadata\`, date range
54
+ - **Filter**: pass \`tag\`, \`path\`, \`path_prefix\`, \`exclude_path_prefix\`, \`search\`, \`metadata\`, date range
55
55
  - **Graph neighborhood**: pass \`near\` to scope results to notes within N hops of an anchor note
56
56
  - **No filters**: returns all notes (paginated)
57
57
 
@@ -123,6 +123,20 @@ Response shape (vault#550 — three variants, pick by what you passed):
123
123
  has_broken_links: { type: "boolean", description: "Presence filter (vault#555): true = only notes with at least one dangling outbound link — a [[wikilink]] or structured `links` target that never resolved to a note; false = only notes with none. Backed by the unresolved_wikilinks table (same data `doctor`/list-unresolved surfaces); safe on a vault where no link has ever gone unresolved (true matches nothing, false is a no-op)." },
124
124
  path: { type: "string", description: "Exact path match (case-insensitive)" },
125
125
  path_prefix: { type: "string", description: "Path prefix match (e.g., 'Projects/')" },
126
+ exclude_path_prefix: {
127
+ oneOf: [
128
+ { type: "string" },
129
+ { type: "array", items: { type: "string" } },
130
+ ],
131
+ description: "Exclude notes whose path matches any of these prefixes (vault#628). Same matching as `path_prefix`. Repeatable. A note with no path is not excluded. First client: `.parachute/` system-space. Alias `excludePathPrefix` is also accepted.",
132
+ },
133
+ excludePathPrefix: {
134
+ oneOf: [
135
+ { type: "string" },
136
+ { type: "array", items: { type: "string" } },
137
+ ],
138
+ description: "Alias for `exclude_path_prefix` (camelCase).",
139
+ },
126
140
  extension: {
127
141
  oneOf: [
128
142
  { type: "string" },
package/core/src/mcp.ts CHANGED
@@ -52,6 +52,7 @@ import {
52
52
  ATTACHMENT_MIME_TYPES,
53
53
  sanitizeAttachmentExtension,
54
54
  mimeForAttachmentExtension,
55
+ contentTypeForAttachmentPath,
55
56
  } from "./attachment/policy.js";
56
57
  import {
57
58
  computeTicketTtlMs,
@@ -818,6 +819,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
818
819
  hasBrokenLinks: params.has_broken_links as boolean | undefined,
819
820
  path: params.path as string | undefined,
820
821
  pathPrefix: params.path_prefix as string | undefined,
822
+ excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
821
823
  extension: params.extension as string | string[] | undefined,
822
824
  metadata: params.metadata as Record<string, unknown> | undefined,
823
825
  createdBy: params.created_by as string | undefined,
@@ -939,6 +941,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
939
941
  hasBrokenLinks: params.has_broken_links as boolean | undefined,
940
942
  path: params.path as string | undefined,
941
943
  pathPrefix: params.path_prefix as string | undefined,
944
+ excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
942
945
  extension: params.extension as string | string[] | undefined,
943
946
  // Same `near[]` neighborhood push-down `search`/structured-query
944
947
  // use — a semantic query can be scoped to a graph neighborhood too.
@@ -985,6 +988,8 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
985
988
  }
986
989
  // Normalize tag param
987
990
  const tags = normalizeTags(params.tag);
991
+ const excludeTagsRaw = params.exclude_tags ?? params.excludeTags ?? params.exclude_tag;
992
+ const excludeTags = normalizeTags(excludeTagsRaw);
988
993
  const mode: SearchMode = searchMode ?? "literal";
989
994
  // "Only whitespace/quotes" (vault#551 edge case): short-circuit
990
995
  // BEFORE ever calling FTS5 — an empty/all-punctuation phrase can
@@ -1004,10 +1009,35 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
1004
1009
  // `invalid_search_syntax`, vault#551) — uncaught on purpose, it
1005
1010
  // propagates to `src/mcp-http.ts`, which maps it to a JSON-RPC
1006
1011
  // error the same way it maps `invalid_query`.
1012
+ // vault#647: forward the same QueryOpts the structured /
1013
+ // semantic branches already pass. Pre-fix only tags/limit/
1014
+ // expand/mode/sort reached searchNotes, so exclude_tags and
1015
+ // date_from were silently dropped (a well-formed result set
1016
+ // answering a different question).
1007
1017
  results = await store.searchNotes(params.search as string, {
1008
1018
  tags,
1009
- limit: (params.limit as number) ?? 50,
1019
+ tagMatch: (params.tag_match as "all" | "any") ?? (tags && tags.length > 1 ? "any" : undefined),
1010
1020
  expand,
1021
+ excludeTags,
1022
+ hasTags: params.has_tags as boolean | undefined,
1023
+ hasLinks: params.has_links as boolean | undefined,
1024
+ hasBrokenLinks: params.has_broken_links as boolean | undefined,
1025
+ path: params.path as string | undefined,
1026
+ pathPrefix: params.path_prefix as string | undefined,
1027
+ excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
1028
+ extension: params.extension as string | string[] | undefined,
1029
+ ids: nearScope ? [...nearScope] : undefined,
1030
+ metadata: params.metadata as Record<string, unknown> | undefined,
1031
+ createdBy: params.created_by as string | undefined,
1032
+ lastUpdatedBy: params.last_updated_by as string | undefined,
1033
+ createdVia: params.created_via as string | undefined,
1034
+ lastUpdatedVia: params.last_updated_via as string | undefined,
1035
+ dateFrom: params.date_from as string | undefined,
1036
+ dateTo: params.date_to as string | undefined,
1037
+ dateFilter: params.date_filter as
1038
+ | { field?: string; from?: string; to?: string }
1039
+ | undefined,
1040
+ limit: (params.limit as number) ?? 50,
1011
1041
  mode,
1012
1042
  sort: params.sort as "asc" | "desc" | undefined,
1013
1043
  });
@@ -1063,6 +1093,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
1063
1093
  hasBrokenLinks: params.has_broken_links as boolean | undefined,
1064
1094
  path: params.path as string | undefined,
1065
1095
  pathPrefix: params.path_prefix as string | undefined,
1096
+ excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
1066
1097
  extension: params.extension as string | string[] | undefined,
1067
1098
  // Push the near-scope into the SQL WHERE so that LIMIT and ORDER
1068
1099
  // BY apply to the neighborhood. Without this, queryNotes would
@@ -1103,8 +1134,8 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
1103
1134
  // requested, so `next_cursor` never surfaces as an honest
1104
1135
  // "more may follow" signal — this is that signal. Mirrors the
1105
1136
  // REST structured-query path in src/routes.ts.
1106
- if (results.length === queryOpts.limit) {
1107
- queryWarnings.push(truncatedResultsWarning(queryOpts.limit));
1137
+ if (queryOpts.offset === undefined && results.length === queryOpts.limit) {
1138
+ queryWarnings.push(truncatedResultsWarning(queryOpts.limit, "mcp"));
1108
1139
  }
1109
1140
  }
1110
1141
  }
@@ -2632,6 +2663,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
2632
2663
  const now = Date.now();
2633
2664
  const expiresAt = now + computeTicketTtlMs(declaredSize);
2634
2665
  const id = generateTicketId();
2666
+ const downloadMime = contentTypeForAttachmentPath(attachment.path);
2635
2667
  const ticket: AttachmentTicket = {
2636
2668
  id,
2637
2669
  kind: "download",
@@ -2639,7 +2671,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
2639
2671
  createdAt: now,
2640
2672
  expiresAt,
2641
2673
  attachmentId: attachment.id,
2642
- mimeType: attachment.mimeType,
2674
+ mimeType: downloadMime,
2643
2675
  sizeBytes: declaredSize,
2644
2676
  };
2645
2677
  await ticketSeam.provider.put(ticket);
@@ -2648,7 +2680,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
2648
2680
  return {
2649
2681
  method: "GET",
2650
2682
  url,
2651
- mime_type: attachment.mimeType,
2683
+ mime_type: downloadMime,
2652
2684
  ...(declaredSize !== undefined ? { size_bytes: declaredSize } : {}),
2653
2685
  expires_at: new Date(expiresAt).toISOString(),
2654
2686
  curl_example: `curl -o downloaded${sanitizeAttachmentExtension(attachment.path) || ""} '${url}'`,
package/core/src/notes.ts CHANGED
@@ -1102,6 +1102,18 @@ export function buildFilterConditions(db: Database, opts: QueryOpts): { conditio
1102
1102
  params.push(opts.pathPrefix + "%");
1103
1103
  }
1104
1104
 
1105
+ // Path-prefix exclusion (vault#628). Mirrors `pathPrefix` matching
1106
+ // (`LIKE prefix || '%'`, SQLite LIKE is ASCII-case-insensitive). NULL
1107
+ // paths are kept — they are not under the prefix. Repeatable: a note
1108
+ // matching ANY listed prefix is dropped.
1109
+ if (opts.excludePathPrefix && opts.excludePathPrefix.length > 0) {
1110
+ for (const prefix of opts.excludePathPrefix) {
1111
+ if (typeof prefix !== "string" || prefix.length === 0) continue;
1112
+ conditions.push("(n.path IS NULL OR n.path NOT LIKE ?)");
1113
+ params.push(prefix + "%");
1114
+ }
1115
+ }
1116
+
1105
1117
  // Extension filter (vault#328). Single string → exact match; array → IN
1106
1118
  // clause. Compared lower-case so a caller passing "CSV" still hits rows
1107
1119
  // stored as "csv". An empty array is a no-op (no filter applied) rather
@@ -1165,10 +1177,11 @@ export function buildFilterConditions(db: Database, opts: QueryOpts): { conditio
1165
1177
  if (opts.metadata) {
1166
1178
  for (const [key, value] of Object.entries(opts.metadata)) {
1167
1179
  if (isOperatorObject(value)) {
1168
- requireIndexedField(db, key);
1180
+ const indexedField = requireIndexedField(db, key);
1169
1181
  const { sql, params: opParams } = buildOperatorClause(
1170
1182
  key,
1171
1183
  value as Record<string, unknown>,
1184
+ indexedField.sqliteType,
1172
1185
  );
1173
1186
  conditions.push(sql);
1174
1187
  params.push(...opParams);
@@ -1656,6 +1669,7 @@ function toQueryHashInputs(opts: QueryOpts): QueryHashInputs {
1656
1669
  hasBrokenLinks: opts.hasBrokenLinks,
1657
1670
  path: opts.path,
1658
1671
  pathPrefix: opts.pathPrefix,
1672
+ excludePathPrefix: opts.excludePathPrefix,
1659
1673
  extension: opts.extension,
1660
1674
  ids: opts.ids,
1661
1675
  metadata: opts.metadata,
@@ -1869,7 +1883,7 @@ function applySearchTitleBoost(
1869
1883
  export function searchNotes(
1870
1884
  db: Database,
1871
1885
  query: string,
1872
- opts?: { tags?: string[]; limit?: number; mode?: SearchMode; sort?: "asc" | "desc" },
1886
+ opts?: QueryOpts & { mode?: SearchMode },
1873
1887
  ): Note[] {
1874
1888
  const limit = typeof opts?.limit === "number" ? opts.limit : 50;
1875
1889
  // Literal-by-default (vault#551): escape the caller's text so FTS5's own
@@ -1929,51 +1943,37 @@ export function searchNotes(
1929
1943
  ? "n.created_at DESC, n.id DESC"
1930
1944
  : "score DESC, n.id ASC";
1931
1945
 
1932
- if (opts?.tags && opts.tags.length > 0) {
1933
- // Canonical-bare-tag guard backstop (vault#XXX) for direct-core callers.
1934
- const searchTags = opts.tags.map(stripTagHash).filter((t) => t !== "");
1935
- if (searchTags.length === 0) {
1936
- // All tag filters collapsed to empty fall through to the untagged
1937
- // search path below (no tag constraint).
1938
- opts = { ...opts, tags: undefined };
1939
- } else {
1940
- try {
1941
- // Tag membership as a semijoin same rationale as queryNotes: a
1942
- // `JOIN note_tags` multiplies rows for multi-tagged notes and forced
1943
- // DISTINCT over full rows. The FTS join itself is 1:1 on rowid.
1944
- const tagPlaceholders = searchTags.map(() => "?").join(", ");
1945
- const rows = db.prepare(`
1946
- SELECT n.*, ${scoreExpr} AS score FROM notes n
1947
- JOIN notes_fts fts ON fts.rowid = n.rowid
1948
- WHERE notes_fts MATCH ?
1949
- AND n.id IN (SELECT note_id FROM note_tags WHERE tag_name IN (${tagPlaceholders}))
1950
- ORDER BY ${orderBy}
1951
- LIMIT ?
1952
- `).all(ftsQuery, ...searchTags, limit) as (NoteRow & { score: number })[];
1953
- return applySearchTitleBoost(notesWithTags(db, rows, scoresById(rows)), mode, query, opts?.sort);
1954
- } catch (err) {
1955
- // Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
1956
- // advanced mode expects it (the caller passed raw syntax); literal
1957
- // mode should never reach it (escaping + control-char sanitization
1958
- // make the query valid) but if the invariant breaks we still want an
1959
- // honest error, not an unstructured 500. A QueryError we threw
1960
- // ourselves (empty-limit validation, etc.) is re-raised untouched.
1961
- if (err instanceof QueryError) throw err;
1962
- throw searchSyntaxError(query, err, mode);
1963
- }
1964
- }
1965
- }
1946
+ // vault#647: compose FTS with the SAME filter builder queryNotes /
1947
+ // semanticSearch use (excludeTags, dateFrom/dateFilter, path, metadata,
1948
+ // …). Pre-fix only `tags` reached the WHERE clause; every other filter
1949
+ // was silently dropped. Historical FTS tag semantics are "any tag
1950
+ // matches" (a single IN (...)); preserve that when the caller didn't
1951
+ // set `tagMatch`. LIMIT still applies AFTER the filters, not to an
1952
+ // unfiltered FTS page.
1953
+ const filterOpts: QueryOpts = {
1954
+ ...(opts ?? {}),
1955
+ tagMatch: opts?.tagMatch ?? (opts?.tags && opts.tags.length > 0 ? "any" : undefined),
1956
+ };
1957
+ const { conditions, params } = buildFilterConditions(db, filterOpts);
1958
+ const extraWhere = conditions.length > 0 ? `AND ${conditions.join(" AND ")}` : "";
1966
1959
 
1967
1960
  try {
1968
1961
  const rows = db.prepare(`
1969
1962
  SELECT n.*, ${scoreExpr} AS score FROM notes n
1970
1963
  JOIN notes_fts fts ON fts.rowid = n.rowid
1971
1964
  WHERE notes_fts MATCH ?
1965
+ ${extraWhere}
1972
1966
  ORDER BY ${orderBy}
1973
1967
  LIMIT ?
1974
- `).all(ftsQuery, limit) as (NoteRow & { score: number })[];
1968
+ `).all(ftsQuery, ...params, limit) as (NoteRow & { score: number })[];
1975
1969
  return applySearchTitleBoost(notesWithTags(db, rows, scoresById(rows)), mode, query, opts?.sort);
1976
1970
  } catch (err) {
1971
+ // Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
1972
+ // advanced mode expects it (the caller passed raw syntax); literal
1973
+ // mode should never reach it (escaping + control-char sanitization
1974
+ // make the query valid) but if the invariant breaks we still want an
1975
+ // honest error, not an unstructured 500. A QueryError we threw
1976
+ // ourselves (empty-limit validation, etc.) is re-raised untouched.
1977
1977
  if (err instanceof QueryError) throw err;
1978
1978
  throw searchSyntaxError(query, err, mode);
1979
1979
  }
@@ -2840,9 +2840,26 @@ export function computeDisplayTitle(content: string | null | undefined): string
2840
2840
  export const LEDE_MAX_LEN = 400;
2841
2841
 
2842
2842
  /**
2843
- * Derive a note's "lede": the first non-empty PARAGRAPH after the title
2843
+ * Opens (or closes) a fenced code block: three or more backticks or tildes.
2844
+ * Capture 1 is the run itself so the scanner can require the CLOSING fence to
2845
+ * use the same character and be at least as long, per CommonMark.
2846
+ */
2847
+ const FENCE_OPEN_RE = /^(`{3,}|~{3,})/;
2848
+ /** A closing fence carries nothing but the run (an info string opens, never closes). */
2849
+ const FENCE_CLOSE_RE = /^(`{3,}|~{3,})\s*$/;
2850
+ /** An ATX heading line: 1-6 `#` followed by whitespace or end-of-line. */
2851
+ const HEADING_LINE_RE = /^#{1,6}(?:\s|$)/;
2852
+ /**
2853
+ * A thematic break (`---`, `***`, `___`) or a setext heading underline
2854
+ * (`---`, `===`). Both are pure markup with no text of their own — the
2855
+ * setext case is why `Title\n---\n\nreal text` used to yield a `---` lede.
2856
+ */
2857
+ const BREAK_LINE_RE = /^(?:-{3,}|\*{3,}|_{3,}|={3,})$/;
2858
+
2859
+ /**
2860
+ * Derive a note's "lede": the first non-empty PROSE PARAGRAPH after the title
2844
2861
  * line (a run of consecutive non-blank lines, whitespace-collapsed to one
2845
- * line, truncated to `LEDE_MAX_LEN` code points). `null` when there's no
2862
+ * line, truncated to `LEDE_MAX_LEN` code points). `null` when there's no such
2846
2863
  * paragraph after the title — a title-only note has no lede to report, and
2847
2864
  * callers must not fall back to repeating the title itself.
2848
2865
  *
@@ -2854,6 +2871,22 @@ export const LEDE_MAX_LEN = 400;
2854
2871
  * what it finds, honestly, using the SAME title-line rule as
2855
2872
  * `computeDisplayTitle` (including its frontmatter skip) so a caller that
2856
2873
  * shows both title and lede sees them agree on where the title ends.
2874
+ *
2875
+ * Non-prose blocks are SKIPPED (vault#616). A note that opens with a code
2876
+ * fence, a section heading, or a horizontal rule used to have that raw markup
2877
+ * returned as its lede — backticks, `## ` markers and all — which contradicted
2878
+ * this function's own "first paragraph" contract and read as garbage in
2879
+ * `expand_mode: "summary"`. The scan now walks past fenced blocks (to the
2880
+ * matching closing fence, or to end-of-content when the fence is never
2881
+ * closed), heading lines, and break/underline lines until it reaches real
2882
+ * text, and returns `null` if it never does. Those same three shapes also
2883
+ * TERMINATE a paragraph already in progress, matching how markdown lets a
2884
+ * fence or heading interrupt a paragraph.
2885
+ *
2886
+ * Deliberately NOT normalized: list blocks. A `- milk` list is genuine prose
2887
+ * content, and how to flatten it into one line (drop the markers? join with
2888
+ * what?) is a rendering decision this function shouldn't make silently — it
2889
+ * reports the list as written.
2857
2890
  */
2858
2891
  export function computeLede(content: string | null | undefined): string | null {
2859
2892
  if (!content) return null;
@@ -2870,14 +2903,51 @@ export function computeLede(content: string | null | undefined): string | null {
2870
2903
  if (titleLine === -1) return null; // no title at all — nothing to find a lede after
2871
2904
 
2872
2905
  let i = titleLine + 1;
2873
- while (i < lines.length && lines[i]!.trim() === "") i++; // skip blank lines after the title
2874
-
2875
2906
  const paragraphLines: string[] = [];
2876
- while (i < lines.length && lines[i]!.trim() !== "") {
2877
- paragraphLines.push(lines[i]!);
2878
- i++;
2907
+
2908
+ // Walk blocks until one of them is prose. Each iteration consumes exactly
2909
+ // one block (blank run, fence, heading, break, or paragraph), so `i` always
2910
+ // advances and the loop terminates.
2911
+ scan: while (i < lines.length) {
2912
+ const trimmed = lines[i]!.trim();
2913
+
2914
+ if (trimmed === "") {
2915
+ i++;
2916
+ continue;
2917
+ }
2918
+
2919
+ const opening = FENCE_OPEN_RE.exec(trimmed);
2920
+ if (opening) {
2921
+ const marker = opening[1]!;
2922
+ i++;
2923
+ while (i < lines.length) {
2924
+ const closing = FENCE_CLOSE_RE.exec(lines[i]!.trim());
2925
+ i++;
2926
+ if (closing && closing[1]![0] === marker[0] && closing[1]!.length >= marker.length) break;
2927
+ }
2928
+ continue; // an unclosed fence runs off the end and leaves i past the last line
2929
+ }
2930
+
2931
+ if (HEADING_LINE_RE.test(trimmed) || BREAK_LINE_RE.test(trimmed)) {
2932
+ i++;
2933
+ continue;
2934
+ }
2935
+
2936
+ // Real text — gather the paragraph, stopping at a blank line or at any
2937
+ // block-level marker that interrupts it.
2938
+ while (i < lines.length) {
2939
+ const line = lines[i]!;
2940
+ const t = line.trim();
2941
+ if (t === "" || FENCE_OPEN_RE.test(t) || HEADING_LINE_RE.test(t) || BREAK_LINE_RE.test(t)) {
2942
+ break scan;
2943
+ }
2944
+ paragraphLines.push(line);
2945
+ i++;
2946
+ }
2947
+ break;
2879
2948
  }
2880
- if (paragraphLines.length === 0) return null; // title-only note
2949
+
2950
+ if (paragraphLines.length === 0) return null; // title-only note, or nothing but markup
2881
2951
 
2882
2952
  const paragraph = paragraphLines.join(" ").replace(/\s+/g, " ").trim();
2883
2953
  if (paragraph === "") return null;
@@ -2938,8 +3008,10 @@ export function toNoteIndex(note: Note): NoteIndex {
2938
3008
  /**
2939
3009
  * Filter metadata on a note/index result based on an include_metadata param.
2940
3010
  * - true / undefined → return as-is (all metadata)
2941
- * - false → strip metadata entirely
2942
- * - string[] → return only those keys (empty array = no filtering)
3011
+ * - false → strip metadata entirely (key omitted — caller asked for none)
3012
+ * - string[] → return only those keys. Empty array = no filtering (all keys).
3013
+ * A non-empty array whose keys miss still keeps `metadata: {}` (vault#600
3014
+ * V1.1 invariant — the key must not vanish from the wire).
2943
3015
  */
2944
3016
  export function filterMetadata(obj: any, includeMetadata: boolean | string[] | undefined): any {
2945
3017
  if (includeMetadata === undefined || includeMetadata === true) return obj;
@@ -2953,7 +3025,7 @@ export function filterMetadata(obj: any, includeMetadata: boolean | string[] | u
2953
3025
  const filtered = Object.fromEntries(
2954
3026
  Object.entries(obj.metadata).filter(([k]) => fields.includes(k)),
2955
3027
  );
2956
- return { ...obj, metadata: Object.keys(filtered).length > 0 ? filtered : undefined };
3028
+ return { ...obj, metadata: filtered };
2957
3029
  }
2958
3030
 
2959
3031
  /**
@@ -16,7 +16,8 @@
16
16
  */
17
17
 
18
18
  import { Database, type SQLQueryBindings } from "bun:sqlite";
19
- import { getIndexedField, type IndexedField } from "./indexed-fields.js";
19
+ import { getIndexedField, type IndexedField, type SqliteType } from "./indexed-fields.js";
20
+ import { inViaJsonEachCast, jsonEachParam } from "./sql-in.js";
20
21
 
21
22
  export const SUPPORTED_OPS = [
22
23
  "eq",
@@ -101,6 +102,52 @@ function toBinding(field: string, op: string, value: unknown): SQLQueryBindings
101
102
  );
102
103
  }
103
104
 
105
+ /**
106
+ * Narrow one `in`/`not_in` element to something `JSON.stringify` can carry
107
+ * into {@link jsonEachParam} (vault#536).
108
+ *
109
+ * Runs `toBinding` first so the primitive-only validation and its error
110
+ * message are unchanged from the one-param-per-value era. Then two shapes
111
+ * need explicit handling, because JSON has no equivalent of what bun:sqlite
112
+ * did implicitly when the value was bound directly:
113
+ *
114
+ * - `boolean` → `1`/`0`, matching the INTEGER a boolean is stored as, so
115
+ * `{ in: [true] }` still matches rows written with `true`. A raw JSON
116
+ * `true` would come out of `json_each` as `1` anyway on current SQLite,
117
+ * but converting here makes the intent explicit rather than dependent on
118
+ * that.
119
+ * - `bigint` → `JSON.stringify` THROWS on it ("Do not know how to serialize
120
+ * a BigInt"). Converted when it fits exactly in a double; otherwise
121
+ * rejected with a real error instead of a crash or a silently wrong
122
+ * comparison. (Unreachable from MCP/REST — `JSON.parse` never yields a
123
+ * bigint — so this only guards a direct in-process caller.)
124
+ */
125
+ function toJsonEachMember(
126
+ field: string,
127
+ op: string,
128
+ value: unknown,
129
+ ): string | number | boolean | null {
130
+ const bound = toBinding(field, op, value);
131
+ if (typeof bound === "boolean") return bound ? 1 : 0;
132
+ if (typeof bound === "bigint") {
133
+ if (bound >= BigInt(Number.MIN_SAFE_INTEGER) && bound <= BigInt(Number.MAX_SAFE_INTEGER)) {
134
+ return Number(bound);
135
+ }
136
+ throw new QueryError(
137
+ `operator "${op}" on metadata field "${field}" got a bigint outside the safe integer range (${bound}); it cannot be compared exactly`,
138
+ "INVALID_OPERATOR_VALUE",
139
+ );
140
+ }
141
+ // `toBinding` is declared as the full `SQLQueryBindings` union (which
142
+ // includes TypedArray/Record shapes it never actually returns), so narrow
143
+ // to what JSON can carry rather than asserting.
144
+ if (bound === null || typeof bound === "string" || typeof bound === "number") return bound;
145
+ throw new QueryError(
146
+ `operator "${op}" on metadata field "${field}" expects a primitive value (string, number, boolean, bigint, or null)`,
147
+ "INVALID_OPERATOR_VALUE",
148
+ );
149
+ }
150
+
104
151
  /**
105
152
  * Best-effort lookup of a metadata field's declared `type` across every tag
106
153
  * schema — used ONLY to sharpen the {@link requireIndexedField} error hint
@@ -176,10 +223,19 @@ export function requireIndexedField(db: Database, field: string): IndexedField {
176
223
  * Build a SQL fragment + bound params for an operator object on an indexed
177
224
  * metadata field. Each operator maps to a single AND clause; an object like
178
225
  * `{ gt: 5, lt: 10 }` composes as `meta_<field> > 5 AND meta_<field> < 10`.
226
+ *
227
+ * `sqliteType` is the field's declared column storage type (from
228
+ * `requireIndexedField(...).sqliteType`, which every caller already
229
+ * resolves). It's needed for `in`/`not_in`: those bind the value-set through
230
+ * a `json_each` subquery, which — unlike a placeholder `IN (?, …)` list —
231
+ * does NOT apply the left column's type affinity to each candidate value, so
232
+ * the set must be CAST to the column's type to keep cross-type matches like
233
+ * numeric `5` against a TEXT-affinity `'5'` (vault#676).
179
234
  */
180
235
  export function buildOperatorClause(
181
236
  field: string,
182
237
  opObj: Record<string, unknown>,
238
+ sqliteType: SqliteType,
183
239
  ): { sql: string; params: SQLQueryBindings[] } {
184
240
  validateOperatorObject(field, opObj);
185
241
  // `field` came from indexed_fields (which validated it via FIELD_NAME_RE
@@ -235,13 +291,39 @@ export function buildOperatorClause(
235
291
  parts.push(op === "in" ? "0" : "1");
236
292
  break;
237
293
  }
238
- const placeholders = value.map(() => "?").join(", ");
294
+ // ONE bound param for the whole set, regardless of its size
295
+ // (vault#536). This used to emit `IN (?, ?, …)` — one param per
296
+ // value — which on Cloudflare Durable Object SQLite (100 bound
297
+ // params per statement, vs bun:sqlite's 999+) turned any `in` array
298
+ // over ~100 elements into a `too many SQL variables` 500 on cloud
299
+ // vaults, while self-host never noticed.
300
+ //
301
+ // This is an EMBEDDED filter inside a paginated statement, so the
302
+ // chunk-and-union approach used for standalone id-lists isn't
303
+ // available — chunking would break the shared LIMIT/OFFSET window.
304
+ // The json_each shape #535 introduced for the `near` neighborhood is
305
+ // the one that works here: bind the array as a single JSON param and
306
+ // let SQLite expand it into a one-column table. See sql-in.ts.
307
+ //
308
+ // Every element still goes through `toBinding`, so the same
309
+ // primitive-only validation (and the same error message) applies as
310
+ // before — the values are just serialized into JSON instead of bound
311
+ // one by one.
312
+ //
313
+ // The set is CAST to the column's declared storage type inside the
314
+ // subquery (`inViaJsonEachCast`). A placeholder `IN (?, …)` list got
315
+ // the left column's affinity applied to each value for free; a
316
+ // `json_each` subquery does not, so without the CAST a numeric `5`
317
+ // silently stopped matching a TEXT-affinity `'5'` (vault#676). The
318
+ // CAST restores that in both directions and covers `in` and `not_in`.
319
+ const members = value.map((v) => toJsonEachMember(field, op, v));
320
+ const inClause = inViaJsonEachCast(sqliteType);
239
321
  if (op === "in") {
240
- parts.push(`${col} IN (${placeholders})`);
322
+ parts.push(`${col} IN ${inClause}`);
241
323
  } else {
242
- parts.push(`(${col} IS NULL OR ${col} NOT IN (${placeholders}))`);
324
+ parts.push(`(${col} IS NULL OR ${col} NOT IN ${inClause})`);
243
325
  }
244
- for (const v of value) params.push(toBinding(field, op, v));
326
+ params.push(jsonEachParam(members));
245
327
  break;
246
328
  }
247
329
  case "exists":
@@ -229,12 +229,20 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
229
229
  * /api/notes` on a >limit-note vault returns the OLDEST rows with zero
230
230
  * signal that the NEWEST ones didn't make it. `?cursor=` (bootstrap or
231
231
  * watermark) sidesteps this entirely — the warning only fires in its
232
- * absence.
232
+ * absence. Explicit `offset` is also exempt (vault#601): offset-paging is
233
+ * deliberate pagination, not an accidental full default page.
233
234
  */
234
- export function truncatedResultsWarning(limit: number): QueryWarning {
235
+ export function truncatedResultsWarning(
236
+ limit: number,
237
+ channel: "rest" | "mcp" = "rest",
238
+ ): QueryWarning {
239
+ const pageHint =
240
+ channel === "mcp"
241
+ ? `Pass cursor: "" to page, or sort: "desc" for newest-first.`
242
+ : "Pass ?cursor= to page, or sort=desc for newest-first.";
235
243
  return {
236
244
  code: "truncated",
237
- message: `${limit} = limit rows returned; there may be more. Pass ?cursor= to page, or sort=desc for newest-first.`,
245
+ message: `${limit} = limit rows returned; there may be more. ${pageHint}`,
238
246
  limit,
239
247
  };
240
248
  }