@openparachute/vault 0.7.6 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -16
- package/core/src/attachment/policy.test.ts +7 -0
- package/core/src/attachment/policy.ts +9 -0
- package/core/src/attachment-tickets-tool.test.ts +15 -0
- package/core/src/conformance.test.ts +78 -0
- package/core/src/conformance.ts +34 -5
- package/core/src/connection-pragmas.test.ts +27 -1
- package/core/src/core.test.ts +88 -3
- package/core/src/cursor.ts +2 -0
- package/core/src/do-param-cap.test.ts +167 -0
- package/core/src/lede.test.ts +60 -0
- package/core/src/mcp-manifest.ts +15 -1
- package/core/src/mcp.ts +37 -5
- package/core/src/notes.ts +120 -48
- package/core/src/query-operators.ts +87 -5
- package/core/src/query-warnings.ts +11 -3
- package/core/src/schema.ts +32 -0
- package/core/src/seed-packs.ts +74 -5
- package/core/src/sql-in.ts +32 -2
- package/core/src/store.ts +16 -49
- package/core/src/test-preload.ts +48 -3
- package/core/src/types.ts +13 -1
- package/core/src/wikilinks.test.ts +57 -0
- package/core/src/wikilinks.ts +63 -29
- package/package.json +1 -1
- package/src/attachment-tickets.test.ts +62 -0
- package/src/attachment-tickets.ts +2 -2
- package/src/cli.ts +36 -8
- package/src/config.ts +34 -1
- package/src/contract-honest-queries.test.ts +33 -1
- package/src/contract-search.test.ts +47 -0
- package/src/embedding/select.ts +16 -3
- package/src/live-match.test.ts +8 -0
- package/src/live-match.ts +15 -0
- package/src/mcp-http.test.ts +12 -0
- package/src/mcp-http.ts +1 -0
- package/src/mcp-tools.ts +51 -17
- package/src/mirror-routes.test.ts +22 -31
- package/src/onboarding-seed.test.ts +68 -0
- package/src/routes.ts +88 -25
- package/src/routing.test.ts +24 -0
- package/src/routing.ts +2 -0
- package/src/subscriptions.ts +18 -2
- package/src/tag-scope-note-tags.test.ts +476 -0
- package/src/tag-scope.ts +73 -5
- package/src/test-home-isolation.test.ts +137 -0
- package/src/test-support/spawn.ts +12 -0
- package/src/transcription/download.test.ts +187 -1
- package/src/transcription/download.ts +149 -2
- package/src/transcription/install-python.test.ts +23 -2
- package/src/transcription/install-python.ts +13 -4
- package/src/vault.test.ts +39 -4
- package/src/version.test.ts +8 -0
- package/src/ws-server.ts +12 -2
package/core/src/lede.test.ts
CHANGED
|
@@ -72,6 +72,66 @@ describe("computeLede", () => {
|
|
|
72
72
|
});
|
|
73
73
|
});
|
|
74
74
|
|
|
75
|
+
describe("markdown blocks that are not prose (vault#616)", () => {
|
|
76
|
+
it("skips a fenced code block and returns the first prose paragraph after it", () => {
|
|
77
|
+
expect(
|
|
78
|
+
computeLede("# Title\n\n```js\nconst x = 1;\n```\n\nThe real lede."),
|
|
79
|
+
).toBe("The real lede.");
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("skips a fenced block that is not blank-line separated from the title", () => {
|
|
83
|
+
expect(computeLede("# Title\n```\nraw\n```\n\nThe real lede.")).toBe("The real lede.");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("skips a tilde-fenced code block", () => {
|
|
87
|
+
expect(computeLede("# Title\n\n~~~\nraw\n~~~\n\nThe real lede.")).toBe("The real lede.");
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("skips a fence whose body contains blank lines", () => {
|
|
91
|
+
expect(
|
|
92
|
+
computeLede("# Title\n\n```\nline one\n\nline two\n```\n\nThe real lede."),
|
|
93
|
+
).toBe("The real lede.");
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("returns null when an unterminated fence runs to end of content", () => {
|
|
97
|
+
expect(computeLede("# Title\n\n```js\nconst x = 1;")).toBeNull();
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("returns null when a code fence is all the content after the title", () => {
|
|
101
|
+
expect(computeLede("# Title\n\n```js\nconst x = 1;\n```")).toBeNull();
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("skips a heading-only block rather than returning its raw marker", () => {
|
|
105
|
+
expect(computeLede("# Title\n\n## Section\n\nThe real lede.")).toBe("The real lede.");
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("skips consecutive headings and fences until it finds prose", () => {
|
|
109
|
+
expect(
|
|
110
|
+
computeLede("# Title\n\n## Section\n\n### Sub\n\n```\ncode\n```\n\nThe real lede."),
|
|
111
|
+
).toBe("The real lede.");
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("returns null when only headings follow the title", () => {
|
|
115
|
+
expect(computeLede("# Title\n\n## Section\n\n### Sub")).toBeNull();
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it("keeps a paragraph whose FIRST line is a heading-marked line but has prose under it", () => {
|
|
119
|
+
// A heading immediately followed by prose with no blank line: markdown
|
|
120
|
+
// ends the heading at the newline, so the prose is the lede.
|
|
121
|
+
expect(computeLede("# Title\n\n## Section\nThe real lede.")).toBe("The real lede.");
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it("skips a setext underline / thematic break block under the title", () => {
|
|
125
|
+
expect(computeLede("Title\n---\n\nThe real lede.")).toBe("The real lede.");
|
|
126
|
+
expect(computeLede("# Title\n\n***\n\nThe real lede.")).toBe("The real lede.");
|
|
127
|
+
expect(computeLede("Title\n===\n\nThe real lede.")).toBe("The real lede.");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("does not mistake a prose line for a thematic break", () => {
|
|
131
|
+
expect(computeLede("# Title\n\n--- and then some prose.")).toBe("--- and then some prose.");
|
|
132
|
+
});
|
|
133
|
+
});
|
|
134
|
+
|
|
75
135
|
describe("length cap", () => {
|
|
76
136
|
it("truncates to LEDE_MAX_LEN code points", () => {
|
|
77
137
|
const longParagraph = "a".repeat(1000);
|
package/core/src/mcp-manifest.ts
CHANGED
|
@@ -51,7 +51,7 @@ export const MCP_TOOL_MANIFEST: readonly McpToolManifestEntry[] = [
|
|
|
51
51
|
description: `Query notes. Returns notes matching the given filters.
|
|
52
52
|
|
|
53
53
|
- **Single note**: pass \`id\` (accepts note ID, path, e.g., "Projects/README", or — as a last-resort fallback when id/path both miss cleanly and exactly one note matches — its H1 title, e.g. "Weekly Review")
|
|
54
|
-
- **Filter**: pass \`tag\`, \`path\`, \`path_prefix\`, \`search\`, \`metadata\`, date range
|
|
54
|
+
- **Filter**: pass \`tag\`, \`path\`, \`path_prefix\`, \`exclude_path_prefix\`, \`search\`, \`metadata\`, date range
|
|
55
55
|
- **Graph neighborhood**: pass \`near\` to scope results to notes within N hops of an anchor note
|
|
56
56
|
- **No filters**: returns all notes (paginated)
|
|
57
57
|
|
|
@@ -123,6 +123,20 @@ Response shape (vault#550 — three variants, pick by what you passed):
|
|
|
123
123
|
has_broken_links: { type: "boolean", description: "Presence filter (vault#555): true = only notes with at least one dangling outbound link — a [[wikilink]] or structured `links` target that never resolved to a note; false = only notes with none. Backed by the unresolved_wikilinks table (same data `doctor`/list-unresolved surfaces); safe on a vault where no link has ever gone unresolved (true matches nothing, false is a no-op)." },
|
|
124
124
|
path: { type: "string", description: "Exact path match (case-insensitive)" },
|
|
125
125
|
path_prefix: { type: "string", description: "Path prefix match (e.g., 'Projects/')" },
|
|
126
|
+
exclude_path_prefix: {
|
|
127
|
+
oneOf: [
|
|
128
|
+
{ type: "string" },
|
|
129
|
+
{ type: "array", items: { type: "string" } },
|
|
130
|
+
],
|
|
131
|
+
description: "Exclude notes whose path matches any of these prefixes (vault#628). Same matching as `path_prefix`. Repeatable. A note with no path is not excluded. First client: `.parachute/` system-space. Alias `excludePathPrefix` is also accepted.",
|
|
132
|
+
},
|
|
133
|
+
excludePathPrefix: {
|
|
134
|
+
oneOf: [
|
|
135
|
+
{ type: "string" },
|
|
136
|
+
{ type: "array", items: { type: "string" } },
|
|
137
|
+
],
|
|
138
|
+
description: "Alias for `exclude_path_prefix` (camelCase).",
|
|
139
|
+
},
|
|
126
140
|
extension: {
|
|
127
141
|
oneOf: [
|
|
128
142
|
{ type: "string" },
|
package/core/src/mcp.ts
CHANGED
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
ATTACHMENT_MIME_TYPES,
|
|
53
53
|
sanitizeAttachmentExtension,
|
|
54
54
|
mimeForAttachmentExtension,
|
|
55
|
+
contentTypeForAttachmentPath,
|
|
55
56
|
} from "./attachment/policy.js";
|
|
56
57
|
import {
|
|
57
58
|
computeTicketTtlMs,
|
|
@@ -818,6 +819,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
818
819
|
hasBrokenLinks: params.has_broken_links as boolean | undefined,
|
|
819
820
|
path: params.path as string | undefined,
|
|
820
821
|
pathPrefix: params.path_prefix as string | undefined,
|
|
822
|
+
excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
|
|
821
823
|
extension: params.extension as string | string[] | undefined,
|
|
822
824
|
metadata: params.metadata as Record<string, unknown> | undefined,
|
|
823
825
|
createdBy: params.created_by as string | undefined,
|
|
@@ -939,6 +941,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
939
941
|
hasBrokenLinks: params.has_broken_links as boolean | undefined,
|
|
940
942
|
path: params.path as string | undefined,
|
|
941
943
|
pathPrefix: params.path_prefix as string | undefined,
|
|
944
|
+
excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
|
|
942
945
|
extension: params.extension as string | string[] | undefined,
|
|
943
946
|
// Same `near[]` neighborhood push-down `search`/structured-query
|
|
944
947
|
// use — a semantic query can be scoped to a graph neighborhood too.
|
|
@@ -985,6 +988,8 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
985
988
|
}
|
|
986
989
|
// Normalize tag param
|
|
987
990
|
const tags = normalizeTags(params.tag);
|
|
991
|
+
const excludeTagsRaw = params.exclude_tags ?? params.excludeTags ?? params.exclude_tag;
|
|
992
|
+
const excludeTags = normalizeTags(excludeTagsRaw);
|
|
988
993
|
const mode: SearchMode = searchMode ?? "literal";
|
|
989
994
|
// "Only whitespace/quotes" (vault#551 edge case): short-circuit
|
|
990
995
|
// BEFORE ever calling FTS5 — an empty/all-punctuation phrase can
|
|
@@ -1004,10 +1009,35 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
1004
1009
|
// `invalid_search_syntax`, vault#551) — uncaught on purpose, it
|
|
1005
1010
|
// propagates to `src/mcp-http.ts`, which maps it to a JSON-RPC
|
|
1006
1011
|
// error the same way it maps `invalid_query`.
|
|
1012
|
+
// vault#647: forward the same QueryOpts the structured /
|
|
1013
|
+
// semantic branches already pass. Pre-fix only tags/limit/
|
|
1014
|
+
// expand/mode/sort reached searchNotes, so exclude_tags and
|
|
1015
|
+
// date_from were silently dropped (a well-formed result set
|
|
1016
|
+
// answering a different question).
|
|
1007
1017
|
results = await store.searchNotes(params.search as string, {
|
|
1008
1018
|
tags,
|
|
1009
|
-
|
|
1019
|
+
tagMatch: (params.tag_match as "all" | "any") ?? (tags && tags.length > 1 ? "any" : undefined),
|
|
1010
1020
|
expand,
|
|
1021
|
+
excludeTags,
|
|
1022
|
+
hasTags: params.has_tags as boolean | undefined,
|
|
1023
|
+
hasLinks: params.has_links as boolean | undefined,
|
|
1024
|
+
hasBrokenLinks: params.has_broken_links as boolean | undefined,
|
|
1025
|
+
path: params.path as string | undefined,
|
|
1026
|
+
pathPrefix: params.path_prefix as string | undefined,
|
|
1027
|
+
excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
|
|
1028
|
+
extension: params.extension as string | string[] | undefined,
|
|
1029
|
+
ids: nearScope ? [...nearScope] : undefined,
|
|
1030
|
+
metadata: params.metadata as Record<string, unknown> | undefined,
|
|
1031
|
+
createdBy: params.created_by as string | undefined,
|
|
1032
|
+
lastUpdatedBy: params.last_updated_by as string | undefined,
|
|
1033
|
+
createdVia: params.created_via as string | undefined,
|
|
1034
|
+
lastUpdatedVia: params.last_updated_via as string | undefined,
|
|
1035
|
+
dateFrom: params.date_from as string | undefined,
|
|
1036
|
+
dateTo: params.date_to as string | undefined,
|
|
1037
|
+
dateFilter: params.date_filter as
|
|
1038
|
+
| { field?: string; from?: string; to?: string }
|
|
1039
|
+
| undefined,
|
|
1040
|
+
limit: (params.limit as number) ?? 50,
|
|
1011
1041
|
mode,
|
|
1012
1042
|
sort: params.sort as "asc" | "desc" | undefined,
|
|
1013
1043
|
});
|
|
@@ -1063,6 +1093,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
1063
1093
|
hasBrokenLinks: params.has_broken_links as boolean | undefined,
|
|
1064
1094
|
path: params.path as string | undefined,
|
|
1065
1095
|
pathPrefix: params.path_prefix as string | undefined,
|
|
1096
|
+
excludePathPrefix: normalizeTags(params.exclude_path_prefix ?? params.excludePathPrefix),
|
|
1066
1097
|
extension: params.extension as string | string[] | undefined,
|
|
1067
1098
|
// Push the near-scope into the SQL WHERE so that LIMIT and ORDER
|
|
1068
1099
|
// BY apply to the neighborhood. Without this, queryNotes would
|
|
@@ -1103,8 +1134,8 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
1103
1134
|
// requested, so `next_cursor` never surfaces as an honest
|
|
1104
1135
|
// "more may follow" signal — this is that signal. Mirrors the
|
|
1105
1136
|
// REST structured-query path in src/routes.ts.
|
|
1106
|
-
if (results.length === queryOpts.limit) {
|
|
1107
|
-
queryWarnings.push(truncatedResultsWarning(queryOpts.limit));
|
|
1137
|
+
if (queryOpts.offset === undefined && results.length === queryOpts.limit) {
|
|
1138
|
+
queryWarnings.push(truncatedResultsWarning(queryOpts.limit, "mcp"));
|
|
1108
1139
|
}
|
|
1109
1140
|
}
|
|
1110
1141
|
}
|
|
@@ -2632,6 +2663,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
2632
2663
|
const now = Date.now();
|
|
2633
2664
|
const expiresAt = now + computeTicketTtlMs(declaredSize);
|
|
2634
2665
|
const id = generateTicketId();
|
|
2666
|
+
const downloadMime = contentTypeForAttachmentPath(attachment.path);
|
|
2635
2667
|
const ticket: AttachmentTicket = {
|
|
2636
2668
|
id,
|
|
2637
2669
|
kind: "download",
|
|
@@ -2639,7 +2671,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
2639
2671
|
createdAt: now,
|
|
2640
2672
|
expiresAt,
|
|
2641
2673
|
attachmentId: attachment.id,
|
|
2642
|
-
mimeType:
|
|
2674
|
+
mimeType: downloadMime,
|
|
2643
2675
|
sizeBytes: declaredSize,
|
|
2644
2676
|
};
|
|
2645
2677
|
await ticketSeam.provider.put(ticket);
|
|
@@ -2648,7 +2680,7 @@ export function generateMcpTools(store: Store, opts?: GenerateMcpToolsOpts): Mcp
|
|
|
2648
2680
|
return {
|
|
2649
2681
|
method: "GET",
|
|
2650
2682
|
url,
|
|
2651
|
-
mime_type:
|
|
2683
|
+
mime_type: downloadMime,
|
|
2652
2684
|
...(declaredSize !== undefined ? { size_bytes: declaredSize } : {}),
|
|
2653
2685
|
expires_at: new Date(expiresAt).toISOString(),
|
|
2654
2686
|
curl_example: `curl -o downloaded${sanitizeAttachmentExtension(attachment.path) || ""} '${url}'`,
|
package/core/src/notes.ts
CHANGED
|
@@ -1102,6 +1102,18 @@ export function buildFilterConditions(db: Database, opts: QueryOpts): { conditio
|
|
|
1102
1102
|
params.push(opts.pathPrefix + "%");
|
|
1103
1103
|
}
|
|
1104
1104
|
|
|
1105
|
+
// Path-prefix exclusion (vault#628). Mirrors `pathPrefix` matching
|
|
1106
|
+
// (`LIKE prefix || '%'`, SQLite LIKE is ASCII-case-insensitive). NULL
|
|
1107
|
+
// paths are kept — they are not under the prefix. Repeatable: a note
|
|
1108
|
+
// matching ANY listed prefix is dropped.
|
|
1109
|
+
if (opts.excludePathPrefix && opts.excludePathPrefix.length > 0) {
|
|
1110
|
+
for (const prefix of opts.excludePathPrefix) {
|
|
1111
|
+
if (typeof prefix !== "string" || prefix.length === 0) continue;
|
|
1112
|
+
conditions.push("(n.path IS NULL OR n.path NOT LIKE ?)");
|
|
1113
|
+
params.push(prefix + "%");
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1105
1117
|
// Extension filter (vault#328). Single string → exact match; array → IN
|
|
1106
1118
|
// clause. Compared lower-case so a caller passing "CSV" still hits rows
|
|
1107
1119
|
// stored as "csv". An empty array is a no-op (no filter applied) rather
|
|
@@ -1165,10 +1177,11 @@ export function buildFilterConditions(db: Database, opts: QueryOpts): { conditio
|
|
|
1165
1177
|
if (opts.metadata) {
|
|
1166
1178
|
for (const [key, value] of Object.entries(opts.metadata)) {
|
|
1167
1179
|
if (isOperatorObject(value)) {
|
|
1168
|
-
requireIndexedField(db, key);
|
|
1180
|
+
const indexedField = requireIndexedField(db, key);
|
|
1169
1181
|
const { sql, params: opParams } = buildOperatorClause(
|
|
1170
1182
|
key,
|
|
1171
1183
|
value as Record<string, unknown>,
|
|
1184
|
+
indexedField.sqliteType,
|
|
1172
1185
|
);
|
|
1173
1186
|
conditions.push(sql);
|
|
1174
1187
|
params.push(...opParams);
|
|
@@ -1656,6 +1669,7 @@ function toQueryHashInputs(opts: QueryOpts): QueryHashInputs {
|
|
|
1656
1669
|
hasBrokenLinks: opts.hasBrokenLinks,
|
|
1657
1670
|
path: opts.path,
|
|
1658
1671
|
pathPrefix: opts.pathPrefix,
|
|
1672
|
+
excludePathPrefix: opts.excludePathPrefix,
|
|
1659
1673
|
extension: opts.extension,
|
|
1660
1674
|
ids: opts.ids,
|
|
1661
1675
|
metadata: opts.metadata,
|
|
@@ -1869,7 +1883,7 @@ function applySearchTitleBoost(
|
|
|
1869
1883
|
export function searchNotes(
|
|
1870
1884
|
db: Database,
|
|
1871
1885
|
query: string,
|
|
1872
|
-
opts?:
|
|
1886
|
+
opts?: QueryOpts & { mode?: SearchMode },
|
|
1873
1887
|
): Note[] {
|
|
1874
1888
|
const limit = typeof opts?.limit === "number" ? opts.limit : 50;
|
|
1875
1889
|
// Literal-by-default (vault#551): escape the caller's text so FTS5's own
|
|
@@ -1929,51 +1943,37 @@ export function searchNotes(
|
|
|
1929
1943
|
? "n.created_at DESC, n.id DESC"
|
|
1930
1944
|
: "score DESC, n.id ASC";
|
|
1931
1945
|
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
const rows = db.prepare(`
|
|
1946
|
-
SELECT n.*, ${scoreExpr} AS score FROM notes n
|
|
1947
|
-
JOIN notes_fts fts ON fts.rowid = n.rowid
|
|
1948
|
-
WHERE notes_fts MATCH ?
|
|
1949
|
-
AND n.id IN (SELECT note_id FROM note_tags WHERE tag_name IN (${tagPlaceholders}))
|
|
1950
|
-
ORDER BY ${orderBy}
|
|
1951
|
-
LIMIT ?
|
|
1952
|
-
`).all(ftsQuery, ...searchTags, limit) as (NoteRow & { score: number })[];
|
|
1953
|
-
return applySearchTitleBoost(notesWithTags(db, rows, scoresById(rows)), mode, query, opts?.sort);
|
|
1954
|
-
} catch (err) {
|
|
1955
|
-
// Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
|
|
1956
|
-
// advanced mode expects it (the caller passed raw syntax); literal
|
|
1957
|
-
// mode should never reach it (escaping + control-char sanitization
|
|
1958
|
-
// make the query valid) but if the invariant breaks we still want an
|
|
1959
|
-
// honest error, not an unstructured 500. A QueryError we threw
|
|
1960
|
-
// ourselves (empty-limit validation, etc.) is re-raised untouched.
|
|
1961
|
-
if (err instanceof QueryError) throw err;
|
|
1962
|
-
throw searchSyntaxError(query, err, mode);
|
|
1963
|
-
}
|
|
1964
|
-
}
|
|
1965
|
-
}
|
|
1946
|
+
// vault#647: compose FTS with the SAME filter builder queryNotes /
|
|
1947
|
+
// semanticSearch use (excludeTags, dateFrom/dateFilter, path, metadata,
|
|
1948
|
+
// …). Pre-fix only `tags` reached the WHERE clause; every other filter
|
|
1949
|
+
// was silently dropped. Historical FTS tag semantics are "any tag
|
|
1950
|
+
// matches" (a single IN (...)); preserve that when the caller didn't
|
|
1951
|
+
// set `tagMatch`. LIMIT still applies AFTER the filters, not to an
|
|
1952
|
+
// unfiltered FTS page.
|
|
1953
|
+
const filterOpts: QueryOpts = {
|
|
1954
|
+
...(opts ?? {}),
|
|
1955
|
+
tagMatch: opts?.tagMatch ?? (opts?.tags && opts.tags.length > 0 ? "any" : undefined),
|
|
1956
|
+
};
|
|
1957
|
+
const { conditions, params } = buildFilterConditions(db, filterOpts);
|
|
1958
|
+
const extraWhere = conditions.length > 0 ? `AND ${conditions.join(" AND ")}` : "";
|
|
1966
1959
|
|
|
1967
1960
|
try {
|
|
1968
1961
|
const rows = db.prepare(`
|
|
1969
1962
|
SELECT n.*, ${scoreExpr} AS score FROM notes n
|
|
1970
1963
|
JOIN notes_fts fts ON fts.rowid = n.rowid
|
|
1971
1964
|
WHERE notes_fts MATCH ?
|
|
1965
|
+
${extraWhere}
|
|
1972
1966
|
ORDER BY ${orderBy}
|
|
1973
1967
|
LIMIT ?
|
|
1974
|
-
`).all(ftsQuery, limit) as (NoteRow & { score: number })[];
|
|
1968
|
+
`).all(ftsQuery, ...params, limit) as (NoteRow & { score: number })[];
|
|
1975
1969
|
return applySearchTitleBoost(notesWithTags(db, rows, scoresById(rows)), mode, query, opts?.sort);
|
|
1976
1970
|
} catch (err) {
|
|
1971
|
+
// Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
|
|
1972
|
+
// advanced mode expects it (the caller passed raw syntax); literal
|
|
1973
|
+
// mode should never reach it (escaping + control-char sanitization
|
|
1974
|
+
// make the query valid) but if the invariant breaks we still want an
|
|
1975
|
+
// honest error, not an unstructured 500. A QueryError we threw
|
|
1976
|
+
// ourselves (empty-limit validation, etc.) is re-raised untouched.
|
|
1977
1977
|
if (err instanceof QueryError) throw err;
|
|
1978
1978
|
throw searchSyntaxError(query, err, mode);
|
|
1979
1979
|
}
|
|
@@ -2840,9 +2840,26 @@ export function computeDisplayTitle(content: string | null | undefined): string
|
|
|
2840
2840
|
export const LEDE_MAX_LEN = 400;
|
|
2841
2841
|
|
|
2842
2842
|
/**
|
|
2843
|
-
*
|
|
2843
|
+
* Opens (or closes) a fenced code block: three or more backticks or tildes.
|
|
2844
|
+
* Capture 1 is the run itself so the scanner can require the CLOSING fence to
|
|
2845
|
+
* use the same character and be at least as long, per CommonMark.
|
|
2846
|
+
*/
|
|
2847
|
+
const FENCE_OPEN_RE = /^(`{3,}|~{3,})/;
|
|
2848
|
+
/** A closing fence carries nothing but the run (an info string opens, never closes). */
|
|
2849
|
+
const FENCE_CLOSE_RE = /^(`{3,}|~{3,})\s*$/;
|
|
2850
|
+
/** An ATX heading line: 1-6 `#` followed by whitespace or end-of-line. */
|
|
2851
|
+
const HEADING_LINE_RE = /^#{1,6}(?:\s|$)/;
|
|
2852
|
+
/**
|
|
2853
|
+
* A thematic break (`---`, `***`, `___`) or a setext heading underline
|
|
2854
|
+
* (`---`, `===`). Both are pure markup with no text of their own — the
|
|
2855
|
+
* setext case is why `Title\n---\n\nreal text` used to yield a `---` lede.
|
|
2856
|
+
*/
|
|
2857
|
+
const BREAK_LINE_RE = /^(?:-{3,}|\*{3,}|_{3,}|={3,})$/;
|
|
2858
|
+
|
|
2859
|
+
/**
|
|
2860
|
+
* Derive a note's "lede": the first non-empty PROSE PARAGRAPH after the title
|
|
2844
2861
|
* line (a run of consecutive non-blank lines, whitespace-collapsed to one
|
|
2845
|
-
* line, truncated to `LEDE_MAX_LEN` code points). `null` when there's no
|
|
2862
|
+
* line, truncated to `LEDE_MAX_LEN` code points). `null` when there's no such
|
|
2846
2863
|
* paragraph after the title — a title-only note has no lede to report, and
|
|
2847
2864
|
* callers must not fall back to repeating the title itself.
|
|
2848
2865
|
*
|
|
@@ -2854,6 +2871,22 @@ export const LEDE_MAX_LEN = 400;
|
|
|
2854
2871
|
* what it finds, honestly, using the SAME title-line rule as
|
|
2855
2872
|
* `computeDisplayTitle` (including its frontmatter skip) so a caller that
|
|
2856
2873
|
* shows both title and lede sees them agree on where the title ends.
|
|
2874
|
+
*
|
|
2875
|
+
* Non-prose blocks are SKIPPED (vault#616). A note that opens with a code
|
|
2876
|
+
* fence, a section heading, or a horizontal rule used to have that raw markup
|
|
2877
|
+
* returned as its lede — backticks, `## ` markers and all — which contradicted
|
|
2878
|
+
* this function's own "first paragraph" contract and read as garbage in
|
|
2879
|
+
* `expand_mode: "summary"`. The scan now walks past fenced blocks (to the
|
|
2880
|
+
* matching closing fence, or to end-of-content when the fence is never
|
|
2881
|
+
* closed), heading lines, and break/underline lines until it reaches real
|
|
2882
|
+
* text, and returns `null` if it never does. Those same three shapes also
|
|
2883
|
+
* TERMINATE a paragraph already in progress, matching how markdown lets a
|
|
2884
|
+
* fence or heading interrupt a paragraph.
|
|
2885
|
+
*
|
|
2886
|
+
* Deliberately NOT normalized: list blocks. A `- milk` list is genuine prose
|
|
2887
|
+
* content, and how to flatten it into one line (drop the markers? join with
|
|
2888
|
+
* what?) is a rendering decision this function shouldn't make silently — it
|
|
2889
|
+
* reports the list as written.
|
|
2857
2890
|
*/
|
|
2858
2891
|
export function computeLede(content: string | null | undefined): string | null {
|
|
2859
2892
|
if (!content) return null;
|
|
@@ -2870,14 +2903,51 @@ export function computeLede(content: string | null | undefined): string | null {
|
|
|
2870
2903
|
if (titleLine === -1) return null; // no title at all — nothing to find a lede after
|
|
2871
2904
|
|
|
2872
2905
|
let i = titleLine + 1;
|
|
2873
|
-
while (i < lines.length && lines[i]!.trim() === "") i++; // skip blank lines after the title
|
|
2874
|
-
|
|
2875
2906
|
const paragraphLines: string[] = [];
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2907
|
+
|
|
2908
|
+
// Walk blocks until one of them is prose. Each iteration consumes exactly
|
|
2909
|
+
// one block (blank run, fence, heading, break, or paragraph), so `i` always
|
|
2910
|
+
// advances and the loop terminates.
|
|
2911
|
+
scan: while (i < lines.length) {
|
|
2912
|
+
const trimmed = lines[i]!.trim();
|
|
2913
|
+
|
|
2914
|
+
if (trimmed === "") {
|
|
2915
|
+
i++;
|
|
2916
|
+
continue;
|
|
2917
|
+
}
|
|
2918
|
+
|
|
2919
|
+
const opening = FENCE_OPEN_RE.exec(trimmed);
|
|
2920
|
+
if (opening) {
|
|
2921
|
+
const marker = opening[1]!;
|
|
2922
|
+
i++;
|
|
2923
|
+
while (i < lines.length) {
|
|
2924
|
+
const closing = FENCE_CLOSE_RE.exec(lines[i]!.trim());
|
|
2925
|
+
i++;
|
|
2926
|
+
if (closing && closing[1]![0] === marker[0] && closing[1]!.length >= marker.length) break;
|
|
2927
|
+
}
|
|
2928
|
+
continue; // an unclosed fence runs off the end and leaves i past the last line
|
|
2929
|
+
}
|
|
2930
|
+
|
|
2931
|
+
if (HEADING_LINE_RE.test(trimmed) || BREAK_LINE_RE.test(trimmed)) {
|
|
2932
|
+
i++;
|
|
2933
|
+
continue;
|
|
2934
|
+
}
|
|
2935
|
+
|
|
2936
|
+
// Real text — gather the paragraph, stopping at a blank line or at any
|
|
2937
|
+
// block-level marker that interrupts it.
|
|
2938
|
+
while (i < lines.length) {
|
|
2939
|
+
const line = lines[i]!;
|
|
2940
|
+
const t = line.trim();
|
|
2941
|
+
if (t === "" || FENCE_OPEN_RE.test(t) || HEADING_LINE_RE.test(t) || BREAK_LINE_RE.test(t)) {
|
|
2942
|
+
break scan;
|
|
2943
|
+
}
|
|
2944
|
+
paragraphLines.push(line);
|
|
2945
|
+
i++;
|
|
2946
|
+
}
|
|
2947
|
+
break;
|
|
2879
2948
|
}
|
|
2880
|
-
|
|
2949
|
+
|
|
2950
|
+
if (paragraphLines.length === 0) return null; // title-only note, or nothing but markup
|
|
2881
2951
|
|
|
2882
2952
|
const paragraph = paragraphLines.join(" ").replace(/\s+/g, " ").trim();
|
|
2883
2953
|
if (paragraph === "") return null;
|
|
@@ -2938,8 +3008,10 @@ export function toNoteIndex(note: Note): NoteIndex {
|
|
|
2938
3008
|
/**
|
|
2939
3009
|
* Filter metadata on a note/index result based on an include_metadata param.
|
|
2940
3010
|
* - true / undefined → return as-is (all metadata)
|
|
2941
|
-
* - false → strip metadata entirely
|
|
2942
|
-
* - string[] → return only those keys
|
|
3011
|
+
* - false → strip metadata entirely (key omitted — caller asked for none)
|
|
3012
|
+
* - string[] → return only those keys. Empty array = no filtering (all keys).
|
|
3013
|
+
* A non-empty array whose keys miss still keeps `metadata: {}` (vault#600
|
|
3014
|
+
* V1.1 invariant — the key must not vanish from the wire).
|
|
2943
3015
|
*/
|
|
2944
3016
|
export function filterMetadata(obj: any, includeMetadata: boolean | string[] | undefined): any {
|
|
2945
3017
|
if (includeMetadata === undefined || includeMetadata === true) return obj;
|
|
@@ -2953,7 +3025,7 @@ export function filterMetadata(obj: any, includeMetadata: boolean | string[] | u
|
|
|
2953
3025
|
const filtered = Object.fromEntries(
|
|
2954
3026
|
Object.entries(obj.metadata).filter(([k]) => fields.includes(k)),
|
|
2955
3027
|
);
|
|
2956
|
-
return { ...obj, metadata:
|
|
3028
|
+
return { ...obj, metadata: filtered };
|
|
2957
3029
|
}
|
|
2958
3030
|
|
|
2959
3031
|
/**
|
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
18
|
import { Database, type SQLQueryBindings } from "bun:sqlite";
|
|
19
|
-
import { getIndexedField, type IndexedField } from "./indexed-fields.js";
|
|
19
|
+
import { getIndexedField, type IndexedField, type SqliteType } from "./indexed-fields.js";
|
|
20
|
+
import { inViaJsonEachCast, jsonEachParam } from "./sql-in.js";
|
|
20
21
|
|
|
21
22
|
export const SUPPORTED_OPS = [
|
|
22
23
|
"eq",
|
|
@@ -101,6 +102,52 @@ function toBinding(field: string, op: string, value: unknown): SQLQueryBindings
|
|
|
101
102
|
);
|
|
102
103
|
}
|
|
103
104
|
|
|
105
|
+
/**
|
|
106
|
+
* Narrow one `in`/`not_in` element to something `JSON.stringify` can carry
|
|
107
|
+
* into {@link jsonEachParam} (vault#536).
|
|
108
|
+
*
|
|
109
|
+
* Runs `toBinding` first so the primitive-only validation and its error
|
|
110
|
+
* message are unchanged from the one-param-per-value era. Then two shapes
|
|
111
|
+
* need explicit handling, because JSON has no equivalent of what bun:sqlite
|
|
112
|
+
* did implicitly when the value was bound directly:
|
|
113
|
+
*
|
|
114
|
+
* - `boolean` → `1`/`0`, matching the INTEGER a boolean is stored as, so
|
|
115
|
+
* `{ in: [true] }` still matches rows written with `true`. A raw JSON
|
|
116
|
+
* `true` would come out of `json_each` as `1` anyway on current SQLite,
|
|
117
|
+
* but converting here makes the intent explicit rather than dependent on
|
|
118
|
+
* that.
|
|
119
|
+
* - `bigint` → `JSON.stringify` THROWS on it ("Do not know how to serialize
|
|
120
|
+
* a BigInt"). Converted when it fits exactly in a double; otherwise
|
|
121
|
+
* rejected with a real error instead of a crash or a silently wrong
|
|
122
|
+
* comparison. (Unreachable from MCP/REST — `JSON.parse` never yields a
|
|
123
|
+
* bigint — so this only guards a direct in-process caller.)
|
|
124
|
+
*/
|
|
125
|
+
function toJsonEachMember(
|
|
126
|
+
field: string,
|
|
127
|
+
op: string,
|
|
128
|
+
value: unknown,
|
|
129
|
+
): string | number | boolean | null {
|
|
130
|
+
const bound = toBinding(field, op, value);
|
|
131
|
+
if (typeof bound === "boolean") return bound ? 1 : 0;
|
|
132
|
+
if (typeof bound === "bigint") {
|
|
133
|
+
if (bound >= BigInt(Number.MIN_SAFE_INTEGER) && bound <= BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
134
|
+
return Number(bound);
|
|
135
|
+
}
|
|
136
|
+
throw new QueryError(
|
|
137
|
+
`operator "${op}" on metadata field "${field}" got a bigint outside the safe integer range (${bound}); it cannot be compared exactly`,
|
|
138
|
+
"INVALID_OPERATOR_VALUE",
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
// `toBinding` is declared as the full `SQLQueryBindings` union (which
|
|
142
|
+
// includes TypedArray/Record shapes it never actually returns), so narrow
|
|
143
|
+
// to what JSON can carry rather than asserting.
|
|
144
|
+
if (bound === null || typeof bound === "string" || typeof bound === "number") return bound;
|
|
145
|
+
throw new QueryError(
|
|
146
|
+
`operator "${op}" on metadata field "${field}" expects a primitive value (string, number, boolean, bigint, or null)`,
|
|
147
|
+
"INVALID_OPERATOR_VALUE",
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
104
151
|
/**
|
|
105
152
|
* Best-effort lookup of a metadata field's declared `type` across every tag
|
|
106
153
|
* schema — used ONLY to sharpen the {@link requireIndexedField} error hint
|
|
@@ -176,10 +223,19 @@ export function requireIndexedField(db: Database, field: string): IndexedField {
|
|
|
176
223
|
* Build a SQL fragment + bound params for an operator object on an indexed
|
|
177
224
|
* metadata field. Each operator maps to a single AND clause; an object like
|
|
178
225
|
* `{ gt: 5, lt: 10 }` composes as `meta_<field> > 5 AND meta_<field> < 10`.
|
|
226
|
+
*
|
|
227
|
+
* `sqliteType` is the field's declared column storage type (from
|
|
228
|
+
* `requireIndexedField(...).sqliteType`, which every caller already
|
|
229
|
+
* resolves). It's needed for `in`/`not_in`: those bind the value-set through
|
|
230
|
+
* a `json_each` subquery, which — unlike a placeholder `IN (?, …)` list —
|
|
231
|
+
* does NOT apply the left column's type affinity to each candidate value, so
|
|
232
|
+
* the set must be CAST to the column's type to keep cross-type matches like
|
|
233
|
+
* numeric `5` against a TEXT-affinity `'5'` (vault#676).
|
|
179
234
|
*/
|
|
180
235
|
export function buildOperatorClause(
|
|
181
236
|
field: string,
|
|
182
237
|
opObj: Record<string, unknown>,
|
|
238
|
+
sqliteType: SqliteType,
|
|
183
239
|
): { sql: string; params: SQLQueryBindings[] } {
|
|
184
240
|
validateOperatorObject(field, opObj);
|
|
185
241
|
// `field` came from indexed_fields (which validated it via FIELD_NAME_RE
|
|
@@ -235,13 +291,39 @@ export function buildOperatorClause(
|
|
|
235
291
|
parts.push(op === "in" ? "0" : "1");
|
|
236
292
|
break;
|
|
237
293
|
}
|
|
238
|
-
|
|
294
|
+
// ONE bound param for the whole set, regardless of its size
|
|
295
|
+
// (vault#536). This used to emit `IN (?, ?, …)` — one param per
|
|
296
|
+
// value — which on Cloudflare Durable Object SQLite (100 bound
|
|
297
|
+
// params per statement, vs bun:sqlite's 999+) turned any `in` array
|
|
298
|
+
// over ~100 elements into a `too many SQL variables` 500 on cloud
|
|
299
|
+
// vaults, while self-host never noticed.
|
|
300
|
+
//
|
|
301
|
+
// This is an EMBEDDED filter inside a paginated statement, so the
|
|
302
|
+
// chunk-and-union approach used for standalone id-lists isn't
|
|
303
|
+
// available — chunking would break the shared LIMIT/OFFSET window.
|
|
304
|
+
// The json_each shape #535 introduced for the `near` neighborhood is
|
|
305
|
+
// the one that works here: bind the array as a single JSON param and
|
|
306
|
+
// let SQLite expand it into a one-column table. See sql-in.ts.
|
|
307
|
+
//
|
|
308
|
+
// Every element still goes through `toBinding`, so the same
|
|
309
|
+
// primitive-only validation (and the same error message) applies as
|
|
310
|
+
// before — the values are just serialized into JSON instead of bound
|
|
311
|
+
// one by one.
|
|
312
|
+
//
|
|
313
|
+
// The set is CAST to the column's declared storage type inside the
|
|
314
|
+
// subquery (`inViaJsonEachCast`). A placeholder `IN (?, …)` list got
|
|
315
|
+
// the left column's affinity applied to each value for free; a
|
|
316
|
+
// `json_each` subquery does not, so without the CAST a numeric `5`
|
|
317
|
+
// silently stopped matching a TEXT-affinity `'5'` (vault#676). The
|
|
318
|
+
// CAST restores that in both directions and covers `in` and `not_in`.
|
|
319
|
+
const members = value.map((v) => toJsonEachMember(field, op, v));
|
|
320
|
+
const inClause = inViaJsonEachCast(sqliteType);
|
|
239
321
|
if (op === "in") {
|
|
240
|
-
parts.push(`${col} IN
|
|
322
|
+
parts.push(`${col} IN ${inClause}`);
|
|
241
323
|
} else {
|
|
242
|
-
parts.push(`(${col} IS NULL OR ${col} NOT IN
|
|
324
|
+
parts.push(`(${col} IS NULL OR ${col} NOT IN ${inClause})`);
|
|
243
325
|
}
|
|
244
|
-
|
|
326
|
+
params.push(jsonEachParam(members));
|
|
245
327
|
break;
|
|
246
328
|
}
|
|
247
329
|
case "exists":
|
|
@@ -229,12 +229,20 @@ export function ignoredParamWarning(param: string, reason: string): QueryWarning
|
|
|
229
229
|
* /api/notes` on a >limit-note vault returns the OLDEST rows with zero
|
|
230
230
|
* signal that the NEWEST ones didn't make it. `?cursor=` (bootstrap or
|
|
231
231
|
* watermark) sidesteps this entirely — the warning only fires in its
|
|
232
|
-
* absence.
|
|
232
|
+
* absence. Explicit `offset` is also exempt (vault#601): offset-paging is
|
|
233
|
+
* deliberate pagination, not an accidental full default page.
|
|
233
234
|
*/
|
|
234
|
-
export function truncatedResultsWarning(
|
|
235
|
+
export function truncatedResultsWarning(
|
|
236
|
+
limit: number,
|
|
237
|
+
channel: "rest" | "mcp" = "rest",
|
|
238
|
+
): QueryWarning {
|
|
239
|
+
const pageHint =
|
|
240
|
+
channel === "mcp"
|
|
241
|
+
? `Pass cursor: "" to page, or sort: "desc" for newest-first.`
|
|
242
|
+
: "Pass ?cursor= to page, or sort=desc for newest-first.";
|
|
235
243
|
return {
|
|
236
244
|
code: "truncated",
|
|
237
|
-
message: `${limit} = limit rows returned; there may be more.
|
|
245
|
+
message: `${limit} = limit rows returned; there may be more. ${pageHint}`,
|
|
238
246
|
limit,
|
|
239
247
|
};
|
|
240
248
|
}
|