@openparachute/vault 0.7.3-rc.12 → 0.7.3-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/core/src/conformance.ts +1 -2
- package/core/src/content-range.test.ts +0 -127
- package/core/src/content-range.ts +0 -100
- package/core/src/contract-typed-index.test.ts +3 -4
- package/core/src/core.test.ts +4 -521
- package/core/src/expand.ts +3 -11
- package/core/src/indexed-fields.test.ts +3 -9
- package/core/src/indexed-fields.ts +1 -9
- package/core/src/mcp.ts +10 -601
- package/core/src/notes.ts +4 -201
- package/core/src/schema-defaults.ts +1 -85
- package/core/src/search-fts-v25.test.ts +1 -9
- package/core/src/search-query.test.ts +0 -42
- package/core/src/search-query.ts +0 -27
- package/core/src/seed-packs.test.ts +1 -117
- package/core/src/seed-packs.ts +1 -217
- package/core/src/store.ts +3 -59
- package/core/src/tag-schemas.ts +12 -27
- package/core/src/types.ts +1 -7
- package/core/src/vault-projection.ts +0 -55
- package/package.json +1 -1
- package/src/add-pack.test.ts +0 -32
- package/src/cli.ts +1 -5
- package/src/contract-errors.test.ts +1 -2
- package/src/mcp-http.ts +4 -33
- package/src/mcp-tools.ts +14 -61
- package/src/onboarding-seed.test.ts +0 -64
- package/src/routes.ts +95 -92
- package/src/routing.ts +0 -17
- package/src/server.ts +0 -7
- package/src/storage.test.ts +1 -200
- package/src/transcription-worker.test.ts +0 -151
- package/src/transcription-worker.ts +52 -113
- package/src/vault.test.ts +11 -48
- package/core/src/attachment/bytes-provider.ts +0 -65
- package/core/src/attachment/policy.test.ts +0 -66
- package/core/src/attachment/policy.ts +0 -131
- package/core/src/attachment/tickets.test.ts +0 -45
- package/core/src/attachment/tickets.ts +0 -117
- package/core/src/attachment-tickets-tool.test.ts +0 -286
- package/core/src/display-title.test.ts +0 -190
- package/core/src/lede.test.ts +0 -96
- package/core/src/search-title-boost.test.ts +0 -125
- package/src/attachment-bytes.ts +0 -68
- package/src/attachment-tickets.test.ts +0 -475
- package/src/attachment-tickets.ts +0 -340
- package/src/read-attachment.test.ts +0 -436
package/core/src/notes.ts
CHANGED
|
@@ -25,7 +25,6 @@ import { computeExpandedTagCounts, loadTagHierarchy, stripTagHash } from "./tag-
|
|
|
25
25
|
import { chunkForInClause, IN_VIA_JSON_EACH, jsonEachParam } from "./sql-in.js";
|
|
26
26
|
import {
|
|
27
27
|
buildLiteralSearchQuery,
|
|
28
|
-
extractLiteralBoostTerms,
|
|
29
28
|
SEARCH_WEIGHT_PATH,
|
|
30
29
|
SEARCH_WEIGHT_CONTENT,
|
|
31
30
|
type SearchMode,
|
|
@@ -1811,61 +1810,6 @@ function searchSyntaxError(rawQuery: string, err: unknown, mode: SearchMode): Qu
|
|
|
1811
1810
|
});
|
|
1812
1811
|
}
|
|
1813
1812
|
|
|
1814
|
-
/**
|
|
1815
|
-
* Post-rank a search result page (title axis, ratified 2026-07-17) so
|
|
1816
|
-
* notes whose display title (first line of content) contains EVERY boost
|
|
1817
|
-
* term outrank the rest — a title-match tier ahead of a body-only-match
|
|
1818
|
-
* tier. Stable partition: within a tier, the incoming order (FTS5
|
|
1819
|
-
* relevance, or an explicit `sort`) is preserved untouched — two notes
|
|
1820
|
-
* that both match, or both don't, keep their relative order.
|
|
1821
|
-
*
|
|
1822
|
-
* No-migration by design (see the search-seam note on `searchNotes`
|
|
1823
|
-
* below): re-orders the already-fetched page in memory instead of adding
|
|
1824
|
-
* a weighted title column to `notes_fts`, which would need a schema
|
|
1825
|
-
* migration + backfill. `notes_fts`'s existing `SEARCH_WEIGHT_PATH` bm25
|
|
1826
|
-
* weighting (above) already boosts by note PATH — a different axis from
|
|
1827
|
-
* the ratified title model (title = first line of CONTENT, not path).
|
|
1828
|
-
* This function is the one that boosts by the content-derived title.
|
|
1829
|
-
*/
|
|
1830
|
-
function boostTitleMatches(notes: Note[], terms: string[]): Note[] {
|
|
1831
|
-
if (terms.length === 0) return notes;
|
|
1832
|
-
return notes
|
|
1833
|
-
.map((note, index) => ({ note, index, titleMatch: titleMatchesAllTerms(note.content, terms) }))
|
|
1834
|
-
.sort((a, b) => {
|
|
1835
|
-
if (a.titleMatch === b.titleMatch) return a.index - b.index; // stable
|
|
1836
|
-
return a.titleMatch ? -1 : 1;
|
|
1837
|
-
})
|
|
1838
|
-
.map((x) => x.note);
|
|
1839
|
-
}
|
|
1840
|
-
|
|
1841
|
-
function titleMatchesAllTerms(content: string | null | undefined, terms: string[]): boolean {
|
|
1842
|
-
const title = computeDisplayTitle(content);
|
|
1843
|
-
if (!title) return false;
|
|
1844
|
-
const lower = title.toLowerCase();
|
|
1845
|
-
return terms.every((t) => lower.includes(t));
|
|
1846
|
-
}
|
|
1847
|
-
|
|
1848
|
-
/**
|
|
1849
|
-
* Apply the title-boost post-rank to a fetched search page. Skipped
|
|
1850
|
-
* whenever it would fight the caller's intent:
|
|
1851
|
-
* - an explicit `sort: "asc"|"desc"` means the caller wants chronological
|
|
1852
|
-
* order, not relevance re-ranking;
|
|
1853
|
-
* - `search_mode: "advanced"` text carries raw FTS5 syntax (boolean
|
|
1854
|
-
* operators, column filters, prefix `*`) — naively tokenizing it into
|
|
1855
|
-
* "terms" would misread operators as content (see
|
|
1856
|
-
* `extractLiteralBoostTerms`'s doc comment). Only literal-mode queries
|
|
1857
|
-
* (the default) get boosted today.
|
|
1858
|
-
*/
|
|
1859
|
-
function applySearchTitleBoost(
|
|
1860
|
-
notes: Note[],
|
|
1861
|
-
mode: SearchMode,
|
|
1862
|
-
rawQuery: string,
|
|
1863
|
-
sort: "asc" | "desc" | undefined,
|
|
1864
|
-
): Note[] {
|
|
1865
|
-
if (sort !== undefined || mode !== "literal") return notes;
|
|
1866
|
-
return boostTitleMatches(notes, extractLiteralBoostTerms(rawQuery));
|
|
1867
|
-
}
|
|
1868
|
-
|
|
1869
1813
|
export function searchNotes(
|
|
1870
1814
|
db: Database,
|
|
1871
1815
|
query: string,
|
|
@@ -1902,14 +1846,6 @@ export function searchNotes(
|
|
|
1902
1846
|
// The weights are our own numeric constants (not user input) interpolated
|
|
1903
1847
|
// directly — bm25()'s weight arguments are positional per-column
|
|
1904
1848
|
// multipliers, not general SQL expressions callers can influence.
|
|
1905
|
-
//
|
|
1906
|
-
// NOTE: "title" here means the note's `path` — this predates the title
|
|
1907
|
-
// axis (ratified 2026-07-17), under which a note's title is its first
|
|
1908
|
-
// CONTENT line, not its path. `applySearchTitleBoost` (below) is the
|
|
1909
|
-
// mechanism for THAT axis: a post-rank pass over the already-scored page,
|
|
1910
|
-
// not a third FTS column. The two are complementary, not redundant — a
|
|
1911
|
-
// note can have a matching path with a non-matching first line, or vice
|
|
1912
|
-
// versa.
|
|
1913
1849
|
const scoreExpr = `(-1.0 * bm25(notes_fts, ${SEARCH_WEIGHT_PATH}, ${SEARCH_WEIGHT_CONTENT}))`;
|
|
1914
1850
|
|
|
1915
1851
|
// `sort` honored under search (vault#551 WS2A item 3): default stays FTS5
|
|
@@ -1950,7 +1886,7 @@ export function searchNotes(
|
|
|
1950
1886
|
ORDER BY ${orderBy}
|
|
1951
1887
|
LIMIT ?
|
|
1952
1888
|
`).all(ftsQuery, ...searchTags, limit) as (NoteRow & { score: number })[];
|
|
1953
|
-
return
|
|
1889
|
+
return notesWithTags(db, rows, scoresById(rows));
|
|
1954
1890
|
} catch (err) {
|
|
1955
1891
|
// Surface EVERY FTS5 error structured, never a raw rethrow (vault#551):
|
|
1956
1892
|
// advanced mode expects it (the caller passed raw syntax); literal
|
|
@@ -1972,7 +1908,7 @@ export function searchNotes(
|
|
|
1972
1908
|
ORDER BY ${orderBy}
|
|
1973
1909
|
LIMIT ?
|
|
1974
1910
|
`).all(ftsQuery, limit) as (NoteRow & { score: number })[];
|
|
1975
|
-
return
|
|
1911
|
+
return notesWithTags(db, rows, scoresById(rows));
|
|
1976
1912
|
} catch (err) {
|
|
1977
1913
|
if (err instanceof QueryError) throw err;
|
|
1978
1914
|
throw searchSyntaxError(query, err, mode);
|
|
@@ -2766,142 +2702,10 @@ export function mergeTags(
|
|
|
2766
2702
|
/** Max code points in a NoteIndex preview. */
|
|
2767
2703
|
export const NOTE_INDEX_PREVIEW_LEN = 120;
|
|
2768
2704
|
|
|
2769
|
-
/** Max code points in a computed `displayTitle` (title axis, ratified 2026-07-17). */
|
|
2770
|
-
export const DISPLAY_TITLE_MAX_LEN = 120;
|
|
2771
|
-
|
|
2772
|
-
/**
|
|
2773
|
-
* Bounded line-count a leading frontmatter block's closing fence is
|
|
2774
|
-
* searched within (see `computeDisplayTitle`). Frontmatter blocks are a
|
|
2775
|
-
* handful of lines in practice; capping the scan keeps a pathological
|
|
2776
|
-
* "content starts with `---` but never closes it" note O(1)-ish rather
|
|
2777
|
-
* than a full-content scan on every derivation.
|
|
2778
|
-
*/
|
|
2779
|
-
const FRONTMATTER_SCAN_LINES = 100;
|
|
2780
|
-
|
|
2781
|
-
/**
|
|
2782
|
-
* Shared by `computeDisplayTitle` and `computeLede`: given `content` already
|
|
2783
|
-
* split into `lines`, return the index of the first line that counts as the
|
|
2784
|
-
* start of the DOCUMENT body — after a leading closed frontmatter block, if
|
|
2785
|
-
* `content` opens with one (see `computeDisplayTitle`'s frontmatter-skip
|
|
2786
|
-
* doc for the full rationale). `0` when there's no leading frontmatter.
|
|
2787
|
-
*/
|
|
2788
|
-
function skipLeadingFrontmatter(lines: string[]): number {
|
|
2789
|
-
if (lines[0]?.trim() !== "---") return 0;
|
|
2790
|
-
const scanLimit = Math.min(lines.length, FRONTMATTER_SCAN_LINES);
|
|
2791
|
-
for (let i = 1; i < scanLimit; i++) {
|
|
2792
|
-
if (lines[i]?.trim() === "---") return i + 1;
|
|
2793
|
-
}
|
|
2794
|
-
return 0;
|
|
2795
|
-
}
|
|
2796
|
-
|
|
2797
|
-
/**
|
|
2798
|
-
* Derive a note's display title: the first non-empty line of `content`,
|
|
2799
|
-
* with a leading markdown heading marker (`#` through `######`) and its
|
|
2800
|
-
* following whitespace stripped, truncated to `DISPLAY_TITLE_MAX_LEN` code
|
|
2801
|
-
* points. `null` when content has no non-empty line (empty or
|
|
2802
|
-
* whitespace/heading-marker-only note).
|
|
2803
|
-
*
|
|
2804
|
-
* The ratified title model (2026-07-17): a note's title IS its first line
|
|
2805
|
-
* — derived from content, NEVER stored as its own column. This function is
|
|
2806
|
-
* the one place that derivation happens; callers (`toNoteIndex` below) call
|
|
2807
|
-
* it fresh at read time. Timestamp-path formatting — what a surface shows
|
|
2808
|
-
* in place of a `null` title — is deliberately NOT this function's job; it
|
|
2809
|
-
* reports the honest content-derived value (or its absence) and leaves
|
|
2810
|
-
* rendering to the caller.
|
|
2811
|
-
*
|
|
2812
|
-
* Frontmatter skip: normal ingestion strips a YAML frontmatter block into
|
|
2813
|
-
* `metadata` before create, so `content` never carries one — but a direct
|
|
2814
|
-
* MCP/REST create can paste raw frontmatter-bearing text
|
|
2815
|
-
* (`---\ntitle: X\n---\n# Real Title`), and the first line of that
|
|
2816
|
-
* DOCUMENT is not the first line of its delimiter. When `content` opens
|
|
2817
|
-
* with a `---` line, derivation starts after the matching CLOSING `---`
|
|
2818
|
-
* (searched within the first `FRONTMATTER_SCAN_LINES` lines). An
|
|
2819
|
-
* unterminated opening `---` falls back to the pre-existing behavior —
|
|
2820
|
-
* scanning from line 0 — so a note whose real first line is literally
|
|
2821
|
-
* `---` isn't mangled.
|
|
2822
|
-
*/
|
|
2823
|
-
export function computeDisplayTitle(content: string | null | undefined): string | null {
|
|
2824
|
-
if (!content) return null;
|
|
2825
|
-
const lines = content.split("\n");
|
|
2826
|
-
const startIndex = skipLeadingFrontmatter(lines);
|
|
2827
|
-
for (let i = startIndex; i < lines.length; i++) {
|
|
2828
|
-
const stripped = lines[i]!.replace(/^#{1,6}\s*/, "").trim();
|
|
2829
|
-
if (stripped === "") continue;
|
|
2830
|
-
// Iterate by Unicode code points so we don't split surrogate pairs mid-character.
|
|
2831
|
-
const codePoints = Array.from(stripped);
|
|
2832
|
-
return codePoints.length > DISPLAY_TITLE_MAX_LEN
|
|
2833
|
-
? codePoints.slice(0, DISPLAY_TITLE_MAX_LEN).join("")
|
|
2834
|
-
: stripped;
|
|
2835
|
-
}
|
|
2836
|
-
return null;
|
|
2837
|
-
}
|
|
2838
|
-
|
|
2839
|
-
/** Max code points in a computed `lede` (summaries-as-content, 2026-07-17 dialogue). */
|
|
2840
|
-
export const LEDE_MAX_LEN = 400;
|
|
2841
|
-
|
|
2842
|
-
/**
|
|
2843
|
-
* Derive a note's "lede": the first non-empty PARAGRAPH after the title
|
|
2844
|
-
* line (a run of consecutive non-blank lines, whitespace-collapsed to one
|
|
2845
|
-
* line, truncated to `LEDE_MAX_LEN` code points). `null` when there's no
|
|
2846
|
-
* paragraph after the title — a title-only note has no lede to report, and
|
|
2847
|
-
* callers must not fall back to repeating the title itself.
|
|
2848
|
-
*
|
|
2849
|
-
* This is the machinery half of a soft convention (2026-07-17 dialogue):
|
|
2850
|
-
* summaries work better as visible CONTENT — a note's opening paragraph —
|
|
2851
|
-
* than as hidden metadata, because visible text gets corrected by the notes
|
|
2852
|
-
* that reference it while hidden metadata rots unnoticed. Nothing validates
|
|
2853
|
-
* that a note actually opens with a title + lede; this function just reports
|
|
2854
|
-
* what it finds, honestly, using the SAME title-line rule as
|
|
2855
|
-
* `computeDisplayTitle` (including its frontmatter skip) so a caller that
|
|
2856
|
-
* shows both title and lede sees them agree on where the title ends.
|
|
2857
|
-
*/
|
|
2858
|
-
export function computeLede(content: string | null | undefined): string | null {
|
|
2859
|
-
if (!content) return null;
|
|
2860
|
-
const lines = content.split("\n");
|
|
2861
|
-
const bodyStart = skipLeadingFrontmatter(lines);
|
|
2862
|
-
|
|
2863
|
-
let titleLine = -1;
|
|
2864
|
-
for (let i = bodyStart; i < lines.length; i++) {
|
|
2865
|
-
if (lines[i]!.replace(/^#{1,6}\s*/, "").trim() !== "") {
|
|
2866
|
-
titleLine = i;
|
|
2867
|
-
break;
|
|
2868
|
-
}
|
|
2869
|
-
}
|
|
2870
|
-
if (titleLine === -1) return null; // no title at all — nothing to find a lede after
|
|
2871
|
-
|
|
2872
|
-
let i = titleLine + 1;
|
|
2873
|
-
while (i < lines.length && lines[i]!.trim() === "") i++; // skip blank lines after the title
|
|
2874
|
-
|
|
2875
|
-
const paragraphLines: string[] = [];
|
|
2876
|
-
while (i < lines.length && lines[i]!.trim() !== "") {
|
|
2877
|
-
paragraphLines.push(lines[i]!);
|
|
2878
|
-
i++;
|
|
2879
|
-
}
|
|
2880
|
-
if (paragraphLines.length === 0) return null; // title-only note
|
|
2881
|
-
|
|
2882
|
-
const paragraph = paragraphLines.join(" ").replace(/\s+/g, " ").trim();
|
|
2883
|
-
if (paragraph === "") return null;
|
|
2884
|
-
|
|
2885
|
-
const codePoints = Array.from(paragraph);
|
|
2886
|
-
return codePoints.length > LEDE_MAX_LEN
|
|
2887
|
-
? codePoints.slice(0, LEDE_MAX_LEN).join("")
|
|
2888
|
-
: paragraph;
|
|
2889
|
-
}
|
|
2890
|
-
|
|
2891
2705
|
/**
|
|
2892
2706
|
* Convert a full Note into its lean index shape:
|
|
2893
|
-
* drops `content`, adds `byteSize
|
|
2894
|
-
*
|
|
2895
|
-
* /notes endpoints, and /graph.
|
|
2896
|
-
*
|
|
2897
|
-
* Perf note: `displayTitle` costs nothing extra here — every caller of this
|
|
2898
|
-
* function already has the full `Note` (content included) in hand by the
|
|
2899
|
-
* time it's called. `queryNotes`'s two-phase page fetch selects a narrow
|
|
2900
|
-
* `id`-only column list for the ORDER BY/pagination phase, but the second
|
|
2901
|
-
* phase (`fetchNotesByIdsOrdered`) always `SELECT *`s the page's full rows
|
|
2902
|
-
* before `toNoteIndex` ever runs — same as `preview`/`byteSize` today. If a
|
|
2903
|
-
* future lean-list path is added that genuinely avoids a content read, it
|
|
2904
|
-
* must NOT call this function (or `toNoteIndex`) on a content-less row.
|
|
2707
|
+
* drops `content`, adds `byteSize` and a whitespace-collapsed `preview`.
|
|
2708
|
+
* Shared between the `query-notes` MCP tool, HTTP /notes endpoints, and /graph.
|
|
2905
2709
|
*/
|
|
2906
2710
|
export function toNoteIndex(note: Note): NoteIndex {
|
|
2907
2711
|
const content = note.content ?? "";
|
|
@@ -2928,7 +2732,6 @@ export function toNoteIndex(note: Note): NoteIndex {
|
|
|
2928
2732
|
metadata: note.metadata,
|
|
2929
2733
|
byteSize,
|
|
2930
2734
|
preview,
|
|
2931
|
-
displayTitle: computeDisplayTitle(note.content),
|
|
2932
2735
|
...(note.score !== undefined ? { score: note.score } : {}),
|
|
2933
2736
|
};
|
|
2934
2737
|
}
|
|
@@ -41,7 +41,6 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import { Database } from "bun:sqlite";
|
|
44
|
-
import { timestampToMs } from "./cursor.js";
|
|
45
44
|
|
|
46
45
|
// ---------------------------------------------------------------------------
|
|
47
46
|
// Types
|
|
@@ -58,7 +57,7 @@ export interface SchemaField {
|
|
|
58
57
|
* `type_mismatch` warning that previously fired on every integer-shaped
|
|
59
58
|
* field because the validator had no `"integer"` case. See vault#310.
|
|
60
59
|
*/
|
|
61
|
-
type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference"
|
|
60
|
+
type?: "string" | "number" | "integer" | "boolean" | "array" | "object" | "reference";
|
|
62
61
|
enum?: string[];
|
|
63
62
|
description?: string;
|
|
64
63
|
/**
|
|
@@ -321,82 +320,6 @@ export function resolveNoteSchemas(
|
|
|
321
320
|
return { effectiveTags, mergedFields, conflicts };
|
|
322
321
|
}
|
|
323
322
|
|
|
324
|
-
/**
|
|
325
|
-
* Normalize `date`-typed field VALUES in `metadata` to canonical UTC ISO
|
|
326
|
-
* form (`Z`-suffixed) BEFORE persisting (vault#date-field-type — mixed-
|
|
327
|
-
* offset corruption, caught in review).
|
|
328
|
-
*
|
|
329
|
-
* The bug: every consumer of an indexed `date` field — `buildOperatorClause`
|
|
330
|
-
* (query-operators.ts), `date_filter` and `order_by` (notes.ts) — compares
|
|
331
|
-
* the generated `meta_<field>` column as raw TEXT. ISO-8601 strings sort
|
|
332
|
-
* correctly under a TEXT compare ONLY when every value shares the same
|
|
333
|
-
* offset representation. `timestampToMs` (cursor.ts) — the validator both
|
|
334
|
-
* `defaultMatchesType` and `valueMatchesType` reuse — correctly ACCEPTS an
|
|
335
|
-
* explicit `±HH:MM` offset (validation was never the gap), but a value like
|
|
336
|
-
* `"2026-07-16T10:00:00+02:00"` (= `08:00Z`) persisted VERBATIM sorts AFTER
|
|
337
|
-
* `"2026-07-16T09:00:00Z"` (= `09:00Z`) under a raw string compare, even
|
|
338
|
-
* though the actual instant is earlier — a mixed-offset vault silently gets
|
|
339
|
-
* wrong range-query/order_by/date_filter results. This is the exact bug
|
|
340
|
-
* class `updated_at` got a dedicated ms-mirror column for (vault#585/#586,
|
|
341
|
-
* see `notes.ts`'s `dateFilter` block) that never extended to user-declared
|
|
342
|
-
* `date` fields.
|
|
343
|
-
*
|
|
344
|
-
* The fix is normalize-on-write, not reject-on-write (matching the
|
|
345
|
-
* "paths are normalized on write" precedent — instant preserved,
|
|
346
|
-
* representation canonicalized) — rejecting offsets would defeat the
|
|
347
|
-
* type's own motivation (calendar integrations and other emitters commonly
|
|
348
|
-
* produce offset timestamps, not always `Z`). Only a FULL timestamp (has a
|
|
349
|
-
* time component) is rewritten, to `new Date(ms).toISOString()` — always
|
|
350
|
-
* UTC, millisecond precision, `Z`-suffixed. A bare `YYYY-MM-DD` value is
|
|
351
|
-
* left untouched: it has no offset to normalize, is already canonical, and
|
|
352
|
-
* prefix-sorts correctly against full timestamps sharing the same calendar
|
|
353
|
-
* day. A value that fails `timestampToMs` is left untouched too — that's a
|
|
354
|
-
* `type_mismatch` for `valueMatchesType` to catch, not this function's job.
|
|
355
|
-
*
|
|
356
|
-
* COPY-ON-WRITE, never mutates the input `metadata` object (vault#date-
|
|
357
|
-
* field-type review round 2) — `core/` is a published library; a direct
|
|
358
|
-
* embedder holding a reference to the object THEY passed in must never see
|
|
359
|
-
* it change out from under them. Returns the SAME reference when nothing
|
|
360
|
-
* needs rewriting (the common case — no unnecessary allocation on the
|
|
361
|
-
* no-op path) or `metadata` is undefined; returns a freshly shallow-copied
|
|
362
|
-
* object, with only the rewritten field(s) replaced, the first time a
|
|
363
|
-
* rewrite is actually needed. Every call site must use the RETURNED value
|
|
364
|
-
* (not assume its input was mutated) — see `store.createNote`/`updateNote`/
|
|
365
|
-
* `createNotes`/`createNoteRaw`, all of which reassign their local
|
|
366
|
-
* `opts`/`updates`/`input` binding from the return rather than relying on a
|
|
367
|
-
* side effect.
|
|
368
|
-
*
|
|
369
|
-
* Called from `store.createNote`/`updateNote`/`createNotes`/`createNoteRaw`
|
|
370
|
-
* — the lowest chokepoint every write path (MCP, REST, import) funnels
|
|
371
|
-
* through — BEFORE the row is persisted, so the value that's validated by
|
|
372
|
-
* `validateNote`/`valueMatchesType` upstream (offset-tolerant, so
|
|
373
|
-
* normalization can't newly fail a write) and the value that's written +
|
|
374
|
-
* echoed back on the response are the SAME normalized string.
|
|
375
|
-
*/
|
|
376
|
-
export function normalizeDateFields(
|
|
377
|
-
resolved: ResolvedSchemas,
|
|
378
|
-
note: { tags?: string[]; metadata?: Record<string, unknown> },
|
|
379
|
-
): Record<string, unknown> | undefined {
|
|
380
|
-
const metadata = note.metadata;
|
|
381
|
-
if (!metadata) return metadata;
|
|
382
|
-
const { mergedFields } = resolveNoteSchemas(resolved, note);
|
|
383
|
-
let copy: Record<string, unknown> | undefined;
|
|
384
|
-
for (const [fieldName, { spec }] of mergedFields) {
|
|
385
|
-
if (spec.type !== "date") continue;
|
|
386
|
-
const value = metadata[fieldName];
|
|
387
|
-
if (typeof value !== "string") continue;
|
|
388
|
-
// Bare `YYYY-MM-DD` (no time component) has no offset to normalize.
|
|
389
|
-
if (/^\d{4}-\d{2}-\d{2}$/.test(value)) continue;
|
|
390
|
-
const ms = timestampToMs(value);
|
|
391
|
-
if (ms === null) continue; // unparseable — valueMatchesType's job, not ours
|
|
392
|
-
const canonical = new Date(ms).toISOString();
|
|
393
|
-
if (canonical === value) continue;
|
|
394
|
-
if (!copy) copy = { ...metadata }; // lazy: only copy once a rewrite is due
|
|
395
|
-
copy[fieldName] = canonical;
|
|
396
|
-
}
|
|
397
|
-
return copy ?? metadata;
|
|
398
|
-
}
|
|
399
|
-
|
|
400
323
|
function fieldSpecsEqual(a: SchemaField, b: SchemaField): boolean {
|
|
401
324
|
if (a.type !== b.type) return false;
|
|
402
325
|
if (!stringArraysEqual(a.enum, b.enum)) return false;
|
|
@@ -467,13 +390,6 @@ function valueMatchesType(value: unknown, type: SchemaField["type"], cardinality
|
|
|
467
390
|
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
468
391
|
}
|
|
469
392
|
return typeof value === "string";
|
|
470
|
-
// `date` validates like `string` — an ISO-8601 date (`YYYY-MM-DD`) or
|
|
471
|
-
// full RFC3339 timestamp. Reuses `cursor.ts`'s `timestampToMs` (the SAME
|
|
472
|
-
// UTC-correct parser `date_filter`'s `updated_at` bound uses), not a
|
|
473
|
-
// second date parser — see tag-schemas.ts's `VALID_FIELD_TYPES` doc
|
|
474
|
-
// comment.
|
|
475
|
-
case "date":
|
|
476
|
-
return typeof value === "string" && timestampToMs(value) !== null;
|
|
477
393
|
}
|
|
478
394
|
}
|
|
479
395
|
|
|
@@ -132,15 +132,7 @@ describe("search — v24 → v25 migration (legacy vault upgrade)", () => {
|
|
|
132
132
|
);
|
|
133
133
|
insert.run(
|
|
134
134
|
"legacy-1",
|
|
135
|
-
|
|
136
|
-
// line — this fixture isolates the PATH-weighted bm25 axis (below)
|
|
137
|
-
// from the separate content-title-boost axis (title axis, ratified
|
|
138
|
-
// 2026-07-17): that boost derives a note's "title" from its FIRST
|
|
139
|
-
// content line, and a single-line note IS its own title in full, so
|
|
140
|
-
// a one-line body mentioning "propolis" would (correctly, under the
|
|
141
|
-
// newer axis) count as a title match and confound this test's
|
|
142
|
-
// path-only signal. See `search-title-boost.test.ts` for that axis.
|
|
143
|
-
"Notes on foraging patterns\n\na dedicated writeup mentioning propolis only once in passing text",
|
|
135
|
+
"a dedicated writeup mentioning propolis only once in passing text",
|
|
144
136
|
"beekeeping-notes",
|
|
145
137
|
"2026-01-01T00:00:00.000Z",
|
|
146
138
|
"2026-01-01T00:00:00.000Z",
|
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
escapeFtsToken,
|
|
5
5
|
buildLiteralSearchQuery,
|
|
6
6
|
isValidSearchMode,
|
|
7
|
-
extractLiteralBoostTerms,
|
|
8
7
|
} from "./search-query.js";
|
|
9
8
|
|
|
10
9
|
describe("escapeFtsToken", () => {
|
|
@@ -148,44 +147,3 @@ describe("isValidSearchMode / SEARCH_MODES", () => {
|
|
|
148
147
|
expect(isValidSearchMode(["literal"])).toBe(false);
|
|
149
148
|
});
|
|
150
149
|
});
|
|
151
|
-
|
|
152
|
-
describe("extractLiteralBoostTerms (search title-boost input, title axis)", () => {
|
|
153
|
-
it("lowercases and whitespace-splits", () => {
|
|
154
|
-
expect(extractLiteralBoostTerms("Budget Review")).toEqual(["budget", "review"]);
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
it("collapses whitespace runs", () => {
|
|
158
|
-
expect(extractLiteralBoostTerms("widgets gadgets")).toEqual(["widgets", "gadgets"]);
|
|
159
|
-
});
|
|
160
|
-
|
|
161
|
-
it("trims leading/trailing whitespace", () => {
|
|
162
|
-
expect(extractLiteralBoostTerms(" widgets ")).toEqual(["widgets"]);
|
|
163
|
-
});
|
|
164
|
-
|
|
165
|
-
it("strips literal quote characters a caller typed, unlike buildLiteralSearchQuery", () => {
|
|
166
|
-
// buildLiteralSearchQuery treats a manually-typed `"` as ordinary
|
|
167
|
-
// content (escaped into the FTS phrase); the boost-term extractor
|
|
168
|
-
// strips it instead, since a title-string comparison shouldn't
|
|
169
|
-
// require the title to contain a literal quote mark.
|
|
170
|
-
expect(extractLiteralBoostTerms(`"eleven-day capping delay"`)).toEqual([
|
|
171
|
-
"eleven-day",
|
|
172
|
-
"capping",
|
|
173
|
-
"delay",
|
|
174
|
-
]);
|
|
175
|
-
});
|
|
176
|
-
|
|
177
|
-
it("preserves punctuation that isn't a quote (hyphen, period, apostrophe)", () => {
|
|
178
|
-
expect(extractLiteralBoostTerms("eleven-day")).toEqual(["eleven-day"]);
|
|
179
|
-
expect(extractLiteralBoostTerms("18.6")).toEqual(["18.6"]);
|
|
180
|
-
expect(extractLiteralBoostTerms("didn't")).toEqual(["didn't"]);
|
|
181
|
-
});
|
|
182
|
-
|
|
183
|
-
it("returns an empty array for blank/whitespace-only input", () => {
|
|
184
|
-
expect(extractLiteralBoostTerms("")).toEqual([]);
|
|
185
|
-
expect(extractLiteralBoostTerms(" ")).toEqual([]);
|
|
186
|
-
});
|
|
187
|
-
|
|
188
|
-
it("sanitizes control characters as separators, matching buildLiteralSearchQuery", () => {
|
|
189
|
-
expect(extractLiteralBoostTerms("hello\x00world")).toEqual(["hello", "world"]);
|
|
190
|
-
});
|
|
191
|
-
});
|
package/core/src/search-query.ts
CHANGED
|
@@ -153,30 +153,3 @@ export function buildLiteralSearchQuery(raw: string): LiteralSearchQuery {
|
|
|
153
153
|
const tokens = cleaned.trim().split(/\s+/).filter(Boolean);
|
|
154
154
|
return { query: tokens.map(escapeFtsToken).join(" "), isEmpty: false };
|
|
155
155
|
}
|
|
156
|
-
|
|
157
|
-
/**
|
|
158
|
-
* Extract lowercase whitespace-delimited terms from raw literal-mode search
|
|
159
|
-
* text, for the search title-boost post-rank (title axis, ratified
|
|
160
|
-
* 2026-07-17) — NOT used to build the FTS5 MATCH query itself (that's
|
|
161
|
-
* `buildLiteralSearchQuery`, above). Sanitizes control characters the same
|
|
162
|
-
* way, plus strips any literal `"` a caller typed (kept as ordinary content
|
|
163
|
-
* by `buildLiteralSearchQuery`, but a display-title substring check
|
|
164
|
-
* shouldn't require the title to contain a literal quote mark).
|
|
165
|
-
*
|
|
166
|
-
* This is a heuristic re-rank signal, not a second index — it deliberately
|
|
167
|
-
* does not attempt FTS5-tokenizer parity (stemming, unicode normalization).
|
|
168
|
-
* It only needs to answer "does the note's first line plausibly contain the
|
|
169
|
-
* query," not reproduce FTS5's match semantics exactly. Restricted to
|
|
170
|
-
* literal-mode callers by convention (see `core/src/notes.ts` `searchNotes`)
|
|
171
|
-
* — `search_mode: "advanced"` text carries FTS5 boolean/column/prefix
|
|
172
|
-
* operators (`AND`, `path:`, `foo*`) that would leak into the term list as
|
|
173
|
-
* false "content" if tokenized the same naive way.
|
|
174
|
-
*/
|
|
175
|
-
export function extractLiteralBoostTerms(raw: string): string[] {
|
|
176
|
-
const cleaned = raw.replace(CONTROL_CHARS, " ").toLowerCase();
|
|
177
|
-
return cleaned
|
|
178
|
-
.trim()
|
|
179
|
-
.split(/\s+/)
|
|
180
|
-
.map((t) => t.replace(/^"+|"+$/g, ""))
|
|
181
|
-
.filter(Boolean);
|
|
182
|
-
}
|
|
@@ -20,10 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
import { describe, test, expect } from "bun:test";
|
|
22
22
|
import {
|
|
23
|
-
ALL_NOTES_VIEW_PATH,
|
|
24
23
|
applySeedPack,
|
|
25
|
-
ARCHIVE_VIEW_PATH,
|
|
26
|
-
ARCHIVED_TAG,
|
|
27
24
|
CAPTURE_ANYTHING_PATH,
|
|
28
25
|
CONNECT_AI_PATH,
|
|
29
26
|
DEFAULT_VAULT_DESCRIPTION,
|
|
@@ -36,17 +33,11 @@ import {
|
|
|
36
33
|
GUIDE_TAG,
|
|
37
34
|
listSeedPacks,
|
|
38
35
|
NOTES_REQUIRED_TAGS,
|
|
39
|
-
PINNED_TAG,
|
|
40
|
-
PINNED_VIEW_PATH,
|
|
41
|
-
RECENT_VIEW_PATH,
|
|
42
36
|
SEED_PACK_NAMES,
|
|
43
|
-
STARTER_ONTOLOGY_PACK,
|
|
44
37
|
SURFACE_STARTER_CONTENT,
|
|
45
38
|
SURFACE_STARTER_PACK,
|
|
46
39
|
SURFACE_STARTER_PATH,
|
|
47
40
|
TAGS_GRAPH_PATH,
|
|
48
|
-
VIEW_TAG,
|
|
49
|
-
VIEWS_PATH_PREFIX,
|
|
50
41
|
WELCOME_PATH,
|
|
51
42
|
welcomePack,
|
|
52
43
|
YOURS_TO_KEEP_PATH,
|
|
@@ -344,119 +335,12 @@ describe("surface-starter pack", () => {
|
|
|
344
335
|
});
|
|
345
336
|
});
|
|
346
337
|
|
|
347
|
-
describe("starter-ontology pack", () => {
|
|
348
|
-
test("exactly four tags, in constitution order: view, archived, pinned, capture", () => {
|
|
349
|
-
expect(STARTER_ONTOLOGY_PACK.name).toBe("starter-ontology");
|
|
350
|
-
expect(STARTER_ONTOLOGY_PACK.tags.map((t) => t.name)).toEqual([
|
|
351
|
-
"view",
|
|
352
|
-
"archived",
|
|
353
|
-
"pinned",
|
|
354
|
-
"capture",
|
|
355
|
-
]);
|
|
356
|
-
});
|
|
357
|
-
|
|
358
|
-
test("opt-in, not seeded by default", () => {
|
|
359
|
-
expect(STARTER_ONTOLOGY_PACK.description).toContain("Opt-in — not seeded by default.");
|
|
360
|
-
});
|
|
361
|
-
|
|
362
|
-
test("view is the one meta tag — carries the kind/query/lane_by/date_field schema", () => {
|
|
363
|
-
expect(VIEW_TAG.name).toBe("view");
|
|
364
|
-
expect(VIEW_TAG.description).toContain("meta tag");
|
|
365
|
-
expect(VIEW_TAG.fields).toBeDefined();
|
|
366
|
-
expect(Object.keys(VIEW_TAG.fields!)).toEqual([
|
|
367
|
-
"kind",
|
|
368
|
-
"query",
|
|
369
|
-
"lane_by",
|
|
370
|
-
"date_field",
|
|
371
|
-
]);
|
|
372
|
-
|
|
373
|
-
const kind = VIEW_TAG.fields!.kind!;
|
|
374
|
-
expect(kind.type).toBe("string");
|
|
375
|
-
expect(kind.enum).toEqual(["list", "board", "calendar", "gallery"]);
|
|
376
|
-
expect(kind.default).toBe("list");
|
|
377
|
-
|
|
378
|
-
const query = VIEW_TAG.fields!.query!;
|
|
379
|
-
expect(query.type).toBe("string");
|
|
380
|
-
|
|
381
|
-
// lane_by / date_field are optional (no default — absent stays absent).
|
|
382
|
-
expect(VIEW_TAG.fields!.lane_by!.default).toBeUndefined();
|
|
383
|
-
expect(VIEW_TAG.fields!.date_field!.default).toBeUndefined();
|
|
384
|
-
});
|
|
385
|
-
|
|
386
|
-
test("archived + pinned are plain tags — no schema", () => {
|
|
387
|
-
expect(ARCHIVED_TAG.name).toBe("archived");
|
|
388
|
-
expect(ARCHIVED_TAG.fields).toBeUndefined();
|
|
389
|
-
expect(ARCHIVED_TAG.description).toContain("out of the flow of the present");
|
|
390
|
-
|
|
391
|
-
expect(PINNED_TAG.name).toBe("pinned");
|
|
392
|
-
expect(PINNED_TAG.fields).toBeUndefined();
|
|
393
|
-
expect(PINNED_TAG.description).toContain("partition");
|
|
394
|
-
});
|
|
395
|
-
|
|
396
|
-
test("capture is reused verbatim from NOTES_REQUIRED_TAGS — byte-equal, no drift risk", () => {
|
|
397
|
-
const capture = STARTER_ONTOLOGY_PACK.tags.find((t) => t.name === "capture");
|
|
398
|
-
expect(capture).toBe(NOTES_REQUIRED_TAGS[0]); // SAME object reference, not just equal content
|
|
399
|
-
expect(capture!.description).toBe(NOTES_REQUIRED_TAGS[0]!.description);
|
|
400
|
-
});
|
|
401
|
-
|
|
402
|
-
test("four seed #view notes mirroring the app's default pages", () => {
|
|
403
|
-
expect(STARTER_ONTOLOGY_PACK.notes.map((n) => n.path)).toEqual([
|
|
404
|
-
ALL_NOTES_VIEW_PATH,
|
|
405
|
-
RECENT_VIEW_PATH,
|
|
406
|
-
PINNED_VIEW_PATH,
|
|
407
|
-
ARCHIVE_VIEW_PATH,
|
|
408
|
-
]);
|
|
409
|
-
for (const note of STARTER_ONTOLOGY_PACK.notes) {
|
|
410
|
-
expect(note.path.startsWith(VIEWS_PATH_PREFIX)).toBe(true);
|
|
411
|
-
expect(note.tags).toEqual(["view"]);
|
|
412
|
-
expect(note.metadata?.kind).toBe("list");
|
|
413
|
-
expect(typeof note.metadata?.query).toBe("string");
|
|
414
|
-
// Every seeded query must itself be valid JSON — a downstream reader
|
|
415
|
-
// parses it as query-notes params.
|
|
416
|
-
expect(() => JSON.parse(note.metadata!.query as string)).not.toThrow();
|
|
417
|
-
}
|
|
418
|
-
});
|
|
419
|
-
|
|
420
|
-
test("All notes / Recent exclude archived explicitly (authoring-time default, not magic)", () => {
|
|
421
|
-
for (const path of [ALL_NOTES_VIEW_PATH, RECENT_VIEW_PATH]) {
|
|
422
|
-
const note = noteAt(STARTER_ONTOLOGY_PACK, path);
|
|
423
|
-
const query = JSON.parse(note.metadata!.query as string);
|
|
424
|
-
expect(query.exclude_tags).toEqual(["archived"]);
|
|
425
|
-
}
|
|
426
|
-
});
|
|
427
|
-
|
|
428
|
-
test("Recent orders by updated_at desc", () => {
|
|
429
|
-
const query = JSON.parse(noteAt(STARTER_ONTOLOGY_PACK, RECENT_VIEW_PATH).metadata!.query as string);
|
|
430
|
-
expect(query.order_by).toBe("updated_at");
|
|
431
|
-
expect(query.sort).toBe("desc");
|
|
432
|
-
});
|
|
433
|
-
|
|
434
|
-
test("Pinned queries tag:pinned; Archive queries tag:archived (the deliberate exception to exclude_tags)", () => {
|
|
435
|
-
const pinnedQuery = JSON.parse(noteAt(STARTER_ONTOLOGY_PACK, PINNED_VIEW_PATH).metadata!.query as string);
|
|
436
|
-
expect(pinnedQuery.tag).toBe("pinned");
|
|
437
|
-
|
|
438
|
-
const archiveQuery = JSON.parse(noteAt(STARTER_ONTOLOGY_PACK, ARCHIVE_VIEW_PATH).metadata!.query as string);
|
|
439
|
-
expect(archiveQuery.tag).toBe("archived");
|
|
440
|
-
expect(archiveQuery.exclude_tags).toBeUndefined();
|
|
441
|
-
});
|
|
442
|
-
|
|
443
|
-
test("no dangling wikilinks within the pack", () => {
|
|
444
|
-
const seededPaths = new Set(STARTER_ONTOLOGY_PACK.notes.map((n) => n.path));
|
|
445
|
-
for (const note of STARTER_ONTOLOGY_PACK.notes) {
|
|
446
|
-
for (const target of wikilinkTargets(note.content)) {
|
|
447
|
-
expect(seededPaths.has(target)).toBe(true);
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
});
|
|
451
|
-
});
|
|
452
|
-
|
|
453
338
|
describe("pack registry", () => {
|
|
454
|
-
test("lists exactly the
|
|
339
|
+
test("lists exactly the three packs, in order", () => {
|
|
455
340
|
expect([...SEED_PACK_NAMES]).toEqual([
|
|
456
341
|
"welcome",
|
|
457
342
|
"getting-started",
|
|
458
343
|
"surface-starter",
|
|
459
|
-
"starter-ontology",
|
|
460
344
|
]);
|
|
461
345
|
expect(listSeedPacks().map((p) => p.name)).toEqual([...SEED_PACK_NAMES]);
|
|
462
346
|
for (const p of listSeedPacks()) {
|