@monoes/monomindcli 2.14.0 → 2.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/monodesign/scripts/context.mjs +1 -1
- package/.claude/skills/monodesign/scripts/detector/detect-antipatterns-browser.js +2 -2
- package/.claude/skills/monodesign/scripts/detector/rules/checks.mjs +2 -2
- package/.claude/skills/monodesign/scripts/live-browser.js +1 -1
- package/bin/cli.js +21 -1
- package/bin/mcp-server.js +21 -1
- package/dist/src/capabilities/cap-documents.d.ts.map +1 -1
- package/dist/src/capabilities/cap-documents.js +23 -0
- package/dist/src/capabilities/cap-documents.js.map +1 -1
- package/dist/src/commands/doc-filters.d.ts +19 -0
- package/dist/src/commands/doc-filters.d.ts.map +1 -0
- package/dist/src/commands/doc-filters.js +58 -0
- package/dist/src/commands/doc-filters.js.map +1 -0
- package/dist/src/commands/doc-library.d.ts +21 -0
- package/dist/src/commands/doc-library.d.ts.map +1 -0
- package/dist/src/commands/doc-library.js +390 -0
- package/dist/src/commands/doc-library.js.map +1 -0
- package/dist/src/commands/doc-list.d.ts +9 -0
- package/dist/src/commands/doc-list.d.ts.map +1 -0
- package/dist/src/commands/doc-list.js +99 -0
- package/dist/src/commands/doc-list.js.map +1 -0
- package/dist/src/commands/doc.d.ts.map +1 -1
- package/dist/src/commands/doc.js +49 -37
- package/dist/src/commands/doc.js.map +1 -1
- package/dist/src/knowledge/capture-envelope.d.ts +80 -0
- package/dist/src/knowledge/capture-envelope.d.ts.map +1 -0
- package/dist/src/knowledge/capture-envelope.js +165 -0
- package/dist/src/knowledge/capture-envelope.js.map +1 -0
- package/dist/src/knowledge/capture-text.d.ts +23 -0
- package/dist/src/knowledge/capture-text.d.ts.map +1 -0
- package/dist/src/knowledge/capture-text.js +49 -0
- package/dist/src/knowledge/capture-text.js.map +1 -0
- package/dist/src/knowledge/citation.d.ts +121 -0
- package/dist/src/knowledge/citation.d.ts.map +1 -0
- package/dist/src/knowledge/citation.js +252 -0
- package/dist/src/knowledge/citation.js.map +1 -0
- package/dist/src/knowledge/document-pipeline.d.ts +60 -0
- package/dist/src/knowledge/document-pipeline.d.ts.map +1 -1
- package/dist/src/knowledge/document-pipeline.js +188 -4
- package/dist/src/knowledge/document-pipeline.js.map +1 -1
- package/dist/src/knowledge/html-extract.d.ts +43 -0
- package/dist/src/knowledge/html-extract.d.ts.map +1 -0
- package/dist/src/knowledge/html-extract.js +286 -0
- package/dist/src/knowledge/html-extract.js.map +1 -0
- package/dist/src/knowledge/html-tags.d.ts +30 -0
- package/dist/src/knowledge/html-tags.d.ts.map +1 -0
- package/dist/src/knowledge/html-tags.js +229 -0
- package/dist/src/knowledge/html-tags.js.map +1 -0
- package/dist/src/knowledge/library.d.ts +106 -0
- package/dist/src/knowledge/library.d.ts.map +1 -0
- package/dist/src/knowledge/library.js +195 -0
- package/dist/src/knowledge/library.js.map +1 -0
- package/dist/src/knowledge/mhtml.d.ts +60 -0
- package/dist/src/knowledge/mhtml.d.ts.map +1 -0
- package/dist/src/knowledge/mhtml.js +154 -0
- package/dist/src/knowledge/mhtml.js.map +1 -0
- package/dist/src/knowledge/related.d.ts +58 -0
- package/dist/src/knowledge/related.d.ts.map +1 -0
- package/dist/src/knowledge/related.js +222 -0
- package/dist/src/knowledge/related.js.map +1 -0
- package/dist/src/knowledge/section-diff.d.ts +48 -0
- package/dist/src/knowledge/section-diff.d.ts.map +1 -0
- package/dist/src/knowledge/section-diff.js +151 -0
- package/dist/src/knowledge/section-diff.js.map +1 -0
- package/dist/src/knowledge/watch.d.ts +92 -0
- package/dist/src/knowledge/watch.d.ts.map +1 -0
- package/dist/src/knowledge/watch.js +221 -0
- package/dist/src/knowledge/watch.js.map +1 -0
- package/dist/src/mcp-server.d.ts.map +1 -1
- package/dist/src/mcp-server.js +44 -87
- package/dist/src/mcp-server.js.map +1 -1
- package/dist/src/mcp-tools/browser-instrument-tools.d.ts +35 -0
- package/dist/src/mcp-tools/browser-instrument-tools.d.ts.map +1 -0
- package/dist/src/mcp-tools/browser-instrument-tools.js +359 -0
- package/dist/src/mcp-tools/browser-instrument-tools.js.map +1 -0
- package/dist/src/mcp-tools/browser-metrics.d.ts +42 -0
- package/dist/src/mcp-tools/browser-metrics.d.ts.map +1 -0
- package/dist/src/mcp-tools/browser-metrics.js +91 -0
- package/dist/src/mcp-tools/browser-metrics.js.map +1 -0
- package/dist/src/mcp-tools/browser-profile-tools.d.ts +16 -0
- package/dist/src/mcp-tools/browser-profile-tools.d.ts.map +1 -0
- package/dist/src/mcp-tools/browser-profile-tools.js +324 -0
- package/dist/src/mcp-tools/browser-profile-tools.js.map +1 -0
- package/dist/src/mcp-tools/browser-session.d.ts +68 -0
- package/dist/src/mcp-tools/browser-session.d.ts.map +1 -0
- package/dist/src/mcp-tools/browser-session.js +224 -0
- package/dist/src/mcp-tools/browser-session.js.map +1 -0
- package/dist/src/mcp-tools/browser-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/browser-tools.js +10 -176
- package/dist/src/mcp-tools/browser-tools.js.map +1 -1
- package/dist/src/mcp-tools/capture-resource-read.d.ts +115 -0
- package/dist/src/mcp-tools/capture-resource-read.d.ts.map +1 -0
- package/dist/src/mcp-tools/capture-resource-read.js +296 -0
- package/dist/src/mcp-tools/capture-resource-read.js.map +1 -0
- package/dist/src/mcp-tools/capture-resource-tools.d.ts +22 -0
- package/dist/src/mcp-tools/capture-resource-tools.d.ts.map +1 -0
- package/dist/src/mcp-tools/capture-resource-tools.js +182 -0
- package/dist/src/mcp-tools/capture-resource-tools.js.map +1 -0
- package/dist/src/mcp-tools/capture-resources.d.ts +142 -0
- package/dist/src/mcp-tools/capture-resources.d.ts.map +1 -0
- package/dist/src/mcp-tools/capture-resources.js +289 -0
- package/dist/src/mcp-tools/capture-resources.js.map +1 -0
- package/dist/src/mcp-tools/index.d.ts +3 -0
- package/dist/src/mcp-tools/index.d.ts.map +1 -1
- package/dist/src/mcp-tools/index.js +6 -0
- package/dist/src/mcp-tools/index.js.map +1 -1
- package/dist/src/mcp-tools/knowledge-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/knowledge-tools.js +9 -1
- package/dist/src/mcp-tools/knowledge-tools.js.map +1 -1
- package/dist/src/mcp-tools/resource-router.d.ts +86 -0
- package/dist/src/mcp-tools/resource-router.d.ts.map +1 -0
- package/dist/src/mcp-tools/resource-router.js +181 -0
- package/dist/src/mcp-tools/resource-router.js.map +1 -0
- package/dist/src/memory/memory-bridge.js +1 -1
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/src/orgrt/session-ledger.d.ts +7 -0
- package/dist/src/orgrt/session-ledger.d.ts.map +1 -1
- package/dist/src/orgrt/session-ledger.js +15 -0
- package/dist/src/orgrt/session-ledger.js.map +1 -1
- package/dist/src/orgrt/session.d.ts.map +1 -1
- package/dist/src/orgrt/session.js +21 -8
- package/dist/src/orgrt/session.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +6 -6
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"library.d.ts","sourceRoot":"","sources":["../../../src/knowledge/library.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,WAAW,aAAa;IAC5B,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,4CAA4C;IAC5C,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;AAE7E,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,QAAQ,EAAE,OAAO,CAAC;CACnB;AAID,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,IAAI,EAAE;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC,GAAG,MAAM,GAAG,SAAS,CAQrB;AAED,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC,GAAG,MAAM,CAET;AAUD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,SAAa,GAAG,MAAM,GAAG,IAAI,CAOpF;AAkBD,iDAAiD;AACjD,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,YAAY,EAClB,MAAM,GAAE,aAAkB,EAC1B,GAAG,SAAa,GACf,OAAO,CAqCT;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,YAAY,EAAE,EACpB,MAAM,GAAE,aAAkB,EAC1B,GAAG,SAAa,GACf,YAAY,EAAE,CAEhB;AAED,wBAAgB,aAAa,CAC3B,IAAI,EAAE,YAAY,EAAE,EACpB,IAAI,GAAE,WAAwB,EAC9B,KAAK,GAAE,KAAK,GAAG,MAAe,GAC7B,YAAY,EAAE,CAsBhB;AAED,uEAAuE;AACvE,wBAAgB,YAAY,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,CAmB7D;AAED,MAAM,WAAW,kBAAmB,SAAQ,aAAa;IACvD,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,6EAA6E;AAC7E,wBAAgB,WAAW,CAAC,IAAI,EAAE,YAAY,EAAE,EAAE,IAAI,GAAE,kBAAuB,GAAG,YAAY,EAAE,CAM/F;AAED,kEAAkE;AAClE,wBAAgB,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG;IACnD,KAAK,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/C,IAAI,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9C,WAAW,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrD,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAClD,CAiBA"}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The capture library (RCL-09) — browsing what has been captured, rather than
|
|
3
|
+
* searching it.
|
|
4
|
+
*
|
|
5
|
+
* `listDocuments` answers "what is indexed". A library answers a different
|
|
6
|
+
* question: everything from this site, captured last week, tagged `reading`,
|
|
7
|
+
* newest first. The facets come from the capture envelope's `meta.json`
|
|
8
|
+
* (RCL-07), which is already on every record — this module is the query layer
|
|
9
|
+
* over them, and the data layer the desktop GUI renders.
|
|
10
|
+
*
|
|
11
|
+
* Rules, once, here:
|
|
12
|
+
* - within a facet values are OR'd, across facets AND'd (`--site a --site b
|
|
13
|
+
* --tag x` = "from a or b, tagged x");
|
|
14
|
+
* - `--site example.com` matches `www.example.com` and `docs.example.com`,
|
|
15
|
+
* because nobody browsing their own library means the bare host only;
|
|
16
|
+
* - a document with no provenance (an ordinary file on disk) sorts by its
|
|
17
|
+
* index time and matches no capture facet — it is not a capture.
|
|
18
|
+
*
|
|
19
|
+
* @module v1/cli/knowledge/library
|
|
20
|
+
*/
|
|
21
|
+
import * as path from 'node:path';
|
|
22
|
+
// ── Facet extraction ───────────────────────────────────────────────
|
|
23
|
+
/** The host a document was captured from, or undefined for a local file. */
|
|
24
|
+
export function documentSite(meta) {
|
|
25
|
+
const url = meta.canonicalUrl ?? meta.provenance?.canonicalUrl ?? meta.provenance?.url;
|
|
26
|
+
if (!url)
|
|
27
|
+
return undefined;
|
|
28
|
+
try {
|
|
29
|
+
return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** When the page was captured; falls back to when it was indexed. */
|
|
36
|
+
export function documentCapturedAt(meta) {
|
|
37
|
+
return meta.provenance?.capturedAt ?? meta.indexedAt ?? '';
|
|
38
|
+
}
|
|
39
|
+
const RELATIVE_RE = /^(\d+(?:\.\d+)?)\s*(m|h|d|w)$/i;
|
|
40
|
+
const UNIT_MS = {
|
|
41
|
+
m: 60_000,
|
|
42
|
+
h: 3_600_000,
|
|
43
|
+
d: 86_400_000,
|
|
44
|
+
w: 604_800_000,
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* A point in time from `2026-09-01`, a full ISO stamp, or an AGE like `7d`
|
|
48
|
+
* (meaning "7 days ago"). Returns null for anything unparseable so a typo
|
|
49
|
+
* cannot silently filter the whole library away.
|
|
50
|
+
*/
|
|
51
|
+
export function parseWhen(value, now = Date.now()) {
|
|
52
|
+
const raw = (value ?? '').trim();
|
|
53
|
+
if (!raw)
|
|
54
|
+
return null;
|
|
55
|
+
const rel = RELATIVE_RE.exec(raw);
|
|
56
|
+
if (rel)
|
|
57
|
+
return now - Number(rel[1]) * UNIT_MS[rel[2].toLowerCase()];
|
|
58
|
+
const parsed = Date.parse(raw.length === 10 ? `${raw}T00:00:00.000Z` : raw);
|
|
59
|
+
return Number.isNaN(parsed) ? null : parsed;
|
|
60
|
+
}
|
|
61
|
+
function siteMatches(host, wanted) {
|
|
62
|
+
if (!host)
|
|
63
|
+
return false;
|
|
64
|
+
return wanted.some((w) => {
|
|
65
|
+
const want = w
|
|
66
|
+
.trim()
|
|
67
|
+
.toLowerCase()
|
|
68
|
+
.replace(/^www\./, '');
|
|
69
|
+
return !!want && (host === want || host.endsWith(`.${want}`));
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
function anyOf(values, wanted) {
|
|
73
|
+
const have = (values ?? []).map((v) => v.toLowerCase());
|
|
74
|
+
return wanted.some((w) => have.includes(w.trim().toLowerCase()));
|
|
75
|
+
}
|
|
76
|
+
/** True when a document belongs in this view. */
|
|
77
|
+
export function matchesLibraryFilter(meta, filter = {}, now = Date.now()) {
|
|
78
|
+
if (filter.scope && meta.scope !== filter.scope)
|
|
79
|
+
return false;
|
|
80
|
+
const prov = meta.provenance;
|
|
81
|
+
const isCapture = !!prov || !!meta.canonicalUrl;
|
|
82
|
+
if (filter.capturedOnly && !isCapture)
|
|
83
|
+
return false;
|
|
84
|
+
if (filter.site?.length && !siteMatches(documentSite(meta), filter.site))
|
|
85
|
+
return false;
|
|
86
|
+
if (filter.tag?.length && !anyOf(prov?.tags, filter.tag))
|
|
87
|
+
return false;
|
|
88
|
+
if (filter.collection?.length &&
|
|
89
|
+
!anyOf(prov?.collection ? [prov.collection] : [], filter.collection))
|
|
90
|
+
return false;
|
|
91
|
+
if (filter.source?.length && !anyOf(prov?.source ? [prov.source] : [], filter.source))
|
|
92
|
+
return false;
|
|
93
|
+
if (filter.since || filter.until) {
|
|
94
|
+
const at = Date.parse(documentCapturedAt(meta));
|
|
95
|
+
const since = parseWhen(filter.since, now);
|
|
96
|
+
const until = parseWhen(filter.until, now);
|
|
97
|
+
if (Number.isNaN(at))
|
|
98
|
+
return false;
|
|
99
|
+
if (since !== null && at < since)
|
|
100
|
+
return false;
|
|
101
|
+
if (until !== null && at > until)
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
if (filter.text) {
|
|
105
|
+
const needle = filter.text.toLowerCase();
|
|
106
|
+
const hay = [
|
|
107
|
+
prov?.title ?? '',
|
|
108
|
+
meta.canonicalUrl ?? prov?.url ?? '',
|
|
109
|
+
meta.filePath,
|
|
110
|
+
prov?.note ?? '',
|
|
111
|
+
]
|
|
112
|
+
.join(' ')
|
|
113
|
+
.toLowerCase();
|
|
114
|
+
if (!hay.includes(needle))
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
export function filterDocuments(docs, filter = {}, now = Date.now()) {
|
|
120
|
+
return docs.filter((d) => matchesLibraryFilter(d, filter, now));
|
|
121
|
+
}
|
|
122
|
+
export function sortDocuments(docs, sort = 'captured', order = 'desc') {
|
|
123
|
+
const dir = order === 'asc' ? 1 : -1;
|
|
124
|
+
const key = (d) => {
|
|
125
|
+
switch (sort) {
|
|
126
|
+
case 'indexed':
|
|
127
|
+
return d.indexedAt ?? '';
|
|
128
|
+
case 'title':
|
|
129
|
+
return (d.provenance?.title ?? path.basename(d.filePath)).toLowerCase();
|
|
130
|
+
case 'path':
|
|
131
|
+
return d.filePath;
|
|
132
|
+
case 'size':
|
|
133
|
+
return d.size ?? 0;
|
|
134
|
+
default:
|
|
135
|
+
return documentCapturedAt(d);
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
return [...docs].sort((a, b) => {
|
|
139
|
+
const ka = key(a);
|
|
140
|
+
const kb = key(b);
|
|
141
|
+
if (ka === kb)
|
|
142
|
+
return a.filePath.localeCompare(b.filePath);
|
|
143
|
+
return ka < kb ? -dir : dir;
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
/** The flat, JSON-friendly row the GUI and `--json` consumers read. */
|
|
147
|
+
export function libraryEntry(meta) {
|
|
148
|
+
const prov = meta.provenance;
|
|
149
|
+
const url = meta.canonicalUrl ?? prov?.canonicalUrl ?? prov?.url;
|
|
150
|
+
return {
|
|
151
|
+
filePath: meta.filePath,
|
|
152
|
+
scope: meta.scope,
|
|
153
|
+
title: prov?.title ?? path.basename(meta.filePath),
|
|
154
|
+
...(url ? { url } : {}),
|
|
155
|
+
...(documentSite(meta) ? { site: documentSite(meta) } : {}),
|
|
156
|
+
capturedAt: documentCapturedAt(meta),
|
|
157
|
+
indexedAt: meta.indexedAt,
|
|
158
|
+
...(prov?.source ? { source: prov.source } : {}),
|
|
159
|
+
...(prov?.collection ? { collection: prov.collection } : {}),
|
|
160
|
+
tags: prov?.tags ?? [],
|
|
161
|
+
chunkCount: meta.chunkCount,
|
|
162
|
+
size: meta.size,
|
|
163
|
+
...(meta.version ? { version: meta.version } : {}),
|
|
164
|
+
captured: !!prov || !!meta.canonicalUrl,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/** Browse the library: filter, sort, cap. Never throws on an empty store. */
|
|
168
|
+
export function listLibrary(docs, opts = {}) {
|
|
169
|
+
const now = opts.now ?? Date.now();
|
|
170
|
+
const filtered = filterDocuments(docs, opts, now);
|
|
171
|
+
const sorted = sortDocuments(filtered, opts.sort ?? 'captured', opts.order ?? 'desc');
|
|
172
|
+
const capped = opts.limit && opts.limit > 0 ? sorted.slice(0, opts.limit) : sorted;
|
|
173
|
+
return capped.map(libraryEntry);
|
|
174
|
+
}
|
|
175
|
+
/** Facet counts for the current view — what a sidebar renders. */
|
|
176
|
+
export function libraryFacets(docs) {
|
|
177
|
+
const tally = (values) => {
|
|
178
|
+
const counts = new Map();
|
|
179
|
+
for (const v of values) {
|
|
180
|
+
if (!v)
|
|
181
|
+
continue;
|
|
182
|
+
counts.set(v, (counts.get(v) ?? 0) + 1);
|
|
183
|
+
}
|
|
184
|
+
return [...counts.entries()]
|
|
185
|
+
.map(([value, count]) => ({ value, count }))
|
|
186
|
+
.sort((a, b) => b.count - a.count || a.value.localeCompare(b.value));
|
|
187
|
+
};
|
|
188
|
+
return {
|
|
189
|
+
sites: tally(docs.map(documentSite)),
|
|
190
|
+
tags: tally(docs.flatMap((d) => d.provenance?.tags ?? [])),
|
|
191
|
+
collections: tally(docs.map((d) => d.provenance?.collection)),
|
|
192
|
+
sources: tally(docs.map((d) => d.provenance?.source)),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
//# sourceMappingURL=library.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"library.js","sourceRoot":"","sources":["../../../src/knowledge/library.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAyClC,sEAAsE;AAEtE,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,IAG5B;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,UAAU,EAAE,YAAY,IAAI,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,kBAAkB,CAAC,IAGlC;IACC,OAAO,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;AAC7D,CAAC;AAED,MAAM,WAAW,GAAG,gCAAgC,CAAC;AACrD,MAAM,OAAO,GAA2B;IACtC,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,SAAS;IACZ,CAAC,EAAE,UAAU;IACb,CAAC,EAAE,WAAW;CACf,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,KAAyB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IACnE,MAAM,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACjC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG;QAAE,OAAO,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5E,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC9C,CAAC;AAED,SAAS,WAAW,CAAC,IAAwB,EAAE,MAAgB;IAC7D,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,CAAC;aACX,IAAI,EAAE;aACN,WAAW,EAAE;aACb,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzB,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,KAAK,CAAC,MAA4B,EAAE,MAAgB;IAC3D,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,oBAAoB,CAClC,IAAkB,EAClB,MAAM,GAAkB,EAAE,EAC1B,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IAEhB,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IAChD,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IACpD,IAAI,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACvF,IAAI,MAAM,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACvE,IACE,MAAM,CAAC,UAAU,EAAE,MAAM;QACzB,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC;QAEpE,OAAO,KAAK,CAAC;IACf,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC;QACnF,OAAO,KAAK,CAAC;IAEf,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACnC,IAAI,KAAK,KAAK,IAAI,IAAI,EAAE,GAAG,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/C,IAAI,KAAK,KAAK,IAAI,IAAI,EAAE,GAAG,KAAK;YAAE,OAAO,KAAK,CAAC;IACjD,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG;YACV,IAAI,EAAE,KAAK,IAAI,EAAE;YACjB,IAAI,CAAC,YAAY,IAAI,IAAI,EAAE,GAAG,IAAI,EAAE;YACpC,IAAI,CAAC,QAAQ;YACb,IAAI,EAAE,IAAI,IAAI,EAAE;SACjB;aACE,IAAI,CAAC,GAAG,CAAC;aACT,WAAW,EAAE,CAAC;QACjB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,IAAoB,EACpB,MAAM,GAAkB,EAAE,EAC1B,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IAEhB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,IAAoB,EACpB,IAAI,GAAgB,UAAU,EAC9B,KAAK,GAAmB,MAAM;IAE9B,MAAM,GAAG,GAAG,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,CAAC,CAAe,EAAmB,EAAE;QAC/C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,SAAS;gBACZ,OAAO,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;YAC3B,KAAK,OAAO;gBACV,OAAO,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC1E,KAAK,MAAM;gBACT,OAAO,CAAC,CAAC,QAAQ,CAAC;YACpB,KAAK,MAAM;gBACT,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YACrB;gBACE,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7B,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAClB,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC3D,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,YAAY,CAAC,IAAkB;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,EAAE,YAAY,IAAI,IAAI,EAAE,GAAG,CAAC;IACjE,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;QAClD,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,UAAU,EAAE,kBAAkB,CAAC,IAAI,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChD,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE;QACtB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,QAAQ,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,YAAY;KACxC,CAAC;AACJ,CAAC;AASD,6EAA6E;AAC7E,MAAM,UAAU,WAAW,CAAC,IAAoB,EAAE,IAAI,GAAuB,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,IAAI,UAAU,EAAE,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACnF,OAAO,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAClC,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,aAAa,CAAC,IAAoB;IAMhD,MAAM,KAAK,GAAG,CAAC,MAAiC,EAAE,EAAE;QAClD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aACzB,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC;IACF,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;QAC1D,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC7D,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;KACtD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MHTML / MHT container parsing (RCL-01).
|
|
3
|
+
*
|
|
4
|
+
* An MHTML archive is an RFC 2557 `multipart/related` MIME message: the page
|
|
5
|
+
* plus every subresource it referenced, each part carrying its own transfer
|
|
6
|
+
* encoding and charset. Chrome's `Page.captureSnapshot` writes them
|
|
7
|
+
* quoted-printable; Internet Explorer and most "save as web archive" tools
|
|
8
|
+
* write base64.
|
|
9
|
+
*
|
|
10
|
+
* Only the ROOT frame is prose. Picking the first `text/html` part instead is
|
|
11
|
+
* how an ad iframe ends up indexed as the article, so the root is resolved
|
|
12
|
+
* from the container's `Snapshot-Content-Location` / `Content-Location` when
|
|
13
|
+
* one is present.
|
|
14
|
+
*
|
|
15
|
+
* Read the file as `latin1` (or pass a Buffer): byte fidelity matters before
|
|
16
|
+
* transfer decoding, and the real charset is only known per part.
|
|
17
|
+
*
|
|
18
|
+
* @module v1/cli/knowledge/mhtml
|
|
19
|
+
*/
|
|
20
|
+
export interface MhtmlPart {
|
|
21
|
+
/** Lowercased MIME type without parameters, e.g. `text/html`. */
|
|
22
|
+
contentType: string;
|
|
23
|
+
/** Charset from the part's Content-Type, lowercased, or null. */
|
|
24
|
+
charset: string | null;
|
|
25
|
+
/** Content-Location, or null. */
|
|
26
|
+
location: string | null;
|
|
27
|
+
/** Lowercased Content-Transfer-Encoding; `binary` when unstated. */
|
|
28
|
+
encoding: string;
|
|
29
|
+
/** Raw bytes after transfer decoding. */
|
|
30
|
+
body: Buffer;
|
|
31
|
+
headers: Record<string, string>;
|
|
32
|
+
}
|
|
33
|
+
export interface ParsedMhtml {
|
|
34
|
+
/** The part holding the page itself, or null when there is no HTML part. */
|
|
35
|
+
root: MhtmlPart | null;
|
|
36
|
+
parts: MhtmlPart[];
|
|
37
|
+
/** Container `Snapshot-Content-Location`/`Content-Location` header. */
|
|
38
|
+
snapshotLocation: string | null;
|
|
39
|
+
/** Container `Subject` header — Chrome writes the page title here. */
|
|
40
|
+
subject: string | null;
|
|
41
|
+
}
|
|
42
|
+
/** Decode a quoted-printable body to bytes: soft line breaks vanish, `=XX`
|
|
43
|
+
* becomes one byte, everything else is its own latin1 byte. */
|
|
44
|
+
export declare function decodeQuotedPrintable(input: string): Buffer;
|
|
45
|
+
/** Decode bytes under a MIME charset, falling back to UTF-8 (then latin1)
|
|
46
|
+
* rather than failing an ingest over an exotic label. */
|
|
47
|
+
export declare function decodeCharset(bytes: Buffer, charset: string | null): string;
|
|
48
|
+
/**
|
|
49
|
+
* Parse an MHTML container. Anything that is not a well-formed multipart
|
|
50
|
+
* message is treated as a single part, so a plain `.html` file renamed to
|
|
51
|
+
* `.mhtml` still ingests.
|
|
52
|
+
*/
|
|
53
|
+
export declare function parseMhtml(raw: string | Buffer): ParsedMhtml;
|
|
54
|
+
/** Parse a container and return its root frame decoded to a string. */
|
|
55
|
+
export declare function mhtmlToHtml(raw: string | Buffer): {
|
|
56
|
+
html: string;
|
|
57
|
+
location: string | null;
|
|
58
|
+
subject: string | null;
|
|
59
|
+
};
|
|
60
|
+
//# sourceMappingURL=mhtml.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mhtml.d.ts","sourceRoot":"","sources":["../../../src/knowledge/mhtml.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,MAAM,WAAW,SAAS;IACxB,iEAAiE;IACjE,WAAW,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,iCAAiC;IACjC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,WAAW;IAC1B,4EAA4E;IAC5E,IAAI,EAAE,SAAS,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,uEAAuE;IACvE,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,sEAAsE;IACtE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;gEACgE;AAChE,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAyB3D;AAaD;0DAC0D;AAC1D,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAU3E;AA4CD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,WAAW,CAiC5D;AAED,uEAAuE;AACvE,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,CAOA"}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MHTML / MHT container parsing (RCL-01).
|
|
3
|
+
*
|
|
4
|
+
* An MHTML archive is an RFC 2557 `multipart/related` MIME message: the page
|
|
5
|
+
* plus every subresource it referenced, each part carrying its own transfer
|
|
6
|
+
* encoding and charset. Chrome's `Page.captureSnapshot` writes them
|
|
7
|
+
* quoted-printable; Internet Explorer and most "save as web archive" tools
|
|
8
|
+
* write base64.
|
|
9
|
+
*
|
|
10
|
+
* Only the ROOT frame is prose. Picking the first `text/html` part instead is
|
|
11
|
+
* how an ad iframe ends up indexed as the article, so the root is resolved
|
|
12
|
+
* from the container's `Snapshot-Content-Location` / `Content-Location` when
|
|
13
|
+
* one is present.
|
|
14
|
+
*
|
|
15
|
+
* Read the file as `latin1` (or pass a Buffer): byte fidelity matters before
|
|
16
|
+
* transfer decoding, and the real charset is only known per part.
|
|
17
|
+
*
|
|
18
|
+
* @module v1/cli/knowledge/mhtml
|
|
19
|
+
*/
|
|
20
|
+
/** Decode a quoted-printable body to bytes: soft line breaks vanish, `=XX`
|
|
21
|
+
* becomes one byte, everything else is its own latin1 byte. */
|
|
22
|
+
export function decodeQuotedPrintable(input) {
|
|
23
|
+
const out = [];
|
|
24
|
+
for (let i = 0; i < input.length; i++) {
|
|
25
|
+
const ch = input[i];
|
|
26
|
+
if (ch !== '=') {
|
|
27
|
+
out.push(input.charCodeAt(i) & 0xff);
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (input[i + 1] === '\r' && input[i + 2] === '\n') {
|
|
31
|
+
i += 2; // soft line break
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (input[i + 1] === '\n') {
|
|
35
|
+
i += 1; // soft line break, LF-only file
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const hex = input.slice(i + 1, i + 3);
|
|
39
|
+
if (/^[0-9a-fA-F]{2}$/.test(hex)) {
|
|
40
|
+
out.push(Number.parseInt(hex, 16));
|
|
41
|
+
i += 2;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
out.push(0x3d); // a lone `=` — keep it
|
|
45
|
+
}
|
|
46
|
+
return Buffer.from(out);
|
|
47
|
+
}
|
|
48
|
+
function decodeBody(raw, encoding) {
|
|
49
|
+
switch (encoding) {
|
|
50
|
+
case 'quoted-printable':
|
|
51
|
+
return decodeQuotedPrintable(raw);
|
|
52
|
+
case 'base64':
|
|
53
|
+
return Buffer.from(raw.replace(/\s+/g, ''), 'base64');
|
|
54
|
+
default:
|
|
55
|
+
return Buffer.from(raw, 'latin1');
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/** Decode bytes under a MIME charset, falling back to UTF-8 (then latin1)
|
|
59
|
+
* rather than failing an ingest over an exotic label. */
|
|
60
|
+
export function decodeCharset(bytes, charset) {
|
|
61
|
+
const label = (charset ?? 'utf-8').trim().toLowerCase();
|
|
62
|
+
for (const candidate of [label, 'utf-8']) {
|
|
63
|
+
try {
|
|
64
|
+
return new TextDecoder(candidate, { fatal: false }).decode(bytes);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
/* unknown label — try the next */
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return bytes.toString('latin1');
|
|
71
|
+
}
|
|
72
|
+
/** Split a MIME chunk into unfolded headers and the body after the first
|
|
73
|
+
* blank line. Continuation lines (leading space/tab) join their header. */
|
|
74
|
+
function splitHeaders(chunk) {
|
|
75
|
+
const sep = /\r?\n\r?\n/.exec(chunk);
|
|
76
|
+
const headerText = sep ? chunk.slice(0, sep.index) : chunk;
|
|
77
|
+
const body = sep ? chunk.slice(sep.index + sep[0].length) : '';
|
|
78
|
+
const headers = {};
|
|
79
|
+
let current = '';
|
|
80
|
+
for (const rawLine of headerText.split(/\r?\n/)) {
|
|
81
|
+
if (/^[ \t]/.test(rawLine) && current) {
|
|
82
|
+
headers[current] += ` ${rawLine.trim()}`;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const colon = rawLine.indexOf(':');
|
|
86
|
+
if (colon === -1)
|
|
87
|
+
continue;
|
|
88
|
+
current = rawLine.slice(0, colon).trim().toLowerCase();
|
|
89
|
+
headers[current] = rawLine.slice(colon + 1).trim();
|
|
90
|
+
}
|
|
91
|
+
return { headers, body };
|
|
92
|
+
}
|
|
93
|
+
function param(headerValue, name) {
|
|
94
|
+
if (!headerValue)
|
|
95
|
+
return null;
|
|
96
|
+
const m = new RegExp(`${name}\\s*=\\s*("[^"]*"|'[^']*'|[^;\\s]+)`, 'i').exec(headerValue);
|
|
97
|
+
return m ? m[1].replace(/^["']|["']$/g, '') : null;
|
|
98
|
+
}
|
|
99
|
+
function toPart(chunk) {
|
|
100
|
+
const { headers, body } = splitHeaders(chunk);
|
|
101
|
+
const contentType = (headers['content-type'] ?? 'text/html').split(';')[0].trim().toLowerCase();
|
|
102
|
+
const encoding = (headers['content-transfer-encoding'] ?? 'binary').trim().toLowerCase();
|
|
103
|
+
return {
|
|
104
|
+
contentType,
|
|
105
|
+
charset: param(headers['content-type'], 'charset')?.toLowerCase() ?? null,
|
|
106
|
+
location: headers['content-location'] ?? null,
|
|
107
|
+
encoding,
|
|
108
|
+
body: decodeBody(body, encoding),
|
|
109
|
+
headers,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Parse an MHTML container. Anything that is not a well-formed multipart
|
|
114
|
+
* message is treated as a single part, so a plain `.html` file renamed to
|
|
115
|
+
* `.mhtml` still ingests.
|
|
116
|
+
*/
|
|
117
|
+
export function parseMhtml(raw) {
|
|
118
|
+
const text = typeof raw === 'string' ? raw : raw.toString('latin1');
|
|
119
|
+
const { headers, body } = splitHeaders(text);
|
|
120
|
+
const snapshotLocation = headers['snapshot-content-location'] ?? headers['content-location'] ?? null;
|
|
121
|
+
const subject = headers.subject ?? null;
|
|
122
|
+
const boundary = param(headers['content-type'], 'boundary');
|
|
123
|
+
let parts;
|
|
124
|
+
if (!boundary) {
|
|
125
|
+
// No boundary: either a single-part message or a bare HTML file. Feeding
|
|
126
|
+
// the whole document through `toPart` covers both — an HTML file has no
|
|
127
|
+
// header block, so everything lands in the body.
|
|
128
|
+
parts = [toPart(/^[a-z-]+:/i.test(text) ? text : `Content-Type: text/html\r\n\r\n${text}`)];
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
const delimiter = `--${boundary}`;
|
|
132
|
+
parts = body
|
|
133
|
+
.split(delimiter)
|
|
134
|
+
.slice(1) // text before the first delimiter is the MIME preamble
|
|
135
|
+
.filter((chunk) => !chunk.startsWith('--')) // the closing `--boundary--`
|
|
136
|
+
.map((chunk) => chunk.replace(/^\r?\n/, ''))
|
|
137
|
+
.map(toPart);
|
|
138
|
+
}
|
|
139
|
+
const html = parts.filter((p) => p.contentType === 'text/html' || p.contentType === 'application/xhtml+xml');
|
|
140
|
+
const root = (snapshotLocation ? html.find((p) => p.location === snapshotLocation) : undefined) ??
|
|
141
|
+
html[0] ??
|
|
142
|
+
null;
|
|
143
|
+
return { root, parts, snapshotLocation, subject };
|
|
144
|
+
}
|
|
145
|
+
/** Parse a container and return its root frame decoded to a string. */
|
|
146
|
+
export function mhtmlToHtml(raw) {
|
|
147
|
+
const parsed = parseMhtml(raw);
|
|
148
|
+
return {
|
|
149
|
+
html: parsed.root ? decodeCharset(parsed.root.body, parsed.root.charset) : '',
|
|
150
|
+
location: parsed.root?.location ?? parsed.snapshotLocation,
|
|
151
|
+
subject: parsed.subject,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
//# sourceMappingURL=mhtml.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mhtml.js","sourceRoot":"","sources":["../../../src/knowledge/mhtml.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AA0BH;gEACgE;AAChE,MAAM,UAAU,qBAAqB,CAAC,KAAa;IACjD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YACrC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB;YAC1B,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC1B,CAAC,IAAI,CAAC,CAAC,CAAC,gCAAgC;YACxC,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,IAAI,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;YACnC,CAAC,IAAI,CAAC,CAAC;YACP,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IACzC,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,QAAgB;IAC/C,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,kBAAkB;YACrB,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC;QACpC,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;QACxD;YACE,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED;0DAC0D;AAC1D,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,OAAsB;IACjE,MAAM,KAAK,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACxD,KAAK,MAAM,SAAS,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,OAAO,IAAI,WAAW,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED;4EAC4E;AAC5E,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3D,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/D,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;YACtC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACzC,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,SAAS;QAC3B,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,KAAK,CAAC,WAA+B,EAAE,IAAY;IAC1D,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC;IAC9B,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,qCAAqC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC1F,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACrD,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChG,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,2BAA2B,CAAC,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzF,OAAO;QACL,WAAW;QACX,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI;QACzE,QAAQ,EAAE,OAAO,CAAC,kBAAkB,CAAC,IAAI,IAAI;QAC7C,QAAQ;QACR,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC;QAChC,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,GAAoB;IAC7C,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,gBAAgB,GACpB,OAAO,CAAC,2BAA2B,CAAC,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC;IAC9E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC;IACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,UAAU,CAAC,CAAC;IAE5D,IAAI,KAAkB,CAAC;IACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,yEAAyE;QACzE,wEAAwE;QACxE,iDAAiD;QACjD,KAAK,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9F,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,KAAK,QAAQ,EAAE,CAAC;QAClC,KAAK,GAAG,IAAI;aACT,KAAK,CAAC,SAAS,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,CAAC,uDAAuD;aAChE,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,6BAA6B;aACxE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAC3C,GAAG,CAAC,MAAM,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CACvB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,WAAW,IAAI,CAAC,CAAC,WAAW,KAAK,uBAAuB,CAClF,CAAC;IACF,MAAM,IAAI,GACR,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,IAAI,CAAC,CAAC,CAAC;QACP,IAAI,CAAC;IAEP,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC;AACpD,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,WAAW,CAAC,GAAoB;IAK9C,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7E,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,IAAI,MAAM,CAAC,gBAAgB;QAC1D,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Related-at-save (RCL-03) — "3 things in your brain relate to this".
|
|
3
|
+
*
|
|
4
|
+
* The extension calls this on EVERY capture, while the user watches. That
|
|
5
|
+
* single fact sets every design decision here:
|
|
6
|
+
*
|
|
7
|
+
* - it never throws. An empty store, a missing embedder, a URL that was
|
|
8
|
+
* never captured — all of them are an empty list, because a capture must
|
|
9
|
+
* not fail because the sidebar had nothing to say;
|
|
10
|
+
* - it is bounded. The vector pass races a timeout (default 2s) and the
|
|
11
|
+
* cheap signals — same URL, same site, read straight off the metadata log —
|
|
12
|
+
* are computed first, so a slow or absent embedder degrades to a fast
|
|
13
|
+
* answer instead of no answer;
|
|
14
|
+
* - the target itself is never in its own results, by path AND by canonical
|
|
15
|
+
* URL, so a re-capture does not "relate" to the version it replaced.
|
|
16
|
+
*
|
|
17
|
+
* Three signals, combined: exact URL (the same page, captured before),
|
|
18
|
+
* same site, and vector similarity over the chunks already embedded.
|
|
19
|
+
*
|
|
20
|
+
* @module v1/cli/knowledge/related
|
|
21
|
+
*/
|
|
22
|
+
export type RelatedReason = 'same-url' | 'same-site' | 'similar';
|
|
23
|
+
export interface RelatedDocument {
|
|
24
|
+
filePath: string;
|
|
25
|
+
scope: string;
|
|
26
|
+
title: string;
|
|
27
|
+
url?: string;
|
|
28
|
+
site?: string;
|
|
29
|
+
/** 0–1. Exact-URL matches lead; similarity and site agreement compound. */
|
|
30
|
+
score: number;
|
|
31
|
+
reasons: RelatedReason[];
|
|
32
|
+
capturedAt?: string;
|
|
33
|
+
/** The most similar passage, when similarity is one of the reasons. */
|
|
34
|
+
excerpt?: string;
|
|
35
|
+
chunkIndex?: number;
|
|
36
|
+
anchor?: string;
|
|
37
|
+
}
|
|
38
|
+
export interface RelatedOptions {
|
|
39
|
+
rootDir?: string;
|
|
40
|
+
scope?: string;
|
|
41
|
+
limit?: number;
|
|
42
|
+
/** Budget for the vector pass. The metadata signals ignore it: they are
|
|
43
|
+
* synchronous file reads. */
|
|
44
|
+
timeoutMs?: number;
|
|
45
|
+
minScore?: number;
|
|
46
|
+
store?: 'project' | 'global' | 'all';
|
|
47
|
+
/** Text to compare against, when the caller already has it (a capture that
|
|
48
|
+
* has not been ingested yet). */
|
|
49
|
+
text?: string;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The documents most related to `target` (a URL, a docId, or an indexed path).
|
|
53
|
+
*
|
|
54
|
+
* Safe to call on every capture: bounded, and empty rather than thrown on any
|
|
55
|
+
* failure.
|
|
56
|
+
*/
|
|
57
|
+
export declare function relatedDocuments(target: string, opts?: RelatedOptions): Promise<RelatedDocument[]>;
|
|
58
|
+
//# sourceMappingURL=related.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"related.d.ts","sourceRoot":"","sources":["../../../src/knowledge/related.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;AAEjE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;kCAC8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,SAAS,GAAG,QAAQ,GAAG,KAAK,CAAC;IACrC;sCACkC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAwFD;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,eAAe,EAAE,CAAC,CAmH5B"}
|