@amsterdamdatalabs/enact-docs-search 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -4
- package/dist/cli.js +182 -54
- package/dist/cli.js.map +1 -1
- package/dist/dependency-errors.d.ts +3 -0
- package/dist/dependency-errors.d.ts.map +1 -0
- package/dist/dependency-errors.js +26 -0
- package/dist/dependency-errors.js.map +1 -0
- package/dist/doctor.d.ts +17 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +88 -0
- package/dist/doctor.js.map +1 -0
- package/dist/embeddings/embedding-service.d.ts.map +1 -1
- package/dist/embeddings/embedding-service.js +2 -3
- package/dist/embeddings/embedding-service.js.map +1 -1
- package/dist/indexing/chunker.d.ts +6 -1
- package/dist/indexing/chunker.d.ts.map +1 -1
- package/dist/indexing/chunker.js +17 -11
- package/dist/indexing/chunker.js.map +1 -1
- package/dist/indexing/embedding-text.d.ts +22 -0
- package/dist/indexing/embedding-text.d.ts.map +1 -0
- package/dist/indexing/embedding-text.js +27 -0
- package/dist/indexing/embedding-text.js.map +1 -0
- package/dist/indexing/frontmatter.d.ts +48 -0
- package/dist/indexing/frontmatter.d.ts.map +1 -0
- package/dist/indexing/frontmatter.js +162 -0
- package/dist/indexing/frontmatter.js.map +1 -0
- package/dist/indexing/indexing-service.d.ts +1 -4
- package/dist/indexing/indexing-service.d.ts.map +1 -1
- package/dist/indexing/indexing-service.js +24 -12
- package/dist/indexing/indexing-service.js.map +1 -1
- package/dist/search/authority.d.ts +31 -0
- package/dist/search/authority.d.ts.map +1 -0
- package/dist/search/authority.js +50 -0
- package/dist/search/authority.js.map +1 -0
- package/dist/search/relevance.d.ts +80 -0
- package/dist/search/relevance.d.ts.map +1 -0
- package/dist/search/relevance.js +119 -0
- package/dist/search/relevance.js.map +1 -0
- package/dist/search/search-service.d.ts +44 -2
- package/dist/search/search-service.d.ts.map +1 -1
- package/dist/search/search-service.js +181 -23
- package/dist/search/search-service.js.map +1 -1
- package/dist/search/snippet.d.ts +11 -0
- package/dist/search/snippet.d.ts.map +1 -1
- package/dist/search/snippet.js +102 -17
- package/dist/search/snippet.js.map +1 -1
- package/dist/storage/document-store.d.ts +47 -5
- package/dist/storage/document-store.d.ts.map +1 -1
- package/dist/storage/document-store.js +94 -16
- package/dist/storage/document-store.js.map +1 -1
- package/dist/storage/persistence.d.ts +6 -10
- package/dist/storage/persistence.d.ts.map +1 -1
- package/dist/storage/persistence.js +20 -20
- package/dist/storage/persistence.js.map +1 -1
- package/dist/storage/vector-store.d.ts +2 -3
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +5 -15
- package/dist/storage/vector-store.js.map +1 -1
- package/dist/types.d.ts +2 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -20
- package/dist/types.js.map +1 -1
- package/package.json +9 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAiBD,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAWtE;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,kBAAmB,GAAG,MAAM,CAOrG;AA8CD,wBAAsB,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,CASvD;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAOzD"}
|
package/dist/doctor.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { nativeDependencyRemediation } from "./dependency-errors.js";
|
|
2
|
+
const ONNX_RUNTIME_MODULE = "onnxruntime-node";
|
|
3
|
+
function errorMessage(error) {
|
|
4
|
+
return error instanceof Error ? error.message : String(error);
|
|
5
|
+
}
|
|
6
|
+
export function buildToolingGuidance(platform) {
|
|
7
|
+
switch (platform) {
|
|
8
|
+
case "darwin":
|
|
9
|
+
return "Install the macOS Command Line Tools with `xcode-select --install`, then reinstall the package.";
|
|
10
|
+
case "win32":
|
|
11
|
+
return "Install Python and Visual Studio Build Tools with the Desktop development with C++ workload, then reinstall the package.";
|
|
12
|
+
case "linux":
|
|
13
|
+
return "Install Python 3, make, and a C/C++ compiler (for example `sudo apt-get install python3 make g++`), then reinstall the package.";
|
|
14
|
+
default:
|
|
15
|
+
return "Install Python 3 and a working C/C++ compiler toolchain for this platform, then reinstall the package.";
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export function formatDoctorFailure(name, error, platform = process.platform) {
|
|
19
|
+
const detail = errorMessage(error);
|
|
20
|
+
const native = /hnsw|onnx|binding|\.node|node-gyp|gyp err|dlopen|native|module/i.test(detail);
|
|
21
|
+
const nativeHint = native
|
|
22
|
+
? ` ${nativeDependencyRemediation(name, error)}${/node-gyp|gyp err|binding\.gyp/i.test(detail) ? ` ${buildToolingGuidance(platform)}` : ""}`
|
|
23
|
+
: " Check that the package dependencies are installed and compatible with this Node.js version.";
|
|
24
|
+
return `${name} failed: ${detail}.${nativeHint}`;
|
|
25
|
+
}
|
|
26
|
+
async function checkTransformers() {
|
|
27
|
+
try {
|
|
28
|
+
const transformers = await import("@huggingface/transformers");
|
|
29
|
+
if (typeof transformers.pipeline !== "function") {
|
|
30
|
+
throw new Error("pipeline export is missing");
|
|
31
|
+
}
|
|
32
|
+
return { name: "Transformers.js", ok: true, detail: "package loaded" };
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
return { name: "Transformers.js", ok: false, detail: formatDoctorFailure("Transformers.js", error) };
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async function checkOnnxRuntime() {
|
|
39
|
+
try {
|
|
40
|
+
const runtime = (await import(ONNX_RUNTIME_MODULE));
|
|
41
|
+
if (typeof runtime.InferenceSession !== "object" && typeof runtime.InferenceSession !== "function") {
|
|
42
|
+
throw new Error("InferenceSession export is missing");
|
|
43
|
+
}
|
|
44
|
+
return { name: "ONNX Runtime", ok: true, detail: "native runtime loaded" };
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
return { name: "ONNX Runtime", ok: false, detail: formatDoctorFailure("ONNX Runtime", error) };
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async function checkHnsw() {
|
|
51
|
+
try {
|
|
52
|
+
const module = (await import("hnswlib-node"));
|
|
53
|
+
const HierarchicalNSW = module.HierarchicalNSW ?? module.default?.HierarchicalNSW;
|
|
54
|
+
if (!HierarchicalNSW)
|
|
55
|
+
throw new Error("HierarchicalNSW export is missing");
|
|
56
|
+
const index = new HierarchicalNSW("cosine", 3);
|
|
57
|
+
index.initIndex(4);
|
|
58
|
+
index.addPoint([1, 0, 0], 0);
|
|
59
|
+
index.addPoint([0, 1, 0], 1);
|
|
60
|
+
const result = index.searchKnn([1, 0, 0], 1);
|
|
61
|
+
if (result.neighbors.length !== 1 || result.neighbors[0] !== 0) {
|
|
62
|
+
throw new Error("native search returned an unexpected result");
|
|
63
|
+
}
|
|
64
|
+
return { name: "hnswlib-node", ok: true, detail: "native index created and searched" };
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
return { name: "hnswlib-node", ok: false, detail: formatDoctorFailure("hnswlib-node", error) };
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export async function runDoctor() {
|
|
71
|
+
const checks = await Promise.all([checkTransformers(), checkOnnxRuntime(), checkHnsw()]);
|
|
72
|
+
return {
|
|
73
|
+
healthy: checks.every((check) => check.ok),
|
|
74
|
+
node: process.version,
|
|
75
|
+
platform: process.platform,
|
|
76
|
+
arch: process.arch,
|
|
77
|
+
checks,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
export function renderDoctor(report) {
|
|
81
|
+
const lines = [
|
|
82
|
+
`Node.js ${report.node} (${report.platform}/${report.arch})`,
|
|
83
|
+
`Status: ${report.healthy ? "healthy" : "failed"}`,
|
|
84
|
+
...report.checks.map((check) => `${check.ok ? "PASS" : "FAIL"} ${check.name}: ${check.detail}`),
|
|
85
|
+
];
|
|
86
|
+
return lines.join("\n");
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=doctor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAyBrE,MAAM,mBAAmB,GAAG,kBAAkB,CAAC;AAE/C,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAyB;IAC5D,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,iGAAiG,CAAC;QAC3G,KAAK,OAAO;YACV,OAAO,0HAA0H,CAAC;QACpI,KAAK,OAAO;YACV,OAAO,iIAAiI,CAAC;QAC3I;YACE,OAAO,wGAAwG,CAAC;IACpH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAc,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ;IAC3F,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,iEAAiE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9F,MAAM,UAAU,GAAG,MAAM;QACvB,CAAC,CAAC,IAAI,2BAA2B,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,gCAAgC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5I,CAAC,CAAC,8FAA8F,CAAC;IACnG,OAAO,GAAG,IAAI,YAAY,MAAM,IAAI,UAAU,EAAE,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,iBAAiB;IAC9B,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC/D,IAAI,OAAO,YAAY,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACzE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,CAAC,iBAAiB,EAAE,KAAK,CAAC,EAAE,CAAC;IACvG,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAsB,CAAC;QACzE,IAAI,OAAO,OAAO,CAAC,gBAAgB,KAAK,QAAQ,IAAI,OAAO,OAAO,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;YACnG,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE,CAAC;IACjG,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAe,CAAC;QAC5D,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC;QAClF,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAE3E,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC/C,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7C,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAC;IACzF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE,CAAC;IACjG,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE,gBAAgB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IACzF,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1C,IAAI,EAAE,OAAO,CAAC,OAAO;QACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM;KACP,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAoB;IAC/C,MAAM,KAAK,GAAG;QACZ,WAAW,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,GAAG;QAC5D,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;KAChG,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-service.d.ts","sourceRoot":"","sources":["../../src/embeddings/embedding-service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedding-service.d.ts","sourceRoot":"","sources":["../../src/embeddings/embedding-service.ts"],"names":[],"mappings":"AAkBA,eAAO,MAAM,QAAQ,4BAA4B,CAAC;AAClD,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC,qBAAa,eAAgB,SAAQ,KAAK;CAAG;AAE7C,MAAM,WAAW,uBAAuB;IACtC,oFAAoF;IACpF,aAAa,EAAE,MAAM,CAAC;IACtB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,qBAAa,gBAAgB;IAIf,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,OAAO,CAAC,iBAAiB,CAA0C;IACnE,OAAO,CAAC,KAAK,CAAS;gBAEO,OAAO,EAAE,uBAAuB;IAEvD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA+CjC,OAAO,IAAI,OAAO;IAIZ,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAqBvD"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { rm } from "node:fs/promises";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
+
import { semanticDependencyMessage } from "../dependency-errors.js";
|
|
3
4
|
export const MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
4
5
|
export const EMBEDDING_DIMENSIONS = 384;
|
|
5
6
|
export class DependencyError extends Error {
|
|
@@ -17,9 +18,7 @@ export class EmbeddingService {
|
|
|
17
18
|
transformers = await import("@huggingface/transformers");
|
|
18
19
|
}
|
|
19
20
|
catch (error) {
|
|
20
|
-
throw new DependencyError("
|
|
21
|
-
"Run `npm install` in the enact-docs-search directory, then retry.\n" +
|
|
22
|
-
`Original error: ${error.message}`);
|
|
21
|
+
throw new DependencyError(semanticDependencyMessage("@huggingface/transformers", error));
|
|
23
22
|
}
|
|
24
23
|
const { pipeline, env } = transformers;
|
|
25
24
|
env.allowLocalModels = true;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-service.js","sourceRoot":"","sources":["../../src/embeddings/embedding-service.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"embedding-service.js","sourceRoot":"","sources":["../../src/embeddings/embedding-service.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAEpE,MAAM,CAAC,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAClD,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAExC,MAAM,OAAO,eAAgB,SAAQ,KAAK;CAAG;AAS7C,MAAM,OAAO,gBAAgB;IAIE;IAHrB,iBAAiB,GAAqC,IAAI,CAAC;IAC3D,KAAK,GAAG,KAAK,CAAC;IAEtB,YAA6B,OAAgC;QAAhC,YAAO,GAAP,OAAO,CAAyB;IAAG,CAAC;IAEjE,KAAK,CAAC,UAAU;QACd,IAAI,YAAY,CAAC;QACjB,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,eAAe,CAAC,yBAAyB,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC;QACvC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC5B,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC;QAC/D,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;QAE1C,IAAI,CAAC;YACH,IAAI,CAAC,iBAAiB,GAAG,CAAC,MAAM,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,EAAE;gBACvE,KAAK,EAAE,MAAM;aACd,CAAC,CAAyC,CAAC;YAC5C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,GAAG,CAAC,iBAAiB,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACH,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBACvF,IAAI,CAAC,iBAAiB,GAAG,CAAC,MAAM,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,EAAE;wBACvE,KAAK,EAAE,MAAM;qBACd,CAAC,CAAyC,CAAC;oBAC5C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;oBAClB,OAAO;gBACT,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,MAAM,IAAI,eAAe,CACvB,uEAAuE;wBACrE,yEAAyE;wBACzE,0DAA0D;wBAC1D,mBAAoB,UAAoB,CAAC,OAAO,EAAE,CACrD,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,MAAM,IAAI,eAAe,CACvB,uEAAuE;gBACrE,4EAA4E;gBAC5E,qCAAqC;gBACrC,mBAAoB,KAAe,CAAC,OAAO,EAAE,CAChD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,iBAAiB,KAAK,IAAI,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,IAAI,EAAE,EAAE;YACtD,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE;YACrD,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAoB,CAAC;QACzC,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,CAAC,GAAG,oBAAoB,CAAC;YACvC,MAAM,GAAG,GAAG,KAAK,GAAG,oBAAoB,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -44,8 +44,13 @@ export declare class Chunker {
|
|
|
44
44
|
private findFirstNonHeadingContent;
|
|
45
45
|
/**
|
|
46
46
|
* Split a document into chunks.
|
|
47
|
+
*
|
|
48
|
+
* `lineOffset` is the number of lines removed from the top of the file
|
|
49
|
+
* before chunking (a stripped frontmatter block). It is added to every
|
|
50
|
+
* `startLine` so citations point at the file on disk, not at the trimmed
|
|
51
|
+
* body this method actually sees.
|
|
47
52
|
*/
|
|
48
|
-
chunk(content: string, filePath: string): Chunk[];
|
|
53
|
+
chunk(content: string, filePath: string, lineOffset?: number): Chunk[];
|
|
49
54
|
private findBreakPoint;
|
|
50
55
|
private findCodeBlockBoundary;
|
|
51
56
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/indexing/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,KAAK;IACpB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,SAAS,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB;AAOD;;;;GAIG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAe;gBAElB,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM;IAI/C;;OAEG;IACH,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE;IAqBjE,OAAO,CAAC,0BAA0B;
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/indexing/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,KAAK;IACpB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,SAAS,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;CACxB;AAOD;;;;GAIG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAe;gBAElB,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM;IAI/C;;OAEG;IACH,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE;IAqBjE,OAAO,CAAC,0BAA0B;IAmClC;;;;;;;OAOG;IACH,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,KAAK,EAAE;IAuEjE,OAAO,CAAC,cAAc;IA8CtB,OAAO,CAAC,qBAAqB;CAoB9B"}
|
package/dist/indexing/chunker.js
CHANGED
|
@@ -57,14 +57,15 @@ export class Chunker {
|
|
|
57
57
|
else if (line.trim() === "") {
|
|
58
58
|
i = lineEnd === -1 ? len : lineEnd + 1;
|
|
59
59
|
}
|
|
60
|
-
else if (line
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
60
|
+
else if (/^-{3,}\s*$/.test(line)) {
|
|
61
|
+
// A standalone rule is decoration, so skip the line and keep looking
|
|
62
|
+
// for the heading that labels this chunk. This branch used to treat a
|
|
63
|
+
// leading `---` as a frontmatter opener and jump to the next `---`,
|
|
64
|
+
// which swallowed the heading directly below it -- and with two rules
|
|
65
|
+
// in a document, labelled the chunk with a heading from further down.
|
|
66
|
+
// Frontmatter no longer reaches the chunker (it is stripped before
|
|
67
|
+
// chunking), so a `---` here can only ever be a horizontal rule.
|
|
68
|
+
i = lineEnd === -1 ? len : lineEnd + 1;
|
|
68
69
|
}
|
|
69
70
|
else {
|
|
70
71
|
break;
|
|
@@ -74,8 +75,13 @@ export class Chunker {
|
|
|
74
75
|
}
|
|
75
76
|
/**
|
|
76
77
|
* Split a document into chunks.
|
|
78
|
+
*
|
|
79
|
+
* `lineOffset` is the number of lines removed from the top of the file
|
|
80
|
+
* before chunking (a stripped frontmatter block). It is added to every
|
|
81
|
+
* `startLine` so citations point at the file on disk, not at the trimmed
|
|
82
|
+
* body this method actually sees.
|
|
77
83
|
*/
|
|
78
|
-
chunk(content, filePath) {
|
|
84
|
+
chunk(content, filePath, lineOffset = 0) {
|
|
79
85
|
const trimmed = content.trim();
|
|
80
86
|
if (!trimmed) {
|
|
81
87
|
return [];
|
|
@@ -89,7 +95,7 @@ export class Chunker {
|
|
|
89
95
|
chunkIndex: 0,
|
|
90
96
|
startPos: 0,
|
|
91
97
|
endPos: normalized.length,
|
|
92
|
-
startLine: 1,
|
|
98
|
+
startLine: 1 + lineOffset,
|
|
93
99
|
headingBreadcrumb: this.getHeadingBreadcrumb(normalized, 0),
|
|
94
100
|
},
|
|
95
101
|
];
|
|
@@ -122,7 +128,7 @@ export class Chunker {
|
|
|
122
128
|
chunkIndex,
|
|
123
129
|
startPos: position,
|
|
124
130
|
endPos: actualEnd,
|
|
125
|
-
startLine: lineNumberAt(normalized, position),
|
|
131
|
+
startLine: lineNumberAt(normalized, position) + lineOffset,
|
|
126
132
|
headingBreadcrumb: this.getHeadingBreadcrumb(normalized, position),
|
|
127
133
|
});
|
|
128
134
|
const advance = actualEnd - position - overlapSize;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/indexing/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA0BH,MAAM,eAAe,GAAiB;IACpC,YAAY,EAAE,IAAI;IAClB,cAAc,EAAE,IAAI;CACrB,CAAC;AAEF;;;;GAIG;AACH,MAAM,OAAO,OAAO;IACV,OAAO,CAAe;IAE9B,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,OAAe,EAAE,QAAgB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,0BAA0B,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACnE,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,YAAY,GAAG,qBAAqB,CAAC;QAE3C,MAAM,QAAQ,GAAsB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;YAE9B,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACzD,CAAC;IAEO,0BAA0B,CAAC,OAAe,EAAE,QAAgB;QAClE,IAAI,CAAC,GAAG,QAAQ,CAAC;QACjB,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACzC,CAAC,EAAE,CAAC;YACN,CAAC;YAED,IAAI,CAAC,IAAI,GAAG;gBAAE,MAAM;YAEpB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAE9D,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC9B,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/indexing/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA0BH,MAAM,eAAe,GAAiB;IACpC,YAAY,EAAE,IAAI;IAClB,cAAc,EAAE,IAAI;CACrB,CAAC;AAEF;;;;GAIG;AACH,MAAM,OAAO,OAAO;IACV,OAAO,CAAe;IAE9B,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,OAAe,EAAE,QAAgB;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,0BAA0B,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACnE,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,YAAY,GAAG,qBAAqB,CAAC;QAE3C,MAAM,QAAQ,GAAsB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;YAE9B,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACzD,CAAC;IAEO,0BAA0B,CAAC,OAAe,EAAE,QAAgB;QAClE,IAAI,CAAC,GAAG,QAAQ,CAAC;QACjB,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACzC,CAAC,EAAE,CAAC;YACN,CAAC;YAED,IAAI,CAAC,IAAI,GAAG;gBAAE,MAAM;YAEpB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAE9D,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC9B,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnC,qEAAqE;gBACrE,sEAAsE;gBACtE,oEAAoE;gBACpE,sEAAsE;gBACtE,sEAAsE;gBACtE,mEAAmE;gBACnE,iEAAiE;gBACjE,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,OAAe,EAAE,QAAgB,EAAE,UAAU,GAAG,CAAC;QACrD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAEvE,IAAI,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YACnD,OAAO;gBACL;oBACE,OAAO,EAAE,UAAU;oBACnB,QAAQ;oBACR,UAAU,EAAE,CAAC;oBACb,QAAQ,EAAE,CAAC;oBACX,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,SAAS,EAAE,CAAC,GAAG,UAAU;oBACzB,iBAAiB,EAAE,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,CAAC,CAAC;iBAC5D;aACF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC;QACjE,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,OAAO,QAAQ,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC;YAE/C,IAAI,SAAS,GAAG,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;gBAC7C,SAAS,CAAC,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBACzD,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;gBACrC,MAAM;YACR,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YAEnF,IAAI,SAAiB,CAAC;YACtB,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBAC3C,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAE3D,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,YAAY;gBACrB,QAAQ;gBACR,UAAU;gBACV,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,YAAY,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,UAAU;gBAC1D,iBAAiB,EAAE,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,QAAQ,CAAC;aACnE,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,CAAC;YACnD,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,UAAU,EAAE,CAAC;YAEb,IAAI,UAAU,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,cAAc,CAAC,OAAe,EAAE,KAAa,EAAE,MAAc;QACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAEjD,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7E,IAAI,iBAAiB,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAO,iBAAiB,CAAC;QAC3B,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAEjD,MAAM,cAAc,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1B,OAAO,KAAK,GAAG,WAAW,GAAG,cAAc,GAAG,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,aAAa,GAAG,WAAW,CAAC;QAClC,IAAI,eAAe,GAAG,CAAC,CAAC,CAAC;QACzB,IAAI,KAAK,CAAC;QACV,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACvD,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9D,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAClD,CAAC;QACD,IAAI,eAAe,KAAK,CAAC,CAAC,EAAE,CAAC;YAC3B,OAAO,KAAK,GAAG,WAAW,GAAG,eAAe,CAAC;QAC/C,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QACrD,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,GAAG,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;QACjD,CAAC;QAED,SAAS,GAAG,IAAI,CAAC,GAAG,CAClB,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,EAC5B,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,EAC7B,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAC9B,CAAC;QACF,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,qBAAqB,CAAC,OAAe,EAAE,KAAa,EAAE,MAAc;QAC1E,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAE3C,MAAM,iBAAiB,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,cAAc,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAE1D,MAAM,kBAAkB,GAAG,iBAAiB,GAAG,CAAC,KAAK,CAAC,CAAC;QAEvD,IAAI,kBAAkB,IAAI,cAAc,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEjD,IAAI,aAAa,KAAK,CAAC,CAAC,IAAI,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBACtE,OAAO,MAAM,GAAG,aAAa,GAAG,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,kEAAkE;AAClE,SAAS,YAAY,CAAC,OAAe,EAAE,MAAc;IACnD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,IAAI,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding input construction.
|
|
3
|
+
*
|
|
4
|
+
* A bare chunk embeds as an orphan: the vector knows what the paragraph says
|
|
5
|
+
* but not which document it belongs to. Prefixing the document's title,
|
|
6
|
+
* description, and heading path puts that identity inside the vector, so the
|
|
7
|
+
* semantic half of the hybrid agrees with the field-weighted keyword half
|
|
8
|
+
* instead of competing with it.
|
|
9
|
+
*/
|
|
10
|
+
export interface EmbeddingTextInput {
|
|
11
|
+
title?: string;
|
|
12
|
+
description?: string;
|
|
13
|
+
headingBreadcrumb?: string[];
|
|
14
|
+
content: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Build the text that is actually embedded for a chunk. Falls back to the
|
|
18
|
+
* bare body when a document carries no frontmatter, so untitled files are
|
|
19
|
+
* neither enriched nor penalised.
|
|
20
|
+
*/
|
|
21
|
+
export declare function buildEmbeddingText(input: EmbeddingTextInput): string;
|
|
22
|
+
//# sourceMappingURL=embedding-text.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-text.d.ts","sourceRoot":"","sources":["../../src/indexing/embedding-text.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,CAYpE"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding input construction.
|
|
3
|
+
*
|
|
4
|
+
* A bare chunk embeds as an orphan: the vector knows what the paragraph says
|
|
5
|
+
* but not which document it belongs to. Prefixing the document's title,
|
|
6
|
+
* description, and heading path puts that identity inside the vector, so the
|
|
7
|
+
* semantic half of the hybrid agrees with the field-weighted keyword half
|
|
8
|
+
* instead of competing with it.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Build the text that is actually embedded for a chunk. Falls back to the
|
|
12
|
+
* bare body when a document carries no frontmatter, so untitled files are
|
|
13
|
+
* neither enriched nor penalised.
|
|
14
|
+
*/
|
|
15
|
+
export function buildEmbeddingText(input) {
|
|
16
|
+
const parts = [];
|
|
17
|
+
const heading = [input.title, input.description].filter(Boolean).join(" — ");
|
|
18
|
+
if (heading)
|
|
19
|
+
parts.push(heading);
|
|
20
|
+
const breadcrumb = (input.headingBreadcrumb ?? []).filter(Boolean).join(" > ");
|
|
21
|
+
if (breadcrumb)
|
|
22
|
+
parts.push(breadcrumb);
|
|
23
|
+
if (parts.length === 0)
|
|
24
|
+
return input.content;
|
|
25
|
+
return `${parts.join("\n")}\n\n${input.content}`;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=embedding-text.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-text.js","sourceRoot":"","sources":["../../src/indexing/embedding-text.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAyB;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7E,IAAI,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEjC,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/E,IAAI,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAE7C,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC;AACnD,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML frontmatter parsing for indexed documents.
|
|
3
|
+
*
|
|
4
|
+
* Frontmatter is metadata, not prose. Indexing it as body text pollutes both
|
|
5
|
+
* the keyword index and the embedding vector, and leaks raw YAML into result
|
|
6
|
+
* snippets. This module splits the two apart once, at parse time, so the rest
|
|
7
|
+
* of the pipeline only ever sees a body plus typed fields.
|
|
8
|
+
*
|
|
9
|
+
* The parser deliberately covers a flat scalar subset -- the fields that drive
|
|
10
|
+
* ranking and display. List and nested values are recognised well enough to be
|
|
11
|
+
* skipped, never well enough to be relied on.
|
|
12
|
+
*/
|
|
13
|
+
/** The document-level fields that ranking and display depend on. */
|
|
14
|
+
export interface DocMeta {
|
|
15
|
+
title?: string;
|
|
16
|
+
description?: string;
|
|
17
|
+
status?: string;
|
|
18
|
+
documentType?: string;
|
|
19
|
+
/** ISO-8601 timestamp as written in the source, unparsed. */
|
|
20
|
+
lastUpdate?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ParsedDocument {
|
|
23
|
+
meta: DocMeta;
|
|
24
|
+
/** The document with its frontmatter block removed. */
|
|
25
|
+
body: string;
|
|
26
|
+
/**
|
|
27
|
+
* Lines the frontmatter block occupied, delimiters included. Callers must
|
|
28
|
+
* add this to any line number derived from `body`, or every citation this
|
|
29
|
+
* tool prints will be short by exactly this much.
|
|
30
|
+
*/
|
|
31
|
+
frontmatterLineCount: number;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Strip a leading YAML frontmatter block and return it as typed fields.
|
|
35
|
+
*
|
|
36
|
+
* A `---` that is not on the very first line is a horizontal rule, not a
|
|
37
|
+
* frontmatter opener, and is left in the body untouched.
|
|
38
|
+
*/
|
|
39
|
+
/**
|
|
40
|
+
* One definition of "line ending", shared with the snippet layer.
|
|
41
|
+
*
|
|
42
|
+
* Two passes that normalise differently disagree about where a line is: the
|
|
43
|
+
* snippet layer split on "\n" only, so a CR-only document was a single line
|
|
44
|
+
* and blanking its frontmatter erased the entire file.
|
|
45
|
+
*/
|
|
46
|
+
export declare function normalizeLineEndings(content: string): string;
|
|
47
|
+
export declare function parseFrontmatter(content: string): ParsedDocument;
|
|
48
|
+
//# sourceMappingURL=frontmatter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frontmatter.d.ts","sourceRoot":"","sources":["../../src/indexing/frontmatter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,oEAAoE;AACpE,MAAM,WAAW,OAAO;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,uDAAuD;IACvD,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAaD;;;;;GAKG;AACH;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAmChE"}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML frontmatter parsing for indexed documents.
|
|
3
|
+
*
|
|
4
|
+
* Frontmatter is metadata, not prose. Indexing it as body text pollutes both
|
|
5
|
+
* the keyword index and the embedding vector, and leaks raw YAML into result
|
|
6
|
+
* snippets. This module splits the two apart once, at parse time, so the rest
|
|
7
|
+
* of the pipeline only ever sees a body plus typed fields.
|
|
8
|
+
*
|
|
9
|
+
* The parser deliberately covers a flat scalar subset -- the fields that drive
|
|
10
|
+
* ranking and display. List and nested values are recognised well enough to be
|
|
11
|
+
* skipped, never well enough to be relied on.
|
|
12
|
+
*/
|
|
13
|
+
const DELIMITER = "---";
|
|
14
|
+
/** Source key -> DocMeta field. Keys absent here are parsed and discarded. */
|
|
15
|
+
const FIELD_MAP = {
|
|
16
|
+
title: "title",
|
|
17
|
+
description: "description",
|
|
18
|
+
status: "status",
|
|
19
|
+
document_type: "documentType",
|
|
20
|
+
last_update: "lastUpdate",
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Strip a leading YAML frontmatter block and return it as typed fields.
|
|
24
|
+
*
|
|
25
|
+
* A `---` that is not on the very first line is a horizontal rule, not a
|
|
26
|
+
* frontmatter opener, and is left in the body untouched.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* One definition of "line ending", shared with the snippet layer.
|
|
30
|
+
*
|
|
31
|
+
* Two passes that normalise differently disagree about where a line is: the
|
|
32
|
+
* snippet layer split on "\n" only, so a CR-only document was a single line
|
|
33
|
+
* and blanking its frontmatter erased the entire file.
|
|
34
|
+
*/
|
|
35
|
+
export function normalizeLineEndings(content) {
|
|
36
|
+
return content.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
37
|
+
}
|
|
38
|
+
export function parseFrontmatter(content) {
|
|
39
|
+
// A leading UTF-8 BOM is invisible in an editor but fails `startsWith`
|
|
40
|
+
// below, which used to disable frontmatter detection outright: the whole
|
|
41
|
+
// YAML block fell through and got indexed as prose, embedding included.
|
|
42
|
+
// Strip it in the same pass as CRLF. It removes one character, not a
|
|
43
|
+
// line, so it cannot shift `frontmatterLineCount` or the body's alignment.
|
|
44
|
+
const normalized = content
|
|
45
|
+
.replace(/\r\n/g, "\n")
|
|
46
|
+
.replace(/\r/g, "\n")
|
|
47
|
+
.replace(/^\uFEFF/, "");
|
|
48
|
+
if (!normalized.startsWith(`${DELIMITER}\n`)) {
|
|
49
|
+
return { meta: {}, body: content, frontmatterLineCount: 0 };
|
|
50
|
+
}
|
|
51
|
+
const lines = normalized.split("\n");
|
|
52
|
+
let closing = -1;
|
|
53
|
+
for (let i = 1; i < lines.length; i++) {
|
|
54
|
+
if (lines[i].trimEnd() === DELIMITER) {
|
|
55
|
+
closing = i;
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// An unterminated block is not frontmatter. Treat the whole file as body
|
|
60
|
+
// rather than silently swallowing it.
|
|
61
|
+
if (closing === -1) {
|
|
62
|
+
return { meta: {}, body: content, frontmatterLineCount: 0 };
|
|
63
|
+
}
|
|
64
|
+
const meta = parseScalars(lines.slice(1, closing));
|
|
65
|
+
const frontmatterLineCount = closing + 1;
|
|
66
|
+
const body = lines.slice(frontmatterLineCount).join("\n");
|
|
67
|
+
return { meta, body, frontmatterLineCount };
|
|
68
|
+
}
|
|
69
|
+
// WHY single-line only: an earlier version treated a value whose first
|
|
70
|
+
// character was a quote but which didn't close on that same line as the
|
|
71
|
+
// start of a multi-line quoted scalar, and kept consuming following lines
|
|
72
|
+
// until one ended with the matching quote. That is not what a genuinely
|
|
73
|
+
// unterminated quote in hand-written frontmatter means -- it means the
|
|
74
|
+
// author forgot to close it. Treating it as "still open" ate every
|
|
75
|
+
// subsequent key (title/description/status/last_update all folded into one
|
|
76
|
+
// value) until either a later line happened to end with that quote
|
|
77
|
+
// character or the block ran out. Real multi-line YAML scalars use `|`/`>`
|
|
78
|
+
// block indicators or explicit indentation continuation, not an unterminated
|
|
79
|
+
// flow quote, so this parser -- which only ever covers the flat scalar
|
|
80
|
+
// subset -- does not need to support that shape at all. Parsing each line
|
|
81
|
+
// independently instead means a bad quote can only ever corrupt its own
|
|
82
|
+
// key's value, never a neighbour's: `no silent drops` for the fields that
|
|
83
|
+
// drive ranking.
|
|
84
|
+
function parseScalars(lines) {
|
|
85
|
+
const meta = {};
|
|
86
|
+
for (const line of lines) {
|
|
87
|
+
// List items and nested mappings belong to the key above them; skip.
|
|
88
|
+
if (/^\s/.test(line) || line.trimStart().startsWith("- "))
|
|
89
|
+
continue;
|
|
90
|
+
const separator = line.indexOf(":");
|
|
91
|
+
if (separator === -1)
|
|
92
|
+
continue;
|
|
93
|
+
const key = line.slice(0, separator).trim();
|
|
94
|
+
const field = FIELD_MAP[key];
|
|
95
|
+
if (!field)
|
|
96
|
+
continue;
|
|
97
|
+
const raw = line.slice(separator + 1).trim();
|
|
98
|
+
if (!raw)
|
|
99
|
+
continue;
|
|
100
|
+
meta[field] = unquote(raw);
|
|
101
|
+
}
|
|
102
|
+
return meta;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Parse `value` as a clean single-line YAML quoted scalar, or return `null`
|
|
106
|
+
* if it isn't one.
|
|
107
|
+
*
|
|
108
|
+
* YAML's two quote styles escape differently, and this walks the value
|
|
109
|
+
* applying whichever rule matches `quote`:
|
|
110
|
+
* - double quotes escape with a backslash (`\"`, `\\`);
|
|
111
|
+
* - single quotes escape a literal quote by doubling it (`''`), and have
|
|
112
|
+
* no backslash-escaping at all -- a backslash is just a character.
|
|
113
|
+
* An escaped quote is skipped, not counted as a delimiter, so a value like
|
|
114
|
+
* `"\"I need X\" the lookup"` (a description that itself quotes a phrase --
|
|
115
|
+
* a real shape in this project's docs corpus) parses cleanly even though it
|
|
116
|
+
* contains inner quote characters.
|
|
117
|
+
*
|
|
118
|
+
* What still returns `null`, deliberately: a value that only *starts* with
|
|
119
|
+
* a quoted phrase (`"why" not the what`) or carries an unescaped embedded
|
|
120
|
+
* quoted phrase (`"a" and "b"`) hits an unescaped quote before the value
|
|
121
|
+
* ends, which is ambiguous -- stripping first/last characters there would
|
|
122
|
+
* guess which quote is "the" closing one, and this parser does not guess.
|
|
123
|
+
* An unterminated value (`"Broken`) never finds a closing quote at all.
|
|
124
|
+
* Either way the raw value is left untouched by the caller.
|
|
125
|
+
*/
|
|
126
|
+
function parseQuotedScalar(value, quote) {
|
|
127
|
+
if (value.length < 2 || value[0] !== quote)
|
|
128
|
+
return null;
|
|
129
|
+
let content = "";
|
|
130
|
+
let i = 1;
|
|
131
|
+
while (i < value.length) {
|
|
132
|
+
const ch = value[i];
|
|
133
|
+
if (quote === '"' && ch === "\\" && i + 1 < value.length) {
|
|
134
|
+
const next = value[i + 1];
|
|
135
|
+
content += next === '"' || next === "\\" ? next : `\\${next}`;
|
|
136
|
+
i += 2;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (quote === "'" && ch === "'" && value[i + 1] === "'") {
|
|
140
|
+
content += "'";
|
|
141
|
+
i += 2;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
if (ch === quote) {
|
|
145
|
+
// An unescaped quote is only a legal closer at the very end.
|
|
146
|
+
return i === value.length - 1 ? content : null;
|
|
147
|
+
}
|
|
148
|
+
content += ch;
|
|
149
|
+
i++;
|
|
150
|
+
}
|
|
151
|
+
return null; // ran off the end without an unescaped closing quote
|
|
152
|
+
}
|
|
153
|
+
function unquote(value) {
|
|
154
|
+
const quote = value[0];
|
|
155
|
+
if (quote === '"' || quote === "'") {
|
|
156
|
+
const parsed = parseQuotedScalar(value, quote);
|
|
157
|
+
if (parsed !== null)
|
|
158
|
+
return parsed;
|
|
159
|
+
}
|
|
160
|
+
return value;
|
|
161
|
+
}
|
|
162
|
+
//# sourceMappingURL=frontmatter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../../src/indexing/frontmatter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAwBH,MAAM,SAAS,GAAG,KAAK,CAAC;AAExB,8EAA8E;AAC9E,MAAM,SAAS,GAAkC;IAC/C,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,aAAa;IAC1B,MAAM,EAAE,QAAQ;IAChB,aAAa,EAAE,cAAc;IAC7B,WAAW,EAAE,YAAY;CAC1B,CAAC;AAEF;;;;;GAKG;AACH;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,uEAAuE;IACvE,yEAAyE;IACzE,wEAAwE;IACxE,qEAAqE;IACrE,2EAA2E;IAC3E,MAAM,UAAU,GAAG,OAAO;SACvB,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;SACpB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE1B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,SAAS,IAAI,CAAC,EAAE,CAAC;QAC7C,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,EAAE,KAAK,SAAS,EAAE,CAAC;YACtC,OAAO,GAAG,CAAC,CAAC;YACZ,MAAM;QACR,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,sCAAsC;IACtC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,oBAAoB,GAAG,OAAO,GAAG,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE1D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;AAC9C,CAAC;AAED,uEAAuE;AACvE,wEAAwE;AACxE,0EAA0E;AAC1E,wEAAwE;AACxE,uEAAuE;AACvE,mEAAmE;AACnE,2EAA2E;AAC3E,mEAAmE;AACnE,2EAA2E;AAC3E,6EAA6E;AAC7E,uEAAuE;AACvE,0EAA0E;AAC1E,wEAAwE;AACxE,0EAA0E;AAC1E,iBAAiB;AACjB,SAAS,YAAY,CAAC,KAAe;IACnC,MAAM,IAAI,GAAY,EAAE,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,qEAAqE;QACrE,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,SAAS;QAE/B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,SAAS;QAErB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,iBAAiB,CAAC,KAAa,EAAE,KAAgB;IACxD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAExD,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,KAAK,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;YAC3B,OAAO,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9D,CAAC,IAAI,CAAC,CAAC;YACP,SAAS;QACX,CAAC;QAED,IAAI,KAAK,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACxD,OAAO,IAAI,GAAG,CAAC;YACf,CAAC,IAAI,CAAC,CAAC;YACP,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;YACjB,6DAA6D;YAC7D,OAAO,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACjD,CAAC;QAED,OAAO,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,CAAC;IACN,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,qDAAqD;AACpE,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/C,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Scope } from "../types.js";
|
|
2
1
|
export interface IndexProgress {
|
|
3
2
|
phase: "scanning" | "text-indexing" | "embedding" | "done";
|
|
4
3
|
filesProcessed: number;
|
|
@@ -7,7 +6,6 @@ export interface IndexProgress {
|
|
|
7
6
|
}
|
|
8
7
|
export type ProgressCallback = (progress: IndexProgress) => void;
|
|
9
8
|
export interface IndexRunResult {
|
|
10
|
-
scope: Scope;
|
|
11
9
|
filesScanned: number;
|
|
12
10
|
filesChanged: number;
|
|
13
11
|
filesRemoved: number;
|
|
@@ -17,10 +15,9 @@ export interface IndexRunResult {
|
|
|
17
15
|
}
|
|
18
16
|
export interface IndexOptions {
|
|
19
17
|
repoPath: string;
|
|
20
|
-
scope: Scope;
|
|
21
18
|
/** Skip vector embedding entirely (keyword-only index, much faster). */
|
|
22
19
|
skipEmbeddings?: boolean;
|
|
23
|
-
/** Ignore the manifest and rebuild every file
|
|
20
|
+
/** Ignore the manifest and rebuild every file from scratch. */
|
|
24
21
|
force?: boolean;
|
|
25
22
|
/** Permit automatic model download when semantic indexing is required. */
|
|
26
23
|
allowRemoteModels?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAqCA,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,CAAC;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAKD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAyN7E"}
|