@jcode.labs/ragmir 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-log.d.ts.map +1 -1
- package/dist/access-log.js +65 -8
- package/dist/access-log.js.map +1 -1
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +29 -3
- package/dist/chunking.js.map +1 -1
- package/dist/cli.js +81 -21
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +73 -29
- package/dist/config.js.map +1 -1
- package/dist/defaults.d.ts +1 -0
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +6 -1
- package/dist/defaults.js.map +1 -1
- package/dist/destroy.d.ts.map +1 -1
- package/dist/destroy.js +23 -1
- package/dist/destroy.js.map +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +40 -9
- package/dist/doctor.js.map +1 -1
- package/dist/embeddings.d.ts +4 -1
- package/dist/embeddings.d.ts.map +1 -1
- package/dist/embeddings.js +62 -14
- package/dist/embeddings.js.map +1 -1
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +48 -4
- package/dist/evaluate.js.map +1 -1
- package/dist/files.d.ts +8 -1
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +44 -8
- package/dist/files.js.map +1 -1
- package/dist/index-diagnostics.d.ts +1 -1
- package/dist/index-diagnostics.d.ts.map +1 -1
- package/dist/index-diagnostics.js +8 -3
- package/dist/index-diagnostics.js.map +1 -1
- package/dist/index-policy.d.ts +3 -0
- package/dist/index-policy.d.ts.map +1 -0
- package/dist/index-policy.js +28 -0
- package/dist/index-policy.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +112 -54
- package/dist/ingest.js.map +1 -1
- package/dist/init.d.ts.map +1 -1
- package/dist/init.js +15 -3
- package/dist/init.js.map +1 -1
- package/dist/limits.d.ts +9 -0
- package/dist/limits.d.ts.map +1 -0
- package/dist/limits.js +25 -0
- package/dist/limits.js.map +1 -0
- package/dist/mcp.d.ts +3 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +71 -14
- package/dist/mcp.js.map +1 -1
- package/dist/parsing.d.ts.map +1 -1
- package/dist/parsing.js +124 -24
- package/dist/parsing.js.map +1 -1
- package/dist/permissions.d.ts +3 -0
- package/dist/permissions.d.ts.map +1 -0
- package/dist/permissions.js +13 -0
- package/dist/permissions.js.map +1 -0
- package/dist/query.d.ts +2 -2
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +198 -39
- package/dist/query.js.map +1 -1
- package/dist/research.d.ts.map +1 -1
- package/dist/research.js +33 -4
- package/dist/research.js.map +1 -1
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +110 -14
- package/dist/security.js.map +1 -1
- package/dist/store.d.ts +3 -0
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +71 -72
- package/dist/store.js.map +1 -1
- package/dist/text.d.ts.map +1 -1
- package/dist/text.js +13 -1
- package/dist/text.js.map +1 -1
- package/dist/types.d.ts +94 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/examples/document-evidence-benchmark/.ragmir/config.json +2 -1
- package/examples/document-evidence-benchmark/README.md +5 -2
- package/examples/document-evidence-benchmark/golden-queries.json +6 -0
- package/examples/document-evidence-benchmark/raw/contracts/pdf-control-evidence.pdf +32 -0
- package/examples/sovereign-rag-demo/.ragmir/config.json +2 -1
- package/examples/sovereign-rag-demo/README.md +9 -1
- package/package.json +3 -3
- package/skills/ragmir/SKILL.md +11 -3
package/dist/doctor.js
CHANGED
|
@@ -27,14 +27,28 @@ export async function doctor(cwd = process.cwd()) {
|
|
|
27
27
|
manifestFound: manifest !== null,
|
|
28
28
|
warning: freshnessWarning,
|
|
29
29
|
};
|
|
30
|
+
const oversizedFiles = countSkippedByReason(auditReport.skippedFiles, "oversized");
|
|
31
|
+
const sensitiveFiles = countSkippedByReason(auditReport.skippedFiles, "sensitive-name");
|
|
32
|
+
const coverageComplete = auditReport.missingFromIndex.length === 0 &&
|
|
33
|
+
auditReport.staleInIndex.length === 0 &&
|
|
34
|
+
auditReport.emptyTextFiles.length === 0 &&
|
|
35
|
+
oversizedFiles === 0;
|
|
36
|
+
const operationalReady = initialized && chunksIndexed > 0 && coverageComplete;
|
|
37
|
+
const indexPolicyCurrent = freshnessWarning === null;
|
|
38
|
+
const privacyCompliant = securityReport.warnings.length === 0;
|
|
30
39
|
const nextSteps = nextActions({
|
|
31
40
|
initialized,
|
|
32
41
|
supportedFiles: auditReport.supportedFiles.length,
|
|
42
|
+
supportedBytes: auditReport.supportedBytes,
|
|
43
|
+
largestFileBytes: auditReport.largestFileBytes,
|
|
33
44
|
skippedFiles: auditReport.skippedFiles.length,
|
|
34
45
|
unsupportedFiles: countSkippedByReason(auditReport.skippedFiles, "unsupported-extension"),
|
|
35
46
|
chunksIndexed,
|
|
36
47
|
missingFromIndex: auditReport.missingFromIndex.length,
|
|
37
48
|
staleInIndex: auditReport.staleInIndex.length,
|
|
49
|
+
emptyTextFiles: auditReport.emptyTextFiles.length,
|
|
50
|
+
oversizedFiles,
|
|
51
|
+
sensitiveFiles,
|
|
38
52
|
warnings: securityReport.warnings.length,
|
|
39
53
|
embeddingProvider: config.embeddingProvider,
|
|
40
54
|
agentKitInstalled,
|
|
@@ -54,21 +68,32 @@ export async function doctor(cwd = process.cwd()) {
|
|
|
54
68
|
transformersAllowRemoteModels: config.transformersAllowRemoteModels,
|
|
55
69
|
redactionEnabled: config.redaction.enabled,
|
|
56
70
|
accessLog: config.accessLog,
|
|
71
|
+
privacyProfile: config.privacyProfile,
|
|
72
|
+
retrievalProfile: config.retrievalProfile,
|
|
57
73
|
supportedFiles: auditReport.supportedFiles.length,
|
|
74
|
+
supportedBytes: auditReport.supportedBytes,
|
|
75
|
+
largestFileBytes: auditReport.largestFileBytes,
|
|
76
|
+
maxFileBytes: config.maxFileBytes,
|
|
58
77
|
skippedFiles: auditReport.skippedFiles.length,
|
|
59
78
|
unsupportedFiles: countSkippedByReason(auditReport.skippedFiles, "unsupported-extension"),
|
|
79
|
+
oversizedFiles,
|
|
80
|
+
sensitiveFiles,
|
|
81
|
+
emptyTextFiles: auditReport.emptyTextFiles.length,
|
|
60
82
|
indexedFiles: auditReport.indexedFiles.length,
|
|
61
83
|
chunksIndexed,
|
|
62
84
|
missingFromIndex: auditReport.missingFromIndex.length,
|
|
63
85
|
staleInIndex: auditReport.staleInIndex.length,
|
|
64
86
|
securityWarnings: securityReport.warnings,
|
|
65
87
|
indexFreshness,
|
|
66
|
-
ready:
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
88
|
+
ready: operationalReady && indexPolicyCurrent && privacyCompliant,
|
|
89
|
+
readiness: {
|
|
90
|
+
operationalReady,
|
|
91
|
+
coverageComplete,
|
|
92
|
+
indexPolicyCurrent,
|
|
93
|
+
privacyCompliant,
|
|
94
|
+
retrievalQualityVerified: false,
|
|
95
|
+
acceptedRisks: config.acceptedRisks,
|
|
96
|
+
},
|
|
72
97
|
nextSteps,
|
|
73
98
|
};
|
|
74
99
|
}
|
|
@@ -98,13 +123,19 @@ function nextActions(input) {
|
|
|
98
123
|
if (input.lexicalScanWarning) {
|
|
99
124
|
steps.push(input.lexicalScanWarning);
|
|
100
125
|
}
|
|
126
|
+
if (input.emptyTextFiles > 0) {
|
|
127
|
+
steps.push(`${input.emptyTextFiles} supported source file(s) produced no indexable text. Configure local OCR for scans/images, convert the files, or add extracted text, then re-ingest.`);
|
|
128
|
+
}
|
|
129
|
+
if (input.oversizedFiles > 0) {
|
|
130
|
+
steps.push(`${input.oversizedFiles} source file(s) exceed maxFileBytes. Run \`rgr audit --unsupported\`, then split the files or raise the limit only after reviewing the memory and parsing risk.`);
|
|
131
|
+
}
|
|
132
|
+
if (input.unsupportedFiles > 0 || input.sensitiveFiles > 0) {
|
|
133
|
+
steps.push("Run `rgr audit --unsupported` to review every intentionally skipped source file.");
|
|
134
|
+
}
|
|
101
135
|
if (input.warnings > 0) {
|
|
102
136
|
steps.push(`Run \`${input.run(["security-audit", "--strict"])}\` and fix the reported warnings.`);
|
|
103
137
|
}
|
|
104
138
|
if (steps.length === 0) {
|
|
105
|
-
if (input.unsupportedFiles > 0) {
|
|
106
|
-
steps.push("Run `rgr audit --unsupported` to inspect files skipped because their type is not supported.");
|
|
107
|
-
}
|
|
108
139
|
if (input.embeddingProvider === "local-hash") {
|
|
109
140
|
steps.push(`For natural-language Q&A, run \`${input.run(["models", "pull", "--enable"])}\`, then run \`${input.run(["ingest", "--rebuild"])}\`.`);
|
|
110
141
|
}
|
package/dist/doctor.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AACxF,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EACL,6BAA6B,EAC7B,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,GACZ,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAGzD,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC9C,MAAM,aAAa,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;IAC5C,MAAM,WAAW,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;IACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;IACxD,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACjE,MAAM,CAAC,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,EAAE,gBAAgB,CAAC,GAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC;QACzB,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC;QACjC,SAAS,CAAC,MAAM,CAAC;QACjB,iBAAiB,CAAC,MAAM,CAAC;QACzB,wBAAwB,CAAC,MAAM,CAAC;KACjC,CAAC,CAAA;IAEJ,MAAM,kBAAkB,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAClG,MAAM,cAAc,GAAG;QACrB,aAAa,EAAE,QAAQ,KAAK,IAAI;QAChC,OAAO,EAAE,gBAAgB;KAC1B,CAAA;
|
|
1
|
+
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AACxF,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EACL,6BAA6B,EAC7B,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,GACZ,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAGzD,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC9C,MAAM,aAAa,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;IAC5C,MAAM,WAAW,GAAG,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;IACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;IACxD,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACjE,MAAM,CAAC,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,EAAE,gBAAgB,CAAC,GAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC;QACzB,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC;QACjC,SAAS,CAAC,MAAM,CAAC;QACjB,iBAAiB,CAAC,MAAM,CAAC;QACzB,wBAAwB,CAAC,MAAM,CAAC;KACjC,CAAC,CAAA;IAEJ,MAAM,kBAAkB,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAClG,MAAM,cAAc,GAAG;QACrB,aAAa,EAAE,QAAQ,KAAK,IAAI;QAChC,OAAO,EAAE,gBAAgB;KAC1B,CAAA;IACD,MAAM,cAAc,GAAG,oBAAoB,CAAC,WAAW,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAClF,MAAM,cAAc,GAAG,oBAAoB,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAA;IACvF,MAAM,gBAAgB,GACpB,WAAW,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC;QACzC,WAAW,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;QACrC,WAAW,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;QACvC,cAAc,KAAK,CAAC,CAAA;IACtB,MAAM,gBAAgB,GAAG,WAAW,IAAI,aAAa,GAAG,CAAC,IAAI,gBAAgB,CAAA;IAC7E,MAAM,kBAAkB,GAAG,gBAAgB,KAAK,IAAI,CAAA;IACpD,MAAM,gBAAgB,GAAG,cAAc,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAA;IAE7D,MAAM,SAAS,GAAG,WAAW,CAAC;QAC5B,WAAW;QACX,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC,MAAM;QACjD,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;QAC9C,YAAY,EAAE,WAAW,CAAC,YAAY,CAAC,MAAM;QAC7C,gBAAgB,EAAE,oBAAoB,CAAC,WAAW,CAAC,YAAY,EAAE,uBAAuB,CAAC;QACzF,aAAa;QACb,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,CAAC,MAAM;QACrD,YAAY,EAAE,WAAW,CAAC,YAAY,CAAC,MAAM;QAC7C,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC,MAAM;QACjD,cAAc;QACd,cAAc;QACd,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,MAAM;QACxC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,iBAAiB;QACjB,gBAAgB;QAChB,kBAAkB;QAClB,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAC,CAAA;IAEF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,WAAW;QACX,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,UAAU,EAAE,OAAO,CAAC,OAAO;QAC3B,iBAAiB;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,6BAA6B,EAAE,MAAM,CAAC,6BAA6B;QACnE,gBAAgB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;QAC1C,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC,MAAM;QACjD,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;QAC9C,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,WAAW,CAAC,YAAY,CAAC,MAAM;QAC7C,gBAAgB,EAAE,oBAAoB,CAAC,WAAW,CAAC,YAAY,EAAE,uBAAuB,CAAC;QACzF,cAAc;QACd,cAAc;QACd,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC,MAAM;QACjD,YAAY,EAAE,WAAW,CAAC,YAAY,CAAC,MAAM;QAC7C,aAAa;QACb,gBAAgB,EAAE,WAAW,CAAC,gBAAgB,CAAC,MAAM;QACrD,YAAY,EAAE,WAAW,CAAC,YAAY,CAAC,MAAM;QAC7C,gBAAgB,EAAE,cAAc,CAAC,QAAQ;QACzC,cAAc;QACd,KAAK,EAAE,gBAAgB,IAAI,kBAAkB,IAAI,gBAAgB;QACjE,SAAS,EAAE;YACT,gBAAgB;YAChB,gBAAgB;YAChB,kBAAkB;YAClB,gBAAgB;YAChB,wBAAwB,EAAE,KAAK;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa;SACpC;QACD,SAAS;KACV,CAAA;AACH,CAAC;AAuBD,SAAS,WAAW,CAAC,KAAsB;IACzC,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,oDAAoD,CAAC,CAAA;QAC7F,OAAO,KAAK,CAAA;IACd,CAAC;IAED,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACR,+GAA+G,CAChH,CAAA;QACH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CACR,kHAAkH,CACnH,CAAA;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QACtF,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,4CAA4C,CAAC,CAAA;QAC/F,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,qDAAqD,CAAC,CAAA;QAC9F,KAAK,CAAC,IAAI,CACR,uKAAuK,CACxK,CAAA;IACH,CAAC;IAED,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CACR,GAAG,KAAK,CAAC,gBAAgB,UAAU,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,sDAAsD,CAC5H,CAAA;IACH,CAAC;IAED,IAAI,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAA;IACtC,CAAC;IAED,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,GAAG,KAAK,CAAC,cAAc,uJAAuJ,CAC/K,CAAA;IACH,CAAC;IAED,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR,GAAG,KAAK,CAAC,cAAc,iKAAiK,CACzL,CAAA;IACH,CAAC;IAED,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAA;IAChG,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC,mCAAmC,CACtF,CAAA;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,IAAI,KAAK,CAAC,iBAAiB,KAAK,YAAY,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CACR,mCAAmC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,kBAAkB,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,KAAK,CACtI,CAAA;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,iCAAiC,CAAC,CAAA;QAC9F,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,wCAAwC,CACvF,CAAA;QACD,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,2CAA2C,CAC5F,CAAA;QACD,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CACR,oGAAoG,CACrG,CAAA;YACD,KAAK,CAAC,IAAI,CACR,8FAA8F,CAC/F,CAAA;QACH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,wDAAwD,CAC9F,CAAA;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAmB;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG;QACpB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QACxF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,oBAAoB,CAAC;KAC3C,CAAA;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,6BAA6B,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAC/B,CAAA;IACD,OAAO,CACL,aAAa,CAAC,KAAK,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC/D,YAAY,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAC1D,CAAA;AACH,CAAC"}
|
package/dist/embeddings.d.ts
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import type { Config } from "./types.js";
|
|
2
|
+
type EmbeddingInputType = "document" | "query";
|
|
2
3
|
export interface PullEmbeddingModelResult {
|
|
3
4
|
embeddingModel: string;
|
|
4
5
|
embeddingModelPath: string;
|
|
5
6
|
}
|
|
6
|
-
export declare function embedTexts(texts: string[], config: Config): Promise<number[][]>;
|
|
7
|
+
export declare function embedTexts(texts: string[], config: Config, inputType?: EmbeddingInputType): Promise<number[][]>;
|
|
7
8
|
export declare function pullEmbeddingModel(config: Config): Promise<PullEmbeddingModelResult>;
|
|
8
9
|
export declare function embedText(text: string, config: Config): Promise<number[]>;
|
|
10
|
+
export declare function prepareEmbeddingText(text: string, model: string, inputType: EmbeddingInputType): string;
|
|
9
11
|
/**
|
|
10
12
|
* Release all cached Transformers.js pipelines. Used by `destroy-index` and on
|
|
11
13
|
* embedding-config changes in long-lived processes (MCP server) so stale model
|
|
12
14
|
* weights are not pinned in memory.
|
|
13
15
|
*/
|
|
14
16
|
export declare function clearTransformersCache(): void;
|
|
17
|
+
export {};
|
|
15
18
|
//# sourceMappingURL=embeddings.d.ts.map
|
package/dist/embeddings.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAuBxC,KAAK,kBAAkB,GAAG,UAAU,GAAG,OAAO,CAAA;AAE9C,MAAM,WAAW,wBAAwB;IACvC,cAAc,EAAE,MAAM,CAAA;IACtB,kBAAkB,EAAE,MAAM,CAAA;CAC3B;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,EAAE,MAAM,EACd,SAAS,GAAE,kBAA+B,GACzC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAUrB;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAe1F;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAM/E;AAqBD,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,kBAAkB,GAC5B,MAAM,CAQR;AA+DD;;;;GAIG;AACH,wBAAgB,sBAAsB,IAAI,IAAI,CAE7C"}
|
package/dist/embeddings.js
CHANGED
|
@@ -4,6 +4,8 @@ import { tokenize } from "./text.js";
|
|
|
4
4
|
const LOCAL_HASH_DIMENSIONS = 384;
|
|
5
5
|
const LONG_TOKEN_MIN_LENGTH = 6;
|
|
6
6
|
const LONG_TOKEN_WEIGHT = 1.4;
|
|
7
|
+
const CHARACTER_NGRAM_LENGTH = 3;
|
|
8
|
+
const CHARACTER_NGRAM_WEIGHT = 0.35;
|
|
7
9
|
/**
|
|
8
10
|
* Maximum number of Transformers.js pipelines kept live in the process. Each
|
|
9
11
|
* pipeline pins ONNX runtime weights (often hundreds of MB), so a long-lived
|
|
@@ -13,14 +15,15 @@ const LONG_TOKEN_WEIGHT = 1.4;
|
|
|
13
15
|
*/
|
|
14
16
|
const MAX_TRANSFORMERS_PIPELINES = 3;
|
|
15
17
|
const transformersPipelines = new Map();
|
|
16
|
-
|
|
18
|
+
let transformersEnvironmentQueue = Promise.resolve();
|
|
19
|
+
export async function embedTexts(texts, config, inputType = "document") {
|
|
17
20
|
if (texts.length === 0) {
|
|
18
21
|
return [];
|
|
19
22
|
}
|
|
20
23
|
if (config.embeddingProvider === "local-hash") {
|
|
21
24
|
return texts.map(localHashEmbedding);
|
|
22
25
|
}
|
|
23
|
-
return embedWithTransformers(texts, config);
|
|
26
|
+
return embedWithTransformers(texts, config, inputType);
|
|
24
27
|
}
|
|
25
28
|
export async function pullEmbeddingModel(config) {
|
|
26
29
|
await mkdir(config.embeddingModelPath, { recursive: true });
|
|
@@ -39,24 +42,35 @@ export async function pullEmbeddingModel(config) {
|
|
|
39
42
|
};
|
|
40
43
|
}
|
|
41
44
|
export async function embedText(text, config) {
|
|
42
|
-
const [embedding] = await embedTexts([text], config);
|
|
45
|
+
const [embedding] = await embedTexts([text], config, "query");
|
|
43
46
|
if (!embedding) {
|
|
44
47
|
throw new Error("No embedding returned for query.");
|
|
45
48
|
}
|
|
46
49
|
return embedding;
|
|
47
50
|
}
|
|
48
|
-
async function embedWithTransformers(texts, config) {
|
|
51
|
+
async function embedWithTransformers(texts, config, inputType) {
|
|
49
52
|
const extractor = await transformersExtractor(config);
|
|
50
|
-
const
|
|
53
|
+
const preparedTexts = texts.map((text) => prepareEmbeddingText(text, config.embeddingModel, inputType));
|
|
54
|
+
const output = await extractor(preparedTexts, { pooling: "mean", normalize: true });
|
|
51
55
|
const rows = tensorToEmbeddingRows(output);
|
|
52
56
|
if (rows.length !== texts.length) {
|
|
53
57
|
throw new Error(`Expected ${texts.length} embeddings, received ${rows.length}.`);
|
|
54
58
|
}
|
|
55
59
|
return rows;
|
|
56
60
|
}
|
|
61
|
+
export function prepareEmbeddingText(text, model, inputType) {
|
|
62
|
+
if (/(^|\/)(multilingual-)?e5-/iu.test(model)) {
|
|
63
|
+
return `${inputType === "query" ? "query" : "passage"}: ${text}`;
|
|
64
|
+
}
|
|
65
|
+
if (inputType === "query" && /(^|\/)mxbai-embed/iu.test(model)) {
|
|
66
|
+
return `Represent this sentence for searching relevant passages: ${text}`;
|
|
67
|
+
}
|
|
68
|
+
return text;
|
|
69
|
+
}
|
|
57
70
|
async function transformersExtractor(config) {
|
|
58
71
|
const key = [
|
|
59
72
|
config.embeddingModel,
|
|
73
|
+
config.embeddingModelRevision,
|
|
60
74
|
config.embeddingModelPath,
|
|
61
75
|
String(config.transformersAllowRemoteModels),
|
|
62
76
|
].join("\n");
|
|
@@ -67,15 +81,36 @@ async function transformersExtractor(config) {
|
|
|
67
81
|
transformersPipelines.set(key, cached);
|
|
68
82
|
return cached;
|
|
69
83
|
}
|
|
70
|
-
const
|
|
71
|
-
transformers.env.localModelPath = config.embeddingModelPath;
|
|
72
|
-
transformers.env.cacheDir = config.embeddingModelPath;
|
|
73
|
-
transformers.env.allowRemoteModels = config.transformersAllowRemoteModels;
|
|
74
|
-
const extractor = (await transformers.pipeline("feature-extraction", config.embeddingModel));
|
|
84
|
+
const extractor = await createTransformersExtractor(config);
|
|
75
85
|
evictTransformersPipeline();
|
|
76
86
|
transformersPipelines.set(key, extractor);
|
|
77
87
|
return extractor;
|
|
78
88
|
}
|
|
89
|
+
async function createTransformersExtractor(config) {
|
|
90
|
+
const creation = transformersEnvironmentQueue.then(async () => {
|
|
91
|
+
const transformers = await import("@huggingface/transformers");
|
|
92
|
+
const previous = {
|
|
93
|
+
localModelPath: transformers.env.localModelPath,
|
|
94
|
+
cacheDir: transformers.env.cacheDir,
|
|
95
|
+
allowRemoteModels: transformers.env.allowRemoteModels,
|
|
96
|
+
};
|
|
97
|
+
transformers.env.localModelPath = config.embeddingModelPath;
|
|
98
|
+
transformers.env.cacheDir = config.embeddingModelPath;
|
|
99
|
+
transformers.env.allowRemoteModels = config.transformersAllowRemoteModels;
|
|
100
|
+
try {
|
|
101
|
+
return (await transformers.pipeline("feature-extraction", config.embeddingModel, {
|
|
102
|
+
revision: config.embeddingModelRevision,
|
|
103
|
+
}));
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
transformers.env.localModelPath = previous.localModelPath;
|
|
107
|
+
transformers.env.cacheDir = previous.cacheDir;
|
|
108
|
+
transformers.env.allowRemoteModels = previous.allowRemoteModels;
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
transformersEnvironmentQueue = creation.then(() => undefined, () => undefined);
|
|
112
|
+
return creation;
|
|
113
|
+
}
|
|
79
114
|
function evictTransformersPipeline() {
|
|
80
115
|
while (transformersPipelines.size >= MAX_TRANSFORMERS_PIPELINES) {
|
|
81
116
|
// Map iteration order is insertion order, so the first key is the least
|
|
@@ -99,10 +134,10 @@ function localHashEmbedding(text) {
|
|
|
99
134
|
const vector = Array.from({ length: LOCAL_HASH_DIMENSIONS }, () => 0);
|
|
100
135
|
const tokens = tokenize(text);
|
|
101
136
|
for (const token of tokens) {
|
|
102
|
-
|
|
103
|
-
const
|
|
104
|
-
|
|
105
|
-
|
|
137
|
+
addHashedFeature(vector, token, tokenWeight(token));
|
|
138
|
+
for (const ngram of characterNgrams(token)) {
|
|
139
|
+
addHashedFeature(vector, `ngram:${ngram}`, CHARACTER_NGRAM_WEIGHT);
|
|
140
|
+
}
|
|
106
141
|
}
|
|
107
142
|
const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
|
|
108
143
|
if (magnitude === 0) {
|
|
@@ -110,6 +145,19 @@ function localHashEmbedding(text) {
|
|
|
110
145
|
}
|
|
111
146
|
return vector.map((value) => value / magnitude);
|
|
112
147
|
}
|
|
148
|
+
function addHashedFeature(vector, feature, weight) {
|
|
149
|
+
const hash = createHash("sha256").update(feature).digest();
|
|
150
|
+
const index = hash.readUInt32BE(0) % LOCAL_HASH_DIMENSIONS;
|
|
151
|
+
const sign = (hash.at(4) ?? 0) % 2 === 0 ? 1 : -1;
|
|
152
|
+
vector[index] = (vector[index] ?? 0) + sign * weight;
|
|
153
|
+
}
|
|
154
|
+
function characterNgrams(token) {
|
|
155
|
+
const characters = [...token];
|
|
156
|
+
if (characters.length < CHARACTER_NGRAM_LENGTH) {
|
|
157
|
+
return [];
|
|
158
|
+
}
|
|
159
|
+
return Array.from({ length: characters.length - CHARACTER_NGRAM_LENGTH + 1 }, (_value, index) => characters.slice(index, index + CHARACTER_NGRAM_LENGTH).join(""));
|
|
160
|
+
}
|
|
113
161
|
function tokenWeight(token) {
|
|
114
162
|
return token.length >= LONG_TOKEN_MIN_LENGTH ? LONG_TOKEN_WEIGHT : 1;
|
|
115
163
|
}
|
package/dist/embeddings.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAGpC,MAAM,qBAAqB,GAAG,GAAG,CAAA;AACjC,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAC/B,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAC7B;;;;;;GAMG;AACH,MAAM,0BAA0B,GAAG,CAAC,CAAA;AACpC,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAiC,CAAA;
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAGpC,MAAM,qBAAqB,GAAG,GAAG,CAAA;AACjC,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAC/B,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAC7B,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAChC,MAAM,sBAAsB,GAAG,IAAI,CAAA;AACnC;;;;;;GAMG;AACH,MAAM,0BAA0B,GAAG,CAAC,CAAA;AACpC,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAiC,CAAA;AACtE,IAAI,4BAA4B,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAA;AAcnE,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,MAAc,EACd,YAAgC,UAAU;IAE1C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,IAAI,MAAM,CAAC,iBAAiB,KAAK,YAAY,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAA;IACtC,CAAC;IAED,OAAO,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc;IACrD,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3D,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC;QAC5C,GAAG,MAAM;QACT,iBAAiB,EAAE,cAAc;QACjC,6BAA6B,EAAE,IAAI;KACpC,CAAC,CAAA;IACF,MAAM,SAAS,CAAC,CAAC,4CAA4C,CAAC,EAAE;QAC9D,OAAO,EAAE,MAAM;QACf,SAAS,EAAE,IAAI;KAChB,CAAC,CAAA;IACF,OAAO;QACL,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;KAC9C,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc;IAC1D,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;IAC7D,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,KAAe,EACf,MAAc,EACd,SAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAA;IACrD,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACvC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,cAAc,EAAE,SAAS,CAAC,CAC7D,CAAA;IACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnF,MAAM,IAAI,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAA;IAE1C,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,MAAM,yBAAyB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAClF,CAAC;IAED,OAAO,IAAI,CAAA;AACb,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,KAAa,EACb,SAA6B;IAE7B,IAAI,6BAA6B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9C,OAAO,GAAG,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,EAAE,CAAA;IAClE,CAAC;IACD,IAAI,SAAS,KAAK,OAAO,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/D,OAAO,4DAA4D,IAAI,EAAE,CAAA;IAC3E,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,MAAc;IACjD,MAAM,GAAG,GAAG;QACV,MAAM,CAAC,cAAc;QACrB,MAAM,CAAC,sBAAsB;QAC7B,MAAM,CAAC,kBAAkB;QACzB,MAAM,CAAC,MAAM,CAAC,6BAA6B,CAAC;KAC7C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC7C,IAAI,MAAM,EAAE,CAAC;QACX,wEAAwE;QACxE,qBAAqB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACjC,qBAAqB,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QACtC,OAAO,MAAM,CAAA;IACf,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,2BAA2B,CAAC,MAAM,CAAC,CAAA;IAC3D,yBAAyB,EAAE,CAAA;IAC3B,qBAAqB,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACzC,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,MAAc;IACvD,MAAM,QAAQ,GAAG,4BAA4B,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QAC5D,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;QAC9D,MAAM,QAAQ,GAAG;YACf,cAAc,EAAE,YAAY,CAAC,GAAG,CAAC,cAAc;YAC/C,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,QAAQ;YACnC,iBAAiB,EAAE,YAAY,CAAC,GAAG,CAAC,iBAAiB;SACtD,CAAA;QACD,YAAY,CAAC,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,kBAAkB,CAAA;QAC3D,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,kBAAkB,CAAA;QACrD,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,6BAA6B,CAAA;QACzE,IAAI,CAAC;YACH,OAAO,CAAC,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC,cAAc,EAAE;gBAC/E,QAAQ,EAAE,MAAM,CAAC,sBAAsB;aACxC,CAAC,CAA0B,CAAA;QAC9B,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,GAAG,CAAC,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAA;YACzD,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAA;YAC7C,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,QAAQ,CAAC,iBAAiB,CAAA;QACjE,CAAC;IACH,CAAC,CAAC,CAAA;IACF,4BAA4B,GAAG,QAAQ,CAAC,IAAI,CAC1C,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAA;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO,qBAAqB,CAAC,IAAI,IAAI,0BAA0B,EAAE,CAAC;QAChE,wEAAwE;QACxE,uBAAuB;QACvB,MAAM,MAAM,GAAG,qBAAqB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;QAClD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAK;QACP,CAAC;QACD,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAC5C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB;IACpC,qBAAqB,CAAC,KAAK,EAAE,CAAA;AAC/B,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAA;QACnD,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,gBAAgB,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QACpE,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;IAClF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,MAAM,CAAA;IACf,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,SAAS,CAAC,CAAA;AACjD,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAgB,EAAE,OAAe,EAAE,MAAc;IACzE,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAA;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAA;IAC1D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACjD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAA;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,MAAM,UAAU,GAAG,CAAC,GAAG,KAAK,CAAC,CAAA;IAC7B,IAAI,UAAU,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC/C,OAAO,EAAE,CAAA;IACX,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,sBAAsB,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAC9F,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CACjE,CAAA;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,KAAK,CAAC,MAAM,IAAI,qBAAqB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAA;AACtE,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAe;IAC5C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAA;IAC5E,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC7B,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,CAAA;IAChB,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAA;AAClF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAO,KAAK,CAAC,MAAM,KAAK,UAAU,CACnC,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAA;AAChF,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;AAC3D,CAAC"}
|
package/dist/evaluate.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAEV,iBAAiB,EACjB,gBAAgB,EAEjB,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAEV,iBAAiB,EACjB,gBAAgB,EAEjB,MAAM,YAAY,CAAA;AAwBnB,wBAAsB,qBAAqB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAiGjG"}
|
package/dist/evaluate.js
CHANGED
|
@@ -10,6 +10,8 @@ const goldenQuerySchema = z
|
|
|
10
10
|
query: z.string().min(1),
|
|
11
11
|
expectedPaths: z.array(z.string().min(1)).min(1),
|
|
12
12
|
expectedCitations: z.array(z.string().min(1)).min(1).optional(),
|
|
13
|
+
includePaths: z.array(z.string().min(1).max(500)).max(20).optional(),
|
|
14
|
+
excludePaths: z.array(z.string().min(1).max(500)).max(20).optional(),
|
|
13
15
|
topK: z.number().int().positive().optional(),
|
|
14
16
|
})
|
|
15
17
|
.strict();
|
|
@@ -31,7 +33,14 @@ export async function evaluateGoldenQueries(options) {
|
|
|
31
33
|
const cases = [];
|
|
32
34
|
for (const goldenQuery of goldenFile.queries) {
|
|
33
35
|
const topK = boundedTopK(goldenQuery.topK ?? defaultTopK, options.maxTopK);
|
|
34
|
-
const
|
|
36
|
+
const startedAt = performance.now();
|
|
37
|
+
const results = await search(goldenQuery.query, {
|
|
38
|
+
cwd,
|
|
39
|
+
topK,
|
|
40
|
+
...(goldenQuery.includePaths === undefined ? {} : { includePaths: goldenQuery.includePaths }),
|
|
41
|
+
...(goldenQuery.excludePaths === undefined ? {} : { excludePaths: goldenQuery.excludePaths }),
|
|
42
|
+
});
|
|
43
|
+
const latencyMs = performance.now() - startedAt;
|
|
35
44
|
const returnedPaths = results.map((result) => result.relativePath);
|
|
36
45
|
const returnedCitations = results.map(citationForResult);
|
|
37
46
|
const expectedCitations = goldenQuery.expectedCitations ?? [];
|
|
@@ -43,7 +52,13 @@ export async function evaluateGoldenQueries(options) {
|
|
|
43
52
|
? returnedCitations.findIndex((citation) => expectedCitations.includes(citation)) + 1
|
|
44
53
|
: returnedPaths.findIndex((resultPath) => goldenQuery.expectedPaths.includes(resultPath)) + 1;
|
|
45
54
|
const reciprocalRank = bestRank > 0 ? 1 / bestRank : 0;
|
|
46
|
-
const
|
|
55
|
+
const returnedValues = requiresExactCitation ? returnedCitations : returnedPaths;
|
|
56
|
+
const uniqueReturnedValues = [...new Set(returnedValues.slice(0, topK))];
|
|
57
|
+
const expectedSet = new Set(expectedValues);
|
|
58
|
+
const matchedRelevant = uniqueReturnedValues.filter((value) => expectedSet.has(value));
|
|
59
|
+
const recall = expectedSet.size === 0 ? 0 : matchedRelevant.length / expectedSet.size;
|
|
60
|
+
const precision = uniqueReturnedValues.length === 0 ? 0 : matchedRelevant.length / uniqueReturnedValues.length;
|
|
61
|
+
const ndcg = ndcgAtK(returnedValues, expectedValues, topK);
|
|
47
62
|
const result = {
|
|
48
63
|
query: goldenQuery.query,
|
|
49
64
|
expectedPaths: goldenQuery.expectedPaths,
|
|
@@ -55,7 +70,10 @@ export async function evaluateGoldenQueries(options) {
|
|
|
55
70
|
hit: requiresExactCitation ? matchedCitations.length > 0 : matchedPaths.length > 0,
|
|
56
71
|
bestRank: bestRank > 0 ? bestRank : null,
|
|
57
72
|
reciprocalRank,
|
|
73
|
+
recall,
|
|
74
|
+
precision,
|
|
58
75
|
ndcg,
|
|
76
|
+
latencyMs,
|
|
59
77
|
};
|
|
60
78
|
if (goldenQuery.id !== undefined) {
|
|
61
79
|
result.id = goldenQuery.id;
|
|
@@ -63,9 +81,16 @@ export async function evaluateGoldenQueries(options) {
|
|
|
63
81
|
if (goldenQuery.expectedCitations !== undefined) {
|
|
64
82
|
result.expectedCitations = goldenQuery.expectedCitations;
|
|
65
83
|
}
|
|
84
|
+
if (goldenQuery.includePaths !== undefined) {
|
|
85
|
+
result.includePaths = goldenQuery.includePaths;
|
|
86
|
+
}
|
|
87
|
+
if (goldenQuery.excludePaths !== undefined) {
|
|
88
|
+
result.excludePaths = goldenQuery.excludePaths;
|
|
89
|
+
}
|
|
66
90
|
cases.push(result);
|
|
67
91
|
}
|
|
68
92
|
const hits = cases.filter((result) => result.hit).length;
|
|
93
|
+
const latencies = cases.map((result) => result.latencyMs).sort((a, b) => a - b);
|
|
69
94
|
await recordAccess(config, {
|
|
70
95
|
action: "evaluate",
|
|
71
96
|
topK: defaultTopK,
|
|
@@ -79,9 +104,13 @@ export async function evaluateGoldenQueries(options) {
|
|
|
79
104
|
total: cases.length,
|
|
80
105
|
hits,
|
|
81
106
|
misses: cases.length - hits,
|
|
82
|
-
|
|
107
|
+
hitRate: hits / cases.length,
|
|
108
|
+
recall: mean(cases.map((result) => result.recall)),
|
|
109
|
+
precision: mean(cases.map((result) => result.precision)),
|
|
83
110
|
meanReciprocalRank: mean(cases.map((result) => result.reciprocalRank)),
|
|
84
111
|
ndcg: mean(cases.map((result) => result.ndcg)),
|
|
112
|
+
p50LatencyMs: percentile(latencies, 0.5),
|
|
113
|
+
p95LatencyMs: percentile(latencies, 0.95),
|
|
85
114
|
cases,
|
|
86
115
|
};
|
|
87
116
|
}
|
|
@@ -111,6 +140,12 @@ function normalizeGoldenQuery(value) {
|
|
|
111
140
|
if (value.topK !== undefined) {
|
|
112
141
|
result.topK = value.topK;
|
|
113
142
|
}
|
|
143
|
+
if (value.includePaths !== undefined) {
|
|
144
|
+
result.includePaths = value.includePaths;
|
|
145
|
+
}
|
|
146
|
+
if (value.excludePaths !== undefined) {
|
|
147
|
+
result.excludePaths = value.excludePaths;
|
|
148
|
+
}
|
|
114
149
|
return result;
|
|
115
150
|
}
|
|
116
151
|
function boundedTopK(topK, maxTopK) {
|
|
@@ -122,16 +157,25 @@ function citationForResult(result) {
|
|
|
122
157
|
function ndcgAtK(returned, expected, topK) {
|
|
123
158
|
const expectedSet = new Set(expected);
|
|
124
159
|
const returnedAtK = returned.slice(0, topK);
|
|
160
|
+
const seen = new Set();
|
|
125
161
|
const dcg = returnedAtK.reduce((score, value, index) => {
|
|
126
|
-
if (!expectedSet.has(value)) {
|
|
162
|
+
if (!expectedSet.has(value) || seen.has(value)) {
|
|
127
163
|
return score;
|
|
128
164
|
}
|
|
165
|
+
seen.add(value);
|
|
129
166
|
return score + 1 / Math.log2(index + 2);
|
|
130
167
|
}, 0);
|
|
131
168
|
const idealMatches = Math.min(expectedSet.size, topK);
|
|
132
169
|
const idealDcg = Array.from({ length: idealMatches }, (_value, index) => 1 / Math.log2(index + 2)).reduce((score, gain) => score + gain, 0);
|
|
133
170
|
return idealDcg === 0 ? 0 : dcg / idealDcg;
|
|
134
171
|
}
|
|
172
|
+
function percentile(sortedValues, quantile) {
|
|
173
|
+
if (sortedValues.length === 0) {
|
|
174
|
+
return 0;
|
|
175
|
+
}
|
|
176
|
+
const index = Math.ceil(quantile * sortedValues.length) - 1;
|
|
177
|
+
return sortedValues[Math.max(0, index)] ?? 0;
|
|
178
|
+
}
|
|
135
179
|
function mean(values) {
|
|
136
180
|
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
137
181
|
}
|
package/dist/evaluate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAQnC,MAAM,iBAAiB,GAAG,CAAC;KACxB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/D,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC;KACD,MAAM,EAAE,CAAA;AAEX,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC;IAC/B,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;SACE,MAAM,CAAC;QACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;QAC5C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;KAC3C,CAAC;SACD,MAAM,EAAE;CACZ,CAAC,CAAA;AAEF,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAA0B;IACpE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAC9D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAChE,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,IAAI,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IACtF,MAAM,KAAK,GAA2B,EAAE,CAAA;IAExC,KAAK,MAAM,WAAW,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,IAAI,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;QAC1E,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,
|
|
1
|
+
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAQnC,MAAM,iBAAiB,GAAG,CAAC;KACxB,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/D,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IACpE,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IACpE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC;KACD,MAAM,EAAE,CAAA;AAEX,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC;IAC/B,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;SACE,MAAM,CAAC;QACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;QAC5C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;KAC3C,CAAC;SACD,MAAM,EAAE;CACZ,CAAC,CAAA;AAEF,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAA0B;IACpE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAC9D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAChE,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,IAAI,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IACtF,MAAM,KAAK,GAA2B,EAAE,CAAA;IAExC,KAAK,MAAM,WAAW,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,IAAI,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;QAC1E,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QACnC,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;YAC9C,GAAG;YACH,IAAI;YACJ,GAAG,CAAC,WAAW,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,CAAC,YAAY,EAAE,CAAC;YAC7F,GAAG,CAAC,WAAW,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,CAAC,YAAY,EAAE,CAAC;SAC9F,CAAC,CAAA;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;QAC/C,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;QAClE,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAA;QACxD,MAAM,iBAAiB,GAAG,WAAW,CAAC,iBAAiB,IAAI,EAAE,CAAA;QAC7D,MAAM,qBAAqB,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAA;QAC1D,MAAM,cAAc,GAAG,qBAAqB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAA;QAC5F,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CACvD,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAC/C,CAAA;QACD,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC7D,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACrC,CAAA;QACD,MAAM,QAAQ,GAAG,qBAAqB;YACpC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC;YACrF,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAA;QAC/F,MAAM,cAAc,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;QACtD,MAAM,cAAc,GAAG,qBAAqB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAA;QAChF,MAAM,oBAAoB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;QACxE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAA;QAC3C,MAAM,eAAe,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;QACtF,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAA;QACrF,MAAM,SAAS,GACb,oBAAoB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAA;QAC9F,MAAM,IAAI,GAAG,OAAO,CAAC,cAAc,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QAE1D,MAAM,MAAM,GAAyB;YACnC,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,aAAa,EAAE,WAAW,CAAC,aAAa;YACxC,IAAI;YACJ,aAAa;YACb,iBAAiB;YACjB,YAAY;YACZ,gBAAgB;YAChB,GAAG,EAAE,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YAClF,QAAQ,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;YACxC,cAAc;YACd,MAAM;YACN,SAAS;YACT,IAAI;YACJ,SAAS;SACV,CAAA;QACD,IAAI,WAAW,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,EAAE,GAAG,WAAW,CAAC,EAAE,CAAA;QAC5B,CAAC;QACD,IAAI,WAAW,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;YAChD,MAAM,CAAC,iBAAiB,GAAG,WAAW,CAAC,iBAAiB,CAAA;QAC1D,CAAC;QACD,IAAI,WAAW,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC3C,MAAM,CAAC,YAAY,GAAG,WAAW,CAAC,YAAY,CAAA;QAChD,CAAC;QACD,IAAI,WAAW,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC3C,MAAM,CAAC,YAAY,GAAG,WAAW,CAAC,YAAY,CAAA;QAChD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACpB,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAA;IACxD,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/E,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,WAAW;QACjB,WAAW,EAAE,KAAK,CAAC,MAAM;KAC1B,CAAC,CAAA;IACF,OAAO;QACL,UAAU;QACV,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,IAAI;QACJ,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI;QAC3B,OAAO,EAAE,IAAI,GAAG,KAAK,CAAC,MAAM;QAC5B,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACxD,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QACtE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC9C,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;QACxC,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC;QACzC,KAAK;KACN,CAAA;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,UAAkB;IAElB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;IAC9C,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAA;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAA;IACpE,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAA;IACzC,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAwC;IACpE,MAAM,MAAM,GAAgB;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,aAAa,EAAE,KAAK,CAAC,aAAa;KACnC,CAAA;IACD,IAAI,KAAK,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAC1C,MAAM,CAAC,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAA;IACpD,CAAC;IACD,IAAI,KAAK,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAA;IACtB,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAA;IAC1C,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAA;IAC1C,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,OAA2B;IAC5D,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;AAC/D,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA4B;IACrD,OAAO,MAAM,CAAC,QAAQ,CAAA;AACxB,CAAC;AAED,SAAS,OAAO,CAAC,QAAkB,EAAE,QAAkB,EAAE,IAAY;IACnE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;QACrD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,OAAO,KAAK,CAAA;QACd,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACf,OAAO,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;IACzC,CAAC,EAAE,CAAC,CAAC,CAAA;IACL,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACzB,EAAE,MAAM,EAAE,YAAY,EAAE,EACxB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAC5C,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,EAAE,CAAC,CAAC,CAAA;IAC1C,OAAO,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,QAAQ,CAAA;AAC5C,CAAC;AAED,SAAS,UAAU,CAAC,YAAsB,EAAE,QAAgB;IAC1D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,CAAA;IACV,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC3D,OAAO,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAA;AAC9C,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;AAChG,CAAC"}
|
package/dist/files.d.ts
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
import type { Config, SkippedSourceFile, SkippedSourceReason, SourceFile, SourceInventory } from "./types.js";
|
|
2
2
|
export declare const OCR_IMAGE_EXTENSIONS: Set<string>;
|
|
3
3
|
export declare const DEFAULT_FAST_GLOB_IGNORES: string[];
|
|
4
|
+
export interface SourceInventoryOptions {
|
|
5
|
+
knownFiles?: ReadonlyMap<string, {
|
|
6
|
+
checksum: string;
|
|
7
|
+
bytes?: number;
|
|
8
|
+
mtimeMs?: number;
|
|
9
|
+
}>;
|
|
10
|
+
}
|
|
4
11
|
export declare const DEFAULT_SUPPORTED_EXTENSIONS: Set<string>;
|
|
5
12
|
export declare function listSourceFiles(config: Config): Promise<SourceFile[]>;
|
|
6
|
-
export declare function inventorySourceFiles(config: Config): Promise<SourceInventory>;
|
|
13
|
+
export declare function inventorySourceFiles(config: Config, options?: SourceInventoryOptions): Promise<SourceInventory>;
|
|
7
14
|
export declare function supportedExtensions(config: Config): Set<string>;
|
|
8
15
|
export declare function summarizeUnsupportedExtensions(skippedFiles: SkippedSourceFile[]): Array<{
|
|
9
16
|
extension: string;
|
package/dist/files.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,MAAM,EACN,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,eAAe,EAChB,MAAM,YAAY,CAAA;AAoCnB,eAAO,MAAM,oBAAoB,aAY/B,CAAA;AA4BF,eAAO,MAAM,yBAAyB,UAKrC,CAAA;AAcD,eAAO,MAAM,4BAA4B,aA8EvC,CAAA;AAEF,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAE3E;AAED,wBAAsB,oBAAoB,
|
|
1
|
+
{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/files.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,MAAM,EACN,iBAAiB,EACjB,mBAAmB,EACnB,UAAU,EACV,eAAe,EAChB,MAAM,YAAY,CAAA;AAoCnB,eAAO,MAAM,oBAAoB,aAY/B,CAAA;AA4BF,eAAO,MAAM,yBAAyB,UAKrC,CAAA;AAcD,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACzF;AAED,eAAO,MAAM,4BAA4B,aA8EvC,CAAA;AAEF,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAE3E;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CAiJ1B;AA4BD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAO/D;AASD,wBAAgB,8BAA8B,CAC5C,YAAY,EAAE,iBAAiB,EAAE,GAChC,KAAK,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAW7C;AA8CD,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAQjE;AAMD,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,mBAAmB,CAAA;CAAE,CAAC,EAC7C,MAAM,EAAE,mBAAmB,GAC1B,MAAM,CAER"}
|
package/dist/files.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { existsSync } from "node:fs";
|
|
2
|
+
import { createReadStream, existsSync } from "node:fs";
|
|
3
3
|
import { readFile, stat } from "node:fs/promises";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import fg from "fast-glob";
|
|
@@ -167,12 +167,25 @@ export const DEFAULT_SUPPORTED_EXTENSIONS = new Set([
|
|
|
167
167
|
export async function listSourceFiles(config) {
|
|
168
168
|
return (await inventorySourceFiles(config)).supportedFiles;
|
|
169
169
|
}
|
|
170
|
-
export async function inventorySourceFiles(config) {
|
|
170
|
+
export async function inventorySourceFiles(config, options = {}) {
|
|
171
171
|
const inputs = await sourceInputs(config);
|
|
172
|
+
const excludedPaths = new Set(inputs.ignorePatterns.length === 0
|
|
173
|
+
? []
|
|
174
|
+
: await fg(inputs.ignorePatterns, {
|
|
175
|
+
cwd: config.projectRoot,
|
|
176
|
+
absolute: true,
|
|
177
|
+
onlyFiles: true,
|
|
178
|
+
dot: true,
|
|
179
|
+
followSymbolicLinks: false,
|
|
180
|
+
unique: true,
|
|
181
|
+
}));
|
|
172
182
|
const files = new Map();
|
|
173
183
|
const skippedFiles = new Map();
|
|
174
184
|
let discoveredFiles = 0;
|
|
175
185
|
const recordSourceFile = async (absolutePath, info, source) => {
|
|
186
|
+
if (excludedPaths.has(absolutePath)) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
176
189
|
const relativePath = path.relative(config.projectRoot, absolutePath);
|
|
177
190
|
if (GENERATED_SOURCE_READMES.has(relativePath)) {
|
|
178
191
|
return;
|
|
@@ -208,7 +221,10 @@ export async function inventorySourceFiles(config) {
|
|
|
208
221
|
});
|
|
209
222
|
return;
|
|
210
223
|
}
|
|
211
|
-
const
|
|
224
|
+
const knownFile = options.knownFiles?.get(relativePath);
|
|
225
|
+
const checksum = knownFile?.bytes === info.size && knownFile.mtimeMs === info.mtimeMs
|
|
226
|
+
? knownFile.checksum
|
|
227
|
+
: await checksumFile(absolutePath);
|
|
212
228
|
files.set(absolutePath, {
|
|
213
229
|
absolutePath,
|
|
214
230
|
relativePath,
|
|
@@ -216,7 +232,7 @@ export async function inventorySourceFiles(config) {
|
|
|
216
232
|
extension,
|
|
217
233
|
bytes: info.size,
|
|
218
234
|
mtimeMs: info.mtimeMs,
|
|
219
|
-
checksum
|
|
235
|
+
checksum,
|
|
220
236
|
});
|
|
221
237
|
};
|
|
222
238
|
for (const root of inputs.roots) {
|
|
@@ -237,7 +253,7 @@ export async function inventorySourceFiles(config) {
|
|
|
237
253
|
unique: true,
|
|
238
254
|
}))
|
|
239
255
|
: [{ path: root, stats: { size: rootInfo.size, mtimeMs: rootInfo.mtimeMs } }];
|
|
240
|
-
|
|
256
|
+
await mapLimit(entries, config.ingestConcurrency, async (entry) => {
|
|
241
257
|
const absolutePath = path.isAbsolute(entry.path) ? entry.path : path.resolve(root, entry.path);
|
|
242
258
|
const info = entry.stats ?? (await stat(absolutePath));
|
|
243
259
|
const relativePath = path.relative(config.projectRoot, absolutePath);
|
|
@@ -245,7 +261,7 @@ export async function inventorySourceFiles(config) {
|
|
|
245
261
|
? path.relative(root, absolutePath) || path.basename(absolutePath)
|
|
246
262
|
: relativePath || path.basename(absolutePath);
|
|
247
263
|
await recordSourceFile(absolutePath, info, source);
|
|
248
|
-
}
|
|
264
|
+
});
|
|
249
265
|
}
|
|
250
266
|
if (inputs.patterns.length > 0) {
|
|
251
267
|
const entries = (await fg(inputs.patterns, {
|
|
@@ -259,14 +275,14 @@ export async function inventorySourceFiles(config) {
|
|
|
259
275
|
stats: true,
|
|
260
276
|
unique: true,
|
|
261
277
|
}));
|
|
262
|
-
|
|
278
|
+
await mapLimit(entries, config.ingestConcurrency, async (entry) => {
|
|
263
279
|
const absolutePath = path.isAbsolute(entry.path)
|
|
264
280
|
? entry.path
|
|
265
281
|
: path.resolve(config.projectRoot, entry.path);
|
|
266
282
|
const info = entry.stats ?? (await stat(absolutePath));
|
|
267
283
|
const relativePath = path.relative(config.projectRoot, absolutePath);
|
|
268
284
|
await recordSourceFile(absolutePath, info, relativePath || path.basename(absolutePath));
|
|
269
|
-
}
|
|
285
|
+
});
|
|
270
286
|
}
|
|
271
287
|
return {
|
|
272
288
|
discoveredFiles,
|
|
@@ -274,6 +290,26 @@ export async function inventorySourceFiles(config) {
|
|
|
274
290
|
skippedFiles: [...skippedFiles.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)),
|
|
275
291
|
};
|
|
276
292
|
}
|
|
293
|
+
async function checksumFile(filePath) {
|
|
294
|
+
const hash = createHash("sha256");
|
|
295
|
+
for await (const chunk of createReadStream(filePath)) {
|
|
296
|
+
hash.update(chunk);
|
|
297
|
+
}
|
|
298
|
+
return hash.digest("hex");
|
|
299
|
+
}
|
|
300
|
+
async function mapLimit(values, limit, worker) {
|
|
301
|
+
let index = 0;
|
|
302
|
+
const workers = Array.from({ length: Math.min(limit, values.length) }, async () => {
|
|
303
|
+
while (index < values.length) {
|
|
304
|
+
const current = values[index];
|
|
305
|
+
index += 1;
|
|
306
|
+
if (current !== undefined) {
|
|
307
|
+
await worker(current);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
});
|
|
311
|
+
await Promise.all(workers);
|
|
312
|
+
}
|
|
277
313
|
export function supportedExtensions(config) {
|
|
278
314
|
return new Set([
|
|
279
315
|
...DEFAULT_SUPPORTED_EXTENSIONS,
|