@jcode.labs/ragmir 2.9.3 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/access-log.d.ts +9 -1
- package/dist/access-log.d.ts.map +1 -1
- package/dist/access-log.js +82 -3
- package/dist/access-log.js.map +1 -1
- package/dist/chunk-stats.d.ts +8 -0
- package/dist/chunk-stats.d.ts.map +1 -0
- package/dist/chunk-stats.js +31 -0
- package/dist/chunk-stats.js.map +1 -0
- package/dist/chunking.d.ts +1 -0
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +58 -31
- package/dist/chunking.js.map +1 -1
- package/dist/cli.js +104 -5
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +16 -0
- package/dist/config.js.map +1 -1
- package/dist/context-resources.d.ts +4 -0
- package/dist/context-resources.d.ts.map +1 -0
- package/dist/context-resources.js +87 -0
- package/dist/context-resources.js.map +1 -0
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +1 -0
- package/dist/defaults.js.map +1 -1
- package/dist/document-structure.d.ts +13 -0
- package/dist/document-structure.d.ts.map +1 -0
- package/dist/document-structure.js +331 -0
- package/dist/document-structure.js.map +1 -0
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +8 -0
- package/dist/evaluate.js.map +1 -1
- package/dist/index-diagnostics.d.ts +1 -1
- package/dist/index-diagnostics.js +1 -1
- package/dist/index-policy.js +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +10 -3
- package/dist/ingest.js.map +1 -1
- package/dist/knowledge-bases.d.ts +4 -0
- package/dist/knowledge-bases.d.ts.map +1 -0
- package/dist/knowledge-bases.js +113 -0
- package/dist/knowledge-bases.js.map +1 -0
- package/dist/mcp-output.d.ts +60 -0
- package/dist/mcp-output.d.ts.map +1 -0
- package/dist/mcp-output.js +270 -0
- package/dist/mcp-output.js.map +1 -0
- package/dist/mcp.d.ts +3 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +136 -18
- package/dist/mcp.js.map +1 -1
- package/dist/preview.d.ts +3 -0
- package/dist/preview.d.ts.map +1 -0
- package/dist/preview.js +115 -0
- package/dist/preview.js.map +1 -0
- package/dist/query.d.ts +2 -1
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +201 -28
- package/dist/query.js.map +1 -1
- package/dist/research.d.ts.map +1 -1
- package/dist/research.js +5 -0
- package/dist/research.js.map +1 -1
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +1 -0
- package/dist/security.js.map +1 -1
- package/dist/setup.js +1 -1
- package/dist/skill.d.ts.map +1 -1
- package/dist/skill.js +46 -4
- package/dist/skill.js.map +1 -1
- package/dist/store.js +2 -2
- package/dist/store.js.map +1 -1
- package/dist/types.d.ts +171 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +3 -3
- package/skills/ragmir/SKILL.md +34 -0
package/README.md
CHANGED
|
@@ -55,7 +55,10 @@ npx rgr audit --unsupported
|
|
|
55
55
|
npx rgr security-audit
|
|
56
56
|
|
|
57
57
|
# Retrieval
|
|
58
|
+
npx rgr preview --path docs --max-chunks 3
|
|
58
59
|
npx rgr search "deployment decision"
|
|
60
|
+
npx rgr search "deployment decision" --explain
|
|
61
|
+
npx rgr search "deployment decision" --context-path "Operations > Deployment"
|
|
59
62
|
npx rgr ask "What evidence supports the deployment decision?"
|
|
60
63
|
npx rgr research "deployment obligations" --compact
|
|
61
64
|
|
|
@@ -88,8 +91,11 @@ Frequently used exports:
|
|
|
88
91
|
| Export | Purpose |
|
|
89
92
|
| --- | --- |
|
|
90
93
|
| `setupProject`, `addSourceEntries` | Initialize project state and select files |
|
|
94
|
+
| `discoverKnowledgeBases`, `knowledgeBaseIdentity` | Route root and nested monorepo bases |
|
|
95
|
+
| `getKnowledgeBaseContext`, `getKnowledgeBaseSourceCatalog` | Give agents bounded readiness and source context |
|
|
91
96
|
| `ingest`, `audit` | Build the index and compare it with files on disk |
|
|
92
|
-
| `
|
|
97
|
+
| `previewChunks` | Inspect redacted chunks and distributions without writing storage |
|
|
98
|
+
| `search`, `ask`, `research`, `expandCitation` | Retrieve or expand cited passages |
|
|
93
99
|
| `doctor`, `securityAudit` | Inspect readiness and local privacy posture |
|
|
94
100
|
| `serveMcp` | Start the read-focused local MCP server |
|
|
95
101
|
| `configurePdfOcr`, `inspectPdfOcr` | Configure and inspect local PDF OCR |
|
|
@@ -105,8 +111,13 @@ npx rgr doctor
|
|
|
105
111
|
```
|
|
106
112
|
|
|
107
113
|
Ragmir writes helper files for the selected clients and points them at the current project. MCP
|
|
108
|
-
exposes status, search, ask, research, audit, evaluation, usage, and
|
|
109
|
-
expose
|
|
114
|
+
exposes status, search, ask, research, exact citation expansion, audit, evaluation, usage, and
|
|
115
|
+
security tools. Retrieval responses have a global byte ceiling and expose metadata-only output
|
|
116
|
+
metrics. The server does not expose index deletion.
|
|
117
|
+
|
|
118
|
+
For a monorepo with root and app-specific bases, run `rgr bases --json` before retrieval when scope
|
|
119
|
+
is unclear. The nearest configured base is active, nested MCP helpers get unique names, and every
|
|
120
|
+
helper pins `RAGMIR_PROJECT_ROOT` so clients cannot drift to a sibling index.
|
|
110
121
|
|
|
111
122
|
## Retrieval modes
|
|
112
123
|
|
|
@@ -147,7 +158,8 @@ local executable, never a shell string or cloud OCR service.
|
|
|
147
158
|
- Generated indexes, models, reports, audio, and access logs belong under ignored `.ragmir/` state.
|
|
148
159
|
- Redaction runs before indexing when configured, but it is not a compliance certification.
|
|
149
160
|
- External extractors and model downloads are opt-in system boundaries.
|
|
150
|
-
- MCP retrieval is bounded
|
|
161
|
+
- MCP retrieval is bounded by `mcpMaxOutputBytes`; access logs contain byte metrics, not query text
|
|
162
|
+
or retrieved passages.
|
|
151
163
|
|
|
152
164
|
Read [configuration and privacy](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md)
|
|
153
165
|
and [security hardening](https://github.com/jcode-works/jcode-ragmir/blob/main/SECURITY-HARDENING.md)
|
package/dist/access-log.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AccessLogAction, AccessLogUsageOptions, AccessLogUsageReport, Config } from "./types.js";
|
|
1
|
+
import type { AccessLogAction, AccessLogUsageOptions, AccessLogUsageReport, Config, McpOutputTool } from "./types.js";
|
|
2
2
|
export interface AccessLogEvent {
|
|
3
3
|
action: AccessLogAction;
|
|
4
4
|
query?: string;
|
|
@@ -6,6 +6,14 @@ export interface AccessLogEvent {
|
|
|
6
6
|
resultCount?: number;
|
|
7
7
|
redactions?: number;
|
|
8
8
|
}
|
|
9
|
+
export interface McpOutputLogEvent {
|
|
10
|
+
tool: McpOutputTool;
|
|
11
|
+
retrievedBytes: number;
|
|
12
|
+
returnedBytes: number;
|
|
13
|
+
compacted: boolean;
|
|
14
|
+
truncated: boolean;
|
|
15
|
+
}
|
|
9
16
|
export declare function recordAccess(config: Config, event: AccessLogEvent): Promise<void>;
|
|
17
|
+
export declare function recordMcpOutput(config: Config, event: McpOutputLogEvent): Promise<void>;
|
|
10
18
|
export declare function accessLogUsageReport(options?: AccessLogUsageOptions): Promise<AccessLogUsageReport>;
|
|
11
19
|
//# sourceMappingURL=access-log.d.ts.map
|
package/dist/access-log.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,
|
|
1
|
+
{"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,EACN,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AAsDD,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBvF;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAyB7F;AA2BD,wBAAsB,oBAAoB,CACxC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAoF/B"}
|
package/dist/access-log.js
CHANGED
|
@@ -13,6 +13,13 @@ const ACCESS_LOG_ACTIONS = [
|
|
|
13
13
|
"destroy-index",
|
|
14
14
|
];
|
|
15
15
|
const ACCESS_LOG_ACTION_SET = new Set(ACCESS_LOG_ACTIONS);
|
|
16
|
+
const MCP_OUTPUT_TOOLS = [
|
|
17
|
+
"ragmir_search",
|
|
18
|
+
"ragmir_ask",
|
|
19
|
+
"ragmir_research",
|
|
20
|
+
"ragmir_expand",
|
|
21
|
+
];
|
|
22
|
+
const MCP_OUTPUT_TOOL_SET = new Set(MCP_OUTPUT_TOOLS);
|
|
16
23
|
const DEFAULT_USAGE_REPORT_DAYS = 7;
|
|
17
24
|
const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
18
25
|
/**
|
|
@@ -41,6 +48,32 @@ export async function recordAccess(config, event) {
|
|
|
41
48
|
// Access logging is best-effort so read-only workspaces do not block local use.
|
|
42
49
|
}
|
|
43
50
|
}
|
|
51
|
+
export async function recordMcpOutput(config, event) {
|
|
52
|
+
if (!config.accessLog) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
await ensurePrivateDirectory(path.dirname(config.accessLogPath));
|
|
57
|
+
await trimAccessLogIfNeeded(config.accessLogPath);
|
|
58
|
+
const line = {
|
|
59
|
+
timestamp: new Date().toISOString(),
|
|
60
|
+
kind: "mcp-output",
|
|
61
|
+
tool: event.tool,
|
|
62
|
+
retrievedBytes: event.retrievedBytes,
|
|
63
|
+
returnedBytes: event.returnedBytes,
|
|
64
|
+
compacted: event.compacted,
|
|
65
|
+
truncated: event.truncated,
|
|
66
|
+
};
|
|
67
|
+
await appendFile(config.accessLogPath, `${JSON.stringify(line)}\n`, {
|
|
68
|
+
encoding: "utf8",
|
|
69
|
+
mode: 0o600,
|
|
70
|
+
});
|
|
71
|
+
await hardenPrivateFile(config.accessLogPath);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
// Output metrics are best-effort and must never block local retrieval.
|
|
75
|
+
}
|
|
76
|
+
}
|
|
44
77
|
async function trimAccessLogIfNeeded(accessLogPath) {
|
|
45
78
|
let size = 0;
|
|
46
79
|
try {
|
|
@@ -75,6 +108,11 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
75
108
|
let invalidLines = 0;
|
|
76
109
|
let resultCountTotal = 0;
|
|
77
110
|
let resultCountEvents = 0;
|
|
111
|
+
let mcpOutputResponses = 0;
|
|
112
|
+
let mcpRetrievedBytes = 0;
|
|
113
|
+
let mcpReturnedBytes = 0;
|
|
114
|
+
let compactedResponses = 0;
|
|
115
|
+
let truncatedResponses = 0;
|
|
78
116
|
let lastEventAt = null;
|
|
79
117
|
if (existsSync(config.accessLogPath)) {
|
|
80
118
|
const lines = (await readFile(config.accessLogPath, "utf8")).split(/\r?\n/u).filter(Boolean);
|
|
@@ -91,6 +129,17 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
91
129
|
continue;
|
|
92
130
|
}
|
|
93
131
|
totalEvents += 1;
|
|
132
|
+
if (isMcpOutputLogLine(event)) {
|
|
133
|
+
mcpOutputResponses += 1;
|
|
134
|
+
mcpRetrievedBytes += event.retrievedBytes;
|
|
135
|
+
mcpReturnedBytes += event.returnedBytes;
|
|
136
|
+
compactedResponses += event.compacted ? 1 : 0;
|
|
137
|
+
truncatedResponses += event.truncated ? 1 : 0;
|
|
138
|
+
if (lastEventAt === null || event.timestamp > lastEventAt) {
|
|
139
|
+
lastEventAt = event.timestamp;
|
|
140
|
+
}
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
94
143
|
eventsByAction[event.action] += 1;
|
|
95
144
|
if (event.queryHash) {
|
|
96
145
|
queryHashes.add(event.queryHash);
|
|
@@ -116,6 +165,15 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
116
165
|
uniqueQueryHashes: queryHashes.size,
|
|
117
166
|
averageResultCount: resultCountEvents === 0 ? null : resultCountTotal / resultCountEvents,
|
|
118
167
|
averageResultCountByAction: averageResultCounts(resultCountsByAction),
|
|
168
|
+
mcpOutput: {
|
|
169
|
+
responses: mcpOutputResponses,
|
|
170
|
+
retrievedBytes: mcpRetrievedBytes,
|
|
171
|
+
returnedBytes: mcpReturnedBytes,
|
|
172
|
+
savedBytes: mcpRetrievedBytes - mcpReturnedBytes,
|
|
173
|
+
reductionRatio: mcpRetrievedBytes === 0 ? null : 1 - mcpReturnedBytes / mcpRetrievedBytes,
|
|
174
|
+
compactedResponses,
|
|
175
|
+
truncatedResponses,
|
|
176
|
+
},
|
|
119
177
|
lastEventAt,
|
|
120
178
|
};
|
|
121
179
|
}
|
|
@@ -201,15 +259,29 @@ function normalizeUsageReportDays(days) {
|
|
|
201
259
|
function parseAccessLogLine(line) {
|
|
202
260
|
try {
|
|
203
261
|
const parsed = JSON.parse(line);
|
|
204
|
-
if (
|
|
205
|
-
return
|
|
262
|
+
if (isAccessLogLine(parsed) || isMcpOutputLogLine(parsed)) {
|
|
263
|
+
return parsed;
|
|
206
264
|
}
|
|
207
|
-
return
|
|
265
|
+
return null;
|
|
208
266
|
}
|
|
209
267
|
catch {
|
|
210
268
|
return null;
|
|
211
269
|
}
|
|
212
270
|
}
|
|
271
|
+
function isMcpOutputLogLine(value) {
|
|
272
|
+
return (typeof value === "object" &&
|
|
273
|
+
value !== null &&
|
|
274
|
+
hasTimestamp(value) &&
|
|
275
|
+
"kind" in value &&
|
|
276
|
+
value.kind === "mcp-output" &&
|
|
277
|
+
"tool" in value &&
|
|
278
|
+
typeof value.tool === "string" &&
|
|
279
|
+
MCP_OUTPUT_TOOL_SET.has(value.tool) &&
|
|
280
|
+
hasNonNegativeNumber(value, "retrievedBytes") &&
|
|
281
|
+
hasNonNegativeNumber(value, "returnedBytes") &&
|
|
282
|
+
hasBoolean(value, "compacted") &&
|
|
283
|
+
hasBoolean(value, "truncated"));
|
|
284
|
+
}
|
|
213
285
|
function isAccessLogLine(value) {
|
|
214
286
|
return (typeof value === "object" &&
|
|
215
287
|
value !== null &&
|
|
@@ -230,4 +302,11 @@ function hasOptionalQueryHash(value) {
|
|
|
230
302
|
function hasOptionalResultCount(value) {
|
|
231
303
|
return !("resultCount" in value) || typeof value.resultCount === "number";
|
|
232
304
|
}
|
|
305
|
+
function hasNonNegativeNumber(value, key) {
|
|
306
|
+
const field = Reflect.get(value, key);
|
|
307
|
+
return typeof field === "number" && Number.isFinite(field) && field >= 0;
|
|
308
|
+
}
|
|
309
|
+
function hasBoolean(value, key) {
|
|
310
|
+
return typeof Reflect.get(value, key) === "boolean";
|
|
311
|
+
}
|
|
233
312
|
//# sourceMappingURL=access-log.js.map
|
package/dist/access-log.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AA4B5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AAEtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;QAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;YAC5F,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAA;QACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC"}
|
|
1
|
+
{"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAiD5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,gBAAgB,GAAoB;IACxC,eAAe;IACf,YAAY;IACZ,iBAAiB;IACjB,eAAe;CAChB,CAAA;AACD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AAC7D,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AAEtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;QAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;YAC5F,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAA;QACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc,EAAE,KAAwB;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;QAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,IAAI,GAAqB;YAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAA;QACD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;YAClE,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAA;QACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;IACzE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,CAAC,CAAA;gBACvB,iBAAiB,IAAI,KAAK,CAAC,cAAc,CAAA;gBACzC,gBAAgB,IAAI,KAAK,CAAC,aAAa,CAAA;gBACvC,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;oBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC/B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,gBAAgB;YAC/B,UAAU,EAAE,iBAAiB,GAAG,gBAAgB;YAChD,cAAc,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,iBAAiB;YACzF,kBAAkB;YAClB,kBAAkB;SACnB;QACD,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,MAAM,IAAI,KAAK;QACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAC9B,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnC,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC;QAC7C,oBAAoB,CAAC,KAAK,EAAE,eAAe,CAAC;QAC5C,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC;QAC9B,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAC/B,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,GAAW;IAC5C,OAAO,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,SAAS,CAAA;AACrD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-stats.d.ts","sourceRoot":"","sources":["../src/chunk-stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5C,UAAU,UAAU;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,GAAG,UAAU,CA2B7E"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export function summarizeChunkStats(chunks) {
|
|
2
|
+
if (chunks.length === 0) {
|
|
3
|
+
return {
|
|
4
|
+
count: 0,
|
|
5
|
+
minChars: 0,
|
|
6
|
+
averageChars: 0,
|
|
7
|
+
p50Chars: 0,
|
|
8
|
+
p95Chars: 0,
|
|
9
|
+
maxChars: 0,
|
|
10
|
+
contextualChunks: 0,
|
|
11
|
+
contextualRatio: 0,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
const lengths = chunks.map((chunk) => chunk.text.length).sort((left, right) => left - right);
|
|
15
|
+
const contextualChunks = chunks.filter((chunk) => chunk.contextPath.trim().length > 0).length;
|
|
16
|
+
return {
|
|
17
|
+
count: chunks.length,
|
|
18
|
+
minChars: lengths[0] ?? 0,
|
|
19
|
+
averageChars: lengths.reduce((sum, length) => sum + length, 0) / lengths.length,
|
|
20
|
+
p50Chars: percentile(lengths, 0.5),
|
|
21
|
+
p95Chars: percentile(lengths, 0.95),
|
|
22
|
+
maxChars: lengths.at(-1) ?? 0,
|
|
23
|
+
contextualChunks,
|
|
24
|
+
contextualRatio: contextualChunks / chunks.length,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function percentile(sortedValues, quantile) {
|
|
28
|
+
const index = Math.ceil(quantile * sortedValues.length) - 1;
|
|
29
|
+
return sortedValues[Math.max(0, index)] ?? 0;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=chunk-stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-stats.js","sourceRoot":"","sources":["../src/chunk-stats.ts"],"names":[],"mappings":"AAOA,MAAM,UAAU,mBAAmB,CAAC,MAA6B;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,CAAC;YACX,YAAY,EAAE,CAAC;YACf,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,CAAC;YACX,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,CAAC;SACnB,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAA;IAC5F,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;IAE7F,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACzB,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/E,QAAQ,EAAE,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;QAClC,QAAQ,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC;QACnC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7B,gBAAgB;QAChB,eAAe,EAAE,gBAAgB,GAAG,MAAM,CAAC,MAAM;KAClD,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,YAAsB,EAAE,QAAgB;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC3D,OAAO,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAA;AAC9C,CAAC"}
|
package/dist/chunking.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
import type { ParsedDocument, TextChunk } from "./types.js";
|
|
2
2
|
export declare function chunkDocument(document: ParsedDocument, chunkSize: number, chunkOverlap: number): TextChunk[];
|
|
3
|
+
export declare function chunkSearchText(chunk: Pick<TextChunk, "contextPath" | "text">): string;
|
|
3
4
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAQ3D,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CA+Db;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CAAC,GAAG,MAAM,CAEtF"}
|
package/dist/chunking.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { markdownFenceSpans, structuralSpans } from "./document-structure.js";
|
|
2
3
|
const PARAGRAPH_BREAK_MIN_RATIO = 0.45;
|
|
3
4
|
const SENTENCE_BREAK_MIN_RATIO = 0.55;
|
|
4
5
|
const LINE_BREAK_MIN_RATIO = 0.65;
|
|
@@ -10,39 +11,61 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
|
10
11
|
}
|
|
11
12
|
const chunks = [];
|
|
12
13
|
const lineStarts = lineStartOffsets(document.text);
|
|
13
|
-
|
|
14
|
+
const structured = structuralSpans(document.text, document.file.extension, chunkSize);
|
|
15
|
+
const regions = structured.length > 0
|
|
16
|
+
? structured
|
|
17
|
+
: [
|
|
18
|
+
{
|
|
19
|
+
charStart: 0,
|
|
20
|
+
charEnd: document.text.length,
|
|
21
|
+
contextPath: "",
|
|
22
|
+
kind: "markdown-section",
|
|
23
|
+
},
|
|
24
|
+
];
|
|
25
|
+
const fences = [".md", ".mdx", ".markdown"].includes(document.file.extension)
|
|
26
|
+
? markdownFenceSpans(document.text)
|
|
27
|
+
: [];
|
|
14
28
|
let chunkIndex = 0;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
29
|
+
for (const region of regions) {
|
|
30
|
+
let cursor = region.charStart;
|
|
31
|
+
while (cursor < region.charEnd) {
|
|
32
|
+
const end = chooseChunkEnd(document.text, cursor, chunkSize, region.charEnd, fences);
|
|
33
|
+
const span = trimmedSpan(document.text, cursor, end);
|
|
34
|
+
if (span.text) {
|
|
35
|
+
const id = createHash("sha256")
|
|
36
|
+
.update(`${document.file.relativePath}:${chunkIndex}:${region.contextPath}:${span.text}`)
|
|
37
|
+
.digest("hex");
|
|
38
|
+
chunks.push({
|
|
39
|
+
id,
|
|
40
|
+
source: document.file.source,
|
|
41
|
+
relativePath: document.file.relativePath,
|
|
42
|
+
chunkIndex,
|
|
43
|
+
contextPath: region.contextPath,
|
|
44
|
+
text: span.text,
|
|
45
|
+
charStart: span.start,
|
|
46
|
+
charEnd: span.end,
|
|
47
|
+
lineStart: lineNumberForOffset(lineStarts, span.start),
|
|
48
|
+
lineEnd: lineNumberForOffset(lineStarts, Math.max(span.start, span.end - 1)),
|
|
49
|
+
...pageRangeForSpan(document, span.start, span.end),
|
|
50
|
+
checksum: document.file.checksum,
|
|
51
|
+
bytes: document.file.bytes,
|
|
52
|
+
mtimeMs: document.file.mtimeMs,
|
|
53
|
+
});
|
|
54
|
+
chunkIndex += 1;
|
|
55
|
+
}
|
|
56
|
+
if (end >= region.charEnd) {
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
cursor = fences.some((fence) => fence.start === end)
|
|
60
|
+
? end
|
|
61
|
+
: Math.max(end - chunkOverlap, cursor + 1, region.charStart);
|
|
38
62
|
}
|
|
39
|
-
if (end >= document.text.length) {
|
|
40
|
-
break;
|
|
41
|
-
}
|
|
42
|
-
cursor = Math.max(end - chunkOverlap, cursor + 1);
|
|
43
63
|
}
|
|
44
64
|
return chunks;
|
|
45
65
|
}
|
|
66
|
+
export function chunkSearchText(chunk) {
|
|
67
|
+
return chunk.contextPath ? `${chunk.contextPath}\n${chunk.text}` : chunk.text;
|
|
68
|
+
}
|
|
46
69
|
function pageRangeForSpan(document, start, end) {
|
|
47
70
|
if (!document.pages || document.pages.length === 0) {
|
|
48
71
|
return {};
|
|
@@ -52,11 +75,15 @@ function pageRangeForSpan(document, start, end) {
|
|
|
52
75
|
const last = overlapping.at(-1);
|
|
53
76
|
return first && last ? { pageStart: first.pageNumber, pageEnd: last.pageNumber } : {};
|
|
54
77
|
}
|
|
55
|
-
function chooseChunkEnd(text, cursor, chunkSize) {
|
|
56
|
-
const hardEnd = Math.min(cursor + chunkSize,
|
|
57
|
-
if (hardEnd ===
|
|
78
|
+
function chooseChunkEnd(text, cursor, chunkSize, regionEnd, fences) {
|
|
79
|
+
const hardEnd = Math.min(cursor + chunkSize, regionEnd);
|
|
80
|
+
if (hardEnd === regionEnd) {
|
|
58
81
|
return hardEnd;
|
|
59
82
|
}
|
|
83
|
+
const intersectedFence = fences.find((fence) => fence.start < hardEnd && fence.end > hardEnd && fence.end <= regionEnd);
|
|
84
|
+
if (intersectedFence && intersectedFence.end - intersectedFence.start <= chunkSize) {
|
|
85
|
+
return intersectedFence.start > cursor ? intersectedFence.start : intersectedFence.end;
|
|
86
|
+
}
|
|
60
87
|
const window = text.slice(cursor, hardEnd);
|
|
61
88
|
const paragraphBreak = window.lastIndexOf("\n\n");
|
|
62
89
|
if (paragraphBreak > chunkSize * PARAGRAPH_BREAK_MIN_RATIO) {
|
package/dist/chunking.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AAG7E,MAAM,yBAAyB,GAAG,IAAI,CAAA;AACtC,MAAM,wBAAwB,GAAG,IAAI,CAAA;AACrC,MAAM,oBAAoB,GAAG,IAAI,CAAA;AACjC,MAAM,0BAA0B,GAAG,IAAI,CAAA;AACvC,MAAM,mBAAmB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE7D,MAAM,UAAU,aAAa,CAC3B,QAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IAClD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IACrF,MAAM,OAAO,GACX,UAAU,CAAC,MAAM,GAAG,CAAC;QACnB,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC;YACE;gBACE,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAC7B,WAAW,EAAE,EAAE;gBACf,IAAI,EAAE,kBAA2B;aAClC;SACF,CAAA;IACP,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3E,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC;QACnC,CAAC,CAAC,EAAE,CAAA;IACN,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,GAAG,MAAM,CAAC,SAAS,CAAA;QAC7B,OAAO,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YACpF,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;YAEpD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC;qBAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;qBACxF,MAAM,CAAC,KAAK,CAAC,CAAA;gBAChB,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE;oBACF,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;oBAC5B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY;oBACxC,UAAU;oBACV,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,KAAK;oBACrB,OAAO,EAAE,IAAI,CAAC,GAAG;oBACjB,SAAS,EAAE,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC;oBACtD,OAAO,EAAE,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;oBAC5E,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;oBACnD,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ;oBAChC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;oBAC1B,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,UAAU,IAAI,CAAC,CAAA;YACjB,CAAC;YAED,IAAI,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC1B,MAAK;YACP,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,GAAG,CAAC;gBAClD,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,EAAE,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;QAChE,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAA8C;IAC5E,OAAO,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;AAC/E,CAAC;AAED,SAAS,gBAAgB,CACvB,QAAwB,EACxB,KAAa,EACb,GAAW;IAEX,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,CAAA;IACX,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,CAAA;IACjG,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAC5B,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/B,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;AACvF,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,MAAc,EACd,SAAiB,EACjB,SAAiB,EACjB,MAA6C;IAE7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,SAAS,CAAC,CAAA;IACvD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,IAAI,KAAK,CAAC,GAAG,GAAG,OAAO,IAAI,KAAK,CAAC,GAAG,IAAI,SAAS,CAClF,CAAA;IACD,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,GAAG,gBAAgB,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;QACnF,OAAO,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAA;IACxF,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC1C,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;IACjD,IAAI,cAAc,GAAG,SAAS,GAAG,yBAAyB,EAAE,CAAC;QAC3D,OAAO,MAAM,GAAG,cAAc,CAAA;IAChC,CAAC;IAED,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IACrD,IAAI,gBAAgB,GAAG,SAAS,GAAG,wBAAwB,EAAE,CAAC;QAC5D,OAAO,MAAM,GAAG,gBAAgB,CAAA;IAClC,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;IAC1C,IAAI,SAAS,GAAG,SAAS,GAAG,oBAAoB,EAAE,CAAC;QACjD,OAAO,MAAM,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAC1C,IAAI,UAAU,GAAG,SAAS,GAAG,0BAA0B,EAAE,CAAC;QACxD,OAAO,MAAM,GAAG,UAAU,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAA;IACZ,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACxC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/F,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAQD,SAAS,WAAW,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW;IAC3D,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,IAAI,UAAU,GAAG,GAAG,CAAA;IACpB,OAAO,YAAY,GAAG,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACzE,YAAY,IAAI,CAAC,CAAA;IACnB,CAAC;IACD,OAAO,UAAU,GAAG,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC3E,UAAU,IAAI,CAAC,CAAA;IACjB,CAAC;IACD,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,GAAG,EAAE,UAAU;QACf,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC;KAC3C,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;IAClB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAoB,EAAE,MAAc;IAC/D,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,IAAI,IAAI,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA;IAChC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;QACf,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;QAChB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAA;AAC9B,CAAC"}
|