@jcode.labs/ragmir 2.17.2 → 2.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -43
- package/dist/access-log.d.ts +10 -0
- package/dist/access-log.d.ts.map +1 -1
- package/dist/access-log.js +119 -49
- package/dist/access-log.js.map +1 -1
- package/dist/chunking.d.ts +5 -1
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +102 -16
- package/dist/chunking.js.map +1 -1
- package/dist/citation.d.ts +18 -0
- package/dist/citation.d.ts.map +1 -0
- package/dist/citation.js +54 -0
- package/dist/citation.js.map +1 -0
- package/dist/cli-entry.d.ts +3 -0
- package/dist/cli-entry.d.ts.map +1 -0
- package/dist/cli-entry.js +16 -0
- package/dist/cli-entry.js.map +1 -0
- package/dist/cli-fast.d.ts +13 -0
- package/dist/cli-fast.d.ts.map +1 -0
- package/dist/cli-fast.js +145 -0
- package/dist/cli-fast.js.map +1 -0
- package/dist/cli-options.d.ts +3 -0
- package/dist/cli-options.d.ts.map +1 -1
- package/dist/cli-options.js +7 -0
- package/dist/cli-options.js.map +1 -1
- package/dist/cli-stdin.d.ts +3 -0
- package/dist/cli-stdin.d.ts.map +1 -0
- package/dist/cli-stdin.js +15 -0
- package/dist/cli-stdin.js.map +1 -0
- package/dist/cli.js +251 -44
- package/dist/cli.js.map +1 -1
- package/dist/client.d.ts +13 -2
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +155 -10
- package/dist/client.js.map +1 -1
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +227 -49
- package/dist/config.js.map +1 -1
- package/dist/context-resources.d.ts +4 -2
- package/dist/context-resources.d.ts.map +1 -1
- package/dist/context-resources.js +55 -24
- package/dist/context-resources.js.map +1 -1
- package/dist/defaults.d.ts +27 -0
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +42 -2
- package/dist/defaults.js.map +1 -1
- package/dist/destroy.d.ts +2 -2
- package/dist/destroy.d.ts.map +1 -1
- package/dist/destroy.js +34 -20
- package/dist/destroy.js.map +1 -1
- package/dist/doctor.d.ts +3 -2
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +86 -41
- package/dist/doctor.js.map +1 -1
- package/dist/durable-file.d.ts +13 -0
- package/dist/durable-file.d.ts.map +1 -0
- package/dist/durable-file.js +56 -0
- package/dist/durable-file.js.map +1 -0
- package/dist/embeddings.d.ts +14 -3
- package/dist/embeddings.d.ts.map +1 -1
- package/dist/embeddings.js +233 -95
- package/dist/embeddings.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/evaluate.d.ts +4 -3
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +611 -110
- package/dist/evaluate.js.map +1 -1
- package/dist/files.d.ts +4 -1
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +119 -33
- package/dist/files.js.map +1 -1
- package/dist/generation-retention.d.ts +52 -0
- package/dist/generation-retention.d.ts.map +1 -0
- package/dist/generation-retention.js +318 -0
- package/dist/generation-retention.js.map +1 -0
- package/dist/index-diagnostics.d.ts +3 -2
- package/dist/index-diagnostics.d.ts.map +1 -1
- package/dist/index-diagnostics.js +32 -3
- package/dist/index-diagnostics.js.map +1 -1
- package/dist/index-policy.d.ts.map +1 -1
- package/dist/index-policy.js +3 -0
- package/dist/index-policy.js.map +1 -1
- package/dist/index-write-lock.d.ts +16 -1
- package/dist/index-write-lock.d.ts.map +1 -1
- package/dist/index-write-lock.js +392 -13
- package/dist/index-write-lock.js.map +1 -1
- package/dist/index.d.ts +12 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/ingest.d.ts +11 -2
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +680 -165
- package/dist/ingest.js.map +1 -1
- package/dist/ingestion-metrics.d.ts +58 -0
- package/dist/ingestion-metrics.d.ts.map +1 -0
- package/dist/ingestion-metrics.js +216 -0
- package/dist/ingestion-metrics.js.map +1 -0
- package/dist/ingestion-state.d.ts +11 -2
- package/dist/ingestion-state.d.ts.map +1 -1
- package/dist/ingestion-state.js +480 -49
- package/dist/ingestion-state.js.map +1 -1
- package/dist/limits.d.ts.map +1 -1
- package/dist/limits.js +11 -1
- package/dist/limits.js.map +1 -1
- package/dist/mcp-output.d.ts +44 -4
- package/dist/mcp-output.d.ts.map +1 -1
- package/dist/mcp-output.js +132 -159
- package/dist/mcp-output.js.map +1 -1
- package/dist/mcp-summaries.d.ts +32 -0
- package/dist/mcp-summaries.d.ts.map +1 -0
- package/dist/mcp-summaries.js +227 -0
- package/dist/mcp-summaries.js.map +1 -0
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +85 -46
- package/dist/mcp.js.map +1 -1
- package/dist/ocr-cache.d.ts +15 -0
- package/dist/ocr-cache.d.ts.map +1 -0
- package/dist/ocr-cache.js +90 -0
- package/dist/ocr-cache.js.map +1 -0
- package/dist/ocr.d.ts +31 -0
- package/dist/ocr.d.ts.map +1 -1
- package/dist/ocr.js +193 -17
- package/dist/ocr.js.map +1 -1
- package/dist/parsing.d.ts.map +1 -1
- package/dist/parsing.js +482 -85
- package/dist/parsing.js.map +1 -1
- package/dist/permissions.js +2 -2
- package/dist/permissions.js.map +1 -1
- package/dist/preview.d.ts.map +1 -1
- package/dist/preview.js +10 -11
- package/dist/preview.js.map +1 -1
- package/dist/quality-report.d.ts +12 -0
- package/dist/quality-report.d.ts.map +1 -0
- package/dist/quality-report.js +146 -0
- package/dist/quality-report.js.map +1 -0
- package/dist/query.d.ts +7 -4
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +371 -284
- package/dist/query.js.map +1 -1
- package/dist/ranking.d.ts +38 -0
- package/dist/ranking.d.ts.map +1 -0
- package/dist/ranking.js +278 -0
- package/dist/ranking.js.map +1 -0
- package/dist/redaction.d.ts +6 -1
- package/dist/redaction.d.ts.map +1 -1
- package/dist/redaction.js +118 -4
- package/dist/redaction.js.map +1 -1
- package/dist/research.d.ts +6 -1
- package/dist/research.d.ts.map +1 -1
- package/dist/research.js +330 -130
- package/dist/research.js.map +1 -1
- package/dist/runtime-info.d.ts +3 -0
- package/dist/runtime-info.d.ts.map +1 -0
- package/dist/runtime-info.js +81 -0
- package/dist/runtime-info.js.map +1 -0
- package/dist/security.d.ts +3 -2
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +242 -45
- package/dist/security.js.map +1 -1
- package/dist/semantic-config.d.ts +7 -1
- package/dist/semantic-config.d.ts.map +1 -1
- package/dist/semantic-config.js +30 -4
- package/dist/semantic-config.js.map +1 -1
- package/dist/setup.js +6 -3
- package/dist/setup.js.map +1 -1
- package/dist/skill.js +34 -12
- package/dist/skill.js.map +1 -1
- package/dist/source-fingerprint-cache.d.ts +20 -0
- package/dist/source-fingerprint-cache.d.ts.map +1 -0
- package/dist/source-fingerprint-cache.js +133 -0
- package/dist/source-fingerprint-cache.js.map +1 -0
- package/dist/storage-maintenance.d.ts +46 -0
- package/dist/storage-maintenance.d.ts.map +1 -0
- package/dist/storage-maintenance.js +329 -0
- package/dist/storage-maintenance.js.map +1 -0
- package/dist/store.d.ts +39 -2
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +559 -24
- package/dist/store.js.map +1 -1
- package/dist/text.d.ts +1 -1
- package/dist/text.d.ts.map +1 -1
- package/dist/text.js +12 -2
- package/dist/text.js.map +1 -1
- package/dist/types.d.ts +414 -5
- package/dist/types.d.ts.map +1 -1
- package/dist/vector-index.d.ts +42 -0
- package/dist/vector-index.d.ts.map +1 -0
- package/dist/vector-index.js +249 -0
- package/dist/vector-index.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/workload.d.ts +22 -0
- package/dist/workload.d.ts.map +1 -0
- package/dist/workload.js +181 -0
- package/dist/workload.js.map +1 -0
- package/examples/document-evidence-benchmark/README.md +25 -63
- package/examples/document-evidence-benchmark/golden-queries.json +1 -1
- package/examples/library-api-demo/README.md +17 -57
- package/examples/sovereign-rag-demo/README.md +37 -97
- package/package.json +36 -11
package/README.md
CHANGED
|
@@ -5,21 +5,18 @@
|
|
|
5
5
|
[](https://www.npmjs.com/package/@jcode.labs/ragmir)
|
|
6
6
|
[](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE)
|
|
7
7
|
|
|
8
|
-
Confidential local RAG for coding agents and Node.js applications.
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
Core indexes the project files you select and retrieves bounded, cited evidence offline by default.
|
|
13
|
-
It does not upload the corpus, call an LLM, or open an HTTP port.
|
|
8
|
+
Confidential local RAG for coding agents and Node.js applications. Core indexes the project files
|
|
9
|
+
you choose and retrieves bounded, cited evidence offline by default. It uploads no corpus, calls no
|
|
10
|
+
LLM, and opens no HTTP port.
|
|
14
11
|
|
|
15
12
|
[Project overview](https://github.com/jcode-works/jcode-ragmir#readme) ·
|
|
16
|
-
[CLI
|
|
17
|
-
[API
|
|
13
|
+
[CLI](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md) ·
|
|
14
|
+
[API](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md) ·
|
|
18
15
|
[Agent integration](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/agent-integration.md)
|
|
19
16
|
|
|
20
17
|
## Install and retrieve
|
|
21
18
|
|
|
22
|
-
Requires Node.js
|
|
19
|
+
Requires Node.js 22 or later.
|
|
23
20
|
|
|
24
21
|
```bash
|
|
25
22
|
npm install --save-dev @jcode.labs/ragmir
|
|
@@ -29,21 +26,22 @@ npx rgr ingest
|
|
|
29
26
|
npx rgr search "Which decision changed the rollout?"
|
|
30
27
|
```
|
|
31
28
|
|
|
32
|
-
|
|
33
|
-
`.ragmir/` state. Ingestion is incremental
|
|
34
|
-
|
|
29
|
+
Generated configuration, indexes, helpers, reports, and metadata-only logs stay under ignored
|
|
30
|
+
`.ragmir/` state. Ingestion is incremental, resumable, bounded by source, chunk, vector, file, batch,
|
|
31
|
+
and concurrency windows, and serialized across local writer processes.
|
|
35
32
|
|
|
36
33
|
## Choose an interface
|
|
37
34
|
|
|
38
|
-
| Interface | Use |
|
|
35
|
+
| Interface | Use it for |
|
|
39
36
|
| --- | --- |
|
|
40
|
-
| `rgr` CLI | Setup, ingest, search, audit, and JSON automation |
|
|
41
|
-
| TypeScript API | Typed retrieval in
|
|
42
|
-
| Local stdio MCP | Bounded context for compatible
|
|
37
|
+
| `rgr` CLI | Setup, ingest, search, audit, maintenance, and JSON automation |
|
|
38
|
+
| TypeScript API | Typed retrieval in scripts and long-running Node.js workers |
|
|
39
|
+
| Local stdio MCP | Bounded, read-focused context for compatible agents |
|
|
43
40
|
|
|
44
|
-
The default `local-hash` provider
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
The default `local-hash` provider works offline with no model download. Enable semantic
|
|
42
|
+
Transformers.js embeddings explicitly with `rgr setup --semantic`, then rebuild. Core remains
|
|
43
|
+
retrieval-only in both modes. Use `explain: true` or `--explain` to inspect lexical and vector
|
|
44
|
+
contributions, fallback decisions, budgets, queue wait, and the active ranking policy.
|
|
47
45
|
|
|
48
46
|
## TypeScript API
|
|
49
47
|
|
|
@@ -58,37 +56,49 @@ try {
|
|
|
58
56
|
timeoutMs: 10_000,
|
|
59
57
|
})
|
|
60
58
|
|
|
61
|
-
for (const result of results)
|
|
62
|
-
console.log(result.citation, result.text)
|
|
63
|
-
}
|
|
59
|
+
for (const result of results) console.log(result.citation, result.text)
|
|
64
60
|
} finally {
|
|
65
61
|
await ragmir.close()
|
|
66
62
|
}
|
|
67
63
|
```
|
|
68
64
|
|
|
69
|
-
Reuse one client per project root in long-running
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
65
|
+
Reuse one client per project root in a long-running process. It owns the local connection, read
|
|
66
|
+
snapshot, active operation lifecycle, metadata-only log flush, and optional embedding-model lease.
|
|
67
|
+
Top-level `ingest`, `search`, `ask`, and `research` functions remain available for one-shot scripts.
|
|
68
|
+
`ask` returns cited context, not generated prose.
|
|
69
|
+
|
|
70
|
+
## Guarantees and boundaries
|
|
71
|
+
|
|
72
|
+
- Citations use source lines only for line-preserving text, plus PDF pages, PPTX slides, XLSX
|
|
73
|
+
sheets and cells, and EPUB spine positions.
|
|
74
|
+
- Rebuilds activate only after row and manifest validation. Interrupted rebuilds leave the previous
|
|
75
|
+
searchable generation active.
|
|
76
|
+
- Exact vector search remains the policy below 100,000 rows. Larger tables use quality-gated IVF-PQ
|
|
77
|
+
with complete coverage and an exact diagnostic mode.
|
|
78
|
+
- Search, embedding, and ingestion use independent bounded queues. Overload and queue deadlines are
|
|
79
|
+
stable retryable errors.
|
|
80
|
+
- Optional OCR processes only blank PDF pages through a configured local executable and private
|
|
81
|
+
resumable cache.
|
|
82
|
+
- `rgr status` and normal `rgr doctor` read compact manifest health. Use `rgr doctor --deep` or
|
|
83
|
+
`rgr audit` for a live source inventory.
|
|
84
|
+
- `rgr security-audit` checks permissions, Git ignore coverage, tracked private paths, redaction,
|
|
85
|
+
and local extractor authority.
|
|
86
|
+
|
|
87
|
+
The [CLI reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md),
|
|
88
|
+
[API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md), and
|
|
89
|
+
[configuration guide](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md)
|
|
90
|
+
hold the complete options and operational detail.
|
|
91
|
+
|
|
92
|
+
## Optional packages
|
|
93
|
+
|
|
94
|
+
- [`@jcode.labs/ragmir-chat`](https://www.npmjs.com/package/@jcode.labs/ragmir-chat) adds cited
|
|
95
|
+
answer generation with a verified local GGUF profile.
|
|
81
96
|
- [`@jcode.labs/ragmir-tts`](https://www.npmjs.com/package/@jcode.labs/ragmir-tts) renders reviewed
|
|
82
97
|
text as local audio or explicit online speech.
|
|
83
98
|
|
|
84
|
-
Core installs and starts without either add-on. A hosted agent receives only
|
|
85
|
-
integration
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
Read the [configuration guide](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md),
|
|
90
|
-
[security hardening guide](https://github.com/jcode-works/jcode-ragmir/blob/main/SECURITY-HARDENING.md),
|
|
91
|
-
and [troubleshooting guide](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/troubleshooting.md)
|
|
92
|
-
before indexing sensitive material.
|
|
99
|
+
Core installs and starts without either add-on. A hosted agent receives only passages your
|
|
100
|
+
integration sends under that provider's data policy; use a local consumer when no passage may leave
|
|
101
|
+
the workstation. Teams synchronize their source folder and tracked configuration, then build one
|
|
102
|
+
local index per developer. See the [team workflow](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/agent-integration.md#team-knowledge-bases).
|
|
93
103
|
|
|
94
104
|
Ragmir Core is open source under the [MIT License](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE).
|
package/dist/access-log.d.ts
CHANGED
|
@@ -14,7 +14,17 @@ export interface McpOutputLogEvent {
|
|
|
14
14
|
truncated: boolean;
|
|
15
15
|
}
|
|
16
16
|
export declare const MAX_USAGE_REPORT_DAYS = 3650;
|
|
17
|
+
export declare const MAX_ACCESS_LOG_PENDING_EVENTS = 1024;
|
|
18
|
+
export interface AccessLogWriterMetrics {
|
|
19
|
+
pendingEvents: number;
|
|
20
|
+
inFlightEvents: number;
|
|
21
|
+
writtenEvents: number;
|
|
22
|
+
droppedEvents: number;
|
|
23
|
+
}
|
|
17
24
|
export declare function recordAccess(config: Config, event: AccessLogEvent): Promise<void>;
|
|
18
25
|
export declare function recordMcpOutput(config: Config, event: McpOutputLogEvent): Promise<void>;
|
|
26
|
+
export declare function flushAccessLog(config: Config): Promise<AccessLogWriterMetrics>;
|
|
27
|
+
export declare function accessLogWriterMetrics(config: Config): AccessLogWriterMetrics;
|
|
19
28
|
export declare function accessLogUsageReport(options?: AccessLogUsageOptions): Promise<AccessLogUsageReport>;
|
|
29
|
+
export declare function accessLogUsageReportWithConfig(config: Config, options?: AccessLogUsageOptions): Promise<AccessLogUsageReport>;
|
|
20
30
|
//# sourceMappingURL=access-log.d.ts.map
|
package/dist/access-log.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,EACN,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AA2CD,eAAO,MAAM,qBAAqB,OAAQ,CAAA;
|
|
1
|
+
{"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,EACN,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AA2CD,eAAO,MAAM,qBAAqB,OAAQ,CAAA;AAW1C,eAAO,MAAM,6BAA6B,OAAQ,CAAA;AAgBlD,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;CACtB;AAKD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjF;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAevF;AAED,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAYpF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,sBAAsB,CAU7E;AA6FD,wBAAsB,oBAAoB,CACxC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAK/B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAiG/B"}
|
package/dist/access-log.js
CHANGED
|
@@ -33,65 +33,112 @@ const MAX_ACCESS_LOG_BYTES = 10 * 1024 * 1024;
|
|
|
33
33
|
/** Number of most recent lines retained when the log exceeds the byte cap. */
|
|
34
34
|
const TRIMMED_ACCESS_LOG_LINES = 50_000;
|
|
35
35
|
const ACCESS_LOG_SALT_FILE = ".ragmir-access-log.salt";
|
|
36
|
-
const
|
|
37
|
-
|
|
36
|
+
export const MAX_ACCESS_LOG_PENDING_EVENTS = 1_024;
|
|
37
|
+
const ACCESS_LOG_WRITE_BATCH_SIZE = 64;
|
|
38
|
+
const accessLogWriters = new Map();
|
|
39
|
+
const accessLogSalts = new Map();
|
|
40
|
+
export function recordAccess(config, event) {
|
|
38
41
|
if (!config.accessLog) {
|
|
39
|
-
return;
|
|
40
|
-
}
|
|
41
|
-
try {
|
|
42
|
-
await withAccessLogWriteQueue(config.accessLogPath, async () => {
|
|
43
|
-
await ensurePrivateDirectory(path.dirname(config.accessLogPath));
|
|
44
|
-
await trimAccessLogIfNeeded(config.accessLogPath);
|
|
45
|
-
await appendFile(config.accessLogPath, `${JSON.stringify(await toLogLine(config, event))}\n`, {
|
|
46
|
-
encoding: "utf8",
|
|
47
|
-
mode: 0o600,
|
|
48
|
-
});
|
|
49
|
-
await hardenPrivateFile(config.accessLogPath);
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
catch {
|
|
53
|
-
// Access logging is best-effort so read-only workspaces do not block local use.
|
|
42
|
+
return Promise.resolve();
|
|
54
43
|
}
|
|
44
|
+
return enqueueAccessLogLine(config.accessLogPath, async () => JSON.stringify(await toLogLine(config, event)));
|
|
55
45
|
}
|
|
56
|
-
export
|
|
46
|
+
export function recordMcpOutput(config, event) {
|
|
57
47
|
if (!config.accessLog) {
|
|
58
|
-
return;
|
|
48
|
+
return Promise.resolve();
|
|
59
49
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
encoding: "utf8",
|
|
75
|
-
mode: 0o600,
|
|
76
|
-
});
|
|
77
|
-
await hardenPrivateFile(config.accessLogPath);
|
|
78
|
-
});
|
|
50
|
+
return enqueueAccessLogLine(config.accessLogPath, async () => JSON.stringify({
|
|
51
|
+
timestamp: new Date().toISOString(),
|
|
52
|
+
kind: "mcp-output",
|
|
53
|
+
tool: event.tool,
|
|
54
|
+
retrievedBytes: event.retrievedBytes,
|
|
55
|
+
returnedBytes: event.returnedBytes,
|
|
56
|
+
compacted: event.compacted,
|
|
57
|
+
truncated: event.truncated,
|
|
58
|
+
}));
|
|
59
|
+
}
|
|
60
|
+
export async function flushAccessLog(config) {
|
|
61
|
+
const state = accessLogWriters.get(config.accessLogPath);
|
|
62
|
+
if (!state) {
|
|
63
|
+
return emptyAccessLogWriterMetrics();
|
|
79
64
|
}
|
|
80
|
-
|
|
81
|
-
|
|
65
|
+
while (state.active || state.pending.length > 0) {
|
|
66
|
+
startAccessLogWriter(config.accessLogPath, state);
|
|
67
|
+
if (state.active) {
|
|
68
|
+
await state.active;
|
|
69
|
+
}
|
|
82
70
|
}
|
|
71
|
+
return accessLogWriterMetrics(config);
|
|
83
72
|
}
|
|
84
|
-
function
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
73
|
+
export function accessLogWriterMetrics(config) {
|
|
74
|
+
const state = accessLogWriters.get(config.accessLogPath);
|
|
75
|
+
return state
|
|
76
|
+
? {
|
|
77
|
+
pendingEvents: state.pending.length,
|
|
78
|
+
inFlightEvents: state.inFlightEvents,
|
|
79
|
+
writtenEvents: state.writtenEvents,
|
|
80
|
+
droppedEvents: state.droppedEvents,
|
|
92
81
|
}
|
|
82
|
+
: emptyAccessLogWriterMetrics();
|
|
83
|
+
}
|
|
84
|
+
function enqueueAccessLogLine(accessLogPath, render) {
|
|
85
|
+
const state = accessLogWriters.get(accessLogPath) ?? {
|
|
86
|
+
pending: [],
|
|
87
|
+
active: null,
|
|
88
|
+
inFlightEvents: 0,
|
|
89
|
+
writtenEvents: 0,
|
|
90
|
+
droppedEvents: 0,
|
|
91
|
+
};
|
|
92
|
+
accessLogWriters.set(accessLogPath, state);
|
|
93
|
+
if (state.pending.length >= MAX_ACCESS_LOG_PENDING_EVENTS) {
|
|
94
|
+
state.droppedEvents += 1;
|
|
95
|
+
return Promise.resolve();
|
|
96
|
+
}
|
|
97
|
+
return new Promise((settle) => {
|
|
98
|
+
state.pending.push({ render, settle });
|
|
99
|
+
startAccessLogWriter(accessLogPath, state);
|
|
93
100
|
});
|
|
94
|
-
|
|
101
|
+
}
|
|
102
|
+
function startAccessLogWriter(accessLogPath, state) {
|
|
103
|
+
if (state.active || state.pending.length === 0) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const batch = state.pending.splice(0, ACCESS_LOG_WRITE_BATCH_SIZE);
|
|
107
|
+
state.inFlightEvents = batch.length;
|
|
108
|
+
const active = writeAccessLogBatch(accessLogPath, batch)
|
|
109
|
+
.then(() => {
|
|
110
|
+
state.writtenEvents += batch.length;
|
|
111
|
+
})
|
|
112
|
+
.catch(() => {
|
|
113
|
+
state.droppedEvents += batch.length;
|
|
114
|
+
})
|
|
115
|
+
.finally(() => {
|
|
116
|
+
for (const entry of batch) {
|
|
117
|
+
entry.settle();
|
|
118
|
+
}
|
|
119
|
+
state.inFlightEvents = 0;
|
|
120
|
+
state.active = null;
|
|
121
|
+
startAccessLogWriter(accessLogPath, state);
|
|
122
|
+
});
|
|
123
|
+
state.active = active;
|
|
124
|
+
}
|
|
125
|
+
async function writeAccessLogBatch(accessLogPath, batch) {
|
|
126
|
+
await ensurePrivateDirectory(path.dirname(accessLogPath));
|
|
127
|
+
await trimAccessLogIfNeeded(accessLogPath);
|
|
128
|
+
const lines = await Promise.all(batch.map((entry) => entry.render()));
|
|
129
|
+
await appendFile(accessLogPath, `${lines.join("\n")}\n`, {
|
|
130
|
+
encoding: "utf8",
|
|
131
|
+
mode: 0o600,
|
|
132
|
+
});
|
|
133
|
+
await hardenPrivateFile(accessLogPath);
|
|
134
|
+
}
|
|
135
|
+
function emptyAccessLogWriterMetrics() {
|
|
136
|
+
return {
|
|
137
|
+
pendingEvents: 0,
|
|
138
|
+
inFlightEvents: 0,
|
|
139
|
+
writtenEvents: 0,
|
|
140
|
+
droppedEvents: 0,
|
|
141
|
+
};
|
|
95
142
|
}
|
|
96
143
|
async function trimAccessLogIfNeeded(accessLogPath) {
|
|
97
144
|
let size = 0;
|
|
@@ -119,6 +166,12 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
119
166
|
const signal = operationSignal(options);
|
|
120
167
|
throwIfAborted(signal);
|
|
121
168
|
const config = await loadConfig(String(options.cwd ?? process.cwd()));
|
|
169
|
+
return accessLogUsageReportWithConfig(config, options);
|
|
170
|
+
}
|
|
171
|
+
export async function accessLogUsageReportWithConfig(config, options = {}) {
|
|
172
|
+
const signal = operationSignal(options);
|
|
173
|
+
throwIfAborted(signal);
|
|
174
|
+
await flushAccessLog(config);
|
|
122
175
|
throwIfAborted(signal);
|
|
123
176
|
const days = normalizeUsageReportDays(options.days);
|
|
124
177
|
const until = new Date();
|
|
@@ -249,6 +302,23 @@ async function hashQuery(config, query) {
|
|
|
249
302
|
}
|
|
250
303
|
async function accessLogSalt(config) {
|
|
251
304
|
const saltPath = path.join(path.dirname(config.accessLogPath), ACCESS_LOG_SALT_FILE);
|
|
305
|
+
const cached = accessLogSalts.get(saltPath);
|
|
306
|
+
if (cached) {
|
|
307
|
+
return cached;
|
|
308
|
+
}
|
|
309
|
+
const loading = loadAccessLogSalt(saltPath);
|
|
310
|
+
accessLogSalts.set(saltPath, loading);
|
|
311
|
+
try {
|
|
312
|
+
return await loading;
|
|
313
|
+
}
|
|
314
|
+
catch (error) {
|
|
315
|
+
if (accessLogSalts.get(saltPath) === loading) {
|
|
316
|
+
accessLogSalts.delete(saltPath);
|
|
317
|
+
}
|
|
318
|
+
throw error;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
async function loadAccessLogSalt(saltPath) {
|
|
252
322
|
try {
|
|
253
323
|
return await readFile(saltPath);
|
|
254
324
|
}
|
package/dist/access-log.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAiD5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,gBAAgB,GAAoB;IACxC,eAAe;IACf,YAAY;IACZ,iBAAiB;IACjB,eAAe;CAChB,CAAA;AACD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AAC7D,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAA;AAC1C,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AACtD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAyB,CAAA;AAE7D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,uBAAuB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;YAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;YACjD,MAAM,UAAU,CACd,MAAM,CAAC,aAAa,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,EACrD;gBACE,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,KAAK;aACZ,CACF,CAAA;YACD,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc,EAAE,KAAwB;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,uBAAuB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;YAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;YACjD,MAAM,IAAI,GAAqB;gBAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,IAAI,EAAE,YAAY;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;aAC3B,CAAA;YACD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;gBAClE,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,KAAK;aACZ,CAAC,CAAA;YACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;IACzE,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC9B,aAAqB,EACrB,SAA8B;IAE9B,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAA;IAC7E,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAC1B,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAA;IACD,oBAAoB,CAAC,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAA;IAChD,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;QACrB,IAAI,oBAAoB,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,OAAO,EAAE,CAAC;YACxD,oBAAoB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC,CAAC,CAAA;IACF,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,IAAI,OAAe,CAAA;QACnB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;QAC9E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,CAAA;QACb,CAAC;QACD,cAAc,CAAC,MAAM,CAAC,CAAA;QACtB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,CAAC,CAAA;gBACvB,iBAAiB,IAAI,KAAK,CAAC,cAAc,CAAA;gBACzC,gBAAgB,IAAI,KAAK,CAAC,aAAa,CAAA;gBACvC,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;oBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC/B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,gBAAgB;YAC/B,UAAU,EAAE,iBAAiB,GAAG,gBAAgB;YAChD,cAAc,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,iBAAiB;YACzF,kBAAkB;YAClB,kBAAkB;SACnB;QACD,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,qBAAqB,EAAE,CAAC;QAC7E,MAAM,IAAI,KAAK,CAAC,sDAAsD,qBAAqB,GAAG,CAAC,CAAA;IACjG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,MAAM,IAAI,KAAK;QACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAC9B,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnC,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC;QAC7C,oBAAoB,CAAC,KAAK,EAAE,eAAe,CAAC;QAC5C,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC;QAC9B,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAC/B,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,GAAW;IAC5C,OAAO,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,SAAS,CAAA;AACrD,CAAC"}
|
|
1
|
+
{"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAiD5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,gBAAgB,GAAoB;IACxC,eAAe;IACf,YAAY;IACZ,iBAAiB;IACjB,eAAe;CAChB,CAAA;AACD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AAC7D,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAA;AAC1C,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AACtD,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,CAAA;AAClD,MAAM,2BAA2B,GAAG,EAAE,CAAA;AAsBtC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAgC,CAAA;AAChE,MAAM,cAAc,GAAG,IAAI,GAAG,EAA2B,CAAA;AAEzD,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IAChE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,OAAO,oBAAoB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE,CAC3D,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAC/C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,KAAwB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,OAAO,oBAAoB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE,CAC3D,IAAI,CAAC,SAAS,CAAC;QACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;KACA,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAc;IACjD,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IACxD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,2BAA2B,EAAE,CAAA;IACtC,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,oBAAoB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;QACjD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,KAAK,CAAC,MAAM,CAAA;QACpB,CAAC;IACH,CAAC;IACD,OAAO,sBAAsB,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IACxD,OAAO,KAAK;QACV,CAAC,CAAC;YACE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YACnC,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC;QACH,CAAC,CAAC,2BAA2B,EAAE,CAAA;AACnC,CAAC;AAED,SAAS,oBAAoB,CAAC,aAAqB,EAAE,MAA6B;IAChF,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI;QACnD,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;KACjB,CAAA;IACD,gBAAgB,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,6BAA6B,EAAE,CAAC;QAC1D,KAAK,CAAC,aAAa,IAAI,CAAC,CAAA;QACxB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;QACtC,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,aAAqB,EAAE,KAA2B;IAC9E,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,OAAM;IACR,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAA;IAClE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,MAAM,CAAA;IACnC,MAAM,MAAM,GAAG,mBAAmB,CAAC,aAAa,EAAE,KAAK,CAAC;SACrD,IAAI,CAAC,GAAG,EAAE;QACT,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;IACrC,CAAC,CAAC;SACD,KAAK,CAAC,GAAG,EAAE;QACV,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;IACrC,CAAC,CAAC;SACD,OAAO,CAAC,GAAG,EAAE;QACZ,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,KAAK,CAAC,MAAM,EAAE,CAAA;QAChB,CAAC;QACD,KAAK,CAAC,cAAc,GAAG,CAAC,CAAA;QACxB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAA;QACnB,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IACJ,KAAK,CAAC,MAAM,GAAG,MAAM,CAAA;AACvB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,aAAqB,EACrB,KAA6B;IAE7B,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAA;IACzD,MAAM,qBAAqB,CAAC,aAAa,CAAC,CAAA;IAC1C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,UAAU,CAAC,aAAa,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACvD,QAAQ,EAAE,MAAM;QAChB,IAAI,EAAE,KAAK;KACZ,CAAC,CAAA;IACF,MAAM,iBAAiB,CAAC,aAAa,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,2BAA2B;IAClC,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;KACjB,CAAA;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,OAAO,8BAA8B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,MAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,cAAc,CAAC,MAAM,CAAC,CAAA;IAC5B,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,IAAI,OAAe,CAAA;QACnB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;QAC9E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,CAAA;QACb,CAAC;QACD,cAAc,CAAC,MAAM,CAAC,CAAA;QACtB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,CAAC,CAAA;gBACvB,iBAAiB,IAAI,KAAK,CAAC,cAAc,CAAA;gBACzC,gBAAgB,IAAI,KAAK,CAAC,aAAa,CAAA;gBACvC,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;oBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC/B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,gBAAgB;YAC/B,UAAU,EAAE,iBAAiB,GAAG,gBAAgB;YAChD,cAAc,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,iBAAiB;YACzF,kBAAkB;YAClB,kBAAkB;SACnB;QACD,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAA;IACf,CAAC;IACD,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC3C,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACrC,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,OAAO,EAAE,CAAC;YAC7C,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACjC,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,qBAAqB,EAAE,CAAC;QAC7E,MAAM,IAAI,KAAK,CAAC,sDAAsD,qBAAqB,GAAG,CAAC,CAAA;IACjG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,MAAM,IAAI,KAAK;QACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAC9B,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnC,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC;QAC7C,oBAAoB,CAAC,KAAK,EAAE,eAAe,CAAC;QAC5C,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC;QAC9B,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAC/B,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,GAAW;IAC5C,OAAO,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,SAAS,CAAA;AACrD,CAAC"}
|
package/dist/chunking.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import type { ParsedDocument, TextChunk } from "./types.js";
|
|
2
|
-
export
|
|
2
|
+
export interface ChunkDocumentOptions {
|
|
3
|
+
maxChunks?: number;
|
|
4
|
+
signal?: AbortSignal;
|
|
5
|
+
}
|
|
6
|
+
export declare function chunkDocument(document: ParsedDocument, chunkSize: number, chunkOverlap: number, options?: ChunkDocumentOptions): TextChunk[];
|
|
3
7
|
export declare function chunkSearchText(chunk: Pick<TextChunk, "contextPath" | "text">): string;
|
|
4
8
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAA4B,SAAS,EAAE,MAAM,YAAY,CAAA;AAQrF,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,oBAAyB,GACjC,SAAS,EAAE,CAyFb;AAwBD,wBAAgB,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CAAC,GAAG,MAAM,CAEtF"}
|
package/dist/chunking.js
CHANGED
|
@@ -1,17 +1,28 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { markdownFenceSpans, structuralSpans } from "./document-structure.js";
|
|
3
|
+
import { throwIfAborted } from "./operation.js";
|
|
3
4
|
const PARAGRAPH_BREAK_MIN_RATIO = 0.45;
|
|
4
5
|
const SENTENCE_BREAK_MIN_RATIO = 0.55;
|
|
5
6
|
const LINE_BREAK_MIN_RATIO = 0.65;
|
|
6
7
|
const WHITESPACE_BREAK_MIN_RATIO = 0.75;
|
|
7
8
|
const SENTENCE_BOUNDARIES = [". ", "? ", "! ", "。", "?", "!"];
|
|
8
|
-
export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
9
|
+
export function chunkDocument(document, chunkSize, chunkOverlap, options = {}) {
|
|
10
|
+
throwIfAborted(options.signal);
|
|
9
11
|
if (!document.text) {
|
|
10
12
|
return [];
|
|
11
13
|
}
|
|
12
14
|
const chunks = [];
|
|
13
|
-
const lineStarts = lineStartOffsets(document.text);
|
|
14
|
-
const
|
|
15
|
+
const lineStarts = lineStartOffsets(document.text, options.signal);
|
|
16
|
+
const locationRegions = sortedCharacterIntervals(document.regions ?? []);
|
|
17
|
+
const pages = sortedCharacterIntervals(document.pages ?? []);
|
|
18
|
+
const sourceRegions = locationRegions.map((region) => ({
|
|
19
|
+
charStart: region.charStart,
|
|
20
|
+
charEnd: region.charEnd,
|
|
21
|
+
contextPath: region.contextPath,
|
|
22
|
+
}));
|
|
23
|
+
const structured = sourceRegions.length > 0
|
|
24
|
+
? sourceRegions
|
|
25
|
+
: structuralSpans(document.text, document.file.extension, chunkSize);
|
|
15
26
|
const regions = structured.length > 0
|
|
16
27
|
? structured
|
|
17
28
|
: [
|
|
@@ -25,16 +36,28 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
|
25
36
|
const fences = [".md", ".mdx", ".markdown"].includes(document.file.extension)
|
|
26
37
|
? markdownFenceSpans(document.text)
|
|
27
38
|
: [];
|
|
39
|
+
const fenceStarts = new Set(fences.map((fence) => fence.start));
|
|
28
40
|
let chunkIndex = 0;
|
|
29
41
|
for (const region of regions) {
|
|
42
|
+
throwIfAborted(options.signal);
|
|
30
43
|
let cursor = region.charStart;
|
|
31
44
|
while (cursor < region.charEnd) {
|
|
45
|
+
throwIfAborted(options.signal);
|
|
32
46
|
const end = chooseChunkEnd(document.text, cursor, chunkSize, region.charEnd, fences);
|
|
33
47
|
const span = trimmedSpan(document.text, cursor, end);
|
|
34
48
|
if (span.text) {
|
|
49
|
+
if (options.maxChunks !== undefined && chunks.length >= options.maxChunks) {
|
|
50
|
+
throw new Error(`Chunk limit of ${options.maxChunks} exceeded for ${document.file.relativePath}. Increase chunkSize or split the source file.`);
|
|
51
|
+
}
|
|
35
52
|
const id = createHash("sha256")
|
|
36
53
|
.update(`${document.file.relativePath}:${chunkIndex}:${region.contextPath}:${span.text}`)
|
|
37
54
|
.digest("hex");
|
|
55
|
+
const lineCoordinates = document.sourceLineCoordinates === false
|
|
56
|
+
? {}
|
|
57
|
+
: {
|
|
58
|
+
lineStart: lineNumberForOffset(lineStarts, span.start),
|
|
59
|
+
lineEnd: lineNumberForOffset(lineStarts, Math.max(span.start, span.end - 1)),
|
|
60
|
+
};
|
|
38
61
|
chunks.push({
|
|
39
62
|
id,
|
|
40
63
|
source: document.file.source,
|
|
@@ -44,9 +67,9 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
|
44
67
|
text: span.text,
|
|
45
68
|
charStart: span.start,
|
|
46
69
|
charEnd: span.end,
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
...
|
|
70
|
+
...lineCoordinates,
|
|
71
|
+
...pageRangeForSpan(pages, span.start, span.end),
|
|
72
|
+
...sourceLocationForSpan(locationRegions, span.start, span.end),
|
|
50
73
|
checksum: document.file.checksum,
|
|
51
74
|
bytes: document.file.bytes,
|
|
52
75
|
mtimeMs: document.file.mtimeMs,
|
|
@@ -56,32 +79,58 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
|
56
79
|
if (end >= region.charEnd) {
|
|
57
80
|
break;
|
|
58
81
|
}
|
|
59
|
-
cursor =
|
|
82
|
+
cursor = fenceStarts.has(end)
|
|
60
83
|
? end
|
|
61
84
|
: Math.max(end - chunkOverlap, cursor + 1, region.charStart);
|
|
62
85
|
}
|
|
63
86
|
}
|
|
64
87
|
return chunks;
|
|
65
88
|
}
|
|
89
|
+
function sourceLocationForSpan(regions, start, end) {
|
|
90
|
+
const region = regions[firstIntervalEndingAfter(regions, start)];
|
|
91
|
+
if (!region || region.charStart >= end) {
|
|
92
|
+
return {};
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
locationKind: region.location.kind,
|
|
96
|
+
locationStart: region.location.start,
|
|
97
|
+
locationEnd: region.location.end,
|
|
98
|
+
...(region.location.label ? { locationLabel: region.location.label } : {}),
|
|
99
|
+
...(region.location.cellStart ? { cellStart: region.location.cellStart } : {}),
|
|
100
|
+
...(region.location.cellEnd ? { cellEnd: region.location.cellEnd } : {}),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
66
103
|
export function chunkSearchText(chunk) {
|
|
67
104
|
return chunk.contextPath ? `${chunk.contextPath}\n${chunk.text}` : chunk.text;
|
|
68
105
|
}
|
|
69
|
-
function pageRangeForSpan(
|
|
70
|
-
if (
|
|
106
|
+
function pageRangeForSpan(pages, start, end) {
|
|
107
|
+
if (pages.length === 0) {
|
|
108
|
+
return {};
|
|
109
|
+
}
|
|
110
|
+
const firstIndex = firstIntervalEndingAfter(pages, start);
|
|
111
|
+
const first = pages[firstIndex];
|
|
112
|
+
if (!first || first.charStart >= end) {
|
|
71
113
|
return {};
|
|
72
114
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
115
|
+
let last = first;
|
|
116
|
+
for (let index = firstIndex + 1; index < pages.length; index += 1) {
|
|
117
|
+
const page = pages[index];
|
|
118
|
+
if (!page || page.charStart >= end) {
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
last = page;
|
|
122
|
+
}
|
|
123
|
+
return { pageStart: first.pageNumber, pageEnd: last.pageNumber };
|
|
77
124
|
}
|
|
78
125
|
function chooseChunkEnd(text, cursor, chunkSize, regionEnd, fences) {
|
|
79
126
|
const hardEnd = Math.min(cursor + chunkSize, regionEnd);
|
|
80
127
|
if (hardEnd === regionEnd) {
|
|
81
128
|
return hardEnd;
|
|
82
129
|
}
|
|
83
|
-
const intersectedFence = fences
|
|
84
|
-
if (intersectedFence &&
|
|
130
|
+
const intersectedFence = fenceAtOffset(fences, hardEnd);
|
|
131
|
+
if (intersectedFence &&
|
|
132
|
+
intersectedFence.end <= regionEnd &&
|
|
133
|
+
intersectedFence.end - intersectedFence.start <= chunkSize) {
|
|
85
134
|
return intersectedFence.start > cursor ? intersectedFence.start : intersectedFence.end;
|
|
86
135
|
}
|
|
87
136
|
const window = text.slice(cursor, hardEnd);
|
|
@@ -128,15 +177,52 @@ function trimmedSpan(text, start, end) {
|
|
|
128
177
|
text: text.slice(trimmedStart, trimmedEnd),
|
|
129
178
|
};
|
|
130
179
|
}
|
|
131
|
-
function lineStartOffsets(text) {
|
|
180
|
+
function lineStartOffsets(text, signal) {
|
|
132
181
|
const starts = [0];
|
|
133
182
|
for (let index = 0; index < text.length; index += 1) {
|
|
183
|
+
if (signal && index % 16_384 === 0) {
|
|
184
|
+
throwIfAborted(signal);
|
|
185
|
+
}
|
|
134
186
|
if (text[index] === "\n" && index + 1 < text.length) {
|
|
135
187
|
starts.push(index + 1);
|
|
136
188
|
}
|
|
137
189
|
}
|
|
138
190
|
return starts;
|
|
139
191
|
}
|
|
192
|
+
function sortedCharacterIntervals(intervals) {
|
|
193
|
+
return [...intervals].sort((left, right) => left.charEnd - right.charEnd || left.charStart - right.charStart);
|
|
194
|
+
}
|
|
195
|
+
function firstIntervalEndingAfter(intervals, offset) {
|
|
196
|
+
let low = 0;
|
|
197
|
+
let high = intervals.length;
|
|
198
|
+
while (low < high) {
|
|
199
|
+
const mid = Math.floor((low + high) / 2);
|
|
200
|
+
const interval = intervals[mid];
|
|
201
|
+
if (interval && interval.charEnd <= offset) {
|
|
202
|
+
low = mid + 1;
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
high = mid;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return low;
|
|
209
|
+
}
|
|
210
|
+
function fenceAtOffset(fences, offset) {
|
|
211
|
+
let low = 0;
|
|
212
|
+
let high = fences.length;
|
|
213
|
+
while (low < high) {
|
|
214
|
+
const mid = Math.floor((low + high) / 2);
|
|
215
|
+
const fence = fences[mid];
|
|
216
|
+
if (fence && fence.end <= offset) {
|
|
217
|
+
low = mid + 1;
|
|
218
|
+
}
|
|
219
|
+
else {
|
|
220
|
+
high = mid;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const fence = fences[low];
|
|
224
|
+
return fence && fence.start < offset && fence.end > offset ? fence : undefined;
|
|
225
|
+
}
|
|
140
226
|
function lineNumberForOffset(lineStarts, offset) {
|
|
141
227
|
let low = 0;
|
|
142
228
|
let high = lineStarts.length - 1;
|