@jcode.labs/ragmir 2.9.2 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -44
- package/dist/access-log.d.ts +9 -1
- package/dist/access-log.d.ts.map +1 -1
- package/dist/access-log.js +82 -3
- package/dist/access-log.js.map +1 -1
- package/dist/chunking.d.ts +1 -0
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +58 -31
- package/dist/chunking.js.map +1 -1
- package/dist/cli.js +3 -0
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +16 -0
- package/dist/config.js.map +1 -1
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +1 -0
- package/dist/defaults.js.map +1 -1
- package/dist/document-structure.d.ts +13 -0
- package/dist/document-structure.d.ts.map +1 -0
- package/dist/document-structure.js +331 -0
- package/dist/document-structure.js.map +1 -0
- package/dist/index-diagnostics.d.ts +1 -1
- package/dist/index-diagnostics.js +1 -1
- package/dist/index-policy.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +3 -2
- package/dist/ingest.js.map +1 -1
- package/dist/mcp-output.d.ts +60 -0
- package/dist/mcp-output.d.ts.map +1 -0
- package/dist/mcp-output.js +270 -0
- package/dist/mcp-output.js.map +1 -0
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +101 -14
- package/dist/mcp.js.map +1 -1
- package/dist/query.d.ts +2 -1
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +132 -8
- package/dist/query.js.map +1 -1
- package/dist/research.d.ts.map +1 -1
- package/dist/research.js +3 -0
- package/dist/research.js.map +1 -1
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +1 -0
- package/dist/security.js.map +1 -1
- package/dist/store.js +2 -2
- package/dist/store.js.map +1 -1
- package/dist/types.d.ts +31 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/examples/document-evidence-benchmark/README.md +71 -17
- package/examples/library-api-demo/README.md +69 -24
- package/examples/sovereign-rag-demo/README.md +90 -46
- package/package.json +3 -3
- package/skills/ragmir/SKILL.md +6 -0
package/README.md
CHANGED
|
@@ -1,50 +1,70 @@
|
|
|
1
1
|
# Ragmir Core
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/@jcode.labs/ragmir)
|
|
4
|
+
[](https://www.npmjs.com/package/@jcode.labs/ragmir)
|
|
5
|
+
[](https://www.npmjs.com/package/@jcode.labs/ragmir)
|
|
6
|
+
[](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE)
|
|
5
7
|
|
|
6
|
-
|
|
7
|
-
knowledge base. It is retrieval-first: the default `local-hash` mode works offline, without an
|
|
8
|
-
account or model download.
|
|
8
|
+
**The CLI, TypeScript API, and local MCP server for cited project retrieval.**
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
`@jcode.labs/ragmir` indexes the files a project selects and returns source-backed passages without
|
|
11
|
+
a hosted document store. The default `local-hash` retrieval path works offline, without an account
|
|
12
|
+
or model download.
|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
[Project overview](https://github.com/jcode-works/jcode-ragmir#readme) ·
|
|
15
|
+
[CLI reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md) ·
|
|
16
|
+
[API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md)
|
|
17
|
+
|
|
18
|
+
## Install Core when you need
|
|
19
|
+
|
|
20
|
+
| Goal | Core interface |
|
|
13
21
|
| --- | --- |
|
|
14
|
-
| Search
|
|
15
|
-
|
|
|
16
|
-
|
|
|
17
|
-
|
|
|
18
|
-
|
|
|
22
|
+
| Search repository documents with citations | `rgr search` |
|
|
23
|
+
| Audit a local knowledge base | `rgr audit`, `rgr doctor`, and `rgr security-audit` |
|
|
24
|
+
| Add retrieval to a Node.js application | Typed `ingest`, `search`, `ask`, and `research` exports |
|
|
25
|
+
| Give an AI agent bounded project context | Local stdio MCP server and generated helpers |
|
|
26
|
+
| Index scanned PDFs | Embedded text first, with optional page-aware local OCR |
|
|
19
27
|
|
|
20
|
-
|
|
21
|
-
|
|
28
|
+
Core retrieves evidence. It does not require a generative model and does not produce an ungrounded
|
|
29
|
+
answer. Add [Ragmir Chat](https://www.npmjs.com/package/@jcode.labs/ragmir-chat) only when local
|
|
30
|
+
GGUF synthesis is useful, or [Ragmir TTS](https://www.npmjs.com/package/@jcode.labs/ragmir-tts)
|
|
31
|
+
when you need audio output.
|
|
22
32
|
|
|
23
|
-
##
|
|
33
|
+
## First cited search
|
|
24
34
|
|
|
25
|
-
|
|
35
|
+
Requires Node.js 20 or later.
|
|
26
36
|
|
|
27
37
|
```bash
|
|
28
38
|
npm install --save-dev @jcode.labs/ragmir
|
|
29
39
|
npx rgr setup
|
|
30
|
-
npx rgr sources add "docs/**/*.md"
|
|
40
|
+
npx rgr sources add "README.md" "docs/**/*.md"
|
|
31
41
|
npx rgr ingest
|
|
32
|
-
npx rgr search "Which decision changed the
|
|
42
|
+
npx rgr search "Which decision changed the rollout?"
|
|
33
43
|
```
|
|
34
44
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
45
|
+
The project now owns an ignored `.ragmir/` directory containing configuration and generated local
|
|
46
|
+
state. Ingestion is incremental, and every result identifies the source path, chunk, line range, and
|
|
47
|
+
PDF page when one is available.
|
|
38
48
|
|
|
39
|
-
##
|
|
49
|
+
## CLI essentials
|
|
40
50
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
51
|
+
```bash
|
|
52
|
+
# Readiness and source coverage
|
|
53
|
+
npx rgr doctor
|
|
54
|
+
npx rgr audit --unsupported
|
|
55
|
+
npx rgr security-audit
|
|
56
|
+
|
|
57
|
+
# Retrieval
|
|
58
|
+
npx rgr search "deployment decision"
|
|
59
|
+
npx rgr ask "What evidence supports the deployment decision?"
|
|
60
|
+
npx rgr research "deployment obligations" --compact
|
|
61
|
+
|
|
62
|
+
# Machine-readable output
|
|
63
|
+
npx rgr search "deployment decision" --json
|
|
64
|
+
```
|
|
45
65
|
|
|
46
|
-
|
|
47
|
-
|
|
66
|
+
`ask` returns cited retrieval context, not LLM synthesis. `research` performs an audit-backed,
|
|
67
|
+
multi-query retrieval pass and reports missing or weak evidence.
|
|
48
68
|
|
|
49
69
|
## TypeScript API
|
|
50
70
|
|
|
@@ -53,39 +73,102 @@ import { ingest, search } from "@jcode.labs/ragmir"
|
|
|
53
73
|
|
|
54
74
|
await ingest({ cwd: process.cwd() })
|
|
55
75
|
|
|
56
|
-
const results = await search("Which decision changed the
|
|
76
|
+
const results = await search("Which decision changed the rollout?", {
|
|
77
|
+
cwd: process.cwd(),
|
|
78
|
+
topK: 5,
|
|
79
|
+
})
|
|
57
80
|
|
|
58
81
|
for (const result of results) {
|
|
59
|
-
console.log(result.
|
|
82
|
+
console.log(result.citation, result.text)
|
|
60
83
|
}
|
|
61
84
|
```
|
|
62
85
|
|
|
63
|
-
|
|
64
|
-
|
|
86
|
+
Frequently used exports:
|
|
87
|
+
|
|
88
|
+
| Export | Purpose |
|
|
89
|
+
| --- | --- |
|
|
90
|
+
| `setupProject`, `addSourceEntries` | Initialize project state and select files |
|
|
91
|
+
| `ingest`, `audit` | Build the index and compare it with files on disk |
|
|
92
|
+
| `search`, `ask`, `research`, `expandCitation` | Retrieve or expand cited passages |
|
|
93
|
+
| `doctor`, `securityAudit` | Inspect readiness and local privacy posture |
|
|
94
|
+
| `serveMcp` | Start the read-focused local MCP server |
|
|
95
|
+
| `configurePdfOcr`, `inspectPdfOcr` | Configure and inspect local PDF OCR |
|
|
65
96
|
|
|
66
|
-
|
|
97
|
+
See the [complete API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md)
|
|
98
|
+
for options and result shapes.
|
|
67
99
|
|
|
68
|
-
|
|
100
|
+
## Connect an AI agent
|
|
69
101
|
|
|
70
102
|
```bash
|
|
71
103
|
npx rgr setup --agents claude,codex,kimi,opencode,cline
|
|
104
|
+
npx rgr doctor
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Ragmir writes helper files for the selected clients and points them at the current project. MCP
|
|
108
|
+
exposes status, search, ask, research, exact citation expansion, audit, evaluation, usage, and
|
|
109
|
+
security tools. Retrieval responses have a global byte ceiling and expose metadata-only output
|
|
110
|
+
metrics. The server does not expose index deletion.
|
|
111
|
+
|
|
112
|
+
## Retrieval modes
|
|
113
|
+
|
|
114
|
+
| Mode | Network requirement | Best for |
|
|
115
|
+
| --- | --- | --- |
|
|
116
|
+
| `local-hash` | None | Default offline retrieval, setup-free projects, and deterministic evaluation |
|
|
117
|
+
| `transformers` | Explicit model preload or download | Higher-quality semantic matching with local inference |
|
|
118
|
+
|
|
119
|
+
Enable semantic retrieval explicitly:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
npx rgr setup --semantic
|
|
123
|
+
npx rgr ingest --rebuild
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
`local-hash` is lexical/hash retrieval. It should not be presented as equivalent to semantic
|
|
127
|
+
embeddings.
|
|
128
|
+
|
|
129
|
+
## Documents and OCR
|
|
130
|
+
|
|
131
|
+
Core parses common source, text, structured-data, Office, OpenDocument, EPUB, HTML, email, notebook,
|
|
132
|
+
and PDF formats. Projects can add custom text extensions. Run `rgr audit --unsupported` to see every
|
|
133
|
+
skipped file and recommendation.
|
|
134
|
+
|
|
135
|
+
For scanned PDFs:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
npx rgr ocr setup --engine auto
|
|
139
|
+
npx rgr ingest --rebuild
|
|
72
140
|
```
|
|
73
141
|
|
|
74
|
-
|
|
75
|
-
|
|
142
|
+
Ragmir prefers embedded PDF text and invokes OCR only for blank pages. OCR runs through a configured
|
|
143
|
+
local executable, never a shell string or cloud OCR service.
|
|
144
|
+
|
|
145
|
+
## Local data and privacy
|
|
76
146
|
|
|
77
|
-
|
|
147
|
+
- Project paths resolve from `cwd` or explicit configuration, not the npm installation directory.
|
|
148
|
+
- Generated indexes, models, reports, audio, and access logs belong under ignored `.ragmir/` state.
|
|
149
|
+
- Redaction runs before indexing when configured, but it is not a compliance certification.
|
|
150
|
+
- External extractors and model downloads are opt-in system boundaries.
|
|
151
|
+
- MCP retrieval is bounded by `mcpMaxOutputBytes`; access logs contain byte metrics, not query text
|
|
152
|
+
or retrieved passages.
|
|
78
153
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
154
|
+
Read [configuration and privacy](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md)
|
|
155
|
+
and [security hardening](https://github.com/jcode-works/jcode-ragmir/blob/main/SECURITY-HARDENING.md)
|
|
156
|
+
before indexing sensitive material.
|
|
157
|
+
|
|
158
|
+
## Runnable examples
|
|
159
|
+
|
|
160
|
+
| Example | Demonstrates |
|
|
161
|
+
| --- | --- |
|
|
162
|
+
| [Sovereign RAG demo](https://github.com/jcode-works/jcode-ragmir/tree/main/packages/ragmir-core/examples/sovereign-rag-demo) | CLI ingestion, search, audit, redaction, and evaluation |
|
|
163
|
+
| [Library API demo](https://github.com/jcode-works/jcode-ragmir/tree/main/packages/ragmir-core/examples/library-api-demo) | The public library surface against fictional files |
|
|
164
|
+
| [Document evidence benchmark](https://github.com/jcode-works/jcode-ragmir/tree/main/packages/ragmir-core/examples/document-evidence-benchmark) | Recall and exact citation evaluation |
|
|
82
165
|
|
|
83
|
-
##
|
|
166
|
+
## Documentation
|
|
84
167
|
|
|
85
|
-
- [Ragmir overview and package comparison](https://github.com/jcode-works/jcode-ragmir#readme)
|
|
86
168
|
- [CLI reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md)
|
|
87
169
|
- [TypeScript API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md)
|
|
88
|
-
- [Configuration
|
|
170
|
+
- [Configuration](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md)
|
|
89
171
|
- [Agent integration](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/agent-integration.md)
|
|
172
|
+
- [Troubleshooting](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/troubleshooting.md)
|
|
90
173
|
|
|
91
|
-
Ragmir is open source under the [MIT License](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE).
|
|
174
|
+
Ragmir Core is open source under the [MIT License](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE).
|
package/dist/access-log.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AccessLogAction, AccessLogUsageOptions, AccessLogUsageReport, Config } from "./types.js";
|
|
1
|
+
import type { AccessLogAction, AccessLogUsageOptions, AccessLogUsageReport, Config, McpOutputTool } from "./types.js";
|
|
2
2
|
export interface AccessLogEvent {
|
|
3
3
|
action: AccessLogAction;
|
|
4
4
|
query?: string;
|
|
@@ -6,6 +6,14 @@ export interface AccessLogEvent {
|
|
|
6
6
|
resultCount?: number;
|
|
7
7
|
redactions?: number;
|
|
8
8
|
}
|
|
9
|
+
export interface McpOutputLogEvent {
|
|
10
|
+
tool: McpOutputTool;
|
|
11
|
+
retrievedBytes: number;
|
|
12
|
+
returnedBytes: number;
|
|
13
|
+
compacted: boolean;
|
|
14
|
+
truncated: boolean;
|
|
15
|
+
}
|
|
9
16
|
export declare function recordAccess(config: Config, event: AccessLogEvent): Promise<void>;
|
|
17
|
+
export declare function recordMcpOutput(config: Config, event: McpOutputLogEvent): Promise<void>;
|
|
10
18
|
export declare function accessLogUsageReport(options?: AccessLogUsageOptions): Promise<AccessLogUsageReport>;
|
|
11
19
|
//# sourceMappingURL=access-log.d.ts.map
|
package/dist/access-log.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,
|
|
1
|
+
{"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,EACN,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AAsDD,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBvF;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAyB7F;AA2BD,wBAAsB,oBAAoB,CACxC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAoF/B"}
|
package/dist/access-log.js
CHANGED
|
@@ -13,6 +13,13 @@ const ACCESS_LOG_ACTIONS = [
|
|
|
13
13
|
"destroy-index",
|
|
14
14
|
];
|
|
15
15
|
const ACCESS_LOG_ACTION_SET = new Set(ACCESS_LOG_ACTIONS);
|
|
16
|
+
const MCP_OUTPUT_TOOLS = [
|
|
17
|
+
"ragmir_search",
|
|
18
|
+
"ragmir_ask",
|
|
19
|
+
"ragmir_research",
|
|
20
|
+
"ragmir_expand",
|
|
21
|
+
];
|
|
22
|
+
const MCP_OUTPUT_TOOL_SET = new Set(MCP_OUTPUT_TOOLS);
|
|
16
23
|
const DEFAULT_USAGE_REPORT_DAYS = 7;
|
|
17
24
|
const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
18
25
|
/**
|
|
@@ -41,6 +48,32 @@ export async function recordAccess(config, event) {
|
|
|
41
48
|
// Access logging is best-effort so read-only workspaces do not block local use.
|
|
42
49
|
}
|
|
43
50
|
}
|
|
51
|
+
export async function recordMcpOutput(config, event) {
|
|
52
|
+
if (!config.accessLog) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
await ensurePrivateDirectory(path.dirname(config.accessLogPath));
|
|
57
|
+
await trimAccessLogIfNeeded(config.accessLogPath);
|
|
58
|
+
const line = {
|
|
59
|
+
timestamp: new Date().toISOString(),
|
|
60
|
+
kind: "mcp-output",
|
|
61
|
+
tool: event.tool,
|
|
62
|
+
retrievedBytes: event.retrievedBytes,
|
|
63
|
+
returnedBytes: event.returnedBytes,
|
|
64
|
+
compacted: event.compacted,
|
|
65
|
+
truncated: event.truncated,
|
|
66
|
+
};
|
|
67
|
+
await appendFile(config.accessLogPath, `${JSON.stringify(line)}\n`, {
|
|
68
|
+
encoding: "utf8",
|
|
69
|
+
mode: 0o600,
|
|
70
|
+
});
|
|
71
|
+
await hardenPrivateFile(config.accessLogPath);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
// Output metrics are best-effort and must never block local retrieval.
|
|
75
|
+
}
|
|
76
|
+
}
|
|
44
77
|
async function trimAccessLogIfNeeded(accessLogPath) {
|
|
45
78
|
let size = 0;
|
|
46
79
|
try {
|
|
@@ -75,6 +108,11 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
75
108
|
let invalidLines = 0;
|
|
76
109
|
let resultCountTotal = 0;
|
|
77
110
|
let resultCountEvents = 0;
|
|
111
|
+
let mcpOutputResponses = 0;
|
|
112
|
+
let mcpRetrievedBytes = 0;
|
|
113
|
+
let mcpReturnedBytes = 0;
|
|
114
|
+
let compactedResponses = 0;
|
|
115
|
+
let truncatedResponses = 0;
|
|
78
116
|
let lastEventAt = null;
|
|
79
117
|
if (existsSync(config.accessLogPath)) {
|
|
80
118
|
const lines = (await readFile(config.accessLogPath, "utf8")).split(/\r?\n/u).filter(Boolean);
|
|
@@ -91,6 +129,17 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
91
129
|
continue;
|
|
92
130
|
}
|
|
93
131
|
totalEvents += 1;
|
|
132
|
+
if (isMcpOutputLogLine(event)) {
|
|
133
|
+
mcpOutputResponses += 1;
|
|
134
|
+
mcpRetrievedBytes += event.retrievedBytes;
|
|
135
|
+
mcpReturnedBytes += event.returnedBytes;
|
|
136
|
+
compactedResponses += event.compacted ? 1 : 0;
|
|
137
|
+
truncatedResponses += event.truncated ? 1 : 0;
|
|
138
|
+
if (lastEventAt === null || event.timestamp > lastEventAt) {
|
|
139
|
+
lastEventAt = event.timestamp;
|
|
140
|
+
}
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
94
143
|
eventsByAction[event.action] += 1;
|
|
95
144
|
if (event.queryHash) {
|
|
96
145
|
queryHashes.add(event.queryHash);
|
|
@@ -116,6 +165,15 @@ export async function accessLogUsageReport(options = {}) {
|
|
|
116
165
|
uniqueQueryHashes: queryHashes.size,
|
|
117
166
|
averageResultCount: resultCountEvents === 0 ? null : resultCountTotal / resultCountEvents,
|
|
118
167
|
averageResultCountByAction: averageResultCounts(resultCountsByAction),
|
|
168
|
+
mcpOutput: {
|
|
169
|
+
responses: mcpOutputResponses,
|
|
170
|
+
retrievedBytes: mcpRetrievedBytes,
|
|
171
|
+
returnedBytes: mcpReturnedBytes,
|
|
172
|
+
savedBytes: mcpRetrievedBytes - mcpReturnedBytes,
|
|
173
|
+
reductionRatio: mcpRetrievedBytes === 0 ? null : 1 - mcpReturnedBytes / mcpRetrievedBytes,
|
|
174
|
+
compactedResponses,
|
|
175
|
+
truncatedResponses,
|
|
176
|
+
},
|
|
119
177
|
lastEventAt,
|
|
120
178
|
};
|
|
121
179
|
}
|
|
@@ -201,15 +259,29 @@ function normalizeUsageReportDays(days) {
|
|
|
201
259
|
function parseAccessLogLine(line) {
|
|
202
260
|
try {
|
|
203
261
|
const parsed = JSON.parse(line);
|
|
204
|
-
if (
|
|
205
|
-
return
|
|
262
|
+
if (isAccessLogLine(parsed) || isMcpOutputLogLine(parsed)) {
|
|
263
|
+
return parsed;
|
|
206
264
|
}
|
|
207
|
-
return
|
|
265
|
+
return null;
|
|
208
266
|
}
|
|
209
267
|
catch {
|
|
210
268
|
return null;
|
|
211
269
|
}
|
|
212
270
|
}
|
|
271
|
+
function isMcpOutputLogLine(value) {
|
|
272
|
+
return (typeof value === "object" &&
|
|
273
|
+
value !== null &&
|
|
274
|
+
hasTimestamp(value) &&
|
|
275
|
+
"kind" in value &&
|
|
276
|
+
value.kind === "mcp-output" &&
|
|
277
|
+
"tool" in value &&
|
|
278
|
+
typeof value.tool === "string" &&
|
|
279
|
+
MCP_OUTPUT_TOOL_SET.has(value.tool) &&
|
|
280
|
+
hasNonNegativeNumber(value, "retrievedBytes") &&
|
|
281
|
+
hasNonNegativeNumber(value, "returnedBytes") &&
|
|
282
|
+
hasBoolean(value, "compacted") &&
|
|
283
|
+
hasBoolean(value, "truncated"));
|
|
284
|
+
}
|
|
213
285
|
function isAccessLogLine(value) {
|
|
214
286
|
return (typeof value === "object" &&
|
|
215
287
|
value !== null &&
|
|
@@ -230,4 +302,11 @@ function hasOptionalQueryHash(value) {
|
|
|
230
302
|
function hasOptionalResultCount(value) {
|
|
231
303
|
return !("resultCount" in value) || typeof value.resultCount === "number";
|
|
232
304
|
}
|
|
305
|
+
function hasNonNegativeNumber(value, key) {
|
|
306
|
+
const field = Reflect.get(value, key);
|
|
307
|
+
return typeof field === "number" && Number.isFinite(field) && field >= 0;
|
|
308
|
+
}
|
|
309
|
+
function hasBoolean(value, key) {
|
|
310
|
+
return typeof Reflect.get(value, key) === "boolean";
|
|
311
|
+
}
|
|
233
312
|
//# sourceMappingURL=access-log.js.map
|
package/dist/access-log.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AA4B5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AAEtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;QAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;YAC5F,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAA;QACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC"}
|
|
1
|
+
{"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAiD5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,gBAAgB,GAAoB;IACxC,eAAe;IACf,YAAY;IACZ,iBAAiB;IACjB,eAAe;CAChB,CAAA;AACD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AAC7D,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AAEtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;QAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;YAC5F,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAA;QACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc,EAAE,KAAwB;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;QAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,IAAI,GAAqB;YAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAA;QACD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;YAClE,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAA;QACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;IACzE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC5F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,CAAC,CAAA;gBACvB,iBAAiB,IAAI,KAAK,CAAC,cAAc,CAAA;gBACzC,gBAAgB,IAAI,KAAK,CAAC,aAAa,CAAA;gBACvC,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;oBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC/B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,gBAAgB;YAC/B,UAAU,EAAE,iBAAiB,GAAG,gBAAgB;YAChD,cAAc,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,iBAAiB;YACzF,kBAAkB;YAClB,kBAAkB;SACnB;QACD,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAClE,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,MAAM,IAAI,KAAK;QACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAC9B,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnC,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC;QAC7C,oBAAoB,CAAC,KAAK,EAAE,eAAe,CAAC;QAC5C,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC;QAC9B,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAC/B,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,GAAW;IAC5C,OAAO,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,SAAS,CAAA;AACrD,CAAC"}
|
package/dist/chunking.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
import type { ParsedDocument, TextChunk } from "./types.js";
|
|
2
2
|
export declare function chunkDocument(document: ParsedDocument, chunkSize: number, chunkOverlap: number): TextChunk[];
|
|
3
|
+
export declare function chunkSearchText(chunk: Pick<TextChunk, "contextPath" | "text">): string;
|
|
3
4
|
//# sourceMappingURL=chunking.d.ts.map
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAQ3D,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CA+Db;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CAAC,GAAG,MAAM,CAEtF"}
|
package/dist/chunking.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { markdownFenceSpans, structuralSpans } from "./document-structure.js";
|
|
2
3
|
const PARAGRAPH_BREAK_MIN_RATIO = 0.45;
|
|
3
4
|
const SENTENCE_BREAK_MIN_RATIO = 0.55;
|
|
4
5
|
const LINE_BREAK_MIN_RATIO = 0.65;
|
|
@@ -10,39 +11,61 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
|
|
|
10
11
|
}
|
|
11
12
|
const chunks = [];
|
|
12
13
|
const lineStarts = lineStartOffsets(document.text);
|
|
13
|
-
|
|
14
|
+
const structured = structuralSpans(document.text, document.file.extension, chunkSize);
|
|
15
|
+
const regions = structured.length > 0
|
|
16
|
+
? structured
|
|
17
|
+
: [
|
|
18
|
+
{
|
|
19
|
+
charStart: 0,
|
|
20
|
+
charEnd: document.text.length,
|
|
21
|
+
contextPath: "",
|
|
22
|
+
kind: "markdown-section",
|
|
23
|
+
},
|
|
24
|
+
];
|
|
25
|
+
const fences = [".md", ".mdx", ".markdown"].includes(document.file.extension)
|
|
26
|
+
? markdownFenceSpans(document.text)
|
|
27
|
+
: [];
|
|
14
28
|
let chunkIndex = 0;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
29
|
+
for (const region of regions) {
|
|
30
|
+
let cursor = region.charStart;
|
|
31
|
+
while (cursor < region.charEnd) {
|
|
32
|
+
const end = chooseChunkEnd(document.text, cursor, chunkSize, region.charEnd, fences);
|
|
33
|
+
const span = trimmedSpan(document.text, cursor, end);
|
|
34
|
+
if (span.text) {
|
|
35
|
+
const id = createHash("sha256")
|
|
36
|
+
.update(`${document.file.relativePath}:${chunkIndex}:${region.contextPath}:${span.text}`)
|
|
37
|
+
.digest("hex");
|
|
38
|
+
chunks.push({
|
|
39
|
+
id,
|
|
40
|
+
source: document.file.source,
|
|
41
|
+
relativePath: document.file.relativePath,
|
|
42
|
+
chunkIndex,
|
|
43
|
+
contextPath: region.contextPath,
|
|
44
|
+
text: span.text,
|
|
45
|
+
charStart: span.start,
|
|
46
|
+
charEnd: span.end,
|
|
47
|
+
lineStart: lineNumberForOffset(lineStarts, span.start),
|
|
48
|
+
lineEnd: lineNumberForOffset(lineStarts, Math.max(span.start, span.end - 1)),
|
|
49
|
+
...pageRangeForSpan(document, span.start, span.end),
|
|
50
|
+
checksum: document.file.checksum,
|
|
51
|
+
bytes: document.file.bytes,
|
|
52
|
+
mtimeMs: document.file.mtimeMs,
|
|
53
|
+
});
|
|
54
|
+
chunkIndex += 1;
|
|
55
|
+
}
|
|
56
|
+
if (end >= region.charEnd) {
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
cursor = fences.some((fence) => fence.start === end)
|
|
60
|
+
? end
|
|
61
|
+
: Math.max(end - chunkOverlap, cursor + 1, region.charStart);
|
|
38
62
|
}
|
|
39
|
-
if (end >= document.text.length) {
|
|
40
|
-
break;
|
|
41
|
-
}
|
|
42
|
-
cursor = Math.max(end - chunkOverlap, cursor + 1);
|
|
43
63
|
}
|
|
44
64
|
return chunks;
|
|
45
65
|
}
|
|
66
|
+
export function chunkSearchText(chunk) {
|
|
67
|
+
return chunk.contextPath ? `${chunk.contextPath}\n${chunk.text}` : chunk.text;
|
|
68
|
+
}
|
|
46
69
|
function pageRangeForSpan(document, start, end) {
|
|
47
70
|
if (!document.pages || document.pages.length === 0) {
|
|
48
71
|
return {};
|
|
@@ -52,11 +75,15 @@ function pageRangeForSpan(document, start, end) {
|
|
|
52
75
|
const last = overlapping.at(-1);
|
|
53
76
|
return first && last ? { pageStart: first.pageNumber, pageEnd: last.pageNumber } : {};
|
|
54
77
|
}
|
|
55
|
-
function chooseChunkEnd(text, cursor, chunkSize) {
|
|
56
|
-
const hardEnd = Math.min(cursor + chunkSize,
|
|
57
|
-
if (hardEnd ===
|
|
78
|
+
function chooseChunkEnd(text, cursor, chunkSize, regionEnd, fences) {
|
|
79
|
+
const hardEnd = Math.min(cursor + chunkSize, regionEnd);
|
|
80
|
+
if (hardEnd === regionEnd) {
|
|
58
81
|
return hardEnd;
|
|
59
82
|
}
|
|
83
|
+
const intersectedFence = fences.find((fence) => fence.start < hardEnd && fence.end > hardEnd && fence.end <= regionEnd);
|
|
84
|
+
if (intersectedFence && intersectedFence.end - intersectedFence.start <= chunkSize) {
|
|
85
|
+
return intersectedFence.start > cursor ? intersectedFence.start : intersectedFence.end;
|
|
86
|
+
}
|
|
60
87
|
const window = text.slice(cursor, hardEnd);
|
|
61
88
|
const paragraphBreak = window.lastIndexOf("\n\n");
|
|
62
89
|
if (paragraphBreak > chunkSize * PARAGRAPH_BREAK_MIN_RATIO) {
|
package/dist/chunking.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AAG7E,MAAM,yBAAyB,GAAG,IAAI,CAAA;AACtC,MAAM,wBAAwB,GAAG,IAAI,CAAA;AACrC,MAAM,oBAAoB,GAAG,IAAI,CAAA;AACjC,MAAM,0BAA0B,GAAG,IAAI,CAAA;AACvC,MAAM,mBAAmB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE7D,MAAM,UAAU,aAAa,CAC3B,QAAwB,EACxB,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IAClD,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IACrF,MAAM,OAAO,GACX,UAAU,CAAC,MAAM,GAAG,CAAC;QACnB,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC;YACE;gBACE,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;gBAC7B,WAAW,EAAE,EAAE;gBACf,IAAI,EAAE,kBAA2B;aAClC;SACF,CAAA;IACP,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3E,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC;QACnC,CAAC,CAAC,EAAE,CAAA;IACN,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,GAAG,MAAM,CAAC,SAAS,CAAA;QAC7B,OAAO,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YACpF,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;YAEpD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC;qBAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;qBACxF,MAAM,CAAC,KAAK,CAAC,CAAA;gBAChB,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE;oBACF,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;oBAC5B,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY;oBACxC,UAAU;oBACV,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,KAAK;oBACrB,OAAO,EAAE,IAAI,CAAC,GAAG;oBACjB,SAAS,EAAE,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC;oBACtD,OAAO,EAAE,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;oBAC5E,GAAG,gBAAgB,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;oBACnD,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ;oBAChC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK;oBAC1B,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,UAAU,IAAI,CAAC,CAAA;YACjB,CAAC;YAED,IAAI,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC1B,MAAK;YACP,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,GAAG,CAAC;gBAClD,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,EAAE,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAA;QAChE,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAA8C;IAC5E,OAAO,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;AAC/E,CAAC;AAED,SAAS,gBAAgB,CACvB,QAAwB,EACxB,KAAa,EACb,GAAW;IAEX,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,CAAA;IACX,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,CAAA;IACjG,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;IAC5B,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/B,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;AACvF,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,MAAc,EACd,SAAiB,EACjB,SAAiB,EACjB,MAA6C;IAE7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,SAAS,CAAC,CAAA;IACvD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAClC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,IAAI,KAAK,CAAC,GAAG,GAAG,OAAO,IAAI,KAAK,CAAC,GAAG,IAAI,SAAS,CAClF,CAAA;IACD,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,GAAG,gBAAgB,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;QACnF,OAAO,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAA;IACxF,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC1C,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;IACjD,IAAI,cAAc,GAAG,SAAS,GAAG,yBAAyB,EAAE,CAAC;QAC3D,OAAO,MAAM,GAAG,cAAc,CAAA;IAChC,CAAC;IAED,MAAM,gBAAgB,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IACrD,IAAI,gBAAgB,GAAG,SAAS,GAAG,wBAAwB,EAAE,CAAC;QAC5D,OAAO,MAAM,GAAG,gBAAgB,CAAA;IAClC,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;IAC1C,IAAI,SAAS,GAAG,SAAS,GAAG,oBAAoB,EAAE,CAAC;QACjD,OAAO,MAAM,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAC1C,IAAI,UAAU,GAAG,SAAS,GAAG,0BAA0B,EAAE,CAAC;QACxD,OAAO,MAAM,GAAG,UAAU,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAA;IACZ,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACxC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/F,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAQD,SAAS,WAAW,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW;IAC3D,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,IAAI,UAAU,GAAG,GAAG,CAAA;IACpB,OAAO,YAAY,GAAG,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACzE,YAAY,IAAI,CAAC,CAAA;IACnB,CAAC;IACD,OAAO,UAAU,GAAG,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC3E,UAAU,IAAI,CAAC,CAAA;IACjB,CAAC;IACD,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,GAAG,EAAE,UAAU;QACf,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC;KAC3C,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;IAClB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,UAAoB,EAAE,MAAc;IAC/D,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,IAAI,IAAI,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA;IAChC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;YACxB,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;QACf,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;QAChB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAA;AAC9B,CAAC"}
|
package/dist/cli.js
CHANGED
|
@@ -599,6 +599,7 @@ program
|
|
|
599
599
|
redactionEnabled: config.redaction.enabled,
|
|
600
600
|
accessLog: config.accessLog,
|
|
601
601
|
mcpMaxTopK: config.mcpMaxTopK,
|
|
602
|
+
mcpMaxOutputBytes: config.mcpMaxOutputBytes,
|
|
602
603
|
topK: config.topK,
|
|
603
604
|
chunkSize: config.chunkSize,
|
|
604
605
|
chunkOverlap: config.chunkOverlap,
|
|
@@ -630,6 +631,7 @@ program
|
|
|
630
631
|
console.log(`redactionEnabled=${config.redaction.enabled}`);
|
|
631
632
|
console.log(`accessLog=${config.accessLog}`);
|
|
632
633
|
console.log(`mcpMaxTopK=${config.mcpMaxTopK}`);
|
|
634
|
+
console.log(`mcpMaxOutputBytes=${config.mcpMaxOutputBytes}`);
|
|
633
635
|
console.log(`topK=${config.topK}`);
|
|
634
636
|
console.log(`chunkSize=${config.chunkSize}`);
|
|
635
637
|
console.log(`chunkOverlap=${config.chunkOverlap}`);
|
|
@@ -669,6 +671,7 @@ program
|
|
|
669
671
|
console.log(`accessLogStoresRawQueries=${report.accessLog.storesRawQueries}`);
|
|
670
672
|
console.log(`storageGitIgnored=${report.storage.gitIgnored}`);
|
|
671
673
|
console.log(`mcpMaxTopK=${report.mcp.maxTopK}`);
|
|
674
|
+
console.log(`mcpMaxOutputBytes=${report.mcp.maxOutputBytes}`);
|
|
672
675
|
console.log(`mcpDestructiveToolsExposed=${report.mcp.destructiveToolsExposed}`);
|
|
673
676
|
for (const warning of report.warnings) {
|
|
674
677
|
console.log(pc.yellow(`warning: ${warning}`));
|