@jcode.labs/ragmir 2.17.2 → 2.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +53 -43
  2. package/dist/access-log.d.ts +10 -0
  3. package/dist/access-log.d.ts.map +1 -1
  4. package/dist/access-log.js +119 -49
  5. package/dist/access-log.js.map +1 -1
  6. package/dist/chunking.d.ts +5 -1
  7. package/dist/chunking.d.ts.map +1 -1
  8. package/dist/chunking.js +102 -16
  9. package/dist/chunking.js.map +1 -1
  10. package/dist/citation.d.ts +18 -0
  11. package/dist/citation.d.ts.map +1 -0
  12. package/dist/citation.js +54 -0
  13. package/dist/citation.js.map +1 -0
  14. package/dist/cli-entry.d.ts +3 -0
  15. package/dist/cli-entry.d.ts.map +1 -0
  16. package/dist/cli-entry.js +16 -0
  17. package/dist/cli-entry.js.map +1 -0
  18. package/dist/cli-fast.d.ts +13 -0
  19. package/dist/cli-fast.d.ts.map +1 -0
  20. package/dist/cli-fast.js +145 -0
  21. package/dist/cli-fast.js.map +1 -0
  22. package/dist/cli-options.d.ts +3 -0
  23. package/dist/cli-options.d.ts.map +1 -1
  24. package/dist/cli-options.js +7 -0
  25. package/dist/cli-options.js.map +1 -1
  26. package/dist/cli-stdin.d.ts +3 -0
  27. package/dist/cli-stdin.d.ts.map +1 -0
  28. package/dist/cli-stdin.js +15 -0
  29. package/dist/cli-stdin.js.map +1 -0
  30. package/dist/cli.js +251 -44
  31. package/dist/cli.js.map +1 -1
  32. package/dist/client.d.ts +13 -2
  33. package/dist/client.d.ts.map +1 -1
  34. package/dist/client.js +155 -10
  35. package/dist/client.js.map +1 -1
  36. package/dist/config.d.ts +6 -0
  37. package/dist/config.d.ts.map +1 -1
  38. package/dist/config.js +227 -49
  39. package/dist/config.js.map +1 -1
  40. package/dist/context-resources.d.ts +4 -2
  41. package/dist/context-resources.d.ts.map +1 -1
  42. package/dist/context-resources.js +55 -24
  43. package/dist/context-resources.js.map +1 -1
  44. package/dist/defaults.d.ts +27 -0
  45. package/dist/defaults.d.ts.map +1 -1
  46. package/dist/defaults.js +42 -2
  47. package/dist/defaults.js.map +1 -1
  48. package/dist/destroy.d.ts +2 -2
  49. package/dist/destroy.d.ts.map +1 -1
  50. package/dist/destroy.js +34 -20
  51. package/dist/destroy.js.map +1 -1
  52. package/dist/doctor.d.ts +3 -2
  53. package/dist/doctor.d.ts.map +1 -1
  54. package/dist/doctor.js +86 -41
  55. package/dist/doctor.js.map +1 -1
  56. package/dist/durable-file.d.ts +13 -0
  57. package/dist/durable-file.d.ts.map +1 -0
  58. package/dist/durable-file.js +56 -0
  59. package/dist/durable-file.js.map +1 -0
  60. package/dist/embeddings.d.ts +14 -3
  61. package/dist/embeddings.d.ts.map +1 -1
  62. package/dist/embeddings.js +233 -95
  63. package/dist/embeddings.js.map +1 -1
  64. package/dist/errors.d.ts +1 -1
  65. package/dist/errors.d.ts.map +1 -1
  66. package/dist/errors.js.map +1 -1
  67. package/dist/evaluate.d.ts +4 -3
  68. package/dist/evaluate.d.ts.map +1 -1
  69. package/dist/evaluate.js +611 -110
  70. package/dist/evaluate.js.map +1 -1
  71. package/dist/files.d.ts +4 -1
  72. package/dist/files.d.ts.map +1 -1
  73. package/dist/files.js +119 -33
  74. package/dist/files.js.map +1 -1
  75. package/dist/generation-retention.d.ts +52 -0
  76. package/dist/generation-retention.d.ts.map +1 -0
  77. package/dist/generation-retention.js +318 -0
  78. package/dist/generation-retention.js.map +1 -0
  79. package/dist/index-diagnostics.d.ts +3 -2
  80. package/dist/index-diagnostics.d.ts.map +1 -1
  81. package/dist/index-diagnostics.js +32 -3
  82. package/dist/index-diagnostics.js.map +1 -1
  83. package/dist/index-policy.d.ts.map +1 -1
  84. package/dist/index-policy.js +3 -0
  85. package/dist/index-policy.js.map +1 -1
  86. package/dist/index-write-lock.d.ts +16 -1
  87. package/dist/index-write-lock.d.ts.map +1 -1
  88. package/dist/index-write-lock.js +392 -13
  89. package/dist/index-write-lock.js.map +1 -1
  90. package/dist/index.d.ts +12 -4
  91. package/dist/index.d.ts.map +1 -1
  92. package/dist/index.js +5 -2
  93. package/dist/index.js.map +1 -1
  94. package/dist/ingest.d.ts +11 -2
  95. package/dist/ingest.d.ts.map +1 -1
  96. package/dist/ingest.js +680 -165
  97. package/dist/ingest.js.map +1 -1
  98. package/dist/ingestion-metrics.d.ts +58 -0
  99. package/dist/ingestion-metrics.d.ts.map +1 -0
  100. package/dist/ingestion-metrics.js +216 -0
  101. package/dist/ingestion-metrics.js.map +1 -0
  102. package/dist/ingestion-state.d.ts +11 -2
  103. package/dist/ingestion-state.d.ts.map +1 -1
  104. package/dist/ingestion-state.js +480 -49
  105. package/dist/ingestion-state.js.map +1 -1
  106. package/dist/limits.d.ts.map +1 -1
  107. package/dist/limits.js +11 -1
  108. package/dist/limits.js.map +1 -1
  109. package/dist/mcp-output.d.ts +44 -4
  110. package/dist/mcp-output.d.ts.map +1 -1
  111. package/dist/mcp-output.js +132 -159
  112. package/dist/mcp-output.js.map +1 -1
  113. package/dist/mcp-summaries.d.ts +32 -0
  114. package/dist/mcp-summaries.d.ts.map +1 -0
  115. package/dist/mcp-summaries.js +227 -0
  116. package/dist/mcp-summaries.js.map +1 -0
  117. package/dist/mcp.d.ts +1 -1
  118. package/dist/mcp.d.ts.map +1 -1
  119. package/dist/mcp.js +85 -46
  120. package/dist/mcp.js.map +1 -1
  121. package/dist/ocr-cache.d.ts +15 -0
  122. package/dist/ocr-cache.d.ts.map +1 -0
  123. package/dist/ocr-cache.js +90 -0
  124. package/dist/ocr-cache.js.map +1 -0
  125. package/dist/ocr.d.ts +31 -0
  126. package/dist/ocr.d.ts.map +1 -1
  127. package/dist/ocr.js +193 -17
  128. package/dist/ocr.js.map +1 -1
  129. package/dist/parsing.d.ts.map +1 -1
  130. package/dist/parsing.js +482 -85
  131. package/dist/parsing.js.map +1 -1
  132. package/dist/permissions.js +2 -2
  133. package/dist/permissions.js.map +1 -1
  134. package/dist/preview.d.ts.map +1 -1
  135. package/dist/preview.js +10 -11
  136. package/dist/preview.js.map +1 -1
  137. package/dist/quality-report.d.ts +12 -0
  138. package/dist/quality-report.d.ts.map +1 -0
  139. package/dist/quality-report.js +146 -0
  140. package/dist/quality-report.js.map +1 -0
  141. package/dist/query.d.ts +7 -4
  142. package/dist/query.d.ts.map +1 -1
  143. package/dist/query.js +371 -284
  144. package/dist/query.js.map +1 -1
  145. package/dist/ranking.d.ts +38 -0
  146. package/dist/ranking.d.ts.map +1 -0
  147. package/dist/ranking.js +278 -0
  148. package/dist/ranking.js.map +1 -0
  149. package/dist/redaction.d.ts +6 -1
  150. package/dist/redaction.d.ts.map +1 -1
  151. package/dist/redaction.js +118 -4
  152. package/dist/redaction.js.map +1 -1
  153. package/dist/research.d.ts +6 -1
  154. package/dist/research.d.ts.map +1 -1
  155. package/dist/research.js +330 -130
  156. package/dist/research.js.map +1 -1
  157. package/dist/runtime-info.d.ts +3 -0
  158. package/dist/runtime-info.d.ts.map +1 -0
  159. package/dist/runtime-info.js +81 -0
  160. package/dist/runtime-info.js.map +1 -0
  161. package/dist/security.d.ts +3 -2
  162. package/dist/security.d.ts.map +1 -1
  163. package/dist/security.js +242 -45
  164. package/dist/security.js.map +1 -1
  165. package/dist/semantic-config.d.ts +7 -1
  166. package/dist/semantic-config.d.ts.map +1 -1
  167. package/dist/semantic-config.js +30 -4
  168. package/dist/semantic-config.js.map +1 -1
  169. package/dist/setup.js +6 -3
  170. package/dist/setup.js.map +1 -1
  171. package/dist/skill.js +34 -12
  172. package/dist/skill.js.map +1 -1
  173. package/dist/source-fingerprint-cache.d.ts +20 -0
  174. package/dist/source-fingerprint-cache.d.ts.map +1 -0
  175. package/dist/source-fingerprint-cache.js +133 -0
  176. package/dist/source-fingerprint-cache.js.map +1 -0
  177. package/dist/storage-maintenance.d.ts +46 -0
  178. package/dist/storage-maintenance.d.ts.map +1 -0
  179. package/dist/storage-maintenance.js +329 -0
  180. package/dist/storage-maintenance.js.map +1 -0
  181. package/dist/store.d.ts +39 -2
  182. package/dist/store.d.ts.map +1 -1
  183. package/dist/store.js +559 -24
  184. package/dist/store.js.map +1 -1
  185. package/dist/text.d.ts +1 -1
  186. package/dist/text.d.ts.map +1 -1
  187. package/dist/text.js +12 -2
  188. package/dist/text.js.map +1 -1
  189. package/dist/types.d.ts +414 -5
  190. package/dist/types.d.ts.map +1 -1
  191. package/dist/vector-index.d.ts +42 -0
  192. package/dist/vector-index.d.ts.map +1 -0
  193. package/dist/vector-index.js +249 -0
  194. package/dist/vector-index.js.map +1 -0
  195. package/dist/version.d.ts +1 -1
  196. package/dist/version.js +1 -1
  197. package/dist/workload.d.ts +22 -0
  198. package/dist/workload.d.ts.map +1 -0
  199. package/dist/workload.js +181 -0
  200. package/dist/workload.js.map +1 -0
  201. package/examples/document-evidence-benchmark/README.md +25 -63
  202. package/examples/document-evidence-benchmark/golden-queries.json +1 -1
  203. package/examples/library-api-demo/README.md +17 -57
  204. package/examples/sovereign-rag-demo/README.md +37 -97
  205. package/package.json +36 -11
package/README.md CHANGED
@@ -5,21 +5,18 @@
5
5
  [![Node.js](https://img.shields.io/node/v/@jcode.labs/ragmir)](https://www.npmjs.com/package/@jcode.labs/ragmir)
6
6
  [![MIT](https://img.shields.io/npm/l/@jcode.labs/ragmir)](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE)
7
7
 
8
- Confidential local RAG for coding agents and Node.js applications.
9
-
10
- *Stop sending confidential documents directly to the cloud.*
11
-
12
- Core indexes the project files you select and retrieves bounded, cited evidence offline by default.
13
- It does not upload the corpus, call an LLM, or open an HTTP port.
8
+ Confidential local RAG for coding agents and Node.js applications. Core indexes the project files
9
+ you choose and retrieves bounded, cited evidence offline by default. It uploads no corpus, calls no
10
+ LLM, and opens no HTTP port.
14
11
 
15
12
  [Project overview](https://github.com/jcode-works/jcode-ragmir#readme) ·
16
- [CLI reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md) ·
17
- [API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md) ·
13
+ [CLI](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md) ·
14
+ [API](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md) ·
18
15
  [Agent integration](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/agent-integration.md)
19
16
 
20
17
  ## Install and retrieve
21
18
 
22
- Requires Node.js 20 or later.
19
+ Requires Node.js 22 or later.
23
20
 
24
21
  ```bash
25
22
  npm install --save-dev @jcode.labs/ragmir
@@ -29,21 +26,22 @@ npx rgr ingest
29
26
  npx rgr search "Which decision changed the rollout?"
30
27
  ```
31
28
 
32
- Setup keeps configuration, the index, helpers, reports, and metadata-only access logs under ignored
33
- `.ragmir/` state. Ingestion is incremental and resumable. Run `npx rgr audit --unsupported` to
34
- compare the selected files with the index and see what was skipped.
29
+ Generated configuration, indexes, helpers, reports, and metadata-only logs stay under ignored
30
+ `.ragmir/` state. Ingestion is incremental, resumable, bounded by source, chunk, vector, file, batch,
31
+ and concurrency windows, and serialized across local writer processes.
35
32
 
36
33
  ## Choose an interface
37
34
 
38
- | Interface | Use |
35
+ | Interface | Use it for |
39
36
  | --- | --- |
40
- | `rgr` CLI | Setup, ingest, search, audit, and JSON automation |
41
- | TypeScript API | Typed retrieval in a script or stateful Node.js worker |
42
- | Local stdio MCP | Bounded context for compatible coding agents |
37
+ | `rgr` CLI | Setup, ingest, search, audit, maintenance, and JSON automation |
38
+ | TypeScript API | Typed retrieval in scripts and long-running Node.js workers |
39
+ | Local stdio MCP | Bounded, read-focused context for compatible agents |
43
40
 
44
- The default `local-hash` provider needs no model download. It is lexical/hash retrieval, not
45
- semantic embeddings. Enable local Transformers.js embeddings explicitly with `rgr setup --semantic`
46
- and rebuild the index.
41
+ The default `local-hash` provider works offline with no model download. Enable semantic
42
+ Transformers.js embeddings explicitly with `rgr setup --semantic`, then rebuild. Core remains
43
+ retrieval-only in both modes. Use `explain: true` or `--explain` to inspect lexical and vector
44
+ contributions, fallback decisions, budgets, queue wait, and the active ranking policy.
47
45
 
48
46
  ## TypeScript API
49
47
 
@@ -58,37 +56,49 @@ try {
58
56
  timeoutMs: 10_000,
59
57
  })
60
58
 
61
- for (const result of results) {
62
- console.log(result.citation, result.text)
63
- }
59
+ for (const result of results) console.log(result.citation, result.text)
64
60
  } finally {
65
61
  await ragmir.close()
66
62
  }
67
63
  ```
68
64
 
69
- Reuse one client per project root in long-running processes and close it during shutdown. Top-level
70
- `ingest`, `search`, `ask`, and `research` functions remain available for one-shot scripts. `ask`
71
- returns cited retrieval context, not a generated answer.
72
-
73
- Core also exports setup, preview, audit, evaluation, source-routing, privacy, OCR, MCP, and agent
74
- integration helpers. See the [complete API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md)
75
- for public options and result types.
76
-
77
- ## Optional packages and boundaries
78
-
79
- - [`@jcode.labs/ragmir-chat`](https://www.npmjs.com/package/@jcode.labs/ragmir-chat) adds local GGUF
80
- answer generation from retrieved passages.
65
+ Reuse one client per project root in a long-running process. It owns the local connection, read
66
+ snapshot, active operation lifecycle, metadata-only log flush, and optional embedding-model lease.
67
+ Top-level `ingest`, `search`, `ask`, and `research` functions remain available for one-shot scripts.
68
+ `ask` returns cited context, not generated prose.
69
+
70
+ ## Guarantees and boundaries
71
+
72
+ - Citations use source lines only for line-preserving text, plus PDF pages, PPTX slides, XLSX
73
+ sheets and cells, and EPUB spine positions.
74
+ - Rebuilds activate only after row and manifest validation. Interrupted rebuilds leave the previous
75
+ searchable generation active.
76
+ - Exact vector search remains the policy below 100,000 rows. Larger tables use quality-gated IVF-PQ
77
+ with complete coverage and an exact diagnostic mode.
78
+ - Search, embedding, and ingestion use independent bounded queues. Overload and queue deadlines are
79
+ stable retryable errors.
80
+ - Optional OCR processes only blank PDF pages through a configured local executable and private
81
+ resumable cache.
82
+ - `rgr status` and normal `rgr doctor` read compact manifest health. Use `rgr doctor --deep` or
83
+ `rgr audit` for a live source inventory.
84
+ - `rgr security-audit` checks permissions, Git ignore coverage, tracked private paths, redaction,
85
+ and local extractor authority.
86
+
87
+ The [CLI reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/cli-reference.md),
88
+ [API reference](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/api-reference.md), and
89
+ [configuration guide](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md)
90
+ hold the complete options and operational detail.
91
+
92
+ ## Optional packages
93
+
94
+ - [`@jcode.labs/ragmir-chat`](https://www.npmjs.com/package/@jcode.labs/ragmir-chat) adds cited
95
+ answer generation with a verified local GGUF profile.
81
96
  - [`@jcode.labs/ragmir-tts`](https://www.npmjs.com/package/@jcode.labs/ragmir-tts) renders reviewed
82
97
  text as local audio or explicit online speech.
83
98
 
84
- Core installs and starts without either add-on. A hosted agent receives only the passages your
85
- integration gives it, under that provider's data policy. Use a local consumer when no passage may
86
- leave the workstation. Ragmir has no hosted storage or cloud-sync layer; teams synchronize source
87
- files separately and build one local index per developer.
88
-
89
- Read the [configuration guide](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/configuration.md),
90
- [security hardening guide](https://github.com/jcode-works/jcode-ragmir/blob/main/SECURITY-HARDENING.md),
91
- and [troubleshooting guide](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/troubleshooting.md)
92
- before indexing sensitive material.
99
+ Core installs and starts without either add-on. A hosted agent receives only passages your
100
+ integration sends under that provider's data policy; use a local consumer when no passage may leave
101
+ the workstation. Teams synchronize their source folder and tracked configuration, then build one
102
+ local index per developer. See the [team workflow](https://github.com/jcode-works/jcode-ragmir/blob/main/docs/agent-integration.md#team-knowledge-bases).
93
103
 
94
104
  Ragmir Core is open source under the [MIT License](https://github.com/jcode-works/jcode-ragmir/blob/main/LICENSE).
@@ -14,7 +14,17 @@ export interface McpOutputLogEvent {
14
14
  truncated: boolean;
15
15
  }
16
16
  export declare const MAX_USAGE_REPORT_DAYS = 3650;
17
+ export declare const MAX_ACCESS_LOG_PENDING_EVENTS = 1024;
18
+ export interface AccessLogWriterMetrics {
19
+ pendingEvents: number;
20
+ inFlightEvents: number;
21
+ writtenEvents: number;
22
+ droppedEvents: number;
23
+ }
17
24
  export declare function recordAccess(config: Config, event: AccessLogEvent): Promise<void>;
18
25
  export declare function recordMcpOutput(config: Config, event: McpOutputLogEvent): Promise<void>;
26
+ export declare function flushAccessLog(config: Config): Promise<AccessLogWriterMetrics>;
27
+ export declare function accessLogWriterMetrics(config: Config): AccessLogWriterMetrics;
19
28
  export declare function accessLogUsageReport(options?: AccessLogUsageOptions): Promise<AccessLogUsageReport>;
29
+ export declare function accessLogUsageReportWithConfig(config: Config, options?: AccessLogUsageOptions): Promise<AccessLogUsageReport>;
20
30
  //# sourceMappingURL=access-log.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,EACN,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AA2CD,eAAO,MAAM,qBAAqB,OAAQ,CAAA;AAa1C,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBvF;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA2B7F;AA8CD,wBAAsB,oBAAoB,CACxC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAiG/B"}
1
+ {"version":3,"file":"access-log.d.ts","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,EACN,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AA2CD,eAAO,MAAM,qBAAqB,OAAQ,CAAA;AAW1C,eAAO,MAAM,6BAA6B,OAAQ,CAAA;AAgBlD,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;CACtB;AAKD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjF;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAevF;AAED,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAYpF;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,sBAAsB,CAU7E;AA6FD,wBAAsB,oBAAoB,CACxC,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAK/B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAiG/B"}
@@ -33,65 +33,112 @@ const MAX_ACCESS_LOG_BYTES = 10 * 1024 * 1024;
33
33
  /** Number of most recent lines retained when the log exceeds the byte cap. */
34
34
  const TRIMMED_ACCESS_LOG_LINES = 50_000;
35
35
  const ACCESS_LOG_SALT_FILE = ".ragmir-access-log.salt";
36
- const accessLogWriteQueues = new Map();
37
- export async function recordAccess(config, event) {
36
+ export const MAX_ACCESS_LOG_PENDING_EVENTS = 1_024;
37
+ const ACCESS_LOG_WRITE_BATCH_SIZE = 64;
38
+ const accessLogWriters = new Map();
39
+ const accessLogSalts = new Map();
40
+ export function recordAccess(config, event) {
38
41
  if (!config.accessLog) {
39
- return;
40
- }
41
- try {
42
- await withAccessLogWriteQueue(config.accessLogPath, async () => {
43
- await ensurePrivateDirectory(path.dirname(config.accessLogPath));
44
- await trimAccessLogIfNeeded(config.accessLogPath);
45
- await appendFile(config.accessLogPath, `${JSON.stringify(await toLogLine(config, event))}\n`, {
46
- encoding: "utf8",
47
- mode: 0o600,
48
- });
49
- await hardenPrivateFile(config.accessLogPath);
50
- });
51
- }
52
- catch {
53
- // Access logging is best-effort so read-only workspaces do not block local use.
42
+ return Promise.resolve();
54
43
  }
44
+ return enqueueAccessLogLine(config.accessLogPath, async () => JSON.stringify(await toLogLine(config, event)));
55
45
  }
56
- export async function recordMcpOutput(config, event) {
46
+ export function recordMcpOutput(config, event) {
57
47
  if (!config.accessLog) {
58
- return;
48
+ return Promise.resolve();
59
49
  }
60
- try {
61
- await withAccessLogWriteQueue(config.accessLogPath, async () => {
62
- await ensurePrivateDirectory(path.dirname(config.accessLogPath));
63
- await trimAccessLogIfNeeded(config.accessLogPath);
64
- const line = {
65
- timestamp: new Date().toISOString(),
66
- kind: "mcp-output",
67
- tool: event.tool,
68
- retrievedBytes: event.retrievedBytes,
69
- returnedBytes: event.returnedBytes,
70
- compacted: event.compacted,
71
- truncated: event.truncated,
72
- };
73
- await appendFile(config.accessLogPath, `${JSON.stringify(line)}\n`, {
74
- encoding: "utf8",
75
- mode: 0o600,
76
- });
77
- await hardenPrivateFile(config.accessLogPath);
78
- });
50
+ return enqueueAccessLogLine(config.accessLogPath, async () => JSON.stringify({
51
+ timestamp: new Date().toISOString(),
52
+ kind: "mcp-output",
53
+ tool: event.tool,
54
+ retrievedBytes: event.retrievedBytes,
55
+ returnedBytes: event.returnedBytes,
56
+ compacted: event.compacted,
57
+ truncated: event.truncated,
58
+ }));
59
+ }
60
+ export async function flushAccessLog(config) {
61
+ const state = accessLogWriters.get(config.accessLogPath);
62
+ if (!state) {
63
+ return emptyAccessLogWriterMetrics();
79
64
  }
80
- catch {
81
- // Output metrics are best-effort and must never block local retrieval.
65
+ while (state.active || state.pending.length > 0) {
66
+ startAccessLogWriter(config.accessLogPath, state);
67
+ if (state.active) {
68
+ await state.active;
69
+ }
82
70
  }
71
+ return accessLogWriterMetrics(config);
83
72
  }
84
- function withAccessLogWriteQueue(accessLogPath, operation) {
85
- const previous = accessLogWriteQueues.get(accessLogPath) ?? Promise.resolve();
86
- const current = previous.then(operation);
87
- const settled = current.then(() => undefined, () => undefined);
88
- accessLogWriteQueues.set(accessLogPath, settled);
89
- void settled.then(() => {
90
- if (accessLogWriteQueues.get(accessLogPath) === settled) {
91
- accessLogWriteQueues.delete(accessLogPath);
73
+ export function accessLogWriterMetrics(config) {
74
+ const state = accessLogWriters.get(config.accessLogPath);
75
+ return state
76
+ ? {
77
+ pendingEvents: state.pending.length,
78
+ inFlightEvents: state.inFlightEvents,
79
+ writtenEvents: state.writtenEvents,
80
+ droppedEvents: state.droppedEvents,
92
81
  }
82
+ : emptyAccessLogWriterMetrics();
83
+ }
84
+ function enqueueAccessLogLine(accessLogPath, render) {
85
+ const state = accessLogWriters.get(accessLogPath) ?? {
86
+ pending: [],
87
+ active: null,
88
+ inFlightEvents: 0,
89
+ writtenEvents: 0,
90
+ droppedEvents: 0,
91
+ };
92
+ accessLogWriters.set(accessLogPath, state);
93
+ if (state.pending.length >= MAX_ACCESS_LOG_PENDING_EVENTS) {
94
+ state.droppedEvents += 1;
95
+ return Promise.resolve();
96
+ }
97
+ return new Promise((settle) => {
98
+ state.pending.push({ render, settle });
99
+ startAccessLogWriter(accessLogPath, state);
93
100
  });
94
- return current;
101
+ }
102
+ function startAccessLogWriter(accessLogPath, state) {
103
+ if (state.active || state.pending.length === 0) {
104
+ return;
105
+ }
106
+ const batch = state.pending.splice(0, ACCESS_LOG_WRITE_BATCH_SIZE);
107
+ state.inFlightEvents = batch.length;
108
+ const active = writeAccessLogBatch(accessLogPath, batch)
109
+ .then(() => {
110
+ state.writtenEvents += batch.length;
111
+ })
112
+ .catch(() => {
113
+ state.droppedEvents += batch.length;
114
+ })
115
+ .finally(() => {
116
+ for (const entry of batch) {
117
+ entry.settle();
118
+ }
119
+ state.inFlightEvents = 0;
120
+ state.active = null;
121
+ startAccessLogWriter(accessLogPath, state);
122
+ });
123
+ state.active = active;
124
+ }
125
+ async function writeAccessLogBatch(accessLogPath, batch) {
126
+ await ensurePrivateDirectory(path.dirname(accessLogPath));
127
+ await trimAccessLogIfNeeded(accessLogPath);
128
+ const lines = await Promise.all(batch.map((entry) => entry.render()));
129
+ await appendFile(accessLogPath, `${lines.join("\n")}\n`, {
130
+ encoding: "utf8",
131
+ mode: 0o600,
132
+ });
133
+ await hardenPrivateFile(accessLogPath);
134
+ }
135
+ function emptyAccessLogWriterMetrics() {
136
+ return {
137
+ pendingEvents: 0,
138
+ inFlightEvents: 0,
139
+ writtenEvents: 0,
140
+ droppedEvents: 0,
141
+ };
95
142
  }
96
143
  async function trimAccessLogIfNeeded(accessLogPath) {
97
144
  let size = 0;
@@ -119,6 +166,12 @@ export async function accessLogUsageReport(options = {}) {
119
166
  const signal = operationSignal(options);
120
167
  throwIfAborted(signal);
121
168
  const config = await loadConfig(String(options.cwd ?? process.cwd()));
169
+ return accessLogUsageReportWithConfig(config, options);
170
+ }
171
+ export async function accessLogUsageReportWithConfig(config, options = {}) {
172
+ const signal = operationSignal(options);
173
+ throwIfAborted(signal);
174
+ await flushAccessLog(config);
122
175
  throwIfAborted(signal);
123
176
  const days = normalizeUsageReportDays(options.days);
124
177
  const until = new Date();
@@ -249,6 +302,23 @@ async function hashQuery(config, query) {
249
302
  }
250
303
  async function accessLogSalt(config) {
251
304
  const saltPath = path.join(path.dirname(config.accessLogPath), ACCESS_LOG_SALT_FILE);
305
+ const cached = accessLogSalts.get(saltPath);
306
+ if (cached) {
307
+ return cached;
308
+ }
309
+ const loading = loadAccessLogSalt(saltPath);
310
+ accessLogSalts.set(saltPath, loading);
311
+ try {
312
+ return await loading;
313
+ }
314
+ catch (error) {
315
+ if (accessLogSalts.get(saltPath) === loading) {
316
+ accessLogSalts.delete(saltPath);
317
+ }
318
+ throw error;
319
+ }
320
+ }
321
+ async function loadAccessLogSalt(saltPath) {
252
322
  try {
253
323
  return await readFile(saltPath);
254
324
  }
@@ -1 +1 @@
1
- {"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAiD5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,gBAAgB,GAAoB;IACxC,eAAe;IACf,YAAY;IACZ,iBAAiB;IACjB,eAAe;CAChB,CAAA;AACD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AAC7D,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAA;AAC1C,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AACtD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAyB,CAAA;AAE7D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,uBAAuB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;YAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;YACjD,MAAM,UAAU,CACd,MAAM,CAAC,aAAa,EACpB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,EACrD;gBACE,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,KAAK;aACZ,CACF,CAAA;YACD,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc,EAAE,KAAwB;IAC5E,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,MAAM,uBAAuB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;YAChE,MAAM,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;YACjD,MAAM,IAAI,GAAqB;gBAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,IAAI,EAAE,YAAY;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;aAC3B,CAAA;YACD,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;gBAClE,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,KAAK;aACZ,CAAC,CAAA;YACF,MAAM,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;IACzE,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC9B,aAAqB,EACrB,SAA8B;IAE9B,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAA;IAC7E,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAC1B,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAA;IACD,oBAAoB,CAAC,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAA;IAChD,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;QACrB,IAAI,oBAAoB,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,OAAO,EAAE,CAAC;YACxD,oBAAoB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC,CAAC,CAAA;IACF,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,IAAI,OAAe,CAAA;QACnB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;QAC9E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,CAAA;QACb,CAAC;QACD,cAAc,CAAC,MAAM,CAAC,CAAA;QACtB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,CAAC,CAAA;gBACvB,iBAAiB,IAAI,KAAK,CAAC,cAAc,CAAA;gBACzC,gBAAgB,IAAI,KAAK,CAAC,aAAa,CAAA;gBACvC,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;oBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC/B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,gBAAgB;YAC/B,UAAU,EAAE,iBAAiB,GAAG,gBAAgB;YAChD,cAAc,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,iBAAiB;YACzF,kBAAkB;YAClB,kBAAkB;SACnB;QACD,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,qBAAqB,EAAE,CAAC;QAC7E,MAAM,IAAI,KAAK,CAAC,sDAAsD,qBAAqB,GAAG,CAAC,CAAA;IACjG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,MAAM,IAAI,KAAK;QACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAC9B,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnC,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC;QAC7C,oBAAoB,CAAC,KAAK,EAAE,eAAe,CAAC;QAC5C,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC;QAC9B,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAC/B,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,GAAW;IAC5C,OAAO,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,SAAS,CAAA;AACrD,CAAC"}
1
+ {"version":3,"file":"access-log.js","sourceRoot":"","sources":["../src/access-log.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACxE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAiD5E,MAAM,kBAAkB,GAAsB;IAC5C,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,UAAU;IACV,UAAU;IACV,eAAe;CAChB,CAAA;AACD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,kBAAkB,CAAC,CAAA;AACjE,MAAM,gBAAgB,GAAoB;IACxC,eAAe;IACf,YAAY;IACZ,iBAAiB;IACjB,eAAe;CAChB,CAAA;AACD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AAC7D,MAAM,yBAAyB,GAAG,CAAC,CAAA;AACnC,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAA;AAC1C,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAChD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AAC7C,8EAA8E;AAC9E,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,oBAAoB,GAAG,yBAAyB,CAAA;AACtD,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,CAAA;AAClD,MAAM,2BAA2B,GAAG,EAAE,CAAA;AAsBtC,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAgC,CAAA;AAChE,MAAM,cAAc,GAAG,IAAI,GAAG,EAA2B,CAAA;AAEzD,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,KAAqB;IAChE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,OAAO,oBAAoB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE,CAC3D,IAAI,CAAC,SAAS,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAC/C,CAAA;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,KAAwB;IACtE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,OAAO,oBAAoB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE,CAC3D,IAAI,CAAC,SAAS,CAAC;QACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;KACA,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAc;IACjD,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IACxD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,2BAA2B,EAAE,CAAA;IACtC,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,oBAAoB,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;QACjD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,KAAK,CAAC,MAAM,CAAA;QACpB,CAAC;IACH,CAAC;IACD,OAAO,sBAAsB,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;IACxD,OAAO,KAAK;QACV,CAAC,CAAC;YACE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;YACnC,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,aAAa,EAAE,KAAK,CAAC,aAAa;SACnC;QACH,CAAC,CAAC,2BAA2B,EAAE,CAAA;AACnC,CAAC;AAED,SAAS,oBAAoB,CAAC,aAAqB,EAAE,MAA6B;IAChF,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI;QACnD,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;KACjB,CAAA;IACD,gBAAgB,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,6BAA6B,EAAE,CAAC;QAC1D,KAAK,CAAC,aAAa,IAAI,CAAC,CAAA;QACxB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;QACtC,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,aAAqB,EAAE,KAA2B;IAC9E,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,OAAM;IACR,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAA;IAClE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,MAAM,CAAA;IACnC,MAAM,MAAM,GAAG,mBAAmB,CAAC,aAAa,EAAE,KAAK,CAAC;SACrD,IAAI,CAAC,GAAG,EAAE;QACT,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;IACrC,CAAC,CAAC;SACD,KAAK,CAAC,GAAG,EAAE;QACV,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;IACrC,CAAC,CAAC;SACD,OAAO,CAAC,GAAG,EAAE;QACZ,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,KAAK,CAAC,MAAM,EAAE,CAAA;QAChB,CAAC;QACD,KAAK,CAAC,cAAc,GAAG,CAAC,CAAA;QACxB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAA;QACnB,oBAAoB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IACJ,KAAK,CAAC,MAAM,GAAG,MAAM,CAAA;AACvB,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,aAAqB,EACrB,KAA6B;IAE7B,MAAM,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAA;IACzD,MAAM,qBAAqB,CAAC,aAAa,CAAC,CAAA;IAC1C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,UAAU,CAAC,aAAa,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QACvD,QAAQ,EAAE,MAAM;QAChB,IAAI,EAAE,KAAK;KACZ,CAAC,CAAA;IACF,MAAM,iBAAiB,CAAC,aAAa,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,2BAA2B;IAClC,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;KACjB,CAAA;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,aAAqB;IACxD,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAM;QACR,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,IAAI,IAAI,IAAI,oBAAoB,EAAE,CAAC;QACjC,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACnE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC,CAAA;IAC9E,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,OAAO,8BAA8B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,MAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IACvC,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,cAAc,CAAC,MAAM,CAAC,CAAA;IAC5B,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,oBAAoB,CAAC,CAAA;IACrE,MAAM,cAAc,GAAG,mBAAmB,EAAE,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,oBAAoB,GAAG,yBAAyB,EAAE,CAAA;IACxD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAA;IACxB,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,IAAI,WAAW,GAAkB,IAAI,CAAA;IAErC,IAAI,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,IAAI,OAAe,CAAA;QACnB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;QAC9E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,CAAA;QACb,CAAC;QACD,cAAc,CAAC,MAAM,CAAC,CAAA;QACtB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,cAAc,CAAC,MAAM,CAAC,CAAA;YACtB,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YACtC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAC7C,IACE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE;gBAC3B,SAAS,GAAG,KAAK,CAAC,OAAO,EAAE,EAC3B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,WAAW,IAAI,CAAC,CAAA;YAChB,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,kBAAkB,IAAI,CAAC,CAAA;gBACvB,iBAAiB,IAAI,KAAK,CAAC,cAAc,CAAA;gBACzC,gBAAgB,IAAI,KAAK,CAAC,aAAa,CAAA;gBACvC,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7C,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;oBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC/B,CAAC;gBACD,SAAQ;YACV,CAAC;YACD,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAClC,CAAC;YACD,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gBAAgB,IAAI,KAAK,CAAC,WAAW,CAAA;gBACrC,iBAAiB,IAAI,CAAC,CAAA;gBACtB,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,CAAA;gBAC7D,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAA;YAChD,CAAC;YACD,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,GAAG,WAAW,EAAE,CAAC;gBAC1D,WAAW,GAAG,KAAK,CAAC,SAAS,CAAA;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,MAAM,CAAC,CAAA;IACtB,OAAO;QACL,gBAAgB,EAAE,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;QAC1B,WAAW;QACX,YAAY;QACZ,cAAc;QACd,iBAAiB,EAAE,WAAW,CAAC,IAAI;QACnC,kBAAkB,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,GAAG,iBAAiB;QACzF,0BAA0B,EAAE,mBAAmB,CAAC,oBAAoB,CAAC;QACrE,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,gBAAgB;YAC/B,UAAU,EAAE,iBAAiB,GAAG,gBAAgB;YAChD,cAAc,EAAE,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,GAAG,iBAAiB;YACzF,kBAAkB;YAClB,kBAAkB;SACnB;QACD,WAAW;KACZ,CAAA;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,OAAO;QACL,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC/B,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACjC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;KACzC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAgD;IAEhD,OAAO;QACL,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;QAClC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,eAAe,EAAE,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;KAC5D,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAuB;IACjD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAqB;IAC5D,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QACzE,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;KAC7B,CAAA;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,oBAAoB,CAAC,CAAA;IACpF,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAA;IACf,CAAC;IACD,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC3C,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACrC,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAA;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,OAAO,EAAE,CAAC;YAC7C,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACjC,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5D,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACjC,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC3B,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,GAAG,EAAE,CAAC;QACN,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,eAAe,EAAE,CAAC;KACnB,CAAA;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAwB;IACxD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,yBAAyB,CAAA;IAClC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,qBAAqB,EAAE,CAAC;QAC7E,MAAM,IAAI,KAAK,CAAC,sDAAsD,qBAAqB,GAAG,CAAC,CAAA;IACjG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,eAAe,CAAC,MAAM,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,MAAM,CAAA;QACf,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,MAAM,IAAI,KAAK;QACf,KAAK,CAAC,IAAI,KAAK,YAAY;QAC3B,MAAM,IAAI,KAAK;QACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAC9B,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnC,oBAAoB,CAAC,KAAK,EAAE,gBAAgB,CAAC;QAC7C,oBAAoB,CAAC,KAAK,EAAE,eAAe,CAAC;QAC5C,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC;QAC9B,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAC/B,CAAA;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,CAAC,KAAK,CAAC;QACnB,SAAS,CAAC,KAAK,CAAC;QAChB,oBAAoB,CAAC,KAAK,CAAC;QAC3B,sBAAsB,CAAC,KAAK,CAAC,CAC9B,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,WAAW,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACpE,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,CACL,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CACjG,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAA;AACvE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,CAAC,CAAC,aAAa,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,CAAA;AAC3E,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,GAAW;IAC5C,OAAO,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,SAAS,CAAA;AACrD,CAAC"}
@@ -1,4 +1,8 @@
1
1
  import type { ParsedDocument, TextChunk } from "./types.js";
2
- export declare function chunkDocument(document: ParsedDocument, chunkSize: number, chunkOverlap: number): TextChunk[];
2
+ export interface ChunkDocumentOptions {
3
+ maxChunks?: number;
4
+ signal?: AbortSignal;
5
+ }
6
+ export declare function chunkDocument(document: ParsedDocument, chunkSize: number, chunkOverlap: number, options?: ChunkDocumentOptions): TextChunk[];
3
7
  export declare function chunkSearchText(chunk: Pick<TextChunk, "contextPath" | "text">): string;
4
8
  //# sourceMappingURL=chunking.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAQ3D,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CA+Db;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CAAC,GAAG,MAAM,CAEtF"}
1
+ {"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAA4B,SAAS,EAAE,MAAM,YAAY,CAAA;AAQrF,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB;AAED,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,oBAAyB,GACjC,SAAS,EAAE,CAyFb;AAwBD,wBAAgB,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CAAC,GAAG,MAAM,CAEtF"}
package/dist/chunking.js CHANGED
@@ -1,17 +1,28 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { markdownFenceSpans, structuralSpans } from "./document-structure.js";
3
+ import { throwIfAborted } from "./operation.js";
3
4
  const PARAGRAPH_BREAK_MIN_RATIO = 0.45;
4
5
  const SENTENCE_BREAK_MIN_RATIO = 0.55;
5
6
  const LINE_BREAK_MIN_RATIO = 0.65;
6
7
  const WHITESPACE_BREAK_MIN_RATIO = 0.75;
7
8
  const SENTENCE_BOUNDARIES = [". ", "? ", "! ", "。", "?", "!"];
8
- export function chunkDocument(document, chunkSize, chunkOverlap) {
9
+ export function chunkDocument(document, chunkSize, chunkOverlap, options = {}) {
10
+ throwIfAborted(options.signal);
9
11
  if (!document.text) {
10
12
  return [];
11
13
  }
12
14
  const chunks = [];
13
- const lineStarts = lineStartOffsets(document.text);
14
- const structured = structuralSpans(document.text, document.file.extension, chunkSize);
15
+ const lineStarts = lineStartOffsets(document.text, options.signal);
16
+ const locationRegions = sortedCharacterIntervals(document.regions ?? []);
17
+ const pages = sortedCharacterIntervals(document.pages ?? []);
18
+ const sourceRegions = locationRegions.map((region) => ({
19
+ charStart: region.charStart,
20
+ charEnd: region.charEnd,
21
+ contextPath: region.contextPath,
22
+ }));
23
+ const structured = sourceRegions.length > 0
24
+ ? sourceRegions
25
+ : structuralSpans(document.text, document.file.extension, chunkSize);
15
26
  const regions = structured.length > 0
16
27
  ? structured
17
28
  : [
@@ -25,16 +36,28 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
25
36
  const fences = [".md", ".mdx", ".markdown"].includes(document.file.extension)
26
37
  ? markdownFenceSpans(document.text)
27
38
  : [];
39
+ const fenceStarts = new Set(fences.map((fence) => fence.start));
28
40
  let chunkIndex = 0;
29
41
  for (const region of regions) {
42
+ throwIfAborted(options.signal);
30
43
  let cursor = region.charStart;
31
44
  while (cursor < region.charEnd) {
45
+ throwIfAborted(options.signal);
32
46
  const end = chooseChunkEnd(document.text, cursor, chunkSize, region.charEnd, fences);
33
47
  const span = trimmedSpan(document.text, cursor, end);
34
48
  if (span.text) {
49
+ if (options.maxChunks !== undefined && chunks.length >= options.maxChunks) {
50
+ throw new Error(`Chunk limit of ${options.maxChunks} exceeded for ${document.file.relativePath}. Increase chunkSize or split the source file.`);
51
+ }
35
52
  const id = createHash("sha256")
36
53
  .update(`${document.file.relativePath}:${chunkIndex}:${region.contextPath}:${span.text}`)
37
54
  .digest("hex");
55
+ const lineCoordinates = document.sourceLineCoordinates === false
56
+ ? {}
57
+ : {
58
+ lineStart: lineNumberForOffset(lineStarts, span.start),
59
+ lineEnd: lineNumberForOffset(lineStarts, Math.max(span.start, span.end - 1)),
60
+ };
38
61
  chunks.push({
39
62
  id,
40
63
  source: document.file.source,
@@ -44,9 +67,9 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
44
67
  text: span.text,
45
68
  charStart: span.start,
46
69
  charEnd: span.end,
47
- lineStart: lineNumberForOffset(lineStarts, span.start),
48
- lineEnd: lineNumberForOffset(lineStarts, Math.max(span.start, span.end - 1)),
49
- ...pageRangeForSpan(document, span.start, span.end),
70
+ ...lineCoordinates,
71
+ ...pageRangeForSpan(pages, span.start, span.end),
72
+ ...sourceLocationForSpan(locationRegions, span.start, span.end),
50
73
  checksum: document.file.checksum,
51
74
  bytes: document.file.bytes,
52
75
  mtimeMs: document.file.mtimeMs,
@@ -56,32 +79,58 @@ export function chunkDocument(document, chunkSize, chunkOverlap) {
56
79
  if (end >= region.charEnd) {
57
80
  break;
58
81
  }
59
- cursor = fences.some((fence) => fence.start === end)
82
+ cursor = fenceStarts.has(end)
60
83
  ? end
61
84
  : Math.max(end - chunkOverlap, cursor + 1, region.charStart);
62
85
  }
63
86
  }
64
87
  return chunks;
65
88
  }
89
+ function sourceLocationForSpan(regions, start, end) {
90
+ const region = regions[firstIntervalEndingAfter(regions, start)];
91
+ if (!region || region.charStart >= end) {
92
+ return {};
93
+ }
94
+ return {
95
+ locationKind: region.location.kind,
96
+ locationStart: region.location.start,
97
+ locationEnd: region.location.end,
98
+ ...(region.location.label ? { locationLabel: region.location.label } : {}),
99
+ ...(region.location.cellStart ? { cellStart: region.location.cellStart } : {}),
100
+ ...(region.location.cellEnd ? { cellEnd: region.location.cellEnd } : {}),
101
+ };
102
+ }
66
103
  export function chunkSearchText(chunk) {
67
104
  return chunk.contextPath ? `${chunk.contextPath}\n${chunk.text}` : chunk.text;
68
105
  }
69
- function pageRangeForSpan(document, start, end) {
70
- if (!document.pages || document.pages.length === 0) {
106
+ function pageRangeForSpan(pages, start, end) {
107
+ if (pages.length === 0) {
108
+ return {};
109
+ }
110
+ const firstIndex = firstIntervalEndingAfter(pages, start);
111
+ const first = pages[firstIndex];
112
+ if (!first || first.charStart >= end) {
71
113
  return {};
72
114
  }
73
- const overlapping = document.pages.filter((page) => page.charStart < end && page.charEnd > start);
74
- const first = overlapping[0];
75
- const last = overlapping.at(-1);
76
- return first && last ? { pageStart: first.pageNumber, pageEnd: last.pageNumber } : {};
115
+ let last = first;
116
+ for (let index = firstIndex + 1; index < pages.length; index += 1) {
117
+ const page = pages[index];
118
+ if (!page || page.charStart >= end) {
119
+ break;
120
+ }
121
+ last = page;
122
+ }
123
+ return { pageStart: first.pageNumber, pageEnd: last.pageNumber };
77
124
  }
78
125
  function chooseChunkEnd(text, cursor, chunkSize, regionEnd, fences) {
79
126
  const hardEnd = Math.min(cursor + chunkSize, regionEnd);
80
127
  if (hardEnd === regionEnd) {
81
128
  return hardEnd;
82
129
  }
83
- const intersectedFence = fences.find((fence) => fence.start < hardEnd && fence.end > hardEnd && fence.end <= regionEnd);
84
- if (intersectedFence && intersectedFence.end - intersectedFence.start <= chunkSize) {
130
+ const intersectedFence = fenceAtOffset(fences, hardEnd);
131
+ if (intersectedFence &&
132
+ intersectedFence.end <= regionEnd &&
133
+ intersectedFence.end - intersectedFence.start <= chunkSize) {
85
134
  return intersectedFence.start > cursor ? intersectedFence.start : intersectedFence.end;
86
135
  }
87
136
  const window = text.slice(cursor, hardEnd);
@@ -128,15 +177,52 @@ function trimmedSpan(text, start, end) {
128
177
  text: text.slice(trimmedStart, trimmedEnd),
129
178
  };
130
179
  }
131
- function lineStartOffsets(text) {
180
+ function lineStartOffsets(text, signal) {
132
181
  const starts = [0];
133
182
  for (let index = 0; index < text.length; index += 1) {
183
+ if (signal && index % 16_384 === 0) {
184
+ throwIfAborted(signal);
185
+ }
134
186
  if (text[index] === "\n" && index + 1 < text.length) {
135
187
  starts.push(index + 1);
136
188
  }
137
189
  }
138
190
  return starts;
139
191
  }
192
+ function sortedCharacterIntervals(intervals) {
193
+ return [...intervals].sort((left, right) => left.charEnd - right.charEnd || left.charStart - right.charStart);
194
+ }
195
+ function firstIntervalEndingAfter(intervals, offset) {
196
+ let low = 0;
197
+ let high = intervals.length;
198
+ while (low < high) {
199
+ const mid = Math.floor((low + high) / 2);
200
+ const interval = intervals[mid];
201
+ if (interval && interval.charEnd <= offset) {
202
+ low = mid + 1;
203
+ }
204
+ else {
205
+ high = mid;
206
+ }
207
+ }
208
+ return low;
209
+ }
210
+ function fenceAtOffset(fences, offset) {
211
+ let low = 0;
212
+ let high = fences.length;
213
+ while (low < high) {
214
+ const mid = Math.floor((low + high) / 2);
215
+ const fence = fences[mid];
216
+ if (fence && fence.end <= offset) {
217
+ low = mid + 1;
218
+ }
219
+ else {
220
+ high = mid;
221
+ }
222
+ }
223
+ const fence = fences[low];
224
+ return fence && fence.start < offset && fence.end > offset ? fence : undefined;
225
+ }
140
226
  function lineNumberForOffset(lineStarts, offset) {
141
227
  let low = 0;
142
228
  let high = lineStarts.length - 1;