@qvac/core 1.0.0-rc.8 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/CHANGELOG.md +243 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/devices.js +6 -18
  8. package/dist-lib/lib/api/execution.d.ts +6 -2
  9. package/dist-lib/lib/api/execution.js +29 -3
  10. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  11. package/dist-lib/lib/api/host-run-log.js +3 -1
  12. package/dist-lib/lib/api/index.d.ts +3 -1
  13. package/dist-lib/lib/api/index.js +17 -6
  14. package/dist-lib/lib/api/members.js +13 -29
  15. package/dist-lib/lib/api/mesh.d.ts +2 -1
  16. package/dist-lib/lib/api/projects.d.ts +1 -1
  17. package/dist-lib/lib/api/projects.js +9 -5
  18. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  19. package/dist-lib/lib/api/rag-label.js +21 -0
  20. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  21. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  22. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  23. package/dist-lib/lib/api/rag-manifest.js +46 -0
  24. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  25. package/dist-lib/lib/api/rag-outline.js +426 -0
  26. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  27. package/dist-lib/lib/api/rag-plugin.js +70 -0
  28. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  29. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  30. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  31. package/dist-lib/lib/api/rag-targets.js +215 -0
  32. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  33. package/dist-lib/lib/api/rag-window.js +46 -0
  34. package/dist-lib/lib/api/rag.d.ts +3 -2
  35. package/dist-lib/lib/api/rag.js +421 -181
  36. package/dist-lib/lib/api/served.js +8 -14
  37. package/dist-lib/lib/api/workspace.d.ts +8 -1
  38. package/dist-lib/lib/api/workspace.js +27 -13
  39. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  40. package/dist-lib/lib/assistant/history.d.ts +11 -2
  41. package/dist-lib/lib/assistant/history.js +74 -4
  42. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  43. package/dist-lib/lib/assistant/media-round.js +111 -0
  44. package/dist-lib/lib/assistant/message.d.ts +7 -0
  45. package/dist-lib/lib/assistant/model-round.d.ts +12 -0
  46. package/dist-lib/lib/assistant/model-round.js +130 -17
  47. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  48. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  49. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  50. package/dist-lib/lib/assistant/round-host.js +142 -8
  51. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  52. package/dist-lib/lib/assistant/upload.js +36 -0
  53. package/dist-lib/lib/assistant-client.d.ts +1 -0
  54. package/dist-lib/lib/assistant-client.js +40 -19
  55. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  56. package/dist-lib/lib/assistant-tree.js +41 -1
  57. package/dist-lib/lib/assistant.d.ts +15 -0
  58. package/dist-lib/lib/assistant.js +474 -89
  59. package/dist-lib/lib/client.d.ts +1 -0
  60. package/dist-lib/lib/client.js +4 -1
  61. package/dist-lib/lib/compaction.js +2 -2
  62. package/dist-lib/lib/core/agent.js +12 -45
  63. package/dist-lib/lib/core/approval-rules.js +1 -8
  64. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  65. package/dist-lib/lib/core/archived-mesh.js +12 -8
  66. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  67. package/dist-lib/lib/core/chat-copy.js +74 -25
  68. package/dist-lib/lib/core/chat-fork.js +15 -6
  69. package/dist-lib/lib/core/chat-group.js +16 -24
  70. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  71. package/dist-lib/lib/core/chat-move.js +94 -3
  72. package/dist-lib/lib/core/chat-order.js +12 -18
  73. package/dist-lib/lib/core/chat.js +38 -31
  74. package/dist-lib/lib/core/chunk.d.ts +2 -2
  75. package/dist-lib/lib/core/chunk.js +18 -25
  76. package/dist-lib/lib/core/collections.d.ts +2 -0
  77. package/dist-lib/lib/core/collections.js +2 -0
  78. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  79. package/dist-lib/lib/core/contract/activity.js +34 -4
  80. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  81. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  82. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  83. package/dist-lib/lib/core/contract/chat.js +127 -5
  84. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  85. package/dist-lib/lib/core/contract/chunks.js +37 -6
  86. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  87. package/dist-lib/lib/core/contract/features.js +22 -0
  88. package/dist-lib/lib/core/contract/index.js +2 -0
  89. package/dist-lib/lib/core/contract/indexing.js +4 -2
  90. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  91. package/dist-lib/lib/core/contract/members.js +16 -10
  92. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  93. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  94. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  95. package/dist-lib/lib/core/contract/projects.js +13 -5
  96. package/dist-lib/lib/core/contract/roles.js +32 -4
  97. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  98. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  99. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  100. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  101. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  102. package/dist-lib/lib/core/contract/workspace.js +40 -7
  103. package/dist-lib/lib/core/contract/writers.js +2 -2
  104. package/dist-lib/lib/core/departures.d.ts +3 -1
  105. package/dist-lib/lib/core/departures.js +3 -0
  106. package/dist-lib/lib/core/devices.js +1 -8
  107. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  108. package/dist-lib/lib/core/files/attachments.js +19 -1
  109. package/dist-lib/lib/core/files/index.d.ts +20 -4
  110. package/dist-lib/lib/core/files/index.js +132 -21
  111. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  112. package/dist-lib/lib/core/files/processing/document.js +7 -0
  113. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  114. package/dist-lib/lib/core/files/processing/index.js +144 -12
  115. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  116. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  117. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  118. package/dist-lib/lib/core/files/rag/build.js +123 -19
  119. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  120. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  121. package/dist-lib/lib/core/files/rag/index.js +13 -1
  122. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  123. package/dist-lib/lib/core/files/rag/load.js +27 -6
  124. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  125. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  126. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  127. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  128. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  129. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  130. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  131. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  132. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  133. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  134. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  135. package/dist-lib/lib/core/files/upload.js +68 -41
  136. package/dist-lib/lib/core/fs-drives.js +14 -26
  137. package/dist-lib/lib/core/invite.d.ts +1 -0
  138. package/dist-lib/lib/core/invite.js +4 -2
  139. package/dist-lib/lib/core/local/files.d.ts +1 -1
  140. package/dist-lib/lib/core/local/files.js +5 -5
  141. package/dist-lib/lib/core/local/index.d.ts +13 -6
  142. package/dist-lib/lib/core/local/index.js +103 -69
  143. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  144. package/dist-lib/lib/core/local/run-logs.js +5 -5
  145. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  146. package/dist-lib/lib/core/mesh-processor.js +87 -0
  147. package/dist-lib/lib/core/mesh.d.ts +48 -5
  148. package/dist-lib/lib/core/mesh.js +318 -129
  149. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  150. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  151. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  152. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  153. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  154. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  155. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  156. package/dist-lib/lib/core/project-keys.js +11 -0
  157. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  158. package/dist-lib/lib/core/public-mesh.js +10 -8
  159. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  160. package/dist-lib/lib/core/run-requests.js +21 -1
  161. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  162. package/dist-lib/lib/core/skill-availability.js +67 -0
  163. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  164. package/dist-lib/lib/core/sync/background.js +2 -1
  165. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  166. package/dist-lib/lib/core/sync/changed.js +1 -1
  167. package/dist-lib/lib/core/sync/index.js +52 -10
  168. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  169. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  170. package/dist-lib/lib/core/sync-status.js +97 -41
  171. package/dist-lib/lib/core/tool-activity.js +6 -12
  172. package/dist-lib/lib/core/tool-configs.js +17 -29
  173. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  174. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  175. package/dist-lib/lib/core/view-read.d.ts +4 -0
  176. package/dist-lib/lib/core/view-read.js +23 -0
  177. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  178. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  179. package/dist-lib/lib/core.d.ts +39 -42
  180. package/dist-lib/lib/core.js +661 -639
  181. package/dist-lib/lib/errors.d.ts +20 -0
  182. package/dist-lib/lib/errors.js +20 -0
  183. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  184. package/dist-lib/lib/llm-model-names.js +1 -1
  185. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  186. package/dist-lib/lib/model-context.js +1 -2
  187. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  188. package/dist-lib/lib/probe-rounds.js +6 -1
  189. package/dist-lib/lib/run-transcript.d.ts +4 -1
  190. package/dist-lib/lib/run-transcript.js +15 -6
  191. package/dist-lib/schema/engine.js +83 -6
  192. package/dist-lib/schema/indexes.d.ts +3 -0
  193. package/dist-lib/schema/indexes.js +5 -0
  194. package/dist-lib/schema/local.js +11 -2
  195. package/dist-lib/schema/qvac.js +49 -5
  196. package/dist-lib/spec/bind.js +34 -0
  197. package/dist-lib/spec/calls.js +5 -0
  198. package/dist-lib/spec/capabilities.d.ts +17 -0
  199. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  200. package/dist-lib/spec/hrpc/index.js +68 -5
  201. package/dist-lib/spec/hrpc/messages.js +260 -23
  202. package/dist-lib/spec/hyperschema/index.js +260 -23
  203. package/dist-lib/spec/hyperschema/schema.json +181 -0
  204. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  205. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  206. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  207. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  208. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  209. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  210. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  211. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  212. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  213. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  214. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  215. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  216. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  217. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  218. package/package.json +75 -70
  219. package/tui/README.md +2 -1
  220. package/tui/app.mjs +28 -7
  221. package/tui/boot.mjs +5 -1
  222. package/tui/lib/attachment-files.mjs +82 -0
  223. package/tui/lib/reopen-meshes.mjs +15 -0
  224. package/tui/main.mjs +4 -43
  225. package/worker/assistant-sidecar.mjs +10 -3
  226. package/worker/stow-entry.mjs +3 -52
@@ -0,0 +1,13 @@
1
+ import type { CachedText } from '../core/files/rag/text-cache.ts';
2
+ export declare const UNREAD: {
3
+ readonly reading: 'still being read';
4
+ readonly noText: 'no readable text';
5
+ readonly unreadable: 'could not be read';
6
+ readonly remote: 'not available on this device';
7
+ };
8
+ export interface ManifestFile {
9
+ readonly title: string;
10
+ readonly detail: string;
11
+ }
12
+ export declare function manifestBlock(files: ReadonlyArray<ManifestFile>): string[];
13
+ export declare function readable(text: CachedText): string;
@@ -0,0 +1,46 @@
1
+ import { parseRagMetadata } from 'rag-extract';
2
+ import { outlineOf } from './rag-outline.js';
3
+ const HEAD_LINES = 200;
4
+ const MIN_HEADING_WORDS = 3;
5
+ const HEADING_CHARS = 80;
6
+ export const UNREAD = {
7
+ reading: 'still being read',
8
+ noText: 'no readable text',
9
+ unreadable: 'could not be read',
10
+ remote: 'not available on this device'
11
+ };
12
+ const LABEL = 'Files in this chat:';
13
+ const MANIFEST_TOKENS = 600;
14
+ const CHARS_PER_TOKEN = 4;
15
+ export function manifestBlock(files) {
16
+ const lines = [LABEL];
17
+ let chars = LABEL.length;
18
+ for (const [at, { title, detail }] of files.entries()) {
19
+ const line = `- ${title} — ${detail}`;
20
+ chars += line.length;
21
+ if (chars > MANIFEST_TOKENS * CHARS_PER_TOKEN) {
22
+ lines.push(`… and ${files.length - at} more files`);
23
+ break;
24
+ }
25
+ lines.push(line);
26
+ }
27
+ return lines;
28
+ }
29
+ export function readable(text) {
30
+ const heading = firstHeading(text.content);
31
+ const size = sizeOf(text);
32
+ return heading ? `${size} — first heading: ${heading}` : size;
33
+ }
34
+ function sizeOf({ content, chunks }) {
35
+ const last = chunks[chunks.length - 1];
36
+ const pages = last ? parseRagMetadata(last.metadataJson)?.page : null;
37
+ if (pages) {
38
+ return `readable, ${pages} pages`;
39
+ }
40
+ return `readable, ${content.split('\n').filter((line) => line.trim()).length} lines`;
41
+ }
42
+ function firstHeading(content) {
43
+ const { headings } = outlineOf(content.split('\n', HEAD_LINES));
44
+ const found = headings.find(({ toc, title }) => !toc && title.split(/\s+/).length >= MIN_HEADING_WORDS);
45
+ return found?.title.slice(0, HEADING_CHARS);
46
+ }
@@ -0,0 +1,36 @@
1
+ export type Range = readonly [number, number];
2
+ export interface Heading {
3
+ readonly line: number;
4
+ readonly size: number;
5
+ readonly title: string;
6
+ readonly number: string | null;
7
+ readonly level: number;
8
+ readonly toc: boolean;
9
+ readonly own: number;
10
+ readonly end: number;
11
+ readonly refs: number;
12
+ }
13
+ export interface Definition {
14
+ readonly term: string;
15
+ readonly line: number;
16
+ readonly refs: number;
17
+ }
18
+ export interface ContentsEntry {
19
+ readonly text: string;
20
+ readonly depth: number;
21
+ readonly line: number | null;
22
+ readonly end: number;
23
+ }
24
+ export interface Outline {
25
+ readonly headings: Heading[];
26
+ readonly contents: ContentsEntry[];
27
+ readonly definitions: Definition[];
28
+ readonly preamble: Range | null;
29
+ readonly noise: number[];
30
+ }
31
+ export declare function outlineOf(lines: ReadonlyArray<string>): Outline;
32
+ export declare function isDefinition(line: string): boolean;
33
+ export declare function isSubstantive(line: string): boolean;
34
+ export declare function outlineEntries(headings: ReadonlyArray<Heading>, maxChars: number): string[];
35
+ export declare function outlineLines(outline: Outline, maxChars: number): string[];
36
+ export declare function occurrences(text: string, term: string): number;
@@ -0,0 +1,426 @@
1
+ const KEYWORDS = 'schedule|part|annex|appendix|chapter|article|art|section|clause';
2
+ const KEYED = new RegExp(`^(${KEYWORDS})\\.?\\s+(\\d+[A-Z]?(?:\\.\\d+)*|[IVXLC]+[A-Z]?)(?![\\p{L}\\p{N}])[\\s.):–-]*(.*)$`, 'iu');
3
+ const NUMBERED = /^(\d{1,3}[A-Z]?(?:\.\d{1,3})*)\.?(?:\s+|\t+)(\S.*)$/u;
4
+ const DEFINES = /^\s*\[?["“]([^"”]{1,80})["”]\s+means\b/;
5
+ const LEADER = /^(.{2,100}?)\s*(?:\.{3,}|…+)\s*\d{1,4}$/;
6
+ const TABBED = /^(.{2,100}?)(?:\t+| {2,})\d{1,4}$/;
7
+ const SPACED = new RegExp(`^((?:\\d{1,3}[A-Z]?(?:\\.\\d{1,3})*\\.?|(?:${KEYWORDS})\\.?)\\s.{1,90}?)\\s\\d{1,4}$`, 'iu');
8
+ const PAGE_MARKER = /^(?:-+\s*)?(?:page\s+)?\d+(?:\s*(?:of|\/)\s*\d+)?(?:\s*-+)?$/i;
9
+ const ROMAN = /^M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$/i;
10
+ const SMALL = new Set('of and the to in on for a an or by with from at as under'.split(' '));
11
+ const CONNECTED = new RegExp(`(?:^|\\s)(?:${[...SMALL].join('|')})$`);
12
+ const WORD = /[\p{L}\p{N}]/u;
13
+ const MAX_HEADING_CHARS = 80;
14
+ const MAX_HEADING_WORDS = 10;
15
+ const MAX_WRAPPED = 5;
16
+ const MIN_TOC = 5;
17
+ const MAX_CONTENTS_GAP = 4;
18
+ const MIN_REPEATS = 3;
19
+ const WRAPPED_WIDTH = 100;
20
+ const NARROW = 0.8;
21
+ const SUBSTANTIVE_WORDS = 4;
22
+ const SUBSTANTIVE_CHARS = 25;
23
+ const MAX_RUNNING_CHARS = 100;
24
+ const SENTENCE = /[.;:!?](?:\s|$)/;
25
+ const PAGE_NUMBER = /(?<![$€£¥\d]\s?)\d+(?!\d)(?!\s?%)/g;
26
+ export function outlineOf(lines) {
27
+ const text = cleaned(lines);
28
+ const found = headingsOf(text);
29
+ const toc = contents(found.filter((heading) => heading.toc));
30
+ const bodyStart = toc.length > 0 ? Math.max(...toc.map(({ line }) => line)) + 1 : 0;
31
+ const body = found.filter((heading) => !heading.toc && heading.line >= bodyStart);
32
+ const all = text.slice(bodyStart).join('\n');
33
+ const cited = citations(all);
34
+ const headings = spans(reconcile(toc, body), text.length).map((heading) => ({
35
+ ...heading,
36
+ refs: heading.toc ? 0 : (cited.get(heading.title) ?? 0)
37
+ }));
38
+ return {
39
+ headings,
40
+ contents: contentsOf(toc, body, text.length),
41
+ definitions: definitionsOf(text, bodyStart, all),
42
+ preamble: preambleOf(text, bodyStart, body),
43
+ noise: lines.flatMap((line, at) => (line.trim() && !text[at] ? [at] : []))
44
+ };
45
+ }
46
+ export function isDefinition(line) {
47
+ return DEFINES.test(line);
48
+ }
49
+ export function isSubstantive(line) {
50
+ const words = line.match(/\p{L}+/gu)?.length ?? 0;
51
+ return words >= SUBSTANTIVE_WORDS && line.trim().length >= SUBSTANTIVE_CHARS && !tocParts(line);
52
+ }
53
+ export function outlineEntries(headings, maxChars) {
54
+ const body = headings.filter(({ toc }) => !toc);
55
+ const all = body.length > 0 ? body : headings;
56
+ const entries = all.map((heading) => ({ heading, text: entryOf(heading) }));
57
+ const numbered = (heading) => heading.number !== null || heading.level === 1;
58
+ const tiers = [() => true, (heading) => heading.level <= 2];
59
+ const fits = (tier) => fitted(entries.filter(({ heading }) => tier(heading)), maxChars) === 0;
60
+ const tier = tiers.find(fits) ?? numbered;
61
+ const kept = [];
62
+ let used = 0;
63
+ for (const { heading, text } of entries) {
64
+ if (!tier(heading) || kept.includes(text) || used + text.length + 1 > maxChars) {
65
+ continue;
66
+ }
67
+ kept.push(text);
68
+ used += text.length + 1;
69
+ }
70
+ const dropped = entries.length - kept.length;
71
+ return dropped > 0 ? [...kept, `… (${dropped} more sections)`] : kept;
72
+ }
73
+ export function outlineLines(outline, maxChars) {
74
+ return outline.contents.length > 0
75
+ ? contentsLines(outline.contents, maxChars)
76
+ : outlineEntries(outline.headings, maxChars);
77
+ }
78
+ function contentsLines(contents, maxChars) {
79
+ const fits = (entries) => fitted(entries, maxChars) === 0;
80
+ const entries = fits(contents) ? contents : contents.filter(({ depth }) => depth <= 1);
81
+ const kept = [];
82
+ let used = 0;
83
+ for (const { text } of entries) {
84
+ if (used + text.length + 1 > maxChars) {
85
+ break;
86
+ }
87
+ kept.push(text);
88
+ used += text.length + 1;
89
+ }
90
+ const dropped = contents.length - kept.length;
91
+ return dropped > 0 ? [...kept, `… (${dropped} more)`] : kept;
92
+ }
93
+ function fitted(entries, maxChars) {
94
+ const total = entries.reduce((sum, { text }) => sum + text.length + 1, 0);
95
+ return Math.max(0, total - maxChars);
96
+ }
97
+ function entryOf({ number, title }) {
98
+ if (!number) {
99
+ return title;
100
+ }
101
+ return /^\d/.test(number) ? `${number}. ${title}` : `${number} ${title}`.trim();
102
+ }
103
+ function cleaned(lines) {
104
+ const text = lines.map((line) => line.trim());
105
+ const repeats = new Map();
106
+ for (const [at, line] of text.entries()) {
107
+ const key = line && !isMarker(line) ? runningKey(line) : null;
108
+ if (key !== null) {
109
+ const seen = repeats.get(key) ?? [];
110
+ seen.push(at);
111
+ repeats.set(key, seen);
112
+ }
113
+ }
114
+ const dropped = new Set();
115
+ for (const at of repeats.values()) {
116
+ if (isRegular(at)) {
117
+ at.forEach((line) => dropped.add(line));
118
+ }
119
+ }
120
+ return text.map((line, at) => (isMarker(line) || dropped.has(at) ? '' : line));
121
+ }
122
+ function runningKey(line) {
123
+ if (line.length > MAX_RUNNING_CHARS || SENTENCE.test(line) || identified(line).number) {
124
+ return null;
125
+ }
126
+ return line.toLowerCase().replace(PAGE_NUMBER, '#');
127
+ }
128
+ function isMarker(line) {
129
+ return (PAGE_MARKER.test(line) ||
130
+ (line.length <= 6 &&
131
+ ROMAN.test(line) &&
132
+ (line === line.toUpperCase() || line === line.toLowerCase())));
133
+ }
134
+ function isRegular(at) {
135
+ const spaced = at.filter((line, i) => line - (at[i - 1] ?? -Infinity) >= MIN_TOC);
136
+ if (spaced.length < MIN_REPEATS) {
137
+ return false;
138
+ }
139
+ const gaps = spaced
140
+ .slice(1)
141
+ .map((line, i) => line - (spaced[i] ?? line))
142
+ .sort((a, b) => a - b);
143
+ const middle = gaps[Math.floor(gaps.length / 2)] ?? 0;
144
+ return gaps.every((gap) => gap >= middle / 2 && gap <= middle * 2);
145
+ }
146
+ function tocParts(line) {
147
+ const text = line.trim();
148
+ const leader = LEADER.exec(text);
149
+ const head = leader?.[1] ?? TABBED.exec(text)?.[1] ?? SPACED.exec(text)?.[1];
150
+ if (!head) {
151
+ return null;
152
+ }
153
+ const parsed = identified(head.trim());
154
+ if (!leader && !parsed.number) {
155
+ return null;
156
+ }
157
+ return parsed;
158
+ }
159
+ function identified(line) {
160
+ const keyed = KEYED.exec(line);
161
+ if (keyed) {
162
+ return { number: `${keyed[1]} ${keyed[2]}`, title: (keyed[3] ?? '').trim() };
163
+ }
164
+ const numbered = NUMBERED.exec(line);
165
+ if (numbered) {
166
+ return { number: numbered[1] ?? null, title: (numbered[2] ?? '').trim() };
167
+ }
168
+ return { number: null, title: line };
169
+ }
170
+ function styleOf(line) {
171
+ const words = line.split(/\s+/).length;
172
+ if (!line ||
173
+ line.length > MAX_HEADING_CHARS ||
174
+ words > MAX_HEADING_WORDS ||
175
+ !/\p{L}{2}/u.test(line) ||
176
+ /^["“[(]|[|]/.test(line)) {
177
+ return null;
178
+ }
179
+ const { number, title } = identified(line);
180
+ const keyed = KEYED.test(line);
181
+ if (number && (title === '' || /^\p{Lu}/u.test(title)) && !/[.;:,]$/.test(title)) {
182
+ return { kind: keyed ? 'keyed' : 'numbered', level: depthOf(number, keyed), number, title };
183
+ }
184
+ const letters = line.replace(/[^\p{L}]/gu, '');
185
+ if (letters.length >= 3 && letters === letters.toUpperCase() && !/[.;:]$/.test(line)) {
186
+ return { kind: 'caps', level: 1, number: null, title: line.replace(/,$/, '') };
187
+ }
188
+ if (!/^\p{Lu}/u.test(line) ||
189
+ /[.:,]$/.test(line) ||
190
+ /[;\t[\]]/.test(line) ||
191
+ letters.length < 3) {
192
+ return null;
193
+ }
194
+ return {
195
+ kind: isTitleCase(line) ? 'title' : 'plain',
196
+ level: isTitleCase(line) ? 2 : 3,
197
+ number: null,
198
+ title: line
199
+ };
200
+ }
201
+ function depthOf(number, keyed) {
202
+ if (!keyed) {
203
+ return number.split('.').length;
204
+ }
205
+ return /^part/i.test(number) ? 2 : 1;
206
+ }
207
+ function isTitleCase(line) {
208
+ return line.split(/\s+/).every((word) => /^[\p{Lu}\d(["“&/–-]/u.test(word) || SMALL.has(word));
209
+ }
210
+ function headingsOf(text) {
211
+ const layout = {
212
+ text,
213
+ styles: text.map(styleOf),
214
+ width: wrapWidth(text),
215
+ next: nextFilled(text)
216
+ };
217
+ const found = [];
218
+ for (let at = 0; at < text.length; at++) {
219
+ const toc = tocParts(text[at] ?? '');
220
+ if (toc) {
221
+ found.push({ line: at, size: 1, title: toc.title, number: toc.number, level: 2, toc: true });
222
+ continue;
223
+ }
224
+ const style = layout.styles[at];
225
+ if (!style || !standsAlone(layout, at)) {
226
+ continue;
227
+ }
228
+ const run = wrapped(layout, at, style);
229
+ const title = [style.title, ...run.map((next) => next.title)].filter(Boolean).join(' ');
230
+ found.push({
231
+ line: at,
232
+ size: 1 + run.length,
233
+ title,
234
+ number: style.number,
235
+ level: style.level,
236
+ toc: false
237
+ });
238
+ at += run.length;
239
+ }
240
+ return found;
241
+ }
242
+ function wrapped({ text, styles, width }, at, head) {
243
+ const hardWrapped = width !== Infinity;
244
+ const run = [];
245
+ for (let next = at + 1; next < text.length && run.length < MAX_WRAPPED - 1; next++) {
246
+ const before = run[run.length - 1] ?? head;
247
+ const candidate = styles[next];
248
+ const line = text[next] ?? '';
249
+ const previous = text[next - 1] ?? '';
250
+ if (!candidate || tocParts(line) || candidate.number) {
251
+ break;
252
+ }
253
+ const same = candidate.kind === before.kind &&
254
+ (before.kind === 'caps' ||
255
+ (before.kind === 'title' &&
256
+ (CONNECTED.test(previous) || (hardWrapped && shortTitle(previous) && shortTitle(line)))));
257
+ if (before.title !== '' && !same) {
258
+ break;
259
+ }
260
+ run.push(candidate);
261
+ }
262
+ return run;
263
+ }
264
+ function shortTitle(line) {
265
+ return line.split(/\s+/).length <= 3;
266
+ }
267
+ function standsAlone({ text, styles, width, next: nexts }, at) {
268
+ const kind = styles[at]?.kind;
269
+ if (kind !== 'title' && kind !== 'plain') {
270
+ return true;
271
+ }
272
+ const line = text[at] ?? '';
273
+ const before = text[at - 1];
274
+ const ended = at === 0 || !before || /[.;:!?)\]”"]$/.test(before) || styles[at - 1] !== null;
275
+ const nextAt = nexts[at] ?? text.length;
276
+ const next = text[nextAt];
277
+ const body = !!next &&
278
+ !/^\p{Ll}/u.test(next) &&
279
+ (kind === 'title' || next.length >= SUBSTANTIVE_CHARS || styles[nextAt] !== null);
280
+ const narrow = width === Infinity || line.length <= width * NARROW;
281
+ return ended && body && narrow;
282
+ }
283
+ function nextFilled(text) {
284
+ const next = new Array(text.length).fill(text.length);
285
+ let following = text.length;
286
+ for (let at = text.length - 1; at >= 0; at--) {
287
+ next[at] = following;
288
+ following = text[at] ? at : following;
289
+ }
290
+ return next;
291
+ }
292
+ function wrapWidth(text) {
293
+ const lengths = text
294
+ .filter(Boolean)
295
+ .map((line) => line.length)
296
+ .sort((a, b) => a - b);
297
+ const p90 = lengths[Math.floor(lengths.length * 0.9)] ?? 0;
298
+ return p90 <= WRAPPED_WIDTH ? p90 : Infinity;
299
+ }
300
+ function contents(entries) {
301
+ let best = [];
302
+ let run = [];
303
+ for (const entry of entries) {
304
+ const last = run[run.length - 1];
305
+ if (last && entry.line - last.line <= MAX_CONTENTS_GAP) {
306
+ run.push(entry);
307
+ }
308
+ else {
309
+ run = [entry];
310
+ }
311
+ if (run.length > best.length) {
312
+ best = run;
313
+ }
314
+ }
315
+ return best.length >= MIN_TOC ? best : [];
316
+ }
317
+ function contentsOf(toc, body, total) {
318
+ let cursor = 0;
319
+ const entries = toc.map((entry) => {
320
+ const depth = entry.number === null ? 1 : depthOf(entry.number, /^\p{L}/u.test(entry.number));
321
+ const key = keyOf(entry.title);
322
+ const at = depth === 1 && key
323
+ ? body.findIndex((heading, i) => i >= cursor && keyOf(heading.title) === key)
324
+ : -1;
325
+ cursor = at >= 0 ? at + 1 : cursor;
326
+ return { text: entryOf(entry), depth, line: body[at]?.line ?? null };
327
+ });
328
+ const lines = entries.flatMap(({ line }) => (line === null ? [] : [line]));
329
+ if (lines.length === 0) {
330
+ return [];
331
+ }
332
+ return entries.map((entry) => ({
333
+ ...entry,
334
+ end: lines.find((line) => line > (entry.line ?? total)) ?? total
335
+ }));
336
+ }
337
+ function reconcile(toc, body) {
338
+ const keys = new Set(body.map(({ title }) => keyOf(title)));
339
+ const numbers = new Map();
340
+ for (const entry of toc) {
341
+ if (entry.number && !numbers.has(keyOf(entry.title))) {
342
+ numbers.set(keyOf(entry.title), entry.number);
343
+ }
344
+ }
345
+ const numbered = new Set(body.filter(({ number }) => number).map(({ title }) => keyOf(title)));
346
+ const bare = new Map();
347
+ for (const heading of body.filter(({ number }) => !number)) {
348
+ bare.set(keyOf(heading.title), (bare.get(keyOf(heading.title)) ?? 0) + 1);
349
+ }
350
+ const borrowed = body.map((heading) => {
351
+ const key = keyOf(heading.title);
352
+ const number = numbers.get(key);
353
+ if (heading.number || !number || numbered.has(key) || bare.get(key) !== 1) {
354
+ return heading;
355
+ }
356
+ return { ...heading, number };
357
+ });
358
+ return [...toc.filter(({ title }) => !keys.has(keyOf(title))), ...borrowed];
359
+ }
360
+ function keyOf(title) {
361
+ return title
362
+ .toLowerCase()
363
+ .replace(/[^\p{L}\p{N}]+/gu, ' ')
364
+ .trim();
365
+ }
366
+ function spans(found, total) {
367
+ const sized = new Array(found.length);
368
+ const open = [];
369
+ let own = total;
370
+ for (let at = found.length - 1; at >= 0; at--) {
371
+ const heading = found[at];
372
+ if (!heading) {
373
+ continue;
374
+ }
375
+ if (heading.toc) {
376
+ sized[at] = { ...heading, own: heading.line + 1, end: heading.line + 1 };
377
+ continue;
378
+ }
379
+ while ((open[open.length - 1]?.level ?? 0) > heading.level) {
380
+ open.pop();
381
+ }
382
+ sized[at] = { ...heading, own, end: open[open.length - 1]?.line ?? total };
383
+ open.push(heading);
384
+ own = heading.line;
385
+ }
386
+ return sized;
387
+ }
388
+ function citations(text) {
389
+ const counts = new Map();
390
+ for (const { 0: whole, 1: title = '', index = 0 } of text.matchAll(/\(([^()\n]+)\)/g)) {
391
+ if (!WORD.test(text[index - 1] ?? ' ') && !WORD.test(text[index + whole.length] ?? ' ')) {
392
+ counts.set(title, (counts.get(title) ?? 0) + 1);
393
+ }
394
+ }
395
+ return counts;
396
+ }
397
+ function definitionsOf(text, from, all) {
398
+ const definitions = [];
399
+ const known = new Set();
400
+ for (const [offset, line] of text.slice(from).entries()) {
401
+ const term = DEFINES.exec(line)?.[1];
402
+ if (term && !known.has(term)) {
403
+ known.add(term);
404
+ definitions.push({ term, line: from + offset, refs: occurrences(all, term) - 1 });
405
+ }
406
+ }
407
+ return definitions;
408
+ }
409
+ export function occurrences(text, term) {
410
+ let count = 0;
411
+ for (let at = text.indexOf(term); at >= 0; at = text.indexOf(term, at + term.length)) {
412
+ if (!WORD.test(text[at - 1] ?? ' ') && !WORD.test(text[at + term.length] ?? ' ')) {
413
+ count++;
414
+ }
415
+ }
416
+ return count;
417
+ }
418
+ function preambleOf(text, from, body) {
419
+ const first = body[0];
420
+ const start = text.findIndex((line, at) => at >= from && line !== '');
421
+ if (!first || start < 0 || first.line <= start) {
422
+ return null;
423
+ }
424
+ const range = [start, first.line];
425
+ return text.slice(...range).some(isSubstantive) ? range : null;
426
+ }
@@ -0,0 +1,20 @@
1
+ import { type PluginVectorIndex, type RagDocumentRef, type VectorHit } from '../core/files/index.ts';
2
+ export interface PluginDocument extends RagDocumentRef {
3
+ readonly index: PluginVectorIndex;
4
+ }
5
+ export interface PluginSearch {
6
+ readonly query: string;
7
+ readonly scopes: readonly string[];
8
+ readonly documents: readonly PluginDocument[];
9
+ readonly scopeDocumentIds: readonly string[];
10
+ readonly budgetTokens?: number | undefined;
11
+ readonly previousQuery?: string | undefined;
12
+ readonly limit?: number | undefined;
13
+ }
14
+ export interface PluginFound {
15
+ readonly hits: readonly VectorHit[];
16
+ readonly targetDocumentId: string | null;
17
+ readonly uncovered?: readonly string[] | undefined;
18
+ readonly outline?: readonly string[] | undefined;
19
+ }
20
+ export declare function searchPlugin(search: PluginSearch): Promise<PluginFound>;
@@ -0,0 +1,70 @@
1
+ import { resolveTargetDocument, selectCoverage } from '../core/files/index.js';
2
+ import { DEFAULT_CHUNK_BUDGET_TOKENS } from 'rag-extract';
3
+ import { rankLexical } from './rag-lexical.js';
4
+ import { classifyRequest, outlineBlock } from './rag-synthesis.js';
5
+ const INJECTED = 5;
6
+ const COVERAGE_CHUNKS = 512;
7
+ export async function searchPlugin(search) {
8
+ const { query, documents, scopeDocumentIds, limit } = search;
9
+ if (limit !== undefined) {
10
+ return { hits: await find(search, { k: limit, focus: documents }), targetDocumentId: null };
11
+ }
12
+ const attached = scopeDocumentIds.length > 0;
13
+ const targetDocumentId = attached ? null : resolveTargetDocument(query, documents);
14
+ const targets = attached ? scopeDocumentIds : [targetDocumentId].flatMap((id) => (id ? [id] : []));
15
+ const { wholeDocument: whole, synthesis } = classifyRequest(query);
16
+ if (targets.length === 0 && !whole) {
17
+ const hits = await find(search, { k: wideK(documents) });
18
+ return {
19
+ ...lexicalFirst(hits, search, { named: targets.length > 0, synthesis }),
20
+ targetDocumentId
21
+ };
22
+ }
23
+ const focused = targets.length
24
+ ? documents.filter((document) => targets.includes(document.documentId))
25
+ : documents;
26
+ if (!whole) {
27
+ const hits = await find(search, { k: wideK(focused), focus: focused, only: attached });
28
+ return {
29
+ ...lexicalFirst(hits, search, { named: targets.length > 0, synthesis }),
30
+ targetDocumentId
31
+ };
32
+ }
33
+ const hits = await find(search, { k: wideK(focused), focus: focused, only: true });
34
+ return { hits: selectCoverage(hits, { budgetTokens: search.budgetTokens }), targetDocumentId };
35
+ }
36
+ function wideK(documents) {
37
+ const chunks = documents.reduce((sum, document) => sum + document.index.chunksOf(document.documentId), 0);
38
+ return Math.min(Math.max(chunks, INJECTED), COVERAGE_CHUNKS);
39
+ }
40
+ function lexicalFirst(hits, search, { named, synthesis }) {
41
+ const budgetTokens = search.budgetTokens ?? DEFAULT_CHUNK_BUDGET_TOKENS;
42
+ const byVector = [...hits].sort((a, b) => b.score - a.score);
43
+ const pick = rankLexical(byVector, search.query, {
44
+ budgetTokens,
45
+ synthesis,
46
+ previousQuery: named ? undefined : search.previousQuery
47
+ });
48
+ return {
49
+ hits: pick?.hits ?? byVector.slice(0, INJECTED),
50
+ uncovered: pick?.uncovered,
51
+ outline: pick ? outlineBlock(pick) : undefined
52
+ };
53
+ }
54
+ async function find(search, { k, focus = [], only = false }) {
55
+ const owner = search.documents[0]?.index;
56
+ if (!owner) {
57
+ return [];
58
+ }
59
+ const allowed = new Set((only ? focus : search.documents).map((document) => document.index.pluginId(document.documentId)));
60
+ const hits = await owner.find({
61
+ text: search.query,
62
+ scopes: search.scopes,
63
+ focus: focus.map((document) => document.index.pluginId(document.documentId)),
64
+ k
65
+ });
66
+ return hits.filter((hit) => allowed.has(hit.artifactId)).map(withoutArtifact);
67
+ }
68
+ function withoutArtifact({ artifactId: _artifactId, ...hit }) {
69
+ return hit;
70
+ }
@@ -0,0 +1,7 @@
1
+ import { type LexicalPick } from './rag-lexical.ts';
2
+ export interface Request {
3
+ readonly synthesis: boolean;
4
+ readonly wholeDocument: boolean;
5
+ }
6
+ export declare function classifyRequest(request: string): Request;
7
+ export declare function outlineBlock({ hits, outline }: LexicalPick): string[] | undefined;