@qvac/core 1.0.0-rc.8 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/CHANGELOG.md +243 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/devices.js +6 -18
  8. package/dist-lib/lib/api/execution.d.ts +6 -2
  9. package/dist-lib/lib/api/execution.js +29 -3
  10. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  11. package/dist-lib/lib/api/host-run-log.js +3 -1
  12. package/dist-lib/lib/api/index.d.ts +3 -1
  13. package/dist-lib/lib/api/index.js +17 -6
  14. package/dist-lib/lib/api/members.js +13 -29
  15. package/dist-lib/lib/api/mesh.d.ts +2 -1
  16. package/dist-lib/lib/api/projects.d.ts +1 -1
  17. package/dist-lib/lib/api/projects.js +9 -5
  18. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  19. package/dist-lib/lib/api/rag-label.js +21 -0
  20. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  21. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  22. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  23. package/dist-lib/lib/api/rag-manifest.js +46 -0
  24. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  25. package/dist-lib/lib/api/rag-outline.js +426 -0
  26. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  27. package/dist-lib/lib/api/rag-plugin.js +70 -0
  28. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  29. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  30. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  31. package/dist-lib/lib/api/rag-targets.js +215 -0
  32. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  33. package/dist-lib/lib/api/rag-window.js +46 -0
  34. package/dist-lib/lib/api/rag.d.ts +3 -2
  35. package/dist-lib/lib/api/rag.js +421 -181
  36. package/dist-lib/lib/api/served.js +8 -14
  37. package/dist-lib/lib/api/workspace.d.ts +8 -1
  38. package/dist-lib/lib/api/workspace.js +27 -13
  39. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  40. package/dist-lib/lib/assistant/history.d.ts +11 -2
  41. package/dist-lib/lib/assistant/history.js +74 -4
  42. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  43. package/dist-lib/lib/assistant/media-round.js +111 -0
  44. package/dist-lib/lib/assistant/message.d.ts +7 -0
  45. package/dist-lib/lib/assistant/model-round.d.ts +12 -0
  46. package/dist-lib/lib/assistant/model-round.js +130 -17
  47. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  48. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  49. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  50. package/dist-lib/lib/assistant/round-host.js +142 -8
  51. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  52. package/dist-lib/lib/assistant/upload.js +36 -0
  53. package/dist-lib/lib/assistant-client.d.ts +1 -0
  54. package/dist-lib/lib/assistant-client.js +40 -19
  55. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  56. package/dist-lib/lib/assistant-tree.js +41 -1
  57. package/dist-lib/lib/assistant.d.ts +15 -0
  58. package/dist-lib/lib/assistant.js +474 -89
  59. package/dist-lib/lib/client.d.ts +1 -0
  60. package/dist-lib/lib/client.js +4 -1
  61. package/dist-lib/lib/compaction.js +2 -2
  62. package/dist-lib/lib/core/agent.js +12 -45
  63. package/dist-lib/lib/core/approval-rules.js +1 -8
  64. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  65. package/dist-lib/lib/core/archived-mesh.js +12 -8
  66. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  67. package/dist-lib/lib/core/chat-copy.js +74 -25
  68. package/dist-lib/lib/core/chat-fork.js +15 -6
  69. package/dist-lib/lib/core/chat-group.js +16 -24
  70. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  71. package/dist-lib/lib/core/chat-move.js +94 -3
  72. package/dist-lib/lib/core/chat-order.js +12 -18
  73. package/dist-lib/lib/core/chat.js +38 -31
  74. package/dist-lib/lib/core/chunk.d.ts +2 -2
  75. package/dist-lib/lib/core/chunk.js +18 -25
  76. package/dist-lib/lib/core/collections.d.ts +2 -0
  77. package/dist-lib/lib/core/collections.js +2 -0
  78. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  79. package/dist-lib/lib/core/contract/activity.js +34 -4
  80. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  81. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  82. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  83. package/dist-lib/lib/core/contract/chat.js +127 -5
  84. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  85. package/dist-lib/lib/core/contract/chunks.js +37 -6
  86. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  87. package/dist-lib/lib/core/contract/features.js +22 -0
  88. package/dist-lib/lib/core/contract/index.js +2 -0
  89. package/dist-lib/lib/core/contract/indexing.js +4 -2
  90. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  91. package/dist-lib/lib/core/contract/members.js +16 -10
  92. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  93. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  94. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  95. package/dist-lib/lib/core/contract/projects.js +13 -5
  96. package/dist-lib/lib/core/contract/roles.js +32 -4
  97. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  98. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  99. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  100. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  101. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  102. package/dist-lib/lib/core/contract/workspace.js +40 -7
  103. package/dist-lib/lib/core/contract/writers.js +2 -2
  104. package/dist-lib/lib/core/departures.d.ts +3 -1
  105. package/dist-lib/lib/core/departures.js +3 -0
  106. package/dist-lib/lib/core/devices.js +1 -8
  107. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  108. package/dist-lib/lib/core/files/attachments.js +19 -1
  109. package/dist-lib/lib/core/files/index.d.ts +20 -4
  110. package/dist-lib/lib/core/files/index.js +132 -21
  111. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  112. package/dist-lib/lib/core/files/processing/document.js +7 -0
  113. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  114. package/dist-lib/lib/core/files/processing/index.js +144 -12
  115. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  116. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  117. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  118. package/dist-lib/lib/core/files/rag/build.js +123 -19
  119. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  120. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  121. package/dist-lib/lib/core/files/rag/index.js +13 -1
  122. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  123. package/dist-lib/lib/core/files/rag/load.js +27 -6
  124. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  125. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  126. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  127. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  128. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  129. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  130. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  131. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  132. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  133. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  134. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  135. package/dist-lib/lib/core/files/upload.js +68 -41
  136. package/dist-lib/lib/core/fs-drives.js +14 -26
  137. package/dist-lib/lib/core/invite.d.ts +1 -0
  138. package/dist-lib/lib/core/invite.js +4 -2
  139. package/dist-lib/lib/core/local/files.d.ts +1 -1
  140. package/dist-lib/lib/core/local/files.js +5 -5
  141. package/dist-lib/lib/core/local/index.d.ts +13 -6
  142. package/dist-lib/lib/core/local/index.js +103 -69
  143. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  144. package/dist-lib/lib/core/local/run-logs.js +5 -5
  145. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  146. package/dist-lib/lib/core/mesh-processor.js +87 -0
  147. package/dist-lib/lib/core/mesh.d.ts +48 -5
  148. package/dist-lib/lib/core/mesh.js +318 -129
  149. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  150. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  151. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  152. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  153. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  154. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  155. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  156. package/dist-lib/lib/core/project-keys.js +11 -0
  157. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  158. package/dist-lib/lib/core/public-mesh.js +10 -8
  159. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  160. package/dist-lib/lib/core/run-requests.js +21 -1
  161. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  162. package/dist-lib/lib/core/skill-availability.js +67 -0
  163. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  164. package/dist-lib/lib/core/sync/background.js +2 -1
  165. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  166. package/dist-lib/lib/core/sync/changed.js +1 -1
  167. package/dist-lib/lib/core/sync/index.js +52 -10
  168. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  169. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  170. package/dist-lib/lib/core/sync-status.js +97 -41
  171. package/dist-lib/lib/core/tool-activity.js +6 -12
  172. package/dist-lib/lib/core/tool-configs.js +17 -29
  173. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  174. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  175. package/dist-lib/lib/core/view-read.d.ts +4 -0
  176. package/dist-lib/lib/core/view-read.js +23 -0
  177. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  178. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  179. package/dist-lib/lib/core.d.ts +39 -42
  180. package/dist-lib/lib/core.js +661 -639
  181. package/dist-lib/lib/errors.d.ts +20 -0
  182. package/dist-lib/lib/errors.js +20 -0
  183. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  184. package/dist-lib/lib/llm-model-names.js +1 -1
  185. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  186. package/dist-lib/lib/model-context.js +1 -2
  187. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  188. package/dist-lib/lib/probe-rounds.js +6 -1
  189. package/dist-lib/lib/run-transcript.d.ts +4 -1
  190. package/dist-lib/lib/run-transcript.js +15 -6
  191. package/dist-lib/schema/engine.js +83 -6
  192. package/dist-lib/schema/indexes.d.ts +3 -0
  193. package/dist-lib/schema/indexes.js +5 -0
  194. package/dist-lib/schema/local.js +11 -2
  195. package/dist-lib/schema/qvac.js +49 -5
  196. package/dist-lib/spec/bind.js +34 -0
  197. package/dist-lib/spec/calls.js +5 -0
  198. package/dist-lib/spec/capabilities.d.ts +17 -0
  199. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  200. package/dist-lib/spec/hrpc/index.js +68 -5
  201. package/dist-lib/spec/hrpc/messages.js +260 -23
  202. package/dist-lib/spec/hyperschema/index.js +260 -23
  203. package/dist-lib/spec/hyperschema/schema.json +181 -0
  204. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  205. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  206. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  207. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  208. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  209. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  210. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  211. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  212. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  213. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  214. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  215. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  216. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  217. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  218. package/package.json +75 -70
  219. package/tui/README.md +2 -1
  220. package/tui/app.mjs +28 -7
  221. package/tui/boot.mjs +5 -1
  222. package/tui/lib/attachment-files.mjs +82 -0
  223. package/tui/lib/reopen-meshes.mjs +15 -0
  224. package/tui/main.mjs +4 -43
  225. package/worker/assistant-sidecar.mjs +10 -3
  226. package/worker/stow-entry.mjs +3 -52
@@ -17,12 +17,15 @@ export interface FileProcessor {
17
17
  readonly input: FileInput;
18
18
  readonly output: 'blob' | 'inline';
19
19
  readonly mediaKind?: string;
20
+ readonly deferChatVectors?: boolean;
20
21
  accepts?(output: QvacKnowledgeArtifact, document: FileRecord): boolean;
21
22
  applies(document: FileRecord): boolean;
22
23
  canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
24
+ text?(document: FileDocument, signal: AbortSignal): Promise<void>;
23
25
  produce(document: FileDocument, signal: AbortSignal): Promise<void>;
24
26
  load?(document: FileDocument, signal: AbortSignal): Promise<void>;
25
27
  evict?(loaded: LoadedArtifact): Promise<void>;
28
+ allLoaded?(): Promise<void>;
26
29
  close(): Promise<boolean>;
27
30
  }
28
31
  export interface FileProcessingOptions {
@@ -44,14 +47,18 @@ export declare class FileProcessing {
44
47
  private readonly _output;
45
48
  private readonly _progress?;
46
49
  private readonly _lanes;
50
+ private readonly _demanded;
51
+ private readonly _vectorDemanded;
47
52
  private readonly _deadlines;
48
53
  private readonly _scheduler;
49
54
  private _scanning;
55
+ private readonly _reads;
50
56
  private _dirty;
51
57
  private _paused;
52
58
  private _closed;
53
59
  private _close;
54
60
  private _indexCleared;
61
+ private _stops;
55
62
  private readonly _onDirty;
56
63
  private readonly _onAll;
57
64
  constructor(mesh: Mesh, options: FileProcessingOptions);
@@ -64,20 +71,26 @@ export declare class FileProcessing {
64
71
  get indexCleared(): boolean;
65
72
  settle(): Promise<void>;
66
73
  enqueue(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
74
+ enqueueVectors(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
67
75
  private _enqueue;
76
+ private _entry;
68
77
  get active(): boolean;
69
78
  pending(id: string): boolean;
70
79
  awaitsBytes(id: string): boolean;
80
+ demand(ids: string[]): () => void;
81
+ private _demandVectors;
71
82
  prioritize(ids: string[]): void;
72
83
  private _shutdown;
73
84
  private _stop;
74
85
  private _kick;
86
+ private _afterFirstLoad;
75
87
  private _scan;
76
88
  private _documents;
77
89
  private _pump;
78
90
  private _forget;
79
91
  private _run;
80
92
  private _recordFailure;
93
+ private _queueProduce;
81
94
  private _queueLoad;
82
95
  private _schedule;
83
96
  private _retry;
@@ -5,6 +5,7 @@ import { IndexingSource } from './source.js';
5
5
  import { fileIds, fileRecord } from '../records/index.js';
6
6
  import { FileDocument } from './document.js';
7
7
  import { errorMessage } from '../../../guards.js';
8
+ const MAX_JOBS = 2;
8
9
  export class FileProcessing {
9
10
  _mesh;
10
11
  _processor;
@@ -14,14 +15,18 @@ export class FileProcessing {
14
15
  _output;
15
16
  _progress;
16
17
  _lanes = [];
18
+ _demanded = new Map();
19
+ _vectorDemanded = new Map();
17
20
  _deadlines = new Map();
18
21
  _scheduler = new DeadlineScheduler(() => this._retry());
19
22
  _scanning = null;
23
+ _reads = new Set();
20
24
  _dirty = false;
21
25
  _paused = false;
22
26
  _closed = false;
23
27
  _close = null;
24
28
  _indexCleared = true;
29
+ _stops = 0;
25
30
  _onDirty = (ids) => {
26
31
  if (ids) {
27
32
  void this.enqueue(ids);
@@ -54,6 +59,9 @@ export class FileProcessing {
54
59
  ready: (id) => this._schedule(mesh.files.processorState(options.processor.id).artifact(id).knowledgeId ?? id, Date.now())
55
60
  }
56
61
  });
62
+ if (options.processor.text) {
63
+ this._lanes.push({ stage: 'text', queue: new Map(), jobs: new Set() });
64
+ }
57
65
  this._lanes.push({ stage: 'produce', queue: new Map(), jobs: new Set() });
58
66
  if (options.processor.load) {
59
67
  this._lanes.push({ stage: 'load', queue: new Map(), jobs: new Set() });
@@ -64,6 +72,7 @@ export class FileProcessing {
64
72
  this._mesh.autobee.on('move-to', this._onAll);
65
73
  this._mesh.autobee.on('writable', this._onAll);
66
74
  this._kick();
75
+ void this._afterFirstLoad();
67
76
  }
68
77
  async suspend() {
69
78
  this._paused = true;
@@ -93,19 +102,46 @@ export class FileProcessing {
93
102
  if (this._closed || this._paused)
94
103
  return;
95
104
  const ids = await this._documents();
96
- for (const lane of this._lanes) {
97
- await this._enqueue(ids, [lane], { wait: true });
105
+ await this.enqueueVectors(ids, { wait: true });
106
+ const loader = this._lanes.find((entry) => entry.stage === 'load');
107
+ if (loader) {
108
+ await this._enqueue(ids, [loader], { wait: true, direct: true });
98
109
  }
99
110
  }
100
111
  enqueue(ids, options = {}) {
101
- return this._enqueue(ids, this._lanes, options);
112
+ const values = [...ids];
113
+ const release = options.wait ? this.demand(values) : null;
114
+ const releaseVectors = options.wait && this._processor.deferChatVectors ? this._demandVectors(values) : null;
115
+ return this._enqueue(values, this._lanes, options).finally(() => {
116
+ release?.();
117
+ releaseVectors?.();
118
+ });
102
119
  }
103
- async _enqueue(ids, lanes, { wait = false } = {}) {
120
+ enqueueVectors(ids, options = {}) {
121
+ const producer = this._lanes.find((entry) => entry.stage === 'produce');
122
+ if (!producer) {
123
+ return Promise.resolve();
124
+ }
125
+ const values = [...ids];
126
+ const release = this.demand(values);
127
+ const releaseVectors = this._demandVectors(values);
128
+ const queued = this._enqueue(values, [producer], { wait: true }).finally(() => {
129
+ release();
130
+ releaseVectors();
131
+ });
132
+ if (options.wait) {
133
+ return queued;
134
+ }
135
+ void queued.catch(noop);
136
+ return Promise.resolve();
137
+ }
138
+ async _enqueue(ids, lanes, { wait = false, direct = false } = {}) {
104
139
  if (this._closed || this._paused)
105
140
  return;
106
141
  const pending = [];
142
+ const entries = new Set(lanes.map((lane) => (direct ? lane : this._entry(lane))));
107
143
  for (const id of ids) {
108
- for (const lane of lanes) {
144
+ for (const lane of entries) {
109
145
  let wakes = lane.queue.get(id);
110
146
  if (!wakes) {
111
147
  lane.queue.set(id, (wakes = []));
@@ -115,11 +151,14 @@ export class FileProcessing {
115
151
  }
116
152
  }
117
153
  }
118
- for (const lane of lanes) {
154
+ for (const lane of entries) {
119
155
  this._pump(lane);
120
156
  }
121
157
  await Promise.all(pending);
122
158
  }
159
+ _entry(lane) {
160
+ return (lane.stage === 'produce' && this._lanes.find((entry) => entry.stage === 'text')) || lane;
161
+ }
123
162
  get active() {
124
163
  return !this._paused && !this._closed && !this._mesh.closing;
125
164
  }
@@ -134,6 +173,39 @@ export class FileProcessing {
134
173
  return (this._source.awaitsBytes((key) => key === id) ||
135
174
  this._output.awaitsBytes((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id));
136
175
  }
176
+ demand(ids) {
177
+ for (const id of ids) {
178
+ this._demanded.set(id, (this._demanded.get(id) ?? 0) + 1);
179
+ }
180
+ this.prioritize(ids);
181
+ return () => {
182
+ for (const id of ids) {
183
+ const count = (this._demanded.get(id) ?? 1) - 1;
184
+ if (count > 0) {
185
+ this._demanded.set(id, count);
186
+ }
187
+ else {
188
+ this._demanded.delete(id);
189
+ }
190
+ }
191
+ };
192
+ }
193
+ _demandVectors(ids) {
194
+ for (const id of ids) {
195
+ this._vectorDemanded.set(id, (this._vectorDemanded.get(id) ?? 0) + 1);
196
+ }
197
+ return () => {
198
+ for (const id of ids) {
199
+ const count = (this._vectorDemanded.get(id) ?? 1) - 1;
200
+ if (count > 0) {
201
+ this._vectorDemanded.set(id, count);
202
+ }
203
+ else {
204
+ this._vectorDemanded.delete(id);
205
+ }
206
+ }
207
+ };
208
+ }
137
209
  prioritize(ids) {
138
210
  for (const lane of this._lanes) {
139
211
  const queued = new Map(lane.queue);
@@ -164,6 +236,7 @@ export class FileProcessing {
164
236
  this._indexCleared = await this._processor.close();
165
237
  }
166
238
  async _stop() {
239
+ this._stops++;
167
240
  this._scheduler.clear();
168
241
  this._deadlines.clear();
169
242
  this._mesh.files.emit('update');
@@ -173,6 +246,7 @@ export class FileProcessing {
173
246
  type: this._closed ? 'file-processing-closed' : 'file-processing-suspended'
174
247
  });
175
248
  }
249
+ await Promise.all([...this._reads].map((snap) => snap.close()));
176
250
  await Promise.all([this._scanning, ...jobs.map((job) => job.done)]);
177
251
  await Promise.all([this._source.cancel(), this._output.cancel()]);
178
252
  for (const lane of this._lanes) {
@@ -198,6 +272,22 @@ export class FileProcessing {
198
272
  }
199
273
  });
200
274
  }
275
+ async _afterFirstLoad() {
276
+ const loader = this._lanes.find((lane) => lane.stage === 'load');
277
+ if (!loader || !this._processor.allLoaded) {
278
+ return;
279
+ }
280
+ const stops = this._stops;
281
+ try {
282
+ await this._enqueue(await this._documents(), [loader], { wait: true });
283
+ if (stops === this._stops && this.active) {
284
+ await this._processor.allLoaded();
285
+ }
286
+ }
287
+ catch (err) {
288
+ this._mesh.log?.warn('rag', 'file processing first load pass failed', { err });
289
+ }
290
+ }
201
291
  async _scan() {
202
292
  while (this._dirty && !this._closed && !this._paused) {
203
293
  this._dirty = false;
@@ -213,20 +303,23 @@ export class FileProcessing {
213
303
  if (this._mesh.closing)
214
304
  return [];
215
305
  const snap = this._mesh.view.snapshot();
306
+ this._reads.add(snap);
216
307
  try {
217
308
  return await fileIds(snap, this._processor.input, this._processor.mediaKind);
218
309
  }
219
310
  finally {
311
+ this._reads.delete(snap);
220
312
  await snap.close();
221
313
  }
222
314
  }
223
315
  _pump(lane) {
224
316
  while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
225
317
  const busy = [...lane.jobs];
226
- if (busy.length && (lane.stage !== 'produce' || busy.length > 1 || !busy[0].admitted)) {
318
+ if (laneFull(lane, busy)) {
227
319
  return;
228
320
  }
229
- const next = [...lane.queue].find(([queued]) => !busy.some((job) => job.id === queued));
321
+ const full = busy.length >= MAX_JOBS;
322
+ const next = [...lane.queue].find(([queued]) => !busy.some((job) => job.id === queued) && (!full || this._demanded.has(queued)));
230
323
  if (!next) {
231
324
  return;
232
325
  }
@@ -244,10 +337,12 @@ export class FileProcessing {
244
337
  job.done = this._run(lane, { id, wakes, signal: abort.signal, admit }).finally(() => {
245
338
  lane.jobs.delete(job);
246
339
  this._pump(lane);
340
+ this._mesh.files.emit('update');
247
341
  });
248
342
  }
249
343
  }
250
344
  async _forget(id) {
345
+ await this._mesh.files.forgetText(id);
251
346
  const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
252
347
  if (loaded) {
253
348
  await this._processor.evict?.(loaded);
@@ -255,8 +350,9 @@ export class FileProcessing {
255
350
  }
256
351
  async _run(lane, { id, wakes, signal, admit }) {
257
352
  let produced = false;
353
+ let record = null;
258
354
  try {
259
- const record = await fileRecord(this._mesh.view, id, this._processor.input);
355
+ record = await fileRecord(this._mesh.view, id, this._processor.input);
260
356
  if (!record && lane.stage === 'load') {
261
357
  await this._forget(id);
262
358
  }
@@ -267,7 +363,7 @@ export class FileProcessing {
267
363
  if (!record || !this._processor.applies(record) || signal.aborted) {
268
364
  return;
269
365
  }
270
- if (lane.stage === 'produce' &&
366
+ if (lane.stage !== 'load' &&
271
367
  (!this._mesh.writable ||
272
368
  !(await this._processor.canProduce(record, {
273
369
  mesh: this._mesh,
@@ -302,8 +398,13 @@ export class FileProcessing {
302
398
  if (produced && this.active) {
303
399
  this._queueLoad(id);
304
400
  }
305
- for (const wake of wakes) {
306
- wake();
401
+ if (lane.stage === 'text' && this.active) {
402
+ this._queueProduce(id, wakes, record);
403
+ }
404
+ else {
405
+ for (const wake of wakes) {
406
+ wake();
407
+ }
307
408
  }
308
409
  this._mesh.files.emit('update');
309
410
  }
@@ -321,6 +422,26 @@ export class FileProcessing {
321
422
  }
322
423
  this._mesh.log?.warn('rag', 'file processing failed', { id, stage, err });
323
424
  }
425
+ _queueProduce(id, wakes, record) {
426
+ const producer = this._lanes.find((entry) => entry.stage === 'produce');
427
+ if (!producer ||
428
+ !record ||
429
+ record.deletedAt ||
430
+ (this._processor.deferChatVectors && record.chatId && !this._vectorDemanded.has(id))) {
431
+ for (const wake of wakes) {
432
+ wake();
433
+ }
434
+ return;
435
+ }
436
+ const queued = producer.queue.get(id);
437
+ if (queued) {
438
+ queued.push(...wakes);
439
+ }
440
+ else {
441
+ producer.queue.set(id, wakes);
442
+ }
443
+ this._pump(producer);
444
+ }
324
445
  _queueLoad(id) {
325
446
  const loader = this._lanes.find((entry) => entry.stage === 'load');
326
447
  if (!loader) {
@@ -356,3 +477,14 @@ export class FileProcessing {
356
477
  }
357
478
  export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.js';
358
479
  export { FileState } from './state.js';
480
+ function noop() { }
481
+ function laneFull(lane, busy) {
482
+ const extracting = busy.some((job) => !job.admitted);
483
+ if (lane.stage === 'text') {
484
+ return busy.length > 0;
485
+ }
486
+ if (lane.stage === 'produce') {
487
+ return extracting || busy.length > MAX_JOBS;
488
+ }
489
+ return extracting;
490
+ }
@@ -10,14 +10,7 @@ export function isClaimLive(row, at) {
10
10
  row.expiresAt <= at + INDEXING_MAX_CLOCK_SKEW_MS + INDEXING_CLAIM_TTL_MS);
11
11
  }
12
12
  export async function getIndexingStatus(store, artifactId) {
13
- const snap = store.view.snapshot();
14
- try {
15
- const status = await snap.get(INDEXING_STATUS, { id: artifactId });
16
- return status;
17
- }
18
- finally {
19
- await snap.close();
20
- }
13
+ return store.view.get(INDEXING_STATUS, { id: artifactId });
21
14
  }
22
15
  async function mustGetIndexingStatus(store, artifactId) {
23
16
  const row = await getIndexingStatus(store, artifactId);
@@ -27,34 +20,20 @@ async function mustGetIndexingStatus(store, artifactId) {
27
20
  return row;
28
21
  }
29
22
  export async function listIndexingByKnowledge(store, { knowledgeId }) {
30
- const snap = store.view.snapshot();
31
- try {
32
- const rows = await snap
33
- .find(INDEXING_STATUS_BY_KNOWLEDGE, {
34
- gte: { knowledgeId },
35
- lte: { knowledgeId }
36
- })
37
- .toArray();
38
- return rows;
39
- }
40
- finally {
41
- await snap.close();
42
- }
23
+ return store.view
24
+ .find(INDEXING_STATUS_BY_KNOWLEDGE, {
25
+ gte: { knowledgeId },
26
+ lte: { knowledgeId }
27
+ })
28
+ .toArray();
43
29
  }
44
30
  export async function listIndexingByChat(store, { chatId }) {
45
- const snap = store.view.snapshot();
46
- try {
47
- const rows = await snap
48
- .find(INDEXING_STATUS_BY_CHAT, {
49
- gte: { chatId },
50
- lte: { chatId }
51
- })
52
- .toArray();
53
- return rows;
54
- }
55
- finally {
56
- await snap.close();
57
- }
31
+ return store.view
32
+ .find(INDEXING_STATUS_BY_CHAT, {
33
+ gte: { chatId },
34
+ lte: { chatId }
35
+ })
36
+ .toArray();
58
37
  }
59
38
  export async function claimIndexing(store, { artifactId, knowledgeId, at }) {
60
39
  if (!at || at < 0) {
@@ -1,5 +1,6 @@
1
1
  import type { AssistantFileProcessorStatus } from '../../../../spec/hyperschema/types.d.ts';
2
2
  import type { IndexingProgress } from '../rag/index.ts';
3
+ import type { CachedText } from '../rag/text-cache.ts';
3
4
  export type LocalWork = Pick<AssistantFileProcessorStatus, 'state' | 'reason' | 'retryAt'>;
4
5
  export type IndexingProgressEntry = Omit<Parameters<IndexingProgress>[0], 'error'> & {
5
6
  error: string | null;
@@ -19,6 +20,7 @@ interface DocumentState {
19
20
  loaded?: LoadedArtifact;
20
21
  sourceMisses?: number;
21
22
  artifactIdMemo?: string;
23
+ prepared?: CachedText;
22
24
  embedFailures?: number;
23
25
  preferredUntil?: number;
24
26
  }
@@ -1,6 +1,7 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
2
  import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
3
- import { type RagArtifactPayload, type RagExtractors } from 'rag-extract';
3
+ import type { PdfPage } from '@qvac/harness/pdf-pages';
4
+ import { type RagArtifactPayload, type RagExtractors, type RagPreparedChunk } from 'rag-extract';
4
5
  export interface BuildEmbedder {
5
6
  model: string;
6
7
  embed(input: {
@@ -13,6 +14,19 @@ export type BuildProgress = (processedChunks: number, totalChunks: number) => vo
13
14
  export declare class EmbedderFailure extends Error {
14
15
  constructor(cause: unknown);
15
16
  }
17
+ export interface ReadPdfPagesOptions {
18
+ readonly signal: AbortSignal;
19
+ readonly beforeOcr?: () => Promise<void>;
20
+ }
21
+ export type ReadPdfPages = (bytes: Uint8Array, options: ReadPdfPagesOptions) => Promise<readonly PdfPage[]>;
22
+ export interface KnowledgeExtractors extends RagExtractors {
23
+ readonly pdfPages?: ReadPdfPages;
24
+ }
25
+ export interface PreparedText {
26
+ readonly content: string;
27
+ readonly chunks: readonly RagPreparedChunk[];
28
+ readonly artifactId?: string | undefined;
29
+ }
16
30
  export interface ArtifactBuildInput {
17
31
  embedder: BuildEmbedder;
18
32
  knowledge: QvacKnowledge;
@@ -22,7 +36,10 @@ export interface ArtifactBuildInput {
22
36
  onProgress?: BuildProgress;
23
37
  onExtracting?: (pages: number, totalPages: number) => Promise<void>;
24
38
  onExtracted?: () => void | Promise<void>;
25
- extractors?: RagExtractors;
39
+ onChunked?: (prepared: PreparedText) => void | Promise<void>;
40
+ kept?: PreparedText | null;
41
+ extractors?: KnowledgeExtractors;
26
42
  }
27
43
  export declare function yieldTurn(signal: AbortSignal): Promise<void>;
44
+ export declare function prepareText(input: ArtifactBuildInput): Promise<PreparedText>;
28
45
  export declare function buildArtifactPayload(input: ArtifactBuildInput): Promise<RagArtifactPayload>;
@@ -1,6 +1,6 @@
1
- import { chunkRagDocuments, extractRagDocumentFromBytes } from 'rag-extract';
1
+ import { chunkRagDocuments, classifyRagDocument, extractRagDocumentFromBytes, RAG_DOCUMENT_KIND } from 'rag-extract';
2
2
  import { AbortedError } from '@qvac/harness/race-abort';
3
- import { errorMessage } from '../../../guards.js';
3
+ import { errorMessage, isRecord } from '../../../guards.js';
4
4
  import { IndexingFailure } from './failure.js';
5
5
  const EMBED_BATCH = 8;
6
6
  export class EmbedderFailure extends Error {
@@ -25,8 +25,17 @@ function pageProgress(onExtracting) {
25
25
  settled: () => chain
26
26
  };
27
27
  }
28
- export async function buildArtifactPayload(input) {
29
- const { embedder, knowledge, artifactId, bytes, signal, onProgress, onExtracted, extractors } = input;
28
+ export async function prepareText(input) {
29
+ if (input.kept) {
30
+ await input.onExtracted?.();
31
+ return input.kept;
32
+ }
33
+ const { knowledge, artifactId, bytes, signal, extractors } = input;
34
+ const pdfPages = extractors?.pdfPages;
35
+ const { fileName, mimeType } = knowledge;
36
+ if (pdfPages && classifyRagDocument({ fileName, mimeType, bytes }) === RAG_DOCUMENT_KIND.pdf) {
37
+ return pageChunks(input, pdfPages);
38
+ }
30
39
  const pages = pageProgress(input.onExtracting);
31
40
  const document = await extractRagDocumentFromBytes({
32
41
  id: artifactId,
@@ -34,37 +43,132 @@ export async function buildArtifactPayload(input) {
34
43
  attachmentId: knowledge.id,
35
44
  chatId: knowledge.chatId ?? undefined,
36
45
  blobId: knowledge.blobId,
37
- fileName: knowledge.fileName,
38
- mimeType: knowledge.mimeType,
46
+ fileName,
47
+ mimeType,
39
48
  bytes
40
49
  }, { extractors, signal, onProgress: pages.report });
41
50
  await pages.settled();
42
51
  await yieldTurn(signal);
43
- await onExtracted?.();
44
- const chunks = chunkRagDocuments([document], {});
45
- if (!document.content.trim() || chunks.length === 0) {
52
+ await input.onExtracted?.();
53
+ const chunks = document.content.trim() ? chunkRagDocuments([document], {}) : [];
54
+ return { content: document.content, chunks };
55
+ }
56
+ async function pageChunks(input, pdfPages) {
57
+ const { knowledge, artifactId, signal, embedder } = input;
58
+ let read = 0;
59
+ const pages = await pdfPages(input.bytes, {
60
+ signal,
61
+ beforeOcr: async () => {
62
+ if (!knowledge.chatId) {
63
+ await embedder.idle?.(signal);
64
+ }
65
+ await input.onExtracting?.(++read, 0);
66
+ }
67
+ });
68
+ await yieldTurn(signal);
69
+ await input.onExtracted?.();
70
+ const unread = pages.filter((page) => page.unread).length;
71
+ if (unread > 0) {
72
+ throw new IndexingFailure('failed', `${knowledge.fileName}: ${unread} of its pages need OCR`);
73
+ }
74
+ const document = {
75
+ id: artifactId,
76
+ artifactId,
77
+ attachmentId: knowledge.id,
78
+ chatId: knowledge.chatId ?? undefined,
79
+ blobId: knowledge.blobId,
80
+ title: knowledge.fileName,
81
+ mimeType: knowledge.mimeType,
82
+ content: ''
83
+ };
84
+ const chunks = withoutShortPages(pages).flatMap(({ page, text }) => chunkRagDocuments([{ ...document, content: text }], {}).map((chunk) => ({ chunk, page })));
85
+ return {
86
+ content: pages.map(({ text }) => text).join('\n\n'),
87
+ chunks: chunks.map(({ chunk, page }, index) => numbered(chunk, { page, index }))
88
+ };
89
+ }
90
+ const SHORT_PAGE_WORDS = 12;
91
+ const UNSPACED = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]/gu;
92
+ const wordsIn = (text) => text.split(/\s+/).filter(Boolean).length + (text.match(UNSPACED)?.length ?? 0);
93
+ const oneParagraph = (text) => text.trim().replace(/\n\s*\n/g, '\n');
94
+ function withoutShortPages(pages) {
95
+ const out = [];
96
+ let held = '';
97
+ for (const { page, text } of pages) {
98
+ const short = wordsIn(text) < SHORT_PAGE_WORDS;
99
+ held = [held, short ? oneParagraph(text) : text].filter(Boolean).join('\n');
100
+ if (short && wordsIn(held) < SHORT_PAGE_WORDS) {
101
+ continue;
102
+ }
103
+ out.push({ page, text: held });
104
+ held = '';
105
+ }
106
+ const last = out.pop();
107
+ if (!last) {
108
+ return pages;
109
+ }
110
+ return [...out, { page: last.page, text: [last.text, held].filter(Boolean).join('\n') }];
111
+ }
112
+ function numbered(chunk, { page, index }) {
113
+ const parsed = JSON.parse(chunk.metadataJson);
114
+ const metadata = isRecord(parsed) ? parsed : {};
115
+ return {
116
+ ...chunk,
117
+ id: `${chunk.id.slice(0, chunk.id.lastIndexOf(':') + 1)}${index}`,
118
+ metadataJson: JSON.stringify({ ...metadata, chunkIndex: index, page })
119
+ };
120
+ }
121
+ export async function buildArtifactPayload(input) {
122
+ const { embedder, knowledge, signal, onProgress } = input;
123
+ const prepared = await prepareText(input);
124
+ const { chunks } = prepared;
125
+ if (chunks.length === 0) {
46
126
  throw new IndexingFailure('no-text', `no extractable text: ${knowledge.fileName}`);
47
127
  }
128
+ await input.onChunked?.(prepared);
48
129
  await onProgress?.(0, chunks.length);
130
+ const positionsByText = new Map();
131
+ for (let i = 0; i < chunks.length; i++) {
132
+ const text = chunks[i].content;
133
+ const positions = positionsByText.get(text);
134
+ if (positions) {
135
+ positions.push(i);
136
+ }
137
+ else {
138
+ positionsByText.set(text, [i]);
139
+ }
140
+ }
141
+ const unique = [...positionsByText];
49
142
  const vectors = [];
50
- for (let start = 0; start < chunks.length; start += EMBED_BATCH) {
143
+ let processed = 0;
144
+ for (let start = 0; start < unique.length; start += EMBED_BATCH) {
51
145
  await yieldTurn(signal);
52
- if (!knowledge.chatId) {
53
- await embedder.idle?.(signal);
146
+ await embedder.idle?.(signal);
147
+ if (signal.aborted) {
148
+ throw new AbortedError();
54
149
  }
55
- const batch = chunks.slice(start, start + EMBED_BATCH);
150
+ const batch = unique.slice(start, start + EMBED_BATCH);
56
151
  let rows;
57
152
  try {
58
- rows = await embedder.embed({ texts: batch.map((c) => c.content), signal });
153
+ rows = await embedder.embed({ texts: batch.map(([text]) => text), signal });
59
154
  }
60
155
  catch (err) {
61
156
  throw new EmbedderFailure(err);
62
157
  }
63
- vectors.push(...rows);
64
- await onProgress?.(Math.min(start + batch.length, chunks.length), chunks.length);
65
- }
66
- if (vectors.length !== chunks.length) {
67
- throw new Error(`embedder returned ${vectors.length} vectors for ${chunks.length} chunks`);
158
+ if (signal.aborted) {
159
+ throw new AbortedError();
160
+ }
161
+ if (rows.length !== batch.length) {
162
+ throw new Error(`embedder returned ${rows.length} vectors for ${batch.length} chunks`);
163
+ }
164
+ for (let i = 0; i < batch.length; i++) {
165
+ const [, positions] = batch[i];
166
+ for (const position of positions) {
167
+ vectors[position] = rows[i];
168
+ }
169
+ processed += positions.length;
170
+ }
171
+ await onProgress?.(processed, chunks.length);
68
172
  }
69
173
  const dimensions = vectors[0]?.length ?? 0;
70
174
  if (dimensions <= 0 || vectors.some((v) => v.length !== dimensions)) {
@@ -33,7 +33,8 @@ export function failureReason(err) {
33
33
  if ((isRecord(err) && err.code === 'ENOSPC') || OUT_OF_SPACE.test(message)) {
34
34
  return 'out-of-space';
35
35
  }
36
- return PASSWORD.test(message) ? 'password-protected' : 'failed';
36
+ const locked = (isRecord(err) && err.code === 'PASSWORD') || PASSWORD.test(message);
37
+ return locked ? 'password-protected' : 'failed';
37
38
  }
38
39
  export function failureDetail(err) {
39
40
  const detail = errorMessage(err);