@qvac/core 1.0.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/CHANGELOG.md +241 -3
  2. package/README.md +1 -0
  3. package/dist-lib/lib/api/attachments.d.ts +1 -1
  4. package/dist-lib/lib/api/attachments.js +11 -0
  5. package/dist-lib/lib/api/chunks.d.ts +4 -2
  6. package/dist-lib/lib/api/chunks.js +6 -3
  7. package/dist-lib/lib/api/device-routes.d.ts +6 -1
  8. package/dist-lib/lib/api/device-routes.js +20 -1
  9. package/dist-lib/lib/api/devices.js +6 -18
  10. package/dist-lib/lib/api/execution.d.ts +6 -2
  11. package/dist-lib/lib/api/execution.js +29 -3
  12. package/dist-lib/lib/api/host-run-log.d.ts +7 -1
  13. package/dist-lib/lib/api/host-run-log.js +3 -1
  14. package/dist-lib/lib/api/index.d.ts +3 -1
  15. package/dist-lib/lib/api/index.js +17 -6
  16. package/dist-lib/lib/api/members.js +13 -29
  17. package/dist-lib/lib/api/mesh.d.ts +2 -1
  18. package/dist-lib/lib/api/projects.d.ts +1 -1
  19. package/dist-lib/lib/api/projects.js +9 -5
  20. package/dist-lib/lib/api/rag-label.d.ts +4 -0
  21. package/dist-lib/lib/api/rag-label.js +21 -0
  22. package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
  23. package/dist-lib/lib/api/rag-lexical.js +1058 -0
  24. package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
  25. package/dist-lib/lib/api/rag-manifest.js +46 -0
  26. package/dist-lib/lib/api/rag-outline.d.ts +36 -0
  27. package/dist-lib/lib/api/rag-outline.js +426 -0
  28. package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
  29. package/dist-lib/lib/api/rag-plugin.js +70 -0
  30. package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
  31. package/dist-lib/lib/api/rag-synthesis.js +102 -0
  32. package/dist-lib/lib/api/rag-targets.d.ts +23 -0
  33. package/dist-lib/lib/api/rag-targets.js +215 -0
  34. package/dist-lib/lib/api/rag-window.d.ts +5 -0
  35. package/dist-lib/lib/api/rag-window.js +46 -0
  36. package/dist-lib/lib/api/rag.d.ts +3 -2
  37. package/dist-lib/lib/api/rag.js +421 -184
  38. package/dist-lib/lib/api/served.js +8 -14
  39. package/dist-lib/lib/api/workspace.d.ts +8 -1
  40. package/dist-lib/lib/api/workspace.js +27 -13
  41. package/dist-lib/lib/assistant/harness.d.ts +9 -1
  42. package/dist-lib/lib/assistant/history.d.ts +11 -2
  43. package/dist-lib/lib/assistant/history.js +74 -4
  44. package/dist-lib/lib/assistant/media-round.d.ts +61 -0
  45. package/dist-lib/lib/assistant/media-round.js +111 -0
  46. package/dist-lib/lib/assistant/message.d.ts +7 -0
  47. package/dist-lib/lib/assistant/model-round.d.ts +19 -1
  48. package/dist-lib/lib/assistant/model-round.js +163 -28
  49. package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
  50. package/dist-lib/lib/assistant/pdf-pages.js +67 -3
  51. package/dist-lib/lib/assistant/round-host.d.ts +22 -0
  52. package/dist-lib/lib/assistant/round-host.js +142 -8
  53. package/dist-lib/lib/assistant/upload.d.ts +12 -0
  54. package/dist-lib/lib/assistant/upload.js +36 -0
  55. package/dist-lib/lib/assistant-client.d.ts +1 -0
  56. package/dist-lib/lib/assistant-client.js +40 -19
  57. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  58. package/dist-lib/lib/assistant-tree.js +48 -1
  59. package/dist-lib/lib/assistant.d.ts +18 -1
  60. package/dist-lib/lib/assistant.js +484 -95
  61. package/dist-lib/lib/client.d.ts +1 -0
  62. package/dist-lib/lib/client.js +4 -1
  63. package/dist-lib/lib/compaction.js +2 -2
  64. package/dist-lib/lib/core/agent.js +12 -45
  65. package/dist-lib/lib/core/approval-rules.js +1 -8
  66. package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
  67. package/dist-lib/lib/core/archived-mesh.js +12 -8
  68. package/dist-lib/lib/core/chat-copy.d.ts +18 -3
  69. package/dist-lib/lib/core/chat-copy.js +74 -25
  70. package/dist-lib/lib/core/chat-fork.js +15 -6
  71. package/dist-lib/lib/core/chat-group.js +16 -24
  72. package/dist-lib/lib/core/chat-move.d.ts +9 -0
  73. package/dist-lib/lib/core/chat-move.js +94 -3
  74. package/dist-lib/lib/core/chat-order.js +12 -18
  75. package/dist-lib/lib/core/chat.js +38 -31
  76. package/dist-lib/lib/core/chunk.d.ts +2 -2
  77. package/dist-lib/lib/core/chunk.js +18 -25
  78. package/dist-lib/lib/core/collections.d.ts +2 -0
  79. package/dist-lib/lib/core/collections.js +2 -0
  80. package/dist-lib/lib/core/contract/activity.d.ts +4 -0
  81. package/dist-lib/lib/core/contract/activity.js +34 -4
  82. package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
  83. package/dist-lib/lib/core/contract/chat-group.js +21 -1
  84. package/dist-lib/lib/core/contract/chat.d.ts +9 -0
  85. package/dist-lib/lib/core/contract/chat.js +127 -5
  86. package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
  87. package/dist-lib/lib/core/contract/chunks.js +37 -6
  88. package/dist-lib/lib/core/contract/features.d.ts +21 -0
  89. package/dist-lib/lib/core/contract/features.js +22 -0
  90. package/dist-lib/lib/core/contract/index.js +2 -0
  91. package/dist-lib/lib/core/contract/indexing.js +4 -2
  92. package/dist-lib/lib/core/contract/knowledge.js +5 -4
  93. package/dist-lib/lib/core/contract/members.js +16 -10
  94. package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
  95. package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
  96. package/dist-lib/lib/core/contract/projects.d.ts +5 -0
  97. package/dist-lib/lib/core/contract/projects.js +13 -5
  98. package/dist-lib/lib/core/contract/roles.js +32 -4
  99. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
  100. package/dist-lib/lib/core/contract/run-requests.js +35 -2
  101. package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
  102. package/dist-lib/lib/core/contract/tool-configs.js +3 -2
  103. package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
  104. package/dist-lib/lib/core/contract/workspace.js +40 -7
  105. package/dist-lib/lib/core/contract/writers.js +2 -2
  106. package/dist-lib/lib/core/departures.d.ts +3 -1
  107. package/dist-lib/lib/core/departures.js +3 -0
  108. package/dist-lib/lib/core/devices.js +1 -8
  109. package/dist-lib/lib/core/files/attachments.d.ts +2 -0
  110. package/dist-lib/lib/core/files/attachments.js +19 -1
  111. package/dist-lib/lib/core/files/index.d.ts +20 -4
  112. package/dist-lib/lib/core/files/index.js +132 -21
  113. package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
  114. package/dist-lib/lib/core/files/processing/document.js +7 -0
  115. package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
  116. package/dist-lib/lib/core/files/processing/index.js +144 -12
  117. package/dist-lib/lib/core/files/processing/protocol.js +13 -34
  118. package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
  119. package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
  120. package/dist-lib/lib/core/files/rag/build.js +123 -19
  121. package/dist-lib/lib/core/files/rag/failure.js +2 -1
  122. package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
  123. package/dist-lib/lib/core/files/rag/index.js +13 -1
  124. package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
  125. package/dist-lib/lib/core/files/rag/load.js +27 -6
  126. package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
  127. package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
  128. package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
  129. package/dist-lib/lib/core/files/rag/produce.js +104 -16
  130. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
  131. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
  132. package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
  133. package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
  134. package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
  135. package/dist-lib/lib/core/files/records/knowledge.js +7 -40
  136. package/dist-lib/lib/core/files/records/reclaim.js +5 -10
  137. package/dist-lib/lib/core/files/upload.js +68 -41
  138. package/dist-lib/lib/core/fs-drives.js +14 -26
  139. package/dist-lib/lib/core/invite.d.ts +1 -0
  140. package/dist-lib/lib/core/invite.js +4 -2
  141. package/dist-lib/lib/core/local/files.d.ts +1 -1
  142. package/dist-lib/lib/core/local/files.js +5 -5
  143. package/dist-lib/lib/core/local/index.d.ts +13 -6
  144. package/dist-lib/lib/core/local/index.js +103 -69
  145. package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
  146. package/dist-lib/lib/core/local/run-logs.js +5 -5
  147. package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
  148. package/dist-lib/lib/core/mesh-processor.js +87 -0
  149. package/dist-lib/lib/core/mesh.d.ts +48 -5
  150. package/dist-lib/lib/core/mesh.js +318 -129
  151. package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
  152. package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
  153. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
  154. package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
  155. package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
  156. package/dist-lib/lib/core/migrations/writer-links.js +32 -0
  157. package/dist-lib/lib/core/project-keys.d.ts +6 -0
  158. package/dist-lib/lib/core/project-keys.js +11 -0
  159. package/dist-lib/lib/core/public-mesh.d.ts +3 -0
  160. package/dist-lib/lib/core/public-mesh.js +10 -8
  161. package/dist-lib/lib/core/run-requests.d.ts +3 -0
  162. package/dist-lib/lib/core/run-requests.js +21 -1
  163. package/dist-lib/lib/core/skill-availability.d.ts +33 -0
  164. package/dist-lib/lib/core/skill-availability.js +67 -0
  165. package/dist-lib/lib/core/sync/background.d.ts +0 -1
  166. package/dist-lib/lib/core/sync/background.js +2 -1
  167. package/dist-lib/lib/core/sync/changed.d.ts +4 -2
  168. package/dist-lib/lib/core/sync/changed.js +1 -1
  169. package/dist-lib/lib/core/sync/index.js +52 -10
  170. package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
  171. package/dist-lib/lib/core/sync/view-updates.js +82 -91
  172. package/dist-lib/lib/core/sync-status.js +97 -41
  173. package/dist-lib/lib/core/tool-activity.js +6 -12
  174. package/dist-lib/lib/core/tool-configs.js +17 -29
  175. package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
  176. package/dist-lib/lib/core/upgrade-summary.js +67 -0
  177. package/dist-lib/lib/core/view-read.d.ts +4 -0
  178. package/dist-lib/lib/core/view-read.js +23 -0
  179. package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
  180. package/dist-lib/lib/core/workspace-upgrade.js +27 -16
  181. package/dist-lib/lib/core.d.ts +45 -42
  182. package/dist-lib/lib/core.js +675 -640
  183. package/dist-lib/lib/errors.d.ts +24 -0
  184. package/dist-lib/lib/errors.js +24 -0
  185. package/dist-lib/lib/llm-model-names.d.ts +1 -1
  186. package/dist-lib/lib/llm-model-names.js +1 -1
  187. package/dist-lib/lib/mesh-scoped-engine.js +4 -0
  188. package/dist-lib/lib/model-context.js +1 -2
  189. package/dist-lib/lib/model-download.d.ts +6 -1
  190. package/dist-lib/lib/model-download.js +16 -3
  191. package/dist-lib/lib/oauth/resolve.js +29 -8
  192. package/dist-lib/lib/oauth/token-exchange-core.js +11 -0
  193. package/dist-lib/lib/probe-rounds.d.ts +7 -2
  194. package/dist-lib/lib/probe-rounds.js +6 -1
  195. package/dist-lib/lib/run-transcript.d.ts +4 -1
  196. package/dist-lib/lib/run-transcript.js +15 -6
  197. package/dist-lib/schema/engine.js +83 -6
  198. package/dist-lib/schema/indexes.d.ts +3 -0
  199. package/dist-lib/schema/indexes.js +5 -0
  200. package/dist-lib/schema/local.js +11 -2
  201. package/dist-lib/schema/qvac.js +49 -5
  202. package/dist-lib/spec/bind.js +34 -0
  203. package/dist-lib/spec/calls.js +5 -0
  204. package/dist-lib/spec/capabilities.d.ts +17 -0
  205. package/dist-lib/spec/hrpc/hrpc.json +65 -0
  206. package/dist-lib/spec/hrpc/index.js +68 -5
  207. package/dist-lib/spec/hrpc/messages.js +260 -23
  208. package/dist-lib/spec/hyperschema/index.js +260 -23
  209. package/dist-lib/spec/hyperschema/schema.json +181 -0
  210. package/dist-lib/spec/hyperschema/types.d.ts +48 -0
  211. package/dist-lib/spec/local/hyperdb/messages.js +270 -224
  212. package/dist-lib/spec/local/hyperschema/index.js +73 -38
  213. package/dist-lib/spec/local/hyperschema/schema.json +31 -0
  214. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
  215. package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
  216. package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
  217. package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
  218. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
  219. package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
  220. package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
  221. package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
  222. package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
  223. package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
  224. package/package.json +82 -71
  225. package/tui/README.md +52 -0
  226. package/tui/app.mjs +69 -12
  227. package/tui/boot.mjs +27 -6
  228. package/tui/cli.mjs +93 -0
  229. package/tui/index.mjs +2 -87
  230. package/tui/lib/attachment-files.mjs +82 -0
  231. package/tui/lib/reopen-meshes.mjs +15 -0
  232. package/tui/lib/updater.mjs +86 -0
  233. package/tui/main.mjs +29 -57
  234. package/worker/assistant-sidecar.mjs +10 -3
  235. package/worker/stow-entry.mjs +3 -52
  236. package/dist-lib/lib/core/chat-project.d.ts +0 -2
  237. package/dist-lib/lib/core/chat-project.js +0 -15
@@ -2,7 +2,8 @@ import type { LoggerInterface } from '@qvac/logging';
2
2
  import ScopeLock from 'scope-lock';
3
3
  import type { AbortSignal } from 'bare-abort-controller';
4
4
  import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
5
- import { type RagExtractors, type VectorIndex } from 'rag-extract';
5
+ import { type VectorIndex } from 'rag-extract';
6
+ import type { KnowledgeExtractors } from './build.ts';
6
7
  import { type IndexingEmbedder } from './produce.ts';
7
8
  import { type IndexingDevice } from './policy.ts';
8
9
  import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
@@ -11,34 +12,41 @@ export interface RagProcessorOptions {
11
12
  readonly model: string;
12
13
  readonly embedder?: IndexingEmbedder;
13
14
  readonly index?: VectorIndex | null;
14
- readonly extractors?: RagExtractors;
15
+ readonly extractors?: KnowledgeExtractors;
15
16
  readonly batches?: ScopeLock;
16
17
  readonly device?: IndexingDevice;
18
+ readonly deferChatVectors?: boolean;
17
19
  readonly log: LoggerInterface | null;
18
20
  }
19
21
  export declare class RagProcessor implements FileProcessor {
20
22
  readonly id = "rag";
21
23
  readonly input = "knowledge";
22
24
  readonly output = "blob";
25
+ readonly deferChatVectors: boolean;
23
26
  readonly version: string;
24
27
  private readonly _producer;
25
28
  private readonly _device?;
26
29
  readonly load?: (document: FileDocument, signal: AbortSignal) => Promise<void>;
30
+ readonly allLoaded?: () => Promise<void>;
27
31
  private readonly _index;
28
32
  private readonly _log;
29
33
  constructor(options: RagProcessorOptions);
30
34
  applies(_document: QvacKnowledge): boolean;
31
35
  canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
36
+ text(document: FileDocument, signal: AbortSignal): Promise<void>;
32
37
  produce(document: FileDocument, signal: AbortSignal): Promise<void>;
33
38
  private _settled;
34
39
  evict(loaded: LoadedArtifact): Promise<void>;
35
40
  close(): Promise<boolean>;
36
41
  }
37
42
  export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
43
+ export { PluginVectorIndex, knowledgeScope, pluginArtifactId } from './plugin-index.ts';
44
+ export type { PluginHit } from './plugin-index.ts';
38
45
  export { joinedIndex } from './joined-index.ts';
39
46
  export { canIndexKnowledge } from './policy.ts';
40
47
  export type { IndexingDevice } from './policy.ts';
41
48
  export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
42
49
  export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
43
- export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
50
+ export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, resolveTargetDocument, retrieveRagKnowledge, searchRagChunks, selectCoverage, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
51
+ export type { KnowledgeExtractors, ReadPdfPages } from './build.ts';
44
52
  export type { RagArtifactPayload, RagDocumentRef, RagExtractors, RagSource, VectorHit, VectorIndex } from 'rag-extract';
@@ -3,15 +3,18 @@ import { pipelineVersionFor } from 'rag-extract';
3
3
  import { AbortedError } from '@qvac/harness/race-abort';
4
4
  import { IndexingPass } from './produce.js';
5
5
  import { HydratePass } from './load.js';
6
+ import { PluginVectorIndex } from './plugin-index.js';
6
7
  import { canIndexKnowledge } from './policy.js';
7
8
  export class RagProcessor {
8
9
  id = 'rag';
9
10
  input = 'knowledge';
10
11
  output = 'blob';
12
+ deferChatVectors;
11
13
  version;
12
14
  _producer;
13
15
  _device;
14
16
  load;
17
+ allLoaded;
15
18
  _index;
16
19
  _log;
17
20
  constructor(options) {
@@ -19,6 +22,7 @@ export class RagProcessor {
19
22
  this._index = options.index ?? null;
20
23
  this._log = options.log;
21
24
  this._device = options.device;
25
+ this.deferChatVectors = options.deferChatVectors ?? true;
22
26
  const lock = options.batches ?? new ScopeLock();
23
27
  const source = options.embedder;
24
28
  const embedder = source && {
@@ -44,6 +48,10 @@ export class RagProcessor {
44
48
  const loader = new HydratePass({ ...options, index: this._index, embedder });
45
49
  this.load = (document, signal) => this._settled(document, loader.run(document, signal));
46
50
  }
51
+ if (this._index instanceof PluginVectorIndex) {
52
+ const index = this._index;
53
+ this.allLoaded = () => index.sweep();
54
+ }
47
55
  }
48
56
  applies(_document) {
49
57
  return true;
@@ -57,6 +65,9 @@ export class RagProcessor {
57
65
  device: host.device
58
66
  })));
59
67
  }
68
+ async text(document, signal) {
69
+ await this._producer?.extract(document, signal);
70
+ }
60
71
  async produce(document, signal) {
61
72
  await this._settled(document, this._producer?.run(document, signal));
62
73
  }
@@ -88,7 +99,8 @@ export class RagProcessor {
88
99
  }
89
100
  }
90
101
  export { MAX_KNOWLEDGE_BYTES } from './limits.js';
102
+ export { PluginVectorIndex, knowledgeScope, pluginArtifactId } from './plugin-index.js';
91
103
  export { joinedIndex } from './joined-index.js';
92
104
  export { canIndexKnowledge } from './policy.js';
93
105
  export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
94
- export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
106
+ export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, resolveTargetDocument, retrieveRagKnowledge, searchRagChunks, selectCoverage, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
@@ -1,13 +1,13 @@
1
1
  import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
2
- import { type RagExtractors, type VectorIndex } from 'rag-extract';
2
+ import { type VectorIndex } from 'rag-extract';
3
3
  import type { FileDocument } from '../processing/index.ts';
4
4
  import type { AbortSignal } from 'bare-abort-controller';
5
- import { type BuildEmbedder } from './build.ts';
5
+ import { type BuildEmbedder, type KnowledgeExtractors } from './build.ts';
6
6
  export interface HydratePassOptions {
7
7
  model: string;
8
8
  index: VectorIndex;
9
9
  embedder?: BuildEmbedder;
10
- extractors?: RagExtractors;
10
+ extractors?: KnowledgeExtractors;
11
11
  }
12
12
  export interface PickArtifactOptions {
13
13
  readonly model: string;
@@ -22,7 +22,10 @@ export declare class HydratePass {
22
22
  private readonly _index;
23
23
  private readonly _embedder;
24
24
  private readonly _extractors;
25
+ private _rebuilding;
26
+ private readonly _rebuildWaiters;
25
27
  constructor(opts: HydratePassOptions);
26
28
  run(document: FileDocument, signal: AbortSignal): Promise<void>;
27
29
  private _rebuild;
30
+ private _rebuildOne;
28
31
  }
@@ -21,6 +21,8 @@ export class HydratePass {
21
21
  _index;
22
22
  _embedder;
23
23
  _extractors;
24
+ _rebuilding = false;
25
+ _rebuildWaiters = new Map();
24
26
  constructor(opts) {
25
27
  this._model = opts.model;
26
28
  this._index = opts.index;
@@ -165,13 +167,32 @@ export class HydratePass {
165
167
  }
166
168
  }
167
169
  async _rebuild(document, artifact, signal) {
170
+ const embedder = this._embedder;
171
+ if (!embedder || (document.artifact(artifact.id).rebuildFailures ?? 0) > EMBED_RETRY_LIMIT) {
172
+ return;
173
+ }
174
+ if (this._rebuilding) {
175
+ this._rebuildWaiters.set(document.record.id, () => document.schedule(Date.now()));
176
+ return;
177
+ }
178
+ this._rebuilding = true;
179
+ document.admit();
180
+ try {
181
+ await this._rebuildOne(document, { artifact, embedder, signal });
182
+ }
183
+ finally {
184
+ this._rebuilding = false;
185
+ const waiters = [...this._rebuildWaiters.values()];
186
+ this._rebuildWaiters.clear();
187
+ for (const wake of waiters) {
188
+ wake();
189
+ }
190
+ }
191
+ }
192
+ async _rebuildOne(document, { artifact, embedder, signal }) {
168
193
  const knowledge = document.record;
169
194
  const local = document.artifact(artifact.id);
170
- if (!this._embedder ||
171
- (local.rebuildFailures ?? 0) > EMBED_RETRY_LIMIT ||
172
- document.paused() ||
173
- document.closed() ||
174
- signal.aborted) {
195
+ if (document.paused() || document.closed() || signal.aborted) {
175
196
  return;
176
197
  }
177
198
  const retryAt = document.output(knowledge.id, knowledge.blobId).blockedUntil();
@@ -200,7 +221,7 @@ export class HydratePass {
200
221
  }
201
222
  document.load({ state: 'running', reason: 'rebuilding local index' });
202
223
  const payload = await buildArtifactPayload({
203
- embedder: this._embedder,
224
+ embedder,
204
225
  extractors: this._extractors,
205
226
  knowledge,
206
227
  artifactId: artifact.id,
@@ -0,0 +1,32 @@
1
+ import type { KnowledgeQuery, KnowledgeService } from '@qvac/harness/lib/contract/knowledge.ts';
2
+ import type { VectorChunk, VectorHit, VectorIndex, VectorIndexInfo } from 'rag-extract';
3
+ export interface PluginHit extends VectorHit {
4
+ readonly artifactId: string;
5
+ }
6
+ export interface PluginIndexOptions {
7
+ readonly mesh: string;
8
+ readonly model: string;
9
+ }
10
+ export declare function knowledgeScope(mesh: string, chatId: string | null | undefined): string;
11
+ export declare function pluginArtifactId(mesh: string, artifactId: string): string;
12
+ export declare class PluginVectorIndex implements VectorIndex {
13
+ private readonly _knowledge;
14
+ private readonly _mesh;
15
+ private readonly _model;
16
+ private readonly _chunks;
17
+ private _dimensions;
18
+ private _listed;
19
+ private readonly _fed;
20
+ constructor(knowledge: KnowledgeService, { mesh, model }: PluginIndexOptions);
21
+ info(): VectorIndexInfo;
22
+ upsert(artifactId: string, chunks: ReadonlyArray<VectorChunk>): Promise<void>;
23
+ remove(artifactId: string): Promise<void>;
24
+ sweep(): Promise<void>;
25
+ scope(chatId: string | null): string;
26
+ pluginId(artifactId: string): string;
27
+ chunksOf(artifactId: string): number;
28
+ find(query: Omit<KnowledgeQuery, 'signal'>): Promise<PluginHit[]>;
29
+ search(): Promise<ReadonlyArray<VectorHit>>;
30
+ clear(): Promise<void>;
31
+ private _listedVersions;
32
+ }
@@ -0,0 +1,127 @@
1
+ import b4a from 'b4a';
2
+ import { contentHash } from 'rag-extract/artifact';
3
+ import { isRecord } from '../../../guards.js';
4
+ export function knowledgeScope(mesh, chatId) {
5
+ return chatId ? `chat:${mesh}:${chatId}` : `kb:${mesh}`;
6
+ }
7
+ export function pluginArtifactId(mesh, artifactId) {
8
+ return `${mesh}/${artifactId}`;
9
+ }
10
+ export class PluginVectorIndex {
11
+ _knowledge;
12
+ _mesh;
13
+ _model;
14
+ _chunks = new Map();
15
+ _dimensions = null;
16
+ _listed = null;
17
+ _fed = new Set();
18
+ constructor(knowledge, { mesh, model }) {
19
+ this._knowledge = knowledge;
20
+ this._mesh = mesh;
21
+ this._model = model;
22
+ }
23
+ info() {
24
+ const chunks = [...this._chunks.values()].reduce((sum, count) => sum + count, 0);
25
+ return { artifacts: this._chunks.size, chunks, dimensions: this._dimensions };
26
+ }
27
+ async upsert(artifactId, chunks) {
28
+ const dimensions = chunks[0]?.embedding.length ?? 0;
29
+ const id = pluginArtifactId(this._mesh, artifactId);
30
+ const version = payloadVersion(chunks);
31
+ this._fed.add(id);
32
+ const held = await this._listedVersions();
33
+ if (held.get(id) !== version) {
34
+ await this._knowledge.upsert({
35
+ artifactId: id,
36
+ version,
37
+ scope: knowledgeScope(this._mesh, chatIdOf(chunks[0])),
38
+ chunks: chunks.map(toKnowledgeChunk),
39
+ embedding: {
40
+ model: this._model,
41
+ dimensions,
42
+ vectors: chunks.map((chunk) => [...chunk.embedding])
43
+ }
44
+ });
45
+ held.set(id, version);
46
+ }
47
+ this._chunks.set(artifactId, chunks.length);
48
+ this._dimensions ??= dimensions || null;
49
+ }
50
+ async remove(artifactId) {
51
+ const id = pluginArtifactId(this._mesh, artifactId);
52
+ this._fed.delete(id);
53
+ await this._knowledge.remove(id);
54
+ (await this._listedVersions()).delete(id);
55
+ this._chunks.delete(artifactId);
56
+ }
57
+ async sweep() {
58
+ const held = await this._listedVersions();
59
+ for (const id of [...held.keys()]) {
60
+ if (!id.startsWith(`${this._mesh}/`) || this._fed.has(id))
61
+ continue;
62
+ held.delete(id);
63
+ await this._knowledge.remove(id);
64
+ }
65
+ }
66
+ scope(chatId) {
67
+ return knowledgeScope(this._mesh, chatId);
68
+ }
69
+ pluginId(artifactId) {
70
+ return pluginArtifactId(this._mesh, artifactId);
71
+ }
72
+ chunksOf(artifactId) {
73
+ return this._chunks.get(artifactId) ?? 0;
74
+ }
75
+ async find(query) {
76
+ const hits = await this._knowledge.search(query);
77
+ return hits.map((hit) => ({
78
+ artifactId: hit.artifactId,
79
+ id: hit.chunkId,
80
+ content: hit.text,
81
+ metadataJson: JSON.stringify(hit.metadata ?? {}),
82
+ score: hit.score
83
+ }));
84
+ }
85
+ search() {
86
+ return Promise.reject(new Error('the knowledge plugin is searched by scope, not by vector'));
87
+ }
88
+ async clear() {
89
+ this._chunks.clear();
90
+ }
91
+ _listedVersions() {
92
+ if (this._listed) {
93
+ return this._listed;
94
+ }
95
+ const listed = this._knowledge
96
+ .status()
97
+ .then(({ artifacts }) => new Map(artifacts.map((artifact) => [artifact.artifactId, artifact.version])));
98
+ listed.catch(() => {
99
+ this._listed = null;
100
+ });
101
+ this._listed = listed;
102
+ return listed;
103
+ }
104
+ }
105
+ function payloadVersion(chunks) {
106
+ const payload = chunks.map((chunk) => [chunk.id, chunk.content, chunk.metadataJson]);
107
+ return contentHash(b4a.from(JSON.stringify(payload)));
108
+ }
109
+ function chatIdOf(chunk) {
110
+ const metadata = metadataOf(chunk);
111
+ return typeof metadata.chatId === 'string' ? metadata.chatId : null;
112
+ }
113
+ function toKnowledgeChunk(chunk) {
114
+ return { id: chunk.id, text: chunk.content, metadata: metadataOf(chunk) };
115
+ }
116
+ function metadataOf(chunk) {
117
+ if (!chunk?.metadataJson) {
118
+ return {};
119
+ }
120
+ try {
121
+ const parsed = JSON.parse(chunk.metadataJson);
122
+ return isRecord(parsed) ? parsed : {};
123
+ }
124
+ catch {
125
+ return {};
126
+ }
127
+ }
@@ -1,6 +1,5 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
- import { type RagExtractors } from 'rag-extract';
3
- import { type BuildEmbedder } from './build.ts';
2
+ import { type BuildEmbedder, type KnowledgeExtractors } from './build.ts';
4
3
  import { type FileDocument } from '../processing/index.ts';
5
4
  export interface IndexingEmbedder extends BuildEmbedder {
6
5
  }
@@ -16,6 +15,9 @@ export declare const EMBED_RETRY_LIMIT = 5;
16
15
  export declare class IndexingPass {
17
16
  private readonly _embedder;
18
17
  private readonly _extractors;
19
- constructor(embedder: IndexingEmbedder, extractors?: RagExtractors);
18
+ constructor(embedder: IndexingEmbedder, extractors?: KnowledgeExtractors);
19
+ extract(document: FileDocument, signal: AbortSignal): Promise<void>;
20
+ private _bytes;
21
+ private _identity;
20
22
  run(document: FileDocument, signal: AbortSignal): Promise<void>;
21
23
  }
@@ -1,5 +1,5 @@
1
1
  import { classifyRagDocument, contentHash, documentIndexArtifactId, encodeRagArtifact, pipelineVersionFor, RAG_DOCUMENT_KIND } from 'rag-extract';
2
- import { buildArtifactPayload, EmbedderFailure, yieldTurn } from './build.js';
2
+ import { buildArtifactPayload, EmbedderFailure, prepareText, yieldTurn } from './build.js';
3
3
  import { isClaimLive } from '../processing/index.js';
4
4
  import { ClaimLostError } from '../processing/claims.js';
5
5
  import { contentFailure, failureDetail, failureError, isContentFailure } from './failure.js';
@@ -12,6 +12,75 @@ export class IndexingPass {
12
12
  this._embedder = embedder;
13
13
  this._extractors = extractors;
14
14
  }
15
+ async extract(document, signal) {
16
+ const knowledge = document.record;
17
+ if (!knowledge.chatId || document.state.artifactIdMemo === UNSUPPORTED) {
18
+ return;
19
+ }
20
+ const kept = await document.keptText();
21
+ if (kept) {
22
+ if (!kept.content.trim()) {
23
+ unreadable(document, { state: 'failed', reason: 'no readable text' });
24
+ }
25
+ return;
26
+ }
27
+ const bytes = await this._bytes(document, signal);
28
+ if (!bytes) {
29
+ return;
30
+ }
31
+ const { kind, refusal } = classify(knowledge, bytes);
32
+ if (refusal) {
33
+ refuse(document, refusal);
34
+ return;
35
+ }
36
+ if (kind === RAG_DOCUMENT_KIND.office && !this._extractors.office) {
37
+ unreadable(document, { state: 'failed', reason: 'extractor unavailable on this device' });
38
+ return;
39
+ }
40
+ const { artifactId } = this._identity(knowledge, bytes);
41
+ const prepared = await prepareText({
42
+ embedder: this._embedder,
43
+ extractors: this._extractors,
44
+ knowledge,
45
+ artifactId,
46
+ bytes,
47
+ signal
48
+ });
49
+ if (prepared.content.trim()) {
50
+ await document.keepText({ ...prepared, artifactId });
51
+ }
52
+ else {
53
+ unreadable(document, { state: 'failed', reason: 'no readable text' });
54
+ }
55
+ }
56
+ async _bytes(document, signal) {
57
+ const artifacts = await document.artifacts();
58
+ if (artifacts.some((row) => row.embedderVersion === pipelineVersionFor(this._embedder.model)) ||
59
+ document.source.blockedUntil() !== null ||
60
+ document.paused() ||
61
+ document.closed()) {
62
+ return null;
63
+ }
64
+ const read = await document.source.read(signal);
65
+ if (read.kind === 'malformed') {
66
+ unreadable(document, { state: 'failed', reason: 'invalid source reference' });
67
+ }
68
+ else if (read.kind === 'missing') {
69
+ document.schedule(read.retryAt);
70
+ document.work({ state: 'pending', reason: 'source unavailable', retryAt: read.retryAt });
71
+ }
72
+ return read.kind === 'bytes' ? read.bytes : null;
73
+ }
74
+ _identity(knowledge, bytes) {
75
+ const sourceHash = contentHash(bytes);
76
+ const artifactId = documentIndexArtifactId({
77
+ attachmentId: knowledge.id,
78
+ sourceBlobId: knowledge.blobId,
79
+ sourceHash,
80
+ embeddingModel: this._embedder.model
81
+ });
82
+ return { sourceHash, artifactId };
83
+ }
15
84
  async run(document, signal) {
16
85
  const knowledge = document.record;
17
86
  const pipelineVersion = pipelineVersionFor(this._embedder.model);
@@ -68,14 +137,7 @@ export class IndexingPass {
68
137
  return;
69
138
  }
70
139
  const bytes = read.bytes;
71
- const kind = classifyRagDocument({
72
- fileName: knowledge.fileName,
73
- mimeType: knowledge.mimeType,
74
- bytes
75
- });
76
- const refusal = contentFailure(knowledge.fileName, bytes, {
77
- supported: kind !== RAG_DOCUMENT_KIND.unsupported
78
- });
140
+ const { kind, refusal } = classify(knowledge, bytes);
79
141
  if (refusal?.reason === 'unsupported' && knowledge.chatId) {
80
142
  local.artifactIdMemo = UNSUPPORTED;
81
143
  document.work({
@@ -98,13 +160,7 @@ export class IndexingPass {
98
160
  });
99
161
  return;
100
162
  }
101
- const sourceHash = contentHash(bytes);
102
- const artifactId = documentIndexArtifactId({
103
- attachmentId: knowledge.id,
104
- sourceBlobId: knowledge.blobId,
105
- sourceHash,
106
- embeddingModel: this._embedder.model
107
- });
163
+ const { sourceHash, artifactId } = this._identity(knowledge, bytes);
108
164
  local.artifactIdMemo = artifactId;
109
165
  if (await document.claims.blockedByStatus(artifactId)) {
110
166
  return;
@@ -127,6 +183,7 @@ export class IndexingPass {
127
183
  if (refusal) {
128
184
  throw refusal;
129
185
  }
186
+ const kept = await document.keptText(artifactId);
130
187
  const payload = await buildArtifactPayload({
131
188
  embedder: this._embedder,
132
189
  extractors: this._extractors,
@@ -134,11 +191,13 @@ export class IndexingPass {
134
191
  artifactId,
135
192
  bytes,
136
193
  signal,
194
+ kept,
137
195
  onExtracting: (pages) => lease.keep(pages === 1),
138
196
  onExtracted: async () => {
139
197
  document.admit();
140
198
  await lease.keep(true);
141
199
  },
200
+ onChunked: kept ? undefined : (prepared) => document.keepText({ ...prepared, artifactId }),
142
201
  onProgress: async (processedChunks, totalChunks) => {
143
202
  lastTotal = totalChunks;
144
203
  document.report({
@@ -186,6 +245,7 @@ export class IndexingPass {
186
245
  });
187
246
  }
188
247
  catch (err) {
248
+ local.prepared = undefined;
189
249
  if (document.closed()) {
190
250
  return;
191
251
  }
@@ -237,3 +297,31 @@ export class IndexingPass {
237
297
  }
238
298
  }
239
299
  }
300
+ function classify(knowledge, bytes) {
301
+ const kind = classifyRagDocument({
302
+ fileName: knowledge.fileName,
303
+ mimeType: knowledge.mimeType,
304
+ bytes
305
+ });
306
+ const refusal = contentFailure(knowledge.fileName, bytes, {
307
+ supported: kind !== RAG_DOCUMENT_KIND.unsupported
308
+ });
309
+ return { kind, refusal };
310
+ }
311
+ function unreadable(document, work) {
312
+ document.state.artifactIdMemo = UNSUPPORTED;
313
+ document.work(work);
314
+ }
315
+ function refuse(document, refusal) {
316
+ if (refusal.reason !== 'unsupported') {
317
+ unreadable(document, { state: 'failed', reason: failureDetail(refusal) });
318
+ return;
319
+ }
320
+ unreadable(document, { state: 'not-applicable', reason: 'unsupported file type' });
321
+ document.report({
322
+ stage: 'failed',
323
+ processedChunks: 0,
324
+ totalChunks: 0,
325
+ error: `${document.record.fileName} is not a supported Knowledge file.`
326
+ });
327
+ }
@@ -1,5 +1,6 @@
1
1
  import type { QvacKnowledge } from '../../../../../spec/mesh/hyperschema/types.d.ts';
2
2
  import type { Mesh } from '../../../mesh.ts';
3
+ import type { KnowledgeExtractors } from '../build.ts';
3
4
  import { type VectorHit } from 'rag-extract';
4
5
  export declare function fullTextBudgetChars(ctxSize: number): number;
5
6
  export declare function retrievalBudgetTokens(ctxSize: number): number | undefined;
@@ -22,5 +23,6 @@ export declare function formatFullDocumentContext(hits: ReadonlyArray<VectorHit>
22
23
  export interface FullTextOptions {
23
24
  readonly budgetChars: number;
24
25
  readonly head?: boolean;
26
+ readonly extractors?: KnowledgeExtractors | undefined;
25
27
  }
26
- export declare function fullTextHits(mesh: Mesh, rows: ReadonlyArray<QvacKnowledge>, { budgetChars, head }: FullTextOptions): Promise<VectorHit[] | null>;
28
+ export declare function fullTextHits(mesh: Mesh, rows: ReadonlyArray<QvacKnowledge>, { budgetChars, head, extractors }: FullTextOptions): Promise<VectorHit[] | null>;