@qvac/core 1.0.0-rc.0 → 1.0.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1203 -1
- package/README.md +56 -45
- package/dist-lib/index.d.ts +3 -2
- package/dist-lib/index.js +1 -1
- package/dist-lib/lib/api/agent-folders.d.ts +22 -0
- package/dist-lib/lib/api/agent-folders.js +115 -0
- package/dist-lib/lib/api/agents.d.ts +20 -2
- package/dist-lib/lib/api/agents.js +128 -24
- package/dist-lib/lib/api/app-config.d.ts +7 -1
- package/dist-lib/lib/api/app-config.js +37 -6
- package/dist-lib/lib/api/approval-rules.d.ts +1 -1
- package/dist-lib/lib/api/approval-rules.js +12 -5
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +14 -2
- package/dist-lib/lib/api/chat-move.d.ts +4 -0
- package/dist-lib/lib/api/chat-move.js +7 -0
- package/dist-lib/lib/api/chat.d.ts +12 -2
- package/dist-lib/lib/api/chat.js +100 -9
- package/dist-lib/lib/api/chunk-delta.d.ts +12 -0
- package/dist-lib/lib/api/chunk-delta.js +115 -0
- package/dist-lib/lib/api/chunks.d.ts +19 -2
- package/dist-lib/lib/api/chunks.js +73 -13
- package/dist-lib/lib/api/config.d.ts +11 -3
- package/dist-lib/lib/api/config.js +51 -5
- package/dist-lib/lib/api/current-mesh.d.ts +13 -5
- package/dist-lib/lib/api/current-mesh.js +15 -11
- package/dist-lib/lib/api/custom-models.d.ts +7 -1
- package/dist-lib/lib/api/custom-models.js +12 -12
- package/dist-lib/lib/api/device-routes.d.ts +14 -0
- package/dist-lib/lib/api/device-routes.js +144 -0
- package/dist-lib/lib/api/devices.d.ts +3 -2
- package/dist-lib/lib/api/devices.js +28 -28
- package/dist-lib/lib/api/diagnostics.d.ts +2 -1
- package/dist-lib/lib/api/diagnostics.js +2 -2
- package/dist-lib/lib/api/execution.d.ts +16 -1
- package/dist-lib/lib/api/execution.js +72 -16
- package/dist-lib/lib/api/files.d.ts +1 -1
- package/dist-lib/lib/api/files.js +65 -8
- package/dist-lib/lib/api/host-run-log.d.ts +2 -41
- package/dist-lib/lib/api/host-run-log.js +35 -83
- package/dist-lib/lib/api/index.d.ts +39 -7
- package/dist-lib/lib/api/index.js +116 -36
- package/dist-lib/lib/api/instrument.d.ts +2 -2
- package/dist-lib/lib/api/instrument.js +24 -48
- package/dist-lib/lib/api/knowledge.d.ts +4 -1
- package/dist-lib/lib/api/knowledge.js +25 -7
- package/dist-lib/lib/api/local-files.d.ts +5 -1
- package/dist-lib/lib/api/local-files.js +9 -5
- package/dist-lib/lib/api/logs.js +3 -3
- package/dist-lib/lib/api/members.d.ts +10 -1
- package/dist-lib/lib/api/members.js +69 -44
- package/dist-lib/lib/api/mesh.d.ts +15 -6
- package/dist-lib/lib/api/mesh.js +13 -9
- package/dist-lib/lib/api/model-download.d.ts +2 -1
- package/dist-lib/lib/api/model-download.js +2 -2
- package/dist-lib/lib/api/network.d.ts +10 -2
- package/dist-lib/lib/api/network.js +8 -2
- package/dist-lib/lib/api/notifications.d.ts +3 -1
- package/dist-lib/lib/api/notifications.js +14 -9
- package/dist-lib/lib/api/projects.d.ts +20 -0
- package/dist-lib/lib/api/projects.js +172 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +17 -0
- package/dist-lib/lib/api/rag-plugin.js +44 -0
- package/dist-lib/lib/api/rag.d.ts +0 -1
- package/dist-lib/lib/api/rag.js +106 -33
- package/dist-lib/lib/api/served.d.ts +17 -0
- package/dist-lib/lib/api/served.js +65 -0
- package/dist-lib/lib/api/sync.d.ts +2 -1
- package/dist-lib/lib/api/sync.js +13 -3
- package/dist-lib/lib/api/tool-activity.d.ts +2 -1
- package/dist-lib/lib/api/tool-activity.js +4 -3
- package/dist-lib/lib/api/tool-configs.d.ts +2 -1
- package/dist-lib/lib/api/tool-configs.js +6 -9
- package/dist-lib/lib/api/upload-frames.d.ts +0 -1
- package/dist-lib/lib/api/upload-frames.js +0 -1
- package/dist-lib/lib/api/watchable.d.ts +3 -2
- package/dist-lib/lib/api/watchable.js +2 -3
- package/dist-lib/lib/api/workspace.d.ts +12 -0
- package/dist-lib/lib/api/workspace.js +61 -0
- package/dist-lib/lib/assistant/fake-models.d.ts +20 -0
- package/dist-lib/lib/assistant/fake-models.js +112 -0
- package/dist-lib/lib/assistant/harness.d.ts +48 -31
- package/dist-lib/lib/assistant/harness.js +51 -10
- package/dist-lib/lib/assistant/history.d.ts +37 -17
- package/dist-lib/lib/assistant/history.js +212 -87
- package/dist-lib/lib/assistant/host-run-log.d.ts +6 -1
- package/dist-lib/lib/assistant/host-run-log.js +24 -36
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +28 -1
- package/dist-lib/lib/assistant/message.js +65 -2
- package/dist-lib/lib/assistant/model-round.d.ts +118 -0
- package/dist-lib/lib/assistant/model-round.js +706 -0
- package/dist-lib/lib/assistant/pdf-budget.d.ts +11 -0
- package/dist-lib/lib/assistant/pdf-budget.js +12 -0
- package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant/round-host.d.ts +65 -0
- package/dist-lib/lib/assistant/round-host.js +332 -0
- package/dist-lib/lib/assistant/skill-encode.d.ts +2 -1
- package/dist-lib/lib/assistant/skill-encode.js +2 -1
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-client.d.ts +5 -1
- package/dist-lib/lib/assistant-client.js +54 -14
- package/dist-lib/lib/assistant-tree.d.ts +14 -3
- package/dist-lib/lib/assistant-tree.js +113 -21
- package/dist-lib/lib/assistant.d.ts +115 -25
- package/dist-lib/lib/assistant.js +2409 -1245
- package/dist-lib/lib/audio-format.js +1 -1
- package/dist-lib/lib/chat-suggestion.d.ts +3 -0
- package/dist-lib/lib/chat-suggestion.js +47 -0
- package/dist-lib/lib/client.d.ts +3 -2
- package/dist-lib/lib/client.js +18 -3
- package/dist-lib/lib/compaction.d.ts +3 -1
- package/dist-lib/lib/compaction.js +18 -4
- package/dist-lib/lib/constants.d.ts +10 -0
- package/dist-lib/lib/constants.js +10 -0
- package/dist-lib/lib/core/agent-copies.d.ts +9 -0
- package/dist-lib/lib/core/agent-copies.js +47 -0
- package/dist-lib/lib/core/agent-hosting.d.ts +25 -0
- package/dist-lib/lib/core/agent-hosting.js +76 -0
- package/dist-lib/lib/core/agent.d.ts +9 -1
- package/dist-lib/lib/core/agent.js +58 -63
- package/dist-lib/lib/core/approval-rules.d.ts +5 -1
- package/dist-lib/lib/core/approval-rules.js +17 -9
- package/dist-lib/lib/core/archived-mesh.d.ts +14 -7
- package/dist-lib/lib/core/archived-mesh.js +75 -59
- package/dist-lib/lib/core/async-iterable.js +1 -1
- package/dist-lib/lib/core/blind-peers.d.ts +1 -0
- package/dist-lib/lib/core/blind-peers.js +1 -0
- package/dist-lib/lib/core/chat-activity.d.ts +1 -1
- package/dist-lib/lib/core/chat-activity.js +45 -34
- package/dist-lib/lib/core/chat-copy.d.ts +42 -0
- package/dist-lib/lib/core/chat-copy.js +404 -0
- package/dist-lib/lib/core/chat-fork.d.ts +16 -0
- package/dist-lib/lib/core/chat-fork.js +246 -0
- package/dist-lib/lib/core/chat-group.js +10 -22
- package/dist-lib/lib/core/chat-move.d.ts +77 -0
- package/dist-lib/lib/core/chat-move.js +601 -0
- package/dist-lib/lib/core/chat-order.d.ts +2 -1
- package/dist-lib/lib/core/chat-order.js +28 -19
- package/dist-lib/lib/core/chat-read.d.ts +4 -0
- package/dist-lib/lib/core/chat-read.js +49 -0
- package/dist-lib/lib/core/chat-search.js +14 -12
- package/dist-lib/lib/core/chat.d.ts +32 -4
- package/dist-lib/lib/core/chat.js +135 -61
- package/dist-lib/lib/core/chunk.d.ts +35 -7
- package/dist-lib/lib/core/chunk.js +271 -72
- package/dist-lib/lib/core/collections.d.ts +41 -1
- package/dist-lib/lib/core/collections.js +41 -1
- package/dist-lib/lib/core/config-value.d.ts +7 -0
- package/dist-lib/lib/core/config-value.js +15 -0
- package/dist-lib/lib/core/contract/activity.d.ts +7 -2
- package/dist-lib/lib/core/contract/activity.js +71 -23
- package/dist-lib/lib/core/contract/agent-join.d.ts +10 -0
- package/dist-lib/lib/core/contract/agent-join.js +104 -0
- package/dist-lib/lib/core/contract/agents.d.ts +5 -0
- package/dist-lib/lib/core/contract/agents.js +142 -3
- package/dist-lib/lib/core/contract/chat.d.ts +4 -0
- package/dist-lib/lib/core/contract/chat.js +53 -3
- package/dist-lib/lib/core/contract/chunks.d.ts +7 -0
- package/dist-lib/lib/core/contract/chunks.js +125 -38
- package/dist-lib/lib/core/contract/devices.js +6 -4
- package/dist-lib/lib/core/contract/index.d.ts +2 -3
- package/dist-lib/lib/core/contract/index.js +38 -11
- package/dist-lib/lib/core/contract/indexing.d.ts +1 -0
- package/dist-lib/lib/core/contract/indexing.js +51 -7
- package/dist-lib/lib/core/contract/invites.d.ts +3 -14
- package/dist-lib/lib/core/contract/invites.js +26 -36
- package/dist-lib/lib/core/contract/joined-meshes.js +4 -1
- package/dist-lib/lib/core/contract/knowledge.d.ts +1 -0
- package/dist-lib/lib/core/contract/knowledge.js +57 -6
- package/dist-lib/lib/core/contract/members.d.ts +3 -1
- package/dist-lib/lib/core/contract/members.js +48 -12
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +3 -3
- package/dist-lib/lib/core/contract/mesh-kind.js +20 -16
- package/dist-lib/lib/core/contract/mesh-name.d.ts +3 -0
- package/dist-lib/lib/core/contract/mesh-name.js +31 -0
- package/dist-lib/lib/core/contract/mesh-rules.d.ts +8 -0
- package/dist-lib/lib/core/contract/mesh-rules.js +35 -0
- package/dist-lib/lib/core/contract/mesh-seeds.d.ts +2 -0
- package/dist-lib/lib/core/contract/mesh-seeds.js +26 -0
- package/dist-lib/lib/core/contract/model-requests.d.ts +26 -0
- package/dist-lib/lib/core/contract/model-requests.js +280 -0
- package/dist-lib/lib/core/contract/oldest-version.d.ts +9 -0
- package/dist-lib/lib/core/contract/oldest-version.js +20 -0
- package/dist-lib/lib/core/contract/open-join.d.ts +2 -0
- package/dist-lib/lib/core/contract/open-join.js +77 -0
- package/dist-lib/lib/core/contract/project-device-keys.d.ts +9 -0
- package/dist-lib/lib/core/contract/project-device-keys.js +38 -0
- package/dist-lib/lib/core/contract/project-host-keys.d.ts +9 -0
- package/dist-lib/lib/core/contract/project-host-keys.js +52 -0
- package/dist-lib/lib/core/contract/project-order.d.ts +2 -0
- package/dist-lib/lib/core/contract/project-order.js +25 -0
- package/dist-lib/lib/core/contract/projects.d.ts +23 -0
- package/dist-lib/lib/core/contract/projects.js +339 -0
- package/dist-lib/lib/core/contract/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/contract/public-mesh.js +31 -0
- package/dist-lib/lib/core/contract/public-seeds.d.ts +9 -0
- package/dist-lib/lib/core/contract/public-seeds.js +43 -0
- package/dist-lib/lib/core/contract/roles.d.ts +10 -0
- package/dist-lib/lib/core/contract/roles.js +123 -0
- package/dist-lib/lib/core/contract/run-requests.d.ts +11 -0
- package/dist-lib/lib/core/contract/run-requests.js +114 -0
- package/dist-lib/lib/core/contract/seed-defaults.d.ts +3 -1
- package/dist-lib/lib/core/contract/seed-defaults.js +17 -4
- package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
- package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
- package/dist-lib/lib/core/contract/skill-secrets.d.ts +3 -0
- package/dist-lib/lib/core/contract/skill-secrets.js +32 -0
- package/dist-lib/lib/core/contract/tool-activity.js +12 -1
- package/dist-lib/lib/core/contract/tool-configs.js +2 -1
- package/dist-lib/lib/core/contract/types.d.ts +2 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +31 -0
- package/dist-lib/lib/core/contract/workspace.js +311 -0
- package/dist-lib/lib/core/contract/writers.js +100 -16
- package/dist-lib/lib/core/deleted-agent-runs.d.ts +8 -0
- package/dist-lib/lib/core/deleted-agent-runs.js +57 -0
- package/dist-lib/lib/core/departures.d.ts +18 -5
- package/dist-lib/lib/core/departures.js +82 -6
- package/dist-lib/lib/core/device-control.d.ts +21 -0
- package/dist-lib/lib/core/device-control.js +91 -0
- package/dist-lib/lib/core/device-seal.d.ts +4 -0
- package/dist-lib/lib/core/device-seal.js +22 -0
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +6 -11
- package/dist-lib/lib/core/document-attachment.d.ts +10 -0
- package/dist-lib/lib/core/document-attachment.js +14 -0
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +23 -7
- package/dist-lib/lib/core/files/index.d.ts +28 -15
- package/dist-lib/lib/core/files/index.js +222 -43
- package/dist-lib/lib/core/files/knowledge.d.ts +10 -1
- package/dist-lib/lib/core/files/knowledge.js +77 -5
- package/dist-lib/lib/core/files/preview/index.js +6 -2
- package/dist-lib/lib/core/files/processing/claims.d.ts +7 -0
- package/dist-lib/lib/core/files/processing/claims.js +33 -7
- package/dist-lib/lib/core/files/processing/document.d.ts +8 -0
- package/dist-lib/lib/core/files/processing/document.js +26 -2
- package/dist-lib/lib/core/files/processing/index.d.ts +12 -3
- package/dist-lib/lib/core/files/processing/index.js +122 -73
- package/dist-lib/lib/core/files/processing/protocol.d.ts +7 -1
- package/dist-lib/lib/core/files/processing/protocol.js +27 -37
- package/dist-lib/lib/core/files/processing/source.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/source.js +12 -4
- package/dist-lib/lib/core/files/processing/state.d.ts +5 -1
- package/dist-lib/lib/core/files/processing/state.js +9 -1
- package/dist-lib/lib/core/files/rag/build.d.ts +15 -2
- package/dist-lib/lib/core/files/rag/build.js +119 -21
- package/dist-lib/lib/core/files/rag/failure.d.ts +20 -0
- package/dist-lib/lib/core/files/rag/failure.js +58 -0
- package/dist-lib/lib/core/files/rag/index.d.ts +21 -5
- package/dist-lib/lib/core/files/rag/index.js +38 -9
- package/dist-lib/lib/core/files/rag/joined-index.d.ts +6 -0
- package/dist-lib/lib/core/files/rag/joined-index.js +35 -0
- package/dist-lib/lib/core/files/rag/limits.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/limits.js +1 -0
- package/dist-lib/lib/core/files/rag/load.d.ts +3 -3
- package/dist-lib/lib/core/files/rag/load.js +6 -3
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +120 -0
- package/dist-lib/lib/core/files/rag/policy.d.ts +10 -1
- package/dist-lib/lib/core/files/rag/policy.js +23 -2
- package/dist-lib/lib/core/files/rag/produce.d.ts +3 -4
- package/dist-lib/lib/core/files/rag/produce.js +31 -18
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +8 -2
- package/dist-lib/lib/core/files/rag/similarity.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/similarity.js +2 -3
- package/dist-lib/lib/core/files/records/chunks.js +22 -1
- package/dist-lib/lib/core/files/records/index.d.ts +1 -1
- package/dist-lib/lib/core/files/records/index.js +1 -1
- package/dist-lib/lib/core/files/records/knowledge.d.ts +12 -4
- package/dist-lib/lib/core/files/records/knowledge.js +34 -30
- package/dist-lib/lib/core/files/records/purge.js +13 -8
- package/dist-lib/lib/core/files/records/reclaim.js +21 -55
- package/dist-lib/lib/core/files/records/routing.d.ts +1 -1
- package/dist-lib/lib/core/files/records/routing.js +6 -2
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/records/sweep.d.ts +3 -0
- package/dist-lib/lib/core/files/records/sweep.js +9 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +2 -0
- package/dist-lib/lib/core/files/store/blob.js +2 -2
- package/dist-lib/lib/core/files/store/index.d.ts +3 -1
- package/dist-lib/lib/core/files/store/index.js +4 -1
- package/dist-lib/lib/core/files/store/reference.d.ts +3 -0
- package/dist-lib/lib/core/files/store/reference.js +12 -2
- package/dist-lib/lib/core/files/store/sweep.js +3 -1
- package/dist-lib/lib/core/files/upload.d.ts +5 -1
- package/dist-lib/lib/core/files/upload.js +117 -51
- package/dist-lib/lib/core/first-event.d.ts +6 -0
- package/dist-lib/lib/core/first-event.js +13 -0
- package/dist-lib/lib/core/fs-drives.d.ts +6 -3
- package/dist-lib/lib/core/fs-drives.js +35 -35
- package/dist-lib/lib/core/helpers/index.d.ts +1 -1
- package/dist-lib/lib/core/helpers/index.js +2 -2
- package/dist-lib/lib/core/id.d.ts +1 -0
- package/dist-lib/lib/core/id.js +3 -0
- package/dist-lib/lib/core/identity.d.ts +3 -1
- package/dist-lib/lib/core/identity.js +25 -4
- package/dist-lib/lib/core/invite.d.ts +6 -2
- package/dist-lib/lib/core/invite.js +22 -8
- package/dist-lib/lib/core/local/files.d.ts +2 -1
- package/dist-lib/lib/core/local/files.js +5 -7
- package/dist-lib/lib/core/local/index.d.ts +93 -8
- package/dist-lib/lib/core/local/index.js +327 -28
- package/dist-lib/lib/core/local/run-logs.d.ts +38 -0
- package/dist-lib/lib/core/local/run-logs.js +132 -0
- package/dist-lib/lib/core/member-key.d.ts +7 -0
- package/dist-lib/lib/core/member-key.js +24 -0
- package/dist-lib/lib/core/membership.d.ts +3 -14
- package/dist-lib/lib/core/membership.js +3 -96
- package/dist-lib/lib/core/mesh.d.ts +132 -29
- package/dist-lib/lib/core/mesh.js +775 -215
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.d.ts +2 -0
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +18 -0
- package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
- package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +8 -6
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +69 -8
- package/dist-lib/lib/core/migrations/writer-links.d.ts +3 -0
- package/dist-lib/lib/core/migrations/writer-links.js +30 -0
- package/dist-lib/lib/core/model-requests.d.ts +13 -0
- package/dist-lib/lib/core/model-requests.js +65 -0
- package/dist-lib/lib/core/next-frame.js +2 -1
- package/dist-lib/lib/core/notification.js +100 -25
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +12 -3
- package/dist-lib/lib/core/pdf/rasterize.js +77 -28
- package/dist-lib/lib/core/peer-duplex.d.ts +34 -0
- package/dist-lib/lib/core/{harness-loan.js → peer-duplex.js} +104 -45
- package/dist-lib/lib/core/project-keys.d.ts +16 -0
- package/dist-lib/lib/core/project-keys.js +65 -0
- package/dist-lib/lib/core/prompt-history.js +4 -4
- package/dist-lib/lib/core/protocol-pairing.d.ts +11 -0
- package/dist-lib/lib/core/protocol-pairing.js +22 -0
- package/dist-lib/lib/core/public-mesh.d.ts +25 -0
- package/dist-lib/lib/core/public-mesh.js +81 -0
- package/dist-lib/lib/core/purge.d.ts +3 -0
- package/dist-lib/lib/core/purge.js +7 -0
- package/dist-lib/lib/core/run-driver.d.ts +17 -0
- package/dist-lib/lib/core/run-driver.js +20 -0
- package/dist-lib/lib/core/run-requests.d.ts +14 -0
- package/dist-lib/lib/core/run-requests.js +78 -0
- package/dist-lib/lib/core/send.d.ts +2 -0
- package/dist-lib/lib/core/send.js +17 -4
- package/dist-lib/lib/core/senders.d.ts +19 -0
- package/dist-lib/lib/core/senders.js +45 -0
- package/dist-lib/lib/core/shared-agents.d.ts +40 -0
- package/dist-lib/lib/core/shared-agents.js +283 -0
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/skill-secrets.d.ts +13 -18
- package/dist-lib/lib/core/skill-secrets.js +93 -72
- package/dist-lib/lib/core/sync/background.d.ts +34 -0
- package/dist-lib/lib/core/sync/background.js +86 -0
- package/dist-lib/lib/core/sync/changed.d.ts +20 -0
- package/dist-lib/lib/core/sync/changed.js +7 -0
- package/dist-lib/lib/core/sync/devices.js +6 -3
- package/dist-lib/lib/core/sync/files.d.ts +1 -0
- package/dist-lib/lib/core/sync/files.js +50 -72
- package/dist-lib/lib/core/sync/index.d.ts +1 -1
- package/dist-lib/lib/core/sync/index.js +193 -81
- package/dist-lib/lib/core/sync/view-updates.d.ts +39 -0
- package/dist-lib/lib/core/sync/view-updates.js +124 -0
- package/dist-lib/lib/core/sync-status.js +99 -43
- package/dist-lib/lib/core/system-info.d.ts +2 -2
- package/dist-lib/lib/core/system-info.js +10 -11
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.d.ts +5 -3
- package/dist-lib/lib/core/tool-configs.js +27 -39
- package/dist-lib/lib/core/transcript-visibility.d.ts +5 -5
- package/dist-lib/lib/core/transcript-visibility.js +27 -31
- package/dist-lib/lib/core/upgrade-agents.d.ts +11 -0
- package/dist-lib/lib/core/upgrade-agents.js +78 -0
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/vault-managed-skills.d.ts +18 -0
- package/dist-lib/lib/core/vault-managed-skills.js +105 -0
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +23 -0
- package/dist-lib/lib/core/workspace-upgrade.js +51 -0
- package/dist-lib/lib/core.d.ts +345 -88
- package/dist-lib/lib/core.js +4275 -1513
- package/dist-lib/lib/credential-pusher.d.ts +17 -7
- package/dist-lib/lib/credential-pusher.js +111 -62
- package/dist-lib/lib/custom-model-pusher.d.ts +2 -4
- package/dist-lib/lib/custom-model-task.d.ts +2 -1
- package/dist-lib/lib/custom-model-task.js +2 -2
- package/dist-lib/lib/device-tree.d.ts +18 -0
- package/dist-lib/lib/device-tree.js +32 -0
- package/dist-lib/lib/diagnostics.d.ts +7 -0
- package/dist-lib/lib/diagnostics.js +22 -0
- package/dist-lib/lib/doctor/checks.d.ts +15 -0
- package/dist-lib/lib/doctor/checks.js +272 -0
- package/dist-lib/lib/doctor/redact.d.ts +4 -0
- package/dist-lib/lib/doctor/redact.js +31 -0
- package/dist-lib/lib/doctor/render.d.ts +8 -0
- package/dist-lib/lib/doctor/render.js +83 -0
- package/dist-lib/lib/doctor/tree.d.ts +5 -0
- package/dist-lib/lib/doctor/tree.js +167 -0
- package/dist-lib/lib/downloads-allowed-pusher.d.ts +8 -0
- package/dist-lib/lib/downloads-allowed-pusher.js +37 -0
- package/dist-lib/lib/errors.d.ts +814 -0
- package/dist-lib/lib/errors.js +748 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +8 -1
- package/dist-lib/lib/fs-drive-bridge.js +312 -31
- package/dist-lib/lib/guards.d.ts +1 -0
- package/dist-lib/lib/guards.js +4 -0
- package/dist-lib/lib/inference-calls.d.ts +1 -2
- package/dist-lib/lib/inference-calls.js +2 -10
- package/dist-lib/lib/integrations/setup-routes.js +7 -5
- package/dist-lib/lib/knowledge-search.d.ts +1 -1
- package/dist-lib/lib/knowledge-search.js +5 -51
- package/dist-lib/lib/kv-cache-ledger.d.ts +22 -0
- package/dist-lib/lib/kv-cache-ledger.js +72 -0
- package/dist-lib/lib/kv-cache-reclaim.d.ts +2 -1
- package/dist-lib/lib/kv-cache-reclaim.js +30 -0
- package/dist-lib/lib/kv-cache-retention.d.ts +36 -0
- package/dist-lib/lib/kv-cache-retention.js +101 -0
- package/dist-lib/lib/latest-frame.d.ts +10 -0
- package/dist-lib/lib/latest-frame.js +48 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/log.d.ts +7 -6
- package/dist-lib/lib/log.js +14 -7
- package/dist-lib/lib/logger-type.d.ts +2 -5
- package/dist-lib/lib/mesh-scoped-engine.js +47 -2
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/model-download-status.d.ts +3 -3
- package/dist-lib/lib/model-download.d.ts +22 -13
- package/dist-lib/lib/model-download.js +106 -37
- package/dist-lib/lib/model-idle-pusher.d.ts +2 -4
- package/dist-lib/lib/model-memory-cap.d.ts +30 -0
- package/dist-lib/lib/model-memory-cap.js +78 -0
- package/dist-lib/lib/models-watch.d.ts +1 -1
- package/dist-lib/lib/models-watch.js +4 -4
- package/dist-lib/lib/oauth/browser-connect.js +3 -2
- package/dist-lib/lib/oauth/flow-core.js +5 -3
- package/dist-lib/lib/oauth/flow.js +1 -1
- package/dist-lib/lib/oauth/loopback.d.ts +1 -0
- package/dist-lib/lib/oauth/loopback.js +57 -12
- package/dist-lib/lib/oauth/providers.js +4 -2
- package/dist-lib/lib/oauth/redirect-types.d.ts +1 -0
- package/dist-lib/lib/oauth/register.js +35 -19
- package/dist-lib/lib/oauth/request.d.ts +10 -0
- package/dist-lib/lib/oauth/request.js +22 -0
- package/dist-lib/lib/oauth/resolve.js +42 -13
- package/dist-lib/lib/oauth/token-exchange-core.js +40 -31
- package/dist-lib/lib/oauth/types.d.ts +3 -0
- package/dist-lib/lib/oauth.d.ts +2 -1
- package/dist-lib/lib/oauth.js +1 -0
- package/dist-lib/lib/persistent-dir.d.ts +1 -0
- package/dist-lib/lib/persistent-dir.js +9 -0
- package/dist-lib/lib/probe-rounds.d.ts +8 -0
- package/dist-lib/lib/probe-rounds.js +10 -0
- package/dist-lib/lib/reconnecting-watch.js +19 -11
- package/dist-lib/lib/run-input.d.ts +14 -0
- package/dist-lib/lib/run-input.js +33 -0
- package/dist-lib/lib/run-notices.d.ts +5 -0
- package/dist-lib/lib/run-notices.js +13 -2
- package/dist-lib/lib/run-replay.d.ts +45 -0
- package/dist-lib/lib/run-replay.js +169 -0
- package/dist-lib/lib/run-transcript.d.ts +13 -12
- package/dist-lib/lib/run-transcript.js +20 -19
- package/dist-lib/lib/suspend-lifecycle.js +15 -5
- package/dist-lib/lib/task.js +6 -4
- package/dist-lib/lib/test-env.d.ts +1 -0
- package/dist-lib/lib/test-env.js +4 -0
- package/dist-lib/schema/build.js +66 -55
- package/dist-lib/schema/config.d.ts +1 -1
- package/dist-lib/schema/config.js +1 -1
- package/dist-lib/schema/content.d.ts +5 -1
- package/dist-lib/schema/content.js +60 -12
- package/dist-lib/schema/engine.d.ts +1 -0
- package/dist-lib/schema/engine.js +941 -152
- package/dist-lib/schema/field-order.js +8 -1
- package/dist-lib/schema/indexes.d.ts +10 -0
- package/dist-lib/schema/indexes.js +53 -3
- package/dist-lib/schema/local.js +83 -5
- package/dist-lib/schema/qvac.d.ts +4 -1
- package/dist-lib/schema/qvac.js +558 -16
- package/dist-lib/spec/bind.js +261 -48
- package/dist-lib/spec/calls.js +56 -5
- package/dist-lib/spec/capabilities.d.ts +129 -20
- package/dist-lib/spec/config/hyperconf/index.js +1 -1
- package/dist-lib/spec/hrpc/hrpc.json +870 -207
- package/dist-lib/spec/hrpc/index.js +969 -347
- package/dist-lib/spec/hrpc/messages.js +7590 -4896
- package/dist-lib/spec/hyperschema/index.js +7590 -4896
- package/dist-lib/spec/hyperschema/schema.json +5316 -3224
- package/dist-lib/spec/hyperschema/types.d.ts +576 -62
- package/dist-lib/spec/local/hyperdb/db.json +95 -1
- package/dist-lib/spec/local/hyperdb/index.js +459 -2
- package/dist-lib/spec/local/hyperdb/messages.js +1588 -488
- package/dist-lib/spec/local/hyperschema/index.js +584 -16
- package/dist-lib/spec/local/hyperschema/schema.json +369 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +88 -2
- package/dist-lib/spec/mesh/hyperdb/db.json +711 -9
- package/dist-lib/spec/mesh/hyperdb/index.js +2769 -7
- package/dist-lib/spec/mesh/hyperdb/messages.js +7140 -1298
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +205 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +579 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +4017 -267
- package/dist-lib/spec/mesh/hyperschema/index.js +4017 -267
- package/dist-lib/spec/mesh/hyperschema/schema.json +2193 -19
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +550 -6
- package/package.json +136 -90
- package/tui/README.md +119 -20
- package/tui/app.mjs +2000 -621
- package/tui/boot.mjs +74 -9
- package/tui/cli.mjs +93 -0
- package/tui/index.mjs +2 -51
- package/tui/lib/connect-sheet.mjs +350 -0
- package/tui/lib/device-target.mjs +64 -0
- package/tui/lib/folders-command.mjs +84 -0
- package/tui/lib/fork-command.mjs +43 -0
- package/tui/lib/knowledge-import.mjs +3 -21
- package/tui/lib/local-path.mjs +21 -0
- package/tui/lib/move-command.mjs +17 -0
- package/tui/lib/render.mjs +30 -1
- package/tui/lib/setup.mjs +59 -25
- package/tui/lib/skill-connect.mjs +145 -0
- package/tui/lib/updater.mjs +86 -0
- package/tui/main.mjs +44 -18
- package/tui/plugins.mjs +158 -0
- package/worker/assistant-sidecar.mjs +28 -4
- package/worker/stow-entry.mjs +3 -50
- package/dist-lib/lib/core/contract/compute.d.ts +0 -11
- package/dist-lib/lib/core/contract/compute.js +0 -48
- package/dist-lib/lib/core/device-invites.d.ts +0 -11
- package/dist-lib/lib/core/device-invites.js +0 -63
- package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
- package/dist-lib/lib/core/files/processing/settle.js +0 -86
- package/dist-lib/lib/core/grant-expiry.d.ts +0 -11
- package/dist-lib/lib/core/grant-expiry.js +0 -66
- package/dist-lib/lib/core/handoff-channel.d.ts +0 -47
- package/dist-lib/lib/core/handoff-channel.js +0 -123
- package/dist-lib/lib/core/harness-loan.d.ts +0 -29
- package/dist-lib/lib/core/host-run-log-store.d.ts +0 -114
- package/dist-lib/lib/core/host-run-log-store.js +0 -976
- package/dist-lib/lib/core/hosted-agents.d.ts +0 -38
- package/dist-lib/lib/core/hosted-agents.js +0 -140
- package/dist-lib/lib/core/inference-calls-handoff.d.ts +0 -20
- package/dist-lib/lib/core/inference-calls-handoff.js +0 -37
- package/dist-lib/lib/core/inference-calls-receiver.d.ts +0 -14
- package/dist-lib/lib/core/inference-calls-receiver.js +0 -42
- package/dist-lib/lib/core/mesh-seed-handoff.d.ts +0 -25
- package/dist-lib/lib/core/mesh-seed-handoff.js +0 -52
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
- package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
- package/dist-lib/lib/core/secret-handoff.d.ts +0 -17
- package/dist-lib/lib/core/secret-handoff.js +0 -29
- package/dist-lib/lib/core/skill-overlay-handoff.d.ts +0 -25
- package/dist-lib/lib/core/skill-overlay-handoff.js +0 -50
- package/dist-lib/lib/core/skill-overlay-receiver.d.ts +0 -8
- package/dist-lib/lib/core/skill-overlay-receiver.js +0 -23
- package/dist-lib/lib/image-cache-dir.d.ts +0 -1
- package/dist-lib/lib/image-cache-dir.js +0 -5
- package/dist-lib/lib/log-levels.d.ts +0 -7
- package/dist-lib/lib/log-levels.js +0 -1
- package/dist-lib/lib/logger.d.ts +0 -5
- package/dist-lib/lib/logger.js +0 -18
- package/dist-lib/lib/oauth/loopback.stub.d.ts +0 -3
- package/dist-lib/lib/oauth/loopback.stub.js +0 -6
|
@@ -1,15 +1,17 @@
|
|
|
1
|
-
import { isInvalidBlobReferenceError, compareBlobReferences } from '../store/index.js';
|
|
1
|
+
import { isInvalidBlobReferenceError, isPendingBlobReference, compareBlobReferences } from '../store/index.js';
|
|
2
2
|
import { AbortController } from 'bare-abort-controller';
|
|
3
3
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
4
4
|
const BLOB_RETRY_MS = 30_000;
|
|
5
5
|
export class IndexingSource {
|
|
6
6
|
_readBytes;
|
|
7
|
+
_log;
|
|
7
8
|
_retryMs;
|
|
8
9
|
_download;
|
|
9
10
|
_downloads = new Map();
|
|
10
11
|
retry;
|
|
11
12
|
constructor(readBytes, opts) {
|
|
12
13
|
this._readBytes = readBytes;
|
|
14
|
+
this._log = opts.log;
|
|
13
15
|
this._retryMs = opts.retryMs ?? BLOB_RETRY_MS;
|
|
14
16
|
this.retry = opts.retries;
|
|
15
17
|
this._download = opts.download;
|
|
@@ -27,6 +29,9 @@ export class IndexingSource {
|
|
|
27
29
|
return this.retry.reconnect(onRetry);
|
|
28
30
|
}
|
|
29
31
|
async read(id, blobRef, signal) {
|
|
32
|
+
if (isPendingBlobReference(blobRef)) {
|
|
33
|
+
return { kind: 'missing', retryAt: Date.now() + this._retryMs };
|
|
34
|
+
}
|
|
30
35
|
try {
|
|
31
36
|
const bytes = await this._readBytes(blobRef, { signal, wait: !this._download });
|
|
32
37
|
this.retry.markFound(id);
|
|
@@ -37,10 +42,10 @@ export class IndexingSource {
|
|
|
37
42
|
throw new AbortedError();
|
|
38
43
|
}
|
|
39
44
|
if (isInvalidBlobReferenceError(err)) {
|
|
40
|
-
|
|
45
|
+
this._log?.warn('rag', 'blob ref is malformed, never readable', { id });
|
|
41
46
|
return { kind: 'malformed' };
|
|
42
47
|
}
|
|
43
|
-
|
|
48
|
+
this._log?.debug('rag', 'blob bytes unavailable', { id, err });
|
|
44
49
|
this.retry.markMissing(id);
|
|
45
50
|
const retryAt = this.retryMiss(id, Date.now());
|
|
46
51
|
this._fetch(id, blobRef);
|
|
@@ -65,6 +70,9 @@ export class IndexingSource {
|
|
|
65
70
|
}
|
|
66
71
|
return false;
|
|
67
72
|
}
|
|
73
|
+
awaitsBytes(matches) {
|
|
74
|
+
return this.pending(matches) || this.retry.missing(matches);
|
|
75
|
+
}
|
|
68
76
|
_fetch(id, reference) {
|
|
69
77
|
const download = this._download;
|
|
70
78
|
if (!download) {
|
|
@@ -88,7 +96,7 @@ export class IndexingSource {
|
|
|
88
96
|
download.ready(id);
|
|
89
97
|
}, (err) => {
|
|
90
98
|
if (!abort.signal.aborted) {
|
|
91
|
-
|
|
99
|
+
this._log?.debug('rag', 'blob download interrupted', { id, err });
|
|
92
100
|
}
|
|
93
101
|
})
|
|
94
102
|
.finally(() => {
|
|
@@ -4,6 +4,9 @@ export type LocalWork = Pick<AssistantFileProcessorStatus, 'state' | 'reason' |
|
|
|
4
4
|
export type IndexingProgressEntry = Omit<Parameters<IndexingProgress>[0], 'error'> & {
|
|
5
5
|
error: string | null;
|
|
6
6
|
};
|
|
7
|
+
export type ChatProgressEntry = IndexingProgressEntry & {
|
|
8
|
+
readonly chatId: string;
|
|
9
|
+
};
|
|
7
10
|
export interface LoadedArtifact {
|
|
8
11
|
artifactId: string;
|
|
9
12
|
blobRef?: string;
|
|
@@ -42,6 +45,7 @@ export declare class FileRetries {
|
|
|
42
45
|
blockedUntil(id: string, now: number): number | null;
|
|
43
46
|
miss(id: string, now: number, retryMs: number): number;
|
|
44
47
|
markMissing(id: string): void;
|
|
48
|
+
missing(matches: (id: string) => boolean): boolean;
|
|
45
49
|
markFound(id: string): void;
|
|
46
50
|
clear(id?: string): void;
|
|
47
51
|
reconnect(onRetry?: (id: string, deadline: number) => void): number | null;
|
|
@@ -58,7 +62,7 @@ export declare class FileState {
|
|
|
58
62
|
forget(id: string): DocumentState | undefined;
|
|
59
63
|
get(id: string): Readonly<DocumentState> | undefined;
|
|
60
64
|
readArtifact(id: string): Readonly<ArtifactState> | undefined;
|
|
61
|
-
progress(chatId: string):
|
|
65
|
+
progress(chatId: string): ChatProgressEntry[];
|
|
62
66
|
clearLoaded(): void;
|
|
63
67
|
retry(id: string): void;
|
|
64
68
|
reset(): void;
|
|
@@ -15,6 +15,13 @@ export class FileRetries {
|
|
|
15
15
|
markMissing(id) {
|
|
16
16
|
this._missing.add(id);
|
|
17
17
|
}
|
|
18
|
+
missing(matches) {
|
|
19
|
+
for (const id of this._missing) {
|
|
20
|
+
if (matches(id))
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
18
25
|
markFound(id) {
|
|
19
26
|
this._missing.delete(id);
|
|
20
27
|
}
|
|
@@ -82,7 +89,7 @@ export class FileState {
|
|
|
82
89
|
return this._artifacts.get(id);
|
|
83
90
|
}
|
|
84
91
|
progress(chatId) {
|
|
85
|
-
return [...this._documents.values()].flatMap((entry) => entry.progress?.chatId === chatId ? [entry.progress] : []);
|
|
92
|
+
return [...this._documents.values()].flatMap((entry) => entry.progress?.chatId === chatId ? [{ ...entry.progress, chatId }] : []);
|
|
86
93
|
}
|
|
87
94
|
clearLoaded() {
|
|
88
95
|
for (const entry of this._documents.values()) {
|
|
@@ -109,6 +116,7 @@ export class FileState {
|
|
|
109
116
|
output.rebuildFailures = undefined;
|
|
110
117
|
this.outputRetries.clear(artifactId);
|
|
111
118
|
}
|
|
119
|
+
entry.artifactIdMemo = undefined;
|
|
112
120
|
}
|
|
113
121
|
reset() {
|
|
114
122
|
this._documents.clear();
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
3
|
+
import type { PdfPage } from '@qvac/harness/pdf-pages';
|
|
3
4
|
import { type RagArtifactPayload, type RagExtractors } from 'rag-extract';
|
|
4
5
|
export interface BuildEmbedder {
|
|
5
6
|
model: string;
|
|
@@ -7,11 +8,20 @@ export interface BuildEmbedder {
|
|
|
7
8
|
texts: string[];
|
|
8
9
|
signal?: AbortSignal;
|
|
9
10
|
}): Promise<number[][]>;
|
|
11
|
+
idle?(signal: AbortSignal): Promise<void>;
|
|
10
12
|
}
|
|
11
|
-
export type BuildProgress = (processedChunks: number, totalChunks: number) => void
|
|
13
|
+
export type BuildProgress = (processedChunks: number, totalChunks: number) => void | Promise<void>;
|
|
12
14
|
export declare class EmbedderFailure extends Error {
|
|
13
15
|
constructor(cause: unknown);
|
|
14
16
|
}
|
|
17
|
+
export interface ReadPdfPagesOptions {
|
|
18
|
+
readonly signal: AbortSignal;
|
|
19
|
+
readonly beforeOcr?: () => Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export type ReadPdfPages = (bytes: Uint8Array, options: ReadPdfPagesOptions) => Promise<readonly PdfPage[]>;
|
|
22
|
+
export interface KnowledgeExtractors extends RagExtractors {
|
|
23
|
+
readonly pdfPages?: ReadPdfPages;
|
|
24
|
+
}
|
|
15
25
|
export interface ArtifactBuildInput {
|
|
16
26
|
embedder: BuildEmbedder;
|
|
17
27
|
knowledge: QvacKnowledge;
|
|
@@ -19,6 +29,9 @@ export interface ArtifactBuildInput {
|
|
|
19
29
|
bytes: Buffer;
|
|
20
30
|
signal: AbortSignal;
|
|
21
31
|
onProgress?: BuildProgress;
|
|
22
|
-
|
|
32
|
+
onExtracting?: (pages: number, totalPages: number) => Promise<void>;
|
|
33
|
+
onExtracted?: () => void | Promise<void>;
|
|
34
|
+
extractors?: KnowledgeExtractors;
|
|
23
35
|
}
|
|
36
|
+
export declare function yieldTurn(signal: AbortSignal): Promise<void>;
|
|
24
37
|
export declare function buildArtifactPayload(input: ArtifactBuildInput): Promise<RagArtifactPayload>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { chunkRagDocuments, extractRagDocumentFromBytes } from 'rag-extract';
|
|
1
|
+
import { chunkRagDocuments, classifyRagDocument, extractRagDocumentFromBytes, RAG_DOCUMENT_KIND } from 'rag-extract';
|
|
2
2
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
3
|
-
import { errorMessage } from '../../../guards.js';
|
|
3
|
+
import { errorMessage, isRecord } from '../../../guards.js';
|
|
4
|
+
import { IndexingFailure } from './failure.js';
|
|
4
5
|
const EMBED_BATCH = 8;
|
|
5
6
|
export class EmbedderFailure extends Error {
|
|
6
7
|
constructor(cause) {
|
|
@@ -8,40 +9,136 @@ export class EmbedderFailure extends Error {
|
|
|
8
9
|
this.name = 'EmbedderFailure';
|
|
9
10
|
}
|
|
10
11
|
}
|
|
11
|
-
export async function
|
|
12
|
-
|
|
12
|
+
export async function yieldTurn(signal) {
|
|
13
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
14
|
+
if (signal.aborted) {
|
|
15
|
+
throw new AbortedError();
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function pageProgress(onExtracting) {
|
|
19
|
+
let chain = Promise.resolve();
|
|
20
|
+
return {
|
|
21
|
+
report: ({ pages, total }) => {
|
|
22
|
+
chain = chain.then(() => onExtracting?.(pages, total));
|
|
23
|
+
chain.catch(noop);
|
|
24
|
+
},
|
|
25
|
+
settled: () => chain
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
async function documentChunks(input) {
|
|
29
|
+
const { knowledge, artifactId, bytes, signal, extractors } = input;
|
|
30
|
+
const pdfPages = extractors?.pdfPages;
|
|
31
|
+
const { fileName, mimeType } = knowledge;
|
|
32
|
+
if (pdfPages && classifyRagDocument({ fileName, mimeType, bytes }) === RAG_DOCUMENT_KIND.pdf) {
|
|
33
|
+
return pageChunks(input, pdfPages);
|
|
34
|
+
}
|
|
35
|
+
const pages = pageProgress(input.onExtracting);
|
|
13
36
|
const document = await extractRagDocumentFromBytes({
|
|
14
37
|
id: artifactId,
|
|
15
38
|
artifactId,
|
|
16
39
|
attachmentId: knowledge.id,
|
|
17
|
-
chatId: knowledge.chatId,
|
|
40
|
+
chatId: knowledge.chatId ?? undefined,
|
|
18
41
|
blobId: knowledge.blobId,
|
|
19
|
-
fileName
|
|
20
|
-
mimeType
|
|
42
|
+
fileName,
|
|
43
|
+
mimeType,
|
|
21
44
|
bytes
|
|
22
|
-
}, { extractors, signal });
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
45
|
+
}, { extractors, signal, onProgress: pages.report });
|
|
46
|
+
await pages.settled();
|
|
47
|
+
await yieldTurn(signal);
|
|
48
|
+
await input.onExtracted?.();
|
|
49
|
+
return document.content.trim() ? chunkRagDocuments([document], {}) : [];
|
|
50
|
+
}
|
|
51
|
+
async function pageChunks(input, pdfPages) {
|
|
52
|
+
const { knowledge, artifactId, signal, embedder } = input;
|
|
53
|
+
let read = 0;
|
|
54
|
+
const pages = await pdfPages(input.bytes, {
|
|
55
|
+
signal,
|
|
56
|
+
beforeOcr: async () => {
|
|
57
|
+
if (!knowledge.chatId) {
|
|
58
|
+
await embedder.idle?.(signal);
|
|
59
|
+
}
|
|
60
|
+
await input.onExtracting?.(++read, 0);
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
await yieldTurn(signal);
|
|
64
|
+
await input.onExtracted?.();
|
|
65
|
+
const unread = pages.filter((page) => page.unread).length;
|
|
66
|
+
if (unread > 0) {
|
|
67
|
+
throw new IndexingFailure('failed', `${knowledge.fileName}: ${unread} of its pages need OCR`);
|
|
68
|
+
}
|
|
69
|
+
const document = {
|
|
70
|
+
id: artifactId,
|
|
71
|
+
artifactId,
|
|
72
|
+
attachmentId: knowledge.id,
|
|
73
|
+
chatId: knowledge.chatId ?? undefined,
|
|
74
|
+
blobId: knowledge.blobId,
|
|
75
|
+
title: knowledge.fileName,
|
|
76
|
+
mimeType: knowledge.mimeType,
|
|
77
|
+
content: ''
|
|
78
|
+
};
|
|
79
|
+
const chunks = pages.flatMap(({ page, text }) => chunkRagDocuments([{ ...document, content: text }], {}).map((chunk) => ({ chunk, page })));
|
|
80
|
+
return chunks.map(({ chunk, page }, index) => numbered(chunk, { page, index }));
|
|
81
|
+
}
|
|
82
|
+
function numbered(chunk, { page, index }) {
|
|
83
|
+
const parsed = JSON.parse(chunk.metadataJson);
|
|
84
|
+
const metadata = isRecord(parsed) ? parsed : {};
|
|
85
|
+
return {
|
|
86
|
+
...chunk,
|
|
87
|
+
id: `${chunk.id.slice(0, chunk.id.lastIndexOf(':') + 1)}${index}`,
|
|
88
|
+
metadataJson: JSON.stringify({ ...metadata, chunkIndex: index, page })
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
export async function buildArtifactPayload(input) {
|
|
92
|
+
const { embedder, knowledge, signal, onProgress } = input;
|
|
93
|
+
const chunks = await documentChunks(input);
|
|
94
|
+
if (chunks.length === 0) {
|
|
95
|
+
throw new IndexingFailure('no-text', `no extractable text: ${knowledge.fileName}`);
|
|
96
|
+
}
|
|
97
|
+
await onProgress?.(0, chunks.length);
|
|
98
|
+
const positionsByText = new Map();
|
|
99
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
100
|
+
const text = chunks[i].content;
|
|
101
|
+
const positions = positionsByText.get(text);
|
|
102
|
+
if (positions) {
|
|
103
|
+
positions.push(i);
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
positionsByText.set(text, [i]);
|
|
107
|
+
}
|
|
26
108
|
}
|
|
27
|
-
|
|
109
|
+
const unique = [...positionsByText];
|
|
28
110
|
const vectors = [];
|
|
29
|
-
|
|
30
|
-
|
|
111
|
+
let processed = 0;
|
|
112
|
+
for (let start = 0; start < unique.length; start += EMBED_BATCH) {
|
|
113
|
+
await yieldTurn(signal);
|
|
114
|
+
if (!knowledge.chatId) {
|
|
115
|
+
await embedder.idle?.(signal);
|
|
116
|
+
}
|
|
117
|
+
if (signal.aborted) {
|
|
31
118
|
throw new AbortedError();
|
|
32
|
-
|
|
119
|
+
}
|
|
120
|
+
const batch = unique.slice(start, start + EMBED_BATCH);
|
|
33
121
|
let rows;
|
|
34
122
|
try {
|
|
35
|
-
rows = await embedder.embed({ texts: batch.map((
|
|
123
|
+
rows = await embedder.embed({ texts: batch.map(([text]) => text), signal });
|
|
36
124
|
}
|
|
37
125
|
catch (err) {
|
|
38
126
|
throw new EmbedderFailure(err);
|
|
39
127
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
128
|
+
if (signal.aborted) {
|
|
129
|
+
throw new AbortedError();
|
|
130
|
+
}
|
|
131
|
+
if (rows.length !== batch.length) {
|
|
132
|
+
throw new Error(`embedder returned ${rows.length} vectors for ${batch.length} chunks`);
|
|
133
|
+
}
|
|
134
|
+
for (let i = 0; i < batch.length; i++) {
|
|
135
|
+
const [, positions] = batch[i];
|
|
136
|
+
for (const position of positions) {
|
|
137
|
+
vectors[position] = rows[i];
|
|
138
|
+
}
|
|
139
|
+
processed += positions.length;
|
|
140
|
+
}
|
|
141
|
+
await onProgress?.(processed, chunks.length);
|
|
45
142
|
}
|
|
46
143
|
const dimensions = vectors[0]?.length ?? 0;
|
|
47
144
|
if (dimensions <= 0 || vectors.some((v) => v.length !== dimensions)) {
|
|
@@ -58,3 +155,4 @@ export async function buildArtifactPayload(input) {
|
|
|
58
155
|
}))
|
|
59
156
|
};
|
|
60
157
|
}
|
|
158
|
+
function noop() { }
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { QvacIndexingStatus } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
|
+
import { INDEXING_FAILURES } from '../../../../schema/engine.ts';
|
|
3
|
+
export type IndexingFailureReason = (typeof INDEXING_FAILURES)[number];
|
|
4
|
+
export interface ContentCheck {
|
|
5
|
+
readonly supported: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface ParsedFailure {
|
|
8
|
+
readonly reason: IndexingFailureReason;
|
|
9
|
+
readonly detail: string | null;
|
|
10
|
+
}
|
|
11
|
+
export declare class IndexingFailure extends Error {
|
|
12
|
+
readonly reason: IndexingFailureReason;
|
|
13
|
+
constructor(reason: IndexingFailureReason, message: string);
|
|
14
|
+
}
|
|
15
|
+
export declare function contentFailure(fileName: string, bytes: Uint8Array, { supported }: ContentCheck): IndexingFailure | null;
|
|
16
|
+
export declare function failureReason(err: unknown): IndexingFailureReason;
|
|
17
|
+
export declare function failureDetail(err: unknown): string;
|
|
18
|
+
export declare function failureError(err: unknown): string;
|
|
19
|
+
export declare function parseFailure(error: string | null | undefined): ParsedFailure;
|
|
20
|
+
export declare function isContentFailure(row: QvacIndexingStatus): boolean;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { INDEXING_FAILURES } from '../../../../schema/engine.js';
|
|
2
|
+
import { errorMessage, isRecord } from '../../../guards.js';
|
|
3
|
+
const ERROR_LIMIT = 300;
|
|
4
|
+
const PASSWORD = /no password given|incorrect password/i;
|
|
5
|
+
const OUT_OF_SPACE = /no space left on device/i;
|
|
6
|
+
const CONTENT_FAILURES = new Set([
|
|
7
|
+
'empty',
|
|
8
|
+
'no-text',
|
|
9
|
+
'password-protected',
|
|
10
|
+
'unsupported'
|
|
11
|
+
]);
|
|
12
|
+
export class IndexingFailure extends Error {
|
|
13
|
+
reason;
|
|
14
|
+
constructor(reason, message) {
|
|
15
|
+
super(message);
|
|
16
|
+
this.name = 'IndexingFailure';
|
|
17
|
+
this.reason = reason;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export function contentFailure(fileName, bytes, { supported }) {
|
|
21
|
+
if (!supported) {
|
|
22
|
+
return new IndexingFailure('unsupported', `${fileName} is not a supported Knowledge file.`);
|
|
23
|
+
}
|
|
24
|
+
if (bytes.length === 0) {
|
|
25
|
+
return new IndexingFailure('empty', `${fileName} is empty`);
|
|
26
|
+
}
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
export function failureReason(err) {
|
|
30
|
+
if (err instanceof IndexingFailure)
|
|
31
|
+
return err.reason;
|
|
32
|
+
const message = errorMessage(err);
|
|
33
|
+
if ((isRecord(err) && err.code === 'ENOSPC') || OUT_OF_SPACE.test(message)) {
|
|
34
|
+
return 'out-of-space';
|
|
35
|
+
}
|
|
36
|
+
const locked = (isRecord(err) && err.code === 'PASSWORD') || PASSWORD.test(message);
|
|
37
|
+
return locked ? 'password-protected' : 'failed';
|
|
38
|
+
}
|
|
39
|
+
export function failureDetail(err) {
|
|
40
|
+
const detail = errorMessage(err);
|
|
41
|
+
return detail.length > ERROR_LIMIT ? `${detail.slice(0, ERROR_LIMIT)}...` : detail;
|
|
42
|
+
}
|
|
43
|
+
export function failureError(err) {
|
|
44
|
+
return `${failureReason(err)}: ${failureDetail(err)}`;
|
|
45
|
+
}
|
|
46
|
+
export function parseFailure(error) {
|
|
47
|
+
if (!error) {
|
|
48
|
+
return { reason: 'failed', detail: null };
|
|
49
|
+
}
|
|
50
|
+
const split = error.indexOf(': ');
|
|
51
|
+
const reason = INDEXING_FAILURES.find((known) => known === error.slice(0, split));
|
|
52
|
+
return reason && split > 0
|
|
53
|
+
? { reason, detail: error.slice(split + 2) }
|
|
54
|
+
: { reason: 'failed', detail: error };
|
|
55
|
+
}
|
|
56
|
+
export function isContentFailure(row) {
|
|
57
|
+
return (!row.chatId && row.status === 'failed' && CONTENT_FAILURES.has(parseFailure(row.error).reason));
|
|
58
|
+
}
|
|
@@ -1,14 +1,21 @@
|
|
|
1
|
+
import type { LoggerInterface } from '@qvac/logging';
|
|
2
|
+
import ScopeLock from 'scope-lock';
|
|
1
3
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
4
|
import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
3
|
-
import { type
|
|
5
|
+
import { type VectorIndex } from 'rag-extract';
|
|
6
|
+
import type { KnowledgeExtractors } from './build.ts';
|
|
4
7
|
import { type IndexingEmbedder } from './produce.ts';
|
|
8
|
+
import { type IndexingDevice } from './policy.ts';
|
|
5
9
|
import type { FileDocument, FileProcessor, ProcessorHost } from '../processing/index.ts';
|
|
6
10
|
import type { LoadedArtifact } from '../processing/state.ts';
|
|
7
11
|
export interface RagProcessorOptions {
|
|
8
12
|
readonly model: string;
|
|
9
13
|
readonly embedder?: IndexingEmbedder;
|
|
10
14
|
readonly index?: VectorIndex | null;
|
|
11
|
-
readonly extractors?:
|
|
15
|
+
readonly extractors?: KnowledgeExtractors;
|
|
16
|
+
readonly batches?: ScopeLock;
|
|
17
|
+
readonly device?: IndexingDevice;
|
|
18
|
+
readonly log: LoggerInterface | null;
|
|
12
19
|
}
|
|
13
20
|
export declare class RagProcessor implements FileProcessor {
|
|
14
21
|
readonly id = "rag";
|
|
@@ -16,18 +23,27 @@ export declare class RagProcessor implements FileProcessor {
|
|
|
16
23
|
readonly output = "blob";
|
|
17
24
|
readonly version: string;
|
|
18
25
|
private readonly _producer;
|
|
26
|
+
private readonly _device?;
|
|
19
27
|
readonly load?: (document: FileDocument, signal: AbortSignal) => Promise<void>;
|
|
28
|
+
readonly allLoaded?: () => Promise<void>;
|
|
20
29
|
private readonly _index;
|
|
30
|
+
private readonly _log;
|
|
21
31
|
constructor(options: RagProcessorOptions);
|
|
22
32
|
applies(_document: QvacKnowledge): boolean;
|
|
23
33
|
canProduce(document: QvacKnowledge, host: ProcessorHost): Promise<boolean>;
|
|
24
34
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
35
|
+
private _settled;
|
|
25
36
|
evict(loaded: LoadedArtifact): Promise<void>;
|
|
26
37
|
close(): Promise<boolean>;
|
|
27
38
|
}
|
|
28
39
|
export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
|
|
29
|
-
export {
|
|
40
|
+
export { PluginVectorIndex, knowledgeScope, pluginArtifactId } from './plugin-index.ts';
|
|
41
|
+
export type { PluginHit } from './plugin-index.ts';
|
|
42
|
+
export { joinedIndex } from './joined-index.ts';
|
|
43
|
+
export { canIndexKnowledge } from './policy.ts';
|
|
44
|
+
export type { IndexingDevice } from './policy.ts';
|
|
30
45
|
export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
|
|
31
|
-
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit } from './retrieval/full-text.ts';
|
|
32
|
-
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
46
|
+
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
|
|
47
|
+
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, resolveTargetDocument, retrieveRagKnowledge, searchRagChunks, selectCoverage, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
48
|
+
export type { KnowledgeExtractors, ReadPdfPages } from './build.ts';
|
|
33
49
|
export type { RagArtifactPayload, RagDocumentRef, RagExtractors, RagSource, VectorHit, VectorIndex } from 'rag-extract';
|
|
@@ -3,22 +3,31 @@ import { pipelineVersionFor } from 'rag-extract';
|
|
|
3
3
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
4
4
|
import { IndexingPass } from './produce.js';
|
|
5
5
|
import { HydratePass } from './load.js';
|
|
6
|
-
import {
|
|
6
|
+
import { PluginVectorIndex } from './plugin-index.js';
|
|
7
|
+
import { canIndexKnowledge } from './policy.js';
|
|
7
8
|
export class RagProcessor {
|
|
8
9
|
id = 'rag';
|
|
9
10
|
input = 'knowledge';
|
|
10
11
|
output = 'blob';
|
|
11
12
|
version;
|
|
12
13
|
_producer;
|
|
14
|
+
_device;
|
|
13
15
|
load;
|
|
16
|
+
allLoaded;
|
|
14
17
|
_index;
|
|
18
|
+
_log;
|
|
15
19
|
constructor(options) {
|
|
16
20
|
this.version = pipelineVersionFor(options.model);
|
|
17
21
|
this._index = options.index ?? null;
|
|
18
|
-
|
|
22
|
+
this._log = options.log;
|
|
23
|
+
this._device = options.device;
|
|
24
|
+
const lock = options.batches ?? new ScopeLock();
|
|
19
25
|
const source = options.embedder;
|
|
20
26
|
const embedder = source && {
|
|
21
27
|
model: source.model,
|
|
28
|
+
idle: async (signal) => {
|
|
29
|
+
await source.idle?.(signal);
|
|
30
|
+
},
|
|
22
31
|
embed: async (input) => {
|
|
23
32
|
await lock.lock();
|
|
24
33
|
try {
|
|
@@ -35,7 +44,11 @@ export class RagProcessor {
|
|
|
35
44
|
this._producer = embedder ? new IndexingPass(embedder, options.extractors) : null;
|
|
36
45
|
if (this._index) {
|
|
37
46
|
const loader = new HydratePass({ ...options, index: this._index, embedder });
|
|
38
|
-
this.load = (document, signal) => loader.run(document, signal);
|
|
47
|
+
this.load = (document, signal) => this._settled(document, loader.run(document, signal));
|
|
48
|
+
}
|
|
49
|
+
if (this._index instanceof PluginVectorIndex) {
|
|
50
|
+
const index = this._index;
|
|
51
|
+
this.allLoaded = () => index.sweep();
|
|
39
52
|
}
|
|
40
53
|
}
|
|
41
54
|
applies(_document) {
|
|
@@ -44,10 +57,24 @@ export class RagProcessor {
|
|
|
44
57
|
async canProduce(document, host) {
|
|
45
58
|
return (!!this._producer &&
|
|
46
59
|
!document.deletedAt &&
|
|
47
|
-
(await
|
|
60
|
+
(await canIndexKnowledge(host.mesh, {
|
|
61
|
+
deviceId: host.deviceId,
|
|
62
|
+
chatId: document.chatId,
|
|
63
|
+
device: host.device
|
|
64
|
+
})));
|
|
48
65
|
}
|
|
49
66
|
async produce(document, signal) {
|
|
50
|
-
await this._producer?.run(document, signal);
|
|
67
|
+
await this._settled(document, this._producer?.run(document, signal));
|
|
68
|
+
}
|
|
69
|
+
async _settled(document, run) {
|
|
70
|
+
try {
|
|
71
|
+
await run;
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
if (!document.record.chatId && this._device?.mobile()) {
|
|
75
|
+
await document.clearFetched();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
51
78
|
}
|
|
52
79
|
async evict(loaded) {
|
|
53
80
|
if (loaded.evict) {
|
|
@@ -61,12 +88,14 @@ export class RagProcessor {
|
|
|
61
88
|
return true;
|
|
62
89
|
}
|
|
63
90
|
catch (err) {
|
|
64
|
-
|
|
91
|
+
this._log?.warn('rag', 'vector index clear failed on close; it must not be reused', { err });
|
|
65
92
|
return false;
|
|
66
93
|
}
|
|
67
94
|
}
|
|
68
95
|
}
|
|
69
96
|
export { MAX_KNOWLEDGE_BYTES } from './limits.js';
|
|
70
|
-
export {
|
|
71
|
-
export {
|
|
72
|
-
export {
|
|
97
|
+
export { PluginVectorIndex, knowledgeScope, pluginArtifactId } from './plugin-index.js';
|
|
98
|
+
export { joinedIndex } from './joined-index.js';
|
|
99
|
+
export { canIndexKnowledge } from './policy.js';
|
|
100
|
+
export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, fullTextAttachmentId, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
|
|
101
|
+
export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, resolveTargetDocument, retrieveRagKnowledge, searchRagChunks, selectCoverage, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type VectorIndex } from 'rag-extract';
|
|
2
|
+
export interface JoinedIndexOptions {
|
|
3
|
+
readonly chatId: string;
|
|
4
|
+
readonly knowledgeBaseIds: ReadonlySet<string>;
|
|
5
|
+
}
|
|
6
|
+
export declare function joinedIndex(indexes: ReadonlyArray<VectorIndex>, { chatId, knowledgeBaseIds }: JoinedIndexOptions): VectorIndex;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { parseRagMetadata } from 'rag-extract';
|
|
2
|
+
export function joinedIndex(indexes, { chatId, knowledgeBaseIds }) {
|
|
3
|
+
const distinct = [...new Set(indexes)];
|
|
4
|
+
const asChat = (candidate) => {
|
|
5
|
+
const metadata = parseRagMetadata(candidate.metadataJson);
|
|
6
|
+
if (!metadata?.attachmentId || !knowledgeBaseIds.has(metadata.attachmentId)) {
|
|
7
|
+
return candidate;
|
|
8
|
+
}
|
|
9
|
+
return { ...candidate, metadataJson: JSON.stringify({ ...metadata, chatId }) };
|
|
10
|
+
};
|
|
11
|
+
return {
|
|
12
|
+
info() {
|
|
13
|
+
const infos = distinct.map((index) => index.info());
|
|
14
|
+
return {
|
|
15
|
+
artifacts: infos.reduce((sum, info) => sum + info.artifacts, 0),
|
|
16
|
+
chunks: infos.reduce((sum, info) => sum + info.chunks, 0),
|
|
17
|
+
dimensions: infos.find((info) => info.dimensions !== null)?.dimensions ?? null
|
|
18
|
+
};
|
|
19
|
+
},
|
|
20
|
+
async search(embedding, { limit, filter }) {
|
|
21
|
+
const scoped = filter && ((candidate) => filter(asChat(candidate)));
|
|
22
|
+
const found = await Promise.all(distinct.map((index) => index.search(embedding, { limit, filter: scoped })));
|
|
23
|
+
return found
|
|
24
|
+
.flat()
|
|
25
|
+
.sort((a, b) => b.score - a.score)
|
|
26
|
+
.slice(0, limit);
|
|
27
|
+
},
|
|
28
|
+
upsert: readOnly,
|
|
29
|
+
remove: readOnly,
|
|
30
|
+
clear: readOnly
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function readOnly() {
|
|
34
|
+
return Promise.reject(new Error('a joined index is read-only'));
|
|
35
|
+
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
|
-
import { type
|
|
2
|
+
import { type VectorIndex } from 'rag-extract';
|
|
3
3
|
import type { FileDocument } from '../processing/index.ts';
|
|
4
4
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
5
|
-
import { type BuildEmbedder } from './build.ts';
|
|
5
|
+
import { type BuildEmbedder, type KnowledgeExtractors } from './build.ts';
|
|
6
6
|
export interface HydratePassOptions {
|
|
7
7
|
model: string;
|
|
8
8
|
index: VectorIndex;
|
|
9
9
|
embedder?: BuildEmbedder;
|
|
10
|
-
extractors?:
|
|
10
|
+
extractors?: KnowledgeExtractors;
|
|
11
11
|
}
|
|
12
12
|
export interface PickArtifactOptions {
|
|
13
13
|
readonly model: string;
|
|
@@ -125,7 +125,10 @@ export class HydratePass {
|
|
|
125
125
|
}
|
|
126
126
|
catch (err) {
|
|
127
127
|
document.artifact(pick.id).failedReference = pick.artifactBlobId;
|
|
128
|
-
|
|
128
|
+
document.log?.warn('rag', 'artifact undecodable, pinned for the session', {
|
|
129
|
+
id: pick.id,
|
|
130
|
+
err
|
|
131
|
+
});
|
|
129
132
|
continue;
|
|
130
133
|
}
|
|
131
134
|
if (document.paused() || document.closed()) {
|
|
@@ -140,7 +143,7 @@ export class HydratePass {
|
|
|
140
143
|
}
|
|
141
144
|
catch (err) {
|
|
142
145
|
document.artifact(pick.id).failedReference = pick.artifactBlobId;
|
|
143
|
-
|
|
146
|
+
document.log?.warn('rag', 'artifact rejected by the vector index', { id: pick.id, err });
|
|
144
147
|
continue;
|
|
145
148
|
}
|
|
146
149
|
const evict = entry && entry.artifactId !== pick.id ? entry.artifactId : undefined;
|
|
@@ -154,7 +157,7 @@ export class HydratePass {
|
|
|
154
157
|
await this._index.remove(evict);
|
|
155
158
|
local.loaded = { artifactId: pick.id, blobRef: pick.artifactBlobId };
|
|
156
159
|
}
|
|
157
|
-
|
|
160
|
+
document.log?.info('rag', 'hydrated knowledge into the vector index', {
|
|
158
161
|
artifactId: pick.id,
|
|
159
162
|
chunks: chunks.length
|
|
160
163
|
});
|