@qvac/core 1.0.0-rc.0 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +852 -1
- package/README.md +40 -81
- package/dist-lib/index.d.ts +0 -1
- package/dist-lib/index.js +0 -1
- package/dist-lib/lib/api/agent-folders.d.ts +22 -0
- package/dist-lib/lib/api/agent-folders.js +115 -0
- package/dist-lib/lib/api/agents.d.ts +19 -2
- package/dist-lib/lib/api/agents.js +121 -24
- package/dist-lib/lib/api/app-config.d.ts +4 -1
- package/dist-lib/lib/api/app-config.js +17 -6
- package/dist-lib/lib/api/attachments.js +3 -2
- package/dist-lib/lib/api/chat-move.d.ts +4 -0
- package/dist-lib/lib/api/chat-move.js +7 -0
- package/dist-lib/lib/api/chat.d.ts +13 -2
- package/dist-lib/lib/api/chat.js +114 -9
- package/dist-lib/lib/api/chunk-delta.d.ts +7 -0
- package/dist-lib/lib/api/chunk-delta.js +33 -0
- package/dist-lib/lib/api/chunks.d.ts +17 -2
- package/dist-lib/lib/api/chunks.js +64 -13
- package/dist-lib/lib/api/config.d.ts +2 -1
- package/dist-lib/lib/api/config.js +3 -2
- package/dist-lib/lib/api/current-mesh.d.ts +13 -5
- package/dist-lib/lib/api/current-mesh.js +15 -11
- package/dist-lib/lib/api/custom-models.d.ts +7 -1
- package/dist-lib/lib/api/custom-models.js +10 -8
- package/dist-lib/lib/api/device-routes.d.ts +9 -0
- package/dist-lib/lib/api/device-routes.js +123 -0
- package/dist-lib/lib/api/devices.d.ts +3 -2
- package/dist-lib/lib/api/devices.js +20 -10
- package/dist-lib/lib/api/diagnostics.d.ts +2 -1
- package/dist-lib/lib/api/diagnostics.js +2 -2
- package/dist-lib/lib/api/execution.d.ts +12 -1
- package/dist-lib/lib/api/execution.js +44 -14
- package/dist-lib/lib/api/host-run-log.d.ts +2 -41
- package/dist-lib/lib/api/host-run-log.js +35 -83
- package/dist-lib/lib/api/index.d.ts +28 -6
- package/dist-lib/lib/api/index.js +101 -33
- package/dist-lib/lib/api/instrument.d.ts +2 -2
- package/dist-lib/lib/api/instrument.js +12 -44
- package/dist-lib/lib/api/knowledge.d.ts +4 -1
- package/dist-lib/lib/api/knowledge.js +25 -7
- package/dist-lib/lib/api/local-files.d.ts +5 -1
- package/dist-lib/lib/api/local-files.js +9 -5
- package/dist-lib/lib/api/logs.js +3 -3
- package/dist-lib/lib/api/members.d.ts +10 -1
- package/dist-lib/lib/api/members.js +64 -17
- package/dist-lib/lib/api/mesh.d.ts +12 -2
- package/dist-lib/lib/api/mesh.js +10 -6
- package/dist-lib/lib/api/model-download.d.ts +2 -1
- package/dist-lib/lib/api/model-download.js +2 -2
- package/dist-lib/lib/api/network.d.ts +10 -2
- package/dist-lib/lib/api/network.js +8 -2
- package/dist-lib/lib/api/notifications.d.ts +3 -1
- package/dist-lib/lib/api/notifications.js +6 -5
- package/dist-lib/lib/api/projects.d.ts +20 -0
- package/dist-lib/lib/api/projects.js +165 -0
- package/dist-lib/lib/api/rag.d.ts +0 -1
- package/dist-lib/lib/api/rag.js +47 -23
- package/dist-lib/lib/api/served.d.ts +17 -0
- package/dist-lib/lib/api/served.js +47 -0
- package/dist-lib/lib/api/sync.d.ts +2 -1
- package/dist-lib/lib/api/sync.js +2 -2
- package/dist-lib/lib/api/tool-activity.d.ts +2 -1
- package/dist-lib/lib/api/tool-activity.js +4 -3
- package/dist-lib/lib/api/tool-configs.d.ts +2 -1
- package/dist-lib/lib/api/tool-configs.js +6 -9
- package/dist-lib/lib/api/upload-frames.d.ts +0 -1
- package/dist-lib/lib/api/upload-frames.js +0 -1
- package/dist-lib/lib/api/watchable.d.ts +3 -2
- package/dist-lib/lib/api/watchable.js +2 -3
- package/dist-lib/lib/api/workspace.d.ts +5 -0
- package/dist-lib/lib/api/workspace.js +48 -0
- package/dist-lib/lib/assistant/fake-models.d.ts +20 -0
- package/dist-lib/lib/assistant/fake-models.js +112 -0
- package/dist-lib/lib/assistant/harness.d.ts +29 -29
- package/dist-lib/lib/assistant/harness.js +34 -6
- package/dist-lib/lib/assistant/history.d.ts +30 -17
- package/dist-lib/lib/assistant/history.js +168 -82
- package/dist-lib/lib/assistant/host-run-log.d.ts +6 -1
- package/dist-lib/lib/assistant/host-run-log.js +24 -36
- package/dist-lib/lib/assistant/message.d.ts +21 -0
- package/dist-lib/lib/assistant/message.js +43 -0
- package/dist-lib/lib/assistant/model-round.d.ts +95 -0
- package/dist-lib/lib/assistant/model-round.js +591 -0
- package/dist-lib/lib/assistant/pdf-budget.d.ts +3 -0
- package/dist-lib/lib/assistant/pdf-budget.js +13 -0
- package/dist-lib/lib/assistant/round-host.d.ts +42 -0
- package/dist-lib/lib/assistant/round-host.js +239 -0
- package/dist-lib/lib/assistant/skill-encode.d.ts +2 -1
- package/dist-lib/lib/assistant/skill-encode.js +2 -1
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-client.d.ts +2 -0
- package/dist-lib/lib/assistant-client.js +4 -3
- package/dist-lib/lib/assistant-tree.d.ts +8 -3
- package/dist-lib/lib/assistant-tree.js +58 -18
- package/dist-lib/lib/assistant.d.ts +76 -20
- package/dist-lib/lib/assistant.js +1659 -1070
- package/dist-lib/lib/audio-format.js +1 -1
- package/dist-lib/lib/chat-suggestion.d.ts +3 -0
- package/dist-lib/lib/chat-suggestion.js +47 -0
- package/dist-lib/lib/client.d.ts +2 -2
- package/dist-lib/lib/constants.d.ts +7 -0
- package/dist-lib/lib/constants.js +7 -0
- package/dist-lib/lib/core/agent-copies.d.ts +9 -0
- package/dist-lib/lib/core/agent-copies.js +40 -0
- package/dist-lib/lib/core/agent-hosting.d.ts +25 -0
- package/dist-lib/lib/core/agent-hosting.js +71 -0
- package/dist-lib/lib/core/agent.d.ts +9 -1
- package/dist-lib/lib/core/agent.js +45 -19
- package/dist-lib/lib/core/archived-mesh.d.ts +11 -6
- package/dist-lib/lib/core/archived-mesh.js +77 -58
- package/dist-lib/lib/core/async-iterable.js +1 -1
- package/dist-lib/lib/core/blind-peers.d.ts +1 -0
- package/dist-lib/lib/core/blind-peers.js +1 -0
- package/dist-lib/lib/core/chat-activity.d.ts +1 -1
- package/dist-lib/lib/core/chat-activity.js +36 -22
- package/dist-lib/lib/core/chat-copy.d.ts +42 -0
- package/dist-lib/lib/core/chat-copy.js +395 -0
- package/dist-lib/lib/core/chat-fork.d.ts +16 -0
- package/dist-lib/lib/core/chat-fork.js +245 -0
- package/dist-lib/lib/core/chat-group.js +8 -7
- package/dist-lib/lib/core/chat-move.d.ts +71 -0
- package/dist-lib/lib/core/chat-move.js +567 -0
- package/dist-lib/lib/core/chat-order.d.ts +2 -1
- package/dist-lib/lib/core/chat-order.js +19 -4
- package/dist-lib/lib/core/chat-project.d.ts +2 -0
- package/dist-lib/lib/core/chat-project.js +15 -0
- package/dist-lib/lib/core/chat-read.d.ts +4 -0
- package/dist-lib/lib/core/chat-read.js +49 -0
- package/dist-lib/lib/core/chat-search.js +14 -12
- package/dist-lib/lib/core/chat.d.ts +34 -4
- package/dist-lib/lib/core/chat.js +122 -39
- package/dist-lib/lib/core/chunk.d.ts +21 -4
- package/dist-lib/lib/core/chunk.js +138 -23
- package/dist-lib/lib/core/collections.d.ts +28 -0
- package/dist-lib/lib/core/collections.js +28 -0
- package/dist-lib/lib/core/contract/agent-join.d.ts +10 -0
- package/dist-lib/lib/core/contract/agent-join.js +108 -0
- package/dist-lib/lib/core/contract/agents.d.ts +5 -0
- package/dist-lib/lib/core/contract/agents.js +141 -3
- package/dist-lib/lib/core/contract/chat.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat.js +42 -3
- package/dist-lib/lib/core/contract/chunks.d.ts +7 -0
- package/dist-lib/lib/core/contract/chunks.js +112 -35
- package/dist-lib/lib/core/contract/index.d.ts +2 -3
- package/dist-lib/lib/core/contract/index.js +35 -11
- package/dist-lib/lib/core/contract/indexing.d.ts +1 -0
- package/dist-lib/lib/core/contract/indexing.js +51 -7
- package/dist-lib/lib/core/contract/invites.d.ts +3 -14
- package/dist-lib/lib/core/contract/invites.js +18 -9
- package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
- package/dist-lib/lib/core/contract/knowledge.d.ts +1 -0
- package/dist-lib/lib/core/contract/knowledge.js +57 -6
- package/dist-lib/lib/core/contract/members.d.ts +3 -1
- package/dist-lib/lib/core/contract/members.js +34 -4
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +3 -3
- package/dist-lib/lib/core/contract/mesh-kind.js +20 -16
- package/dist-lib/lib/core/contract/mesh-name.d.ts +3 -0
- package/dist-lib/lib/core/contract/mesh-name.js +31 -0
- package/dist-lib/lib/core/contract/mesh-rules.d.ts +8 -0
- package/dist-lib/lib/core/contract/mesh-rules.js +31 -0
- package/dist-lib/lib/core/contract/mesh-seeds.d.ts +2 -0
- package/dist-lib/lib/core/contract/mesh-seeds.js +26 -0
- package/dist-lib/lib/core/contract/model-requests.d.ts +26 -0
- package/dist-lib/lib/core/contract/model-requests.js +279 -0
- package/dist-lib/lib/core/contract/oldest-version.d.ts +9 -0
- package/dist-lib/lib/core/contract/oldest-version.js +20 -0
- package/dist-lib/lib/core/contract/open-join.d.ts +2 -0
- package/dist-lib/lib/core/contract/open-join.js +77 -0
- package/dist-lib/lib/core/contract/project-device-keys.d.ts +7 -0
- package/dist-lib/lib/core/contract/project-device-keys.js +29 -0
- package/dist-lib/lib/core/contract/project-host-keys.d.ts +7 -0
- package/dist-lib/lib/core/contract/project-host-keys.js +40 -0
- package/dist-lib/lib/core/contract/project-order.d.ts +2 -0
- package/dist-lib/lib/core/contract/project-order.js +25 -0
- package/dist-lib/lib/core/contract/projects.d.ts +21 -0
- package/dist-lib/lib/core/contract/projects.js +328 -0
- package/dist-lib/lib/core/contract/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/contract/public-mesh.js +31 -0
- package/dist-lib/lib/core/contract/public-seeds.d.ts +9 -0
- package/dist-lib/lib/core/contract/public-seeds.js +43 -0
- package/dist-lib/lib/core/contract/roles.d.ts +10 -0
- package/dist-lib/lib/core/contract/roles.js +120 -0
- package/dist-lib/lib/core/contract/run-requests.d.ts +10 -0
- package/dist-lib/lib/core/contract/run-requests.js +75 -0
- package/dist-lib/lib/core/contract/seed-defaults.d.ts +3 -1
- package/dist-lib/lib/core/contract/seed-defaults.js +17 -4
- package/dist-lib/lib/core/contract/skill-secrets.d.ts +3 -0
- package/dist-lib/lib/core/contract/skill-secrets.js +32 -0
- package/dist-lib/lib/core/contract/tool-activity.js +12 -1
- package/dist-lib/lib/core/contract/tool-configs.js +2 -1
- package/dist-lib/lib/core/contract/types.d.ts +2 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +31 -0
- package/dist-lib/lib/core/contract/workspace.js +317 -0
- package/dist-lib/lib/core/contract/writers.js +82 -12
- package/dist-lib/lib/core/departures.d.ts +16 -5
- package/dist-lib/lib/core/departures.js +79 -6
- package/dist-lib/lib/core/device-control.d.ts +21 -0
- package/dist-lib/lib/core/device-control.js +91 -0
- package/dist-lib/lib/core/device-seal.d.ts +4 -0
- package/dist-lib/lib/core/device-seal.js +22 -0
- package/dist-lib/lib/core/devices.js +1 -1
- package/dist-lib/lib/core/document-attachment.d.ts +7 -0
- package/dist-lib/lib/core/document-attachment.js +9 -0
- package/dist-lib/lib/core/files/attachments.js +5 -4
- package/dist-lib/lib/core/files/index.d.ts +21 -11
- package/dist-lib/lib/core/files/index.js +114 -28
- package/dist-lib/lib/core/files/knowledge.d.ts +10 -1
- package/dist-lib/lib/core/files/knowledge.js +77 -5
- package/dist-lib/lib/core/files/preview/index.js +6 -2
- package/dist-lib/lib/core/files/processing/claims.d.ts +7 -0
- package/dist-lib/lib/core/files/processing/claims.js +33 -7
- package/dist-lib/lib/core/files/processing/document.d.ts +8 -0
- package/dist-lib/lib/core/files/processing/document.js +26 -2
- package/dist-lib/lib/core/files/processing/index.d.ts +7 -2
- package/dist-lib/lib/core/files/processing/index.js +91 -71
- package/dist-lib/lib/core/files/processing/protocol.d.ts +7 -1
- package/dist-lib/lib/core/files/processing/protocol.js +14 -3
- package/dist-lib/lib/core/files/processing/source.d.ts +3 -0
- package/dist-lib/lib/core/files/processing/source.js +9 -4
- package/dist-lib/lib/core/files/processing/state.d.ts +4 -1
- package/dist-lib/lib/core/files/processing/state.js +2 -1
- package/dist-lib/lib/core/files/rag/build.d.ts +4 -1
- package/dist-lib/lib/core/files/rag/build.js +18 -7
- package/dist-lib/lib/core/files/rag/failure.d.ts +20 -0
- package/dist-lib/lib/core/files/rag/failure.js +57 -0
- package/dist-lib/lib/core/files/rag/index.d.ts +12 -1
- package/dist-lib/lib/core/files/rag/index.js +29 -7
- package/dist-lib/lib/core/files/rag/joined-index.d.ts +6 -0
- package/dist-lib/lib/core/files/rag/joined-index.js +35 -0
- package/dist-lib/lib/core/files/rag/limits.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/limits.js +1 -0
- package/dist-lib/lib/core/files/rag/load.js +6 -3
- package/dist-lib/lib/core/files/rag/policy.d.ts +10 -1
- package/dist-lib/lib/core/files/rag/policy.js +23 -2
- package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +30 -18
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +1 -1
- package/dist-lib/lib/core/files/rag/similarity.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/similarity.js +2 -3
- package/dist-lib/lib/core/files/records/chunks.js +22 -1
- package/dist-lib/lib/core/files/records/index.d.ts +1 -1
- package/dist-lib/lib/core/files/records/index.js +1 -1
- package/dist-lib/lib/core/files/records/knowledge.d.ts +11 -1
- package/dist-lib/lib/core/files/records/knowledge.js +40 -3
- package/dist-lib/lib/core/files/records/purge.js +13 -8
- package/dist-lib/lib/core/files/records/reclaim.js +20 -48
- package/dist-lib/lib/core/files/records/routing.d.ts +1 -1
- package/dist-lib/lib/core/files/records/routing.js +6 -2
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/records/sweep.d.ts +3 -0
- package/dist-lib/lib/core/files/records/sweep.js +9 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +2 -0
- package/dist-lib/lib/core/files/store/blob.js +2 -2
- package/dist-lib/lib/core/files/store/index.d.ts +3 -1
- package/dist-lib/lib/core/files/store/index.js +4 -1
- package/dist-lib/lib/core/files/store/reference.d.ts +3 -0
- package/dist-lib/lib/core/files/store/reference.js +12 -2
- package/dist-lib/lib/core/files/store/sweep.js +3 -1
- package/dist-lib/lib/core/files/upload.d.ts +5 -1
- package/dist-lib/lib/core/files/upload.js +53 -14
- package/dist-lib/lib/core/fs-drives.d.ts +6 -3
- package/dist-lib/lib/core/fs-drives.js +30 -18
- package/dist-lib/lib/core/id.d.ts +1 -0
- package/dist-lib/lib/core/id.js +3 -0
- package/dist-lib/lib/core/identity.d.ts +3 -1
- package/dist-lib/lib/core/identity.js +25 -4
- package/dist-lib/lib/core/invite.d.ts +4 -0
- package/dist-lib/lib/core/invite.js +19 -6
- package/dist-lib/lib/core/local/files.d.ts +2 -1
- package/dist-lib/lib/core/local/files.js +5 -7
- package/dist-lib/lib/core/local/index.d.ts +76 -6
- package/dist-lib/lib/core/local/index.js +209 -19
- package/dist-lib/lib/core/local/run-logs.d.ts +38 -0
- package/dist-lib/lib/core/local/run-logs.js +132 -0
- package/dist-lib/lib/core/member-key.d.ts +7 -0
- package/dist-lib/lib/core/member-key.js +24 -0
- package/dist-lib/lib/core/membership.d.ts +3 -14
- package/dist-lib/lib/core/membership.js +3 -96
- package/dist-lib/lib/core/mesh.d.ts +94 -28
- package/dist-lib/lib/core/mesh.js +527 -144
- package/dist-lib/lib/core/migrations/agent-tools-scope.d.ts +2 -0
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +24 -0
- package/dist-lib/lib/core/migrations/file-index.js +2 -4
- package/dist-lib/lib/core/migrations/index.js +6 -2
- package/dist-lib/lib/core/model-requests.d.ts +13 -0
- package/dist-lib/lib/core/model-requests.js +65 -0
- package/dist-lib/lib/core/next-frame.js +2 -1
- package/dist-lib/lib/core/notification.js +4 -2
- package/dist-lib/lib/core/pdf/rasterize.d.ts +2 -2
- package/dist-lib/lib/core/pdf/rasterize.js +5 -6
- package/dist-lib/lib/core/peer-duplex.d.ts +34 -0
- package/dist-lib/lib/core/{harness-loan.js → peer-duplex.js} +47 -40
- package/dist-lib/lib/core/project-keys.d.ts +13 -0
- package/dist-lib/lib/core/project-keys.js +56 -0
- package/dist-lib/lib/core/prompt-history.js +4 -4
- package/dist-lib/lib/core/protocol-pairing.d.ts +11 -0
- package/dist-lib/lib/core/protocol-pairing.js +22 -0
- package/dist-lib/lib/core/public-mesh.d.ts +26 -0
- package/dist-lib/lib/core/public-mesh.js +88 -0
- package/dist-lib/lib/core/purge.d.ts +3 -0
- package/dist-lib/lib/core/purge.js +7 -0
- package/dist-lib/lib/core/run-driver.d.ts +17 -0
- package/dist-lib/lib/core/run-driver.js +20 -0
- package/dist-lib/lib/core/run-requests.d.ts +11 -0
- package/dist-lib/lib/core/run-requests.js +59 -0
- package/dist-lib/lib/core/send.d.ts +2 -0
- package/dist-lib/lib/core/send.js +17 -4
- package/dist-lib/lib/core/senders.d.ts +18 -0
- package/dist-lib/lib/core/senders.js +52 -0
- package/dist-lib/lib/core/shared-agents.d.ts +39 -0
- package/dist-lib/lib/core/shared-agents.js +255 -0
- package/dist-lib/lib/core/skill-secrets.d.ts +10 -16
- package/dist-lib/lib/core/skill-secrets.js +90 -72
- package/dist-lib/lib/core/sync/background.d.ts +47 -0
- package/dist-lib/lib/core/sync/background.js +162 -0
- package/dist-lib/lib/core/sync/changed.d.ts +20 -0
- package/dist-lib/lib/core/sync/changed.js +7 -0
- package/dist-lib/lib/core/sync/devices.js +1 -1
- package/dist-lib/lib/core/sync/files.d.ts +1 -0
- package/dist-lib/lib/core/sync/files.js +50 -72
- package/dist-lib/lib/core/sync/index.d.ts +1 -1
- package/dist-lib/lib/core/sync/index.js +154 -81
- package/dist-lib/lib/core/sync/view-updates.d.ts +33 -0
- package/dist-lib/lib/core/sync/view-updates.js +133 -0
- package/dist-lib/lib/core/system-info.d.ts +2 -2
- package/dist-lib/lib/core/system-info.js +10 -11
- package/dist-lib/lib/core/tool-configs.d.ts +5 -3
- package/dist-lib/lib/core/tool-configs.js +11 -11
- package/dist-lib/lib/core/transcript-visibility.d.ts +5 -5
- package/dist-lib/lib/core/transcript-visibility.js +27 -31
- package/dist-lib/lib/core/upgrade-agents.d.ts +11 -0
- package/dist-lib/lib/core/upgrade-agents.js +78 -0
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +22 -0
- package/dist-lib/lib/core/workspace-upgrade.js +50 -0
- package/dist-lib/lib/core.d.ts +249 -79
- package/dist-lib/lib/core.js +3369 -1460
- package/dist-lib/lib/credential-pusher.d.ts +6 -0
- package/dist-lib/lib/credential-pusher.js +77 -60
- package/dist-lib/lib/custom-model-pusher.d.ts +2 -4
- package/dist-lib/lib/custom-model-task.d.ts +2 -1
- package/dist-lib/lib/custom-model-task.js +2 -2
- package/dist-lib/lib/diagnostics.d.ts +7 -0
- package/dist-lib/lib/diagnostics.js +22 -0
- package/dist-lib/lib/doctor/checks.d.ts +15 -0
- package/dist-lib/lib/doctor/checks.js +272 -0
- package/dist-lib/lib/doctor/redact.d.ts +4 -0
- package/dist-lib/lib/doctor/redact.js +31 -0
- package/dist-lib/lib/doctor/render.d.ts +8 -0
- package/dist-lib/lib/doctor/render.js +83 -0
- package/dist-lib/lib/doctor/tree.d.ts +5 -0
- package/dist-lib/lib/doctor/tree.js +167 -0
- package/dist-lib/lib/downloads-allowed-pusher.d.ts +8 -0
- package/dist-lib/lib/downloads-allowed-pusher.js +37 -0
- package/dist-lib/lib/errors.d.ts +790 -0
- package/dist-lib/lib/errors.js +724 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +1 -1
- package/dist-lib/lib/fs-drive-bridge.js +4 -1
- package/dist-lib/lib/guards.d.ts +1 -0
- package/dist-lib/lib/guards.js +4 -0
- package/dist-lib/lib/inference-calls.d.ts +1 -2
- package/dist-lib/lib/inference-calls.js +2 -10
- package/dist-lib/lib/knowledge-search.js +13 -18
- package/dist-lib/lib/kv-cache-reclaim.d.ts +2 -1
- package/dist-lib/lib/kv-cache-reclaim.js +30 -0
- package/dist-lib/lib/log.d.ts +7 -6
- package/dist-lib/lib/log.js +14 -7
- package/dist-lib/lib/logger-type.d.ts +2 -5
- package/dist-lib/lib/mesh-scoped-engine.js +42 -2
- package/dist-lib/lib/model-download-status.d.ts +3 -3
- package/dist-lib/lib/model-download.d.ts +11 -12
- package/dist-lib/lib/model-download.js +63 -22
- package/dist-lib/lib/model-idle-pusher.d.ts +2 -4
- package/dist-lib/lib/model-memory-cap.d.ts +27 -0
- package/dist-lib/lib/model-memory-cap.js +69 -0
- package/dist-lib/lib/models-watch.d.ts +1 -1
- package/dist-lib/lib/models-watch.js +4 -4
- package/dist-lib/lib/oauth/browser-connect.js +3 -2
- package/dist-lib/lib/oauth/flow-core.js +5 -3
- package/dist-lib/lib/oauth/flow.js +1 -1
- package/dist-lib/lib/oauth/loopback.d.ts +1 -0
- package/dist-lib/lib/oauth/loopback.js +57 -12
- package/dist-lib/lib/oauth/providers.js +4 -2
- package/dist-lib/lib/oauth/redirect-types.d.ts +1 -0
- package/dist-lib/lib/oauth/register.js +35 -19
- package/dist-lib/lib/oauth/request.d.ts +10 -0
- package/dist-lib/lib/oauth/request.js +22 -0
- package/dist-lib/lib/oauth/token-exchange-core.js +31 -33
- package/dist-lib/lib/oauth/types.d.ts +3 -0
- package/dist-lib/lib/oauth.d.ts +2 -1
- package/dist-lib/lib/oauth.js +1 -0
- package/dist-lib/lib/persistent-dir.d.ts +1 -0
- package/dist-lib/lib/persistent-dir.js +9 -0
- package/dist-lib/lib/probe-rounds.d.ts +3 -0
- package/dist-lib/lib/probe-rounds.js +5 -0
- package/dist-lib/lib/run-input.d.ts +14 -0
- package/dist-lib/lib/run-input.js +33 -0
- package/dist-lib/lib/run-notices.d.ts +3 -0
- package/dist-lib/lib/run-notices.js +8 -2
- package/dist-lib/lib/run-replay.d.ts +32 -0
- package/dist-lib/lib/run-replay.js +121 -0
- package/dist-lib/lib/run-transcript.d.ts +11 -12
- package/dist-lib/lib/run-transcript.js +16 -18
- package/dist-lib/lib/suspend-lifecycle.js +15 -5
- package/dist-lib/lib/task.js +6 -4
- package/dist-lib/lib/test-env.d.ts +1 -0
- package/dist-lib/lib/test-env.js +4 -0
- package/dist-lib/schema/build.js +3 -1
- package/dist-lib/schema/config.d.ts +1 -1
- package/dist-lib/schema/config.js +1 -1
- package/dist-lib/schema/content.d.ts +5 -1
- package/dist-lib/schema/content.js +58 -11
- package/dist-lib/schema/engine.d.ts +1 -0
- package/dist-lib/schema/engine.js +709 -116
- package/dist-lib/schema/field-order.js +2 -1
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +30 -3
- package/dist-lib/schema/local.js +72 -4
- package/dist-lib/schema/qvac.d.ts +4 -1
- package/dist-lib/schema/qvac.js +364 -12
- package/dist-lib/spec/bind.js +209 -48
- package/dist-lib/spec/calls.js +45 -5
- package/dist-lib/spec/capabilities.d.ts +98 -18
- package/dist-lib/spec/config/hyperconf/index.js +1 -1
- package/dist-lib/spec/hrpc/hrpc.json +726 -206
- package/dist-lib/spec/hrpc/index.js +832 -345
- package/dist-lib/spec/hrpc/messages.js +6906 -4832
- package/dist-lib/spec/hyperschema/index.js +6906 -4832
- package/dist-lib/spec/hyperschema/schema.json +4744 -3139
- package/dist-lib/spec/hyperschema/types.d.ts +461 -61
- package/dist-lib/spec/local/hyperdb/db.json +95 -1
- package/dist-lib/spec/local/hyperdb/index.js +459 -2
- package/dist-lib/spec/local/hyperdb/messages.js +1506 -484
- package/dist-lib/spec/local/hyperschema/index.js +542 -13
- package/dist-lib/spec/local/hyperschema/schema.json +319 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +78 -2
- package/dist-lib/spec/mesh/hyperdb/db.json +371 -5
- package/dist-lib/spec/mesh/hyperdb/index.js +1564 -5
- package/dist-lib/spec/mesh/hyperdb/messages.js +6023 -1246
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +187 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +528 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +3449 -217
- package/dist-lib/spec/mesh/hyperschema/index.js +3449 -217
- package/dist-lib/spec/mesh/hyperschema/schema.json +1865 -19
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +470 -6
- package/package.json +50 -35
- package/worker/assistant-sidecar.mjs +15 -2
- package/worker/stow-entry.mjs +2 -0
- package/dist-lib/lib/core/contract/compute.d.ts +0 -11
- package/dist-lib/lib/core/contract/compute.js +0 -48
- package/dist-lib/lib/core/device-invites.d.ts +0 -11
- package/dist-lib/lib/core/device-invites.js +0 -63
- package/dist-lib/lib/core/grant-expiry.d.ts +0 -11
- package/dist-lib/lib/core/grant-expiry.js +0 -66
- package/dist-lib/lib/core/handoff-channel.d.ts +0 -47
- package/dist-lib/lib/core/handoff-channel.js +0 -123
- package/dist-lib/lib/core/harness-loan.d.ts +0 -29
- package/dist-lib/lib/core/host-run-log-store.d.ts +0 -114
- package/dist-lib/lib/core/host-run-log-store.js +0 -976
- package/dist-lib/lib/core/hosted-agents.d.ts +0 -38
- package/dist-lib/lib/core/hosted-agents.js +0 -140
- package/dist-lib/lib/core/inference-calls-handoff.d.ts +0 -20
- package/dist-lib/lib/core/inference-calls-handoff.js +0 -37
- package/dist-lib/lib/core/inference-calls-receiver.d.ts +0 -14
- package/dist-lib/lib/core/inference-calls-receiver.js +0 -42
- package/dist-lib/lib/core/mesh-seed-handoff.d.ts +0 -25
- package/dist-lib/lib/core/mesh-seed-handoff.js +0 -52
- package/dist-lib/lib/core/secret-handoff.d.ts +0 -17
- package/dist-lib/lib/core/secret-handoff.js +0 -29
- package/dist-lib/lib/core/skill-overlay-handoff.d.ts +0 -25
- package/dist-lib/lib/core/skill-overlay-handoff.js +0 -50
- package/dist-lib/lib/core/skill-overlay-receiver.d.ts +0 -8
- package/dist-lib/lib/core/skill-overlay-receiver.js +0 -23
- package/dist-lib/lib/image-cache-dir.d.ts +0 -1
- package/dist-lib/lib/image-cache-dir.js +0 -5
- package/dist-lib/lib/log-levels.d.ts +0 -7
- package/dist-lib/lib/log-levels.js +0 -1
- package/dist-lib/lib/logger.d.ts +0 -5
- package/dist-lib/lib/logger.js +0 -18
- package/dist-lib/lib/oauth/loopback.stub.d.ts +0 -3
- package/dist-lib/lib/oauth/loopback.stub.js +0 -6
- package/tui/README.md +0 -142
- package/tui/app.mjs +0 -6939
- package/tui/boot.mjs +0 -111
- package/tui/index.mjs +0 -53
- package/tui/lib/boot-timing.mjs +0 -41
- package/tui/lib/busy-words.mjs +0 -116
- package/tui/lib/export.mjs +0 -194
- package/tui/lib/knowledge-command.mjs +0 -190
- package/tui/lib/knowledge-import.mjs +0 -171
- package/tui/lib/render.mjs +0 -196
- package/tui/lib/save-file.mjs +0 -88
- package/tui/lib/select.mjs +0 -81
- package/tui/lib/selection.mjs +0 -18
- package/tui/lib/setup.mjs +0 -101
- package/tui/lib/sidebar.mjs +0 -63
- package/tui/lib/splash-art.mjs +0 -107
- package/tui/lib/splash.mjs +0 -18
- package/tui/lib/upload-frames.mjs +0 -9
- package/tui/lib/usage.mjs +0 -59
- package/tui/lib/wrap.mjs +0 -143
- package/tui/main.mjs +0 -264
|
@@ -14,7 +14,9 @@ export class FileDocument {
|
|
|
14
14
|
schedule;
|
|
15
15
|
closed;
|
|
16
16
|
paused;
|
|
17
|
+
admit;
|
|
17
18
|
_output;
|
|
19
|
+
_fetched = false;
|
|
18
20
|
_progress;
|
|
19
21
|
_processor;
|
|
20
22
|
constructor(mesh, options) {
|
|
@@ -24,11 +26,12 @@ export class FileDocument {
|
|
|
24
26
|
this._processor = options.processor;
|
|
25
27
|
const state = mesh.files.processorState(options.processor.id);
|
|
26
28
|
this.state = state.document(record.id);
|
|
27
|
-
this.source = fileSource(options.source, { id: record.id, reference: record.blobId });
|
|
29
|
+
this.source = this._fetching(fileSource(options.source, { id: record.id, reference: record.blobId }));
|
|
28
30
|
this._output = options.output;
|
|
29
31
|
this.schedule = options.schedule;
|
|
30
32
|
this.closed = () => options.closed() || !!mesh.closing;
|
|
31
33
|
this.paused = options.paused;
|
|
34
|
+
this.admit = options.admit ?? (() => { });
|
|
32
35
|
this._progress = options.progress;
|
|
33
36
|
const claims = new IndexingClaims({
|
|
34
37
|
mesh,
|
|
@@ -41,10 +44,14 @@ export class FileDocument {
|
|
|
41
44
|
isFailed: (id) => claims.isFailed(id),
|
|
42
45
|
blockedByStatus: (id) => claims.blockedByStatus(id),
|
|
43
46
|
claim: (id) => claims.claim(id, record.id),
|
|
47
|
+
lease: (id, claimId) => claims.lease(id, claimId),
|
|
44
48
|
settle: (id) => claims.settle(id, record.id)
|
|
45
49
|
};
|
|
46
50
|
this.ownership = new ArtifactOwnership(mesh, state);
|
|
47
51
|
}
|
|
52
|
+
get log() {
|
|
53
|
+
return this._mesh.log;
|
|
54
|
+
}
|
|
48
55
|
get localCoreKey() {
|
|
49
56
|
return b4a.toHex(this._mesh.blobs.core.key);
|
|
50
57
|
}
|
|
@@ -56,7 +63,24 @@ export class FileDocument {
|
|
|
56
63
|
}
|
|
57
64
|
output(id, reference) {
|
|
58
65
|
this.artifact(id).knowledgeId = this.record.id;
|
|
59
|
-
return fileSource(this._output, { id, reference });
|
|
66
|
+
return this._fetching(fileSource(this._output, { id, reference }));
|
|
67
|
+
}
|
|
68
|
+
async clearFetched() {
|
|
69
|
+
if (this._fetched) {
|
|
70
|
+
await this._mesh.files.clearFetched(this.record.blobId, this.record.id);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
_fetching(source) {
|
|
74
|
+
return {
|
|
75
|
+
...source,
|
|
76
|
+
read: async (signal) => {
|
|
77
|
+
const read = await source.read(signal);
|
|
78
|
+
if (read.kind === 'bytes') {
|
|
79
|
+
this._fetched = true;
|
|
80
|
+
}
|
|
81
|
+
return read;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
60
84
|
}
|
|
61
85
|
artifacts() {
|
|
62
86
|
return this._mesh.files.artifacts(this.record.id);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import { type IndexingProgress } from '../rag/index.ts';
|
|
3
|
+
import type { IndexingDevice } from '../rag/policy.ts';
|
|
3
4
|
import { type FileInput, type FileRecord } from '../records/index.ts';
|
|
4
5
|
import type { Mesh } from '../../mesh.ts';
|
|
5
6
|
import { FileDocument } from './document.ts';
|
|
@@ -8,6 +9,7 @@ import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/ty
|
|
|
8
9
|
export interface ProcessorHost {
|
|
9
10
|
readonly mesh: Mesh;
|
|
10
11
|
readonly deviceId: Buffer | null;
|
|
12
|
+
readonly device?: IndexingDevice;
|
|
11
13
|
}
|
|
12
14
|
export interface FileProcessor {
|
|
13
15
|
readonly id: string;
|
|
@@ -26,6 +28,7 @@ export interface FileProcessor {
|
|
|
26
28
|
export interface FileProcessingOptions {
|
|
27
29
|
readonly processor: FileProcessor;
|
|
28
30
|
readonly deviceId: Buffer | null;
|
|
31
|
+
readonly device?: IndexingDevice;
|
|
29
32
|
readonly retryMs?: number;
|
|
30
33
|
readonly progress?: IndexingProgress;
|
|
31
34
|
}
|
|
@@ -36,6 +39,7 @@ export declare class FileProcessing {
|
|
|
36
39
|
private readonly _mesh;
|
|
37
40
|
private readonly _processor;
|
|
38
41
|
private readonly _deviceId;
|
|
42
|
+
private readonly _device?;
|
|
39
43
|
private readonly _source;
|
|
40
44
|
private readonly _output;
|
|
41
45
|
private readonly _progress?;
|
|
@@ -55,6 +59,7 @@ export declare class FileProcessing {
|
|
|
55
59
|
suspend(): Promise<void>;
|
|
56
60
|
resume(): void;
|
|
57
61
|
reconnect(): void;
|
|
62
|
+
rescan(): void;
|
|
58
63
|
close(): Promise<void>;
|
|
59
64
|
get indexCleared(): boolean;
|
|
60
65
|
settle(): Promise<void>;
|
|
@@ -70,14 +75,14 @@ export declare class FileProcessing {
|
|
|
70
75
|
private _documents;
|
|
71
76
|
private _pump;
|
|
72
77
|
private _forget;
|
|
73
|
-
private
|
|
78
|
+
private _run;
|
|
74
79
|
private _recordFailure;
|
|
75
80
|
private _queueLoad;
|
|
76
81
|
private _schedule;
|
|
77
82
|
private _retry;
|
|
78
83
|
}
|
|
79
84
|
export type { FileDocument } from './document.ts';
|
|
80
|
-
export { isClaimLive, listIndexingByChat } from './protocol.ts';
|
|
85
|
+
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.ts';
|
|
81
86
|
export { FileState } from './state.ts';
|
|
82
87
|
export type { IndexingProgressEntry, LocalWork } from './state.ts';
|
|
83
88
|
export { settleFiles } from './settle.ts';
|
|
@@ -9,6 +9,7 @@ export class FileProcessing {
|
|
|
9
9
|
_mesh;
|
|
10
10
|
_processor;
|
|
11
11
|
_deviceId;
|
|
12
|
+
_device;
|
|
12
13
|
_source;
|
|
13
14
|
_output;
|
|
14
15
|
_progress;
|
|
@@ -34,15 +35,18 @@ export class FileProcessing {
|
|
|
34
35
|
this._mesh = mesh;
|
|
35
36
|
this._processor = options.processor;
|
|
36
37
|
this._deviceId = options.deviceId;
|
|
38
|
+
this._device = options.device;
|
|
37
39
|
this._progress = options.progress;
|
|
38
40
|
const readBytes = mesh.files.readBytes.bind(mesh.files);
|
|
39
41
|
const fetch = mesh.files.download.bind(mesh.files);
|
|
40
42
|
this._source = new IndexingSource(readBytes, {
|
|
43
|
+
log: mesh.log,
|
|
41
44
|
retryMs: options.retryMs,
|
|
42
45
|
retries: mesh.files.processorState(options.processor.id).sourceRetries,
|
|
43
46
|
download: { fetch, ready: (id) => this._schedule(id, Date.now()) }
|
|
44
47
|
});
|
|
45
48
|
this._output = new IndexingSource(readBytes, {
|
|
49
|
+
log: mesh.log,
|
|
46
50
|
retryMs: options.retryMs,
|
|
47
51
|
retries: mesh.files.processorState(options.processor.id).outputRetries,
|
|
48
52
|
download: {
|
|
@@ -50,9 +54,9 @@ export class FileProcessing {
|
|
|
50
54
|
ready: (id) => this._schedule(mesh.files.processorState(options.processor.id).artifact(id).knowledgeId ?? id, Date.now())
|
|
51
55
|
}
|
|
52
56
|
});
|
|
53
|
-
this._lanes.push({ stage: 'produce', queue: new Map(),
|
|
57
|
+
this._lanes.push({ stage: 'produce', queue: new Map(), jobs: new Set() });
|
|
54
58
|
if (options.processor.load) {
|
|
55
|
-
this._lanes.push({ stage: 'load', queue: new Map(),
|
|
59
|
+
this._lanes.push({ stage: 'load', queue: new Map(), jobs: new Set() });
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
start() {
|
|
@@ -75,6 +79,9 @@ export class FileProcessing {
|
|
|
75
79
|
this._source.reconnect((id, deadline) => this._schedule(id, deadline));
|
|
76
80
|
this._output.reconnect((id, deadline) => this._schedule(this._mesh.files.processorState(this._processor.id).artifact(id).knowledgeId ?? id, deadline));
|
|
77
81
|
}
|
|
82
|
+
rescan() {
|
|
83
|
+
this._kick();
|
|
84
|
+
}
|
|
78
85
|
close() {
|
|
79
86
|
this._close ??= this._shutdown();
|
|
80
87
|
return this._close;
|
|
@@ -119,7 +126,7 @@ export class FileProcessing {
|
|
|
119
126
|
pending(id) {
|
|
120
127
|
return (!this._paused &&
|
|
121
128
|
!this._closed &&
|
|
122
|
-
(this._lanes.some((lane) => lane.
|
|
129
|
+
(this._lanes.some((lane) => lane.queue.has(id) || [...lane.jobs].some((job) => job.id === id)) ||
|
|
123
130
|
this._source.pending((key) => key === id) ||
|
|
124
131
|
this._output.pending((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id)));
|
|
125
132
|
}
|
|
@@ -153,12 +160,13 @@ export class FileProcessing {
|
|
|
153
160
|
this._scheduler.clear();
|
|
154
161
|
this._deadlines.clear();
|
|
155
162
|
this._mesh.files.emit('update');
|
|
156
|
-
|
|
157
|
-
|
|
163
|
+
const jobs = this._lanes.flatMap((lane) => [...lane.jobs]);
|
|
164
|
+
for (const job of jobs) {
|
|
165
|
+
job.abort.abort({
|
|
158
166
|
type: this._closed ? 'file-processing-closed' : 'file-processing-suspended'
|
|
159
167
|
});
|
|
160
168
|
}
|
|
161
|
-
await Promise.all([this._scanning, ...
|
|
169
|
+
await Promise.all([this._scanning, ...jobs.map((job) => job.done)]);
|
|
162
170
|
await Promise.all([this._source.cancel(), this._output.cancel()]);
|
|
163
171
|
for (const lane of this._lanes) {
|
|
164
172
|
for (const wakes of lane.queue.values()) {
|
|
@@ -190,7 +198,7 @@ export class FileProcessing {
|
|
|
190
198
|
await this.enqueue(await this._documents());
|
|
191
199
|
}
|
|
192
200
|
catch (err) {
|
|
193
|
-
|
|
201
|
+
this._mesh.log?.warn('rag', 'file processing scan failed', { err });
|
|
194
202
|
}
|
|
195
203
|
}
|
|
196
204
|
}
|
|
@@ -206,13 +214,31 @@ export class FileProcessing {
|
|
|
206
214
|
}
|
|
207
215
|
}
|
|
208
216
|
_pump(lane) {
|
|
209
|
-
|
|
210
|
-
|
|
217
|
+
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
218
|
+
const busy = [...lane.jobs];
|
|
219
|
+
if (busy.length && (lane.stage !== 'produce' || busy.length > 1 || !busy[0].admitted)) {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
const next = [...lane.queue].find(([queued]) => !busy.some((job) => job.id === queued));
|
|
223
|
+
if (!next) {
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
const [id, wakes] = next;
|
|
227
|
+
lane.queue.delete(id);
|
|
228
|
+
const abort = new AbortController();
|
|
229
|
+
const job = { id, abort, admitted: false, done: Promise.resolve() };
|
|
230
|
+
lane.jobs.add(job);
|
|
231
|
+
const admit = () => {
|
|
232
|
+
if (job.admitted)
|
|
233
|
+
return;
|
|
234
|
+
job.admitted = true;
|
|
235
|
+
this._pump(lane);
|
|
236
|
+
};
|
|
237
|
+
job.done = this._run(lane, { id, wakes, signal: abort.signal, admit }).finally(() => {
|
|
238
|
+
lane.jobs.delete(job);
|
|
239
|
+
this._pump(lane);
|
|
240
|
+
});
|
|
211
241
|
}
|
|
212
|
-
lane.running = this._drain(lane).finally(() => {
|
|
213
|
-
lane.running = null;
|
|
214
|
-
this._pump(lane);
|
|
215
|
-
});
|
|
216
242
|
}
|
|
217
243
|
async _forget(id) {
|
|
218
244
|
const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
|
|
@@ -220,65 +246,59 @@ export class FileProcessing {
|
|
|
220
246
|
await this._processor.evict?.(loaded);
|
|
221
247
|
}
|
|
222
248
|
}
|
|
223
|
-
async
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
lane.
|
|
228
|
-
|
|
229
|
-
let produced = false;
|
|
230
|
-
try {
|
|
231
|
-
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
232
|
-
if (!record && lane.stage === 'load') {
|
|
233
|
-
await this._forget(id);
|
|
234
|
-
}
|
|
235
|
-
if (!record || record.deletedAt) {
|
|
236
|
-
await this._source.cancel((key) => key === id);
|
|
237
|
-
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
238
|
-
}
|
|
239
|
-
if (!record || !this._processor.applies(record) || lane.abort.signal.aborted) {
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
if (lane.stage === 'produce' &&
|
|
243
|
-
(!this._mesh.writable ||
|
|
244
|
-
!(await this._processor.canProduce(record, {
|
|
245
|
-
mesh: this._mesh,
|
|
246
|
-
deviceId: this._deviceId
|
|
247
|
-
})))) {
|
|
248
|
-
continue;
|
|
249
|
-
}
|
|
250
|
-
const document = new FileDocument(this._mesh, {
|
|
251
|
-
record,
|
|
252
|
-
processor: this._processor,
|
|
253
|
-
source: this._source,
|
|
254
|
-
output: this._output,
|
|
255
|
-
schedule: (deadline) => this._schedule(id, deadline),
|
|
256
|
-
closed: () => this._closed,
|
|
257
|
-
paused: () => this._paused,
|
|
258
|
-
progress: this._progress
|
|
259
|
-
});
|
|
260
|
-
if (lane.abort.signal.aborted) {
|
|
261
|
-
continue;
|
|
262
|
-
}
|
|
263
|
-
await this._processor[lane.stage]?.(document, lane.abort.signal);
|
|
264
|
-
produced = lane.stage === 'produce';
|
|
249
|
+
async _run(lane, { id, wakes, signal, admit }) {
|
|
250
|
+
let produced = false;
|
|
251
|
+
try {
|
|
252
|
+
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
253
|
+
if (!record && lane.stage === 'load') {
|
|
254
|
+
await this._forget(id);
|
|
265
255
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
}
|
|
256
|
+
if (!record || record.deletedAt) {
|
|
257
|
+
await this._source.cancel((key) => key === id);
|
|
258
|
+
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
270
259
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
260
|
+
if (!record || !this._processor.applies(record) || signal.aborted) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
if (lane.stage === 'produce' &&
|
|
264
|
+
(!this._mesh.writable ||
|
|
265
|
+
!(await this._processor.canProduce(record, {
|
|
266
|
+
mesh: this._mesh,
|
|
267
|
+
deviceId: this._deviceId,
|
|
268
|
+
device: this._device
|
|
269
|
+
})))) {
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
const document = new FileDocument(this._mesh, {
|
|
273
|
+
record,
|
|
274
|
+
processor: this._processor,
|
|
275
|
+
source: this._source,
|
|
276
|
+
output: this._output,
|
|
277
|
+
schedule: (deadline) => this._schedule(id, deadline),
|
|
278
|
+
closed: () => this._closed,
|
|
279
|
+
paused: () => this._paused,
|
|
280
|
+
progress: this._progress,
|
|
281
|
+
admit
|
|
282
|
+
});
|
|
283
|
+
if (signal.aborted) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
await this._processor[lane.stage]?.(document, signal);
|
|
287
|
+
produced = lane.stage === 'produce';
|
|
288
|
+
}
|
|
289
|
+
catch (err) {
|
|
290
|
+
if (!signal.aborted) {
|
|
291
|
+
this._recordFailure(id, { stage: lane.stage, err });
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
finally {
|
|
295
|
+
if (produced && this.active) {
|
|
296
|
+
this._queueLoad(id);
|
|
297
|
+
}
|
|
298
|
+
for (const wake of wakes) {
|
|
299
|
+
wake();
|
|
281
300
|
}
|
|
301
|
+
this._mesh.files.emit('update');
|
|
282
302
|
}
|
|
283
303
|
}
|
|
284
304
|
_recordFailure(id, { stage, err }) {
|
|
@@ -292,7 +312,7 @@ export class FileProcessing {
|
|
|
292
312
|
else {
|
|
293
313
|
this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
|
|
294
314
|
}
|
|
295
|
-
|
|
315
|
+
this._mesh.log?.warn('rag', 'file processing failed', { id, stage, err });
|
|
296
316
|
}
|
|
297
317
|
_queueLoad(id) {
|
|
298
318
|
const loader = this._lanes.find((entry) => entry.stage === 'load');
|
|
@@ -327,6 +347,6 @@ export class FileProcessing {
|
|
|
327
347
|
void this.enqueue(ids);
|
|
328
348
|
}
|
|
329
349
|
}
|
|
330
|
-
export { isClaimLive, listIndexingByChat } from './protocol.js';
|
|
350
|
+
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.js';
|
|
331
351
|
export { FileState } from './state.js';
|
|
332
352
|
export { settleFiles } from './settle.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { QvacIndexingResolution, QvacIndexingStatus } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
2
|
import type { Mesh } from '../../mesh.ts';
|
|
3
3
|
import type { ChatScope, KnowledgeScope } from '../records/knowledge.ts';
|
|
4
|
-
export { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS } from '../../contract/indexing.ts';
|
|
4
|
+
export { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS, INDEXING_STALE_MS } from '../../contract/indexing.ts';
|
|
5
5
|
export declare function isClaimLive(row: QvacIndexingStatus, at: number): boolean;
|
|
6
6
|
export interface ClaimIndexingInput {
|
|
7
7
|
artifactId: string;
|
|
@@ -12,6 +12,11 @@ export interface IndexingClaimResult {
|
|
|
12
12
|
readonly row: QvacIndexingStatus;
|
|
13
13
|
readonly won: boolean;
|
|
14
14
|
}
|
|
15
|
+
export interface ProgressIndexingInput {
|
|
16
|
+
readonly artifactId: string;
|
|
17
|
+
readonly claimId: string;
|
|
18
|
+
readonly at: number;
|
|
19
|
+
}
|
|
15
20
|
export interface ResolveIndexingInput {
|
|
16
21
|
readonly artifactId: string;
|
|
17
22
|
readonly claimId: string;
|
|
@@ -26,4 +31,5 @@ export declare function getIndexingStatus(store: Mesh, artifactId: string): Prom
|
|
|
26
31
|
export declare function listIndexingByKnowledge(store: Mesh, { knowledgeId }: KnowledgeScope): Promise<QvacIndexingStatus[]>;
|
|
27
32
|
export declare function listIndexingByChat(store: Mesh, { chatId }: ChatScope): Promise<QvacIndexingStatus[]>;
|
|
28
33
|
export declare function claimIndexing(store: Mesh, { artifactId, knowledgeId, at }: ClaimIndexingInput): Promise<IndexingClaimResult>;
|
|
34
|
+
export declare function progressIndexing(store: Mesh, { artifactId, claimId, at }: ProgressIndexingInput): Promise<boolean>;
|
|
29
35
|
export declare function resolveIndexing(store: Mesh, { artifactId, claimId, status, error }: ResolveIndexingInput): Promise<IndexingResolutionResult>;
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { INDEXING_MAX_CLOCK_SKEW_MS } from '../../contract/indexing.js';
|
|
1
|
+
import { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS } from '../../contract/indexing.js';
|
|
2
2
|
import { randomId } from '../../id.js';
|
|
3
|
-
|
|
3
|
+
import b4a from 'b4a';
|
|
4
|
+
export { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS, INDEXING_STALE_MS } from '../../contract/indexing.js';
|
|
4
5
|
import { INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, INDEXING_STATUS_BY_KNOWLEDGE } from '../../collections.js';
|
|
5
6
|
export function isClaimLive(row, at) {
|
|
6
|
-
return (row.status === 'indexing' &&
|
|
7
|
+
return (row.status === 'indexing' &&
|
|
8
|
+
row.expiresAt > at &&
|
|
9
|
+
row.at <= at + INDEXING_MAX_CLOCK_SKEW_MS &&
|
|
10
|
+
row.expiresAt <= at + INDEXING_MAX_CLOCK_SKEW_MS + INDEXING_CLAIM_TTL_MS);
|
|
7
11
|
}
|
|
8
12
|
export async function getIndexingStatus(store, artifactId) {
|
|
9
13
|
const snap = store.view.snapshot();
|
|
@@ -62,6 +66,13 @@ export async function claimIndexing(store, { artifactId, knowledgeId, at }) {
|
|
|
62
66
|
const won = row.status === 'indexing' && row.claimId === claimId;
|
|
63
67
|
return { row, won };
|
|
64
68
|
}
|
|
69
|
+
export async function progressIndexing(store, { artifactId, claimId, at }) {
|
|
70
|
+
await store.dispatch('progress-indexing', { id: artifactId, claimId, at });
|
|
71
|
+
const row = await getIndexingStatus(store, artifactId);
|
|
72
|
+
return (row?.status === 'indexing' &&
|
|
73
|
+
row.claimId === claimId &&
|
|
74
|
+
b4a.equals(row.claimedBy, store.localWriterKey));
|
|
75
|
+
}
|
|
65
76
|
export async function resolveIndexing(store, { artifactId, claimId, status, error }) {
|
|
66
77
|
await store.dispatch('resolve-indexing', { id: artifactId, claimId, status, error });
|
|
67
78
|
const row = await mustGetIndexingStatus(store, artifactId);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LoggerInterface } from '@qvac/logging';
|
|
1
2
|
import { type BlobStore } from '../store/index.ts';
|
|
2
3
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
3
4
|
import type { FileRetries } from './state.ts';
|
|
@@ -11,6 +12,7 @@ export type SourceRead = {
|
|
|
11
12
|
kind: 'malformed';
|
|
12
13
|
};
|
|
13
14
|
export interface IndexingSourceOptions {
|
|
15
|
+
readonly log: LoggerInterface | null;
|
|
14
16
|
readonly retryMs?: number;
|
|
15
17
|
readonly retries: FileRetries;
|
|
16
18
|
readonly download?: {
|
|
@@ -20,6 +22,7 @@ export interface IndexingSourceOptions {
|
|
|
20
22
|
}
|
|
21
23
|
export declare class IndexingSource {
|
|
22
24
|
private readonly _readBytes;
|
|
25
|
+
private readonly _log;
|
|
23
26
|
private readonly _retryMs;
|
|
24
27
|
private readonly _download;
|
|
25
28
|
private readonly _downloads;
|
|
@@ -1,15 +1,17 @@
|
|
|
1
|
-
import { isInvalidBlobReferenceError, compareBlobReferences } from '../store/index.js';
|
|
1
|
+
import { isInvalidBlobReferenceError, isPendingBlobReference, compareBlobReferences } from '../store/index.js';
|
|
2
2
|
import { AbortController } from 'bare-abort-controller';
|
|
3
3
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
4
4
|
const BLOB_RETRY_MS = 30_000;
|
|
5
5
|
export class IndexingSource {
|
|
6
6
|
_readBytes;
|
|
7
|
+
_log;
|
|
7
8
|
_retryMs;
|
|
8
9
|
_download;
|
|
9
10
|
_downloads = new Map();
|
|
10
11
|
retry;
|
|
11
12
|
constructor(readBytes, opts) {
|
|
12
13
|
this._readBytes = readBytes;
|
|
14
|
+
this._log = opts.log;
|
|
13
15
|
this._retryMs = opts.retryMs ?? BLOB_RETRY_MS;
|
|
14
16
|
this.retry = opts.retries;
|
|
15
17
|
this._download = opts.download;
|
|
@@ -27,6 +29,9 @@ export class IndexingSource {
|
|
|
27
29
|
return this.retry.reconnect(onRetry);
|
|
28
30
|
}
|
|
29
31
|
async read(id, blobRef, signal) {
|
|
32
|
+
if (isPendingBlobReference(blobRef)) {
|
|
33
|
+
return { kind: 'missing', retryAt: Date.now() + this._retryMs };
|
|
34
|
+
}
|
|
30
35
|
try {
|
|
31
36
|
const bytes = await this._readBytes(blobRef, { signal, wait: !this._download });
|
|
32
37
|
this.retry.markFound(id);
|
|
@@ -37,10 +42,10 @@ export class IndexingSource {
|
|
|
37
42
|
throw new AbortedError();
|
|
38
43
|
}
|
|
39
44
|
if (isInvalidBlobReferenceError(err)) {
|
|
40
|
-
|
|
45
|
+
this._log?.warn('rag', 'blob ref is malformed, never readable', { id });
|
|
41
46
|
return { kind: 'malformed' };
|
|
42
47
|
}
|
|
43
|
-
|
|
48
|
+
this._log?.debug('rag', 'blob bytes unavailable', { id, err });
|
|
44
49
|
this.retry.markMissing(id);
|
|
45
50
|
const retryAt = this.retryMiss(id, Date.now());
|
|
46
51
|
this._fetch(id, blobRef);
|
|
@@ -88,7 +93,7 @@ export class IndexingSource {
|
|
|
88
93
|
download.ready(id);
|
|
89
94
|
}, (err) => {
|
|
90
95
|
if (!abort.signal.aborted) {
|
|
91
|
-
|
|
96
|
+
this._log?.debug('rag', 'blob download interrupted', { id, err });
|
|
92
97
|
}
|
|
93
98
|
})
|
|
94
99
|
.finally(() => {
|
|
@@ -4,6 +4,9 @@ export type LocalWork = Pick<AssistantFileProcessorStatus, 'state' | 'reason' |
|
|
|
4
4
|
export type IndexingProgressEntry = Omit<Parameters<IndexingProgress>[0], 'error'> & {
|
|
5
5
|
error: string | null;
|
|
6
6
|
};
|
|
7
|
+
export type ChatProgressEntry = IndexingProgressEntry & {
|
|
8
|
+
readonly chatId: string;
|
|
9
|
+
};
|
|
7
10
|
export interface LoadedArtifact {
|
|
8
11
|
artifactId: string;
|
|
9
12
|
blobRef?: string;
|
|
@@ -58,7 +61,7 @@ export declare class FileState {
|
|
|
58
61
|
forget(id: string): DocumentState | undefined;
|
|
59
62
|
get(id: string): Readonly<DocumentState> | undefined;
|
|
60
63
|
readArtifact(id: string): Readonly<ArtifactState> | undefined;
|
|
61
|
-
progress(chatId: string):
|
|
64
|
+
progress(chatId: string): ChatProgressEntry[];
|
|
62
65
|
clearLoaded(): void;
|
|
63
66
|
retry(id: string): void;
|
|
64
67
|
reset(): void;
|
|
@@ -82,7 +82,7 @@ export class FileState {
|
|
|
82
82
|
return this._artifacts.get(id);
|
|
83
83
|
}
|
|
84
84
|
progress(chatId) {
|
|
85
|
-
return [...this._documents.values()].flatMap((entry) => entry.progress?.chatId === chatId ? [entry.progress] : []);
|
|
85
|
+
return [...this._documents.values()].flatMap((entry) => entry.progress?.chatId === chatId ? [{ ...entry.progress, chatId }] : []);
|
|
86
86
|
}
|
|
87
87
|
clearLoaded() {
|
|
88
88
|
for (const entry of this._documents.values()) {
|
|
@@ -109,6 +109,7 @@ export class FileState {
|
|
|
109
109
|
output.rebuildFailures = undefined;
|
|
110
110
|
this.outputRetries.clear(artifactId);
|
|
111
111
|
}
|
|
112
|
+
entry.artifactIdMemo = undefined;
|
|
112
113
|
}
|
|
113
114
|
reset() {
|
|
114
115
|
this._documents.clear();
|
|
@@ -7,8 +7,9 @@ export interface BuildEmbedder {
|
|
|
7
7
|
texts: string[];
|
|
8
8
|
signal?: AbortSignal;
|
|
9
9
|
}): Promise<number[][]>;
|
|
10
|
+
idle?(signal: AbortSignal): Promise<void>;
|
|
10
11
|
}
|
|
11
|
-
export type BuildProgress = (processedChunks: number, totalChunks: number) => void
|
|
12
|
+
export type BuildProgress = (processedChunks: number, totalChunks: number) => void | Promise<void>;
|
|
12
13
|
export declare class EmbedderFailure extends Error {
|
|
13
14
|
constructor(cause: unknown);
|
|
14
15
|
}
|
|
@@ -19,6 +20,8 @@ export interface ArtifactBuildInput {
|
|
|
19
20
|
bytes: Buffer;
|
|
20
21
|
signal: AbortSignal;
|
|
21
22
|
onProgress?: BuildProgress;
|
|
23
|
+
onExtracted?: () => void | Promise<void>;
|
|
22
24
|
extractors?: RagExtractors;
|
|
23
25
|
}
|
|
26
|
+
export declare function yieldTurn(signal: AbortSignal): Promise<void>;
|
|
24
27
|
export declare function buildArtifactPayload(input: ArtifactBuildInput): Promise<RagArtifactPayload>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { chunkRagDocuments, extractRagDocumentFromBytes } from 'rag-extract';
|
|
2
2
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
3
3
|
import { errorMessage } from '../../../guards.js';
|
|
4
|
+
import { IndexingFailure } from './failure.js';
|
|
4
5
|
const EMBED_BATCH = 8;
|
|
5
6
|
export class EmbedderFailure extends Error {
|
|
6
7
|
constructor(cause) {
|
|
@@ -8,27 +9,37 @@ export class EmbedderFailure extends Error {
|
|
|
8
9
|
this.name = 'EmbedderFailure';
|
|
9
10
|
}
|
|
10
11
|
}
|
|
12
|
+
export async function yieldTurn(signal) {
|
|
13
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
14
|
+
if (signal.aborted) {
|
|
15
|
+
throw new AbortedError();
|
|
16
|
+
}
|
|
17
|
+
}
|
|
11
18
|
export async function buildArtifactPayload(input) {
|
|
12
|
-
const { embedder, knowledge, artifactId, bytes, signal, onProgress, extractors } = input;
|
|
19
|
+
const { embedder, knowledge, artifactId, bytes, signal, onProgress, onExtracted, extractors } = input;
|
|
13
20
|
const document = await extractRagDocumentFromBytes({
|
|
14
21
|
id: artifactId,
|
|
15
22
|
artifactId,
|
|
16
23
|
attachmentId: knowledge.id,
|
|
17
|
-
chatId: knowledge.chatId,
|
|
24
|
+
chatId: knowledge.chatId ?? undefined,
|
|
18
25
|
blobId: knowledge.blobId,
|
|
19
26
|
fileName: knowledge.fileName,
|
|
20
27
|
mimeType: knowledge.mimeType,
|
|
21
28
|
bytes
|
|
22
29
|
}, { extractors, signal });
|
|
30
|
+
await yieldTurn(signal);
|
|
31
|
+
await onExtracted?.();
|
|
23
32
|
const chunks = chunkRagDocuments([document], {});
|
|
24
33
|
if (!document.content.trim() || chunks.length === 0) {
|
|
25
|
-
throw new
|
|
34
|
+
throw new IndexingFailure('no-text', `no extractable text: ${knowledge.fileName}`);
|
|
26
35
|
}
|
|
27
|
-
onProgress?.(0, chunks.length);
|
|
36
|
+
await onProgress?.(0, chunks.length);
|
|
28
37
|
const vectors = [];
|
|
29
38
|
for (let start = 0; start < chunks.length; start += EMBED_BATCH) {
|
|
30
|
-
|
|
31
|
-
|
|
39
|
+
await yieldTurn(signal);
|
|
40
|
+
if (!knowledge.chatId) {
|
|
41
|
+
await embedder.idle?.(signal);
|
|
42
|
+
}
|
|
32
43
|
const batch = chunks.slice(start, start + EMBED_BATCH);
|
|
33
44
|
let rows;
|
|
34
45
|
try {
|
|
@@ -38,7 +49,7 @@ export async function buildArtifactPayload(input) {
|
|
|
38
49
|
throw new EmbedderFailure(err);
|
|
39
50
|
}
|
|
40
51
|
vectors.push(...rows);
|
|
41
|
-
onProgress?.(Math.min(start + batch.length, chunks.length), chunks.length);
|
|
52
|
+
await onProgress?.(Math.min(start + batch.length, chunks.length), chunks.length);
|
|
42
53
|
}
|
|
43
54
|
if (vectors.length !== chunks.length) {
|
|
44
55
|
throw new Error(`embedder returned ${vectors.length} vectors for ${chunks.length} chunks`);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { QvacIndexingStatus } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
|
+
import { INDEXING_FAILURES } from '../../../../schema/engine.ts';
|
|
3
|
+
export type IndexingFailureReason = (typeof INDEXING_FAILURES)[number];
|
|
4
|
+
export interface ContentCheck {
|
|
5
|
+
readonly supported: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface ParsedFailure {
|
|
8
|
+
readonly reason: IndexingFailureReason;
|
|
9
|
+
readonly detail: string | null;
|
|
10
|
+
}
|
|
11
|
+
export declare class IndexingFailure extends Error {
|
|
12
|
+
readonly reason: IndexingFailureReason;
|
|
13
|
+
constructor(reason: IndexingFailureReason, message: string);
|
|
14
|
+
}
|
|
15
|
+
export declare function contentFailure(fileName: string, bytes: Uint8Array, { supported }: ContentCheck): IndexingFailure | null;
|
|
16
|
+
export declare function failureReason(err: unknown): IndexingFailureReason;
|
|
17
|
+
export declare function failureDetail(err: unknown): string;
|
|
18
|
+
export declare function failureError(err: unknown): string;
|
|
19
|
+
export declare function parseFailure(error: string | null | undefined): ParsedFailure;
|
|
20
|
+
export declare function isContentFailure(row: QvacIndexingStatus): boolean;
|