@qvac/core 1.0.0-rc.1 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1008 -1
- package/README.md +40 -81
- package/dist-lib/index.d.ts +1 -1
- package/dist-lib/index.js +1 -1
- package/dist-lib/lib/api/agent-folders.d.ts +22 -0
- package/dist-lib/lib/api/agent-folders.js +115 -0
- package/dist-lib/lib/api/agents.d.ts +19 -2
- package/dist-lib/lib/api/agents.js +121 -24
- package/dist-lib/lib/api/app-config.d.ts +4 -1
- package/dist-lib/lib/api/app-config.js +17 -6
- package/dist-lib/lib/api/attachments.js +3 -2
- package/dist-lib/lib/api/chat-move.d.ts +4 -0
- package/dist-lib/lib/api/chat-move.js +7 -0
- package/dist-lib/lib/api/chat.d.ts +13 -2
- package/dist-lib/lib/api/chat.js +114 -9
- package/dist-lib/lib/api/chunk-delta.d.ts +12 -0
- package/dist-lib/lib/api/chunk-delta.js +115 -0
- package/dist-lib/lib/api/chunks.d.ts +17 -2
- package/dist-lib/lib/api/chunks.js +70 -13
- package/dist-lib/lib/api/config.d.ts +6 -2
- package/dist-lib/lib/api/config.js +47 -5
- package/dist-lib/lib/api/current-mesh.d.ts +13 -5
- package/dist-lib/lib/api/current-mesh.js +15 -11
- package/dist-lib/lib/api/custom-models.d.ts +7 -1
- package/dist-lib/lib/api/custom-models.js +12 -12
- package/dist-lib/lib/api/device-routes.d.ts +9 -0
- package/dist-lib/lib/api/device-routes.js +123 -0
- package/dist-lib/lib/api/devices.d.ts +3 -2
- package/dist-lib/lib/api/devices.js +23 -11
- package/dist-lib/lib/api/diagnostics.d.ts +2 -1
- package/dist-lib/lib/api/diagnostics.js +2 -2
- package/dist-lib/lib/api/execution.d.ts +12 -1
- package/dist-lib/lib/api/execution.js +44 -14
- package/dist-lib/lib/api/files.d.ts +1 -1
- package/dist-lib/lib/api/files.js +65 -8
- package/dist-lib/lib/api/host-run-log.d.ts +2 -41
- package/dist-lib/lib/api/host-run-log.js +35 -83
- package/dist-lib/lib/api/index.d.ts +28 -6
- package/dist-lib/lib/api/index.js +101 -33
- package/dist-lib/lib/api/instrument.d.ts +2 -2
- package/dist-lib/lib/api/instrument.js +24 -48
- package/dist-lib/lib/api/knowledge.d.ts +4 -1
- package/dist-lib/lib/api/knowledge.js +25 -7
- package/dist-lib/lib/api/local-files.d.ts +5 -1
- package/dist-lib/lib/api/local-files.js +9 -5
- package/dist-lib/lib/api/logs.js +3 -3
- package/dist-lib/lib/api/members.d.ts +10 -1
- package/dist-lib/lib/api/members.js +67 -26
- package/dist-lib/lib/api/mesh.d.ts +12 -4
- package/dist-lib/lib/api/mesh.js +12 -8
- package/dist-lib/lib/api/model-download.d.ts +2 -1
- package/dist-lib/lib/api/model-download.js +2 -2
- package/dist-lib/lib/api/network.d.ts +10 -2
- package/dist-lib/lib/api/network.js +8 -2
- package/dist-lib/lib/api/notifications.d.ts +3 -1
- package/dist-lib/lib/api/notifications.js +14 -9
- package/dist-lib/lib/api/projects.d.ts +20 -0
- package/dist-lib/lib/api/projects.js +169 -0
- package/dist-lib/lib/api/rag.d.ts +0 -1
- package/dist-lib/lib/api/rag.js +71 -34
- package/dist-lib/lib/api/served.d.ts +17 -0
- package/dist-lib/lib/api/served.js +48 -0
- package/dist-lib/lib/api/sync.d.ts +2 -1
- package/dist-lib/lib/api/sync.js +2 -2
- package/dist-lib/lib/api/tool-activity.d.ts +2 -1
- package/dist-lib/lib/api/tool-activity.js +4 -3
- package/dist-lib/lib/api/tool-configs.d.ts +2 -1
- package/dist-lib/lib/api/tool-configs.js +6 -9
- package/dist-lib/lib/api/upload-frames.d.ts +0 -1
- package/dist-lib/lib/api/upload-frames.js +0 -1
- package/dist-lib/lib/api/watchable.d.ts +3 -2
- package/dist-lib/lib/api/watchable.js +2 -3
- package/dist-lib/lib/api/workspace.d.ts +5 -0
- package/dist-lib/lib/api/workspace.js +47 -0
- package/dist-lib/lib/assistant/fake-models.d.ts +20 -0
- package/dist-lib/lib/assistant/fake-models.js +112 -0
- package/dist-lib/lib/assistant/harness.d.ts +37 -31
- package/dist-lib/lib/assistant/harness.js +51 -10
- package/dist-lib/lib/assistant/history.d.ts +30 -17
- package/dist-lib/lib/assistant/history.js +179 -87
- package/dist-lib/lib/assistant/host-run-log.d.ts +6 -1
- package/dist-lib/lib/assistant/host-run-log.js +24 -36
- package/dist-lib/lib/assistant/message.d.ts +27 -1
- package/dist-lib/lib/assistant/message.js +65 -2
- package/dist-lib/lib/assistant/model-round.d.ts +100 -0
- package/dist-lib/lib/assistant/model-round.js +619 -0
- package/dist-lib/lib/assistant/pdf-budget.d.ts +11 -0
- package/dist-lib/lib/assistant/pdf-budget.js +12 -0
- package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant/round-host.d.ts +42 -0
- package/dist-lib/lib/assistant/round-host.js +241 -0
- package/dist-lib/lib/assistant/skill-encode.d.ts +2 -1
- package/dist-lib/lib/assistant/skill-encode.js +2 -1
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-client.d.ts +4 -1
- package/dist-lib/lib/assistant-client.js +27 -8
- package/dist-lib/lib/assistant-tree.d.ts +10 -3
- package/dist-lib/lib/assistant-tree.js +63 -20
- package/dist-lib/lib/assistant.d.ts +93 -24
- package/dist-lib/lib/assistant.js +2048 -1231
- package/dist-lib/lib/audio-format.js +1 -1
- package/dist-lib/lib/chat-suggestion.d.ts +3 -0
- package/dist-lib/lib/chat-suggestion.js +47 -0
- package/dist-lib/lib/client.d.ts +2 -2
- package/dist-lib/lib/client.js +10 -1
- package/dist-lib/lib/compaction.d.ts +3 -1
- package/dist-lib/lib/compaction.js +18 -4
- package/dist-lib/lib/constants.d.ts +10 -0
- package/dist-lib/lib/constants.js +10 -0
- package/dist-lib/lib/core/agent-copies.d.ts +9 -0
- package/dist-lib/lib/core/agent-copies.js +40 -0
- package/dist-lib/lib/core/agent-hosting.d.ts +25 -0
- package/dist-lib/lib/core/agent-hosting.js +71 -0
- package/dist-lib/lib/core/agent.d.ts +9 -1
- package/dist-lib/lib/core/agent.js +45 -19
- package/dist-lib/lib/core/archived-mesh.d.ts +11 -6
- package/dist-lib/lib/core/archived-mesh.js +77 -58
- package/dist-lib/lib/core/async-iterable.js +1 -1
- package/dist-lib/lib/core/blind-peers.d.ts +1 -0
- package/dist-lib/lib/core/blind-peers.js +1 -0
- package/dist-lib/lib/core/chat-activity.d.ts +1 -1
- package/dist-lib/lib/core/chat-activity.js +45 -34
- package/dist-lib/lib/core/chat-copy.d.ts +42 -0
- package/dist-lib/lib/core/chat-copy.js +395 -0
- package/dist-lib/lib/core/chat-fork.d.ts +16 -0
- package/dist-lib/lib/core/chat-fork.js +246 -0
- package/dist-lib/lib/core/chat-group.js +8 -7
- package/dist-lib/lib/core/chat-move.d.ts +75 -0
- package/dist-lib/lib/core/chat-move.js +598 -0
- package/dist-lib/lib/core/chat-order.d.ts +2 -1
- package/dist-lib/lib/core/chat-order.js +19 -4
- package/dist-lib/lib/core/chat-project.d.ts +2 -0
- package/dist-lib/lib/core/chat-project.js +15 -0
- package/dist-lib/lib/core/chat-read.d.ts +4 -0
- package/dist-lib/lib/core/chat-read.js +49 -0
- package/dist-lib/lib/core/chat-search.js +14 -12
- package/dist-lib/lib/core/chat.d.ts +33 -4
- package/dist-lib/lib/core/chat.js +122 -39
- package/dist-lib/lib/core/chunk.d.ts +35 -7
- package/dist-lib/lib/core/chunk.js +250 -45
- package/dist-lib/lib/core/collections.d.ts +40 -1
- package/dist-lib/lib/core/collections.js +40 -1
- package/dist-lib/lib/core/config-value.d.ts +6 -0
- package/dist-lib/lib/core/config-value.js +14 -0
- package/dist-lib/lib/core/contract/activity.d.ts +3 -2
- package/dist-lib/lib/core/contract/activity.js +39 -21
- package/dist-lib/lib/core/contract/agent-join.d.ts +10 -0
- package/dist-lib/lib/core/contract/agent-join.js +104 -0
- package/dist-lib/lib/core/contract/agents.d.ts +5 -0
- package/dist-lib/lib/core/contract/agents.js +142 -3
- package/dist-lib/lib/core/contract/chat.d.ts +4 -0
- package/dist-lib/lib/core/contract/chat.js +53 -3
- package/dist-lib/lib/core/contract/chunks.d.ts +7 -0
- package/dist-lib/lib/core/contract/chunks.js +125 -38
- package/dist-lib/lib/core/contract/devices.js +6 -4
- package/dist-lib/lib/core/contract/index.d.ts +2 -3
- package/dist-lib/lib/core/contract/index.js +36 -11
- package/dist-lib/lib/core/contract/indexing.d.ts +1 -0
- package/dist-lib/lib/core/contract/indexing.js +51 -7
- package/dist-lib/lib/core/contract/invites.d.ts +3 -14
- package/dist-lib/lib/core/contract/invites.js +26 -36
- package/dist-lib/lib/core/contract/joined-meshes.js +3 -1
- package/dist-lib/lib/core/contract/knowledge.d.ts +1 -0
- package/dist-lib/lib/core/contract/knowledge.js +57 -6
- package/dist-lib/lib/core/contract/members.d.ts +3 -1
- package/dist-lib/lib/core/contract/members.js +34 -4
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +3 -3
- package/dist-lib/lib/core/contract/mesh-kind.js +20 -16
- package/dist-lib/lib/core/contract/mesh-name.d.ts +3 -0
- package/dist-lib/lib/core/contract/mesh-name.js +31 -0
- package/dist-lib/lib/core/contract/mesh-rules.d.ts +8 -0
- package/dist-lib/lib/core/contract/mesh-rules.js +35 -0
- package/dist-lib/lib/core/contract/mesh-seeds.d.ts +2 -0
- package/dist-lib/lib/core/contract/mesh-seeds.js +26 -0
- package/dist-lib/lib/core/contract/model-requests.d.ts +26 -0
- package/dist-lib/lib/core/contract/model-requests.js +280 -0
- package/dist-lib/lib/core/contract/oldest-version.d.ts +9 -0
- package/dist-lib/lib/core/contract/oldest-version.js +20 -0
- package/dist-lib/lib/core/contract/open-join.d.ts +2 -0
- package/dist-lib/lib/core/contract/open-join.js +77 -0
- package/dist-lib/lib/core/contract/project-device-keys.d.ts +7 -0
- package/dist-lib/lib/core/contract/project-device-keys.js +38 -0
- package/dist-lib/lib/core/contract/project-host-keys.d.ts +7 -0
- package/dist-lib/lib/core/contract/project-host-keys.js +41 -0
- package/dist-lib/lib/core/contract/project-order.d.ts +2 -0
- package/dist-lib/lib/core/contract/project-order.js +25 -0
- package/dist-lib/lib/core/contract/projects.d.ts +21 -0
- package/dist-lib/lib/core/contract/projects.js +327 -0
- package/dist-lib/lib/core/contract/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/contract/public-mesh.js +31 -0
- package/dist-lib/lib/core/contract/public-seeds.d.ts +9 -0
- package/dist-lib/lib/core/contract/public-seeds.js +43 -0
- package/dist-lib/lib/core/contract/roles.d.ts +10 -0
- package/dist-lib/lib/core/contract/roles.js +121 -0
- package/dist-lib/lib/core/contract/run-requests.d.ts +10 -0
- package/dist-lib/lib/core/contract/run-requests.js +81 -0
- package/dist-lib/lib/core/contract/seed-defaults.d.ts +3 -1
- package/dist-lib/lib/core/contract/seed-defaults.js +17 -4
- package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
- package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
- package/dist-lib/lib/core/contract/skill-secrets.d.ts +3 -0
- package/dist-lib/lib/core/contract/skill-secrets.js +32 -0
- package/dist-lib/lib/core/contract/tool-activity.js +12 -1
- package/dist-lib/lib/core/contract/tool-configs.js +2 -1
- package/dist-lib/lib/core/contract/types.d.ts +2 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +31 -0
- package/dist-lib/lib/core/contract/workspace.js +311 -0
- package/dist-lib/lib/core/contract/writers.js +100 -16
- package/dist-lib/lib/core/departures.d.ts +16 -5
- package/dist-lib/lib/core/departures.js +79 -6
- package/dist-lib/lib/core/device-control.d.ts +21 -0
- package/dist-lib/lib/core/device-control.js +91 -0
- package/dist-lib/lib/core/device-seal.d.ts +4 -0
- package/dist-lib/lib/core/device-seal.js +22 -0
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +5 -3
- package/dist-lib/lib/core/document-attachment.d.ts +10 -0
- package/dist-lib/lib/core/document-attachment.js +14 -0
- package/dist-lib/lib/core/files/attachments.js +8 -7
- package/dist-lib/lib/core/files/index.d.ts +27 -14
- package/dist-lib/lib/core/files/index.js +220 -43
- package/dist-lib/lib/core/files/knowledge.d.ts +10 -1
- package/dist-lib/lib/core/files/knowledge.js +77 -5
- package/dist-lib/lib/core/files/preview/index.js +6 -2
- package/dist-lib/lib/core/files/processing/claims.d.ts +7 -0
- package/dist-lib/lib/core/files/processing/claims.js +33 -7
- package/dist-lib/lib/core/files/processing/document.d.ts +8 -0
- package/dist-lib/lib/core/files/processing/document.js +26 -2
- package/dist-lib/lib/core/files/processing/index.d.ts +8 -3
- package/dist-lib/lib/core/files/processing/index.js +99 -73
- package/dist-lib/lib/core/files/processing/protocol.d.ts +7 -1
- package/dist-lib/lib/core/files/processing/protocol.js +14 -3
- package/dist-lib/lib/core/files/processing/source.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/source.js +12 -4
- package/dist-lib/lib/core/files/processing/state.d.ts +5 -1
- package/dist-lib/lib/core/files/processing/state.js +9 -1
- package/dist-lib/lib/core/files/rag/build.d.ts +5 -1
- package/dist-lib/lib/core/files/rag/build.js +32 -8
- package/dist-lib/lib/core/files/rag/failure.d.ts +20 -0
- package/dist-lib/lib/core/files/rag/failure.js +57 -0
- package/dist-lib/lib/core/files/rag/index.d.ts +13 -2
- package/dist-lib/lib/core/files/rag/index.js +30 -8
- package/dist-lib/lib/core/files/rag/joined-index.d.ts +6 -0
- package/dist-lib/lib/core/files/rag/joined-index.js +35 -0
- package/dist-lib/lib/core/files/rag/limits.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/limits.js +1 -0
- package/dist-lib/lib/core/files/rag/load.js +6 -3
- package/dist-lib/lib/core/files/rag/policy.d.ts +10 -1
- package/dist-lib/lib/core/files/rag/policy.js +23 -2
- package/dist-lib/lib/core/files/rag/produce.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +31 -18
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +8 -2
- package/dist-lib/lib/core/files/rag/similarity.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/similarity.js +2 -3
- package/dist-lib/lib/core/files/records/chunks.js +22 -1
- package/dist-lib/lib/core/files/records/index.d.ts +1 -1
- package/dist-lib/lib/core/files/records/index.js +1 -1
- package/dist-lib/lib/core/files/records/knowledge.d.ts +11 -1
- package/dist-lib/lib/core/files/records/knowledge.js +40 -3
- package/dist-lib/lib/core/files/records/purge.js +13 -8
- package/dist-lib/lib/core/files/records/reclaim.js +20 -48
- package/dist-lib/lib/core/files/records/routing.d.ts +1 -1
- package/dist-lib/lib/core/files/records/routing.js +6 -2
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/records/sweep.d.ts +3 -0
- package/dist-lib/lib/core/files/records/sweep.js +9 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +2 -0
- package/dist-lib/lib/core/files/store/blob.js +2 -2
- package/dist-lib/lib/core/files/store/index.d.ts +3 -1
- package/dist-lib/lib/core/files/store/index.js +4 -1
- package/dist-lib/lib/core/files/store/reference.d.ts +3 -0
- package/dist-lib/lib/core/files/store/reference.js +12 -2
- package/dist-lib/lib/core/files/store/sweep.js +3 -1
- package/dist-lib/lib/core/files/upload.d.ts +5 -1
- package/dist-lib/lib/core/files/upload.js +53 -14
- package/dist-lib/lib/core/first-event.d.ts +6 -0
- package/dist-lib/lib/core/first-event.js +13 -0
- package/dist-lib/lib/core/fs-drives.d.ts +6 -3
- package/dist-lib/lib/core/fs-drives.js +30 -18
- package/dist-lib/lib/core/id.d.ts +1 -0
- package/dist-lib/lib/core/id.js +3 -0
- package/dist-lib/lib/core/identity.d.ts +3 -1
- package/dist-lib/lib/core/identity.js +25 -4
- package/dist-lib/lib/core/invite.d.ts +6 -2
- package/dist-lib/lib/core/invite.js +22 -8
- package/dist-lib/lib/core/local/files.d.ts +2 -1
- package/dist-lib/lib/core/local/files.js +5 -7
- package/dist-lib/lib/core/local/index.d.ts +86 -7
- package/dist-lib/lib/core/local/index.js +312 -27
- package/dist-lib/lib/core/local/run-logs.d.ts +38 -0
- package/dist-lib/lib/core/local/run-logs.js +132 -0
- package/dist-lib/lib/core/member-key.d.ts +7 -0
- package/dist-lib/lib/core/member-key.js +24 -0
- package/dist-lib/lib/core/membership.d.ts +3 -14
- package/dist-lib/lib/core/membership.js +3 -96
- package/dist-lib/lib/core/mesh.d.ts +107 -30
- package/dist-lib/lib/core/mesh.js +625 -171
- package/dist-lib/lib/core/migrations/agent-tools-scope.d.ts +2 -0
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +24 -0
- package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
- package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +8 -6
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +69 -8
- package/dist-lib/lib/core/model-requests.d.ts +13 -0
- package/dist-lib/lib/core/model-requests.js +65 -0
- package/dist-lib/lib/core/next-frame.js +2 -1
- package/dist-lib/lib/core/notification.js +100 -25
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +12 -3
- package/dist-lib/lib/core/pdf/rasterize.js +77 -28
- package/dist-lib/lib/core/peer-duplex.d.ts +34 -0
- package/dist-lib/lib/core/{harness-loan.js → peer-duplex.js} +104 -45
- package/dist-lib/lib/core/project-keys.d.ts +13 -0
- package/dist-lib/lib/core/project-keys.js +56 -0
- package/dist-lib/lib/core/prompt-history.js +4 -4
- package/dist-lib/lib/core/protocol-pairing.d.ts +11 -0
- package/dist-lib/lib/core/protocol-pairing.js +22 -0
- package/dist-lib/lib/core/public-mesh.d.ts +26 -0
- package/dist-lib/lib/core/public-mesh.js +94 -0
- package/dist-lib/lib/core/purge.d.ts +3 -0
- package/dist-lib/lib/core/purge.js +7 -0
- package/dist-lib/lib/core/run-driver.d.ts +17 -0
- package/dist-lib/lib/core/run-driver.js +20 -0
- package/dist-lib/lib/core/run-requests.d.ts +11 -0
- package/dist-lib/lib/core/run-requests.js +58 -0
- package/dist-lib/lib/core/send.d.ts +2 -0
- package/dist-lib/lib/core/send.js +17 -4
- package/dist-lib/lib/core/senders.d.ts +19 -0
- package/dist-lib/lib/core/senders.js +45 -0
- package/dist-lib/lib/core/shared-agents.d.ts +40 -0
- package/dist-lib/lib/core/shared-agents.js +283 -0
- package/dist-lib/lib/core/skill-secrets.d.ts +10 -16
- package/dist-lib/lib/core/skill-secrets.js +90 -72
- package/dist-lib/lib/core/sync/background.d.ts +35 -0
- package/dist-lib/lib/core/sync/background.js +85 -0
- package/dist-lib/lib/core/sync/changed.d.ts +20 -0
- package/dist-lib/lib/core/sync/changed.js +7 -0
- package/dist-lib/lib/core/sync/devices.js +6 -3
- package/dist-lib/lib/core/sync/files.d.ts +1 -0
- package/dist-lib/lib/core/sync/files.js +50 -72
- package/dist-lib/lib/core/sync/index.d.ts +1 -1
- package/dist-lib/lib/core/sync/index.js +164 -81
- package/dist-lib/lib/core/sync/view-updates.d.ts +33 -0
- package/dist-lib/lib/core/sync/view-updates.js +133 -0
- package/dist-lib/lib/core/sync-status.js +3 -3
- package/dist-lib/lib/core/system-info.d.ts +2 -2
- package/dist-lib/lib/core/system-info.js +10 -11
- package/dist-lib/lib/core/tool-configs.d.ts +5 -3
- package/dist-lib/lib/core/tool-configs.js +11 -11
- package/dist-lib/lib/core/transcript-visibility.d.ts +5 -5
- package/dist-lib/lib/core/transcript-visibility.js +27 -31
- package/dist-lib/lib/core/upgrade-agents.d.ts +11 -0
- package/dist-lib/lib/core/upgrade-agents.js +78 -0
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +22 -0
- package/dist-lib/lib/core/workspace-upgrade.js +50 -0
- package/dist-lib/lib/core.d.ts +303 -84
- package/dist-lib/lib/core.js +3952 -1511
- package/dist-lib/lib/credential-pusher.d.ts +17 -7
- package/dist-lib/lib/credential-pusher.js +111 -62
- package/dist-lib/lib/custom-model-pusher.d.ts +2 -4
- package/dist-lib/lib/custom-model-task.d.ts +2 -1
- package/dist-lib/lib/custom-model-task.js +2 -2
- package/dist-lib/lib/device-tree.d.ts +18 -0
- package/dist-lib/lib/device-tree.js +32 -0
- package/dist-lib/lib/diagnostics.d.ts +7 -0
- package/dist-lib/lib/diagnostics.js +22 -0
- package/dist-lib/lib/doctor/checks.d.ts +15 -0
- package/dist-lib/lib/doctor/checks.js +272 -0
- package/dist-lib/lib/doctor/redact.d.ts +4 -0
- package/dist-lib/lib/doctor/redact.js +31 -0
- package/dist-lib/lib/doctor/render.d.ts +8 -0
- package/dist-lib/lib/doctor/render.js +83 -0
- package/dist-lib/lib/doctor/tree.d.ts +5 -0
- package/dist-lib/lib/doctor/tree.js +167 -0
- package/dist-lib/lib/downloads-allowed-pusher.d.ts +8 -0
- package/dist-lib/lib/downloads-allowed-pusher.js +37 -0
- package/dist-lib/lib/errors.d.ts +794 -0
- package/dist-lib/lib/errors.js +728 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +8 -1
- package/dist-lib/lib/fs-drive-bridge.js +312 -31
- package/dist-lib/lib/guards.d.ts +1 -0
- package/dist-lib/lib/guards.js +4 -0
- package/dist-lib/lib/inference-calls.d.ts +1 -2
- package/dist-lib/lib/inference-calls.js +2 -10
- package/dist-lib/lib/integrations/setup-routes.js +7 -5
- package/dist-lib/lib/knowledge-search.d.ts +1 -1
- package/dist-lib/lib/knowledge-search.js +5 -51
- package/dist-lib/lib/kv-cache-reclaim.d.ts +2 -1
- package/dist-lib/lib/kv-cache-reclaim.js +30 -0
- package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
- package/dist-lib/lib/kv-cache-retention.js +98 -0
- package/dist-lib/lib/latest-frame.d.ts +10 -0
- package/dist-lib/lib/latest-frame.js +48 -0
- package/dist-lib/lib/log.d.ts +7 -6
- package/dist-lib/lib/log.js +14 -7
- package/dist-lib/lib/logger-type.d.ts +2 -5
- package/dist-lib/lib/mesh-scoped-engine.js +42 -2
- package/dist-lib/lib/model-download-status.d.ts +3 -3
- package/dist-lib/lib/model-download.d.ts +16 -12
- package/dist-lib/lib/model-download.js +90 -34
- package/dist-lib/lib/model-idle-pusher.d.ts +2 -4
- package/dist-lib/lib/model-memory-cap.d.ts +27 -0
- package/dist-lib/lib/model-memory-cap.js +69 -0
- package/dist-lib/lib/models-watch.d.ts +1 -1
- package/dist-lib/lib/models-watch.js +4 -4
- package/dist-lib/lib/oauth/browser-connect.js +3 -2
- package/dist-lib/lib/oauth/flow-core.js +5 -3
- package/dist-lib/lib/oauth/flow.js +1 -1
- package/dist-lib/lib/oauth/loopback.d.ts +1 -0
- package/dist-lib/lib/oauth/loopback.js +57 -12
- package/dist-lib/lib/oauth/providers.js +4 -2
- package/dist-lib/lib/oauth/redirect-types.d.ts +1 -0
- package/dist-lib/lib/oauth/register.js +35 -19
- package/dist-lib/lib/oauth/request.d.ts +10 -0
- package/dist-lib/lib/oauth/request.js +22 -0
- package/dist-lib/lib/oauth/resolve.js +13 -5
- package/dist-lib/lib/oauth/token-exchange-core.js +31 -33
- package/dist-lib/lib/oauth/types.d.ts +3 -0
- package/dist-lib/lib/oauth.d.ts +2 -1
- package/dist-lib/lib/oauth.js +1 -0
- package/dist-lib/lib/persistent-dir.d.ts +1 -0
- package/dist-lib/lib/persistent-dir.js +9 -0
- package/dist-lib/lib/probe-rounds.d.ts +3 -0
- package/dist-lib/lib/probe-rounds.js +5 -0
- package/dist-lib/lib/reconnecting-watch.js +19 -11
- package/dist-lib/lib/run-input.d.ts +14 -0
- package/dist-lib/lib/run-input.js +33 -0
- package/dist-lib/lib/run-notices.d.ts +5 -0
- package/dist-lib/lib/run-notices.js +13 -2
- package/dist-lib/lib/run-replay.d.ts +44 -0
- package/dist-lib/lib/run-replay.js +158 -0
- package/dist-lib/lib/run-transcript.d.ts +11 -12
- package/dist-lib/lib/run-transcript.js +16 -18
- package/dist-lib/lib/suspend-lifecycle.js +15 -5
- package/dist-lib/lib/task.js +6 -4
- package/dist-lib/lib/test-env.d.ts +1 -0
- package/dist-lib/lib/test-env.js +4 -0
- package/dist-lib/schema/build.js +66 -55
- package/dist-lib/schema/config.d.ts +1 -1
- package/dist-lib/schema/config.js +1 -1
- package/dist-lib/schema/content.d.ts +5 -1
- package/dist-lib/schema/content.js +60 -12
- package/dist-lib/schema/engine.d.ts +1 -0
- package/dist-lib/schema/engine.js +780 -155
- package/dist-lib/schema/field-order.js +8 -1
- package/dist-lib/schema/indexes.d.ts +7 -0
- package/dist-lib/schema/indexes.js +48 -3
- package/dist-lib/schema/local.js +80 -5
- package/dist-lib/schema/qvac.d.ts +4 -1
- package/dist-lib/schema/qvac.js +474 -14
- package/dist-lib/spec/bind.js +212 -48
- package/dist-lib/spec/calls.js +46 -5
- package/dist-lib/spec/capabilities.d.ts +100 -18
- package/dist-lib/spec/config/hyperconf/index.js +1 -1
- package/dist-lib/spec/hrpc/hrpc.json +739 -206
- package/dist-lib/spec/hrpc/index.js +844 -345
- package/dist-lib/spec/hrpc/messages.js +3906 -1749
- package/dist-lib/spec/hyperschema/index.js +3906 -1749
- package/dist-lib/spec/hyperschema/schema.json +4809 -3119
- package/dist-lib/spec/hyperschema/types.d.ts +482 -62
- package/dist-lib/spec/local/hyperdb/db.json +95 -1
- package/dist-lib/spec/local/hyperdb/index.js +459 -2
- package/dist-lib/spec/local/hyperdb/messages.js +1565 -487
- package/dist-lib/spec/local/hyperschema/index.js +573 -16
- package/dist-lib/spec/local/hyperschema/schema.json +354 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +85 -2
- package/dist-lib/spec/mesh/hyperdb/db.json +577 -7
- package/dist-lib/spec/mesh/hyperdb/index.js +2224 -6
- package/dist-lib/spec/mesh/hyperdb/messages.js +6476 -1266
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +193 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +545 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +3655 -265
- package/dist-lib/spec/mesh/hyperschema/index.js +3655 -265
- package/dist-lib/spec/mesh/hyperschema/schema.json +1974 -19
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +498 -6
- package/package.json +64 -43
- package/worker/assistant-sidecar.mjs +18 -2
- package/worker/stow-entry.mjs +2 -0
- package/dist-lib/lib/core/contract/compute.d.ts +0 -11
- package/dist-lib/lib/core/contract/compute.js +0 -48
- package/dist-lib/lib/core/device-invites.d.ts +0 -11
- package/dist-lib/lib/core/device-invites.js +0 -63
- package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
- package/dist-lib/lib/core/files/processing/settle.js +0 -86
- package/dist-lib/lib/core/grant-expiry.d.ts +0 -11
- package/dist-lib/lib/core/grant-expiry.js +0 -66
- package/dist-lib/lib/core/handoff-channel.d.ts +0 -47
- package/dist-lib/lib/core/handoff-channel.js +0 -123
- package/dist-lib/lib/core/harness-loan.d.ts +0 -29
- package/dist-lib/lib/core/host-run-log-store.d.ts +0 -114
- package/dist-lib/lib/core/host-run-log-store.js +0 -976
- package/dist-lib/lib/core/hosted-agents.d.ts +0 -38
- package/dist-lib/lib/core/hosted-agents.js +0 -140
- package/dist-lib/lib/core/inference-calls-handoff.d.ts +0 -20
- package/dist-lib/lib/core/inference-calls-handoff.js +0 -37
- package/dist-lib/lib/core/inference-calls-receiver.d.ts +0 -14
- package/dist-lib/lib/core/inference-calls-receiver.js +0 -42
- package/dist-lib/lib/core/mesh-seed-handoff.d.ts +0 -25
- package/dist-lib/lib/core/mesh-seed-handoff.js +0 -52
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
- package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
- package/dist-lib/lib/core/secret-handoff.d.ts +0 -17
- package/dist-lib/lib/core/secret-handoff.js +0 -29
- package/dist-lib/lib/core/skill-overlay-handoff.d.ts +0 -25
- package/dist-lib/lib/core/skill-overlay-handoff.js +0 -50
- package/dist-lib/lib/core/skill-overlay-receiver.d.ts +0 -8
- package/dist-lib/lib/core/skill-overlay-receiver.js +0 -23
- package/dist-lib/lib/image-cache-dir.d.ts +0 -1
- package/dist-lib/lib/image-cache-dir.js +0 -5
- package/dist-lib/lib/log-levels.d.ts +0 -7
- package/dist-lib/lib/log-levels.js +0 -1
- package/dist-lib/lib/logger.d.ts +0 -5
- package/dist-lib/lib/logger.js +0 -18
- package/dist-lib/lib/oauth/loopback.stub.d.ts +0 -3
- package/dist-lib/lib/oauth/loopback.stub.js +0 -6
- package/tui/README.md +0 -142
- package/tui/app.mjs +0 -6939
- package/tui/boot.mjs +0 -111
- package/tui/index.mjs +0 -53
- package/tui/lib/boot-timing.mjs +0 -41
- package/tui/lib/busy-words.mjs +0 -116
- package/tui/lib/export.mjs +0 -194
- package/tui/lib/knowledge-command.mjs +0 -190
- package/tui/lib/knowledge-import.mjs +0 -171
- package/tui/lib/render.mjs +0 -196
- package/tui/lib/save-file.mjs +0 -88
- package/tui/lib/select.mjs +0 -81
- package/tui/lib/selection.mjs +0 -18
- package/tui/lib/setup.mjs +0 -101
- package/tui/lib/sidebar.mjs +0 -63
- package/tui/lib/splash-art.mjs +0 -107
- package/tui/lib/splash.mjs +0 -18
- package/tui/lib/upload-frames.mjs +0 -9
- package/tui/lib/usage.mjs +0 -59
- package/tui/lib/wrap.mjs +0 -143
- package/tui/main.mjs +0 -264
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
import { listKnowledge, removeKnowledge } from './records/index.js';
|
|
2
|
-
import { MAX_KNOWLEDGE_BYTES } from './rag/index.js';
|
|
1
|
+
import { addKnowledge, assertKnowledgeRoom, hasLiveSource, listKnowledge, removeKnowledge } from './records/index.js';
|
|
2
|
+
import { MAX_KNOWLEDGE_BYTES, isSupportedRagDocument } from './rag/index.js';
|
|
3
|
+
import { requireChatIn } from '../chunk.js';
|
|
4
|
+
import { randomId } from '../id.js';
|
|
5
|
+
import { CoreError } from '../../errors.js';
|
|
3
6
|
import { uploadFile } from './upload.js';
|
|
7
|
+
import { KNOWLEDGE_CAP_ROW } from '../contract/knowledge.js';
|
|
8
|
+
import { KNOWLEDGE_CAPS } from '../collections.js';
|
|
4
9
|
export { MAX_KNOWLEDGE_BYTES };
|
|
5
|
-
export function knowledgeFiles(mesh, { maxBytes = MAX_KNOWLEDGE_BYTES, inactivityMs, deviceId } = {}) {
|
|
10
|
+
export function knowledgeFiles(mesh, { maxBytes = MAX_KNOWLEDGE_BYTES, inactivityMs, deviceId, mobile } = {}) {
|
|
6
11
|
const list = async ({ chatId }) => ({
|
|
7
12
|
knowledge: await listKnowledge(mesh, { chatId })
|
|
8
13
|
});
|
|
@@ -22,10 +27,77 @@ export function knowledgeFiles(mesh, { maxBytes = MAX_KNOWLEDGE_BYTES, inactivit
|
|
|
22
27
|
}
|
|
23
28
|
},
|
|
24
29
|
downloadKnowledge: async function* ({ blobId }) {
|
|
25
|
-
|
|
30
|
+
let fetched = false;
|
|
31
|
+
try {
|
|
32
|
+
for await (const frame of mesh.files.read(blobId)) {
|
|
33
|
+
fetched = true;
|
|
34
|
+
yield frame;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
if (fetched && mobile?.() && (await hasLiveSource(mesh, null, blobId))) {
|
|
39
|
+
await mesh.files.clearFetched(blobId, blobId);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
26
42
|
},
|
|
27
43
|
listKnowledge: list,
|
|
28
44
|
listIndexing: indexingList,
|
|
29
|
-
removeKnowledge: async ({ knowledgeId, at }) => removeKnowledge(mesh, { knowledgeId, at })
|
|
45
|
+
removeKnowledge: async ({ knowledgeId, at }) => removeKnowledge(mesh, { knowledgeId, at }),
|
|
46
|
+
fileKnowledge: (request) => fileKnowledge(mesh, request)
|
|
30
47
|
};
|
|
31
48
|
}
|
|
49
|
+
async function fileKnowledge(mesh, { chatId, fileId }) {
|
|
50
|
+
if (mesh.role !== 'project') {
|
|
51
|
+
throw new CoreError('KNOWLEDGE_BASE_NOT_A_PROJECT');
|
|
52
|
+
}
|
|
53
|
+
await requireChatIn(mesh, chatId);
|
|
54
|
+
const file = await mesh.files.resolve({ chatId, fileId });
|
|
55
|
+
if (!file) {
|
|
56
|
+
throw new CoreError('KNOWLEDGE_NOT_FOUND', [fileId]);
|
|
57
|
+
}
|
|
58
|
+
if (!isSupportedRagDocument(file)) {
|
|
59
|
+
throw new CoreError('KNOWLEDGE_TYPE_UNSUPPORTED', [file.fileName]);
|
|
60
|
+
}
|
|
61
|
+
await assertKnowledgeRoom(mesh, file.byteLength);
|
|
62
|
+
const blobId = await copyBlob(mesh, file.blobId);
|
|
63
|
+
const id = randomId();
|
|
64
|
+
const { fileName, mimeType, byteLength } = file;
|
|
65
|
+
try {
|
|
66
|
+
return await addKnowledge(mesh, { id, fileName, mimeType, byteLength, blobId });
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
await mesh.files.clear(blobId, id).catch(noop);
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async function copyBlob(mesh, source) {
|
|
74
|
+
let blobId = null;
|
|
75
|
+
for await (const frame of mesh.files.write(bytesOf(mesh.files.read(source, { wait: false })))) {
|
|
76
|
+
if ('blobId' in frame) {
|
|
77
|
+
blobId = mesh.files.reference(frame.blobId);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (blobId === null) {
|
|
81
|
+
throw new Error('blob write ended without its blobId frame');
|
|
82
|
+
}
|
|
83
|
+
return blobId;
|
|
84
|
+
}
|
|
85
|
+
function noop() { }
|
|
86
|
+
async function* bytesOf(frames) {
|
|
87
|
+
for await (const { data } of frames) {
|
|
88
|
+
yield data;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
export async function setKnowledgeCap(mesh, bytes) {
|
|
92
|
+
if (mesh.role !== 'project') {
|
|
93
|
+
throw new CoreError('KNOWLEDGE_BASE_NOT_A_PROJECT');
|
|
94
|
+
}
|
|
95
|
+
if (bytes < 1) {
|
|
96
|
+
throw new CoreError('KNOWLEDGE_CAP_INVALID');
|
|
97
|
+
}
|
|
98
|
+
const current = await mesh.view.get(KNOWLEDGE_CAPS, { id: KNOWLEDGE_CAP_ROW });
|
|
99
|
+
await mesh.dispatch('set-knowledge-cap', {
|
|
100
|
+
bytes,
|
|
101
|
+
setAt: Math.max(Date.now(), (current?.setAt ?? 0) + 1)
|
|
102
|
+
});
|
|
103
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import crypto from 'hypercore-crypto';
|
|
3
3
|
import { processorArtifactId } from '../records/index.js';
|
|
4
|
-
import { parseBlobReference } from '../store/index.js';
|
|
4
|
+
import { isPendingBlobReference, parseBlobReference } from '../store/index.js';
|
|
5
5
|
import { IMAGE_MIMES } from '../../image/normalize.js';
|
|
6
6
|
import { previewEncoder } from './image.js';
|
|
7
7
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
@@ -30,7 +30,8 @@ export class PreviewProcessor {
|
|
|
30
30
|
IMAGE_MIMES.has(file.mimeType.toLowerCase()));
|
|
31
31
|
}
|
|
32
32
|
async canProduce(file, host) {
|
|
33
|
-
|
|
33
|
+
const chat = file.chatId ? await getChat(host.mesh, file.chatId) : null;
|
|
34
|
+
return !file.deletedAt && !chat?.deletedAt;
|
|
34
35
|
}
|
|
35
36
|
accepts(output, file) {
|
|
36
37
|
if (output.embedderVersion !== this.version)
|
|
@@ -151,6 +152,9 @@ export function previewId(file) {
|
|
|
151
152
|
});
|
|
152
153
|
}
|
|
153
154
|
function previewSource(file) {
|
|
155
|
+
if (isPendingBlobReference(file.blobId)) {
|
|
156
|
+
return hash([file.blobId]);
|
|
157
|
+
}
|
|
154
158
|
const { blobId, coreKey } = parseBlobReference(file.blobId);
|
|
155
159
|
return hash([
|
|
156
160
|
coreKey ?? '',
|
|
@@ -11,6 +11,12 @@ export interface IndexingClaimsOptions {
|
|
|
11
11
|
export interface IndexingClaim {
|
|
12
12
|
readonly claimId: string;
|
|
13
13
|
}
|
|
14
|
+
export interface IndexingLease {
|
|
15
|
+
keep(force?: boolean): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
export declare class ClaimLostError extends Error {
|
|
18
|
+
constructor(artifactId: string);
|
|
19
|
+
}
|
|
14
20
|
export declare class IndexingClaims {
|
|
15
21
|
private readonly _mesh;
|
|
16
22
|
private readonly _schedule;
|
|
@@ -22,6 +28,7 @@ export declare class IndexingClaims {
|
|
|
22
28
|
markFailed(artifactId: string): void;
|
|
23
29
|
isFailed(artifactId: string): boolean;
|
|
24
30
|
claim(artifactId: string, knowledgeId: string): Promise<IndexingClaim | null>;
|
|
31
|
+
lease(artifactId: string, claimId: string): IndexingLease;
|
|
25
32
|
blockedByStatus(artifactId: string): Promise<boolean>;
|
|
26
33
|
settle(artifactId: string, knowledgeId: string): Promise<void>;
|
|
27
34
|
}
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import { claimIndexing, getIndexingStatus, isClaimLive, resolveIndexing } from './protocol.js';
|
|
1
|
+
import { INDEXING_STALE_MS, claimIndexing, getIndexingStatus, isClaimLive, progressIndexing, resolveIndexing } from './protocol.js';
|
|
2
|
+
import { isContentFailure } from '../rag/failure.js';
|
|
3
|
+
export class ClaimLostError extends Error {
|
|
4
|
+
constructor(artifactId) {
|
|
5
|
+
super(`indexing claim taken over: ${artifactId}`);
|
|
6
|
+
this.name = 'ClaimLostError';
|
|
7
|
+
}
|
|
8
|
+
}
|
|
2
9
|
export class IndexingClaims {
|
|
3
10
|
_mesh;
|
|
4
11
|
_schedule;
|
|
@@ -42,18 +49,37 @@ export class IndexingClaims {
|
|
|
42
49
|
!(await getIndexingStatus(this._mesh, artifactId))) {
|
|
43
50
|
return { claimId: '' };
|
|
44
51
|
}
|
|
45
|
-
|
|
52
|
+
this._mesh.log?.debug('rag', 'claim not placed', { id: artifactId, err });
|
|
46
53
|
return null;
|
|
47
54
|
}
|
|
48
55
|
}
|
|
56
|
+
lease(artifactId, claimId) {
|
|
57
|
+
let last = Date.now();
|
|
58
|
+
return {
|
|
59
|
+
keep: async (force = false) => {
|
|
60
|
+
const at = Date.now();
|
|
61
|
+
if (!claimId || (!force && at - last < INDEXING_STALE_MS / 3)) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
last = at;
|
|
65
|
+
if (!(await progressIndexing(this._mesh, { artifactId, claimId, at }))) {
|
|
66
|
+
throw new ClaimLostError(artifactId);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
}
|
|
49
71
|
async blockedByStatus(artifactId) {
|
|
50
|
-
if (this.isFailed(artifactId))
|
|
51
|
-
return true;
|
|
52
72
|
const row = await getIndexingStatus(this._mesh, artifactId);
|
|
73
|
+
if (this.isFailed(artifactId)) {
|
|
74
|
+
if (row) {
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
this._state.artifact(artifactId).terminal = false;
|
|
78
|
+
}
|
|
53
79
|
if (!row) {
|
|
54
80
|
return false;
|
|
55
81
|
}
|
|
56
|
-
if (row.status === 'ready') {
|
|
82
|
+
if (row.status === 'ready' || isContentFailure(row)) {
|
|
57
83
|
return true;
|
|
58
84
|
}
|
|
59
85
|
if (isClaimLive(row, Date.now())) {
|
|
@@ -87,11 +113,11 @@ export class IndexingClaims {
|
|
|
87
113
|
status: 'ready'
|
|
88
114
|
});
|
|
89
115
|
if (applied) {
|
|
90
|
-
|
|
116
|
+
this._mesh.log?.info('rag', 'repaired to ready', { id: artifactId });
|
|
91
117
|
}
|
|
92
118
|
}
|
|
93
119
|
catch (err) {
|
|
94
|
-
|
|
120
|
+
this._mesh.log?.debug('rag', 'repair resolve dropped', { id: artifactId, err });
|
|
95
121
|
}
|
|
96
122
|
}
|
|
97
123
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LoggerInterface } from '@qvac/logging';
|
|
1
2
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
3
|
import { type IndexingProgress } from '../rag/index.ts';
|
|
3
4
|
import { IndexingClaims } from './claims.ts';
|
|
@@ -17,6 +18,7 @@ export interface FileDocumentOptions {
|
|
|
17
18
|
readonly closed: () => boolean;
|
|
18
19
|
readonly paused: () => boolean;
|
|
19
20
|
readonly progress?: IndexingProgress;
|
|
21
|
+
readonly admit?: () => void;
|
|
20
22
|
}
|
|
21
23
|
interface SourceReference {
|
|
22
24
|
readonly id: string;
|
|
@@ -32,16 +34,20 @@ export declare class FileDocument {
|
|
|
32
34
|
isFailed: IndexingClaims['isFailed'];
|
|
33
35
|
blockedByStatus: IndexingClaims['blockedByStatus'];
|
|
34
36
|
claim: (id: string) => ReturnType<IndexingClaims['claim']>;
|
|
37
|
+
lease: IndexingClaims['lease'];
|
|
35
38
|
settle: (id: string) => Promise<void>;
|
|
36
39
|
};
|
|
37
40
|
readonly ownership: ArtifactOwnership;
|
|
38
41
|
readonly schedule: (deadline: number) => void;
|
|
39
42
|
readonly closed: () => boolean;
|
|
40
43
|
readonly paused: () => boolean;
|
|
44
|
+
readonly admit: () => void;
|
|
41
45
|
private readonly _output;
|
|
46
|
+
private _fetched;
|
|
42
47
|
private readonly _progress?;
|
|
43
48
|
private readonly _processor;
|
|
44
49
|
constructor(mesh: Mesh, options: FileDocumentOptions);
|
|
50
|
+
get log(): LoggerInterface | null;
|
|
45
51
|
get localCoreKey(): string;
|
|
46
52
|
artifact(id: string): import("./state.ts").ArtifactState;
|
|
47
53
|
readArtifact(id: string): Readonly<import("./state.ts").ArtifactState> | undefined;
|
|
@@ -51,6 +57,8 @@ export declare class FileDocument {
|
|
|
51
57
|
miss: () => number;
|
|
52
58
|
clear: () => void;
|
|
53
59
|
};
|
|
60
|
+
clearFetched(): Promise<void>;
|
|
61
|
+
private _fetching;
|
|
54
62
|
artifacts(): Promise<import("../../../../spec/index").QvacKnowledgeArtifact[]>;
|
|
55
63
|
statuses(): Promise<import("../../../../spec/index").QvacIndexingStatus[]>;
|
|
56
64
|
current(): Promise<FileRecord | null>;
|
|
@@ -14,7 +14,9 @@ export class FileDocument {
|
|
|
14
14
|
schedule;
|
|
15
15
|
closed;
|
|
16
16
|
paused;
|
|
17
|
+
admit;
|
|
17
18
|
_output;
|
|
19
|
+
_fetched = false;
|
|
18
20
|
_progress;
|
|
19
21
|
_processor;
|
|
20
22
|
constructor(mesh, options) {
|
|
@@ -24,11 +26,12 @@ export class FileDocument {
|
|
|
24
26
|
this._processor = options.processor;
|
|
25
27
|
const state = mesh.files.processorState(options.processor.id);
|
|
26
28
|
this.state = state.document(record.id);
|
|
27
|
-
this.source = fileSource(options.source, { id: record.id, reference: record.blobId });
|
|
29
|
+
this.source = this._fetching(fileSource(options.source, { id: record.id, reference: record.blobId }));
|
|
28
30
|
this._output = options.output;
|
|
29
31
|
this.schedule = options.schedule;
|
|
30
32
|
this.closed = () => options.closed() || !!mesh.closing;
|
|
31
33
|
this.paused = options.paused;
|
|
34
|
+
this.admit = options.admit ?? (() => { });
|
|
32
35
|
this._progress = options.progress;
|
|
33
36
|
const claims = new IndexingClaims({
|
|
34
37
|
mesh,
|
|
@@ -41,10 +44,14 @@ export class FileDocument {
|
|
|
41
44
|
isFailed: (id) => claims.isFailed(id),
|
|
42
45
|
blockedByStatus: (id) => claims.blockedByStatus(id),
|
|
43
46
|
claim: (id) => claims.claim(id, record.id),
|
|
47
|
+
lease: (id, claimId) => claims.lease(id, claimId),
|
|
44
48
|
settle: (id) => claims.settle(id, record.id)
|
|
45
49
|
};
|
|
46
50
|
this.ownership = new ArtifactOwnership(mesh, state);
|
|
47
51
|
}
|
|
52
|
+
get log() {
|
|
53
|
+
return this._mesh.log;
|
|
54
|
+
}
|
|
48
55
|
get localCoreKey() {
|
|
49
56
|
return b4a.toHex(this._mesh.blobs.core.key);
|
|
50
57
|
}
|
|
@@ -56,7 +63,24 @@ export class FileDocument {
|
|
|
56
63
|
}
|
|
57
64
|
output(id, reference) {
|
|
58
65
|
this.artifact(id).knowledgeId = this.record.id;
|
|
59
|
-
return fileSource(this._output, { id, reference });
|
|
66
|
+
return this._fetching(fileSource(this._output, { id, reference }));
|
|
67
|
+
}
|
|
68
|
+
async clearFetched() {
|
|
69
|
+
if (this._fetched) {
|
|
70
|
+
await this._mesh.files.clearFetched(this.record.blobId, this.record.id);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
_fetching(source) {
|
|
74
|
+
return {
|
|
75
|
+
...source,
|
|
76
|
+
read: async (signal) => {
|
|
77
|
+
const read = await source.read(signal);
|
|
78
|
+
if (read.kind === 'bytes') {
|
|
79
|
+
this._fetched = true;
|
|
80
|
+
}
|
|
81
|
+
return read;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
60
84
|
}
|
|
61
85
|
artifacts() {
|
|
62
86
|
return this._mesh.files.artifacts(this.record.id);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import { type IndexingProgress } from '../rag/index.ts';
|
|
3
|
+
import type { IndexingDevice } from '../rag/policy.ts';
|
|
3
4
|
import { type FileInput, type FileRecord } from '../records/index.ts';
|
|
4
5
|
import type { Mesh } from '../../mesh.ts';
|
|
5
6
|
import { FileDocument } from './document.ts';
|
|
@@ -8,6 +9,7 @@ import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/ty
|
|
|
8
9
|
export interface ProcessorHost {
|
|
9
10
|
readonly mesh: Mesh;
|
|
10
11
|
readonly deviceId: Buffer | null;
|
|
12
|
+
readonly device?: IndexingDevice;
|
|
11
13
|
}
|
|
12
14
|
export interface FileProcessor {
|
|
13
15
|
readonly id: string;
|
|
@@ -26,6 +28,7 @@ export interface FileProcessor {
|
|
|
26
28
|
export interface FileProcessingOptions {
|
|
27
29
|
readonly processor: FileProcessor;
|
|
28
30
|
readonly deviceId: Buffer | null;
|
|
31
|
+
readonly device?: IndexingDevice;
|
|
29
32
|
readonly retryMs?: number;
|
|
30
33
|
readonly progress?: IndexingProgress;
|
|
31
34
|
}
|
|
@@ -36,6 +39,7 @@ export declare class FileProcessing {
|
|
|
36
39
|
private readonly _mesh;
|
|
37
40
|
private readonly _processor;
|
|
38
41
|
private readonly _deviceId;
|
|
42
|
+
private readonly _device?;
|
|
39
43
|
private readonly _source;
|
|
40
44
|
private readonly _output;
|
|
41
45
|
private readonly _progress?;
|
|
@@ -55,6 +59,7 @@ export declare class FileProcessing {
|
|
|
55
59
|
suspend(): Promise<void>;
|
|
56
60
|
resume(): void;
|
|
57
61
|
reconnect(): void;
|
|
62
|
+
rescan(): void;
|
|
58
63
|
close(): Promise<void>;
|
|
59
64
|
get indexCleared(): boolean;
|
|
60
65
|
settle(): Promise<void>;
|
|
@@ -62,6 +67,7 @@ export declare class FileProcessing {
|
|
|
62
67
|
private _enqueue;
|
|
63
68
|
get active(): boolean;
|
|
64
69
|
pending(id: string): boolean;
|
|
70
|
+
awaitsBytes(id: string): boolean;
|
|
65
71
|
prioritize(ids: string[]): void;
|
|
66
72
|
private _shutdown;
|
|
67
73
|
private _stop;
|
|
@@ -70,14 +76,13 @@ export declare class FileProcessing {
|
|
|
70
76
|
private _documents;
|
|
71
77
|
private _pump;
|
|
72
78
|
private _forget;
|
|
73
|
-
private
|
|
79
|
+
private _run;
|
|
74
80
|
private _recordFailure;
|
|
75
81
|
private _queueLoad;
|
|
76
82
|
private _schedule;
|
|
77
83
|
private _retry;
|
|
78
84
|
}
|
|
79
85
|
export type { FileDocument } from './document.ts';
|
|
80
|
-
export { isClaimLive, listIndexingByChat } from './protocol.ts';
|
|
86
|
+
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.ts';
|
|
81
87
|
export { FileState } from './state.ts';
|
|
82
88
|
export type { IndexingProgressEntry, LocalWork } from './state.ts';
|
|
83
|
-
export { settleFiles } from './settle.ts';
|
|
@@ -9,6 +9,7 @@ export class FileProcessing {
|
|
|
9
9
|
_mesh;
|
|
10
10
|
_processor;
|
|
11
11
|
_deviceId;
|
|
12
|
+
_device;
|
|
12
13
|
_source;
|
|
13
14
|
_output;
|
|
14
15
|
_progress;
|
|
@@ -34,15 +35,18 @@ export class FileProcessing {
|
|
|
34
35
|
this._mesh = mesh;
|
|
35
36
|
this._processor = options.processor;
|
|
36
37
|
this._deviceId = options.deviceId;
|
|
38
|
+
this._device = options.device;
|
|
37
39
|
this._progress = options.progress;
|
|
38
40
|
const readBytes = mesh.files.readBytes.bind(mesh.files);
|
|
39
41
|
const fetch = mesh.files.download.bind(mesh.files);
|
|
40
42
|
this._source = new IndexingSource(readBytes, {
|
|
43
|
+
log: mesh.log,
|
|
41
44
|
retryMs: options.retryMs,
|
|
42
45
|
retries: mesh.files.processorState(options.processor.id).sourceRetries,
|
|
43
46
|
download: { fetch, ready: (id) => this._schedule(id, Date.now()) }
|
|
44
47
|
});
|
|
45
48
|
this._output = new IndexingSource(readBytes, {
|
|
49
|
+
log: mesh.log,
|
|
46
50
|
retryMs: options.retryMs,
|
|
47
51
|
retries: mesh.files.processorState(options.processor.id).outputRetries,
|
|
48
52
|
download: {
|
|
@@ -50,9 +54,9 @@ export class FileProcessing {
|
|
|
50
54
|
ready: (id) => this._schedule(mesh.files.processorState(options.processor.id).artifact(id).knowledgeId ?? id, Date.now())
|
|
51
55
|
}
|
|
52
56
|
});
|
|
53
|
-
this._lanes.push({ stage: 'produce', queue: new Map(),
|
|
57
|
+
this._lanes.push({ stage: 'produce', queue: new Map(), jobs: new Set() });
|
|
54
58
|
if (options.processor.load) {
|
|
55
|
-
this._lanes.push({ stage: 'load', queue: new Map(),
|
|
59
|
+
this._lanes.push({ stage: 'load', queue: new Map(), jobs: new Set() });
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
start() {
|
|
@@ -75,6 +79,9 @@ export class FileProcessing {
|
|
|
75
79
|
this._source.reconnect((id, deadline) => this._schedule(id, deadline));
|
|
76
80
|
this._output.reconnect((id, deadline) => this._schedule(this._mesh.files.processorState(this._processor.id).artifact(id).knowledgeId ?? id, deadline));
|
|
77
81
|
}
|
|
82
|
+
rescan() {
|
|
83
|
+
this._kick();
|
|
84
|
+
}
|
|
78
85
|
close() {
|
|
79
86
|
this._close ??= this._shutdown();
|
|
80
87
|
return this._close;
|
|
@@ -119,16 +126,23 @@ export class FileProcessing {
|
|
|
119
126
|
pending(id) {
|
|
120
127
|
return (!this._paused &&
|
|
121
128
|
!this._closed &&
|
|
122
|
-
(this._lanes.some((lane) => lane.
|
|
129
|
+
(this._lanes.some((lane) => lane.queue.has(id) || [...lane.jobs].some((job) => job.id === id)) ||
|
|
123
130
|
this._source.pending((key) => key === id) ||
|
|
124
131
|
this._output.pending((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id)));
|
|
125
132
|
}
|
|
133
|
+
awaitsBytes(id) {
|
|
134
|
+
return (this._source.awaitsBytes((key) => key === id) ||
|
|
135
|
+
this._output.awaitsBytes((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id));
|
|
136
|
+
}
|
|
126
137
|
prioritize(ids) {
|
|
127
138
|
for (const lane of this._lanes) {
|
|
128
139
|
const queued = new Map(lane.queue);
|
|
129
140
|
lane.queue.clear();
|
|
130
141
|
for (const id of ids) {
|
|
131
|
-
|
|
142
|
+
const wakes = queued.get(id);
|
|
143
|
+
if (wakes) {
|
|
144
|
+
lane.queue.set(id, wakes);
|
|
145
|
+
}
|
|
132
146
|
}
|
|
133
147
|
for (const [id, wakes] of queued) {
|
|
134
148
|
if (!lane.queue.has(id)) {
|
|
@@ -153,12 +167,13 @@ export class FileProcessing {
|
|
|
153
167
|
this._scheduler.clear();
|
|
154
168
|
this._deadlines.clear();
|
|
155
169
|
this._mesh.files.emit('update');
|
|
156
|
-
|
|
157
|
-
|
|
170
|
+
const jobs = this._lanes.flatMap((lane) => [...lane.jobs]);
|
|
171
|
+
for (const job of jobs) {
|
|
172
|
+
job.abort.abort({
|
|
158
173
|
type: this._closed ? 'file-processing-closed' : 'file-processing-suspended'
|
|
159
174
|
});
|
|
160
175
|
}
|
|
161
|
-
await Promise.all([this._scanning, ...
|
|
176
|
+
await Promise.all([this._scanning, ...jobs.map((job) => job.done)]);
|
|
162
177
|
await Promise.all([this._source.cancel(), this._output.cancel()]);
|
|
163
178
|
for (const lane of this._lanes) {
|
|
164
179
|
for (const wakes of lane.queue.values()) {
|
|
@@ -190,7 +205,7 @@ export class FileProcessing {
|
|
|
190
205
|
await this.enqueue(await this._documents());
|
|
191
206
|
}
|
|
192
207
|
catch (err) {
|
|
193
|
-
|
|
208
|
+
this._mesh.log?.warn('rag', 'file processing scan failed', { err });
|
|
194
209
|
}
|
|
195
210
|
}
|
|
196
211
|
}
|
|
@@ -206,13 +221,31 @@ export class FileProcessing {
|
|
|
206
221
|
}
|
|
207
222
|
}
|
|
208
223
|
_pump(lane) {
|
|
209
|
-
|
|
210
|
-
|
|
224
|
+
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
225
|
+
const busy = [...lane.jobs];
|
|
226
|
+
if (busy.length && (lane.stage !== 'produce' || busy.length > 1 || !busy[0].admitted)) {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
const next = [...lane.queue].find(([queued]) => !busy.some((job) => job.id === queued));
|
|
230
|
+
if (!next) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
const [id, wakes] = next;
|
|
234
|
+
lane.queue.delete(id);
|
|
235
|
+
const abort = new AbortController();
|
|
236
|
+
const job = { id, abort, admitted: false, done: Promise.resolve() };
|
|
237
|
+
lane.jobs.add(job);
|
|
238
|
+
const admit = () => {
|
|
239
|
+
if (job.admitted)
|
|
240
|
+
return;
|
|
241
|
+
job.admitted = true;
|
|
242
|
+
this._pump(lane);
|
|
243
|
+
};
|
|
244
|
+
job.done = this._run(lane, { id, wakes, signal: abort.signal, admit }).finally(() => {
|
|
245
|
+
lane.jobs.delete(job);
|
|
246
|
+
this._pump(lane);
|
|
247
|
+
});
|
|
211
248
|
}
|
|
212
|
-
lane.running = this._drain(lane).finally(() => {
|
|
213
|
-
lane.running = null;
|
|
214
|
-
this._pump(lane);
|
|
215
|
-
});
|
|
216
249
|
}
|
|
217
250
|
async _forget(id) {
|
|
218
251
|
const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
|
|
@@ -220,65 +253,59 @@ export class FileProcessing {
|
|
|
220
253
|
await this._processor.evict?.(loaded);
|
|
221
254
|
}
|
|
222
255
|
}
|
|
223
|
-
async
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
lane.
|
|
228
|
-
|
|
229
|
-
let produced = false;
|
|
230
|
-
try {
|
|
231
|
-
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
232
|
-
if (!record && lane.stage === 'load') {
|
|
233
|
-
await this._forget(id);
|
|
234
|
-
}
|
|
235
|
-
if (!record || record.deletedAt) {
|
|
236
|
-
await this._source.cancel((key) => key === id);
|
|
237
|
-
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
238
|
-
}
|
|
239
|
-
if (!record || !this._processor.applies(record) || lane.abort.signal.aborted) {
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
if (lane.stage === 'produce' &&
|
|
243
|
-
(!this._mesh.writable ||
|
|
244
|
-
!(await this._processor.canProduce(record, {
|
|
245
|
-
mesh: this._mesh,
|
|
246
|
-
deviceId: this._deviceId
|
|
247
|
-
})))) {
|
|
248
|
-
continue;
|
|
249
|
-
}
|
|
250
|
-
const document = new FileDocument(this._mesh, {
|
|
251
|
-
record,
|
|
252
|
-
processor: this._processor,
|
|
253
|
-
source: this._source,
|
|
254
|
-
output: this._output,
|
|
255
|
-
schedule: (deadline) => this._schedule(id, deadline),
|
|
256
|
-
closed: () => this._closed,
|
|
257
|
-
paused: () => this._paused,
|
|
258
|
-
progress: this._progress
|
|
259
|
-
});
|
|
260
|
-
if (lane.abort.signal.aborted) {
|
|
261
|
-
continue;
|
|
262
|
-
}
|
|
263
|
-
await this._processor[lane.stage]?.(document, lane.abort.signal);
|
|
264
|
-
produced = lane.stage === 'produce';
|
|
256
|
+
async _run(lane, { id, wakes, signal, admit }) {
|
|
257
|
+
let produced = false;
|
|
258
|
+
try {
|
|
259
|
+
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
260
|
+
if (!record && lane.stage === 'load') {
|
|
261
|
+
await this._forget(id);
|
|
265
262
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
}
|
|
263
|
+
if (!record || record.deletedAt) {
|
|
264
|
+
await this._source.cancel((key) => key === id);
|
|
265
|
+
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
270
266
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
267
|
+
if (!record || !this._processor.applies(record) || signal.aborted) {
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
if (lane.stage === 'produce' &&
|
|
271
|
+
(!this._mesh.writable ||
|
|
272
|
+
!(await this._processor.canProduce(record, {
|
|
273
|
+
mesh: this._mesh,
|
|
274
|
+
deviceId: this._deviceId,
|
|
275
|
+
device: this._device
|
|
276
|
+
})))) {
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
const document = new FileDocument(this._mesh, {
|
|
280
|
+
record,
|
|
281
|
+
processor: this._processor,
|
|
282
|
+
source: this._source,
|
|
283
|
+
output: this._output,
|
|
284
|
+
schedule: (deadline) => this._schedule(id, deadline),
|
|
285
|
+
closed: () => this._closed,
|
|
286
|
+
paused: () => this._paused,
|
|
287
|
+
progress: this._progress,
|
|
288
|
+
admit
|
|
289
|
+
});
|
|
290
|
+
if (signal.aborted) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
await this._processor[lane.stage]?.(document, signal);
|
|
294
|
+
produced = lane.stage === 'produce';
|
|
295
|
+
}
|
|
296
|
+
catch (err) {
|
|
297
|
+
if (!signal.aborted) {
|
|
298
|
+
this._recordFailure(id, { stage: lane.stage, err });
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
finally {
|
|
302
|
+
if (produced && this.active) {
|
|
303
|
+
this._queueLoad(id);
|
|
304
|
+
}
|
|
305
|
+
for (const wake of wakes) {
|
|
306
|
+
wake();
|
|
281
307
|
}
|
|
308
|
+
this._mesh.files.emit('update');
|
|
282
309
|
}
|
|
283
310
|
}
|
|
284
311
|
_recordFailure(id, { stage, err }) {
|
|
@@ -292,7 +319,7 @@ export class FileProcessing {
|
|
|
292
319
|
else {
|
|
293
320
|
this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
|
|
294
321
|
}
|
|
295
|
-
|
|
322
|
+
this._mesh.log?.warn('rag', 'file processing failed', { id, stage, err });
|
|
296
323
|
}
|
|
297
324
|
_queueLoad(id) {
|
|
298
325
|
const loader = this._lanes.find((entry) => entry.stage === 'load');
|
|
@@ -327,6 +354,5 @@ export class FileProcessing {
|
|
|
327
354
|
void this.enqueue(ids);
|
|
328
355
|
}
|
|
329
356
|
}
|
|
330
|
-
export { isClaimLive, listIndexingByChat } from './protocol.js';
|
|
357
|
+
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.js';
|
|
331
358
|
export { FileState } from './state.js';
|
|
332
|
-
export { settleFiles } from './settle.js';
|