@qvac/core 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +262 -1
- package/README.md +1 -1
- package/dist-lib/lib/api/agents.js +3 -2
- package/dist-lib/lib/api/app-config.d.ts +1 -1
- package/dist-lib/lib/api/app-config.js +54 -15
- package/dist-lib/lib/api/attachments.d.ts +6 -5
- package/dist-lib/lib/api/attachments.js +4 -2
- package/dist-lib/lib/api/chat.d.ts +7 -3
- package/dist-lib/lib/api/chat.js +4 -4
- package/dist-lib/lib/api/chunks.js +3 -3
- package/dist-lib/lib/api/config.js +2 -2
- package/dist-lib/lib/api/current-mesh.d.ts +3 -4
- package/dist-lib/lib/api/current-mesh.js +9 -4
- package/dist-lib/lib/api/custom-models.d.ts +8 -1
- package/dist-lib/lib/api/custom-models.js +53 -16
- package/dist-lib/lib/api/devices.d.ts +14 -5
- package/dist-lib/lib/api/devices.js +14 -9
- package/dist-lib/lib/api/diagnostics.d.ts +1 -0
- package/dist-lib/lib/api/diagnostics.js +1 -1
- package/dist-lib/lib/api/effective-ctx-size.d.ts +6 -1
- package/dist-lib/lib/api/effective-ctx-size.js +5 -3
- package/dist-lib/lib/api/execution.js +9 -2
- package/dist-lib/lib/api/files.js +2 -1
- package/dist-lib/lib/api/index.d.ts +7 -1
- package/dist-lib/lib/api/index.js +101 -101
- package/dist-lib/lib/api/instrument.js +29 -20
- package/dist-lib/lib/api/knowledge.d.ts +6 -5
- package/dist-lib/lib/api/knowledge.js +8 -6
- package/dist-lib/lib/api/local-files.js +1 -1
- package/dist-lib/lib/api/logs.js +6 -3
- package/dist-lib/lib/api/members.js +22 -13
- package/dist-lib/lib/api/mesh.d.ts +7 -2
- package/dist-lib/lib/api/mesh.js +10 -5
- package/dist-lib/lib/api/model-download.js +1 -1
- package/dist-lib/lib/api/network.d.ts +1 -0
- package/dist-lib/lib/api/network.js +1 -1
- package/dist-lib/lib/api/notifications.js +23 -13
- package/dist-lib/lib/api/rag.d.ts +6 -1
- package/dist-lib/lib/api/rag.js +20 -11
- package/dist-lib/lib/api/range.d.ts +2 -2
- package/dist-lib/lib/api/sync.js +1 -1
- package/dist-lib/lib/api/tool-activity.js +7 -3
- package/dist-lib/lib/api/tool-configs.js +2 -4
- package/dist-lib/lib/api/upload-frames.d.ts +7 -8
- package/dist-lib/lib/api/upload-frames.js +2 -1
- package/dist-lib/lib/api/watchable.d.ts +6 -1
- package/dist-lib/lib/api/watchable.js +23 -14
- package/dist-lib/lib/assistant/fetch-body.js +4 -2
- package/dist-lib/lib/assistant/harness.d.ts +72 -24
- package/dist-lib/lib/assistant/harness.js +53 -15
- package/dist-lib/lib/assistant/history.d.ts +9 -7
- package/dist-lib/lib/assistant/history.js +77 -42
- package/dist-lib/lib/assistant/message.d.ts +5 -3
- package/dist-lib/lib/assistant/message.js +4 -2
- package/dist-lib/lib/assistant/skill-encode.d.ts +3 -1
- package/dist-lib/lib/assistant/skill-encode.js +52 -62
- package/dist-lib/lib/assistant-client.js +7 -4
- package/dist-lib/lib/assistant-tree.js +36 -11
- package/dist-lib/lib/assistant.d.ts +37 -12
- package/dist-lib/lib/assistant.js +977 -373
- package/dist-lib/lib/audio-format.d.ts +1 -1
- package/dist-lib/lib/audio-format.js +1 -1
- package/dist-lib/lib/chat-title.js +9 -4
- package/dist-lib/lib/client.js +3 -2
- package/dist-lib/lib/collabora/convert.d.ts +2 -2
- package/dist-lib/lib/collabora/convert.js +6 -3
- package/dist-lib/lib/collabora/entry.js +9 -5
- package/dist-lib/lib/collabora/protocol.d.ts +2 -2
- package/dist-lib/lib/collabora/protocol.js +14 -7
- package/dist-lib/lib/collabora/worker.js +14 -7
- package/dist-lib/lib/compaction.d.ts +5 -6
- package/dist-lib/lib/compaction.js +14 -7
- package/dist-lib/lib/constants.d.ts +2 -0
- package/dist-lib/lib/constants.js +4 -0
- package/dist-lib/lib/core/account-recovery.d.ts +4 -3
- package/dist-lib/lib/core/account-recovery.js +7 -4
- package/dist-lib/lib/core/agent.d.ts +0 -1
- package/dist-lib/lib/core/agent.js +18 -16
- package/dist-lib/lib/core/approval-rules.d.ts +7 -6
- package/dist-lib/lib/core/approval-rules.js +12 -6
- package/dist-lib/lib/core/archived-mesh.d.ts +7 -5
- package/dist-lib/lib/core/archived-mesh.js +25 -22
- package/dist-lib/lib/core/async-iterable.js +10 -5
- package/dist-lib/lib/core/attachment-link.d.ts +2 -2
- package/dist-lib/lib/core/audio-attachment.js +1 -1
- package/dist-lib/lib/core/blind-peers.d.ts +2 -2
- package/dist-lib/lib/core/chat-activity.d.ts +2 -2
- package/dist-lib/lib/core/chat-activity.js +42 -21
- package/dist-lib/lib/core/chat-group.d.ts +8 -6
- package/dist-lib/lib/core/chat-group.js +17 -10
- package/dist-lib/lib/core/chat-order.d.ts +11 -9
- package/dist-lib/lib/core/chat-order.js +1 -3
- package/dist-lib/lib/core/chat-search.d.ts +4 -4
- package/dist-lib/lib/core/chat-search.js +38 -24
- package/dist-lib/lib/core/chat.d.ts +20 -14
- package/dist-lib/lib/core/chat.js +42 -28
- package/dist-lib/lib/core/chunk.d.ts +20 -24
- package/dist-lib/lib/core/chunk.js +34 -22
- package/dist-lib/lib/core/collections.d.ts +69 -0
- package/dist-lib/lib/core/collections.js +69 -0
- package/dist-lib/lib/core/contract/activity.d.ts +8 -2
- package/dist-lib/lib/core/contract/activity.js +39 -11
- package/dist-lib/lib/core/contract/agents.js +3 -2
- package/dist-lib/lib/core/contract/approval-rules.js +1 -1
- package/dist-lib/lib/core/contract/attestations.js +1 -1
- package/dist-lib/lib/core/contract/chat-group.js +1 -1
- package/dist-lib/lib/core/contract/chat.js +1 -1
- package/dist-lib/lib/core/contract/chunks.js +16 -9
- package/dist-lib/lib/core/contract/compute.js +2 -1
- package/dist-lib/lib/core/contract/devices.js +10 -6
- package/dist-lib/lib/core/contract/grant-order.d.ts +5 -0
- package/dist-lib/lib/core/contract/grant-order.js +20 -0
- package/dist-lib/lib/core/contract/index.d.ts +1 -1
- package/dist-lib/lib/core/contract/index.js +27 -2
- package/dist-lib/lib/core/contract/indexing.js +3 -3
- package/dist-lib/lib/core/contract/invites.d.ts +16 -0
- package/dist-lib/lib/core/contract/invites.js +256 -0
- package/dist-lib/lib/core/contract/joined-meshes.js +22 -22
- package/dist-lib/lib/core/contract/knowledge.js +5 -5
- package/dist-lib/lib/core/contract/members.d.ts +15 -0
- package/dist-lib/lib/core/contract/members.js +116 -31
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +6 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +49 -0
- package/dist-lib/lib/core/contract/seal.js +1 -1
- package/dist-lib/lib/core/contract/seed-defaults.js +1 -3
- package/dist-lib/lib/core/contract/tool-activity.js +5 -5
- package/dist-lib/lib/core/contract/tool-configs.js +1 -2
- package/dist-lib/lib/core/contract/writers.d.ts +13 -0
- package/dist-lib/lib/core/contract/writers.js +181 -24
- package/dist-lib/lib/core/departures.js +16 -8
- package/dist-lib/lib/core/device-attestation.d.ts +5 -0
- package/dist-lib/lib/core/device-attestation.js +24 -0
- package/dist-lib/lib/core/device-invites.d.ts +2 -0
- package/dist-lib/lib/core/device-invites.js +29 -7
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +52 -2
- package/dist-lib/lib/core/document-attachment.d.ts +20 -11
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.d.ts +2 -5
- package/dist-lib/lib/core/files/attachments.js +6 -3
- package/dist-lib/lib/core/files/index.d.ts +44 -52
- package/dist-lib/lib/core/files/index.js +142 -93
- package/dist-lib/lib/core/files/knowledge.d.ts +2 -5
- package/dist-lib/lib/core/files/local-index.js +9 -5
- package/dist-lib/lib/core/files/preview/image.d.ts +0 -1
- package/dist-lib/lib/core/files/preview/image.js +9 -6
- package/dist-lib/lib/core/files/preview/index.d.ts +8 -11
- package/dist-lib/lib/core/files/preview/index.js +20 -15
- package/dist-lib/lib/core/files/processing/claims.d.ts +6 -5
- package/dist-lib/lib/core/files/processing/claims.js +16 -8
- package/dist-lib/lib/core/files/processing/document.d.ts +21 -50
- package/dist-lib/lib/core/files/processing/index.d.ts +18 -13
- package/dist-lib/lib/core/files/processing/index.js +65 -37
- package/dist-lib/lib/core/files/processing/ownership.d.ts +8 -1
- package/dist-lib/lib/core/files/processing/ownership.js +1 -1
- package/dist-lib/lib/core/files/processing/protocol.d.ts +19 -19
- package/dist-lib/lib/core/files/processing/protocol.js +5 -5
- package/dist-lib/lib/core/files/processing/settle.d.ts +10 -7
- package/dist-lib/lib/core/files/processing/settle.js +22 -11
- package/dist-lib/lib/core/files/processing/source.js +14 -8
- package/dist-lib/lib/core/files/processing/state.d.ts +13 -11
- package/dist-lib/lib/core/files/processing/state.js +19 -12
- package/dist-lib/lib/core/files/rag/build.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/build.js +2 -1
- package/dist-lib/lib/core/files/rag/document-types.d.ts +4 -6
- package/dist-lib/lib/core/files/rag/index.d.ts +10 -13
- package/dist-lib/lib/core/files/rag/index.js +3 -2
- package/dist-lib/lib/core/files/rag/load.d.ts +10 -9
- package/dist-lib/lib/core/files/rag/load.js +26 -13
- package/dist-lib/lib/core/files/rag/policy.js +2 -1
- package/dist-lib/lib/core/files/rag/produce.js +33 -19
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +30 -10
- package/dist-lib/lib/core/files/records/chunks.js +4 -2
- package/dist-lib/lib/core/files/records/knowledge.d.ts +10 -11
- package/dist-lib/lib/core/files/records/knowledge.js +9 -8
- package/dist-lib/lib/core/files/records/linkage.js +6 -3
- package/dist-lib/lib/core/files/records/output.d.ts +5 -4
- package/dist-lib/lib/core/files/records/output.js +8 -4
- package/dist-lib/lib/core/files/records/reclaim.js +41 -25
- package/dist-lib/lib/core/files/records/routing.d.ts +4 -6
- package/dist-lib/lib/core/files/records/routing.js +15 -8
- package/dist-lib/lib/core/files/records/source.d.ts +5 -4
- package/dist-lib/lib/core/files/records/source.js +20 -14
- package/dist-lib/lib/core/files/records/sweep.js +1 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +10 -10
- package/dist-lib/lib/core/files/store/blob.js +25 -19
- package/dist-lib/lib/core/files/store/index.d.ts +1 -6
- package/dist-lib/lib/core/files/store/index.js +4 -4
- package/dist-lib/lib/core/files/store/inline.d.ts +2 -2
- package/dist-lib/lib/core/files/store/inline.js +6 -3
- package/dist-lib/lib/core/files/store/reference.d.ts +6 -9
- package/dist-lib/lib/core/files/store/reference.js +25 -17
- package/dist-lib/lib/core/files/store/server.d.ts +7 -6
- package/dist-lib/lib/core/files/store/server.js +22 -11
- package/dist-lib/lib/core/files/store/sweep.js +13 -7
- package/dist-lib/lib/core/files/upload-input.d.ts +4 -12
- package/dist-lib/lib/core/files/upload-input.js +10 -5
- package/dist-lib/lib/core/files/upload.d.ts +27 -24
- package/dist-lib/lib/core/files/upload.js +74 -51
- package/dist-lib/lib/core/fs-drives.js +4 -2
- package/dist-lib/lib/core/grant-expiry.d.ts +11 -0
- package/dist-lib/lib/core/grant-expiry.js +66 -0
- package/dist-lib/lib/core/handoff-channel.d.ts +2 -1
- package/dist-lib/lib/core/handoff-channel.js +36 -15
- package/dist-lib/lib/core/harness-loan.js +35 -17
- package/dist-lib/lib/core/helpers/index.d.ts +2 -1
- package/dist-lib/lib/core/helpers/index.js +11 -6
- package/dist-lib/lib/core/hosted-agents.js +19 -12
- package/dist-lib/lib/core/id.js +1 -1
- package/dist-lib/lib/core/identity.d.ts +1 -0
- package/dist-lib/lib/core/identity.js +11 -3
- package/dist-lib/lib/core/image/cache-sweep.d.ts +12 -10
- package/dist-lib/lib/core/image/cache-sweep.js +11 -6
- package/dist-lib/lib/core/image/normalize.d.ts +6 -0
- package/dist-lib/lib/core/image/normalize.js +12 -2
- package/dist-lib/lib/core/image/sniff.js +42 -21
- package/dist-lib/lib/core/inference-calls-receiver.js +7 -4
- package/dist-lib/lib/core/invite-grants.d.ts +20 -0
- package/dist-lib/lib/core/invite-grants.js +46 -0
- package/dist-lib/lib/core/invite.d.ts +3 -0
- package/dist-lib/lib/core/invite.js +6 -3
- package/dist-lib/lib/core/local/files.js +20 -19
- package/dist-lib/lib/core/local/index.d.ts +70 -45
- package/dist-lib/lib/core/local/index.js +157 -126
- package/dist-lib/lib/core/membership.d.ts +15 -2
- package/dist-lib/lib/core/membership.js +95 -20
- package/dist-lib/lib/core/mesh-seed-handoff.d.ts +12 -4
- package/dist-lib/lib/core/mesh-seed-handoff.js +23 -5
- package/dist-lib/lib/core/mesh.d.ts +57 -30
- package/dist-lib/lib/core/mesh.js +232 -78
- package/dist-lib/lib/core/migrations/agent-catalog.js +3 -2
- package/dist-lib/lib/core/migrations/agent-indexes.js +3 -3
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +2 -0
- package/dist-lib/lib/core/migrations/device-indexes.js +7 -0
- package/dist-lib/lib/core/migrations/file-index.js +7 -4
- package/dist-lib/lib/core/migrations/index.d.ts +4 -3
- package/dist-lib/lib/core/migrations/index.js +3 -1
- package/dist-lib/lib/core/migrations/membership.js +9 -4
- package/dist-lib/lib/core/migrations/seal.js +3 -2
- package/dist-lib/lib/core/min-version.js +4 -2
- package/dist-lib/lib/core/next-frame.d.ts +5 -4
- package/dist-lib/lib/core/next-frame.js +6 -3
- package/dist-lib/lib/core/notification.d.ts +5 -1
- package/dist-lib/lib/core/notification.js +89 -77
- package/dist-lib/lib/core/ota-config.js +2 -1
- package/dist-lib/lib/core/pdf/rasterize.d.ts +18 -0
- package/dist-lib/lib/core/pdf/rasterize.js +57 -0
- package/dist-lib/lib/core/prompt-history.js +4 -2
- package/dist-lib/lib/core/range.d.ts +4 -4
- package/dist-lib/lib/core/range.js +12 -6
- package/dist-lib/lib/core/rank.d.ts +5 -4
- package/dist-lib/lib/core/rank.js +2 -1
- package/dist-lib/lib/core/send.d.ts +8 -5
- package/dist-lib/lib/core/send.js +9 -3
- package/dist-lib/lib/core/skill-overlay-receiver.js +7 -4
- package/dist-lib/lib/core/skill-secrets.js +16 -9
- package/dist-lib/lib/core/sync/chunks.js +2 -1
- package/dist-lib/lib/core/sync/devices.js +3 -1
- package/dist-lib/lib/core/sync/files.d.ts +2 -4
- package/dist-lib/lib/core/sync/files.js +28 -20
- package/dist-lib/lib/core/sync/index.d.ts +2 -5
- package/dist-lib/lib/core/sync/index.js +33 -17
- package/dist-lib/lib/core/sync-status.js +10 -7
- package/dist-lib/lib/core/system-info.js +4 -2
- package/dist-lib/lib/core/tool-activity.d.ts +4 -3
- package/dist-lib/lib/core/tool-activity.js +1 -1
- package/dist-lib/lib/core/tool-configs.js +12 -7
- package/dist-lib/lib/core.d.ts +107 -45
- package/dist-lib/lib/core.js +1765 -595
- package/dist-lib/lib/credential-pusher.d.ts +12 -17
- package/dist-lib/lib/credential-pusher.js +61 -42
- package/dist-lib/lib/custom-model-pusher.d.ts +7 -3
- package/dist-lib/lib/custom-model-pusher.js +15 -7
- package/dist-lib/lib/custom-model-task.d.ts +1 -0
- package/dist-lib/lib/custom-model-task.js +12 -0
- package/dist-lib/lib/deadline.js +4 -2
- package/dist-lib/lib/diagnostics.d.ts +1 -5
- package/dist-lib/lib/frame-readable.d.ts +3 -0
- package/dist-lib/lib/frame-readable.js +3 -0
- package/dist-lib/lib/fs-drive-bridge.js +24 -14
- package/dist-lib/lib/fs-grants.js +13 -8
- package/dist-lib/lib/guards.d.ts +3 -0
- package/dist-lib/lib/guards.js +9 -0
- package/dist-lib/lib/inference-calls.d.ts +28 -23
- package/dist-lib/lib/inference-calls.js +25 -13
- package/dist-lib/lib/integrations/mcp-providers.d.ts +2 -2
- package/dist-lib/lib/integrations/setup-routes.d.ts +2 -2
- package/dist-lib/lib/knowledge-search.d.ts +6 -5
- package/dist-lib/lib/knowledge-search.js +26 -18
- package/dist-lib/lib/kv-cache-reclaim.js +4 -2
- package/dist-lib/lib/kv-cache.js +8 -4
- package/dist-lib/lib/legacy-core-domain.d.ts +42 -42
- package/dist-lib/lib/log-buffer.js +2 -1
- package/dist-lib/lib/log.js +25 -15
- package/dist-lib/lib/mesh-scoped-engine.js +14 -4
- package/dist-lib/lib/model-catalog-revision.js +6 -3
- package/dist-lib/lib/model-clamp.js +6 -3
- package/dist-lib/lib/model-context.d.ts +8 -3
- package/dist-lib/lib/model-context.js +13 -5
- package/dist-lib/lib/model-download-status.js +4 -2
- package/dist-lib/lib/model-download.d.ts +12 -4
- package/dist-lib/lib/model-download.js +200 -103
- package/dist-lib/lib/model-idle-pusher.d.ts +10 -0
- package/dist-lib/lib/model-idle-pusher.js +41 -0
- package/dist-lib/lib/oauth/browser-connect.d.ts +10 -8
- package/dist-lib/lib/oauth/browser-connect.js +33 -20
- package/dist-lib/lib/oauth/client.d.ts +2 -2
- package/dist-lib/lib/oauth/flow-core.js +23 -9
- package/dist-lib/lib/oauth/flow.d.ts +1 -3
- package/dist-lib/lib/oauth/flow.js +9 -5
- package/dist-lib/lib/oauth/keys.d.ts +4 -4
- package/dist-lib/lib/oauth/keys.js +18 -12
- package/dist-lib/lib/oauth/loopback.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.js +147 -107
- package/dist-lib/lib/oauth/loopback.stub.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.stub.js +1 -1
- package/dist-lib/lib/oauth/pkce.d.ts +4 -2
- package/dist-lib/lib/oauth/pkce.js +10 -0
- package/dist-lib/lib/oauth/providers.js +5 -5
- package/dist-lib/lib/oauth/redirect-types.d.ts +12 -7
- package/dist-lib/lib/oauth/register.js +3 -2
- package/dist-lib/lib/oauth/registry.d.ts +2 -2
- package/dist-lib/lib/oauth/registry.js +2 -1
- package/dist-lib/lib/oauth/resolve.d.ts +30 -10
- package/dist-lib/lib/oauth/resolve.js +12 -6
- package/dist-lib/lib/oauth/token-exchange-core.d.ts +12 -4
- package/dist-lib/lib/oauth/token-exchange-core.js +1 -1
- package/dist-lib/lib/oauth/types.d.ts +12 -12
- package/dist-lib/lib/oauth.d.ts +4 -5
- package/dist-lib/lib/oauth.js +2 -3
- package/dist-lib/lib/reconnecting-watch.d.ts +1 -1
- package/dist-lib/lib/reconnecting-watch.js +6 -6
- package/dist-lib/lib/retry-budget.js +6 -3
- package/dist-lib/lib/run-notices.d.ts +2 -0
- package/dist-lib/lib/run-notices.js +7 -3
- package/dist-lib/lib/run-transcript.d.ts +2 -4
- package/dist-lib/lib/run-transcript.js +4 -2
- package/dist-lib/lib/skill-readiness.d.ts +1 -0
- package/dist-lib/lib/skill-readiness.js +17 -2
- package/dist-lib/lib/skill-types.d.ts +16 -16
- package/dist-lib/lib/suspend-lifecycle.d.ts +4 -4
- package/dist-lib/lib/suspend-lifecycle.js +10 -5
- package/dist-lib/lib/task.d.ts +5 -1
- package/dist-lib/lib/task.js +29 -12
- package/dist-lib/lib/unref.d.ts +3 -2
- package/dist-lib/lib/unref.js +2 -1
- package/dist-lib/lib/wav-duration.js +10 -5
- package/dist-lib/lib/write-queue.d.ts +5 -4
- package/dist-lib/lib/write-queue.js +2 -1
- package/dist-lib/schema/build.js +16 -8
- package/dist-lib/schema/config.d.ts +3 -4
- package/dist-lib/schema/config.js +1 -1
- package/dist-lib/schema/content.d.ts +18 -2
- package/dist-lib/schema/content.js +4 -1
- package/dist-lib/schema/engine.d.ts +3 -2
- package/dist-lib/schema/engine.js +192 -12
- package/dist-lib/schema/field-order.js +22 -11
- package/dist-lib/schema/indexes.d.ts +22 -0
- package/dist-lib/schema/{helpers.js → indexes.js} +19 -57
- package/dist-lib/schema/local.d.ts +3 -2
- package/dist-lib/schema/local.js +9 -3
- package/dist-lib/schema/media-kind.d.ts +8 -0
- package/dist-lib/schema/media-kind.js +52 -0
- package/dist-lib/schema/qvac.d.ts +12 -9
- package/dist-lib/schema/qvac.js +132 -7
- package/dist-lib/spec/bind.js +21 -0
- package/dist-lib/spec/calls.js +7 -0
- package/dist-lib/spec/capabilities.d.ts +18 -4
- package/dist-lib/spec/config/hyperconf/index.js +1 -1
- package/dist-lib/spec/hrpc/hrpc.json +93 -2
- package/dist-lib/spec/hrpc/index.js +89 -5
- package/dist-lib/spec/hrpc/messages.js +1568 -941
- package/dist-lib/spec/hyperschema/index.js +1568 -941
- package/dist-lib/spec/hyperschema/schema.json +531 -1
- package/dist-lib/spec/hyperschema/types.d.ts +145 -1
- package/dist-lib/spec/local/hyperdb/index.js +1 -1
- package/dist-lib/spec/local/hyperdb/messages.js +92 -20
- package/dist-lib/spec/local/hyperschema/index.js +47 -11
- package/dist-lib/spec/local/hyperschema/schema.json +32 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +7 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +131 -3
- package/dist-lib/spec/mesh/hyperdb/index.js +548 -5
- package/dist-lib/spec/mesh/hyperdb/messages.js +2444 -893
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +37 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +104 -2
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +1035 -11
- package/dist-lib/spec/mesh/hyperschema/index.js +1035 -11
- package/dist-lib/spec/mesh/hyperschema/schema.json +553 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +143 -1
- package/package.json +11 -5
- package/tui/app.mjs +2624 -986
- package/tui/boot.mjs +17 -46
- package/tui/index.mjs +10 -5
- package/tui/lib/boot-timing.mjs +8 -8
- package/tui/lib/busy-words.mjs +21 -22
- package/tui/lib/export.mjs +100 -32
- package/tui/lib/knowledge-command.mjs +64 -19
- package/tui/lib/knowledge-import.mjs +52 -18
- package/tui/lib/render.mjs +65 -33
- package/tui/lib/save-file.mjs +30 -13
- package/tui/lib/select.mjs +23 -5
- package/tui/lib/selection.mjs +6 -4
- package/tui/lib/setup.mjs +11 -19
- package/tui/lib/sidebar.mjs +23 -13
- package/tui/lib/splash.mjs +6 -7
- package/tui/lib/upload-frames.mjs +1 -2
- package/tui/lib/usage.mjs +13 -9
- package/tui/lib/wrap.mjs +35 -6
- package/tui/main.mjs +54 -78
- package/worker/assistant-sidecar.mjs +5 -10
- package/worker/stow-entry.mjs +7 -4
- package/dist-lib/lib/custom-model-source.d.ts +0 -15
- package/dist-lib/lib/custom-model-source.js +0 -46
- package/dist-lib/lib/oauth/common.d.ts +0 -38
- package/dist-lib/lib/oauth/common.js +0 -34
- package/dist-lib/schema/helpers.d.ts +0 -32
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
|
-
import { Buffer } from 'buffer';
|
|
3
2
|
import crypto from 'hypercore-crypto';
|
|
4
3
|
import { processorArtifactId } from '../records/index.js';
|
|
5
4
|
import { parseBlobReference } from '../store/index.js';
|
|
@@ -7,6 +6,7 @@ import { IMAGE_MIMES } from '../../image/normalize.js';
|
|
|
7
6
|
import { previewEncoder } from './image.js';
|
|
8
7
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
9
8
|
import { getChat } from '../../chat.js';
|
|
9
|
+
import { errorMessage } from '../../../guards.js';
|
|
10
10
|
export { previewEncoder } from './image.js';
|
|
11
11
|
export const PREVIEW_VERSION = 'processor:preview:1';
|
|
12
12
|
const RETRY_MS = 30_000;
|
|
@@ -52,8 +52,9 @@ export class PreviewProcessor {
|
|
|
52
52
|
await document.claims.settle(id);
|
|
53
53
|
return;
|
|
54
54
|
}
|
|
55
|
-
if (document.claims.isFailed(id))
|
|
55
|
+
if (document.claims.isFailed(id)) {
|
|
56
56
|
return;
|
|
57
|
+
}
|
|
57
58
|
const encode = await this._encoder();
|
|
58
59
|
if (!encode) {
|
|
59
60
|
document.work({ state: 'pending', reason: 'image processor unavailable on this device' });
|
|
@@ -78,8 +79,9 @@ export class PreviewProcessor {
|
|
|
78
79
|
document.schedule(retryAt);
|
|
79
80
|
return;
|
|
80
81
|
}
|
|
81
|
-
if (await document.claims.blockedByStatus(id))
|
|
82
|
+
if (await document.claims.blockedByStatus(id)) {
|
|
82
83
|
return;
|
|
84
|
+
}
|
|
83
85
|
const source = await document.source.read(signal);
|
|
84
86
|
if (source.kind !== 'bytes') {
|
|
85
87
|
const retryAt = source.kind === 'missing' ? source.retryAt : undefined;
|
|
@@ -88,21 +90,25 @@ export class PreviewProcessor {
|
|
|
88
90
|
reason: 'source unavailable',
|
|
89
91
|
retryAt
|
|
90
92
|
});
|
|
91
|
-
if (retryAt)
|
|
93
|
+
if (retryAt) {
|
|
92
94
|
document.schedule(retryAt);
|
|
95
|
+
}
|
|
93
96
|
return;
|
|
94
97
|
}
|
|
95
|
-
if (signal.aborted || document.paused() || document.closed())
|
|
98
|
+
if (signal.aborted || document.paused() || document.closed()) {
|
|
96
99
|
return;
|
|
100
|
+
}
|
|
97
101
|
const claim = await document.claims.claim(id);
|
|
98
|
-
if (!claim)
|
|
102
|
+
if (!claim) {
|
|
99
103
|
return;
|
|
104
|
+
}
|
|
100
105
|
document.work({ state: 'running' });
|
|
101
106
|
try {
|
|
102
107
|
const bytes = await encode(source.bytes, signal);
|
|
103
108
|
const current = await document.current();
|
|
104
|
-
if (signal.aborted)
|
|
109
|
+
if (signal.aborted) {
|
|
105
110
|
throw new AbortedError();
|
|
111
|
+
}
|
|
106
112
|
if (!current || current.deletedAt || previewId(current) !== id) {
|
|
107
113
|
throw new Error('source no longer available');
|
|
108
114
|
}
|
|
@@ -121,18 +127,16 @@ export class PreviewProcessor {
|
|
|
121
127
|
}
|
|
122
128
|
catch (error) {
|
|
123
129
|
const interrupted = signal.aborted || document.paused() || document.closed();
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
: error instanceof Error
|
|
127
|
-
? error.message
|
|
128
|
-
: String(error);
|
|
130
|
+
const detail = errorMessage(error);
|
|
131
|
+
const reason = interrupted ? 'interrupted: suspend' : detail;
|
|
129
132
|
document.work({ state: interrupted ? 'pending' : 'failed', reason });
|
|
130
|
-
if (!interrupted)
|
|
133
|
+
if (!interrupted) {
|
|
131
134
|
document.claims.markFailed(id);
|
|
135
|
+
}
|
|
132
136
|
if (claim.claimId && !document.closed()) {
|
|
133
137
|
await document
|
|
134
138
|
.resolve({ artifactId: id, claimId: claim.claimId, status: 'failed', error: reason })
|
|
135
|
-
.catch(
|
|
139
|
+
.catch(noop);
|
|
136
140
|
}
|
|
137
141
|
}
|
|
138
142
|
}
|
|
@@ -157,5 +161,6 @@ function previewSource(file) {
|
|
|
157
161
|
]);
|
|
158
162
|
}
|
|
159
163
|
function hash(values) {
|
|
160
|
-
return b4a.
|
|
164
|
+
return b4a.toHex(crypto.hash(b4a.from(JSON.stringify(values))));
|
|
161
165
|
}
|
|
166
|
+
function noop() { }
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import type { Mesh } from '../../mesh.ts';
|
|
2
2
|
import type { FileState } from './state.ts';
|
|
3
|
-
export
|
|
3
|
+
export interface IndexingClaimsOptions {
|
|
4
4
|
mesh: Mesh;
|
|
5
5
|
schedule: (deadline: number) => void;
|
|
6
6
|
paused: () => boolean;
|
|
7
7
|
closed: () => boolean;
|
|
8
8
|
state?: FileState;
|
|
9
9
|
advisory?: boolean;
|
|
10
|
-
}
|
|
10
|
+
}
|
|
11
|
+
export interface IndexingClaim {
|
|
12
|
+
readonly claimId: string;
|
|
13
|
+
}
|
|
11
14
|
export declare class IndexingClaims {
|
|
12
15
|
private readonly _mesh;
|
|
13
16
|
private readonly _schedule;
|
|
@@ -18,9 +21,7 @@ export declare class IndexingClaims {
|
|
|
18
21
|
constructor(opts: IndexingClaimsOptions);
|
|
19
22
|
markFailed(artifactId: string): void;
|
|
20
23
|
isFailed(artifactId: string): boolean;
|
|
21
|
-
claim(artifactId: string, knowledgeId: string): Promise<
|
|
22
|
-
claimId: string;
|
|
23
|
-
} | null>;
|
|
24
|
+
claim(artifactId: string, knowledgeId: string): Promise<IndexingClaim | null>;
|
|
24
25
|
blockedByStatus(artifactId: string): Promise<boolean>;
|
|
25
26
|
settle(artifactId: string, knowledgeId: string): Promise<void>;
|
|
26
27
|
}
|
|
@@ -27,10 +27,12 @@ export class IndexingClaims {
|
|
|
27
27
|
knowledgeId,
|
|
28
28
|
at: Date.now()
|
|
29
29
|
});
|
|
30
|
-
if (won)
|
|
30
|
+
if (won) {
|
|
31
31
|
return { claimId: row.claimId };
|
|
32
|
-
|
|
32
|
+
}
|
|
33
|
+
if (isClaimLive(row, Date.now())) {
|
|
33
34
|
this._schedule(row.expiresAt);
|
|
35
|
+
}
|
|
34
36
|
return null;
|
|
35
37
|
}
|
|
36
38
|
catch (err) {
|
|
@@ -48,10 +50,12 @@ export class IndexingClaims {
|
|
|
48
50
|
if (this.isFailed(artifactId))
|
|
49
51
|
return true;
|
|
50
52
|
const row = await getIndexingStatus(this._mesh, artifactId);
|
|
51
|
-
if (!row)
|
|
53
|
+
if (!row) {
|
|
52
54
|
return false;
|
|
53
|
-
|
|
55
|
+
}
|
|
56
|
+
if (row.status === 'ready') {
|
|
54
57
|
return true;
|
|
58
|
+
}
|
|
55
59
|
if (isClaimLive(row, Date.now())) {
|
|
56
60
|
this._schedule(row.expiresAt);
|
|
57
61
|
return true;
|
|
@@ -62,25 +66,29 @@ export class IndexingClaims {
|
|
|
62
66
|
if (this._paused() || this._closed())
|
|
63
67
|
return;
|
|
64
68
|
const row = await getIndexingStatus(this._mesh, artifactId);
|
|
65
|
-
if (row?.status === 'ready')
|
|
69
|
+
if (row?.status === 'ready') {
|
|
66
70
|
return;
|
|
71
|
+
}
|
|
67
72
|
if (row && isClaimLive(row, Date.now())) {
|
|
68
73
|
this._schedule(row.expiresAt);
|
|
69
74
|
return;
|
|
70
75
|
}
|
|
71
|
-
if (this._paused() || this._closed())
|
|
76
|
+
if (this._paused() || this._closed()) {
|
|
72
77
|
return;
|
|
78
|
+
}
|
|
73
79
|
const claim = await this.claim(artifactId, knowledgeId);
|
|
74
|
-
if (!claim?.claimId)
|
|
80
|
+
if (!claim?.claimId) {
|
|
75
81
|
return;
|
|
82
|
+
}
|
|
76
83
|
try {
|
|
77
84
|
const { applied } = await resolveIndexing(this._mesh, {
|
|
78
85
|
artifactId,
|
|
79
86
|
claimId: claim.claimId,
|
|
80
87
|
status: 'ready'
|
|
81
88
|
});
|
|
82
|
-
if (applied)
|
|
89
|
+
if (applied) {
|
|
83
90
|
logger.info('rag', 'repaired to ready', { id: artifactId });
|
|
91
|
+
}
|
|
84
92
|
}
|
|
85
93
|
catch (err) {
|
|
86
94
|
logger.debug('rag', 'repair resolve dropped', { id: artifactId, err });
|
|
@@ -1,14 +1,27 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { BlobId } from 'hyperblobs';
|
|
3
2
|
import { type IndexingProgress } from '../rag/index.ts';
|
|
4
3
|
import { IndexingClaims } from './claims.ts';
|
|
5
|
-
import { ArtifactOwnership } from './ownership.ts';
|
|
4
|
+
import { ArtifactOwnership, type WrittenArtifact } from './ownership.ts';
|
|
6
5
|
import { resolveIndexing } from './protocol.ts';
|
|
7
6
|
import type { IndexingSource } from './source.ts';
|
|
8
7
|
import { type FileRecord } from '../records/index.ts';
|
|
9
8
|
import type { Mesh } from '../../mesh.ts';
|
|
10
9
|
import type { LocalWork } from './state.ts';
|
|
11
10
|
import type { FileProcessor } from './index.ts';
|
|
11
|
+
export interface FileDocumentOptions {
|
|
12
|
+
readonly record: FileRecord;
|
|
13
|
+
readonly processor: FileProcessor;
|
|
14
|
+
readonly source: IndexingSource;
|
|
15
|
+
readonly output: IndexingSource;
|
|
16
|
+
readonly schedule: (deadline: number) => void;
|
|
17
|
+
readonly closed: () => boolean;
|
|
18
|
+
readonly paused: () => boolean;
|
|
19
|
+
readonly progress?: IndexingProgress;
|
|
20
|
+
}
|
|
21
|
+
interface SourceReference {
|
|
22
|
+
readonly id: string;
|
|
23
|
+
readonly reference: string;
|
|
24
|
+
}
|
|
12
25
|
export declare class FileDocument {
|
|
13
26
|
private readonly _mesh;
|
|
14
27
|
readonly record: FileRecord;
|
|
@@ -28,43 +41,10 @@ export declare class FileDocument {
|
|
|
28
41
|
private readonly _output;
|
|
29
42
|
private readonly _progress?;
|
|
30
43
|
private readonly _processor;
|
|
31
|
-
constructor(mesh: Mesh, options:
|
|
32
|
-
record: FileRecord;
|
|
33
|
-
processor: FileProcessor;
|
|
34
|
-
source: IndexingSource;
|
|
35
|
-
output: IndexingSource;
|
|
36
|
-
schedule: (deadline: number) => void;
|
|
37
|
-
closed: () => boolean;
|
|
38
|
-
paused: () => boolean;
|
|
39
|
-
progress?: IndexingProgress;
|
|
40
|
-
});
|
|
44
|
+
constructor(mesh: Mesh, options: FileDocumentOptions);
|
|
41
45
|
get localCoreKey(): string;
|
|
42
|
-
artifact(id: string):
|
|
43
|
-
|
|
44
|
-
terminal?: boolean;
|
|
45
|
-
failedReference?: string;
|
|
46
|
-
unavailable?: {
|
|
47
|
-
blobRef: string;
|
|
48
|
-
repeated: boolean;
|
|
49
|
-
};
|
|
50
|
-
rebuildFailures?: number;
|
|
51
|
-
published?: {
|
|
52
|
-
artifactBlobId: string;
|
|
53
|
-
};
|
|
54
|
-
};
|
|
55
|
-
readArtifact(id: string): Readonly<{
|
|
56
|
-
knowledgeId?: string;
|
|
57
|
-
terminal?: boolean;
|
|
58
|
-
failedReference?: string;
|
|
59
|
-
unavailable?: {
|
|
60
|
-
blobRef: string;
|
|
61
|
-
repeated: boolean;
|
|
62
|
-
};
|
|
63
|
-
rebuildFailures?: number;
|
|
64
|
-
published?: {
|
|
65
|
-
artifactBlobId: string;
|
|
66
|
-
};
|
|
67
|
-
}> | undefined;
|
|
46
|
+
artifact(id: string): import("./state.ts").ArtifactState;
|
|
47
|
+
readArtifact(id: string): Readonly<import("./state.ts").ArtifactState> | undefined;
|
|
68
48
|
output(id: string, reference: string): {
|
|
69
49
|
read: (signal?: AbortSignal) => Promise<import("./source.ts").SourceRead>;
|
|
70
50
|
blockedUntil: () => number | null;
|
|
@@ -75,23 +55,14 @@ export declare class FileDocument {
|
|
|
75
55
|
statuses(): Promise<import("../../../../spec/index").QvacIndexingStatus[]>;
|
|
76
56
|
current(): Promise<FileRecord | null>;
|
|
77
57
|
publish(input: Omit<Parameters<Mesh['files']['publish']>[0], 'knowledgeId'>): Promise<import("../../../../spec/index").QvacKnowledgeArtifact>;
|
|
78
|
-
resolve(input: Parameters<typeof resolveIndexing>[1]): Promise<
|
|
79
|
-
|
|
80
|
-
applied: boolean;
|
|
81
|
-
}>;
|
|
82
|
-
write(bytes: Buffer): Promise<{
|
|
83
|
-
blobRef: BlobId;
|
|
84
|
-
artifactBlobId: string;
|
|
85
|
-
}>;
|
|
58
|
+
resolve(input: Parameters<typeof resolveIndexing>[1]): Promise<import("./protocol.ts").IndexingResolutionResult>;
|
|
59
|
+
write(bytes: Buffer): Promise<WrittenArtifact>;
|
|
86
60
|
inline(bytes: Buffer, mimeType: string): string;
|
|
87
61
|
work(state: LocalWork): void;
|
|
88
62
|
load(state: LocalWork, artifactId?: string): void;
|
|
89
63
|
report(update: Omit<Parameters<IndexingProgress>[0], 'knowledgeId' | 'chatId'>): void;
|
|
90
64
|
}
|
|
91
|
-
declare function fileSource(source: IndexingSource, { id, reference }: {
|
|
92
|
-
id: string;
|
|
93
|
-
reference: string;
|
|
94
|
-
}): {
|
|
65
|
+
declare function fileSource(source: IndexingSource, { id, reference }: SourceReference): {
|
|
95
66
|
read: (signal?: AbortSignal) => Promise<import("./source.ts").SourceRead>;
|
|
96
67
|
blockedUntil: () => number | null;
|
|
97
68
|
miss: () => number;
|
|
@@ -4,6 +4,10 @@ import { type FileInput, type FileRecord } from '../records/index.ts';
|
|
|
4
4
|
import type { Mesh } from '../../mesh.ts';
|
|
5
5
|
import { FileDocument } from './document.ts';
|
|
6
6
|
import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
7
|
+
export interface ProcessorHost {
|
|
8
|
+
readonly mesh: Mesh;
|
|
9
|
+
readonly deviceId: Buffer | null;
|
|
10
|
+
}
|
|
7
11
|
export interface FileProcessor {
|
|
8
12
|
readonly id: string;
|
|
9
13
|
readonly version: string;
|
|
@@ -12,14 +16,20 @@ export interface FileProcessor {
|
|
|
12
16
|
readonly mediaKind?: string;
|
|
13
17
|
accepts?(output: QvacKnowledgeArtifact, document: FileRecord): boolean;
|
|
14
18
|
applies(document: FileRecord): boolean;
|
|
15
|
-
canProduce(document: FileRecord, host:
|
|
16
|
-
mesh: Mesh;
|
|
17
|
-
deviceId: Buffer | null;
|
|
18
|
-
}): boolean | Promise<boolean>;
|
|
19
|
+
canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
|
|
19
20
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
20
21
|
load?(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
21
22
|
close(): Promise<boolean>;
|
|
22
23
|
}
|
|
24
|
+
export interface FileProcessingOptions {
|
|
25
|
+
readonly processor: FileProcessor;
|
|
26
|
+
readonly deviceId: Buffer | null;
|
|
27
|
+
readonly retryMs?: number;
|
|
28
|
+
readonly progress?: IndexingProgress;
|
|
29
|
+
}
|
|
30
|
+
export interface EnqueueOptions {
|
|
31
|
+
readonly wait?: boolean;
|
|
32
|
+
}
|
|
23
33
|
export declare class FileProcessing {
|
|
24
34
|
private readonly _mesh;
|
|
25
35
|
private readonly _processor;
|
|
@@ -38,12 +48,7 @@ export declare class FileProcessing {
|
|
|
38
48
|
private _indexCleared;
|
|
39
49
|
private readonly _onDirty;
|
|
40
50
|
private readonly _onAll;
|
|
41
|
-
constructor(mesh: Mesh, options:
|
|
42
|
-
processor: FileProcessor;
|
|
43
|
-
deviceId: Buffer | null;
|
|
44
|
-
retryMs?: number;
|
|
45
|
-
progress?: IndexingProgress;
|
|
46
|
-
});
|
|
51
|
+
constructor(mesh: Mesh, options: FileProcessingOptions);
|
|
47
52
|
start(): void;
|
|
48
53
|
suspend(): Promise<void>;
|
|
49
54
|
resume(): void;
|
|
@@ -51,9 +56,7 @@ export declare class FileProcessing {
|
|
|
51
56
|
close(): Promise<void>;
|
|
52
57
|
get indexCleared(): boolean;
|
|
53
58
|
settle(): Promise<void>;
|
|
54
|
-
enqueue(ids: Iterable<string>, options?:
|
|
55
|
-
wait?: boolean;
|
|
56
|
-
}): Promise<void>;
|
|
59
|
+
enqueue(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
|
|
57
60
|
private _enqueue;
|
|
58
61
|
get active(): boolean;
|
|
59
62
|
pending(id: string): boolean;
|
|
@@ -65,6 +68,8 @@ export declare class FileProcessing {
|
|
|
65
68
|
private _documents;
|
|
66
69
|
private _pump;
|
|
67
70
|
private _drain;
|
|
71
|
+
private _recordFailure;
|
|
72
|
+
private _queueLoad;
|
|
68
73
|
private _schedule;
|
|
69
74
|
private _retry;
|
|
70
75
|
}
|
|
@@ -4,6 +4,7 @@ import { DeadlineScheduler } from '../../../deadline.js';
|
|
|
4
4
|
import { IndexingSource } from './source.js';
|
|
5
5
|
import { fileIds, fileRecord } from '../records/index.js';
|
|
6
6
|
import { FileDocument } from './document.js';
|
|
7
|
+
import { errorMessage } from '../../../guards.js';
|
|
7
8
|
export class FileProcessing {
|
|
8
9
|
_mesh;
|
|
9
10
|
_processor;
|
|
@@ -21,10 +22,12 @@ export class FileProcessing {
|
|
|
21
22
|
_close = null;
|
|
22
23
|
_indexCleared = true;
|
|
23
24
|
_onDirty = (ids) => {
|
|
24
|
-
if (ids)
|
|
25
|
+
if (ids) {
|
|
25
26
|
void this.enqueue(ids);
|
|
26
|
-
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
27
29
|
this._kick();
|
|
30
|
+
}
|
|
28
31
|
};
|
|
29
32
|
_onAll = () => this._kick();
|
|
30
33
|
constructor(mesh, options) {
|
|
@@ -83,8 +86,9 @@ export class FileProcessing {
|
|
|
83
86
|
if (this._closed || this._paused)
|
|
84
87
|
return;
|
|
85
88
|
const ids = await this._documents();
|
|
86
|
-
for (const lane of this._lanes)
|
|
89
|
+
for (const lane of this._lanes) {
|
|
87
90
|
await this._enqueue(ids, [lane], { wait: true });
|
|
91
|
+
}
|
|
88
92
|
}
|
|
89
93
|
enqueue(ids, options = {}) {
|
|
90
94
|
return this._enqueue(ids, this._lanes, options);
|
|
@@ -96,14 +100,17 @@ export class FileProcessing {
|
|
|
96
100
|
for (const id of ids) {
|
|
97
101
|
for (const lane of lanes) {
|
|
98
102
|
let wakes = lane.queue.get(id);
|
|
99
|
-
if (!wakes)
|
|
103
|
+
if (!wakes) {
|
|
100
104
|
lane.queue.set(id, (wakes = []));
|
|
101
|
-
|
|
105
|
+
}
|
|
106
|
+
if (wait) {
|
|
102
107
|
pending.push(new Promise((resolve) => wakes.push(resolve)));
|
|
108
|
+
}
|
|
103
109
|
}
|
|
104
110
|
}
|
|
105
|
-
for (const lane of lanes)
|
|
111
|
+
for (const lane of lanes) {
|
|
106
112
|
this._pump(lane);
|
|
113
|
+
}
|
|
107
114
|
await Promise.all(pending);
|
|
108
115
|
}
|
|
109
116
|
get active() {
|
|
@@ -120,11 +127,14 @@ export class FileProcessing {
|
|
|
120
127
|
for (const lane of this._lanes) {
|
|
121
128
|
const queued = new Map(lane.queue);
|
|
122
129
|
lane.queue.clear();
|
|
123
|
-
for (const id of ids)
|
|
130
|
+
for (const id of ids) {
|
|
124
131
|
lane.queue.set(id, queued.get(id) ?? []);
|
|
125
|
-
|
|
126
|
-
|
|
132
|
+
}
|
|
133
|
+
for (const [id, wakes] of queued) {
|
|
134
|
+
if (!lane.queue.has(id)) {
|
|
127
135
|
lane.queue.set(id, wakes);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
128
138
|
this._pump(lane);
|
|
129
139
|
}
|
|
130
140
|
}
|
|
@@ -134,8 +144,9 @@ export class FileProcessing {
|
|
|
134
144
|
this._mesh.autobee.off('move-to', this._onAll);
|
|
135
145
|
this._mesh.autobee.off('writable', this._onAll);
|
|
136
146
|
await this._stop();
|
|
137
|
-
if (this._processor.load)
|
|
147
|
+
if (this._processor.load) {
|
|
138
148
|
this._mesh.files.clearLoaded(this._processor.id);
|
|
149
|
+
}
|
|
139
150
|
this._indexCleared = await this._processor.close();
|
|
140
151
|
}
|
|
141
152
|
async _stop() {
|
|
@@ -150,9 +161,11 @@ export class FileProcessing {
|
|
|
150
161
|
await Promise.all([this._scanning, ...this._lanes.map((lane) => lane.running)]);
|
|
151
162
|
await Promise.all([this._source.cancel(), this._output.cancel()]);
|
|
152
163
|
for (const lane of this._lanes) {
|
|
153
|
-
for (const wakes of lane.queue.values())
|
|
154
|
-
for (const wake of wakes)
|
|
164
|
+
for (const wakes of lane.queue.values()) {
|
|
165
|
+
for (const wake of wakes) {
|
|
155
166
|
wake();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
156
169
|
lane.queue.clear();
|
|
157
170
|
}
|
|
158
171
|
}
|
|
@@ -160,12 +173,14 @@ export class FileProcessing {
|
|
|
160
173
|
if (this._closed || this._paused)
|
|
161
174
|
return;
|
|
162
175
|
this._dirty = true;
|
|
163
|
-
if (this._scanning)
|
|
176
|
+
if (this._scanning) {
|
|
164
177
|
return;
|
|
178
|
+
}
|
|
165
179
|
this._scanning = this._scan().finally(() => {
|
|
166
180
|
this._scanning = null;
|
|
167
|
-
if (this._dirty)
|
|
181
|
+
if (this._dirty) {
|
|
168
182
|
this._kick();
|
|
183
|
+
}
|
|
169
184
|
});
|
|
170
185
|
}
|
|
171
186
|
async _scan() {
|
|
@@ -201,7 +216,7 @@ export class FileProcessing {
|
|
|
201
216
|
}
|
|
202
217
|
async _drain(lane) {
|
|
203
218
|
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
204
|
-
const [id, wakes] = lane.queue
|
|
219
|
+
const [[id, wakes]] = lane.queue;
|
|
205
220
|
lane.queue.delete(id);
|
|
206
221
|
lane.current = id;
|
|
207
222
|
lane.abort = new AbortController();
|
|
@@ -212,8 +227,9 @@ export class FileProcessing {
|
|
|
212
227
|
await this._source.cancel((key) => key === id);
|
|
213
228
|
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
214
229
|
}
|
|
215
|
-
if (!record || !this._processor.applies(record) || lane.abort.signal.aborted)
|
|
230
|
+
if (!record || !this._processor.applies(record) || lane.abort.signal.aborted) {
|
|
216
231
|
continue;
|
|
232
|
+
}
|
|
217
233
|
if (lane.stage === 'produce' &&
|
|
218
234
|
(!this._mesh.writable ||
|
|
219
235
|
!(await this._processor.canProduce(record, {
|
|
@@ -232,49 +248,60 @@ export class FileProcessing {
|
|
|
232
248
|
paused: () => this._paused,
|
|
233
249
|
progress: this._progress
|
|
234
250
|
});
|
|
235
|
-
if (lane.abort.signal.aborted)
|
|
251
|
+
if (lane.abort.signal.aborted) {
|
|
236
252
|
continue;
|
|
253
|
+
}
|
|
237
254
|
await this._processor[lane.stage]?.(document, lane.abort.signal);
|
|
238
255
|
produced = lane.stage === 'produce';
|
|
239
256
|
}
|
|
240
257
|
catch (err) {
|
|
241
258
|
if (!lane.abort.signal.aborted) {
|
|
242
|
-
|
|
243
|
-
if (lane.stage === 'load') {
|
|
244
|
-
this._mesh.files.load(id, { state: 'failed', reason }, {
|
|
245
|
-
processorId: this._processor.id,
|
|
246
|
-
artifactId: this._mesh.files.hydratedArtifact(id) ?? undefined
|
|
247
|
-
});
|
|
248
|
-
}
|
|
249
|
-
else {
|
|
250
|
-
this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
|
|
251
|
-
}
|
|
252
|
-
logger.warn('rag', 'file processing failed', { id, stage: lane.stage, err });
|
|
259
|
+
this._recordFailure(id, { stage: lane.stage, err });
|
|
253
260
|
}
|
|
254
261
|
}
|
|
255
262
|
finally {
|
|
256
263
|
lane.abort = null;
|
|
257
264
|
lane.current = undefined;
|
|
258
265
|
if (produced && this.active) {
|
|
259
|
-
|
|
260
|
-
if (loader) {
|
|
261
|
-
if (!loader.queue.has(id))
|
|
262
|
-
loader.queue.set(id, []);
|
|
263
|
-
this._pump(loader);
|
|
264
|
-
}
|
|
266
|
+
this._queueLoad(id);
|
|
265
267
|
}
|
|
266
|
-
for (const wake of wakes)
|
|
268
|
+
for (const wake of wakes) {
|
|
267
269
|
wake();
|
|
270
|
+
}
|
|
268
271
|
this._mesh.files.emit('update');
|
|
269
272
|
}
|
|
270
273
|
}
|
|
271
274
|
}
|
|
275
|
+
_recordFailure(id, { stage, err }) {
|
|
276
|
+
const reason = errorMessage(err);
|
|
277
|
+
if (stage === 'load') {
|
|
278
|
+
this._mesh.files.load(id, { state: 'failed', reason }, {
|
|
279
|
+
processorId: this._processor.id,
|
|
280
|
+
artifactId: this._mesh.files.hydratedArtifact(id) ?? undefined
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
else {
|
|
284
|
+
this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
|
|
285
|
+
}
|
|
286
|
+
logger.warn('rag', 'file processing failed', { id, stage, err });
|
|
287
|
+
}
|
|
288
|
+
_queueLoad(id) {
|
|
289
|
+
const loader = this._lanes.find((entry) => entry.stage === 'load');
|
|
290
|
+
if (!loader) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (!loader.queue.has(id)) {
|
|
294
|
+
loader.queue.set(id, []);
|
|
295
|
+
}
|
|
296
|
+
this._pump(loader);
|
|
297
|
+
}
|
|
272
298
|
_schedule(id, deadline) {
|
|
273
299
|
if (this._closed || this._paused)
|
|
274
300
|
return;
|
|
275
301
|
const previous = this._deadlines.get(id);
|
|
276
|
-
if (previous === undefined || deadline < previous)
|
|
302
|
+
if (previous === undefined || deadline < previous) {
|
|
277
303
|
this._deadlines.set(id, deadline);
|
|
304
|
+
}
|
|
278
305
|
this._scheduler.schedule(deadline);
|
|
279
306
|
}
|
|
280
307
|
_retry() {
|
|
@@ -284,8 +311,9 @@ export class FileProcessing {
|
|
|
284
311
|
this._deadlines.delete(id);
|
|
285
312
|
ids.push(id);
|
|
286
313
|
}
|
|
287
|
-
else
|
|
314
|
+
else {
|
|
288
315
|
this._scheduler.schedule(deadline);
|
|
316
|
+
}
|
|
289
317
|
}
|
|
290
318
|
void this.enqueue(ids);
|
|
291
319
|
}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import type { BlobId } from 'hyperblobs';
|
|
2
2
|
import type { Mesh } from '../../mesh.ts';
|
|
3
3
|
import type { FileState } from './state.ts';
|
|
4
|
+
export interface WrittenArtifact {
|
|
5
|
+
readonly blobRef: BlobId;
|
|
6
|
+
readonly artifactBlobId: string;
|
|
7
|
+
}
|
|
8
|
+
export interface PublishedArtifact extends WrittenArtifact {
|
|
9
|
+
readonly landedBlobId: string;
|
|
10
|
+
}
|
|
4
11
|
export declare class ArtifactOwnership {
|
|
5
12
|
private readonly _mesh;
|
|
6
13
|
private readonly _state;
|
|
7
14
|
constructor(mesh: Mesh, state?: FileState);
|
|
8
|
-
onPublished(artifactId: string, blobRef
|
|
15
|
+
onPublished(artifactId: string, { blobRef, artifactBlobId, landedBlobId }: PublishedArtifact): Promise<void>;
|
|
9
16
|
reclaimSuperseded(artifactId: string, winnerBlobId: string): Promise<void>;
|
|
10
17
|
clearUnreferenced(artifactId: string, blobRef: BlobId): Promise<void>;
|
|
11
18
|
}
|
|
@@ -7,7 +7,7 @@ export class ArtifactOwnership {
|
|
|
7
7
|
this._mesh = mesh;
|
|
8
8
|
this._state = state;
|
|
9
9
|
}
|
|
10
|
-
async onPublished(artifactId, blobRef, artifactBlobId, landedBlobId) {
|
|
10
|
+
async onPublished(artifactId, { blobRef, artifactBlobId, landedBlobId }) {
|
|
11
11
|
if (compareBlobReferences(landedBlobId, artifactBlobId, b4a.toHex(this._mesh.blobs.core.key)) ===
|
|
12
12
|
false) {
|
|
13
13
|
await this._mesh.files.clear(this._mesh.files.reference(blobRef), artifactId).catch(noop);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { QvacIndexingResolution, QvacIndexingStatus } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
2
|
import type { Mesh } from '../../mesh.ts';
|
|
3
|
+
import type { ChatScope, KnowledgeScope } from '../records/knowledge.ts';
|
|
3
4
|
export { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS } from '../../contract/indexing.ts';
|
|
4
5
|
export declare function isClaimLive(row: QvacIndexingStatus, at: number): boolean;
|
|
5
6
|
export interface ClaimIndexingInput {
|
|
@@ -7,23 +8,22 @@ export interface ClaimIndexingInput {
|
|
|
7
8
|
knowledgeId: string;
|
|
8
9
|
at: number;
|
|
9
10
|
}
|
|
11
|
+
export interface IndexingClaimResult {
|
|
12
|
+
readonly row: QvacIndexingStatus;
|
|
13
|
+
readonly won: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface ResolveIndexingInput {
|
|
16
|
+
readonly artifactId: string;
|
|
17
|
+
readonly claimId: string;
|
|
18
|
+
readonly status: QvacIndexingResolution;
|
|
19
|
+
readonly error?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface IndexingResolutionResult {
|
|
22
|
+
readonly row: QvacIndexingStatus;
|
|
23
|
+
readonly applied: boolean;
|
|
24
|
+
}
|
|
10
25
|
export declare function getIndexingStatus(store: Mesh, artifactId: string): Promise<QvacIndexingStatus | null>;
|
|
11
|
-
export declare function listIndexingByKnowledge(store: Mesh, { knowledgeId }:
|
|
12
|
-
|
|
13
|
-
}): Promise<
|
|
14
|
-
export declare function
|
|
15
|
-
chatId: string;
|
|
16
|
-
}): Promise<QvacIndexingStatus[]>;
|
|
17
|
-
export declare function claimIndexing(store: Mesh, { artifactId, knowledgeId, at }: ClaimIndexingInput): Promise<{
|
|
18
|
-
row: QvacIndexingStatus;
|
|
19
|
-
won: boolean;
|
|
20
|
-
}>;
|
|
21
|
-
export declare function resolveIndexing(store: Mesh, { artifactId, claimId, status, error }: {
|
|
22
|
-
artifactId: string;
|
|
23
|
-
claimId: string;
|
|
24
|
-
status: QvacIndexingResolution;
|
|
25
|
-
error?: string;
|
|
26
|
-
}): Promise<{
|
|
27
|
-
row: QvacIndexingStatus;
|
|
28
|
-
applied: boolean;
|
|
29
|
-
}>;
|
|
26
|
+
export declare function listIndexingByKnowledge(store: Mesh, { knowledgeId }: KnowledgeScope): Promise<QvacIndexingStatus[]>;
|
|
27
|
+
export declare function listIndexingByChat(store: Mesh, { chatId }: ChatScope): Promise<QvacIndexingStatus[]>;
|
|
28
|
+
export declare function claimIndexing(store: Mesh, { artifactId, knowledgeId, at }: ClaimIndexingInput): Promise<IndexingClaimResult>;
|
|
29
|
+
export declare function resolveIndexing(store: Mesh, { artifactId, claimId, status, error }: ResolveIndexingInput): Promise<IndexingResolutionResult>;
|