@qvac/core 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -1
- package/dist-lib/lib/api/agents.js +3 -2
- package/dist-lib/lib/api/app-config.d.ts +1 -1
- package/dist-lib/lib/api/app-config.js +54 -15
- package/dist-lib/lib/api/attachments.d.ts +6 -5
- package/dist-lib/lib/api/attachments.js +4 -2
- package/dist-lib/lib/api/chat.d.ts +7 -3
- package/dist-lib/lib/api/chat.js +4 -4
- package/dist-lib/lib/api/chunks.js +3 -3
- package/dist-lib/lib/api/config.js +2 -2
- package/dist-lib/lib/api/current-mesh.d.ts +3 -4
- package/dist-lib/lib/api/current-mesh.js +9 -4
- package/dist-lib/lib/api/custom-models.d.ts +8 -1
- package/dist-lib/lib/api/custom-models.js +53 -16
- package/dist-lib/lib/api/devices.d.ts +14 -5
- package/dist-lib/lib/api/devices.js +13 -8
- package/dist-lib/lib/api/diagnostics.d.ts +1 -0
- package/dist-lib/lib/api/diagnostics.js +1 -1
- package/dist-lib/lib/api/effective-ctx-size.d.ts +6 -1
- package/dist-lib/lib/api/effective-ctx-size.js +5 -3
- package/dist-lib/lib/api/execution.js +2 -1
- package/dist-lib/lib/api/files.js +2 -1
- package/dist-lib/lib/api/index.d.ts +5 -1
- package/dist-lib/lib/api/index.js +101 -102
- package/dist-lib/lib/api/instrument.js +29 -20
- package/dist-lib/lib/api/knowledge.d.ts +6 -5
- package/dist-lib/lib/api/knowledge.js +8 -6
- package/dist-lib/lib/api/local-files.js +1 -1
- package/dist-lib/lib/api/logs.js +6 -3
- package/dist-lib/lib/api/members.js +22 -13
- package/dist-lib/lib/api/mesh.js +3 -3
- package/dist-lib/lib/api/model-download.js +1 -1
- package/dist-lib/lib/api/network.d.ts +1 -0
- package/dist-lib/lib/api/network.js +1 -1
- package/dist-lib/lib/api/notifications.js +23 -13
- package/dist-lib/lib/api/rag.d.ts +6 -1
- package/dist-lib/lib/api/rag.js +20 -11
- package/dist-lib/lib/api/range.d.ts +2 -2
- package/dist-lib/lib/api/sync.js +1 -1
- package/dist-lib/lib/api/tool-activity.js +7 -3
- package/dist-lib/lib/api/tool-configs.js +2 -4
- package/dist-lib/lib/api/upload-frames.d.ts +7 -8
- package/dist-lib/lib/api/upload-frames.js +2 -1
- package/dist-lib/lib/api/watchable.d.ts +6 -1
- package/dist-lib/lib/api/watchable.js +23 -14
- package/dist-lib/lib/assistant/fetch-body.js +4 -2
- package/dist-lib/lib/assistant/harness.d.ts +48 -27
- package/dist-lib/lib/assistant/harness.js +40 -16
- package/dist-lib/lib/assistant/history.d.ts +9 -7
- package/dist-lib/lib/assistant/history.js +77 -42
- package/dist-lib/lib/assistant/message.d.ts +5 -3
- package/dist-lib/lib/assistant/message.js +4 -2
- package/dist-lib/lib/assistant/skill-encode.d.ts +3 -1
- package/dist-lib/lib/assistant/skill-encode.js +52 -62
- package/dist-lib/lib/assistant-client.js +7 -4
- package/dist-lib/lib/assistant-tree.js +27 -11
- package/dist-lib/lib/assistant.d.ts +31 -11
- package/dist-lib/lib/assistant.js +846 -390
- package/dist-lib/lib/audio-format.d.ts +1 -1
- package/dist-lib/lib/audio-format.js +1 -1
- package/dist-lib/lib/chat-title.js +9 -4
- package/dist-lib/lib/client.js +3 -2
- package/dist-lib/lib/collabora/convert.d.ts +2 -2
- package/dist-lib/lib/collabora/convert.js +6 -3
- package/dist-lib/lib/collabora/entry.js +9 -5
- package/dist-lib/lib/collabora/protocol.d.ts +2 -2
- package/dist-lib/lib/collabora/protocol.js +14 -7
- package/dist-lib/lib/collabora/worker.js +14 -7
- package/dist-lib/lib/compaction.d.ts +5 -6
- package/dist-lib/lib/compaction.js +14 -7
- package/dist-lib/lib/core/account-recovery.d.ts +4 -3
- package/dist-lib/lib/core/account-recovery.js +7 -4
- package/dist-lib/lib/core/agent.js +17 -15
- package/dist-lib/lib/core/approval-rules.d.ts +7 -6
- package/dist-lib/lib/core/approval-rules.js +12 -6
- package/dist-lib/lib/core/archived-mesh.d.ts +7 -5
- package/dist-lib/lib/core/archived-mesh.js +25 -22
- package/dist-lib/lib/core/async-iterable.js +10 -5
- package/dist-lib/lib/core/attachment-link.d.ts +2 -2
- package/dist-lib/lib/core/audio-attachment.js +1 -1
- package/dist-lib/lib/core/blind-peers.d.ts +2 -2
- package/dist-lib/lib/core/chat-activity.d.ts +2 -2
- package/dist-lib/lib/core/chat-activity.js +42 -21
- package/dist-lib/lib/core/chat-group.d.ts +8 -6
- package/dist-lib/lib/core/chat-group.js +17 -10
- package/dist-lib/lib/core/chat-order.d.ts +11 -9
- package/dist-lib/lib/core/chat-order.js +1 -3
- package/dist-lib/lib/core/chat-search.d.ts +4 -4
- package/dist-lib/lib/core/chat-search.js +38 -24
- package/dist-lib/lib/core/chat.d.ts +20 -14
- package/dist-lib/lib/core/chat.js +50 -31
- package/dist-lib/lib/core/chunk.d.ts +19 -23
- package/dist-lib/lib/core/chunk.js +27 -21
- package/dist-lib/lib/core/collections.d.ts +70 -0
- package/dist-lib/lib/core/collections.js +70 -0
- package/dist-lib/lib/core/contract/activity.d.ts +6 -2
- package/dist-lib/lib/core/contract/activity.js +15 -12
- package/dist-lib/lib/core/contract/agents.js +3 -2
- package/dist-lib/lib/core/contract/approval-rules.js +1 -1
- package/dist-lib/lib/core/contract/attestations.js +1 -1
- package/dist-lib/lib/core/contract/chat-group.js +1 -1
- package/dist-lib/lib/core/contract/chat.d.ts +2 -0
- package/dist-lib/lib/core/contract/chat.js +67 -1
- package/dist-lib/lib/core/contract/chunks.js +15 -9
- package/dist-lib/lib/core/contract/compute.d.ts +1 -1
- package/dist-lib/lib/core/contract/compute.js +4 -2
- package/dist-lib/lib/core/contract/devices.js +6 -5
- package/dist-lib/lib/core/contract/grant-order.js +5 -6
- package/dist-lib/lib/core/contract/index.js +4 -2
- package/dist-lib/lib/core/contract/indexing.js +3 -3
- package/dist-lib/lib/core/contract/invites.d.ts +8 -3
- package/dist-lib/lib/core/contract/invites.js +52 -26
- package/dist-lib/lib/core/contract/joined-meshes.js +3 -3
- package/dist-lib/lib/core/contract/knowledge.js +5 -5
- package/dist-lib/lib/core/contract/members.d.ts +13 -2
- package/dist-lib/lib/core/contract/members.js +46 -25
- package/dist-lib/lib/core/contract/mesh-kind.js +9 -8
- package/dist-lib/lib/core/contract/seal.js +1 -1
- package/dist-lib/lib/core/contract/seed-defaults.js +1 -3
- package/dist-lib/lib/core/contract/tool-activity.js +10 -5
- package/dist-lib/lib/core/contract/tool-configs.js +1 -2
- package/dist-lib/lib/core/contract/writers.d.ts +6 -1
- package/dist-lib/lib/core/contract/writers.js +55 -34
- package/dist-lib/lib/core/departures.js +16 -8
- package/dist-lib/lib/core/device-invites.js +3 -2
- package/dist-lib/lib/core/devices.js +6 -3
- package/dist-lib/lib/core/document-attachment.d.ts +20 -11
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.d.ts +2 -5
- package/dist-lib/lib/core/files/attachments.js +6 -3
- package/dist-lib/lib/core/files/index.d.ts +46 -52
- package/dist-lib/lib/core/files/index.js +149 -94
- package/dist-lib/lib/core/files/knowledge.d.ts +2 -5
- package/dist-lib/lib/core/files/local-index.d.ts +4 -1
- package/dist-lib/lib/core/files/local-index.js +9 -5
- package/dist-lib/lib/core/files/preview/image.d.ts +0 -1
- package/dist-lib/lib/core/files/preview/image.js +9 -6
- package/dist-lib/lib/core/files/preview/index.d.ts +8 -11
- package/dist-lib/lib/core/files/preview/index.js +20 -15
- package/dist-lib/lib/core/files/processing/claims.d.ts +6 -5
- package/dist-lib/lib/core/files/processing/claims.js +16 -8
- package/dist-lib/lib/core/files/processing/document.d.ts +21 -50
- package/dist-lib/lib/core/files/processing/index.d.ts +21 -13
- package/dist-lib/lib/core/files/processing/index.js +74 -37
- package/dist-lib/lib/core/files/processing/ownership.d.ts +8 -1
- package/dist-lib/lib/core/files/processing/ownership.js +1 -1
- package/dist-lib/lib/core/files/processing/protocol.d.ts +19 -19
- package/dist-lib/lib/core/files/processing/protocol.js +5 -5
- package/dist-lib/lib/core/files/processing/settle.d.ts +10 -7
- package/dist-lib/lib/core/files/processing/settle.js +22 -11
- package/dist-lib/lib/core/files/processing/source.js +14 -8
- package/dist-lib/lib/core/files/processing/state.d.ts +14 -11
- package/dist-lib/lib/core/files/processing/state.js +24 -12
- package/dist-lib/lib/core/files/rag/build.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/build.js +2 -1
- package/dist-lib/lib/core/files/rag/document-types.d.ts +4 -6
- package/dist-lib/lib/core/files/rag/index.d.ts +12 -13
- package/dist-lib/lib/core/files/rag/index.js +9 -2
- package/dist-lib/lib/core/files/rag/load.d.ts +10 -9
- package/dist-lib/lib/core/files/rag/load.js +26 -13
- package/dist-lib/lib/core/files/rag/policy.js +2 -1
- package/dist-lib/lib/core/files/rag/produce.js +33 -19
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +30 -10
- package/dist-lib/lib/core/files/records/chunks.js +4 -2
- package/dist-lib/lib/core/files/records/index.d.ts +1 -0
- package/dist-lib/lib/core/files/records/index.js +1 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +10 -11
- package/dist-lib/lib/core/files/records/knowledge.js +9 -8
- package/dist-lib/lib/core/files/records/linkage.js +6 -3
- package/dist-lib/lib/core/files/records/output.d.ts +5 -4
- package/dist-lib/lib/core/files/records/output.js +8 -4
- package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
- package/dist-lib/lib/core/files/records/purge.js +108 -0
- package/dist-lib/lib/core/files/records/reclaim.js +43 -25
- package/dist-lib/lib/core/files/records/routing.d.ts +4 -6
- package/dist-lib/lib/core/files/records/routing.js +15 -8
- package/dist-lib/lib/core/files/records/source.d.ts +5 -4
- package/dist-lib/lib/core/files/records/source.js +20 -14
- package/dist-lib/lib/core/files/records/sweep.js +1 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +11 -10
- package/dist-lib/lib/core/files/store/blob.js +54 -19
- package/dist-lib/lib/core/files/store/index.d.ts +2 -6
- package/dist-lib/lib/core/files/store/index.js +10 -5
- package/dist-lib/lib/core/files/store/inline.d.ts +2 -2
- package/dist-lib/lib/core/files/store/inline.js +6 -3
- package/dist-lib/lib/core/files/store/reference.d.ts +6 -9
- package/dist-lib/lib/core/files/store/reference.js +25 -17
- package/dist-lib/lib/core/files/store/server.d.ts +7 -6
- package/dist-lib/lib/core/files/store/server.js +22 -11
- package/dist-lib/lib/core/files/store/sweep.js +13 -7
- package/dist-lib/lib/core/files/upload-input.d.ts +4 -12
- package/dist-lib/lib/core/files/upload-input.js +10 -5
- package/dist-lib/lib/core/files/upload.d.ts +27 -24
- package/dist-lib/lib/core/files/upload.js +74 -51
- package/dist-lib/lib/core/fs-drives.js +4 -2
- package/dist-lib/lib/core/grant-expiry.js +12 -7
- package/dist-lib/lib/core/handoff-channel.js +27 -13
- package/dist-lib/lib/core/harness-loan.js +35 -17
- package/dist-lib/lib/core/helpers/index.js +6 -2
- package/dist-lib/lib/core/hosted-agents.js +19 -12
- package/dist-lib/lib/core/id.js +1 -1
- package/dist-lib/lib/core/identity.js +6 -3
- package/dist-lib/lib/core/image/cache-sweep.d.ts +12 -10
- package/dist-lib/lib/core/image/cache-sweep.js +11 -6
- package/dist-lib/lib/core/image/normalize.d.ts +6 -0
- package/dist-lib/lib/core/image/normalize.js +12 -2
- package/dist-lib/lib/core/image/sniff.js +42 -21
- package/dist-lib/lib/core/inference-calls-receiver.js +7 -4
- package/dist-lib/lib/core/invite-grants.js +4 -2
- package/dist-lib/lib/core/invite.js +2 -1
- package/dist-lib/lib/core/local/files.d.ts +2 -0
- package/dist-lib/lib/core/local/files.js +51 -19
- package/dist-lib/lib/core/local/index.d.ts +69 -46
- package/dist-lib/lib/core/local/index.js +146 -129
- package/dist-lib/lib/core/membership.d.ts +12 -2
- package/dist-lib/lib/core/membership.js +55 -33
- package/dist-lib/lib/core/mesh-seed-handoff.js +7 -1
- package/dist-lib/lib/core/mesh.d.ts +37 -27
- package/dist-lib/lib/core/mesh.js +131 -73
- package/dist-lib/lib/core/migrations/agent-catalog.js +3 -2
- package/dist-lib/lib/core/migrations/agent-indexes.js +3 -3
- package/dist-lib/lib/core/migrations/file-index.js +7 -4
- package/dist-lib/lib/core/migrations/index.d.ts +4 -3
- package/dist-lib/lib/core/migrations/membership.js +9 -4
- package/dist-lib/lib/core/migrations/seal.js +3 -2
- package/dist-lib/lib/core/min-version.js +4 -2
- package/dist-lib/lib/core/next-frame.d.ts +5 -4
- package/dist-lib/lib/core/next-frame.js +6 -3
- package/dist-lib/lib/core/notification.d.ts +5 -1
- package/dist-lib/lib/core/notification.js +89 -77
- package/dist-lib/lib/core/ota-config.js +2 -1
- package/dist-lib/lib/core/pdf/rasterize.d.ts +18 -0
- package/dist-lib/lib/core/pdf/rasterize.js +57 -0
- package/dist-lib/lib/core/prompt-history.js +4 -2
- package/dist-lib/lib/core/range.d.ts +4 -4
- package/dist-lib/lib/core/range.js +12 -6
- package/dist-lib/lib/core/rank.d.ts +5 -4
- package/dist-lib/lib/core/rank.js +2 -1
- package/dist-lib/lib/core/send.d.ts +4 -4
- package/dist-lib/lib/core/send.js +6 -3
- package/dist-lib/lib/core/skill-overlay-receiver.js +7 -4
- package/dist-lib/lib/core/skill-secrets.js +16 -9
- package/dist-lib/lib/core/sync/chunks.js +2 -1
- package/dist-lib/lib/core/sync/devices.js +3 -1
- package/dist-lib/lib/core/sync/files.d.ts +2 -4
- package/dist-lib/lib/core/sync/files.js +28 -20
- package/dist-lib/lib/core/sync/index.d.ts +2 -5
- package/dist-lib/lib/core/sync/index.js +33 -15
- package/dist-lib/lib/core/sync-status.js +10 -7
- package/dist-lib/lib/core/system-info.js +4 -2
- package/dist-lib/lib/core/tool-activity.d.ts +4 -3
- package/dist-lib/lib/core/tool-activity.js +1 -1
- package/dist-lib/lib/core/tool-configs.js +12 -7
- package/dist-lib/lib/core.d.ts +38 -26
- package/dist-lib/lib/core.js +1009 -541
- package/dist-lib/lib/credential-pusher.d.ts +20 -20
- package/dist-lib/lib/credential-pusher.js +65 -43
- package/dist-lib/lib/custom-model-pusher.d.ts +7 -3
- package/dist-lib/lib/custom-model-pusher.js +15 -7
- package/dist-lib/lib/custom-model-task.d.ts +1 -0
- package/dist-lib/lib/custom-model-task.js +12 -0
- package/dist-lib/lib/deadline.js +4 -2
- package/dist-lib/lib/diagnostics.d.ts +1 -5
- package/dist-lib/lib/frame-readable.d.ts +3 -0
- package/dist-lib/lib/frame-readable.js +3 -0
- package/dist-lib/lib/fs-drive-bridge.js +24 -14
- package/dist-lib/lib/fs-grants.js +13 -8
- package/dist-lib/lib/guards.d.ts +3 -0
- package/dist-lib/lib/guards.js +9 -0
- package/dist-lib/lib/inference-calls.d.ts +28 -23
- package/dist-lib/lib/inference-calls.js +25 -13
- package/dist-lib/lib/integrations/mcp-providers.d.ts +2 -2
- package/dist-lib/lib/integrations/setup-routes.d.ts +2 -2
- package/dist-lib/lib/knowledge-search.d.ts +6 -5
- package/dist-lib/lib/knowledge-search.js +26 -18
- package/dist-lib/lib/kv-cache-reclaim.js +4 -2
- package/dist-lib/lib/kv-cache.js +8 -4
- package/dist-lib/lib/legacy-core-domain.d.ts +42 -42
- package/dist-lib/lib/log-buffer.js +2 -1
- package/dist-lib/lib/log.js +25 -15
- package/dist-lib/lib/mesh-scoped-engine.js +12 -4
- package/dist-lib/lib/model-catalog-revision.js +6 -3
- package/dist-lib/lib/model-clamp.js +6 -3
- package/dist-lib/lib/model-context.d.ts +8 -3
- package/dist-lib/lib/model-context.js +13 -5
- package/dist-lib/lib/model-download-status.js +4 -2
- package/dist-lib/lib/model-download.d.ts +12 -4
- package/dist-lib/lib/model-download.js +200 -103
- package/dist-lib/lib/model-idle-pusher.d.ts +2 -2
- package/dist-lib/lib/model-idle-pusher.js +3 -4
- package/dist-lib/lib/oauth/browser-connect.d.ts +10 -8
- package/dist-lib/lib/oauth/browser-connect.js +33 -20
- package/dist-lib/lib/oauth/client.d.ts +2 -2
- package/dist-lib/lib/oauth/flow-core.js +23 -9
- package/dist-lib/lib/oauth/flow.d.ts +1 -3
- package/dist-lib/lib/oauth/flow.js +9 -5
- package/dist-lib/lib/oauth/keys.d.ts +4 -4
- package/dist-lib/lib/oauth/keys.js +18 -12
- package/dist-lib/lib/oauth/loopback.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.js +147 -107
- package/dist-lib/lib/oauth/loopback.stub.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.stub.js +1 -1
- package/dist-lib/lib/oauth/pkce.d.ts +4 -2
- package/dist-lib/lib/oauth/pkce.js +10 -0
- package/dist-lib/lib/oauth/providers.js +5 -5
- package/dist-lib/lib/oauth/redirect-types.d.ts +12 -7
- package/dist-lib/lib/oauth/register.js +3 -2
- package/dist-lib/lib/oauth/registry.d.ts +2 -2
- package/dist-lib/lib/oauth/registry.js +2 -1
- package/dist-lib/lib/oauth/resolve.d.ts +30 -10
- package/dist-lib/lib/oauth/resolve.js +12 -6
- package/dist-lib/lib/oauth/token-exchange-core.d.ts +12 -4
- package/dist-lib/lib/oauth/token-exchange-core.js +1 -1
- package/dist-lib/lib/oauth/types.d.ts +12 -12
- package/dist-lib/lib/oauth.d.ts +4 -5
- package/dist-lib/lib/oauth.js +2 -3
- package/dist-lib/lib/reconnecting-watch.d.ts +1 -1
- package/dist-lib/lib/reconnecting-watch.js +6 -6
- package/dist-lib/lib/retry-budget.js +6 -3
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +5 -2
- package/dist-lib/lib/run-transcript.d.ts +2 -4
- package/dist-lib/lib/run-transcript.js +4 -2
- package/dist-lib/lib/skill-readiness.js +4 -2
- package/dist-lib/lib/skill-types.d.ts +16 -16
- package/dist-lib/lib/suspend-lifecycle.d.ts +4 -4
- package/dist-lib/lib/suspend-lifecycle.js +10 -5
- package/dist-lib/lib/task.js +22 -12
- package/dist-lib/lib/unref.d.ts +3 -2
- package/dist-lib/lib/unref.js +2 -1
- package/dist-lib/lib/wav-duration.js +10 -5
- package/dist-lib/lib/write-queue.d.ts +5 -4
- package/dist-lib/lib/write-queue.js +2 -1
- package/dist-lib/schema/build.js +10 -6
- package/dist-lib/schema/config.d.ts +2 -3
- package/dist-lib/schema/content.d.ts +17 -2
- package/dist-lib/schema/content.js +2 -1
- package/dist-lib/schema/engine.d.ts +3 -2
- package/dist-lib/schema/engine.js +95 -6
- package/dist-lib/schema/field-order.js +22 -11
- package/dist-lib/schema/indexes.d.ts +22 -0
- package/dist-lib/schema/{helpers.js → indexes.js} +14 -57
- package/dist-lib/schema/local.d.ts +3 -2
- package/dist-lib/schema/local.js +7 -2
- package/dist-lib/schema/media-kind.d.ts +8 -0
- package/dist-lib/schema/media-kind.js +52 -0
- package/dist-lib/schema/qvac.d.ts +12 -9
- package/dist-lib/schema/qvac.js +18 -4
- package/dist-lib/spec/bind.js +12 -0
- package/dist-lib/spec/calls.js +4 -0
- package/dist-lib/spec/capabilities.d.ts +12 -4
- package/dist-lib/spec/hrpc/hrpc.json +54 -2
- package/dist-lib/spec/hrpc/index.js +53 -5
- package/dist-lib/spec/hrpc/messages.js +783 -494
- package/dist-lib/spec/hyperschema/index.js +783 -494
- package/dist-lib/spec/hyperschema/schema.json +247 -1
- package/dist-lib/spec/hyperschema/types.d.ts +69 -1
- package/dist-lib/spec/local/hyperdb/index.js +1 -1
- package/dist-lib/spec/local/hyperdb/messages.js +82 -14
- package/dist-lib/spec/local/hyperschema/index.js +42 -8
- package/dist-lib/spec/local/hyperschema/schema.json +27 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +6 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
- package/dist-lib/spec/mesh/hyperdb/index.js +72 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +400 -221
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +117 -2
- package/dist-lib/spec/mesh/hyperschema/index.js +117 -2
- package/dist-lib/spec/mesh/hyperschema/schema.json +65 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +19 -1
- package/package.json +8 -4
- package/tui/app.mjs +2424 -941
- package/tui/boot.mjs +18 -49
- package/tui/index.mjs +10 -5
- package/tui/lib/boot-timing.mjs +8 -8
- package/tui/lib/busy-words.mjs +21 -22
- package/tui/lib/export.mjs +100 -32
- package/tui/lib/knowledge-command.mjs +64 -19
- package/tui/lib/knowledge-import.mjs +52 -18
- package/tui/lib/render.mjs +65 -33
- package/tui/lib/save-file.mjs +30 -13
- package/tui/lib/select.mjs +23 -5
- package/tui/lib/selection.mjs +6 -4
- package/tui/lib/setup.mjs +11 -19
- package/tui/lib/sidebar.mjs +23 -13
- package/tui/lib/splash.mjs +6 -7
- package/tui/lib/upload-frames.mjs +1 -2
- package/tui/lib/usage.mjs +13 -9
- package/tui/lib/wrap.mjs +35 -6
- package/tui/main.mjs +54 -78
- package/worker/assistant-sidecar.mjs +5 -10
- package/worker/stow-entry.mjs +7 -4
- package/dist-lib/lib/custom-model-source.d.ts +0 -15
- package/dist-lib/lib/custom-model-source.js +0 -46
- package/dist-lib/lib/oauth/common.d.ts +0 -38
- package/dist-lib/lib/oauth/common.js +0 -34
- package/dist-lib/schema/helpers.d.ts +0 -33
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
|
-
import { Buffer } from 'buffer';
|
|
3
2
|
import crypto from 'hypercore-crypto';
|
|
4
3
|
import { processorArtifactId } from '../records/index.js';
|
|
5
4
|
import { parseBlobReference } from '../store/index.js';
|
|
@@ -7,6 +6,7 @@ import { IMAGE_MIMES } from '../../image/normalize.js';
|
|
|
7
6
|
import { previewEncoder } from './image.js';
|
|
8
7
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
9
8
|
import { getChat } from '../../chat.js';
|
|
9
|
+
import { errorMessage } from '../../../guards.js';
|
|
10
10
|
export { previewEncoder } from './image.js';
|
|
11
11
|
export const PREVIEW_VERSION = 'processor:preview:1';
|
|
12
12
|
const RETRY_MS = 30_000;
|
|
@@ -52,8 +52,9 @@ export class PreviewProcessor {
|
|
|
52
52
|
await document.claims.settle(id);
|
|
53
53
|
return;
|
|
54
54
|
}
|
|
55
|
-
if (document.claims.isFailed(id))
|
|
55
|
+
if (document.claims.isFailed(id)) {
|
|
56
56
|
return;
|
|
57
|
+
}
|
|
57
58
|
const encode = await this._encoder();
|
|
58
59
|
if (!encode) {
|
|
59
60
|
document.work({ state: 'pending', reason: 'image processor unavailable on this device' });
|
|
@@ -78,8 +79,9 @@ export class PreviewProcessor {
|
|
|
78
79
|
document.schedule(retryAt);
|
|
79
80
|
return;
|
|
80
81
|
}
|
|
81
|
-
if (await document.claims.blockedByStatus(id))
|
|
82
|
+
if (await document.claims.blockedByStatus(id)) {
|
|
82
83
|
return;
|
|
84
|
+
}
|
|
83
85
|
const source = await document.source.read(signal);
|
|
84
86
|
if (source.kind !== 'bytes') {
|
|
85
87
|
const retryAt = source.kind === 'missing' ? source.retryAt : undefined;
|
|
@@ -88,21 +90,25 @@ export class PreviewProcessor {
|
|
|
88
90
|
reason: 'source unavailable',
|
|
89
91
|
retryAt
|
|
90
92
|
});
|
|
91
|
-
if (retryAt)
|
|
93
|
+
if (retryAt) {
|
|
92
94
|
document.schedule(retryAt);
|
|
95
|
+
}
|
|
93
96
|
return;
|
|
94
97
|
}
|
|
95
|
-
if (signal.aborted || document.paused() || document.closed())
|
|
98
|
+
if (signal.aborted || document.paused() || document.closed()) {
|
|
96
99
|
return;
|
|
100
|
+
}
|
|
97
101
|
const claim = await document.claims.claim(id);
|
|
98
|
-
if (!claim)
|
|
102
|
+
if (!claim) {
|
|
99
103
|
return;
|
|
104
|
+
}
|
|
100
105
|
document.work({ state: 'running' });
|
|
101
106
|
try {
|
|
102
107
|
const bytes = await encode(source.bytes, signal);
|
|
103
108
|
const current = await document.current();
|
|
104
|
-
if (signal.aborted)
|
|
109
|
+
if (signal.aborted) {
|
|
105
110
|
throw new AbortedError();
|
|
111
|
+
}
|
|
106
112
|
if (!current || current.deletedAt || previewId(current) !== id) {
|
|
107
113
|
throw new Error('source no longer available');
|
|
108
114
|
}
|
|
@@ -121,18 +127,16 @@ export class PreviewProcessor {
|
|
|
121
127
|
}
|
|
122
128
|
catch (error) {
|
|
123
129
|
const interrupted = signal.aborted || document.paused() || document.closed();
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
: error instanceof Error
|
|
127
|
-
? error.message
|
|
128
|
-
: String(error);
|
|
130
|
+
const detail = errorMessage(error);
|
|
131
|
+
const reason = interrupted ? 'interrupted: suspend' : detail;
|
|
129
132
|
document.work({ state: interrupted ? 'pending' : 'failed', reason });
|
|
130
|
-
if (!interrupted)
|
|
133
|
+
if (!interrupted) {
|
|
131
134
|
document.claims.markFailed(id);
|
|
135
|
+
}
|
|
132
136
|
if (claim.claimId && !document.closed()) {
|
|
133
137
|
await document
|
|
134
138
|
.resolve({ artifactId: id, claimId: claim.claimId, status: 'failed', error: reason })
|
|
135
|
-
.catch(
|
|
139
|
+
.catch(noop);
|
|
136
140
|
}
|
|
137
141
|
}
|
|
138
142
|
}
|
|
@@ -157,5 +161,6 @@ function previewSource(file) {
|
|
|
157
161
|
]);
|
|
158
162
|
}
|
|
159
163
|
function hash(values) {
|
|
160
|
-
return b4a.
|
|
164
|
+
return b4a.toHex(crypto.hash(b4a.from(JSON.stringify(values))));
|
|
161
165
|
}
|
|
166
|
+
function noop() { }
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import type { Mesh } from '../../mesh.ts';
|
|
2
2
|
import type { FileState } from './state.ts';
|
|
3
|
-
export
|
|
3
|
+
export interface IndexingClaimsOptions {
|
|
4
4
|
mesh: Mesh;
|
|
5
5
|
schedule: (deadline: number) => void;
|
|
6
6
|
paused: () => boolean;
|
|
7
7
|
closed: () => boolean;
|
|
8
8
|
state?: FileState;
|
|
9
9
|
advisory?: boolean;
|
|
10
|
-
}
|
|
10
|
+
}
|
|
11
|
+
export interface IndexingClaim {
|
|
12
|
+
readonly claimId: string;
|
|
13
|
+
}
|
|
11
14
|
export declare class IndexingClaims {
|
|
12
15
|
private readonly _mesh;
|
|
13
16
|
private readonly _schedule;
|
|
@@ -18,9 +21,7 @@ export declare class IndexingClaims {
|
|
|
18
21
|
constructor(opts: IndexingClaimsOptions);
|
|
19
22
|
markFailed(artifactId: string): void;
|
|
20
23
|
isFailed(artifactId: string): boolean;
|
|
21
|
-
claim(artifactId: string, knowledgeId: string): Promise<
|
|
22
|
-
claimId: string;
|
|
23
|
-
} | null>;
|
|
24
|
+
claim(artifactId: string, knowledgeId: string): Promise<IndexingClaim | null>;
|
|
24
25
|
blockedByStatus(artifactId: string): Promise<boolean>;
|
|
25
26
|
settle(artifactId: string, knowledgeId: string): Promise<void>;
|
|
26
27
|
}
|
|
@@ -27,10 +27,12 @@ export class IndexingClaims {
|
|
|
27
27
|
knowledgeId,
|
|
28
28
|
at: Date.now()
|
|
29
29
|
});
|
|
30
|
-
if (won)
|
|
30
|
+
if (won) {
|
|
31
31
|
return { claimId: row.claimId };
|
|
32
|
-
|
|
32
|
+
}
|
|
33
|
+
if (isClaimLive(row, Date.now())) {
|
|
33
34
|
this._schedule(row.expiresAt);
|
|
35
|
+
}
|
|
34
36
|
return null;
|
|
35
37
|
}
|
|
36
38
|
catch (err) {
|
|
@@ -48,10 +50,12 @@ export class IndexingClaims {
|
|
|
48
50
|
if (this.isFailed(artifactId))
|
|
49
51
|
return true;
|
|
50
52
|
const row = await getIndexingStatus(this._mesh, artifactId);
|
|
51
|
-
if (!row)
|
|
53
|
+
if (!row) {
|
|
52
54
|
return false;
|
|
53
|
-
|
|
55
|
+
}
|
|
56
|
+
if (row.status === 'ready') {
|
|
54
57
|
return true;
|
|
58
|
+
}
|
|
55
59
|
if (isClaimLive(row, Date.now())) {
|
|
56
60
|
this._schedule(row.expiresAt);
|
|
57
61
|
return true;
|
|
@@ -62,25 +66,29 @@ export class IndexingClaims {
|
|
|
62
66
|
if (this._paused() || this._closed())
|
|
63
67
|
return;
|
|
64
68
|
const row = await getIndexingStatus(this._mesh, artifactId);
|
|
65
|
-
if (row?.status === 'ready')
|
|
69
|
+
if (row?.status === 'ready') {
|
|
66
70
|
return;
|
|
71
|
+
}
|
|
67
72
|
if (row && isClaimLive(row, Date.now())) {
|
|
68
73
|
this._schedule(row.expiresAt);
|
|
69
74
|
return;
|
|
70
75
|
}
|
|
71
|
-
if (this._paused() || this._closed())
|
|
76
|
+
if (this._paused() || this._closed()) {
|
|
72
77
|
return;
|
|
78
|
+
}
|
|
73
79
|
const claim = await this.claim(artifactId, knowledgeId);
|
|
74
|
-
if (!claim?.claimId)
|
|
80
|
+
if (!claim?.claimId) {
|
|
75
81
|
return;
|
|
82
|
+
}
|
|
76
83
|
try {
|
|
77
84
|
const { applied } = await resolveIndexing(this._mesh, {
|
|
78
85
|
artifactId,
|
|
79
86
|
claimId: claim.claimId,
|
|
80
87
|
status: 'ready'
|
|
81
88
|
});
|
|
82
|
-
if (applied)
|
|
89
|
+
if (applied) {
|
|
83
90
|
logger.info('rag', 'repaired to ready', { id: artifactId });
|
|
91
|
+
}
|
|
84
92
|
}
|
|
85
93
|
catch (err) {
|
|
86
94
|
logger.debug('rag', 'repair resolve dropped', { id: artifactId, err });
|
|
@@ -1,14 +1,27 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { BlobId } from 'hyperblobs';
|
|
3
2
|
import { type IndexingProgress } from '../rag/index.ts';
|
|
4
3
|
import { IndexingClaims } from './claims.ts';
|
|
5
|
-
import { ArtifactOwnership } from './ownership.ts';
|
|
4
|
+
import { ArtifactOwnership, type WrittenArtifact } from './ownership.ts';
|
|
6
5
|
import { resolveIndexing } from './protocol.ts';
|
|
7
6
|
import type { IndexingSource } from './source.ts';
|
|
8
7
|
import { type FileRecord } from '../records/index.ts';
|
|
9
8
|
import type { Mesh } from '../../mesh.ts';
|
|
10
9
|
import type { LocalWork } from './state.ts';
|
|
11
10
|
import type { FileProcessor } from './index.ts';
|
|
11
|
+
export interface FileDocumentOptions {
|
|
12
|
+
readonly record: FileRecord;
|
|
13
|
+
readonly processor: FileProcessor;
|
|
14
|
+
readonly source: IndexingSource;
|
|
15
|
+
readonly output: IndexingSource;
|
|
16
|
+
readonly schedule: (deadline: number) => void;
|
|
17
|
+
readonly closed: () => boolean;
|
|
18
|
+
readonly paused: () => boolean;
|
|
19
|
+
readonly progress?: IndexingProgress;
|
|
20
|
+
}
|
|
21
|
+
interface SourceReference {
|
|
22
|
+
readonly id: string;
|
|
23
|
+
readonly reference: string;
|
|
24
|
+
}
|
|
12
25
|
export declare class FileDocument {
|
|
13
26
|
private readonly _mesh;
|
|
14
27
|
readonly record: FileRecord;
|
|
@@ -28,43 +41,10 @@ export declare class FileDocument {
|
|
|
28
41
|
private readonly _output;
|
|
29
42
|
private readonly _progress?;
|
|
30
43
|
private readonly _processor;
|
|
31
|
-
constructor(mesh: Mesh, options:
|
|
32
|
-
record: FileRecord;
|
|
33
|
-
processor: FileProcessor;
|
|
34
|
-
source: IndexingSource;
|
|
35
|
-
output: IndexingSource;
|
|
36
|
-
schedule: (deadline: number) => void;
|
|
37
|
-
closed: () => boolean;
|
|
38
|
-
paused: () => boolean;
|
|
39
|
-
progress?: IndexingProgress;
|
|
40
|
-
});
|
|
44
|
+
constructor(mesh: Mesh, options: FileDocumentOptions);
|
|
41
45
|
get localCoreKey(): string;
|
|
42
|
-
artifact(id: string):
|
|
43
|
-
|
|
44
|
-
terminal?: boolean;
|
|
45
|
-
failedReference?: string;
|
|
46
|
-
unavailable?: {
|
|
47
|
-
blobRef: string;
|
|
48
|
-
repeated: boolean;
|
|
49
|
-
};
|
|
50
|
-
rebuildFailures?: number;
|
|
51
|
-
published?: {
|
|
52
|
-
artifactBlobId: string;
|
|
53
|
-
};
|
|
54
|
-
};
|
|
55
|
-
readArtifact(id: string): Readonly<{
|
|
56
|
-
knowledgeId?: string;
|
|
57
|
-
terminal?: boolean;
|
|
58
|
-
failedReference?: string;
|
|
59
|
-
unavailable?: {
|
|
60
|
-
blobRef: string;
|
|
61
|
-
repeated: boolean;
|
|
62
|
-
};
|
|
63
|
-
rebuildFailures?: number;
|
|
64
|
-
published?: {
|
|
65
|
-
artifactBlobId: string;
|
|
66
|
-
};
|
|
67
|
-
}> | undefined;
|
|
46
|
+
artifact(id: string): import("./state.ts").ArtifactState;
|
|
47
|
+
readArtifact(id: string): Readonly<import("./state.ts").ArtifactState> | undefined;
|
|
68
48
|
output(id: string, reference: string): {
|
|
69
49
|
read: (signal?: AbortSignal) => Promise<import("./source.ts").SourceRead>;
|
|
70
50
|
blockedUntil: () => number | null;
|
|
@@ -75,23 +55,14 @@ export declare class FileDocument {
|
|
|
75
55
|
statuses(): Promise<import("../../../../spec/index").QvacIndexingStatus[]>;
|
|
76
56
|
current(): Promise<FileRecord | null>;
|
|
77
57
|
publish(input: Omit<Parameters<Mesh['files']['publish']>[0], 'knowledgeId'>): Promise<import("../../../../spec/index").QvacKnowledgeArtifact>;
|
|
78
|
-
resolve(input: Parameters<typeof resolveIndexing>[1]): Promise<
|
|
79
|
-
|
|
80
|
-
applied: boolean;
|
|
81
|
-
}>;
|
|
82
|
-
write(bytes: Buffer): Promise<{
|
|
83
|
-
blobRef: BlobId;
|
|
84
|
-
artifactBlobId: string;
|
|
85
|
-
}>;
|
|
58
|
+
resolve(input: Parameters<typeof resolveIndexing>[1]): Promise<import("./protocol.ts").IndexingResolutionResult>;
|
|
59
|
+
write(bytes: Buffer): Promise<WrittenArtifact>;
|
|
86
60
|
inline(bytes: Buffer, mimeType: string): string;
|
|
87
61
|
work(state: LocalWork): void;
|
|
88
62
|
load(state: LocalWork, artifactId?: string): void;
|
|
89
63
|
report(update: Omit<Parameters<IndexingProgress>[0], 'knowledgeId' | 'chatId'>): void;
|
|
90
64
|
}
|
|
91
|
-
declare function fileSource(source: IndexingSource, { id, reference }: {
|
|
92
|
-
id: string;
|
|
93
|
-
reference: string;
|
|
94
|
-
}): {
|
|
65
|
+
declare function fileSource(source: IndexingSource, { id, reference }: SourceReference): {
|
|
95
66
|
read: (signal?: AbortSignal) => Promise<import("./source.ts").SourceRead>;
|
|
96
67
|
blockedUntil: () => number | null;
|
|
97
68
|
miss: () => number;
|
|
@@ -3,7 +3,12 @@ import { type IndexingProgress } from '../rag/index.ts';
|
|
|
3
3
|
import { type FileInput, type FileRecord } from '../records/index.ts';
|
|
4
4
|
import type { Mesh } from '../../mesh.ts';
|
|
5
5
|
import { FileDocument } from './document.ts';
|
|
6
|
+
import type { LoadedArtifact } from './state.ts';
|
|
6
7
|
import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
8
|
+
export interface ProcessorHost {
|
|
9
|
+
readonly mesh: Mesh;
|
|
10
|
+
readonly deviceId: Buffer | null;
|
|
11
|
+
}
|
|
7
12
|
export interface FileProcessor {
|
|
8
13
|
readonly id: string;
|
|
9
14
|
readonly version: string;
|
|
@@ -12,14 +17,21 @@ export interface FileProcessor {
|
|
|
12
17
|
readonly mediaKind?: string;
|
|
13
18
|
accepts?(output: QvacKnowledgeArtifact, document: FileRecord): boolean;
|
|
14
19
|
applies(document: FileRecord): boolean;
|
|
15
|
-
canProduce(document: FileRecord, host:
|
|
16
|
-
mesh: Mesh;
|
|
17
|
-
deviceId: Buffer | null;
|
|
18
|
-
}): boolean | Promise<boolean>;
|
|
20
|
+
canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
|
|
19
21
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
20
22
|
load?(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
23
|
+
evict?(loaded: LoadedArtifact): Promise<void>;
|
|
21
24
|
close(): Promise<boolean>;
|
|
22
25
|
}
|
|
26
|
+
export interface FileProcessingOptions {
|
|
27
|
+
readonly processor: FileProcessor;
|
|
28
|
+
readonly deviceId: Buffer | null;
|
|
29
|
+
readonly retryMs?: number;
|
|
30
|
+
readonly progress?: IndexingProgress;
|
|
31
|
+
}
|
|
32
|
+
export interface EnqueueOptions {
|
|
33
|
+
readonly wait?: boolean;
|
|
34
|
+
}
|
|
23
35
|
export declare class FileProcessing {
|
|
24
36
|
private readonly _mesh;
|
|
25
37
|
private readonly _processor;
|
|
@@ -38,12 +50,7 @@ export declare class FileProcessing {
|
|
|
38
50
|
private _indexCleared;
|
|
39
51
|
private readonly _onDirty;
|
|
40
52
|
private readonly _onAll;
|
|
41
|
-
constructor(mesh: Mesh, options:
|
|
42
|
-
processor: FileProcessor;
|
|
43
|
-
deviceId: Buffer | null;
|
|
44
|
-
retryMs?: number;
|
|
45
|
-
progress?: IndexingProgress;
|
|
46
|
-
});
|
|
53
|
+
constructor(mesh: Mesh, options: FileProcessingOptions);
|
|
47
54
|
start(): void;
|
|
48
55
|
suspend(): Promise<void>;
|
|
49
56
|
resume(): void;
|
|
@@ -51,9 +58,7 @@ export declare class FileProcessing {
|
|
|
51
58
|
close(): Promise<void>;
|
|
52
59
|
get indexCleared(): boolean;
|
|
53
60
|
settle(): Promise<void>;
|
|
54
|
-
enqueue(ids: Iterable<string>, options?:
|
|
55
|
-
wait?: boolean;
|
|
56
|
-
}): Promise<void>;
|
|
61
|
+
enqueue(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
|
|
57
62
|
private _enqueue;
|
|
58
63
|
get active(): boolean;
|
|
59
64
|
pending(id: string): boolean;
|
|
@@ -64,7 +69,10 @@ export declare class FileProcessing {
|
|
|
64
69
|
private _scan;
|
|
65
70
|
private _documents;
|
|
66
71
|
private _pump;
|
|
72
|
+
private _forget;
|
|
67
73
|
private _drain;
|
|
74
|
+
private _recordFailure;
|
|
75
|
+
private _queueLoad;
|
|
68
76
|
private _schedule;
|
|
69
77
|
private _retry;
|
|
70
78
|
}
|
|
@@ -4,6 +4,7 @@ import { DeadlineScheduler } from '../../../deadline.js';
|
|
|
4
4
|
import { IndexingSource } from './source.js';
|
|
5
5
|
import { fileIds, fileRecord } from '../records/index.js';
|
|
6
6
|
import { FileDocument } from './document.js';
|
|
7
|
+
import { errorMessage } from '../../../guards.js';
|
|
7
8
|
export class FileProcessing {
|
|
8
9
|
_mesh;
|
|
9
10
|
_processor;
|
|
@@ -21,10 +22,12 @@ export class FileProcessing {
|
|
|
21
22
|
_close = null;
|
|
22
23
|
_indexCleared = true;
|
|
23
24
|
_onDirty = (ids) => {
|
|
24
|
-
if (ids)
|
|
25
|
+
if (ids) {
|
|
25
26
|
void this.enqueue(ids);
|
|
26
|
-
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
27
29
|
this._kick();
|
|
30
|
+
}
|
|
28
31
|
};
|
|
29
32
|
_onAll = () => this._kick();
|
|
30
33
|
constructor(mesh, options) {
|
|
@@ -83,8 +86,9 @@ export class FileProcessing {
|
|
|
83
86
|
if (this._closed || this._paused)
|
|
84
87
|
return;
|
|
85
88
|
const ids = await this._documents();
|
|
86
|
-
for (const lane of this._lanes)
|
|
89
|
+
for (const lane of this._lanes) {
|
|
87
90
|
await this._enqueue(ids, [lane], { wait: true });
|
|
91
|
+
}
|
|
88
92
|
}
|
|
89
93
|
enqueue(ids, options = {}) {
|
|
90
94
|
return this._enqueue(ids, this._lanes, options);
|
|
@@ -96,14 +100,17 @@ export class FileProcessing {
|
|
|
96
100
|
for (const id of ids) {
|
|
97
101
|
for (const lane of lanes) {
|
|
98
102
|
let wakes = lane.queue.get(id);
|
|
99
|
-
if (!wakes)
|
|
103
|
+
if (!wakes) {
|
|
100
104
|
lane.queue.set(id, (wakes = []));
|
|
101
|
-
|
|
105
|
+
}
|
|
106
|
+
if (wait) {
|
|
102
107
|
pending.push(new Promise((resolve) => wakes.push(resolve)));
|
|
108
|
+
}
|
|
103
109
|
}
|
|
104
110
|
}
|
|
105
|
-
for (const lane of lanes)
|
|
111
|
+
for (const lane of lanes) {
|
|
106
112
|
this._pump(lane);
|
|
113
|
+
}
|
|
107
114
|
await Promise.all(pending);
|
|
108
115
|
}
|
|
109
116
|
get active() {
|
|
@@ -120,11 +127,14 @@ export class FileProcessing {
|
|
|
120
127
|
for (const lane of this._lanes) {
|
|
121
128
|
const queued = new Map(lane.queue);
|
|
122
129
|
lane.queue.clear();
|
|
123
|
-
for (const id of ids)
|
|
130
|
+
for (const id of ids) {
|
|
124
131
|
lane.queue.set(id, queued.get(id) ?? []);
|
|
125
|
-
|
|
126
|
-
|
|
132
|
+
}
|
|
133
|
+
for (const [id, wakes] of queued) {
|
|
134
|
+
if (!lane.queue.has(id)) {
|
|
127
135
|
lane.queue.set(id, wakes);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
128
138
|
this._pump(lane);
|
|
129
139
|
}
|
|
130
140
|
}
|
|
@@ -134,8 +144,9 @@ export class FileProcessing {
|
|
|
134
144
|
this._mesh.autobee.off('move-to', this._onAll);
|
|
135
145
|
this._mesh.autobee.off('writable', this._onAll);
|
|
136
146
|
await this._stop();
|
|
137
|
-
if (this._processor.load)
|
|
147
|
+
if (this._processor.load) {
|
|
138
148
|
this._mesh.files.clearLoaded(this._processor.id);
|
|
149
|
+
}
|
|
139
150
|
this._indexCleared = await this._processor.close();
|
|
140
151
|
}
|
|
141
152
|
async _stop() {
|
|
@@ -150,9 +161,11 @@ export class FileProcessing {
|
|
|
150
161
|
await Promise.all([this._scanning, ...this._lanes.map((lane) => lane.running)]);
|
|
151
162
|
await Promise.all([this._source.cancel(), this._output.cancel()]);
|
|
152
163
|
for (const lane of this._lanes) {
|
|
153
|
-
for (const wakes of lane.queue.values())
|
|
154
|
-
for (const wake of wakes)
|
|
164
|
+
for (const wakes of lane.queue.values()) {
|
|
165
|
+
for (const wake of wakes) {
|
|
155
166
|
wake();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
156
169
|
lane.queue.clear();
|
|
157
170
|
}
|
|
158
171
|
}
|
|
@@ -160,12 +173,14 @@ export class FileProcessing {
|
|
|
160
173
|
if (this._closed || this._paused)
|
|
161
174
|
return;
|
|
162
175
|
this._dirty = true;
|
|
163
|
-
if (this._scanning)
|
|
176
|
+
if (this._scanning) {
|
|
164
177
|
return;
|
|
178
|
+
}
|
|
165
179
|
this._scanning = this._scan().finally(() => {
|
|
166
180
|
this._scanning = null;
|
|
167
|
-
if (this._dirty)
|
|
181
|
+
if (this._dirty) {
|
|
168
182
|
this._kick();
|
|
183
|
+
}
|
|
169
184
|
});
|
|
170
185
|
}
|
|
171
186
|
async _scan() {
|
|
@@ -199,21 +214,31 @@ export class FileProcessing {
|
|
|
199
214
|
this._pump(lane);
|
|
200
215
|
});
|
|
201
216
|
}
|
|
217
|
+
async _forget(id) {
|
|
218
|
+
const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
|
|
219
|
+
if (loaded) {
|
|
220
|
+
await this._processor.evict?.(loaded);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
202
223
|
async _drain(lane) {
|
|
203
224
|
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
204
|
-
const [id, wakes] = lane.queue
|
|
225
|
+
const [[id, wakes]] = lane.queue;
|
|
205
226
|
lane.queue.delete(id);
|
|
206
227
|
lane.current = id;
|
|
207
228
|
lane.abort = new AbortController();
|
|
208
229
|
let produced = false;
|
|
209
230
|
try {
|
|
210
231
|
const record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
232
|
+
if (!record && lane.stage === 'load') {
|
|
233
|
+
await this._forget(id);
|
|
234
|
+
}
|
|
211
235
|
if (!record || record.deletedAt) {
|
|
212
236
|
await this._source.cancel((key) => key === id);
|
|
213
237
|
await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
|
|
214
238
|
}
|
|
215
|
-
if (!record || !this._processor.applies(record) || lane.abort.signal.aborted)
|
|
239
|
+
if (!record || !this._processor.applies(record) || lane.abort.signal.aborted) {
|
|
216
240
|
continue;
|
|
241
|
+
}
|
|
217
242
|
if (lane.stage === 'produce' &&
|
|
218
243
|
(!this._mesh.writable ||
|
|
219
244
|
!(await this._processor.canProduce(record, {
|
|
@@ -232,49 +257,60 @@ export class FileProcessing {
|
|
|
232
257
|
paused: () => this._paused,
|
|
233
258
|
progress: this._progress
|
|
234
259
|
});
|
|
235
|
-
if (lane.abort.signal.aborted)
|
|
260
|
+
if (lane.abort.signal.aborted) {
|
|
236
261
|
continue;
|
|
262
|
+
}
|
|
237
263
|
await this._processor[lane.stage]?.(document, lane.abort.signal);
|
|
238
264
|
produced = lane.stage === 'produce';
|
|
239
265
|
}
|
|
240
266
|
catch (err) {
|
|
241
267
|
if (!lane.abort.signal.aborted) {
|
|
242
|
-
|
|
243
|
-
if (lane.stage === 'load') {
|
|
244
|
-
this._mesh.files.load(id, { state: 'failed', reason }, {
|
|
245
|
-
processorId: this._processor.id,
|
|
246
|
-
artifactId: this._mesh.files.hydratedArtifact(id) ?? undefined
|
|
247
|
-
});
|
|
248
|
-
}
|
|
249
|
-
else {
|
|
250
|
-
this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
|
|
251
|
-
}
|
|
252
|
-
logger.warn('rag', 'file processing failed', { id, stage: lane.stage, err });
|
|
268
|
+
this._recordFailure(id, { stage: lane.stage, err });
|
|
253
269
|
}
|
|
254
270
|
}
|
|
255
271
|
finally {
|
|
256
272
|
lane.abort = null;
|
|
257
273
|
lane.current = undefined;
|
|
258
274
|
if (produced && this.active) {
|
|
259
|
-
|
|
260
|
-
if (loader) {
|
|
261
|
-
if (!loader.queue.has(id))
|
|
262
|
-
loader.queue.set(id, []);
|
|
263
|
-
this._pump(loader);
|
|
264
|
-
}
|
|
275
|
+
this._queueLoad(id);
|
|
265
276
|
}
|
|
266
|
-
for (const wake of wakes)
|
|
277
|
+
for (const wake of wakes) {
|
|
267
278
|
wake();
|
|
279
|
+
}
|
|
268
280
|
this._mesh.files.emit('update');
|
|
269
281
|
}
|
|
270
282
|
}
|
|
271
283
|
}
|
|
284
|
+
_recordFailure(id, { stage, err }) {
|
|
285
|
+
const reason = errorMessage(err);
|
|
286
|
+
if (stage === 'load') {
|
|
287
|
+
this._mesh.files.load(id, { state: 'failed', reason }, {
|
|
288
|
+
processorId: this._processor.id,
|
|
289
|
+
artifactId: this._mesh.files.hydratedArtifact(id) ?? undefined
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
else {
|
|
293
|
+
this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
|
|
294
|
+
}
|
|
295
|
+
logger.warn('rag', 'file processing failed', { id, stage, err });
|
|
296
|
+
}
|
|
297
|
+
_queueLoad(id) {
|
|
298
|
+
const loader = this._lanes.find((entry) => entry.stage === 'load');
|
|
299
|
+
if (!loader) {
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (!loader.queue.has(id)) {
|
|
303
|
+
loader.queue.set(id, []);
|
|
304
|
+
}
|
|
305
|
+
this._pump(loader);
|
|
306
|
+
}
|
|
272
307
|
_schedule(id, deadline) {
|
|
273
308
|
if (this._closed || this._paused)
|
|
274
309
|
return;
|
|
275
310
|
const previous = this._deadlines.get(id);
|
|
276
|
-
if (previous === undefined || deadline < previous)
|
|
311
|
+
if (previous === undefined || deadline < previous) {
|
|
277
312
|
this._deadlines.set(id, deadline);
|
|
313
|
+
}
|
|
278
314
|
this._scheduler.schedule(deadline);
|
|
279
315
|
}
|
|
280
316
|
_retry() {
|
|
@@ -284,8 +320,9 @@ export class FileProcessing {
|
|
|
284
320
|
this._deadlines.delete(id);
|
|
285
321
|
ids.push(id);
|
|
286
322
|
}
|
|
287
|
-
else
|
|
323
|
+
else {
|
|
288
324
|
this._scheduler.schedule(deadline);
|
|
325
|
+
}
|
|
289
326
|
}
|
|
290
327
|
void this.enqueue(ids);
|
|
291
328
|
}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import type { BlobId } from 'hyperblobs';
|
|
2
2
|
import type { Mesh } from '../../mesh.ts';
|
|
3
3
|
import type { FileState } from './state.ts';
|
|
4
|
+
export interface WrittenArtifact {
|
|
5
|
+
readonly blobRef: BlobId;
|
|
6
|
+
readonly artifactBlobId: string;
|
|
7
|
+
}
|
|
8
|
+
export interface PublishedArtifact extends WrittenArtifact {
|
|
9
|
+
readonly landedBlobId: string;
|
|
10
|
+
}
|
|
4
11
|
export declare class ArtifactOwnership {
|
|
5
12
|
private readonly _mesh;
|
|
6
13
|
private readonly _state;
|
|
7
14
|
constructor(mesh: Mesh, state?: FileState);
|
|
8
|
-
onPublished(artifactId: string, blobRef
|
|
15
|
+
onPublished(artifactId: string, { blobRef, artifactBlobId, landedBlobId }: PublishedArtifact): Promise<void>;
|
|
9
16
|
reclaimSuperseded(artifactId: string, winnerBlobId: string): Promise<void>;
|
|
10
17
|
clearUnreferenced(artifactId: string, blobRef: BlobId): Promise<void>;
|
|
11
18
|
}
|
|
@@ -7,7 +7,7 @@ export class ArtifactOwnership {
|
|
|
7
7
|
this._mesh = mesh;
|
|
8
8
|
this._state = state;
|
|
9
9
|
}
|
|
10
|
-
async onPublished(artifactId, blobRef, artifactBlobId, landedBlobId) {
|
|
10
|
+
async onPublished(artifactId, { blobRef, artifactBlobId, landedBlobId }) {
|
|
11
11
|
if (compareBlobReferences(landedBlobId, artifactBlobId, b4a.toHex(this._mesh.blobs.core.key)) ===
|
|
12
12
|
false) {
|
|
13
13
|
await this._mesh.files.clear(this._mesh.files.reference(blobRef), artifactId).catch(noop);
|