@qvac/core 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -1
- package/dist-lib/lib/api/agents.js +3 -2
- package/dist-lib/lib/api/app-config.d.ts +1 -1
- package/dist-lib/lib/api/app-config.js +54 -15
- package/dist-lib/lib/api/attachments.d.ts +6 -5
- package/dist-lib/lib/api/attachments.js +4 -2
- package/dist-lib/lib/api/chat.d.ts +7 -3
- package/dist-lib/lib/api/chat.js +4 -4
- package/dist-lib/lib/api/chunks.js +3 -3
- package/dist-lib/lib/api/config.js +2 -2
- package/dist-lib/lib/api/current-mesh.d.ts +3 -4
- package/dist-lib/lib/api/current-mesh.js +9 -4
- package/dist-lib/lib/api/custom-models.d.ts +8 -1
- package/dist-lib/lib/api/custom-models.js +53 -16
- package/dist-lib/lib/api/devices.d.ts +14 -5
- package/dist-lib/lib/api/devices.js +13 -8
- package/dist-lib/lib/api/diagnostics.d.ts +1 -0
- package/dist-lib/lib/api/diagnostics.js +1 -1
- package/dist-lib/lib/api/effective-ctx-size.d.ts +6 -1
- package/dist-lib/lib/api/effective-ctx-size.js +5 -3
- package/dist-lib/lib/api/execution.js +2 -1
- package/dist-lib/lib/api/files.js +2 -1
- package/dist-lib/lib/api/index.d.ts +5 -1
- package/dist-lib/lib/api/index.js +101 -102
- package/dist-lib/lib/api/instrument.js +29 -20
- package/dist-lib/lib/api/knowledge.d.ts +6 -5
- package/dist-lib/lib/api/knowledge.js +8 -6
- package/dist-lib/lib/api/local-files.js +1 -1
- package/dist-lib/lib/api/logs.js +6 -3
- package/dist-lib/lib/api/members.js +22 -13
- package/dist-lib/lib/api/mesh.js +3 -3
- package/dist-lib/lib/api/model-download.js +1 -1
- package/dist-lib/lib/api/network.d.ts +1 -0
- package/dist-lib/lib/api/network.js +1 -1
- package/dist-lib/lib/api/notifications.js +23 -13
- package/dist-lib/lib/api/rag.d.ts +6 -1
- package/dist-lib/lib/api/rag.js +20 -11
- package/dist-lib/lib/api/range.d.ts +2 -2
- package/dist-lib/lib/api/sync.js +1 -1
- package/dist-lib/lib/api/tool-activity.js +7 -3
- package/dist-lib/lib/api/tool-configs.js +2 -4
- package/dist-lib/lib/api/upload-frames.d.ts +7 -8
- package/dist-lib/lib/api/upload-frames.js +2 -1
- package/dist-lib/lib/api/watchable.d.ts +6 -1
- package/dist-lib/lib/api/watchable.js +23 -14
- package/dist-lib/lib/assistant/fetch-body.js +4 -2
- package/dist-lib/lib/assistant/harness.d.ts +48 -27
- package/dist-lib/lib/assistant/harness.js +40 -16
- package/dist-lib/lib/assistant/history.d.ts +9 -7
- package/dist-lib/lib/assistant/history.js +77 -42
- package/dist-lib/lib/assistant/message.d.ts +5 -3
- package/dist-lib/lib/assistant/message.js +4 -2
- package/dist-lib/lib/assistant/skill-encode.d.ts +3 -1
- package/dist-lib/lib/assistant/skill-encode.js +52 -62
- package/dist-lib/lib/assistant-client.js +7 -4
- package/dist-lib/lib/assistant-tree.js +27 -11
- package/dist-lib/lib/assistant.d.ts +31 -11
- package/dist-lib/lib/assistant.js +846 -390
- package/dist-lib/lib/audio-format.d.ts +1 -1
- package/dist-lib/lib/audio-format.js +1 -1
- package/dist-lib/lib/chat-title.js +9 -4
- package/dist-lib/lib/client.js +3 -2
- package/dist-lib/lib/collabora/convert.d.ts +2 -2
- package/dist-lib/lib/collabora/convert.js +6 -3
- package/dist-lib/lib/collabora/entry.js +9 -5
- package/dist-lib/lib/collabora/protocol.d.ts +2 -2
- package/dist-lib/lib/collabora/protocol.js +14 -7
- package/dist-lib/lib/collabora/worker.js +14 -7
- package/dist-lib/lib/compaction.d.ts +5 -6
- package/dist-lib/lib/compaction.js +14 -7
- package/dist-lib/lib/core/account-recovery.d.ts +4 -3
- package/dist-lib/lib/core/account-recovery.js +7 -4
- package/dist-lib/lib/core/agent.js +17 -15
- package/dist-lib/lib/core/approval-rules.d.ts +7 -6
- package/dist-lib/lib/core/approval-rules.js +12 -6
- package/dist-lib/lib/core/archived-mesh.d.ts +7 -5
- package/dist-lib/lib/core/archived-mesh.js +25 -22
- package/dist-lib/lib/core/async-iterable.js +10 -5
- package/dist-lib/lib/core/attachment-link.d.ts +2 -2
- package/dist-lib/lib/core/audio-attachment.js +1 -1
- package/dist-lib/lib/core/blind-peers.d.ts +2 -2
- package/dist-lib/lib/core/chat-activity.d.ts +2 -2
- package/dist-lib/lib/core/chat-activity.js +42 -21
- package/dist-lib/lib/core/chat-group.d.ts +8 -6
- package/dist-lib/lib/core/chat-group.js +17 -10
- package/dist-lib/lib/core/chat-order.d.ts +11 -9
- package/dist-lib/lib/core/chat-order.js +1 -3
- package/dist-lib/lib/core/chat-search.d.ts +4 -4
- package/dist-lib/lib/core/chat-search.js +38 -24
- package/dist-lib/lib/core/chat.d.ts +20 -14
- package/dist-lib/lib/core/chat.js +50 -31
- package/dist-lib/lib/core/chunk.d.ts +19 -23
- package/dist-lib/lib/core/chunk.js +27 -21
- package/dist-lib/lib/core/collections.d.ts +70 -0
- package/dist-lib/lib/core/collections.js +70 -0
- package/dist-lib/lib/core/contract/activity.d.ts +6 -2
- package/dist-lib/lib/core/contract/activity.js +15 -12
- package/dist-lib/lib/core/contract/agents.js +3 -2
- package/dist-lib/lib/core/contract/approval-rules.js +1 -1
- package/dist-lib/lib/core/contract/attestations.js +1 -1
- package/dist-lib/lib/core/contract/chat-group.js +1 -1
- package/dist-lib/lib/core/contract/chat.d.ts +2 -0
- package/dist-lib/lib/core/contract/chat.js +67 -1
- package/dist-lib/lib/core/contract/chunks.js +15 -9
- package/dist-lib/lib/core/contract/compute.d.ts +1 -1
- package/dist-lib/lib/core/contract/compute.js +4 -2
- package/dist-lib/lib/core/contract/devices.js +6 -5
- package/dist-lib/lib/core/contract/grant-order.js +5 -6
- package/dist-lib/lib/core/contract/index.js +4 -2
- package/dist-lib/lib/core/contract/indexing.js +3 -3
- package/dist-lib/lib/core/contract/invites.d.ts +8 -3
- package/dist-lib/lib/core/contract/invites.js +52 -26
- package/dist-lib/lib/core/contract/joined-meshes.js +3 -3
- package/dist-lib/lib/core/contract/knowledge.js +5 -5
- package/dist-lib/lib/core/contract/members.d.ts +13 -2
- package/dist-lib/lib/core/contract/members.js +46 -25
- package/dist-lib/lib/core/contract/mesh-kind.js +9 -8
- package/dist-lib/lib/core/contract/seal.js +1 -1
- package/dist-lib/lib/core/contract/seed-defaults.js +1 -3
- package/dist-lib/lib/core/contract/tool-activity.js +10 -5
- package/dist-lib/lib/core/contract/tool-configs.js +1 -2
- package/dist-lib/lib/core/contract/writers.d.ts +6 -1
- package/dist-lib/lib/core/contract/writers.js +55 -34
- package/dist-lib/lib/core/departures.js +16 -8
- package/dist-lib/lib/core/device-invites.js +3 -2
- package/dist-lib/lib/core/devices.js +6 -3
- package/dist-lib/lib/core/document-attachment.d.ts +20 -11
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.d.ts +2 -5
- package/dist-lib/lib/core/files/attachments.js +6 -3
- package/dist-lib/lib/core/files/index.d.ts +46 -52
- package/dist-lib/lib/core/files/index.js +149 -94
- package/dist-lib/lib/core/files/knowledge.d.ts +2 -5
- package/dist-lib/lib/core/files/local-index.d.ts +4 -1
- package/dist-lib/lib/core/files/local-index.js +9 -5
- package/dist-lib/lib/core/files/preview/image.d.ts +0 -1
- package/dist-lib/lib/core/files/preview/image.js +9 -6
- package/dist-lib/lib/core/files/preview/index.d.ts +8 -11
- package/dist-lib/lib/core/files/preview/index.js +20 -15
- package/dist-lib/lib/core/files/processing/claims.d.ts +6 -5
- package/dist-lib/lib/core/files/processing/claims.js +16 -8
- package/dist-lib/lib/core/files/processing/document.d.ts +21 -50
- package/dist-lib/lib/core/files/processing/index.d.ts +21 -13
- package/dist-lib/lib/core/files/processing/index.js +74 -37
- package/dist-lib/lib/core/files/processing/ownership.d.ts +8 -1
- package/dist-lib/lib/core/files/processing/ownership.js +1 -1
- package/dist-lib/lib/core/files/processing/protocol.d.ts +19 -19
- package/dist-lib/lib/core/files/processing/protocol.js +5 -5
- package/dist-lib/lib/core/files/processing/settle.d.ts +10 -7
- package/dist-lib/lib/core/files/processing/settle.js +22 -11
- package/dist-lib/lib/core/files/processing/source.js +14 -8
- package/dist-lib/lib/core/files/processing/state.d.ts +14 -11
- package/dist-lib/lib/core/files/processing/state.js +24 -12
- package/dist-lib/lib/core/files/rag/build.d.ts +2 -2
- package/dist-lib/lib/core/files/rag/build.js +2 -1
- package/dist-lib/lib/core/files/rag/document-types.d.ts +4 -6
- package/dist-lib/lib/core/files/rag/index.d.ts +12 -13
- package/dist-lib/lib/core/files/rag/index.js +9 -2
- package/dist-lib/lib/core/files/rag/load.d.ts +10 -9
- package/dist-lib/lib/core/files/rag/load.js +26 -13
- package/dist-lib/lib/core/files/rag/policy.js +2 -1
- package/dist-lib/lib/core/files/rag/produce.js +33 -19
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +30 -10
- package/dist-lib/lib/core/files/records/chunks.js +4 -2
- package/dist-lib/lib/core/files/records/index.d.ts +1 -0
- package/dist-lib/lib/core/files/records/index.js +1 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +10 -11
- package/dist-lib/lib/core/files/records/knowledge.js +9 -8
- package/dist-lib/lib/core/files/records/linkage.js +6 -3
- package/dist-lib/lib/core/files/records/output.d.ts +5 -4
- package/dist-lib/lib/core/files/records/output.js +8 -4
- package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
- package/dist-lib/lib/core/files/records/purge.js +108 -0
- package/dist-lib/lib/core/files/records/reclaim.js +43 -25
- package/dist-lib/lib/core/files/records/routing.d.ts +4 -6
- package/dist-lib/lib/core/files/records/routing.js +15 -8
- package/dist-lib/lib/core/files/records/source.d.ts +5 -4
- package/dist-lib/lib/core/files/records/source.js +20 -14
- package/dist-lib/lib/core/files/records/sweep.js +1 -3
- package/dist-lib/lib/core/files/store/blob.d.ts +11 -10
- package/dist-lib/lib/core/files/store/blob.js +54 -19
- package/dist-lib/lib/core/files/store/index.d.ts +2 -6
- package/dist-lib/lib/core/files/store/index.js +10 -5
- package/dist-lib/lib/core/files/store/inline.d.ts +2 -2
- package/dist-lib/lib/core/files/store/inline.js +6 -3
- package/dist-lib/lib/core/files/store/reference.d.ts +6 -9
- package/dist-lib/lib/core/files/store/reference.js +25 -17
- package/dist-lib/lib/core/files/store/server.d.ts +7 -6
- package/dist-lib/lib/core/files/store/server.js +22 -11
- package/dist-lib/lib/core/files/store/sweep.js +13 -7
- package/dist-lib/lib/core/files/upload-input.d.ts +4 -12
- package/dist-lib/lib/core/files/upload-input.js +10 -5
- package/dist-lib/lib/core/files/upload.d.ts +27 -24
- package/dist-lib/lib/core/files/upload.js +74 -51
- package/dist-lib/lib/core/fs-drives.js +4 -2
- package/dist-lib/lib/core/grant-expiry.js +12 -7
- package/dist-lib/lib/core/handoff-channel.js +27 -13
- package/dist-lib/lib/core/harness-loan.js +35 -17
- package/dist-lib/lib/core/helpers/index.js +6 -2
- package/dist-lib/lib/core/hosted-agents.js +19 -12
- package/dist-lib/lib/core/id.js +1 -1
- package/dist-lib/lib/core/identity.js +6 -3
- package/dist-lib/lib/core/image/cache-sweep.d.ts +12 -10
- package/dist-lib/lib/core/image/cache-sweep.js +11 -6
- package/dist-lib/lib/core/image/normalize.d.ts +6 -0
- package/dist-lib/lib/core/image/normalize.js +12 -2
- package/dist-lib/lib/core/image/sniff.js +42 -21
- package/dist-lib/lib/core/inference-calls-receiver.js +7 -4
- package/dist-lib/lib/core/invite-grants.js +4 -2
- package/dist-lib/lib/core/invite.js +2 -1
- package/dist-lib/lib/core/local/files.d.ts +2 -0
- package/dist-lib/lib/core/local/files.js +51 -19
- package/dist-lib/lib/core/local/index.d.ts +69 -46
- package/dist-lib/lib/core/local/index.js +146 -129
- package/dist-lib/lib/core/membership.d.ts +12 -2
- package/dist-lib/lib/core/membership.js +55 -33
- package/dist-lib/lib/core/mesh-seed-handoff.js +7 -1
- package/dist-lib/lib/core/mesh.d.ts +37 -27
- package/dist-lib/lib/core/mesh.js +131 -73
- package/dist-lib/lib/core/migrations/agent-catalog.js +3 -2
- package/dist-lib/lib/core/migrations/agent-indexes.js +3 -3
- package/dist-lib/lib/core/migrations/file-index.js +7 -4
- package/dist-lib/lib/core/migrations/index.d.ts +4 -3
- package/dist-lib/lib/core/migrations/membership.js +9 -4
- package/dist-lib/lib/core/migrations/seal.js +3 -2
- package/dist-lib/lib/core/min-version.js +4 -2
- package/dist-lib/lib/core/next-frame.d.ts +5 -4
- package/dist-lib/lib/core/next-frame.js +6 -3
- package/dist-lib/lib/core/notification.d.ts +5 -1
- package/dist-lib/lib/core/notification.js +89 -77
- package/dist-lib/lib/core/ota-config.js +2 -1
- package/dist-lib/lib/core/pdf/rasterize.d.ts +18 -0
- package/dist-lib/lib/core/pdf/rasterize.js +57 -0
- package/dist-lib/lib/core/prompt-history.js +4 -2
- package/dist-lib/lib/core/range.d.ts +4 -4
- package/dist-lib/lib/core/range.js +12 -6
- package/dist-lib/lib/core/rank.d.ts +5 -4
- package/dist-lib/lib/core/rank.js +2 -1
- package/dist-lib/lib/core/send.d.ts +4 -4
- package/dist-lib/lib/core/send.js +6 -3
- package/dist-lib/lib/core/skill-overlay-receiver.js +7 -4
- package/dist-lib/lib/core/skill-secrets.js +16 -9
- package/dist-lib/lib/core/sync/chunks.js +2 -1
- package/dist-lib/lib/core/sync/devices.js +3 -1
- package/dist-lib/lib/core/sync/files.d.ts +2 -4
- package/dist-lib/lib/core/sync/files.js +28 -20
- package/dist-lib/lib/core/sync/index.d.ts +2 -5
- package/dist-lib/lib/core/sync/index.js +33 -15
- package/dist-lib/lib/core/sync-status.js +10 -7
- package/dist-lib/lib/core/system-info.js +4 -2
- package/dist-lib/lib/core/tool-activity.d.ts +4 -3
- package/dist-lib/lib/core/tool-activity.js +1 -1
- package/dist-lib/lib/core/tool-configs.js +12 -7
- package/dist-lib/lib/core.d.ts +38 -26
- package/dist-lib/lib/core.js +1009 -541
- package/dist-lib/lib/credential-pusher.d.ts +20 -20
- package/dist-lib/lib/credential-pusher.js +65 -43
- package/dist-lib/lib/custom-model-pusher.d.ts +7 -3
- package/dist-lib/lib/custom-model-pusher.js +15 -7
- package/dist-lib/lib/custom-model-task.d.ts +1 -0
- package/dist-lib/lib/custom-model-task.js +12 -0
- package/dist-lib/lib/deadline.js +4 -2
- package/dist-lib/lib/diagnostics.d.ts +1 -5
- package/dist-lib/lib/frame-readable.d.ts +3 -0
- package/dist-lib/lib/frame-readable.js +3 -0
- package/dist-lib/lib/fs-drive-bridge.js +24 -14
- package/dist-lib/lib/fs-grants.js +13 -8
- package/dist-lib/lib/guards.d.ts +3 -0
- package/dist-lib/lib/guards.js +9 -0
- package/dist-lib/lib/inference-calls.d.ts +28 -23
- package/dist-lib/lib/inference-calls.js +25 -13
- package/dist-lib/lib/integrations/mcp-providers.d.ts +2 -2
- package/dist-lib/lib/integrations/setup-routes.d.ts +2 -2
- package/dist-lib/lib/knowledge-search.d.ts +6 -5
- package/dist-lib/lib/knowledge-search.js +26 -18
- package/dist-lib/lib/kv-cache-reclaim.js +4 -2
- package/dist-lib/lib/kv-cache.js +8 -4
- package/dist-lib/lib/legacy-core-domain.d.ts +42 -42
- package/dist-lib/lib/log-buffer.js +2 -1
- package/dist-lib/lib/log.js +25 -15
- package/dist-lib/lib/mesh-scoped-engine.js +12 -4
- package/dist-lib/lib/model-catalog-revision.js +6 -3
- package/dist-lib/lib/model-clamp.js +6 -3
- package/dist-lib/lib/model-context.d.ts +8 -3
- package/dist-lib/lib/model-context.js +13 -5
- package/dist-lib/lib/model-download-status.js +4 -2
- package/dist-lib/lib/model-download.d.ts +12 -4
- package/dist-lib/lib/model-download.js +200 -103
- package/dist-lib/lib/model-idle-pusher.d.ts +2 -2
- package/dist-lib/lib/model-idle-pusher.js +3 -4
- package/dist-lib/lib/oauth/browser-connect.d.ts +10 -8
- package/dist-lib/lib/oauth/browser-connect.js +33 -20
- package/dist-lib/lib/oauth/client.d.ts +2 -2
- package/dist-lib/lib/oauth/flow-core.js +23 -9
- package/dist-lib/lib/oauth/flow.d.ts +1 -3
- package/dist-lib/lib/oauth/flow.js +9 -5
- package/dist-lib/lib/oauth/keys.d.ts +4 -4
- package/dist-lib/lib/oauth/keys.js +18 -12
- package/dist-lib/lib/oauth/loopback.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.js +147 -107
- package/dist-lib/lib/oauth/loopback.stub.d.ts +2 -2
- package/dist-lib/lib/oauth/loopback.stub.js +1 -1
- package/dist-lib/lib/oauth/pkce.d.ts +4 -2
- package/dist-lib/lib/oauth/pkce.js +10 -0
- package/dist-lib/lib/oauth/providers.js +5 -5
- package/dist-lib/lib/oauth/redirect-types.d.ts +12 -7
- package/dist-lib/lib/oauth/register.js +3 -2
- package/dist-lib/lib/oauth/registry.d.ts +2 -2
- package/dist-lib/lib/oauth/registry.js +2 -1
- package/dist-lib/lib/oauth/resolve.d.ts +30 -10
- package/dist-lib/lib/oauth/resolve.js +12 -6
- package/dist-lib/lib/oauth/token-exchange-core.d.ts +12 -4
- package/dist-lib/lib/oauth/token-exchange-core.js +1 -1
- package/dist-lib/lib/oauth/types.d.ts +12 -12
- package/dist-lib/lib/oauth.d.ts +4 -5
- package/dist-lib/lib/oauth.js +2 -3
- package/dist-lib/lib/reconnecting-watch.d.ts +1 -1
- package/dist-lib/lib/reconnecting-watch.js +6 -6
- package/dist-lib/lib/retry-budget.js +6 -3
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +5 -2
- package/dist-lib/lib/run-transcript.d.ts +2 -4
- package/dist-lib/lib/run-transcript.js +4 -2
- package/dist-lib/lib/skill-readiness.js +4 -2
- package/dist-lib/lib/skill-types.d.ts +16 -16
- package/dist-lib/lib/suspend-lifecycle.d.ts +4 -4
- package/dist-lib/lib/suspend-lifecycle.js +10 -5
- package/dist-lib/lib/task.js +22 -12
- package/dist-lib/lib/unref.d.ts +3 -2
- package/dist-lib/lib/unref.js +2 -1
- package/dist-lib/lib/wav-duration.js +10 -5
- package/dist-lib/lib/write-queue.d.ts +5 -4
- package/dist-lib/lib/write-queue.js +2 -1
- package/dist-lib/schema/build.js +10 -6
- package/dist-lib/schema/config.d.ts +2 -3
- package/dist-lib/schema/content.d.ts +17 -2
- package/dist-lib/schema/content.js +2 -1
- package/dist-lib/schema/engine.d.ts +3 -2
- package/dist-lib/schema/engine.js +95 -6
- package/dist-lib/schema/field-order.js +22 -11
- package/dist-lib/schema/indexes.d.ts +22 -0
- package/dist-lib/schema/{helpers.js → indexes.js} +14 -57
- package/dist-lib/schema/local.d.ts +3 -2
- package/dist-lib/schema/local.js +7 -2
- package/dist-lib/schema/media-kind.d.ts +8 -0
- package/dist-lib/schema/media-kind.js +52 -0
- package/dist-lib/schema/qvac.d.ts +12 -9
- package/dist-lib/schema/qvac.js +18 -4
- package/dist-lib/spec/bind.js +12 -0
- package/dist-lib/spec/calls.js +4 -0
- package/dist-lib/spec/capabilities.d.ts +12 -4
- package/dist-lib/spec/hrpc/hrpc.json +54 -2
- package/dist-lib/spec/hrpc/index.js +53 -5
- package/dist-lib/spec/hrpc/messages.js +783 -494
- package/dist-lib/spec/hyperschema/index.js +783 -494
- package/dist-lib/spec/hyperschema/schema.json +247 -1
- package/dist-lib/spec/hyperschema/types.d.ts +69 -1
- package/dist-lib/spec/local/hyperdb/index.js +1 -1
- package/dist-lib/spec/local/hyperdb/messages.js +82 -14
- package/dist-lib/spec/local/hyperschema/index.js +42 -8
- package/dist-lib/spec/local/hyperschema/schema.json +27 -1
- package/dist-lib/spec/local/hyperschema/types.d.ts +6 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
- package/dist-lib/spec/mesh/hyperdb/index.js +72 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +400 -221
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
- package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +117 -2
- package/dist-lib/spec/mesh/hyperschema/index.js +117 -2
- package/dist-lib/spec/mesh/hyperschema/schema.json +65 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +19 -1
- package/package.json +8 -4
- package/tui/app.mjs +2424 -941
- package/tui/boot.mjs +18 -49
- package/tui/index.mjs +10 -5
- package/tui/lib/boot-timing.mjs +8 -8
- package/tui/lib/busy-words.mjs +21 -22
- package/tui/lib/export.mjs +100 -32
- package/tui/lib/knowledge-command.mjs +64 -19
- package/tui/lib/knowledge-import.mjs +52 -18
- package/tui/lib/render.mjs +65 -33
- package/tui/lib/save-file.mjs +30 -13
- package/tui/lib/select.mjs +23 -5
- package/tui/lib/selection.mjs +6 -4
- package/tui/lib/setup.mjs +11 -19
- package/tui/lib/sidebar.mjs +23 -13
- package/tui/lib/splash.mjs +6 -7
- package/tui/lib/upload-frames.mjs +1 -2
- package/tui/lib/usage.mjs +13 -9
- package/tui/lib/wrap.mjs +35 -6
- package/tui/main.mjs +54 -78
- package/worker/assistant-sidecar.mjs +5 -10
- package/worker/stow-entry.mjs +7 -4
- package/dist-lib/lib/custom-model-source.d.ts +0 -15
- package/dist-lib/lib/custom-model-source.js +0 -46
- package/dist-lib/lib/oauth/common.d.ts +0 -38
- package/dist-lib/lib/oauth/common.js +0 -34
- package/dist-lib/schema/helpers.d.ts +0 -33
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { fileKeyOf } from '../local-index.js';
|
|
3
|
+
import { referencedBlobs } from './sweep.js';
|
|
4
|
+
import { CHUNKS_BY_CHAT, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
|
|
5
|
+
export async function reclaimPurgedFiles(mesh) {
|
|
6
|
+
const index = mesh.fileIndex;
|
|
7
|
+
if (!index) {
|
|
8
|
+
return 0;
|
|
9
|
+
}
|
|
10
|
+
const rows = await index.list();
|
|
11
|
+
const purged = await purgedChatIds(mesh, rows);
|
|
12
|
+
const candidates = rows.filter((row) => !!row.chatId && purged.has(row.chatId));
|
|
13
|
+
if (candidates.length === 0) {
|
|
14
|
+
return 0;
|
|
15
|
+
}
|
|
16
|
+
const owner = ownerOf(mesh);
|
|
17
|
+
const referenced = await referencedKeys(mesh, owner);
|
|
18
|
+
const cores = new Map();
|
|
19
|
+
let reclaimed = 0;
|
|
20
|
+
for (const row of candidates) {
|
|
21
|
+
if (mesh.closing)
|
|
22
|
+
break;
|
|
23
|
+
if (referenced.has(keyId(row)))
|
|
24
|
+
continue;
|
|
25
|
+
if (!(await mesh.files.clear(row.blobId, row.attachmentId ?? row.blobId))) {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
await index.forget(row);
|
|
29
|
+
cores.set(b4a.toHex(row.coreKey), row.coreKey);
|
|
30
|
+
reclaimed++;
|
|
31
|
+
}
|
|
32
|
+
if (cores.size > 0 && !mesh.closing) {
|
|
33
|
+
await mesh.files.compact([...cores.values()]);
|
|
34
|
+
}
|
|
35
|
+
return reclaimed;
|
|
36
|
+
}
|
|
37
|
+
export async function chatReferences(mesh, chatId) {
|
|
38
|
+
const references = [];
|
|
39
|
+
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
40
|
+
const snap = mesh.view.snapshot();
|
|
41
|
+
try {
|
|
42
|
+
for await (const chunk of snap.find(CHUNKS_BY_CHAT, inChat)) {
|
|
43
|
+
if (chunk.type === 'attachment' && chunk.attachment) {
|
|
44
|
+
references.push(chunk.attachment.blobId);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
for (const row of await snap.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
|
|
48
|
+
references.push(row.blobId);
|
|
49
|
+
const ofKnowledge = { gte: { knowledgeId: row.id }, lte: { knowledgeId: row.id } };
|
|
50
|
+
for await (const artifact of snap.find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, ofKnowledge)) {
|
|
51
|
+
references.push(artifact.artifactBlobId);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await snap.close();
|
|
57
|
+
}
|
|
58
|
+
return references;
|
|
59
|
+
}
|
|
60
|
+
export async function reclaimReferences(mesh, references) {
|
|
61
|
+
if (references.length === 0)
|
|
62
|
+
return;
|
|
63
|
+
const owner = ownerOf(mesh);
|
|
64
|
+
const referenced = await referencedKeys(mesh, owner);
|
|
65
|
+
const cores = new Map();
|
|
66
|
+
for (const reference of references) {
|
|
67
|
+
if (mesh.closing)
|
|
68
|
+
return;
|
|
69
|
+
const key = fileKeyOf(reference, owner);
|
|
70
|
+
if (!key || referenced.has(keyId(key))) {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (!(await mesh.files.clear(reference, reference))) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
await mesh.fileIndex?.forget(key);
|
|
77
|
+
cores.set(b4a.toHex(key.coreKey), key.coreKey);
|
|
78
|
+
}
|
|
79
|
+
if (cores.size > 0) {
|
|
80
|
+
void mesh.files.compact([...cores.values()]);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
async function purgedChatIds(mesh, rows) {
|
|
84
|
+
const purged = new Set();
|
|
85
|
+
const chatIds = rows.map((row) => row.chatId).filter((id) => !!id);
|
|
86
|
+
for (const chatId of new Set(chatIds)) {
|
|
87
|
+
if (await mesh.view.get(PURGED_CHATS, { chatId })) {
|
|
88
|
+
purged.add(chatId);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return purged;
|
|
92
|
+
}
|
|
93
|
+
function ownerOf(mesh) {
|
|
94
|
+
return { meshKey: mesh.key, ownCoreKey: mesh.blobs.core.key };
|
|
95
|
+
}
|
|
96
|
+
async function referencedKeys(mesh, owner) {
|
|
97
|
+
const keys = new Set();
|
|
98
|
+
for (const reference of await referencedBlobs(mesh)) {
|
|
99
|
+
const key = fileKeyOf(reference, owner);
|
|
100
|
+
if (key) {
|
|
101
|
+
keys.add(keyId(key));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return keys;
|
|
105
|
+
}
|
|
106
|
+
function keyId(key) {
|
|
107
|
+
return `${b4a.toHex(key.coreKey)}/${key.blockOffset}`;
|
|
108
|
+
}
|
|
@@ -3,6 +3,7 @@ import { removeChatAttachments, removeChatKnowledge } from '../../chat.js';
|
|
|
3
3
|
import { listChunks } from '../../chunk.js';
|
|
4
4
|
import { getKnowledge, listKnowledgeArtifactsByKnowledge } from './knowledge.js';
|
|
5
5
|
import { syncAttachmentChunk } from './chunks.js';
|
|
6
|
+
import { CHATS, CHUNK_COUNTERS, KNOWLEDGE, PURGED_CHATS } from '../../collections.js';
|
|
6
7
|
export async function syncKnowledgeTombstone(mesh, row) {
|
|
7
8
|
if (!row.deletedAt)
|
|
8
9
|
return;
|
|
@@ -19,35 +20,21 @@ export async function syncKnowledgeTombstone(mesh, row) {
|
|
|
19
20
|
}
|
|
20
21
|
export async function syncArtifactArrival(mesh, row) {
|
|
21
22
|
const knowledge = await getKnowledge(mesh, row.knowledgeId);
|
|
22
|
-
if (!knowledge?.deletedAt)
|
|
23
|
+
if (!knowledge?.deletedAt) {
|
|
23
24
|
return;
|
|
25
|
+
}
|
|
24
26
|
await mesh.files.clear(row.artifactBlobId, row.id);
|
|
25
27
|
}
|
|
26
28
|
export async function sweepTombstonedBlobs(mesh, deviceId = null) {
|
|
27
29
|
if (deviceId && mesh.writable) {
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if (mesh.closing)
|
|
32
|
-
return;
|
|
33
|
-
if (!chat.deletedAt)
|
|
34
|
-
continue;
|
|
35
|
-
await removeChatAttachments(mesh, deviceId, chat.id, chat.deletedAt);
|
|
36
|
-
await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
catch (error) {
|
|
40
|
-
logger.warn('sync', 'chat-tombstone sweep pass failed - continuing with local reclaims', {
|
|
41
|
-
error
|
|
42
|
-
});
|
|
43
|
-
}
|
|
44
|
-
finally {
|
|
45
|
-
await chats.close();
|
|
30
|
+
const finished = await recascadeTombstonedChats(mesh, deviceId);
|
|
31
|
+
if (!finished) {
|
|
32
|
+
return;
|
|
46
33
|
}
|
|
47
34
|
}
|
|
48
35
|
const knowledge = mesh.view.snapshot();
|
|
49
36
|
try {
|
|
50
|
-
for await (const row of knowledge.find(
|
|
37
|
+
for await (const row of knowledge.find(KNOWLEDGE)) {
|
|
51
38
|
if (mesh.closing)
|
|
52
39
|
return;
|
|
53
40
|
await syncKnowledgeTombstone(mesh, row);
|
|
@@ -58,13 +45,12 @@ export async function sweepTombstonedBlobs(mesh, deviceId = null) {
|
|
|
58
45
|
}
|
|
59
46
|
const counters = mesh.view.snapshot();
|
|
60
47
|
try {
|
|
61
|
-
for await (const counter of counters.find(
|
|
48
|
+
for await (const counter of counters.find(CHUNK_COUNTERS)) {
|
|
62
49
|
if (mesh.closing)
|
|
63
50
|
return;
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
await syncAttachmentChunk(mesh, chunk);
|
|
51
|
+
const finished = await syncChatAttachmentChunks(mesh, counter.chatId);
|
|
52
|
+
if (!finished) {
|
|
53
|
+
return;
|
|
68
54
|
}
|
|
69
55
|
}
|
|
70
56
|
}
|
|
@@ -72,3 +58,35 @@ export async function sweepTombstonedBlobs(mesh, deviceId = null) {
|
|
|
72
58
|
await counters.close();
|
|
73
59
|
}
|
|
74
60
|
}
|
|
61
|
+
async function recascadeTombstonedChats(mesh, deviceId) {
|
|
62
|
+
const chats = mesh.view.snapshot();
|
|
63
|
+
try {
|
|
64
|
+
for await (const chat of chats.find(CHATS)) {
|
|
65
|
+
if (mesh.closing) {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
if (!chat.deletedAt || (await chats.get(PURGED_CHATS, { chatId: chat.id }))) {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
await removeChatAttachments(mesh, chat.id, { deviceId, at: chat.deletedAt });
|
|
72
|
+
await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
logger.warn('sync', 'chat-tombstone sweep pass failed - continuing with local reclaims', {
|
|
77
|
+
error
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
await chats.close();
|
|
82
|
+
}
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
async function syncChatAttachmentChunks(mesh, chatId) {
|
|
86
|
+
for (const chunk of await listChunks(mesh, { chatId, type: 'attachment' })) {
|
|
87
|
+
if (mesh.closing)
|
|
88
|
+
return false;
|
|
89
|
+
await syncAttachmentChunk(mesh, chunk);
|
|
90
|
+
}
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import type { QvacChunkContentAttachment } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
1
|
+
import type { QvacChunkContentAttachment, QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
2
2
|
import type { Mesh } from '../../mesh.ts';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
}): boolean;
|
|
7
|
-
export declare function routeAttachmentToKnowledge(mesh: Mesh, chatId: string, attachment: QvacChunkContentAttachment): Promise<import("../../../../spec/index").QvacKnowledge>;
|
|
3
|
+
import type { AttachmentFile } from '../../document-attachment.ts';
|
|
4
|
+
export declare function isKnowledgeRoutedAttachment(input: AttachmentFile): boolean;
|
|
5
|
+
export declare function routeAttachmentToKnowledge(mesh: Mesh, chatId: string, attachment: QvacChunkContentAttachment): Promise<QvacKnowledge>;
|
|
8
6
|
export declare function hasLiveSource(mesh: Mesh, chatId: string, reference: string): Promise<boolean>;
|
|
9
7
|
interface AttachTurnChunk {
|
|
10
8
|
readonly id: string;
|
|
@@ -22,42 +22,49 @@ export async function routeAttachmentToKnowledge(mesh, chatId, attachment) {
|
|
|
22
22
|
});
|
|
23
23
|
try {
|
|
24
24
|
const chat = await getChat(mesh, chatId);
|
|
25
|
-
if (chat && !chat.deletedAt && !chat.ragEnabled)
|
|
26
|
-
await setChatRagEnabled(mesh, chatId, true);
|
|
25
|
+
if (chat && !chat.deletedAt && !chat.ragEnabled) {
|
|
26
|
+
await setChatRagEnabled(mesh, chatId, { ragEnabled: true });
|
|
27
|
+
}
|
|
27
28
|
}
|
|
28
29
|
catch (err) {
|
|
29
|
-
await removeKnowledge(mesh, { knowledgeId: attachment.id, at: Date.now() }).catch(
|
|
30
|
+
await removeKnowledge(mesh, { knowledgeId: attachment.id, at: Date.now() }).catch(noop);
|
|
30
31
|
throw err;
|
|
31
32
|
}
|
|
32
33
|
return knowledge;
|
|
33
34
|
}
|
|
34
35
|
export async function hasLiveSource(mesh, chatId, reference) {
|
|
35
36
|
const same = (blobId) => compareBlobReferences(blobId, reference, b4a.toHex(mesh.blobs.core.key)) !== false;
|
|
36
|
-
if ((await listKnowledge(mesh, { chatId })).some((row) => same(row.blobId)))
|
|
37
|
+
if ((await listKnowledge(mesh, { chatId })).some((row) => same(row.blobId))) {
|
|
37
38
|
return true;
|
|
39
|
+
}
|
|
38
40
|
return (await listChunks(mesh, { chatId, type: 'attachment' })).some(({ attachment }) => attachment && !attachment.deletedAt && same(attachment.blobId));
|
|
39
41
|
}
|
|
40
42
|
export function attachTurnDocumentIds(chunks) {
|
|
41
43
|
const attachments = [];
|
|
42
44
|
const userMessages = [];
|
|
43
45
|
for (const chunk of chunks) {
|
|
44
|
-
if (chunk.type === 'attachment')
|
|
46
|
+
if (chunk.type === 'attachment') {
|
|
45
47
|
attachments.push(chunk);
|
|
46
|
-
|
|
48
|
+
}
|
|
49
|
+
else if (chunk.userMessage) {
|
|
47
50
|
userMessages.push(chunk);
|
|
51
|
+
}
|
|
48
52
|
}
|
|
49
53
|
const last = userMessages[userMessages.length - 1];
|
|
50
|
-
if (!last)
|
|
54
|
+
if (!last) {
|
|
51
55
|
return [];
|
|
56
|
+
}
|
|
52
57
|
const bound = groupAttachmentsByMessage(attachments, userMessages).get(last.id) ?? [];
|
|
53
58
|
const ids = [];
|
|
54
59
|
for (const chunk of bound) {
|
|
55
60
|
const ref = chunk.attachment;
|
|
56
|
-
if (!ref?.id || !ref.fileName || !ref.mimeType || ref.deletedAt)
|
|
61
|
+
if (!ref?.id || !ref.fileName || !ref.mimeType || ref.deletedAt) {
|
|
57
62
|
continue;
|
|
63
|
+
}
|
|
58
64
|
if (isKnowledgeRoutedAttachment({ fileName: ref.fileName, mimeType: ref.mimeType })) {
|
|
59
65
|
ids.push(ref.id);
|
|
60
66
|
}
|
|
61
67
|
}
|
|
62
68
|
return ids;
|
|
63
69
|
}
|
|
70
|
+
function noop() { }
|
|
@@ -10,7 +10,8 @@ export declare function attachmentRecord(chunk: QvacChunk): FileRecord | null;
|
|
|
10
10
|
export declare function fileRecord(view: HyperDB, id: string, input: FileInput): Promise<FileRecord | null>;
|
|
11
11
|
export declare function fileIds(view: HyperDB, input: FileInput, mediaKind?: string): Promise<string[]>;
|
|
12
12
|
export declare function claimSource(view: HyperDB, id: string): Promise<FileRecord | null>;
|
|
13
|
-
export
|
|
14
|
-
chatId: string;
|
|
15
|
-
fileId: string;
|
|
16
|
-
}
|
|
13
|
+
export interface ChatFileKey {
|
|
14
|
+
readonly chatId: string;
|
|
15
|
+
readonly fileId: string;
|
|
16
|
+
}
|
|
17
|
+
export declare function scopedFile(view: HyperDB, { chatId, fileId }: ChatFileKey): Promise<FileRecord | null>;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { mediaKindOrdinal } from '../../../../schema/
|
|
1
|
+
import { mediaKindOrdinal } from '../../../../schema/media-kind.js';
|
|
2
|
+
import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS, CHUNKS_BY_CHAT, KNOWLEDGE } from '../../collections.js';
|
|
2
3
|
export function attachmentRecord(chunk) {
|
|
3
4
|
const file = chunk.attachment;
|
|
4
|
-
if (chunk.type !== 'attachment' || !file)
|
|
5
|
+
if (chunk.type !== 'attachment' || !file) {
|
|
5
6
|
return null;
|
|
7
|
+
}
|
|
6
8
|
return {
|
|
7
9
|
...file,
|
|
8
10
|
chatId: chunk.chatId,
|
|
@@ -17,48 +19,52 @@ export function attachmentRecord(chunk) {
|
|
|
17
19
|
}
|
|
18
20
|
export async function fileRecord(view, id, input) {
|
|
19
21
|
if (input === 'knowledge') {
|
|
20
|
-
const row = await view.get(
|
|
22
|
+
const row = await view.get(KNOWLEDGE, { id });
|
|
21
23
|
return row && { ...row, kind: input };
|
|
22
24
|
}
|
|
23
|
-
const chunk = await view.get(
|
|
25
|
+
const chunk = await view.get(CHUNKS, { id });
|
|
24
26
|
return chunk && attachmentRecord(chunk);
|
|
25
27
|
}
|
|
26
28
|
export async function fileIds(view, input, mediaKind) {
|
|
27
29
|
if (input === 'knowledge') {
|
|
28
|
-
return (await view.find(
|
|
30
|
+
return (await view.find(KNOWLEDGE).toArray()).map((row) => row.id);
|
|
29
31
|
}
|
|
30
32
|
const ids = [];
|
|
31
33
|
const chunks = mediaKind
|
|
32
|
-
? view.find(
|
|
34
|
+
? view.find(ATTACHMENTS_BY_MEDIA_KIND, {
|
|
33
35
|
gte: { mediaKind: mediaKindOrdinal(mediaKind) },
|
|
34
36
|
lte: { mediaKind: mediaKindOrdinal(mediaKind) }
|
|
35
37
|
})
|
|
36
|
-
: view.find(
|
|
38
|
+
: view.find(CHUNKS);
|
|
37
39
|
for await (const chunk of chunks) {
|
|
38
|
-
if (chunk.type === 'attachment' && chunk.id === chunk.attachment?.id)
|
|
40
|
+
if (chunk.type === 'attachment' && chunk.id === chunk.attachment?.id) {
|
|
39
41
|
ids.push(chunk.id);
|
|
42
|
+
}
|
|
40
43
|
}
|
|
41
44
|
return ids;
|
|
42
45
|
}
|
|
43
46
|
export async function claimSource(view, id) {
|
|
44
47
|
const knowledge = await fileRecord(view, id, 'knowledge');
|
|
45
|
-
if (knowledge)
|
|
48
|
+
if (knowledge) {
|
|
46
49
|
return knowledge.deletedAt ? null : knowledge;
|
|
50
|
+
}
|
|
47
51
|
const file = await fileRecord(view, id, 'attachment');
|
|
48
52
|
return file?.id === id && file.preview && !file.deletedAt ? file : null;
|
|
49
53
|
}
|
|
50
54
|
export async function scopedFile(view, { chatId, fileId }) {
|
|
51
|
-
const chat = await view.get(
|
|
52
|
-
if (chat?.deletedAt)
|
|
55
|
+
const chat = await view.get(CHATS, { id: chatId });
|
|
56
|
+
if (chat?.deletedAt) {
|
|
53
57
|
return null;
|
|
54
|
-
|
|
58
|
+
}
|
|
59
|
+
const direct = await view.get(CHUNKS, { id: fileId });
|
|
55
60
|
if (direct?.chatId === chatId && direct.attachment?.id === fileId) {
|
|
56
61
|
return direct.attachment.deletedAt ? null : attachmentRecord(direct);
|
|
57
62
|
}
|
|
58
63
|
const row = await fileRecord(view, fileId, 'knowledge');
|
|
59
|
-
if (row)
|
|
64
|
+
if (row) {
|
|
60
65
|
return row.chatId === chatId && !row.deletedAt ? row : null;
|
|
61
|
-
|
|
66
|
+
}
|
|
67
|
+
for await (const chunk of view.find(CHUNKS_BY_CHAT, {
|
|
62
68
|
gte: { chatId },
|
|
63
69
|
lte: { chatId }
|
|
64
70
|
})) {
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
const KNOWLEDGE = '@qvac/knowledge';
|
|
3
|
-
const KNOWLEDGE_ARTIFACTS = '@qvac/knowledge-artifacts';
|
|
1
|
+
import { CHUNKS, KNOWLEDGE, KNOWLEDGE_ARTIFACTS } from '../../collections.js';
|
|
4
2
|
export async function referencedBlobs(mesh) {
|
|
5
3
|
const references = [];
|
|
6
4
|
const snapshot = mesh.view.snapshot();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BlobId } from 'hyperblobs';
|
|
2
2
|
import type Hyperblobs from 'hyperblobs';
|
|
3
3
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
4
|
+
import { type NextFrameOptions } from '../../next-frame.ts';
|
|
4
5
|
export interface BlobHost {
|
|
5
6
|
readonly blobs: Hyperblobs;
|
|
6
7
|
openBlobs(key: Buffer): Promise<Hyperblobs>;
|
|
@@ -12,18 +13,18 @@ export type WriteProgress = {
|
|
|
12
13
|
blobId: BlobId;
|
|
13
14
|
byteLength: number;
|
|
14
15
|
};
|
|
15
|
-
export declare function writeBlob(blobs: Hyperblobs, chunks: AsyncIterable<Uint8Array>, options?:
|
|
16
|
-
signal?: AbortSignal;
|
|
17
|
-
inactivityMs?: number;
|
|
18
|
-
}): AsyncGenerator<WriteProgress>;
|
|
16
|
+
export declare function writeBlob(blobs: Hyperblobs, chunks: AsyncIterable<Uint8Array>, options?: NextFrameOptions): AsyncGenerator<WriteProgress>;
|
|
19
17
|
export declare function stringifyBlobId(blobs: Hyperblobs, id: BlobId): string;
|
|
20
|
-
export
|
|
18
|
+
export interface ReadBlobOptions {
|
|
21
19
|
readonly wait?: boolean;
|
|
22
20
|
readonly timeout?: number;
|
|
23
21
|
readonly signal?: AbortSignal;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
22
|
+
}
|
|
23
|
+
export interface BlobReadChunk {
|
|
24
|
+
readonly data: Buffer;
|
|
25
|
+
readonly bytesRead: number;
|
|
26
|
+
readonly byteLength: number;
|
|
27
|
+
}
|
|
28
|
+
export declare function readBlob(blobs: Hyperblobs, blobId: BlobId, options?: ReadBlobOptions): AsyncGenerator<BlobReadChunk>;
|
|
29
29
|
export declare function clearBlob(mesh: BlobHost, blobId: string, id: string): Promise<boolean>;
|
|
30
|
+
export declare function compactBlobCores(mesh: BlobHost, coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|
|
@@ -18,21 +18,12 @@ export async function* writeBlob(blobs, chunks, options = {}) {
|
|
|
18
18
|
const chunk = next.value;
|
|
19
19
|
bytesWritten += chunk.length;
|
|
20
20
|
if (!ws.write(chunk)) {
|
|
21
|
-
|
|
22
|
-
const drained = new Promise((resolve) => {
|
|
23
|
-
onDrain = () => resolve();
|
|
24
|
-
ws.once('drain', onDrain);
|
|
25
|
-
});
|
|
26
|
-
try {
|
|
27
|
-
await Promise.race([drained, done]);
|
|
28
|
-
}
|
|
29
|
-
finally {
|
|
30
|
-
ws.off('drain', onDrain);
|
|
31
|
-
}
|
|
21
|
+
await drained(ws, done);
|
|
32
22
|
}
|
|
33
23
|
next = await nextFrame(it, options);
|
|
34
|
-
if (next.done)
|
|
24
|
+
if (next.done) {
|
|
35
25
|
break;
|
|
26
|
+
}
|
|
36
27
|
yield { bytesWritten };
|
|
37
28
|
}
|
|
38
29
|
ws.end();
|
|
@@ -45,19 +36,32 @@ export async function* writeBlob(blobs, chunks, options = {}) {
|
|
|
45
36
|
void it.return?.().catch(noop);
|
|
46
37
|
ws.end();
|
|
47
38
|
const settled = await done.then(() => true, () => false);
|
|
48
|
-
if (settled && ws.id.blockLength > 0)
|
|
39
|
+
if (settled && ws.id.blockLength > 0) {
|
|
49
40
|
await blobs.clear(ws.id).catch(noop);
|
|
41
|
+
}
|
|
50
42
|
}
|
|
51
43
|
}
|
|
52
44
|
}
|
|
45
|
+
async function drained(ws, done) {
|
|
46
|
+
const drain = Promise.withResolvers();
|
|
47
|
+
const onDrain = () => drain.resolve();
|
|
48
|
+
ws.once('drain', onDrain);
|
|
49
|
+
try {
|
|
50
|
+
await Promise.race([drain.promise, done]);
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
ws.off('drain', onDrain);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
53
56
|
const noop = () => { };
|
|
54
57
|
export function stringifyBlobId(blobs, id) {
|
|
55
58
|
return JSON.stringify({ ...id, coreKey: b4a.toHex(blobs.core.key) });
|
|
56
59
|
}
|
|
57
60
|
export async function* readBlob(blobs, blobId, options = {}) {
|
|
58
61
|
const { signal, ...readOptions } = options;
|
|
59
|
-
if (signal?.aborted)
|
|
62
|
+
if (signal?.aborted) {
|
|
60
63
|
throw new AbortedError();
|
|
64
|
+
}
|
|
61
65
|
const rs = blobs.createReadStream(blobId, readOptions);
|
|
62
66
|
const abort = () => rs.destroy(new AbortedError());
|
|
63
67
|
signal?.addEventListener('abort', abort, { once: true });
|
|
@@ -75,9 +79,11 @@ export async function* readBlob(blobs, blobId, options = {}) {
|
|
|
75
79
|
}
|
|
76
80
|
export async function clearBlob(mesh, blobId, id) {
|
|
77
81
|
try {
|
|
78
|
-
|
|
82
|
+
const reference = parseBlobReference(blobId);
|
|
83
|
+
if (reference.inline) {
|
|
79
84
|
return true;
|
|
80
|
-
|
|
85
|
+
}
|
|
86
|
+
await clearBlobRef(mesh, reference);
|
|
81
87
|
return true;
|
|
82
88
|
}
|
|
83
89
|
catch (err) {
|
|
@@ -85,13 +91,13 @@ export async function clearBlob(mesh, blobId, id) {
|
|
|
85
91
|
return false;
|
|
86
92
|
}
|
|
87
93
|
}
|
|
88
|
-
async function clearBlobRef(mesh,
|
|
89
|
-
const { coreKey,
|
|
94
|
+
async function clearBlobRef(mesh, reference) {
|
|
95
|
+
const { coreKey, blobId: id } = reference;
|
|
90
96
|
if (!coreKey || coreKey === b4a.toHex(mesh.blobs.core.key)) {
|
|
91
97
|
await mesh.blobs.clear(id);
|
|
92
98
|
return;
|
|
93
99
|
}
|
|
94
|
-
const remote = await mesh.openBlobs(
|
|
100
|
+
const remote = await mesh.openBlobs(b4a.from(coreKey, 'hex'));
|
|
95
101
|
try {
|
|
96
102
|
await remote.clear(id);
|
|
97
103
|
}
|
|
@@ -99,3 +105,32 @@ async function clearBlobRef(mesh, blobId) {
|
|
|
99
105
|
await remote.close();
|
|
100
106
|
}
|
|
101
107
|
}
|
|
108
|
+
function compactable(core) {
|
|
109
|
+
return 'compact' in core && typeof core.compact === 'function';
|
|
110
|
+
}
|
|
111
|
+
async function compact(core) {
|
|
112
|
+
if (!compactable(core)) {
|
|
113
|
+
throw new Error('this hypercore cannot compact');
|
|
114
|
+
}
|
|
115
|
+
await core.compact();
|
|
116
|
+
}
|
|
117
|
+
export async function compactBlobCores(mesh, coreKeys) {
|
|
118
|
+
for (const key of coreKeys) {
|
|
119
|
+
try {
|
|
120
|
+
if (b4a.equals(key, mesh.blobs.core.key)) {
|
|
121
|
+
await compact(mesh.blobs.core);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const remote = await mesh.openBlobs(key);
|
|
125
|
+
try {
|
|
126
|
+
await compact(remote.core);
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
await remote.close();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
logger.warn('sync', 'blob compaction failed', { coreKey: b4a.toHex(key), err });
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { Buffer } from 'buffer';
|
|
2
1
|
import EventEmitter from 'bare-events';
|
|
3
2
|
import type Hyperblobs from 'hyperblobs';
|
|
4
3
|
import { stringifyBlobId, writeBlob, type BlobHost } from './blob.ts';
|
|
@@ -23,14 +22,11 @@ export declare class BlobStore extends EventEmitter {
|
|
|
23
22
|
}>;
|
|
24
23
|
reference(id: Parameters<typeof stringifyBlobId>[1]): string;
|
|
25
24
|
inline(bytes: Buffer, mimeType: string): string;
|
|
26
|
-
read(reference: string, options?: Parameters<typeof readBlobReference>[2]): AsyncGenerator<
|
|
27
|
-
data: Buffer;
|
|
28
|
-
bytesRead: number;
|
|
29
|
-
byteLength: number;
|
|
30
|
-
}, any, any>;
|
|
25
|
+
read(reference: string, options?: Parameters<typeof readBlobReference>[2]): AsyncGenerator<import("./blob.ts").BlobReadChunk, any, any>;
|
|
31
26
|
readBytes(reference: string, options?: Parameters<typeof readBlobReferenceBytes>[2]): Promise<Buffer<ArrayBufferLike>>;
|
|
32
27
|
download(reference: string, options: Parameters<typeof downloadBlobReference>[2]): Promise<void>;
|
|
33
28
|
clear(reference: string, id: string): Promise<boolean>;
|
|
29
|
+
compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
|
|
34
30
|
available(reference: string): Promise<boolean>;
|
|
35
31
|
sweep(references: string[], coreLength: number): Promise<void>;
|
|
36
32
|
close(): Promise<void>;
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
|
-
import { Buffer } from 'buffer';
|
|
3
2
|
import EventEmitter from 'bare-events';
|
|
4
3
|
import { sweepUnreferenced } from './sweep.js';
|
|
5
4
|
import { inlineReference } from './inline.js';
|
|
6
|
-
import { clearBlob, stringifyBlobId, writeBlob } from './blob.js';
|
|
5
|
+
import { clearBlob, compactBlobCores, stringifyBlobId, writeBlob } from './blob.js';
|
|
7
6
|
import { parseBlobReference, readBlobReference, readBlobReferenceBytes, downloadBlobReference } from './reference.js';
|
|
8
7
|
export { CoreBlobServer } from './server.js';
|
|
9
8
|
export { compareBlobReferences, isInvalidBlobReferenceError, parseBlobReference, parseBlobReferenceCoreKey } from './reference.js';
|
|
@@ -45,18 +44,24 @@ export class BlobStore extends EventEmitter {
|
|
|
45
44
|
clear(reference, id) {
|
|
46
45
|
return clearBlob(this, reference, id);
|
|
47
46
|
}
|
|
47
|
+
async compact(coreKeys) {
|
|
48
|
+
if (this._closed)
|
|
49
|
+
return;
|
|
50
|
+
await compactBlobCores(this, coreKeys);
|
|
51
|
+
}
|
|
48
52
|
async available(reference) {
|
|
49
53
|
if (this._closed)
|
|
50
54
|
return false;
|
|
51
55
|
try {
|
|
52
56
|
const { blobId, coreKey, inline } = parseBlobReference(reference);
|
|
53
|
-
if (inline)
|
|
57
|
+
if (inline) {
|
|
54
58
|
return true;
|
|
59
|
+
}
|
|
55
60
|
let blobs = this.blobs;
|
|
56
|
-
if (coreKey && coreKey !== b4a.
|
|
61
|
+
if (coreKey && coreKey !== b4a.toHex(blobs.core.key)) {
|
|
57
62
|
let reader = this._readers.get(coreKey);
|
|
58
63
|
if (!reader) {
|
|
59
|
-
reader = this.openBlobs(
|
|
64
|
+
reader = this.openBlobs(b4a.from(coreKey, 'hex')).then((remote) => {
|
|
60
65
|
remote.core.on('download', this._updated);
|
|
61
66
|
return remote;
|
|
62
67
|
});
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export declare const MAX_INLINE_BYTES: number;
|
|
2
2
|
export declare const MAX_REFERENCE_CHARS: number;
|
|
3
|
-
export
|
|
3
|
+
export interface InlinePayload {
|
|
4
4
|
mimeType: string;
|
|
5
5
|
data: string;
|
|
6
|
-
}
|
|
6
|
+
}
|
|
7
7
|
export declare function inlineReference(bytes: Buffer, mimeType: string): string;
|
|
8
8
|
export declare function readInline(record: Record<string, unknown>): InlinePayload | undefined;
|
|
@@ -2,8 +2,9 @@ import b4a from 'b4a';
|
|
|
2
2
|
export const MAX_INLINE_BYTES = 512 * 1024;
|
|
3
3
|
export const MAX_REFERENCE_CHARS = 4 * Math.ceil(MAX_INLINE_BYTES / 3) + 1024;
|
|
4
4
|
export function inlineReference(bytes, mimeType) {
|
|
5
|
-
if (bytes.length > MAX_INLINE_BYTES)
|
|
5
|
+
if (bytes.length > MAX_INLINE_BYTES) {
|
|
6
6
|
throw new Error('inline output exceeds 512 KiB');
|
|
7
|
+
}
|
|
7
8
|
return JSON.stringify({
|
|
8
9
|
blockOffset: 0,
|
|
9
10
|
blockLength: 0,
|
|
@@ -30,9 +31,11 @@ export function readInline(record) {
|
|
|
30
31
|
}
|
|
31
32
|
const { data, mimeType } = inline;
|
|
32
33
|
const bytes = b4a.from(data, 'base64');
|
|
33
|
-
if (bytes.length > MAX_INLINE_BYTES)
|
|
34
|
+
if (bytes.length > MAX_INLINE_BYTES) {
|
|
34
35
|
throw new Error('inline output exceeds 512 KiB');
|
|
35
|
-
|
|
36
|
+
}
|
|
37
|
+
if (b4a.toString(bytes, 'base64') !== data) {
|
|
36
38
|
throw new TypeError('invalid inline output');
|
|
39
|
+
}
|
|
37
40
|
return { mimeType, data };
|
|
38
41
|
}
|