@ixo/oracle-runtime-workers 0.13.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -10
- package/package.json +13 -8
- package/src/artifacts/config.ts +158 -0
- package/src/artifacts/crypto.ts +94 -0
- package/src/artifacts/routes.ts +73 -0
- package/src/artifacts/store.ts +238 -0
- package/src/artifacts/sweep.ts +200 -0
- package/src/artifacts/tool.ts +103 -0
- package/src/artifacts/viewer-page.ts +614 -0
- package/src/attachments/download.ts +168 -45
- package/src/attachments/index.ts +9 -0
- package/src/attachments/magic.ts +33 -0
- package/src/attachments/pipeline.ts +105 -49
- package/src/attachments/sandbox-archive.ts +9 -8
- package/src/attachments/view-cache.ts +142 -0
- package/src/attachments/view.ts +53 -12
- package/src/channels/auth.ts +310 -0
- package/src/channels/contract.ts +246 -0
- package/src/channels/turns.ts +280 -0
- package/src/core/budgeted-llm.ts +223 -0
- package/src/core/capability-router.ts +273 -0
- package/src/core/context-budget.ts +97 -10
- package/src/core/env.ts +111 -0
- package/src/core/execution-profile.ts +17 -0
- package/src/core/index.ts +267 -9
- package/src/core/llm.ts +30 -9
- package/src/core/main-agent-types.ts +41 -1
- package/src/core/main-agent.ts +367 -65
- package/src/core/manifest.ts +34 -0
- package/src/core/meta-tools.ts +128 -18
- package/src/core/middlewares/capability-gate.ts +130 -12
- package/src/core/middlewares/context-guard.ts +55 -24
- package/src/core/middlewares/index.ts +20 -2
- package/src/core/middlewares/page-context.ts +46 -2
- package/src/core/middlewares/return-direct-first.ts +47 -0
- package/src/core/middlewares/summarization.ts +152 -10
- package/src/core/middlewares/tool-execution.ts +323 -0
- package/src/core/middlewares/tool-repetition-guard.ts +202 -35
- package/src/core/middlewares/turn-boundary.ts +162 -0
- package/src/core/openrouter-pricing.ts +155 -22
- package/src/core/prompt-composer.ts +112 -7
- package/src/core/registries.ts +253 -48
- package/src/core/request-admission.ts +159 -0
- package/src/core/runtime-context.ts +107 -8
- package/src/core/state.ts +1 -1
- package/src/core/sub-agent-fallback.ts +21 -2
- package/src/core/subagent-as-tool.ts +78 -43
- package/src/core/tool-access.ts +189 -0
- package/src/core/tool-scheduler.ts +125 -0
- package/src/core/turn-budget.ts +205 -0
- package/src/core/turn-time-note.ts +43 -0
- package/src/core/user-kv.ts +446 -0
- package/src/core/wrap-plugin-tool.ts +20 -0
- package/src/delivery/plan.ts +278 -0
- package/src/delivery/profile.ts +161 -0
- package/src/delivery/prompt.ts +27 -0
- package/src/delivery/schema.ts +61 -0
- package/src/delivery/shaper.ts +389 -0
- package/src/delivery/types.ts +84 -0
- package/src/do/ambient.ts +51 -15
- package/src/do/contracts.ts +170 -6
- package/src/do/idle-eviction.ts +38 -0
- package/src/do/matrix-turn-ledger.ts +24 -5
- package/src/do/result-store-test-do.ts +137 -3
- package/src/do/result-store.ts +239 -58
- package/src/do/room-mirror.ts +29 -0
- package/src/do/run-buffer.ts +64 -9
- package/src/do/run-coordinator.ts +451 -76
- package/src/do/run-request.ts +2 -0
- package/src/do/run-store-test-do.ts +301 -1
- package/src/do/run-store.ts +385 -43
- package/src/do/secrets-adapter.ts +22 -3
- package/src/do/sse-stream.ts +60 -9
- package/src/do/transcript.ts +48 -5
- package/src/do/turn-body.ts +105 -5
- package/src/do/ucan-service.ts +204 -34
- package/src/do/user-oracle-do.ts +2003 -274
- package/src/feedback/config.ts +127 -0
- package/src/feedback/contract.ts +148 -0
- package/src/feedback/linear-sink.ts +285 -0
- package/src/feedback/privacy.ts +150 -0
- package/src/feedback/reservation.ts +298 -0
- package/src/feedback/submit.ts +247 -0
- package/src/index.ts +138 -6
- package/src/llm/byo-adapter.ts +27 -6
- package/src/llm/byo-catalog.ts +13 -0
- package/src/llm/byo-model-fallback.ts +248 -0
- package/src/llm/byo-service.ts +45 -8
- package/src/llm/provider-error.ts +139 -2
- package/src/matrix/gateway-do.ts +504 -108
- package/src/matrix/group-chat.ts +66 -23
- package/src/matrix/inbox-store.ts +16 -1
- package/src/matrix/ingest.ts +79 -14
- package/src/matrix/media-deadline.ts +130 -0
- package/src/matrix/replay-format.ts +5 -3
- package/src/matrix/reply-parts.ts +92 -0
- package/src/matrix/room-alias.ts +96 -0
- package/src/matrix/user-homeserver.ts +166 -7
- package/src/owner-store/ixo-vfs-store.ts +170 -43
- package/src/owner-store/migrating-store.ts +31 -15
- package/src/owner-store/types.ts +18 -3
- package/src/plugin-api/index.ts +43 -2
- package/src/plugin-api/oracle-plugin.ts +35 -0
- package/src/plugin-api/request-admission.ts +41 -0
- package/src/plugin-api/tool-helper.ts +25 -1
- package/src/plugin-api/tool-plane.ts +92 -0
- package/src/plugin-api/types.ts +194 -6
- package/src/plugins/agui/agui.plugin.ts +31 -16
- package/src/plugins/composio/composio-tools.ts +57 -6
- package/src/plugins/composio/composio-ucan.ts +3 -2
- package/src/plugins/composio/composio.plugin.ts +2 -0
- package/src/plugins/delegated-capabilities.ts +25 -0
- package/src/plugins/editor/content-session.ts +173 -83
- package/src/plugins/editor/content-tools.ts +76 -17
- package/src/plugins/editor/editor-mx.ts +28 -9
- package/src/plugins/editor/editor.plugin.ts +3 -1
- package/src/plugins/editor/failures.ts +28 -7
- package/src/plugins/editor/prompts.ts +2 -1
- package/src/plugins/editor/provider.ts +553 -40
- package/src/plugins/editor/standalone-editor-tool.ts +61 -18
- package/src/plugins/firecrawl/firecrawl-tools.ts +13 -5
- package/src/plugins/flows/action-metadata.ts +111 -11
- package/src/plugins/flows/actions.ts +18 -0
- package/src/plugins/flows/decision-context.ts +31 -0
- package/src/plugins/flows/edit.ts +223 -52
- package/src/plugins/flows/errors.ts +22 -1
- package/src/plugins/flows/flow-doc.ts +92 -9
- package/src/plugins/flows/flows.plugin.ts +11 -5
- package/src/plugins/flows/forms.ts +34 -0
- package/src/plugins/flows/index.ts +2 -0
- package/src/plugins/flows/input-policy.ts +140 -0
- package/src/plugins/flows/prompts.ts +14 -11
- package/src/plugins/flows/read.ts +49 -9
- package/src/plugins/flows/ref-syntax.ts +129 -0
- package/src/plugins/flows/test-homeserver.ts +245 -0
- package/src/plugins/flows/tools/authoring.ts +216 -112
- package/src/plugins/flows/tools/discovery.ts +4 -1
- package/src/plugins/flows/tools/forms.ts +21 -2
- package/src/plugins/flows/tools/settings.ts +34 -13
- package/src/plugins/flows/translator.ts +13 -10
- package/src/plugins/flows/types.ts +167 -42
- package/src/plugins/index.ts +11 -1
- package/src/plugins/ixo-transaction/chain-ids.ts +43 -0
- package/src/plugins/ixo-transaction/index.ts +2 -0
- package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +119 -0
- package/src/plugins/ixo-transaction/receipts.ts +99 -0
- package/src/plugins/ixo-transaction/tools.ts +329 -0
- package/src/plugins/ixo-transaction/wallet-signing.ts +202 -0
- package/src/plugins/mcp-tool-adapter.ts +23 -2
- package/src/plugins/memory/memory-tools.ts +7 -2
- package/src/plugins/memory/memory-ucan.ts +3 -2
- package/src/plugins/memory/memory.plugin.ts +2 -0
- package/src/plugins/pod-creator/blueprint-store.ts +163 -0
- package/src/plugins/pod-creator/blueprint-types.ts +61 -0
- package/src/plugins/pod-creator/bounded-map.ts +87 -0
- package/src/plugins/pod-creator/capsule-content-client.ts +285 -0
- package/src/plugins/pod-creator/chain-gateway.ts +110 -0
- package/src/plugins/pod-creator/config.ts +85 -0
- package/src/plugins/pod-creator/create-session-store.ts +180 -0
- package/src/plugins/pod-creator/create-tools.ts +538 -0
- package/src/plugins/pod-creator/design-pod-roles.ts +111 -0
- package/src/plugins/pod-creator/index.ts +65 -0
- package/src/plugins/pod-creator/orchestration-tools.ts +172 -0
- package/src/plugins/pod-creator/pod-creator.plugin.ts +190 -0
- package/src/plugins/pod-creator/stage.ts +201 -0
- package/src/plugins/pod-creator/sub-agents.ts +189 -0
- package/src/plugins/portal/action-log.ts +21 -1
- package/src/plugins/portal/declared-tools.ts +90 -0
- package/src/plugins/portal/portal.plugin.ts +42 -16
- package/src/plugins/sandbox/sandbox-bridge.ts +79 -10
- package/src/plugins/sandbox/sandbox-mcp.ts +3 -2
- package/src/plugins/sandbox/sandbox-write-blob.ts +11 -1
- package/src/plugins/sandbox/sandbox.plugin.ts +8 -16
- package/src/plugins/tasks/manifest.ts +1 -1
- package/src/plugins/tasks/middleware.ts +20 -7
- package/src/plugins/tasks/tasks-tools.ts +30 -3
- package/src/plugins/tasks/tasks.plugin.ts +64 -11
- package/src/plugins/user-preferences/schema.ts +12 -1
- package/src/plugins/user-preferences/user-preferences-store.ts +38 -13
- package/src/plugins/user-preferences/user-preferences-tool.ts +3 -2
- package/src/plugins/vfs/vfs-client.ts +250 -71
- package/src/plugins/vfs/vfs-content.ts +30 -11
- package/src/plugins/vfs/vfs-errors.ts +22 -0
- package/src/plugins/vfs/vfs-sandbox-tools.ts +12 -5
- package/src/plugins/vfs/vfs-tools.ts +60 -20
- package/src/realtime/frontend-call-registry.ts +228 -81
- package/src/realtime/realtime-endpoint.ts +215 -44
- package/src/realtime/session-socket-hub.ts +47 -2
- package/src/secrets/secrets-service.ts +9 -3
- package/src/shell/app.ts +562 -82
- package/src/shell/auth.ts +131 -37
- package/src/shell/turn-body-cap.ts +11 -13
- package/src/sqlite/blob-compactor.ts +61 -28
- package/src/sqlite/database.ts +156 -90
- package/src/sqlite/do-vfs.ts +416 -129
- package/src/sqlite/page-tier.ts +133 -40
- package/src/sqlite/serialization.ts +40 -0
- package/src/sqlite/sessions-store.ts +10 -2
- package/src/sqlite/sqlite-saver.ts +139 -19
- package/src/sqlite/user-kv-store.ts +317 -0
- package/src/tasks/schedule.ts +201 -12
- package/src/tasks/scheduler.ts +586 -98
- package/src/tasks/spec.ts +67 -3
- package/src/tasks/store.ts +292 -27
- package/src/tasks/topic-deliverables.ts +70 -0
|
@@ -64,16 +64,22 @@ const IID_SERVICES_QUERY = `
|
|
|
64
64
|
}
|
|
65
65
|
`;
|
|
66
66
|
|
|
67
|
+
/** Upper bound on one Blocksync lookup unless the caller passes its own signal. */
|
|
68
|
+
export const BLOCKSYNC_LOOKUP_TIMEOUT_MS = 5_000;
|
|
69
|
+
|
|
67
70
|
/**
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
+
* What Blocksync knows about a DID's homeserver: `indexed: false` when it
|
|
72
|
+
* has no record of the DID (not indexed yet, or unknown), otherwise the
|
|
73
|
+
* server its document names (null for none). Throws on transport failure —
|
|
74
|
+
* including no answer within `BLOCKSYNC_LOOKUP_TIMEOUT_MS`, or `signal`
|
|
75
|
+
* aborting first — so callers can decide whether to fall back.
|
|
71
76
|
*/
|
|
72
|
-
export async function
|
|
77
|
+
export async function fetchUserMatrixHomeServer(
|
|
73
78
|
blocksyncGraphqlUrl: string,
|
|
74
79
|
userDid: string,
|
|
75
80
|
fetchImpl: typeof fetch = fetch,
|
|
76
|
-
|
|
81
|
+
signal: AbortSignal = AbortSignal.timeout(BLOCKSYNC_LOOKUP_TIMEOUT_MS),
|
|
82
|
+
): Promise<{ indexed: false } | { indexed: true; serverName: string | null }> {
|
|
77
83
|
const res = await fetchImpl(blocksyncGraphqlUrl, {
|
|
78
84
|
method: 'POST',
|
|
79
85
|
headers: { 'content-type': 'application/json' },
|
|
@@ -81,6 +87,7 @@ export async function fetchUserMatrixServerName(
|
|
|
81
87
|
query: IID_SERVICES_QUERY,
|
|
82
88
|
variables: { id: userDid },
|
|
83
89
|
}),
|
|
90
|
+
signal,
|
|
84
91
|
});
|
|
85
92
|
if (!res.ok) {
|
|
86
93
|
throw new Error(
|
|
@@ -98,9 +105,161 @@ export async function fetchUserMatrixServerName(
|
|
|
98
105
|
);
|
|
99
106
|
}
|
|
100
107
|
const node = body.data?.iids?.nodes?.[0];
|
|
101
|
-
if (!node) return
|
|
108
|
+
if (!node) return { indexed: false };
|
|
102
109
|
const services: DidService[] = Array.isArray(node.service)
|
|
103
110
|
? node.service
|
|
104
111
|
: [];
|
|
105
|
-
return matrixServerNameFromServices(services);
|
|
112
|
+
return { indexed: true, serverName: matrixServerNameFromServices(services) };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Resolve a user's Matrix server name from Blocksync. Returns null when the
|
|
117
|
+
* DID is unknown or carries no MatrixHomeServer service; throws like
|
|
118
|
+
* `fetchUserMatrixHomeServer`.
|
|
119
|
+
*/
|
|
120
|
+
export async function fetchUserMatrixServerName(
|
|
121
|
+
blocksyncGraphqlUrl: string,
|
|
122
|
+
userDid: string,
|
|
123
|
+
fetchImpl: typeof fetch = fetch,
|
|
124
|
+
signal?: AbortSignal,
|
|
125
|
+
): Promise<string | null> {
|
|
126
|
+
const answer = await fetchUserMatrixHomeServer(
|
|
127
|
+
blocksyncGraphqlUrl,
|
|
128
|
+
userDid,
|
|
129
|
+
fetchImpl,
|
|
130
|
+
signal,
|
|
131
|
+
);
|
|
132
|
+
return answer.indexed ? answer.serverName : null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* How long "the DID document names no homeserver" is served from the cache.
|
|
137
|
+
* Short: a user who registers one (or whose registration Blocksync indexes
|
|
138
|
+
* late) must not be held to the default for the full TTL.
|
|
139
|
+
*/
|
|
140
|
+
export const UNREGISTERED_CACHE_TTL_MS = 5 * 60_000;
|
|
141
|
+
|
|
142
|
+
/** A resolved server name as the gateway caches it (`hs:<did>` in its storage). */
|
|
143
|
+
export interface CachedUserServerName {
|
|
144
|
+
serverName: string;
|
|
145
|
+
/** When Blocksync answered (epoch ms). */
|
|
146
|
+
at: number;
|
|
147
|
+
/**
|
|
148
|
+
* Blocksync answered that the DID document names no homeserver;
|
|
149
|
+
* `serverName` is then the default it stood for when cached. Served for
|
|
150
|
+
* `UNREGISTERED_CACHE_TTL_MS` only.
|
|
151
|
+
*/
|
|
152
|
+
unregistered?: true;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Where a `lookupUserServerName` answer came from:
|
|
157
|
+
* - `cache` — a cached Blocksync answer younger than the TTL;
|
|
158
|
+
* - `blocksync` — Blocksync answered just now (and the answer was cached);
|
|
159
|
+
* - `stale` — Blocksync failed and an EXPIRED cached answer was used;
|
|
160
|
+
* - `unregistered` — the DID document names no homeserver, or Blocksync has
|
|
161
|
+
* no record of the DID yet: the default;
|
|
162
|
+
* - `unconfigured` — no Blocksync URL is configured: the default.
|
|
163
|
+
*/
|
|
164
|
+
export type UserServerNameSource =
|
|
165
|
+
| 'cache'
|
|
166
|
+
| 'blocksync'
|
|
167
|
+
| 'stale'
|
|
168
|
+
| 'unregistered'
|
|
169
|
+
| 'unconfigured';
|
|
170
|
+
|
|
171
|
+
export interface UserServerNameLookup {
|
|
172
|
+
serverName: string;
|
|
173
|
+
source: UserServerNameSource;
|
|
174
|
+
/** The Blocksync failure a `stale` answer stands in for. */
|
|
175
|
+
error?: unknown;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export interface UserServerNameLookupDeps {
|
|
179
|
+
blocksyncGraphqlUrl: string | undefined;
|
|
180
|
+
/** The answer for a DID that names no homeserver (the oracle's own server). */
|
|
181
|
+
defaultServerName: string;
|
|
182
|
+
/** How long a Blocksync answer is served from the cache without asking again. */
|
|
183
|
+
ttlMs: number;
|
|
184
|
+
readCache(userDid: string): Promise<CachedUserServerName | undefined>;
|
|
185
|
+
writeCache(userDid: string, entry: CachedUserServerName): Promise<void>;
|
|
186
|
+
now?: () => number;
|
|
187
|
+
fetchImpl?: typeof fetch;
|
|
188
|
+
/** Ends the Blocksync request early; without one it is bounded by `BLOCKSYNC_LOOKUP_TIMEOUT_MS`. */
|
|
189
|
+
signal?: AbortSignal;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Blocksync could not be asked and nothing was ever cached: there is no answer to give. */
|
|
193
|
+
export class UserServerNameUnavailableError extends Error {
|
|
194
|
+
constructor(userDid: string, cause: unknown) {
|
|
195
|
+
super(
|
|
196
|
+
`could not resolve ${userDid}'s homeserver from Blocksync and nothing is cached`,
|
|
197
|
+
{ cause },
|
|
198
|
+
);
|
|
199
|
+
this.name = 'UserServerNameUnavailableError';
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* The server a user's DID document registers, through a TTL cache over
|
|
205
|
+
* Blocksync. A failed Blocksync request is not an answer: it is served from
|
|
206
|
+
* the cached entry even when that has expired — a homeserver registration
|
|
207
|
+
* rarely changes, while treating the failure as "the oracle's own server"
|
|
208
|
+
* would make every user registered elsewhere look foreign for as long as
|
|
209
|
+
* Blocksync is down. With nothing cached it throws
|
|
210
|
+
* `UserServerNameUnavailableError`, and each caller decides: the sender check
|
|
211
|
+
* gives no verdict, the room-alias lookup falls back to the oracle's server.
|
|
212
|
+
* "Registers no homeserver" is an answer too, cached for the short
|
|
213
|
+
* `UNREGISTERED_CACHE_TTL_MS` (a DID without the service is a normal case,
|
|
214
|
+
* not one to ask about on every message). A DID Blocksync has no record of
|
|
215
|
+
* gets the default without anything being cached: it may be indexed any
|
|
216
|
+
* moment.
|
|
217
|
+
*/
|
|
218
|
+
export async function lookupUserServerName(
|
|
219
|
+
userDid: string,
|
|
220
|
+
deps: UserServerNameLookupDeps,
|
|
221
|
+
): Promise<UserServerNameLookup> {
|
|
222
|
+
const now = deps.now ?? Date.now;
|
|
223
|
+
const cached = await deps.readCache(userDid);
|
|
224
|
+
if (
|
|
225
|
+
cached &&
|
|
226
|
+
now() - cached.at <
|
|
227
|
+
(cached.unregistered ? UNREGISTERED_CACHE_TTL_MS : deps.ttlMs)
|
|
228
|
+
)
|
|
229
|
+
return cached.unregistered
|
|
230
|
+
? { serverName: deps.defaultServerName, source: 'unregistered' }
|
|
231
|
+
: { serverName: cached.serverName, source: 'cache' };
|
|
232
|
+
if (!deps.blocksyncGraphqlUrl)
|
|
233
|
+
return { serverName: deps.defaultServerName, source: 'unconfigured' };
|
|
234
|
+
let answer: Awaited<ReturnType<typeof fetchUserMatrixHomeServer>>;
|
|
235
|
+
try {
|
|
236
|
+
answer = await fetchUserMatrixHomeServer(
|
|
237
|
+
deps.blocksyncGraphqlUrl,
|
|
238
|
+
userDid,
|
|
239
|
+
deps.fetchImpl,
|
|
240
|
+
deps.signal,
|
|
241
|
+
);
|
|
242
|
+
} catch (err) {
|
|
243
|
+
if (cached)
|
|
244
|
+
return {
|
|
245
|
+
serverName: cached.unregistered
|
|
246
|
+
? deps.defaultServerName
|
|
247
|
+
: cached.serverName,
|
|
248
|
+
source: 'stale',
|
|
249
|
+
error: err,
|
|
250
|
+
};
|
|
251
|
+
throw new UserServerNameUnavailableError(userDid, err);
|
|
252
|
+
}
|
|
253
|
+
if (!answer.indexed)
|
|
254
|
+
return { serverName: deps.defaultServerName, source: 'unregistered' };
|
|
255
|
+
if (!answer.serverName) {
|
|
256
|
+
await deps.writeCache(userDid, {
|
|
257
|
+
serverName: deps.defaultServerName,
|
|
258
|
+
at: now(),
|
|
259
|
+
unregistered: true,
|
|
260
|
+
});
|
|
261
|
+
return { serverName: deps.defaultServerName, source: 'unregistered' };
|
|
262
|
+
}
|
|
263
|
+
await deps.writeCache(userDid, { serverName: answer.serverName, at: now() });
|
|
264
|
+
return { serverName: answer.serverName, source: 'blocksync' };
|
|
106
265
|
}
|
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
* `POST /upload` + `PATCH /upload/:id` (tus, larger files), `GET
|
|
25
25
|
* /files/:id/content` (bytes, streamed), `POST /batch/delete`, `POST
|
|
26
26
|
* /batch/move`. The file is never `PUT` (a new VERSION, capped at 50 by the
|
|
27
|
-
* VFS): every flush uploads to a temp path and swaps it in — see
|
|
27
|
+
* VFS): every flush uploads to a temp path and swaps it in with moves — see
|
|
28
|
+
* `save()`. The VFS refuses to move onto an occupied path (per-item 409
|
|
29
|
+
* "Destination occupied"), so the old file is moved aside first.
|
|
28
30
|
*
|
|
29
31
|
* Two VFS semantics this store must respect:
|
|
30
32
|
* - `/.oracles` is a dot-folder, so the file is HIDDEN by the VFS dotfile
|
|
@@ -38,7 +40,8 @@
|
|
|
38
40
|
* `stat()` lists under the short literal `/.oracles` prefix and matches
|
|
39
41
|
* the exact path client-side.
|
|
40
42
|
*/
|
|
41
|
-
import { abilityCovers
|
|
43
|
+
import { abilityCovers } from '../core/runtime-context';
|
|
44
|
+
import { type WorkersUcanService } from '../do/ucan-service';
|
|
42
45
|
import { isNetworkError, withRetry } from './retry';
|
|
43
46
|
import { tusUpload } from './tus-upload';
|
|
44
47
|
import {
|
|
@@ -135,6 +138,12 @@ export class VfsRequestError extends Error {
|
|
|
135
138
|
* once complete; anything still carrying the infix is a crashed flush.
|
|
136
139
|
*/
|
|
137
140
|
export const TEMP_PATH_INFIX = '.uploading-';
|
|
141
|
+
/**
|
|
142
|
+
* The previous file is moved to `<path>.replaced-<ts>` while the new one
|
|
143
|
+
* moves into place, and deleted once it has. Anything still carrying the
|
|
144
|
+
* infix is a complete earlier copy a crashed or failed swap left behind.
|
|
145
|
+
*/
|
|
146
|
+
export const ASIDE_PATH_INFIX = '.replaced-';
|
|
138
147
|
/** tus part size: a multiple of 64 KiB, ≥ 5 MiB (the VFS minimum) — and the peak upload buffer. */
|
|
139
148
|
export const TUS_PART_BYTES = 5 * 1024 * 1024;
|
|
140
149
|
/** Gzipped files up to one part go through the single-request `POST /files`. */
|
|
@@ -205,7 +214,7 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
205
214
|
}
|
|
206
215
|
|
|
207
216
|
async head(): Promise<{ etag: string } | null> {
|
|
208
|
-
const stat = await this.
|
|
217
|
+
const stat = await this.ownerCopy();
|
|
209
218
|
return stat ? { etag: this.etagOf(stat) } : null;
|
|
210
219
|
}
|
|
211
220
|
|
|
@@ -216,7 +225,7 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
216
225
|
* on the first one.
|
|
217
226
|
*/
|
|
218
227
|
async load(): Promise<OwnerCopy | null> {
|
|
219
|
-
const stat = await this.
|
|
228
|
+
const stat = await this.ownerCopy();
|
|
220
229
|
if (!stat) return null;
|
|
221
230
|
const res = await this.request(
|
|
222
231
|
'fs/read',
|
|
@@ -231,24 +240,29 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
231
240
|
}
|
|
232
241
|
|
|
233
242
|
/**
|
|
234
|
-
*
|
|
243
|
+
* Replace the file by moves, never a new version, and never leave the
|
|
244
|
+
* user without a complete upstream copy:
|
|
235
245
|
*
|
|
236
|
-
* 1.
|
|
237
|
-
* 2. take the gzipped length from `hints` (the object measured it in the
|
|
246
|
+
* 1. take the gzipped length from `hints` (the object measured it in the
|
|
238
247
|
* same pass as its checksum) or, without a hint, count it in a pass
|
|
239
248
|
* of its own (the VFS needs `Upload-Length` at session creation —
|
|
240
249
|
* `Upload-Defer-Length` is not supported — and gzip output is
|
|
241
250
|
* deterministic for identical input);
|
|
242
|
-
*
|
|
251
|
+
* 2. upload gzip → a TEMP path next to the real one: one `POST /files`
|
|
243
252
|
* when it fits in a single part, else a tus session in 5 MiB parts
|
|
244
253
|
* (`tusUpload`, resumable per part);
|
|
245
|
-
*
|
|
246
|
-
* file
|
|
254
|
+
* 3. move the old file aside (`.replaced-<ts>`), then the temp file
|
|
255
|
+
* into place; when that move fails for good, the old file is moved
|
|
256
|
+
* back before the error is thrown;
|
|
257
|
+
* 4. only once the new file is at the path: delete the old copy and any
|
|
258
|
+
* leftovers of earlier flushes (`.uploading-` / `.replaced-` files).
|
|
259
|
+
* A leftover may be the only complete copy (a swap that failed after
|
|
260
|
+
* the old file was moved aside), so none is deleted earlier.
|
|
247
261
|
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
* overhead, whatever the file size.
|
|
262
|
+
* The path is empty only between the two moves. Every request is retried
|
|
263
|
+
* with backoff; a step that still fails throws, the working copy stays
|
|
264
|
+
* dirty and the next tick starts over. Peak memory is one part (5 MiB)
|
|
265
|
+
* plus fixed overhead, whatever the file size.
|
|
252
266
|
*/
|
|
253
267
|
async save(
|
|
254
268
|
snapshot: FileSnapshot,
|
|
@@ -256,20 +270,12 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
256
270
|
): Promise<SaveResult> {
|
|
257
271
|
const listing = await this.listOracleFiles();
|
|
258
272
|
const existing = listing.filter((f) => f.path === this.path);
|
|
259
|
-
const
|
|
260
|
-
f.path.startsWith(`${this.path}${TEMP_PATH_INFIX}`),
|
|
261
|
-
);
|
|
262
|
-
if (stale.length > 0) {
|
|
263
|
-
await this.batchDelete(stale.map((f) => f.id)).catch((err: unknown) => {
|
|
264
|
-
console.warn(
|
|
265
|
-
`[owner-store] could not delete ${stale.length} stale temp upload(s): ${describe(err)}`,
|
|
266
|
-
);
|
|
267
|
-
});
|
|
268
|
-
}
|
|
273
|
+
const leftovers = listing.filter((f) => this.isLeftover(f.path));
|
|
269
274
|
|
|
270
275
|
const gzLength =
|
|
271
276
|
hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
|
|
272
|
-
const
|
|
277
|
+
const stamp = Date.now().toString(36);
|
|
278
|
+
const tempPath = `${this.path}${TEMP_PATH_INFIX}${stamp}`;
|
|
273
279
|
const uploaded =
|
|
274
280
|
gzLength <= SINGLE_SHOT_MAX_BYTES
|
|
275
281
|
? await this.uploadSingleShot(tempPath, snapshot)
|
|
@@ -281,27 +287,100 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
281
287
|
);
|
|
282
288
|
}
|
|
283
289
|
|
|
284
|
-
|
|
285
|
-
|
|
290
|
+
const aside: VfsFileStat[] = [];
|
|
291
|
+
for (const [i, file] of existing.entries()) {
|
|
292
|
+
const asidePath = `${this.path}${ASIDE_PATH_INFIX}${stamp}${i > 0 ? `-${i}` : ''}`;
|
|
293
|
+
await this.moveWithRetry(
|
|
294
|
+
file.id,
|
|
295
|
+
asidePath,
|
|
296
|
+
'moving the previous state file aside',
|
|
297
|
+
);
|
|
298
|
+
aside.push({ ...file, path: asidePath });
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
await this.moveIntoPlace(tempId, stamp, aside);
|
|
302
|
+
} catch (error) {
|
|
303
|
+
await this.restoreAside(aside);
|
|
304
|
+
throw error;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
const stale = [...leftovers, ...aside].filter((f) => f.id !== tempId);
|
|
308
|
+
if (stale.length > 0) {
|
|
309
|
+
await withRetry(() => this.batchDelete(stale.map((f) => f.id)), {
|
|
286
310
|
isRetryable: isRetryableRequest,
|
|
287
311
|
...this.retryOptions(),
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
312
|
+
}).catch((err: unknown) => {
|
|
313
|
+
console.warn(
|
|
314
|
+
`[owner-store] could not delete ${stale.length} earlier cop${stale.length === 1 ? 'y' : 'ies'} of ${this.path} (the next flush retries): ${describe(err)}`,
|
|
315
|
+
);
|
|
292
316
|
});
|
|
293
317
|
}
|
|
318
|
+
const etag =
|
|
319
|
+
uploaded.contentHash ??
|
|
320
|
+
(await this.statPath(this.path))?.checksum ??
|
|
321
|
+
tempId;
|
|
322
|
+
return { etag, bytes: gzLength };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* The file at the path or, when a swap failed after moving the previous
|
|
327
|
+
* file aside (and could not put it back), the newest set-aside copy — the
|
|
328
|
+
* last complete file this store landed. Never an `.uploading-` temp: its
|
|
329
|
+
* swap did not finish, so it is not the owner copy. `save()` removes the
|
|
330
|
+
* set-aside copies only after its own file is at the path.
|
|
331
|
+
*/
|
|
332
|
+
private async ownerCopy(): Promise<VfsFileStat | null> {
|
|
333
|
+
const listing = await this.listOracleFiles();
|
|
334
|
+
const exact = listing.find((f) => f.path === this.path);
|
|
335
|
+
if (exact) return exact;
|
|
336
|
+
const prefix = `${this.path}${ASIDE_PATH_INFIX}`;
|
|
337
|
+
let newest: { file: VfsFileStat; at: number } | null = null;
|
|
338
|
+
for (const file of listing) {
|
|
339
|
+
if (!file.path.startsWith(prefix)) continue;
|
|
340
|
+
// `<stamp>` or `<stamp>-<n>`: the stamp is the save's base-36 time.
|
|
341
|
+
const at = Number.parseInt(
|
|
342
|
+
file.path.slice(prefix.length).split('-')[0] ?? '',
|
|
343
|
+
36,
|
|
344
|
+
);
|
|
345
|
+
if (!Number.isFinite(at)) continue;
|
|
346
|
+
if (newest === null || at > newest.at) newest = { file, at };
|
|
347
|
+
}
|
|
348
|
+
return newest?.file ?? null;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** A crashed or failed flush's temp upload or set-aside copy of this file. */
|
|
352
|
+
private isLeftover(path: string): boolean {
|
|
353
|
+
return (
|
|
354
|
+
path.startsWith(`${this.path}${TEMP_PATH_INFIX}`) ||
|
|
355
|
+
path.startsWith(`${this.path}${ASIDE_PATH_INFIX}`)
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Move the uploaded temp file to the real path. A "Destination occupied"
|
|
361
|
+
* answer means something still sits there (a concurrent or crashed
|
|
362
|
+
* flush): it is moved aside too (`aside` collects it, so a failure
|
|
363
|
+
* restores it and success deletes it) and the move retried.
|
|
364
|
+
*/
|
|
365
|
+
private async moveIntoPlace(
|
|
366
|
+
tempId: string,
|
|
367
|
+
stamp: string,
|
|
368
|
+
aside: VfsFileStat[],
|
|
369
|
+
): Promise<void> {
|
|
294
370
|
await withRetry(
|
|
295
371
|
async () => {
|
|
296
372
|
try {
|
|
297
373
|
await this.move(tempId, this.path);
|
|
298
374
|
} catch (err) {
|
|
299
|
-
// Someone (a crashed earlier flush) still occupies the path: clear it
|
|
300
|
-
// and let the retry move again.
|
|
301
375
|
if (err instanceof VfsRequestError && err.status === 409) {
|
|
302
376
|
const occupant = await this.statPath(this.path);
|
|
303
|
-
|
|
304
|
-
|
|
377
|
+
// An earlier attempt committed but its answer was lost.
|
|
378
|
+
if (occupant?.id === tempId) return;
|
|
379
|
+
if (occupant) {
|
|
380
|
+
const asidePath = `${this.path}${ASIDE_PATH_INFIX}${stamp}-o${aside.length}`;
|
|
381
|
+
await this.move(occupant.id, asidePath);
|
|
382
|
+
aside.push({ ...occupant, path: asidePath });
|
|
383
|
+
}
|
|
305
384
|
}
|
|
306
385
|
throw err;
|
|
307
386
|
}
|
|
@@ -317,11 +396,61 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
317
396
|
),
|
|
318
397
|
},
|
|
319
398
|
);
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* The new file could not be moved into place: put the previous one back
|
|
403
|
+
* so the path keeps a complete copy. Best effort — if this fails too the
|
|
404
|
+
* set-aside copy stays where it is, and the next flush (which only
|
|
405
|
+
* deletes leftovers after its own file landed) cleans it up.
|
|
406
|
+
*/
|
|
407
|
+
private async restoreAside(aside: VfsFileStat[]): Promise<void> {
|
|
408
|
+
const previous = aside[0];
|
|
409
|
+
if (previous === undefined) return;
|
|
410
|
+
try {
|
|
411
|
+
await this.moveWithRetry(
|
|
412
|
+
previous.id,
|
|
413
|
+
this.path,
|
|
414
|
+
'restoring the previous state file',
|
|
415
|
+
);
|
|
416
|
+
} catch (err) {
|
|
417
|
+
console.error(
|
|
418
|
+
`[owner-store] ${this.path} is empty: the previous copy stays at ${previous.path} until the next flush (${describe(err)})`,
|
|
419
|
+
);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
private async moveWithRetry(
|
|
424
|
+
id: string,
|
|
425
|
+
destinationPath: string,
|
|
426
|
+
what: string,
|
|
427
|
+
): Promise<void> {
|
|
428
|
+
await withRetry(
|
|
429
|
+
async () => {
|
|
430
|
+
try {
|
|
431
|
+
await this.move(id, destinationPath);
|
|
432
|
+
} catch (err) {
|
|
433
|
+
// The VFS checks the file's version on every move: a retry of a
|
|
434
|
+
// move that committed but lost its answer gets a 409 although the
|
|
435
|
+
// file is already where it was sent.
|
|
436
|
+
if (
|
|
437
|
+
err instanceof VfsRequestError &&
|
|
438
|
+
err.status === 409 &&
|
|
439
|
+
(await this.statPath(destinationPath))?.id === id
|
|
440
|
+
)
|
|
441
|
+
return;
|
|
442
|
+
throw err;
|
|
443
|
+
}
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
isRetryable: isRetryableRequest,
|
|
447
|
+
...this.retryOptions(),
|
|
448
|
+
onRetry: (err, attempt, delay) =>
|
|
449
|
+
console.warn(
|
|
450
|
+
`[owner-store] ${what} failed (${describe(err)}) — retry ${attempt} in ${delay} ms`,
|
|
451
|
+
),
|
|
452
|
+
},
|
|
453
|
+
);
|
|
325
454
|
}
|
|
326
455
|
|
|
327
456
|
/** ≤ one part: buffer the gzip once and `POST /files` it (retried as a whole). */
|
|
@@ -386,9 +515,7 @@ export class IxoVfsOwnerStore implements OwnerStore {
|
|
|
386
515
|
async remove(): Promise<void> {
|
|
387
516
|
const listing = await this.listOracleFiles();
|
|
388
517
|
const mine = listing.filter(
|
|
389
|
-
(f) =>
|
|
390
|
-
f.path === this.path ||
|
|
391
|
-
f.path.startsWith(`${this.path}${TEMP_PATH_INFIX}`),
|
|
518
|
+
(f) => f.path === this.path || this.isLeftover(f.path),
|
|
392
519
|
);
|
|
393
520
|
if (mine.length === 0) return;
|
|
394
521
|
await this.batchDelete(mine.map((f) => f.id));
|
|
@@ -20,12 +20,13 @@
|
|
|
20
20
|
* Matrix copy so no second copy of the user's history lingers.
|
|
21
21
|
*/
|
|
22
22
|
import { VfsNoDelegationError } from './ixo-vfs-store';
|
|
23
|
-
import
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
23
|
+
import {
|
|
24
|
+
NotSqliteFileError,
|
|
25
|
+
type FileSnapshot,
|
|
26
|
+
type OwnerCopy,
|
|
27
|
+
type OwnerStore,
|
|
28
|
+
type SaveHints,
|
|
29
|
+
type SaveResult,
|
|
29
30
|
} from './types';
|
|
30
31
|
|
|
31
32
|
export interface MigratingOwnerStoreOptions {
|
|
@@ -77,13 +78,22 @@ export class MigratingOwnerStore implements OwnerStore {
|
|
|
77
78
|
}
|
|
78
79
|
if (fromVfs) return fromVfs;
|
|
79
80
|
|
|
80
|
-
|
|
81
|
+
// A legacy read that FAILS is not "no legacy copy": a new, empty working
|
|
82
|
+
// copy would hide the user's history and, once a turn dirtied it, be
|
|
83
|
+
// flushed to VFS as their system of record. So the failure propagates
|
|
84
|
+
// and the boot retries. The one exception is a copy proven unusable (its
|
|
85
|
+
// whole header arrived and is not SQLite): no retry can load it.
|
|
86
|
+
let fromLegacy: OwnerCopy | null;
|
|
87
|
+
try {
|
|
88
|
+
fromLegacy = await this.legacy.load();
|
|
89
|
+
} catch (err) {
|
|
90
|
+
if (!(err instanceof NotSqliteFileError && err.proven)) throw err;
|
|
81
91
|
this.log(
|
|
82
|
-
'
|
|
83
|
-
`[owner-store] legacy Matrix
|
|
92
|
+
'error',
|
|
93
|
+
`[owner-store] legacy Matrix copy is unusable, treating it as absent: ${err.message}`,
|
|
84
94
|
);
|
|
85
|
-
|
|
86
|
-
}
|
|
95
|
+
fromLegacy = null;
|
|
96
|
+
}
|
|
87
97
|
if (!fromLegacy) {
|
|
88
98
|
// Without a delegation VFS could not be READ, so "no legacy copy" is
|
|
89
99
|
// not "no file": an existing user whose grant lapsed would otherwise
|
|
@@ -146,18 +156,24 @@ export class MigratingOwnerStore implements OwnerStore {
|
|
|
146
156
|
// Only fall through to legacy when VFS positively has no file.
|
|
147
157
|
const vfs = await this.primary.head();
|
|
148
158
|
if (vfs) return vfs;
|
|
149
|
-
return this.legacy.head()
|
|
159
|
+
return this.legacy.head();
|
|
150
160
|
}
|
|
151
161
|
|
|
152
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* The legacy Matrix copy regardless of what VFS holds (see
|
|
164
|
+
* `OwnerStore.loadLegacy`). Null means there is none, or one proven
|
|
165
|
+
* unusable; a read that failed throws, so the caller can tell "absent"
|
|
166
|
+
* from "unknown" and look again later.
|
|
167
|
+
*/
|
|
153
168
|
async loadLegacy(): Promise<OwnerCopy | null> {
|
|
154
169
|
try {
|
|
155
170
|
const copy = await this.legacy.load();
|
|
156
171
|
return copy ? { ...copy, fromLegacy: true } : null;
|
|
157
172
|
} catch (err) {
|
|
173
|
+
if (!(err instanceof NotSqliteFileError && err.proven)) throw err;
|
|
158
174
|
this.log(
|
|
159
|
-
'
|
|
160
|
-
`[owner-store] legacy Matrix
|
|
175
|
+
'error',
|
|
176
|
+
`[owner-store] legacy Matrix copy is unusable, treating it as absent: ${err.message}`,
|
|
161
177
|
);
|
|
162
178
|
return null;
|
|
163
179
|
}
|
package/src/owner-store/types.ts
CHANGED
|
@@ -189,6 +189,23 @@ const SQLITE_HEADER_BYTES = 16;
|
|
|
189
189
|
* again as it writes; this is what lets a store reject a corrupt or foreign
|
|
190
190
|
* upload before anything downstream starts.
|
|
191
191
|
*/
|
|
192
|
+
/**
|
|
193
|
+
* A stream that does not open with the SQLite header. `proven` when the whole
|
|
194
|
+
* header arrived and differs: those bytes will never load. A shorter read may
|
|
195
|
+
* be a transfer cut short, so it proves nothing about the stored file.
|
|
196
|
+
*/
|
|
197
|
+
export class NotSqliteFileError extends Error {
|
|
198
|
+
readonly proven: boolean;
|
|
199
|
+
|
|
200
|
+
constructor(what: string, headerBytesRead: number) {
|
|
201
|
+
super(
|
|
202
|
+
`${what} is not a SQLite file (${headerBytesRead} header byte(s) read)`,
|
|
203
|
+
);
|
|
204
|
+
this.name = 'NotSqliteFileError';
|
|
205
|
+
this.proven = headerBytesRead >= SQLITE_HEADER_BYTES;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
192
209
|
export async function assertSqliteStream(
|
|
193
210
|
stream: ReadableStream<Uint8Array>,
|
|
194
211
|
what: string,
|
|
@@ -196,9 +213,7 @@ export async function assertSqliteStream(
|
|
|
196
213
|
const { head, rest } = await peekStream(stream, SQLITE_HEADER_BYTES);
|
|
197
214
|
if (isSqliteFile(head)) return rest;
|
|
198
215
|
await rest.cancel(`${what} is not a SQLite file`).catch(() => undefined);
|
|
199
|
-
throw new
|
|
200
|
-
`${what} is not a SQLite file (${head.byteLength} header byte(s) read)`,
|
|
201
|
-
);
|
|
216
|
+
throw new NotSqliteFileError(what, head.byteLength);
|
|
202
217
|
}
|
|
203
218
|
|
|
204
219
|
function concat(parts: Uint8Array[]): Uint8Array {
|