@ouraihub/aether-worker 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -4
- package/src/app.ts +52 -16
- package/src/context.ts +9 -3
- package/src/conversations/action.ts +31 -8
- package/src/conversations/index.ts +144 -141
- package/src/conversations/model.ts +36 -11
- package/src/conversations/routes.ts +101 -0
- package/src/coordinator-helpers.ts +131 -0
- package/src/coordinator.ts +54 -126
- package/src/credentials.ts +22 -4
- package/src/files.ts +17 -5
- package/src/host.ts +10 -2
- package/src/identity.ts +54 -25
- package/src/images/index.ts +39 -12
- package/src/logger.ts +15 -4
- package/src/mcp/auth.ts +65 -17
- package/src/mcp/client.ts +89 -24
- package/src/mcp/common.ts +7 -3
- package/src/mcp/index.ts +74 -21
- package/src/mcp/oauth.ts +82 -23
- package/src/mcp/types.ts +9 -2
- package/src/outbound.ts +23 -8
- package/src/providers/catalog.ts +63 -0
- package/src/providers/http.ts +88 -0
- package/src/providers/index.ts +103 -163
- package/src/providers/model-options.ts +28 -0
- package/src/runtime-input.ts +44 -0
- package/src/runtime-tools.ts +298 -0
- package/src/runtime.ts +105 -264
- package/src/search/index.ts +89 -23
- package/src/skills/persistence.ts +18 -5
- package/src/skills/service.ts +60 -13
- package/src/storage.ts +108 -13
- package/src/testing/fakes.ts +97 -0
package/src/search/index.ts
CHANGED
|
@@ -83,7 +83,12 @@ function parseProvider(value: unknown): ProviderRecord {
|
|
|
83
83
|
: { basicAuthUsername: string(data.basicAuthUsername, 'basicAuthUsername') }),
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
|
-
function scope(
|
|
86
|
+
function scope(
|
|
87
|
+
actor: Actor,
|
|
88
|
+
capability: SearchCap,
|
|
89
|
+
provider: string,
|
|
90
|
+
keyId: string,
|
|
91
|
+
): CredentialScope {
|
|
87
92
|
return {
|
|
88
93
|
applicationId: actor.applicationId,
|
|
89
94
|
tenantId: actor.tenantId,
|
|
@@ -114,7 +119,11 @@ export class SearchService {
|
|
|
114
119
|
provider: string,
|
|
115
120
|
): Promise<{ document: StoredDocument | null; value: ProviderRecord }> {
|
|
116
121
|
providerFor(capability, provider)
|
|
117
|
-
const document = await this.options.documents.get(
|
|
122
|
+
const document = await this.options.documents.get(
|
|
123
|
+
actor,
|
|
124
|
+
'search.providers',
|
|
125
|
+
partition(capability, provider),
|
|
126
|
+
)
|
|
118
127
|
return { document, value: document ? parseProvider(document.value) : { keys: [] } }
|
|
119
128
|
}
|
|
120
129
|
async preferences(actor: Actor): Promise<SearchPreferences> {
|
|
@@ -141,11 +150,20 @@ export class SearchService {
|
|
|
141
150
|
providerFor(capability, provider)
|
|
142
151
|
return provider
|
|
143
152
|
}
|
|
144
|
-
private async open(
|
|
145
|
-
|
|
153
|
+
private async open(
|
|
154
|
+
actor: Actor,
|
|
155
|
+
capability: SearchCap,
|
|
156
|
+
provider: string,
|
|
157
|
+
key: KeyRecord,
|
|
158
|
+
): Promise<string> {
|
|
159
|
+
const bytes = await this.options.credentials.open(
|
|
160
|
+
scope(actor, capability, provider, key.id),
|
|
161
|
+
key.sealed,
|
|
162
|
+
)
|
|
146
163
|
try {
|
|
147
164
|
const value = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
|
|
148
|
-
if (!value.trim() || value.length > 4096)
|
|
165
|
+
if (!value.trim() || value.length > 4096)
|
|
166
|
+
throw new KitFailure('needs_config', 'Search credential unavailable')
|
|
149
167
|
return value
|
|
150
168
|
} finally {
|
|
151
169
|
bytes.fill(0)
|
|
@@ -180,7 +198,8 @@ export class SearchService {
|
|
|
180
198
|
...(value.baseUrl ? { baseUrl: value.baseUrl } : {}),
|
|
181
199
|
...(value.basicAuthUsername ? { basicAuthUsername: value.basicAuthUsername } : {}),
|
|
182
200
|
keys,
|
|
183
|
-
ready:
|
|
201
|
+
ready:
|
|
202
|
+
(config.keyMode !== 'required' || readable > 0) && (!value.keys.length || readable > 0),
|
|
184
203
|
})
|
|
185
204
|
}
|
|
186
205
|
return {
|
|
@@ -200,13 +219,15 @@ export class SearchService {
|
|
|
200
219
|
const baseUrl = string(data.baseUrl, 'baseUrl').trim()
|
|
201
220
|
if (new TextEncoder().encode(string(data.baseUrl, 'baseUrl')).length > 8192)
|
|
202
221
|
throw badRequest('baseUrl', 'too_long', 8192)
|
|
203
|
-
if (provider !== 'fetch' && baseUrl)
|
|
222
|
+
if (provider !== 'fetch' && baseUrl)
|
|
223
|
+
await this.options.checkTarget(validateTarget(baseUrl), signal)
|
|
204
224
|
delete value.baseUrl
|
|
205
225
|
if (provider !== 'fetch' && baseUrl) value.baseUrl = baseUrl
|
|
206
226
|
}
|
|
207
227
|
if (data.basicAuthUsername !== undefined) {
|
|
208
228
|
const username = string(data.basicAuthUsername, 'basicAuthUsername').trim()
|
|
209
|
-
if (new TextEncoder().encode(username).length > 200)
|
|
229
|
+
if (new TextEncoder().encode(username).length > 200)
|
|
230
|
+
throw badRequest('basicAuthUsername', 'too_long', 200)
|
|
210
231
|
if (!spec.basicAuth && username) throw badRequest('basicAuthUsername', 'unsupported')
|
|
211
232
|
delete value.basicAuthUsername
|
|
212
233
|
if (spec.basicAuth && username) value.basicAuthUsername = username
|
|
@@ -220,12 +241,22 @@ export class SearchService {
|
|
|
220
241
|
)
|
|
221
242
|
return this.settings(actor, capability)
|
|
222
243
|
}
|
|
223
|
-
async setDefault(
|
|
244
|
+
async setDefault(
|
|
245
|
+
actor: Actor,
|
|
246
|
+
capability: SearchCap,
|
|
247
|
+
provider: string,
|
|
248
|
+
): Promise<SearchSettingsDto> {
|
|
224
249
|
providerFor(capability, provider)
|
|
225
250
|
const old = await this.options.documents.get(actor, 'search.defaults', capability)
|
|
226
|
-
await this.options.documents.compareAndSet(
|
|
227
|
-
|
|
228
|
-
|
|
251
|
+
await this.options.documents.compareAndSet(
|
|
252
|
+
actor,
|
|
253
|
+
'search.defaults',
|
|
254
|
+
capability,
|
|
255
|
+
old?.revision ?? null,
|
|
256
|
+
{
|
|
257
|
+
provider,
|
|
258
|
+
},
|
|
259
|
+
)
|
|
229
260
|
return this.settings(actor, capability)
|
|
230
261
|
}
|
|
231
262
|
async saveKey(
|
|
@@ -241,7 +272,8 @@ export class SearchService {
|
|
|
241
272
|
if (!plaintext || plaintext.length > 4096) throw badRequest('value', 'invalid', 4096)
|
|
242
273
|
const { document, value } = await this.stored(actor, capability, provider)
|
|
243
274
|
const existing = keyId === undefined ? undefined : value.keys.find((key) => key.id === keyId)
|
|
244
|
-
if (keyId !== undefined && !existing)
|
|
275
|
+
if (keyId !== undefined && !existing)
|
|
276
|
+
throw new KitFailure('not_found', 'Search key not found', { field: 'key' })
|
|
245
277
|
if (!existing && value.keys.length >= 16) throw badRequest('keys', 'too_many', 16)
|
|
246
278
|
const id = keyId ?? crypto.randomUUID()
|
|
247
279
|
const bytes = new TextEncoder().encode(plaintext)
|
|
@@ -262,7 +294,12 @@ export class SearchService {
|
|
|
262
294
|
)
|
|
263
295
|
return this.settings(actor, capability)
|
|
264
296
|
}
|
|
265
|
-
async removeKey(
|
|
297
|
+
async removeKey(
|
|
298
|
+
actor: Actor,
|
|
299
|
+
capability: SearchCap,
|
|
300
|
+
provider: string,
|
|
301
|
+
keyId: string,
|
|
302
|
+
): Promise<SearchSettingsDto> {
|
|
266
303
|
const { document, value } = await this.stored(actor, capability, provider)
|
|
267
304
|
if (!value.keys.some((key) => key.id === keyId))
|
|
268
305
|
throw new KitFailure('not_found', 'Search key not found', { field: 'key' })
|
|
@@ -298,7 +335,10 @@ export class SearchService {
|
|
|
298
335
|
const { document, value } = await this.stored(actor, capability, provider)
|
|
299
336
|
if (!document) return
|
|
300
337
|
for (const key of value.keys)
|
|
301
|
-
key.sealed = await this.options.credentials.rotate(
|
|
338
|
+
key.sealed = await this.options.credentials.rotate(
|
|
339
|
+
scope(actor, capability, provider, key.id),
|
|
340
|
+
key.sealed,
|
|
341
|
+
)
|
|
302
342
|
await this.options.documents.compareAndSet(
|
|
303
343
|
actor,
|
|
304
344
|
'search.providers',
|
|
@@ -307,7 +347,11 @@ export class SearchService {
|
|
|
307
347
|
value,
|
|
308
348
|
)
|
|
309
349
|
}
|
|
310
|
-
async resolve(
|
|
350
|
+
async resolve(
|
|
351
|
+
actor: Actor,
|
|
352
|
+
capability: SearchCap,
|
|
353
|
+
explicitProvider?: string,
|
|
354
|
+
): Promise<ResolvedSearchConfig> {
|
|
311
355
|
const provider = explicitProvider ?? (await this.selected(actor, capability))
|
|
312
356
|
const { config } = providerFor(capability, provider)
|
|
313
357
|
const { value } = await this.stored(actor, capability, provider)
|
|
@@ -339,12 +383,29 @@ export class SearchService {
|
|
|
339
383
|
}
|
|
340
384
|
async fetch(actor: Actor, rawUrl: string, signal?: AbortSignal): Promise<SearchPage> {
|
|
341
385
|
const url = validateTarget(rawUrl)
|
|
342
|
-
return fetchPage(
|
|
386
|
+
return fetchPage(
|
|
387
|
+
await this.resolve(actor, 'fetch'),
|
|
388
|
+
url,
|
|
389
|
+
this.options.outbound,
|
|
390
|
+
this.options.checkTarget,
|
|
391
|
+
signal,
|
|
392
|
+
)
|
|
343
393
|
}
|
|
344
|
-
async check(
|
|
394
|
+
async check(
|
|
395
|
+
actor: Actor,
|
|
396
|
+
capability: SearchCap,
|
|
397
|
+
provider: string,
|
|
398
|
+
signal?: AbortSignal,
|
|
399
|
+
): Promise<{ count: number }> {
|
|
345
400
|
const snapshot = await this.resolve(actor, capability, provider)
|
|
346
401
|
if (capability === 'fetch') {
|
|
347
|
-
await fetchPage(
|
|
402
|
+
await fetchPage(
|
|
403
|
+
snapshot,
|
|
404
|
+
'https://example.com',
|
|
405
|
+
this.options.outbound,
|
|
406
|
+
this.options.checkTarget,
|
|
407
|
+
signal,
|
|
408
|
+
)
|
|
348
409
|
return { count: 1 }
|
|
349
410
|
}
|
|
350
411
|
const result = await searchQuery(snapshot, 'web search', this.options.outbound, signal)
|
|
@@ -374,7 +435,8 @@ export class SearchService {
|
|
|
374
435
|
search: 'POST',
|
|
375
436
|
fetch: 'POST',
|
|
376
437
|
}
|
|
377
|
-
if (!methods[route] || request.method !== methods[route])
|
|
438
|
+
if (!methods[route] || request.method !== methods[route])
|
|
439
|
+
throw badRequest('route', 'unsupported')
|
|
378
440
|
if (url.search) throw badRequest('query', 'unsupported')
|
|
379
441
|
let raw: unknown
|
|
380
442
|
try {
|
|
@@ -403,8 +465,10 @@ export class SearchService {
|
|
|
403
465
|
const data = input(raw, names)
|
|
404
466
|
const capability = cap(data.capability)
|
|
405
467
|
const provider = string(data.provider, 'provider')
|
|
406
|
-
if (route === 'default')
|
|
407
|
-
|
|
468
|
+
if (route === 'default')
|
|
469
|
+
return Response.json(await this.setDefault(actor, capability, provider))
|
|
470
|
+
if (route === 'check')
|
|
471
|
+
return Response.json(await this.check(actor, capability, provider, signal))
|
|
408
472
|
if (route === 'keys/reveal')
|
|
409
473
|
return Response.json(
|
|
410
474
|
await this.reveal(
|
|
@@ -417,7 +481,9 @@ export class SearchService {
|
|
|
417
481
|
),
|
|
418
482
|
)
|
|
419
483
|
if (request.method === 'DELETE')
|
|
420
|
-
return Response.json(
|
|
484
|
+
return Response.json(
|
|
485
|
+
await this.removeKey(actor, capability, provider, string(data.keyId, 'keyId')),
|
|
486
|
+
)
|
|
421
487
|
return Response.json(
|
|
422
488
|
await this.saveKey(
|
|
423
489
|
actor,
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import type { SkillMeta } from '@ouraihub/aether-contracts'
|
|
2
2
|
import { KitFailure } from '@ouraihub/aether-core'
|
|
3
3
|
import { digest, sha256 } from '@ouraihub/aether-core/skills/content'
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
parseCatalog,
|
|
6
|
+
type SkillCatalog,
|
|
7
|
+
type SkillRecord,
|
|
8
|
+
} from '@ouraihub/aether-core/skills/service'
|
|
5
9
|
import { exact, object, revision, text } from '@ouraihub/aether-core/skills/validation'
|
|
6
10
|
import type { Actor } from '../context.js'
|
|
7
11
|
import type { FileStore } from '../files.js'
|
|
@@ -97,15 +101,23 @@ function decode(value: unknown): number[] {
|
|
|
97
101
|
}
|
|
98
102
|
export function serializeSkill(skill: SkillRecord): Uint8Array {
|
|
99
103
|
return new TextEncoder().encode(
|
|
100
|
-
JSON.stringify({
|
|
104
|
+
JSON.stringify({
|
|
105
|
+
...skill,
|
|
106
|
+
files: skill.files.map((file) => ({ ...file, bytes: encode(file.bytes) })),
|
|
107
|
+
}),
|
|
101
108
|
)
|
|
102
109
|
}
|
|
103
|
-
export async function loadSkills(
|
|
110
|
+
export async function loadSkills(
|
|
111
|
+
files: FileStore,
|
|
112
|
+
actor: Actor,
|
|
113
|
+
value: unknown,
|
|
114
|
+
): Promise<LoadedSkills> {
|
|
104
115
|
const refs = parseRefs(value)
|
|
105
116
|
const catalog: SkillRecord[] = []
|
|
106
117
|
for (const ref of refs) {
|
|
107
118
|
const blob = await files.get(actor, ref.blobId)
|
|
108
|
-
if (!blob || blob.mime !== 'application/json')
|
|
119
|
+
if (!blob || blob.mime !== 'application/json')
|
|
120
|
+
throw new KitFailure('internal', 'Skill package is unavailable')
|
|
109
121
|
let parsed: unknown
|
|
110
122
|
try {
|
|
111
123
|
parsed = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(blob.data))
|
|
@@ -133,7 +145,8 @@ export async function loadSkills(files: FileStore, actor: Actor, value: unknown)
|
|
|
133
145
|
if ((await sha256(new Uint8Array(file.bytes))) !== file.sha256)
|
|
134
146
|
throw new KitFailure('internal', 'Skill file integrity failure')
|
|
135
147
|
}
|
|
136
|
-
if ((await digest(skill, skill.files)) !== skill.digest)
|
|
148
|
+
if ((await digest(skill, skill.files)) !== skill.digest)
|
|
149
|
+
throw new KitFailure('internal', 'Skill integrity failure')
|
|
137
150
|
catalog.push(skill)
|
|
138
151
|
}
|
|
139
152
|
return { catalog, refs }
|
package/src/skills/service.ts
CHANGED
|
@@ -2,18 +2,35 @@ import type { ManualSave, SkillRef } from '@ouraihub/aether-contracts'
|
|
|
2
2
|
import { badRequest, KitFailure } from '@ouraihub/aether-core'
|
|
3
3
|
import { readSkillSource } from '@ouraihub/aether-core/skills/package'
|
|
4
4
|
import * as skills from '@ouraihub/aether-core/skills/service'
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
exact,
|
|
7
|
+
object,
|
|
8
|
+
parseManualSave,
|
|
9
|
+
revision,
|
|
10
|
+
strings,
|
|
11
|
+
text,
|
|
12
|
+
} from '@ouraihub/aether-core/skills/validation'
|
|
6
13
|
import type { Actor, WorkerContext } from '../context.js'
|
|
7
14
|
import type { DocumentStore } from '../storage.js'
|
|
8
15
|
import type { FileStore } from '../files.js'
|
|
9
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
loadSkills,
|
|
18
|
+
parseRefs,
|
|
19
|
+
serializeSkill,
|
|
20
|
+
type LoadedSkills,
|
|
21
|
+
type SkillBlobRef,
|
|
22
|
+
} from './persistence.js'
|
|
10
23
|
|
|
11
24
|
/** All catalog mutations commit in one actor-scoped CAS; conflicts never overwrite newer content. */
|
|
12
25
|
export class SkillsService {
|
|
13
26
|
private readonly store: DocumentStore
|
|
14
27
|
private readonly blobs: FileStore
|
|
15
28
|
private readonly builtins: readonly skills.BuiltinSkill[]
|
|
16
|
-
constructor(
|
|
29
|
+
constructor(
|
|
30
|
+
store: DocumentStore,
|
|
31
|
+
blobs: FileStore,
|
|
32
|
+
builtins: readonly skills.BuiltinSkill[] = [],
|
|
33
|
+
) {
|
|
17
34
|
this.store = store
|
|
18
35
|
this.blobs = blobs
|
|
19
36
|
this.builtins = builtins
|
|
@@ -35,7 +52,10 @@ export class SkillsService {
|
|
|
35
52
|
continue
|
|
36
53
|
}
|
|
37
54
|
const blobId = crypto.randomUUID()
|
|
38
|
-
await this.blobs.put(actor, blobId, {
|
|
55
|
+
await this.blobs.put(actor, blobId, {
|
|
56
|
+
mime: 'application/json',
|
|
57
|
+
data: serializeSkill(skill),
|
|
58
|
+
})
|
|
39
59
|
created.push(blobId)
|
|
40
60
|
refs.push({ ...skills.metadata(skill), blobId, digest: skill.digest })
|
|
41
61
|
}
|
|
@@ -49,9 +69,15 @@ export class SkillsService {
|
|
|
49
69
|
throw error
|
|
50
70
|
}
|
|
51
71
|
}
|
|
52
|
-
private async load(
|
|
72
|
+
private async load(
|
|
73
|
+
actor: Actor,
|
|
74
|
+
now: number,
|
|
75
|
+
): Promise<LoadedSkills & { revision: number | null }> {
|
|
53
76
|
const document = await this.store.get(actor, 'skills', 'catalog')
|
|
54
|
-
const current =
|
|
77
|
+
const current =
|
|
78
|
+
document === null
|
|
79
|
+
? { catalog: [], refs: [] }
|
|
80
|
+
: await loadSkills(this.blobs, actor, document.value)
|
|
55
81
|
const synced = await skills.syncBuiltins(current.catalog, this.builtins, now)
|
|
56
82
|
if (synced === current.catalog) return { ...current, revision: document?.revision ?? null }
|
|
57
83
|
const saved = await this.commit(actor, current, synced, document?.revision ?? null)
|
|
@@ -71,7 +97,8 @@ export class SkillsService {
|
|
|
71
97
|
if (this.builtins.length !== 0) return skills.list((await this.load(actor, now)).catalog)
|
|
72
98
|
const document = await this.store.get(actor, 'skills', 'catalog')
|
|
73
99
|
const refs = document === null ? [] : parseRefs(document.value)
|
|
74
|
-
if (refs.some((ref) => ref.origin === 'builtin'))
|
|
100
|
+
if (refs.some((ref) => ref.origin === 'builtin'))
|
|
101
|
+
return skills.list((await this.load(actor, now)).catalog)
|
|
75
102
|
return refs.map((ref) => ({
|
|
76
103
|
id: ref.id,
|
|
77
104
|
name: ref.name,
|
|
@@ -101,9 +128,18 @@ export class SkillsService {
|
|
|
101
128
|
const content = await readSkillSource(source)
|
|
102
129
|
return skills.preview((await this.load(actor, now)).catalog, content)
|
|
103
130
|
}
|
|
104
|
-
async install(
|
|
131
|
+
async install(
|
|
132
|
+
actor: Actor,
|
|
133
|
+
source: unknown,
|
|
134
|
+
confirmedDigest: string,
|
|
135
|
+
replace: boolean,
|
|
136
|
+
newId: string,
|
|
137
|
+
now: number,
|
|
138
|
+
) {
|
|
105
139
|
const content = await readSkillSource(source)
|
|
106
|
-
return this.mutate(actor, now, (catalog) =>
|
|
140
|
+
return this.mutate(actor, now, (catalog) =>
|
|
141
|
+
skills.install(catalog, content, confirmedDigest, replace, newId, now),
|
|
142
|
+
)
|
|
107
143
|
}
|
|
108
144
|
async save(actor: Actor, input: ManualSave, newId: string, now: number) {
|
|
109
145
|
const checked = parseManualSave(input)
|
|
@@ -132,11 +168,21 @@ export class SkillsService {
|
|
|
132
168
|
const id = decodeURIComponent(parts[0] ?? '')
|
|
133
169
|
if (parts.length === 1) return Response.json(await this.get(actor, id, now))
|
|
134
170
|
const expected = revision(Number(url.searchParams.get('revision')))
|
|
135
|
-
if (parts.length === 2 && parts[1] === 'files')
|
|
171
|
+
if (parts.length === 2 && parts[1] === 'files')
|
|
172
|
+
return Response.json(await this.files(actor, id, expected, now))
|
|
136
173
|
if (parts.length === 2 && parts[1] === 'file') {
|
|
137
|
-
const bytes = await this.file(
|
|
174
|
+
const bytes = await this.file(
|
|
175
|
+
actor,
|
|
176
|
+
id,
|
|
177
|
+
expected,
|
|
178
|
+
text(url.searchParams.get('path'), 'path'),
|
|
179
|
+
now,
|
|
180
|
+
)
|
|
138
181
|
return new Response(bytes, {
|
|
139
|
-
headers: {
|
|
182
|
+
headers: {
|
|
183
|
+
'Content-Type': 'application/octet-stream',
|
|
184
|
+
'X-Content-Type-Options': 'nosniff',
|
|
185
|
+
},
|
|
140
186
|
})
|
|
141
187
|
}
|
|
142
188
|
}
|
|
@@ -149,7 +195,8 @@ export class SkillsService {
|
|
|
149
195
|
} catch {
|
|
150
196
|
throw badRequest('body', 'invalid_json')
|
|
151
197
|
}
|
|
152
|
-
if (path === '/save')
|
|
198
|
+
if (path === '/save')
|
|
199
|
+
return Response.json(await this.save(actor, parseManualSave(raw), crypto.randomUUID(), now))
|
|
153
200
|
const body = object(raw, 'body')
|
|
154
201
|
if (path === '/preview') {
|
|
155
202
|
exact(body, ['source'], 'body')
|
package/src/storage.ts
CHANGED
|
@@ -13,7 +13,11 @@ export interface DocumentListOptions {
|
|
|
13
13
|
}
|
|
14
14
|
export interface DocumentStore {
|
|
15
15
|
get(owner: Actor, collection: string, key: string): Promise<StoredDocument | null>
|
|
16
|
-
list(
|
|
16
|
+
list(
|
|
17
|
+
owner: Actor,
|
|
18
|
+
collection: string,
|
|
19
|
+
options?: DocumentListOptions,
|
|
20
|
+
): Promise<readonly StoredDocument[]>
|
|
17
21
|
compareAndSet(
|
|
18
22
|
owner: Actor,
|
|
19
23
|
collection: string,
|
|
@@ -32,19 +36,34 @@ function revision(value: number): void {
|
|
|
32
36
|
if (!Number.isSafeInteger(value) || value < 1 || value >= Number.MAX_SAFE_INTEGER)
|
|
33
37
|
throw badRequest('revision', 'invalid')
|
|
34
38
|
}
|
|
35
|
-
function scope(
|
|
39
|
+
function scope(
|
|
40
|
+
owner: Actor,
|
|
41
|
+
collection: string,
|
|
42
|
+
): readonly [string, string, string, string, string] {
|
|
36
43
|
identifier(collection)
|
|
37
|
-
return [
|
|
44
|
+
return [
|
|
45
|
+
owner.applicationId,
|
|
46
|
+
JSON.stringify(owner.tenantId),
|
|
47
|
+
owner.subjectId,
|
|
48
|
+
ownerKey(owner),
|
|
49
|
+
collection,
|
|
50
|
+
]
|
|
38
51
|
}
|
|
39
52
|
export async function validateD1Schema(database: D1Database): Promise<void> {
|
|
40
53
|
let row: unknown
|
|
41
54
|
try {
|
|
42
55
|
row = await database.prepare('SELECT version FROM aether_schema WHERE singleton = 1').first()
|
|
43
56
|
} catch {
|
|
44
|
-
throw new KitFailure(
|
|
57
|
+
throw new KitFailure(
|
|
58
|
+
'incompatible_db',
|
|
59
|
+
'Apply the supported Aether public migration before startup',
|
|
60
|
+
)
|
|
45
61
|
}
|
|
46
62
|
if (typeof row !== 'object' || row === null || !('version' in row) || row.version !== 1) {
|
|
47
|
-
throw new KitFailure(
|
|
63
|
+
throw new KitFailure(
|
|
64
|
+
'incompatible_db',
|
|
65
|
+
'Apply the supported Aether public migration before startup',
|
|
66
|
+
)
|
|
48
67
|
}
|
|
49
68
|
}
|
|
50
69
|
/** D1 rows stay below its 2,000,000 byte limit. Large functional documents live
|
|
@@ -56,7 +75,10 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
56
75
|
readonly #database: D1Database
|
|
57
76
|
readonly #blobs: R2Bucket | undefined
|
|
58
77
|
readonly #maxBytes: number
|
|
59
|
-
constructor(
|
|
78
|
+
constructor(
|
|
79
|
+
database: D1Database,
|
|
80
|
+
options: { readonly blobs?: R2Bucket; readonly maxDocumentBytes?: number } = {},
|
|
81
|
+
) {
|
|
60
82
|
this.#database = database
|
|
61
83
|
this.#blobs = options.blobs
|
|
62
84
|
this.#maxBytes = options.maxDocumentBytes ?? 64 * 1024 * 1024
|
|
@@ -85,10 +107,14 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
85
107
|
if (typeof row.value_json === 'string' && row.blob_key === null) json = row.value_json
|
|
86
108
|
else if (row.value_json === null && typeof row.blob_key === 'string' && this.#blobs) {
|
|
87
109
|
const prefix = await this.#blobPrefix(owner)
|
|
88
|
-
if (
|
|
110
|
+
if (
|
|
111
|
+
!row.blob_key.startsWith(prefix) ||
|
|
112
|
+
!/^[a-f0-9-]{36}$/u.test(row.blob_key.slice(prefix.length))
|
|
113
|
+
)
|
|
89
114
|
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
90
115
|
const blob = await this.#blobs.get(row.blob_key)
|
|
91
|
-
if (!blob || blob.size > this.#maxBytes)
|
|
116
|
+
if (!blob || blob.size > this.#maxBytes)
|
|
117
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
92
118
|
const bytes = await blob.arrayBuffer()
|
|
93
119
|
if (bytes.byteLength !== blob.size || bytes.byteLength > this.#maxBytes)
|
|
94
120
|
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
@@ -113,9 +139,14 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
113
139
|
.first()
|
|
114
140
|
return row === null ? null : this.#parse(owner, row)
|
|
115
141
|
}
|
|
116
|
-
async list(
|
|
142
|
+
async list(
|
|
143
|
+
owner: Actor,
|
|
144
|
+
collection: string,
|
|
145
|
+
options: DocumentListOptions = {},
|
|
146
|
+
): Promise<readonly StoredDocument[]> {
|
|
117
147
|
const limit = options.limit ?? MAX_LIST
|
|
118
|
-
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST)
|
|
148
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST)
|
|
149
|
+
throw badRequest('limit', 'invalid', MAX_LIST)
|
|
119
150
|
if (options.after !== undefined) identifier(options.after)
|
|
120
151
|
const result = await this.#database
|
|
121
152
|
.prepare(
|
|
@@ -123,7 +154,8 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
123
154
|
)
|
|
124
155
|
.bind(...scope(owner, collection), options.after ?? '', limit)
|
|
125
156
|
.all()
|
|
126
|
-
if (!result.success || !Array.isArray(result.results))
|
|
157
|
+
if (!result.success || !Array.isArray(result.results))
|
|
158
|
+
throw new KitFailure('internal', 'Document listing failed')
|
|
127
159
|
const documents: StoredDocument[] = []
|
|
128
160
|
// Sequential hydration bounds R2 concurrency/memory to one document at a time.
|
|
129
161
|
for (const row of result.results) documents.push(await this.#parse(owner, row))
|
|
@@ -151,7 +183,8 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
151
183
|
let inline: string | null = json
|
|
152
184
|
let blobKey: string | null = null
|
|
153
185
|
if (bytes.byteLength > INLINE_BYTES) {
|
|
154
|
-
if (!this.#blobs)
|
|
186
|
+
if (!this.#blobs)
|
|
187
|
+
throw new KitFailure('needs_config', 'Large documents require an R2 blob binding')
|
|
155
188
|
blobKey = `${await this.#blobPrefix(owner)}${crypto.randomUUID()}`
|
|
156
189
|
const written = await this.#blobs.put(blobKey, bytes, {
|
|
157
190
|
httpMetadata: { contentType: 'application/json' },
|
|
@@ -184,7 +217,12 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
184
217
|
const parsed: unknown = JSON.parse(json)
|
|
185
218
|
return { key, revision: row.revision, value: parsed }
|
|
186
219
|
}
|
|
187
|
-
async remove(
|
|
220
|
+
async remove(
|
|
221
|
+
owner: Actor,
|
|
222
|
+
collection: string,
|
|
223
|
+
key: string,
|
|
224
|
+
expectedRevision: number,
|
|
225
|
+
): Promise<void> {
|
|
188
226
|
identifier(key)
|
|
189
227
|
revision(expectedRevision)
|
|
190
228
|
const row = await this.#database
|
|
@@ -196,3 +234,60 @@ export class D1DocumentStore implements DocumentStore {
|
|
|
196
234
|
if (row === null) throw new KitFailure('conflict', 'Document revision conflict')
|
|
197
235
|
}
|
|
198
236
|
}
|
|
237
|
+
|
|
238
|
+
export interface RetryOptions {
|
|
239
|
+
readonly attempts?: number
|
|
240
|
+
readonly baseMs?: number
|
|
241
|
+
/** Injected for deterministic tests; defaults to a real timer. */
|
|
242
|
+
readonly sleep?: (ms: number) => Promise<unknown>
|
|
243
|
+
readonly random?: () => number
|
|
244
|
+
}
|
|
245
|
+
/** Marks the store call inside a `retryCas` attempt; only its revision conflicts are retried. */
|
|
246
|
+
export type CasCall = <T>(write: () => Promise<T>) => Promise<T>
|
|
247
|
+
class RevisionConflict extends Error {}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Runs `attempt` until its CAS write succeeds. Each attempt must reload and re-apply its change
|
|
251
|
+
* to the newest revision. Only a conflict raised by the `cas`-wrapped store call is retried;
|
|
252
|
+
* semantic conflicts from the attempt itself (stale preparation, finished turn) surface at once.
|
|
253
|
+
* Retries back off exponentially with jitter, so contending writers spread out instead of
|
|
254
|
+
* exhausting their attempts in lockstep.
|
|
255
|
+
*/
|
|
256
|
+
export async function retryCas<T>(
|
|
257
|
+
attempt: (cas: CasCall) => Promise<T>,
|
|
258
|
+
exhausted: () => KitFailure,
|
|
259
|
+
options: RetryOptions = {},
|
|
260
|
+
): Promise<T> {
|
|
261
|
+
const attempts = options.attempts ?? 8,
|
|
262
|
+
baseMs = options.baseMs ?? 5,
|
|
263
|
+
random = options.random ?? Math.random
|
|
264
|
+
const sleep =
|
|
265
|
+
options.sleep ??
|
|
266
|
+
((ms: number) => {
|
|
267
|
+
const { promise, resolve } = Promise.withResolvers<void>()
|
|
268
|
+
setTimeout(resolve, ms)
|
|
269
|
+
return promise
|
|
270
|
+
})
|
|
271
|
+
const cas: CasCall = async (write) => {
|
|
272
|
+
try {
|
|
273
|
+
return await write()
|
|
274
|
+
} catch (error) {
|
|
275
|
+
if (
|
|
276
|
+
error instanceof KitFailure &&
|
|
277
|
+
error.code === 'conflict' &&
|
|
278
|
+
error.details?.reason === undefined
|
|
279
|
+
)
|
|
280
|
+
throw new RevisionConflict()
|
|
281
|
+
throw error
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (let i = 0; i < attempts; i++) {
|
|
285
|
+
try {
|
|
286
|
+
return await attempt(cas)
|
|
287
|
+
} catch (error) {
|
|
288
|
+
if (!(error instanceof RevisionConflict)) throw error
|
|
289
|
+
}
|
|
290
|
+
if (i < attempts - 1) await sleep(baseMs * 2 ** i * (0.5 + random()))
|
|
291
|
+
}
|
|
292
|
+
throw exhausted()
|
|
293
|
+
}
|