@ouraihub/aether-worker 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +789 -0
- package/migrations/0001.sql +22 -0
- package/package.json +38 -3
- package/src/app.ts +363 -0
- package/src/cloudflare.ts +49 -0
- package/src/context.ts +55 -0
- package/src/conversations/action.ts +257 -0
- package/src/conversations/index.ts +480 -0
- package/src/conversations/model.ts +427 -0
- package/src/conversations/routes.ts +101 -0
- package/src/coordinator-helpers.ts +131 -0
- package/src/coordinator.ts +488 -0
- package/src/credentials.ts +171 -0
- package/src/files.ts +117 -0
- package/src/host.ts +25 -0
- package/src/identity.ts +227 -0
- package/src/images/index.ts +73 -0
- package/src/index.ts +30 -0
- package/src/logger.ts +98 -0
- package/src/mcp/auth.ts +398 -0
- package/src/mcp/client.ts +491 -0
- package/src/mcp/common.ts +124 -0
- package/src/mcp/index.ts +292 -0
- package/src/mcp/oauth.ts +343 -0
- package/src/mcp/types.ts +81 -0
- package/src/outbound.ts +320 -0
- package/src/providers/catalog.ts +63 -0
- package/src/providers/http.ts +88 -0
- package/src/providers/index.ts +445 -0
- package/src/providers/model-options.ts +28 -0
- package/src/runtime-input.ts +44 -0
- package/src/runtime-tools.ts +298 -0
- package/src/runtime.ts +476 -0
- package/src/search/index.ts +497 -0
- package/src/skills/index.ts +1 -0
- package/src/skills/persistence.ts +153 -0
- package/src/skills/service.ts +250 -0
- package/src/storage.ts +293 -0
- package/src/testing/fakes.ts +97 -0
- package/README.md +0 -3
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import type { ManualSave, SkillRef } from '@ouraihub/aether-contracts'
|
|
2
|
+
import { badRequest, KitFailure } from '@ouraihub/aether-core'
|
|
3
|
+
import { readSkillSource } from '@ouraihub/aether-core/skills/package'
|
|
4
|
+
import * as skills from '@ouraihub/aether-core/skills/service'
|
|
5
|
+
import {
|
|
6
|
+
exact,
|
|
7
|
+
object,
|
|
8
|
+
parseManualSave,
|
|
9
|
+
revision,
|
|
10
|
+
strings,
|
|
11
|
+
text,
|
|
12
|
+
} from '@ouraihub/aether-core/skills/validation'
|
|
13
|
+
import type { Actor, WorkerContext } from '../context.js'
|
|
14
|
+
import type { DocumentStore } from '../storage.js'
|
|
15
|
+
import type { FileStore } from '../files.js'
|
|
16
|
+
import {
|
|
17
|
+
loadSkills,
|
|
18
|
+
parseRefs,
|
|
19
|
+
serializeSkill,
|
|
20
|
+
type LoadedSkills,
|
|
21
|
+
type SkillBlobRef,
|
|
22
|
+
} from './persistence.js'
|
|
23
|
+
|
|
24
|
+
/** All catalog mutations commit in one actor-scoped CAS; conflicts never overwrite newer content. */
|
|
25
|
+
export class SkillsService {
|
|
26
|
+
private readonly store: DocumentStore
|
|
27
|
+
private readonly blobs: FileStore
|
|
28
|
+
private readonly builtins: readonly skills.BuiltinSkill[]
|
|
29
|
+
constructor(
|
|
30
|
+
store: DocumentStore,
|
|
31
|
+
blobs: FileStore,
|
|
32
|
+
builtins: readonly skills.BuiltinSkill[] = [],
|
|
33
|
+
) {
|
|
34
|
+
this.store = store
|
|
35
|
+
this.blobs = blobs
|
|
36
|
+
this.builtins = builtins
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
private async commit(
|
|
40
|
+
actor: Actor,
|
|
41
|
+
previous: LoadedSkills,
|
|
42
|
+
next: skills.SkillCatalog,
|
|
43
|
+
expected: number | null,
|
|
44
|
+
): Promise<{ refs: readonly SkillBlobRef[]; revision: number }> {
|
|
45
|
+
const created: string[] = []
|
|
46
|
+
const refs: SkillBlobRef[] = []
|
|
47
|
+
try {
|
|
48
|
+
for (const skill of next) {
|
|
49
|
+
const oldRef = previous.refs.find((ref) => ref.id === skill.id)
|
|
50
|
+
if (oldRef && previous.catalog.find((entry) => entry.id === skill.id) === skill) {
|
|
51
|
+
refs.push(oldRef)
|
|
52
|
+
continue
|
|
53
|
+
}
|
|
54
|
+
const blobId = crypto.randomUUID()
|
|
55
|
+
await this.blobs.put(actor, blobId, {
|
|
56
|
+
mime: 'application/json',
|
|
57
|
+
data: serializeSkill(skill),
|
|
58
|
+
})
|
|
59
|
+
created.push(blobId)
|
|
60
|
+
refs.push({ ...skills.metadata(skill), blobId, digest: skill.digest })
|
|
61
|
+
}
|
|
62
|
+
const saved = await this.store.compareAndSet(actor, 'skills', 'catalog', expected, refs)
|
|
63
|
+
return { refs, revision: saved.revision }
|
|
64
|
+
} catch (error) {
|
|
65
|
+
// Only a definite rejection proves these blobs are not live catalog references.
|
|
66
|
+
if (error instanceof KitFailure && error.code === 'conflict') {
|
|
67
|
+
await Promise.all(created.map((id) => this.blobs.remove(actor, id)))
|
|
68
|
+
}
|
|
69
|
+
throw error
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
private async load(
|
|
73
|
+
actor: Actor,
|
|
74
|
+
now: number,
|
|
75
|
+
): Promise<LoadedSkills & { revision: number | null }> {
|
|
76
|
+
const document = await this.store.get(actor, 'skills', 'catalog')
|
|
77
|
+
const current =
|
|
78
|
+
document === null
|
|
79
|
+
? { catalog: [], refs: [] }
|
|
80
|
+
: await loadSkills(this.blobs, actor, document.value)
|
|
81
|
+
const synced = await skills.syncBuiltins(current.catalog, this.builtins, now)
|
|
82
|
+
if (synced === current.catalog) return { ...current, revision: document?.revision ?? null }
|
|
83
|
+
const saved = await this.commit(actor, current, synced, document?.revision ?? null)
|
|
84
|
+
return { catalog: synced, ...saved }
|
|
85
|
+
}
|
|
86
|
+
private async mutate(
|
|
87
|
+
actor: Actor,
|
|
88
|
+
now: number,
|
|
89
|
+
operation: (catalog: skills.SkillCatalog) => Promise<skills.SkillCatalog> | skills.SkillCatalog,
|
|
90
|
+
) {
|
|
91
|
+
const current = await this.load(actor, now)
|
|
92
|
+
const next = await operation(current.catalog)
|
|
93
|
+
await this.commit(actor, current, next, current.revision)
|
|
94
|
+
return skills.list(next)
|
|
95
|
+
}
|
|
96
|
+
async list(actor: Actor, now: number) {
|
|
97
|
+
if (this.builtins.length !== 0) return skills.list((await this.load(actor, now)).catalog)
|
|
98
|
+
const document = await this.store.get(actor, 'skills', 'catalog')
|
|
99
|
+
const refs = document === null ? [] : parseRefs(document.value)
|
|
100
|
+
if (refs.some((ref) => ref.origin === 'builtin'))
|
|
101
|
+
return skills.list((await this.load(actor, now)).catalog)
|
|
102
|
+
return refs.map((ref) => ({
|
|
103
|
+
id: ref.id,
|
|
104
|
+
name: ref.name,
|
|
105
|
+
description: ref.description,
|
|
106
|
+
version: ref.version,
|
|
107
|
+
origin: ref.origin,
|
|
108
|
+
revision: ref.revision,
|
|
109
|
+
bodyChars: ref.bodyChars,
|
|
110
|
+
updatedAt: ref.updatedAt,
|
|
111
|
+
hasScripts: ref.hasScripts,
|
|
112
|
+
fileCount: ref.fileCount,
|
|
113
|
+
}))
|
|
114
|
+
}
|
|
115
|
+
async get(actor: Actor, id: string, now: number) {
|
|
116
|
+
return skills.get((await this.load(actor, now)).catalog, id)
|
|
117
|
+
}
|
|
118
|
+
async files(actor: Actor, id: string, expected: number, now: number) {
|
|
119
|
+
return skills.files((await this.load(actor, now)).catalog, id, expected)
|
|
120
|
+
}
|
|
121
|
+
async file(actor: Actor, id: string, expected: number, path: string, now: number) {
|
|
122
|
+
return skills.file((await this.load(actor, now)).catalog, id, expected, path)
|
|
123
|
+
}
|
|
124
|
+
async snapshot(actor: Actor, refs: readonly SkillRef[], now: number) {
|
|
125
|
+
return skills.snapshot((await this.load(actor, now)).catalog, refs)
|
|
126
|
+
}
|
|
127
|
+
async preview(actor: Actor, source: unknown, now: number) {
|
|
128
|
+
const content = await readSkillSource(source)
|
|
129
|
+
return skills.preview((await this.load(actor, now)).catalog, content)
|
|
130
|
+
}
|
|
131
|
+
async install(
|
|
132
|
+
actor: Actor,
|
|
133
|
+
source: unknown,
|
|
134
|
+
confirmedDigest: string,
|
|
135
|
+
replace: boolean,
|
|
136
|
+
newId: string,
|
|
137
|
+
now: number,
|
|
138
|
+
) {
|
|
139
|
+
const content = await readSkillSource(source)
|
|
140
|
+
return this.mutate(actor, now, (catalog) =>
|
|
141
|
+
skills.install(catalog, content, confirmedDigest, replace, newId, now),
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
async save(actor: Actor, input: ManualSave, newId: string, now: number) {
|
|
145
|
+
const checked = parseManualSave(input)
|
|
146
|
+
return this.mutate(actor, now, (catalog) => skills.save(catalog, checked, newId, now))
|
|
147
|
+
}
|
|
148
|
+
async duplicate(actor: Actor, id: string, newName: string, newId: string, now: number) {
|
|
149
|
+
return this.mutate(actor, now, (catalog) => skills.duplicate(catalog, id, newName, newId, now))
|
|
150
|
+
}
|
|
151
|
+
async reorder(actor: Actor, ids: readonly string[], now: number) {
|
|
152
|
+
return this.mutate(actor, now, (catalog) => skills.reorder(catalog, ids))
|
|
153
|
+
}
|
|
154
|
+
async remove(actor: Actor, id: string, now: number) {
|
|
155
|
+
return this.mutate(actor, now, (catalog) => skills.remove(catalog, id))
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async handle(request: Request, context: WorkerContext): Promise<Response | null> {
|
|
159
|
+
const url = new URL(request.url)
|
|
160
|
+
const base = '/api/v1/skills'
|
|
161
|
+
if (url.pathname !== base && !url.pathname.startsWith(`${base}/`)) return null
|
|
162
|
+
const { actor, now } = context
|
|
163
|
+
if (context.signal.aborted) throw new KitFailure('cancelled', 'Cancelled')
|
|
164
|
+
const path = url.pathname.slice(base.length)
|
|
165
|
+
if (request.method === 'GET') {
|
|
166
|
+
if (path === '') return Response.json(await this.list(actor, now))
|
|
167
|
+
const parts = path.slice(1).split('/')
|
|
168
|
+
const id = decodeURIComponent(parts[0] ?? '')
|
|
169
|
+
if (parts.length === 1) return Response.json(await this.get(actor, id, now))
|
|
170
|
+
const expected = revision(Number(url.searchParams.get('revision')))
|
|
171
|
+
if (parts.length === 2 && parts[1] === 'files')
|
|
172
|
+
return Response.json(await this.files(actor, id, expected, now))
|
|
173
|
+
if (parts.length === 2 && parts[1] === 'file') {
|
|
174
|
+
const bytes = await this.file(
|
|
175
|
+
actor,
|
|
176
|
+
id,
|
|
177
|
+
expected,
|
|
178
|
+
text(url.searchParams.get('path'), 'path'),
|
|
179
|
+
now,
|
|
180
|
+
)
|
|
181
|
+
return new Response(bytes, {
|
|
182
|
+
headers: {
|
|
183
|
+
'Content-Type': 'application/octet-stream',
|
|
184
|
+
'X-Content-Type-Options': 'nosniff',
|
|
185
|
+
},
|
|
186
|
+
})
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (request.method === 'DELETE' && /^\/[^/]+$/u.test(path))
|
|
190
|
+
return Response.json(await this.remove(actor, decodeURIComponent(path.slice(1)), now))
|
|
191
|
+
if (request.method === 'POST') {
|
|
192
|
+
let raw: unknown
|
|
193
|
+
try {
|
|
194
|
+
raw = await request.json()
|
|
195
|
+
} catch {
|
|
196
|
+
throw badRequest('body', 'invalid_json')
|
|
197
|
+
}
|
|
198
|
+
if (path === '/save')
|
|
199
|
+
return Response.json(await this.save(actor, parseManualSave(raw), crypto.randomUUID(), now))
|
|
200
|
+
const body = object(raw, 'body')
|
|
201
|
+
if (path === '/preview') {
|
|
202
|
+
exact(body, ['source'], 'body')
|
|
203
|
+
return Response.json(await this.preview(actor, body.source, now))
|
|
204
|
+
}
|
|
205
|
+
if (path === '/install') {
|
|
206
|
+
exact(body, ['source', 'confirmedDigest', 'replace'], 'body')
|
|
207
|
+
if (typeof body.replace !== 'boolean') throw badRequest('replace', 'invalid')
|
|
208
|
+
return Response.json(
|
|
209
|
+
await this.install(
|
|
210
|
+
actor,
|
|
211
|
+
body.source,
|
|
212
|
+
text(body.confirmedDigest, 'confirmedDigest'),
|
|
213
|
+
body.replace,
|
|
214
|
+
crypto.randomUUID(),
|
|
215
|
+
now,
|
|
216
|
+
),
|
|
217
|
+
)
|
|
218
|
+
}
|
|
219
|
+
if (path === '/reorder') {
|
|
220
|
+
exact(body, ['ids'], 'body')
|
|
221
|
+
return Response.json(await this.reorder(actor, strings(body.ids, 'ids'), now))
|
|
222
|
+
}
|
|
223
|
+
if (path === '/snapshot') {
|
|
224
|
+
exact(body, ['refs'], 'body')
|
|
225
|
+
const refs: unknown = body.refs
|
|
226
|
+
if (!Array.isArray(refs)) throw badRequest('refs', 'invalid')
|
|
227
|
+
const checked = refs.map((value: unknown) => {
|
|
228
|
+
const ref = object(value, 'ref')
|
|
229
|
+
exact(ref, ['id', 'revision'], 'ref')
|
|
230
|
+
return { id: text(ref.id, 'id'), revision: revision(ref.revision) }
|
|
231
|
+
})
|
|
232
|
+
return Response.json(await this.snapshot(actor, checked, now))
|
|
233
|
+
}
|
|
234
|
+
const duplicate = /^\/([^/]+)\/duplicate$/u.exec(path)
|
|
235
|
+
if (duplicate?.[1] !== undefined) {
|
|
236
|
+
exact(body, ['newName'], 'body')
|
|
237
|
+
return Response.json(
|
|
238
|
+
await this.duplicate(
|
|
239
|
+
actor,
|
|
240
|
+
decodeURIComponent(duplicate[1]),
|
|
241
|
+
text(body.newName, 'newName'),
|
|
242
|
+
crypto.randomUUID(),
|
|
243
|
+
now,
|
|
244
|
+
),
|
|
245
|
+
)
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
throw badRequest('route', 'unsupported')
|
|
249
|
+
}
|
|
250
|
+
}
|
package/src/storage.ts
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { badRequest, KitFailure } from '@ouraihub/aether-core'
|
|
2
|
+
import { ownerKey, type Actor } from './context.ts'
|
|
3
|
+
import type { D1Database, R2Bucket } from './cloudflare.ts'
|
|
4
|
+
|
|
5
|
+
export interface StoredDocument {
|
|
6
|
+
readonly key: string
|
|
7
|
+
readonly revision: number
|
|
8
|
+
readonly value: unknown
|
|
9
|
+
}
|
|
10
|
+
export interface DocumentListOptions {
|
|
11
|
+
readonly after?: string
|
|
12
|
+
readonly limit?: number
|
|
13
|
+
}
|
|
14
|
+
export interface DocumentStore {
|
|
15
|
+
get(owner: Actor, collection: string, key: string): Promise<StoredDocument | null>
|
|
16
|
+
list(
|
|
17
|
+
owner: Actor,
|
|
18
|
+
collection: string,
|
|
19
|
+
options?: DocumentListOptions,
|
|
20
|
+
): Promise<readonly StoredDocument[]>
|
|
21
|
+
compareAndSet(
|
|
22
|
+
owner: Actor,
|
|
23
|
+
collection: string,
|
|
24
|
+
key: string,
|
|
25
|
+
expectedRevision: number | null,
|
|
26
|
+
value: unknown,
|
|
27
|
+
): Promise<StoredDocument>
|
|
28
|
+
remove(owner: Actor, collection: string, key: string, expectedRevision: number): Promise<void>
|
|
29
|
+
}
|
|
30
|
+
const INLINE_BYTES = 256 * 1024
|
|
31
|
+
const MAX_LIST = 200
|
|
32
|
+
function identifier(value: string): void {
|
|
33
|
+
if (!value || value.length > 256) throw badRequest('document', 'invalid_identifier')
|
|
34
|
+
}
|
|
35
|
+
function revision(value: number): void {
|
|
36
|
+
if (!Number.isSafeInteger(value) || value < 1 || value >= Number.MAX_SAFE_INTEGER)
|
|
37
|
+
throw badRequest('revision', 'invalid')
|
|
38
|
+
}
|
|
39
|
+
function scope(
|
|
40
|
+
owner: Actor,
|
|
41
|
+
collection: string,
|
|
42
|
+
): readonly [string, string, string, string, string] {
|
|
43
|
+
identifier(collection)
|
|
44
|
+
return [
|
|
45
|
+
owner.applicationId,
|
|
46
|
+
JSON.stringify(owner.tenantId),
|
|
47
|
+
owner.subjectId,
|
|
48
|
+
ownerKey(owner),
|
|
49
|
+
collection,
|
|
50
|
+
]
|
|
51
|
+
}
|
|
52
|
+
export async function validateD1Schema(database: D1Database): Promise<void> {
|
|
53
|
+
let row: unknown
|
|
54
|
+
try {
|
|
55
|
+
row = await database.prepare('SELECT version FROM aether_schema WHERE singleton = 1').first()
|
|
56
|
+
} catch {
|
|
57
|
+
throw new KitFailure(
|
|
58
|
+
'incompatible_db',
|
|
59
|
+
'Apply the supported Aether public migration before startup',
|
|
60
|
+
)
|
|
61
|
+
}
|
|
62
|
+
if (typeof row !== 'object' || row === null || !('version' in row) || row.version !== 1) {
|
|
63
|
+
throw new KitFailure(
|
|
64
|
+
'incompatible_db',
|
|
65
|
+
'Apply the supported Aether public migration before startup',
|
|
66
|
+
)
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** D1 rows stay below its 2,000,000 byte limit. Large functional documents live
|
|
70
|
+
* in immutable scoped R2 blobs; credentials inside documents MUST already be sealed.
|
|
71
|
+
* Tombstones reserve deleted keys. Superseded/uncertain-commit blobs stay for explicit
|
|
72
|
+
* deployment GC: deleting them inline would race readers or a committed SQL response.
|
|
73
|
+
*/
|
|
74
|
+
export class D1DocumentStore implements DocumentStore {
|
|
75
|
+
readonly #database: D1Database
|
|
76
|
+
readonly #blobs: R2Bucket | undefined
|
|
77
|
+
readonly #maxBytes: number
|
|
78
|
+
constructor(
|
|
79
|
+
database: D1Database,
|
|
80
|
+
options: { readonly blobs?: R2Bucket; readonly maxDocumentBytes?: number } = {},
|
|
81
|
+
) {
|
|
82
|
+
this.#database = database
|
|
83
|
+
this.#blobs = options.blobs
|
|
84
|
+
this.#maxBytes = options.maxDocumentBytes ?? 64 * 1024 * 1024
|
|
85
|
+
if (!Number.isSafeInteger(this.#maxBytes) || this.#maxBytes < INLINE_BYTES)
|
|
86
|
+
throw badRequest('maxDocumentBytes', 'invalid')
|
|
87
|
+
}
|
|
88
|
+
async #blobPrefix(owner: Actor): Promise<string> {
|
|
89
|
+
const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(ownerKey(owner)))
|
|
90
|
+
return `aether/documents/v1/${Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join('')}/`
|
|
91
|
+
}
|
|
92
|
+
async #parse(owner: Actor, row: unknown): Promise<StoredDocument> {
|
|
93
|
+
if (
|
|
94
|
+
typeof row !== 'object' ||
|
|
95
|
+
row === null ||
|
|
96
|
+
!('key' in row) ||
|
|
97
|
+
typeof row.key !== 'string' ||
|
|
98
|
+
!('revision' in row) ||
|
|
99
|
+
typeof row.revision !== 'number' ||
|
|
100
|
+
!Number.isSafeInteger(row.revision) ||
|
|
101
|
+
row.revision < 1 ||
|
|
102
|
+
!('value_json' in row) ||
|
|
103
|
+
!('blob_key' in row)
|
|
104
|
+
)
|
|
105
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
106
|
+
let json: string
|
|
107
|
+
if (typeof row.value_json === 'string' && row.blob_key === null) json = row.value_json
|
|
108
|
+
else if (row.value_json === null && typeof row.blob_key === 'string' && this.#blobs) {
|
|
109
|
+
const prefix = await this.#blobPrefix(owner)
|
|
110
|
+
if (
|
|
111
|
+
!row.blob_key.startsWith(prefix) ||
|
|
112
|
+
!/^[a-f0-9-]{36}$/u.test(row.blob_key.slice(prefix.length))
|
|
113
|
+
)
|
|
114
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
115
|
+
const blob = await this.#blobs.get(row.blob_key)
|
|
116
|
+
if (!blob || blob.size > this.#maxBytes)
|
|
117
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
118
|
+
const bytes = await blob.arrayBuffer()
|
|
119
|
+
if (bytes.byteLength !== blob.size || bytes.byteLength > this.#maxBytes)
|
|
120
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
121
|
+
json = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
|
|
122
|
+
} else throw new KitFailure('internal', 'Stored document is unreadable')
|
|
123
|
+
if (new TextEncoder().encode(json).byteLength > this.#maxBytes)
|
|
124
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
125
|
+
try {
|
|
126
|
+
const value: unknown = JSON.parse(json)
|
|
127
|
+
return { key: row.key, revision: row.revision, value }
|
|
128
|
+
} catch {
|
|
129
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
async get(owner: Actor, collection: string, key: string): Promise<StoredDocument | null> {
|
|
133
|
+
identifier(key)
|
|
134
|
+
const row = await this.#database
|
|
135
|
+
.prepare(
|
|
136
|
+
'SELECT key, revision, value_json, blob_key FROM aether_documents WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND key = ? AND deleted = 0',
|
|
137
|
+
)
|
|
138
|
+
.bind(...scope(owner, collection), key)
|
|
139
|
+
.first()
|
|
140
|
+
return row === null ? null : this.#parse(owner, row)
|
|
141
|
+
}
|
|
142
|
+
async list(
|
|
143
|
+
owner: Actor,
|
|
144
|
+
collection: string,
|
|
145
|
+
options: DocumentListOptions = {},
|
|
146
|
+
): Promise<readonly StoredDocument[]> {
|
|
147
|
+
const limit = options.limit ?? MAX_LIST
|
|
148
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST)
|
|
149
|
+
throw badRequest('limit', 'invalid', MAX_LIST)
|
|
150
|
+
if (options.after !== undefined) identifier(options.after)
|
|
151
|
+
const result = await this.#database
|
|
152
|
+
.prepare(
|
|
153
|
+
'SELECT key, revision, value_json, blob_key FROM aether_documents WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND deleted = 0 AND key > ? ORDER BY key LIMIT ?',
|
|
154
|
+
)
|
|
155
|
+
.bind(...scope(owner, collection), options.after ?? '', limit)
|
|
156
|
+
.all()
|
|
157
|
+
if (!result.success || !Array.isArray(result.results))
|
|
158
|
+
throw new KitFailure('internal', 'Document listing failed')
|
|
159
|
+
const documents: StoredDocument[] = []
|
|
160
|
+
// Sequential hydration bounds R2 concurrency/memory to one document at a time.
|
|
161
|
+
for (const row of result.results) documents.push(await this.#parse(owner, row))
|
|
162
|
+
return documents
|
|
163
|
+
}
|
|
164
|
+
async compareAndSet(
|
|
165
|
+
owner: Actor,
|
|
166
|
+
collection: string,
|
|
167
|
+
key: string,
|
|
168
|
+
expectedRevision: number | null,
|
|
169
|
+
value: unknown,
|
|
170
|
+
): Promise<StoredDocument> {
|
|
171
|
+
identifier(key)
|
|
172
|
+
if (expectedRevision !== null) revision(expectedRevision)
|
|
173
|
+
const values = scope(owner, collection)
|
|
174
|
+
let json: string | undefined
|
|
175
|
+
try {
|
|
176
|
+
json = JSON.stringify(value)
|
|
177
|
+
} catch {
|
|
178
|
+
throw badRequest('document', 'invalid_json')
|
|
179
|
+
}
|
|
180
|
+
if (json === undefined) throw badRequest('document', 'invalid_json')
|
|
181
|
+
const bytes = new TextEncoder().encode(json)
|
|
182
|
+
if (bytes.byteLength > this.#maxBytes) throw badRequest('document', 'too_large', this.#maxBytes)
|
|
183
|
+
let inline: string | null = json
|
|
184
|
+
let blobKey: string | null = null
|
|
185
|
+
if (bytes.byteLength > INLINE_BYTES) {
|
|
186
|
+
if (!this.#blobs)
|
|
187
|
+
throw new KitFailure('needs_config', 'Large documents require an R2 blob binding')
|
|
188
|
+
blobKey = `${await this.#blobPrefix(owner)}${crypto.randomUUID()}`
|
|
189
|
+
const written = await this.#blobs.put(blobKey, bytes, {
|
|
190
|
+
httpMetadata: { contentType: 'application/json' },
|
|
191
|
+
onlyIf: { etagDoesNotMatch: '*' },
|
|
192
|
+
})
|
|
193
|
+
if (written === null) throw new KitFailure('conflict', 'Document blob already exists')
|
|
194
|
+
inline = null
|
|
195
|
+
}
|
|
196
|
+
const row =
|
|
197
|
+
expectedRevision === null
|
|
198
|
+
? await this.#database
|
|
199
|
+
.prepare(
|
|
200
|
+
'INSERT INTO aether_documents (application_id, tenant_key, subject_id, owner_key, collection, key, revision, value_json, blob_key, deleted, updated_at) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, 0, ?) ON CONFLICT DO NOTHING RETURNING key, revision, value_json, blob_key',
|
|
201
|
+
)
|
|
202
|
+
.bind(...values, key, inline, blobKey, Date.now())
|
|
203
|
+
.first()
|
|
204
|
+
: await this.#database
|
|
205
|
+
.prepare(
|
|
206
|
+
'UPDATE aether_documents SET revision = revision + 1, value_json = ?, blob_key = ?, updated_at = ? WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND key = ? AND revision = ? AND deleted = 0 RETURNING key, revision, value_json, blob_key',
|
|
207
|
+
)
|
|
208
|
+
.bind(inline, blobKey, Date.now(), ...values, key, expectedRevision)
|
|
209
|
+
.first()
|
|
210
|
+
if (row === null) {
|
|
211
|
+
if (blobKey && this.#blobs) await this.#blobs.delete(blobKey)
|
|
212
|
+
throw new KitFailure('conflict', 'Document revision conflict')
|
|
213
|
+
}
|
|
214
|
+
// SQL RETURNING is authoritative; do not reread a row that another writer can change.
|
|
215
|
+
if (typeof row !== 'object' || !('revision' in row) || typeof row.revision !== 'number')
|
|
216
|
+
throw new KitFailure('internal', 'Stored document is unreadable')
|
|
217
|
+
const parsed: unknown = JSON.parse(json)
|
|
218
|
+
return { key, revision: row.revision, value: parsed }
|
|
219
|
+
}
|
|
220
|
+
async remove(
|
|
221
|
+
owner: Actor,
|
|
222
|
+
collection: string,
|
|
223
|
+
key: string,
|
|
224
|
+
expectedRevision: number,
|
|
225
|
+
): Promise<void> {
|
|
226
|
+
identifier(key)
|
|
227
|
+
revision(expectedRevision)
|
|
228
|
+
const row = await this.#database
|
|
229
|
+
.prepare(
|
|
230
|
+
'UPDATE aether_documents SET revision = revision + 1, value_json = NULL, blob_key = NULL, deleted = 1, updated_at = ? WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND key = ? AND revision = ? AND deleted = 0 RETURNING key',
|
|
231
|
+
)
|
|
232
|
+
.bind(Date.now(), ...scope(owner, collection), key, expectedRevision)
|
|
233
|
+
.first()
|
|
234
|
+
if (row === null) throw new KitFailure('conflict', 'Document revision conflict')
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export interface RetryOptions {
|
|
239
|
+
readonly attempts?: number
|
|
240
|
+
readonly baseMs?: number
|
|
241
|
+
/** Injected for deterministic tests; defaults to a real timer. */
|
|
242
|
+
readonly sleep?: (ms: number) => Promise<unknown>
|
|
243
|
+
readonly random?: () => number
|
|
244
|
+
}
|
|
245
|
+
/** Marks the store call inside a `retryCas` attempt; only its revision conflicts are retried. */
|
|
246
|
+
export type CasCall = <T>(write: () => Promise<T>) => Promise<T>
|
|
247
|
+
class RevisionConflict extends Error {}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Runs `attempt` until its CAS write succeeds. Each attempt must reload and re-apply its change
|
|
251
|
+
* to the newest revision. Only a conflict raised by the `cas`-wrapped store call is retried;
|
|
252
|
+
* semantic conflicts from the attempt itself (stale preparation, finished turn) surface at once.
|
|
253
|
+
* Retries back off exponentially with jitter, so contending writers spread out instead of
|
|
254
|
+
* exhausting their attempts in lockstep.
|
|
255
|
+
*/
|
|
256
|
+
export async function retryCas<T>(
|
|
257
|
+
attempt: (cas: CasCall) => Promise<T>,
|
|
258
|
+
exhausted: () => KitFailure,
|
|
259
|
+
options: RetryOptions = {},
|
|
260
|
+
): Promise<T> {
|
|
261
|
+
const attempts = options.attempts ?? 8,
|
|
262
|
+
baseMs = options.baseMs ?? 5,
|
|
263
|
+
random = options.random ?? Math.random
|
|
264
|
+
const sleep =
|
|
265
|
+
options.sleep ??
|
|
266
|
+
((ms: number) => {
|
|
267
|
+
const { promise, resolve } = Promise.withResolvers<void>()
|
|
268
|
+
setTimeout(resolve, ms)
|
|
269
|
+
return promise
|
|
270
|
+
})
|
|
271
|
+
const cas: CasCall = async (write) => {
|
|
272
|
+
try {
|
|
273
|
+
return await write()
|
|
274
|
+
} catch (error) {
|
|
275
|
+
if (
|
|
276
|
+
error instanceof KitFailure &&
|
|
277
|
+
error.code === 'conflict' &&
|
|
278
|
+
error.details?.reason === undefined
|
|
279
|
+
)
|
|
280
|
+
throw new RevisionConflict()
|
|
281
|
+
throw error
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (let i = 0; i < attempts; i++) {
|
|
285
|
+
try {
|
|
286
|
+
return await attempt(cas)
|
|
287
|
+
} catch (error) {
|
|
288
|
+
if (!(error instanceof RevisionConflict)) throw error
|
|
289
|
+
}
|
|
290
|
+
if (i < attempts - 1) await sleep(baseMs * 2 ** i * (0.5 + random()))
|
|
291
|
+
}
|
|
292
|
+
throw exhausted()
|
|
293
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// In-memory stand-ins for the Durable Object and document storage ports, shared by unit tests and
|
|
2
|
+
// the scenario runner. They hold everything in maps and clone on every read and write, so callers
|
|
3
|
+
// never alias stored values.
|
|
4
|
+
import { KitFailure } from '@ouraihub/aether-core'
|
|
5
|
+
import type { DurableObjectState, DurableObjectStorage } from '../cloudflare.js'
|
|
6
|
+
import type { Actor } from '../context.js'
|
|
7
|
+
import type { DocumentStore, StoredDocument } from '../storage.js'
|
|
8
|
+
|
|
9
|
+
/** Durable Object storage; `writes` records every stored value in order, `alarm` the scheduled alarm. */
|
|
10
|
+
export class MemoryStorage implements DurableObjectStorage {
|
|
11
|
+
readonly records = new Map<string, unknown>()
|
|
12
|
+
readonly writes: unknown[] = []
|
|
13
|
+
alarm: number | null = null
|
|
14
|
+
async get<T = unknown>(key: string): Promise<T | undefined> {
|
|
15
|
+
return structuredClone(this.records.get(key)) as T | undefined
|
|
16
|
+
}
|
|
17
|
+
async put<T>(key: string, value: T): Promise<void> {
|
|
18
|
+
this.records.set(key, structuredClone(value))
|
|
19
|
+
this.writes.push(structuredClone(value))
|
|
20
|
+
}
|
|
21
|
+
async delete(key: string): Promise<boolean> {
|
|
22
|
+
return this.records.delete(key)
|
|
23
|
+
}
|
|
24
|
+
async setAlarm(value: number | Date): Promise<void> {
|
|
25
|
+
this.alarm = Number(value)
|
|
26
|
+
}
|
|
27
|
+
async getAlarm(): Promise<number | null> {
|
|
28
|
+
return this.alarm
|
|
29
|
+
}
|
|
30
|
+
async deleteAlarm(): Promise<void> {
|
|
31
|
+
this.alarm = null
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Durable Object state; `drain()` resolves once every `waitUntil` task (including ones they add) settled. */
|
|
36
|
+
export class MemoryState implements DurableObjectState {
|
|
37
|
+
readonly storage: MemoryStorage
|
|
38
|
+
readonly tasks = new Set<Promise<unknown>>()
|
|
39
|
+
constructor(storage = new MemoryStorage()) {
|
|
40
|
+
this.storage = storage
|
|
41
|
+
}
|
|
42
|
+
blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T> {
|
|
43
|
+
return callback()
|
|
44
|
+
}
|
|
45
|
+
waitUntil(promise: Promise<unknown>): void {
|
|
46
|
+
this.tasks.add(promise)
|
|
47
|
+
void promise.finally(() => this.tasks.delete(promise)).catch(() => undefined)
|
|
48
|
+
}
|
|
49
|
+
async drain(): Promise<void> {
|
|
50
|
+
while (this.tasks.size) await Promise.all([...this.tasks])
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Owner-scoped document store with revision compare-and-set; `failConversations` fails conversation writes. */
|
|
55
|
+
export class MemoryDocuments implements DocumentStore {
|
|
56
|
+
readonly documents = new Map<string, StoredDocument>()
|
|
57
|
+
failConversations = false
|
|
58
|
+
private id(owner: Actor, collection: string, key: string): string {
|
|
59
|
+
return JSON.stringify([owner.applicationId, owner.tenantId, owner.subjectId, collection, key])
|
|
60
|
+
}
|
|
61
|
+
async get(owner: Actor, collection: string, key: string): Promise<StoredDocument | null> {
|
|
62
|
+
return structuredClone(this.documents.get(this.id(owner, collection, key)) ?? null)
|
|
63
|
+
}
|
|
64
|
+
async list(owner: Actor, collection: string): Promise<readonly StoredDocument[]> {
|
|
65
|
+
const prefix = JSON.stringify([
|
|
66
|
+
owner.applicationId,
|
|
67
|
+
owner.tenantId,
|
|
68
|
+
owner.subjectId,
|
|
69
|
+
collection,
|
|
70
|
+
]).slice(0, -1)
|
|
71
|
+
return [...this.documents.entries()]
|
|
72
|
+
.filter(([key]) => key.startsWith(prefix))
|
|
73
|
+
.map(([, value]) => structuredClone(value))
|
|
74
|
+
}
|
|
75
|
+
async compareAndSet(
|
|
76
|
+
owner: Actor,
|
|
77
|
+
collection: string,
|
|
78
|
+
key: string,
|
|
79
|
+
expected: number | null,
|
|
80
|
+
value: unknown,
|
|
81
|
+
): Promise<StoredDocument> {
|
|
82
|
+
if (collection === 'conversations' && this.failConversations)
|
|
83
|
+
throw new KitFailure('internal', 'secret-token database failure')
|
|
84
|
+
const id = this.id(owner, collection, key),
|
|
85
|
+
old = this.documents.get(id)
|
|
86
|
+
if ((old?.revision ?? null) !== expected) throw new KitFailure('conflict', 'Concurrent update')
|
|
87
|
+
const saved = { key, revision: (old?.revision ?? 0) + 1, value: structuredClone(value) }
|
|
88
|
+
this.documents.set(id, saved)
|
|
89
|
+
return structuredClone(saved)
|
|
90
|
+
}
|
|
91
|
+
async remove(owner: Actor, collection: string, key: string, expected: number): Promise<void> {
|
|
92
|
+
const id = this.id(owner, collection, key)
|
|
93
|
+
if (this.documents.get(id)?.revision !== expected)
|
|
94
|
+
throw new KitFailure('conflict', 'Concurrent update')
|
|
95
|
+
this.documents.delete(id)
|
|
96
|
+
}
|
|
97
|
+
}
|
package/README.md
DELETED