@ouraihub/aether-worker 0.0.0-stage → 0.1.0

package/src/skills/service.ts

@@ -0,0 +1,203 @@
+import type { ManualSave, SkillRef } from '@ouraihub/aether-contracts'
+import { badRequest, KitFailure } from '@ouraihub/aether-core'
+import { readSkillSource } from '@ouraihub/aether-core/skills/package'
+import * as skills from '@ouraihub/aether-core/skills/service'
+import { exact, object, parseManualSave, revision, strings, text } from '@ouraihub/aether-core/skills/validation'
+import type { Actor, WorkerContext } from '../context.js'
+import type { DocumentStore } from '../storage.js'
+import type { FileStore } from '../files.js'
+import { loadSkills, parseRefs, serializeSkill, type LoadedSkills, type SkillBlobRef } from './persistence.js'
+
+/** All catalog mutations commit in one actor-scoped CAS; conflicts never overwrite newer content. */
+export class SkillsService {
+  private readonly store: DocumentStore
+  private readonly blobs: FileStore
+  private readonly builtins: readonly skills.BuiltinSkill[]
+  constructor(store: DocumentStore, blobs: FileStore, builtins: readonly skills.BuiltinSkill[] = []) {
+    this.store = store
+    this.blobs = blobs
+    this.builtins = builtins
+  }
+
+  private async commit(
+    actor: Actor,
+    previous: LoadedSkills,
+    next: skills.SkillCatalog,
+    expected: number | null,
+  ): Promise<{ refs: readonly SkillBlobRef[]; revision: number }> {
+    const created: string[] = []
+    const refs: SkillBlobRef[] = []
+    try {
+      for (const skill of next) {
+        const oldRef = previous.refs.find((ref) => ref.id === skill.id)
+        if (oldRef && previous.catalog.find((entry) => entry.id === skill.id) === skill) {
+          refs.push(oldRef)
+          continue
+        }
+        const blobId = crypto.randomUUID()
+        await this.blobs.put(actor, blobId, { mime: 'application/json', data: serializeSkill(skill) })
+        created.push(blobId)
+        refs.push({ ...skills.metadata(skill), blobId, digest: skill.digest })
+      }
+      const saved = await this.store.compareAndSet(actor, 'skills', 'catalog', expected, refs)
+      return { refs, revision: saved.revision }
+    } catch (error) {
+      // Only a definite rejection proves these blobs are not live catalog references.
+      if (error instanceof KitFailure && error.code === 'conflict') {
+        await Promise.all(created.map((id) => this.blobs.remove(actor, id)))
+      }
+      throw error
+    }
+  }
+  private async load(actor: Actor, now: number): Promise<LoadedSkills & { revision: number | null }> {
+    const document = await this.store.get(actor, 'skills', 'catalog')
+    const current = document === null ? { catalog: [], refs: [] } : await loadSkills(this.blobs, actor, document.value)
+    const synced = await skills.syncBuiltins(current.catalog, this.builtins, now)
+    if (synced === current.catalog) return { ...current, revision: document?.revision ?? null }
+    const saved = await this.commit(actor, current, synced, document?.revision ?? null)
+    return { catalog: synced, ...saved }
+  }
+  private async mutate(
+    actor: Actor,
+    now: number,
+    operation: (catalog: skills.SkillCatalog) => Promise<skills.SkillCatalog> | skills.SkillCatalog,
+  ) {
+    const current = await this.load(actor, now)
+    const next = await operation(current.catalog)
+    await this.commit(actor, current, next, current.revision)
+    return skills.list(next)
+  }
+  async list(actor: Actor, now: number) {
+    if (this.builtins.length !== 0) return skills.list((await this.load(actor, now)).catalog)
+    const document = await this.store.get(actor, 'skills', 'catalog')
+    const refs = document === null ? [] : parseRefs(document.value)
+    if (refs.some((ref) => ref.origin === 'builtin')) return skills.list((await this.load(actor, now)).catalog)
+    return refs.map((ref) => ({
+      id: ref.id,
+      name: ref.name,
+      description: ref.description,
+      version: ref.version,
+      origin: ref.origin,
+      revision: ref.revision,
+      bodyChars: ref.bodyChars,
+      updatedAt: ref.updatedAt,
+      hasScripts: ref.hasScripts,
+      fileCount: ref.fileCount,
+    }))
+  }
+  async get(actor: Actor, id: string, now: number) {
+    return skills.get((await this.load(actor, now)).catalog, id)
+  }
+  async files(actor: Actor, id: string, expected: number, now: number) {
+    return skills.files((await this.load(actor, now)).catalog, id, expected)
+  }
+  async file(actor: Actor, id: string, expected: number, path: string, now: number) {
+    return skills.file((await this.load(actor, now)).catalog, id, expected, path)
+  }
+  async snapshot(actor: Actor, refs: readonly SkillRef[], now: number) {
+    return skills.snapshot((await this.load(actor, now)).catalog, refs)
+  }
+  async preview(actor: Actor, source: unknown, now: number) {
+    const content = await readSkillSource(source)
+    return skills.preview((await this.load(actor, now)).catalog, content)
+  }
+  async install(actor: Actor, source: unknown, confirmedDigest: string, replace: boolean, newId: string, now: number) {
+    const content = await readSkillSource(source)
+    return this.mutate(actor, now, (catalog) => skills.install(catalog, content, confirmedDigest, replace, newId, now))
+  }
+  async save(actor: Actor, input: ManualSave, newId: string, now: number) {
+    const checked = parseManualSave(input)
+    return this.mutate(actor, now, (catalog) => skills.save(catalog, checked, newId, now))
+  }
+  async duplicate(actor: Actor, id: string, newName: string, newId: string, now: number) {
+    return this.mutate(actor, now, (catalog) => skills.duplicate(catalog, id, newName, newId, now))
+  }
+  async reorder(actor: Actor, ids: readonly string[], now: number) {
+    return this.mutate(actor, now, (catalog) => skills.reorder(catalog, ids))
+  }
+  async remove(actor: Actor, id: string, now: number) {
+    return this.mutate(actor, now, (catalog) => skills.remove(catalog, id))
+  }
+
+  async handle(request: Request, context: WorkerContext): Promise<Response | null> {
+    const url = new URL(request.url)
+    const base = '/api/v1/skills'
+    if (url.pathname !== base && !url.pathname.startsWith(`${base}/`)) return null
+    const { actor, now } = context
+    if (context.signal.aborted) throw new KitFailure('cancelled', 'Cancelled')
+    const path = url.pathname.slice(base.length)
+    if (request.method === 'GET') {
+      if (path === '') return Response.json(await this.list(actor, now))
+      const parts = path.slice(1).split('/')
+      const id = decodeURIComponent(parts[0] ?? '')
+      if (parts.length === 1) return Response.json(await this.get(actor, id, now))
+      const expected = revision(Number(url.searchParams.get('revision')))
+      if (parts.length === 2 && parts[1] === 'files') return Response.json(await this.files(actor, id, expected, now))
+      if (parts.length === 2 && parts[1] === 'file') {
+        const bytes = await this.file(actor, id, expected, text(url.searchParams.get('path'), 'path'), now)
+        return new Response(bytes, {
+          headers: { 'Content-Type': 'application/octet-stream', 'X-Content-Type-Options': 'nosniff' },
+        })
+      }
+    }
+    if (request.method === 'DELETE' && /^\/[^/]+$/u.test(path))
+      return Response.json(await this.remove(actor, decodeURIComponent(path.slice(1)), now))
+    if (request.method === 'POST') {
+      let raw: unknown
+      try {
+        raw = await request.json()
+      } catch {
+        throw badRequest('body', 'invalid_json')
+      }
+      if (path === '/save') return Response.json(await this.save(actor, parseManualSave(raw), crypto.randomUUID(), now))
+      const body = object(raw, 'body')
+      if (path === '/preview') {
+        exact(body, ['source'], 'body')
+        return Response.json(await this.preview(actor, body.source, now))
+      }
+      if (path === '/install') {
+        exact(body, ['source', 'confirmedDigest', 'replace'], 'body')
+        if (typeof body.replace !== 'boolean') throw badRequest('replace', 'invalid')
+        return Response.json(
+          await this.install(
+            actor,
+            body.source,
+            text(body.confirmedDigest, 'confirmedDigest'),
+            body.replace,
+            crypto.randomUUID(),
+            now,
+          ),
+        )
+      }
+      if (path === '/reorder') {
+        exact(body, ['ids'], 'body')
+        return Response.json(await this.reorder(actor, strings(body.ids, 'ids'), now))
+      }
+      if (path === '/snapshot') {
+        exact(body, ['refs'], 'body')
+        const refs: unknown = body.refs
+        if (!Array.isArray(refs)) throw badRequest('refs', 'invalid')
+        const checked = refs.map((value: unknown) => {
+          const ref = object(value, 'ref')
+          exact(ref, ['id', 'revision'], 'ref')
+          return { id: text(ref.id, 'id'), revision: revision(ref.revision) }
+        })
+        return Response.json(await this.snapshot(actor, checked, now))
+      }
+      const duplicate = /^\/([^/]+)\/duplicate$/u.exec(path)
+      if (duplicate?.[1] !== undefined) {
+        exact(body, ['newName'], 'body')
+        return Response.json(
+          await this.duplicate(
+            actor,
+            decodeURIComponent(duplicate[1]),
+            text(body.newName, 'newName'),
+            crypto.randomUUID(),
+            now,
+          ),
+        )
+      }
+    }
+    throw badRequest('route', 'unsupported')
+  }
+}

package/src/storage.ts

@@ -0,0 +1,198 @@
+import { badRequest, KitFailure } from '@ouraihub/aether-core'
+import { ownerKey, type Actor } from './context.ts'
+import type { D1Database, R2Bucket } from './cloudflare.ts'
+
+export interface StoredDocument {
+  readonly key: string
+  readonly revision: number
+  readonly value: unknown
+}
+export interface DocumentListOptions {
+  readonly after?: string
+  readonly limit?: number
+}
+export interface DocumentStore {
+  get(owner: Actor, collection: string, key: string): Promise<StoredDocument | null>
+  list(owner: Actor, collection: string, options?: DocumentListOptions): Promise<readonly StoredDocument[]>
+  compareAndSet(
+    owner: Actor,
+    collection: string,
+    key: string,
+    expectedRevision: number | null,
+    value: unknown,
+  ): Promise<StoredDocument>
+  remove(owner: Actor, collection: string, key: string, expectedRevision: number): Promise<void>
+}
+const INLINE_BYTES = 256 * 1024
+const MAX_LIST = 200
+function identifier(value: string): void {
+  if (!value || value.length > 256) throw badRequest('document', 'invalid_identifier')
+}
+function revision(value: number): void {
+  if (!Number.isSafeInteger(value) || value < 1 || value >= Number.MAX_SAFE_INTEGER)
+    throw badRequest('revision', 'invalid')
+}
+function scope(owner: Actor, collection: string): readonly [string, string, string, string, string] {
+  identifier(collection)
+  return [owner.applicationId, JSON.stringify(owner.tenantId), owner.subjectId, ownerKey(owner), collection]
+}
+export async function validateD1Schema(database: D1Database): Promise<void> {
+  let row: unknown
+  try {
+    row = await database.prepare('SELECT version FROM aether_schema WHERE singleton = 1').first()
+  } catch {
+    throw new KitFailure('incompatible_db', 'Apply the supported Aether public migration before startup')
+  }
+  if (typeof row !== 'object' || row === null || !('version' in row) || row.version !== 1) {
+    throw new KitFailure('incompatible_db', 'Apply the supported Aether public migration before startup')
+  }
+}
+/** D1 rows stay below its 2,000,000 byte limit. Large functional documents live
+ * in immutable scoped R2 blobs; credentials inside documents MUST already be sealed.
+ * Tombstones reserve deleted keys. Superseded/uncertain-commit blobs stay for explicit
+ * deployment GC: deleting them inline would race readers or a committed SQL response.
+ */
+export class D1DocumentStore implements DocumentStore {
+  readonly #database: D1Database
+  readonly #blobs: R2Bucket | undefined
+  readonly #maxBytes: number
+  constructor(database: D1Database, options: { readonly blobs?: R2Bucket; readonly maxDocumentBytes?: number } = {}) {
+    this.#database = database
+    this.#blobs = options.blobs
+    this.#maxBytes = options.maxDocumentBytes ?? 64 * 1024 * 1024
+    if (!Number.isSafeInteger(this.#maxBytes) || this.#maxBytes < INLINE_BYTES)
+      throw badRequest('maxDocumentBytes', 'invalid')
+  }
+  async #blobPrefix(owner: Actor): Promise<string> {
+    const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(ownerKey(owner)))
+    return `aether/documents/v1/${Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join('')}/`
+  }
+  async #parse(owner: Actor, row: unknown): Promise<StoredDocument> {
+    if (
+      typeof row !== 'object' ||
+      row === null ||
+      !('key' in row) ||
+      typeof row.key !== 'string' ||
+      !('revision' in row) ||
+      typeof row.revision !== 'number' ||
+      !Number.isSafeInteger(row.revision) ||
+      row.revision < 1 ||
+      !('value_json' in row) ||
+      !('blob_key' in row)
+    )
+      throw new KitFailure('internal', 'Stored document is unreadable')
+    let json: string
+    if (typeof row.value_json === 'string' && row.blob_key === null) json = row.value_json
+    else if (row.value_json === null && typeof row.blob_key === 'string' && this.#blobs) {
+      const prefix = await this.#blobPrefix(owner)
+      if (!row.blob_key.startsWith(prefix) || !/^[a-f0-9-]{36}$/u.test(row.blob_key.slice(prefix.length)))
+        throw new KitFailure('internal', 'Stored document is unreadable')
+      const blob = await this.#blobs.get(row.blob_key)
+      if (!blob || blob.size > this.#maxBytes) throw new KitFailure('internal', 'Stored document is unreadable')
+      const bytes = await blob.arrayBuffer()
+      if (bytes.byteLength !== blob.size || bytes.byteLength > this.#maxBytes)
+        throw new KitFailure('internal', 'Stored document is unreadable')
+      json = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
+    } else throw new KitFailure('internal', 'Stored document is unreadable')
+    if (new TextEncoder().encode(json).byteLength > this.#maxBytes)
+      throw new KitFailure('internal', 'Stored document is unreadable')
+    try {
+      const value: unknown = JSON.parse(json)
+      return { key: row.key, revision: row.revision, value }
+    } catch {
+      throw new KitFailure('internal', 'Stored document is unreadable')
+    }
+  }
+  async get(owner: Actor, collection: string, key: string): Promise<StoredDocument | null> {
+    identifier(key)
+    const row = await this.#database
+      .prepare(
+        'SELECT key, revision, value_json, blob_key FROM aether_documents WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND key = ? AND deleted = 0',
+      )
+      .bind(...scope(owner, collection), key)
+      .first()
+    return row === null ? null : this.#parse(owner, row)
+  }
+  async list(owner: Actor, collection: string, options: DocumentListOptions = {}): Promise<readonly StoredDocument[]> {
+    const limit = options.limit ?? MAX_LIST
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST) throw badRequest('limit', 'invalid', MAX_LIST)
+    if (options.after !== undefined) identifier(options.after)
+    const result = await this.#database
+      .prepare(
+        'SELECT key, revision, value_json, blob_key FROM aether_documents WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND deleted = 0 AND key > ? ORDER BY key LIMIT ?',
+      )
+      .bind(...scope(owner, collection), options.after ?? '', limit)
+      .all()
+    if (!result.success || !Array.isArray(result.results)) throw new KitFailure('internal', 'Document listing failed')
+    const documents: StoredDocument[] = []
+    // Sequential hydration bounds R2 concurrency/memory to one document at a time.
+    for (const row of result.results) documents.push(await this.#parse(owner, row))
+    return documents
+  }
+  async compareAndSet(
+    owner: Actor,
+    collection: string,
+    key: string,
+    expectedRevision: number | null,
+    value: unknown,
+  ): Promise<StoredDocument> {
+    identifier(key)
+    if (expectedRevision !== null) revision(expectedRevision)
+    const values = scope(owner, collection)
+    let json: string | undefined
+    try {
+      json = JSON.stringify(value)
+    } catch {
+      throw badRequest('document', 'invalid_json')
+    }
+    if (json === undefined) throw badRequest('document', 'invalid_json')
+    const bytes = new TextEncoder().encode(json)
+    if (bytes.byteLength > this.#maxBytes) throw badRequest('document', 'too_large', this.#maxBytes)
+    let inline: string | null = json
+    let blobKey: string | null = null
+    if (bytes.byteLength > INLINE_BYTES) {
+      if (!this.#blobs) throw new KitFailure('needs_config', 'Large documents require an R2 blob binding')
+      blobKey = `${await this.#blobPrefix(owner)}${crypto.randomUUID()}`
+      const written = await this.#blobs.put(blobKey, bytes, {
+        httpMetadata: { contentType: 'application/json' },
+        onlyIf: { etagDoesNotMatch: '*' },
+      })
+      if (written === null) throw new KitFailure('conflict', 'Document blob already exists')
+      inline = null
+    }
+    const row =
+      expectedRevision === null
+        ? await this.#database
+            .prepare(
+              'INSERT INTO aether_documents (application_id, tenant_key, subject_id, owner_key, collection, key, revision, value_json, blob_key, deleted, updated_at) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, 0, ?) ON CONFLICT DO NOTHING RETURNING key, revision, value_json, blob_key',
+            )
+            .bind(...values, key, inline, blobKey, Date.now())
+            .first()
+        : await this.#database
+            .prepare(
+              'UPDATE aether_documents SET revision = revision + 1, value_json = ?, blob_key = ?, updated_at = ? WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND key = ? AND revision = ? AND deleted = 0 RETURNING key, revision, value_json, blob_key',
+            )
+            .bind(inline, blobKey, Date.now(), ...values, key, expectedRevision)
+            .first()
+    if (row === null) {
+      if (blobKey && this.#blobs) await this.#blobs.delete(blobKey)
+      throw new KitFailure('conflict', 'Document revision conflict')
+    }
+    // SQL RETURNING is authoritative; do not reread a row that another writer can change.
+    if (typeof row !== 'object' || !('revision' in row) || typeof row.revision !== 'number')
+      throw new KitFailure('internal', 'Stored document is unreadable')
+    const parsed: unknown = JSON.parse(json)
+    return { key, revision: row.revision, value: parsed }
+  }
+  async remove(owner: Actor, collection: string, key: string, expectedRevision: number): Promise<void> {
+    identifier(key)
+    revision(expectedRevision)
+    const row = await this.#database
+      .prepare(
+        'UPDATE aether_documents SET revision = revision + 1, value_json = NULL, blob_key = NULL, deleted = 1, updated_at = ? WHERE application_id = ? AND tenant_key = ? AND subject_id = ? AND owner_key = ? AND collection = ? AND key = ? AND revision = ? AND deleted = 0 RETURNING key',
+      )
+      .bind(Date.now(), ...scope(owner, collection), key, expectedRevision)
+      .first()
+    if (row === null) throw new KitFailure('conflict', 'Document revision conflict')
+  }
+}

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.