@ixo/oracle-runtime-workers 0.2.0

package/src/owner-store/migrating-store.ts

@@ -0,0 +1,185 @@
+/**
+ * The default owner store: **IXO VFS is the system of record**; the user's
+ * Matrix room media is only a read-only *legacy* source, checked when VFS has
+ * no file yet so users coming from the Node runtime keep their history.
+ *
+ *   load():  VFS first. If absent there but present in Matrix media, import
+ *            the legacy file AND write it to VFS immediately — from that
+ *            moment the VFS copy is authoritative and Matrix is never
+ *            consulted again (head() sees the VFS file).
+ *   save():  VFS, and only VFS. Nothing is ever written to Matrix media any
+ *            more: a failed VFS write (no `ixo:filesystem` delegation yet, a
+ *            VFS outage) propagates, and the object keeps its working copy
+ *            dirty in Durable Object storage and retries.
+ *   head():  VFS if a file exists there, else legacy.
+ *   remove(): both (the user asked to be forgotten).
+ *   removeLegacyCopy(etag): once the VFS copy is confirmed (its etag matches
+ *            what the caller just uploaded or loaded), redact the legacy
+ *            Matrix copy so no second copy of the user's history lingers.
+ */
+import { VfsNoDelegationError } from './ixo-vfs-store';
+import {
+  bytesOfStream,
+  snapshotOfBytes,
+  streamOfBytes,
+  type FileSnapshot,
+  type OwnerCopy,
+  type OwnerStore,
+  type SaveResult,
+} from './types';
+
+export interface MigratingOwnerStoreOptions {
+  /** System of record (IXO VFS). */
+  primary: OwnerStore;
+  /** Read-mostly legacy source (Matrix room media). */
+  legacy: OwnerStore;
+  log?: (level: 'log' | 'warn' | 'error', message: string) => void;
+}
+
+export class MigratingOwnerStore implements OwnerStore {
+  readonly kind = 'vfs' as const;
+  private readonly primary: OwnerStore;
+  private readonly legacy: OwnerStore;
+  private readonly log: (
+    level: 'log' | 'warn' | 'error',
+    message: string,
+  ) => void;
+
+  constructor(opts: MigratingOwnerStoreOptions) {
+    this.primary = opts.primary;
+    this.legacy = opts.legacy;
+    this.log =
+      opts.log ??
+      ((level, message) => {
+        // eslint-disable-next-line no-console -- console IS the logger on Workers
+        console[level](message);
+      });
+  }
+
+  async load(): Promise<OwnerCopy | null> {
+    // "Not on VFS yet" (no ixo:filesystem delegation deposited) means *no VFS
+    // copy* — fall through to the legacy source so the user's boot works and
+    // their Matrix history stays reachable. Any OTHER failure (store/VFS
+    // outage) still throws: silently serving a stale legacy copy to a user
+    // whose system of record IS the VFS would fork their history.
+    let fromVfs: OwnerCopy | null;
+    let noDelegation: VfsNoDelegationError | null = null;
+    try {
+      fromVfs = await this.primary.load();
+    } catch (err) {
+      if (!(err instanceof VfsNoDelegationError)) throw err;
+      this.log(
+        'warn',
+        `[owner-store] ${err.message} — this user is not on VFS yet; checking legacy Matrix media`,
+      );
+      noDelegation = err;
+      fromVfs = null;
+    }
+    if (fromVfs) return fromVfs;
+
+    const fromLegacy = await this.legacy.load().catch((err) => {
+      this.log(
+        'warn',
+        `[owner-store] legacy Matrix load failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      return null;
+    });
+    if (!fromLegacy) {
+      // Without a delegation VFS could not be READ, so "no legacy copy" is
+      // not "no file": an existing user whose grant lapsed would otherwise
+      // start with an empty history (and later flush it over the real
+      // copy). Only a successful VFS listing may say the user has no file.
+      if (noDelegation) throw noDelegation;
+      return null;
+    }
+
+    // One-time migration: the legacy file becomes the first VFS version. The
+    // legacy copy is already in memory (gateway RPCs hand over bytes), so
+    // re-opening it for the VFS upload costs nothing extra.
+    const legacyBytes = await bytesOfStream(fromLegacy.stream);
+    try {
+      const { etag } = await this.primary.save(snapshotOfBytes(legacyBytes));
+      this.log(
+        'log',
+        `[owner-store] migrated ${legacyBytes.byteLength} bytes from Matrix media to VFS (${etag})`,
+      );
+      return { stream: streamOfBytes(legacyBytes), etag };
+    } catch (err) {
+      this.log(
+        'warn',
+        `[owner-store] VFS migration write failed — serving the legacy Matrix copy for now: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      return { stream: streamOfBytes(legacyBytes), etag: fromLegacy.etag };
+    }
+  }
+
+  /**
+   * Write to the system of record. Every failure propagates — including
+   * "not on VFS yet" (no delegation) — so the object keeps the copy dirty
+   * and retries; Matrix media is never written.
+   */
+  async save(snapshot: FileSnapshot): Promise<SaveResult> {
+    return this.primary.save(snapshot);
+  }
+
+  /**
+   * Delete the legacy Matrix copy once VFS is confirmed to hold the file:
+   * the VFS `head()` etag must equal `expectedEtag` (the etag the caller just
+   * uploaded or loaded), otherwise nothing is touched. Returns true when a
+   * legacy copy existed and was removed.
+   */
+  async removeLegacyCopy(expectedEtag: string): Promise<boolean> {
+    const legacyHead = await this.legacy.head();
+    if (!legacyHead) return false;
+    const vfsHead = await this.primary.head();
+    if (!vfsHead || vfsHead.etag !== expectedEtag) {
+      this.log(
+        'warn',
+        `[owner-store] legacy Matrix copy kept: VFS copy not confirmed (have ${vfsHead?.etag ?? 'none'}, expected ${expectedEtag})`,
+      );
+      return false;
+    }
+    await this.legacy.remove('Migrated to IXO VFS');
+    this.log(
+      'log',
+      `[owner-store] removed the legacy Matrix copy (${legacyHead.etag}); VFS ${vfsHead.etag} is the only copy now`,
+    );
+    return true;
+  }
+
+  async head(): Promise<{ etag: string } | null> {
+    // Errors PROPAGATE: a caller must be able to tell "the file is absent"
+    // (null) from "could not check" (throw). Swallowing a VFS error to null
+    // here would let a boot read a transient outage as an upstream deletion.
+    // Only fall through to legacy when VFS positively has no file.
+    const vfs = await this.primary.head();
+    if (vfs) return vfs;
+    return this.legacy.head().catch(() => null);
+  }
+
+  /** The legacy Matrix copy regardless of what VFS holds (see `OwnerStore.loadLegacy`). */
+  async loadLegacy(): Promise<OwnerCopy | null> {
+    return this.legacy.load().catch((err) => {
+      this.log(
+        'warn',
+        `[owner-store] legacy Matrix load failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      return null;
+    });
+  }
+
+  async remove(): Promise<void> {
+    await this.primary.remove().catch((err) => {
+      this.log(
+        'warn',
+        `[owner-store] VFS remove failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    });
+    await this.legacy.remove().catch((err) => {
+      this.log(
+        'warn',
+        `[owner-store] legacy Matrix remove failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    });
+  }
+}

package/src/owner-store/owner-copy-errors.ts

@@ -0,0 +1,175 @@
+/**
+ * Why a user's owner copy could not be loaded on a cold boot, classified so
+ * the object retries only what can succeed on its own and the shell can
+ * answer with an honest status code:
+ *
+ *   - `NO_VFS_DELEGATION` (403, not retryable): the user has not deposited an
+ *     `ixo:filesystem` delegation for this oracle. Retrying cannot help; the
+ *     client must deposit the grant and try again.
+ *   - `VFS_AUTH_FAILED` (403, not retryable): the VFS rejected the oracle's
+ *     credentials (401/403). Something is misconfigured; retrying cannot help.
+ *   - `OWNER_COPY_UNAVAILABLE` (503, retryable): the UCAN store or the VFS was
+ *     unreachable or failing (network, 5xx, 429, timeout). The object already
+ *     retried with backoff; the next request tries again from scratch.
+ *
+ * In every case nothing was written — the boot is not memoised on failure.
+ */
+import {
+  VfsNoDelegationError,
+  VfsRequestError,
+  VfsStoreUnavailableError,
+} from './ixo-vfs-store';
+
+export type OwnerCopyErrorCode =
+  | 'NO_VFS_DELEGATION'
+  | 'VFS_AUTH_FAILED'
+  | 'OWNER_COPY_UNAVAILABLE';
+
+/** Backoff between cold-boot load attempts: three retries, ~7 s in total. */
+export const OWNER_COPY_LOAD_RETRY_DELAYS_MS: readonly number[] = [
+  1_000, 2_000, 4_000,
+];
+
+export class OwnerCopyUnavailableError extends Error {
+  readonly code: OwnerCopyErrorCode;
+
+  readonly httpStatus: 403 | 503;
+
+  readonly retryable: boolean;
+
+  readonly userDid: string;
+
+  constructor(userDid: string, cause: unknown, attempts: number) {
+    const code = classifyOwnerCopyError(cause);
+    const detail = cause instanceof Error ? cause.message : String(cause);
+    super(messageFor(code, detail, attempts));
+    this.name = 'OwnerCopyUnavailableError';
+    this.code = code;
+    this.httpStatus = code === 'OWNER_COPY_UNAVAILABLE' ? 503 : 403;
+    this.retryable = code === 'OWNER_COPY_UNAVAILABLE';
+    this.userDid = userDid;
+  }
+}
+
+/** Whether a failed load may succeed if simply tried again shortly. */
+export function isRetryableOwnerCopyError(error: unknown): boolean {
+  return classifyOwnerCopyError(error) === 'OWNER_COPY_UNAVAILABLE';
+}
+
+export function classifyOwnerCopyError(error: unknown): OwnerCopyErrorCode {
+  if (error instanceof VfsNoDelegationError) return 'NO_VFS_DELEGATION';
+  if (
+    error instanceof VfsRequestError &&
+    (error.status === 401 || error.status === 403)
+  )
+    return 'VFS_AUTH_FAILED';
+  return 'OWNER_COPY_UNAVAILABLE';
+}
+
+function messageFor(
+  code: OwnerCopyErrorCode,
+  detail: string,
+  attempts: number,
+): string {
+  switch (code) {
+    case 'NO_VFS_DELEGATION':
+      return `Your oracle data cannot be loaded because this oracle has no file-storage permission from you yet. Deposit an ixo:filesystem delegation for it, then try again. Nothing was changed. (${detail})`;
+    case 'VFS_AUTH_FAILED':
+      return `Your oracle data cannot be loaded because your file storage rejected the oracle's credentials. Nothing was changed. (${detail})`;
+    default:
+      return `Your oracle data could not be loaded from your file storage after ${attempts} attempt(s). Nothing was changed — please try again in a moment. (${detail})`;
+  }
+}
+
+// Re-exported so the shell can map the transient store failure too.
+export { VfsStoreUnavailableError };
+
+/** What the shell needs to answer a request whose owner copy could not load. */
+export interface OwnerCopyFailure {
+  code: OwnerCopyErrorCode;
+  httpStatus: 403 | 503;
+  retryable: boolean;
+  message: string;
+}
+
+const RPC_ENVELOPE_TAG = 'owner-copy';
+
+interface RpcEnvelope extends OwnerCopyFailure {
+  oracleError: typeof RPC_ENVELOPE_TAG;
+}
+
+/**
+ * Durable Object RPC serialises a thrown error as a plain `Error` carrying
+ * only its message: the class, `name` and custom fields are gone on the
+ * caller's side. The user object therefore throws the failure as a JSON
+ * envelope in the message, and the shell / turn handler read it back with
+ * `parseOwnerCopyFailure`. Inside the object `instanceof` still works.
+ */
+export function toRpcError(failure: OwnerCopyUnavailableError): Error {
+  const envelope: RpcEnvelope = {
+    oracleError: RPC_ENVELOPE_TAG,
+    code: failure.code,
+    httpStatus: failure.httpStatus,
+    retryable: failure.retryable,
+    message: failure.message,
+  };
+  const err = new Error(JSON.stringify(envelope));
+  err.name = failure.name;
+  return err;
+}
+
+function isEnvelope(value: unknown): value is RpcEnvelope {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    (value as { oracleError?: unknown }).oracleError === RPC_ENVELOPE_TAG &&
+    typeof (value as { code?: unknown }).code === 'string' &&
+    typeof (value as { message?: unknown }).message === 'string'
+  );
+}
+
+/**
+ * Recover the typed failure from anything the object may have thrown: the
+ * class itself (same isolate), its RPC envelope (across the stub), or the
+ * raw store errors when they escape before classification. `null` for
+ * everything else (a genuine 500).
+ */
+export function parseOwnerCopyFailure(error: unknown): OwnerCopyFailure | null {
+  if (error instanceof OwnerCopyUnavailableError) {
+    return {
+      code: error.code,
+      httpStatus: error.httpStatus,
+      retryable: error.retryable,
+      message: error.message,
+    };
+  }
+  if (!(error instanceof Error)) return null;
+  if (error.message.startsWith('{')) {
+    try {
+      const parsed: unknown = JSON.parse(error.message);
+      if (isEnvelope(parsed)) {
+        const { code, httpStatus, retryable, message } = parsed;
+        return { code, httpStatus, retryable, message };
+      }
+    } catch {
+      // not an envelope
+    }
+  }
+  if (error.name === 'VfsNoDelegationError') {
+    return {
+      code: 'NO_VFS_DELEGATION',
+      httpStatus: 403,
+      retryable: false,
+      message: error.message,
+    };
+  }
+  if (error.name === 'VfsStoreUnavailableError') {
+    return {
+      code: 'OWNER_COPY_UNAVAILABLE',
+      httpStatus: 503,
+      retryable: true,
+      message: error.message,
+    };
+  }
+  return null;
+}

package/src/owner-store/retry.ts

@@ -0,0 +1,45 @@
+/**
+ * Bounded retry with fixed backoff for owner-store requests. A flush that
+ * still fails after these attempts is left for the next alarm tick — the
+ * working copy stays dirty, nothing is lost.
+ */
+
+export const DEFAULT_RETRY_DELAYS_MS: readonly number[] = [
+  2_000, 5_000, 15_000,
+];
+
+export interface RetryOptions {
+  /** Delay before retry n (length = number of retries). */
+  delaysMs?: readonly number[];
+  isRetryable: (error: unknown) => boolean;
+  sleep?: (ms: number) => Promise<void>;
+  onRetry?: (error: unknown, attempt: number, delayMs: number) => void;
+}
+
+export async function withRetry<T>(
+  op: (attempt: number) => Promise<T>,
+  options: RetryOptions,
+): Promise<T> {
+  const delays = options.delaysMs ?? DEFAULT_RETRY_DELAYS_MS;
+  const sleep =
+    options.sleep ??
+    ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await op(attempt);
+    } catch (error) {
+      const delay = delays[attempt];
+      if (delay === undefined || !options.isRetryable(error)) throw error;
+      options.onRetry?.(error, attempt + 1, delay);
+      await sleep(delay);
+    }
+  }
+}
+
+/** Network-level failures (`fetch` rejections, aborts) are always retryable. */
+export function isNetworkError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  if (error.name === 'AbortError' || error.name === 'TimeoutError') return true;
+  // workerd/undici surface connection failures as TypeError('fetch failed' | 'Network connection lost' | ...).
+  return error.name === 'TypeError';
+}