@ixo/oracle-runtime-workers 0.13.0 → 0.16.0

package/src/attachments/sandbox-archive.ts

@@ -33,14 +33,15 @@ export async function uploadToSandbox(
   const safeFilename = sanitizeSandboxPath(filename);
   const safePath = sanitizeSandboxPath(destPath);
   const formData = new FormData();
-  // Copy into a fresh ArrayBuffer-backed view so the Blob never sees a
-  // SharedArrayBuffer-typed slice (the DOM lib types reject those).
-  const copy = new Uint8Array(bytes.byteLength);
-  copy.set(bytes);
-  formData.set(
-    'file',
-    new File([copy.buffer], safeFilename, { type: mimetype }),
-  );
+  // A view over a plain ArrayBuffer goes into the Blob as it is (no second
+  // copy of a file of up to 25 MB); only a SharedArrayBuffer-backed one —
+  // which the Blob types reject — is copied.
+  const buffer = bytes.buffer;
+  const part =
+    buffer instanceof ArrayBuffer
+      ? new Uint8Array(buffer, bytes.byteOffset, bytes.byteLength)
+      : Uint8Array.from(bytes);
+  formData.set('file', new File([part], safeFilename, { type: mimetype }));
   formData.set('path', safePath);
   const response = await fetchImpl(`${baseUrl}/artifacts/upload`, {
     method: 'POST',

package/src/attachments/view-cache.ts

@@ -0,0 +1,142 @@
+/**
+ * Text a `view_attachment` call extracted, kept in the user's own SQLite
+ * file per (session, attachment reference, extraction model) so a later
+ * view of the same file costs neither a download nor a helper-model call.
+ * Only Matrix media is cached: an `mxc://` URI or event never changes,
+ * whereas what an http(s) URL serves can. Entries expire after
+ * `VIEW_CACHE_MAX_AGE_MS`, a session keeps its newest
+ * `VIEW_CACHE_MAX_PER_SESSION`, both enforced on write, and a deleted
+ * session takes its entries with it (`forgetSession`).
+ */
+import type { DoSqliteDatabase } from '../sqlite/database';
+import type { Logger } from '../plugin-api/types';
+
+/** One session's cache, as `viewAttachment` sees it. */
+export interface AttachmentTextCache {
+  get(ref: string, model: string): Promise<string | undefined>;
+  put(ref: string, model: string, text: string): Promise<void>;
+}
+
+/**
+ * The longest text kept per entry (characters). Documents are extracted up
+ * to `MAX_TEXT_LENGTH` (50 000) plus a header, so this covers them; a longer
+ * description is not cached and is extracted again on the next view.
+ */
+export const VIEW_CACHE_MAX_CHARS = 64 * 1024;
+
+/** How long an entry is served and kept. */
+export const VIEW_CACHE_MAX_AGE_MS = 30 * 24 * 3600 * 1000;
+
+/** Entries a session keeps (the newest). */
+export const VIEW_CACHE_MAX_PER_SESSION = 50;
+
+/** What an http(s) URL serves can change between views. */
+function isCacheable(ref: string): boolean {
+  return !/^https?:\/\//i.test(ref);
+}
+
+/** The "model" key of plain text decoded locally (no helper model involved). */
+export const LOCAL_TEXT_EXTRACTION = 'local-text';
+
+export class AttachmentTextCacheStore {
+  private setupPromise: Promise<void> | undefined;
+
+  constructor(
+    private readonly db: Pick<DoSqliteDatabase, 'run' | 'get'>,
+    private readonly log: Pick<Logger, 'warn'>,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  private setup(): Promise<void> {
+    this.setupPromise ??= this.createTables().catch((error: unknown) => {
+      this.setupPromise = undefined;
+      throw error;
+    });
+    return this.setupPromise;
+  }
+
+  private async createTables(): Promise<void> {
+    await this.db.run(
+      `CREATE TABLE IF NOT EXISTS attachment_text_cache (
+         session_id TEXT NOT NULL,
+         ref TEXT NOT NULL,
+         model TEXT NOT NULL,
+         text TEXT NOT NULL,
+         created_at INTEGER NOT NULL,
+         UNIQUE (session_id, ref, model)
+       )`,
+    );
+    await this.db.run(
+      `CREATE INDEX IF NOT EXISTS idx_attachment_text_cache_age
+       ON attachment_text_cache(created_at)`,
+    );
+    await this.db.run(
+      `CREATE INDEX IF NOT EXISTS idx_attachment_text_cache_session
+       ON attachment_text_cache(session_id, created_at)`,
+    );
+  }
+
+  /**
+   * The cache of one session. Its failures are logged and treated as a miss
+   * (or a skipped write): a cache must never fail the view itself.
+   */
+  forSession(sessionId: string): AttachmentTextCache {
+    return {
+      get: async (ref, model) => {
+        if (!isCacheable(ref)) return undefined;
+        try {
+          await this.setup();
+          const row = await this.db.get<{ text: string }>(
+            `SELECT text FROM attachment_text_cache
+             WHERE session_id = ? AND ref = ? AND model = ? AND created_at > ?`,
+            [sessionId, ref, model, this.now() - VIEW_CACHE_MAX_AGE_MS],
+          );
+          return row?.text;
+        } catch (error) {
+          this.log.warn(
+            `[attachments] view cache read failed: ${error instanceof Error ? error.message : String(error)}`,
+          );
+          return undefined;
+        }
+      },
+      put: async (ref, model, text) => {
+        if (!isCacheable(ref) || text.length > VIEW_CACHE_MAX_CHARS) return;
+        try {
+          await this.setup();
+          const now = this.now();
+          await this.db.run(
+            `INSERT OR REPLACE INTO attachment_text_cache
+               (session_id, ref, model, text, created_at) VALUES (?, ?, ?, ?, ?)`,
+            [sessionId, ref, model, text, now],
+          );
+          await this.db.run(
+            `DELETE FROM attachment_text_cache WHERE created_at <= ?`,
+            [now - VIEW_CACHE_MAX_AGE_MS],
+          );
+          await this.db.run(
+            `DELETE FROM attachment_text_cache
+             WHERE session_id = ? AND rowid NOT IN (
+               SELECT rowid FROM attachment_text_cache WHERE session_id = ?
+               ORDER BY created_at DESC, rowid DESC LIMIT ?
+             )`,
+            [sessionId, sessionId, VIEW_CACHE_MAX_PER_SESSION],
+          );
+        } catch (error) {
+          this.log.warn(
+            `[attachments] view cache write failed: ${error instanceof Error ? error.message : String(error)}`,
+          );
+        }
+      },
+    };
+  }
+
+  /** Drop every entry of a deleted session; returns how many went. */
+  async forgetSession(sessionId: string): Promise<number> {
+    await this.setup();
+    const { changes } = await this.db.run(
+      `DELETE FROM attachment_text_cache WHERE session_id = ?`,
+      [sessionId],
+    );
+    return changes;
+  }
+}

package/src/attachments/view.ts

@@ -3,15 +3,19 @@
  * lane behind payload retention (`retention.ts`). The same routing as the
  * turn pipeline applies: when the current model reads the modality the bytes
  * come back as a native content block, otherwise the helper (`vision` role)
- * model turns them into text; plain text is decoded locally.
+ * model turns them into text; plain text is decoded locally. Text results
+ * are cached per session (`deps.cache`); native views are not (they are the
+ * bytes themselves).
  */
 import { classifyAttachment } from './classify';
 import type { NativeAttachment } from './content-blocks';
 import { loadAttachmentBytes, type MatrixMediaSource } from './download';
 import { extractText, type ExtractionProvider } from './extract';
-import { categorizeFile, verifyMagicBytes } from './magic';
+import { categorizeFile, isPlainTextType, verifyMagicBytes } from './magic';
+import { attachmentRef } from './retention';
 import { routeAttachment } from './route';
 import { sanitizeAttachmentFilename } from './sanitize';
+import { LOCAL_TEXT_EXTRACTION, type AttachmentTextCache } from './view-cache';
 import type { AttachmentInput } from './types';
 import type { ModelInputCapabilities } from '../core/llm';
 import type { AttachmentMeta } from '../sqlite/serialization';
@@ -31,6 +35,11 @@ export interface ViewAttachmentDeps {
   fetchImpl?: typeof fetch;
   signal?: AbortSignal;
   logger: Logger;
+  /**
+   * The session's cache of extracted text (`AttachmentTextCacheStore`):
+   * a hit costs no download and no helper-model call.
+   */
+  cache?: AttachmentTextCache;
 }
 
 /** What the `view_attachment` tool sees on the host: session-scoped access. */
@@ -62,20 +71,25 @@ export async function viewAttachment(
   const attachment = toInput(meta);
   const kind = classifyAttachment(attachment);
   const strategy = routeAttachment(kind, deps.caps);
-  const { bytes, mimetype } = await loadAttachmentBytes(
-    attachment,
-    deps.roomId,
-    deps.source,
-    { fetchImpl: deps.fetchImpl, signal: deps.signal },
-  );
-  deps.logger.log(
-    `[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
-  );
   if (strategy === 'send-native') {
+    const { bytes, mimetype, sniffed } = await loadAttachmentBytes(
+      attachment,
+      deps.roomId,
+      deps.source,
+      { fetchImpl: deps.fetchImpl, signal: deps.signal },
+    );
+    deps.logger.log(
+      `[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
+    );
     const loaded = classifyAttachment({
       mimetype,
-      filename: attachment.filename,
+      filename: sniffed ? '' : attachment.filename,
     });
+    // Bytes that are not what the file claims never go back as a block.
+    if (loaded !== kind)
+      throw new Error(
+        `File content mismatch: claimed ${attachment.mimetype} but detected ${mimetype}`,
+      );
     return {
       kind: 'native',
       native: {
@@ -86,6 +100,7 @@ export async function viewAttachment(
       },
     };
   }
+
   const category = categorizeFile(attachment.mimetype);
   if (category === 'unsupported') {
     return {
@@ -93,6 +108,31 @@ export async function viewAttachment(
       text: `[File "${sanitizeAttachmentFilename(attachment.filename)}" (${attachment.mimetype}) is not a supported file type and could not be processed]`,
     };
   }
+  // Text extracted earlier in this session by the same model is final.
+  const ref = attachmentRef(meta);
+  const model =
+    category === 'document' && isPlainTextType(attachment.mimetype)
+      ? LOCAL_TEXT_EXTRACTION
+      : deps.extraction?.model;
+  const cacheKey = deps.cache && ref && model ? { ref, model } : undefined;
+  if (cacheKey) {
+    const cached = await deps.cache?.get(cacheKey.ref, cacheKey.model);
+    if (cached !== undefined) {
+      deps.logger.log(
+        `[attachments] view "${attachment.filename}" → cached text (${cacheKey.model})`,
+      );
+      return { kind: 'text', text: cached };
+    }
+  }
+  const { bytes, mimetype } = await loadAttachmentBytes(
+    attachment,
+    deps.roomId,
+    deps.source,
+    { fetchImpl: deps.fetchImpl, signal: deps.signal },
+  );
+  deps.logger.log(
+    `[attachments] view "${attachment.filename}" (${mimetype}) kind=${kind} → ${strategy} (${bytes.length} bytes)`,
+  );
   verifyMagicBytes(bytes, category, attachment, (m) =>
     deps.logger.warn(`[attachments] ${m}`),
   );
@@ -103,5 +143,6 @@ export async function viewAttachment(
     deps.extraction,
     { fetchImpl: deps.fetchImpl, signal: deps.signal },
   );
+  if (cacheKey) await deps.cache?.put(cacheKey.ref, cacheKey.model, text);
   return { kind: 'text', text };
 }

package/src/channels/auth.ts

@@ -0,0 +1,310 @@
+import {
+  createCompositeDIDResolver,
+  createWebDIDResolver,
+  createUCANValidator,
+  defineCapability,
+  parseDelegation,
+  Schema,
+  type DIDKeyResolver,
+} from '@ixo/ucan';
+import { z } from 'zod';
+import type { OracleWorkerEnv, TurnIdentity } from '../do/contracts';
+import { RunAttemptDeferred } from '../do/run-coordinator';
+import {
+  IXO_DID_RESOLUTION_CACHE_TTL_MS,
+  sharedIxoDIDResolver,
+} from '../shell/auth';
+import {
+  ChannelError,
+  type ChannelTurnInput,
+  channelRequestHash,
+} from './contract';
+
+export const ChannelInvoke = defineCapability({
+  can: 'ixo:channel/invoke',
+  protocol: 'ixo:',
+  supportWildcards: false,
+  nb: {
+    provider: Schema.string(),
+    bindingRevision: Schema.integer(),
+    oracleDid: Schema.string(),
+  },
+  derives: (claimed, delegated) =>
+    claimed.nb?.provider === delegated.nb?.provider &&
+    claimed.nb?.bindingRevision === delegated.nb?.bindingRevision &&
+    claimed.nb?.oracleDid === delegated.nb?.oracleDid
+      ? { ok: {} }
+      : {
+          error: new Error('Channel caveats cannot change across a delegation'),
+        },
+});
+
+export interface ChannelAuthConfig {
+  oracleDid: string;
+  channelDid: string;
+  didResolver: DIDKeyResolver;
+}
+
+export async function authenticateChannel(
+  headers: Headers,
+  raw: string,
+  input: ChannelTurnInput,
+  config: ChannelAuthConfig,
+): Promise<TurnIdentity> {
+  const authorization = headers.get('authorization');
+  const token = authorization?.match(/^Bearer ([A-Za-z0-9+/=_-]+)$/i)?.[1];
+  if (
+    !token ||
+    token.length > 32_000 ||
+    headers.get('x-auth-type')?.toLowerCase() !== 'ucan'
+  )
+    throw new ChannelError(
+      401,
+      'A channel UCAN invocation is required',
+      'unauthorized',
+    );
+  const validator = await createUCANValidator({
+    serverDid: config.oracleDid,
+    rootIssuers: ['*'],
+    didResolver: config.didResolver,
+    requireExpiration: true,
+    // No replay marks: a poll repeats its exact body and may reuse an
+    // invocation, also concurrently. The durable receipt keyed by
+    // (user, binding, requestId) and the request hash in the invocation's
+    // facts make every repeat return the same run instead of a new one.
+    invocationStore: {
+      has: async () => false,
+      add: async () => undefined,
+      addIfAbsent: async () => true,
+    },
+  });
+  const result = await validator.validate(
+    token,
+    ChannelInvoke,
+    `ixo:channel:${input.bindingId}`,
+  );
+  const chain = result.proofChain;
+  const userDid = chain?.[0];
+  if (
+    !result.ok ||
+    !userDid?.startsWith('did:ixo:') ||
+    chain?.length !== 2 ||
+    chain[1] !== config.channelDid ||
+    result.invoker !== config.channelDid
+  )
+    throw new ChannelError(
+      401,
+      'Invalid user-rooted channel invocation',
+      'unauthorized',
+    );
+  const now = Math.floor(Date.now() / 1000);
+  const invocation = await parseDelegation(token);
+  const proof = invocation.proofs[0];
+  if (
+    invocation.proofs.length !== 1 ||
+    !proof ||
+    !('issuer' in proof) ||
+    proof.proofs.length !== 0 ||
+    proof.issuer.did() !== userDid ||
+    proof.audience.did() !== config.channelDid ||
+    proof.capabilities.length !== 1 ||
+    proof.capabilities[0]?.can !== 'ixo:channel/invoke' ||
+    proof.capabilities[0]?.with !== `ixo:channel:${input.bindingId}` ||
+    !Number.isFinite(proof.expiration) ||
+    !proof.expiration ||
+    proof.expiration > now + 300
+  )
+    throw new ChannelError(
+      401,
+      'A bounded, direct user channel grant is required',
+      'unauthorized',
+    );
+  if (
+    !result.expiration ||
+    result.expiration <= now ||
+    result.expiration > now + 60 ||
+    !Number.isFinite(invocation.expiration) ||
+    !invocation.expiration ||
+    invocation.expiration > now + 60
+  )
+    throw new ChannelError(
+      401,
+      'Channel invocations must expire within 60 seconds',
+      'unauthorized',
+    );
+  const nb = result.capability?.nb;
+  if (
+    result.capability?.can !== 'ixo:channel/invoke' ||
+    result.capability.with !== `ixo:channel:${input.bindingId}` ||
+    nb?.provider !== input.provider ||
+    nb.bindingRevision !== input.bindingRevision ||
+    nb.oracleDid !== config.oracleDid
+  )
+    throw new ChannelError(
+      403,
+      'Channel scope does not match this request',
+      'scope_mismatch',
+    );
+  const hash = await channelRequestHash(raw);
+  if (
+    result.facts?.length !== 1 ||
+    result.facts[0]?.requestId !== input.requestId ||
+    result.facts[0]?.requestHash !== hash
+  )
+    throw new ChannelError(
+      403,
+      'Invocation does not authorize this request body',
+      'scope_mismatch',
+    );
+  return {
+    userDid,
+    channel: {
+      callerDid: config.channelDid,
+      provider: input.provider,
+      bindingId: input.bindingId,
+      bindingRevision: input.bindingRevision,
+    },
+  };
+}
+
+/**
+ * did:web documents (a channel service identity) resolved for every channel
+ * request, shared per isolate: successful lookups are cached as long as
+ * did:ixo keys, failures never, and concurrent lookups share one request.
+ */
+const channelWebResolver = createWebDIDResolver({
+  cacheTtlMs: IXO_DID_RESOLUTION_CACHE_TTL_MS,
+});
+
+export function channelAuthConfig(
+  env: Pick<
+    OracleWorkerEnv,
+    'CHANNEL_SERVICE_DID' | 'BLOCKSYNC_GRAPHQL_URL' | 'ORACLE_DID'
+  >,
+): ChannelAuthConfig {
+  if (!env.CHANNEL_SERVICE_DID)
+    throw new ChannelError(
+      503,
+      'Channels are not configured',
+      'not_configured',
+    );
+  const resolver = createCompositeDIDResolver([
+    sharedIxoDIDResolver(env.BLOCKSYNC_GRAPHQL_URL),
+    channelWebResolver,
+  ]);
+  return {
+    oracleDid: env.ORACLE_DID,
+    channelDid: env.CHANNEL_SERVICE_DID,
+    didResolver: async (did) => {
+      if (
+        did === env.CHANNEL_SERVICE_DID ||
+        did === env.ORACLE_DID ||
+        did.startsWith('did:ixo:')
+      )
+        return resolver(did);
+      return {
+        error: {
+          name: 'DIDKeyResolutionError',
+          did,
+          message: 'Unapproved channel identity',
+        },
+      };
+    },
+  };
+}
+
+const BindingVerdict = z.strictObject({ active: z.boolean() });
+
+export async function assertActiveChannelBinding(
+  identity: TurnIdentity,
+  env: Pick<
+    OracleWorkerEnv,
+    'ORACLE_DID' | 'AUTH_HUB_CHANNEL_SERVICE_KEY' | 'AUTH_HUB' | 'AUTH_HUB_URL'
+  >,
+): Promise<void> {
+  if (!identity.channel)
+    throw new ChannelError(
+      403,
+      'Channel identity is required',
+      'identity_required',
+    );
+  if (!env.AUTH_HUB_CHANNEL_SERVICE_KEY || (!env.AUTH_HUB && !env.AUTH_HUB_URL))
+    throw new ChannelError(
+      503,
+      'Channel binding validation is not configured',
+      'not_configured',
+    );
+  const url = new URL(
+    '/api/internal/channels/validate-binding',
+    env.AUTH_HUB_URL ?? 'https://auth-hub.internal',
+  );
+  if (url.protocol !== 'https:')
+    throw new ChannelError(503, 'Auth Hub requires HTTPS', 'not_configured');
+  const request = new Request(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'x-channels-service-key': env.AUTH_HUB_CHANNEL_SERVICE_KEY,
+    },
+    body: JSON.stringify({
+      bindingId: identity.channel.bindingId,
+      bindingRevision: identity.channel.bindingRevision,
+      userDid: identity.userDid,
+      provider: identity.channel.provider,
+      oracleDid: env.ORACLE_DID,
+    }),
+    signal: AbortSignal.timeout(10_000),
+    redirect: 'manual',
+  });
+  let response: Response;
+  try {
+    response = await (env.AUTH_HUB
+      ? env.AUTH_HUB.fetch(request)
+      : fetch(request));
+  } catch {
+    throw new ChannelError(
+      503,
+      'Channel binding validation is unavailable',
+      'binding_check_unavailable',
+    );
+  }
+  if (!response.ok)
+    throw new ChannelError(
+      503,
+      'Channel binding validation is unavailable',
+      'binding_check_unavailable',
+    );
+  const parsed = BindingVerdict.safeParse(await response.json());
+  if (!parsed.success)
+    throw new ChannelError(
+      503,
+      'Invalid channel binding validation response',
+      'binding_check_unavailable',
+    );
+  if (!parsed.data.active)
+    throw new ChannelError(
+      403,
+      'Channel binding is inactive',
+      'binding_inactive',
+    );
+}
+
+/**
+ * The binding check before a channel attempt executes (a queued run being
+ * dequeued, a recovery attempt). An inactive binding (403) ends the run. An
+ * unavailable or unconfigured validator (503) defers the attempt to the run
+ * coordinator's recovery backoff: nothing has executed, and one Auth Hub
+ * outage must not lose the user's message.
+ */
+export async function assertChannelAttemptAllowed(
+  identity: TurnIdentity,
+  env: Parameters<typeof assertActiveChannelBinding>[1],
+): Promise<void> {
+  try {
+    await assertActiveChannelBinding(identity, env);
+  } catch (error) {
+    if (error instanceof ChannelError && error.status === 503)
+      throw new RunAttemptDeferred(error.message, { cause: error });
+    throw error;
+  }
+}