@ixo/oracle-runtime-workers 0.30.0 → 0.31.0

package/src/channels/turns.ts

@@ -162,6 +162,14 @@ const CHANNEL_TOPIC_LOOKBACK = 5;
 /** Ids per statement: a Durable Object binds at most 100 parameters. */
 const CHANNEL_IDS_PER_STATEMENT = 50;
 
+/**
+ * The text the gateway sends for a file with no caption: `[photo]`,
+ * `[video]`, `[audio]` or `[document: <name>]`.
+ */
+export function isAttachmentPlaceholder(message: string): boolean {
+  return /^\[(?:photo|video|audio)\]$|^\[document: [^\n]*\]$/.test(message);
+}
+
 function clipCodePoints(text: string, limit: number): string {
   const points = Array.from(text);
   return points.length > limit ? points.slice(0, limit).join('') : text;
@@ -1463,7 +1471,14 @@ export class ChannelTurns {
           assignment.destination = { kind: 'session', sessionId: session };
       } else if (mode === 'on')
         assignment.destination = { kind: 'clarify', text: ambiguousThreadText };
-    } else {
+    } else if (
+      input.attachments?.length &&
+      isAttachmentPlaceholder(input.message)
+    )
+      // A file with no caption, only the gateway's placeholder: nothing to
+      // classify, so it stays in the current conversation.
+      assignment.reason = 'attachment-placeholder';
+    else {
       const candidates = await this.routingCandidates(input, baseline);
       if (candidates.length) {
         const evidence = {
@@ -1516,6 +1531,10 @@ export class ChannelTurns {
         } else assignment.reason = 'shadow';
       }
     }
+    // A clarifying question is answered instead of the message, so the files
+    // would be dropped: a turn that carries files stays where it is.
+    if (input.attachments?.length && assignment.destination.kind === 'clarify')
+      assignment.destination = { kind: 'session', sessionId: baseline };
     await this.serialize(async () => {
       await this.assertNotCancelled(input);
       await this.db.run(
@@ -1861,6 +1880,9 @@ export class ChannelTurns {
         bindingId: input.bindingId,
         bindingRevision: input.bindingRevision,
         remoteMessageRef: input.remoteMessageRef,
+        ...(input.attachments?.length
+          ? { attachments: input.attachments }
+          : {}),
         ...(replyContext ? { replyContext } : {}),
         ...(input.replyTo ? { replyTo: input.replyTo } : {}),
         ...(routing?.destination.kind === 'clarify'

package/src/core/main-agent-types.ts

@@ -9,6 +9,7 @@ import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
 import type { BaseCheckpointSaver } from '@langchain/langgraph';
 import type { ReactAgent } from 'langchain';
 import { z } from 'zod';
+import { ChannelAttachment } from '../channels/contract';
 import type {
   ChatOpenAIFields,
   MergedConfig,
@@ -48,6 +49,7 @@ export const mainAgentRequestContextSchema = z.object({
         provider: z.string(),
         bindingId: z.string(),
         bindingRevision: z.number().int().positive(),
+        attachments: z.array(ChannelAttachment).optional(),
       })
       .optional(),
     wsId: z.string().optional(),

package/src/core/runtime-context.ts

@@ -1,3 +1,4 @@
+import type { ChannelAttachment } from '../channels/contract';
 import {
   UNAVAILABLE_DECISION_EVALUATOR,
   type DecisionEvaluateOptions,
@@ -330,6 +331,8 @@ export interface RuntimeSessionContext {
     provider: string;
     bindingId: string;
     bindingRevision: number;
+    /** Files sent with this message (see `RuntimeContext.session.channel`). */
+    attachments?: readonly ChannelAttachment[];
   };
   roomId?: string;
   surface?: SessionSurface;

package/src/delivery/plan.ts

@@ -165,7 +165,9 @@ export function draftReplyPlan(input: {
     for (const call of step.toolCalls) {
       if (
         call.name === 'request_hub_action' ||
-        call.name === 'request_chain_transaction'
+        call.name === 'request_chain_transaction' ||
+        // The Yoma plugin's application and claim cards.
+        call.name === 'yoma_form'
       ) {
         try {
           const result = JSON.parse(input.toolResults.get(call.id) ?? 'null');

package/src/do/contracts.ts

@@ -1,4 +1,5 @@
 import type {
+  ChannelAttachment,
   ChannelControlInput,
   ChannelControlOutcome,
   ChannelReplyContext,
@@ -400,6 +401,8 @@ export interface TurnRequest {
     replyTo?: ChannelReplyReference;
     /** Host-generated direct reply: no agent or tools run for an ambiguous destination. */
     routingClarification?: string;
+    /** Files the person sent with this message, as the gateway serves them. */
+    attachments?: ChannelAttachment[];
   };
   executionProfile?: TaskExecutionProfile;
   identity: TurnIdentity;

package/src/do/turn-attachments.ts

@@ -0,0 +1,103 @@
+/**
+ * The attachments of one turn attempt: which files the run reads, how a
+ * channel file's `channel:` reference becomes its link (server-side, never
+ * through the model), and when nothing is downloaded at all.
+ */
+import {
+  attachmentMetaFor,
+  channelAttachmentRef,
+  channelMediaOrigins,
+  createChannelMediaSource,
+  parseAttachmentInputs,
+  type AttachmentInput,
+  type ChannelAttachmentLinkStore,
+  type ChannelMediaLink,
+  type ChannelMediaSource,
+  type PreparedAttachments,
+} from '../attachments';
+import { parseChannelServiceDids } from '../channels/auth';
+import {
+  channelAttachmentInputs,
+  type ChannelAttachment,
+} from '../channels/contract';
+import type { AttachmentMeta } from '../sqlite/serialization';
+
+/** The part of a turn request the attachment inputs come from. */
+interface TurnFiles {
+  client: string;
+  attachments?: unknown;
+  channel?: { bindingId: string; attachments?: readonly ChannelAttachment[] };
+}
+
+/**
+ * What the run reads: a channel turn's files by their `channel:` reference
+ * (from the gateway's list on the request), otherwise the request body's
+ * attachments as the Portal and Matrix send them. A body can never name a
+ * `channel:` reference itself.
+ */
+export function turnAttachmentInputs(
+  req: TurnFiles,
+  bodyAttachments: unknown,
+): AttachmentInput[] {
+  const files = req.client === 'channel' ? req.channel?.attachments : undefined;
+  if (files?.length) return channelAttachmentInputs(files);
+  return parseAttachmentInputs(bodyAttachments ?? req.attachments);
+}
+
+/** A channel turn's files as the links the owner object resolves references to. */
+export function channelTurnLinks(
+  req: TurnFiles,
+): Array<ChannelMediaLink & { ref: string }> {
+  if (req.client !== 'channel' || !req.channel?.attachments?.length) return [];
+  const bindingId = req.channel.bindingId;
+  return req.channel.attachments.map((file) => ({
+    ref: channelAttachmentRef(file.sha256),
+    url: file.url,
+    sha256: file.sha256,
+    bindingId,
+  }));
+}
+
+/**
+ * The channel source of a turn in `sessionId`: a reference resolves to the
+ * turn's own files first, then to the links kept for the session; only the
+ * gateways of `CHANNEL_SERVICE_DID` are fetched from.
+ */
+export function turnChannelMediaSource(input: {
+  sessionId: string;
+  turnLinks: ReadonlyArray<ChannelMediaLink & { ref: string }>;
+  store: Pick<ChannelAttachmentLinkStore, 'get'> | null;
+  channelServiceDid: string | undefined;
+}): ChannelMediaSource {
+  return createChannelMediaSource({
+    origins: channelMediaOrigins(
+      parseChannelServiceDids(input.channelServiceDid),
+    ),
+    resolve: async (ref) => {
+      const own = input.turnLinks.find((link) => link.ref === ref);
+      if (own) return own;
+      return input.store?.get(input.sessionId, ref);
+    },
+  });
+}
+
+/**
+ * The turn's prepared attachments, or none: a resumed attempt hands the
+ * graph no new input (the person's message, with its files, is already in
+ * the checkpoint), so nothing is downloaded again; its metadata is still
+ * known for `view_attachment`.
+ */
+export async function prepareTurnAttachments(input: {
+  inputs: readonly AttachmentInput[];
+  resumed: boolean;
+  prepare: () => Promise<PreparedAttachments>;
+}): Promise<{ prepared: PreparedAttachments | null; metas: AttachmentMeta[] }> {
+  if (input.inputs.length === 0) return { prepared: null, metas: [] };
+  if (input.resumed)
+    return {
+      prepared: null,
+      metas: input.inputs.map((attachment) => attachmentMetaFor(attachment)),
+    };
+  const prepared = await input.prepare();
+  return { prepared, metas: prepared.metas };
+}

package/src/do/user-oracle-do.ts

@@ -46,7 +46,11 @@ import type {
   ActionProposalSubject,
 } from '@ixo/ixo-transaction/action-proposals';
 import { ChannelTurns } from '../channels/turns';
-import { HubActionClient } from '../hub-actions/client';
+import {
+  HubActionClient,
+  type HubOperationStatus,
+  type HubReviewStatus,
+} from '../hub-actions/client';
 import { chainTxOperation, prepareHubAction } from '../hub-actions/request';
 import {
   bankReceiptSummary,
@@ -126,6 +130,7 @@ import {
   offloadInlinePayloads,
   parseAttachmentInputs,
   prepareAttachments,
+  ChannelAttachmentLinkStore,
   type AttachmentViewSurface,
   type MatrixMediaSource,
   type ViewAttachmentDeps,
@@ -135,6 +140,12 @@ import {
   readBytesCapped,
   AttachmentTextCacheStore,
 } from '../attachments';
+import {
+  channelTurnLinks,
+  prepareTurnAttachments,
+  turnAttachmentInputs,
+  turnChannelMediaSource,
+} from './turn-attachments';
 import type { RuntimeCore } from '../core';
 import {
   createLlmAdapter,
@@ -430,6 +441,7 @@ import {
 } from './ucan-service';
 import type {
   ChainTxMessageInput,
+  HubActionStatusRead,
   UcanDelegation,
   UserContextData,
 } from '../plugin-api/types';
@@ -564,6 +576,7 @@ function hasUserContext(value: unknown): boolean {
 function admissionApplies(req: TurnRequest): boolean {
   return (
     !req.attachments?.length &&
+    !req.channel?.attachments?.length &&
     !req.taskRunId &&
     !req.sessionId.startsWith(TASK_SESSION_PREFIX) &&
     req.roomKind !== 'group'
@@ -1058,6 +1071,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
     private lastActionPruneAt: number | null = null;
     /** Text `view_attachment` extracted, per session (`attachments/view-cache.ts`). */
     private attachmentTextCache: AttachmentTextCacheStore | null = null;
+    /** The links behind the `channel:` references of each session's files. */
+    private channelAttachmentLinks: ChannelAttachmentLinkStore | null = null;
     /** Large texts of document ingestion, in memory (a few, minutes). */
     private readonly ingestTextLru = new IngestTextLru();
     /**
@@ -2019,6 +2034,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       });
       await this.resultStore.setup();
       this.attachmentTextCache = new AttachmentTextCacheStore(liveDb, console);
+      this.channelAttachmentLinks = new ChannelAttachmentLinkStore(
+        liveDb,
+        console,
+      );
       // Constructing it touches no SQL. Only an observe turn records (and so
       // creates `domain_context_runs`); a session delete never creates it but
       // still removes rows an earlier observe deployment left behind.
@@ -2273,8 +2292,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
                 }) => this.requestHubAction(input),
                 chainTransaction: (input) =>
                   this.chainTransactionHubAction(input),
-                status: (ownerDid: string, actionId: string) =>
-                  this.hubActionStatusForTool(ownerDid, actionId),
+                status: (
+                  ownerDid: string,
+                  actionId: string,
+                  opts?: { timeoutMs?: number },
+                ) => this.hubActionStatusForTool(ownerDid, actionId, opts),
               },
             }
           : {}),
@@ -2366,6 +2388,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       this.lastHubOptions = null;
       this.topicProposals = null;
       this.attachmentTextCache = null;
+      this.channelAttachmentLinks = null;
       this.domainContextStore = null;
       // Its store is bound to the database just dropped: a caller that sees
       // no scheduler boots again instead of reading a closed connection.
@@ -3329,7 +3352,10 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       // The gateway parses with @ixo/action-protocol 0.4.1, whose bank
       // receipt is exact: the 0.5.0 `denom`/`amountBaseUnits` fields are
       // dropped here (the owner-facing amount line comes from
-      // hub_action_status instead), so the emitted shape stays 0.4.1.
+      // hub_action_status instead), so the emitted shape stays 0.4.1. Every
+      // other receipt goes as Auth Hub signed it, a failure's `detail`
+      // included: the gateway confirms an outcome by comparing the whole
+      // receipt with Auth Hub's copy. Only what the model sees drops it.
       const hubActionOutcomes = (
         await this.hubOutcomesForRun(ownerDid, result.sessionId, result.runId)
       ).map((outcome) => ({ ...outcome, receipt: v1Receipt(outcome.receipt) }));
@@ -3613,24 +3639,58 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
     }
 
     /**
-     * `hub_action_status`: the outcome plus, for a bank send, the
-     * owner-facing line (amount from the token stored at preparation).
+     * `ctx.hubActions.status` (and `hub_action_status`): Auth Hub's review
+     * and operation status, the outcome and, for a bank send, the
+     * owner-facing line (amount from the token stored at preparation). An
+     * action this object never prepared is `held: false`, without a Hub
+     * read.
      */
     private async hubActionStatusForTool(
       ownerDid: string,
       actionId: string,
-    ): Promise<{ outcome: ActionOutcomeV1 | null; summary: string | null }> {
-      const outcome = await this.statusHubAction(ownerDid, actionId);
-      if (!outcome) return { outcome, summary: null };
-      const token = await this.hubActionStore!.token(ownerDid, actionId);
-      return { outcome, summary: bankReceiptSummary(outcome.receipt, token) };
+      opts: { timeoutMs?: number } = {},
+    ): Promise<HubActionStatusRead> {
+      const read = await this.statusHubAction(
+        ownerDid,
+        actionId,
+        opts.timeoutMs,
+      );
+      if (!read)
+        return {
+          held: false,
+          reviewStatus: null,
+          operationStatus: null,
+          outcome: null,
+          summary: null,
+        };
+      const token = read.outcome
+        ? await this.hubActionStore!.token(ownerDid, actionId)
+        : null;
+      return {
+        held: true,
+        reviewStatus: read.reviewStatus,
+        operationStatus: read.operationStatus,
+        outcome: read.outcome,
+        summary: read.outcome
+          ? bankReceiptSummary(read.outcome.receipt, token)
+          : null,
+      };
     }
 
+    /**
+     * One authenticated status read of a held action (null: not held), its
+     * outcome recorded when new. An outcome for another revision than the
+     * one Auth Hub reports is not taken (`outcome: null`).
+     */
     private async statusHubAction(
       ownerDid: string,
       actionId: string,
       timeoutMs?: number,
-    ): Promise<ActionOutcomeV1 | null> {
+    ): Promise<{
+      reviewStatus: HubReviewStatus;
+      operationStatus: HubOperationStatus;
+      outcome: ActionOutcomeV1 | null;
+    } | null> {
       const held = await this.hubActionStore?.get(ownerDid, actionId);
       if (!held || !this.hubActionClient) return null;
       const status = await this.hubActionClient.status(actionId, timeoutMs);
@@ -3639,16 +3699,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         canonicalJson(status.reference) !== canonicalJson(held.reference)
       )
         throw new Error('Hub status does not match the prepared action');
+      const read = {
+        reviewStatus: status.reviewStatus,
+        operationStatus: status.operationStatus,
+      };
       if (status.outcome) {
         if (
           status.revision !== status.outcome.authorizedRevision ||
           status.digest !== status.outcome.authorizedDigest
         )
-          return null;
+          return { ...read, outcome: null };
         if (await this.hubActionStore!.recordOutcome(status.outcome))
           this.markDirty();
       }
-      return status.outcome;
+      return { ...read, outcome: status.outcome };
     }
 
     /**
@@ -3687,11 +3751,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
           // refreshed writes it, at most one per poll.
           this.markDirty();
           try {
-            const fresh = await this.statusHubAction(
-              ownerDid,
-              row.reference.id,
-              HUB_POLL_TIMEOUT_MS,
-            );
+            const fresh = (
+              await this.statusHubAction(
+                ownerDid,
+                row.reference.id,
+                HUB_POLL_TIMEOUT_MS,
+              )
+            )?.outcome;
             if (fresh && (!latest || fresh.sequence > latest.sequence))
               latest = fresh;
           } catch (error) {
@@ -3848,6 +3914,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
               `[user-do] could not delete the cached attachment text of ${sessionId}: ${err instanceof Error ? err.message : String(err)}`,
             );
           });
+        await this.channelAttachmentLinks
+          ?.forgetSession(sessionId)
+          .catch((err: unknown) => {
+            console.warn(
+              `[user-do] could not delete the channel file links of ${sessionId}; they expire with the links: ${err instanceof Error ? err.message : String(err)}`,
+            );
+          });
         await this.resultStore
           ?.deleteForSession(sessionId)
           .catch((err: unknown) => {
@@ -7038,6 +7111,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       signal: AbortSignal;
       /** Attachments of the turn now running (not necessarily persisted yet). */
       turnMetas: readonly AttachmentMeta[];
+      /** The links of this turn's channel files (earlier ones come from the store). */
+      channelLinks: ReturnType<typeof channelTurnLinks>;
     }): Promise<AttachmentViewSurface> {
       const saver = this.saver!;
       const offloaded = await saver.hasThreadMessageMatching(input.sessionId, {
@@ -7048,6 +7123,12 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         findPersistedAttachmentMeta(saver, input.sessionId, ref);
       const viewDeps = (): ViewAttachmentDeps => ({
         source: this.matrixMediaSource(),
+        channel: turnChannelMediaSource({
+          sessionId: input.sessionId,
+          turnLinks: input.channelLinks,
+          store: this.channelAttachmentLinks,
+          channelServiceDid: this.env.CHANNEL_SERVICE_DID,
+        }),
         extraction: this.extractionProvider(input.ambientLlm, input.platform),
         caps: getModelCapabilities(input.model),
         roomId: input.roomId,
@@ -7390,32 +7471,47 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       // native capabilities, send images/files inline when it reads them,
       // otherwise the `vision` role turns them into text on the platform key;
       // originals are archived to the user's sandbox when one is reachable.
-      const attachmentInputs = parseAttachmentInputs(
-        body.attachments ?? req.attachments,
-      );
+      // A channel turn's files are read by their `channel:` reference; their
+      // links stay on the server (the run request and the session's link
+      // store), out of every message and note.
+      const attachmentInputs = turnAttachmentInputs(req, body.attachments);
       const attachmentModel =
         effectiveModel && isAllowedModel(effectiveModel)
           ? effectiveModel
           : getDefaultModelId(core.validatedEnv);
-      const prepared =
-        attachmentInputs.length > 0
-          ? await prepareAttachments(
-              {
-                text: body.message,
-                attachments: attachmentInputs,
-                roomId: sessionRoomId,
-                model: attachmentModel,
-                caps: getModelCapabilities(attachmentModel),
-              },
-              {
-                source: this.matrixMediaSource(),
-                extraction: this.extractionProvider(baseAmbient.llm, core.llm),
-                sandbox: await this.sandboxArchiveConfig(req.identity.userDid),
-                signal: abortController.signal,
-                logger: console,
-              },
-            )
-          : null;
+      const turnLinks = channelTurnLinks(req);
+      const channelMedia = turnLinks.length
+        ? turnChannelMediaSource({
+            sessionId: req.sessionId,
+            turnLinks,
+            store: this.channelAttachmentLinks,
+            channelServiceDid: this.env.CHANNEL_SERVICE_DID,
+          })
+        : undefined;
+      if (turnLinks.length && !run.resumed)
+        await this.channelAttachmentLinks?.put(req.sessionId, turnLinks);
+      const { prepared, metas: turnMetas } = await prepareTurnAttachments({
+        inputs: attachmentInputs,
+        resumed: run.resumed,
+        prepare: async () =>
+          prepareAttachments(
+            {
+              text: body.message,
+              attachments: attachmentInputs,
+              roomId: sessionRoomId,
+              model: attachmentModel,
+              caps: getModelCapabilities(attachmentModel),
+            },
+            {
+              source: this.matrixMediaSource(),
+              ...(channelMedia ? { channel: channelMedia } : {}),
+              extraction: this.extractionProvider(baseAmbient.llm, core.llm),
+              sandbox: await this.sandboxArchiveConfig(req.identity.userDid),
+              signal: abortController.signal,
+              logger: console,
+            },
+          ),
+      });
       if (prepared) {
         console.log(
           `[user-do] attachments: ${attachmentInputs.length} attached, ${prepared.extracted.length} extracted, ${Array.isArray(prepared.content) ? prepared.content.length - 1 : 0} native`,
@@ -7439,7 +7535,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
             ambientLlm: baseAmbient.llm,
             platform: core.llm,
             signal: abortController.signal,
-            turnMetas: prepared?.metas ?? [],
+            turnMetas,
+            channelLinks: turnLinks,
           });
 
       // Request metadata (editor room, space, session run, entity) → state,
@@ -7773,6 +7870,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
                     provider: req.channel.provider,
                     bindingId: req.channel.bindingId,
                     bindingRevision: req.channel.bindingRevision,
+                    ...(req.channel.attachments?.length
+                      ? { attachments: req.channel.attachments }
+                      : {}),
                   },
                 }
               : {}),