@stigmer/runner 3.8.0 → 3.10.0

package/src/activities/execute-deep-agent/__tests__/vision-input.test.ts

@@ -0,0 +1,152 @@
+/**
+ * Proves the T04 vision input path end-to-end through the REAL framework
+ * stack: the exact `{ role: "user", content: [...] }` shape that setup.ts
+ * builds is driven through `createDeepAgent` (the production graph factory,
+ * default middleware included) and the test asserts what the chat model's
+ * `_generate` actually received.
+ *
+ * This is the offline stand-in for the deferred live provider probes (T01
+ * A4/A5): it cannot prove Anthropic renders the pixels, but it proves the
+ * image blocks survive the LangGraph message reducer and the full deepagents
+ * middleware stack byte-identically — the part of the path we own.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdtemp, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BaseMessage } from "@langchain/core/messages";
+import { HumanMessage } from "@langchain/core/messages";
+import type { ChatResult } from "@langchain/core/outputs";
+import { MemorySaver } from "@langchain/langgraph";
+import { createDeepAgent } from "deepagents";
+import { createCasCaptureBackend } from "../cas-capture-backend.js";
+import { CasCaptureObserver } from "../cas-capture-observer.js";
+import {
+  ScriptedModel,
+  type ScriptSelector,
+} from "../__test-utils__/scripted-model.js";
+import {
+  DEEP_AGENT_VISION_PROFILE,
+  VisionBudget,
+  toLangChainImageBlocks,
+  type LangChainContentBlock,
+  type VisionImage,
+} from "../../../shared/attachment-vision.js";
+
+/**
+ * A ScriptedModel that records every `_generate` input. `bindTools` must
+ * return a NEW capturing instance sharing the same capture array — the base
+ * class re-instantiates itself on bind, which would silently drop the spy.
+ */
+class CapturingModel extends ScriptedModel {
+  constructor(
+    private readonly selectFn: ScriptSelector,
+    private readonly captured: BaseMessage[][],
+  ) {
+    super(selectFn);
+  }
+
+  // Reports as ChatAnthropic so createDeepAgent engages the same
+  // anthropic-prompt-caching middleware a production run gets — the stack the
+  // image blocks must survive.
+  override getName(): string {
+    return "ChatAnthropic";
+  }
+
+  override bindTools(tools: unknown[]): this {
+    const next = new CapturingModel(this.selectFn, this.captured);
+    next.toolNames = (tools as Array<{ name?: string }>).map((t) => t?.name ?? "");
+    return next as unknown as this;
+  }
+
+  override async _generate(messages: BaseMessage[]): Promise<ChatResult> {
+    this.captured.push(messages);
+    return super._generate(messages);
+  }
+}
+
+function makeVisionImage(filename: string): VisionImage {
+  const pngBytes = Buffer.concat([
+    Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+    Buffer.alloc(56, 0xab),
+  ]);
+  const outcome = new VisionBudget(DEEP_AGENT_VISION_PROFILE).offer(
+    filename,
+    "image/png",
+    pngBytes,
+  );
+  if (outcome.kind !== "accepted") throw new Error("fixture PNG must be accepted");
+  return outcome.image;
+}
+
+describe("deep-agent vision input through the real graph", () => {
+  let root: string;
+
+  beforeEach(async () => {
+    root = await mkdtemp(join(tmpdir(), "vision-input-"));
+  });
+
+  afterEach(async () => {
+    await rm(root, { recursive: true, force: true });
+  });
+
+  /**
+   * `content` is exactly what setup.ts puts on the user message: a plain
+   * string, or the content-block array built by toLangChainImageBlocks plus
+   * the composed text block.
+   */
+  async function invokeGraphWith(
+    content: string | LangChainContentBlock[],
+  ): Promise<BaseMessage[][]> {
+    const captured: BaseMessage[][] = [];
+    const script: ScriptSelector = () => ({ toolCalls: [], done: "seen" });
+
+    const observer = new CasCaptureObserver({ rootDir: root, isIgnored: async () => false });
+    const backend = await createCasCaptureBackend({ rootDir: root, observer, shellEnv: {} });
+
+    const agent = await createDeepAgent({
+      model: new CapturingModel(script, captured),
+      checkpointer: new MemorySaver() as never,
+      backend,
+    } as Parameters<typeof createDeepAgent>[0]);
+
+    // The EXACT input shape setup.ts constructs (plain role/content dict, not
+    // a HumanMessage instance — the reducer does the coercion in production).
+    await agent.invoke(
+      { messages: [{ role: "user", content }] },
+      { configurable: { thread_id: "vision-thread" }, recursionLimit: 10 },
+    );
+    return captured;
+  }
+
+  it("delivers image_url blocks (labels first, text last) to the model byte-identically", async () => {
+    const image = makeVisionImage("photo.png");
+    const content: LangChainContentBlock[] = [
+      ...toLangChainImageBlocks([image]),
+      { type: "text", text: "what is in this image?" },
+    ];
+
+    const captured = await invokeGraphWith(content);
+
+    expect(captured.length).toBeGreaterThan(0);
+    const human = captured[0].find((m): m is HumanMessage => m instanceof HumanMessage);
+    expect(human).toBeDefined();
+    expect(human!.content).toEqual([
+      { type: "text", text: "Image 1: photo.png" },
+      {
+        type: "image_url",
+        image_url: { url: `data:image/png;base64,${image.base64}` },
+      },
+      { type: "text", text: "what is in this image?" },
+    ]);
+  });
+
+  it("keeps a plain-string user message untouched (the no-attachment path)", async () => {
+    const captured = await invokeGraphWith("just text, no images");
+
+    const human = captured[0].find((m): m is HumanMessage => m instanceof HumanMessage);
+    expect(human).toBeDefined();
+    expect(human!.content).toBe("just text, no images");
+  });
+});

package/src/activities/execute-deep-agent/attachment-injector.ts

@@ -20,6 +20,11 @@ import { basename, posix } from "node:path";
 import type { Attachment } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/spec_pb";
 import type { WorkspaceBackend } from "../../shared/workspace/types.js";
 import type { ArtifactStorage } from "../../shared/artifact-storage.js";
+import type {
+  VisionBudget,
+  VisionDegradedReason,
+  VisionImage,
+} from "../../shared/attachment-vision.js";
 
 // ── Constants ────────────────────────────────────────────────────────
 
@@ -36,6 +41,15 @@ export interface InjectedFile {
   readonly filename: string;
   readonly path: string;
   readonly sizeBytes: number;
+  /** Present when the attachment was accepted into the turn's vision payload. */
+  readonly vision?: VisionImage;
+  /**
+   * Present when the attachment was plausibly an image but could not ride
+   * inline (see {@link VisionDegradedReason}) — disclosed in the system prompt
+   * so the agent never silently ignores a photo the user believes it can see.
+   * Attachments that were never image-shaped carry neither field.
+   */
+  readonly visionDegraded?: VisionDegradedReason;
 }
 
 export interface InjectAttachmentsOptions {
@@ -50,6 +64,13 @@ export interface InjectAttachmentsOptions {
    */
   readonly storage: ArtifactStorage | undefined;
   readonly isLocalMode: boolean;
+  /**
+   * The turn's vision selector (attachment-vision.ts owns all policy).
+   * `undefined` disables inline image delivery; file materialization is
+   * identical either way — vision is strictly additive. Archives (`extract`)
+   * are never offered: an image inside a ZIP has no attachment-level bytes.
+   */
+  readonly visionBudget?: VisionBudget;
 }
 
 export interface ZipEntryInfo {
@@ -240,7 +261,7 @@ export function validateZipForExtraction(
  * On any failure, throws AttachmentInjectionError with an actionable message.
  */
 export async function injectAttachments(opts: InjectAttachmentsOptions): Promise<InjectedFile[]> {
-  const { backend, attachments, storage, isLocalMode } = opts;
+  const { backend, attachments, storage, isLocalMode, visionBudget } = opts;
 
   if (attachments.length === 0) return [];
 
@@ -262,10 +283,17 @@ export async function injectAttachments(opts: InjectAttachmentsOptions): Promise
       injectedFiles.push(...extracted);
     } else {
       await backend.writeFileBuffer(mountPath, content);
+      const filename = attachment.filename || basename(mountPath);
+      // The bytes are already in hand for the workspace write — offer them to
+      // the vision budget before they go out of scope (the sniff decides
+      // eligibility; the budget owns every size/count rule).
+      const vision = visionBudget?.offer(filename, attachment.contentType, content);
       injectedFiles.push({
-        filename: attachment.filename || basename(mountPath),
+        filename,
         path: mountPath,
         sizeBytes: content.length,
+        ...(vision?.kind === "accepted" ? { vision: vision.image } : {}),
+        ...(vision?.kind === "degraded" ? { visionDegraded: vision.reason } : {}),
       });
     }
   }
@@ -313,20 +341,53 @@ function resolveMountPath(attachment: Attachment): string {
         "mountPath resolves to an empty path after removing leading slashes",
       );
     }
+    // A caller-supplied mount path is untrusted. Stripping leading slashes does
+    // not stop `..` segments from climbing out of the workspace root on the
+    // non-`.stigmer/` branch (the `.stigmer/`-routed branch is already guarded
+    // by LocalWorkspaceBackend.resolvePath). Reject any path that normalizes to
+    // an escape before it reaches the backend write.
+    if (escapesRoot(cleaned)) {
+      throw new AttachmentInjectionError(
+        attachment.filename,
+        `mount path '${attachment.mountPath}' escapes the workspace root`,
+      );
+    }
     return cleaned;
   }
 
-  const filename = attachment.filename || deriveFilename(attachment.storageKey);
-  if (!filename) {
+  const rawName = attachment.filename || deriveFilename(attachment.storageKey);
+  if (!rawName) {
     throw new AttachmentInjectionError(
       "(unknown)",
       "attachment has neither filename nor storageKey — cannot determine mount path",
     );
   }
 
+  // The filename is untrusted; reduce it to a single path component so it lands
+  // directly under the inputs prefix rather than steering the write elsewhere.
+  const filename = posix.basename(rawName);
+  if (filename === "" || filename === "." || filename === "..") {
+    throw new AttachmentInjectionError(
+      rawName,
+      `filename '${rawName}' does not yield a usable name for materialization`,
+    );
+  }
+
   return `${DEFAULT_INPUTS_PREFIX}/${filename}`;
 }
 
+// escapesRoot reports whether a workspace-relative mount path would climb out
+// of the workspace root once normalized. Mount paths are forward-slash,
+// workspace-relative strings, so posix normalization is the correct semantics.
+function escapesRoot(relPath: string): boolean {
+  const normalized = posix.normalize(relPath);
+  return (
+    normalized === ".." ||
+    normalized.startsWith("../") ||
+    posix.isAbsolute(normalized)
+  );
+}
+
 function deriveFilename(storageKey: string): string {
   if (!storageKey) return "";
   const parts = storageKey.split("/");

package/src/activities/execute-deep-agent/hitl.ts

@@ -37,10 +37,15 @@ const ACTION_MAP: ReadonlyMap<ApprovalAction, string> = new Map([
   [ApprovalAction.REJECT, "reject"],
 ]);
 
-export interface ResumeResult {
-  readonly graphInput: Command | Record<string, unknown>;
-  readonly isResumeFromApproval: boolean;
-}
+/**
+ * Either a genuine approval resume carrying the `Command(resume)` payload, or
+ * not one — in which case the caller uses `setup.langgraphInput` (the single
+ * construction site of the turn's user message; this module deliberately does
+ * NOT build a second copy that could drift from it).
+ */
+export type ResumeResult =
+  | { readonly isResumeFromApproval: true; readonly graphInput: Command }
+  | { readonly isResumeFromApproval: false };
 
 export interface GraphStateSnapshot {
   readonly values: Record<string, unknown>;
@@ -71,8 +76,8 @@ export interface InterruptValue {
  * Resolve the resume input for a reinvocation after approval.
  *
  * Returns a `Command(resume=...)` if there are pending interrupts with
- * matching approval decisions, or a fresh user message input if this is
- * not a resume scenario.
+ * matching approval decisions; otherwise reports "not a resume" and the
+ * caller falls back to `setup.langgraphInput`.
  *
  * The graph checkpoint snapshot is read once by the caller and passed in: the
  * same snapshot also decides whether status must be seeded from the persisted
@@ -82,22 +87,15 @@ export interface InterruptValue {
 export function resolveResumeInput(
   execution: AgentExecution,
   graphState: GraphStateSnapshot,
-  userMessage: string,
 ): ResumeResult {
   const pendingInterrupts = extractPendingInterrupts(graphState);
   if (pendingInterrupts.length === 0) {
-    return {
-      graphInput: { messages: [{ role: "user", content: userMessage }] },
-      isResumeFromApproval: false,
-    };
+    return { isResumeFromApproval: false };
   }
 
   const decisions = extractApprovalDecisions(execution);
   if (decisions.size === 0) {
-    return {
-      graphInput: { messages: [{ role: "user", content: userMessage }] },
-      isResumeFromApproval: false,
-    };
+    return { isResumeFromApproval: false };
   }
 
   const resumeDict: Record<string, { action: string; comment?: string }> = {};
@@ -117,10 +115,7 @@ export function resolveResumeInput(
   }
 
   if (Object.keys(resumeDict).length === 0) {
-    return {
-      graphInput: { messages: [{ role: "user", content: userMessage }] },
-      isResumeFromApproval: false,
-    };
+    return { isResumeFromApproval: false };
   }
 
   console.log(

package/src/activities/execute-deep-agent/index.ts

@@ -161,12 +161,11 @@ export function createDeepAgentActivities(config: Config) {
           unattended: setup.unattended,
         });
 
-        const resume = resolveResumeInput(
-          setup.execution,
-          graphState,
-          setup.execution.spec!.message,
-        );
+        const resume = resolveResumeInput(setup.execution, graphState);
 
+        // Not an approval resume -> setup.langgraphInput, the single
+        // construction site of the turn's user message (string or multimodal
+        // content blocks when the turn carries inline images).
         const effectiveInput = resume.isResumeFromApproval
           ? resume.graphInput
           : setup.langgraphInput;

package/src/activities/execute-deep-agent/prompt-builder.ts

@@ -17,6 +17,10 @@ import {
   type SenderIdentity,
 } from "../../shared/sender-identity.js";
 import { formatSessionContextText } from "../../shared/session-context.js";
+import {
+  visionDisclosureLines,
+  type NotViewableEntry,
+} from "../../shared/attachment-vision.js";
 import { PLAN_MODE_DIRECTIVE } from "../../shared/plan-mode-prompt.js";
 import {
   buildImplementPlanDirective,
@@ -108,6 +112,12 @@ export interface PromptBuilderInput {
   workspaceFileRefs: string[];
   workspaceRoot: string;
   injectedFiles: InjectedFile[];
+  /**
+   * Vision facts about this turn's attachments (T04): which images the model
+   * sees inline in the user message and which degraded to path-only.
+   * Rendered inside the Input Files section.
+   */
+  vision?: VisionPromptInfo;
   /**
    * The execution's interaction mode. PLAN appends the shared plan-mode
    * directive so the model knows the turn's deliverable is a plan document.
@@ -158,6 +168,17 @@ export interface InjectedFile {
   size?: number | null;
 }
 
+/**
+ * Which images ride the user message inline (in send order) and which
+ * degraded to the file-pointer story — the shared vision wording
+ * (attachment-vision.ts) keeps this prompt and the Cursor harness's
+ * input-files section telling the agent the same thing.
+ */
+export interface VisionPromptInfo {
+  inlineFilenames: string[];
+  notViewable: NotViewableEntry[];
+}
+
 /**
  * Assemble the full system prompt from base instructions and contextual
  * sections. Pure function with no I/O.
@@ -196,7 +217,7 @@ export function buildEnhancedSystemPrompt(input: PromptBuilderInput): string {
   }
 
   if (input.injectedFiles.length > 0) {
-    prompt += buildInjectedFilesSection(input.injectedFiles);
+    prompt += buildInjectedFilesSection(input.injectedFiles, input.vision);
   }
 
   if (input.senderIdentity) {
@@ -374,7 +395,10 @@ function buildReferencedFilesSection(
   return section;
 }
 
-function buildInjectedFilesSection(files: InjectedFile[]): string {
+function buildInjectedFilesSection(
+  files: InjectedFile[],
+  vision?: VisionPromptInfo,
+): string {
   let section = "\n\n## Input Files\n\n";
   section +=
     "The following files have been provided as read-only reference " +
@@ -390,5 +414,16 @@ function buildInjectedFilesSection(files: InjectedFile[]): string {
     section += `- \`${f.path}\`${sizeInfo}\n`;
   }
 
+  // The vision lines (shared wording, attachment-vision.ts) tell the model
+  // which of these files it can already SEE inline in the user message versus
+  // which degraded to path-only — without them an agent silently ignores a
+  // photo the user believes it can see.
+  if (vision) {
+    const lines = visionDisclosureLines(vision.inlineFilenames, vision.notViewable);
+    if (lines.length > 0) {
+      section += "\n" + lines.join("\n") + "\n";
+    }
+  }
+
   return section;
 }

package/src/activities/execute-deep-agent/setup.ts

@@ -73,7 +73,7 @@ import type { ApprovalGateConfig } from "../../middleware/approval-gate.js";
 import { deriveExecutionFingerprintKey } from "../../shared/approval-fingerprint.js";
 import { getRunnerHitlMasterSecret } from "../../shared/fingerprint-secret.js";
 import { getModelPricing, ensureLoaded as ensurePricingLoaded } from "../../shared/model-pricing.js";
-import { getDefaultModel } from "../../shared/model-registry.js";
+import { getDefaultModel, getModelVisionCapability } from "../../shared/model-registry.js";
 import { buildChatModel } from "../../shared/model-client.js";
 import {
   loadArtifactStorageConfig,
@@ -99,6 +99,12 @@ import {
 } from "../../shared/skill-writer.js";
 import { filterSkills, SKILL_COUNT_THRESHOLD } from "../../shared/skill-relevance.js";
 import { injectAttachments } from "./attachment-injector.js";
+import {
+  DEEP_AGENT_VISION_PROFILE,
+  VisionBudget,
+  toLangChainImageBlocks,
+  type NotViewableEntry,
+} from "../../shared/attachment-vision.js";
 import { transformAndCompileSubagents } from "./subagent-transformer.js";
 import {
   resolveRecursionLimit,
@@ -508,14 +514,46 @@ export async function performSetup(deps: SetupDependencies): Promise<SetupResult
       timing.mark("resolve_skills");
     }
 
-    // Step 7c: Inject attachments
+    // Step 7c: Inject attachments. The vision budget rides along so image
+    // attachments are selected for inline delivery while their bytes are
+    // already in hand (attachment-vision.ts owns all policy). The budget
+    // also carries the root model's registry vision capability — the root
+    // model is the only one that ever receives inline image blocks
+    // (sub-agents get a fresh task description, summarization reuses this
+    // same model instance) — so a model flagged `vision: false` degrades
+    // images honestly instead of shipping a payload its provider rejects.
     const attachments = execution.spec!.attachments || [];
+    const visionBudget = new VisionBudget(DEEP_AGENT_VISION_PROFILE, {
+      modelVision: await getModelVisionCapability(modelName),
+    });
     const injectedFiles = await injectAttachments({
       backend: workspaceBackend,
       attachments,
       storage: artifactStorage,
       isLocalMode: config.mode === "local",
+      visionBudget,
     });
+    // Vision facts, derived once from the single injection result: the images
+    // the model will see inline (in attachment order) and the ones that
+    // degraded to path-only, disclosed in the system prompt's Input Files
+    // section.
+    const visionImages = injectedFiles.flatMap((f) => (f.vision ? [f.vision] : []));
+    const visionNotViewable: NotViewableEntry[] = injectedFiles.flatMap((f) =>
+      f.visionDegraded ? [{ path: f.path, reason: f.visionDegraded }] : [],
+    );
+    const visionPromptInfo = visionImages.length > 0 || visionNotViewable.length > 0
+      ? {
+          inlineFilenames: visionImages.map((v) => v.filename),
+          notViewable: visionNotViewable,
+        }
+      : undefined;
+    if (visionPromptInfo) {
+      console.log(
+        `[attachment-vision] execution=${executionId} inline=${visionImages.length} ` +
+        `(${visionImages.reduce((n, v) => n + v.byteSize, 0)} bytes) ` +
+        `degraded=${JSON.stringify(visionNotViewable.map((d) => `${d.path}:${d.reason}`))}`,
+      );
+    }
     timing.mark("inject_attachments");
 
     // Step 8: Build enhanced system prompt
@@ -535,6 +573,7 @@ export async function performSetup(deps: SetupDependencies): Promise<SetupResult
       workspaceFileRefs: execution.spec!.workspaceFileRefs || [],
       workspaceRoot: workspaceBackend.rootDir,
       injectedFiles,
+      vision: visionPromptInfo,
       interactionMode: execution.spec!.executionConfig?.interactionMode,
       buildFromPlan: execution.spec!.executionConfig?.buildFromPlan,
       contextBridge: readContextBridge(session.spec!.metadata),
@@ -784,8 +823,24 @@ export async function performSetup(deps: SetupDependencies): Promise<SetupResult
     if (outputSchema) {
       userMessage += `\n\n---\nIMPORTANT: When your analysis is complete, provide your findings as structured output matching the required schema. The system will capture your structured response automatically.`;
     }
+    // With inline images the user message becomes a multimodal content-block
+    // array — image blocks (with filename labels) FIRST, the composed text
+    // last, per Anthropic's images-before-text guidance. Without images the
+    // content stays a plain string, byte-identical to the pre-vision shape.
+    // The LangGraph messages reducer coerces either form into a HumanMessage
+    // untouched (verified against the installed @langchain/langgraph).
     const langgraphInput = {
-      messages: [{ role: "user", content: userMessage }],
+      messages: [
+        {
+          role: "user",
+          content: visionImages.length > 0
+            ? [
+                ...toLangChainImageBlocks(visionImages),
+                { type: "text", text: userMessage },
+              ]
+            : userMessage,
+        },
+      ],
     };
 
     const langgraphConfig: Record<string, unknown> = {

package/src/runner-manager.ts

@@ -332,6 +332,25 @@ export async function createStigmerRunnerManager(
 
   const activities = await createAllActivities(config);
   markBoot("activities_imported");
+  // Surface the resolved artifact store at boot (see runner.ts for rationale):
+  // a path/type misconfiguration is the #285 failure class, and it should be
+  // visible here rather than only when a read fails mid-execution.
+  try {
+    const { loadArtifactStorageConfig } = await import(
+      "./shared/artifact-storage.js"
+    );
+    const artifactCfg = loadArtifactStorageConfig(config);
+    console.log(
+      `[runner-manager] Artifact store: type=${artifactCfg.type}` +
+        (artifactCfg.type === "local"
+          ? ` | root=${artifactCfg.localPath}`
+          : ` | proxy=${artifactCfg.proxyEndpoint ?? "(unset)"}`),
+    );
+  } catch (err) {
+    console.warn(
+      `[runner-manager] Artifact store: could not resolve config for boot log: ${err}`,
+    );
+  }
   const payloadCodec = await createPayloadCodec(config);
 
   const connection = await NativeConnection.connect({

package/src/runner.ts

@@ -282,6 +282,25 @@ export async function createStigmerRunner(
       `Mode: ${config.mode} | ` +
       `Max concurrency: ${config.maxConcurrentActivities}`,
   );
+  // Surface the resolved artifact store at boot so a path/type misconfiguration
+  // (the #285 class of failure) is visible immediately instead of only when an
+  // attachment or offload read fails minutes into an execution.
+  try {
+    const { loadArtifactStorageConfig } = await import(
+      "./shared/artifact-storage.js"
+    );
+    const artifactCfg = loadArtifactStorageConfig(config);
+    console.log(
+      `[runner] Artifact store: type=${artifactCfg.type}` +
+        (artifactCfg.type === "local"
+          ? ` | root=${artifactCfg.localPath}`
+          : ` | proxy=${artifactCfg.proxyEndpoint ?? "(unset)"}`),
+    );
+  } catch (err) {
+    console.warn(
+      `[runner] Artifact store: could not resolve config for boot log: ${err}`,
+    );
+  }
 
   const payloadCodec = await createPayloadCodec(config);