create-walle 0.9.21 → 0.9.23

package/template/wall-e/coding/stream-processor.js

@@ -13,6 +13,55 @@ const { ArtifactStore } = require('./artifact-store');
 const { normalizeResponse, transformRequest, providerId } = require('./provider-transform');
 const { Confidence, EventName, FailureClass, Provenance, createLaneEvent } = require('./lane-events');
 const { reviewExternalActionGateway } = require('../external-action-gateway');
+const { isSuccessfulVerificationToolCall, normalizeToolCallEvidence } = require('./acceptance-contract');
+const { RuntimeToolExecutor } = require('../runtime/tool-executor');
+const { buildRuntimeToolResultEnvelope } = require('../runtime/tool-result-envelope');
+const { attachProjectionFlags } = require('../runtime/transcript-projection');
+const { isRetryable, parseRetryDelay } = require('../llm/retry');
+const { decorateProviderError } = require('../llm/provider-error');
+
+// Transient provider failures (e.g. a "fetch failed" network blip) shouldn't kill
+// a whole coding turn. We retry at stream *establishment* only — see
+// streamWithEstablishmentRetry — which covers both non-streaming providers
+// (streamFromChat) and any native chatStream provider, without risking duplicated
+// output from a mid-stream retry.
+const CODING_PROVIDER_RETRIES = 2;
+
+/**
+ * Retry a stream's establishment (the first event) on transient errors, then
+ * drain the rest without retry. A failure before the first event is yielded is a
+ * connection-level failure and is safe to retry; a failure after output has begun
+ * is not (retrying would duplicate already-emitted deltas).
+ *
+ * @param {() => AsyncIterable} makeStream  Creates a fresh stream per attempt.
+ * @param {{ signal?: AbortSignal, retries?: number }} [opts]
+ */
+async function* streamWithEstablishmentRetry(makeStream, { signal, retries = CODING_PROVIDER_RETRIES } = {}) {
+  let attempt = 0;
+  for (;;) {
+    const iterator = makeStream()[Symbol.asyncIterator]();
+    let first;
+    try {
+      first = await iterator.next();
+    } catch (err) {
+      const status = err?.status || err?.code || 0;
+      if (!signal?.aborted && attempt < retries && isRetryable(status, err?.message || '')) {
+        attempt += 1;
+        await new Promise((resolve) => setTimeout(resolve, parseRetryDelay(null, attempt - 1)));
+        continue;
+      }
+      throw err;
+    }
+    if (first.done) return;
+    yield first.value;
+    for (;;) {
+      // Mid-stream: never retry — propagate so the caller fails the turn cleanly.
+      const next = await iterator.next();
+      if (next.done) return;
+      yield next.value;
+    }
+  }
+}
 
 async function* streamFromChat(provider, request) {
   const response = normalizeResponse(recoverAllowedTextToolCalls(await provider.chat(request), request.tools), {
@@ -72,9 +121,10 @@ class StreamProcessor extends EventEmitter {
       temperature,
       signal,
     });
-    const stream = typeof this.provider.chatStream === 'function'
-      ? this.provider.chatStream(request)
-      : streamFromChat(this.provider, request);
+    const makeStream = typeof this.provider.chatStream === 'function'
+      ? () => this.provider.chatStream(request)
+      : () => streamFromChat(this.provider, request);
+    const stream = streamWithEstablishmentRetry(makeStream, { signal });
 
     const assistantMessageId = newId('assistant');
     const threadId = String(requestOptions.threadId || sessionId);
@@ -153,9 +203,25 @@ class StreamProcessor extends EventEmitter {
         await this._handleEvent(event, state, { sessionId, cwd, tools });
       }
       if (state.toolCalls.length > 0 && this.toolExecutor) {
-        for (const call of state.toolCalls) {
-          await this._runTool(call, state, { sessionId, cwd });
-        }
+        const executor = new RuntimeToolExecutor({
+          toolSpecs: tools,
+          executeToolCall: async (call) => this._runTool(call, state, { sessionId, cwd }),
+          runtimeProfile: {
+            profileId: 'coding',
+            permissionProfile: 'coding-workspace',
+            persistenceProfile: 'jsonl-transcript',
+            outputContract: 'coding-acceptance',
+            toolsets: ['coding', 'memory', 'mcp', 'skills', 'local'],
+          },
+          now: this.now,
+        });
+        await executor.run(state.toolCalls, {
+          sessionId,
+          cwd,
+          provider: state.provider,
+          model: state.model,
+        });
+        orderToolResultsByCalls(state, state.toolCalls);
       }
       if (this.snapshotService?.captureStepFinish) {
         const snapshot = await this.snapshotService.captureStepFinish({ sessionId, cwd, messageId: assistantMessageId });
@@ -164,8 +230,19 @@ class StreamProcessor extends EventEmitter {
       state.status = 'finished';
     } catch (err) {
       state.status = 'error';
-      state.errors.push(err.message);
-      await this._record(sessionId, cwd, 'error', { message: err.message });
+      // Classify the raw provider error (e.g. "fetch failed") into a friendly,
+      // actionable message — parity with the chat path. The raw message is kept
+      // as rawError for debugging; providerError carries the rich block + action.
+      const { providerError } = decorateProviderError(err, { provider: state.provider, model: state.model });
+      const friendly = providerError.userMessage || err.message;
+      const rawError = err.message;
+      // Keep the RAW message in state.errors: the orchestrator pattern-matches it
+      // (isProviderFailureRecoverableByCli) to decide CLI fallback. Surface the
+      // friendly, classified text on the user-facing record/lane/emit, and expose
+      // providerError on the returned turn so the orchestrator can render it too.
+      state.errors.push(rawError);
+      state.providerError = providerError;
+      await this._record(sessionId, cwd, 'error', { message: friendly, providerError, rawError });
       await this._laneEvent({
         name: EventName.SESSION_FAILED,
         sessionId,
@@ -174,11 +251,11 @@ class StreamProcessor extends EventEmitter {
         model: state.model,
         runId: assistantMessageId,
         failureClass: FailureClass.PROVIDER,
-        reason: err.message,
+        reason: friendly,
         confidence: Confidence.HIGH,
-        data: { error: err.message },
+        data: { error: friendly, providerError, rawError },
       });
-      this._emit({ type: 'error', sessionId, messageId: assistantMessageId, error: err.message });
+      this._emit({ type: 'error', sessionId, messageId: assistantMessageId, error: friendly, providerError });
     } finally {
       await this._runtimeItem(sessionId, cwd, {
         threadId,
@@ -336,6 +413,7 @@ class StreamProcessor extends EventEmitter {
   }
 
   async _runTool(call, state, { sessionId, cwd }) {
+    const toolStartedAt = Date.now();
     try {
       await this._runtimeItem(sessionId, cwd, {
         threadId: state.threadId,
@@ -346,13 +424,30 @@ class StreamProcessor extends EventEmitter {
         name: call.name,
         data: { input: call.input },
       });
+      this._emit({
+        type: 'tool_call',
+        id: call.id,
+        toolCallId: call.id,
+        tool: call.name,
+        name: call.name,
+        summary: `Running ${call.name}`,
+        input: call.input,
+      });
       const gatewayReview = reviewExternalActionGateway({
         toolName: call.name,
         input: call.input,
       });
       if (!gatewayReview.admitted) {
         const result = gatewayReview.result;
-        state.toolResults.push({ toolCallId: call.id, name: call.name, result });
+        const envelope = buildRuntimeToolResultEnvelope({
+          call,
+          spec: { name: call.name },
+          result,
+          startedAt: toolStartedAt,
+          endedAt: Date.now(),
+          now: this.now,
+        });
+        state.toolResults.push({ toolCallId: call.id, name: call.name, result, envelope });
         await this._runtimeItem(sessionId, cwd, {
           threadId: state.threadId,
           turnId: state.turnId,
@@ -360,7 +455,7 @@ class StreamProcessor extends EventEmitter {
           role: 'tool',
           status: 'completed',
           name: call.name,
-          data: { result },
+          data: { result, envelope },
         });
         await this._record(sessionId, cwd, 'tool', {
           state: 'blocked',
@@ -368,6 +463,17 @@ class StreamProcessor extends EventEmitter {
           name: call.name,
           input: call.input,
           result,
+          envelope,
+        });
+        this._emit({
+          type: 'tool_result',
+          id: call.id,
+          toolCallId: call.id,
+          tool: call.name,
+          name: call.name,
+          summary: `Blocked ${call.name}`,
+          result,
+          error: result?.reason || result?.error || '',
         });
         await this._laneEvent({
           name: EventName.TOOL_FINISHED,
@@ -388,15 +494,105 @@ class StreamProcessor extends EventEmitter {
           name: call.name,
           input: call.input,
         });
-        const permission = await this.permissionService.authorize({
-          sessionId,
-          tool: call.name,
-          input: call.input,
-          cwd,
-          projectRoot: cwd,
-          headless: this.headless,
-          metadata: { toolCallId: call.id },
+        // When a request actually needs the user (interactive ask), surface it: flip
+        // the lane to "waiting" so the session reads as "Needs You" (not a silent hang
+        // while the agent waits indefinitely), and persist a durable, actionable
+        // approval card part (carrying permId) that survives reload. onAsked fires
+        // synchronously inside ask(), before authorize() blocks.
+        const permEvents = this.permissionService.events;
+        let askedPermId = null;
+        const onAsked = (entry) => {
+          if (!entry || String(entry.metadata?.toolCallId || '') !== String(call.id)) return;
+          askedPermId = entry.id;
+          this._laneEvent({
+            name: EventName.SESSION_WAITING,
+            sessionId,
+            cwd,
+            provider: state.provider,
+            model: state.model,
+            runId: state.messageId,
+            confidence: Confidence.HIGH,
+            reason: 'Waiting for your approval',
+            data: { toolCallId: call.id, name: call.name, permId: entry.id, waitingFor: 'permission' },
+          });
+          this._record(sessionId, cwd, 'permission_request', {
+            permId: entry.id,
+            permissionContextId: entry.permissionContextId || '',
+            toolCallId: call.id,
+            tool: call.name,
+            permission: entry.permission || '',
+            patterns: entry.patterns || (entry.pattern ? [entry.pattern] : []),
+            command: (call.input && call.input.command) || '',
+            input: call.input,
+            reason: entry.metadata?.evaluated?.reason || 'Requires approval',
+          });
+          // Live event so a watching client renders the approval card immediately
+          // (the durable part above covers reload). Shape matches the chat permission
+          // card the CTM frontend already knows how to render.
+          this._emit({
+            type: 'permission_request',
+            sessionId,
+            permId: entry.id,
+            tool: call.name,
+            command: (call.input && call.input.command) || '',
+            args: Array.isArray(call.input && call.input.args) ? call.input.args : [],
+            input: call.input,
+            reason: entry.metadata?.evaluated?.reason || 'Requires approval',
+          });
+        };
+        if (permEvents?.on) permEvents.on('permission.asked', onAsked);
+        let permission;
+        try {
+          permission = await this.permissionService.authorize({
+            sessionId,
+            tool: call.name,
+            input: call.input,
+            cwd,
+            projectRoot: cwd,
+            headless: this.headless,
+            metadata: { toolCallId: call.id },
+          });
+        } finally {
+          if (permEvents?.off) permEvents.off('permission.asked', onAsked);
+        }
+        await this._record(sessionId, cwd, 'tool', {
+          state: permission.decision === 'allow' ? 'permission_allowed' : 'permission_denied',
+          toolCallId: call.id,
+          name: call.name,
+          permissionContextId: permission.permissionContextId || '',
+          decision: permission.decision,
+          source: permission.source || '',
+          reason: permission.reason || permission.message || '',
         });
+        // If a waiting card was surfaced, clear it and return the lane to running.
+        if (askedPermId) {
+          await this._record(sessionId, cwd, 'permission_resolved', {
+            permId: askedPermId,
+            toolCallId: call.id,
+            tool: call.name,
+            decision: permission.decision,
+            source: permission.source || '',
+          });
+          // Live event so a watching client clears/marks the card without a reload.
+          this._emit({
+            type: 'permission_resolved',
+            sessionId,
+            permId: askedPermId,
+            tool: call.name,
+            decision: permission.decision,
+            source: permission.source || '',
+          });
+          this._laneEvent({
+            name: EventName.SESSION_RUNNING,
+            sessionId,
+            cwd,
+            provider: state.provider,
+            model: state.model,
+            runId: state.messageId,
+            confidence: Confidence.HIGH,
+            data: { toolCallId: call.id, name: call.name },
+          });
+        }
         if (permission.decision !== 'allow') {
           throw new Error(`Permission denied: ${permission.reason || permission.message || permission.decision}`);
         }
@@ -429,11 +625,20 @@ class StreamProcessor extends EventEmitter {
       });
       let result = await this.toolExecutor(call, { sessionId, cwd, model: state.model, provider: state.provider });
       await this._recordDiagnostics(sessionId, cwd, call, result);
+      const typedArtifacts = this._storeTypedArtifacts(result, { sessionId, cwd, call });
       const routed = this._routeToolOutput(result, { sessionId, cwd, call });
       result = routed.result;
+      const envelope = buildRuntimeToolResultEnvelope({
+        call,
+        spec: { name: call.name },
+        result,
+        startedAt: toolStartedAt,
+        endedAt: Date.now(),
+        now: this.now,
+      });
       if (isEditTool(call.name) && !result?.error) state.hadEdit = true;
       if (isSuccessfulTestCommand(call, result)) state.verified = true;
-      state.toolResults.push({ toolCallId: call.id, name: call.name, result });
+      state.toolResults.push({ toolCallId: call.id, name: call.name, result, envelope });
       if (routed.artifact) {
         await this._record(sessionId, cwd, 'artifact', {
           type: 'tool_output',
@@ -442,6 +647,14 @@ class StreamProcessor extends EventEmitter {
           artifact: routed.artifact,
         });
       }
+      for (const artifact of typedArtifacts) {
+        await this._record(sessionId, cwd, 'artifact', {
+          type: artifact.kind || 'artifact',
+          toolCallId: call.id,
+          name: call.name,
+          artifact,
+        });
+      }
       await this._runtimeItem(sessionId, cwd, {
         threadId: state.threadId,
         turnId: state.turnId,
@@ -449,13 +662,23 @@ class StreamProcessor extends EventEmitter {
         role: 'tool',
         status: 'completed',
         name: call.name,
-        data: { result },
+        data: { result, envelope },
       });
       await this._record(sessionId, cwd, 'tool', {
         state: 'completed',
         toolCallId: call.id,
         name: call.name,
         result,
+        envelope,
+      });
+      this._emit({
+        type: 'tool_result',
+        id: call.id,
+        toolCallId: call.id,
+        tool: call.name,
+        name: call.name,
+        summary: `Completed ${call.name}`,
+        result,
       });
       await this._laneEvent({
         name: EventName.TOOL_FINISHED,
@@ -468,8 +691,17 @@ class StreamProcessor extends EventEmitter {
         data: { toolCallId: call.id, name: call.name },
       });
     } catch (err) {
+      const envelope = buildRuntimeToolResultEnvelope({
+        call,
+        spec: { name: call.name },
+        result: { error: err.message },
+        error: err,
+        startedAt: toolStartedAt,
+        endedAt: Date.now(),
+        now: this.now,
+      });
       state.toolErrors.push(err.message);
-      state.toolResults.push({ toolCallId: call.id, name: call.name, error: err.message });
+      state.toolResults.push({ toolCallId: call.id, name: call.name, error: err.message, envelope });
       await this._runtimeItem(sessionId, cwd, {
         threadId: state.threadId,
         turnId: state.turnId,
@@ -477,13 +709,23 @@ class StreamProcessor extends EventEmitter {
         role: 'tool',
         status: 'failed',
         name: call.name,
-        data: { error: err.message },
+        data: { error: err.message, envelope },
       });
       await this._record(sessionId, cwd, 'tool', {
         state: 'error',
         toolCallId: call.id,
         name: call.name,
         error: err.message,
+        envelope,
+      });
+      this._emit({
+        type: 'tool_result',
+        id: call.id,
+        toolCallId: call.id,
+        tool: call.name,
+        name: call.name,
+        summary: `Failed ${call.name}`,
+        error: err.message,
       });
       await this._laneEvent({
         name: EventName.TOOL_FAILED,
@@ -509,6 +751,36 @@ class StreamProcessor extends EventEmitter {
     });
   }
 
+  _storeTypedArtifacts(result, { sessionId, call }) {
+    const descriptors = extractTypedArtifactDescriptors(result, call);
+    if (!descriptors.length) return [];
+    const stored = [];
+    const seen = new Set();
+    for (const descriptor of descriptors) {
+      try {
+        const artifact = this.artifactStore?.storeArtifact
+          ? this.artifactStore.storeArtifact({
+            sessionId,
+            toolCallId: call.id,
+            toolName: call.name,
+            kind: descriptor.kind,
+            sourcePath: descriptor.path || descriptor.sourcePath,
+            content: descriptor.content,
+            mimeType: descriptor.mimeType,
+            bytes: descriptor.bytes,
+            sha256: descriptor.sha256,
+            metadata: descriptor.metadata || {},
+          })
+          : descriptor;
+        const key = artifact.artifactId || `${artifact.kind}:${artifact.path}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        stored.push(artifact);
+      } catch {}
+    }
+    return stored;
+  }
+
   async _recordDiagnostics(sessionId, cwd, call, result) {
     const diagnostics = extractDiagnostics(result);
     for (const diagnostic of diagnostics) {
@@ -529,7 +801,7 @@ class StreamProcessor extends EventEmitter {
   }
 
   async _record(sessionId, cwd, partType, data) {
-    const payload = {
+    const payload = attachProjectionFlags({
       type: 'walle_part',
       provider: 'walle',
       sessionId,
@@ -537,7 +809,7 @@ class StreamProcessor extends EventEmitter {
       timestamp: this.now(),
       partType,
       data,
-    };
+    }, partType);
     if (this.transcript?.appendPart) return this.transcript.appendPart(payload);
     if (this.transcript?.append) return this.transcript.append(payload);
     return null;
@@ -561,7 +833,7 @@ class StreamProcessor extends EventEmitter {
 
   async _runtimePart(sessionId, cwd, partType, event = {}) {
     if (!this.transcript) return null;
-    const payload = {
+    const payload = attachProjectionFlags({
       type: 'walle_part',
       provider: 'walle',
       sessionId,
@@ -573,7 +845,7 @@ class StreamProcessor extends EventEmitter {
         threadId: event.threadId || sessionId,
         ...event,
       },
-    };
+    }, partType);
     if (partType === 'runtime_thread' && this.transcript.appendRuntimeThread) return this.transcript.appendRuntimeThread(payload.data, payload);
     if (partType === 'runtime_turn' && this.transcript.appendRuntimeTurn) return this.transcript.appendRuntimeTurn(payload.data, payload);
     if (partType === 'runtime_item' && this.transcript.appendRuntimeItem) return this.transcript.appendRuntimeItem(payload.data, payload);
@@ -604,12 +876,18 @@ function isEditTool(name) {
   return ['edit_file', 'write_file', 'apply_patch', 'multi_edit'].includes(name);
 }
 
+function orderToolResultsByCalls(state, calls = []) {
+  if (!state?.toolResults || !Array.isArray(state.toolResults)) return;
+  const order = new Map((calls || []).map((call, index) => [call.id, index]));
+  state.toolResults.sort((a, b) => {
+    const ai = order.has(a.toolCallId) ? order.get(a.toolCallId) : Number.MAX_SAFE_INTEGER;
+    const bi = order.has(b.toolCallId) ? order.get(b.toolCallId) : Number.MAX_SAFE_INTEGER;
+    return ai - bi;
+  });
+}
+
 function isSuccessfulTestCommand(call, result) {
-  if (call?.name !== 'run_shell') return false;
-  const command = String(call.input?.command || '');
-  if (!/\b(test|spec|jest|mocha|pytest|npm\s+test|node\s+test\.js)\b/i.test(command)) return false;
-  if (result?.error || result?.exitCode) return false;
-  return true;
+  return isSuccessfulVerificationToolCall(normalizeToolCallEvidence(call, result));
 }
 
 function extractDiagnostics(result) {
@@ -639,9 +917,85 @@ function extractDiagnostics(result) {
   return diagnostics;
 }
 
+function extractTypedArtifactDescriptors(result, call = {}) {
+  const descriptors = [];
+  const push = (descriptor) => {
+    if (!descriptor || typeof descriptor !== 'object') return;
+    const kind = String(descriptor.kind || descriptor.type || '').trim();
+    const artifactPath = descriptor.path || descriptor.file_path || descriptor.sourcePath || '';
+    if (!kind || (!artifactPath && descriptor.content === undefined)) return;
+    descriptors.push({
+      ...descriptor,
+      kind,
+      path: artifactPath,
+      mimeType: descriptor.mimeType || descriptor.mime_type || mimeTypeForKind(kind, artifactPath),
+      metadata: descriptor.metadata || {},
+    });
+  };
+  if (result && typeof result === 'object') {
+    if (result.artifact) push(result.artifact);
+    if (Array.isArray(result.artifacts)) result.artifacts.forEach(push);
+    if (call.name === 'browser_screenshot' && result.ok && result.path) {
+      push({
+        kind: 'screenshot',
+        path: result.path,
+        mimeType: 'image/png',
+        bytes: result.size || result.bytes,
+        metadata: {
+          url: result.url,
+          viewport: result.viewport,
+          width: result.width,
+          height: result.height,
+        },
+      });
+    }
+    if ((call.name === 'make_pdf' || call.name === 'pdf_info') && result.ok && result.path) {
+      push({
+        kind: 'pdf',
+        path: result.path,
+        mimeType: 'application/pdf',
+        bytes: result.bytes,
+        sha256: result.sha256,
+        metadata: { page_count: result.page_count },
+      });
+    }
+    if ((call.name === 'make_pdf' || call.name === 'pdf_render_pages') && Array.isArray(result.preview_paths)) {
+      result.preview_paths.forEach((previewPath, index) => push({
+        kind: 'pdf_page_preview',
+        path: previewPath,
+        mimeType: 'image/jpeg',
+        metadata: {
+          source_pdf: result.path,
+          page: result.pages?.start ? result.pages.start + index : index + 1,
+          page_count: result.page_count,
+        },
+      }));
+    }
+  }
+  const seen = new Set();
+  return descriptors.filter((descriptor) => {
+    const key = `${descriptor.kind}:${descriptor.path || ''}:${descriptor.sha256 || ''}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function mimeTypeForKind(kind, artifactPath) {
+  if (kind === 'pdf') return 'application/pdf';
+  if (kind === 'screenshot') return 'image/png';
+  if (kind === 'pdf_page_preview') return 'image/jpeg';
+  const text = String(artifactPath || '').toLowerCase();
+  if (text.endsWith('.pdf')) return 'application/pdf';
+  if (text.endsWith('.png')) return 'image/png';
+  if (text.endsWith('.jpg') || text.endsWith('.jpeg')) return 'image/jpeg';
+  return 'application/octet-stream';
+}
+
 module.exports = {
   StreamProcessor,
   streamFromChat,
   isSuccessfulTestCommand,
   extractDiagnostics,
+  extractTypedArtifactDescriptors,
 };