@agent-native/core 0.97.0 → 0.98.0

package/corpus/core/scripts/materialize-source-corpus.mjs

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import { execFileSync } from "node:child_process";
+import { randomBytes } from "node:crypto";
 import {
   copyFileSync,
   existsSync,
@@ -7,6 +8,7 @@ import {
   mkdirSync,
   readFileSync,
   readdirSync,
+  renameSync,
   rmSync,
   writeFileSync,
 } from "node:fs";
@@ -113,6 +115,14 @@ const excludedFileSuffixes = [
   ".tsbuildinfo",
 ];
 
+// Matches both the "corpus" output dir itself and the unique per-process
+// temp dirs materializeSourceCorpus() swaps into place (see
+// swapCorpusDirIntoPlace), so a non-git fallback filesystem walk never
+// recurses into the corpus output it is currently producing.
+function isCorpusOutputDirName(name) {
+  return name === "corpus" || name.startsWith("corpus.tmp-");
+}
+
 function shouldSkipFile(name) {
   if (excludedFileNames.has(name)) return true;
   if (name === ".env") return true;
@@ -125,7 +135,14 @@ function shouldSkipFile(name) {
 
 function shouldSkipRelativePath(relativePath) {
   const segments = relativePath.split("/");
-  if (segments.some((segment) => excludedDirNames.has(segment))) return true;
+  if (
+    segments.some(
+      (segment) =>
+        excludedDirNames.has(segment) || isCorpusOutputDirName(segment),
+    )
+  ) {
+    return true;
+  }
   const name = segments[segments.length - 1];
   return shouldSkipFile(name);
 }
@@ -190,7 +207,12 @@ function listFilesystemFiles(absRoot, relRoot) {
     const abs = join(absRoot, entry.name);
     const rel = `${relRoot}/${entry.name}`.split("\\").join("/");
     if (entry.isDirectory()) {
-      if (excludedDirNames.has(entry.name)) continue;
+      if (
+        excludedDirNames.has(entry.name) ||
+        isCorpusOutputDirName(entry.name)
+      ) {
+        continue;
+      }
       files.push(...listFilesystemFiles(abs, rel));
     } else if (entry.isFile()) {
       files.push(rel);
@@ -210,7 +232,7 @@ function sourceFilesFor(rootRel) {
     .sort();
 }
 
-function copySourceFiles(rootRel, targetName) {
+function copySourceFiles(rootRel, targetName, baseDir) {
   const files = sourceFilesFor(rootRel);
   const prefix = `${rootRel}/`;
   let copied = 0;
@@ -220,7 +242,7 @@ function copySourceFiles(rootRel, targetName) {
     const sourcePath = join(repoRoot, file);
     if (!existsSync(sourcePath)) continue;
     if (!lstatSync(sourcePath).isFile()) continue;
-    const targetPath = join(corpusDir, targetName, relToRoot);
+    const targetPath = join(baseDir, targetName, relToRoot);
     mkdirSync(dirname(targetPath), { recursive: true });
     copyFileSync(sourcePath, targetPath);
     copied += 1;
@@ -228,7 +250,7 @@ function copySourceFiles(rootRel, targetName) {
   return { files: copied };
 }
 
-function writeCorpusReadme(stats) {
+function writeCorpusReadme(stats, baseDir) {
   const lines = [
     "# Agent Native Source Corpus",
     "",
@@ -263,24 +285,88 @@ function writeCorpusReadme(stats) {
     `- template files: ${stats.templateFiles}`,
     "",
   ];
-  writeFileSync(join(corpusDir, "README.md"), lines.join("\n"));
+  writeFileSync(join(baseDir, "README.md"), lines.join("\n"));
+}
+
+const swapMaxAttempts = 5;
+const swapRetryDelayMs = 40;
+
+// Synchronous sleep with no subprocess dependency, so retrying stays
+// cross-platform (Windows CI has no `sleep` binary under cmd.exe).
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+// `corpusDir` is only ever replaced wholesale by a fully-populated temp dir
+// (see swapCorpusDirIntoPlace), so its presence + README.md proves *some*
+// materialize run -- ours or a concurrent one -- finished successfully.
+export function looksLikeMaterializedCorpus(dir) {
+  return existsSync(join(dir, "README.md"));
+}
+
+// Concurrent `materializeSourceCorpus()` runs (e.g. two overlapping
+// `scripts/dev-lazy.ts` prebuilds racing on the same checkout) used to both
+// `rmSync`/repopulate the shared `corpusDir` directly, which could throw
+// ENOTEMPTY out of the recursive rm/rename when one process's writes landed
+// mid-walk of another's, crashing dev-lazy startup entirely. Build into a
+// unique-per-process temp dir instead (nothing else can touch that path),
+// then swap it into place with a small bounded retry that treats "a
+// concurrent run already produced an equivalent corpus" as success rather
+// than a fatal error.
+export function swapCorpusDirIntoPlace(tempDir, targetDir = corpusDir) {
+  for (let attempt = 1; attempt <= swapMaxAttempts; attempt += 1) {
+    try {
+      rmSync(targetDir, { recursive: true, force: true });
+    } catch {
+      // A concurrent rename may already be repopulating targetDir; let the
+      // renameSync below decide the outcome instead of failing here.
+    }
+    try {
+      renameSync(tempDir, targetDir);
+      return true;
+    } catch (error) {
+      const code = error && error.code;
+      const concurrentWriter =
+        code === "ENOTEMPTY" ||
+        code === "EEXIST" ||
+        code === "ENOENT" ||
+        code === "EPERM";
+      if (!concurrentWriter) throw error;
+      if (looksLikeMaterializedCorpus(targetDir)) {
+        rmSync(tempDir, { recursive: true, force: true });
+        return false;
+      }
+      if (attempt < swapMaxAttempts) {
+        sleepSync(swapRetryDelayMs * attempt);
+        continue;
+      }
+      throw error;
+    }
+  }
+  return false;
 }
 
 export function materializeSourceCorpus() {
-  rmSync(corpusDir, { recursive: true, force: true });
-  mkdirSync(corpusDir, { recursive: true });
+  const tempDir = `${corpusDir}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`;
+  rmSync(tempDir, { recursive: true, force: true });
+  mkdirSync(tempDir, { recursive: true });
 
-  const coreStats = copySourceFiles("packages/core", "core");
-  const templateStats = copySourceFiles("templates", "templates");
+  const coreStats = copySourceFiles("packages/core", "core", tempDir);
+  const templateStats = copySourceFiles("templates", "templates", tempDir);
+
+  writeCorpusReadme(
+    { coreFiles: coreStats.files, templateFiles: templateStats.files },
+    tempDir,
+  );
 
-  writeCorpusReadme({
-    coreFiles: coreStats.files,
-    templateFiles: templateStats.files,
-  });
+  const applied = swapCorpusDirIntoPlace(tempDir);
 
   const size = relative(packageDir, corpusDir);
+  const note = applied
+    ? ""
+    : " (accepted a concurrent run's equivalent corpus)";
   console.log(
-    `[agent-native] Materialized source corpus at ${size} (${coreStats.files} core files, ${templateStats.files} template files).`,
+    `[agent-native] Materialized source corpus at ${size} (${coreStats.files} core files, ${templateStats.files} template files).${note}`,
   );
 }
 

package/corpus/core/src/action.ts

@@ -307,6 +307,25 @@ interface DefineActionWithSchema<
   schema: TSchema;
   /** Legacy parameters — ignored when `schema` is provided. */
   parameters?: never;
+  /**
+   * Optional alternate Standard Schema (Zod, Valibot, ArkType) used ONLY to
+   * build the tool definition advertised to the model — the JSON Schema that
+   * lands in the Claude `tools` array (and MCP/A2A tool listings, which read
+   * the same `tool.parameters`). Runtime validation always runs against
+   * `schema` above via the normal `wrapWithValidation` path; setting this
+   * never weakens validation and never changes `run()`'s argument type.
+   *
+   * Use this when the full input schema is much richer than what the model
+   * needs to see up front — the canonical example is a deep discriminated
+   * union of block/shape types where a per-call catalog lookup tool (e.g.
+   * `get-plan-blocks`) already teaches the full field shapes. Advertise a
+   * compact version (e.g. an enum of valid `type` values plus a note to call
+   * the lookup tool) instead of embedding every variant's fields in every
+   * request. Invalid calls still get the full, actionable validation error
+   * (missing/invalid fields, received args) from `schema` — this only trims
+   * what is proactively shown, not what is accepted or checked.
+   */
+  agentInputSchema?: StandardSchemaV1;
   /** Optional Standard Schema-compatible schema (Zod, Valibot, ArkType) the
    *  action's RETURN value is validated against AFTER `run()` resolves. Borrowed
    *  from Mastra/Flue structured-output. When omitted, behavior is byte-for-byte
@@ -437,6 +456,10 @@ interface DefineActionWithParams<
   parameters?: TParams;
   /** Standard Schema — not used in this overload. */
   schema?: never;
+  /** Advertised-only schema override — not used in this overload (no runtime
+   *  schema to advertise a compact alternative for). See the schema overload
+   *  above. */
+  agentInputSchema?: never;
   /** Optional Standard Schema-compatible schema the action's RETURN value is
    *  validated against AFTER `run()` resolves. See the schema overload above.
    *  When omitted, behavior is byte-for-byte unchanged. */
@@ -632,6 +655,25 @@ export function defineAction(options: any) {
     };
   }
 
+  // `agentInputSchema` swaps in a compact JSON Schema for the ADVERTISED tool
+  // definition only (what the model/MCP/A2A tool listings see). It never
+  // touches validation below — `wrapWithValidation` is always called with
+  // `options.schema`, the full schema, as the source of truth for what's
+  // accepted. Passing the same (compact) `toolParameters` into it only
+  // affects the top-level "Expected: { ... }" hint text and gateway string
+  // coercion, both of which only look at top-level property names/types —
+  // unaffected by trimming a nested union, so this stays safe either way.
+  if (
+    hasSchema &&
+    options.agentInputSchema &&
+    "~standard" in options.agentInputSchema
+  ) {
+    toolParameters = schemaToJsonSchema(
+      options.agentInputSchema,
+      options.description,
+    );
+  }
+
   // Wrap run() with INPUT validation when schema is provided.
   // Pass toolParameters so the validation error can echo the expected signature
   // (required vs optional fields) and help the caller self-correct.

package/corpus/core/src/agent/engine/builder-engine.ts

@@ -222,6 +222,9 @@ class BuilderEngine implements AgentEngine {
         opts.maxOutputTokens,
         opts.model,
       ),
+      ...(typeof opts.temperature === "number"
+        ? { temperature: opts.temperature }
+        : {}),
       ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),
     };
 

package/corpus/core/src/agent/engine/context-directives-transform.ts

@@ -0,0 +1,69 @@
+import { applyContextDirectives } from "../context-xray/apply-directives.js";
+import { loadContextDirectives } from "../context-xray/directives-store.js";
+import {
+  buildManifest,
+  writeContextManifest,
+} from "../context-xray/manifest.js";
+import { computeProtectedSegmentIds } from "../context-xray/segments.js";
+import type { EngineMessage } from "./types.js";
+
+/**
+ * Context X-Ray transform for one `runAgentLoop` iteration: loads any
+ * directives for the thread (evict/pin/restore), applies them to the raw
+ * message history, and best-effort persists a manifest describing what was
+ * sent to the model.
+ *
+ * Returns the transformed messages, or the original `messages` array
+ * unchanged (by reference) if the transform throws — the transform is a
+ * hardening/observability layer, never a gate, so a failure here must not
+ * break the turn. The manifest write is fire-and-forget (not awaited) so it
+ * never adds latency to the model-call path.
+ *
+ * Moved verbatim out of `runAgentLoop`'s per-iteration setup — behavior
+ * unchanged. Caller is still responsible for gating this on `threadId` being
+ * present and for the separate Observational Memory pass that runs after it.
+ */
+export async function applyContextXrayTransformForIteration(opts: {
+  threadId: string;
+  ownerEmail?: string | null;
+  turnId?: string;
+  model: string;
+  messages: EngineMessage[];
+}): Promise<EngineMessage[]> {
+  const { threadId, ownerEmail, turnId, model, messages } = opts;
+  try {
+    const directives = await loadContextDirectives(threadId, {
+      ownerEmail: ownerEmail ?? null,
+    });
+    const protectedSegmentIds = computeProtectedSegmentIds(messages);
+    const { messages: transformedMessages, appliedStatus } =
+      applyContextDirectives(messages, directives, {
+        protectedSegmentIds,
+      });
+    const manifest = await buildManifest({
+      threadId,
+      ...(turnId ? { turnId } : {}),
+      model,
+      rawMessages: messages,
+      sentMessages: transformedMessages,
+      appliedStatus,
+      directives,
+      protectedSegmentIds,
+      source: "structured",
+      enforceable: true,
+    });
+    void writeContextManifest(threadId, manifest).catch((err) => {
+      console.warn(
+        "[context-xray] failed to write manifest:",
+        err instanceof Error ? err.message : String(err),
+      );
+    });
+    return transformedMessages;
+  } catch (err) {
+    console.warn(
+      "[context-xray] context transform skipped:",
+      err instanceof Error ? err.message : String(err),
+    );
+    return messages;
+  }
+}

package/corpus/core/src/agent/engine/continuation-dispatch-retry.ts

@@ -0,0 +1,168 @@
+import type {
+  ChainServerDrivenContinuationDeps,
+  ContinuationDispatchBudget,
+} from "../production-agent.js";
+
+/**
+ * The subset of `ChainServerDrivenContinuationDeps` this stage needs,
+ * already resolved to their non-optional defaults (mirrors how
+ * `chainServerDrivenContinuation` builds its local `d` object).
+ */
+export type ContinuationDispatchRetryDeps = Required<
+  Pick<
+    ChainServerDrivenContinuationDeps,
+    | "sleep"
+    | "updateRunHeartbeat"
+    | "fireInternalDispatch"
+    | "readBackgroundRunClaim"
+  >
+>;
+
+/**
+ * Attempts to deliver one background-continuation dispatch, retrying with
+ * backoff up to `dispatchBudget.maxDispatchAttempts` times.
+ *
+ * Moved verbatim out of `chainServerDrivenContinuation`'s "TRANSACTIONAL
+ * HANDOFF" section — behavior, ordering, and error handling unchanged. The
+ * caller is still responsible for: minting `nextRunId`, inserting the
+ * successor row, building `dispatchBody`, and handling the
+ * deferred-recovery / fatal-error paths after this returns.
+ *
+ * `isLoopProtectionDispatchError` and `maxNestedSelfDispatchDepth` are
+ * passed in (rather than imported) so this module has no runtime import
+ * back on `production-agent.js` — only the type-only import above, which
+ * TypeScript erases.
+ */
+export async function attemptContinuationDispatch(params: {
+  event: unknown;
+  chainViaDurableBackground: boolean;
+  backgroundContinuationCount: number;
+  nextRunId: string;
+  nextRowInserted: boolean;
+  continuationDispatchPath: string;
+  dispatchBody: Record<string, unknown>;
+  dispatchBudget: ContinuationDispatchBudget;
+  isLoopProtectionDispatchError: (err: unknown) => boolean;
+  maxNestedSelfDispatchDepth: number;
+  deps: ContinuationDispatchRetryDeps;
+}): Promise<{
+  dispatched: boolean;
+  lastDispatchErr: unknown;
+  nestedDepthExceeded: boolean;
+}> {
+  const {
+    event,
+    chainViaDurableBackground,
+    backgroundContinuationCount,
+    nextRunId,
+    nextRowInserted,
+    continuationDispatchPath,
+    dispatchBody,
+    dispatchBudget,
+    isLoopProtectionDispatchError,
+    maxNestedSelfDispatchDepth,
+    deps: d,
+  } = params;
+  const maxDispatchAttempts = dispatchBudget.maxDispatchAttempts;
+  const dispatchResponseTimeoutMs = dispatchBudget.dispatchResponseTimeoutMs;
+
+  let dispatched = false;
+  let lastDispatchErr: unknown;
+  // Proactive nested-chain safety margin — see `MAX_NESTED_SELF_DISPATCH_DEPTH`.
+  // Skip the nested dispatch attempt entirely once this segment's hop count
+  // reaches the cap; a nested attempt at this depth is expected to trip
+  // Netlify's loop protection, so there is nothing to gain by trying it and
+  // burning this worker's remaining wall clock on a doomed call. Falls
+  // straight into the same deferred-recovery path below as an exhausted
+  // retry budget.
+  const nestedDepthExceeded =
+    backgroundContinuationCount >= maxNestedSelfDispatchDepth;
+  if (nestedDepthExceeded) {
+    lastDispatchErr = new Error(
+      `proactive nested-dispatch depth cap reached (backgroundContinuationCount=${backgroundContinuationCount} >= MAX_NESTED_SELF_DISPATCH_DEPTH=${maxNestedSelfDispatchDepth}) — deferring to the unclaimed-background-run sweep instead of risking Netlify loop protection`,
+    );
+  }
+  for (
+    let attempt = 0;
+    !nestedDepthExceeded && attempt < maxDispatchAttempts && !dispatched;
+    attempt++
+  ) {
+    try {
+      if (attempt > 0) {
+        // Uncapped budgets (durable-background and true-foreground) keep
+        // the original `500 * 2 ** attempt` schedule unchanged. The capped
+        // budget (proven-in-background-function) instead uses a schedule
+        // starting at 500ms and doubling per gap, capped at
+        // `backoffCapMs` — see `resolveContinuationDispatchBudget` for why
+        // this stays well inside the worker's remaining wall clock even
+        // with 5 attempts.
+        const backoffMs = Number.isFinite(dispatchBudget.backoffCapMs)
+          ? Math.min(500 * 2 ** (attempt - 1), dispatchBudget.backoffCapMs)
+          : 500 * 2 ** attempt;
+        await d.sleep(backoffMs);
+        // Keep the pre-inserted successor row visibly alive while we
+        // retry: the awaited attempts + backoff can outlast
+        // UNCLAIMED_BACKGROUND_RUN_GRACE_MS (25s), and without a fresh
+        // heartbeat the unclaimed-run reaper / sweep could error a handoff
+        // we are still delivering.
+        if (nextRowInserted) {
+          await d.updateRunHeartbeat(nextRunId).catch(() => {});
+        }
+      }
+      await d.fireInternalDispatch({
+        event,
+        // Durable chain: same path resolution as the initial dispatch —
+        // on hosted Netlify the background function's DEFAULT url (no
+        // custom config.path; async via background:true; never shadowed
+        // because /.netlify/* is excluded from the /* catch-all) so each
+        // chunk keeps the 15-min budget; off-Netlify the in-process
+        // framework route. Foreground self-chain: always the framework
+        // `_process-run` route on the regular function (see the fn doc).
+        path: continuationDispatchPath,
+        taskId: nextRunId,
+        body: dispatchBody,
+        awaitResponse: true,
+        responseTimeoutMs: dispatchResponseTimeoutMs,
+      });
+      dispatched = true;
+    } catch (dispatchErr) {
+      lastDispatchErr = dispatchErr;
+      // Regular-function targets (foreground self-chain) respond only
+      // after the successor chunk FINISHES, so an await timeout is not
+      // proof of a dead handoff. The successor's ATOMIC CLAIM is: a row
+      // that left `dispatch_mode='background'` (claimed) or is already
+      // terminal proves the handoff landed — stop retrying. A duplicate
+      // delivery would lose the claim and no-op anyway; skipping it saves
+      // wall-clock this close to the invocation deadline.
+      if (!chainViaDurableBackground && nextRowInserted) {
+        const claim = await d
+          .readBackgroundRunClaim(nextRunId)
+          .catch(() => null);
+        if (
+          claim &&
+          ((claim.dispatchMode && claim.dispatchMode !== "background") ||
+            (claim.status && claim.status !== "running"))
+        ) {
+          dispatched = true;
+          break;
+        }
+      }
+      console.error(
+        `[agent-chat] background continuation dispatch attempt ${attempt + 1} failed:`,
+        dispatchErr instanceof Error ? dispatchErr.message : dispatchErr,
+      );
+      // Netlify loop protection (see `isLoopProtectionDispatchError`) is a
+      // property of this same live, nested call chain — retrying the exact
+      // same nested self-dispatch within the next few seconds will not
+      // change that, so further attempts are a guaranteed-doomed use of
+      // this worker's remaining wall clock. Stop immediately (instead of
+      // burning the full `maxDispatchAttempts` budget) and fall into the
+      // same deferred-recovery path below, which hands the successor to the
+      // sweep — a genuinely different, non-nested invocation.
+      if (isLoopProtectionDispatchError(dispatchErr)) {
+        break;
+      }
+    }
+  }
+  return { dispatched, lastDispatchErr, nestedDepthExceeded };
+}

package/corpus/core/src/agent/engine/tool-call-journal-seed.ts

@@ -0,0 +1,85 @@
+import { getCurrentTurnEventsForThread } from "../run-store.js";
+import {
+  classifyToolCallJournal,
+  type ToolCallJournal,
+} from "../tool-call-journal.js";
+
+/**
+ * Minimal shape `runAgentLoop` needs from a prior tool call — kept local
+ * (rather than importing `AgentLoopToolCallSummary` from `production-agent.js`)
+ * so this module has no runtime dependency back on the file that calls it.
+ */
+export interface PriorTurnToolCallSummary {
+  name: string;
+  input: unknown;
+}
+
+/** Minimal shape `runAgentLoop` needs from a prior tool result. See
+ * `PriorTurnToolCallSummary` for why this is a local shape, not an import. */
+export interface PriorTurnToolResultSummary {
+  name: string;
+  content: string;
+  isError: boolean;
+}
+
+/**
+ * Tool-call journal hard-block (resume safety). Snapshot the per-turn journal
+ * ONCE here, before any tool runs in this chunk, so it reflects only PRIOR
+ * run chunks of this logical turn. A write tool whose exact call already
+ * completed in an earlier interrupted chunk must not re-fire its side effect;
+ * when matched, `runToolCall` returns the journaled result instead of
+ * executing.
+ *
+ * Loaded eagerly (not lazily mid-loop) so the current chunk's own
+ * asynchronously-persisted tool_done events can never leak in and make a
+ * same-chunk call wrongly short-circuit. Best-effort: any ledger failure
+ * leaves the journal empty and all calls run normally. Fresh first-turn calls
+ * see an empty journal and are unaffected.
+ *
+ * Also returns the prior chunks' tool calls/results so the caller can seed
+ * its own `toolCallHistory` / `toolResultHistory` accumulators — final
+ * response guards must see successful reads from earlier chunks, not only
+ * tools executed after the latest handoff. Otherwise a guard can reject a
+ * grounded answer (or a successfully-created artifact) after the
+ * evidence-producing query completed in a predecessor run.
+ *
+ * Moved verbatim out of `runAgentLoop`'s per-turn setup — behavior unchanged.
+ */
+export async function loadPriorTurnToolCallJournal(
+  threadId: string | undefined,
+): Promise<{
+  toolCallJournal: ToolCallJournal | null;
+  priorToolCalls: PriorTurnToolCallSummary[];
+  priorToolResults: PriorTurnToolResultSummary[];
+}> {
+  const priorToolCalls: PriorTurnToolCallSummary[] = [];
+  const priorToolResults: PriorTurnToolResultSummary[] = [];
+  let toolCallJournal: ToolCallJournal | null = null;
+  if (!threadId) {
+    return { toolCallJournal, priorToolCalls, priorToolResults };
+  }
+  try {
+    const priorEvents = await getCurrentTurnEventsForThread(threadId);
+    if (priorEvents.length > 0) {
+      for (const event of priorEvents) {
+        if (event.type === "tool_start") {
+          priorToolCalls.push({
+            name: event.tool,
+            input: event.input,
+          });
+        } else if (event.type === "tool_done") {
+          priorToolResults.push({
+            name: event.tool,
+            content: event.result,
+            isError: event.isError === true,
+          });
+        }
+      }
+      toolCallJournal = classifyToolCallJournal(priorEvents);
+    }
+  } catch {
+    // Journal is a hardening layer, never a gate — a failed ledger read just
+    // means no hard-block this turn.
+  }
+  return { toolCallJournal, priorToolCalls, priorToolResults };
+}