@tangle-network/agent-runtime 0.11.0 → 0.12.0

package/dist/agent.d.ts

@@ -1,5 +1,6 @@
 import * as _tangle_network_agent_eval from '@tangle-network/agent-eval';
-import { FindingSubject, TraceAnalystKindSpec, AnalystFinding } from '@tangle-network/agent-eval';
+import { FindingSubject, TraceAnalystKindSpec, AnalystFinding, TraceStore, RunCompleteHook, FeedbackLabel, FeedbackTrajectoryStore } from '@tangle-network/agent-eval';
+import { R as RuntimeStreamEvent } from './types-afLuHk1G.js';
 import { I as ImprovementAdapter, K as KnowledgeAdapter, a as RunAnalystLoopResult } from './types-D_MXrmJP.js';
 
 /**
@@ -230,16 +231,61 @@ interface JudgeConfig<TRunOutput> {
 }
 interface AgentRuntime<TPersona, TRunOutput> {
     /**
-     * Invoke the agent against one persona. Returns the structured run
-     * output the rubric will score.
+     * Invoke the agent against one persona. Returns BOTH:
+     *   - `events`: an `AsyncIterable<RuntimeStreamEvent>` the chat-centric
+     *     product consumes verbatim (SSE / WebSocket / inline render).
+     *     **Streaming is mandatory — never collapse this to a single Promise.**
+     *     The agent's existing `runChatTurn` (or equivalent async generator)
+     *     plugs in here directly.
+     *   - `output`: a `Promise<TRunOutput>` resolved AFTER the event stream
+     *     drains. The eval substrate awaits this for rubric scoring; chat
+     *     products usually ignore it (they already rendered incrementally).
      *
-     * `ctx.emitter` is the substrate-threaded `TraceEmitter` — agents
-     * SHOULD record their LLM calls / tool calls through it for capture
-     * integrity. `ctx.deadlineMs` is wall-clock; the runtime SHOULD
-     * honour it for graceful cancel.
+     * Implementation contract:
+     *   1. `act` MUST return immediately (synchronous construction of the
+     *      `events` iterator + the `output` promise).
+     *   2. Iterating `events` drives the underlying LLM/tool calls — the
+     *      caller chooses when to consume.
+     *   3. `output` resolves only after the iterator yields its terminal
+     *      event (typically `task_end`); see `collectAgentRun` helper.
+     *
+     * `ctx.emitter` is the substrate-threaded `TraceEmitter` — runtimes
+     * SHOULD record LLM/tool spans through it for capture integrity.
+     * `ctx.deadlineMs` is wall-clock; the runtime SHOULD honour for graceful
+     * cancel. `ctx.signal` is the standard abort signal.
      */
-    act: (persona: TPersona, ctx: AgentRunContext) => Promise<TRunOutput>;
+    act: (persona: TPersona, ctx: AgentRunContext) => AgentRunInvocation<TRunOutput>;
+}
+interface AgentRunInvocation<TRunOutput> {
+    /** Live stream of typed runtime events. Consumed by chat UX directly. */
+    events: AsyncIterable<RuntimeStreamEvent>;
+    /** Final structured output the rubric scores. Resolves after `events` drains. */
+    output: Promise<TRunOutput>;
 }
+/**
+ * Stub for agents whose `runtime.act` is not yet wired to the substrate's
+ * eval path. Preserves the streaming contract (empty event stream + a
+ * rejected `output` promise that tells the caller exactly what to fix).
+ *
+ * Per-vertical manifests usually start with this stub and replace it with
+ * the agent's real streaming runtime (`runChatTurn` or equivalent) once
+ * the eval path consumes the manifest end-to-end.
+ */
+declare function unimplementedAgentRun<TRunOutput = unknown>(reason?: string): AgentRunInvocation<TRunOutput>;
+/**
+ * Drain `act`'s `events` into an array AND await its `output`. Useful for
+ * eval / outcome-measurement code paths that don't care about live
+ * rendering. The events array is preserved so the substrate can inspect
+ * tool calls / readiness / questions retrospectively.
+ *
+ * IMPORTANT: chat-centric UX MUST NOT call this — it defeats streaming
+ * (no incremental render). Use `for await (const ev of invocation.events)`
+ * directly in the chat surface.
+ */
+declare function collectAgentRun<TRunOutput>(invocation: AgentRunInvocation<TRunOutput>): Promise<{
+    events: ReadonlyArray<RuntimeStreamEvent>;
+    output: TRunOutput;
+}>;
 interface AgentRunContext {
     /** Substrate-managed trace emitter. */
     emitter: _tangle_network_agent_eval.TraceEmitter;
@@ -534,4 +580,166 @@ declare function measureOutcome<TProposal, TEdit>(result: RunAnalystLoopResult<T
     outcome: OutcomeMeasurement;
 }>;
 
-export { type AgentManifest, AgentManifestError, type AgentRubric, type AgentRunContext, type AgentRuntime, type AgentSurfaces, type AnalystConfig, type AutoApplyPolicy, type CreateSurfaceImprovementAdapterOpts, type CreateSurfaceKnowledgeAdapterOpts, type DraftPatchInput, type DraftPatchOutput, type JudgeConfig, type KnowledgeAdapterDeps, type OutcomeMeasurement, type OutcomeMeasurementOpts, type ResolvedSurface, type RubricDimension, type SurfaceImprovementEdit, type SurfaceValidationIssue, createSurfaceImprovementAdapter, createSurfaceKnowledgeAdapter, defineAgent, measureOutcome, renderSurfaceIssues, resolveSubjectPath, validateSurfaces };
+/**
+ * `createProductionTraceSink` — the production-side capture primitive
+ * every vertical agent's chat handler wires in once.
+ *
+ * Closes the data-leak: until now, every production chat session emitted
+ * zero replayable trace data. Eval runs captured everything; production
+ * captured nothing. RL training corpora, research analyses, and the
+ * self-improvement loop all ran on synthetic personas. This primitive
+ * makes every real user conversation a piece of data the downstream
+ * channels (Prime Intellect, GEPA, research, canaries, analyst loop)
+ * can consume.
+ *
+ * Wiring (per agent, ~10 lines in the production chat handler):
+ *
+ *   ```ts
+ *   const sink = createProductionTraceSink({
+ *     projectId: 'tax-agent',
+ *     otlp: { endpoint: env.LANGFUSE_OTEL_ENDPOINT, authHeader: env.LANGFUSE_OTEL_AUTH },
+ *     runRecordStore: drizzleRunRecordStore(db),
+ *     feedbackStore: drizzleFeedbackStore(db),
+ *   })
+ *
+ *   const emitter = new TraceEmitter(sink.traceStore, {
+ *     onRunComplete: [sink.onRunComplete],
+ *   })
+ *   await emitter.startRun({
+ *     scenarioId: sessionId,
+ *     projectId: 'tax-agent',
+ *     layer: 'app-runtime',
+ *   })
+ *   // ... existing chat flow, with LLM/tool spans emitted ...
+ *   await emitter.endRun({ pass, score })
+ *   // sink.onRunComplete fires automatically:
+ *   //   1. composes RunRecord, persists to runRecordStore
+ *   //   2. exports run as OTLP, POSTs to Langfuse
+ *   //   3. logs failures (does NOT throw — never crashes the chat handler)
+ *   ```
+ *
+ * Separately, the agent's feedback endpoint calls `sink.recordFeedback`
+ * to write user thumbs-up/thumbs-down (or richer labels) into the
+ * FeedbackTrajectory store — the corpus DPO/KTO trainers consume.
+ *
+ * Cloudflare Worker semantics: the sink buffers spans in memory through
+ * the request lifetime (via agent-eval's `InMemoryTraceStore`).
+ * `onRunComplete` is awaited (typically inside `ctx.waitUntil`) so the
+ * worker stays alive long enough to flush. The OTLP POST is fire-and-
+ * forget — failures are logged but never surface to the chat user.
+ */
+
+interface ProductionTraceSinkOpts {
+    /**
+     * Stable agent identifier — appears in OTLP `service.name`, every
+     * RunRecord row, every FeedbackTrajectory row. MUST match the
+     * agent's repo name to keep cross-repo telemetry joinable.
+     */
+    projectId: string;
+    /**
+     * OTLP forwarding target. Typically Langfuse's HTTP receiver. Omit to
+     * disable OTLP export (RunRecord persistence still works).
+     *
+     * `authHeader` is the full header value (e.g. `Basic <base64>`); the
+     * sink does NOT base64-encode for you.
+     */
+    otlp?: {
+        endpoint: string;
+        authHeader?: string;
+        /** Optional resource attributes merged into every span batch. */
+        resourceAttributes?: Record<string, string | number | boolean>;
+    };
+    /**
+     * Durable RunRecord persistence. Per-vertical agents implement this
+     * over their own DB (Drizzle / D1 / Postgres). Optional — when omitted,
+     * RunRecords stay in-memory and are lost when the worker isolate ends.
+     */
+    runRecordStore?: ProductionRunRecordStore;
+    /**
+     * Durable feedback persistence. Used by `recordFeedback`; agents wire
+     * their thumbs-up/down + free-text feedback endpoints to call into the
+     * sink, which writes a `FeedbackLabel` into a `FeedbackTrajectory`.
+     *
+     * Optional — when omitted, `recordFeedback` is a no-op.
+     */
+    feedbackStore?: FeedbackTrajectoryStore;
+    /**
+     * Pluggable fetch — defaults to globalThis.fetch. Tests inject a
+     * mocked fetch.
+     */
+    fetch?: typeof fetch;
+    /**
+     * Pluggable structured logger — defaults to console.warn for failures.
+     * The sink NEVER throws on flush failure; it logs and returns.
+     */
+    log?: (msg: string, fields?: Record<string, unknown>) => void;
+}
+/**
+ * Durable per-agent RunRecord persistence. Each vertical implements over
+ * its own DB. The sink calls `append` once per `endRun`.
+ */
+interface ProductionRunRecordStore {
+    append(record: ProductionRunRecord): Promise<void>;
+}
+/**
+ * Minimal canonical row the sink composes on `endRun`. Per-agent DB
+ * adapters extend with their own fields; the sink only writes what
+ * the runtime canonically captures.
+ */
+interface ProductionRunRecord {
+    runId: string;
+    projectId: string;
+    scenarioId: string;
+    variantId?: string;
+    startedAt: string;
+    endedAt: string;
+    status: 'completed' | 'failed' | 'aborted';
+    pass?: boolean;
+    score?: number;
+    failureClass?: string;
+    notes?: string;
+    /** Echoed back from `emitter.startRun({ tags })`. */
+    tags?: Record<string, string>;
+    /** Span row count — useful for diagnostics. */
+    spanCount: number;
+}
+interface ProductionTraceSink {
+    /**
+     * The TraceStore the agent's `TraceEmitter` writes to. In-memory by
+     * design: spans accumulate through the chat session, flush at
+     * `onRunComplete`. The runtime never reads from this store directly —
+     * the sink reads from it during the flush step.
+     */
+    traceStore: TraceStore;
+    /**
+     * Hook the agent passes into
+     * `new TraceEmitter(store, { onRunComplete: [sink.onRunComplete] })`.
+     * Fires once per chat session at `endRun` time. Composes the
+     * RunRecord, persists, and ships OTLP. Errors are logged, never thrown.
+     */
+    onRunComplete: RunCompleteHook;
+    /**
+     * Append a user feedback label (thumbs-up/down, correction, severity)
+     * to the FeedbackTrajectory for a completed run. Creates a minimal
+     * trajectory anchored to the run if one doesn't exist; appends the
+     * label if it does. No-op when `feedbackStore` is undefined.
+     *
+     * Returns the trajectory id (existing or freshly created) for the
+     * agent's API to link back to the session, or `null` on no-op /
+     * error.
+     */
+    recordFeedback(input: RecordFeedbackInput): Promise<string | null>;
+}
+interface RecordFeedbackInput {
+    /** Run id from the original `emitter.startRun`. */
+    runId: string;
+    /** The user-supplied feedback label. */
+    label: FeedbackLabel;
+    /** Optional scenario id (mirrors the run's). */
+    scenarioId?: string;
+    /** Optional pre-existing trajectory id if the agent tracks them separately. */
+    trajectoryId?: string;
+}
+declare function createProductionTraceSink(opts: ProductionTraceSinkOpts): ProductionTraceSink;
+
+export { type AgentManifest, AgentManifestError, type AgentRubric, type AgentRunContext, type AgentRunInvocation, type AgentRuntime, type AgentSurfaces, type AnalystConfig, type AutoApplyPolicy, type CreateSurfaceImprovementAdapterOpts, type CreateSurfaceKnowledgeAdapterOpts, type DraftPatchInput, type DraftPatchOutput, type JudgeConfig, type KnowledgeAdapterDeps, type OutcomeMeasurement, type OutcomeMeasurementOpts, type ProductionRunRecord, type ProductionRunRecordStore, type ProductionTraceSink, type ProductionTraceSinkOpts, type RecordFeedbackInput, type ResolvedSurface, type RubricDimension, type SurfaceImprovementEdit, type SurfaceValidationIssue, collectAgentRun, createProductionTraceSink, createSurfaceImprovementAdapter, createSurfaceKnowledgeAdapter, defineAgent, measureOutcome, renderSurfaceIssues, resolveSubjectPath, unimplementedAgentRun, validateSurfaces };

package/dist/agent.js

@@ -6,49 +6,67 @@ import {
 import { existsSync } from "fs";
 import { isAbsolute, join } from "path";
 function resolveSubjectPath(subject, surfaces, repoRoot) {
-  const rel = relativePathForSubject(subject, surfaces);
-  if (rel === null) return null;
-  const abs = isAbsolute(rel) ? rel : join(repoRoot, rel);
-  const exists = existsSync(abs);
+  const candidates = candidatePathsForSubject(subject, surfaces);
+  if (candidates.length === 0) return null;
+  for (const rel of candidates) {
+    const abs = isAbsolute(rel) ? rel : join(repoRoot, rel);
+    if (existsSync(abs)) {
+      return { absolutePath: abs, repoRelativePath: rel, exists: true, intent: "edit-existing" };
+    }
+  }
+  const fallback = candidates[0];
+  const fallbackAbs = isAbsolute(fallback) ? fallback : join(repoRoot, fallback);
   return {
-    absolutePath: abs,
-    repoRelativePath: rel,
-    exists,
-    intent: exists ? "edit-existing" : "create-new"
+    absolutePath: fallbackAbs,
+    repoRelativePath: fallback,
+    exists: false,
+    intent: "create-new"
   };
 }
-function relativePathForSubject(subject, surfaces) {
+function candidatePathsForSubject(subject, surfaces) {
   switch (subject.kind) {
     case "knowledge.wiki":
     case "knowledge.stale":
-      return join(surfaces.knowledge, `${subject.slug}.md`);
+      return [join(surfaces.knowledge, `${subject.slug}.md`)];
     case "knowledge.claim":
-      return join(surfaces.knowledge, "claims", `${slugify(subject.topic)}.md`);
+      return [join(surfaces.knowledge, "claims", `${slugify(subject.topic)}.md`)];
     case "knowledge.raw":
-      return join(surfaces.knowledge, "raw", `${subject.sourceId}.md`);
-    case "system-prompt":
-      return join(surfaces.systemPrompt, `${slugify(subject.section)}.md`);
+      return [join(surfaces.knowledge, "raw", `${subject.sourceId}.md`)];
+    case "system-prompt": {
+      const slug = slugify(subject.section);
+      return [
+        join(surfaces.systemPrompt, `${slug}.md`),
+        join(surfaces.systemPrompt, slug, "SKILL.md"),
+        join(surfaces.systemPrompt, slug, "index.md")
+      ];
+    }
     case "tool-doc":
-      return subject.aspect ? join(surfaces.tools, subject.tool, `${slugify(subject.aspect)}.md`) : join(surfaces.tools, subject.tool, "README.md");
+      if (subject.aspect) {
+        return [join(surfaces.tools, subject.tool, `${slugify(subject.aspect)}.md`)];
+      }
+      return [
+        join(surfaces.tools, subject.tool, "README.md"),
+        join(surfaces.tools, `${subject.tool}.md`)
+      ];
     case "new-tool":
-      return join(surfaces.tools, subject.name, "README.md");
+      return [join(surfaces.tools, subject.name, "README.md")];
     case "rag":
-      if (!surfaces.rag) return null;
-      return join(surfaces.rag, subject.corpus, `${subject.docId}.md`);
+      if (!surfaces.rag) return [];
+      return [join(surfaces.rag, subject.corpus, `${subject.docId}.md`)];
     case "memory":
-      if (!surfaces.memory) return null;
-      return join(surfaces.memory, `${slugify(subject.key)}.json`);
+      if (!surfaces.memory) return [];
+      return [join(surfaces.memory, `${slugify(subject.key)}.json`)];
     case "scaffolding":
-      if (!surfaces.scaffolding) return null;
-      return join(surfaces.scaffolding, `${slugify(subject.concern)}.md`);
+      if (!surfaces.scaffolding) return [];
+      return [join(surfaces.scaffolding, `${slugify(subject.concern)}.md`)];
     case "output-schema":
-      if (!surfaces.outputSchema) return null;
-      return surfaces.outputSchema;
+      if (!surfaces.outputSchema) return [];
+      return [surfaces.outputSchema];
     case "websearch.outdated":
     case "prior-run-summary":
-      return null;
+      return [];
     case "cluster":
-      return null;
+      return [];
   }
 }
 function slugify(s) {
@@ -63,11 +81,7 @@ function validateSurfaces(surfaces, repoRoot) {
     "knowledge"
   ];
   const fileSurfaces = ["rubric"];
-  const optionalDirSurfaces = [
-    "scaffolding",
-    "memory",
-    "rag"
-  ];
+  const optionalDirSurfaces = ["scaffolding", "memory", "rag"];
   const optionalFileSurfaces = ["outputSchema"];
   for (const key of dirSurfaces) {
     const p = surfaces[key];
@@ -119,6 +133,19 @@ function renderSurfaceIssues(issues, repoRoot) {
 }
 
 // src/agent/define-agent.ts
+function unimplementedAgentRun(reason = "AgentRuntime.act is not yet wired for this manifest") {
+  return {
+    events: (async function* empty() {
+    })(),
+    output: Promise.reject(new Error(reason))
+  };
+}
+async function collectAgentRun(invocation) {
+  const events = [];
+  for await (const ev of invocation.events) events.push(ev);
+  const output = await invocation.output;
+  return { events, output };
+}
 var AgentManifestError = class extends Error {
   constructor(message, agentId, issues = []) {
     super(message);
@@ -155,8 +182,8 @@ function defineAgent(manifest) {
 }
 
 // src/agent/improvement-adapter.ts
-import { readFileSync } from "fs";
 import { spawnSync } from "child_process";
+import { readFileSync } from "fs";
 import {
   parseFindingSubject
 } from "@tangle-network/agent-eval";
@@ -254,7 +281,9 @@ function createSurfaceImprovementAdapter(opts) {
         return { applied, warnings };
       }
       if (mode === "open-pr" && !opts.ghRepo) {
-        warnings.push("createSurfaceImprovementAdapter: mode=open-pr requires `ghRepo`; falling back to no-op");
+        warnings.push(
+          "createSurfaceImprovementAdapter: mode=open-pr requires `ghRepo`; falling back to no-op"
+        );
         return { applied, warnings };
       }
       for (const edit of edits) {
@@ -306,7 +335,9 @@ function openPullRequest(paths, edits, repoRoot, ghRepo, baseBranch) {
     `Automated analyst-loop edits \u2014 review carefully before merge.`,
     "",
     `Source findings:`,
-    ...edits.map((e) => `  - ${e.sourceFindingId} (confidence ${e.confidence.toFixed(2)}, severity ${e.severity})`),
+    ...edits.map(
+      (e) => `  - ${e.sourceFindingId} (confidence ${e.confidence.toFixed(2)}, severity ${e.severity})`
+    ),
     "",
     "Rationales:",
     ...edits.map((e) => `
@@ -462,14 +493,124 @@ function meanComposite(rows) {
   if (rows.length === 0) return 0;
   return rows.reduce((acc, r) => acc + r.composite, 0) / rows.length;
 }
+
+// src/agent/production-trace-sink.ts
+import {
+  exportRunAsOtlp,
+  InMemoryTraceStore
+} from "@tangle-network/agent-eval";
+function createProductionTraceSink(opts) {
+  const log = opts.log ?? defaultLog;
+  const fetchImpl = opts.fetch ?? globalThis.fetch;
+  const traceStore = new InMemoryTraceStore();
+  const onRunComplete = async (ctx) => {
+    if (opts.runRecordStore) {
+      try {
+        const record = await composeRunRecord(traceStore, ctx, opts.projectId);
+        await opts.runRecordStore.append(record);
+      } catch (err) {
+        log("runRecordStore.append failed", {
+          runId: ctx.runId,
+          error: err instanceof Error ? err.message : String(err)
+        });
+      }
+    }
+    if (opts.otlp) {
+      try {
+        const resourceAttrs = {
+          "service.name": opts.projectId,
+          ...opts.otlp.resourceAttributes ?? {}
+        };
+        const otlpPayload = await exportRunAsOtlp(traceStore, ctx.runId, resourceAttrs);
+        const headers = { "content-type": "application/json" };
+        if (opts.otlp.authHeader) headers.authorization = opts.otlp.authHeader;
+        const res = await fetchImpl(opts.otlp.endpoint, {
+          method: "POST",
+          headers,
+          body: JSON.stringify(otlpPayload)
+        });
+        if (!res.ok) {
+          log("OTLP POST non-2xx", {
+            runId: ctx.runId,
+            status: res.status,
+            endpoint: opts.otlp.endpoint
+          });
+        }
+      } catch (err) {
+        log("OTLP POST threw", {
+          runId: ctx.runId,
+          error: err instanceof Error ? err.message : String(err),
+          endpoint: opts.otlp.endpoint
+        });
+      }
+    }
+  };
+  const recordFeedback = async (input) => {
+    if (!opts.feedbackStore) return null;
+    const trajectoryId = input.trajectoryId ?? `traj-${input.runId}`;
+    try {
+      const existing = await opts.feedbackStore.get(trajectoryId);
+      if (existing) {
+        await opts.feedbackStore.appendLabel(trajectoryId, input.label);
+        return trajectoryId;
+      }
+      await opts.feedbackStore.save({
+        id: trajectoryId,
+        projectId: opts.projectId,
+        scenarioId: input.scenarioId ?? input.runId,
+        task: { intent: "chat", context: { runId: input.runId } },
+        attempts: [],
+        labels: [input.label],
+        createdAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      return trajectoryId;
+    } catch (err) {
+      log("feedbackStore write failed", {
+        runId: input.runId,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      return null;
+    }
+  };
+  return { traceStore, onRunComplete, recordFeedback };
+}
+async function composeRunRecord(store, ctx, projectId) {
+  const run = await store.getRun(ctx.runId);
+  const spans = await store.spans({ runId: ctx.runId });
+  const now = Date.now();
+  const startedAtMs = run?.startedAt ?? now;
+  const endedAtMs = run?.endedAt ?? now;
+  return {
+    runId: ctx.runId,
+    projectId,
+    scenarioId: run?.scenarioId ?? ctx.runId,
+    variantId: run?.variantId,
+    startedAt: new Date(startedAtMs).toISOString(),
+    endedAt: new Date(endedAtMs).toISOString(),
+    status: ctx.status,
+    pass: ctx.outcome?.pass,
+    score: ctx.outcome?.score,
+    failureClass: ctx.outcome?.failureClass,
+    notes: ctx.outcome?.notes,
+    tags: run?.tags,
+    spanCount: spans.length
+  };
+}
+function defaultLog(msg, fields) {
+  if (fields) console.warn(`[production-trace-sink] ${msg}`, fields);
+  else console.warn(`[production-trace-sink] ${msg}`);
+}
 export {
   AgentManifestError,
+  collectAgentRun,
+  createProductionTraceSink,
   createSurfaceImprovementAdapter,
   createSurfaceKnowledgeAdapter,
   defineAgent,
   measureOutcome,
   renderSurfaceIssues,
   resolveSubjectPath,
+  unimplementedAgentRun,
   validateSurfaces
 };
 //# sourceMappingURL=agent.js.map