@kindgi/api 0.1.4 → 0.1.5-rc.0

package/src/routes/judged-suites.ts

@@ -4,7 +4,7 @@
 import { type Context, Hono } from 'hono';
 
 import { type Principal, ref, tuplesForCreate } from '@kindgi/authz';
-import type { Cursor, ProjectId, TenantId, UserId } from '@kindgi/types';
+import type { Cursor, ProjectId, ScopeSegment, TenantId, UserId } from '@kindgi/types';
 
 import { statusFor, toWireError } from '../errors.js';
 import type {
@@ -22,6 +22,8 @@ import type {
 import type { Authorizer } from '../middleware/authorize.js';
 import type { AppEnv } from '../types.js';
 import { clampLimit } from './pagination.js';
+import { projectMismatch } from './project-mismatch.js';
+import { parseSegmentsBody } from './segments.js';
 
 /** The most cases a test set built from judgments holds. */
 export const MAX_JUDGED_CASES = 1000;
@@ -66,35 +68,13 @@ export function judgedSuitesRouter(
       );
     }
 
-    const built = await buildCases(judgments, tenantId, body);
     const principal = c.get('principal') as Principal | undefined;
     const creatorUserId =
       principal?.actor.kind === 'user' ? (principal.actor.id as UserId) : undefined;
-    const outcome = await suites.publish({
-      tenantId,
-      projectId: body.projectId,
-      suite: {
-        id: suiteId,
-        tenantId,
-        version: body.version,
-        kind: 'judged',
-        ...(body.description !== undefined && { description: body.description }),
-        spec: {
-          source: 'judgments',
-          // Where the judgments came from: a comparison's summary names it.
-          projectId: body.projectId,
-          query: body.query,
-          caseCount: built.cases.length,
-          truncated: built.truncated,
-          builtAt: new Date().toISOString(),
-        },
-      },
-      enqueueTuples: (id) =>
-        tuplesForCreate(
-          { kind: 'eval_suite', id, tenantId, projectId: body.projectId },
-          creatorUserId,
-        ),
-    });
+    const outcome = await buildJudgedSuite(
+      { suites, judgments, cases },
+      { tenantId, suiteId, ...body, ...(creatorUserId !== undefined && { creatorUserId }) },
+    );
     if (outcome.kind === 'already-registered') {
       return fail(
         c,
@@ -105,14 +85,16 @@ export function judgedSuitesRouter(
     if (outcome.kind === 'project-not-found') {
       return fail(c, 'bad-input', `\`projectId\` "${body.projectId}" is not a project here.`);
     }
-    await cases.putCases({ tenantId, suiteId, version: body.version, cases: built.cases });
+    if (outcome.kind === 'project-mismatch') {
+      return projectMismatch(c, 'eval-suite', suiteId, outcome.projectId, 'build');
+    }
     c.status(201);
     return c.json({
       suiteId,
       version: body.version,
       kind: 'judged',
-      caseCount: built.cases.length,
-      truncated: built.truncated,
+      caseCount: outcome.caseCount,
+      truncated: outcome.truncated,
     });
   });
 
@@ -144,6 +126,87 @@ export function judgedSuitesRouter(
   return r;
 }
 
+/** Which judged runs a test set is built from, and which of their judgments count. */
+export interface JudgedSuiteQuery {
+  readonly agentId?: string;
+  readonly agentVersion?: string;
+  readonly flowId?: string;
+  readonly since?: string;
+  readonly until?: string;
+  readonly judgeClassIds?: readonly string[];
+  readonly minJudgments?: number;
+  /** Only runs started in this segment path or below it. */
+  readonly segments?: readonly ScopeSegment[];
+}
+
+export interface BuildJudgedSuiteInput {
+  readonly tenantId: TenantId;
+  readonly suiteId: string;
+  readonly version: string;
+  /** The project whose judged runs it takes, and the suite's project. */
+  readonly projectId: ProjectId;
+  readonly description?: string;
+  readonly query: JudgedSuiteQuery;
+  /** The user who built it, made the suite's creator. */
+  readonly creatorUserId?: UserId;
+}
+
+export type BuildJudgedSuiteOutcome =
+  | { readonly kind: 'ok'; readonly caseCount: number; readonly truncated: boolean }
+  | { readonly kind: 'already-registered' }
+  | { readonly kind: 'project-not-found' }
+  /** The suite belongs to another project: nothing is written. */
+  | { readonly kind: 'project-mismatch'; readonly projectId: ProjectId };
+
+/**
+ * Build a test set and publish it as a `judged` suite version, as
+ * `POST /v1/eval-suites/{suiteId}/versions/from-judgments` does: the
+ * judged runs the query picks (newest first, at most `MAX_JUDGED_CASES`),
+ * each a case with its items' judgments summed up. The binding must list
+ * judged runs (`listJudgedRuns`).
+ */
+export async function buildJudgedSuite(
+  deps: {
+    readonly suites: EvalSuiteRegistryBinding;
+    readonly judgments: JudgmentRegistryBinding;
+    readonly cases: EvalCaseStoreBinding;
+  },
+  input: BuildJudgedSuiteInput,
+): Promise<BuildJudgedSuiteOutcome> {
+  const { tenantId, suiteId, version, projectId } = input;
+  const built = await buildCases(deps.judgments, tenantId, input);
+  const outcome = await deps.suites.publish({
+    tenantId,
+    projectId,
+    suite: {
+      id: suiteId,
+      tenantId,
+      version,
+      kind: 'judged',
+      ...(input.description !== undefined && { description: input.description }),
+      spec: {
+        source: 'judgments',
+        // Where the judgments came from: a comparison's summary names it.
+        projectId,
+        query: input.query,
+        caseCount: built.cases.length,
+        truncated: built.truncated,
+        builtAt: new Date().toISOString(),
+      },
+    },
+    enqueueTuples: (id) =>
+      tuplesForCreate({ kind: 'eval_suite', id, tenantId, projectId }, input.creatorUserId),
+  });
+  if (outcome.kind === 'already-registered' || outcome.kind === 'project-not-found') {
+    return { kind: outcome.kind };
+  }
+  if (outcome.kind === 'project-mismatch') {
+    return { kind: 'project-mismatch', projectId: outcome.projectId };
+  }
+  await deps.cases.putCases({ tenantId, suiteId, version, cases: built.cases });
+  return { kind: 'ok', caseCount: built.cases.length, truncated: built.truncated };
+}
+
 function fail(c: Context<AppEnv>, code: string, message: string): Response {
   c.status(statusFor(code) as never);
   return c.json(toWireError({ code, message }, c.get('requestId')));
@@ -156,15 +219,7 @@ interface BuildBody {
   readonly projectId: ProjectId;
   readonly description?: string;
   /** The judged-run filter, as recorded in the suite's spec. */
-  readonly query: {
-    readonly agentId?: string;
-    readonly agentVersion?: string;
-    readonly flowId?: string;
-    readonly since?: string;
-    readonly until?: string;
-    readonly judgeClassIds?: readonly string[];
-    readonly minJudgments?: number;
-  };
+  readonly query: JudgedSuiteQuery;
 }
 
 /** Each class's weight, read once (an unclassified judgment counts 1; a missing class too). */
@@ -186,7 +241,7 @@ function classWeights(
 
 function judgedRunFilter(
   tenantId: TenantId,
-  body: BuildBody,
+  body: Pick<BuildBody, 'projectId' | 'query'>,
 ): Omit<JudgedRunListInput, 'limit' | 'cursor'> {
   const { judgeClassIds: _ids, minJudgments: _min, ...runFilter } = body.query;
   return { tenantId, projectId: body.projectId, ...runFilter };
@@ -196,7 +251,7 @@ function judgedRunFilter(
 async function buildCases(
   judgments: JudgmentRegistryBinding,
   tenantId: TenantId,
-  body: BuildBody,
+  body: Pick<BuildBody, 'projectId' | 'query'>,
 ): Promise<{ readonly cases: JudgedEvalCase[]; readonly truncated: boolean }> {
   const list = judgments.listJudgedRuns as NonNullable<JudgmentRegistryBinding['listJudgedRuns']>;
   const weightOf = classWeights(judgments, tenantId);
@@ -277,7 +332,16 @@ async function summarize(
     totalWeight,
     restricted,
     reasons: judgments.flatMap((j) =>
-      j.reason !== undefined ? [{ verdict: j.verdict, reason: j.reason }] : [],
+      j.reason !== undefined
+        ? [
+            {
+              verdict: j.verdict,
+              reason: j.reason,
+              ...(j.judgeClassId !== undefined && { judgeClassId: j.judgeClassId }),
+              ...(j.restricted === true && { restricted: true as const }),
+            },
+          ]
+        : [],
     ),
   };
 }
@@ -335,7 +399,13 @@ function parseQuery(b: Record<string, unknown>): BuildBody['query'] | string {
   if (typeof strings === 'string') return strings;
   const counted = parseJudgmentFilters(b);
   if (typeof counted === 'string') return counted;
-  return { ...strings, ...counted };
+  const segments = parseSegmentsBody(b.segments);
+  if (segments.kind === 'err') return segments.message;
+  return {
+    ...strings,
+    ...counted,
+    ...(segments.segments !== undefined && { segments: segments.segments }),
+  };
 }
 
 function parseBuildBody(raw: unknown): Parsed<BuildBody> {

package/src/routes/judgment-context.ts

@@ -8,6 +8,7 @@ import {
   readGateDecision,
 } from '@kindgi/agents';
 import type { JournalEntry, RunBinding } from '@kindgi/runtime';
+import { TOOL_ENV_RECORD_KEY, toolCallRecordKey } from '@kindgi/tools';
 import type { ConversationId, RunId, TenantId } from '@kindgi/types';
 
 import type { JudgedRunContext } from '../judgment-binding.js';
@@ -53,28 +54,97 @@ async function readHistory(
   };
 }
 
-type JournalParts = Pick<JudgedRunContext, 'retrieved' | 'sessionApproval'>;
+type JournalParts = Pick<
+  JudgedRunContext,
+  'retrieved' | 'recalled' | 'sessionApproval' | 'toolEnv'
+>;
 
-/** What the turn's journal says it retrieved, and how its session approval was decided. */
+/** A record's value when it's env values: an object whose values are strings. */
+function envValues(value: unknown): Readonly<Record<string, string>> | undefined {
+  const o = obj(value);
+  if (o === undefined) return undefined;
+  return Object.values(o).every((v) => typeof v === 'string')
+    ? (o as Record<string, string>)
+    : undefined;
+}
+
+/**
+ * The env values each tool's calls were sent, by tool id, from a run's
+ * journal, added to `into`: for each call given (an agent turn's, with its
+ * call id; a flow's tool step, without), its recorded env
+ * (`toolCallRecordKey(…, TOOL_ENV_RECORD_KEY)`). A tool's first call wins.
+ */
+export function addToolEnv(
+  entries: readonly JournalEntry[],
+  calls: readonly { readonly toolId: string; readonly callId?: string }[],
+  into: Record<string, Readonly<Record<string, string>>>,
+): void {
+  const recorded = new Map<string, unknown>();
+  for (const e of entries) {
+    const payload = obj(e.payload);
+    const key = payload?.key;
+    if (e.kind === 'value.recorded' && typeof key === 'string' && !recorded.has(key)) {
+      recorded.set(key, payload?.value);
+    }
+  }
+  for (const call of calls) {
+    if (into[call.toolId] !== undefined) continue;
+    const env = envValues(
+      recorded.get(
+        toolCallRecordKey({
+          toolId: call.toolId,
+          key: TOOL_ENV_RECORD_KEY,
+          ...(call.callId !== undefined && { callId: call.callId }),
+        }),
+      ),
+    );
+    if (env !== undefined) into[call.toolId] = env;
+  }
+}
+
+/** An agent turn's tool calls, from its output: each `tool` message's tool and call id. */
+export function turnCallIds(
+  output: unknown,
+): readonly { readonly toolId: string; readonly callId: string }[] {
+  const appended = obj(output)?.appended;
+  if (!Array.isArray(appended)) return [];
+  return appended.flatMap((m) => {
+    const toolCall = obj(obj(m)?.toolCall);
+    const toolId = toolCall?.toolId;
+    const callId = toolCall?.invocationId;
+    return obj(m)?.role === 'tool' && typeof toolId === 'string' && typeof callId === 'string'
+      ? [{ toolId, callId }]
+      : [];
+  });
+}
+
+/** What the turn's journal says it retrieved and recalled, and how its session approval was decided. */
 async function readJournalParts(
   runBinding: RunBinding,
   tenantId: TenantId,
   runId: string,
+  output: unknown,
 ): Promise<JournalParts> {
   const journal = await runBinding.readJournal(tenantId, runId as RunId);
   if (journal.kind === 'err') return {};
   const entries = journal.value;
+  const toolEnv: Record<string, Readonly<Record<string, string>>> = {};
+  addToolEnv(entries, turnCallIds(output), toolEnv);
   const step = entries.find(
     (e) =>
       e.kind === 'step.completed' &&
       typeof e.nodeId === 'string' &&
       (e.nodeId === RUN_RETRIEVALS_NODE || e.nodeId.endsWith(`/${RUN_RETRIEVALS_NODE}`)),
   );
-  const retrieved = obj(obj(step?.payload)?.output)?.retrieved;
+  const stepOutput = obj(obj(step?.payload)?.output);
+  const retrieved = stepOutput?.retrieved;
+  const recalled = stepOutput?.recalled;
   const sessionApproval = sessionApprovalOf(entries);
   return {
     ...(retrieved !== undefined && { retrieved }),
+    ...(recalled !== undefined && { recalled }),
     ...(sessionApproval !== undefined && { sessionApproval }),
+    ...(Object.keys(toolEnv).length > 0 && { toolEnv }),
   };
 }
 
@@ -102,7 +172,8 @@ function sessionApprovalOf(
 /**
  * What a judged agent turn read besides its input, so it can be replayed
  * faithfully later: the conversation before the turn, what its
- * retrievals returned, and the decision at its session approval gate. Best effort and read-only: a part that can't be
+ * retrievals returned, the decision at its session approval gate, and the
+ * env values its tools were sent. Best effort and read-only: a part that can't be
  * read is left out (the judgment never fails over it). `undefined` for
  * a flow run, or when nothing could be read.
  */
@@ -121,7 +192,7 @@ export async function captureTurnContext(input: {
           () => ({}),
         )
       : Promise.resolve({}),
-    readJournalParts(input.runBinding, input.tenantId, input.runId).catch(() => ({})),
+    readJournalParts(input.runBinding, input.tenantId, input.runId, input.output).catch(() => ({})),
   ]);
   const context: JudgedRunContext = { ...history, ...journal };
   return Object.keys(context).length > 0 ? context : undefined;

package/src/routes/judgment-flow-context.ts

@@ -22,6 +22,7 @@ import type {
   JudgedRunContext,
   JudgedToolCall,
 } from '../judgment-binding.js';
+import { addToolEnv, turnCallIds } from './judgment-context.js';
 
 /** The most tool calls a judged flow run keeps. */
 export const MAX_JUDGED_FLOW_CALLS = 500;
@@ -94,6 +95,8 @@ function scopeOf(loopContext: unknown): string | undefined {
 class Collector {
   readonly calls: JudgedToolCall[] = [];
   readonly steps: JudgedFlowStep[] = [];
+  /** The env values each tool's calls were sent, by tool id (`JudgedRunContext.toolEnv`). */
+  readonly toolEnv: Record<string, Readonly<Record<string, string>>> = {};
   truncated = false;
 
   add(call: JudgedToolCall): void {
@@ -127,6 +130,11 @@ async function nodeCalls(
   ]);
   const tools = toolNodes(flow);
   if (journal.kind === 'err' || tools.size === 0) return;
+  addToolEnv(
+    journal.value,
+    [...new Set(tools.values())].map((toolId) => ({ toolId })),
+    c.toolEnv,
+  );
   const inputs = new Map<string, unknown>();
   for (const e of journal.value) {
     if (typeof e.nodeId !== 'string' || !tools.has(e.nodeId)) continue;
@@ -147,7 +155,7 @@ async function nodeCalls(
   }
 }
 
-/** An agent step's turn: its calls, and what it retrieved. */
+/** An agent step's turn: its calls, and what it retrieved and recalled. */
 async function stepTurn(r: Reader, c: Collector, child: KernelRunRecord): Promise<void> {
   const agent = child.agent;
   if (agent === undefined) return;
@@ -156,6 +164,7 @@ async function stepTurn(r: Reader, c: Collector, child: KernelRunRecord): Promis
   const nodeId = (child.parentNodeId as unknown as string | null | undefined) ?? undefined;
   const scope = child.parentScope ?? undefined;
   const retrieved = obj(output)?.retrieved;
+  const recalled = obj(output)?.recalled;
   c.steps.push({
     runId: child.runId as unknown as string,
     ...(nodeId !== undefined && { nodeId }),
@@ -163,6 +172,7 @@ async function stepTurn(r: Reader, c: Collector, child: KernelRunRecord): Promis
     agentId: agent.id,
     agentVersion: agent.version,
     ...(retrieved !== undefined && { retrieved }),
+    ...(recalled !== undefined && { recalled }),
   });
   for (const call of turnToolCalls(output)) {
     c.add({
@@ -172,6 +182,10 @@ async function stepTurn(r: Reader, c: Collector, child: KernelRunRecord): Promis
       ...call,
     });
   }
+  const calls = turnCallIds(output);
+  if (calls.length === 0) return;
+  const journal = await r.runBinding.readJournal(r.tenantId, child.runId);
+  if (journal.kind === 'ok') addToolEnv(journal.value, calls, c.toolEnv);
 }
 
 /** A run's child runs (agent steps' turns, sub-flows); none when they can't be read. */
@@ -241,5 +255,5 @@ export async function captureFlowContext(input: {
     steps: c.steps,
     ...(c.truncated && { truncated: true }),
   };
-  return { flow };
+  return { flow, ...(Object.keys(c.toolEnv).length > 0 && { toolEnv: c.toolEnv }) };
 }

package/src/routes/judgments.ts

@@ -6,7 +6,7 @@ import { type Context, Hono } from 'hono';
 import type { ConversationBinding } from '@kindgi/agents';
 import { REVIEWER_ROLE_RANK, type ReviewerRole, ref } from '@kindgi/authz';
 import type { RunBinding } from '@kindgi/runtime';
-import type { Cursor, ProjectId, RunId, TenantId } from '@kindgi/types';
+import type { Cursor, ProjectId, RunId, ScopeSegment, TenantId } from '@kindgi/types';
 
 import { statusFor, toWireError } from '../errors.js';
 import type { FlowRegistryBinding } from '../flow-binding.js';
@@ -105,6 +105,9 @@ export function judgmentsRouter(
         subject,
         input: run.input,
         output: run.output,
+        // Which segment the run was in, kept with the copy: a test set for a
+        // segment takes only its runs.
+        segments: run.segments,
         ...(context !== undefined && { context }),
       },
       item: body.item,
@@ -395,7 +398,11 @@ export function judgeClassesRouter(
 type Prepared =
   | {
       readonly kind: 'ok';
-      readonly run: { readonly input: unknown; readonly output: unknown };
+      readonly run: {
+        readonly input: unknown;
+        readonly output: unknown;
+        readonly segments: readonly ScopeSegment[];
+      };
       readonly subject: JudgedSubject;
       readonly projectId: ProjectId;
       readonly itemValue?: unknown;
@@ -430,6 +437,13 @@ async function prepareJudgment(
   ) {
     return err('permission-denied', `Not allowed to judge run "${body.runId}".`);
   }
+  // An erasure cleared its content: there's nothing to judge (T273 M-5).
+  if (run.contentErasedAt !== undefined) {
+    return err(
+      'run-erased',
+      `Run "${body.runId}" was erased (a person's words were removed): there's nothing to judge.`,
+    );
+  }
   if (run.status !== 'completed' || run.output === undefined || run.output === null) {
     return err(
       'run-not-finished',
@@ -462,7 +476,7 @@ async function prepareJudgment(
       restricted = true;
     }
   }
-  const copy = { input: run.input, output: run.output };
+  const copy = { input: run.input, output: run.output, segments: run.segments ?? [] };
   const extra = {
     ...(run.agent !== undefined && { conversationId: run.agent.conversationId as string }),
     ...(restricted && { restricted: true as const }),

package/src/routes/mcp.ts

@@ -133,8 +133,15 @@ export function mcpRouter(
       ...(scopeParsed.scope !== undefined && { scope: scopeParsed.scope }),
       ...(scopeParsed.inherit !== undefined && { inherit: scopeParsed.inherit }),
     });
+    // Only what the caller may read (T243 A), as `GET …/:id` asks.
+    const visible =
+      authorizer === undefined
+        ? page.data
+        : await authorizer.filterByCan(c, 'read', page.data, (a) =>
+            ref('mcp_endpoint', a.endpointId as unknown as string),
+          );
     return c.json({
-      data: page.data.map(serializeEndpoint),
+      data: visible.map(serializeEndpoint),
       hasMore: page.nextCursor !== undefined,
       ...(page.nextCursor !== undefined && { nextCursor: page.nextCursor as unknown as string }),
     });
@@ -624,6 +631,7 @@ function serializeEndpoint(e: MCPEndpoint): Record<string, unknown> {
     ...(e.secretRef !== undefined && { secretRef: e.secretRef }),
     ...(e.instructions !== undefined && { instructions: e.instructions }),
     ...(e.metadata !== undefined && { metadata: e.metadata }),
+    ...(e.sendTraceparent !== undefined && { sendTraceparent: e.sendTraceparent }),
   };
 }
 
@@ -636,6 +644,7 @@ const REGISTER_BODY_FIELDS = new Set([
   'secretRef',
   'instructions',
   'metadata',
+  'sendTraceparent',
   'scopeKind',
   'scopeId',
 ]);
@@ -736,6 +745,24 @@ function validateMCPEndpoint(
       },
     };
   }
+  if (b.sendTraceparent !== undefined && typeof b.sendTraceparent !== 'boolean') {
+    return {
+      kind: 'err',
+      error: {
+        message: `endpoint "${b.endpointId}" sendTraceparent must be true or false`,
+        reason: 'invalid-send-traceparent',
+      },
+    };
+  }
+  if (b.sendTraceparent === true && b.transport === 'stdio') {
+    return {
+      kind: 'err',
+      error: {
+        message: `endpoint "${b.endpointId}" sendTraceparent needs an HTTP transport: a stdio server gets no headers`,
+        reason: 'invalid-send-traceparent',
+      },
+    };
+  }
 
   const value: MCPEndpoint = {
     endpointId: b.endpointId,
@@ -747,6 +774,7 @@ function validateMCPEndpoint(
     ...(b.metadata !== undefined && {
       metadata: b.metadata as Readonly<Record<string, unknown>>,
     }),
+    ...(b.sendTraceparent !== undefined && { sendTraceparent: b.sendTraceparent }),
   };
   return { kind: 'ok', value };
 }