@agent-native/core 0.173.1 → 0.173.2-nightly-20260825161643

package/corpus/templates/content/parity/README.md

@@ -10,7 +10,7 @@ PR 2.2 adds two executable tiers:
   existing action tests, make no model calls, and require no private provider
   credentials.
 - Gated agent evals run through `agent-native eval parity`. They are opt-in via
-  `CONTENT_PARITY_EVALS=1`, capped to four initial scenarios, and should be
+  `CONTENT_PARITY_EVALS=1`, kept to a small explicit scenario set, and should be
   reserved for manual or nightly checks.
 
 ## Deterministic Checks
@@ -33,14 +33,23 @@ cd templates/content
 CONTENT_PARITY_EVALS=1 ANTHROPIC_API_KEY=... ./node_modules/.bin/agent-native eval parity
 ```
 
+The database-create property-preservation regression has a dedicated
+fixture-only runner so it can inspect model-produced arguments without loading
+or executing unrelated Content actions:
+
+```bash
+CONTENT_PARITY_EVALS=1 pnpm eval:property-preservation
+```
+
 With `CONTENT_PARITY_EVALS` unset, parity evals return skipped rows and do not
 call the agent runner. The CLI still exits `0`, but both readable and JSON
 reports mark each row with `status: "skipped"` and a `skipReason` such as
 `Skipped because CONTENT_PARITY_EVALS is unset`.
 
-With the gate set, the eval files run the four PR 2.2 scenarios:
+With the gate set, the eval files include these scenarios:
 
 - `database-source-scope`
+- `database-create-property-preservation`
 - `document-search-edit`
 - `local-file-source-truth`
 - `builder-source-review-readonly`

package/corpus/templates/content/parity/eval-scenarios.ts

@@ -8,9 +8,61 @@ export interface ParityEvalScenario {
   prompt: string;
   successSignals: string[];
   expectedTools?: string[];
+  expectedPropertyValues?: Record<string, unknown>;
+  expectedPropertyTypes?: Record<string, string>;
+  expectedCreateEnvelope?: {
+    target: {
+      spaceId: string;
+      databaseId: string;
+      databaseDocumentId: string;
+    };
+    expectedSchemaRevision: string;
+    idempotencyKey: string;
+    title: string;
+  };
 }
 
 export const parityEvalScenarios: ParityEvalScenario[] = [
+  {
+    id: "database-create-property-preservation",
+    title: "Database create property preservation",
+    capabilityIds: ["database.rows"],
+    gateEnv: "CONTENT_PARITY_EVALS",
+    defaultState: "skipped",
+    requiresPrivateCredentials: false,
+    prompt:
+      "Find the fixture Content database titled PR #3314 feedback — do not treat 3314 as its database ID. Discover its exact database target and mutation contract through Content actions, then create exactly one row titled [FIXTURE] preserve explicit properties with idempotency key fixture-create-property-preservation-v1. Set Status to status-cannot-verify and Evidence to Baseline fixture preserve-me using the discovered immutable property IDs and property types. Omit authority identity because the authenticated server owns it. No Blocks value was requested. Call add-database-item once, then report its result truthfully.",
+    successSignals: [
+      "Discovers the exact database and mutation contract before creating.",
+      "Uses add-database-item once for the discovered stable target without authoring authority identity.",
+      "Preserves both explicitly requested writable property values.",
+      "Does not invent a Blocks value or another property.",
+      "Reports an action failure rather than claiming a row was created if the fixture is unavailable.",
+    ],
+    expectedTools: [
+      "list-content-databases",
+      "get-content-database",
+      "add-database-item",
+    ],
+    expectedPropertyValues: {
+      fixture_status_property: "status-cannot-verify",
+      fixture_evidence_property: "Baseline fixture preserve-me",
+    },
+    expectedPropertyTypes: {
+      fixture_status_property: "status",
+      fixture_evidence_property: "text",
+    },
+    expectedCreateEnvelope: {
+      target: {
+        spaceId: "fixture_personal_space",
+        databaseId: "fixture_feedback_database",
+        databaseDocumentId: "fixture_feedback_document",
+      },
+      expectedSchemaRevision: "fixture_schema_revision",
+      idempotencyKey: "fixture-create-property-preservation-v1",
+      title: "[FIXTURE] preserve explicit properties",
+    },
+  },
   {
     id: "database-bulk-row-reliability",
     title: "Bulk database row reliability",

package/corpus/templates/content/parity/run-database-create-property-preservation.ts

@@ -0,0 +1,140 @@
+import { createAgentRunner, runEvals } from "@agent-native/core/eval";
+
+import addDatabaseItem from "../actions/add-database-item.ts";
+import getContentDatabase from "../actions/get-content-database.ts";
+import listContentDatabases from "../actions/list-content-databases.ts";
+import { parityEvalScenarios } from "./eval-scenarios.ts";
+import { scenarioToEval } from "./scenario-to-eval.ts";
+
+const scenario = parityEvalScenarios.find(
+  (candidate) => candidate.id === "database-create-property-preservation",
+);
+if (!scenario) {
+  throw new Error("Missing database create property preservation scenario.");
+}
+
+const FIXTURE_SPACE_ID = "fixture_personal_space";
+const FIXTURE_DATABASE_ID = "fixture_feedback_database";
+const FIXTURE_DOCUMENT_ID = "fixture_feedback_document";
+const FIXTURE_SCHEMA_REVISION = "fixture_schema_revision";
+const FIXTURE_DATABASE_TITLE = "PR #3314 feedback";
+
+const evalCase = scenarioToEval(scenario);
+evalCase.scorers = evalCase.scorers.filter(
+  (scorer) =>
+    scorer.name === "expected_tools" ||
+    scorer.name === "expected_property_values",
+);
+
+const runner = await createAgentRunner({
+  actions: {
+    "list-content-databases": {
+      ...listContentDatabases,
+      run: async () => ({
+        databases: [
+          {
+            databaseId: FIXTURE_DATABASE_ID,
+            documentId: FIXTURE_DOCUMENT_ID,
+            spaceId: FIXTURE_SPACE_ID,
+            title: FIXTURE_DATABASE_TITLE,
+            description: "",
+          },
+        ],
+        pagination: {
+          offset: 0,
+          limit: 50,
+          totalItems: 1,
+          returnedItems: 1,
+          hasMore: false,
+          nextOffset: null,
+        },
+      }),
+    },
+    "get-content-database": {
+      ...getContentDatabase,
+      run: async () => ({
+        database: {
+          id: FIXTURE_DATABASE_ID,
+          documentId: FIXTURE_DOCUMENT_ID,
+          spaceId: FIXTURE_SPACE_ID,
+          title: FIXTURE_DATABASE_TITLE,
+          naturalKeyPropertyId: null,
+          viewConfig: {
+            activeViewId: "fixture_view",
+            views: [],
+            sorts: [],
+            filters: [],
+            columnWidths: {},
+          },
+          createdAt: new Date(0).toISOString(),
+          updatedAt: new Date(0).toISOString(),
+        },
+        properties: [],
+        items: [],
+        source: null,
+        mutationContract: {
+          target: {
+            authorityScope: {
+              kind: "personal",
+              id: "fixture-owner@example.com",
+            },
+            spaceId: FIXTURE_SPACE_ID,
+            databaseId: FIXTURE_DATABASE_ID,
+            databaseDocumentId: FIXTURE_DOCUMENT_ID,
+          },
+          schemaRevision: FIXTURE_SCHEMA_REVISION,
+          naturalKeyPropertyId: null,
+          properties: [
+            {
+              id: "fixture_status_property",
+              name: "Status",
+              type: "status",
+              writable: true,
+              sourceManaged: false,
+              acceptedShape: null,
+              options: {
+                options: [
+                  {
+                    id: "status-cannot-verify",
+                    name: "Cannot verify",
+                    color: "gray",
+                  },
+                ],
+              },
+            },
+            {
+              id: "fixture_evidence_property",
+              name: "Evidence",
+              type: "text",
+              writable: true,
+              sourceManaged: false,
+              acceptedShape: null,
+              options: {},
+            },
+          ],
+        },
+      }),
+    },
+    "add-database-item": {
+      ...addDatabaseItem,
+      run: async (input) => ({ fixtureOnly: true, received: input }),
+    },
+  },
+  systemPrompt:
+    "You are Content's AI document assistant. Use the registered Content actions and preserve exact user-supplied target constraints, property IDs, and property values. Never invent fields or claim an action succeeded when it failed.",
+});
+
+const report = await runEvals([evalCase], runner, { persist: false });
+console.log(
+  JSON.stringify(
+    {
+      engine: runner.engine.name,
+      model: runner.model,
+      report,
+    },
+    null,
+    2,
+  ),
+);
+
+process.exitCode = report.failed === 0 ? 0 : 1;

package/corpus/templates/content/parity/scenario-to-eval.ts

@@ -30,6 +30,387 @@ function expectedToolScorer(expectedTools: string[]) {
   });
 }
 
+function hasExactKeys(
+  record: Record<string, unknown>,
+  expectedKeys: readonly string[],
+): boolean {
+  const actualKeys = Object.keys(record).sort();
+  const sortedExpectedKeys = [...expectedKeys].sort();
+  return (
+    actualKeys.length === sortedExpectedKeys.length &&
+    actualKeys.every((key, index) => key === sortedExpectedKeys[index])
+  );
+}
+
+function analyzePropertyValues(input: unknown): {
+  received: Record<string, unknown>;
+  receivedTypes: Record<string, string>;
+  invalid: string[];
+} {
+  if (!input || typeof input !== "object" || Array.isArray(input)) {
+    return {
+      received: {},
+      receivedTypes: {},
+      invalid: ["tool input is not an object"],
+    };
+  }
+  const record = input as Record<string, unknown>;
+  const hasEntries = record.propertyEntries !== undefined;
+  const hasValues = record.propertyValues !== undefined;
+  if (hasEntries && hasValues) {
+    return {
+      received: {},
+      receivedTypes: {},
+      invalid: ["propertyEntries and propertyValues were both provided"],
+    };
+  }
+  if (hasValues) {
+    if (
+      !record.propertyValues ||
+      typeof record.propertyValues !== "object" ||
+      Array.isArray(record.propertyValues)
+    ) {
+      return {
+        received: {},
+        receivedTypes: {},
+        invalid: ["propertyValues is not a record"],
+      };
+    }
+    return {
+      received: record.propertyValues as Record<string, unknown>,
+      receivedTypes: {},
+      invalid: ["propertyValues bypassed the typed agent input"],
+    };
+  }
+  if (!hasEntries || !Array.isArray(record.propertyEntries)) {
+    return {
+      received: {},
+      receivedTypes: {},
+      invalid: ["propertyEntries is not an array"],
+    };
+  }
+
+  const received: Record<string, unknown> = Object.create(null) as Record<
+    string,
+    unknown
+  >;
+  const receivedTypes: Record<string, string> = Object.create(null) as Record<
+    string,
+    string
+  >;
+  const invalid: string[] = [];
+  for (const entry of record.propertyEntries) {
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+      invalid.push("propertyEntries contains a non-object entry");
+      continue;
+    }
+    const { propertyId, propertyType, value } = entry as Record<
+      string,
+      unknown
+    >;
+    if (
+      !hasExactKeys(entry as Record<string, unknown>, [
+        "propertyId",
+        "propertyType",
+        "value",
+      ])
+    ) {
+      invalid.push(
+        "propertyEntries contains an entry with unrecognized fields",
+      );
+      continue;
+    }
+    if (typeof propertyId !== "string" || propertyId.length === 0) {
+      invalid.push("propertyEntries contains an invalid propertyId");
+      continue;
+    }
+    if (typeof propertyType !== "string" || propertyType.length === 0) {
+      invalid.push(
+        `propertyEntries contains an invalid type for ${propertyId}`,
+      );
+      continue;
+    }
+    if (Object.prototype.hasOwnProperty.call(received, propertyId)) {
+      invalid.push(`propertyEntries contains duplicate ID ${propertyId}`);
+      continue;
+    }
+    received[propertyId] = value;
+    receivedTypes[propertyId] = propertyType;
+  }
+  return { received, receivedTypes, invalid };
+}
+
+const databaseRowMutationTools = new Set([
+  "add-database-item",
+  "update-database-item",
+  "upsert-database-item-by-key",
+  "duplicate-database-items",
+  "remove-database-items",
+]);
+
+function matchesCreateEnvelope(
+  input: Record<string, unknown>,
+  expected: NonNullable<ParityEvalScenario["expectedCreateEnvelope"]>,
+): boolean {
+  const target = input.target;
+  if (!target || typeof target !== "object" || Array.isArray(target)) {
+    return false;
+  }
+  const actualTarget = target as Record<string, unknown>;
+  return (
+    hasExactKeys(input, [
+      "target",
+      "expectedSchemaRevision",
+      "idempotencyKey",
+      "title",
+      input.propertyEntries === undefined
+        ? "propertyValues"
+        : "propertyEntries",
+    ]) &&
+    hasExactKeys(actualTarget, [
+      "spaceId",
+      "databaseId",
+      "databaseDocumentId",
+    ]) &&
+    actualTarget.spaceId === expected.target.spaceId &&
+    actualTarget.databaseId === expected.target.databaseId &&
+    actualTarget.databaseDocumentId === expected.target.databaseDocumentId &&
+    input.expectedSchemaRevision === expected.expectedSchemaRevision &&
+    input.idempotencyKey === expected.idempotencyKey &&
+    input.title === expected.title
+  );
+}
+
+function parseToolResultJson(result: string | undefined): unknown {
+  if (!result) return undefined;
+  try {
+    return JSON.parse(result);
+  } catch {
+    // coercion-ok: an unparseable tool result fails the same "did not supply
+    // the expected target/contract" scorer checks below as a missing one —
+    // both are non-passing evidence, not a distinction the eval needs to make.
+    return undefined;
+  }
+}
+
+function expectedPropertyValuesScorer(
+  expected: Record<string, unknown>,
+  expectedTypes: Record<string, string> | undefined,
+  expectedEnvelope?: ParityEvalScenario["expectedCreateEnvelope"],
+) {
+  return createScorer<
+    AgentRunOutput,
+    {
+      received: Record<string, unknown>;
+      missing: string[];
+      unexpected: string[];
+      invalid: string[];
+      mutationCalls: string[];
+    }
+  >({
+    name: "expected_property_values",
+    analyze(run) {
+      const mutationCalls = (run.toolCallDetails ?? [])
+        .filter((call) => databaseRowMutationTools.has(call.name))
+        .map((call) => call.name);
+      const createCalls = (run.toolCallDetails ?? []).filter(
+        (call) => call.name === "add-database-item",
+      );
+      const analysis = analyzePropertyValues(createCalls[0]?.input);
+      const invalid = [...analysis.invalid];
+      if (createCalls.length !== 1) {
+        invalid.push(
+          `expected exactly one add-database-item call, received ${createCalls.length}`,
+        );
+      }
+      if (mutationCalls.length !== 1) {
+        invalid.push(
+          `expected exactly one row mutation, received ${mutationCalls.length}`,
+        );
+      }
+      const orderedCalls = run.toolCalls;
+      const listIndex = orderedCalls.indexOf("list-content-databases");
+      const inspectIndex = orderedCalls.indexOf("get-content-database");
+      const createIndex = orderedCalls.indexOf("add-database-item");
+      if (
+        listIndex < 0 ||
+        inspectIndex <= listIndex ||
+        createIndex <= inspectIndex
+      ) {
+        invalid.push("database discovery did not precede the create mutation");
+      }
+      const listCall = (run.toolCallDetails ?? []).find(
+        (call) => call.name === "list-content-databases",
+      );
+      const inspectCall = (run.toolCallDetails ?? []).find(
+        (call) => call.name === "get-content-database",
+      );
+      const createCall = createCalls[0];
+      const listInput = listCall?.input as Record<string, unknown> | undefined;
+      const inspectInput = inspectCall?.input as
+        | Record<string, unknown>
+        | undefined;
+      if (
+        expectedEnvelope &&
+        (listInput?.title !== "PR #3314 feedback" ||
+          inspectInput?.databaseId !== expectedEnvelope.target.databaseId)
+      ) {
+        invalid.push(
+          "discovery did not resolve the requested title to the exact create target",
+        );
+      }
+      if (
+        !listCall?.completed ||
+        listCall.isError ||
+        !inspectCall?.completed ||
+        inspectCall.isError
+      ) {
+        invalid.push("database discovery calls did not complete successfully");
+      }
+      if (
+        listCall?.completedAtEventIndex === undefined ||
+        inspectCall?.startedAtEventIndex === undefined ||
+        listCall.completedAtEventIndex >= inspectCall.startedAtEventIndex ||
+        inspectCall.completedAtEventIndex === undefined ||
+        createCall?.startedAtEventIndex === undefined ||
+        inspectCall.completedAtEventIndex >= createCall.startedAtEventIndex
+      ) {
+        invalid.push(
+          "database discovery results were not available before dependent calls started",
+        );
+      }
+      if (expectedEnvelope) {
+        const listResult = parseToolResultJson(listCall?.result) as
+          | { databases?: Array<Record<string, unknown>> }
+          | undefined;
+        const discoveredDatabase = listResult?.databases?.find(
+          (database) =>
+            database.databaseId === expectedEnvelope.target.databaseId,
+        );
+        const discoveredDocumentId =
+          discoveredDatabase?.documentId ??
+          discoveredDatabase?.databaseDocumentId;
+        if (
+          !discoveredDatabase ||
+          discoveredDocumentId !== expectedEnvelope.target.databaseDocumentId ||
+          discoveredDatabase.spaceId !== expectedEnvelope.target.spaceId
+        ) {
+          invalid.push(
+            "list-content-databases result did not supply the expected create target",
+          );
+        }
+        const inspectResult = parseToolResultJson(inspectCall?.result) as
+          | {
+              mutationContract?: {
+                target?: Record<string, unknown>;
+                schemaRevision?: string;
+                expectedSchemaRevision?: string;
+                properties?: Array<Record<string, unknown>>;
+              };
+            }
+          | undefined;
+        const contractTarget = inspectResult?.mutationContract?.target;
+        const contractSchemaRevision =
+          inspectResult?.mutationContract?.schemaRevision ??
+          inspectResult?.mutationContract?.expectedSchemaRevision;
+        if (
+          !contractTarget ||
+          contractTarget.spaceId !== expectedEnvelope.target.spaceId ||
+          contractTarget.databaseId !== expectedEnvelope.target.databaseId ||
+          contractTarget.databaseDocumentId !==
+            expectedEnvelope.target.databaseDocumentId ||
+          contractSchemaRevision !== expectedEnvelope.expectedSchemaRevision
+        ) {
+          invalid.push(
+            "get-content-database result did not supply the expected mutation contract",
+          );
+        }
+        const discoveredProperties = new Map(
+          (inspectResult?.mutationContract?.properties ?? []).map(
+            (property) => [property.id, property],
+          ),
+        );
+        for (const [propertyId, expectedType] of Object.entries(
+          expectedTypes ?? {},
+        )) {
+          const property = discoveredProperties.get(propertyId);
+          if (
+            property?.type !== expectedType ||
+            property.writable !== true ||
+            property.sourceManaged === true
+          ) {
+            invalid.push(
+              `discovery did not supply writable property ${propertyId} with type ${expectedType}`,
+            );
+          }
+        }
+      }
+      for (const [propertyId, expectedType] of Object.entries(
+        expectedTypes ?? {},
+      )) {
+        if (analysis.receivedTypes[propertyId] !== expectedType) {
+          invalid.push(
+            `property ${propertyId} did not declare discovered type ${expectedType}`,
+          );
+        }
+      }
+      const createInput = createCalls[0]?.input;
+      if (
+        expectedEnvelope &&
+        (!createInput ||
+          typeof createInput !== "object" ||
+          Array.isArray(createInput) ||
+          !matchesCreateEnvelope(
+            createInput as Record<string, unknown>,
+            expectedEnvelope,
+          ))
+      ) {
+        invalid.push(
+          "create target, schema revision, idempotency key, or title did not match the fixture",
+        );
+      }
+      if (
+        !createCalls[0]?.completed ||
+        createCalls[0]?.completedSideEffect !== true ||
+        createCalls[0]?.isError
+      ) {
+        invalid.push("add-database-item did not complete successfully");
+      }
+      if (!run.ok) {
+        invalid.push("agent run did not complete successfully");
+      }
+      const received = analysis.received;
+      const missing = Object.entries(expected)
+        .filter(([propertyId, value]) => received[propertyId] !== value)
+        .map(([propertyId]) => propertyId);
+      const unexpected = Object.keys(received).filter(
+        (propertyId) =>
+          !Object.prototype.hasOwnProperty.call(expected, propertyId),
+      );
+      return { received, missing, unexpected, invalid, mutationCalls };
+    },
+    generateScore({ missing, unexpected, invalid }) {
+      return missing.length === 0 &&
+        unexpected.length === 0 &&
+        invalid.length === 0
+        ? 1
+        : 0;
+    },
+    generateReason({
+      analysis: { received, missing, unexpected, invalid, mutationCalls },
+    }) {
+      if (
+        missing.length === 0 &&
+        unexpected.length === 0 &&
+        invalid.length === 0
+      ) {
+        return "Agent preserved every expected property ID and exact value without inventing another property.";
+      }
+      return `Received propertyValues ${JSON.stringify(received)}; mutations: ${mutationCalls.join(", ") || "none"}; missing or changed: ${missing.join(", ") || "none"}; unexpected: ${unexpected.join(", ") || "none"}; invalid: ${invalid.join("; ") || "none"}`;
+    },
+  });
+}
+
 export function scenarioToEval(scenario: ParityEvalScenario): Eval {
   const name = `content-parity:${scenario.id}`;
 
@@ -52,6 +433,15 @@ export function scenarioToEval(scenario: ParityEvalScenario): Eval {
       ...(scenario.expectedTools?.length
         ? [expectedToolScorer(scenario.expectedTools)]
         : []),
+      ...(scenario.expectedPropertyValues
+        ? [
+            expectedPropertyValuesScorer(
+              scenario.expectedPropertyValues,
+              scenario.expectedPropertyTypes,
+              scenario.expectedCreateEnvelope,
+            ),
+          ]
+        : []),
     ],
   });
 }

package/dist/agent/run-store.js

@@ -2246,7 +2246,6 @@ export function resolveErroredRunTerminalEvent(run) {
                 type: "error",
                 error: detail || "The agent run failed.",
                 ...(code && code !== "unknown" ? { errorCode: code } : {}),
-                recoverable: true,
             },
             shouldPersist: true,
         };

package/dist/eval/agent-runner.js

@@ -52,19 +52,41 @@ export async function createAgentRunner(config) {
         const messages = toEngineMessages(input);
         let text = "";
         const toolCalls = [];
+        const toolCallDetails = [];
         let ok = true;
         let error;
+        let eventIndex = 0;
         const controller = new AbortController();
         const timer = setTimeout(() => controller.abort(), timeoutMs);
         const started = Date.now();
         const send = (event) => {
+            const currentEventIndex = eventIndex++;
             switch (event.type) {
                 case "text":
                     text += event.text;
                     break;
                 case "tool_start":
                     toolCalls.push(event.tool);
+                    toolCallDetails.push({
+                        name: event.tool,
+                        id: event.id,
+                        input: event.input,
+                        startedAtEventIndex: currentEventIndex,
+                    });
+                    break;
+                case "tool_done": {
+                    const detail = event.id
+                        ? toolCallDetails.find((call) => call.id === event.id)
+                        : toolCallDetails.find((call) => call.name === event.tool && !call.completed);
+                    if (detail) {
+                        detail.completed = true;
+                        detail.completedAtEventIndex = currentEventIndex;
+                        detail.completedSideEffect = event.completedSideEffect;
+                        detail.isError = event.isError === true;
+                        detail.result = event.result;
+                    }
                     break;
+                }
                 case "error":
                     ok = false;
                     error = event.error;
@@ -95,6 +117,7 @@ export async function createAgentRunner(config) {
         return {
             text,
             toolCalls,
+            toolCallDetails: toolCallDetails.map(({ id: _id, ...detail }) => detail),
             ok,
             error,
             runId,