@valerypopoff/trivet 2.0.1

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2026 Val P.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,12 @@
+# @valerypopoff/trivet
+
+Graph-oriented testing utilities for Rivet 2.0 projects.
+
+## Development
+
+```bash
+yarn workspace @valerypopoff/trivet run build
+yarn workspace @valerypopoff/trivet run lint
+```
+
+See the root [README](../../README.md) and [package docs](../../developer-docs/PACKAGES.md) for the current workspace contract.

package/dist/cjs/bundle.cjs

@@ -0,0 +1,381 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  DummyNativeApi: () => DummyNativeApi,
+  createTestGraphRunner: () => createTestGraphRunner,
+  deserializeTestCase: () => deserializeTestCase,
+  deserializeTestSuite: () => deserializeTestSuite,
+  deserializeTrivetData: () => deserializeTrivetData,
+  deserializeTrivetDataFromString: () => deserializeTrivetDataFromString,
+  runTrivet: () => runTrivet,
+  serializeTestCase: () => serializeTestCase,
+  serializeTestSuite: () => serializeTestSuite,
+  serializeTrivetData: () => serializeTrivetData,
+  serializeTrivetDataToString: () => serializeTrivetDataToString,
+  validateTestCaseFormat: () => validateTestCaseFormat,
+  validateValidationGraphFormat: () => validateValidationGraphFormat
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/trivetTypes.ts
+var import_rivet2_core = require("@valerypopoff/rivet2-core");
+
+// src/serialization/serialization_v1.ts
+var yaml = __toESM(require("yaml"), 1);
+function serializeTestCase(testCase) {
+  return {
+    id: testCase.id,
+    input: testCase.input,
+    expectedOutput: testCase.expectedOutput
+  };
+}
+function deserializeTestCase(data) {
+  return {
+    id: data.id,
+    input: data.input,
+    expectedOutput: data.expectedOutput
+  };
+}
+function serializeTestSuite(testSuite) {
+  return {
+    id: testSuite.id,
+    name: testSuite.name,
+    description: testSuite.description,
+    testGraph: testSuite.testGraph,
+    validationGraph: testSuite.validationGraph,
+    testCases: testSuite.testCases.map((testCase) => serializeTestCase(testCase))
+  };
+}
+function deserializeTestSuite(data) {
+  return {
+    id: data.id,
+    name: data.name,
+    description: data.description,
+    testGraph: data.testGraph,
+    validationGraph: data.validationGraph,
+    testCases: data.testCases.map((testCase) => deserializeTestCase(testCase))
+  };
+}
+function serializeTrivetData(data) {
+  return {
+    version: 1,
+    testSuites: data.testSuites.map((testSuite) => serializeTestSuite(testSuite))
+  };
+}
+function deserializeTrivetData(data) {
+  if (data.version !== 1) {
+    throw new Error(`Unsupported version: ${data.version}`);
+  }
+  return {
+    testSuites: data.testSuites.map((testSuite) => deserializeTestSuite(testSuite))
+  };
+}
+function serializeTrivetDataToString(data) {
+  return yaml.stringify(serializeTrivetData(data), null, 2);
+}
+function deserializeTrivetDataFromString(data) {
+  return deserializeTrivetData(yaml.parse(data));
+}
+
+// src/api.ts
+var import_rivet2_core2 = require("@valerypopoff/rivet2-core");
+var import_lodash_es = require("lodash");
+var TRUTHY_STRINGS = /* @__PURE__ */ new Set(["true", "TRUE"]);
+function validateOutput(v) {
+  switch (v.type) {
+    case "boolean":
+      return v.value;
+    case "string":
+      return TRUTHY_STRINGS.has(v.value);
+    default:
+      throw new Error(`Unexpected output type: ${v.type}`);
+  }
+}
+var DummyNativeApi = class {
+  readdir(path, baseDir, options) {
+    throw new Error(`Method not implemented. ${path}  ${baseDir} ${options}`);
+  }
+  readTextFile(path, baseDir) {
+    throw new Error(`Method not implemented. ${path}  ${baseDir}`);
+  }
+  readBinaryFile(path, baseDir) {
+    throw new Error(`Method not implemented. ${path}  ${baseDir}`);
+  }
+  writeTextFile(path, data, baseDir) {
+    throw new Error(`Method not implemented. ${path}  ${baseDir} ${data}`);
+  }
+  exec(command, args, options) {
+    throw new Error(`Method not implemented. ${command}  ${args} ${options}`);
+  }
+};
+function createTestGraphRunner(opts) {
+  return async (project, graphId, inputs) => {
+    const processor = new import_rivet2_core2.GraphProcessor(project, graphId, import_rivet2_core2.globalRivetNodeRegistry);
+    processor.executor = opts.executor;
+    const resolvedContextValues = {};
+    const outputs = await processor.processGraph(
+      {
+        nativeApi: new DummyNativeApi(),
+        tokenizer: new import_rivet2_core2.GptTokenizerTokenizer(),
+        settings: (0, import_rivet2_core2.resolveProcessSettings)({ openAiKey: opts.openAiKey })
+      },
+      inputs,
+      resolvedContextValues
+    );
+    return outputs;
+  };
+}
+async function runTrivet(opts) {
+  var _a;
+  const { project, testSuites, runGraph, onUpdate, iterationCount = 1 } = opts;
+  const graphsById = (0, import_lodash_es.keyBy)(project.graphs, (g) => {
+    var _a2;
+    return ((_a2 = g.metadata) == null ? void 0 : _a2.id) ?? "";
+  });
+  const trivetResults = {
+    testSuiteResults: [],
+    iterationCount
+  };
+  for (const testSuite of testSuites) {
+    const testGraph = graphsById[testSuite.testGraph];
+    const validationGraph = graphsById[testSuite.validationGraph];
+    if (testGraph === void 0) {
+      (0, import_rivet2_core2.logRuntimeWarn)("Missing Trivet test graph; skipping.", { testGraphId: testSuite.testGraph });
+      continue;
+    }
+    if (validationGraph === void 0) {
+      (0, import_rivet2_core2.logRuntimeWarn)("Missing Trivet validation graph; skipping.", { validationGraphId: testSuite.validationGraph });
+      continue;
+    }
+    const validationOutputNodesById = (0, import_lodash_es.keyBy)(
+      validationGraph.nodes.filter((n) => n.type === "graphOutput"),
+      (n) => n.data.id
+    );
+    const testCaseResults = [];
+    onUpdate == null ? void 0 : onUpdate({
+      ...trivetResults,
+      testSuiteResults: [
+        ...trivetResults.testSuiteResults,
+        {
+          id: testSuite.id,
+          testGraph: testSuite.testGraph,
+          validationGraph: testSuite.validationGraph,
+          name: testSuite.name ?? "Test",
+          description: testSuite.description ?? "It should pass.",
+          testCaseResults: testCaseResults.slice(),
+          passing: testCaseResults.every((r) => r.passing)
+        }
+      ]
+    });
+    for (const testCase of testSuite.testCases) {
+      for (let i = 0; i < iterationCount; i++) {
+        try {
+          const resolvedInputs = (0, import_lodash_es.mapValues)(testCase.input, import_rivet2_core2.inferType);
+          const startTime = Date.now();
+          const outputs = await runGraph(project, testGraph.metadata.id, resolvedInputs);
+          const duration = Date.now() - startTime;
+          const costOutput = outputs.cost;
+          const cost = costOutput && costOutput.type === "number" ? costOutput.value : 0;
+          (0, import_rivet2_core2.logRuntimeInfo)("Ran Trivet test graph", {
+            testSuiteId: testSuite.id,
+            testCaseId: testCase.id,
+            iteration: i + 1,
+            duration,
+            outputCount: Object.keys(outputs).length
+          });
+          (0, import_rivet2_core2.logRuntimeDebug)("Trivet test graph output summary", {
+            testSuiteId: testSuite.id,
+            testCaseId: testCase.id,
+            iteration: i + 1,
+            outputs: (0, import_rivet2_core2.summarizePortMapForLog)(outputs)
+          });
+          const validationInputs = {
+            input: {
+              type: "object",
+              value: testCase.input
+            },
+            expectedOutput: {
+              type: "object",
+              value: testCase.expectedOutput
+            },
+            output: {
+              type: "object",
+              value: (0, import_lodash_es.mapValues)(outputs, (dataValue) => dataValue.value)
+            }
+          };
+          (0, import_rivet2_core2.logRuntimeInfo)("Running Trivet validation graph", {
+            testSuiteId: testSuite.id,
+            testCaseId: testCase.id,
+            iteration: i + 1,
+            inputCount: Object.keys(validationInputs).length
+          });
+          (0, import_rivet2_core2.logRuntimeDebug)("Trivet validation input summary", {
+            testSuiteId: testSuite.id,
+            testCaseId: testCase.id,
+            iteration: i + 1,
+            inputs: (0, import_rivet2_core2.summarizePortMapForLog)(validationInputs)
+          });
+          const validationOutputs = (0, import_lodash_es.omit)(
+            await runGraph(project, validationGraph.metadata.id, validationInputs),
+            "cost"
+          );
+          const validationResults = Object.entries(validationOutputs).map(([outputId, result]) => {
+            const node = validationOutputNodesById[outputId];
+            if (node === void 0) {
+              throw new Error("Missing node for validation");
+            }
+            const valid = validateOutput(result);
+            return {
+              id: node.id,
+              title: node.title ?? "Validation",
+              description: node.description ?? "It should be valid.",
+              valid
+            };
+          });
+          testCaseResults.push({
+            id: testCase.id,
+            iteration: i + 1,
+            passing: validationResults.every((r) => r.valid),
+            message: ((_a = validationResults.find((r) => !r.valid)) == null ? void 0 : _a.description) ?? "PASS",
+            outputs: (0, import_lodash_es.mapValues)((0, import_lodash_es.omit)(outputs, "cost"), (dataValue) => dataValue.value),
+            duration,
+            cost
+          });
+        } catch (err) {
+          testCaseResults.push({
+            id: testCase.id,
+            iteration: i + 1,
+            passing: false,
+            message: "An error occurred",
+            outputs: {},
+            duration: 0,
+            cost: 0,
+            error: err
+          });
+        }
+        let existingTestSuite = trivetResults.testSuiteResults.find((ts) => ts.id === testSuite.id);
+        if (existingTestSuite == null) {
+          existingTestSuite = {
+            id: testSuite.id,
+            testGraph: testSuite.testGraph,
+            validationGraph: testSuite.validationGraph,
+            name: testSuite.name ?? "Test",
+            description: testSuite.description ?? "It should pass.",
+            testCaseResults: [],
+            passing: false
+          };
+          trivetResults.testSuiteResults.push(existingTestSuite);
+        }
+        existingTestSuite.testCaseResults = testCaseResults.slice();
+        existingTestSuite.passing = testCaseResults.every((r) => r.passing);
+        onUpdate == null ? void 0 : onUpdate((0, import_lodash_es.cloneDeep)(trivetResults));
+      }
+    }
+  }
+  return trivetResults;
+}
+
+// src/validateValidationGraphFormat.ts
+var import_lodash_es2 = require("lodash");
+var import_rivet2_core3 = require("@valerypopoff/rivet2-core");
+function validateValidationGraphFormat(validationGraph) {
+  const inputNodes = validationGraph.nodes.filter((n) => n.type === "graphInput");
+  const inputNodesById = (0, import_lodash_es2.keyBy)(inputNodes, (n) => n.data.id);
+  const validationResult = {
+    valid: true,
+    errorMessages: []
+  };
+  if (inputNodesById.input == null) {
+    validationResult.valid = false;
+    validationResult.errorMessages.push('Must have an input node with id "input"');
+  }
+  if (inputNodesById.expectedOutput == null) {
+    validationResult.valid = false;
+    validationResult.errorMessages.push('Must have an input node with id "expectedOutput"');
+  }
+  if (inputNodesById.output == null) {
+    validationResult.valid = false;
+    validationResult.errorMessages.push('Must have an input node with id "output"');
+  }
+  const outputNodes = validationGraph.nodes.filter((n) => n.type === "graphOutput");
+  if (outputNodes.length === 0) {
+    validationResult.valid = false;
+    validationResult.errorMessages.push(
+      "Must have at least one output node (that returns true/false as string or boolean)"
+    );
+  }
+  const invalidOutputs = outputNodes.filter(
+    (n) => n.data.dataType !== "boolean" && n.data.dataType !== "string" && n.data.dataType !== "any"
+  );
+  if (invalidOutputs.length > 0) {
+    validationResult.valid = false;
+    validationResult.errorMessages.push(
+      `Output nodes must return boolean or string, but found ${invalidOutputs.map((n) => n.data.dataType).join(", ")}`
+    );
+  }
+  return validationResult;
+}
+
+// src/validateTestCaseFormat.ts
+var import_rivet2_core4 = require("@valerypopoff/rivet2-core");
+function validateTestCaseFormat(testGraph, testCase) {
+  const inputNodes = testGraph.nodes.filter((n) => n.type === "graphInput");
+  const outputNodes = testGraph.nodes.filter((n) => n.type === "graphOutput");
+  const inputNodeIds = inputNodes.map((n) => n.data.id);
+  const outputNodeIds = outputNodes.map((n) => n.data.id);
+  const tcInputIds = new Set(Object.keys(testCase.input));
+  const missingInputIds = inputNodeIds.filter((id) => !tcInputIds.has(id));
+  const tcOutputIds = new Set(Object.keys(testCase.expectedOutput));
+  const missingOutputIds = outputNodeIds.filter((id) => !tcOutputIds.has(id));
+  return {
+    valid: missingInputIds.length === 0 && missingOutputIds.length === 0,
+    missingInputIds,
+    missingOutputIds
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DummyNativeApi,
+  createTestGraphRunner,
+  deserializeTestCase,
+  deserializeTestSuite,
+  deserializeTrivetData,
+  deserializeTrivetDataFromString,
+  runTrivet,
+  serializeTestCase,
+  serializeTestSuite,
+  serializeTrivetData,
+  serializeTrivetDataToString,
+  validateTestCaseFormat,
+  validateValidationGraphFormat
+});
+//# sourceMappingURL=bundle.cjs.map

package/dist/cjs/bundle.cjs.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/index.ts", "../../src/trivetTypes.ts", "../../src/serialization/serialization_v1.ts", "../../src/api.ts", "../../src/validateValidationGraphFormat.ts", "../../src/validateTestCaseFormat.ts"],
+  "sourcesContent": ["export * from './trivetTypes.js';\nexport * from './serialization/serialization_v1.js';\nexport * from './api.js';\nexport * from './validateValidationGraphFormat.js';\nexport * from './validateTestCaseFormat.js';\n", "import { type GraphId, type GraphInputs, type GraphOutputs, type Project } from '@valerypopoff/rivet2-core';\n\nexport type TrivetGraphRunner = (project: Project, graphId: GraphId, inputs: GraphInputs) => Promise<GraphOutputs>;\n\nexport interface TrivetOpts {\n  project: Project;\n  testSuites: TrivetTestSuite[];\n\n  /** Runs each test in each suite N times. Defaults to just 1. A test passes if all iterations pass. */\n  iterationCount?: number;\n\n  runGraph: TrivetGraphRunner;\n  onUpdate?: (results: TrivetResults) => void;\n}\n\nexport interface TrivetTestSuite {\n  id: string;\n  name?: string;\n  description?: string;\n  testGraph: string;\n  validationGraph: string;\n  testCases: TrivetTestCase[];\n}\n\nexport interface TrivetTestCase {\n  id: string;\n  input: Record<string, unknown>;\n  expectedOutput: Record<string, unknown>;\n}\n\nexport interface TrivetResults {\n  testSuiteResults: TrivetTestSuiteResult[];\n  iterationCount: number;\n}\n\nexport interface TrivetTestSuiteResult {\n  id: string;\n  testGraph: string;\n  validationGraph: string;\n  name: string;\n  description: string;\n  testCaseResults: TrivetTestCaseResult[];\n  passing: boolean;\n}\n\nexport interface TrivetTestCaseResult {\n  id: string;\n  iteration: number;\n  passing: boolean;\n  message: string;\n  outputs: Record<string, unknown>;\n  duration: number;\n  cost: number;\n  error?: Error | string | any;\n}\n\nexport type TrivetData = {\n  testSuites: TrivetTestSuite[];\n};\n", "import * as yaml from 'yaml';\nimport { type TrivetData, type TrivetTestCase, type TrivetTestSuite } from '../index.js';\n\nexport interface SerializedTrivetTestSuite {\n  id: string;\n  name?: string;\n  description?: string;\n  testGraph: string;\n  validationGraph: string;\n  testCases: SerializedTrivetTestCase[];\n}\n\nexport interface SerializedTrivetTestCase {\n  id: string;\n  input: Record<string, unknown>;\n  expectedOutput: Record<string, unknown>;\n}\n\nexport interface SerializedTrivetData {\n  version: 1;\n  testSuites: SerializedTrivetTestSuite[];\n}\n\nexport function serializeTestCase(testCase: TrivetTestCase): SerializedTrivetTestCase {\n  return {\n    id: testCase.id,\n    input: testCase.input,\n    expectedOutput: testCase.expectedOutput,\n  };\n}\n\nexport function deserializeTestCase(data: SerializedTrivetTestCase): TrivetTestCase {\n  return {\n    id: data.id,\n    input: data.input,\n    expectedOutput: data.expectedOutput,\n  };\n}\n\nexport function serializeTestSuite(testSuite: TrivetTestSuite): SerializedTrivetTestSuite {\n  return {\n    id: testSuite.id,\n    name: testSuite.name,\n    description: testSuite.description,\n    testGraph: testSuite.testGraph,\n    validationGraph: testSuite.validationGraph,\n    testCases: testSuite.testCases.map((testCase) => serializeTestCase(testCase)),\n  };\n}\n\nexport function deserializeTestSuite(data: SerializedTrivetTestSuite): TrivetTestSuite {\n  return {\n    id: data.id,\n    name: data.name,\n    description: data.description,\n    testGraph: data.testGraph,\n    validationGraph: data.validationGraph,\n    testCases: data.testCases.map((testCase) => deserializeTestCase(testCase)),\n  };\n}\n\nexport function serializeTrivetData(data: TrivetData): SerializedTrivetData {\n  return {\n    version: 1,\n    testSuites: data.testSuites.map((testSuite) => serializeTestSuite(testSuite)),\n  };\n}\n\nexport function deserializeTrivetData(data: SerializedTrivetData): TrivetData {\n  if (data.version !== 1) {\n    throw new Error(`Unsupported version: ${data.version}`);\n  }\n\n  return {\n    testSuites: data.testSuites.map((testSuite) => deserializeTestSuite(testSuite)),\n  };\n}\n\nexport function serializeTrivetDataToString(data: TrivetData): string {\n  return yaml.stringify(serializeTrivetData(data), null, 2);\n}\n\nexport function deserializeTrivetDataFromString(data: string): TrivetData {\n  return deserializeTrivetData(yaml.parse(data));\n}\n", "import {\n  type DataValue,\n  type NativeApi,\n  type BaseDir,\n  type ReadDirOptions,\n  GraphProcessor,\n  globalRivetNodeRegistry,\n  type GraphOutputNode,\n  GptTokenizerTokenizer,\n  inferType,\n  logRuntimeDebug,\n  logRuntimeInfo,\n  logRuntimeWarn,\n  resolveProcessSettings,\n  summarizePortMapForLog,\n} from '@valerypopoff/rivet2-core';\nimport { cloneDeep, keyBy, mapValues, omit } from 'lodash-es';\nimport {\n  type TrivetGraphRunner,\n  type TrivetOpts,\n  type TrivetResults,\n  type TrivetTestCaseResult,\n} from './trivetTypes.js';\n\nconst TRUTHY_STRINGS = new Set(['true', 'TRUE']);\n\nfunction validateOutput(v: DataValue) {\n  switch (v.type) {\n    case 'boolean':\n      return v.value;\n    case 'string':\n      return TRUTHY_STRINGS.has(v.value);\n    default:\n      throw new Error(`Unexpected output type: ${v.type}`);\n  }\n}\n\nexport class DummyNativeApi implements NativeApi {\n  readdir(path: string, baseDir?: BaseDir | undefined, options?: ReadDirOptions | undefined): Promise<string[]> {\n    throw new Error(`Method not implemented. ${path}  ${baseDir} ${options}`);\n  }\n  readTextFile(path: string, baseDir?: BaseDir | undefined): Promise<string> {\n    throw new Error(`Method not implemented. ${path}  ${baseDir}`);\n  }\n  readBinaryFile(path: string, baseDir?: BaseDir | undefined): Promise<Blob> {\n    throw new Error(`Method not implemented. ${path}  ${baseDir}`);\n  }\n  writeTextFile(path: string, data: string, baseDir?: BaseDir | undefined): Promise<void> {\n    throw new Error(`Method not implemented. ${path}  ${baseDir} ${data}`);\n  }\n  exec(command: string, args: string[], options?: { cwd?: string | undefined } | undefined): Promise<void> {\n    throw new Error(`Method not implemented. ${command}  ${args} ${options}`);\n  }\n}\n\nexport function createTestGraphRunner(opts: { openAiKey: string; executor?: 'nodejs' | 'browser' }): TrivetGraphRunner {\n  return async (project, graphId, inputs) => {\n    const processor = new GraphProcessor(project, graphId, globalRivetNodeRegistry);\n    processor.executor = opts.executor;\n    const resolvedContextValues: Record<string, DataValue> = {};\n    const outputs = await processor.processGraph(\n      {\n        nativeApi: new DummyNativeApi(),\n        tokenizer: new GptTokenizerTokenizer(),\n        settings: resolveProcessSettings({ openAiKey: opts.openAiKey }),\n      },\n      inputs,\n      resolvedContextValues,\n    );\n    return outputs;\n  };\n}\n\nexport async function runTrivet(opts: TrivetOpts): Promise<TrivetResults> {\n  const { project, testSuites, runGraph, onUpdate, iterationCount = 1 } = opts;\n\n  const graphsById = keyBy(project.graphs, (g) => g.metadata?.id ?? '');\n\n  const trivetResults: TrivetResults = {\n    testSuiteResults: [],\n    iterationCount,\n  };\n\n  for (const testSuite of testSuites) {\n    const testGraph = graphsById[testSuite.testGraph];\n    const validationGraph = graphsById[testSuite.validationGraph];\n    if (testGraph === undefined) {\n      logRuntimeWarn('Missing Trivet test graph; skipping.', { testGraphId: testSuite.testGraph });\n      continue;\n    }\n    if (validationGraph === undefined) {\n      logRuntimeWarn('Missing Trivet validation graph; skipping.', { validationGraphId: testSuite.validationGraph });\n      continue;\n    }\n\n    const validationOutputNodesById = keyBy(\n      validationGraph.nodes.filter((n): n is GraphOutputNode => n.type === 'graphOutput'),\n      (n) => n.data.id,\n    );\n\n    const testCaseResults: TrivetTestCaseResult[] = [];\n\n    onUpdate?.({\n      ...trivetResults,\n      testSuiteResults: [\n        ...trivetResults.testSuiteResults,\n        {\n          id: testSuite.id,\n          testGraph: testSuite.testGraph,\n          validationGraph: testSuite.validationGraph,\n          name: testSuite.name ?? 'Test',\n          description: testSuite.description ?? 'It should pass.',\n          testCaseResults: testCaseResults.slice(),\n          passing: testCaseResults.every((r) => r.passing),\n        },\n      ],\n    });\n\n    for (const testCase of testSuite.testCases) {\n      for (let i = 0; i < iterationCount; i++) {\n        try {\n          const resolvedInputs: Record<string, DataValue> = mapValues(testCase.input, inferType);\n          const startTime = Date.now();\n          const outputs = await runGraph(project, testGraph.metadata!.id!, resolvedInputs);\n          const duration = Date.now() - startTime;\n          const costOutput = outputs.cost;\n          const cost = costOutput && costOutput.type === 'number' ? costOutput.value : 0;\n\n          logRuntimeInfo('Ran Trivet test graph', {\n            testSuiteId: testSuite.id,\n            testCaseId: testCase.id,\n            iteration: i + 1,\n            duration,\n            outputCount: Object.keys(outputs).length,\n          });\n          logRuntimeDebug('Trivet test graph output summary', {\n            testSuiteId: testSuite.id,\n            testCaseId: testCase.id,\n            iteration: i + 1,\n            outputs: summarizePortMapForLog(outputs),\n          });\n\n          const validationInputs: Record<string, DataValue> = {\n            input: {\n              type: 'object',\n              value: testCase.input,\n            },\n            expectedOutput: {\n              type: 'object',\n              value: testCase.expectedOutput,\n            },\n            output: {\n              type: 'object',\n              value: mapValues(outputs, (dataValue) => dataValue.value),\n            },\n          };\n\n          logRuntimeInfo('Running Trivet validation graph', {\n            testSuiteId: testSuite.id,\n            testCaseId: testCase.id,\n            iteration: i + 1,\n            inputCount: Object.keys(validationInputs).length,\n          });\n          logRuntimeDebug('Trivet validation input summary', {\n            testSuiteId: testSuite.id,\n            testCaseId: testCase.id,\n            iteration: i + 1,\n            inputs: summarizePortMapForLog(validationInputs),\n          });\n\n          const validationOutputs = omit(\n            await runGraph(project, validationGraph.metadata!.id!, validationInputs),\n            'cost',\n          );\n\n          const validationResults = Object.entries(validationOutputs).map(([outputId, result]) => {\n            const node = validationOutputNodesById[outputId];\n            if (node === undefined) {\n              throw new Error('Missing node for validation');\n            }\n            const valid = validateOutput(result);\n            return {\n              id: node.id,\n              title: node.title ?? 'Validation',\n              description: node.description ?? 'It should be valid.',\n              valid,\n            };\n          });\n          testCaseResults.push({\n            id: testCase.id,\n            iteration: i + 1,\n            passing: validationResults.every((r) => r.valid),\n            message: validationResults.find((r) => !r.valid)?.description ?? 'PASS',\n            outputs: mapValues(omit(outputs, 'cost'), (dataValue) => dataValue.value),\n            duration,\n            cost,\n          });\n        } catch (err) {\n          testCaseResults.push({\n            id: testCase.id,\n            iteration: i + 1,\n            passing: false,\n            message: 'An error occurred',\n            outputs: {},\n            duration: 0,\n            cost: 0,\n            error: err,\n          });\n        }\n\n        let existingTestSuite = trivetResults.testSuiteResults.find((ts) => ts.id === testSuite.id);\n        if (existingTestSuite == null) {\n          existingTestSuite = {\n            id: testSuite.id,\n            testGraph: testSuite.testGraph,\n            validationGraph: testSuite.validationGraph,\n            name: testSuite.name ?? 'Test',\n            description: testSuite.description ?? 'It should pass.',\n            testCaseResults: [],\n            passing: false,\n          };\n          trivetResults.testSuiteResults.push(existingTestSuite);\n        }\n        existingTestSuite.testCaseResults = testCaseResults.slice();\n        existingTestSuite.passing = testCaseResults.every((r) => r.passing);\n\n        onUpdate?.(cloneDeep(trivetResults));\n      }\n    }\n  }\n  return trivetResults;\n}\n", "import { keyBy } from 'lodash-es';\nimport { type GraphInputNode, type NodeGraph } from '@valerypopoff/rivet2-core';\n\nexport function validateValidationGraphFormat(validationGraph: NodeGraph): { valid: boolean; errorMessages: string[] } {\n  const inputNodes = validationGraph.nodes.filter((n): n is GraphInputNode => n.type === 'graphInput');\n  const inputNodesById = keyBy(inputNodes, (n) => n.data.id);\n  const validationResult = {\n    valid: true,\n    errorMessages: [] as string[],\n  };\n  if (inputNodesById.input == null) {\n    validationResult.valid = false;\n    validationResult.errorMessages.push('Must have an input node with id \"input\"');\n  }\n  if (inputNodesById.expectedOutput == null) {\n    validationResult.valid = false;\n    validationResult.errorMessages.push('Must have an input node with id \"expectedOutput\"');\n  }\n  if (inputNodesById.output == null) {\n    validationResult.valid = false;\n    validationResult.errorMessages.push('Must have an input node with id \"output\"');\n  }\n  const outputNodes = validationGraph.nodes.filter((n): n is GraphInputNode => n.type === 'graphOutput');\n  if (outputNodes.length === 0) {\n    validationResult.valid = false;\n    validationResult.errorMessages.push(\n      'Must have at least one output node (that returns true/false as string or boolean)',\n    );\n  }\n  const invalidOutputs = outputNodes.filter(\n    (n) => n.data.dataType !== 'boolean' && n.data.dataType !== 'string' && n.data.dataType !== 'any',\n  );\n  if (invalidOutputs.length > 0) {\n    validationResult.valid = false;\n    validationResult.errorMessages.push(\n      `Output nodes must return boolean or string, but found ${invalidOutputs.map((n) => n.data.dataType).join(', ')}`,\n    );\n  }\n  return validationResult;\n}\n", "import { type GraphInputNode, type GraphOutputNode, type NodeGraph } from '@valerypopoff/rivet2-core';\nimport { type TrivetTestCase } from './trivetTypes.js';\n\nexport function validateTestCaseFormat(\n  testGraph: NodeGraph,\n  testCase: TrivetTestCase,\n): {\n  valid: boolean;\n  missingInputIds: string[];\n  missingOutputIds: string[];\n} {\n  const inputNodes = testGraph.nodes.filter((n): n is GraphInputNode => n.type === 'graphInput');\n  const outputNodes = testGraph.nodes.filter((n): n is GraphOutputNode => n.type === 'graphOutput');\n  const inputNodeIds = inputNodes.map((n) => n.data.id);\n  const outputNodeIds = outputNodes.map((n) => n.data.id);\n\n  const tcInputIds = new Set(Object.keys(testCase.input));\n  const missingInputIds = inputNodeIds.filter((id) => !tcInputIds.has(id));\n\n  const tcOutputIds = new Set(Object.keys(testCase.expectedOutput));\n  const missingOutputIds = outputNodeIds.filter((id) => !tcOutputIds.has(id));\n\n  return {\n    valid: missingInputIds.length === 0 && missingOutputIds.length === 0,\n    missingInputIds,\n    missingOutputIds,\n  };\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,yBAAgF;;;ACAhF,WAAsB;AAuBf,SAAS,kBAAkB,UAAoD;AACpF,SAAO;AAAA,IACL,IAAI,SAAS;AAAA,IACb,OAAO,SAAS;AAAA,IAChB,gBAAgB,SAAS;AAAA,EAC3B;AACF;AAEO,SAAS,oBAAoB,MAAgD;AAClF,SAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT,OAAO,KAAK;AAAA,IACZ,gBAAgB,KAAK;AAAA,EACvB;AACF;AAEO,SAAS,mBAAmB,WAAuD;AACxF,SAAO;AAAA,IACL,IAAI,UAAU;AAAA,IACd,MAAM,UAAU;AAAA,IAChB,aAAa,UAAU;AAAA,IACvB,WAAW,UAAU;AAAA,IACrB,iBAAiB,UAAU;AAAA,IAC3B,WAAW,UAAU,UAAU,IAAI,CAAC,aAAa,kBAAkB,QAAQ,CAAC;AAAA,EAC9E;AACF;AAEO,SAAS,qBAAqB,MAAkD;AACrF,SAAO;AAAA,IACL,IAAI,KAAK;AAAA,IACT,MAAM,KAAK;AAAA,IACX,aAAa,KAAK;AAAA,IAClB,WAAW,KAAK;AAAA,IAChB,iBAAiB,KAAK;AAAA,IACtB,WAAW,KAAK,UAAU,IAAI,CAAC,aAAa,oBAAoB,QAAQ,CAAC;AAAA,EAC3E;AACF;AAEO,SAAS,oBAAoB,MAAwC;AAC1E,SAAO;AAAA,IACL,SAAS;AAAA,IACT,YAAY,KAAK,WAAW,IAAI,CAAC,cAAc,mBAAmB,SAAS,CAAC;AAAA,EAC9E;AACF;AAEO,SAAS,sBAAsB,MAAwC;AAC5E,MAAI,KAAK,YAAY,GAAG;AACtB,UAAM,IAAI,MAAM,wBAAwB,KAAK,OAAO,EAAE;AAAA,EACxD;AAEA,SAAO;AAAA,IACL,YAAY,KAAK,WAAW,IAAI,CAAC,cAAc,qBAAqB,SAAS,CAAC;AAAA,EAChF;AACF;AAEO,SAAS,4BAA4B,MAA0B;AACpE,SAAY,eAAU,oBAAoB,IAAI,GAAG,MAAM,CAAC;AAC1D;AAEO,SAAS,gCAAgC,MAA0B;AACxE,SAAO,sBAA2B,WAAM,IAAI,CAAC;AAC/C;;;ACpFA,IAAAA,sBAeO;AACP,uBAAkD;AAQlD,IAAM,iBAAiB,oBAAI,IAAI,CAAC,QAAQ,MAAM,CAAC;AAE/C,SAAS,eAAe,GAAc;AACpC,UAAQ,EAAE,MAAM;AAAA,IACd,KAAK;AACH,aAAO,EAAE;AAAA,IACX,KAAK;AACH,aAAO,eAAe,IAAI,EAAE,KAAK;AAAA,IACnC;AACE,YAAM,IAAI,MAAM,2BAA2B,EAAE,IAAI,EAAE;AAAA,EACvD;AACF;AAEO,IAAM,iBAAN,MAA0C;AAAA,EAC/C,QAAQ,MAAc,SAA+B,SAAyD;AAC5G,UAAM,IAAI,MAAM,2BAA2B,IAAI,KAAK,OAAO,IAAI,OAAO,EAAE;AAAA,EAC1E;AAAA,EACA,aAAa,MAAc,SAAgD;AACzE,UAAM,IAAI,MAAM,2BAA2B,IAAI,KAAK,OAAO,EAAE;AAAA,EAC/D;AAAA,EACA,eAAe,MAAc,SAA8C;AACzE,UAAM,IAAI,MAAM,2BAA2B,IAAI,KAAK,OAAO,EAAE;AAAA,EAC/D;AAAA,EACA,cAAc,MAAc,MAAc,SAA8C;AACtF,UAAM,IAAI,MAAM,2BAA2B,IAAI,KAAK,OAAO,IAAI,IAAI,EAAE;AAAA,EACvE;AAAA,EACA,KAAK,SAAiB,MAAgB,SAAmE;AACvG,UAAM,IAAI,MAAM,2BAA2B,OAAO,KAAK,IAAI,IAAI,OAAO,EAAE;AAAA,EAC1E;AACF;AAEO,SAAS,sBAAsB,MAAiF;AACrH,SAAO,OAAO,SAAS,SAAS,WAAW;AACzC,UAAM,YAAY,IAAI,mCAAe,SAAS,SAAS,2CAAuB;AAC9E,cAAU,WAAW,KAAK;AAC1B,UAAM,wBAAmD,CAAC;AAC1D,UAAM,UAAU,MAAM,UAAU;AAAA,MAC9B;AAAA,QACE,WAAW,IAAI,eAAe;AAAA,QAC9B,WAAW,IAAI,0CAAsB;AAAA,QACrC,cAAU,4CAAuB,EAAE,WAAW,KAAK,UAAU,CAAC;AAAA,MAChE;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,UAAU,MAA0C;AAzE1E;AA0EE,QAAM,EAAE,SAAS,YAAY,UAAU,UAAU,iBAAiB,EAAE,IAAI;AAExE,QAAM,iBAAa,wBAAM,QAAQ,QAAQ,CAAC,MAAG;AA5E/C,QAAAC;AA4EkD,aAAAA,MAAA,EAAE,aAAF,gBAAAA,IAAY,OAAM;AAAA,GAAE;AAEpE,QAAM,gBAA+B;AAAA,IACnC,kBAAkB,CAAC;AAAA,IACnB;AAAA,EACF;AAEA,aAAW,aAAa,YAAY;AAClC,UAAM,YAAY,WAAW,UAAU,SAAS;AAChD,UAAM,kBAAkB,WAAW,UAAU,eAAe;AAC5D,QAAI,cAAc,QAAW;AAC3B,8CAAe,wCAAwC,EAAE,aAAa,UAAU,UAAU,CAAC;AAC3F;AAAA,IACF;AACA,QAAI,oBAAoB,QAAW;AACjC,8CAAe,8CAA8C,EAAE,mBAAmB,UAAU,gBAAgB,CAAC;AAC7G;AAAA,IACF;AAEA,UAAM,gCAA4B;AAAA,MAChC,gBAAgB,MAAM,OAAO,CAAC,MAA4B,EAAE,SAAS,aAAa;AAAA,MAClF,CAAC,MAAM,EAAE,KAAK;AAAA,IAChB;AAEA,UAAM,kBAA0C,CAAC;AAEjD,yCAAW;AAAA,MACT,GAAG;AAAA,MACH,kBAAkB;AAAA,QAChB,GAAG,cAAc;AAAA,QACjB;AAAA,UACE,IAAI,UAAU;AAAA,UACd,WAAW,UAAU;AAAA,UACrB,iBAAiB,UAAU;AAAA,UAC3B,MAAM,UAAU,QAAQ;AAAA,UACxB,aAAa,UAAU,eAAe;AAAA,UACtC,iBAAiB,gBAAgB,MAAM;AAAA,UACvC,SAAS,gBAAgB,MAAM,CAAC,MAAM,EAAE,OAAO;AAAA,QACjD;AAAA,MACF;AAAA,IACF;AAEA,eAAW,YAAY,UAAU,WAAW;AAC1C,eAAS,IAAI,GAAG,IAAI,gBAAgB,KAAK;AACvC,YAAI;AACF,gBAAM,qBAA4C,4BAAU,SAAS,OAAO,6BAAS;AACrF,gBAAM,YAAY,KAAK,IAAI;AAC3B,gBAAM,UAAU,MAAM,SAAS,SAAS,UAAU,SAAU,IAAK,cAAc;AAC/E,gBAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,gBAAM,aAAa,QAAQ;AAC3B,gBAAM,OAAO,cAAc,WAAW,SAAS,WAAW,WAAW,QAAQ;AAE7E,kDAAe,yBAAyB;AAAA,YACtC,aAAa,UAAU;AAAA,YACvB,YAAY,SAAS;AAAA,YACrB,WAAW,IAAI;AAAA,YACf;AAAA,YACA,aAAa,OAAO,KAAK,OAAO,EAAE;AAAA,UACpC,CAAC;AACD,mDAAgB,oCAAoC;AAAA,YAClD,aAAa,UAAU;AAAA,YACvB,YAAY,SAAS;AAAA,YACrB,WAAW,IAAI;AAAA,YACf,aAAS,4CAAuB,OAAO;AAAA,UACzC,CAAC;AAED,gBAAM,mBAA8C;AAAA,YAClD,OAAO;AAAA,cACL,MAAM;AAAA,cACN,OAAO,SAAS;AAAA,YAClB;AAAA,YACA,gBAAgB;AAAA,cACd,MAAM;AAAA,cACN,OAAO,SAAS;AAAA,YAClB;AAAA,YACA,QAAQ;AAAA,cACN,MAAM;AAAA,cACN,WAAO,4BAAU,SAAS,CAAC,cAAc,UAAU,KAAK;AAAA,YAC1D;AAAA,UACF;AAEA,kDAAe,mCAAmC;AAAA,YAChD,aAAa,UAAU;AAAA,YACvB,YAAY,SAAS;AAAA,YACrB,WAAW,IAAI;AAAA,YACf,YAAY,OAAO,KAAK,gBAAgB,EAAE;AAAA,UAC5C,CAAC;AACD,mDAAgB,mCAAmC;AAAA,YACjD,aAAa,UAAU;AAAA,YACvB,YAAY,SAAS;AAAA,YACrB,WAAW,IAAI;AAAA,YACf,YAAQ,4CAAuB,gBAAgB;AAAA,UACjD,CAAC;AAED,gBAAM,wBAAoB;AAAA,YACxB,MAAM,SAAS,SAAS,gBAAgB,SAAU,IAAK,gBAAgB;AAAA,YACvE;AAAA,UACF;AAEA,gBAAM,oBAAoB,OAAO,QAAQ,iBAAiB,EAAE,IAAI,CAAC,CAAC,UAAU,MAAM,MAAM;AACtF,kBAAM,OAAO,0BAA0B,QAAQ;AAC/C,gBAAI,SAAS,QAAW;AACtB,oBAAM,IAAI,MAAM,6BAA6B;AAAA,YAC/C;AACA,kBAAM,QAAQ,eAAe,MAAM;AACnC,mBAAO;AAAA,cACL,IAAI,KAAK;AAAA,cACT,OAAO,KAAK,SAAS;AAAA,cACrB,aAAa,KAAK,eAAe;AAAA,cACjC;AAAA,YACF;AAAA,UACF,CAAC;AACD,0BAAgB,KAAK;AAAA,YACnB,IAAI,SAAS;AAAA,YACb,WAAW,IAAI;AAAA,YACf,SAAS,kBAAkB,MAAM,CAAC,MAAM,EAAE,KAAK;AAAA,YAC/C,WAAS,uBAAkB,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,MAAtC,mBAAyC,gBAAe;AAAA,YACjE,aAAS,gCAAU,uBAAK,SAAS,MAAM,GAAG,CAAC,cAAc,UAAU,KAAK;AAAA,YACxE;AAAA,YACA;AAAA,UACF,CAAC;AAAA,QACH,SAAS,KAAK;AACZ,0BAAgB,KAAK;AAAA,YACnB,IAAI,SAAS;AAAA,YACb,WAAW,IAAI;AAAA,YACf,SAAS;AAAA,YACT,SAAS;AAAA,YACT,SAAS,CAAC;AAAA,YACV,UAAU;AAAA,YACV,MAAM;AAAA,YACN,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAEA,YAAI,oBAAoB,cAAc,iBAAiB,KAAK,CAAC,OAAO,GAAG,OAAO,UAAU,EAAE;AAC1F,YAAI,qBAAqB,MAAM;AAC7B,8BAAoB;AAAA,YAClB,IAAI,UAAU;AAAA,YACd,WAAW,UAAU;AAAA,YACrB,iBAAiB,UAAU;AAAA,YAC3B,MAAM,UAAU,QAAQ;AAAA,YACxB,aAAa,UAAU,eAAe;AAAA,YACtC,iBAAiB,CAAC;AAAA,YAClB,SAAS;AAAA,UACX;AACA,wBAAc,iBAAiB,KAAK,iBAAiB;AAAA,QACvD;AACA,0BAAkB,kBAAkB,gBAAgB,MAAM;AAC1D,0BAAkB,UAAU,gBAAgB,MAAM,CAAC,MAAM,EAAE,OAAO;AAElE,iDAAW,4BAAU,aAAa;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;;;ACvOA,IAAAC,oBAAsB;AACtB,IAAAC,sBAAoD;AAE7C,SAAS,8BAA8B,iBAAyE;AACrH,QAAM,aAAa,gBAAgB,MAAM,OAAO,CAAC,MAA2B,EAAE,SAAS,YAAY;AACnG,QAAM,qBAAiB,yBAAM,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE;AACzD,QAAM,mBAAmB;AAAA,IACvB,OAAO;AAAA,IACP,eAAe,CAAC;AAAA,EAClB;AACA,MAAI,eAAe,SAAS,MAAM;AAChC,qBAAiB,QAAQ;AACzB,qBAAiB,cAAc,KAAK,yCAAyC;AAAA,EAC/E;AACA,MAAI,eAAe,kBAAkB,MAAM;AACzC,qBAAiB,QAAQ;AACzB,qBAAiB,cAAc,KAAK,kDAAkD;AAAA,EACxF;AACA,MAAI,eAAe,UAAU,MAAM;AACjC,qBAAiB,QAAQ;AACzB,qBAAiB,cAAc,KAAK,0CAA0C;AAAA,EAChF;AACA,QAAM,cAAc,gBAAgB,MAAM,OAAO,CAAC,MAA2B,EAAE,SAAS,aAAa;AACrG,MAAI,YAAY,WAAW,GAAG;AAC5B,qBAAiB,QAAQ;AACzB,qBAAiB,cAAc;AAAA,MAC7B;AAAA,IACF;AAAA,EACF;AACA,QAAM,iBAAiB,YAAY;AAAA,IACjC,CAAC,MAAM,EAAE,KAAK,aAAa,aAAa,EAAE,KAAK,aAAa,YAAY,EAAE,KAAK,aAAa;AAAA,EAC9F;AACA,MAAI,eAAe,SAAS,GAAG;AAC7B,qBAAiB,QAAQ;AACzB,qBAAiB,cAAc;AAAA,MAC7B,yDAAyD,eAAe,IAAI,CAAC,MAAM,EAAE,KAAK,QAAQ,EAAE,KAAK,IAAI,CAAC;AAAA,IAChH;AAAA,EACF;AACA,SAAO;AACT;;;ACvCA,IAAAC,sBAA0E;AAGnE,SAAS,uBACd,WACA,UAKA;AACA,QAAM,aAAa,UAAU,MAAM,OAAO,CAAC,MAA2B,EAAE,SAAS,YAAY;AAC7F,QAAM,cAAc,UAAU,MAAM,OAAO,CAAC,MAA4B,EAAE,SAAS,aAAa;AAChG,QAAM,eAAe,WAAW,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;AACpD,QAAM,gBAAgB,YAAY,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;AAEtD,QAAM,aAAa,IAAI,IAAI,OAAO,KAAK,SAAS,KAAK,CAAC;AACtD,QAAM,kBAAkB,aAAa,OAAO,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;AAEvE,QAAM,cAAc,IAAI,IAAI,OAAO,KAAK,SAAS,cAAc,CAAC;AAChE,QAAM,mBAAmB,cAAc,OAAO,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;AAE1E,SAAO;AAAA,IACL,OAAO,gBAAgB,WAAW,KAAK,iBAAiB,WAAW;AAAA,IACnE;AAAA,IACA;AAAA,EACF;AACF;",
+  "names": ["import_rivet2_core", "_a", "import_lodash_es", "import_rivet2_core", "import_rivet2_core"]
+}

package/dist/esm/api.js

@@ -0,0 +1,184 @@
+import { GraphProcessor, globalRivetNodeRegistry, GptTokenizerTokenizer, inferType, logRuntimeDebug, logRuntimeInfo, logRuntimeWarn, resolveProcessSettings, summarizePortMapForLog, } from '@valerypopoff/rivet2-core';
+import { cloneDeep, keyBy, mapValues, omit } from 'lodash-es';
+import {} from './trivetTypes.js';
+const TRUTHY_STRINGS = new Set(['true', 'TRUE']);
+function validateOutput(v) {
+    switch (v.type) {
+        case 'boolean':
+            return v.value;
+        case 'string':
+            return TRUTHY_STRINGS.has(v.value);
+        default:
+            throw new Error(`Unexpected output type: ${v.type}`);
+    }
+}
+export class DummyNativeApi {
+    readdir(path, baseDir, options) {
+        throw new Error(`Method not implemented. ${path}  ${baseDir} ${options}`);
+    }
+    readTextFile(path, baseDir) {
+        throw new Error(`Method not implemented. ${path}  ${baseDir}`);
+    }
+    readBinaryFile(path, baseDir) {
+        throw new Error(`Method not implemented. ${path}  ${baseDir}`);
+    }
+    writeTextFile(path, data, baseDir) {
+        throw new Error(`Method not implemented. ${path}  ${baseDir} ${data}`);
+    }
+    exec(command, args, options) {
+        throw new Error(`Method not implemented. ${command}  ${args} ${options}`);
+    }
+}
+export function createTestGraphRunner(opts) {
+    return async (project, graphId, inputs) => {
+        const processor = new GraphProcessor(project, graphId, globalRivetNodeRegistry);
+        processor.executor = opts.executor;
+        const resolvedContextValues = {};
+        const outputs = await processor.processGraph({
+            nativeApi: new DummyNativeApi(),
+            tokenizer: new GptTokenizerTokenizer(),
+            settings: resolveProcessSettings({ openAiKey: opts.openAiKey }),
+        }, inputs, resolvedContextValues);
+        return outputs;
+    };
+}
+export async function runTrivet(opts) {
+    const { project, testSuites, runGraph, onUpdate, iterationCount = 1 } = opts;
+    const graphsById = keyBy(project.graphs, (g) => g.metadata?.id ?? '');
+    const trivetResults = {
+        testSuiteResults: [],
+        iterationCount,
+    };
+    for (const testSuite of testSuites) {
+        const testGraph = graphsById[testSuite.testGraph];
+        const validationGraph = graphsById[testSuite.validationGraph];
+        if (testGraph === undefined) {
+            logRuntimeWarn('Missing Trivet test graph; skipping.', { testGraphId: testSuite.testGraph });
+            continue;
+        }
+        if (validationGraph === undefined) {
+            logRuntimeWarn('Missing Trivet validation graph; skipping.', { validationGraphId: testSuite.validationGraph });
+            continue;
+        }
+        const validationOutputNodesById = keyBy(validationGraph.nodes.filter((n) => n.type === 'graphOutput'), (n) => n.data.id);
+        const testCaseResults = [];
+        onUpdate?.({
+            ...trivetResults,
+            testSuiteResults: [
+                ...trivetResults.testSuiteResults,
+                {
+                    id: testSuite.id,
+                    testGraph: testSuite.testGraph,
+                    validationGraph: testSuite.validationGraph,
+                    name: testSuite.name ?? 'Test',
+                    description: testSuite.description ?? 'It should pass.',
+                    testCaseResults: testCaseResults.slice(),
+                    passing: testCaseResults.every((r) => r.passing),
+                },
+            ],
+        });
+        for (const testCase of testSuite.testCases) {
+            for (let i = 0; i < iterationCount; i++) {
+                try {
+                    const resolvedInputs = mapValues(testCase.input, inferType);
+                    const startTime = Date.now();
+                    const outputs = await runGraph(project, testGraph.metadata.id, resolvedInputs);
+                    const duration = Date.now() - startTime;
+                    const costOutput = outputs.cost;
+                    const cost = costOutput && costOutput.type === 'number' ? costOutput.value : 0;
+                    logRuntimeInfo('Ran Trivet test graph', {
+                        testSuiteId: testSuite.id,
+                        testCaseId: testCase.id,
+                        iteration: i + 1,
+                        duration,
+                        outputCount: Object.keys(outputs).length,
+                    });
+                    logRuntimeDebug('Trivet test graph output summary', {
+                        testSuiteId: testSuite.id,
+                        testCaseId: testCase.id,
+                        iteration: i + 1,
+                        outputs: summarizePortMapForLog(outputs),
+                    });
+                    const validationInputs = {
+                        input: {
+                            type: 'object',
+                            value: testCase.input,
+                        },
+                        expectedOutput: {
+                            type: 'object',
+                            value: testCase.expectedOutput,
+                        },
+                        output: {
+                            type: 'object',
+                            value: mapValues(outputs, (dataValue) => dataValue.value),
+                        },
+                    };
+                    logRuntimeInfo('Running Trivet validation graph', {
+                        testSuiteId: testSuite.id,
+                        testCaseId: testCase.id,
+                        iteration: i + 1,
+                        inputCount: Object.keys(validationInputs).length,
+                    });
+                    logRuntimeDebug('Trivet validation input summary', {
+                        testSuiteId: testSuite.id,
+                        testCaseId: testCase.id,
+                        iteration: i + 1,
+                        inputs: summarizePortMapForLog(validationInputs),
+                    });
+                    const validationOutputs = omit(await runGraph(project, validationGraph.metadata.id, validationInputs), 'cost');
+                    const validationResults = Object.entries(validationOutputs).map(([outputId, result]) => {
+                        const node = validationOutputNodesById[outputId];
+                        if (node === undefined) {
+                            throw new Error('Missing node for validation');
+                        }
+                        const valid = validateOutput(result);
+                        return {
+                            id: node.id,
+                            title: node.title ?? 'Validation',
+                            description: node.description ?? 'It should be valid.',
+                            valid,
+                        };
+                    });
+                    testCaseResults.push({
+                        id: testCase.id,
+                        iteration: i + 1,
+                        passing: validationResults.every((r) => r.valid),
+                        message: validationResults.find((r) => !r.valid)?.description ?? 'PASS',
+                        outputs: mapValues(omit(outputs, 'cost'), (dataValue) => dataValue.value),
+                        duration,
+                        cost,
+                    });
+                }
+                catch (err) {
+                    testCaseResults.push({
+                        id: testCase.id,
+                        iteration: i + 1,
+                        passing: false,
+                        message: 'An error occurred',
+                        outputs: {},
+                        duration: 0,
+                        cost: 0,
+                        error: err,
+                    });
+                }
+                let existingTestSuite = trivetResults.testSuiteResults.find((ts) => ts.id === testSuite.id);
+                if (existingTestSuite == null) {
+                    existingTestSuite = {
+                        id: testSuite.id,
+                        testGraph: testSuite.testGraph,
+                        validationGraph: testSuite.validationGraph,
+                        name: testSuite.name ?? 'Test',
+                        description: testSuite.description ?? 'It should pass.',
+                        testCaseResults: [],
+                        passing: false,
+                    };
+                    trivetResults.testSuiteResults.push(existingTestSuite);
+                }
+                existingTestSuite.testCaseResults = testCaseResults.slice();
+                existingTestSuite.passing = testCaseResults.every((r) => r.passing);
+                onUpdate?.(cloneDeep(trivetResults));
+            }
+        }
+    }
+    return trivetResults;
+}

package/dist/esm/index.js

@@ -0,0 +1,5 @@
+export * from './trivetTypes.js';
+export * from './serialization/serialization_v1.js';
+export * from './api.js';
+export * from './validateValidationGraphFormat.js';
+export * from './validateTestCaseFormat.js';

package/dist/esm/serialization/serialization_v1.js

@@ -0,0 +1,56 @@
+import * as yaml from 'yaml';
+import {} from '../index.js';
+export function serializeTestCase(testCase) {
+    return {
+        id: testCase.id,
+        input: testCase.input,
+        expectedOutput: testCase.expectedOutput,
+    };
+}
+export function deserializeTestCase(data) {
+    return {
+        id: data.id,
+        input: data.input,
+        expectedOutput: data.expectedOutput,
+    };
+}
+export function serializeTestSuite(testSuite) {
+    return {
+        id: testSuite.id,
+        name: testSuite.name,
+        description: testSuite.description,
+        testGraph: testSuite.testGraph,
+        validationGraph: testSuite.validationGraph,
+        testCases: testSuite.testCases.map((testCase) => serializeTestCase(testCase)),
+    };
+}
+export function deserializeTestSuite(data) {
+    return {
+        id: data.id,
+        name: data.name,
+        description: data.description,
+        testGraph: data.testGraph,
+        validationGraph: data.validationGraph,
+        testCases: data.testCases.map((testCase) => deserializeTestCase(testCase)),
+    };
+}
+export function serializeTrivetData(data) {
+    return {
+        version: 1,
+        testSuites: data.testSuites.map((testSuite) => serializeTestSuite(testSuite)),
+    };
+}
+export function deserializeTrivetData(data) {
+    if (data.version !== 1) {
+        throw new Error(`Unsupported version: ${data.version}`);
+    }
+    return {
+        testSuites: data.testSuites.map((testSuite) => deserializeTestSuite(testSuite)),
+    };
+}
+export function serializeTrivetDataToString(data) {
+    return yaml.stringify(serializeTrivetData(data), null, 2);
+}
+export function deserializeTrivetDataFromString(data) {
+    return deserializeTrivetData(yaml.parse(data));
+}

package/dist/esm/trivetTypes.js

@@ -0,0 +1 @@
+import {} from '@valerypopoff/rivet2-core';

package/dist/esm/validateTestCaseFormat.js

@@ -0,0 +1,17 @@
+import {} from '@valerypopoff/rivet2-core';
+import {} from './trivetTypes.js';
+export function validateTestCaseFormat(testGraph, testCase) {
+    const inputNodes = testGraph.nodes.filter((n) => n.type === 'graphInput');
+    const outputNodes = testGraph.nodes.filter((n) => n.type === 'graphOutput');
+    const inputNodeIds = inputNodes.map((n) => n.data.id);
+    const outputNodeIds = outputNodes.map((n) => n.data.id);
+    const tcInputIds = new Set(Object.keys(testCase.input));
+    const missingInputIds = inputNodeIds.filter((id) => !tcInputIds.has(id));
+    const tcOutputIds = new Set(Object.keys(testCase.expectedOutput));
+    const missingOutputIds = outputNodeIds.filter((id) => !tcOutputIds.has(id));
+    return {
+        valid: missingInputIds.length === 0 && missingOutputIds.length === 0,
+        missingInputIds,
+        missingOutputIds,
+    };
+}

package/dist/esm/validateValidationGraphFormat.js

@@ -0,0 +1,33 @@
+import { keyBy } from 'lodash-es';
+import {} from '@valerypopoff/rivet2-core';
+export function validateValidationGraphFormat(validationGraph) {
+    const inputNodes = validationGraph.nodes.filter((n) => n.type === 'graphInput');
+    const inputNodesById = keyBy(inputNodes, (n) => n.data.id);
+    const validationResult = {
+        valid: true,
+        errorMessages: [],
+    };
+    if (inputNodesById.input == null) {
+        validationResult.valid = false;
+        validationResult.errorMessages.push('Must have an input node with id "input"');
+    }
+    if (inputNodesById.expectedOutput == null) {
+        validationResult.valid = false;
+        validationResult.errorMessages.push('Must have an input node with id "expectedOutput"');
+    }
+    if (inputNodesById.output == null) {
+        validationResult.valid = false;
+        validationResult.errorMessages.push('Must have an input node with id "output"');
+    }
+    const outputNodes = validationGraph.nodes.filter((n) => n.type === 'graphOutput');
+    if (outputNodes.length === 0) {
+        validationResult.valid = false;
+        validationResult.errorMessages.push('Must have at least one output node (that returns true/false as string or boolean)');
+    }
+    const invalidOutputs = outputNodes.filter((n) => n.data.dataType !== 'boolean' && n.data.dataType !== 'string' && n.data.dataType !== 'any');
+    if (invalidOutputs.length > 0) {
+        validationResult.valid = false;
+        validationResult.errorMessages.push(`Output nodes must return boolean or string, but found ${invalidOutputs.map((n) => n.data.dataType).join(', ')}`);
+    }
+    return validationResult;
+}

package/dist/types/api.d.ts

@@ -0,0 +1,16 @@
+import { type NativeApi, type BaseDir, type ReadDirOptions } from '@valerypopoff/rivet2-core';
+import { type TrivetGraphRunner, type TrivetOpts, type TrivetResults } from './trivetTypes.js';
+export declare class DummyNativeApi implements NativeApi {
+    readdir(path: string, baseDir?: BaseDir | undefined, options?: ReadDirOptions | undefined): Promise<string[]>;
+    readTextFile(path: string, baseDir?: BaseDir | undefined): Promise<string>;
+    readBinaryFile(path: string, baseDir?: BaseDir | undefined): Promise<Blob>;
+    writeTextFile(path: string, data: string, baseDir?: BaseDir | undefined): Promise<void>;
+    exec(command: string, args: string[], options?: {
+        cwd?: string | undefined;
+    } | undefined): Promise<void>;
+}
+export declare function createTestGraphRunner(opts: {
+    openAiKey: string;
+    executor?: 'nodejs' | 'browser';
+}): TrivetGraphRunner;
+export declare function runTrivet(opts: TrivetOpts): Promise<TrivetResults>;

package/dist/types/index.d.ts

@@ -0,0 +1,5 @@
+export * from './trivetTypes.js';
+export * from './serialization/serialization_v1.js';
+export * from './api.js';
+export * from './validateValidationGraphFormat.js';
+export * from './validateTestCaseFormat.js';

package/dist/types/serialization/serialization_v1.d.ts

@@ -0,0 +1,26 @@
+import { type TrivetData, type TrivetTestCase, type TrivetTestSuite } from '../index.js';
+export interface SerializedTrivetTestSuite {
+    id: string;
+    name?: string;
+    description?: string;
+    testGraph: string;
+    validationGraph: string;
+    testCases: SerializedTrivetTestCase[];
+}
+export interface SerializedTrivetTestCase {
+    id: string;
+    input: Record<string, unknown>;
+    expectedOutput: Record<string, unknown>;
+}
+export interface SerializedTrivetData {
+    version: 1;
+    testSuites: SerializedTrivetTestSuite[];
+}
+export declare function serializeTestCase(testCase: TrivetTestCase): SerializedTrivetTestCase;
+export declare function deserializeTestCase(data: SerializedTrivetTestCase): TrivetTestCase;
+export declare function serializeTestSuite(testSuite: TrivetTestSuite): SerializedTrivetTestSuite;
+export declare function deserializeTestSuite(data: SerializedTrivetTestSuite): TrivetTestSuite;
+export declare function serializeTrivetData(data: TrivetData): SerializedTrivetData;
+export declare function deserializeTrivetData(data: SerializedTrivetData): TrivetData;
+export declare function serializeTrivetDataToString(data: TrivetData): string;
+export declare function deserializeTrivetDataFromString(data: string): TrivetData;

package/dist/types/trivetTypes.d.ts

@@ -0,0 +1,49 @@
+import { type GraphId, type GraphInputs, type GraphOutputs, type Project } from '@valerypopoff/rivet2-core';
+export type TrivetGraphRunner = (project: Project, graphId: GraphId, inputs: GraphInputs) => Promise<GraphOutputs>;
+export interface TrivetOpts {
+    project: Project;
+    testSuites: TrivetTestSuite[];
+    /** Runs each test in each suite N times. Defaults to just 1. A test passes if all iterations pass. */
+    iterationCount?: number;
+    runGraph: TrivetGraphRunner;
+    onUpdate?: (results: TrivetResults) => void;
+}
+export interface TrivetTestSuite {
+    id: string;
+    name?: string;
+    description?: string;
+    testGraph: string;
+    validationGraph: string;
+    testCases: TrivetTestCase[];
+}
+export interface TrivetTestCase {
+    id: string;
+    input: Record<string, unknown>;
+    expectedOutput: Record<string, unknown>;
+}
+export interface TrivetResults {
+    testSuiteResults: TrivetTestSuiteResult[];
+    iterationCount: number;
+}
+export interface TrivetTestSuiteResult {
+    id: string;
+    testGraph: string;
+    validationGraph: string;
+    name: string;
+    description: string;
+    testCaseResults: TrivetTestCaseResult[];
+    passing: boolean;
+}
+export interface TrivetTestCaseResult {
+    id: string;
+    iteration: number;
+    passing: boolean;
+    message: string;
+    outputs: Record<string, unknown>;
+    duration: number;
+    cost: number;
+    error?: Error | string | any;
+}
+export type TrivetData = {
+    testSuites: TrivetTestSuite[];
+};

package/dist/types/validateTestCaseFormat.d.ts

@@ -0,0 +1,7 @@
+import { type NodeGraph } from '@valerypopoff/rivet2-core';
+import { type TrivetTestCase } from './trivetTypes.js';
+export declare function validateTestCaseFormat(testGraph: NodeGraph, testCase: TrivetTestCase): {
+    valid: boolean;
+    missingInputIds: string[];
+    missingOutputIds: string[];
+};

package/dist/types/validateValidationGraphFormat.d.ts

@@ -0,0 +1,5 @@
+import { type NodeGraph } from '@valerypopoff/rivet2-core';
+export declare function validateValidationGraphFormat(validationGraph: NodeGraph): {
+    valid: boolean;
+    errorMessages: string[];
+};

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@valerypopoff/trivet",
+  "version": "2.0.1",
+  "license": "MIT",
+  "main": "dist/cjs/bundle.cjs",
+  "module": "dist/esm/index.js",
+  "types": "dist/types/index.d.ts",
+  "type": "module",
+  "exports": {
+    "node": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/cjs/bundle.cjs"
+    },
+    "browser": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/cjs/bundle.cjs"
+    },
+    "default": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/cjs/bundle.cjs"
+    }
+  },
+  "files": [
+    "dist/cjs/**",
+    "dist/esm/**",
+    "dist/types/**"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/valerypopoff/rivet2.0.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@valerypopoff/rivet2-core": "^2.0.1",
+    "lodash-es": "^4.17.21",
+    "yaml": "^2.3.3"
+  }
+}