@indexnetwork/protocol 0.3.1 → 0.3.2

package/dist/support/tests/llm-assert.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Assert that `output` satisfies the given `criteria` according to an LLM judge.
+ * Throws an error (with reasoning embedded) if the assertion fails.
+ * Uses the SMARTEST_VERIFIER_MODEL env var (default: google/gemini-2.5-flash).
+ *
+ * @param output - The value produced by the system under test.
+ * @param criteria - Natural language description of what the output must satisfy.
+ * @throws {Error} If the LLM judge determines the output does not meet the criteria.
+ */
+export declare function assertLLM(output: unknown, criteria: string): Promise<void>;
+//# sourceMappingURL=llm-assert.d.ts.map

package/dist/support/tests/llm-assert.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-assert.d.ts","sourceRoot":"","sources":["../../../src/support/tests/llm-assert.ts"],"names":[],"mappings":"AAeA;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhF"}

package/dist/support/tests/llm-assert.js

@@ -0,0 +1,43 @@
+import { ChatOpenAI } from "@langchain/openai";
+import { HumanMessage, SystemMessage } from "@langchain/core/messages";
+import { z } from "zod";
+const JUDGE_SYSTEM_PROMPT = `You are a test oracle for an AI system. Given the output of a system under test and evaluation criteria, determine whether the output passes or fails.
+
+Return JSON with two fields:
+- pass: true if the output satisfies the criteria, false otherwise
+- reasoning: concise explanation of your judgment (1-3 sentences)`;
+const judgeOutputSchema = z.object({
+    pass: z.boolean(),
+    reasoning: z.string(),
+});
+/**
+ * Assert that `output` satisfies the given `criteria` according to an LLM judge.
+ * Throws an error (with reasoning embedded) if the assertion fails.
+ * Uses the SMARTEST_VERIFIER_MODEL env var (default: google/gemini-2.5-flash).
+ *
+ * @param output - The value produced by the system under test.
+ * @param criteria - Natural language description of what the output must satisfy.
+ * @throws {Error} If the LLM judge determines the output does not meet the criteria.
+ */
+export async function assertLLM(output, criteria) {
+    const modelId = process.env.SMARTEST_VERIFIER_MODEL ?? "google/gemini-2.5-flash";
+    const model = new ChatOpenAI({
+        model: modelId,
+        apiKey: process.env.OPENROUTER_API_KEY,
+        configuration: {
+            baseURL: process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1",
+        },
+        temperature: 0,
+        maxTokens: 512,
+    });
+    const structured = model.withStructuredOutput(judgeOutputSchema, { name: "llm_judge" });
+    const userMessage = `Output:\n${JSON.stringify(output, null, 2)}\n\nCriteria:\n${criteria}`;
+    const result = await structured.invoke([
+        new SystemMessage(JUDGE_SYSTEM_PROMPT),
+        new HumanMessage(userMessage),
+    ]);
+    if (!result.pass) {
+        throw new Error(`LLM assertion failed: ${result.reasoning}`);
+    }
+}
+//# sourceMappingURL=llm-assert.js.map

package/dist/support/tests/llm-assert.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-assert.js","sourceRoot":"","sources":["../../../src/support/tests/llm-assert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,mBAAmB,GAAG;;;;kEAIsC,CAAC;AAEnE,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACjB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAe,EAAE,QAAgB;IAC/D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,yBAAyB,CAAC;IAEjF,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QAC3B,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAmB;QACvC,aAAa,EAAE;YACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,8BAA8B;SAC3E;QACD,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,GAAG;KACf,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,KAAK,CAAC,oBAAoB,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAExF,MAAM,WAAW,GAAG,YAAY,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,kBAAkB,QAAQ,EAAE,CAAC;IAE5F,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC;QACrC,IAAI,aAAa,CAAC,mBAAmB,CAAC;QACtC,IAAI,YAAY,CAAC,WAAW,CAAC;KAC9B,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indexnetwork/protocol",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -14,6 +14,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
+    "test": "bun test",
     "prepublishOnly": "bun run build"
   },
   "dependencies": {