@nirmos/sdk 0.3.3 → 0.3.5

package/examples/evaluation-quickstart.ts

@@ -0,0 +1,255 @@
+import { Nirmos } from "@nirmos/sdk";
+
+declare const process: { env: Record<string, string | undefined> };
+
+function requireId(value: unknown, label: string): string {
+  if (typeof value !== "string" || !value) throw new Error(`Missing ${label}`);
+  return value;
+}
+
+export async function evaluationQuickstart(
+  client: Nirmos,
+  options: {
+    providerId: string;
+    modelId: string;
+    cleanup?: boolean;
+    pollMs?: number;
+    maxPolls?: number;
+  },
+) {
+  let datasetId: string | undefined;
+  let suiteId: string | undefined;
+  let evaluatorId: string | undefined;
+  let runId: string | undefined;
+  let terminal = false;
+  let submissionStarted = false;
+  try {
+    const dataset = await client.evaluations.datasets.create({
+      name: "SDK two-case demo",
+    });
+    datasetId = requireId(dataset.dataset.id, "dataset ID");
+    const datasetVersionId = requireId(
+      dataset.initialVersion?.id,
+      "dataset version ID",
+    );
+    console.log({ datasetId, datasetVersionId });
+    for (const column of [
+      { key: "question", name: "Question", role: "input" as const },
+      { key: "expected", name: "Expected", role: "expected" as const },
+      { key: "topic", name: "Topic", role: "metadata" as const },
+    ]) {
+      await client.evaluations.datasets.columns.create(datasetId, {
+        datasetVersionId,
+        ...column,
+        dataType: "text",
+        required: true,
+      });
+    }
+    for (const data of [
+      {
+        question: "Reply with exactly Paris and nothing else.",
+        expected: "Paris",
+        topic: "cities",
+      },
+      {
+        question: "Reply with exactly Tokyo and nothing else.",
+        expected: "Tokyo",
+        topic: "cities",
+      },
+    ]) {
+      await client.evaluations.datasets.cases.create(datasetId, {
+        datasetVersionId,
+        data,
+        tags: ["sdk-demo"],
+      });
+    }
+    await client.evaluations.datasets.versions.freeze(
+      datasetId,
+      datasetVersionId,
+    );
+    const suite = await client.evaluations.suites.create({
+      name: "SDK exact-match demo",
+      datasetVersionId,
+    });
+    suiteId = requireId(suite.id, "suite ID");
+    console.log({ suiteId });
+    const target = await client.evaluations.columns.create({
+      suiteId,
+      key: "candidate",
+      name: "Candidate",
+      category: "target",
+      enabled: true,
+      config: { role: "candidate", dataType: "output" },
+      target: {
+        type: "model",
+        name: "Demo model",
+        providerId: options.providerId,
+        modelId: options.modelId,
+        inputMapping: { user_message: "dataset.question" },
+      },
+    });
+    // The target's logical ID is distinct from the immutable version used in runs.
+    const targetId = requireId(target.target?.id, "target ID");
+    const targetVersionId = requireId(
+      target.targetVersion?.id,
+      "target version ID",
+    );
+    const evaluator = await client.evaluations.evaluators.create({
+      name: "SDK exact match",
+      version: {
+        type: "exact_match",
+        category: "deterministic",
+        scope: "output",
+        inputSchema: [
+          { key: "actual", type: "any", required: true },
+          { key: "expected", type: "any", required: true },
+        ],
+        config: {
+          caseSensitive: true,
+          trimWhitespace: true,
+          normalizeWhitespace: false,
+        },
+        outputType: "boolean",
+        scoreDirection: "none",
+      },
+    });
+    evaluatorId = requireId(evaluator.evaluator.id, "evaluator ID");
+    const evaluatorVersionId = requireId(
+      evaluator.version?.id,
+      "evaluator version ID",
+    );
+    console.log({ evaluatorId, targetVersionId, evaluatorVersionId });
+    await client.evaluations.columns.create({
+      suiteId,
+      key: "exact_match",
+      name: "Exact match",
+      category: "evaluator",
+      enabled: true,
+      evaluatorVersionId,
+      config: {
+        inputMapping: {
+          actual: { source: "target", path: "output" },
+          expected: { source: "dataset", key: "expected" },
+        },
+      },
+    });
+    const experiment = await client.evaluations.experiments.prepare({
+      suiteId,
+      datasetVersionId,
+      targetVersionIds: [targetVersionId],
+      evaluatorVersionIds: [evaluatorVersionId],
+      trialCount: 1,
+    });
+    const experimentId = requireId(experiment.id, "experiment ID");
+    // Persist this input and submissionId if implementing replay across process restarts.
+    const submissionId = crypto.randomUUID();
+    console.log({ experimentId, submissionId });
+    submissionStarted = true;
+    const created = await client.evaluations.runs.create(
+      {
+        experimentId,
+        cases: "all",
+        targetIds: [targetVersionId],
+        evaluatorIds: [evaluatorVersionId],
+        trialCount: 1,
+      },
+      { idempotencyKey: submissionId, maxRetries: 0 },
+    );
+    runId = created.id;
+    console.log({ runId });
+    for (let attempt = 0; attempt < (options.maxPolls ?? 60); attempt++) {
+      const run = await client.evaluations.runs.get(runId, {
+        timeoutMs: 10_000,
+        maxRetries: 0,
+      });
+      if (["COMPLETED", "FAILED", "CANCELLED"].includes(run.status)) {
+        terminal = true;
+        const results = await client.evaluations.runs.results(runId, {
+          page: 1,
+          pageSize: 25,
+        });
+        for (const trial of results.data) {
+          console.log({
+            caseId: trial.caseId,
+            trialStatus: trial.status,
+            error: trial.error,
+          });
+          for (const score of trial.scores) {
+            console.log({
+              evaluatorVersionId: score.evaluatorVersionId,
+              status: score.status,
+              value: score.value,
+              passed: score.passed,
+              error: score.error,
+            });
+          }
+        }
+        // Two cases × one target × one trial fit on one page.
+        if (run.status !== "COMPLETED")
+          throw new Error(`Run ${runId}: ${run.status}`);
+        if (
+          results.data.length !== 2 ||
+          results.data.some(
+            (trial) =>
+              trial.status !== "COMPLETED" ||
+              trial.scores.length !== 1 ||
+              trial.scores.some(
+                (score) => score.status !== "COMPLETED" || score.error,
+              ),
+          )
+        ) {
+          throw new Error(
+            "Run ended with missing or failed trials/scores; inspect results",
+          );
+        }
+        // Runtime GET returns persisted aggregates after terminal processing.
+        const summary = run.summary;
+        if (!summary || typeof summary.overallPassRate !== "number")
+          throw new Error("Missing aggregate scores; inspect the terminal run");
+        console.log({ overallPassRate: summary.overallPassRate, byEvaluator: summary.byEvaluator });
+        return { runId, results, summary }; // passed:false is a valid evaluation outcome.
+      }
+      await new Promise((resolve) =>
+        setTimeout(resolve, options.pollMs ?? 2000),
+      );
+    }
+    await client.evaluations.runs.cancel(runId);
+    throw new Error(
+      `Polling limit reached; cancellation requested for ${runId}. Confirm terminal state before cleanup.`,
+    );
+  } finally {
+    // Never archive inputs while background execution might still use them.
+    if (options.cleanup && (!submissionStarted || terminal)) {
+      const cleanup = await Promise.allSettled([
+        ...(suiteId ? [client.evaluations.suites.archive(suiteId)] : []),
+        ...(evaluatorId
+          ? [client.evaluations.evaluators.archive(evaluatorId)]
+          : []),
+        ...(datasetId ? [client.evaluations.datasets.archive(datasetId)] : []),
+      ]);
+      for (const result of cleanup)
+        if (result.status === "rejected")
+          console.error("Archive failed:", result.reason);
+    }
+  }
+}
+
+if (process.env.NIRMOS_RUN_EVALUATION_EXAMPLE === "1") {
+  const key = requireId(process.env.NIRMOS_API_KEY, "NIRMOS_API_KEY");
+  await evaluationQuickstart(
+    new Nirmos({
+      auth: { type: "api-key", key },
+      apiBaseUrl: process.env.NIRMOS_API_URL,
+      runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
+      maxRetries: 0,
+    }),
+    {
+      providerId: requireId(
+        process.env.NIRMOS_PROVIDER_ID,
+        "NIRMOS_PROVIDER_ID",
+      ),
+      modelId: requireId(process.env.NIRMOS_MODEL_ID, "NIRMOS_MODEL_ID"),
+      cleanup: process.env.EVALUATION_CLEANUP === "1",
+    },
+  );
+}

package/examples/evaluation-trace-import.mjs

@@ -0,0 +1,81 @@
+// Generated from evaluation-trace-import.ts. Do not edit.
+import { APIError, Nirmos } from "@nirmos/sdk";
+/** Destination draft needs required JSON columns named input and output. */
+export async function importGatewayTrace(client, options) {
+    const response = await client.gateway.chat.create({
+        model: options.model,
+        messages: [{ role: "user", content: "What is the capital of France?" }],
+    }, { maxRetries: 0 });
+    const requestId = response.metadata.requestId;
+    if (!requestId)
+        throw new Error("Gateway response omitted public request ID");
+    const deadline = Date.now() + (options.maxWaitMs ?? 30_000);
+    while (true) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0)
+            throw new Error(`Trace ${requestId} unavailable after bounded wait; check logging and retention`);
+        try {
+            // Logs API accepts the same public request ID. Existence confirms persistence,
+            // not eligibility: import performs the authoritative payload/operation checks.
+            await client.logs.get(requestId, {
+                timeoutMs: Math.min(5_000, remaining),
+                maxRetries: 0,
+            });
+            break;
+        }
+        catch (error) {
+            if (!(error instanceof APIError) || error.status !== 404)
+                throw error;
+            await new Promise((resolve) => setTimeout(resolve, Math.min(options.pollMs ?? 1_000, Math.max(0, deadline - Date.now()))));
+        }
+    }
+    const imported = await client.evaluations.datasets.importTrace({
+        requestLogId: requestId,
+        datasetVersionId: options.datasetVersionId,
+        mapping: { input: "input", output: "output" },
+        tags: ["gateway-regression"],
+    }, { maxRetries: 0 });
+    const datasetCase = await client.evaluations.datasets.cases.get(imported.case.id);
+    return { requestId, imported, datasetCase };
+}
+if (process.env.NIRMOS_RUN_TRACE_EXAMPLE === "1") {
+    const apiKey = process.env.NIRMOS_API_KEY;
+    const model = process.env.NIRMOS_CHAT_MODEL;
+    if (!apiKey || !model)
+        throw new Error("Set NIRMOS_API_KEY and NIRMOS_CHAT_MODEL");
+    const client = new Nirmos({
+        auth: { type: "api-key", key: apiKey },
+        apiBaseUrl: process.env.NIRMOS_API_URL,
+        runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
+        maxRetries: 0,
+    });
+    const dataset = await client.evaluations.datasets.create({
+        name: "Gateway trace demo",
+    });
+    try {
+        for (const key of ["input", "output"]) {
+            await client.evaluations.datasets.columns.create(dataset.dataset.id, {
+                datasetVersionId: dataset.initialVersion.id,
+                key,
+                name: key,
+                dataType: "json",
+                role: key === "input" ? "input" : "expected",
+                required: true,
+            });
+        }
+        const result = await importGatewayTrace(client, {
+            datasetVersionId: dataset.initialVersion.id,
+            model,
+        });
+        console.log(result.requestId, result.imported.case.id, result.datasetCase.values);
+    }
+    catch (error) {
+        if (error instanceof APIError)
+            console.error(error.code, error.message);
+        throw error;
+    }
+    finally {
+        if (process.env.EVALUATION_CLEANUP === "1")
+            await client.evaluations.datasets.archive(dataset.dataset.id);
+    }
+}

package/examples/evaluation-trace-import.ts

@@ -0,0 +1,105 @@
+import { APIError, Nirmos } from "@nirmos/sdk";
+
+declare const process: { env: Record<string, string | undefined> };
+
+/** Destination draft needs required JSON columns named input and output. */
+export async function importGatewayTrace(
+  client: Nirmos,
+  options: {
+    datasetVersionId: string;
+    model: string;
+    maxWaitMs?: number;
+    pollMs?: number;
+  },
+) {
+  const response = await client.gateway.chat.create(
+    {
+      model: options.model,
+      messages: [{ role: "user", content: "What is the capital of France?" }],
+    },
+    { maxRetries: 0 },
+  );
+  const requestId = response.metadata.requestId;
+  if (!requestId) throw new Error("Gateway response omitted public request ID");
+  const deadline = Date.now() + (options.maxWaitMs ?? 30_000);
+  while (true) {
+    const remaining = deadline - Date.now();
+    if (remaining <= 0)
+      throw new Error(
+        `Trace ${requestId} unavailable after bounded wait; check logging and retention`,
+      );
+    try {
+      // Logs API accepts the same public request ID. Existence confirms persistence,
+      // not eligibility: import performs the authoritative payload/operation checks.
+      await client.logs.get(requestId, {
+        timeoutMs: Math.min(5_000, remaining),
+        maxRetries: 0,
+      });
+      break;
+    } catch (error) {
+      if (!(error instanceof APIError) || error.status !== 404) throw error;
+      await new Promise((resolve) =>
+        setTimeout(
+          resolve,
+          Math.min(options.pollMs ?? 1_000, Math.max(0, deadline - Date.now())),
+        ),
+      );
+    }
+  }
+  const imported = await client.evaluations.datasets.importTrace(
+    {
+      requestLogId: requestId,
+      datasetVersionId: options.datasetVersionId,
+      mapping: { input: "input", output: "output" },
+      tags: ["gateway-regression"],
+    },
+    { maxRetries: 0 },
+  );
+  const datasetCase = await client.evaluations.datasets.cases.get(
+    imported.case.id,
+  );
+  return { requestId, imported, datasetCase };
+}
+
+if (process.env.NIRMOS_RUN_TRACE_EXAMPLE === "1") {
+  const apiKey = process.env.NIRMOS_API_KEY;
+  const model = process.env.NIRMOS_CHAT_MODEL;
+  if (!apiKey || !model)
+    throw new Error("Set NIRMOS_API_KEY and NIRMOS_CHAT_MODEL");
+  const client = new Nirmos({
+    auth: { type: "api-key", key: apiKey },
+    apiBaseUrl: process.env.NIRMOS_API_URL,
+    runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
+    maxRetries: 0,
+  });
+  const dataset = await client.evaluations.datasets.create({
+    name: "Gateway trace demo",
+  });
+  try {
+    for (const key of ["input", "output"]) {
+      await client.evaluations.datasets.columns.create(dataset.dataset.id, {
+        datasetVersionId: dataset.initialVersion.id,
+        key,
+        name: key,
+        dataType: "json",
+        role: key === "input" ? "input" : "expected",
+        required: true,
+      });
+    }
+    const result = await importGatewayTrace(client, {
+      datasetVersionId: dataset.initialVersion.id,
+      model,
+    });
+    console.log(
+      result.requestId,
+      result.imported.case.id,
+      result.datasetCase.values,
+    );
+  } catch (error) {
+    if (error instanceof APIError) console.error(error.code, error.message);
+    throw error;
+  } finally {
+    if (process.env.EVALUATION_CLEANUP === "1")
+      await client.evaluations.datasets.archive(dataset.dataset.id);
+  }
+}

package/examples/tsconfig.json

@@ -1,5 +1,5 @@
 {
   "extends": "../tsconfig.json",
   "compilerOptions": { "noEmit": true, "rootDir": ".." },
-  "include": ["memory-quickstart.ts"]
+  "include": ["*.ts"]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nirmos/sdk",
-  "version": "0.3.3",
+  "version": "0.3.5",
   "description": "Official TypeScript SDK for the Nirmos AI platform.",
   "license": "MIT",
   "type": "module",

package/src/core/version.ts

@@ -1,2 +1,2 @@
 // Generated from package.json by scripts/generate-version.mjs.
-export const SDK_VERSION = "0.3.3";
+export const SDK_VERSION = "0.3.5";

package/src/evaluations/types.ts

@@ -0,0 +1,18 @@
+import type { Requests } from "../management/contracts.js";
+
+/** Built-in evaluator versions are discriminated by type. Configs follow the API schemas. */
+export type EvaluationEvaluatorVersion = Requests["createEvaluatorVersion"];
+export type EvaluationTargetConfig = NonNullable<
+  Requests["createEvaluationColumn"]["target"]
+>;
+export type EvaluationColumnCreate = Requests["createEvaluationColumn"];
+export type EvaluationInputBinding = NonNullable<
+  NonNullable<EvaluationColumnCreate["config"]>["inputMapping"]
+>[string];
+export type EvaluationSegmentCriteria = Requests["createSegment"]["criteria"];
+
+export type EvaluationEvaluatorType = EvaluationEvaluatorVersion["type"];
+export type EvaluationEvaluatorConfig<T extends EvaluationEvaluatorType> =
+  Extract<EvaluationEvaluatorVersion, { type: T }>["config"];
+export type EvaluationExpression =
+  EvaluationEvaluatorConfig<"custom_expression">["expression"];

package/src/index.ts

@@ -150,3 +150,14 @@ export {
   validatePromptVariableValues,
   validatePromptSettings,
 } from "./prompts/variables.js";
+
+export type {
+  EvaluationEvaluatorVersion,
+  EvaluationEvaluatorType,
+  EvaluationEvaluatorConfig,
+  EvaluationExpression,
+  EvaluationTargetConfig,
+  EvaluationColumnCreate,
+  EvaluationInputBinding,
+  EvaluationSegmentCriteria,
+} from "./evaluations/types.js";