@nirmos/sdk 0.3.2 → 0.3.4

package/examples/evaluation-quickstart.ts

@@ -0,0 +1,250 @@
+import { Nirmos } from "@nirmos/sdk";
+
+declare const process: { env: Record<string, string | undefined> };
+
+function requireId(value: unknown, label: string): string {
+  if (typeof value !== "string" || !value) throw new Error(`Missing ${label}`);
+  return value;
+}
+
+export async function evaluationQuickstart(
+  client: Nirmos,
+  options: {
+    providerId: string;
+    modelId: string;
+    cleanup?: boolean;
+    pollMs?: number;
+    maxPolls?: number;
+  },
+) {
+  let datasetId: string | undefined;
+  let suiteId: string | undefined;
+  let evaluatorId: string | undefined;
+  let runId: string | undefined;
+  let terminal = false;
+  let submissionStarted = false;
+  try {
+    const dataset = await client.evaluations.datasets.create({
+      name: "SDK two-case demo",
+    });
+    datasetId = requireId(dataset.dataset.id, "dataset ID");
+    const datasetVersionId = requireId(
+      dataset.initialVersion?.id,
+      "dataset version ID",
+    );
+    console.log({ datasetId, datasetVersionId });
+    for (const column of [
+      { key: "question", name: "Question", role: "input" as const },
+      { key: "expected", name: "Expected", role: "expected" as const },
+      { key: "topic", name: "Topic", role: "metadata" as const },
+    ]) {
+      await client.evaluations.datasets.columns.create(datasetId, {
+        datasetVersionId,
+        ...column,
+        dataType: "text",
+        required: true,
+      });
+    }
+    for (const data of [
+      {
+        question: "Reply with exactly Paris and nothing else.",
+        expected: "Paris",
+        topic: "cities",
+      },
+      {
+        question: "Reply with exactly Tokyo and nothing else.",
+        expected: "Tokyo",
+        topic: "cities",
+      },
+    ]) {
+      await client.evaluations.datasets.cases.create(datasetId, {
+        datasetVersionId,
+        data,
+        tags: ["sdk-demo"],
+      });
+    }
+    await client.evaluations.datasets.versions.freeze(
+      datasetId,
+      datasetVersionId,
+    );
+    const suite = await client.evaluations.suites.create({
+      name: "SDK exact-match demo",
+      datasetVersionId,
+    });
+    suiteId = requireId(suite.id, "suite ID");
+    console.log({ suiteId });
+    const target = await client.evaluations.columns.create({
+      suiteId,
+      key: "candidate",
+      name: "Candidate",
+      category: "target",
+      enabled: true,
+      config: { role: "candidate", dataType: "output" },
+      target: {
+        type: "model",
+        name: "Demo model",
+        providerId: options.providerId,
+        modelId: options.modelId,
+        inputMapping: { user_message: "dataset.question" },
+      },
+    });
+    // The target's logical ID is distinct from the immutable version used in runs.
+    const targetId = requireId(target.target?.id, "target ID");
+    const targetVersionId = requireId(
+      target.targetVersion?.id,
+      "target version ID",
+    );
+    const evaluator = await client.evaluations.evaluators.create({
+      name: "SDK exact match",
+      version: {
+        type: "exact_match",
+        category: "deterministic",
+        scope: "output",
+        inputSchema: [
+          { key: "actual", type: "any", required: true },
+          { key: "expected", type: "any", required: true },
+        ],
+        config: {
+          caseSensitive: true,
+          trimWhitespace: true,
+          normalizeWhitespace: false,
+        },
+        outputType: "boolean",
+        scoreDirection: "none",
+      },
+    });
+    evaluatorId = requireId(evaluator.evaluator.id, "evaluator ID");
+    const evaluatorVersionId = requireId(
+      evaluator.version?.id,
+      "evaluator version ID",
+    );
+    console.log({ evaluatorId, targetVersionId, evaluatorVersionId });
+    await client.evaluations.columns.create({
+      suiteId,
+      key: "exact_match",
+      name: "Exact match",
+      category: "evaluator",
+      enabled: true,
+      evaluatorVersionId,
+      config: {
+        inputMapping: {
+          actual: { source: "target", path: "output" },
+          expected: { source: "dataset", key: "expected" },
+        },
+      },
+    });
+    const experiment = await client.evaluations.experiments.prepare({
+      suiteId,
+      datasetVersionId,
+      targetVersionIds: [targetVersionId],
+      evaluatorVersionIds: [evaluatorVersionId],
+      trialCount: 1,
+    });
+    const experimentId = requireId(experiment.id, "experiment ID");
+    // Persist this input and submissionId if implementing replay across process restarts.
+    const submissionId = crypto.randomUUID();
+    console.log({ experimentId, submissionId });
+    submissionStarted = true;
+    const created = await client.evaluations.runs.create(
+      {
+        experimentId,
+        cases: "all",
+        targetIds: [targetVersionId],
+        evaluatorIds: [evaluatorVersionId],
+        trialCount: 1,
+      },
+      { idempotencyKey: submissionId, maxRetries: 0 },
+    );
+    runId = created.id;
+    console.log({ runId });
+    for (let attempt = 0; attempt < (options.maxPolls ?? 60); attempt++) {
+      const run = await client.evaluations.runs.get(runId, {
+        timeoutMs: 10_000,
+        maxRetries: 0,
+      });
+      if (["COMPLETED", "FAILED", "CANCELLED"].includes(run.status)) {
+        terminal = true;
+        const results = await client.evaluations.runs.results(runId, {
+          page: 1,
+          pageSize: 25,
+        });
+        for (const trial of results.data) {
+          console.log({
+            caseId: trial.caseId,
+            trialStatus: trial.status,
+            error: trial.error,
+          });
+          for (const score of trial.scores) {
+            console.log({
+              evaluatorVersionId: score.evaluatorVersionId,
+              status: score.status,
+              value: score.value,
+              passed: score.passed,
+              error: score.error,
+            });
+          }
+        }
+        // Two cases × one target × one trial fit on one page.
+        if (run.status !== "COMPLETED")
+          throw new Error(`Run ${runId}: ${run.status}`);
+        if (
+          results.data.length !== 2 ||
+          results.data.some(
+            (trial) =>
+              trial.status !== "COMPLETED" ||
+              trial.scores.length !== 1 ||
+              trial.scores.some(
+                (score) => score.status !== "COMPLETED" || score.error,
+              ),
+          )
+        ) {
+          throw new Error(
+            "Run ended with missing or failed trials/scores; inspect results",
+          );
+        }
+        return { runId, results }; // passed:false is a valid evaluation outcome.
+      }
+      await new Promise((resolve) =>
+        setTimeout(resolve, options.pollMs ?? 2000),
+      );
+    }
+    await client.evaluations.runs.cancel(runId);
+    throw new Error(
+      `Polling limit reached; cancellation requested for ${runId}. Confirm terminal state before cleanup.`,
+    );
+  } finally {
+    // Never archive inputs while background execution might still use them.
+    if (options.cleanup && (!submissionStarted || terminal)) {
+      const cleanup = await Promise.allSettled([
+        ...(suiteId ? [client.evaluations.suites.archive(suiteId)] : []),
+        ...(evaluatorId
+          ? [client.evaluations.evaluators.archive(evaluatorId)]
+          : []),
+        ...(datasetId ? [client.evaluations.datasets.archive(datasetId)] : []),
+      ]);
+      for (const result of cleanup)
+        if (result.status === "rejected")
+          console.error("Archive failed:", result.reason);
+    }
+  }
+}
+
+if (process.env.NIRMOS_RUN_EVALUATION_EXAMPLE === "1") {
+  const key = requireId(process.env.NIRMOS_API_KEY, "NIRMOS_API_KEY");
+  await evaluationQuickstart(
+    new Nirmos({
+      auth: { type: "api-key", key },
+      apiBaseUrl: process.env.NIRMOS_API_URL,
+      runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
+      maxRetries: 0,
+    }),
+    {
+      providerId: requireId(
+        process.env.NIRMOS_PROVIDER_ID,
+        "NIRMOS_PROVIDER_ID",
+      ),
+      modelId: requireId(process.env.NIRMOS_MODEL_ID, "NIRMOS_MODEL_ID"),
+      cleanup: process.env.EVALUATION_CLEANUP === "1",
+    },
+  );
+}

package/examples/evaluation-trace-import.mjs

@@ -0,0 +1,81 @@
+// Generated from evaluation-trace-import.ts. Do not edit.
+import { APIError, Nirmos } from "@nirmos/sdk";
+/** Destination draft needs required JSON columns named input and output. */
+export async function importGatewayTrace(client, options) {
+    const response = await client.gateway.chat.create({
+        model: options.model,
+        messages: [{ role: "user", content: "What is the capital of France?" }],
+    }, { maxRetries: 0 });
+    const requestId = response.metadata.requestId;
+    if (!requestId)
+        throw new Error("Gateway response omitted public request ID");
+    const deadline = Date.now() + (options.maxWaitMs ?? 30_000);
+    while (true) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0)
+            throw new Error(`Trace ${requestId} unavailable after bounded wait; check logging and retention`);
+        try {
+            // Logs API accepts the same public request ID. Existence confirms persistence,
+            // not eligibility: import performs the authoritative payload/operation checks.
+            await client.logs.get(requestId, {
+                timeoutMs: Math.min(5_000, remaining),
+                maxRetries: 0,
+            });
+            break;
+        }
+        catch (error) {
+            if (!(error instanceof APIError) || error.status !== 404)
+                throw error;
+            await new Promise((resolve) => setTimeout(resolve, Math.min(options.pollMs ?? 1_000, Math.max(0, deadline - Date.now()))));
+        }
+    }
+    const imported = await client.evaluations.datasets.importTrace({
+        requestLogId: requestId,
+        datasetVersionId: options.datasetVersionId,
+        mapping: { input: "input", output: "output" },
+        tags: ["gateway-regression"],
+    }, { maxRetries: 0 });
+    const datasetCase = await client.evaluations.datasets.cases.get(imported.case.id);
+    return { requestId, imported, datasetCase };
+}
+if (process.env.NIRMOS_RUN_TRACE_EXAMPLE === "1") {
+    const apiKey = process.env.NIRMOS_API_KEY;
+    const model = process.env.NIRMOS_CHAT_MODEL;
+    if (!apiKey || !model)
+        throw new Error("Set NIRMOS_API_KEY and NIRMOS_CHAT_MODEL");
+    const client = new Nirmos({
+        auth: { type: "api-key", key: apiKey },
+        apiBaseUrl: process.env.NIRMOS_API_URL,
+        runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
+        maxRetries: 0,
+    });
+    const dataset = await client.evaluations.datasets.create({
+        name: "Gateway trace demo",
+    });
+    try {
+        for (const key of ["input", "output"]) {
+            await client.evaluations.datasets.columns.create(dataset.dataset.id, {
+                datasetVersionId: dataset.initialVersion.id,
+                key,
+                name: key,
+                dataType: "json",
+                role: key === "input" ? "input" : "expected",
+                required: true,
+            });
+        }
+        const result = await importGatewayTrace(client, {
+            datasetVersionId: dataset.initialVersion.id,
+            model,
+        });
+        console.log(result.requestId, result.imported.case.id, result.datasetCase.values);
+    }
+    catch (error) {
+        if (error instanceof APIError)
+            console.error(error.code, error.message);
+        throw error;
+    }
+    finally {
+        if (process.env.EVALUATION_CLEANUP === "1")
+            await client.evaluations.datasets.archive(dataset.dataset.id);
+    }
+}

package/examples/evaluation-trace-import.ts

@@ -0,0 +1,105 @@
+import { APIError, Nirmos } from "@nirmos/sdk";
+
+declare const process: { env: Record<string, string | undefined> };
+
+/** Destination draft needs required JSON columns named input and output. */
+export async function importGatewayTrace(
+  client: Nirmos,
+  options: {
+    datasetVersionId: string;
+    model: string;
+    maxWaitMs?: number;
+    pollMs?: number;
+  },
+) {
+  const response = await client.gateway.chat.create(
+    {
+      model: options.model,
+      messages: [{ role: "user", content: "What is the capital of France?" }],
+    },
+    { maxRetries: 0 },
+  );
+  const requestId = response.metadata.requestId;
+  if (!requestId) throw new Error("Gateway response omitted public request ID");
+  const deadline = Date.now() + (options.maxWaitMs ?? 30_000);
+  while (true) {
+    const remaining = deadline - Date.now();
+    if (remaining <= 0)
+      throw new Error(
+        `Trace ${requestId} unavailable after bounded wait; check logging and retention`,
+      );
+    try {
+      // Logs API accepts the same public request ID. Existence confirms persistence,
+      // not eligibility: import performs the authoritative payload/operation checks.
+      await client.logs.get(requestId, {
+        timeoutMs: Math.min(5_000, remaining),
+        maxRetries: 0,
+      });
+      break;
+    } catch (error) {
+      if (!(error instanceof APIError) || error.status !== 404) throw error;
+      await new Promise((resolve) =>
+        setTimeout(
+          resolve,
+          Math.min(options.pollMs ?? 1_000, Math.max(0, deadline - Date.now())),
+        ),
+      );
+    }
+  }
+  const imported = await client.evaluations.datasets.importTrace(
+    {
+      requestLogId: requestId,
+      datasetVersionId: options.datasetVersionId,
+      mapping: { input: "input", output: "output" },
+      tags: ["gateway-regression"],
+    },
+    { maxRetries: 0 },
+  );
+  const datasetCase = await client.evaluations.datasets.cases.get(
+    imported.case.id,
+  );
+  return { requestId, imported, datasetCase };
+}
+
+if (process.env.NIRMOS_RUN_TRACE_EXAMPLE === "1") {
+  const apiKey = process.env.NIRMOS_API_KEY;
+  const model = process.env.NIRMOS_CHAT_MODEL;
+  if (!apiKey || !model)
+    throw new Error("Set NIRMOS_API_KEY and NIRMOS_CHAT_MODEL");
+  const client = new Nirmos({
+    auth: { type: "api-key", key: apiKey },
+    apiBaseUrl: process.env.NIRMOS_API_URL,
+    runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
+    maxRetries: 0,
+  });
+  const dataset = await client.evaluations.datasets.create({
+    name: "Gateway trace demo",
+  });
+  try {
+    for (const key of ["input", "output"]) {
+      await client.evaluations.datasets.columns.create(dataset.dataset.id, {
+        datasetVersionId: dataset.initialVersion.id,
+        key,
+        name: key,
+        dataType: "json",
+        role: key === "input" ? "input" : "expected",
+        required: true,
+      });
+    }
+    const result = await importGatewayTrace(client, {
+      datasetVersionId: dataset.initialVersion.id,
+      model,
+    });
+    console.log(
+      result.requestId,
+      result.imported.case.id,
+      result.datasetCase.values,
+    );
+  } catch (error) {
+    if (error instanceof APIError) console.error(error.code, error.message);
+    throw error;
+  } finally {
+    if (process.env.EVALUATION_CLEANUP === "1")
+      await client.evaluations.datasets.archive(dataset.dataset.id);
+  }
+}

package/examples/memory-quickstart.mjs

@@ -0,0 +1,199 @@
+// Generated from memory-quickstart.ts. Do not edit.
+import { APIError, Nirmos, } from "@nirmos/sdk";
+async function all(load) {
+    const items = [];
+    let cursor;
+    do {
+        const page = await load(cursor);
+        items.push(...page.items);
+        cursor = page.nextCursor ?? undefined;
+    } while (cursor);
+    return items;
+}
+/** Server-side only. Persist eventKey with the source event before first ingestion. */
+export async function memoryQuickstart(client, options) {
+    const pause = () => new Promise((resolve) => setTimeout(resolve, options.paceMs ?? 20000));
+    const provider = (await all((cursor) => client.providers.list({ cursor, limit: 100 }))).find((p) => p.slug === "google");
+    if (!provider)
+        throw new Error("Google provider missing from this deployment's catalog.");
+    // Optional SDK setup; dashboard Gateway Credentials → Add credential → Google is equivalent.
+    if (options.googleApiKey)
+        await client.providerCredentials.create({
+            name: "Memory quickstart Google",
+            providerId: provider.id,
+            credentialType: "api_key",
+            secretData: { apiKey: options.googleApiKey },
+            isDefault: true,
+        });
+    const models = await all((cursor) => client.models.list({ providerId: provider.id, limit: 100, cursor }));
+    const text = models.find((m) => m.slug === "google-gemini-2-5-flash-lite" && m.availability);
+    const embedding = models.find((m) => m.slug === "google-gemini-embedding-001" && m.availability);
+    if (!text || !embedding)
+        throw new Error("Configure an active Google credential and verify both models in models.list(). Availability does not guarantee provider quota.");
+    if (!embedding.knowledgeSupportedDimensions?.includes(768))
+        throw new Error("Selected embedding model does not support 768 dimensions in this deployment.");
+    const textModel = `${provider.slug}/${text.id}`;
+    const policyInput = {
+        policyKey: `${options.eventKey}-policy`,
+        name: "Memory quickstart",
+        version: 1,
+        status: "active",
+        captureTypes: [
+            "preference",
+            "fact",
+            "project_fact",
+            "decision",
+            "constraint",
+            "commitment",
+            "event",
+            "temporary_state",
+        ],
+        allowInferredMemories: false,
+        minimumConfidence: 0.7,
+        minimumFutureUtility: 0.5,
+        defaultMemoryTTLSeconds: 86400,
+        extractionConfig: {
+            model: textModel,
+            reconciliationModel: textModel,
+            profileModel: textModel,
+            observationModel: textModel,
+            indexing: {
+                model: `${provider.slug}/${embedding.id}`,
+                dimensions: 768,
+                distanceMetric: "cosine",
+            },
+        },
+        privacyConfig: {
+            allowPII: false,
+            allowedSensitivities: ["unclassified", "public", "internal"],
+        },
+        retentionConfig: {
+            episodeContent: "full",
+            evidenceExcerpts: true,
+            candidateContent: false,
+        },
+    };
+    const existing = await client.memory.policies.list({
+        policyKey: policyInput.policyKey,
+        limit: 100,
+    });
+    const policy = existing.items.find((p) => p.version === 1) ??
+        (await client.memory.policies.create(policyInput));
+    const spaceKey = `${options.eventKey}/customer-123`;
+    const spaces = await all((cursor) => client.memory.spaces.list({
+        externalSubjectId: "customer-123",
+        cursor,
+        limit: 100,
+    }));
+    const space = spaces.find((s) => s.spaceKey === spaceKey) ??
+        (await client.memory.spaces.create({
+            spaceKey,
+            name: "Disposable customer memory",
+            scopeType: "user",
+            scopeId: "customer-123",
+            externalSubjectId: "customer-123",
+            policyId: policy.id,
+        }));
+    const input = {
+        memorySpaceId: space.id,
+        conversationId: options.eventKey,
+        idempotencyKey: options.eventKey,
+        messages: [
+            {
+                id: `${options.eventKey}-message`,
+                role: "user",
+                content: "I prefer important notifications through email rather than WhatsApp.",
+            },
+        ],
+    };
+    let result;
+    try {
+        result = await client.memory.ingest(input, {
+            idempotencyKey: options.eventKey,
+            maxRetries: 0,
+        });
+    }
+    catch (error) {
+        if (error instanceof APIError) {
+            // Safe identifiers/categories only; never print secrets, prompts or memory content.
+            console.error({
+                code: error.code,
+                status: error.status,
+                retryable: error.retryable,
+                retryAfterMs: error.retryAfterMs,
+                requestId: error.requestId,
+                diagnostics: error.details,
+            });
+        }
+        // After correcting configuration/quota, call ingest(input) with the SAME event identity.
+        // Do not blindly retry, mutate the input, or generate a fresh key.
+        throw error;
+    }
+    console.log({
+        runId: result.memoryRunId,
+        accepted: result.acceptedCount,
+        rejected: result.rejectedCount,
+        indexing: result.indexing,
+    });
+    const run = await client.memory.runs.get(result.memoryRunId);
+    if (result.indexing.failed)
+        throw new Error(`Indexing incomplete for run ${run.id}; inspect diagnostics and repair through memory.reindex().`);
+    // A completed event replays without model execution or another memory write.
+    const replay = await client.memory.ingest(input, {
+        idempotencyKey: options.eventKey,
+        maxRetries: 0,
+    });
+    if (replay.memoryRunId !== result.memoryRunId || !replay.replayed)
+        throw new Error("Expected the original completed event to replay.");
+    console.log({ replayed: replay.replayed, runId: replay.memoryRunId });
+    await pause();
+    const search = await client.memory.search({
+        memorySpaceId: space.id,
+        query: "How should I send important notifications?",
+        relevance: "relevant",
+    });
+    console.log({ relevantMatches: search.memories.length });
+    await pause();
+    const context = await client.memory.context({
+        memorySpaceId: space.id,
+        messages: [{ role: "user", content: "How should you notify me?" }],
+    });
+    console.log({ contextMemories: context.memories.length });
+    await pause();
+    await client.memory.profiles.generate({ memorySpaceId: space.id });
+    const profile = await client.memory.profiles.get(space.id);
+    const items = await client.memory.items.list({
+        memorySpaceId: space.id,
+        limit: 25,
+    });
+    const first = items.items[0];
+    if (first) {
+        await client.memory.items.update(first.id, { importance: 0.8 });
+        await client.memory.items.evidence(first.id, { limit: 25 });
+        // Explicit cleanup only for this disposable example's space.
+        if (options.cleanup)
+            await client.memory.items.delete(first.id);
+    }
+    if (options.cleanup) {
+        await client.memory.spaces.delete(space.id);
+        await client.memory.erase(space.id);
+        await client.memory.policies.delete(policy.id);
+    }
+    return { space, policy, result, search, context, profile };
+}
+if (process.env.NIRMOS_RUN_MEMORY_EXAMPLE === "1") {
+    const token = process.env.NIRMOS_API_KEY;
+    const eventKey = process.env.MEMORY_EVENT_KEY;
+    if (!token || !eventKey)
+        throw new Error("Set NIRMOS_API_KEY and a persisted MEMORY_EVENT_KEY.");
+    await memoryQuickstart(new Nirmos({
+        auth: { type: "api-key", key: token },
+        apiBaseUrl: process.env.NIRMOS_API_BASE_URL ?? "http://localhost:4001",
+        runtimeBaseUrl: process.env.NIRMOS_RUNTIME_BASE_URL ?? "http://localhost:4000",
+        maxRetries: 0,
+    }), {
+        eventKey,
+        googleApiKey: process.env.GOOGLE_API_KEY,
+        cleanup: process.env.MEMORY_CLEANUP === "1",
+    });
+}