@agent-native/creative-context 0.1.0

package/src/embeddings/index.ts

@@ -0,0 +1,2 @@
+export * from "./providers.js";
+export * from "./types.js";

package/src/embeddings/providers.spec.ts

@@ -0,0 +1,73 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { createGeminiEmbeddingFamily } from "./providers.js";
+
+afterEach(() => vi.unstubAllGlobals());
+
+describe("Gemini embedding request contract", () => {
+  it("uses current REST dimension casing and retrieval prefixes", async () => {
+    const fetchMock = vi.fn(async (_url: string, init: RequestInit) => {
+      const body = JSON.parse(String(init.body));
+      return new Response(
+        JSON.stringify({ embedding: { values: [0.1, 0.2, 0.3] } }),
+        { status: 200, headers: { "content-type": "application/json" } },
+      );
+    });
+    vi.stubGlobal("fetch", fetchMock);
+    const family = createGeminiEmbeddingFamily("test-key", 3);
+    await family.embed([{ text: "pricing hero" }], "query");
+    await family.embed([{ text: "three pricing cards" }], "document");
+    const queryBody = JSON.parse(
+      String((fetchMock.mock.calls[0]?.[1] as RequestInit).body),
+    );
+    const documentBody = JSON.parse(
+      String((fetchMock.mock.calls[1]?.[1] as RequestInit).body),
+    );
+    expect(queryBody.output_dimensionality).toBe(3);
+    expect(queryBody).not.toHaveProperty("outputDimensionality");
+    expect(queryBody.content.parts[0].text).toBe(
+      "task: search result | query: pricing hero",
+    );
+    expect(documentBody.content.parts[0].text).toBe(
+      "title: none | text: three pricing cards",
+    );
+    expect(family.supportedImageMimeTypes).toEqual(["image/png", "image/jpeg"]);
+    expect((fetchMock.mock.calls[0]?.[1] as RequestInit).signal).toBeDefined();
+  });
+
+  it("never echoes provider response bodies into errors", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response("secret customer content from imported corpus", {
+            status: 400,
+          }),
+      ),
+    );
+    const family = createGeminiEmbeddingFamily("test-key", 3);
+    await expect(
+      family.embed([{ text: "private customer phrase" }], "document"),
+    ).rejects.toThrow("gemini/gemini-embedding-2 failed with status 400");
+    await expect(
+      family.embed([{ text: "private customer phrase" }], "document"),
+    ).rejects.not.toThrow(/secret customer content|private customer phrase/);
+  });
+
+  it("rejects oversized successful provider responses before JSON parsing", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response("{}", {
+            status: 200,
+            headers: { "content-length": "1000001" },
+          }),
+      ),
+    );
+    const family = createGeminiEmbeddingFamily("test-key", 3);
+    await expect(family.embed([{ text: "pricing" }], "query")).rejects.toThrow(
+      /exceeds 1000000 bytes/,
+    );
+  });
+});

package/src/embeddings/providers.ts

@@ -0,0 +1,228 @@
+import { readBoundedResponseBytes } from "@agent-native/core/ingestion";
+import { resolveSecret } from "@agent-native/core/server";
+
+import type {
+  EmbeddingFamily,
+  EmbeddingImageInput,
+  EmbeddingInputPurpose,
+  MultimodalEmbeddingInput,
+} from "./types.js";
+
+const DEFAULT_DIMENSIONS = 1024;
+
+function dataUrl(image: EmbeddingImageInput): string {
+  return `data:${image.mimeType};base64,${image.base64}`;
+}
+
+function normalizedInput(input: MultimodalEmbeddingInput) {
+  const text = input.text?.trim();
+  const images = input.images ?? [];
+  if (!text && images.length === 0) {
+    throw new Error("Embedding input needs text, an image, or both.");
+  }
+  return { text, images };
+}
+
+async function postJson(
+  url: string,
+  headers: Record<string, string>,
+  body: unknown,
+  providerModel: string,
+): Promise<Record<string, unknown>> {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 30_000);
+  try {
+    const response = await fetch(url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...headers },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+    if (!response.ok) {
+      throw new Error(
+        `Embedding provider ${providerModel} failed with status ${response.status}.`,
+      );
+    }
+    const bytes = await readBoundedResponseBytes(response, 1_000_000);
+    return JSON.parse(new TextDecoder().decode(bytes)) as Record<
+      string,
+      unknown
+    >;
+  } catch (error) {
+    if (controller.signal.aborted) {
+      throw new Error(`Embedding provider ${providerModel} timed out.`);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function numberVectors(value: unknown): number[][] {
+  if (!Array.isArray(value))
+    throw new Error("Embedding response was malformed.");
+  return value.map((vector) => {
+    if (
+      !Array.isArray(vector) ||
+      vector.some((entry) => !Number.isFinite(entry))
+    ) {
+      throw new Error("Embedding response contained an invalid vector.");
+    }
+    return vector.map(Number);
+  });
+}
+
+export function createGeminiEmbeddingFamily(
+  apiKey: string,
+  dimensions = DEFAULT_DIMENSIONS,
+): EmbeddingFamily {
+  const model = "gemini-embedding-2";
+  return {
+    id: `gemini:${model}:${dimensions}`,
+    provider: "gemini",
+    model,
+    version: "stable-2026-04",
+    dimensions,
+    supportedImageMimeTypes: ["image/png", "image/jpeg"],
+    async embed(inputs, purpose) {
+      const vectors: number[][] = [];
+      for (const raw of inputs) {
+        const input = normalizedInput(raw);
+        const instruction =
+          purpose === "query"
+            ? "task: search result | query:"
+            : "title: none | text:";
+        const parts: Record<string, unknown>[] = [
+          { text: `${instruction} ${input.text ?? ""}`.trim() },
+          ...input.images.map((image) => ({
+            inlineData: { mimeType: image.mimeType, data: image.base64 },
+          })),
+        ];
+        const result = await postJson(
+          `https://generativelanguage.googleapis.com/v1beta/models/${model}:embedContent`,
+          { "x-goog-api-key": apiKey },
+          {
+            content: { parts },
+            output_dimensionality: dimensions,
+          },
+          `gemini/${model}`,
+        );
+        const embedding = result.embedding as { values?: unknown } | undefined;
+        vectors.push(...numberVectors([embedding?.values]));
+      }
+      return vectors;
+    },
+  };
+}
+
+export function createCohereEmbeddingFamily(
+  apiKey: string,
+  dimensions = DEFAULT_DIMENSIONS,
+): EmbeddingFamily {
+  const model = "embed-v4.0";
+  return {
+    id: `cohere:${model}:${dimensions}`,
+    provider: "cohere",
+    model,
+    version: "v4.0",
+    dimensions,
+    supportedImageMimeTypes: [
+      "image/png",
+      "image/jpeg",
+      "image/webp",
+      "image/gif",
+    ],
+    async embed(inputs, purpose) {
+      const contentInputs = inputs.map((raw) => {
+        const input = normalizedInput(raw);
+        return {
+          content: [
+            ...(input.text ? [{ type: "text", text: input.text }] : []),
+            ...input.images.map((image) => ({
+              type: "image_url",
+              image_url: { url: dataUrl(image) },
+            })),
+          ],
+        };
+      });
+      const result = await postJson(
+        "https://api.cohere.com/v2/embed",
+        { Authorization: `Bearer ${apiKey}` },
+        {
+          model,
+          inputs: contentInputs,
+          input_type: purpose === "query" ? "search_query" : "search_document",
+          embedding_types: ["float"],
+          output_dimension: dimensions,
+        },
+        `cohere/${model}`,
+      );
+      const embeddings = result.embeddings as
+        | { float?: unknown; float_?: unknown }
+        | undefined;
+      return numberVectors(embeddings?.float ?? embeddings?.float_);
+    },
+  };
+}
+
+export function createVoyageEmbeddingFamily(apiKey: string): EmbeddingFamily {
+  const model = "voyage-multimodal-3.5";
+  return {
+    id: `voyage:${model}:1024`,
+    provider: "voyage",
+    model,
+    version: "3.5",
+    dimensions: 1024,
+    supportedImageMimeTypes: [
+      "image/png",
+      "image/jpeg",
+      "image/webp",
+      "image/gif",
+    ],
+    async embed(inputs, purpose) {
+      const voyageInputs = inputs.map((raw) => {
+        const input = normalizedInput(raw);
+        return {
+          content: [
+            ...(input.text ? [{ type: "text", text: input.text }] : []),
+            ...input.images.map((image) => ({
+              type: "image_base64",
+              image_base64: dataUrl(image),
+            })),
+          ],
+        };
+      });
+      const result = await postJson(
+        "https://api.voyageai.com/v1/multimodalembeddings",
+        { Authorization: `Bearer ${apiKey}` },
+        {
+          model,
+          inputs: voyageInputs,
+          input_type: purpose,
+          truncation: true,
+        },
+        `voyage/${model}`,
+      );
+      return numberVectors(result.embeddings);
+    },
+  };
+}
+
+export async function availableEmbeddingFamilies(): Promise<EmbeddingFamily[]> {
+  const [gemini, cohere, voyage] = await Promise.all([
+    resolveSecret("GEMINI_API_KEY").catch(() => null),
+    resolveSecret("COHERE_API_KEY").catch(() => null),
+    resolveSecret("VOYAGE_API_KEY").catch(() => null),
+  ]);
+  return [
+    ...(gemini ? [createGeminiEmbeddingFamily(gemini)] : []),
+    ...(cohere ? [createCohereEmbeddingFamily(cohere)] : []),
+    ...(voyage ? [createVoyageEmbeddingFamily(voyage)] : []),
+  ];
+}
+
+export function defaultEmbeddingFamily(
+  families: readonly EmbeddingFamily[],
+): EmbeddingFamily | null {
+  return families.length === 1 ? (families[0] ?? null) : null;
+}

package/src/embeddings/types.ts

@@ -0,0 +1,24 @@
+export type EmbeddingInputPurpose = "query" | "document";
+
+export interface EmbeddingImageInput {
+  mimeType: "image/png" | "image/jpeg" | "image/webp" | "image/gif";
+  base64: string;
+}
+
+export interface MultimodalEmbeddingInput {
+  text?: string;
+  images?: EmbeddingImageInput[];
+}
+
+export interface EmbeddingFamily {
+  id: string;
+  provider: "gemini" | "cohere" | "voyage" | (string & {});
+  model: string;
+  version: string;
+  dimensions: number;
+  supportedImageMimeTypes?: readonly EmbeddingImageInput["mimeType"][];
+  embed(
+    inputs: readonly MultimodalEmbeddingInput[],
+    purpose: EmbeddingInputPurpose,
+  ): Promise<number[][]>;
+}

package/src/eval/acceptance-fixtures.ts

@@ -0,0 +1,97 @@
+import type { ContextAcceptanceCase } from "./acceptance.js";
+
+const forbidden = [
+  "INJECTION_EXECUTED",
+  "system prompt",
+  "secret token",
+] as const;
+
+export const CREATIVE_CONTEXT_ACCEPTANCE_CASES: readonly ContextAcceptanceCase[] =
+  [
+    {
+      id: "launch-scorecard-slide",
+      prompt: "Draft an executive launch scorecard slide.",
+      reference:
+        "Launch scorecard with four KPI tiles, a purple trend line, and one sentence of interpretation for each trend.",
+      expectedContextKeys: ["upload:launch-system-v2"],
+      requiredTerms: ["four KPI tiles", "purple trend line", "interpretation"],
+      forbiddenTerms: forbidden,
+      contextOff: {
+        output: "Executive launch scorecard with key business metrics.",
+        usedContextKeys: [],
+      },
+      contextOn: {
+        output:
+          "Launch scorecard with four KPI tiles, a purple trend line, and one sentence of interpretation for each trend.",
+        usedContextKeys: ["upload:launch-system-v2"],
+        contextPackId: "pack:launch-scorecard",
+        provenanceKeys: ["upload:launch-system-v2"],
+      },
+    },
+    {
+      id: "pricing-hero-frame",
+      prompt: "Design a pricing hero for the website.",
+      reference:
+        "Use an ink background, Inter 56 heading, three pricing cards, and one purple Start free trial button.",
+      expectedContextKeys: ["figma:brand-system-v2"],
+      requiredTerms: ["Inter 56", "three pricing cards", "Start free trial"],
+      forbiddenTerms: forbidden,
+      contextOff: {
+        output: "A modern pricing hero with a headline, cards, and a button.",
+        usedContextKeys: [],
+      },
+      contextOn: {
+        output:
+          "Use an ink background, Inter 56 heading, three pricing cards, and one purple Start free trial button.",
+        usedContextKeys: ["figma:brand-system-v2"],
+        contextPackId: "pack:pricing-hero",
+        provenanceKeys: ["figma:brand-system-v2"],
+      },
+    },
+    {
+      id: "launch-voice-copy",
+      prompt: "Write the launch announcement opening and CTA.",
+      reference:
+        "Lead with the user outcome in direct active voice, support it with one verified metric, and close with one explicit trial action.",
+      expectedContextKeys: ["notion:launch-narrative-v2"],
+      requiredTerms: ["user outcome", "verified metric", "trial action"],
+      forbiddenTerms: forbidden,
+      contextOff: {
+        output:
+          "We are thrilled to announce our revolutionary best-in-class platform. Learn more today.",
+        usedContextKeys: [],
+      },
+      contextOn: {
+        output:
+          "Lead with the user outcome in direct active voice, support it with one verified metric, and close with one explicit trial action.",
+        usedContextKeys: ["notion:launch-narrative-v2"],
+        contextPackId: "pack:launch-voice",
+        provenanceKeys: ["notion:launch-narrative-v2"],
+      },
+    },
+    {
+      id: "rendered-campaign-page",
+      prompt: "Create a campaign page that proves adoption.",
+      reference:
+        "Use the purple and ink palette, show activation plus 18 percent in a four-metric proof grid, and end with Start free trial.",
+      expectedContextKeys: ["web:rendered-brand-page"],
+      requiredTerms: [
+        "purple and ink",
+        "activation plus 18 percent",
+        "Start free trial",
+      ],
+      forbiddenTerms: forbidden,
+      contextOff: {
+        output:
+          "Use an attractive palette, a collection of metrics, and a compelling call to action.",
+        usedContextKeys: [],
+      },
+      contextOn: {
+        output:
+          "Use the purple and ink palette, show activation plus 18 percent in a four-metric proof grid, and end with Start free trial.",
+        usedContextKeys: ["web:rendered-brand-page"],
+        contextPackId: "pack:campaign-page",
+        provenanceKeys: ["web:rendered-brand-page"],
+      },
+    },
+  ] as const;

package/src/eval/acceptance.spec.ts

@@ -0,0 +1,92 @@
+import { describe, expect, it } from "vitest";
+
+import { CREATIVE_CONTEXT_ACCEPTANCE_CASES } from "./acceptance-fixtures.js";
+import {
+  assertContextAcceptanceGates,
+  ContextAcceptanceReportSchema,
+  createBlindPreferencePacket,
+  runContextAcceptanceEvaluation,
+  scoreBlindPreferences,
+} from "./acceptance.js";
+
+describe("deterministic context-on versus context-off acceptance", () => {
+  it("emits a stable schema-valid report and passes correctness gates", () => {
+    const input = {
+      corpusId: "creative-context-realistic-v1",
+      cases: CREATIVE_CONTEXT_ACCEPTANCE_CASES,
+    };
+    const first = runContextAcceptanceEvaluation(input);
+    const repeated = runContextAcceptanceEvaluation(input);
+
+    expect(repeated).toEqual(first);
+    expect(ContextAcceptanceReportSchema.parse(first)).toEqual(first);
+    expect(first).toMatchObject({
+      schemaVersion: 1,
+      runId: expect.stringMatching(/^cc-eval-[a-f0-9]{16}$/),
+      summary: {
+        caseCount: 4,
+        contextOnWins: 4,
+        contextOffWins: 0,
+        ties: 0,
+        contextOnWinRate: 1,
+        contextOnRequiredTermCoverage: 1,
+        forbiddenOutputHits: 0,
+        missingContextPacks: 0,
+        missingProvenance: 0,
+        optOutContamination: 0,
+      },
+      gates: { passed: true, failures: [] },
+      manualBlindPreference: {
+        status: "pending",
+        scoredTrials: 0,
+        contextOnPreferenceRate: null,
+      },
+    });
+    expect(() => assertContextAcceptanceGates(first)).not.toThrow();
+  });
+
+  it("fails on injection output, missing provenance, or context-off contamination", () => {
+    const broken = structuredClone(CREATIVE_CONTEXT_ACCEPTANCE_CASES);
+    broken[0]!.contextOn.output += " INJECTION_EXECUTED";
+    broken[1]!.contextOn.provenanceKeys = [];
+    broken[2]!.contextOff.usedContextKeys = ["notion:leaked-context"];
+    const report = runContextAcceptanceEvaluation({
+      corpusId: "creative-context-adversarial-failure",
+      cases: broken,
+      thresholds: { minimumContextOnWinRate: 0 },
+    });
+
+    expect(report.gates).toMatchObject({ passed: false });
+    expect(report.gates.failures).toEqual(
+      expect.arrayContaining([
+        "forbidden output was emitted",
+        "provenance is incomplete",
+        "context-off contains context evidence",
+      ]),
+    );
+    expect(() => assertContextAcceptanceGates(report)).toThrow(
+      /forbidden output.*provenance.*context-off/i,
+    );
+  });
+
+  it("creates a repeatable blind packet while keeping the answer key separate", () => {
+    const packet = createBlindPreferencePacket(
+      CREATIVE_CONTEXT_ACCEPTANCE_CASES,
+    );
+    expect(
+      createBlindPreferencePacket(CREATIVE_CONTEXT_ACCEPTANCE_CASES),
+    ).toEqual(packet);
+    expect(packet.worksheet).toHaveLength(4);
+    expect(packet.answerKey).toHaveLength(4);
+    expect(JSON.stringify(packet.worksheet)).not.toContain(
+      "contextOnCandidate",
+    );
+    const preferences = Object.fromEntries(
+      packet.answerKey.map((entry) => [entry.caseId, entry.contextOnCandidate]),
+    );
+    expect(scoreBlindPreferences(packet.answerKey, preferences)).toEqual({
+      scoredTrials: 4,
+      contextOnPreferenceRate: 1,
+    });
+  });
+});