@agent-native/creative-context 0.1.0

package/src/eval/fixtures.ts

@@ -0,0 +1,136 @@
+import type { RetrievalEvalTask } from "./index.js";
+
+export interface CreativeContextGoldDocument {
+  key: string;
+  kind: "slide" | "figma-frame" | "notion-section" | "web-page" | "image";
+  title: string;
+  text: string;
+  imageBase64?: string;
+  owner: "personal" | "organization" | "other-organization";
+  status: "active" | "deprecated";
+  revisionOf?: string;
+}
+
+export const CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR4nGOI9n+JFTEMLQkAOpFkwUU6BmIAAAAASUVORK5CYII=";
+export const CREATIVE_CONTEXT_INK_IMAGE_BASE64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEUlEQVR4nGPg5hbAihiGlgQAtbUJgThTKnUAAAAASUVORK5CYII=";
+
+export const CREATIVE_CONTEXT_GOLD_DOCUMENTS: readonly CreativeContextGoldDocument[] =
+  [
+    {
+      key: "slide:metrics-v2",
+      kind: "slide",
+      title: "Weekly metrics scorecard",
+      text: "Dark scorecard with four KPI tiles, a purple trend line, and a concise takeaway.",
+      owner: "organization",
+      status: "active",
+      revisionOf: "slide:metrics-v1",
+    },
+    {
+      key: "slide:metrics-v1",
+      kind: "slide",
+      title: "Weekly metrics scorecard old",
+      text: "Previous scorecard with four KPI tiles.",
+      owner: "organization",
+      status: "deprecated",
+    },
+    {
+      key: "figma:pricing-hero",
+      kind: "figma-frame",
+      title: "Pricing hero",
+      text: "1440 by 820 dark hero, Inter 56 heading, three pricing cards, primary trial button.",
+      imageBase64: CREATIVE_CONTEXT_INK_IMAGE_BASE64,
+      owner: "organization",
+      status: "active",
+    },
+    {
+      key: "notion:dense-guidelines",
+      kind: "notion-section",
+      title: "Dense launch system guidance",
+      text: "For launch narratives, lead with the user outcome, name the concrete workflow, support claims with one metric, keep headings under eight words, use direct active voice, avoid inflated superlatives, preserve the primary purple and ink palette, use compact scorecards for quantitative proof, and close with one explicit trial action. Product screenshots should center the working surface, annotations should identify the changed behavior, and supporting copy should remain subordinate to the artifact. For executive reviews, pair each KPI with a trend and a one-sentence interpretation. For campaign variants, retain the same hierarchy while adapting the evidence and call to action to the audience segment.",
+      owner: "organization",
+      status: "active",
+    },
+    {
+      key: "notion:voice",
+      kind: "notion-section",
+      title: "Launch writing principles",
+      text: "Direct, concise, optimistic product writing. Prefer concrete verbs and short sentences.",
+      owner: "organization",
+      status: "active",
+    },
+    {
+      key: "web:palette",
+      kind: "web-page",
+      title: "Brand home page",
+      text: "Primary #5B4FE9, ink #0B0B10, Inter headings, twelve-pixel radii.",
+      owner: "organization",
+      status: "active",
+    },
+    {
+      key: "image:campaign",
+      kind: "image",
+      title: "Launch campaign hero",
+      text: "Purple gradient product hero with a centered device render and compact white headline.",
+      imageBase64: CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64,
+      owner: "organization",
+      status: "active",
+    },
+    {
+      key: "private:other-org",
+      kind: "slide",
+      title: "Confidential acquisition plan",
+      text: "Restricted content owned by another organization.",
+      owner: "other-organization",
+      status: "active",
+    },
+  ] as const;
+
+export const CREATIVE_CONTEXT_GOLD_TASKS: readonly RetrievalEvalTask[] = [
+  {
+    id: "exact-metrics-layout",
+    query: { text: "weekly metrics scorecard with four KPI tiles" },
+    relevantKeys: ["slide:metrics-v2"],
+    forbiddenKeys: ["private:other-org"],
+  },
+  {
+    id: "visual-pricing-match",
+    query: { text: "dark pricing hero with three cards" },
+    relevantKeys: ["figma:pricing-hero"],
+    forbiddenKeys: ["private:other-org"],
+  },
+  {
+    id: "voice-match",
+    query: { text: "direct concise optimistic launch copy" },
+    relevantKeys: ["notion:voice"],
+    forbiddenKeys: ["private:other-org"],
+  },
+  {
+    id: "text-to-image-campaign-match",
+    query: { text: "purple campaign hero with centered device" },
+    relevantKeys: ["image:campaign"],
+    forbiddenKeys: ["private:other-org"],
+  },
+  {
+    id: "image-to-image-campaign-match",
+    query: {
+      images: [
+        {
+          mimeType: "image/png",
+          base64: CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64,
+        },
+      ],
+    },
+    relevantKeys: ["image:campaign"],
+    forbiddenKeys: ["private:other-org"],
+  },
+  {
+    id: "dense-guidance-match",
+    query: {
+      text: "executive launch KPI trend interpretation active voice purple ink compact scorecard",
+    },
+    relevantKeys: ["notion:dense-guidelines"],
+    forbiddenKeys: ["private:other-org"],
+  },
+] as const;

package/src/eval/index.ts

@@ -0,0 +1,291 @@
+import type {
+  EmbeddingFamily,
+  MultimodalEmbeddingInput,
+} from "../embeddings/types.js";
+
+export {
+  assertContextAcceptanceGates,
+  ContextAcceptanceReportSchema,
+  createBlindPreferencePacket,
+  runContextAcceptanceEvaluation,
+  scoreBlindPreferences,
+  type BlindPreferenceAnswerKey,
+  type BlindPreferenceWorksheetRow,
+  type ContextAcceptanceArtifact,
+  type ContextAcceptanceCase,
+  type ContextAcceptanceReport,
+  type ContextAcceptanceThresholds,
+} from "./acceptance.js";
+export { CREATIVE_CONTEXT_ACCEPTANCE_CASES } from "./acceptance-fixtures.js";
+
+export {
+  CREATIVE_CONTEXT_INK_IMAGE_BASE64,
+  CREATIVE_CONTEXT_GOLD_DOCUMENTS,
+  CREATIVE_CONTEXT_GOLD_TASKS,
+  CREATIVE_CONTEXT_PURPLE_IMAGE_BASE64,
+  type CreativeContextGoldDocument,
+} from "./fixtures.js";
+
+export interface RetrievalEvalTask {
+  id: string;
+  query: MultimodalEmbeddingInput;
+  relevantKeys: readonly string[];
+  forbiddenKeys?: readonly string[];
+}
+
+export interface RetrievalEvalMetrics {
+  taskCount: number;
+  top5Recall: number;
+  meanReciprocalRank: number;
+  permissionLeaks: number;
+}
+
+export interface ContextQualityTrial {
+  id: string;
+  preferred: "context-on" | "context-off" | "tie";
+  baselineEditDistance: number;
+  contextEditDistance: number;
+}
+
+export interface ContextQualityMetrics {
+  trialCount: number;
+  contextPreferenceRate: number;
+  meanEditDistanceReduction: number;
+}
+
+export interface CreativeContextCorrectnessEvidence {
+  permissionLeaks: number;
+  generationsMissingPacks: number;
+  generationsMissingProvenance: number;
+  importsResumable: boolean;
+  importsIdempotent: boolean;
+  importsDeterministic: boolean;
+  revocationRemovesAccess: boolean;
+  lexicalLaneRebuilds: boolean;
+  vectorLaneRebuilds: boolean;
+  connectorsPassing: readonly string[];
+  consumersUsingReuseLadder: readonly string[];
+  nativeCodeRetrievalPassing: readonly string[];
+  nativeCloneFidelityPassing: readonly string[];
+  nativeCloneVersionsPinned: boolean;
+  supportedNativeElementsEditable: boolean;
+  runtimeExcludesFullResolutionReferenceRenders: boolean;
+  contextOptOutExcludesAllContext: boolean;
+  contextOnBeatsContextOff: boolean;
+}
+
+const REQUIRED_CONNECTORS = [
+  "upload",
+  "website",
+  "figma",
+  "notion",
+  "google-slides",
+] as const;
+const REQUIRED_CONSUMERS = ["slides", "assets", "design", "content"] as const;
+const REQUIRED_NATIVE_CONSUMERS = ["slides", "design"] as const;
+
+export function evaluateRankings(
+  tasks: readonly RetrievalEvalTask[],
+  rankings: Readonly<Record<string, readonly string[]>>,
+): RetrievalEvalMetrics {
+  if (tasks.length === 0) {
+    return {
+      taskCount: 0,
+      top5Recall: 0,
+      meanReciprocalRank: 0,
+      permissionLeaks: 0,
+    };
+  }
+  let top5 = 0;
+  let reciprocalRank = 0;
+  let permissionLeaks = 0;
+  for (const task of tasks) {
+    const ranking = rankings[task.id] ?? [];
+    const relevant = new Set(task.relevantKeys);
+    if (ranking.slice(0, 5).some((key) => relevant.has(key))) top5 += 1;
+    const first = ranking.findIndex((key) => relevant.has(key));
+    if (first >= 0) reciprocalRank += 1 / (first + 1);
+    const forbidden = new Set(task.forbiddenKeys ?? []);
+    permissionLeaks += ranking.filter((key) => forbidden.has(key)).length;
+  }
+  return {
+    taskCount: tasks.length,
+    top5Recall: top5 / tasks.length,
+    meanReciprocalRank: reciprocalRank / tasks.length,
+    permissionLeaks,
+  };
+}
+
+export function evaluateContextQuality(
+  trials: readonly ContextQualityTrial[],
+): ContextQualityMetrics {
+  if (!trials.length) {
+    return {
+      trialCount: 0,
+      contextPreferenceRate: 0,
+      meanEditDistanceReduction: 0,
+    };
+  }
+  const preferred = trials.filter(
+    (trial) => trial.preferred === "context-on",
+  ).length;
+  const reduction = trials.reduce((total, trial) => {
+    if (trial.baselineEditDistance <= 0) return total;
+    return (
+      total +
+      (trial.baselineEditDistance - trial.contextEditDistance) /
+        trial.baselineEditDistance
+    );
+  }, 0);
+  return {
+    trialCount: trials.length,
+    contextPreferenceRate: preferred / trials.length,
+    meanEditDistanceReduction: reduction / trials.length,
+  };
+}
+
+export function creativeContextCompletionFailures(
+  evidence: CreativeContextCorrectnessEvidence,
+): string[] {
+  const failures: string[] = [];
+  if (evidence.permissionLeaks !== 0) failures.push("permission leaks");
+  if (evidence.generationsMissingPacks !== 0)
+    failures.push("generations missing context packs");
+  if (evidence.generationsMissingProvenance !== 0)
+    failures.push("generations missing provenance");
+  if (!evidence.importsResumable) failures.push("imports are not resumable");
+  if (!evidence.importsIdempotent) failures.push("imports are not idempotent");
+  if (!evidence.importsDeterministic)
+    failures.push("imports are not deterministic");
+  if (!evidence.revocationRemovesAccess)
+    failures.push("revocation does not remove retrieval access");
+  if (!evidence.lexicalLaneRebuilds)
+    failures.push("lexical retrieval cannot rebuild from canonical data");
+  if (!evidence.vectorLaneRebuilds)
+    failures.push("vector retrieval cannot rebuild from canonical data");
+  const connectorSet = new Set(evidence.connectorsPassing);
+  const missingConnectors = REQUIRED_CONNECTORS.filter(
+    (connector) => !connectorSet.has(connector),
+  );
+  if (missingConnectors.length)
+    failures.push(`connectors missing: ${missingConnectors.join(", ")}`);
+  const consumerSet = new Set(evidence.consumersUsingReuseLadder);
+  const missingConsumers = REQUIRED_CONSUMERS.filter(
+    (consumer) => !consumerSet.has(consumer),
+  );
+  if (missingConsumers.length)
+    failures.push(`reuse ladder missing: ${missingConsumers.join(", ")}`);
+  const nativeRetrievalSet = new Set(evidence.nativeCodeRetrievalPassing);
+  const missingNativeRetrieval = REQUIRED_NATIVE_CONSUMERS.filter(
+    (consumer) => !nativeRetrievalSet.has(consumer),
+  );
+  if (missingNativeRetrieval.length) {
+    failures.push(
+      `native code retrieval missing: ${missingNativeRetrieval.join(", ")}`,
+    );
+  }
+  const nativeFidelitySet = new Set(evidence.nativeCloneFidelityPassing);
+  const missingNativeFidelity = REQUIRED_NATIVE_CONSUMERS.filter(
+    (consumer) => !nativeFidelitySet.has(consumer),
+  );
+  if (missingNativeFidelity.length) {
+    failures.push(
+      `native clone fidelity missing: ${missingNativeFidelity.join(", ")}`,
+    );
+  }
+  if (!evidence.nativeCloneVersionsPinned) {
+    failures.push("native clone versions are not pinned");
+  }
+  if (!evidence.supportedNativeElementsEditable) {
+    failures.push("supported native elements are not editable");
+  }
+  if (!evidence.runtimeExcludesFullResolutionReferenceRenders) {
+    failures.push("runtime context includes full-resolution reference renders");
+  }
+  if (!evidence.contextOptOutExcludesAllContext)
+    failures.push("context opt-out does not exclude all context");
+  if (!evidence.contextOnBeatsContextOff)
+    failures.push("context-on does not beat context-off");
+  return failures;
+}
+
+export function assertCreativeContextCompletionGates(
+  evidence: CreativeContextCorrectnessEvidence,
+): void {
+  const failures = creativeContextCompletionFailures(evidence);
+  if (failures.length) {
+    throw new Error(
+      `Creative context completion gates failed: ${failures.join("; ")}`,
+    );
+  }
+}
+
+function cosine(left: readonly number[], right: readonly number[]): number {
+  if (left.length !== right.length) return Number.NEGATIVE_INFINITY;
+  let dot = 0;
+  let leftNorm = 0;
+  let rightNorm = 0;
+  for (let index = 0; index < left.length; index += 1) {
+    const a = left[index] ?? 0;
+    const b = right[index] ?? 0;
+    dot += a * b;
+    leftNorm += a * a;
+    rightNorm += b * b;
+  }
+  if (!leftNorm || !rightNorm) return Number.NEGATIVE_INFINITY;
+  return dot / Math.sqrt(leftNorm * rightNorm);
+}
+
+export async function bakeOffEmbeddingFamilies(input: {
+  families: readonly EmbeddingFamily[];
+  documents: Readonly<Record<string, MultimodalEmbeddingInput>>;
+  tasks: readonly RetrievalEvalTask[];
+  allowedDocumentKeys?: readonly string[];
+  topK?: number;
+}): Promise<{ family: EmbeddingFamily; metrics: RetrievalEvalMetrics }[]> {
+  const documentEntries = Object.entries(input.documents);
+  const allowed = input.allowedDocumentKeys
+    ? new Set(input.allowedDocumentKeys)
+    : null;
+  const results = await Promise.all(
+    input.families.map(async (family) => {
+      const documentVectors = await family.embed(
+        documentEntries.map(([, value]) => value),
+        "document",
+      );
+      const queryVectors = await family.embed(
+        input.tasks.map((task) => task.query),
+        "query",
+      );
+      const rankings: Record<string, string[]> = {};
+      input.tasks.forEach((task, taskIndex) => {
+        const query = queryVectors[taskIndex] ?? [];
+        rankings[task.id] = documentEntries
+          .filter(([key]) => !allowed || allowed.has(key))
+          .map(([key], index) => ({
+            key,
+            score: cosine(query, documentVectors[index] ?? []),
+          }))
+          .sort((a, b) => b.score - a.score || a.key.localeCompare(b.key))
+          .slice(0, Math.max(5, input.topK ?? 20))
+          .map((entry) => entry.key);
+      });
+      return { family, metrics: evaluateRankings(input.tasks, rankings) };
+    }),
+  );
+  return results.sort(
+    (a, b) =>
+      a.metrics.permissionLeaks - b.metrics.permissionLeaks ||
+      b.metrics.top5Recall - a.metrics.top5Recall ||
+      b.metrics.meanReciprocalRank - a.metrics.meanReciprocalRank,
+  );
+}
+
+export function embeddingBakeoffPasses(metrics: RetrievalEvalMetrics): boolean {
+  return (
+    metrics.taskCount >= 6 &&
+    metrics.permissionLeaks === 0 &&
+    metrics.top5Recall >= 0.8 &&
+    metrics.meanReciprocalRank >= 0.35
+  );
+}

package/src/eval/library-flow.acceptance.spec.tsx

@@ -0,0 +1,141 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+
+import { renderToStaticMarkup } from "react-dom/server";
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@agent-native/core/client", () => ({
+  getBrowserTabId: () => "acceptance-tab",
+  readClientAppState: vi.fn(),
+  setClientAppState: vi.fn(),
+  useActionMutation: vi.fn(),
+  useActionQuery: vi.fn(),
+  useChangeVersion: () => 0,
+  useT: () => (key: string) =>
+    ({
+      "creativeContext.off": "Off",
+      "creativeContext.automatic": "Automatic",
+    })[key] ?? key,
+}));
+
+const { CreativeContextChip } =
+  await import("../client/CreativeContextChip.js");
+const { normalizeCreativeContextState } =
+  await import("../client/application-state.js");
+const { UploadContextConnector } = await import("../connectors/upload.js");
+
+describe("credential-free Library acceptance", () => {
+  it("imports usable PPTX items, displays their pack chip, and structurally opts out", async () => {
+    const bytes = new Uint8Array(
+      await readFile(
+        new URL("./fixtures/launch-system-v2.pptx", import.meta.url),
+      ),
+    );
+    const handle = {
+      id: "acceptance-pptx",
+      provider: "fixture",
+      opaque: true as const,
+      encrypted: true,
+      mimeType:
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    };
+    const config = {
+      items: [
+        {
+          id: "launch-system",
+          title: "Launch system",
+          fileName: "launch-system.pptx",
+          mimeType: handle.mimeType,
+          blobHandle: handle,
+        },
+      ],
+    };
+    const context = {
+      appId: "content",
+      ownerEmail: "owner@example.com",
+      readPrivateBlob: async () => ({ data: bytes, handle }),
+      putPrivateBlob: async (input: {
+        data: Uint8Array;
+        mimeType?: string;
+      }) => ({
+        id: createHash("sha256").update(input.data).digest("hex").slice(0, 20),
+        provider: "fixture",
+        opaque: true as const,
+        encrypted: true,
+        mimeType: input.mimeType,
+      }),
+    } as never;
+    const connector = new UploadContextConnector();
+    const inventory = await connector.inventory(
+      { sourceId: "library-source", config },
+      context,
+    );
+    const imported = await connector.fetch(
+      {
+        sourceId: "library-source",
+        config,
+        item: inventory.items[0]!,
+      },
+      context,
+    );
+
+    expect(imported.items).toHaveLength(4);
+    expect(imported.items.every((item) => item.parseStatus === "parsed")).toBe(
+      true,
+    );
+    expect(imported.items[0]).toMatchObject({
+      externalId: "launch-system",
+      metadata: {
+        childExternalIds: [
+          "launch-system:slide-1",
+          "launch-system:slide-2",
+          "launch-system:slide-3",
+        ],
+      },
+    });
+    expect(imported.items[1]?.chunks?.length).toBeGreaterThan(0);
+
+    const pack = {
+      id: "pack:launch-system",
+      name: "Launch system",
+      description: "Imported approved launch evidence",
+      derivedFromPackId: null,
+      brandDnaVersionId: null,
+      contextMode: "auto",
+      request: {},
+      memberCount: imported.items.length,
+      pinned: true,
+      archivedAt: null,
+      visibility: "private" as const,
+      createdAt: "2026-07-16T00:00:00.000Z",
+    };
+    const enabledMarkup = renderToStaticMarkup(
+      <CreativeContextChip
+        state={{
+          contextMode: "auto",
+          currentPackId: pack.id,
+          pinnedPackId: pack.id,
+        }}
+        packs={[pack]}
+      />,
+    );
+    expect(enabledMarkup).toContain("Launch system");
+    expect(enabledMarkup).toContain("Imported approved launch evidence");
+
+    const off = normalizeCreativeContextState({
+      contextMode: "off",
+      currentPackId: pack.id,
+      pinnedPackId: pack.id,
+    });
+    expect(off).toEqual({
+      contextMode: "off",
+      currentPackId: null,
+      pinnedPackId: null,
+    });
+    const offMarkup = renderToStaticMarkup(
+      <CreativeContextChip state={off} packs={[pack]} />,
+    );
+    expect(offMarkup).toContain("Off");
+    expect(offMarkup).not.toContain("Launch system");
+  });
+});

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./types.js";
+export * from "./native-artifact.js";
+export * from "./native-artifact-reassembly.js";
+export * from "./media-url.js";