@agent-native/creative-context 0.1.0

package/src/jobs/index.ts

@@ -0,0 +1,40 @@
+export { runContextImportJob } from "./import-runner.js";
+export { contextImportProgressReporter } from "./progress.js";
+export { creativeContextImportJobPersistence } from "./store-adapter.js";
+export {
+  enqueueCreativeContextDailyMaintenance,
+  processCreativeContextBackgroundJob,
+  processDueCreativeContextBackgroundJobs,
+  registerCreativeContextBackgroundDispatcher,
+  type CreativeContextBackgroundDispatch,
+  type CreativeContextBackgroundDispatcher,
+} from "./background-worker.js";
+export {
+  rebuildFtsBatch,
+  rebuildVectorBatch,
+  type RebuildBatchResult,
+} from "./rebuild.js";
+export {
+  CREATIVE_CONTEXT_BACKGROUND_PROCESSOR_ROUTE,
+  CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE,
+  createCreativeContextWorkerPlugin,
+  startCreativeContextDailyMaintenance,
+  startCreativeContextImportSweep,
+} from "./server-worker.js";
+export {
+  dispatchCreativeContextImportJob,
+  processCreativeContextImportJob,
+  processDueCreativeContextImportJobs,
+  registerCreativeContextImportContinuationDispatcher,
+  type CreativeContextImportContinuationDispatcher,
+  type CreativeContextImportDispatch,
+  type ProcessCreativeContextImportJobOptions,
+} from "./worker.js";
+export type {
+  ContextImportCheckpoint,
+  ContextImportJobPatch,
+  ContextImportJobPersistence,
+  ContextImportProgressReporter,
+  RunContextImportJobOptions,
+  RunContextImportJobResult,
+} from "./types.js";

package/src/jobs/progress.ts

@@ -0,0 +1,37 @@
+import {
+  completeRun,
+  getRun,
+  startRun,
+  updateRunProgress,
+} from "@agent-native/core/progress";
+
+import type { ContextImportProgressReporter } from "./types.js";
+
+export const contextImportProgressReporter: ContextImportProgressReporter = {
+  async start(input) {
+    const existing = await getRun(input.id, input.owner);
+    if (existing) {
+      await updateRunProgress(input.id, input.owner, {
+        status: "running",
+        percent: null,
+        step: input.step,
+        metadata: input.metadata,
+      });
+      return;
+    }
+    await startRun(input);
+  },
+  async update(input) {
+    await updateRunProgress(input.id, input.owner, {
+      percent: input.percent,
+      step: input.step,
+      metadata: input.metadata,
+    });
+  },
+  async complete(input) {
+    await completeRun(input.id, input.owner, input.status, {
+      step: input.step,
+      metadata: input.metadata,
+    });
+  },
+};

package/src/jobs/rebuild.ts

@@ -0,0 +1,245 @@
+import { getDbExec, isPostgres } from "@agent-native/core/db";
+import { inArray } from "drizzle-orm";
+
+import { availableEmbeddingFamilies } from "../embeddings/providers.js";
+import type { EmbeddingFamily } from "../embeddings/types.js";
+import { upsertPostgresFtsDocument } from "../search/postgres-fts.js";
+import { getCreativeContext } from "../server/context.js";
+import { sha256 } from "../store/helpers.js";
+import {
+  createJob,
+  getActiveEmbeddingSet,
+  getCreativeContextItem,
+  listAccessibleSearchDocuments,
+  recordEmbeddingMetadata,
+  type AccessibleSearchDocument,
+} from "../store/index.js";
+import type { ContextJob } from "../types.js";
+import { PGVECTOR_REQUIRED_MESSAGE } from "../vector/pgvector.js";
+
+const MAX_REBUILD_BATCH = 500;
+const EMBEDDING_BATCH = 16;
+
+export interface RebuildBatchResult {
+  processed: number;
+  indexed: number;
+  afterChunkId: string | null;
+  hasMore: boolean;
+  lane: "portable-lexical" | "postgres-fts" | "pgvector";
+  embeddingSetId?: string;
+  mediaQueued?: number;
+}
+
+function requestedItemIds(job: ContextJob): string[] | undefined {
+  const value = job.request.itemIds;
+  return Array.isArray(value)
+    ? value.filter(
+        (entry): entry is string =>
+          typeof entry === "string" && entry.length > 0,
+      )
+    : undefined;
+}
+
+function rebuildLimit(job: ContextJob): number {
+  const eagerLimit = Number(job.budget?.eagerLimit ?? MAX_REBUILD_BATCH);
+  return Math.max(
+    1,
+    Math.min(
+      MAX_REBUILD_BATCH,
+      Number.isFinite(eagerLimit) ? Math.floor(eagerLimit) : MAX_REBUILD_BATCH,
+    ),
+  );
+}
+
+async function loadBatch(job: ContextJob): Promise<AccessibleSearchDocument[]> {
+  const checkpoint = job.checkpoint ?? {};
+  return listAccessibleSearchDocuments({
+    sourceIds: job.sourceId ? [job.sourceId] : undefined,
+    itemIds: requestedItemIds(job),
+    afterChunkId:
+      typeof checkpoint.afterChunkId === "string"
+        ? checkpoint.afterChunkId
+        : undefined,
+    limit: rebuildLimit(job),
+  });
+}
+
+async function markIndexed(documents: AccessibleSearchDocument[]) {
+  const itemIds = [...new Set(documents.map((document) => document.itemId))];
+  if (!itemIds.length) return;
+  const { getDb, schema } = getCreativeContext();
+  await getDb()
+    .update(schema.contextItems)
+    .set({ indexState: "indexed" })
+    .where(inArray(schema.contextItems.id, itemIds));
+}
+
+export async function rebuildFtsBatch(
+  job: ContextJob,
+): Promise<RebuildBatchResult> {
+  const documents = await loadBatch(job);
+  let indexed = 0;
+  for (const document of documents) {
+    const stored = await upsertPostgresFtsDocument(getDbExec(), {
+      chunkId: document.chunkId!,
+      itemVersionId: document.itemVersionId,
+      title: document.title,
+      summary: document.summary,
+      body: document.body,
+    });
+    if (stored) indexed += 1;
+  }
+  await markIndexed(documents);
+  return {
+    processed: documents.length,
+    indexed,
+    afterChunkId: documents.at(-1)?.chunkId ?? null,
+    hasMore: documents.length === rebuildLimit(job),
+    lane: isPostgres() ? "postgres-fts" : "portable-lexical",
+  };
+}
+
+async function resolveEmbeddingFamily(): Promise<{
+  family: EmbeddingFamily;
+  set: NonNullable<Awaited<ReturnType<typeof getActiveEmbeddingSet>>>;
+}> {
+  const families = await availableEmbeddingFamilies();
+  const set = await getActiveEmbeddingSet();
+  if (!set) {
+    throw new Error(
+      "Run the embedding bakeoff and persist its winner before rebuilding vectors.",
+    );
+  }
+  const family = set
+    ? (families.find(
+        (candidate) =>
+          candidate.id === set!.family &&
+          candidate.model === set!.model &&
+          candidate.version === set!.version,
+      ) ?? null)
+    : null;
+  if (!family) {
+    throw new Error(
+      set
+        ? `Embedding credentials for active family ${set.family} are unavailable.`
+        : "Run the embedding bakeoff before rebuilding vectors.",
+    );
+  }
+  if (set.dimensions !== family.dimensions) {
+    throw new Error("Active embedding set dimensions do not match its family.");
+  }
+  return { family, set };
+}
+
+function embeddingText(document: AccessibleSearchDocument): string {
+  return [document.title, document.summary, document.body]
+    .filter(Boolean)
+    .join("\n")
+    .slice(0, 24_000);
+}
+
+export async function rebuildVectorBatch(
+  job: ContextJob,
+): Promise<RebuildBatchResult> {
+  if (!isPostgres() || !getCreativeContext().vectorAdapter) {
+    throw new Error(PGVECTOR_REQUIRED_MESSAGE);
+  }
+  const documents = await loadBatch(job);
+  if (!documents.length) {
+    return {
+      processed: 0,
+      indexed: 0,
+      afterChunkId: null,
+      hasMore: false,
+      lane: "pgvector",
+    };
+  }
+  const { family, set } = await resolveEmbeddingFamily();
+  const vectorAdapter = getCreativeContext().vectorAdapter!;
+  let indexed = 0;
+  for (let offset = 0; offset < documents.length; offset += EMBEDDING_BATCH) {
+    const batch = documents.slice(offset, offset + EMBEDDING_BATCH);
+    const texts = batch.map(embeddingText);
+    const vectors = await family.embed(
+      texts.map((text) => ({ text })),
+      "document",
+    );
+    if (vectors.length !== batch.length) {
+      throw new Error(
+        "Embedding provider returned an incomplete rebuild batch.",
+      );
+    }
+    for (let index = 0; index < batch.length; index += 1) {
+      const document = batch[index]!;
+      const vector = vectors[index]!;
+      if (vector.length !== set.dimensions) {
+        throw new Error(
+          `Embedding provider returned ${vector.length} dimensions; expected ${set.dimensions}.`,
+        );
+      }
+      const vectorKey = `creative-context:${set.id}:chunk:${document.chunkId}`;
+      const stored = await vectorAdapter.upsert({
+        embeddingId: vectorKey,
+        embeddingSetId: set.id,
+        vector,
+      });
+      await recordEmbeddingMetadata({
+        embeddingSetId: set.id,
+        itemId: document.itemId,
+        itemVersionId: document.itemVersionId,
+        chunkId: document.chunkId!,
+        targetType: "chunk",
+        targetId: document.chunkId!,
+        vectorKey: stored.vectorKey,
+        dimensions: vector.length,
+        checksum: await sha256(texts[index]!),
+      });
+      indexed += 1;
+    }
+  }
+  await markIndexed(documents);
+  const firstChunkVersions = [
+    ...new Map(
+      documents
+        .filter((document) => document.chunkOrdinal === 0)
+        .map((document) => [document.itemVersionId, document]),
+    ).values(),
+  ];
+  const details = await Promise.all(
+    firstChunkVersions.map((document) =>
+      getCreativeContextItem(document.itemId, document.itemVersionId),
+    ),
+  );
+  const mediaIds = [
+    ...new Set(
+      details.flatMap((detail) =>
+        detail ? detail.media.map((media) => media.id) : [],
+      ),
+    ),
+  ];
+  if (mediaIds.length) {
+    await createJob({
+      sourceId: job.sourceId ?? undefined,
+      kind: "enrich-media",
+      request: {
+        operation: "rebuild-media-embeddings",
+        mediaIds,
+        embeddingSetId: set.id,
+      },
+      progressTotal: mediaIds.length,
+      budget: {
+        eagerLimit: Math.min(25, mediaIds.length),
+        remainingMode: "durable-continuation",
+      },
+    });
+  }
+  return {
+    processed: documents.length,
+    indexed,
+    afterChunkId: documents.at(-1)?.chunkId ?? null,
+    hasMore: documents.length === rebuildLimit(job),
+    lane: "pgvector",
+    embeddingSetId: set.id,
+    mediaQueued: mediaIds.length,
+  };
+}

package/src/jobs/server-worker.spec.ts

@@ -0,0 +1,159 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+  fireInternalDispatch: vi.fn(async () => {}),
+  getH3App: vi.fn(),
+  readBody: vi.fn(),
+  registerDispatcher: vi.fn(),
+  processImport: vi.fn(),
+  processDue: vi.fn(async () => ({ discovered: 0, dispatched: 0, failed: 0 })),
+  h3Use: vi.fn(),
+  verifyInternalToken: vi.fn(() => true),
+  isLocalDatabase: vi.fn(() => true),
+}));
+
+vi.mock("@agent-native/core/db", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("@agent-native/core/db")>()),
+  isLocalDatabase: mocks.isLocalDatabase,
+}));
+
+vi.mock("@agent-native/core/server", () => ({
+  awaitBootstrap: vi.fn(async () => {}),
+  extractInternalBearerToken: vi.fn(() => "token"),
+  fireInternalDispatch: mocks.fireInternalDispatch,
+  FRAMEWORK_ROUTE_PREFIX: "/_agent-native",
+  getH3App: mocks.getH3App,
+  readBody: mocks.readBody,
+  verifyInternalToken: mocks.verifyInternalToken,
+}));
+
+vi.mock("./worker.js", () => ({
+  processCreativeContextImportJob: mocks.processImport,
+  processDueCreativeContextImportJobs: mocks.processDue,
+  registerCreativeContextImportContinuationDispatcher: mocks.registerDispatcher,
+}));
+
+vi.mock("./background-worker.js", () => ({
+  enqueueCreativeContextDailyMaintenance: vi.fn(async () => ({
+    discovered: 0,
+    queued: 0,
+    failed: 0,
+  })),
+  processCreativeContextBackgroundJob: vi.fn(),
+  processDueCreativeContextBackgroundJobs: vi.fn(async () => ({
+    discovered: 0,
+    dispatched: 0,
+    failed: 0,
+  })),
+  registerCreativeContextBackgroundDispatcher: vi.fn(),
+}));
+
+const {
+  CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE,
+  createCreativeContextWorkerPlugin,
+} = await import("./server-worker.js");
+
+describe("creative context hosted worker", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+    vi.setSystemTime(new Date("2026-07-16T17:00:00.000Z"));
+    mocks.fireInternalDispatch.mockClear();
+    mocks.registerDispatcher.mockClear();
+    mocks.processDue.mockClear();
+    mocks.h3Use.mockClear();
+    mocks.verifyInternalToken.mockReset();
+    mocks.verifyInternalToken.mockReturnValue(true);
+    mocks.isLocalDatabase.mockReset();
+    mocks.isLocalDatabase.mockReturnValue(true);
+    mocks.getH3App.mockReturnValue({ use: mocks.h3Use });
+    mocks.readBody.mockReset();
+    vi.unstubAllEnvs();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("mounts the processor, sweeps durable jobs, and schedules a future quota resume", async () => {
+    const nitroApp = {};
+    await createCreativeContextWorkerPlugin({ appId: "slides" })(nitroApp);
+
+    expect(mocks.h3Use).toHaveBeenCalledWith(
+      CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE,
+      expect.any(Function),
+    );
+    expect(mocks.processDue).toHaveBeenCalledWith({ appId: "slides" });
+    const dispatcher = mocks.registerDispatcher.mock.calls[0]?.[0] as (
+      input: Record<string, unknown>,
+    ) => Promise<void>;
+    await dispatcher({
+      jobId: "job-1",
+      ownerEmail: "owner@example.com",
+      appId: "slides",
+      resumeAt: "2026-07-16T17:01:00.000Z",
+    });
+    expect(mocks.fireInternalDispatch).not.toHaveBeenCalled();
+
+    await vi.advanceTimersByTimeAsync(60_000);
+    expect(mocks.fireInternalDispatch).toHaveBeenCalledWith({
+      path: CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE,
+      taskId: "job-1",
+      body: expect.objectContaining({
+        jobId: "job-1",
+        ownerEmail: "owner@example.com",
+        appId: "slides",
+      }),
+    });
+    expect(mocks.processDue).toHaveBeenCalledTimes(2);
+  });
+
+  it("keeps a caller-supplied dispatcher when requested", async () => {
+    await createCreativeContextWorkerPlugin({
+      appId: "content",
+      registerDispatcher: false,
+    })({});
+    expect(mocks.registerDispatcher).not.toHaveBeenCalled();
+  });
+
+  it("rejects invalid signed dispatches", async () => {
+    vi.stubEnv("A2A_SECRET", "configured-secret");
+    mocks.verifyInternalToken.mockReturnValue(false);
+    mocks.readBody.mockResolvedValue({
+      jobId: "job-invalid",
+      ownerEmail: "owner@example.com",
+    });
+    await createCreativeContextWorkerPlugin({ appId: "design" })({});
+    const handler = mocks.h3Use.mock.calls.at(-1)?.[1] as (
+      event: unknown,
+    ) => Promise<Response>;
+    const response = await handler({
+      req: {
+        method: "POST",
+        headers: new Headers({ authorization: "Bearer invalid" }),
+      },
+    });
+    expect(response.status).toBe(401);
+    expect(mocks.processImport).not.toHaveBeenCalled();
+  });
+
+  it("fails closed without A2A_SECRET outside local development", async () => {
+    vi.stubEnv("NODE_ENV", "production");
+    mocks.isLocalDatabase.mockReturnValue(false);
+    mocks.readBody.mockResolvedValue({
+      jobId: "job-hosted",
+      ownerEmail: "owner@example.com",
+    });
+    await createCreativeContextWorkerPlugin({ appId: "assets" })({});
+    const handler = mocks.h3Use.mock.calls.at(-1)?.[1] as (
+      event: unknown,
+    ) => Promise<Response>;
+    const response = await handler({
+      req: { method: "POST", headers: new Headers() },
+    });
+    expect(response.status).toBe(503);
+    await expect(response.json()).resolves.toMatchObject({
+      error: expect.stringContaining("A2A_SECRET"),
+    });
+    expect(mocks.processImport).not.toHaveBeenCalled();
+  });
+});