@agent-native/creative-context 0.1.0

package/src/jobs/server-worker.ts

@@ -0,0 +1,344 @@
+import { isLocalDatabase } from "@agent-native/core/db";
+import {
+  awaitBootstrap,
+  extractInternalBearerToken,
+  fireInternalDispatch,
+  FRAMEWORK_ROUTE_PREFIX,
+  getH3App,
+  readBody,
+  verifyInternalToken,
+  type NitroPluginDef,
+} from "@agent-native/core/server";
+
+import {
+  enqueueCreativeContextDailyMaintenance,
+  processCreativeContextBackgroundJob,
+  processDueCreativeContextBackgroundJobs,
+  registerCreativeContextBackgroundDispatcher,
+} from "./background-worker.js";
+import {
+  processDueCreativeContextImportJobs,
+  processCreativeContextImportJob,
+  registerCreativeContextImportContinuationDispatcher,
+  type CreativeContextImportDispatch,
+} from "./worker.js";
+
+export const CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE = `${FRAMEWORK_ROUTE_PREFIX}/creative-context/process-import`;
+export const CREATIVE_CONTEXT_BACKGROUND_PROCESSOR_ROUTE = `${FRAMEWORK_ROUTE_PREFIX}/creative-context/process-background`;
+
+const mountedApps = new WeakSet<object>();
+const delayedDispatches = new Map<string, ReturnType<typeof setTimeout>>();
+const sweepTimers = new Map<string, ReturnType<typeof setInterval>>();
+const maintenanceTimers = new Map<string, ReturnType<typeof setInterval>>();
+const DEFAULT_SWEEP_INTERVAL_MS = 60_000;
+const DEFAULT_MAINTENANCE_INTERVAL_MS = 24 * 60 * 60_000;
+
+export function createCreativeContextWorkerPlugin(input: {
+  appId: string;
+  registerDispatcher?: boolean;
+}): NitroPluginDef {
+  const appId = input.appId.trim();
+  if (!appId) throw new Error("appId is required.");
+  return async (nitroApp: object) => {
+    if (mountedApps.has(nitroApp)) return;
+    mountedApps.add(nitroApp);
+    await awaitBootstrap(nitroApp);
+    if (input.registerDispatcher !== false) {
+      registerCreativeContextImportContinuationDispatcher((dispatch) =>
+        scheduleHostedDispatch({ ...dispatch, appId }),
+      );
+    }
+    registerCreativeContextBackgroundDispatcher((dispatch) =>
+      scheduleHostedBackgroundDispatch({ ...dispatch, appId }),
+    );
+    startCreativeContextImportSweep({ appId });
+    startCreativeContextDailyMaintenance({ appId });
+    const h3App = getH3App(nitroApp);
+    h3App.use(CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE, async (event: any) => {
+      if (event?.req?.method !== "POST") {
+        return json({ error: "Method not allowed" }, 405);
+      }
+      const body = asDispatch(await readBody(event), appId);
+      if (!body) return json({ error: "Invalid import dispatch" }, 400);
+      const token = extractInternalBearerToken(
+        event.req.headers?.get?.("authorization") ??
+          event.node?.req?.headers?.authorization,
+      );
+      if (!process.env.A2A_SECRET && !unsignedDispatchAllowed()) {
+        return json(
+          {
+            error:
+              "A2A_SECRET is required for hosted creative-context background processing.",
+          },
+          503,
+        );
+      }
+      if (
+        process.env.A2A_SECRET &&
+        (!token || !verifyInternalToken(body.jobId, token))
+      ) {
+        return json({ error: "Invalid internal dispatch token" }, 401);
+      }
+      if (resumeTime(body) > Date.now()) {
+        await scheduleHostedDispatch(body);
+        return json({ accepted: true, deferredUntil: body.resumeAt }, 202);
+      }
+      const work = processCreativeContextImportJob(body);
+      const waitUntil = resolveWaitUntil(event);
+      if (waitUntil) {
+        waitUntil(work);
+        return json({ accepted: true }, 202);
+      }
+      try {
+        const result = await work;
+        return json({
+          accepted: true,
+          yielded: result.yielded,
+          reason: result.reason,
+          status: result.job.status,
+        });
+      } catch (error) {
+        console.error(
+          "[creative-context] hosted import processor failed:",
+          error instanceof Error ? error.message : error,
+        );
+        return json({ error: "Creative context import failed" }, 500);
+      }
+    });
+    h3App.use(
+      CREATIVE_CONTEXT_BACKGROUND_PROCESSOR_ROUTE,
+      async (event: any) => {
+        if (event?.req?.method !== "POST") {
+          return json({ error: "Method not allowed" }, 405);
+        }
+        const body = asDispatch(await readBody(event), appId);
+        if (!body) return json({ error: "Invalid background dispatch" }, 400);
+        const token = extractInternalBearerToken(
+          event.req.headers?.get?.("authorization") ??
+            event.node?.req?.headers?.authorization,
+        );
+        if (!process.env.A2A_SECRET && !unsignedDispatchAllowed()) {
+          return json(
+            {
+              error:
+                "A2A_SECRET is required for hosted creative-context background processing.",
+            },
+            503,
+          );
+        }
+        if (
+          process.env.A2A_SECRET &&
+          (!token || !verifyInternalToken(body.jobId, token))
+        ) {
+          return json({ error: "Invalid internal dispatch token" }, 401);
+        }
+        if (resumeTime(body) > Date.now()) {
+          await scheduleHostedBackgroundDispatch(body);
+          return json({ accepted: true, deferredUntil: body.resumeAt }, 202);
+        }
+        const work = processCreativeContextBackgroundJob(body);
+        const waitUntil = resolveWaitUntil(event);
+        if (waitUntil) {
+          waitUntil(work);
+          return json({ accepted: true }, 202);
+        }
+        try {
+          const job = await work;
+          return json({ accepted: true, status: job.status });
+        } catch (error) {
+          console.error(
+            "[creative-context] hosted background processor failed:",
+            error instanceof Error ? error.message : error,
+          );
+          return json({ error: "Creative context background job failed" }, 500);
+        }
+      },
+    );
+  };
+}
+
+export function startCreativeContextDailyMaintenance(input: {
+  appId: string;
+  intervalMs?: number;
+}): () => void {
+  const appId = input.appId.trim();
+  if (!appId) throw new Error("appId is required.");
+  if (!maintenanceTimers.has(appId)) {
+    const run = () => {
+      void enqueueCreativeContextDailyMaintenance({ appId }).catch((error) => {
+        console.error(
+          "[creative-context] daily maintenance enqueue failed:",
+          error instanceof Error ? error.message : error,
+        );
+      });
+    };
+    run();
+    const timer = setInterval(
+      run,
+      Math.max(60_000, input.intervalMs ?? DEFAULT_MAINTENANCE_INTERVAL_MS),
+    );
+    timer.unref?.();
+    maintenanceTimers.set(appId, timer);
+  }
+  return () => {
+    const timer = maintenanceTimers.get(appId);
+    if (timer) clearInterval(timer);
+    maintenanceTimers.delete(appId);
+  };
+}
+
+function unsignedDispatchAllowed(): boolean {
+  return process.env.NODE_ENV !== "production" && isLocalDatabase();
+}
+
+export function startCreativeContextImportSweep(input: {
+  appId: string;
+  intervalMs?: number;
+}): () => void {
+  const appId = input.appId.trim();
+  if (!appId) throw new Error("appId is required.");
+  const existing = sweepTimers.get(appId);
+  if (!existing) {
+    const run = () => {
+      void Promise.all([
+        processDueCreativeContextImportJobs({ appId }),
+        processDueCreativeContextBackgroundJobs({ appId }),
+      ]).catch((error) => {
+        console.error(
+          "[creative-context] due job sweep failed:",
+          error instanceof Error ? error.message : error,
+        );
+      });
+    };
+    run();
+    const timer = setInterval(
+      run,
+      Math.max(1_000, input.intervalMs ?? DEFAULT_SWEEP_INTERVAL_MS),
+    );
+    timer.unref?.();
+    sweepTimers.set(appId, timer);
+  }
+  return () => {
+    const timer = sweepTimers.get(appId);
+    if (timer) clearInterval(timer);
+    sweepTimers.delete(appId);
+  };
+}
+
+async function scheduleHostedDispatch(
+  dispatch: CreativeContextImportDispatch,
+): Promise<void> {
+  const delay = Math.max(0, resumeTime(dispatch) - Date.now());
+  if (delay > 0) {
+    const existing = delayedDispatches.get(dispatch.jobId);
+    if (existing) clearTimeout(existing);
+    const timer = setTimeout(
+      () => {
+        delayedDispatches.delete(dispatch.jobId);
+        void fireHostedDispatch(dispatch).catch((error) => {
+          console.error(
+            `[creative-context] delayed import dispatch ${dispatch.jobId} failed:`,
+            error instanceof Error ? error.message : error,
+          );
+        });
+      },
+      Math.min(delay, 2_147_000_000),
+    );
+    timer.unref?.();
+    delayedDispatches.set(dispatch.jobId, timer);
+    return;
+  }
+  await fireHostedDispatch(dispatch);
+}
+
+async function fireHostedDispatch(
+  dispatch: CreativeContextImportDispatch,
+): Promise<void> {
+  await fireInternalDispatch({
+    path: CREATIVE_CONTEXT_IMPORT_PROCESSOR_ROUTE,
+    taskId: dispatch.jobId,
+    body: { ...dispatch },
+  });
+}
+
+async function scheduleHostedBackgroundDispatch(
+  dispatch: CreativeContextImportDispatch,
+): Promise<void> {
+  const delay = Math.max(0, resumeTime(dispatch) - Date.now());
+  if (delay > 0) {
+    const key = `background:${dispatch.jobId}`;
+    const existing = delayedDispatches.get(key);
+    if (existing) clearTimeout(existing);
+    const timer = setTimeout(
+      () => {
+        delayedDispatches.delete(key);
+        void fireInternalDispatch({
+          path: CREATIVE_CONTEXT_BACKGROUND_PROCESSOR_ROUTE,
+          taskId: dispatch.jobId,
+          body: { ...dispatch },
+        }).catch((error) => {
+          console.error(
+            `[creative-context] delayed background dispatch ${dispatch.jobId} failed:`,
+            error instanceof Error ? error.message : error,
+          );
+        });
+      },
+      Math.min(delay, 2_147_000_000),
+    );
+    timer.unref?.();
+    delayedDispatches.set(key, timer);
+    return;
+  }
+  await fireInternalDispatch({
+    path: CREATIVE_CONTEXT_BACKGROUND_PROCESSOR_ROUTE,
+    taskId: dispatch.jobId,
+    body: { ...dispatch },
+  });
+}
+
+function asDispatch(
+  value: unknown,
+  appId: string,
+): CreativeContextImportDispatch | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const record = value as Record<string, unknown>;
+  const jobId = text(record.jobId) ?? text(record.taskId);
+  const ownerEmail = text(record.ownerEmail);
+  if (!jobId || !ownerEmail) return null;
+  return {
+    jobId,
+    ownerEmail,
+    orgId: text(record.orgId) ?? null,
+    appId,
+    resumeAt: text(record.resumeAt) ?? null,
+  };
+}
+
+function resolveWaitUntil(
+  event: any,
+): ((promise: Promise<unknown>) => void) | null {
+  const candidate =
+    event?.context?.cloudflare?.context?.waitUntil ??
+    event?.context?.waitUntil ??
+    event?.waitUntil;
+  return typeof candidate === "function"
+    ? (promise) => candidate.call(event.context ?? event, promise)
+    : null;
+}
+
+function resumeTime(input: CreativeContextImportDispatch): number {
+  if (!input.resumeAt) return 0;
+  const timestamp = Date.parse(input.resumeAt);
+  return Number.isFinite(timestamp) ? timestamp : 0;
+}
+
+function text(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function json(value: unknown, status = 200): Response {
+  return new Response(JSON.stringify(value), {
+    status,
+    headers: { "content-type": "application/json" },
+  });
+}

package/src/jobs/store-adapter.ts

@@ -0,0 +1,52 @@
+import {
+  claimJobLease,
+  createJob,
+  getJob,
+  ingestItems,
+  inferBrandDnaProposalFromCorpus,
+  markSourceContainerOwnerVerified,
+  releaseJobLease,
+  reconcileSourceInventory,
+  renewJobLease,
+  updateLeasedJob,
+  upsertSourceInventory,
+  updateContextSource,
+} from "../store/index.js";
+import type { ContextImportJobPersistence } from "./types.js";
+
+export const creativeContextImportJobPersistence: ContextImportJobPersistence =
+  {
+    getJob,
+    claimJobLease,
+    async renewJobLease(input) {
+      return Boolean(await renewJobLease(input));
+    },
+    releaseJobLease,
+    updateLeasedJob,
+    upsertSourceInventory,
+    ingestItems,
+    reconcileSourceInventory,
+    markSourceContainerOwnerVerified,
+    async updateSourceSyncCursor(sourceId, syncCursor) {
+      await updateContextSource(sourceId, { syncCursor });
+    },
+    async inferBrandDnaProposal(input) {
+      const result = await inferBrandDnaProposalFromCorpus(input);
+      return "preview" in result
+        ? { preview: result.preview }
+        : { reason: result.reason };
+    },
+    async enqueueMediaEnrichment(input) {
+      const job = await createJob({
+        sourceId: input.sourceId,
+        kind: "enrich-media",
+        request: { operation: "enrich-media", mediaIds: input.mediaIds },
+        progressTotal: input.mediaIds.length,
+        budget: {
+          eagerLimit: input.eagerLimit,
+          remainingMode: "pending-on-demand",
+        },
+      });
+      return { jobId: job.id };
+    },
+  };

package/src/jobs/types.ts

@@ -0,0 +1,147 @@
+import type {
+  IngestionBudgetLimits,
+  IngestionCheckpoint,
+} from "@agent-native/core/ingestion";
+
+import type { ContextImportConnectorRegistry } from "../connectors/registry.js";
+import type {
+  ContextConnectorExecutionContext,
+  ContextConnectorInventoryItem,
+} from "../connectors/types.js";
+import type {
+  ContextIngestBatch,
+  ContextIngestResult,
+  ContextImportInferenceResult,
+  ContextJob,
+  ContextSource,
+} from "../types.js";
+
+export interface ContextImportCheckpoint extends IngestionCheckpoint {
+  inventory: ContextConnectorInventoryItem[];
+  failures: Array<{ externalId: string; error: string }>;
+  ingested: number;
+  created: number;
+  versioned: number;
+  unchanged: number;
+  inventoryTotal: number;
+  selectionApplied: boolean;
+  deferred: number;
+  hydrationReasons: Record<string, string>;
+  discoveredExternalIds: string[];
+  syncHighWater: string | null;
+}
+
+export interface ContextImportJobPatch {
+  status?: ContextJob["status"];
+  progressCurrent?: number;
+  progressTotal?: number | null;
+  checkpoint?: Record<string, unknown> | null;
+  result?: Record<string, unknown> | null;
+  error?: string | null;
+  startedAt?: string | null;
+  completedAt?: string | null;
+  nextResumeAt?: string | null;
+  budget?: Record<string, unknown> | null;
+}
+
+export interface ContextImportJobPersistence {
+  getJob(jobId: string): Promise<ContextJob | null>;
+  claimJobLease(input: {
+    jobId: string;
+    owner: string;
+    token: string;
+    expiresAt: string;
+  }): Promise<ContextJob | null>;
+  renewJobLease(input: {
+    jobId: string;
+    token: string;
+    expiresAt: string;
+  }): Promise<boolean>;
+  releaseJobLease(input: { jobId: string; token: string }): Promise<boolean>;
+  updateLeasedJob(input: {
+    jobId: string;
+    leaseToken: string;
+    patch: ContextImportJobPatch;
+  }): Promise<ContextJob | null>;
+  upsertSourceInventory?(input: {
+    sourceId: string;
+    items: ContextConnectorInventoryItem[];
+    completedAt?: string;
+  }): Promise<{
+    sourceId: string;
+    received: number;
+    created: number;
+    updated: number;
+    itemIds: string[];
+  }>;
+  ingestItems(batch: ContextIngestBatch): Promise<ContextIngestResult>;
+  reconcileSourceInventory?(input: {
+    sourceId: string;
+    presentExternalIds: string[];
+    completedAt?: string;
+  }): Promise<{ removed: number; restored: number }>;
+  markSourceContainerOwnerVerified?(sourceId: string): Promise<void>;
+  updateSourceSyncCursor?(sourceId: string, syncCursor: string): Promise<void>;
+  inferBrandDnaProposal?(input: {
+    sourceId: string;
+    profileId?: string;
+  }): Promise<{
+    preview?: ContextImportInferenceResult["brandDnaProposal"];
+    reason?: string;
+  }>;
+  enqueueMediaEnrichment?(input: {
+    sourceId: string;
+    mediaIds: string[];
+    eagerLimit: number;
+  }): Promise<{ jobId: string }>;
+}
+
+export interface ContextImportProgressReporter {
+  start(input: {
+    id: string;
+    owner: string;
+    title: string;
+    step: string;
+    metadata: Record<string, unknown>;
+  }): Promise<void>;
+  update(input: {
+    id: string;
+    owner: string;
+    percent: number | null;
+    step: string;
+    metadata: Record<string, unknown>;
+  }): Promise<void>;
+  complete(input: {
+    id: string;
+    owner: string;
+    status: "succeeded" | "failed" | "cancelled";
+    step: string;
+    metadata: Record<string, unknown>;
+  }): Promise<void>;
+}
+
+export interface RunContextImportJobOptions {
+  jobId: string;
+  source: ContextSource;
+  workerId: string;
+  persistence: ContextImportJobPersistence;
+  connectors: ContextImportConnectorRegistry;
+  connectorContext: ContextConnectorExecutionContext;
+  progress?: ContextImportProgressReporter;
+  limits?: Partial<
+    IngestionBudgetLimits & {
+      batchSize: number;
+      inventoryPageSize: number;
+      maxInventoryItems: number;
+      leaseTtlMs: number;
+    }
+  >;
+  continueOnItemError?: boolean;
+  now?: () => number;
+}
+
+export interface RunContextImportJobResult {
+  job: ContextJob;
+  yielded: boolean;
+  reason: "runtime" | "items" | "batches" | "quota" | "lease" | null;
+}

package/src/jobs/worker.ts

@@ -0,0 +1,152 @@
+import { runWithRequestContext } from "@agent-native/core/server";
+
+import { getCreativeContext } from "../server/context.js";
+import {
+  getContextSource,
+  listDueContextImportJobDispatches,
+} from "../store/index.js";
+import { runContextImportJob } from "./import-runner.js";
+import { creativeContextImportJobPersistence } from "./store-adapter.js";
+import type {
+  ContextImportJobPersistence,
+  RunContextImportJobOptions,
+  RunContextImportJobResult,
+} from "./types.js";
+
+export interface CreativeContextImportDispatch {
+  jobId: string;
+  ownerEmail: string;
+  orgId: string | null;
+  appId: string;
+  resumeAt?: string | null;
+}
+
+export type CreativeContextImportContinuationDispatcher = (
+  input: CreativeContextImportDispatch,
+) => Promise<void>;
+
+export interface ProcessCreativeContextImportJobOptions extends CreativeContextImportDispatch {
+  workerId?: string;
+  persistence?: ContextImportJobPersistence;
+  limits?: RunContextImportJobOptions["limits"];
+  dispatchContinuation?: CreativeContextImportContinuationDispatcher;
+}
+
+let configuredDispatcher: CreativeContextImportContinuationDispatcher | null =
+  null;
+
+export function registerCreativeContextImportContinuationDispatcher(
+  dispatcher: CreativeContextImportContinuationDispatcher,
+): () => void {
+  configuredDispatcher = dispatcher;
+  return () => {
+    if (configuredDispatcher === dispatcher) configuredDispatcher = null;
+  };
+}
+
+export async function dispatchCreativeContextImportJob(
+  input: CreativeContextImportDispatch,
+): Promise<void> {
+  const dispatcher = configuredDispatcher ?? localContinuationDispatcher;
+  await dispatcher(input);
+}
+
+export async function processDueCreativeContextImportJobs(input: {
+  appId: string;
+  limit?: number;
+}): Promise<{ discovered: number; dispatched: number; failed: number }> {
+  const due = await listDueContextImportJobDispatches(input);
+  let dispatched = 0;
+  let failed = 0;
+  for (const job of due) {
+    try {
+      await dispatchCreativeContextImportJob(job);
+      dispatched++;
+    } catch (error) {
+      failed++;
+      console.error(
+        `[creative-context] failed to dispatch due import ${job.jobId}:`,
+        error instanceof Error ? error.message : error,
+      );
+    }
+  }
+  return { discovered: due.length, dispatched, failed };
+}
+
+export async function processCreativeContextImportJob(
+  options: ProcessCreativeContextImportJobOptions,
+): Promise<RunContextImportJobResult> {
+  return runWithRequestContext(
+    {
+      userEmail: options.ownerEmail,
+      ...(options.orgId ? { orgId: options.orgId } : {}),
+    },
+    async () => {
+      const configured = getCreativeContext();
+      const source = await getContextSourceForJob(
+        options.jobId,
+        options.persistence ?? creativeContextImportJobPersistence,
+      );
+      const result = await runContextImportJob({
+        jobId: options.jobId,
+        source,
+        workerId:
+          options.workerId ??
+          `creative-context:${process.pid}:${Math.random().toString(36).slice(2)}`,
+        persistence: options.persistence ?? creativeContextImportJobPersistence,
+        connectors: configured.connectors,
+        connectorContext: {
+          ...configured.connectorContext,
+          appId: configured.connectorContext.appId || options.appId,
+          ownerEmail:
+            configured.connectorContext.ownerEmail ?? options.ownerEmail,
+        },
+        limits: options.limits,
+      });
+      if (result.yielded && result.reason !== "lease") {
+        await (
+          options.dispatchContinuation ??
+          configuredDispatcher ??
+          localContinuationDispatcher
+        )({
+          jobId: options.jobId,
+          ownerEmail: options.ownerEmail,
+          orgId: options.orgId,
+          appId: options.appId,
+          resumeAt: result.job.nextResumeAt,
+        });
+      }
+      return result;
+    },
+  );
+}
+
+async function getContextSourceForJob(
+  jobId: string,
+  persistence: ContextImportJobPersistence,
+) {
+  const job = await persistence.getJob(jobId);
+  if (!job?.sourceId)
+    throw new Error("Creative context import job has no source.");
+  const source = await getContextSource(job.sourceId);
+  if (!source) throw new Error("Creative context source was not found.");
+  return source;
+}
+
+async function localContinuationDispatcher(
+  input: CreativeContextImportDispatch,
+): Promise<void> {
+  const resumeAt = input.resumeAt
+    ? new Date(input.resumeAt).getTime()
+    : Date.now();
+  const delay = Math.max(0, Math.min(2_147_000_000, resumeAt - Date.now()));
+  const timer = setTimeout(() => {
+    void processCreativeContextImportJob(input).catch((error) => {
+      console.error(
+        "[creative-context] local import continuation failed:",
+        error instanceof Error ? error.message : error,
+      );
+    });
+  }, delay);
+  timer.unref?.();
+}