@m6d/cortex-server 2.3.0 → 2.4.0

package/src/lib/adapters/database/postgres/threads.ts

@@ -2,13 +2,19 @@
 // Drizzle types query builders per dialect. See DatabaseAdapter in ../index.ts.
 import { and, desc, eq } from "drizzle-orm";
 import { attachments, threads } from "@/db/schema.pg";
+import { threadDeleteEvent, threadUpsertEvent } from "@/cc/sync-events";
 import type { Thread } from "@/types";
 import type { ThreadContextMeta } from "@/ai/context/types";
 import type { DatabaseAdapter } from "@/adapters/database/index";
 import type { PostgresDb } from "./client";
+import { enqueueSyncEvents } from "./outbox";
 import type { StorageAdapter } from "@/adapters/storage/index";
 
-export function createThreadsRepository(db: PostgresDb, storage?: StorageAdapter) {
+export function createThreadsRepository(
+    db: PostgresDb,
+    storage?: StorageAdapter,
+    enqueue: typeof enqueueSyncEvents = enqueueSyncEvents,
+) {
     return {
         async list(userId, agentId) {
             return await db
@@ -28,9 +34,16 @@ export function createThreadsRepository(db: PostgresDb, storage?: StorageAdapter
             return result.length ? (result[0] as Thread) : null;
         },
 
-        async create(userId, agentId) {
-            const result = await db.insert(threads).values({ userId, agentId }).returning();
-            return result[0] as Thread;
+        async create(userId, agentId, options) {
+            return await db.transaction(async function (tx) {
+                const result = await tx
+                    .insert(threads)
+                    .values({ userId, agentId, isTest: options?.isTest ?? false })
+                    .returning();
+                const thread = result[0] as Thread;
+                await enqueue(tx, [threadUpsertEvent(thread)]);
+                return thread;
+            });
         },
 
         /** Object storage has no foreign keys, so attachment paths must be swept before row cascade. */
@@ -46,13 +59,20 @@ export function createThreadsRepository(db: PostgresDb, storage?: StorageAdapter
                 await storage.delete(attachmentPaths);
             }
 
-            // attachments.message_id has no cascade, so those rows go first.
-            await db.delete(attachments).where(eq(attachments.threadId, threadId)).execute();
+            await db.transaction(async function (tx) {
+                // attachments.message_id has no cascade, so those rows go first.
+                await tx.delete(attachments).where(eq(attachments.threadId, threadId)).execute();
 
-            await db
-                .delete(threads)
-                .where(and(eq(threads.id, threadId), eq(threads.userId, userId)))
-                .execute();
+                const deleted = await tx
+                    .delete(threads)
+                    .where(and(eq(threads.id, threadId), eq(threads.userId, userId)))
+                    .returning({ id: threads.id });
+                // The ownership filter can match nothing; a delete event for
+                // someone else's thread id must not touch cc's mirror.
+                if (deleted.length) {
+                    await enqueue(tx, [threadDeleteEvent(threadId)]);
+                }
+            });
         },
 
         async touch(threadId) {
@@ -66,7 +86,15 @@ export function createThreadsRepository(db: PostgresDb, storage?: StorageAdapter
         },
 
         async updateTitle(threadId, title) {
-            await db.update(threads).set({ title }).where(eq(threads.id, threadId)).execute();
+            await db.transaction(async function (tx) {
+                const result = await tx
+                    .update(threads)
+                    .set({ title })
+                    .where(eq(threads.id, threadId))
+                    .returning();
+                const thread = result[0];
+                if (thread) await enqueue(tx, [threadUpsertEvent(thread)]);
+            });
         },
 
         async updateSession(threadId, session) {

package/src/lib/ai/finish-turn.ts

@@ -32,7 +32,7 @@ export async function finishTurn(options: FinishTurnOptions) {
     if (!persistedThread) return;
 
     await config.db.messages.upsert(thread.id, messages);
-    await persistCapturedSteps(capturedSteps, assistantMessageId, config);
+    await persistCapturedSteps(capturedSteps, assistantMessageId, thread.id, config);
 
     config.onStreamFinish?.({ messages: threadMessages, isAborted });
 
@@ -55,12 +55,14 @@ export async function finishTurn(options: FinishTurnOptions) {
 async function persistCapturedSteps(
     capturedSteps: CapturedStep[],
     messageId: string | undefined,
+    threadId: string,
     config: ResolvedCortexAgentConfig,
 ) {
     if (capturedSteps.length === 0 || !messageId) return;
 
     try {
         await config.db.llmRequests.insert(
+            threadId,
             capturedSteps.map((step, index) => ({
                 messageId,
                 stepNumber: index,

package/src/lib/ai/prompt.ts

@@ -15,7 +15,9 @@ export async function resolveSession(
 ) {
     let session = thread.session;
 
-    if (!session && config.loadSessionData) {
+    // Test threads carry cc-minted tokens the customer's session hook has
+    // never seen — do not hand those to it.
+    if (!session && config.loadSessionData && !thread.isTest) {
         session = await config.loadSessionData(token);
         // Persist to DB for future cache hits
         await config.db.threads.updateSession(thread.id, session);

package/src/lib/auth/middleware.ts

@@ -3,11 +3,33 @@ import { getCookie } from "hono/cookie";
 import { HTTPException } from "hono/http-exception";
 import { createRemoteJWKSet, jwtVerify } from "jose";
 import type { AppEnv, AuthedAppEnv } from "@/types";
-import type { CortexConfig } from "@/config";
+import type { ControlCenterConfig, CortexConfig } from "@/config";
+import { isPlaygroundToken, verifyPlaygroundToken } from "./playground";
 
-export function createUserLoaderMiddleware(authConfig: CortexConfig["auth"]) {
+export function createUserLoaderMiddleware(
+    authConfig: CortexConfig["auth"],
+    controlCenter?: ControlCenterConfig,
+) {
     if (!authConfig) {
         return createMiddleware<AppEnv>(async (c, next) => {
+            // Even without primary auth, a cc playground token names a real
+            // caller: verifying it keeps playground users from collapsing
+            // into the shared anonymous identity (and thread namespace).
+            const header = c.req.header("Authorization");
+            const token = header?.startsWith("Bearer ") ? header.slice(7) : null;
+            if (token && isPlaygroundToken(controlCenter, token)) {
+                try {
+                    const payload = await verifyPlaygroundToken(controlCenter!, token);
+                    if (payload.sub) {
+                        c.set("user", { id: payload.sub, token });
+                        await next();
+                        return;
+                    }
+                } catch {
+                    // Invalid playground token: same anonymous fallback as
+                    // the primary path's failed verification.
+                }
+            }
             c.set("user", { id: "00000000-0000-0000-0000-000000000000", token: "" });
             await next();
         });
@@ -45,12 +67,16 @@ export function createUserLoaderMiddleware(authConfig: CortexConfig["auth"]) {
         }
 
         try {
-            const { payload } = await jwtVerify(token, jwks, {
-                issuer: authConfig.issuer,
-                // Absorbs clock skew between the IdP and this server; jose
-                // rejects non-finite values, so this must stay a real number.
-                clockTolerance: 60,
-            });
+            // The unverified `iss` claim routes the token to its issuer's
+            // keys; each is then fully verified against that issuer's JWKS.
+            const { payload } = isPlaygroundToken(controlCenter, token)
+                ? await verifyPlaygroundToken(controlCenter!, token).then((p) => ({ payload: p }))
+                : await jwtVerify(token, jwks, {
+                      issuer: authConfig.issuer,
+                      // Absorbs clock skew between the IdP and this server; jose
+                      // rejects non-finite values, so this must stay a real number.
+                      clockTolerance: 60,
+                  });
 
             if (payload.sub) {
                 c.set("user", { id: payload.sub, token });

package/src/lib/auth/playground.ts

@@ -0,0 +1,72 @@
+import { createRemoteJWKSet, decodeJwt, jwtVerify } from "jose";
+import type { ControlCenterConfig } from "@/config";
+
+/*
+ * cc as a secondary issuer (opt-in via `controlCenter.playground`): its
+ * short-lived end-user JWTs verify against `{url}/api/auth/jwks`, and its
+ * signed `X-Cortex-Playground` header marks threads as test data. The remote
+ * JWKS set is cached per cc URL — jose already caches the fetched keys and
+ * refetches on unknown-kid, so one set per URL is all the state needed.
+ */
+
+const jwksByUrl = new Map<string, ReturnType<typeof createRemoteJWKSet>>();
+
+function ccJwks(config: ControlCenterConfig) {
+    const url = `${config.url.replace(/\/+$/, "")}/api/auth/jwks`;
+    let jwks = jwksByUrl.get(url);
+    if (!jwks) {
+        jwks = createRemoteJWKSet(new URL(url));
+        jwksByUrl.set(url, jwks);
+    }
+    return jwks;
+}
+
+export function ccIssuer(config: ControlCenterConfig) {
+    return config.playgroundIssuer ?? config.url.replace(/\/+$/, "");
+}
+
+/**
+ * True when the token's (unverified) `iss` names the playground-enabled cc —
+ * the routing decision only; verification still happens against the JWKS.
+ */
+export function isPlaygroundToken(controlCenter: ControlCenterConfig | undefined, token: string) {
+    if (!controlCenter?.playground) return false;
+    try {
+        return decodeJwt(token).iss === ccIssuer(controlCenter);
+    } catch {
+        return false;
+    }
+}
+
+export async function verifyPlaygroundToken(controlCenter: ControlCenterConfig, token: string) {
+    const { payload } = await jwtVerify(token, ccJwks(controlCenter), {
+        issuer: ccIssuer(controlCenter),
+        clockTolerance: 60,
+    });
+    // cc gates this claim on its playground permission, so a token any other
+    // cc user minted cannot authenticate here even with a valid signature.
+    if (payload.playground !== true) {
+        throw new Error("cc token lacks the playground claim");
+    }
+    return payload;
+}
+
+/**
+ * Verifies the cc-signed `X-Cortex-Playground` header. Works for both
+ * cc-minted and pasted-real-user-token modes: the header always carries the
+ * caller's own cc JWT, independent of the Authorization token.
+ */
+export async function isPlaygroundRequest(
+    controlCenter: ControlCenterConfig | undefined,
+    request: Request,
+) {
+    if (!controlCenter?.playground) return false;
+    const header = request.headers.get("x-cortex-playground");
+    if (!header) return false;
+    try {
+        await verifyPlaygroundToken(controlCenter, header);
+        return true;
+    } catch {
+        return false;
+    }
+}

package/src/lib/cc/client.ts

@@ -9,8 +9,9 @@ import {
     searchKnowledgeResponseSchema,
     searchServicesResponseSchema,
     searchToolsResponseSchema,
+    syncResponseSchema,
 } from "./types";
-import type { ExecuteRequest, ResolveRequest, SearchRequest } from "./types";
+import type { ExecuteRequest, ResolveRequest, SearchRequest, SyncRequest } from "./types";
 
 type ControlCenterFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
 
@@ -190,6 +191,25 @@ export class ControlCenterClient {
         }
     }
 
+    /**
+     * Pushes one outbox batch to `POST /api/runtime/sync` (server-level, not
+     * under an agent). Returns true — the flusher's license to delete the
+     * rows — only when cc itself confirms the whole batch: a 2xx whose body
+     * parses as the sync response and counts every event. A gateway serving
+     * HTML with a 200, or a partial count, keeps the rows queued for retry.
+     */
+    public async sync(request: SyncRequest, abortSignal?: AbortSignal) {
+        const response = await this.postResponse(
+            "/api/runtime/sync",
+            request,
+            undefined,
+            abortSignal,
+        );
+        if (!response?.ok) return false;
+        const result = await this.parse(response, syncResponseSchema);
+        return result?.accepted === request.events.length;
+    }
+
     private agentPath(agentId: string, suffix: string) {
         return `/api/runtime/agents/${encodeURIComponent(agentId)}${suffix}`;
     }

package/src/lib/cc/flusher.ts

@@ -0,0 +1,147 @@
+import { randomUUID } from "node:crypto";
+import type { Redis } from "ioredis";
+import type { DatabaseAdapter } from "@/adapters/database/index";
+import type { ControlCenterClient } from "./client";
+
+const TICK_MS = 10_000;
+const HEARTBEAT_MS = 60_000;
+const BATCH_SIZE = 500;
+// cc rejects runtime bodies over 256 KiB; the envelope around the events is
+// tiny, so this leaves comfortable headroom.
+const MAX_BATCH_BYTES = 192 * 1024;
+// A lease outlives any tick by construction: the sync POST hard-aborts at
+// the client's 15s request timeout, and peek/ack are local DB calls, so a
+// flush cannot still be in flight when the lease lapses.
+const LEASE_MS = 60_000;
+
+/**
+ * Drains the sync outbox to cc every 10s: peek → sync → ack on success, rows
+ * stay put on failure so the next tick retries. A quiet outbox still POSTs an
+ * empty batch once a minute — that is the heartbeat behind cc's online badge.
+ *
+ * With Redis configured (multi-replica deployments), a per-tick lease keeps
+ * exactly one replica flushing at a time, so batches cannot overlap or land
+ * out of order. Without Redis the deployment is single-process and the
+ * in-flight flag alone suffices.
+ */
+type OutboxRow = Awaited<ReturnType<DatabaseAdapter["outbox"]["peek"]>>[number];
+type SyncEvent = OutboxRow["event"];
+
+/** A contract-valid stand-in for a message too large for cc's request cap:
+ * enough text for the transcript, `content` reduced to a truncation marker.
+ * The full message stays in the server database. Only messages can grow
+ * unbounded; anything else oversized has no meaningful reduction. */
+export function shrinkOversizedEvent(event: SyncEvent) {
+    if (event.type !== "message.upsert") return null;
+    return {
+        ...event,
+        text: event.text === null ? null : event.text.slice(0, 4096),
+        content: { truncated: true },
+    } satisfies SyncEvent;
+}
+
+/** Longest row prefix whose serialized events stay under cc's request body
+ * cap, or null when the head row alone can never fit (a poison pill that
+ * would wedge the queue if kept). */
+export function sliceBatch(rows: OutboxRow[], maxBytes = MAX_BATCH_BYTES) {
+    const batch = new Array<OutboxRow>();
+    let bytes = 0;
+    for (const row of rows) {
+        const size = Buffer.byteLength(JSON.stringify(row.event));
+        if (batch.length === 0 && size > maxBytes) return null;
+        if (bytes + size > maxBytes) break;
+        batch.push(row);
+        bytes += size;
+    }
+    return batch;
+}
+
+export function startSyncFlusher(db: DatabaseAdapter, client: ControlCenterClient, redis?: Redis) {
+    const instanceId = randomUUID();
+    // Keyed by cc target + key hash so only replicas of this deployment
+    // contend; unrelated deployments sharing a Redis flush independently.
+    const leaseKey = `cortex:cc-sync-flusher:${client.cacheKey}`;
+    let inFlight = false;
+
+    async function withLease(run: () => Promise<void>) {
+        if (!redis) return run();
+        const acquired = await redis.set(leaseKey, instanceId, "PX", LEASE_MS, "NX");
+        if (acquired !== "OK") return;
+        try {
+            await run();
+        } finally {
+            // Only the owner releases; an expired lease belongs to whoever
+            // claimed it next, so a plain DEL would steal it.
+            const release = `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end`;
+            await redis.eval(release, 1, leaseKey, instanceId);
+        }
+    }
+
+    let lastSyncedAt = 0;
+    let epoch: string | undefined;
+
+    async function flush() {
+        const rows = await db.outbox.peek(BATCH_SIZE);
+        if (rows.length === 0 && Date.now() - lastSyncedAt < HEARTBEAT_MS) return;
+
+        // The epoch names this database generation; a recreated database
+        // mints a new one, so cc knows its restarted outbox ids are not a
+        // stale sender. Minted once, then cached for the process lifetime.
+        epoch ??= await db.outbox.epoch();
+
+        // Batches are additionally bounded by serialized size so they fit
+        // cc's request body cap. A lone event that can never fit is mirrored
+        // in truncated form when possible; only an unshrinkable one is
+        // dropped (the server DB stays system of record), so it cannot wedge
+        // every later event and heartbeat behind it.
+        let batch = sliceBatch(rows);
+        if (batch === null) {
+            const oversized = rows[0]!;
+            const shrunk = shrinkOversizedEvent(oversized.event);
+            if (shrunk === null) {
+                console.error(
+                    `[cortex-server] sync event ${oversized.id} exceeds cc's request size cap; dropping it from the mirror`,
+                );
+                await db.outbox.ack([oversized.id]);
+                return;
+            }
+            batch = [{ id: oversized.id, event: shrunk }];
+        }
+
+        // Rows are peeked id-asc, so the last id is the batch's fence: cc
+        // discards any same-epoch batch at or below its per-server
+        // high-water mark, so even a flush that somehow outlives its lease
+        // (a stalled process, a response cc finishes applying after the
+        // client timed out) cannot overwrite state a newer flush already
+        // mirrored.
+        const last = batch.at(-1);
+        const accepted = await client.sync(
+            last
+                ? { events: batch.map(({ event }) => event), epoch, seq: last.id }
+                : { events: [], epoch },
+        );
+        if (!accepted) return;
+
+        lastSyncedAt = Date.now();
+        await db.outbox.ack(batch.map(({ id }) => id));
+    }
+
+    async function tick() {
+        if (inFlight) return;
+        inFlight = true;
+        try {
+            await withLease(flush);
+        } catch (err) {
+            console.error("[cortex-server] cc sync flush failed:", err);
+        } finally {
+            inFlight = false;
+        }
+    }
+
+    const interval = setInterval(() => void tick(), TICK_MS);
+    void tick();
+
+    return function stop() {
+        clearInterval(interval);
+    };
+}

package/src/lib/cc/sync-events.ts

@@ -0,0 +1,65 @@
+import { randomUUID } from "node:crypto";
+import type { SyncEvent } from "@cortex/contracts/runtime";
+import type { ChatMessage, Thread, TokenUsage } from "@/types";
+
+/*
+ * Contract event constructors shared by both dialect repositories, so the
+ * mssql/postgres twins only differ in SQL. Events are built once, inside the
+ * write's transaction, and replayed batches reuse the ids stamped here —
+ * that is what makes cc's (serverId, id) upserts idempotent.
+ */
+
+export function threadUpsertEvent(thread: Thread) {
+    return {
+        type: "thread.upsert",
+        id: thread.id,
+        agentSlug: thread.agentId,
+        userId: thread.userId,
+        title: thread.title,
+        isTest: thread.isTest,
+        createdAt: thread.createdAt.toISOString(),
+        updatedAt: thread.updatedAt.toISOString(),
+    } satisfies SyncEvent;
+}
+
+export function threadDeleteEvent(threadId: string) {
+    return { type: "thread.delete", id: threadId } satisfies SyncEvent;
+}
+
+export function messageUpsertEvent(input: {
+    threadId: string;
+    message: ChatMessage;
+    text: string | null | undefined;
+    ordinal: number;
+    createdAt: Date;
+}) {
+    return {
+        type: "message.upsert",
+        id: input.message.id,
+        threadId: input.threadId,
+        role: input.message.role,
+        text: input.text ?? null,
+        ordinal: input.ordinal,
+        content: input.message as unknown as Record<string, unknown>,
+        createdAt: input.createdAt.toISOString(),
+    } satisfies SyncEvent;
+}
+
+export function usageRecordEvent(input: {
+    threadId: string;
+    messageId: string;
+    stepNumber: number;
+    tokenUsage: TokenUsage;
+}) {
+    return {
+        type: "usage.record",
+        // llm_requests ids are DB-generated and never read back; the event
+        // carries its own id, minted once at enqueue time.
+        id: randomUUID(),
+        messageId: input.messageId,
+        threadId: input.threadId,
+        stepNumber: input.stepNumber,
+        tokenUsage: input.tokenUsage,
+        createdAt: new Date().toISOString(),
+    } satisfies SyncEvent;
+}

package/src/lib/cc/types.ts

@@ -12,11 +12,14 @@ export {
     searchKnowledgeResponseSchema,
     searchServicesResponseSchema,
     searchToolsResponseSchema,
+    syncResponseSchema,
     type ExecuteRequest,
     type ResolveRequest,
     type ResolveResponse,
     type RuntimeAgentConfig,
     type SearchRequest,
+    type SyncEvent,
+    type SyncRequest,
     type ToolEmbed,
 } from "@cortex/contracts/runtime";
 

package/src/lib/config.ts

@@ -42,6 +42,17 @@ type RerankerConfig = {
 export type ControlCenterConfig = {
     url: string;
     apiKey: string;
+    /**
+     * Trust cc as a secondary token issuer (its playground). cc-minted
+     * end-user JWTs then authenticate against `{url}/api/auth/jwks`, and a
+     * cc-signed `X-Cortex-Playground` header marks new threads as test data.
+     */
+    playground?: boolean;
+    /**
+     * The `iss` claim cc stamps (its Better Auth `baseURL`), when it differs
+     * from the URL this server dials. Defaults to `url`.
+     */
+    playgroundIssuer?: string;
 };
 
 export type KnowledgeConfig = {

package/src/lib/db/migrations/mssql/20260829050547_friendly_red_shift/migration.sql

@@ -0,0 +1,8 @@
+CREATE TABLE [ai].[sync_outbox] (
+	[id] bigint IDENTITY(1, 1),
+	[event] nvarchar(max) NOT NULL,
+	[created_at] datetime2 NOT NULL,
+	CONSTRAINT [sync_outbox_pkey] PRIMARY KEY([id])
+);
+--> statement-breakpoint
+ALTER TABLE [ai].[threads] ADD [is_test] bit NOT NULL CONSTRAINT [threads_is_test_default] DEFAULT ((0));