@agent-native/core 0.111.4 → 0.112.0

package/corpus/templates/brain/jobs/process-ingest-queue.ts

@@ -1,9 +1,11 @@
+import { getDbExec } from "@agent-native/core/db";
 import { runWithRequestContext } from "@agent-native/core/server/request-context";
 import { and, asc, eq, isNull, lte, or } from "drizzle-orm";
 
 import { getDb, schema } from "../server/db/index.js";
 import {
   nowIso,
+  nanoid,
   parseJson,
   readBrainAgentGuidance,
   stableJson,
@@ -15,6 +17,7 @@ type SourceRow = typeof schema.brainSources.$inferSelect;
 
 export interface DistillationAgentContext {
   queue: QueueRow;
+  claimToken: string;
   capture: CaptureRow;
   source: SourceRow;
   payload: Record<string, unknown>;
@@ -91,7 +94,26 @@ async function loadCaptureAndSource(row: QueueRow) {
   return { capture, source };
 }
 
-async function markFailed(row: QueueRow, message: string, payload: object) {
+function claimedRowCondition(row: QueueRow, claimToken?: string) {
+  return claimToken
+    ? and(
+        eq(schema.brainIngestQueue.id, row.id),
+        eq(schema.brainIngestQueue.status, "processing"),
+        eq(schema.brainIngestQueue.leaseToken, claimToken),
+      )
+    : and(
+        eq(schema.brainIngestQueue.id, row.id),
+        eq(schema.brainIngestQueue.status, row.status),
+        eq(schema.brainIngestQueue.updatedAt, row.updatedAt),
+      );
+}
+
+async function markFailed(
+  row: QueueRow,
+  message: string,
+  payload: object,
+  claimToken?: string,
+) {
   const now = nowIso();
   await getDb()
     .update(schema.brainIngestQueue)
@@ -99,15 +121,18 @@ async function markFailed(row: QueueRow, message: string, payload: object) {
       status: "failed",
       payloadJson: stableJson(payload),
       error: message,
+      leaseToken: null,
+      leaseExpiresAt: null,
       updatedAt: now,
     })
-    .where(eq(schema.brainIngestQueue.id, row.id));
+    .where(claimedRowCondition(row, claimToken));
 }
 
 async function requeueDistillation(
   row: QueueRow,
   message: string,
   payload: object,
+  claimToken: string,
 ) {
   const now = nowIso();
   await getDb()
@@ -117,24 +142,61 @@ async function requeueDistillation(
       payloadJson: stableJson(payload),
       error: message,
       runAfter: recheckAt(now),
+      leaseToken: null,
+      leaseExpiresAt: null,
       updatedAt: now,
     })
-    .where(eq(schema.brainIngestQueue.id, row.id));
+    .where(claimedRowCondition(row, claimToken));
 }
 
-async function claimForHeadlessRunner(row: QueueRow, payload: object) {
+export async function claimForHeadlessRunner(row: QueueRow, payload: object) {
   const now = nowIso();
-  await getDb()
-    .update(schema.brainIngestQueue)
-    .set({
-      status: "processing",
-      attempts: row.attempts + 1,
-      payloadJson: stableJson(payload),
-      error: null,
-      runAfter: null,
-      updatedAt: now,
-    })
-    .where(eq(schema.brainIngestQueue.id, row.id));
+  const claimToken = nanoid(16);
+  const leaseExpiresAt = new Date(
+    Date.parse(now) + STALE_PROCESSING_MS,
+  ).toISOString();
+  const result = await getDbExec().execute({
+    sql: `UPDATE brain_ingest_queue
+      SET status = ?, attempts = ?, payload_json = ?, error = NULL,
+          run_after = NULL, lease_token = ?, lease_expires_at = ?, updated_at = ?
+      WHERE id = ? AND status = ? AND updated_at = ?`,
+    args: [
+      "processing",
+      row.attempts + 1,
+      stableJson(payload),
+      claimToken,
+      leaseExpiresAt,
+      now,
+      row.id,
+      row.status,
+      row.updatedAt,
+    ],
+  });
+  return result.rowsAffected > 0 ? claimToken : null;
+}
+
+async function runDeterministicOperation(
+  row: QueueRow,
+  context: NonNullable<Awaited<ReturnType<typeof loadCaptureAndSource>>>,
+) {
+  if (row.operation === "search-index") {
+    const { indexBrainCapture } = await import("../server/lib/search-index.js");
+    await indexBrainCapture(context.capture.id);
+    return;
+  }
+  if (row.operation === "search-unindex") {
+    const { unindexBrainCapture } =
+      await import("../server/lib/search-index.js");
+    await unindexBrainCapture(context.capture.id);
+    return;
+  }
+  if (row.operation === "slack-thread-refresh") {
+    const { refreshSlackThreadCapture } =
+      await import("../server/lib/connectors.js");
+    await refreshSlackThreadCapture(context.source, row.payloadJson);
+    return;
+  }
+  throw new Error(`Unsupported ingest queue operation: ${row.operation}`);
 }
 
 async function latestQueueRow(rowId: string) {
@@ -146,6 +208,19 @@ async function latestQueueRow(rowId: string) {
   return updated;
 }
 
+async function markOperationDone(row: QueueRow, claimToken: string) {
+  await getDb()
+    .update(schema.brainIngestQueue)
+    .set({
+      status: "done",
+      leaseToken: null,
+      leaseExpiresAt: null,
+      error: null,
+      updatedAt: nowIso(),
+    })
+    .where(claimedRowCondition(row, claimToken));
+}
+
 function buildDistillationMessage(
   context: DistillationAgentContext,
   guidance: Awaited<ReturnType<typeof readBrainAgentGuidance>>["guidance"],
@@ -176,7 +251,7 @@ function buildDistillationMessage(
     "1. Call get-capture with includeRawContent=true for this capture id when exact quote validation is needed.",
     "2. Extract only durable company knowledge with exact source quotes.",
     "3. Call write-knowledge for supported entries or proposals.",
-    "4. Call mark-capture-distilled when finished, or mark ignored if excluded.",
+    `4. Call mark-capture-distilled with captureId=${context.capture.id}, queueId=${context.queue.id}, and claimToken=${context.claimToken} when finished, or mark ignored if excluded.`,
   ]
     .filter(Boolean)
     .join("\n");
@@ -260,11 +335,13 @@ export async function processBrainIngestQueueOnce(
       const payload = parseJson<Record<string, unknown>>(row.payloadJson, {});
       if (row.operation === "distill") {
         if (!options.runDistillation) {
-          await claimForHeadlessRunner(row, payload);
+          const claimToken = await claimForHeadlessRunner(row, payload);
+          if (!claimToken) continue;
           await requeueDistillation(
             row,
             "Distillation is still queued; no distillation worker completed this item.",
             { ...payload, lastDistillationCheckAt: now },
+            claimToken,
           );
           deferred.push(row.id);
           continue;
@@ -288,7 +365,8 @@ export async function processBrainIngestQueueOnce(
               ? payload.headlessClaimCount + 1
               : 1,
         };
-        await claimForHeadlessRunner(row, nextPayload);
+        const claimToken = await claimForHeadlessRunner(row, nextPayload);
+        if (!claimToken) continue;
 
         try {
           const runner =
@@ -300,7 +378,14 @@ export async function processBrainIngestQueueOnce(
             },
             () =>
               runner({
-                queue: row,
+                queue: {
+                  ...row,
+                  attempts: row.attempts + 1,
+                  payloadJson: stableJson(nextPayload),
+                  status: "processing",
+                  leaseToken: claimToken,
+                },
+                claimToken,
                 capture: contextRows.capture,
                 source: contextRows.source,
                 payload: nextPayload,
@@ -310,16 +395,23 @@ export async function processBrainIngestQueueOnce(
           const message = err instanceof Error ? err.message : String(err);
           const failedPermanently = row.attempts + 1 >= MAX_ATTEMPTS;
           if (failedPermanently) {
-            await markFailed(row, message, {
-              ...nextPayload,
-              failedAt: nowIso(),
-            });
+            await markFailed(
+              row,
+              message,
+              { ...nextPayload, failedAt: nowIso() },
+              claimToken,
+            );
             failed.push(row.id);
           } else {
-            await requeueDistillation(row, message, {
-              ...nextPayload,
-              lastHeadlessDistillationErrorAt: nowIso(),
-            });
+            await requeueDistillation(
+              row,
+              message,
+              {
+                ...nextPayload,
+                lastHeadlessDistillationErrorAt: nowIso(),
+              },
+              claimToken,
+            );
             deferred.push(row.id);
           }
           continue;
@@ -333,6 +425,7 @@ export async function processBrainIngestQueueOnce(
             row,
             "Headless distillation agent did not mark this capture distilled or ignored.",
             { ...nextPayload, failedAt: nowIso() },
+            claimToken,
           );
           failed.push(row.id);
         } else {
@@ -340,22 +433,163 @@ export async function processBrainIngestQueueOnce(
             row,
             "Headless distillation agent did not mark this capture distilled or ignored.",
             { ...nextPayload, lastHeadlessDistillationAt: nowIso() },
+            claimToken,
           );
           deferred.push(row.id);
         }
         continue;
       }
-
-      await claimForHeadlessRunner(row, payload);
-      await markFailed(
-        row,
-        `Unsupported ingest queue operation: ${row.operation}`,
-        {
+      if (row.operation === "search-unindex") {
+        if (!row.captureId) {
+          await markFailed(row, "Search unindex operation had no capture id.", {
+            ...payload,
+            failedAt: now,
+          });
+          failed.push(row.id);
+          continue;
+        }
+        const claimToken = await claimForHeadlessRunner(row, payload);
+        if (!claimToken) continue;
+        try {
+          const { unindexBrainCapture } =
+            await import("../server/lib/search-index.js");
+          await unindexBrainCapture(row.captureId);
+          await markOperationDone(row, claimToken);
+          processed.push(row.id);
+        } catch (error) {
+          await markFailed(
+            row,
+            error instanceof Error ? error.message : String(error),
+            { ...payload, failedAt: nowIso() },
+            claimToken,
+          );
+          failed.push(row.id);
+        }
+        continue;
+      }
+      if (row.operation === "slack-thread-refresh") {
+        const [source] = row.sourceId
+          ? await db
+              .select()
+              .from(schema.brainSources)
+              .where(eq(schema.brainSources.id, row.sourceId))
+              .limit(1)
+          : [];
+        if (!source) {
+          await markFailed(row, "Slack refresh source was missing.", {
+            ...payload,
+            failedAt: now,
+          });
+          failed.push(row.id);
+          continue;
+        }
+        const claimToken = await claimForHeadlessRunner(row, payload);
+        if (!claimToken) continue;
+        try {
+          const { refreshSlackThreadCapture } =
+            await import("../server/lib/connectors.js");
+          await runWithRequestContext(
+            {
+              userEmail: source.ownerEmail,
+              orgId: source.orgId ?? undefined,
+            },
+            () => refreshSlackThreadCapture(source, row.payloadJson),
+          );
+          await markOperationDone(row, claimToken);
+          processed.push(row.id);
+        } catch (error) {
+          await markFailed(
+            row,
+            error instanceof Error ? error.message : String(error),
+            { ...payload, failedAt: nowIso() },
+            claimToken,
+          );
+          failed.push(row.id);
+        }
+        continue;
+      }
+      if (row.operation === "sync") {
+        const [source] = row.sourceId
+          ? await db
+              .select()
+              .from(schema.brainSources)
+              .where(eq(schema.brainSources.id, row.sourceId))
+              .limit(1)
+          : [];
+        if (!source) {
+          await markFailed(row, "Source sync operation had no source.", {
+            ...payload,
+            failedAt: now,
+          });
+          failed.push(row.id);
+          continue;
+        }
+        const claimToken = await claimForHeadlessRunner(row, payload);
+        if (!claimToken) continue;
+        try {
+          const { runConnectorSync } =
+            await import("../server/lib/connectors.js");
+          await runWithRequestContext(
+            {
+              userEmail: source.ownerEmail,
+              orgId: source.orgId ?? undefined,
+            },
+            () => runConnectorSync(source),
+          );
+          await markOperationDone(row, claimToken);
+          processed.push(row.id);
+        } catch (error) {
+          await markFailed(
+            row,
+            error instanceof Error ? error.message : String(error),
+            { ...payload, failedAt: nowIso() },
+            claimToken,
+          );
+          failed.push(row.id);
+        }
+        continue;
+      }
+      const contextRows = await loadCaptureAndSource(row);
+      if (!contextRows) {
+        await markFailed(row, "Queue capture or source was missing.", {
           ...payload,
           failedAt: now,
-        },
-      );
-      failed.push(row.id);
+        });
+        failed.push(row.id);
+        continue;
+      }
+      const claimToken = await claimForHeadlessRunner(row, payload);
+      if (!claimToken) continue;
+      try {
+        await runWithRequestContext(
+          {
+            userEmail: contextRows.source.ownerEmail,
+            orgId: contextRows.source.orgId ?? undefined,
+          },
+          () => runDeterministicOperation(row, contextRows),
+        );
+        await markOperationDone(row, claimToken);
+        processed.push(row.id);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (row.attempts + 1 >= MAX_ATTEMPTS) {
+          await markFailed(
+            row,
+            message,
+            { ...payload, failedAt: nowIso() },
+            claimToken,
+          );
+          failed.push(row.id);
+        } else {
+          await requeueDistillation(
+            row,
+            message,
+            { ...payload, lastOperationErrorAt: nowIso() },
+            claimToken,
+          );
+          deferred.push(row.id);
+        }
+      }
     }
 
     return { processed, deferred, failed };

package/corpus/templates/brain/package.json

@@ -14,6 +14,7 @@
     "demo:seed": "agent-native action seed-demo-data",
     "demo:eval": "agent-native action run-demo-eval",
     "retrieval:eval": "agent-native action run-retrieval-eval",
+    "privacy:eval": "vitest --run evals/privacy-leakage.test.ts --config vitest.config.ts",
     "eval:ci": "sh -c 'set -e; db=\"./data/brain-evals-ci-$$.db\"; cleanup() { rm -f \"$db\" \"$db-shm\" \"$db-wal\"; }; trap cleanup EXIT; cleanup; DATABASE_URL=\"file:$db\" AGENT_USER_EMAIL=\"brain-evals@agent-native.test\" agent-native action run-demo-eval; DATABASE_URL=\"file:$db\" AGENT_USER_EMAIL=\"brain-evals@agent-native.test\" agent-native action run-retrieval-eval'",
     "script": "agent-native script"
   },

package/corpus/templates/brain/server/db/index.ts

@@ -13,6 +13,8 @@ registerShareableResource({
   displayName: "Brain Source",
   titleColumn: "title",
   getResourcePath: (source) => `/sources/${source.id}`,
+  allowPublic: false,
+  requireOrgMemberForUserShares: true,
   getDb,
 });
 
@@ -23,6 +25,8 @@ registerShareableResource({
   displayName: "Brain Knowledge",
   titleColumn: "title",
   getResourcePath: (knowledge) => `/knowledge/${knowledge.id}`,
+  allowPublic: false,
+  requireOrgMemberForUserShares: true,
   getDb,
 });
 
@@ -33,5 +37,19 @@ registerShareableResource({
   displayName: "Brain Proposal",
   titleColumn: "title",
   getResourcePath: (proposal) => `/proposals/${proposal.id}`,
+  allowPublic: false,
+  requireOrgMemberForUserShares: true,
+  getDb,
+});
+
+registerShareableResource({
+  type: "brain-project",
+  resourceTable: schema.brainProjects,
+  sharesTable: schema.brainProjectShares,
+  displayName: "Brain Project",
+  titleColumn: "title",
+  getResourcePath: (project) => `/search?projectId=${project.id}`,
+  allowPublic: false,
+  requireOrgMemberForUserShares: true,
   getDb,
 });