@agent-native/core 0.111.4 → 0.112.0

package/corpus/templates/brain/server/lib/brain-health.ts

@@ -1,11 +1,17 @@
 import { getSetting, putSetting } from "@agent-native/core/settings";
-import { accessFilter } from "@agent-native/core/sharing";
+import { accessFilter, assertAccess } from "@agent-native/core/sharing";
 import { listWorkspaceConnectionProviderCatalogForApp } from "@agent-native/core/workspace-connections";
-import { and, desc, eq, inArray, ne } from "drizzle-orm";
+import { and, desc, eq, inArray, isNull, ne, or } from "drizzle-orm";
 
 import { getDb, schema } from "../db/index.js";
 import { nextBrainSourceSyncAt } from "../jobs/sync-sources.js";
-import { parseJson } from "./brain.js";
+import { listAccessibleAudienceIds } from "./audiences.js";
+import { parseJson, readBrainSettings } from "./brain.js";
+import { brainPrivacyReadiness } from "./privacy-readiness.js";
+import {
+  BRAIN_SEARCH_INDEX_VERSION,
+  BRAIN_SENSITIVITY_POLICY_VERSION,
+} from "./search-index-contracts.js";
 import { redactSensitiveText } from "./search.js";
 
 const APP_ID = "brain";
@@ -139,6 +145,7 @@ function sourceConfig(source: SourceRow) {
 function sourceHasSlackChannels(source: SourceRow) {
   if (source.provider !== "slack") return false;
   const config = sourceConfig(source);
+  if (config.includePublicChannels === true) return true;
   const values = [
     config.channelIds,
     config.channels,
@@ -314,6 +321,19 @@ export async function readBrainHealth() {
     )
     .orderBy(desc(schema.brainSources.updatedAt));
   const sourceIds = sourceRows.map((source) => source.id);
+  const audienceIds = await listAccessibleAudienceIds(sourceIds);
+  const adminSourceIds = (
+    await Promise.all(
+      sourceIds.map(async (sourceId) => {
+        try {
+          await assertAccess("brain-source", sourceId, "admin");
+          return sourceId;
+        } catch {
+          return null;
+        }
+      }),
+    )
+  ).filter((sourceId): sourceId is string => Boolean(sourceId));
   const syncRows = sourceIds.length
     ? await db
         .select()
@@ -329,46 +349,146 @@ export async function readBrainHealth() {
     }
   }
 
-  const captureRows = sourceIds.length
-    ? await db
-        .select({
-          sourceId: schema.brainRawCaptures.sourceId,
-          status: schema.brainRawCaptures.status,
-          createdAt: schema.brainRawCaptures.createdAt,
-          capturedAt: schema.brainRawCaptures.capturedAt,
-        })
-        .from(schema.brainRawCaptures)
-        .where(inArray(schema.brainRawCaptures.sourceId, sourceIds))
-    : [];
+  const captureRows =
+    sourceIds.length && audienceIds.length
+      ? await db
+          .select({
+            sourceId: schema.brainRawCaptures.sourceId,
+            status: schema.brainRawCaptures.status,
+            sensitivityDisposition:
+              schema.brainRawCaptures.sensitivityDisposition,
+            createdAt: schema.brainRawCaptures.createdAt,
+            capturedAt: schema.brainRawCaptures.capturedAt,
+          })
+          .from(schema.brainRawCaptures)
+          .innerJoin(
+            schema.brainCaptureAudiences,
+            eq(
+              schema.brainCaptureAudiences.captureId,
+              schema.brainRawCaptures.id,
+            ),
+          )
+          .where(
+            and(
+              inArray(schema.brainRawCaptures.sourceId, sourceIds),
+              eq(schema.brainRawCaptures.sensitivityDisposition, "allowed"),
+              inArray(schema.brainCaptureAudiences.audienceId, audienceIds),
+            ),
+          )
+      : [];
   const proposalRows = await db
     .select({ status: schema.brainProposals.status })
     .from(schema.brainProposals)
-    .where(accessFilter(schema.brainProposals, schema.brainProposalShares));
+    .where(
+      and(
+        accessFilter(schema.brainProposals, schema.brainProposalShares),
+        or(
+          isNull(schema.brainProposals.captureId),
+          audienceIds.length
+            ? inArray(schema.brainProposals.audienceId, audienceIds)
+            : undefined,
+        ),
+      ),
+    );
   const knowledgeRows = await db
     .select({ status: schema.brainKnowledge.status })
     .from(schema.brainKnowledge)
-    .where(accessFilter(schema.brainKnowledge, schema.brainKnowledgeShares));
-  const queueRows = await db
-    .select({
-      status: schema.brainIngestQueue.status,
-      updatedAt: schema.brainIngestQueue.updatedAt,
-      sourceId: schema.brainSources.id,
-    })
-    .from(schema.brainIngestQueue)
-    .innerJoin(
-      schema.brainRawCaptures,
-      eq(schema.brainIngestQueue.captureId, schema.brainRawCaptures.id),
-    )
-    .innerJoin(
-      schema.brainSources,
-      eq(schema.brainRawCaptures.sourceId, schema.brainSources.id),
-    )
     .where(
       and(
-        eq(schema.brainIngestQueue.operation, "distill"),
-        accessFilter(schema.brainSources, schema.brainSourceShares),
+        accessFilter(schema.brainKnowledge, schema.brainKnowledgeShares),
+        or(
+          isNull(schema.brainKnowledge.captureId),
+          audienceIds.length
+            ? inArray(schema.brainKnowledge.audienceId, audienceIds)
+            : undefined,
+        ),
       ),
     );
+  const queueRows = audienceIds.length
+    ? await db
+        .select({
+          operation: schema.brainIngestQueue.operation,
+          status: schema.brainIngestQueue.status,
+          updatedAt: schema.brainIngestQueue.updatedAt,
+          sourceId: schema.brainSources.id,
+        })
+        .from(schema.brainIngestQueue)
+        .innerJoin(
+          schema.brainRawCaptures,
+          eq(schema.brainIngestQueue.captureId, schema.brainRawCaptures.id),
+        )
+        .innerJoin(
+          schema.brainSources,
+          eq(schema.brainRawCaptures.sourceId, schema.brainSources.id),
+        )
+        .innerJoin(
+          schema.brainCaptureAudiences,
+          eq(
+            schema.brainCaptureAudiences.captureId,
+            schema.brainRawCaptures.id,
+          ),
+        )
+        .where(
+          and(
+            accessFilter(schema.brainSources, schema.brainSourceShares),
+            eq(schema.brainRawCaptures.sensitivityDisposition, "allowed"),
+            inArray(schema.brainCaptureAudiences.audienceId, audienceIds),
+          ),
+        )
+    : [];
+  const [sensitivityRows, artifactRows, embeddingRows] = await Promise.all([
+    adminSourceIds.length
+      ? db
+          .select({
+            disposition: schema.brainSensitivityEvents.disposition,
+            confidenceBand: schema.brainSensitivityEvents.confidenceBand,
+          })
+          .from(schema.brainSensitivityEvents)
+          .innerJoin(
+            schema.brainSources,
+            eq(schema.brainSensitivityEvents.sourceId, schema.brainSources.id),
+          )
+          .where(
+            and(
+              accessFilter(schema.brainSources, schema.brainSourceShares),
+              inArray(schema.brainSensitivityEvents.sourceId, adminSourceIds),
+            ),
+          )
+      : Promise.resolve([]),
+    audienceIds.length
+      ? db
+          .select({
+            captureId: schema.brainSearchArtifacts.captureId,
+            status: schema.brainSearchArtifacts.status,
+          })
+          .from(schema.brainSearchArtifacts)
+          .innerJoin(
+            schema.brainSources,
+            eq(schema.brainSearchArtifacts.sourceId, schema.brainSources.id),
+          )
+          .where(
+            and(
+              accessFilter(schema.brainSources, schema.brainSourceShares),
+              inArray(schema.brainSearchArtifacts.audienceId, audienceIds),
+            ),
+          )
+      : Promise.resolve([]),
+    audienceIds.length
+      ? db
+          .select({ status: schema.brainSearchEmbeddings.status })
+          .from(schema.brainSearchEmbeddings)
+          .innerJoin(
+            schema.brainSources,
+            eq(schema.brainSearchEmbeddings.sourceId, schema.brainSources.id),
+          )
+          .where(
+            and(
+              accessFilter(schema.brainSources, schema.brainSourceShares),
+              inArray(schema.brainSearchEmbeddings.audienceId, audienceIds),
+            ),
+          )
+      : Promise.resolve([]),
+  ]);
 
   const configuredCounts = new Map<string, number>();
   for (const source of sourceRows) {
@@ -417,13 +537,51 @@ export async function readBrainHealth() {
     ...sourceRows.map((source) => ({ status: source.status })),
   ]);
   const captureCounts = countStatuses(captureRows);
+  const captureDispositionCounts = countStatuses(
+    captureRows.map((capture) => ({
+      status: capture.sensitivityDisposition,
+    })),
+  );
   const proposalCounts = countStatuses(proposalRows);
   const knowledgeCounts = countStatuses(knowledgeRows);
-  const staleQueue = queueRows.filter(isStaleProcessing).length;
-  const failedQueue = queueRows.filter((row) => row.status === "failed").length;
-  const pendingQueue = queueRows.filter(
+  const distillationQueueRows = queueRows.filter(
+    (row) => row.operation === "distill",
+  );
+  const searchQueueRows = queueRows.filter(
+    (row) =>
+      row.operation === "search-index" || row.operation === "search-unindex",
+  );
+  const staleQueue = distillationQueueRows.filter(isStaleProcessing).length;
+  const failedQueue = distillationQueueRows.filter(
+    (row) => row.status === "failed",
+  ).length;
+  const pendingQueue = distillationQueueRows.filter(
     (row) => row.status === "queued" || row.status === "processing",
   ).length;
+  const staleSearchQueue = searchQueueRows.filter(isStaleProcessing).length;
+  const failedSearchQueue = searchQueueRows.filter(
+    (row) => row.status === "failed",
+  ).length;
+  const pendingSearchQueue = searchQueueRows.filter(
+    (row) => row.status === "queued" || row.status === "processing",
+  ).length;
+  const sensitivityCounts = countStatuses(
+    sensitivityRows.map((row) => ({ status: row.disposition })),
+  );
+  const sensitivityConfidenceCounts = countStatuses(
+    sensitivityRows.map((row) => ({ status: row.confidenceBand })),
+  );
+  const artifactCounts = countStatuses(artifactRows);
+  const embeddingCounts = countStatuses(embeddingRows);
+  const allowedCaptureCount = captureDispositionCounts.allowed ?? 0;
+  const activeArtifactCount = artifactCounts.active ?? 0;
+  const indexedCaptureCount = new Set(
+    artifactRows
+      .filter((artifact) => artifact.status === "active")
+      .map((artifact) => artifact.captureId),
+  ).size;
+  const privacySettings = await readBrainSettings();
+  const privacyClassifier = brainPrivacyReadiness(privacySettings);
   const lastEval = await readBrainEvalSnapshot();
 
   const realSources = sourceSummaries.filter((source) => !source.demo);
@@ -489,6 +647,9 @@ export async function readBrainHealth() {
   ]);
 
   const nextSteps = [
+    !privacyClassifier.configured
+      ? "Configure Brain's approved privacy classifier. Until then, only deterministic-clean captures are stored; uncertain content is quarantined."
+      : null,
     ...setupSteps
       .filter((step) => step.status === "next")
       .map((step) => step.detail),
@@ -563,6 +724,23 @@ export async function readBrainHealth() {
         captureRows.map((capture) => capture.capturedAt ?? capture.createdAt),
       ),
       counts: captureCounts,
+      sensitivityDispositionCounts: captureDispositionCounts,
+    },
+    privacy: {
+      classifier: {
+        ...privacyClassifier,
+      },
+      events: {
+        total: sensitivityRows.length,
+        suppressed: sensitivityCounts.suppressed ?? 0,
+        quarantined: sensitivityCounts.quarantined ?? 0,
+        released: sensitivityCounts.released ?? 0,
+        discarded: sensitivityCounts.discarded ?? 0,
+        expired: sensitivityCounts.expired ?? 0,
+        counts: sensitivityCounts,
+        confidenceCounts: sensitivityConfidenceCounts,
+      },
+      policyVersion: BRAIN_SENSITIVITY_POLICY_VERSION,
     },
     proposals: {
       pending: proposalCounts.pending ?? 0,
@@ -583,8 +761,43 @@ export async function readBrainHealth() {
       pending: pendingQueue,
       failed: failedQueue,
       stale: staleQueue,
-      total: queueRows.length,
-      counts: countStatuses(queueRows),
+      total: distillationQueueRows.length,
+      counts: countStatuses(distillationQueueRows),
+    },
+    searchIndex: {
+      indexVersion: BRAIN_SEARCH_INDEX_VERSION,
+      coverage: {
+        eligibleCaptures: allowedCaptureCount,
+        activeArtifacts: activeArtifactCount,
+        indexedCaptures: indexedCaptureCount,
+        missingCaptures: Math.max(0, allowedCaptureCount - indexedCaptureCount),
+        percent:
+          allowedCaptureCount === 0
+            ? 100
+            : Math.round((indexedCaptureCount / allowedCaptureCount) * 10_000) /
+              100,
+      },
+      artifacts: {
+        total: artifactRows.length,
+        active: artifactCounts.active ?? 0,
+        stale: artifactCounts.stale ?? 0,
+        deleted: artifactCounts.deleted ?? 0,
+        counts: artifactCounts,
+      },
+      embeddings: {
+        total: embeddingRows.length,
+        active: embeddingCounts.active ?? 0,
+        stale: embeddingCounts.stale ?? 0,
+        deleted: embeddingCounts.deleted ?? 0,
+        counts: embeddingCounts,
+      },
+      queue: {
+        pending: pendingSearchQueue,
+        failed: failedSearchQueue,
+        stale: staleSearchQueue,
+        total: searchQueueRows.length,
+        counts: countStatuses(searchQueueRows),
+      },
     },
     retrieval: {
       lastEval,