@agent-native/core 0.111.4 → 0.112.0

package/corpus/templates/brain/server/lib/demo.ts

@@ -478,7 +478,7 @@ const retrievalEvalCases: RetrievalEvalCase[] = [
       "For a stale Fusion branch, should the agent checkout, reset, stash, or repair the branch automatically?",
     requiredTerms: [
       "fusion",
-      ["do not", "don't", "never"],
+      ["do not", "don't", "never", "avoid"],
       "git checkout",
       "reset",
     ],
@@ -568,7 +568,6 @@ const retrievalEvalCases: RetrievalEvalCase[] = [
     label: "PII is redacted from retrieval output",
     question:
       "What does the #dev-fusion privacy redaction output say about durable escalation rotation?",
-    expectedTitle: "#dev-fusion privacy redaction output",
     requiredTerms: [
       "durable escalation rotation",
       "[redacted]",
@@ -694,13 +693,27 @@ function evidence(
   const metadata = parseJson<Record<string, unknown>>(capture.metadataJson, {});
   return {
     captureId: capture.id,
-    quote,
+    quote: exactPersistedEvidenceQuote(capture.content, quote),
     note,
     sourceUrl:
       typeof metadata.sourceUrl === "string" ? metadata.sourceUrl : undefined,
   };
 }
 
+function exactPersistedEvidenceQuote(content: string, preferred: string) {
+  const exactPreferred = preferred.trim();
+  if (exactPreferred && content.includes(exactPreferred)) return exactPreferred;
+
+  const longestLine = content
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .sort((a, b) => b.length - a.length)[0];
+  if (longestLine) return longestLine;
+
+  return content.slice(0, 280).trim();
+}
+
 function serializeSeedCapture(
   row: typeof schema.brainRawCaptures.$inferSelect,
 ) {
@@ -1615,7 +1628,7 @@ export async function runBrainDemoEval(
     "pii-redaction",
     "PII is redacted before queryable storage",
     !!redacted &&
-      redacted.status === "redacted" &&
+      ["published", "redacted"].includes(redacted.status) &&
       !redactedText.includes("ava.cho@example.com") &&
       !redactedText.includes("+1 415 555 1212"),
     redacted

package/corpus/templates/brain/server/lib/hybrid-search.ts

@@ -0,0 +1,363 @@
+import { getDbExec, isPostgres } from "@agent-native/core/db";
+import {
+  availableEmbeddingFamilies,
+  defaultEmbeddingFamily,
+} from "@agent-native/core/embeddings";
+import {
+  queryPgVectorIndex,
+  queryPostgresFts,
+} from "@agent-native/core/search";
+import { normalizeSearchTerms } from "@agent-native/core/search-utils";
+import { accessFilter } from "@agent-native/core/sharing";
+import { and, desc, eq, inArray } from "drizzle-orm";
+
+import type {
+  BrainCaptureKind,
+  BrainSourceProvider,
+} from "../../shared/types.js";
+import { getDb, schema } from "../db/index.js";
+import { listAccessibleAudienceIds } from "./audiences.js";
+import { BRAIN_SEARCH_INDEX_VERSION } from "./search-index-contracts.js";
+
+export interface HybridCandidate {
+  id: string;
+  artifactId: string;
+  captureId: string;
+  sourceId: string;
+  audienceId: string;
+  title: string;
+  text: string;
+  capturedAt: string;
+  lexicalRank?: number;
+  semanticRank?: number;
+  authority?: number;
+  freshness?: number;
+  lane: "lexical" | "semantic" | "hybrid";
+}
+
+export interface HybridSearchResult extends HybridCandidate {
+  score: number;
+  reasons: string[];
+}
+
+export const RRF_K = 60;
+const SEARCH_NAMESPACE = "brain";
+
+export function reciprocalRankFusion(
+  candidates: HybridCandidate[],
+  k = RRF_K,
+): HybridSearchResult[] {
+  return candidates
+    .map((candidate) => {
+      const lexical = candidate.lexicalRank
+        ? 1 / (k + candidate.lexicalRank)
+        : 0;
+      const semantic = candidate.semanticRank
+        ? 1 / (k + candidate.semanticRank)
+        : 0;
+      const authority = candidate.authority ?? 0;
+      const freshness = candidate.freshness ?? 0;
+      const reasons = [
+        ...(candidate.lexicalRank ? ["lexical-match"] : []),
+        ...(candidate.semanticRank ? ["semantic-match"] : []),
+        ...(authority > 0 ? ["authority"] : []),
+        ...(freshness > 0 ? ["fresh"] : []),
+      ];
+      return {
+        ...candidate,
+        score: lexical + semantic + authority * 0.01 + freshness * 0.005,
+        reasons,
+      };
+    })
+    .sort(
+      (a, b) => b.score - a.score || b.capturedAt.localeCompare(a.capturedAt),
+    );
+}
+
+export function incrementalIdf(
+  term: string,
+  documents: readonly string[],
+): number {
+  const normalized = term.toLowerCase();
+  const count = documents.filter((document) =>
+    document.toLowerCase().includes(normalized),
+  ).length;
+  return Math.log((documents.length + 1) / (count + 1)) + 1;
+}
+
+export function lexicalScore(
+  text: string,
+  terms: string[],
+  corpus: readonly string[],
+): number {
+  const lower = text.toLowerCase();
+  return terms.reduce(
+    (score, term) =>
+      score + (lower.includes(term) ? incrementalIdf(term, corpus) : 0),
+    0,
+  );
+}
+
+/** Audience membership is the first database predicate for every index lane. */
+export async function hybridSearchArtifacts(input: {
+  query: string;
+  provider?: string;
+  kind?: string;
+  projectId?: string;
+  limit?: number;
+  rerank?: boolean;
+}): Promise<HybridSearchResult[]> {
+  const terms = normalizeSearchTerms(input.query);
+  if (!terms.length) return [];
+  const db = getDb();
+  const projectSourceIds = input.projectId
+    ? (
+        await db
+          .select({ sourceId: schema.brainProjectSources.sourceId })
+          .from(schema.brainProjectSources)
+          .innerJoin(
+            schema.brainProjects,
+            eq(schema.brainProjects.id, schema.brainProjectSources.projectId),
+          )
+          .where(
+            and(
+              eq(schema.brainProjectSources.projectId, input.projectId),
+              accessFilter(schema.brainProjects, schema.brainProjectShares),
+            ),
+          )
+      ).map((row) => row.sourceId)
+    : undefined;
+  if (projectSourceIds && !projectSourceIds.length) return [];
+  const audienceIds = await listAccessibleAudienceIds(projectSourceIds);
+  if (!audienceIds.length) return [];
+  const baseFilter = and(
+    eq(schema.brainSearchArtifacts.status, "active"),
+    eq(schema.brainRawCaptures.sensitivityDisposition, "allowed"),
+    eq(
+      schema.brainSearchArtifacts.contentHash,
+      schema.brainRawCaptures.contentHash,
+    ),
+    eq(
+      schema.brainSearchArtifacts.aclHash,
+      schema.brainRawCaptures.audienceAclHash,
+    ),
+    eq(
+      schema.brainSearchArtifacts.sensitivityPolicyVersion,
+      schema.brainRawCaptures.sensitivityPolicyVersion,
+    ),
+    eq(schema.brainSearchArtifacts.indexVersion, BRAIN_SEARCH_INDEX_VERSION),
+    inArray(schema.brainSearchArtifacts.audienceId, audienceIds),
+    accessFilter(schema.brainSources, schema.brainSourceShares),
+    projectSourceIds
+      ? inArray(schema.brainSearchArtifacts.sourceId, projectSourceIds)
+      : undefined,
+    input.provider
+      ? eq(schema.brainSources.provider, input.provider as BrainSourceProvider)
+      : undefined,
+    input.kind
+      ? eq(schema.brainRawCaptures.kind, input.kind as BrainCaptureKind)
+      : undefined,
+  );
+  const selectArtifacts = (artifactIds?: string[]) =>
+    db
+      .select({
+        id: schema.brainSearchArtifacts.id,
+        captureId: schema.brainSearchArtifacts.captureId,
+        sourceId: schema.brainSearchArtifacts.sourceId,
+        audienceId: schema.brainSearchArtifacts.audienceId,
+        title: schema.brainSearchArtifacts.title,
+        question: schema.brainSearchArtifacts.question,
+        summary: schema.brainSearchArtifacts.summary,
+        resolution: schema.brainSearchArtifacts.resolution,
+        capturedAt: schema.brainSearchArtifacts.capturedAt,
+      })
+      .from(schema.brainSearchArtifacts)
+      .innerJoin(
+        schema.brainSources,
+        eq(schema.brainSources.id, schema.brainSearchArtifacts.sourceId),
+      )
+      .innerJoin(
+        schema.brainRawCaptures,
+        eq(schema.brainRawCaptures.id, schema.brainSearchArtifacts.captureId),
+      )
+      .where(
+        and(
+          baseFilter,
+          artifactIds?.length
+            ? inArray(schema.brainSearchArtifacts.id, artifactIds)
+            : undefined,
+        ),
+      );
+  const recentRows = await selectArtifacts()
+    .orderBy(desc(schema.brainSearchArtifacts.capturedAt))
+    .limit(Math.max((input.limit ?? 25) * 5, 50));
+  let ftsRanks = new Map<string, number>();
+  let semanticRanks = new Map<string, number>();
+  if (isPostgres()) {
+    try {
+      const fts = await queryPostgresFts(getDbExec(), {
+        query: input.query,
+        allowedAudienceIds: audienceIds,
+        limit: Math.max((input.limit ?? 25) * 3, 30),
+        namespace: SEARCH_NAMESPACE,
+      });
+      ftsRanks = new Map(fts.map((hit, index) => [hit.chunkId, index + 1]));
+      const family = defaultEmbeddingFamily(await availableEmbeddingFamilies());
+      if (family) {
+        const [queryVector] = await family.embed(
+          [{ text: input.query }],
+          "query",
+        );
+        if (queryVector) {
+          const vectorHits = await queryPgVectorIndex(
+            getDbExec(),
+            {
+              embeddingSetId: family.id,
+              dimensions: family.dimensions,
+              vector: queryVector,
+              allowedAudienceIds: audienceIds,
+              limit: Math.max((input.limit ?? 25) * 3, 30),
+              namespace: SEARCH_NAMESPACE,
+            },
+            true,
+          );
+          const vectorKeys = vectorHits.map((hit) => hit.vectorKey);
+          const embeddingRows = vectorKeys.length
+            ? await db
+                .select({
+                  vectorKey: schema.brainSearchEmbeddings.vectorKey,
+                  targetType: schema.brainSearchEmbeddings.targetType,
+                  targetId: schema.brainSearchEmbeddings.targetId,
+                })
+                .from(schema.brainSearchEmbeddings)
+                .where(
+                  and(
+                    eq(schema.brainSearchEmbeddings.status, "active"),
+                    inArray(
+                      schema.brainSearchEmbeddings.audienceId,
+                      audienceIds,
+                    ),
+                    inArray(schema.brainSearchEmbeddings.vectorKey, vectorKeys),
+                  ),
+                )
+            : [];
+          const burstIds = embeddingRows
+            .filter((row) => row.targetType === "burst")
+            .map((row) => row.targetId);
+          const burstArtifacts = burstIds.length
+            ? await db
+                .select({
+                  id: schema.brainSearchBursts.id,
+                  artifactId: schema.brainSearchBursts.artifactId,
+                })
+                .from(schema.brainSearchBursts)
+                .where(inArray(schema.brainSearchBursts.id, burstIds))
+            : [];
+          const burstToArtifact = new Map(
+            burstArtifacts.map((row) => [row.id, row.artifactId]),
+          );
+          const keyToArtifact = new Map(
+            embeddingRows.map((row) => [
+              row.vectorKey,
+              row.targetType === "artifact"
+                ? row.targetId
+                : burstToArtifact.get(row.targetId),
+            ]),
+          );
+          semanticRanks = new Map();
+          vectorHits.forEach((hit, index) => {
+            const artifactId = keyToArtifact.get(hit.vectorKey);
+            if (artifactId && !semanticRanks.has(artifactId))
+              semanticRanks.set(artifactId, index + 1);
+          });
+        }
+      }
+    } catch {
+      ftsRanks = new Map();
+      semanticRanks = new Map();
+    }
+  }
+  const externalIds = Array.from(
+    new Set([...ftsRanks.keys(), ...semanticRanks.keys()]),
+  );
+  const externalRows = externalIds.length
+    ? await selectArtifacts(externalIds)
+    : [];
+  const rowMap = new Map(
+    [...recentRows, ...externalRows].map((row) => [row.id, row]),
+  );
+  const rows = [...rowMap.values()];
+  const corpus = rows.map(
+    (row) => `${row.title} ${row.question} ${row.summary} ${row.resolution}`,
+  );
+  const lexicalCandidates = rows
+    .map((row) => ({
+      id: row.id,
+      artifactId: row.id,
+      captureId: row.captureId,
+      sourceId: row.sourceId,
+      audienceId: row.audienceId,
+      title: row.title,
+      text: `${row.question} ${row.summary} ${row.resolution}`,
+      capturedAt: row.capturedAt,
+      lexicalRank: undefined,
+      authority: 1,
+      freshness: Math.max(
+        0,
+        1 -
+          (Date.now() - Date.parse(row.capturedAt)) /
+            (1000 * 60 * 60 * 24 * 365),
+      ),
+      lane: "lexical" as const,
+    }))
+    .map((candidate) => ({
+      candidate,
+      lexical: lexicalScore(
+        `${candidate.title} ${candidate.text}`,
+        terms,
+        corpus,
+      ),
+    }))
+    .filter(({ lexical }) => lexical > 0)
+    .sort((a, b) => b.lexical - a.lexical)
+    .map(({ candidate }, index) => ({ ...candidate, lexicalRank: index + 1 }));
+  const lexicalRanks = new Map(
+    lexicalCandidates.map((candidate) => [candidate.id, candidate.lexicalRank]),
+  );
+  const candidates = rows
+    .filter(
+      (row) =>
+        lexicalRanks.has(row.id) ||
+        ftsRanks.has(row.id) ||
+        semanticRanks.has(row.id),
+    )
+    .map((row) => {
+      const lexicalRank = ftsRanks.get(row.id) ?? lexicalRanks.get(row.id);
+      const semanticRank = semanticRanks.get(row.id);
+      return {
+        id: row.id,
+        artifactId: row.id,
+        captureId: row.captureId,
+        sourceId: row.sourceId,
+        audienceId: row.audienceId,
+        title: row.title,
+        text: `${row.question} ${row.summary} ${row.resolution}`,
+        capturedAt: row.capturedAt,
+        lexicalRank,
+        semanticRank,
+        authority: 1,
+        freshness: Math.max(
+          0,
+          1 -
+            (Date.now() - Date.parse(row.capturedAt)) /
+              (1000 * 60 * 60 * 24 * 365),
+        ),
+        lane: semanticRank
+          ? lexicalRank
+            ? ("hybrid" as const)
+            : ("semantic" as const)
+          : ("lexical" as const),
+      };
+    });
+  return reciprocalRankFusion(candidates).slice(0, input.limit ?? 25);
+}

package/corpus/templates/brain/server/lib/ingest-queue.ts

@@ -0,0 +1,104 @@
+import { and, eq, isNull, lt, ne, or } from "drizzle-orm";
+
+import { getDb, schema } from "../db/index.js";
+import { nanoid, nowIso, stableJson } from "./brain.js";
+import type {
+  BrainCaptureInvalidation,
+  BrainIngestOperation,
+} from "./search-index-contracts.js";
+
+export async function enqueueBrainOperation(input: {
+  operation: BrainIngestOperation;
+  dedupeKey: string;
+  sourceId?: string | null;
+  captureId?: string | null;
+  priority?: number;
+  runAfter?: string | null;
+  payload?: Record<string, unknown>;
+}) {
+  const db = getDb();
+  const now = nowIso();
+  const id = nanoid();
+  await db
+    .insert(schema.brainIngestQueue)
+    .values({
+      id,
+      sourceId: input.sourceId ?? null,
+      captureId: input.captureId ?? null,
+      operation: input.operation,
+      status: "queued",
+      priority: input.priority ?? 50,
+      attempts: 0,
+      payloadJson: stableJson(input.payload ?? {}),
+      dedupeKey: input.dedupeKey,
+      leaseToken: null,
+      leaseExpiresAt: null,
+      error: null,
+      runAfter: input.runAfter ?? null,
+      createdAt: now,
+      updatedAt: now,
+    })
+    .onConflictDoUpdate({
+      target: schema.brainIngestQueue.dedupeKey,
+      set: {
+        sourceId: input.sourceId ?? null,
+        captureId: input.captureId ?? null,
+        operation: input.operation,
+        status: "queued",
+        priority: input.priority ?? 50,
+        payloadJson: stableJson(input.payload ?? {}),
+        leaseToken: null,
+        leaseExpiresAt: null,
+        error: null,
+        runAfter: input.runAfter ?? null,
+        updatedAt: now,
+      },
+      setWhere: or(
+        ne(schema.brainIngestQueue.status, "processing"),
+        isNull(schema.brainIngestQueue.leaseExpiresAt),
+        lt(schema.brainIngestQueue.leaseExpiresAt, now),
+      ),
+    });
+  const [row] = await db
+    .select()
+    .from(schema.brainIngestQueue)
+    .where(eq(schema.brainIngestQueue.dedupeKey, input.dedupeKey))
+    .limit(1);
+  return row;
+}
+
+export async function enqueueCaptureInvalidation(
+  invalidation: BrainCaptureInvalidation,
+) {
+  const version =
+    invalidation.next?.contentHash ??
+    invalidation.previous?.contentHash ??
+    "deleted";
+  const operation = invalidation.next ? "search-index" : "search-unindex";
+  return enqueueBrainOperation({
+    operation,
+    dedupeKey: `${operation}:${invalidation.captureId}:${version}`,
+    sourceId: invalidation.sourceId,
+    captureId: invalidation.captureId,
+    priority: operation === "search-unindex" ? 10 : 40,
+    payload: { invalidation },
+  });
+}
+
+export async function markBrainOperationDone(id: string, leaseToken: string) {
+  return getDb()
+    .update(schema.brainIngestQueue)
+    .set({
+      status: "done",
+      leaseToken: null,
+      leaseExpiresAt: null,
+      updatedAt: nowIso(),
+    })
+    .where(
+      and(
+        eq(schema.brainIngestQueue.id, id),
+        eq(schema.brainIngestQueue.status, "processing"),
+        eq(schema.brainIngestQueue.leaseToken, leaseToken),
+      ),
+    );
+}

package/corpus/templates/brain/server/lib/privacy-readiness.ts

@@ -0,0 +1,24 @@
+import type { BrainSettings } from "../../shared/types.js";
+
+export interface BrainPrivacyReadiness {
+  configured: boolean;
+  model: string | null;
+  engine: string | null;
+  warning: string | null;
+}
+
+export function brainPrivacyReadiness(
+  settings: BrainSettings,
+): BrainPrivacyReadiness {
+  const model = settings.privacyClassifierModel?.trim() || null;
+  const engine = settings.privacyClassifierEngine?.trim() || null;
+  const configured = Boolean(model && engine);
+  return {
+    configured,
+    model: configured ? model : null,
+    engine: configured ? engine : null,
+    warning: configured
+      ? null
+      : "No approved privacy classifier is configured. Deterministic-clean captures may be stored; uncertain captures are quarantined and deterministic hard-category captures are suppressed. Neither can be searched, cited, distilled, or exposed to agents.",
+  };
+}

package/corpus/templates/brain/server/lib/search-index-contracts.ts

@@ -0,0 +1,72 @@
+export const BRAIN_SEARCH_INDEX_VERSION = "1";
+export const BRAIN_SENSITIVITY_POLICY_VERSION = "1";
+
+export type BrainSensitivityCategory =
+  | "performance"
+  | "discipline"
+  | "termination"
+  | "layoff-reorg"
+  | "compensation"
+  | "recruiting"
+  | "health-accommodation"
+  | "investigation"
+  | "privileged-legal"
+  | "secret-credential"
+  | "personal";
+
+export type BrainSensitivityDisposition =
+  | "allowed"
+  | "suppressed"
+  | "quarantined";
+
+export interface BrainSafeSegment {
+  id: string;
+  authorKey?: string;
+  capturedAt: string;
+  sourceUrl?: string;
+  text: string;
+  reactionCount: number;
+}
+
+export interface BrainSensitivityDecision {
+  disposition: BrainSensitivityDisposition;
+  categories: BrainSensitivityCategory[];
+  confidenceBand: "deterministic" | "high" | "medium" | "uncertain";
+  policyVersion: string;
+  safeSegments: BrainSafeSegment[];
+  safeContent: string;
+  classifier: "deterministic" | "approved-model";
+}
+
+export interface BrainAudienceAssignment {
+  audienceId: string;
+  aclHash: string;
+  kind: "org" | "slack-private-channel" | "meeting" | "restricted";
+}
+
+export interface BrainSearchStalenessKey {
+  contentHash: string;
+  indexVersion: string;
+  sensitivityPolicyVersion: string;
+  aclHash: string;
+}
+
+export type BrainIngestOperation =
+  | "distill"
+  | "sync"
+  | "search-index"
+  | "search-unindex"
+  | "slack-thread-refresh";
+
+export interface BrainCaptureInvalidation {
+  captureId: string;
+  sourceId: string;
+  reason:
+    | "content-changed"
+    | "sensitivity-changed"
+    | "access-changed"
+    | "source-deleted"
+    | "upstream-deleted";
+  previous?: Partial<BrainSearchStalenessKey>;
+  next?: Partial<BrainSearchStalenessKey>;
+}