@agent-native/core 0.111.3 → 0.112.0

package/corpus/templates/brain/server/lib/search.ts

@@ -11,7 +11,9 @@ import { and, desc, eq, inArray, or, sql, type SQL } from "drizzle-orm";
 
 import type { BrainEvidence } from "../../shared/types.js";
 import { getDb, schema } from "../db/index.js";
+import { listAccessibleAudienceIds } from "./audiences.js";
 import { parseJson, safeCitationUrl } from "./brain.js";
+import { hybridSearchArtifacts } from "./hybrid-search.js";
 
 export type UniversalSearchType = "knowledge" | "capture" | "source";
 
@@ -35,10 +37,14 @@ export interface UniversalSearchResult {
     captureTitle?: string | null;
     quote?: string | null;
     sourceUrl?: string | null;
+    preview?: string | null;
+    verbatim?: boolean;
   } | null;
   confidence: number | null;
   updatedAt: string;
   score: number;
+  reasons?: string[];
+  lane?: "lexical" | "semantic" | "hybrid" | "like";
 }
 
 export type FederatedDelegationTarget = "analytics" | "mail" | "dispatch";
@@ -364,12 +370,27 @@ async function searchKnowledgeResults(
   terms: string[],
   limit: number,
 ): Promise<UniversalSearchResult[]> {
+  const audienceIds = await listAccessibleAudienceIds();
+  const accessibleSourceExists = sql`exists (
+    select 1 from ${schema.brainSources}
+    where ${schema.brainSources.id} = ${schema.brainKnowledge.sourceId}
+      and ${accessFilter(schema.brainSources, schema.brainSourceShares)}
+  )`;
   const rows = await getDb()
     .select()
     .from(schema.brainKnowledge)
     .where(
       and(
         accessFilter(schema.brainKnowledge, schema.brainKnowledgeShares),
+        or(
+          sql`${schema.brainKnowledge.captureId} IS NULL`,
+          audienceIds.length
+            ? and(
+                accessibleSourceExists,
+                inArray(schema.brainKnowledge.audienceId, audienceIds),
+              )
+            : undefined,
+        ),
         eq(schema.brainKnowledge.status, "published"),
         anyColumnMatches(
           [
@@ -437,6 +458,8 @@ async function searchCaptureResults(
   terms: string[],
   limit: number,
 ): Promise<UniversalSearchResult[]> {
+  const audienceIds = await listAccessibleAudienceIds();
+  if (!audienceIds.length) return [];
   const accessibleSourceExists = sql`exists (
     select 1 from ${schema.brainSources}
     where ${schema.brainSources.id} = ${schema.brainRawCaptures.sourceId}
@@ -448,6 +471,12 @@ async function searchCaptureResults(
     .where(
       and(
         accessibleSourceExists,
+        eq(schema.brainRawCaptures.sensitivityDisposition, "allowed"),
+        sql`exists (
+          select 1 from ${schema.brainCaptureAudiences}
+          where ${schema.brainCaptureAudiences.captureId} = ${schema.brainRawCaptures.id}
+            and ${inArray(schema.brainCaptureAudiences.audienceId, audienceIds)}
+        )`,
         anyColumnMatches(
           [
             schema.brainRawCaptures.title,
@@ -481,7 +510,9 @@ async function searchCaptureResults(
         citation: {
           captureId: row.id,
           captureTitle: redactSensitiveText(row.title),
-          quote: snippet,
+          quote: null,
+          preview: snippet,
+          verbatim: false,
           sourceUrl,
         },
         confidence: null,
@@ -560,23 +591,99 @@ export async function searchEverythingRows(args: {
   query: string;
   type?: UniversalSearchType | "all";
   provider?: string;
+  kind?: string;
+  projectId?: string;
   status?: string;
   limit?: number;
 }): Promise<UniversalSearchResult[]> {
   const terms = normalizeSearchTerms(args.query);
   if (!terms.length) return [];
+  const projectSourceIds = args.projectId
+    ? (
+        await getDb()
+          .select({ sourceId: schema.brainProjectSources.sourceId })
+          .from(schema.brainProjectSources)
+          .innerJoin(
+            schema.brainProjects,
+            eq(schema.brainProjects.id, schema.brainProjectSources.projectId),
+          )
+          .where(
+            and(
+              eq(schema.brainProjectSources.projectId, args.projectId),
+              accessFilter(schema.brainProjects, schema.brainProjectShares),
+            ),
+          )
+      ).map((row) => row.sourceId)
+    : undefined;
+  if (projectSourceIds && !projectSourceIds.length) return [];
+  const projectSourceSet = projectSourceIds
+    ? new Set(projectSourceIds)
+    : undefined;
   const limit = args.limit ?? 25;
   const perTypeLimit = Math.max(limit, 10);
   const searches: Array<Promise<UniversalSearchResult[]>> = [];
   if (!args.type || args.type === "all" || args.type === "knowledge") {
     searches.push(searchKnowledgeResults(terms, perTypeLimit));
   }
-  if (!args.type || args.type === "all" || args.type === "capture") {
+  if (
+    (!args.type || args.type === "all" || args.type === "capture") &&
+    !args.kind
+  ) {
     searches.push(searchCaptureResults(terms, perTypeLimit));
   }
   if (!args.type || args.type === "all" || args.type === "source") {
     searches.push(searchSourceResults(terms, perTypeLimit));
   }
+  if (!args.type || args.type === "all" || args.type === "capture") {
+    searches.push(
+      hybridSearchArtifacts({
+        query: args.query,
+        provider: args.provider,
+        kind: args.kind,
+        projectId: args.projectId,
+        limit: perTypeLimit,
+      })
+        .then(async (artifacts) => {
+          const sources = await accessibleSourceMap(
+            artifacts.map((artifact) => artifact.sourceId),
+          );
+          return artifacts.flatMap((artifact) => {
+            const source = sources.get(artifact.sourceId);
+            if (!source) return [];
+            const snippet = redactSensitiveText(
+              buildSnippet(artifact.text, terms),
+            );
+            return [
+              {
+                type: "capture" as const,
+                id: artifact.captureId,
+                title: redactSensitiveText(artifact.title),
+                snippet,
+                summary: snippet,
+                status: "indexed",
+                provider: source.provider,
+                source: serializeSourceInfo(source),
+                sourceUrl: null,
+                citation: {
+                  captureId: artifact.captureId,
+                  captureTitle: redactSensitiveText(artifact.title),
+                  quote: null,
+                  preview: snippet,
+                  verbatim: false,
+                  sourceUrl: null,
+                },
+                confidence: null,
+                updatedAt: artifact.capturedAt,
+                score: artifact.score * 100,
+                reasons: artifact.reasons,
+                lane: artifact.lane,
+              },
+            ];
+          });
+        })
+        .catch(() => []),
+    );
+  }
   const provider = args.provider?.toLowerCase();
   const status = args.status?.toLowerCase();
   const results = (await Promise.all(searches))
@@ -590,7 +697,10 @@ export async function searchEverythingRows(args: {
       const resultStatus = result.status.toLowerCase();
       const providerMatches = !provider || resultProvider === provider;
       const statusMatches = !status || resultStatus === status;
-      return providerMatches && statusMatches;
+      const projectMatches =
+        !projectSourceSet ||
+        Boolean(result.source?.id && projectSourceSet.has(result.source.id));
+      return providerMatches && statusMatches && projectMatches;
     })
     .sort(
       (a, b) =>
@@ -598,7 +708,13 @@ export async function searchEverythingRows(args: {
         Date.parse(b.updatedAt) - Date.parse(a.updatedAt) ||
         a.title.localeCompare(b.title),
     );
-  return results.slice(0, limit);
+  const deduped = new Map<string, UniversalSearchResult>();
+  for (const result of results) {
+    const key = `${result.type}:${result.id}`;
+    const current = deduped.get(key);
+    if (!current || result.score > current.score) deduped.set(key, result);
+  }
+  return Array.from(deduped.values()).slice(0, limit);
 }
 
 async function readBrainSourceProviderCoverage(): Promise<

package/corpus/templates/brain/server/lib/sensitivity-policy.ts

@@ -0,0 +1,174 @@
+import { z } from "zod";
+
+import {
+  BRAIN_SENSITIVITY_POLICY_VERSION,
+  type BrainSafeSegment,
+  type BrainSensitivityCategory,
+  type BrainSensitivityDecision,
+} from "./search-index-contracts.js";
+
+export const MAX_CLASSIFIER_OUTPUT_CHARS = 80_000;
+export const classifierDecisionSchema = z
+  .object({
+    disposition: z.enum(["allowed", "suppressed", "quarantined"]),
+    categories: z
+      .array(
+        z.enum([
+          "performance",
+          "discipline",
+          "termination",
+          "layoff-reorg",
+          "compensation",
+          "recruiting",
+          "health-accommodation",
+          "investigation",
+          "privileged-legal",
+          "secret-credential",
+          "personal",
+        ]),
+      )
+      .max(12),
+    safeContent: z.string().max(MAX_CLASSIFIER_OUTPUT_CHARS),
+    safeSegments: z
+      .array(
+        z.object({
+          text: z.string().min(1).max(8_000),
+          sourceUrl: z.string().url().optional(),
+        }),
+      )
+      .max(50),
+  })
+  .strict();
+
+const HARD_CATEGORY_PATTERNS: ReadonlyArray<
+  readonly [BrainSensitivityCategory, RegExp]
+> = [
+  [
+    "performance",
+    /\b(performance review|under[- ]?perform(?:er|ing)?|low performer|performance concern)\b/i,
+  ],
+  [
+    "discipline",
+    /\b(performance improvement plan|start (?:a )?pip|(?:place(?:d)?|put) (?:them|her|him|the employee) on (?:a )?pip|written warning|disciplinary action|final warning)\b/i,
+  ],
+  [
+    "termination",
+    /\b(employment termination|terminate the employee|terminat(?:e|ed|ion) (?:their|his|her|the employee'?s?) employment|fired|fire\s+(?:them|her|him)|severance (?:agreement|package|pay))\b/i,
+  ],
+  [
+    "layoff-reorg",
+    /\b(layoffs?|rif|reorg(?:anization)?|rightsizing|roles? impacted|role eliminat(?:ion|ed)|workforce reduction)\b/i,
+  ],
+  [
+    "compensation",
+    /\b(compensation (?:review|discussion|adjustment|change)|salary|(?:annual|signing|performance|retention) bonus|bonus (?:amount|payout|target)|equity grant|stock options?|pay band|payroll|pay raise|salary increase|comp adjustment)\b/i,
+  ],
+  [
+    "recruiting",
+    /\b(candidate|applicant|interview(?:er|ing)?|recruit(?:er|ing)?|resume|résumé|reference check|job offer|shortlist|hiring panel)\b/i,
+  ],
+  [
+    "health-accommodation",
+    /\b(health condition|medical (?:condition|diagnosis|leave|record|accommodation)|doctor(?:'s)? note|disability accommodation|workplace accommodation|fmla|mental health (?:condition|leave|accommodation)|pregnan(?:t|cy))\b/i,
+  ],
+  [
+    "investigation",
+    /\b(investigation|investigat(?:e|ing|ed)|harassment complaint|misconduct|whistleblower|ethics complaint)\b/i,
+  ],
+  [
+    "privileged-legal",
+    /\b(attorney[- ]client|legal privilege|privileged and confidential|outside counsel|litigation hold)\b/i,
+  ],
+  [
+    "secret-credential",
+    /\b(?:password|passcode|secret|api[- ]?key|access[- ]?token|private[- ]?key)\s*[:=]|\b(?:sk|pk|rk|ghp|gho|ghu|github_pat)[_-][A-Za-z0-9_=-]{12,}\b/i,
+  ],
+];
+
+const PERSONAL_PATTERN =
+  /\b(home address|social security|ssn|birthday|spouse|husband|wife|children?)\b/i;
+const PROMPT_INJECTION_PATTERN =
+  /\b(ignore (?:previous|all|privacy)|override (?:the |all )?(?:policy|rules)|reveal (?:the )?(?:secret|private)|persist every)\b/i;
+export interface DeterministicSensitivityScreen {
+  categories: BrainSensitivityCategory[];
+  sensitiveLines: string[];
+  safeLines: string[];
+}
+
+export function screenSensitivityDeterministically(
+  content: string,
+): DeterministicSensitivityScreen {
+  const categories = new Set<BrainSensitivityCategory>();
+  const sensitiveLines: string[] = [];
+  const safeLines: string[] = [];
+
+  for (const rawLine of content.split(/\r?\n/g)) {
+    const line = rawLine.trim();
+    if (!line) continue;
+    let sensitive = false;
+    for (const [category, pattern] of HARD_CATEGORY_PATTERNS) {
+      if (!pattern.test(line)) continue;
+      categories.add(category);
+      sensitive = true;
+    }
+    if (PERSONAL_PATTERN.test(line)) {
+      categories.add("personal");
+      sensitive = true;
+    }
+    if (PROMPT_INJECTION_PATTERN.test(line)) sensitive = true;
+    (sensitive ? sensitiveLines : safeLines).push(line);
+  }
+
+  return { categories: [...categories], sensitiveLines, safeLines };
+}
+
+export function fallbackSensitivityDecision(
+  content: string,
+  capturedAt: string,
+): BrainSensitivityDecision {
+  const screen = screenSensitivityDeterministically(content);
+  const safeContent = screen.safeLines
+    .join("\n")
+    .slice(0, MAX_CLASSIFIER_OUTPUT_CHARS);
+  const safeSegments: BrainSafeSegment[] = safeContent
+    ? [{ id: "safe-1", capturedAt, text: safeContent, reactionCount: 0 }]
+    : [];
+  const disposition =
+    screen.categories.length || screen.sensitiveLines.length || !safeContent
+      ? "quarantined"
+      : "allowed";
+  return {
+    disposition,
+    categories: screen.categories,
+    confidenceBand:
+      disposition === "allowed" || screen.categories.length
+        ? "deterministic"
+        : "uncertain",
+    policyVersion: BRAIN_SENSITIVITY_POLICY_VERSION,
+    safeSegments,
+    safeContent,
+    classifier: "deterministic",
+  };
+}
+
+export function deterministicQuarantineDecision(
+  content: string,
+  capturedAt: string,
+): BrainSensitivityDecision | null {
+  const screen = screenSensitivityDeterministically(content);
+  if (!screen.categories.length) return null;
+  const safeContent = screen.safeLines
+    .join("\n")
+    .slice(0, MAX_CLASSIFIER_OUTPUT_CHARS);
+  return {
+    disposition: "suppressed",
+    categories: screen.categories,
+    confidenceBand: "deterministic",
+    policyVersion: BRAIN_SENSITIVITY_POLICY_VERSION,
+    safeSegments: safeContent
+      ? [{ id: "safe-1", capturedAt, text: safeContent, reactionCount: 0 }]
+      : [],
+    safeContent,
+    classifier: "deterministic",
+  };
+}

package/corpus/templates/brain/server/lib/slack-events.ts

@@ -0,0 +1,281 @@
+import crypto from "node:crypto";
+
+import { and, eq } from "drizzle-orm";
+
+import { getDb, schema } from "../db/index.js";
+import { parseJson, retireUpstreamDeletedCapture } from "./brain.js";
+import { enqueueBrainOperation } from "./ingest-queue.js";
+import { resolveSourceCredential } from "./source-credentials.js";
+
+export interface SlackEventsEnvelope {
+  type?: string;
+  challenge?: string;
+  team_id?: string;
+  api_app_id?: string;
+  event?: {
+    type?: string;
+    subtype?: string;
+    channel?: string;
+    ts?: string;
+    thread_ts?: string;
+    deleted_ts?: string;
+    previous_message?: { ts?: string; thread_ts?: string };
+    item?: { type?: string; channel?: string; ts?: string };
+  };
+}
+
+type SlackSource = typeof schema.brainSources.$inferSelect;
+
+function objectValue(value: unknown): Record<string, unknown> {
+  return value && typeof value === "object"
+    ? (value as Record<string, unknown>)
+    : {};
+}
+
+function stringValue(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function configuredString(
+  config: Record<string, unknown>,
+  keys: string[],
+): string | undefined {
+  for (const candidate of [config, objectValue(config.slack)]) {
+    for (const key of keys) {
+      const value = stringValue(candidate[key]);
+      if (value) return value;
+    }
+  }
+  return undefined;
+}
+
+function configuredChannelIds(config: Record<string, unknown>) {
+  const result = new Set<string>();
+  for (const candidate of [config, objectValue(config.slack)]) {
+    for (const key of [
+      "channelIds",
+      "channels",
+      "allowedChannels",
+      "allowlistedChannels",
+      "allowList",
+    ]) {
+      const raw = candidate[key];
+      const values =
+        typeof raw === "string"
+          ? raw.split(",")
+          : Array.isArray(raw)
+            ? raw.filter((value): value is string => typeof value === "string")
+            : [];
+      for (const value of values) {
+        const id = value.trim().replace(/^#/, "");
+        if (/^[CG][A-Z0-9]+$/i.test(id)) result.add(id);
+      }
+    }
+  }
+  return result;
+}
+
+function includesPublicChannels(config: Record<string, unknown>) {
+  for (const candidate of [config, objectValue(config.slack)]) {
+    const value = candidate.includePublicChannels;
+    if (value === true) return true;
+    if (
+      typeof value === "string" &&
+      ["true", "1", "yes", "on"].includes(value.toLowerCase())
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export function slackEventDirective(payload: SlackEventsEnvelope) {
+  const event = payload.event;
+  if (!event) return null;
+  if (event.type === "message") {
+    const retiring = event.subtype === "message_deleted";
+    const threadTs = retiring
+      ? (event.previous_message?.thread_ts ??
+        event.previous_message?.ts ??
+        event.deleted_ts)
+      : (event.thread_ts ?? event.ts);
+    return event.channel && threadTs
+      ? {
+          action: retiring ? ("retire" as const) : ("refresh" as const),
+          channelId: event.channel,
+          threadTs,
+        }
+      : null;
+  }
+  if (
+    (event.type === "reaction_added" || event.type === "reaction_removed") &&
+    event.item?.type === "message" &&
+    event.item.channel &&
+    event.item.ts
+  ) {
+    return {
+      action: "refresh" as const,
+      channelId: event.item.channel,
+      threadTs: event.item.ts,
+    };
+  }
+  return null;
+}
+
+function sourceMatchesEvent(
+  source: SlackSource,
+  payload: SlackEventsEnvelope,
+  location: { channelId: string; threadTs: string } | null,
+) {
+  const config = parseJson<Record<string, unknown>>(source.configJson, {});
+  const teamId = configuredString(config, ["slackTeamId", "teamId"]);
+  const appId = configuredString(config, ["slackAppId", "appId"]);
+  if (!teamId || teamId !== payload.team_id) return false;
+  if (appId && appId !== payload.api_app_id) return false;
+  if (!location) return payload.type === "url_verification";
+  return (
+    configuredChannelIds(config).has(location.channelId) ||
+    (includesPublicChannels(config) && /^C[A-Z0-9]+$/i.test(location.channelId))
+  );
+}
+
+export function parseSlackEventsEnvelope(
+  rawBody: string,
+): SlackEventsEnvelope | null {
+  try {
+    const parsed = JSON.parse(rawBody) as unknown;
+    return parsed && typeof parsed === "object"
+      ? (parsed as SlackEventsEnvelope)
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+export function verifySlackEventSignature(input: {
+  rawBody: string;
+  timestamp?: string;
+  signature?: string;
+  signingSecret?: string;
+  nowMs?: number;
+}) {
+  if (!input.timestamp || !input.signature || !input.signingSecret)
+    return false;
+  const timestamp = Number.parseInt(input.timestamp, 10);
+  if (!Number.isFinite(timestamp)) return false;
+  if (Math.abs((input.nowMs ?? Date.now()) / 1_000 - timestamp) > 300)
+    return false;
+  const expected = `v0=${crypto
+    .createHmac("sha256", input.signingSecret)
+    .update(`v0:${input.timestamp}:${input.rawBody}`)
+    .digest("hex")}`;
+  try {
+    return crypto.timingSafeEqual(
+      Buffer.from(input.signature),
+      Buffer.from(expected),
+    );
+  } catch {
+    return false;
+  }
+}
+
+async function matchingSlackSource(
+  payload: SlackEventsEnvelope,
+  location: { channelId: string; threadTs: string } | null,
+) {
+  const sources = await getDb()
+    .select()
+    .from(schema.brainSources)
+    // guard:allow-unscoped — Slack's signed Events API has no user session;
+    // each candidate must match its configured workspace and channel before use.
+    .where(
+      and(
+        eq(schema.brainSources.provider, "slack"),
+        eq(schema.brainSources.status, "active"),
+      ),
+    );
+  return (
+    sources.find((source) => sourceMatchesEvent(source, payload, location)) ??
+    null
+  );
+}
+
+async function sourceSigningSecret(source: SlackSource) {
+  const config = parseJson<Record<string, unknown>>(source.configJson, {});
+  return resolveSourceCredential({
+    provider: "slack",
+    key: "SLACK_SIGNING_SECRET",
+    ctx: { userEmail: source.ownerEmail, orgId: source.orgId },
+    workspaceConnectionId: configuredString(config, ["workspaceConnectionId"]),
+  });
+}
+
+export async function retireSlackThreadCapture(input: {
+  sourceId: string;
+  channelId: string;
+  threadTs: string;
+}) {
+  return retireUpstreamDeletedCapture({
+    sourceId: input.sourceId,
+    externalId: `slack:${input.channelId}:${input.threadTs}`,
+    provider: "slack",
+  });
+}
+
+/**
+ * Verify a Brain Slack event and either enqueue an ID-only refresh or retire a
+ * deleted upstream thread. The raw event body is never stored or logged.
+ */
+export async function enqueueSlackThreadRefreshFromEvent(input: {
+  rawBody: string;
+  timestamp?: string;
+  signature?: string;
+}): Promise<
+  | { status: "invalid" | "ignored" | "queued"; challenge?: string }
+  | { status: "missing-signing-secret" }
+> {
+  const payload = parseSlackEventsEnvelope(input.rawBody);
+  if (!payload?.team_id) return { status: "invalid" };
+  const directive = slackEventDirective(payload);
+  const source = await matchingSlackSource(payload, directive);
+  if (!source) return { status: "ignored" };
+  const signingSecret = await sourceSigningSecret(source);
+  if (!signingSecret) return { status: "missing-signing-secret" };
+  if (
+    !verifySlackEventSignature({
+      rawBody: input.rawBody,
+      timestamp: input.timestamp,
+      signature: input.signature,
+      signingSecret,
+    })
+  ) {
+    return { status: "invalid" };
+  }
+  if (
+    payload.type === "url_verification" &&
+    typeof payload.challenge === "string"
+  ) {
+    return { status: "queued", challenge: payload.challenge };
+  }
+  if (!directive) return { status: "ignored" };
+  if (directive.action === "retire") {
+    await retireSlackThreadCapture({
+      sourceId: source.id,
+      channelId: directive.channelId,
+      threadTs: directive.threadTs,
+    });
+    return { status: "queued" };
+  }
+  await enqueueBrainOperation({
+    operation: "slack-thread-refresh",
+    dedupeKey: `slack-thread-refresh:${source.id}:${directive.channelId}:${directive.threadTs}`,
+    sourceId: source.id,
+    priority: 20,
+    payload: {
+      teamId: payload.team_id,
+      channelId: directive.channelId,
+      threadTs: directive.threadTs,
+    },
+  });
+  return { status: "queued" };
+}

package/corpus/templates/brain/server/onboarding.ts

@@ -0,0 +1,29 @@
+import { registerOnboardingStep } from "@agent-native/core/onboarding";
+
+import { readBrainSettings } from "./lib/brain.js";
+import { brainPrivacyReadiness } from "./lib/privacy-readiness.js";
+
+registerOnboardingStep({
+  id: "brain-privacy-classifier",
+  order: 16,
+  required: false,
+  title: "Configure Brain privacy classification",
+  description:
+    "Choose the approved model and engine that review captures before storage. Until configured, deterministic-clean content can be stored but uncertain content is quarantined and unavailable to search or agents.",
+  methods: [
+    {
+      id: "settings",
+      kind: "link",
+      primary: true,
+      label: "Open Brain privacy settings",
+      payload: { url: "/settings", external: false },
+    },
+  ],
+  isComplete: async () => {
+    try {
+      return brainPrivacyReadiness(await readBrainSettings()).configured;
+    } catch {
+      return false;
+    }
+  },
+});