@opengeni/documents 0.2.31 → 0.2.36

package/src/index.ts

@@ -4,8 +4,11 @@ import type {
   CreateDocumentBaseRequest,
   Document,
   DocumentBase,
+  DocumentCuration,
+  DocumentCurationStatus,
   DocumentSearchMode,
   DocumentSearchResult,
+  DocumentVisibility,
   FileAsset,
   KnowledgeSourceKind,
 } from "@opengeni/contracts";
@@ -13,7 +16,7 @@ import { requireFile, withRlsContext, withWorkspaceRls, type Database } from "@o
 import * as schema from "@opengeni/db/schema";
 import type { ObjectStorage } from "@opengeni/storage";
 import { LiteParse } from "@llamaindex/liteparse";
-import { and, asc, desc, eq, inArray, sql, type SQL } from "drizzle-orm";
+import { and, asc, desc, eq, inArray, or, sql, type SQL } from "drizzle-orm";
 import OpenAI from "openai";
 
 export const DEFAULT_DOCUMENT_PARSER = "liteparse";
@@ -21,6 +24,17 @@ export const DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
 export const DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
 export const DEFAULT_DOCUMENT_CHUNK_SIZE = 1200;
 export const DEFAULT_DOCUMENT_CHUNK_OVERLAP = 160;
+export const DEFAULT_DOCUMENT_CURATION_MODEL = "gpt-4o-mini";
+// Curator input is a preview, not the whole document — enough to name and
+// classify without paying for a full-document prompt on every drop.
+export const DOCUMENT_CURATION_MAX_INPUT_CHARS = 24_000;
+// A base move is applied automatically only at or above this curator
+// confidence; below it the suggestion is surfaced for human review instead.
+export const DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE = 0.75;
+// The per-workspace default base: where knowledge drops land (created on
+// first drop) and stay unless configured curation files them into a topical base.
+export const DEFAULT_BASE_NAME = "Default";
+export const DEFAULT_BASE_DESCRIPTION = "Default base for dropped files and notes.";
 
 export type ParsedDocument = {
   text: string;
@@ -48,10 +62,59 @@ export type DocumentEmbedder = {
   embedQuery: (text: string) => Promise<number[]>;
 };
 
+export type DocumentCurationCandidateBase = {
+  id: string;
+  name: string;
+  description: string | null;
+};
+
+export type DocumentCurationInput = {
+  /** Parsed document text, clipped to DOCUMENT_CURATION_MAX_INPUT_CHARS. */
+  text: string;
+  filename: string;
+  /** Current (usually filename-derived) title. */
+  title: string;
+  /** Candidate bases the document could be filed into (never its current base). */
+  bases: DocumentCurationCandidateBase[];
+};
+
+export type DocumentCurationOutcome = {
+  title: string | null;
+  summary: string | null;
+  sourceKind: KnowledgeSourceKind | null;
+  topics: string[];
+  targetBaseId: string | null;
+  confidence: number;
+  reason: string | null;
+};
+
+export type DocumentCurator = {
+  model: string;
+  curate: (input: DocumentCurationInput) => Promise<DocumentCurationOutcome>;
+};
+
 export type DocumentServices = {
   parser: DocumentParser;
   chunker: DocumentChunker;
   embedder: DocumentEmbedder;
+  /** Optional: names/summarizes/categorizes dropped documents during indexing. */
+  curator?: DocumentCurator | undefined;
+};
+
+/**
+ * Read-scoping for document queries. Fail-closed: when a caller supplies no
+ * filter, private documents are invisible (only their creator may see them,
+ * and only by passing their subject id).
+ */
+export type DocumentAccessFilter = {
+  /** Grant subject id of the human viewer; null/undefined hides private docs. */
+  viewerSubjectId?: string | null | undefined;
+  /**
+   * Agent retrieval surface: only agent-enabled documents. Workspace-visible
+   * documents are available to every agent; private documents are available
+   * only when the agent carries the creating subject as its viewer subject.
+   */
+  agentOnly?: boolean | undefined;
 };
 
 export type DocumentSearchInput = {
@@ -62,6 +125,7 @@ export type DocumentSearchInput = {
   mode?: DocumentSearchMode | undefined;
   sourceKinds?: KnowledgeSourceKind[] | undefined;
   aclTags?: string[] | undefined;
+  access?: DocumentAccessFilter | undefined;
 };
 
 export type DocumentIndexHooks = {
@@ -221,6 +285,169 @@ export class DeterministicEmbeddingProvider implements DocumentEmbedder {
   }
 }
 
+/**
+ * Deterministic no-network curation: first meaningful line becomes the title,
+ * the opening text becomes the summary, and the kind is guessed from the
+ * filename/content type. Never proposes a base move (confidence 0). Used as
+ * the `heuristic` provider and as the in-pipeline fallback when the LLM
+ * curator fails — a drop must always end up named and summarized.
+ */
+export function heuristicCuration(
+  input: DocumentCurationInput,
+  contentType = "application/octet-stream",
+): DocumentCurationOutcome {
+  const firstLine = input.text
+    .split("\n")
+    .map((line) => line.replace(/^[#>\s*-]+/, "").trim())
+    .find((line) => line.length >= 3);
+  const title = (firstLine ?? input.title).slice(0, 120).trim() || input.title;
+  const summaryWindow = input.text.replace(/\s+/g, " ").trim().slice(0, 360);
+  const summary =
+    summaryWindow.length === 360 ? `${summaryWindow.slice(0, 357)}...` : summaryWindow;
+  return {
+    title,
+    summary: summary || null,
+    sourceKind: heuristicSourceKind(input.filename, contentType),
+    topics: [],
+    targetBaseId: null,
+    confidence: 0,
+    reason: null,
+  };
+}
+
+function heuristicSourceKind(rawFilename: string, rawContentType: string): KnowledgeSourceKind {
+  const filename = rawFilename.toLowerCase();
+  const contentType = rawContentType.toLowerCase();
+  if (filename.endsWith(".eml") || contentType === "message/rfc822") return "email";
+  if (filename.endsWith(".vtt") || filename.endsWith(".srt") || filename.includes("transcript")) {
+    return "meeting_transcript";
+  }
+  if (contentType === "text/html") return "web";
+  if (contentType === "application/pdf" || filename.endsWith(".docx") || filename.endsWith(".md")) {
+    return "document";
+  }
+  return "manual_upload";
+}
+
+export class HeuristicCurationProvider implements DocumentCurator {
+  readonly model = "heuristic";
+
+  async curate(input: DocumentCurationInput): Promise<DocumentCurationOutcome> {
+    return heuristicCuration(input);
+  }
+}
+
+const CURATION_SYSTEM_PROMPT = [
+  "You organize a team knowledge base. Given the beginning of a dropped document",
+  "and the list of existing collections (bases), return STRICT JSON with keys:",
+  '"title" (concise, specific, <= 120 chars, no filename extensions),',
+  '"summary" (2-3 sentences, plain prose, what the document is and why it matters),',
+  '"sourceKind" (one of: manual_upload, meeting_transcript, repository, email, chat, document, web, other),',
+  '"topics" (3-6 short lowercase tags),',
+  '"targetBaseId" (the id of the best-fitting existing base, or null if none fits),',
+  '"confidence" (0..1 — how sure you are the document belongs in targetBaseId),',
+  '"reason" (one sentence explaining the filing choice).',
+  "Only pick a targetBaseId from the provided list. If the document fits no base",
+  "well, return targetBaseId null and confidence 0. Respond with JSON only.",
+].join(" ");
+
+export class OpenAICurationProvider implements DocumentCurator {
+  private client: OpenAI | null = null;
+  private readonly apiKey: string | undefined;
+  private readonly baseURL: string | undefined;
+  private readonly defaultHeaders: Record<string, string> | undefined;
+  private readonly defaultQuery: Record<string, string> | undefined;
+  readonly model: string;
+
+  constructor(args: {
+    apiKey?: string | undefined;
+    baseURL?: string | undefined;
+    defaultHeaders?: Record<string, string> | undefined;
+    defaultQuery?: Record<string, string> | undefined;
+    model?: string | undefined;
+  }) {
+    this.apiKey = args.apiKey ?? process.env.OPENAI_API_KEY;
+    this.baseURL = args.baseURL;
+    this.defaultHeaders = args.defaultHeaders;
+    this.defaultQuery = args.defaultQuery;
+    this.model = args.model ?? DEFAULT_DOCUMENT_CURATION_MODEL;
+  }
+
+  async curate(input: DocumentCurationInput): Promise<DocumentCurationOutcome> {
+    const response = await this.openai().chat.completions.create({
+      model: this.model,
+      response_format: { type: "json_object" },
+      messages: [
+        { role: "system", content: CURATION_SYSTEM_PROMPT },
+        {
+          role: "user",
+          content: JSON.stringify({
+            filename: input.filename,
+            currentTitle: input.title,
+            bases: input.bases,
+            text: input.text,
+          }),
+        },
+      ],
+    });
+    const raw = response.choices[0]?.message?.content;
+    if (!raw) {
+      throw new Error("curation model returned no content");
+    }
+    return parseCurationOutcome(raw, input.bases);
+  }
+
+  private openai(): OpenAI {
+    if (!this.apiKey) {
+      throw new Error("OpenAI document curation requires an API key");
+    }
+    this.client ??= new OpenAI({
+      apiKey: this.apiKey,
+      ...(this.baseURL ? { baseURL: this.baseURL } : {}),
+      ...(this.defaultQuery ? { defaultQuery: this.defaultQuery } : {}),
+      ...(this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}),
+    });
+    return this.client;
+  }
+}
+
+/** Parse + clamp model output; a targetBaseId outside the candidate list is dropped. */
+export function parseCurationOutcome(
+  raw: string,
+  bases: DocumentCurationCandidateBase[],
+): DocumentCurationOutcome {
+  const parsed: unknown = JSON.parse(raw);
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error("curation model returned non-object JSON");
+  }
+  const record = parsed as Record<string, unknown>;
+  const knownBase = bases.find((base) => base.id === record.targetBaseId);
+  const confidence =
+    typeof record.confidence === "number" && Number.isFinite(record.confidence)
+      ? Math.min(1, Math.max(0, record.confidence))
+      : 0;
+  return {
+    title:
+      cleanString(typeof record.title === "string" ? record.title.slice(0, 200) : null) ?? null,
+    summary:
+      cleanString(typeof record.summary === "string" ? record.summary.slice(0, 2000) : null) ??
+      null,
+    sourceKind:
+      typeof record.sourceKind === "string"
+        ? normalizeKnowledgeSourceKind(record.sourceKind)
+        : null,
+    topics: cleanStringArray(
+      Array.isArray(record.topics)
+        ? record.topics.filter((topic): topic is string => typeof topic === "string").slice(0, 8)
+        : [],
+    ).map((topic) => topic.toLowerCase().slice(0, 60)),
+    targetBaseId: knownBase?.id ?? null,
+    confidence: knownBase ? confidence : 0,
+    reason:
+      cleanString(typeof record.reason === "string" ? record.reason.slice(0, 500) : null) ?? null,
+  };
+}
+
 export function createDocumentServices(
   settings?: Settings,
   overrides: Partial<DocumentServices> = {},
@@ -244,9 +471,24 @@ export function createDocumentServices(
             model: settings?.documentEmbeddingModel ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL,
             dimensions,
           })),
+    curator: overrides.curator ?? createDocumentCurator(settings),
   };
 }
 
+function createDocumentCurator(settings?: Settings): DocumentCurator | undefined {
+  const provider = settings?.documentCurationProvider ?? "openai";
+  if (provider === "none") return undefined;
+  if (provider === "heuristic") return new HeuristicCurationProvider();
+  const embeddingConfig = documentOpenAIEmbeddingConfig(settings);
+  return new OpenAICurationProvider({
+    apiKey: settings?.documentCurationApiKey ?? embeddingConfig.apiKey,
+    baseURL: settings?.documentCurationBaseUrl ?? embeddingConfig.baseURL,
+    defaultHeaders: embeddingConfig.defaultHeaders,
+    defaultQuery: embeddingConfig.defaultQuery,
+    model: settings?.documentCurationModel ?? DEFAULT_DOCUMENT_CURATION_MODEL,
+  });
+}
+
 export function documentOpenAIEmbeddingConfig(settings?: Settings): {
   apiKey?: string | undefined;
   baseURL?: string | undefined;
@@ -354,9 +596,65 @@ export async function getDocumentBase(
   });
 }
 
+/**
+ * Find-or-create the workspace's Default base — where knowledge drops land
+ * before configured curation files them elsewhere. Matched by name
+ * (case-insensitive) so a user-created "Default" is adopted rather than
+ * duplicated.
+ */
+export async function ensureDefaultBase(
+  db: Database,
+  input: { accountId: string; workspaceId: string },
+): Promise<DocumentBase> {
+  return await withRlsContext(
+    db,
+    { accountId: input.accountId, workspaceId: input.workspaceId },
+    async (scopedDb) =>
+      await scopedDb.transaction(async (tx) => {
+        const defaultName = sql`lower(btrim(${schema.documentBases.name})) = ${DEFAULT_BASE_NAME.toLowerCase()}`;
+        const [existing] = await tx
+          .select()
+          .from(schema.documentBases)
+          .where(and(eq(schema.documentBases.workspaceId, input.workspaceId), defaultName))
+          .limit(1);
+        if (existing) return mapDocumentBase(existing);
+
+        // The partial unique index is the serialization point for concurrent
+        // first drops. A losing insert re-reads the winner instead of
+        // surfacing a duplicate-base error.
+        const [inserted] = await tx
+          .insert(schema.documentBases)
+          .values({
+            accountId: input.accountId,
+            workspaceId: input.workspaceId,
+            name: DEFAULT_BASE_NAME,
+            description: DEFAULT_BASE_DESCRIPTION,
+          })
+          .onConflictDoNothing()
+          .returning();
+        if (inserted) return mapDocumentBase(inserted);
+
+        const [raced] = await tx
+          .select()
+          .from(schema.documentBases)
+          .where(and(eq(schema.documentBases.workspaceId, input.workspaceId), defaultName))
+          .limit(1);
+        if (raced) return mapDocumentBase(raced);
+        throw new Error("Failed to create Default document base");
+      }),
+  );
+}
+
 export async function addDocumentToBase(
   db: Database,
-  input: AddDocumentRequest & { accountId: string; workspaceId: string; baseId: string },
+  input: AddDocumentRequest & {
+    accountId: string;
+    workspaceId: string;
+    baseId: string;
+    createdBy?: string | null | undefined;
+    curationStatus?: DocumentCurationStatus | undefined;
+    access?: DocumentAccessFilter | undefined;
+  },
 ): Promise<Document> {
   return await withRlsContext(
     db,
@@ -365,6 +663,9 @@ export async function addDocumentToBase(
       const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
       if (!base) throw new Error(`Document base not found: ${input.baseId}`);
       const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
+      if (input.visibility === "private" && !cleanString(input.createdBy ?? null)) {
+        throw new Error("private documents require a creating subject");
+      }
       const now = new Date();
       const [existing] = await scopedDb
         .select()
@@ -378,9 +679,14 @@ export async function addDocumentToBase(
         )
         .limit(1);
       if (existing) {
+        if (!documentMatchesAccess(existing, input.access)) {
+          throw new Error(`Document not found: ${existing.id}`);
+        }
         // Idempotent re-add: refresh caller-supplied source metadata on the
         // existing row instead of silently discarding it (aclTags especially —
-        // a re-add that tightens tags must not be a no-op).
+        // a re-add that tightens tags must not be a no-op). Access policy is
+        // deliberately inherited: re-adding a known file is not an implicit
+        // visibility/agent-policy update that another manager can exploit.
         const [updated] = await scopedDb
           .update(schema.documents)
           .set({
@@ -400,6 +706,7 @@ export async function addDocumentToBase(
             and(
               eq(schema.documents.workspaceId, input.workspaceId),
               eq(schema.documents.id, existing.id),
+              ...documentAccessConditions(input.access),
             ),
           )
           .returning();
@@ -424,6 +731,10 @@ export async function addDocumentToBase(
           sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
           sourceVersion: cleanString(input.sourceVersion) ?? null,
           aclTags: cleanStringArray(input.aclTags),
+          visibility: input.visibility ?? "workspace",
+          agentAccess: input.agentAccess ?? true,
+          createdBy: input.createdBy ?? null,
+          curationStatus: input.curationStatus ?? "none",
           updatedAt: now,
         })
         .returning();
@@ -433,9 +744,107 @@ export async function addDocumentToBase(
   );
 }
 
+/**
+ * Move a document and its indexed chunks to another base. With no explicit
+ * target, applies the stored curation suggestion. A 'suggested' or 'pending'
+ * document that gets moved counts as filed.
+ */
+export async function moveDocumentToBase(
+  db: Database,
+  input: {
+    accountId: string;
+    workspaceId: string;
+    documentId: string;
+    targetBaseId?: string | null | undefined;
+    access?: DocumentAccessFilter | undefined;
+  },
+): Promise<Document> {
+  return await withRlsContext(
+    db,
+    { accountId: input.accountId, workspaceId: input.workspaceId },
+    async (scopedDb) => {
+      const [row] = await scopedDb
+        .select()
+        .from(schema.documents)
+        .where(
+          and(
+            eq(schema.documents.workspaceId, input.workspaceId),
+            eq(schema.documents.id, input.documentId),
+            ...documentAccessConditions(input.access),
+          ),
+        )
+        .limit(1);
+      if (!row) throw new Error(`Document not found: ${input.documentId}`);
+      const suggestion = (row.curation as { suggestedBaseId?: string | null } | null)
+        ?.suggestedBaseId;
+      const targetBaseId = input.targetBaseId ?? suggestion;
+      if (!targetBaseId) {
+        throw new Error("document has no suggested base; pass targetBaseId");
+      }
+      if (targetBaseId === row.baseId) return mapDocument(row);
+      const base = await getDocumentBase(scopedDb, input.workspaceId, targetBaseId);
+      if (!base) throw new Error(`Document base not found: ${targetBaseId}`);
+      const [conflict] = await scopedDb
+        .select({ id: schema.documents.id })
+        .from(schema.documents)
+        .where(
+          and(
+            eq(schema.documents.workspaceId, input.workspaceId),
+            eq(schema.documents.baseId, targetBaseId),
+            eq(schema.documents.fileId, row.fileId),
+          ),
+        )
+        .limit(1);
+      if (conflict) {
+        throw new Error("a document for this file already exists in the target base");
+      }
+      const now = new Date();
+      const moved = await scopedDb.transaction(async (tx) => {
+        const [updated] = await tx
+          .update(schema.documents)
+          .set({
+            baseId: targetBaseId,
+            ...(row.curationStatus === "suggested" || row.curationStatus === "pending"
+              ? { curationStatus: "auto_filed" }
+              : {}),
+            updatedAt: now,
+          })
+          .where(
+            and(
+              eq(schema.documents.workspaceId, input.workspaceId),
+              eq(schema.documents.id, input.documentId),
+              ...documentAccessConditions(input.access),
+            ),
+          )
+          .returning();
+        if (updated) {
+          await tx
+            .update(schema.documentChunks)
+            .set({ baseId: targetBaseId })
+            .where(
+              and(
+                eq(schema.documentChunks.workspaceId, input.workspaceId),
+                eq(schema.documentChunks.documentId, input.documentId),
+              ),
+            );
+        }
+        return updated;
+      });
+      if (!moved) throw new Error(`Document not found: ${input.documentId}`);
+      return mapDocument(moved);
+    },
+  );
+}
+
 export async function deleteDocumentFromBase(
   db: Database,
-  input: { accountId: string; workspaceId: string; baseId: string; documentId: string },
+  input: {
+    accountId: string;
+    workspaceId: string;
+    baseId: string;
+    documentId: string;
+    access?: DocumentAccessFilter | undefined;
+  },
 ): Promise<void> {
   await withRlsContext(
     db,
@@ -448,6 +857,7 @@ export async function deleteDocumentFromBase(
           and(
             eq(schema.documents.workspaceId, input.workspaceId),
             eq(schema.documents.id, input.documentId),
+            ...documentAccessConditions(input.access),
           ),
         )
         .limit(1);
@@ -463,6 +873,7 @@ export async function deleteDocumentFromBase(
           and(
             eq(schema.documents.workspaceId, input.workspaceId),
             eq(schema.documents.id, input.documentId),
+            ...documentAccessConditions(input.access),
           ),
         );
     },
@@ -473,13 +884,18 @@ export async function listDocuments(
   db: Database,
   workspaceId: string,
   baseId: string,
+  access?: DocumentAccessFilter,
 ): Promise<Document[]> {
   return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     const rows = await scopedDb
       .select()
       .from(schema.documents)
       .where(
-        and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.baseId, baseId)),
+        and(
+          eq(schema.documents.workspaceId, workspaceId),
+          eq(schema.documents.baseId, baseId),
+          ...documentAccessConditions(access),
+        ),
       )
       .orderBy(asc(schema.documents.createdAt));
     return rows.map(mapDocument);
@@ -490,13 +906,18 @@ export async function getDocument(
   db: Database,
   workspaceId: string,
   documentId: string,
+  access?: DocumentAccessFilter,
 ): Promise<Document | null> {
   return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     const [row] = await scopedDb
       .select()
       .from(schema.documents)
       .where(
-        and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
+        and(
+          eq(schema.documents.workspaceId, workspaceId),
+          eq(schema.documents.id, documentId),
+          ...documentAccessConditions(access),
+        ),
       )
       .limit(1);
     return row ? mapDocument(row) : null;
@@ -507,6 +928,7 @@ export async function queueDocumentForReindex(
   db: Database,
   workspaceId: string,
   documentId: string,
+  access?: DocumentAccessFilter,
 ): Promise<Document> {
   return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     const [row] = await scopedDb
@@ -517,7 +939,11 @@ export async function queueDocumentForReindex(
         updatedAt: new Date(),
       })
       .where(
-        and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
+        and(
+          eq(schema.documents.workspaceId, workspaceId),
+          eq(schema.documents.id, documentId),
+          ...documentAccessConditions(access),
+        ),
       )
       .returning();
     if (!row) throw new Error(`Document not found: ${documentId}`);
@@ -533,7 +959,7 @@ export async function indexDocumentNow(
   services: DocumentServices = createDocumentServices(),
   hooks: DocumentIndexHooks = {},
 ): Promise<Document> {
-  const [document] = await withWorkspaceRls(
+  const [loadedDocument] = await withWorkspaceRls(
     db,
     workspaceId,
     async (scopedDb) =>
@@ -545,7 +971,8 @@ export async function indexDocumentNow(
         )
         .limit(1),
   );
-  if (!document) throw new Error(`Document not found: ${documentId}`);
+  if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
+  let document: DocumentRow = loadedDocument;
   const file = await requireReadyFile(db, workspaceId, document.fileId);
   await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     await scopedDb
@@ -563,6 +990,13 @@ export async function indexDocumentNow(
   try {
     const bytes = await objectStorage.getFileBytes(file);
     const parsed = await services.parser.parse(bytes, file);
+    // Knowledge drops (curationStatus 'pending') are curated between parse and
+    // chunking when a provider is enabled, so chunk metadata and base placement
+    // reflect the curated truth. Disabled curation leaves caller metadata intact;
+    // enabled-provider failures remain fail-soft through the heuristic fallback.
+    if (document.curationStatus === "pending") {
+      document = await curateDroppedDocument(db, services, document, parsed, file);
+    }
     const chunks = services.chunker.chunk(parsed, file);
     await hooks.beforeEmbed?.({
       accountId: document.accountId,
@@ -658,11 +1092,147 @@ export async function indexDocumentNow(
     if (!failed) throw error;
     return mapDocument(failed);
   }
-  const updated = await getDocument(db, workspaceId, documentId);
+  // Internal indexing must be able to return a private document to the caller
+  // that created/queued it. Public reads remain fail-closed when no subject is
+  // supplied; the creator subject is the document's frozen access principal.
+  const updated = await getDocument(db, workspaceId, documentId, {
+    viewerSubjectId: document.createdBy,
+  });
   if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
   return updated;
 }
 
+type DocumentRow = typeof schema.documents.$inferSelect;
+
+async function curateDroppedDocument(
+  db: Database,
+  services: DocumentServices,
+  document: DocumentRow,
+  parsed: ParsedDocument,
+  file: FileAsset,
+): Promise<DocumentRow> {
+  // `none` is an explicit disabled-curation policy. Do not silently replace
+  // it with heuristics: indexing still proceeds, but the drop remains an
+  // ordinary uncured document with its caller-supplied title and metadata.
+  if (!services.curator) {
+    const [updated] = await withWorkspaceRls(
+      db,
+      document.workspaceId,
+      async (scopedDb) =>
+        await scopedDb
+          .update(schema.documents)
+          .set({
+            curationStatus: "none",
+            summary: null,
+            topics: [],
+            curation: null,
+            updatedAt: new Date(),
+          })
+          .where(
+            and(
+              eq(schema.documents.workspaceId, document.workspaceId),
+              eq(schema.documents.id, document.id),
+            ),
+          )
+          .returning(),
+    );
+    return updated ?? document;
+  }
+  const bases = await listDocumentBases(db, document.workspaceId);
+  const candidates: DocumentCurationCandidateBase[] = bases
+    .filter((base) => base.id !== document.baseId)
+    .map((base) => ({ id: base.id, name: base.name, description: base.description }));
+  const input: DocumentCurationInput = {
+    text: parsed.text.slice(0, DOCUMENT_CURATION_MAX_INPUT_CHARS),
+    filename: file.filename,
+    title: document.title,
+    bases: candidates,
+  };
+  let outcome: DocumentCurationOutcome;
+  let model: string;
+  let failure: string | null = null;
+  try {
+    outcome = await services.curator.curate(input);
+    model = services.curator.model;
+  } catch (error) {
+    failure = error instanceof Error ? error.message : String(error);
+    console.warn("document curation failed; applying heuristic fallback", {
+      workspaceId: document.workspaceId,
+      documentId: document.id,
+      error: failure,
+    });
+    outcome = heuristicCuration(input, file.contentType);
+    model = "heuristic";
+  }
+  const suggestedBase = candidates.find((base) => base.id === outcome.targetBaseId) ?? null;
+  let moveToBaseId: string | null = null;
+  if (
+    suggestedBase &&
+    failure === null &&
+    outcome.confidence >= DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE
+  ) {
+    // The (workspace, base, file) unique index means a same-file twin already
+    // in the target base blocks the move; keep it as a suggestion instead.
+    const conflict = await withWorkspaceRls(
+      db,
+      document.workspaceId,
+      async (scopedDb) =>
+        await scopedDb
+          .select({ id: schema.documents.id })
+          .from(schema.documents)
+          .where(
+            and(
+              eq(schema.documents.workspaceId, document.workspaceId),
+              eq(schema.documents.baseId, suggestedBase.id),
+              eq(schema.documents.fileId, document.fileId),
+            ),
+          )
+          .limit(1),
+    );
+    if (conflict.length === 0) {
+      moveToBaseId = suggestedBase.id;
+    }
+  }
+  const curation: DocumentCuration = {
+    suggestedBaseId: suggestedBase?.id ?? null,
+    suggestedBaseName: suggestedBase?.name ?? null,
+    confidence: outcome.confidence,
+    reason: failure ? `curation failed (${failure}); heuristic fallback applied` : outcome.reason,
+    originalTitle: document.title,
+    model,
+  };
+  const curationStatus: DocumentCurationStatus = failure
+    ? "failed"
+    : moveToBaseId
+      ? "auto_filed"
+      : "suggested";
+  const [updated] = await withWorkspaceRls(
+    db,
+    document.workspaceId,
+    async (scopedDb) =>
+      await scopedDb
+        .update(schema.documents)
+        .set({
+          title: outcome.title ?? document.title,
+          summary: outcome.summary,
+          topics: outcome.topics,
+          ...(outcome.sourceKind ? { sourceKind: outcome.sourceKind } : {}),
+          ...(moveToBaseId ? { baseId: moveToBaseId } : {}),
+          curationStatus,
+          curation,
+          updatedAt: new Date(),
+        })
+        .where(
+          and(
+            eq(schema.documents.workspaceId, document.workspaceId),
+            eq(schema.documents.id, document.id),
+          ),
+        )
+        .returning(),
+  );
+  return updated ?? document;
+}
+
 export async function searchDocuments(
   db: Database,
   input: DocumentSearchInput,
@@ -794,6 +1364,7 @@ export async function getDocumentChunk(
   db: Database,
   workspaceId: string,
   chunkId: string,
+  access?: DocumentAccessFilter,
 ): Promise<DocumentSearchResult | null> {
   const [row] = await withWorkspaceRls(
     db,
@@ -826,6 +1397,7 @@ export async function getDocumentChunk(
             eq(schema.documentChunks.workspaceId, workspaceId),
             eq(schema.documentChunks.id, chunkId),
             eq(schema.documents.status, "ready"),
+            ...documentAccessConditions(access),
           ),
         )
         .limit(1),
@@ -849,10 +1421,65 @@ type CombinedSearchRow = SearchRowBase & {
   keywordScore: number | null;
 };
 
+/**
+ * Visibility/agent scoping shared by every document read path. Fail-closed:
+ * with no filter supplied, private documents are invisible.
+ */
+function documentAccessConditions(access: DocumentAccessFilter | undefined): SQL[] {
+  if (access?.agentOnly) {
+    const viewer = access.viewerSubjectId;
+    const visibility = viewer
+      ? or(eq(schema.documents.visibility, "workspace"), eq(schema.documents.createdBy, viewer))
+      : eq(schema.documents.visibility, "workspace");
+    return [eq(schema.documents.agentAccess, true), ...(visibility ? [visibility] : [])];
+  }
+  const viewer = access?.viewerSubjectId;
+  if (viewer) {
+    const condition = or(
+      eq(schema.documents.visibility, "workspace"),
+      eq(schema.documents.createdBy, viewer),
+    );
+    return condition ? [condition] : [];
+  }
+  return [eq(schema.documents.visibility, "workspace")];
+}
+
+function documentMatchesAccess(
+  document: Pick<DocumentAccessRecord, "visibility" | "createdBy" | "agentAccess">,
+  access: DocumentAccessFilter | undefined,
+): boolean {
+  if (access?.agentOnly) {
+    return (
+      document.agentAccess &&
+      (document.visibility !== "private" ||
+        (!!access.viewerSubjectId && document.createdBy === access.viewerSubjectId))
+    );
+  }
+  return canViewDocument(document, access?.viewerSubjectId);
+}
+
+/** Whether a single already-fetched document is readable by this human viewer. */
+export function canViewDocument(
+  document: Pick<DocumentAccessRecord, "visibility" | "createdBy">,
+  viewerSubjectId: string | null | undefined,
+): boolean {
+  return (
+    document.visibility !== "private" ||
+    (!!viewerSubjectId && document.createdBy === viewerSubjectId)
+  );
+}
+
+type DocumentAccessRecord = {
+  visibility: string;
+  createdBy: string | null;
+  agentAccess: boolean;
+};
+
 function documentSearchConditions(input: DocumentSearchInput, embeddingModel?: string): SQL[] {
   const conditions: SQL[] = [
     eq(schema.documents.status, "ready"),
     eq(schema.documentChunks.workspaceId, input.workspaceId),
+    ...documentAccessConditions(input.access),
   ];
   if (embeddingModel) {
     conditions.push(eq(schema.documentChunks.embeddingModel, embeddingModel));
@@ -1195,7 +1822,30 @@ function mapDocument(row: typeof schema.documents.$inferSelect): Document {
     sourceUpdatedAt: row.sourceUpdatedAt?.toISOString() ?? null,
     sourceVersion: row.sourceVersion,
     aclTags: cleanStringArray(row.aclTags),
+    visibility: normalizeDocumentVisibility(row.visibility),
+    createdBy: row.createdBy,
+    agentAccess: row.agentAccess,
+    summary: row.summary,
+    topics: cleanStringArray(row.topics),
+    curationStatus: normalizeDocumentCurationStatus(row.curationStatus),
+    curation: (row.curation as DocumentCuration | null) ?? null,
     createdAt: row.createdAt.toISOString(),
     updatedAt: row.updatedAt.toISOString(),
   };
 }
+
+function normalizeDocumentVisibility(value: string): DocumentVisibility {
+  return value === "private" ? "private" : "workspace";
+}
+
+function normalizeDocumentCurationStatus(value: string): DocumentCurationStatus {
+  switch (value) {
+    case "pending":
+    case "suggested":
+    case "auto_filed":
+    case "failed":
+      return value;
+    default:
+      return "none";
+  }
+}