@opengeni/documents 0.2.30 → 0.2.36

package/dist/index.js

@@ -2,13 +2,18 @@
 import { requireFile, withRlsContext, withWorkspaceRls } from "@opengeni/db";
 import * as schema from "@opengeni/db/schema";
 import { LiteParse } from "@llamaindex/liteparse";
-import { and, asc, desc, eq, inArray, sql } from "drizzle-orm";
+import { and, asc, desc, eq, inArray, or, sql } from "drizzle-orm";
 import OpenAI from "openai";
 var DEFAULT_DOCUMENT_PARSER = "liteparse";
 var DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
 var DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
 var DEFAULT_DOCUMENT_CHUNK_SIZE = 1200;
 var DEFAULT_DOCUMENT_CHUNK_OVERLAP = 160;
+var DEFAULT_DOCUMENT_CURATION_MODEL = "gpt-4o-mini";
+var DOCUMENT_CURATION_MAX_INPUT_CHARS = 24e3;
+var DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE = 0.75;
+var DEFAULT_BASE_NAME = "Default";
+var DEFAULT_BASE_DESCRIPTION = "Default base for dropped files and notes.";
 var LiteParseDocumentParser = class {
   name = DEFAULT_DOCUMENT_PARSER;
   parseQueue = Promise.resolve();
@@ -132,6 +137,123 @@ var DeterministicEmbeddingProvider = class {
     return deterministicEmbedding(text, this.dimensions);
   }
 };
+function heuristicCuration(input, contentType = "application/octet-stream") {
+  const firstLine = input.text.split("\n").map((line) => line.replace(/^[#>\s*-]+/, "").trim()).find((line) => line.length >= 3);
+  const title = (firstLine ?? input.title).slice(0, 120).trim() || input.title;
+  const summaryWindow = input.text.replace(/\s+/g, " ").trim().slice(0, 360);
+  const summary = summaryWindow.length === 360 ? `${summaryWindow.slice(0, 357)}...` : summaryWindow;
+  return {
+    title,
+    summary: summary || null,
+    sourceKind: heuristicSourceKind(input.filename, contentType),
+    topics: [],
+    targetBaseId: null,
+    confidence: 0,
+    reason: null
+  };
+}
+function heuristicSourceKind(rawFilename, rawContentType) {
+  const filename = rawFilename.toLowerCase();
+  const contentType = rawContentType.toLowerCase();
+  if (filename.endsWith(".eml") || contentType === "message/rfc822") return "email";
+  if (filename.endsWith(".vtt") || filename.endsWith(".srt") || filename.includes("transcript")) {
+    return "meeting_transcript";
+  }
+  if (contentType === "text/html") return "web";
+  if (contentType === "application/pdf" || filename.endsWith(".docx") || filename.endsWith(".md")) {
+    return "document";
+  }
+  return "manual_upload";
+}
+var HeuristicCurationProvider = class {
+  model = "heuristic";
+  async curate(input) {
+    return heuristicCuration(input);
+  }
+};
+var CURATION_SYSTEM_PROMPT = [
+  "You organize a team knowledge base. Given the beginning of a dropped document",
+  "and the list of existing collections (bases), return STRICT JSON with keys:",
+  '"title" (concise, specific, <= 120 chars, no filename extensions),',
+  '"summary" (2-3 sentences, plain prose, what the document is and why it matters),',
+  '"sourceKind" (one of: manual_upload, meeting_transcript, repository, email, chat, document, web, other),',
+  '"topics" (3-6 short lowercase tags),',
+  '"targetBaseId" (the id of the best-fitting existing base, or null if none fits),',
+  '"confidence" (0..1 \u2014 how sure you are the document belongs in targetBaseId),',
+  '"reason" (one sentence explaining the filing choice).',
+  "Only pick a targetBaseId from the provided list. If the document fits no base",
+  "well, return targetBaseId null and confidence 0. Respond with JSON only."
+].join(" ");
+var OpenAICurationProvider = class {
+  client = null;
+  apiKey;
+  baseURL;
+  defaultHeaders;
+  defaultQuery;
+  model;
+  constructor(args) {
+    this.apiKey = args.apiKey ?? process.env.OPENAI_API_KEY;
+    this.baseURL = args.baseURL;
+    this.defaultHeaders = args.defaultHeaders;
+    this.defaultQuery = args.defaultQuery;
+    this.model = args.model ?? DEFAULT_DOCUMENT_CURATION_MODEL;
+  }
+  async curate(input) {
+    const response = await this.openai().chat.completions.create({
+      model: this.model,
+      response_format: { type: "json_object" },
+      messages: [
+        { role: "system", content: CURATION_SYSTEM_PROMPT },
+        {
+          role: "user",
+          content: JSON.stringify({
+            filename: input.filename,
+            currentTitle: input.title,
+            bases: input.bases,
+            text: input.text
+          })
+        }
+      ]
+    });
+    const raw = response.choices[0]?.message?.content;
+    if (!raw) {
+      throw new Error("curation model returned no content");
+    }
+    return parseCurationOutcome(raw, input.bases);
+  }
+  openai() {
+    if (!this.apiKey) {
+      throw new Error("OpenAI document curation requires an API key");
+    }
+    this.client ??= new OpenAI({
+      apiKey: this.apiKey,
+      ...this.baseURL ? { baseURL: this.baseURL } : {},
+      ...this.defaultQuery ? { defaultQuery: this.defaultQuery } : {},
+      ...this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}
+    });
+    return this.client;
+  }
+};
+function parseCurationOutcome(raw, bases) {
+  const parsed = JSON.parse(raw);
+  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+    throw new Error("curation model returned non-object JSON");
+  }
+  const record = parsed;
+  const knownBase = bases.find((base) => base.id === record.targetBaseId);
+  const confidence = typeof record.confidence === "number" && Number.isFinite(record.confidence) ? Math.min(1, Math.max(0, record.confidence)) : 0;
+  return {
+    title: cleanString(typeof record.title === "string" ? record.title.slice(0, 200) : null) ?? null,
+    summary: cleanString(typeof record.summary === "string" ? record.summary.slice(0, 2e3) : null) ?? null,
+    sourceKind: typeof record.sourceKind === "string" ? normalizeKnowledgeSourceKind(record.sourceKind) : null,
+    topics: cleanStringArray(
+      Array.isArray(record.topics) ? record.topics.filter((topic) => typeof topic === "string").slice(0, 8) : []
+    ).map((topic) => topic.toLowerCase().slice(0, 60)),
+    targetBaseId: knownBase?.id ?? null,
+    confidence: knownBase ? confidence : 0,
+    reason: cleanString(typeof record.reason === "string" ? record.reason.slice(0, 500) : null) ?? null
+  };
+}
 function createDocumentServices(settings, overrides = {}) {
   const dimensions = settings?.documentEmbeddingDimensions ?? DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS;
   const openAIEmbeddingConfig = documentOpenAIEmbeddingConfig(settings);
@@ -145,9 +267,23 @@ function createDocumentServices(settings, overrides = {}) {
       ...openAIEmbeddingConfig,
       model: settings?.documentEmbeddingModel ?? DEFAULT_DOCUMENT_EMBEDDING_MODEL,
       dimensions
-    }))
+    })),
+    curator: overrides.curator ?? createDocumentCurator(settings)
   };
 }
+function createDocumentCurator(settings) {
+  const provider = settings?.documentCurationProvider ?? "openai";
+  if (provider === "none") return void 0;
+  if (provider === "heuristic") return new HeuristicCurationProvider();
+  const embeddingConfig = documentOpenAIEmbeddingConfig(settings);
+  return new OpenAICurationProvider({
+    apiKey: settings?.documentCurationApiKey ?? embeddingConfig.apiKey,
+    baseURL: settings?.documentCurationBaseUrl ?? embeddingConfig.baseURL,
+    defaultHeaders: embeddingConfig.defaultHeaders,
+    defaultQuery: embeddingConfig.defaultQuery,
+    model: settings?.documentCurationModel ?? DEFAULT_DOCUMENT_CURATION_MODEL
+  });
+}
 function documentOpenAIEmbeddingConfig(settings) {
   if (!settings) return {};
   if (settings.documentEmbeddingApiKey || settings.documentEmbeddingBaseUrl) {
@@ -215,6 +351,27 @@ async function getDocumentBase(db, workspaceId, baseId) {
     return row ? mapDocumentBase(row) : null;
   });
 }
+async function ensureDefaultBase(db, input) {
+  return await withRlsContext(
+    db,
+    { accountId: input.accountId, workspaceId: input.workspaceId },
+    async (scopedDb) => await scopedDb.transaction(async (tx) => {
+      const defaultName = sql`lower(btrim(${schema.documentBases.name})) = ${DEFAULT_BASE_NAME.toLowerCase()}`;
+      const [existing] = await tx.select().from(schema.documentBases).where(and(eq(schema.documentBases.workspaceId, input.workspaceId), defaultName)).limit(1);
+      if (existing) return mapDocumentBase(existing);
+      const [inserted] = await tx.insert(schema.documentBases).values({
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        name: DEFAULT_BASE_NAME,
+        description: DEFAULT_BASE_DESCRIPTION
+      }).onConflictDoNothing().returning();
+      if (inserted) return mapDocumentBase(inserted);
+      const [raced] = await tx.select().from(schema.documentBases).where(and(eq(schema.documentBases.workspaceId, input.workspaceId), defaultName)).limit(1);
+      if (raced) return mapDocumentBase(raced);
+      throw new Error("Failed to create Default document base");
+    })
+  );
+}
 async function addDocumentToBase(db, input) {
   return await withRlsContext(
     db,
@@ -223,6 +380,9 @@ async function addDocumentToBase(db, input) {
       const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
       if (!base) throw new Error(`Document base not found: ${input.baseId}`);
       const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
+      if (input.visibility === "private" && !cleanString(input.createdBy ?? null)) {
+        throw new Error("private documents require a creating subject");
+      }
       const now = /* @__PURE__ */ new Date();
       const [existing] = await scopedDb.select().from(schema.documents).where(
         and(
@@ -232,6 +392,9 @@ async function addDocumentToBase(db, input) {
         )
       ).limit(1);
       if (existing) {
+        if (!documentMatchesAccess(existing, input.access)) {
+          throw new Error(`Document not found: ${existing.id}`);
+        }
         const [updated] = await scopedDb.update(schema.documents).set({
           title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? existing.title,
           ...input.sourceKind !== void 0 ? { sourceKind: input.sourceKind } : {},
@@ -247,7 +410,8 @@ async function addDocumentToBase(db, input) {
         }).where(
           and(
             eq(schema.documents.workspaceId, input.workspaceId),
-            eq(schema.documents.id, existing.id)
+            eq(schema.documents.id, existing.id),
+            ...documentAccessConditions(input.access)
           )
         ).returning();
         return mapDocument(updated ?? existing);
@@ -269,6 +433,10 @@ async function addDocumentToBase(db, input) {
         sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
         sourceVersion: cleanString(input.sourceVersion) ?? null,
         aclTags: cleanStringArray(input.aclTags),
+        visibility: input.visibility ?? "workspace",
+        agentAccess: input.agentAccess ?? true,
+        createdBy: input.createdBy ?? null,
+        curationStatus: input.curationStatus ?? "none",
         updatedAt: now
       }).returning();
       if (!row) throw new Error("Failed to create document");
@@ -276,6 +444,65 @@ async function addDocumentToBase(db, input) {
     }
   );
 }
+async function moveDocumentToBase(db, input) {
+  return await withRlsContext(
+    db,
+    { accountId: input.accountId, workspaceId: input.workspaceId },
+    async (scopedDb) => {
+      const [row] = await scopedDb.select().from(schema.documents).where(
+        and(
+          eq(schema.documents.workspaceId, input.workspaceId),
+          eq(schema.documents.id, input.documentId),
+          ...documentAccessConditions(input.access)
+        )
+      ).limit(1);
+      if (!row) throw new Error(`Document not found: ${input.documentId}`);
+      const suggestion = row.curation?.suggestedBaseId;
+      const targetBaseId = input.targetBaseId ?? suggestion;
+      if (!targetBaseId) {
+        throw new Error("document has no suggested base; pass targetBaseId");
+      }
+      if (targetBaseId === row.baseId) return mapDocument(row);
+      const base = await getDocumentBase(scopedDb, input.workspaceId, targetBaseId);
+      if (!base) throw new Error(`Document base not found: ${targetBaseId}`);
+      const [conflict] = await scopedDb.select({ id: schema.documents.id }).from(schema.documents).where(
+        and(
+          eq(schema.documents.workspaceId, input.workspaceId),
+          eq(schema.documents.baseId, targetBaseId),
+          eq(schema.documents.fileId, row.fileId)
+        )
+      ).limit(1);
+      if (conflict) {
+        throw new Error("a document for this file already exists in the target base");
+      }
+      const now = /* @__PURE__ */ new Date();
+      const moved = await scopedDb.transaction(async (tx) => {
+        const [updated] = await tx.update(schema.documents).set({
+          baseId: targetBaseId,
+          ...row.curationStatus === "suggested" || row.curationStatus === "pending" ? { curationStatus: "auto_filed" } : {},
+          updatedAt: now
+        }).where(
+          and(
+            eq(schema.documents.workspaceId, input.workspaceId),
+            eq(schema.documents.id, input.documentId),
+            ...documentAccessConditions(input.access)
+          )
+        ).returning();
+        if (updated) {
+          await tx.update(schema.documentChunks).set({ baseId: targetBaseId }).where(
+            and(
+              eq(schema.documentChunks.workspaceId, input.workspaceId),
+              eq(schema.documentChunks.documentId, input.documentId)
+            )
+          );
+        }
+        return updated;
+      });
+      if (!moved) throw new Error(`Document not found: ${input.documentId}`);
+      return mapDocument(moved);
+    }
+  );
+}
 async function deleteDocumentFromBase(db, input) {
   await withRlsContext(
     db,
@@ -284,7 +511,8 @@ async function deleteDocumentFromBase(db, input) {
       const [document] = await scopedDb.select().from(schema.documents).where(
         and(
           eq(schema.documents.workspaceId, input.workspaceId),
-          eq(schema.documents.id, input.documentId)
+          eq(schema.documents.id, input.documentId),
+          ...documentAccessConditions(input.access)
         )
       ).limit(1);
       if (!document) {
@@ -296,50 +524,64 @@ async function deleteDocumentFromBase(db, input) {
       await scopedDb.delete(schema.documents).where(
         and(
           eq(schema.documents.workspaceId, input.workspaceId),
-          eq(schema.documents.id, input.documentId)
+          eq(schema.documents.id, input.documentId),
+          ...documentAccessConditions(input.access)
         )
       );
     }
   );
 }
-async function listDocuments(db, workspaceId, baseId) {
+async function listDocuments(db, workspaceId, baseId, access) {
   return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     const rows = await scopedDb.select().from(schema.documents).where(
-      and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.baseId, baseId))
+      and(
+        eq(schema.documents.workspaceId, workspaceId),
+        eq(schema.documents.baseId, baseId),
+        ...documentAccessConditions(access)
+      )
     ).orderBy(asc(schema.documents.createdAt));
     return rows.map(mapDocument);
   });
 }
-async function getDocument(db, workspaceId, documentId) {
+async function getDocument(db, workspaceId, documentId, access) {
   return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     const [row] = await scopedDb.select().from(schema.documents).where(
-      and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))
+      and(
+        eq(schema.documents.workspaceId, workspaceId),
+        eq(schema.documents.id, documentId),
+        ...documentAccessConditions(access)
+      )
     ).limit(1);
     return row ? mapDocument(row) : null;
   });
 }
-async function queueDocumentForReindex(db, workspaceId, documentId) {
+async function queueDocumentForReindex(db, workspaceId, documentId, access) {
   return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     const [row] = await scopedDb.update(schema.documents).set({
       status: "queued",
       error: null,
       updatedAt: /* @__PURE__ */ new Date()
     }).where(
-      and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))
+      and(
+        eq(schema.documents.workspaceId, workspaceId),
+        eq(schema.documents.id, documentId),
+        ...documentAccessConditions(access)
+      )
     ).returning();
     if (!row) throw new Error(`Document not found: ${documentId}`);
     return mapDocument(row);
   });
 }
 async function indexDocumentNow(db, objectStorage, workspaceId, documentId, services = createDocumentServices(), hooks = {}) {
-  const [document] = await withWorkspaceRls(
+  const [loadedDocument] = await withWorkspaceRls(
     db,
     workspaceId,
     async (scopedDb) => await scopedDb.select().from(schema.documents).where(
       and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))
     ).limit(1)
   );
-  if (!document) throw new Error(`Document not found: ${documentId}`);
+  if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
+  let document = loadedDocument;
   const file = await requireReadyFile(db, workspaceId, document.fileId);
   await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
     await scopedDb.update(schema.documents).set({
@@ -354,6 +596,9 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
   try {
     const bytes = await objectStorage.getFileBytes(file);
     const parsed = await services.parser.parse(bytes, file);
+    if (document.curationStatus === "pending") {
+      document = await curateDroppedDocument(db, services, document, parsed, file);
+    }
     const chunks = services.chunker.chunk(parsed, file);
     await hooks.beforeEmbed?.({
       accountId: document.accountId,
@@ -438,10 +683,104 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
     if (!failed) throw error;
     return mapDocument(failed);
   }
-  const updated = await getDocument(db, workspaceId, documentId);
+  const updated = await getDocument(db, workspaceId, documentId, {
+    viewerSubjectId: document.createdBy
+  });
   if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
   return updated;
 }
+async function curateDroppedDocument(db, services, document, parsed, file) {
+  if (!services.curator) {
+    const [updated2] = await withWorkspaceRls(
+      db,
+      document.workspaceId,
+      async (scopedDb) => await scopedDb.update(schema.documents).set({
+        curationStatus: "none",
+        summary: null,
+        topics: [],
+        curation: null,
+        updatedAt: /* @__PURE__ */ new Date()
+      }).where(
+        and(
+          eq(schema.documents.workspaceId, document.workspaceId),
+          eq(schema.documents.id, document.id)
+        )
+      ).returning()
+    );
+    return updated2 ?? document;
+  }
+  const bases = await listDocumentBases(db, document.workspaceId);
+  const candidates = bases.filter((base) => base.id !== document.baseId).map((base) => ({ id: base.id, name: base.name, description: base.description }));
+  const input = {
+    text: parsed.text.slice(0, DOCUMENT_CURATION_MAX_INPUT_CHARS),
+    filename: file.filename,
+    title: document.title,
+    bases: candidates
+  };
+  let outcome;
+  let model;
+  let failure = null;
+  try {
+    outcome = await services.curator.curate(input);
+    model = services.curator.model;
+  } catch (error) {
+    failure = error instanceof Error ? error.message : String(error);
+    console.warn("document curation failed; applying heuristic fallback", {
+      workspaceId: document.workspaceId,
+      documentId: document.id,
+      error: failure
+    });
+    outcome = heuristicCuration(input, file.contentType);
+    model = "heuristic";
+  }
+  const suggestedBase = candidates.find((base) => base.id === outcome.targetBaseId) ?? null;
+  let moveToBaseId = null;
+  if (suggestedBase && failure === null && outcome.confidence >= DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE) {
+    const conflict = await withWorkspaceRls(
+      db,
+      document.workspaceId,
+      async (scopedDb) => await scopedDb.select({ id: schema.documents.id }).from(schema.documents).where(
+        and(
+          eq(schema.documents.workspaceId, document.workspaceId),
+          eq(schema.documents.baseId, suggestedBase.id),
+          eq(schema.documents.fileId, document.fileId)
+        )
+      ).limit(1)
+    );
+    if (conflict.length === 0) {
+      moveToBaseId = suggestedBase.id;
+    }
+  }
+  const curation = {
+    suggestedBaseId: suggestedBase?.id ?? null,
+    suggestedBaseName: suggestedBase?.name ?? null,
+    confidence: outcome.confidence,
+    reason: failure ? `curation failed (${failure}); heuristic fallback applied` : outcome.reason,
+    originalTitle: document.title,
+    model
+  };
+  const curationStatus = failure ? "failed" : moveToBaseId ? "auto_filed" : "suggested";
+  const [updated] = await withWorkspaceRls(
+    db,
+    document.workspaceId,
+    async (scopedDb) => await scopedDb.update(schema.documents).set({
+      title: outcome.title ?? document.title,
+      summary: outcome.summary,
+      topics: outcome.topics,
+      ...outcome.sourceKind ? { sourceKind: outcome.sourceKind } : {},
+      ...moveToBaseId ? { baseId: moveToBaseId } : {},
+      curationStatus,
+      curation,
+      updatedAt: /* @__PURE__ */ new Date()
+    }).where(
+      and(
+        eq(schema.documents.workspaceId, document.workspaceId),
+        eq(schema.documents.id, document.id)
+      )
+    ).returning()
+  );
+  return updated ?? document;
+}
 async function searchDocuments(db, input, services = createDocumentServices()) {
   const mode = input.mode ?? "hybrid";
   const limit = Math.min(Math.max(input.limit ?? 5, 1), 50);
@@ -539,7 +878,7 @@ async function keywordSearchDocuments(db, input, limit) {
     keywordScore: normalizeKeywordScore(Number(row.rank))
   }));
 }
-async function getDocumentChunk(db, workspaceId, chunkId) {
+async function getDocumentChunk(db, workspaceId, chunkId, access) {
   const [row] = await withWorkspaceRls(
     db,
     workspaceId,
@@ -565,7 +904,8 @@ async function getDocumentChunk(db, workspaceId, chunkId) {
       and(
         eq(schema.documentChunks.workspaceId, workspaceId),
         eq(schema.documentChunks.id, chunkId),
-        eq(schema.documents.status, "ready")
+        eq(schema.documents.status, "ready"),
+        ...documentAccessConditions(access)
       )
     ).limit(1)
   );
@@ -578,10 +918,36 @@ async function getDocumentChunk(db, workspaceId, chunkId) {
     keywordScore: null
   };
 }
+function documentAccessConditions(access) {
+  if (access?.agentOnly) {
+    const viewer2 = access.viewerSubjectId;
+    const visibility = viewer2 ? or(eq(schema.documents.visibility, "workspace"), eq(schema.documents.createdBy, viewer2)) : eq(schema.documents.visibility, "workspace");
+    return [eq(schema.documents.agentAccess, true), ...visibility ? [visibility] : []];
+  }
+  const viewer = access?.viewerSubjectId;
+  if (viewer) {
+    const condition = or(
+      eq(schema.documents.visibility, "workspace"),
+      eq(schema.documents.createdBy, viewer)
+    );
+    return condition ? [condition] : [];
+  }
+  return [eq(schema.documents.visibility, "workspace")];
+}
+function documentMatchesAccess(document, access) {
+  if (access?.agentOnly) {
+    return document.agentAccess && (document.visibility !== "private" || !!access.viewerSubjectId && document.createdBy === access.viewerSubjectId);
+  }
+  return canViewDocument(document, access?.viewerSubjectId);
+}
+function canViewDocument(document, viewerSubjectId) {
+  return document.visibility !== "private" || !!viewerSubjectId && document.createdBy === viewerSubjectId;
+}
 function documentSearchConditions(input, embeddingModel) {
   const conditions = [
     eq(schema.documents.status, "ready"),
-    eq(schema.documentChunks.workspaceId, input.workspaceId)
+    eq(schema.documentChunks.workspaceId, input.workspaceId),
+    ...documentAccessConditions(input.access)
   ];
   if (embeddingModel) {
     conditions.push(eq(schema.documentChunks.embeddingModel, embeddingModel));
@@ -822,33 +1188,66 @@ function mapDocument(row) {
     sourceUpdatedAt: row.sourceUpdatedAt?.toISOString() ?? null,
     sourceVersion: row.sourceVersion,
     aclTags: cleanStringArray(row.aclTags),
+    visibility: normalizeDocumentVisibility(row.visibility),
+    createdBy: row.createdBy,
+    agentAccess: row.agentAccess,
+    summary: row.summary,
+    topics: cleanStringArray(row.topics),
+    curationStatus: normalizeDocumentCurationStatus(row.curationStatus),
+    curation: row.curation ?? null,
     createdAt: row.createdAt.toISOString(),
     updatedAt: row.updatedAt.toISOString()
   };
 }
+function normalizeDocumentVisibility(value) {
+  return value === "private" ? "private" : "workspace";
+}
+function normalizeDocumentCurationStatus(value) {
+  switch (value) {
+    case "pending":
+    case "suggested":
+    case "auto_filed":
+    case "failed":
+      return value;
+    default:
+      return "none";
+  }
+}
 export {
+  DEFAULT_BASE_DESCRIPTION,
+  DEFAULT_BASE_NAME,
   DEFAULT_DOCUMENT_CHUNK_OVERLAP,
   DEFAULT_DOCUMENT_CHUNK_SIZE,
+  DEFAULT_DOCUMENT_CURATION_MODEL,
   DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS,
   DEFAULT_DOCUMENT_EMBEDDING_MODEL,
   DEFAULT_DOCUMENT_PARSER,
+  DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE,
+  DOCUMENT_CURATION_MAX_INPUT_CHARS,
   DeterministicEmbeddingProvider,
+  HeuristicCurationProvider,
   LiteParseDocumentParser,
+  OpenAICurationProvider,
   OpenAIEmbeddingProvider,
   RecursiveTextChunker,
   addDocumentToBase,
+  canViewDocument,
   chunkText,
   createDocumentBase,
   createDocumentServices,
   deleteDocumentFromBase,
   deterministicEmbedding,
   documentOpenAIEmbeddingConfig,
+  ensureDefaultBase,
   getDocument,
   getDocumentBase,
   getDocumentChunk,
+  heuristicCuration,
   indexDocumentNow,
   listDocumentBases,
   listDocuments,
+  moveDocumentToBase,
+  parseCurationOutcome,
   parseDocumentBytes,
   queueDocumentForReindex,
   searchDocuments