@opengeni/documents 0.8.28-canary.0 → 0.8.28-canary.2

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+export { knowledgeIndexChunks } from "./knowledge-index.js";
+export type { KnowledgeIndexChunk } from "./knowledge-index.js";
 import type { Settings } from "@opengeni/config";
 import { type AddDocumentRequest, CreateDocumentBaseRequest, Document, DocumentAuthorityKind, DocumentBase, DocumentCurationStatus, DocumentSearchMode, DocumentSearchResult, DocumentStatus, DocumentVisibility, FileAsset, KnowledgeBrowseResponse, KnowledgeRecord, KnowledgeSearchResponse, KnowledgeSourceKind, ListIndexedDocumentsResponse, type ReclassifyDocumentAuthorityRequest, type RunDocumentDefaultCollectionBackfillRequest } from "@opengeni/contracts";
 import { type Database } from "@opengeni/db";
@@ -201,14 +203,6 @@ export type EffectiveKnowledgeBrowseInput = {
     limit?: number | undefined;
     agentAuthority?: AgentDocumentAuthorityContext | undefined;
 };
-export type DocumentIndexHooks = {
-    beforeEmbed?: (input: {
-        accountId: string;
-        workspaceId: string;
-        documentId: string;
-        chunkCount: number;
-    }) => Promise<void>;
-};
 export declare class LiteParseDocumentParser implements DocumentParser {
     readonly name = "liteparse";
     private parseQueue;
@@ -536,7 +530,7 @@ export declare function getDocumentOriginalFile(db: Database, input: {
  */
 export declare function getDocumentForIndexing(db: Database, workspaceId: string, documentId: string): Promise<Document | null>;
 export declare function queueDocumentForReindex(db: Database, workspaceId: string, documentId: string, access?: DocumentAccessFilter, organizationAuthorityGranted?: boolean): Promise<Document>;
-export declare function indexDocumentNow(db: Database, objectStorage: ObjectStorage, workspaceId: string, documentId: string, services?: DocumentServices, hooks?: DocumentIndexHooks, access?: DocumentAccessFilter): Promise<Document>;
+export declare function indexDocumentNow(db: Database, objectStorage: ObjectStorage, workspaceId: string, documentId: string, services?: DocumentServices, access?: DocumentAccessFilter): Promise<Document>;
 export declare function searchDocuments(db: Database, input: DocumentSearchInput, services?: Pick<DocumentServices, "embedder">): Promise<DocumentSearchResult[]>;
 export type DocumentCandidateRelevanceFloor = {
     vectorScore: number;

package/dist/index.js

@@ -1,3 +1,34 @@
+// src/knowledge-index.ts
+function* knowledgeIndexChunks(entry) {
+  const field = entry.content.length ? "content" : "title";
+  const text = field === "content" ? entry.content : entry.title;
+  const size = 1200;
+  const overlap = 160;
+  let start = 0;
+  let index = 0;
+  const scalarBoundary = (at) => {
+    const code = text.charCodeAt(at);
+    const previous = text.charCodeAt(at - 1);
+    return code >= 56320 && code <= 57343 && previous >= 55296 && previous <= 56319 ? at - 1 : at;
+  };
+  while (start < text.length) {
+    const end = scalarBoundary(Math.min(start + size, text.length));
+    const excerpt = text.slice(start, end);
+    yield {
+      index,
+      field,
+      start,
+      end,
+      text: excerpt,
+      embeddingInput: field === "content" ? `${entry.title}
+${excerpt}` : excerpt
+    };
+    if (end === text.length) return;
+    start = scalarBoundary(end - overlap);
+    index += 1;
+  }
+}
+
 // src/index.ts
 import {
   KnowledgeProviderCitation,
@@ -13,6 +44,8 @@ import {
   OrganizationDocumentAuthorityReclassification
 } from "@opengeni/contracts";
 import {
+  claimKnowledgeDocumentPreparation,
+  completeKnowledgeDocumentPreparation,
   createPersonalDocumentAuthority,
   getFilesForSubject,
   resolveDocumentOriginalFileForSubject,
@@ -232,12 +265,12 @@ var LiteParseDocumentParser = class {
   name = DEFAULT_DOCUMENT_PARSER;
   parseQueue = Promise.resolve();
   async parse(bytes, file) {
-    const text = isTextLike(file) ? Buffer.from(bytes).toString("utf8").replace(/\0/g, " ").trim() : await this.parseWithLiteParse(bytes);
+    const text = isTextLike(file) ? Buffer.from(bytes).toString("utf8") : await this.parseWithLiteParse(bytes);
     if (!text.trim()) {
       throw new Error(`Parsed document is empty: ${file.filename}`);
     }
     return {
-      text: text.trim(),
+      text,
       metadata: {
         parser: this.name,
         filename: file.filename,
@@ -251,7 +284,7 @@ var LiteParseDocumentParser = class {
       const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1, quiet: true });
       const result = await parser.parse(Buffer.from(bytes));
       const text = typeof result?.text === "string" ? result.text : "";
-      return text.replace(/\0/g, " ").trim();
+      return text;
     });
   }
   async enqueueParse(task) {
@@ -1434,7 +1467,7 @@ function assertOrganizationDocumentAuthority(authorityKind, organizationAuthorit
     throw new Error("organization document mutations require exact account authority");
   }
 }
-async function indexDocumentNow(db, objectStorage, workspaceId, documentId, services = createDocumentServices(), hooks = {}, access) {
+async function indexDocumentNow(db, objectStorage, workspaceId, documentId, services = createDocumentServices(), access) {
   const [loadedDocument] = await withDocumentRls(
     db,
     workspaceId,
@@ -1444,13 +1477,7 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
     ).limit(1)
   );
   if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
-  let document = loadedDocument;
-  const file = await requireReadyFile(db, {
-    accountId: document.accountId,
-    workspaceId,
-    subjectId: cleanString(document.createdBy) ?? null,
-    fileId: document.fileId
-  });
+  const document = loadedDocument;
   await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
     await scopedDb.update(schema.documents).set({
       status: "indexing",
@@ -1462,88 +1489,46 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
     );
   });
   try {
-    const object = await retryWhileMissing(
-      async () => objectStorage.getObjectBytes(file.objectKey)
-    );
-    if (!object) throw new Error("document source object is missing");
-    const bytes = object.bytes;
-    const parsed = await services.parser.parse(bytes, file);
-    if (document.curationStatus === "pending") {
-      document = await curateDroppedDocument(db, services, document, parsed, file);
-    }
-    const chunks = services.chunker.chunk(parsed, file);
-    await hooks.beforeEmbed?.({
+    const identity = {
       accountId: document.accountId,
-      workspaceId: document.workspaceId,
+      workspaceId,
       documentId,
-      chunkCount: chunks.length
-    });
-    const embeddings = await services.embedder.embedMany(chunks.map((chunk) => chunk.text));
-    if (embeddings.length !== chunks.length) {
-      throw new Error(
-        `Embedding provider returned ${embeddings.length} embeddings for ${chunks.length} chunks`
-      );
+      fileId: document.fileId
+    };
+    const preparation = await claimKnowledgeDocumentPreparation(db, identity);
+    let chunkCount = 0;
+    if (preparation.status === "prepare") {
+      const file = preparation.file;
+      const object = await retryWhileMissing(() => objectStorage.getObjectBytes(file.objectKey));
+      if (!object) throw new Error("document source object is missing");
+      const bytes = object.bytes;
+      if (bytes.length !== file.sizeBytes) throw new Error("document source size changed");
+      const sourceVersion = createHash("sha256").update(bytes).digest("hex");
+      if (file.sha256 && file.sha256 !== sourceVersion)
+        throw new Error("document source hash changed");
+      const parsed = await services.parser.parse(bytes, file);
+      await completeKnowledgeDocumentPreparation(db, {
+        ...identity,
+        leaseId: preparation.leaseId,
+        title: document.title,
+        content: parsed.text,
+        sourceVersion
+      });
+      for (const _chunk of knowledgeIndexChunks({ title: document.title, content: parsed.text }))
+        chunkCount++;
     }
-    await withDocumentRls(
-      db,
-      workspaceId,
-      access,
-      async (scopedDb) => await scopedDb.transaction(async (tx) => {
-        await tx.delete(schema.documentChunks).where(
-          and(
-            eq(schema.documentChunks.workspaceId, workspaceId),
-            eq(schema.documentChunks.documentId, documentId)
-          )
-        );
-        if (chunks.length > 0) {
-          await tx.insert(schema.documentChunks).values(
-            chunks.map((chunk, index) => ({
-              accountId: document.accountId,
-              workspaceId: document.workspaceId,
-              documentId,
-              baseId: document.baseId,
-              fileId: file.id,
-              authorityKind: document.authorityKind,
-              authorityWorkspaceId: document.authorityWorkspaceId,
-              authoritySubjectId: document.authoritySubjectId,
-              chunkIndex: index,
-              text: chunk.text,
-              metadata: {
-                ...chunk.metadata,
-                documentTitle: document.title,
-                sourceKind: document.sourceKind,
-                sourceUri: document.sourceUri,
-                sourceExternalId: document.sourceExternalId,
-                sourceTitle: document.sourceTitle,
-                sourceAuthor: document.sourceAuthor,
-                sourceCreatedAt: document.sourceCreatedAt?.toISOString() ?? null,
-                sourceUpdatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
-                sourceVersion: document.sourceVersion,
-                aclTags: document.aclTags
-              },
-              embedding: validateEmbedding(
-                embeddings[index] ?? [],
-                services.embedder.dimensions,
-                services.embedder.model
-              ),
-              embeddingModel: services.embedder.model
-            }))
-          );
-        }
-        await tx.update(schema.documents).set({
-          status: "ready",
-          parser: services.parser.name,
-          chunkCount: chunks.length,
-          error: null,
-          updatedAt: /* @__PURE__ */ new Date()
-        }).where(
-          and(
-            eq(schema.documents.workspaceId, workspaceId),
-            eq(schema.documents.id, documentId)
-          )
-        );
-      })
-    );
+    await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
+      await scopedDb.update(schema.documents).set({
+        status: "ready",
+        parser: services.parser.name,
+        chunkCount,
+        error: null,
+        ...document.curationStatus === "pending" ? { curationStatus: "none" } : {},
+        updatedAt: /* @__PURE__ */ new Date()
+      }).where(
+        and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))
+      );
+    });
   } catch (error) {
     const [failed] = await withDocumentRls(
       db,
@@ -1564,105 +1549,6 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
   if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
   return updated;
 }
-async function curateDroppedDocument(db, services, document, parsed, file) {
-  if (!services.curator) {
-    const [updated2] = await withDocumentRls(
-      db,
-      document.workspaceId,
-      { viewerSubjectId: document.authoritySubjectId },
-      async (scopedDb) => await scopedDb.update(schema.documents).set({
-        curationStatus: "none",
-        summary: null,
-        topics: [],
-        curation: null,
-        updatedAt: /* @__PURE__ */ new Date()
-      }).where(
-        and(
-          eq(schema.documents.workspaceId, document.workspaceId),
-          eq(schema.documents.id, document.id)
-        )
-      ).returning()
-    );
-    return updated2 ?? document;
-  }
-  const bases = await listDocumentBases(db, document.workspaceId);
-  const candidates = bases.filter((base) => base.id !== document.baseId).map((base) => ({
-    id: base.id,
-    name: base.name,
-    description: base.description
-  }));
-  const input = {
-    text: parsed.text.slice(0, DOCUMENT_CURATION_MAX_INPUT_CHARS),
-    filename: file.filename,
-    title: document.title,
-    bases: candidates
-  };
-  let outcome;
-  let model;
-  let failure = null;
-  try {
-    outcome = await services.curator.curate(input);
-    model = services.curator.model;
-  } catch (error) {
-    failure = error instanceof Error ? error.message : String(error);
-    console.warn("document curation failed; applying heuristic fallback", {
-      workspaceId: document.workspaceId,
-      documentId: document.id,
-      error: failure
-    });
-    outcome = heuristicCuration(input, file.contentType);
-    model = "heuristic";
-  }
-  const suggestedBase = candidates.find((base) => base.id === outcome.targetBaseId) ?? null;
-  let moveToBaseId = null;
-  if (suggestedBase && failure === null && outcome.confidence >= DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE) {
-    const conflict = await withDocumentRls(
-      db,
-      document.workspaceId,
-      { viewerSubjectId: document.authoritySubjectId },
-      async (scopedDb) => await scopedDb.select({ id: schema.documents.id }).from(schema.documents).where(
-        and(
-          eq(schema.documents.workspaceId, document.workspaceId),
-          eq(schema.documents.baseId, suggestedBase.id),
-          eq(schema.documents.fileId, document.fileId)
-        )
-      ).limit(1)
-    );
-    if (conflict.length === 0) {
-      moveToBaseId = suggestedBase.id;
-    }
-  }
-  const curation = {
-    suggestedBaseId: suggestedBase?.id ?? null,
-    suggestedBaseName: suggestedBase?.name ?? null,
-    confidence: outcome.confidence,
-    reason: failure ? `curation failed (${failure}); heuristic fallback applied` : outcome.reason,
-    originalTitle: document.title,
-    model
-  };
-  const curationStatus = failure ? "failed" : moveToBaseId ? "auto_filed" : "suggested";
-  const [updated] = await withDocumentRls(
-    db,
-    document.workspaceId,
-    { viewerSubjectId: document.authoritySubjectId },
-    async (scopedDb) => await scopedDb.update(schema.documents).set({
-      title: outcome.title ?? document.title,
-      summary: outcome.summary,
-      topics: outcome.topics,
-      ...outcome.sourceKind ? { sourceKind: outcome.sourceKind } : {},
-      ...moveToBaseId ? { baseId: moveToBaseId } : {},
-      curationStatus,
-      curation,
-      updatedAt: /* @__PURE__ */ new Date()
-    }).where(
-      and(
-        eq(schema.documents.workspaceId, document.workspaceId),
-        eq(schema.documents.id, document.id)
-      )
-    ).returning()
-  );
-  return updated ?? document;
-}
 async function searchDocuments(db, input, services = createDocumentServices()) {
   return (await searchDocumentCandidates(db, input, services)).results;
 }
@@ -3145,6 +3031,7 @@ export {
   getEffectiveKnowledgeRecord,
   heuristicCuration,
   indexDocumentNow,
+  knowledgeIndexChunks,
   listAccessibleDocuments,
   listDocumentAuthorityReclassifications,
   listDocumentBases,