@opengeni/documents 0.5.18 → 0.5.32

package/dist/index.js

@@ -8,9 +8,8 @@ import {
   withWorkspaceSubjectRls
 } from "@opengeni/db";
 import * as schema from "@opengeni/db/schema";
-import { LiteParse } from "@llamaindex/liteparse";
-import { and, asc, desc, eq, inArray, or, sql } from "drizzle-orm";
-import OpenAI from "openai";
+import { createHash } from "crypto";
+import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql } from "drizzle-orm";
 var DEFAULT_DOCUMENT_PARSER = "liteparse";
 var DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
 var DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
@@ -19,6 +18,7 @@ var DEFAULT_DOCUMENT_CHUNK_OVERLAP = 160;
 var DEFAULT_DOCUMENT_CURATION_MODEL = "gpt-4o-mini";
 var DOCUMENT_CURATION_MAX_INPUT_CHARS = 24e3;
 var DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES = 1024;
+var DOCUMENT_INDEX_CHECKPOINT_MAX_CHARS = 1024;
 var DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE = 0.75;
 var DEFAULT_BASE_NAME = "Default";
 var DEFAULT_BASE_DESCRIPTION = "Default base for dropped files and notes.";
@@ -44,6 +44,7 @@ var LiteParseDocumentParser = class {
   }
   async parseWithLiteParse(bytes) {
     return await this.enqueueParse(async () => {
+      const { LiteParse } = await import("@llamaindex/liteparse");
       const parser = new LiteParse({ ocrEnabled: true, numWorkers: 1 });
       const result = await parser.parse(Buffer.from(bytes), true);
       const text = typeof result?.text === "string" ? result.text : "";
@@ -85,7 +86,7 @@ var RecursiveTextChunker = class {
   }
 };
 var OpenAIEmbeddingProvider = class {
-  client = null;
+  clientPromise = null;
   apiKey;
   baseURL;
   constructor(args) {
@@ -105,7 +106,7 @@ var OpenAIEmbeddingProvider = class {
     const out = [];
     for (let start = 0; start < texts.length; start += 64) {
       const batch = texts.slice(start, start + 64);
-      const response = await this.openai().embeddings.create({
+      const response = await (await this.openai()).embeddings.create({
         model: this.model,
         input: batch,
         dimensions: this.dimensions
@@ -123,17 +124,19 @@ var OpenAIEmbeddingProvider = class {
     }
     return embedding;
   }
-  openai() {
+  async openai() {
     if (!this.apiKey) {
       throw new Error("OpenAI document embeddings require an API key");
     }
-    this.client ??= new OpenAI({
-      apiKey: this.apiKey,
-      ...this.baseURL ? { baseURL: this.baseURL } : {},
-      ...this.defaultQuery ? { defaultQuery: this.defaultQuery } : {},
-      ...this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}
-    });
-    return this.client;
+    this.clientPromise ??= import("openai").then(
+      ({ default: OpenAIClient }) => new OpenAIClient({
+        apiKey: this.apiKey,
+        ...this.baseURL ? { baseURL: this.baseURL } : {},
+        ...this.defaultQuery ? { defaultQuery: this.defaultQuery } : {},
+        ...this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}
+      })
+    );
+    return await this.clientPromise;
   }
 };
 var DeterministicEmbeddingProvider = class {
@@ -196,7 +199,7 @@ var CURATION_SYSTEM_PROMPT = [
   "well, return targetBaseId null and confidence 0. Respond with JSON only."
 ].join(" ");
 var OpenAICurationProvider = class {
-  client = null;
+  clientPromise = null;
   apiKey;
   baseURL;
   defaultHeaders;
@@ -210,7 +213,7 @@ var OpenAICurationProvider = class {
     this.model = args.model ?? DEFAULT_DOCUMENT_CURATION_MODEL;
   }
   async curate(input) {
-    const response = await this.openai().chat.completions.create({
+    const response = await (await this.openai()).chat.completions.create({
       model: this.model,
       response_format: { type: "json_object" },
       messages: [
@@ -232,17 +235,19 @@ var OpenAICurationProvider = class {
     }
     return parseCurationOutcome(raw, input.bases);
   }
-  openai() {
+  async openai() {
     if (!this.apiKey) {
       throw new Error("OpenAI document curation requires an API key");
     }
-    this.client ??= new OpenAI({
-      apiKey: this.apiKey,
-      ...this.baseURL ? { baseURL: this.baseURL } : {},
-      ...this.defaultQuery ? { defaultQuery: this.defaultQuery } : {},
-      ...this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}
-    });
-    return this.client;
+    this.clientPromise ??= import("openai").then(
+      ({ default: OpenAIClient }) => new OpenAIClient({
+        apiKey: this.apiKey,
+        ...this.baseURL ? { baseURL: this.baseURL } : {},
+        ...this.defaultQuery ? { defaultQuery: this.defaultQuery } : {},
+        ...this.defaultHeaders ? { defaultHeaders: this.defaultHeaders } : {}
+      })
+    );
+    return await this.clientPromise;
   }
 };
 function parseCurationOutcome(raw, bases) {
@@ -534,15 +539,24 @@ async function addDocumentToBase(db, input) {
       const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
       if (!base) throw new Error(`Document base not found: ${input.baseId}`);
       const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
+      const knowledgeSourceIdentity = cleanString(input.knowledgeSourceIdentity ?? null);
+      if (knowledgeSourceIdentity && knowledgeSourceIdentity.length > 512) {
+        throw new Error("knowledge source document identity exceeds 512 characters");
+      }
       const now = /* @__PURE__ */ new Date();
       const [existing] = await scopedDb.select().from(schema.documents).where(
         and(
           eq(schema.documents.workspaceId, input.workspaceId),
-          eq(schema.documents.baseId, input.baseId),
-          eq(schema.documents.fileId, input.fileId)
+          ...knowledgeSourceIdentity ? [eq(schema.documents.knowledgeSourceIdentity, knowledgeSourceIdentity)] : [
+            eq(schema.documents.baseId, input.baseId),
+            eq(schema.documents.fileId, input.fileId)
+          ]
         )
       ).limit(1);
       if (existing) {
+        if (knowledgeSourceIdentity && existing.fileId !== input.fileId) {
+          throw new Error("knowledge source document identity is bound to different content");
+        }
         if (!documentMatchesAccess(existing, input.workspaceId, input.access)) {
           throw new Error(`Document not found: ${existing.id}`);
         }
@@ -587,6 +601,7 @@ async function addDocumentToBase(db, input) {
         sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
         sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
         sourceVersion: cleanString(input.sourceVersion) ?? null,
+        knowledgeSourceIdentity,
         aclTags: cleanStringArray(input.aclTags),
         authorityKind: authority.kind,
         authorityWorkspaceId: authority.workspaceId,
@@ -630,7 +645,7 @@ async function moveDocumentToBase(db, input) {
         and(
           eq(schema.documents.workspaceId, input.workspaceId),
           eq(schema.documents.baseId, targetBaseId),
-          eq(schema.documents.fileId, row.fileId)
+          ...row.knowledgeSourceIdentity ? [eq(schema.documents.knowledgeSourceIdentity, row.knowledgeSourceIdentity)] : [eq(schema.documents.fileId, row.fileId)]
         )
       ).limit(1);
       if (conflict) {
@@ -710,6 +725,101 @@ async function listDocuments(db, workspaceId, baseId, access) {
     return rows.map(mapDocument);
   });
 }
+async function listEffectiveIndexedDocuments(db, input) {
+  const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
+  const limit = input.limit ?? 50;
+  if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
+    throw new Error("indexed document list limit must be between 1 and 100");
+  }
+  const afterSequence = input.checkpoint ? decodeDocumentIndexCheckpoint(input.checkpoint, {
+    accountId: input.accountId,
+    workspaceId: input.workspaceId,
+    initiatingSubjectId
+  }) : 0n;
+  const access = {
+    agentOnly: true,
+    viewerSubjectId: initiatingSubjectId
+  };
+  const rows = await withDocumentAccountRls(
+    db,
+    input.accountId,
+    input.workspaceId,
+    access,
+    async (scopedDb) => await scopedDb.select().from(schema.documents).where(
+      and(
+        eq(schema.documents.accountId, input.accountId),
+        eq(schema.documents.status, "ready"),
+        isNotNull(schema.documents.indexSequence),
+        gt(schema.documents.indexSequence, afterSequence),
+        ...documentAccessConditions(input.workspaceId, access)
+      )
+    ).orderBy(asc(schema.documents.indexSequence)).limit(limit + 1)
+  );
+  const hasMore = rows.length > limit;
+  const pageRows = rows.slice(0, limit);
+  const nextSequence = pageRows.at(-1)?.indexSequence ?? afterSequence;
+  if (nextSequence === null) {
+    throw new Error("ready document is missing its index sequence");
+  }
+  return {
+    documents: pageRows.map(mapIndexedDocumentSummary),
+    nextCheckpoint: encodeDocumentIndexCheckpoint({
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      initiatingSubjectId,
+      sequence: nextSequence
+    }),
+    hasMore
+  };
+}
+function encodeDocumentIndexCheckpoint(input) {
+  if (input.sequence < 0n) throw new Error("document index checkpoint sequence is invalid");
+  return Buffer.from(
+    JSON.stringify({
+      v: 1,
+      s: documentIndexCheckpointScope(input),
+      q: input.sequence.toString()
+    }),
+    "utf8"
+  ).toString("base64url");
+}
+function decodeDocumentIndexCheckpoint(value, scope) {
+  try {
+    if (!value || value.length > DOCUMENT_INDEX_CHECKPOINT_MAX_CHARS) {
+      throw new Error("checkpoint length");
+    }
+    const bytes = Buffer.from(value, "base64url");
+    if (bytes.toString("base64url") !== value) throw new Error("checkpoint encoding");
+    const parsed = JSON.parse(bytes.toString("utf8"));
+    if (Object.keys(parsed).sort().join(",") !== "q,s,v" || parsed.v !== 1 || typeof parsed.s !== "string" || typeof parsed.q !== "string" || !/^(0|[1-9][0-9]*)$/.test(parsed.q)) {
+      throw new Error("checkpoint payload");
+    }
+    if (parsed.s !== documentIndexCheckpointScope(scope)) {
+      throw new Error("document index checkpoint belongs to a different workspace or subject");
+    }
+    return BigInt(parsed.q);
+  } catch (error) {
+    if (error instanceof Error && error.message === "document index checkpoint belongs to a different workspace or subject") {
+      throw error;
+    }
+    throw new Error("invalid document index checkpoint", { cause: error });
+  }
+}
+function documentIndexCheckpointScope(input) {
+  return createHash("sha256").update("opengeni:document-index-checkpoint:v1\0").update(input.accountId).update("\0").update(input.workspaceId).update("\0").update(canonicalEffectiveDocumentSubject(input.initiatingSubjectId)).digest("hex");
+}
+function canonicalEffectiveDocumentSubject(value) {
+  const subjectId = cleanString(value);
+  if (!subjectId || subjectId !== value) {
+    throw new Error("effective document retrieval requires an initiating subject");
+  }
+  if (new TextEncoder().encode(subjectId).byteLength > DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES) {
+    throw new Error(
+      `effective document initiating subject exceeds ${DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES} UTF-8 bytes`
+    );
+  }
+  return subjectId;
+}
 async function getDocument(db, workspaceId, documentId, access) {
   return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
     const [row] = await scopedDb.select().from(schema.documents).where(
@@ -1508,6 +1618,40 @@ function mapDocument(row) {
     updatedAt: row.updatedAt.toISOString()
   };
 }
+function mapIndexedDocumentSummary(row) {
+  if (row.indexSequence === null || row.indexedAt === null) {
+    throw new Error(`Ready document is missing index completion metadata: ${row.id}`);
+  }
+  return {
+    id: row.id,
+    title: row.title,
+    parser: row.parser,
+    chunkCount: row.chunkCount,
+    indexedAt: row.indexedAt.toISOString(),
+    summary: row.summary,
+    topics: cleanStringArray(row.topics),
+    source: {
+      kind: normalizeKnowledgeSourceKind(row.sourceKind),
+      uri: row.sourceUri,
+      externalId: row.sourceExternalId,
+      title: row.sourceTitle,
+      author: row.sourceAuthor,
+      createdAt: row.sourceCreatedAt?.toISOString() ?? null,
+      updatedAt: row.sourceUpdatedAt?.toISOString() ?? null,
+      version: row.sourceVersion
+    },
+    provenance: {
+      ingestionWorkspaceId: row.workspaceId,
+      baseId: row.baseId,
+      fileId: row.fileId,
+      authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
+      authorityWorkspaceId: row.authorityWorkspaceId,
+      authoritySubjectId: row.authoritySubjectId,
+      createdBy: row.createdBy,
+      createdAt: row.createdAt.toISOString()
+    }
+  };
+}
 function normalizeDocumentAuthorityKind(value) {
   switch (value) {
     case "organization":
@@ -1543,6 +1687,7 @@ export {
   DOCUMENT_AUTHORITY_SUBJECT_MAX_BYTES,
   DOCUMENT_CURATION_AUTO_FILE_CONFIDENCE,
   DOCUMENT_CURATION_MAX_INPUT_CHARS,
+  DOCUMENT_INDEX_CHECKPOINT_MAX_CHARS,
   DeterministicEmbeddingProvider,
   HeuristicCurationProvider,
   LiteParseDocumentParser,
@@ -1554,9 +1699,11 @@ export {
   chunkText,
   createDocumentBase,
   createDocumentServices,
+  decodeDocumentIndexCheckpoint,
   deleteDocumentFromBase,
   deterministicEmbedding,
   documentOpenAIEmbeddingConfig,
+  encodeDocumentIndexCheckpoint,
   ensureDefaultBase,
   getDocument,
   getDocumentBase,
@@ -1567,6 +1714,7 @@ export {
   listDocumentBases,
   listDocumentBasesEnsuringDefault,
   listDocuments,
+  listEffectiveIndexedDocuments,
   moveDocumentToBase,
   parseCurationOutcome,
   parseDocumentBytes,