@opengeni/documents 0.6.9-canary.0 → 0.8.2-canary.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/documents",
-  "version": "0.6.9-canary.0",
+  "version": "0.8.2-canary.0",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -40,10 +40,10 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "^1.5.3",
-    "@opengeni/config": "^0.19.0-canary.0",
-    "@opengeni/contracts": "^2.2.0-canary.0",
-    "@opengeni/db": "^3.1.0-canary.0",
-    "@opengeni/storage": "^0.2.105-canary.0",
+    "@opengeni/config": "^0.20.1-canary.0",
+    "@opengeni/contracts": "^2.5.0-canary.0",
+    "@opengeni/db": "^3.4.0-canary.0",
+    "@opengeni/storage": "^0.2.108-canary.0",
     "drizzle-orm": "^0.45.2",
     "openai": "6.47.0"
   }

package/src/index.ts

@@ -26,6 +26,13 @@ import {
   type ReclassifyDocumentAuthorityRequest,
   type RunDocumentDefaultCollectionBackfillRequest,
   DocumentDefaultCollectionBackfill,
+  DocumentDefaultCollectionBackfillAudit,
+  DocumentDefaultCollectionBackfillOperationAudit,
+  DocumentDefaultCollectionBackfillReceiptAudit,
+  DocumentDefaultCollectionBackfillRunAudit,
+  ListDocumentDefaultCollectionBackfillRunsResponse,
+  ListOrganizationDocumentAuthorityReclassificationsResponse,
+  OrganizationDocumentAuthorityReclassification,
 } from "@opengeni/contracts";
 import {
   createPersonalDocumentAuthority,
@@ -197,6 +204,15 @@ export type RunDocumentDefaultCollectionBackfillInput =
     accountAdminAuthorization: DocumentAccountAdminAuthorization;
   };
 
+export type DocumentMigrationAuditInput = {
+  accountId: string;
+  workspaceId: string;
+  actorSubjectId: string;
+  accountAdminAuthorization: DocumentAccountAdminAuthorization;
+  limit?: number | undefined;
+  cursor?: string | undefined;
+};
+
 export type AgentDocumentAuthorityContext = {
   sessionId: string;
   attemptId: string;
@@ -243,6 +259,7 @@ export type DocumentSearchInput = {
   limit?: number | undefined;
   mode?: DocumentSearchMode | undefined;
   sourceKinds?: KnowledgeSourceKind[] | undefined;
+  authorityKinds?: DocumentAuthorityKind[] | undefined;
   aclTags?: string[] | undefined;
   access?: DocumentAccessFilter | undefined;
 };
@@ -1023,6 +1040,264 @@ export async function runDocumentDefaultCollectionBackfill(
   );
 }
 
+export async function listDocumentDefaultCollectionBackfillRuns(
+  db: Database,
+  input: DocumentMigrationAuditInput,
+) {
+  const limit = validateDocumentMigrationAuditLimit(input.limit);
+  const cursor = input.cursor
+    ? decodeDocumentMigrationAuditCursor(input.cursor, ["backfill-runs", input.accountId], true)
+    : null;
+  return await withDocumentAccountRls(
+    db,
+    input.accountId,
+    input.workspaceId,
+    { viewerSubjectId: input.actorSubjectId },
+    async (scopedDb) => {
+      const command = {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        actorSubjectId: input.actorSubjectId,
+        accountAdminAuthorization: input.accountAdminAuthorization,
+        limit: limit + 1,
+        beforeStartedAt: cursor?.timestamp ?? null,
+        beforeRunId: cursor?.id ?? null,
+      };
+      const rows = (await scopedDb.execute(sql`
+        SELECT list_document_default_collection_backfill_runs(
+          ${JSON.stringify(command)}::jsonb
+        ) AS result
+      `)) as unknown as Array<{ result: unknown }>;
+      const parsed = rows.map((row) => DocumentDefaultCollectionBackfillRunAudit.parse(row.result));
+      const hasMore = parsed.length > limit;
+      const runs = parsed.slice(0, limit);
+      const tail = hasMore ? runs.at(-1) : null;
+      return ListDocumentDefaultCollectionBackfillRunsResponse.parse({
+        runs,
+        hasMore,
+        nextCursor: tail
+          ? encodeDocumentMigrationAuditCursor(["backfill-runs", input.accountId], {
+              timestamp: tail.startedAt,
+              id: tail.runId,
+            })
+          : null,
+      });
+    },
+  );
+}
+
+export async function getDocumentDefaultCollectionBackfillAudit(
+  db: Database,
+  input: DocumentMigrationAuditInput & {
+    runId: string;
+    operationCursor?: string | undefined;
+    receiptCursor?: string | undefined;
+  },
+) {
+  const limit = validateDocumentMigrationAuditLimit(input.limit);
+  const operationCursor = input.operationCursor
+    ? decodeDocumentMigrationAuditCursor(
+        input.operationCursor,
+        ["backfill-operations", input.accountId, input.runId],
+        true,
+      )
+    : null;
+  const receiptCursor = input.receiptCursor
+    ? decodeDocumentMigrationAuditCursor(
+        input.receiptCursor,
+        ["backfill-receipts", input.accountId, input.runId],
+        false,
+      )
+    : null;
+  return await withDocumentAccountRls(
+    db,
+    input.accountId,
+    input.workspaceId,
+    { viewerSubjectId: input.actorSubjectId },
+    async (scopedDb) => {
+      const command = {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        actorSubjectId: input.actorSubjectId,
+        accountAdminAuthorization: input.accountAdminAuthorization,
+        runId: input.runId,
+        limit,
+        operationBeforeCreatedAt: operationCursor?.timestamp ?? null,
+        operationBeforeId: operationCursor?.id ?? null,
+        receiptAfterWorkspaceId: receiptCursor?.id ?? null,
+      };
+      const rows = (await scopedDb.execute(sql`
+        SELECT get_document_default_collection_backfill_audit(
+          ${JSON.stringify(command)}::jsonb
+        ) AS result
+      `)) as unknown as Array<{ result: unknown }>;
+      const raw = rows[0]?.result;
+      if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+        throw new Error("document Default collection backfill audit returned no result");
+      }
+      const value = raw as Record<string, unknown>;
+      const run = DocumentDefaultCollectionBackfillRunAudit.parse(value.run);
+      const operations = Array.isArray(value.operations)
+        ? value.operations.map((operation) =>
+            DocumentDefaultCollectionBackfillOperationAudit.parse(operation),
+          )
+        : [];
+      const receipts = Array.isArray(value.receipts)
+        ? value.receipts.map((receipt) =>
+            DocumentDefaultCollectionBackfillReceiptAudit.parse(receipt),
+          )
+        : [];
+      const operationsHasMore = operations.length > limit;
+      const receiptsHasMore = receipts.length > limit;
+      const operationPage = operations.slice(0, limit);
+      const receiptPage = receipts.slice(0, limit);
+      const operationTail = operationsHasMore ? operationPage.at(-1) : null;
+      const receiptTail = receiptsHasMore ? receiptPage.at(-1) : null;
+      return DocumentDefaultCollectionBackfillAudit.parse({
+        run,
+        operations: operationPage,
+        receipts: receiptPage,
+        operationsHasMore,
+        operationsNextCursor: operationTail
+          ? encodeDocumentMigrationAuditCursor(
+              ["backfill-operations", input.accountId, input.runId],
+              {
+                timestamp: operationTail.createdAt,
+                id: operationTail.operationId,
+              },
+            )
+          : null,
+        receiptsHasMore,
+        receiptsNextCursor: receiptTail
+          ? encodeDocumentMigrationAuditCursor(
+              ["backfill-receipts", input.accountId, input.runId],
+              { timestamp: null, id: receiptTail.workspaceId },
+            )
+          : null,
+      });
+    },
+  );
+}
+
+export async function listOrganizationDocumentAuthorityReclassifications(
+  db: Database,
+  input: DocumentMigrationAuditInput,
+) {
+  const limit = validateDocumentMigrationAuditLimit(input.limit);
+  const cursor = input.cursor
+    ? decodeDocumentMigrationAuditCursor(
+        input.cursor,
+        ["organization-reclassifications", input.accountId],
+        true,
+      )
+    : null;
+  return await withDocumentAccountRls(
+    db,
+    input.accountId,
+    input.workspaceId,
+    { viewerSubjectId: input.actorSubjectId },
+    async (scopedDb) => {
+      const command = {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        actorSubjectId: input.actorSubjectId,
+        accountAdminAuthorization: input.accountAdminAuthorization,
+        limit: limit + 1,
+        beforeCreatedAt: cursor?.timestamp ?? null,
+        beforeOperationId: cursor?.id ?? null,
+      };
+      const rows = (await scopedDb.execute(sql`
+        SELECT list_organization_document_authority_reclassifications(
+          ${JSON.stringify(command)}::jsonb
+        ) AS result
+      `)) as unknown as Array<{ result: unknown }>;
+      const parsed = rows.map((row) =>
+        OrganizationDocumentAuthorityReclassification.parse(row.result),
+      );
+      const hasMore = parsed.length > limit;
+      const receipts = parsed.slice(0, limit);
+      const tail = hasMore ? receipts.at(-1) : null;
+      return ListOrganizationDocumentAuthorityReclassificationsResponse.parse({
+        receipts,
+        hasMore,
+        nextCursor: tail
+          ? encodeDocumentMigrationAuditCursor(
+              ["organization-reclassifications", input.accountId],
+              { timestamp: tail.createdAt, id: tail.operationId },
+            )
+          : null,
+      });
+    },
+  );
+}
+
+function validateDocumentMigrationAuditLimit(value: number | undefined): number {
+  const limit = value ?? 50;
+  if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
+    throw new Error("document migration audit limit must be between 1 and 100");
+  }
+  return limit;
+}
+
+type DocumentMigrationAuditCursor = { timestamp: string | null; id: string };
+
+function documentMigrationAuditCursorScope(parts: string[]): string {
+  return createHash("sha256")
+    .update(JSON.stringify(["document-migration-audit", 1, ...parts]), "utf8")
+    .digest("hex")
+    .slice(0, 32);
+}
+
+function encodeDocumentMigrationAuditCursor(
+  parts: string[],
+  cursor: DocumentMigrationAuditCursor,
+): string {
+  return Buffer.from(
+    JSON.stringify({
+      v: 1,
+      s: documentMigrationAuditCursorScope(parts),
+      t: cursor.timestamp,
+      i: cursor.id,
+    }),
+    "utf8",
+  ).toString("base64url");
+}
+
+function decodeDocumentMigrationAuditCursor(
+  value: string,
+  parts: string[],
+  requireTimestamp: boolean,
+): DocumentMigrationAuditCursor {
+  try {
+    if (!value || value.length > 1_024) throw new Error("cursor length");
+    const bytes = Buffer.from(value, "base64url");
+    if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
+    const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
+    if (
+      Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
+      parsed.v !== 1 ||
+      parsed.s !== documentMigrationAuditCursorScope(parts) ||
+      typeof parsed.i !== "string" ||
+      !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(
+        parsed.i,
+      ) ||
+      (requireTimestamp
+        ? typeof parsed.t !== "string" || !Number.isFinite(new Date(parsed.t).getTime())
+        : parsed.t !== null)
+    ) {
+      throw new Error("cursor payload");
+    }
+    return {
+      timestamp: typeof parsed.t === "string" ? parsed.t : null,
+      id: parsed.i,
+    };
+  } catch (error) {
+    throw new Error("invalid document migration audit cursor", {
+      cause: error,
+    });
+  }
+}
+
 export async function reclassifyDocumentAuthority(
   db: Database,
   input: ReclassifyDocumentAuthorityInput,
@@ -1170,7 +1445,9 @@ function decodeDocumentAuthorityReclassificationCursor(
     }
     return { createdAt: parsed.t, operationId: parsed.i };
   } catch (error) {
-    throw new Error("invalid document authority receipt cursor", { cause: error });
+    throw new Error("invalid document authority receipt cursor", {
+      cause: error,
+    });
   }
 }
 
@@ -1622,7 +1899,11 @@ export function encodeDocumentIndexCheckpoint(input: {
 
 export function decodeDocumentIndexCheckpoint(
   value: string,
-  scope: { accountId: string; workspaceId: string; initiatingSubjectId: string },
+  scope: {
+    accountId: string;
+    workspaceId: string;
+    initiatingSubjectId: string;
+  },
 ): bigint {
   try {
     if (!value || value.length > DOCUMENT_INDEX_CHECKPOINT_MAX_CHARS) {
@@ -1996,7 +2277,11 @@ async function curateDroppedDocument(
   const bases = await listDocumentBases(db, document.workspaceId);
   const candidates: DocumentCurationCandidateBase[] = bases
     .filter((base) => base.id !== document.baseId)
-    .map((base) => ({ id: base.id, name: base.name, description: base.description }));
+    .map((base) => ({
+      id: base.id,
+      name: base.name,
+      description: base.description,
+    }));
   const input: DocumentCurationInput = {
     text: parsed.text.slice(0, DOCUMENT_CURATION_MAX_INPUT_CHARS),
     filename: file.filename,
@@ -2188,6 +2473,7 @@ export async function searchEffectiveDocuments(
       limit: input.limit,
       mode: input.mode,
       sourceKinds: input.sourceKinds,
+      authorityKinds: input.authorityKinds,
       aclTags: input.aclTags,
       // Construct the lower-level access filter here instead of spreading the
       // caller input, so an untyped/legacy access override is always ignored.
@@ -2230,6 +2516,7 @@ export async function searchEffectiveKnowledge(
       ...(input.baseIds ? { baseIds: input.baseIds } : {}),
       ...(input.mode ? { mode: input.mode } : {}),
       ...(input.sourceKinds ? { sourceKinds: input.sourceKinds } : {}),
+      ...(input.authorityKinds ? { authorityKinds: input.authorityKinds } : {}),
       ...(input.aclTags ? { aclTags: input.aclTags } : {}),
       access,
     },
@@ -2273,6 +2560,9 @@ export async function searchEffectiveKnowledge(
               ranked.map((result) => result.chunkId),
             ),
             eq(schema.documents.status, "ready"),
+            ...(input.authorityKinds?.length
+              ? [inArray(schema.documents.authorityKind, input.authorityKinds)]
+              : []),
             ...documentAccessConditions(input.workspaceId, access),
           ),
         ),
@@ -2808,7 +3098,10 @@ export async function browseEffectiveKnowledge(
     async (scopedDb) => {
       if (parent) {
         const [authorizedParent] = await scopedDb
-          .select({ id: schema.documents.id, indexSequence: schema.documents.indexSequence })
+          .select({
+            id: schema.documents.id,
+            indexSequence: schema.documents.indexSequence,
+          })
           .from(schema.documents)
           .where(
             and(
@@ -2820,7 +3113,10 @@ export async function browseEffectiveKnowledge(
           )
           .limit(1);
         if (!authorizedParent) {
-          return selectKnowledgeBrowseRecords({ entries: [], hasMoreAfterEntries: false });
+          return selectKnowledgeBrowseRecords({
+            entries: [],
+            hasMoreAfterEntries: false,
+          });
         }
         if (authorizedParent.indexSequence === null) {
           throw new Error("ready knowledge document is missing its index revision");
@@ -2937,7 +3233,11 @@ export function encodeKnowledgeBrowseCursor(
   if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
   const version = knowledgeBrowseCursorVersion(scope);
   return Buffer.from(
-    JSON.stringify({ v: version, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
+    JSON.stringify({
+      v: version,
+      s: knowledgeBrowseCursorScope(scope),
+      q: position.toString(),
+    }),
     "utf8",
   ).toString("base64url");
 }
@@ -3017,7 +3317,10 @@ function parseKnowledgeRecordId(value: string): {
       value,
     );
   if (!match) throw new Error("invalid knowledge record id");
-  return { kind: match[1] as "document" | "document_chunk", id: match[2]!.toLowerCase() };
+  return {
+    kind: match[1] as "document" | "document_chunk",
+    id: match[2]!.toLowerCase(),
+  };
 }
 
 function knowledgeDocumentRecord(
@@ -3621,6 +3924,9 @@ function documentSearchConditions(input: DocumentSearchInput, embeddingModel?: s
   if (input.sourceKinds?.length) {
     conditions.push(inArray(schema.documents.sourceKind, input.sourceKinds));
   }
+  if (input.authorityKinds?.length) {
+    conditions.push(inArray(schema.documents.authorityKind, input.authorityKinds));
+  }
   const aclTags = cleanStringArray(input.aclTags);
   if (aclTags.length > 0) {
     conditions.push(sql`${schema.documents.aclTags} @> ${JSON.stringify(aclTags)}::jsonb`);