@opengeni/documents 0.5.18 → 0.5.32

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/documents",
-  "version": "0.5.18",
+  "version": "0.5.32",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -23,6 +23,10 @@
     "./google-drive": {
       "types": "./dist/google-drive.d.ts",
       "import": "./dist/google-drive.js"
+    },
+    "./atlassian": {
+      "types": "./dist/atlassian.d.ts",
+      "import": "./dist/atlassian.js"
     }
   },
   "publishConfig": {
@@ -36,11 +40,11 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "^1.5.3",
-    "@opengeni/config": "^0.12.1",
-    "@opengeni/contracts": "^0.40.0",
-    "@opengeni/db": "^0.28.9",
-    "@opengeni/storage": "^0.2.75",
+    "@opengeni/config": "^0.13.2",
+    "@opengeni/contracts": "^0.44.1",
+    "@opengeni/db": "^0.31.1",
+    "@opengeni/storage": "^0.2.87",
     "drizzle-orm": "^0.45.2",
-    "openai": "6.36.0"
+    "openai": "6.47.0"
   }
 }

package/src/atlassian.ts

@@ -0,0 +1,250 @@
+import type { ScopedKnowledgeScope } from "@opengeni/contracts";
+import {
+  connectorDestinationDocumentAuthority,
+  resolveConnectorDocumentDestination,
+  type ConnectorDocumentDestination,
+} from "@opengeni/contracts/connector-destinations";
+import type { AtlassianSelectedSource } from "@opengeni/contracts/atlassian";
+
+export const ATLASSIAN_PROVIDER_KEY = "atlassian" as const;
+
+export type AtlassianKnowledgeSourceIdentity = {
+  providerKey: typeof ATLASSIAN_PROVIDER_KEY;
+  externalTenantId: string;
+  externalSourceId: string;
+  sourceKind: "atlassian-jira-project" | "atlassian-confluence-space";
+  sourceUri: string;
+  scope: ScopedKnowledgeScope;
+};
+
+export type AtlassianInventoryProviderItem = {
+  id: string;
+  key: string;
+  title: string;
+  version: string | null;
+  createdAt: string | null;
+  updatedAt: string | null;
+  webUrl: string;
+};
+
+export type AtlassianInventoryPage = {
+  items: AtlassianInventoryProviderItem[];
+  nextCursor: string | null;
+};
+
+export type AtlassianInventoryEntry = {
+  externalObjectId: string;
+  externalVersionId: string | null;
+  sourceId: string;
+  parentFolderId: string;
+  driveId: null;
+  title: string;
+  mimeType: "text/markdown";
+  modifiedTime: string | null;
+  createdTime: string | null;
+  sourceUri: string;
+  transfer: {
+    action: "download";
+    contentType: "text/markdown";
+    filename: string;
+    declaredBytes: null;
+  };
+};
+
+export type AtlassianInventoryStopReason =
+  | "api_request_limit"
+  | "elapsed_time_limit"
+  | "item_limit"
+  | "provider_error";
+
+export type AtlassianInventoryCheckpoint = {
+  version: 1;
+  cloudId: string;
+  sourceId: string;
+  cursor: string | null;
+  itemCount: number;
+  apiRequestCount: number;
+};
+
+export function atlassianKnowledgeScope(
+  destination: ConnectorDocumentDestination,
+): ScopedKnowledgeScope {
+  const authority = connectorDestinationDocumentAuthority(destination);
+  if (authority.authorityKind === "organization") {
+    return { kind: "organization", workspaceId: null, subjectId: null };
+  }
+  if (authority.authorityKind === "workspace") {
+    if (!authority.authorityWorkspaceId) throw new Error("workspace authority is missing");
+    return { kind: "workspace", workspaceId: authority.authorityWorkspaceId, subjectId: null };
+  }
+  if (!authority.authorityWorkspaceId || !authority.authoritySubjectId) {
+    throw new Error("personal authority is missing");
+  }
+  return {
+    kind: "personal",
+    workspaceId: authority.authorityWorkspaceId,
+    subjectId: authority.authoritySubjectId,
+  };
+}
+
+export function atlassianKnowledgeSourceIdentity(input: {
+  source: AtlassianSelectedSource;
+  accountId: string;
+  workspaceId: string;
+  connectionSubjectId: string;
+}): AtlassianKnowledgeSourceIdentity {
+  const destination = resolveConnectorDocumentDestination(input.source.destination, {
+    accountId: input.accountId,
+    workspaceId: input.workspaceId,
+    connectionSubjectId: input.connectionSubjectId,
+  });
+  const siteUrl = new URL(input.source.siteUrl);
+  const sourceUri =
+    input.source.kind === "jira_project"
+      ? new URL(`/jira/software/c/projects/${encodeURIComponent(input.source.key)}`, siteUrl)
+      : new URL(`/wiki/spaces/${encodeURIComponent(input.source.key)}`, siteUrl);
+  return {
+    providerKey: ATLASSIAN_PROVIDER_KEY,
+    externalTenantId: bounded(input.source.cloudId, 256, "cloudId"),
+    externalSourceId: bounded(input.source.id, 300, "source.id"),
+    sourceKind:
+      input.source.kind === "jira_project"
+        ? "atlassian-jira-project"
+        : "atlassian-confluence-space",
+    sourceUri: sourceUri.toString(),
+    scope: atlassianKnowledgeScope(destination),
+  };
+}
+
+export async function inventoryAtlassianSource(input: {
+  cloudId: string;
+  source: AtlassianSelectedSource;
+  limits: { maxItems: number; maxApiRequests: number; maxElapsedMs: number; pageSize: number };
+  checkpoint: Record<string, unknown> | null;
+  listPage: (cursor: string | null, pageSize: number) => Promise<AtlassianInventoryPage>;
+}): Promise<{
+  status: "complete" | "paused";
+  stopReason: AtlassianInventoryStopReason | null;
+  entries: AtlassianInventoryEntry[];
+  checkpoint: AtlassianInventoryCheckpoint | null;
+  providerRequests: number;
+  elapsedMs: number;
+}> {
+  const startedAt = Date.now();
+  const restored = parseCheckpoint(input.checkpoint, input.cloudId, input.source.id);
+  let cursor = restored?.cursor ?? null;
+  let itemCount = restored?.itemCount ?? 0;
+  let apiRequestCount = restored?.apiRequestCount ?? 0;
+  const entries: AtlassianInventoryEntry[] = [];
+
+  while (true) {
+    if (Date.now() - startedAt >= input.limits.maxElapsedMs) return paused("elapsed_time_limit");
+    if (apiRequestCount >= input.limits.maxApiRequests) return paused("api_request_limit");
+    if (itemCount >= input.limits.maxItems) return paused("item_limit");
+
+    let page: AtlassianInventoryPage;
+    try {
+      page = await input.listPage(
+        cursor,
+        Math.min(input.limits.pageSize, input.limits.maxItems - itemCount),
+      );
+    } catch {
+      return paused("provider_error");
+    }
+    apiRequestCount += 1;
+    for (const item of page.items) {
+      if (itemCount >= input.limits.maxItems) return paused("item_limit");
+      entries.push(atlassianEntry(input.source, item));
+      itemCount += 1;
+    }
+    cursor = page.nextCursor;
+    if (!cursor) {
+      return {
+        status: "complete",
+        stopReason: null,
+        entries,
+        checkpoint: null,
+        providerRequests: apiRequestCount - (restored?.apiRequestCount ?? 0),
+        elapsedMs: Date.now() - startedAt,
+      };
+    }
+  }
+
+  function paused(stopReason: AtlassianInventoryStopReason) {
+    return {
+      status: "paused" as const,
+      stopReason,
+      entries,
+      checkpoint: {
+        version: 1 as const,
+        cloudId: input.cloudId,
+        sourceId: input.source.id,
+        cursor,
+        itemCount,
+        apiRequestCount,
+      },
+      providerRequests: apiRequestCount - (restored?.apiRequestCount ?? 0),
+      elapsedMs: Date.now() - startedAt,
+    };
+  }
+}
+
+function atlassianEntry(
+  source: AtlassianSelectedSource,
+  item: AtlassianInventoryProviderItem,
+): AtlassianInventoryEntry {
+  const title = bounded(item.title, 1024, "title");
+  return {
+    externalObjectId: bounded(`${source.kind}:${item.id}`, 512, "item.id"),
+    externalVersionId: item.version,
+    sourceId: source.id,
+    parentFolderId: source.resourceId,
+    driveId: null,
+    title,
+    mimeType: "text/markdown",
+    modifiedTime: item.updatedAt,
+    createdTime: item.createdAt,
+    sourceUri: new URL(item.webUrl).toString(),
+    transfer: {
+      action: "download",
+      contentType: "text/markdown",
+      filename: `${safeFilename(item.key || title)}.md`,
+      declaredBytes: null,
+    },
+  };
+}
+
+function parseCheckpoint(
+  value: Record<string, unknown> | null,
+  cloudId: string,
+  sourceId: string,
+): AtlassianInventoryCheckpoint | null {
+  if (!value) return null;
+  if (
+    value.version !== 1 ||
+    value.cloudId !== cloudId ||
+    value.sourceId !== sourceId ||
+    (value.cursor !== null && typeof value.cursor !== "string") ||
+    !Number.isSafeInteger(value.itemCount) ||
+    !Number.isSafeInteger(value.apiRequestCount)
+  ) {
+    throw new Error("invalid Atlassian inventory checkpoint");
+  }
+  return value as AtlassianInventoryCheckpoint;
+}
+
+function bounded(value: string, max: number, label: string): string {
+  const normalized = value.trim();
+  if (!normalized || normalized.length > max) throw new Error(`invalid ${label}`);
+  return normalized;
+}
+
+function safeFilename(value: string): string {
+  return (
+    value
+      .normalize("NFKC")
+      .replace(/[\\/:*?"<>|\0-\x1f]/g, "_")
+      .trim()
+      .slice(0, 220) || "item"
+  );
+}

package/src/google-drive.ts

@@ -262,6 +262,7 @@ export function googleDriveKnowledgeScope(
 
 export function googleDriveKnowledgeSourceIdentity(input: {
   googlePermissionId: string;
+  googleEmail?: string | undefined;
   source: Pick<GoogleDriveSelectedSource, "id" | "driveId" | "destination" | "targetScope">;
   accountId?: string | undefined;
   workspaceId: string;
@@ -270,7 +271,11 @@ export function googleDriveKnowledgeSourceIdentity(input: {
   initiatingSubjectId?: string | undefined;
 }): GoogleDriveKnowledgeSourceIdentity {
   const sourceId = driveId(input.source.id, "source.id");
-  const externalTenantId = normalizedGooglePermissionId(input.googlePermissionId);
+  // permissionId identifies a principal, not a Google tenant. Consumer
+  // accounts intentionally share one provider tenant partition; Workspace
+  // accounts use their normalized hosted domain. Per-principal authority stays
+  // on the immutable connection owner binding.
+  const externalTenantId = googleTenantIdentity(input.googleEmail);
   const sourceDriveId = nullableDriveId(input.source.driveId, "source.driveId");
   const destination = resolveConnectorDocumentDestination(input.source.destination, {
     accountId: input.accountId ?? input.workspaceId,
@@ -295,6 +300,19 @@ export function googleDriveKnowledgeSourceIdentity(input: {
   };
 }
 
+function googleTenantIdentity(email: string | undefined): string {
+  if (email === undefined) return "google-consumer";
+  const normalized = email.trim().toLowerCase();
+  const separator = normalized.lastIndexOf("@");
+  if (separator <= 0 || separator === normalized.length - 1) {
+    throw new Error("googleEmail must be a normalized email address");
+  }
+  const domain = normalized.slice(separator + 1);
+  return domain === "gmail.com" || domain === "googlemail.com"
+    ? "google-consumer"
+    : `google-workspace:${domain}`;
+}
+
 export function planGoogleDriveTransfer(
   item: GoogleDriveInventoryProviderItem,
   maxFileBytes: number,
@@ -340,6 +358,7 @@ export function planGoogleDriveTransfer(
 
 export async function inventoryGoogleDriveSource(input: {
   googlePermissionId: string;
+  googleEmail?: string | undefined;
   source: GoogleDriveSelectedSource;
   accountId?: string | undefined;
   workspaceId: string;
@@ -355,6 +374,7 @@ export async function inventoryGoogleDriveSource(input: {
   const googlePermissionId = normalizedGooglePermissionId(input.googlePermissionId);
   const source = googleDriveKnowledgeSourceIdentity({
     googlePermissionId,
+    googleEmail: input.googleEmail,
     source: input.source,
     ...(input.accountId ? { accountId: input.accountId } : {}),
     workspaceId: input.workspaceId,