@xopcai/xopc 0.0.210 → 0.0.213

package/dist/src/connectors/connected-source-ingestion.js

@@ -1,6 +1,9 @@
+import { createLogger } from "../utils/logger/index.js";
+import { init_logger } from "../utils/logger.js";
 import { getConnectorInstallation, listConnectorConnections } from "../storage/sqlite/connector-repository.js";
 import "../storage/sqlite/index.js";
 import { ComposioSessionsAdapter } from "./composio-sessions.js";
+import { decodeConnectedSourceCursor, encodeConnectedSourceCursor } from "./connected-source-cursor.js";
 import { scopeForComposioAction } from "./composio.js";
 import { getConnectorDefinition } from "./catalog.js";
 import { getConnectorInstance } from "./instances.js";
@@ -10,42 +13,21 @@ import { ConnectedKnowledgePipeline } from "../knowledge/connected-knowledge-pip
 import { LocalFolderKnowledgeSourceAdapter } from "../knowledge/local-folder-adapter.js";
 import "../knowledge/index.js";
 import { normalizeConnectedSourceResult } from "./connected-source-normalizers.js";
+import { sanitizeConnectedSourceValue } from "./connected-source-sanitization.js";
 import { createHash } from "node:crypto";
 //#region src/connectors/connected-source-ingestion.ts
+init_logger();
+const log = createLogger("ConnectedSourceIngestion");
 const COMPOSIO_CONNECTED_SOURCE_TOOLKITS = new Set([
 	"gmail",
 	"googlecalendar",
 	"googledrive",
-	"googledocs",
-	"googlesheets",
-	"notion",
-	"slack",
 	"github",
-	"linear",
-	"jira",
-	"outlook",
-	"microsoft_teams",
-	"one_drive",
-	"excel"
+	"linear"
 ]);
-const SENSITIVE_KEY_RE = /(?:^|_)(?:api_?key|token|secret|password|passwd|authorization|credential|private_?key)(?:$|_)/i;
-const INLINE_SECRET_PATTERNS = [
-	/\bsk-[A-Za-z0-9_-]{8,}\b/g,
-	/\bghp_[A-Za-z0-9]{20,}\b/g,
-	/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g,
-	/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/g
-];
 function asRecord(value) {
 	return value && typeof value === "object" && !Array.isArray(value) ? value : null;
 }
-function sanitizeSourceValue(value, depth = 0) {
-	if (depth > 12) return "[MaxDepth]";
-	if (typeof value === "string") return INLINE_SECRET_PATTERNS.reduce((text, pattern) => text.replace(pattern, "[REDACTED]"), value);
-	if (Array.isArray(value)) return value.map((item) => sanitizeSourceValue(item, depth + 1));
-	const record = asRecord(value);
-	if (!record) return value;
-	return Object.fromEntries(Object.entries(record).map(([key, nested]) => [key, SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeSourceValue(nested, depth + 1)]));
-}
 function boundedPersonValue(value, maxLength = 320) {
 	return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
 }
@@ -66,7 +48,7 @@ function personEntities(record) {
 		const nested = asRecord(value);
 		if (!nested) return [];
 		const name = boundedPersonValue(nested.name, 160) || boundedPersonValue(nested.displayName, 160);
-		const email = boundedPersonValue(nested.email);
+		const email = boundedPersonValue(nested.email) || boundedPersonValue(nested.emailAddress);
 		const username = boundedPersonValue(nested.username, 160);
 		return name || email || username ? [{
 			role,
@@ -83,6 +65,7 @@ function personEntities(record) {
 		"user",
 		"username",
 		"owner",
+		"owners",
 		"assignee",
 		"attendees",
 		"participants"
@@ -108,14 +91,37 @@ function peopleSignals(entities) {
 function connectionIdentities(identity) {
 	return [...new Set(Object.values(identity).filter((value) => typeof value === "string").map((value) => value.trim().toLowerCase()).filter(Boolean))];
 }
+function resultPayload(value) {
+	const root = asRecord(value) ?? {};
+	return asRecord(root.data) ?? root;
+}
+function paginationValue(result, ...keys) {
+	const payload = resultPayload(result);
+	const pagination = asRecord(payload.pagination) ?? asRecord(payload.pageInfo) ?? {};
+	for (const key of keys) {
+		const value = payload[key] ?? pagination[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+}
+function isPayloadTooLargeError(error) {
+	const details = asRecord(error);
+	const status = details?.status ?? details?.statusCode;
+	const code = details?.code;
+	const message = error instanceof Error ? error.message : String(error);
+	return String(status) === "413" || String(code) === "4345" || /\b413\b|ToolRouterV2_PayloadTooLarge|payload is too large/i.test(message);
+}
+function maxResultsValue(args) {
+	const value = args.max_results;
+	return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : void 0;
+}
 function resultToSourceItems(input) {
 	return normalizeConnectedSourceResult(input).slice(0, 200).map((entity) => {
 		const record = entity.value;
 		const people = personEntities(record);
 		const ownerIdentities = connectionIdentities(input.connectionIdentity);
 		const observedIdentities = peopleSignals(people).map((value) => value.toLowerCase());
-		const actorAttributed = entity.metadata.actorAttributed === true || input.toolkit === "gmail" || input.toolkit === "googlecalendar" || input.toolkit === "github" && observedIdentities.some((value) => ownerIdentities.includes(value));
-		const serialized = JSON.stringify(sanitizeSourceValue(entity.value), null, 2);
+		const actorAttributed = entity.metadata.actorAttributed === true || input.toolkit === "gmail" || input.toolkit === "googlecalendar" || ["github", "googledrive"].includes(input.toolkit) && observedIdentities.some((value) => ownerIdentities.includes(value));
+		const serialized = JSON.stringify(sanitizeConnectedSourceValue(entity.value), null, 2);
 		const normalizedText = serialized.length > 8e3 ? `${serialized.slice(0, 8e3)}\n…` : serialized;
 		const contentHash = createHash("sha256").update(serialized).digest("hex");
 		return {
@@ -144,56 +150,113 @@ function resultToSourceItems(input) {
 			sensitivity: "personal",
 			retentionClass: "bounded",
 			synthesisPipeline: "connected_knowledge",
-			synthesisStatus: input.streamKind === "inventory" ? "ignored" : entity.synthesisStatus
+			synthesisStatus: input.streamKind === "inventory" ? "ignored" : entity.synthesisStatus,
+			deletedAt: entity.deletedAt
 		};
 	});
 }
 var ComposioKnowledgeSourceAdapter = class {
 	kind = "composio";
+	adaptiveMaxResults;
 	constructor(input) {
 		this.input = input;
 	}
 	async pull(pull) {
-		const execution = await this.input.adapter.executeWithPolicy({
-			context: {
-				principalId: "local-owner",
-				toolkits: [this.input.toolkit]
-			},
-			installation: {
-				...this.input.installation,
-				maxScope: "read",
-				confirmationPolicy: "never"
-			},
-			connection: this.input.connection,
+		const cursor = decodeConnectedSourceCursor(pull.cursor);
+		const scanStartedAt = cursor?.scanStartedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+		const warnings = pull.cursor && !cursor ? ["Stored connected-source cursor was invalid and was replaced."] : [];
+		const execute = async (cursorValue) => {
+			const configuredArgs = this.input.buildArguments?.({
+				...pull,
+				cursor: cursorValue
+			}) ?? this.input.arguments;
+			const configuredMaxResults = maxResultsValue(configuredArgs);
+			let args = this.adaptiveMaxResults && configuredMaxResults && configuredMaxResults > this.adaptiveMaxResults ? {
+				...configuredArgs,
+				max_results: this.adaptiveMaxResults
+			} : configuredArgs;
+			for (;;) try {
+				return await this.input.adapter.executeWithPolicy({
+					context: {
+						principalId: "local-owner",
+						toolkits: [this.input.toolkit]
+					},
+					installation: {
+						...this.input.installation,
+						maxScope: "read",
+						confirmationPolicy: "never"
+					},
+					connection: this.input.connection,
+					agentId: this.input.agentId,
+					action: {
+						connectorId: this.input.connectorId,
+						actionId: this.input.actionId,
+						toolkit: this.input.toolkit,
+						scope: "read",
+						curated: true,
+						cachedAt: (/* @__PURE__ */ new Date()).toISOString()
+					},
+					args,
+					confirmed: true
+				});
+			} catch (error) {
+				const currentMaxResults = maxResultsValue(args);
+				if (!isPayloadTooLargeError(error) || !currentMaxResults || currentMaxResults <= 1) throw error;
+				const reducedMaxResults = Math.max(1, Math.floor(currentMaxResults / 2));
+				this.adaptiveMaxResults = reducedMaxResults;
+				log.info({
+					connectorId: this.input.connectorId,
+					actionId: this.input.actionId,
+					previousMaxResults: currentMaxResults,
+					maxResults: reducedMaxResults,
+					phase: "payload_too_large_retry"
+				}, `Retrying connector action with page size ${reducedMaxResults}`);
+				args = {
+					...args,
+					max_results: reducedMaxResults
+				};
+			}
+		};
+		let execution;
+		try {
+			execution = await execute(pull.cursor);
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			if (this.input.toolkit !== "googlecalendar" || !cursor?.syncToken || !/\b410\b|sync.?token/i.test(message)) throw error;
+			warnings.push("Expired Google Calendar sync token was replaced with a full sync.");
+			execution = await execute(void 0);
+		}
+		if (execution.decision !== "allowed") throw new Error(execution.reason);
+		const items = resultToSourceItems({
+			result: execution.result,
+			sourceInstanceId: pull.instanceId,
+			collectionScope: pull.collectionScope,
+			streamKind: this.input.streamKind,
+			connectorId: this.input.connectorId,
+			connectionId: this.input.connection.id,
+			actionId: this.input.actionId,
+			toolkit: this.input.toolkit,
 			agentId: this.input.agentId,
-			action: {
-				connectorId: this.input.connectorId,
-				actionId: this.input.actionId,
-				toolkit: this.input.toolkit,
-				scope: "read",
-				curated: true,
-				cachedAt: (/* @__PURE__ */ new Date()).toISOString()
-			},
-			args: this.input.buildArguments?.(pull) ?? this.input.arguments,
-			confirmed: true
+			workspaceId: this.input.workspaceId,
+			connectionIdentity: this.input.connection.identity
 		});
-		if (execution.decision !== "allowed") throw new Error(execution.reason);
+		const nextPageToken = paginationValue(execution.result, "nextPageToken", "next_page_token");
+		const nextSyncToken = paginationValue(execution.result, "nextSyncToken", "next_sync_token");
+		const completeSnapshot = this.input.streamKind === "inventory" && (this.input.toolkit === "github" || !cursor?.checkpoint);
 		return {
-			items: resultToSourceItems({
-				result: execution.result,
-				sourceInstanceId: pull.instanceId,
-				collectionScope: pull.collectionScope,
-				streamKind: this.input.streamKind,
-				connectorId: this.input.connectorId,
-				connectionId: this.input.connection.id,
-				actionId: this.input.actionId,
-				toolkit: this.input.toolkit,
-				agentId: this.input.agentId,
-				workspaceId: this.input.workspaceId,
-				connectionIdentity: this.input.connection.identity
+			items,
+			nextCursor: nextPageToken ? encodeConnectedSourceCursor({
+				checkpoint: cursor?.checkpoint,
+				scanStartedAt,
+				pageToken: nextPageToken,
+				syncToken: cursor?.syncToken
+			}) : encodeConnectedSourceCursor({
+				checkpoint: scanStartedAt,
+				...nextSyncToken ? { syncToken: nextSyncToken } : {}
 			}),
-			nextCursor: (/* @__PURE__ */ new Date()).toISOString(),
-			warnings: []
+			hasMore: Boolean(nextPageToken),
+			warnings,
+			...completeSnapshot ? { snapshotExternalIds: items.map((item) => item.externalId) } : {}
 		};
 	}
 };

package/dist/src/connectors/connected-source-normalizers.d.ts

@@ -4,6 +4,7 @@ export type ConnectedSourceEntity = {
     itemType: string;
     occurredAt?: string;
     sourceUpdatedAt?: string;
+    deletedAt?: string;
     value: Record<string, unknown>;
     metadata: Record<string, unknown>;
     synthesisStatus: KnowledgeSynthesisStatus;

package/dist/src/connectors/connected-source-normalizers.js

@@ -65,11 +65,13 @@ function normalizeCalendar(result) {
 		const externalId = text(event, "id", "eventId", "event_id");
 		if (!externalId) return [];
 		const occurredAt = time(event.start ?? event.startTime ?? event.start_time);
+		const sourceUpdatedAt = time(event.updated ?? event.updatedAt ?? event.updated_at);
 		return [{
 			externalId,
 			itemType: "calendar_event",
 			occurredAt,
-			sourceUpdatedAt: time(event.updated ?? event.updatedAt ?? event.updated_at),
+			sourceUpdatedAt,
+			...event.status === "cancelled" ? { deletedAt: sourceUpdatedAt ?? (/* @__PURE__ */ new Date()).toISOString() } : {},
 			value: compact({
 				id: externalId,
 				title: text(event, "summary", "title", "name"),
@@ -88,6 +90,34 @@ function normalizeCalendar(result) {
 		}];
 	});
 }
+function normalizeGoogleDrive(result) {
+	return rows(payload(result), "files", "items").flatMap((file) => {
+		const externalId = text(file, "id", "fileId", "file_id");
+		const name = text(file, "name", "title");
+		if (!externalId || !name || file.trashed === true) return [];
+		const modifiedAt = time(file.modifiedTime ?? file.modified_time ?? file.modifiedDate);
+		return [{
+			externalId,
+			itemType: "cloud_document",
+			occurredAt: modifiedAt ?? time(file.createdTime ?? file.created_time ?? file.createdDate),
+			sourceUpdatedAt: modifiedAt,
+			value: compact({
+				id: externalId,
+				title: name,
+				mimeType: text(file, "mimeType", "mime_type"),
+				modifiedAt,
+				owners: file.owners,
+				webViewLink: text(file, "webViewLink", "web_view_link")
+			}),
+			metadata: {
+				observationKind: "document_metadata",
+				logicalEventKey: `googledrive:file:${externalId}`,
+				mimeType: text(file, "mimeType", "mime_type")
+			},
+			synthesisStatus: "ignored"
+		}];
+	});
+}
 function repositoryName(row) {
 	const explicit = text(record(row.repository) ?? record(row.repo), "full_name", "name") ?? text(row, "repository_full_name", "full_name", "repo");
 	if (explicit) return explicit;
@@ -178,8 +208,9 @@ function normalizeExternalTasks(result, toolkit) {
 function normalizeConnectedSourceResult(input) {
 	if (input.toolkit === "gmail") return normalizeGmail(input.result);
 	if (input.toolkit === "googlecalendar") return normalizeCalendar(input.result);
+	if (input.toolkit === "googledrive") return normalizeGoogleDrive(input.result);
 	if (input.toolkit === "github") return normalizeGitHub(input.result, input.actionId);
-	if (input.toolkit === "linear" || input.toolkit === "jira") return normalizeExternalTasks(input.result, input.toolkit);
+	if (input.toolkit === "linear") return normalizeExternalTasks(input.result, input.toolkit);
 	return [];
 }
 //#endregion

package/dist/src/connectors/connected-source-sanitization.d.ts

@@ -0,0 +1 @@
+export declare function sanitizeConnectedSourceValue(value: unknown, depth?: number): unknown;

package/dist/src/connectors/connected-source-sanitization.js

@@ -0,0 +1,21 @@
+//#region src/connectors/connected-source-sanitization.ts
+const SENSITIVE_KEY_RE = /(?:^|_)(?:api_?key|token|secret|password|passwd|authorization|credential|private_?key)(?:$|_)/i;
+const INLINE_SECRET_PATTERNS = [
+	/\bsk-[A-Za-z0-9_-]{8,}\b/g,
+	/\bghp_[A-Za-z0-9]{20,}\b/g,
+	/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g,
+	/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/g
+];
+function record(value) {
+	return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+}
+function sanitizeConnectedSourceValue(value, depth = 0) {
+	if (depth > 12) return "[MaxDepth]";
+	if (typeof value === "string") return INLINE_SECRET_PATTERNS.reduce((text, pattern) => text.replace(pattern, "[REDACTED]"), value);
+	if (Array.isArray(value)) return value.map((item) => sanitizeConnectedSourceValue(item, depth + 1));
+	const source = record(value);
+	if (!source) return value;
+	return Object.fromEntries(Object.entries(source).map(([key, nested]) => [key, SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeConnectedSourceValue(nested, depth + 1)]));
+}
+//#endregion
+export { sanitizeConnectedSourceValue };

package/dist/src/connectors/connector-identity.js

@@ -23,6 +23,13 @@ function normalizeConnectorIdentity(toolkit, result) {
 		profileUrl: text(row, ["html_url"])
 	};
 	if (toolkit === "gmail") return { email: text(row, ["emailAddress", "email"]) };
+	if (toolkit === "googledrive") {
+		const user = object(row.user);
+		return {
+			email: text(user, ["emailAddress", "email"]) ?? text(row, ["emailAddress", "email"]),
+			displayName: text(user, ["displayName", "name"])
+		};
+	}
 	if (toolkit === "slack") return {
 		enterpriseId: text(row, ["enterprise_id", "enterpriseId"]),
 		workspaceId: text(row, ["team_id", "teamId"]),
@@ -62,9 +69,9 @@ function connectorIdentityKey(toolkit, identity) {
 		if (!workspaceId || !subjectId) return void 0;
 		return `slack:${text(identity, ["enterpriseId", "enterprise_id"]) ?? "-"}:${workspaceId}:${subjectId}`;
 	}
-	if (normalizedToolkit === "gmail") {
+	if (normalizedToolkit === "gmail" || normalizedToolkit === "googledrive") {
 		const email = text(identity, ["email"]);
-		return email ? `gmail:${email.toLowerCase()}` : void 0;
+		return email ? `${normalizedToolkit}:${email.toLowerCase()}` : void 0;
 	}
 	if (normalizedToolkit === "github") {
 		const username = text(identity, ["username"]);
@@ -75,6 +82,7 @@ function mergeConnectorIdentity(toolkit, current, incoming) {
 	const normalizedToolkit = toolkit.trim().toLowerCase();
 	if (![
 		"gmail",
+		"googledrive",
 		"github",
 		"slack"
 	].includes(normalizedToolkit)) return {

package/dist/src/connectors/content-enrichment.d.ts

@@ -0,0 +1,29 @@
+import { ComposioSessionsAdapter } from './composio-sessions.js';
+type ContentReadAdapter = Pick<ComposioSessionsAdapter, 'executeWithPolicy'>;
+export type ConnectedContentCandidate = {
+    sourceItemId: string;
+    sourceInstanceId: string;
+    toolkit: 'gmail' | 'googledrive';
+    title: string;
+    occurredAt?: string;
+    mimeType?: string;
+};
+export type ConnectedContentReadResult = {
+    requested: number;
+    completed: number;
+    failed: Array<{
+        sourceItemId: string;
+        error: string;
+    }>;
+    recordIds: string[];
+};
+export declare function listConnectedContentCandidates(options: {
+    agentId: string;
+    limit?: number;
+}): ConnectedContentCandidate[];
+export declare function readConnectedContent(input: {
+    sourceItemIds: string[];
+    agentId: string;
+    adapterFactory?: (downloadDirectory: string) => ContentReadAdapter;
+}): Promise<ConnectedContentReadResult>;
+export {};