@xopcai/xopc 0.0.210 → 0.0.211

package/dist/src/config/schema.js

@@ -834,7 +834,7 @@ activityDetailDefault: z.enum([
 			"always",
 			"inbound",
 			"tagged"
-		]).default("inbound"),
+		]).default("off"),
 		fallback: TTSFallbackConfigSchema.optional(),
 		maxTextLength: z.number().int().min(1).default(512),
 		timeoutMs: z.number().int().min(1e3).max(18e4).default(6e4),

package/dist/src/connectors/composio-sessions.d.ts

@@ -122,6 +122,7 @@ export declare class ComposioSessionsAdapter {
     constructor(options?: {
         resolver?: CredentialResolver;
         clientFactory?: ClientFactory;
+        fileDownloadDir?: string;
     });
     createSession(context: ComposioSessionContext): Promise<ComposioSessionLike>;
     getToolkitAuthState(toolkit: string): Promise<ComposioToolkitAuthState>;

package/dist/src/connectors/composio-sessions.js

@@ -103,8 +103,9 @@ var ComposioSessionsAdapter = class {
 				apiKey,
 				baseURL: process.env.XOPC_COMPOSIO_BASE_URL?.trim() || void 0,
 				allowTracking: false,
-				dangerouslyAllowAutoUploadDownloadFiles: false,
+				dangerouslyAllowAutoUploadDownloadFiles: Boolean(options.fileDownloadDir),
 				fileUploadDirs: false,
+				...options.fileDownloadDir ? { fileDownloadDir: options.fileDownloadDir } : {},
 				provider: new PiProvider(),
 				host: "xopc"
 			});

package/dist/src/connectors/composio.js

@@ -92,14 +92,19 @@ const COMPOSIO_CURATED_ACTIONS = {
 	},
 	googlecalendar: {
 		GOOGLECALENDAR_FIND_EVENT: "read",
-		GOOGLECALENDAR_LIST_EVENTS: "read",
+		GOOGLECALENDAR_SYNC_EVENTS: "read",
 		GOOGLECALENDAR_CREATE_EVENT: "write",
 		GOOGLECALENDAR_UPDATE_EVENT: "write",
 		GOOGLECALENDAR_DELETE_EVENT: "admin"
 	},
 	googledrive: {
-		GOOGLEDRIVE_SEARCH_FILES: "read",
-		GOOGLEDRIVE_GET_FILE: "read",
+		GOOGLEDRIVE_FIND_FILE: "read",
+		GOOGLEDRIVE_GET_ABOUT: "read",
+		GOOGLEDRIVE_GET_FILE_METADATA: "read",
+		GOOGLEDRIVE_DOWNLOAD_FILE: "read",
+		GOOGLEDRIVE_EXPORT_GOOGLE_WORKSPACE_FILE: "read",
+		GOOGLEDRIVE_GET_CHANGES_START_PAGE_TOKEN: "read",
+		GOOGLEDRIVE_LIST_CHANGES: "read",
 		GOOGLEDRIVE_CREATE_FILE: "write",
 		GOOGLEDRIVE_UPDATE_FILE: "write",
 		GOOGLEDRIVE_DELETE_FILE: "admin"

package/dist/src/connectors/connected-source-cursor.d.ts

@@ -0,0 +1,9 @@
+export type ConnectedSourceCursor = {
+    version: 1;
+    checkpoint?: string;
+    scanStartedAt?: string;
+    pageToken?: string;
+    syncToken?: string;
+};
+export declare function decodeConnectedSourceCursor(value: string | undefined): ConnectedSourceCursor | undefined;
+export declare function encodeConnectedSourceCursor(cursor: Omit<ConnectedSourceCursor, 'version'>): string;

package/dist/src/connectors/connected-source-cursor.js

@@ -0,0 +1,30 @@
+//#region src/connectors/connected-source-cursor.ts
+function optionalText(value) {
+	return typeof value === "string" && value.trim() ? value.trim() : void 0;
+}
+function decodeConnectedSourceCursor(value) {
+	if (!value) return void 0;
+	try {
+		const parsed = JSON.parse(value);
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
+		const row = parsed;
+		if (row.version !== 1) return void 0;
+		return {
+			version: 1,
+			...optionalText(row.checkpoint) ? { checkpoint: optionalText(row.checkpoint) } : {},
+			...optionalText(row.scanStartedAt) ? { scanStartedAt: optionalText(row.scanStartedAt) } : {},
+			...optionalText(row.pageToken) ? { pageToken: optionalText(row.pageToken) } : {},
+			...optionalText(row.syncToken) ? { syncToken: optionalText(row.syncToken) } : {}
+		};
+	} catch {
+		return;
+	}
+}
+function encodeConnectedSourceCursor(cursor) {
+	return JSON.stringify({
+		version: 1,
+		...cursor
+	});
+}
+//#endregion
+export { decodeConnectedSourceCursor, encodeConnectedSourceCursor };

package/dist/src/connectors/connected-source-ingestion.js

@@ -1,51 +1,29 @@
 import { getConnectorInstallation, listConnectorConnections } from "../storage/sqlite/connector-repository.js";
 import "../storage/sqlite/index.js";
 import { ComposioSessionsAdapter } from "./composio-sessions.js";
+import { decodeConnectedSourceCursor, encodeConnectedSourceCursor } from "./connected-source-cursor.js";
 import { scopeForComposioAction } from "./composio.js";
 import { getConnectorDefinition } from "./catalog.js";
-import { getConnectorInstance } from "./instances.js";
 import { getWorkspacePath } from "../config/workspace-path-helpers.js";
+import { getConnectorInstance } from "./instances.js";
 import { KnowledgeIngestionService } from "../knowledge/ingestion-service.js";
 import { ConnectedKnowledgePipeline } from "../knowledge/connected-knowledge-pipeline.js";
 import { LocalFolderKnowledgeSourceAdapter } from "../knowledge/local-folder-adapter.js";
 import "../knowledge/index.js";
 import { normalizeConnectedSourceResult } from "./connected-source-normalizers.js";
+import { sanitizeConnectedSourceValue } from "./connected-source-sanitization.js";
 import { createHash } from "node:crypto";
 //#region src/connectors/connected-source-ingestion.ts
 const COMPOSIO_CONNECTED_SOURCE_TOOLKITS = new Set([
 	"gmail",
 	"googlecalendar",
 	"googledrive",
-	"googledocs",
-	"googlesheets",
-	"notion",
-	"slack",
 	"github",
-	"linear",
-	"jira",
-	"outlook",
-	"microsoft_teams",
-	"one_drive",
-	"excel"
+	"linear"
 ]);
-const SENSITIVE_KEY_RE = /(?:^|_)(?:api_?key|token|secret|password|passwd|authorization|credential|private_?key)(?:$|_)/i;
-const INLINE_SECRET_PATTERNS = [
-	/\bsk-[A-Za-z0-9_-]{8,}\b/g,
-	/\bghp_[A-Za-z0-9]{20,}\b/g,
-	/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g,
-	/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/g
-];
 function asRecord(value) {
 	return value && typeof value === "object" && !Array.isArray(value) ? value : null;
 }
-function sanitizeSourceValue(value, depth = 0) {
-	if (depth > 12) return "[MaxDepth]";
-	if (typeof value === "string") return INLINE_SECRET_PATTERNS.reduce((text, pattern) => text.replace(pattern, "[REDACTED]"), value);
-	if (Array.isArray(value)) return value.map((item) => sanitizeSourceValue(item, depth + 1));
-	const record = asRecord(value);
-	if (!record) return value;
-	return Object.fromEntries(Object.entries(record).map(([key, nested]) => [key, SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeSourceValue(nested, depth + 1)]));
-}
 function boundedPersonValue(value, maxLength = 320) {
 	return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
 }
@@ -66,7 +44,7 @@ function personEntities(record) {
 		const nested = asRecord(value);
 		if (!nested) return [];
 		const name = boundedPersonValue(nested.name, 160) || boundedPersonValue(nested.displayName, 160);
-		const email = boundedPersonValue(nested.email);
+		const email = boundedPersonValue(nested.email) || boundedPersonValue(nested.emailAddress);
 		const username = boundedPersonValue(nested.username, 160);
 		return name || email || username ? [{
 			role,
@@ -83,6 +61,7 @@ function personEntities(record) {
 		"user",
 		"username",
 		"owner",
+		"owners",
 		"assignee",
 		"attendees",
 		"participants"
@@ -108,14 +87,26 @@ function peopleSignals(entities) {
 function connectionIdentities(identity) {
 	return [...new Set(Object.values(identity).filter((value) => typeof value === "string").map((value) => value.trim().toLowerCase()).filter(Boolean))];
 }
+function resultPayload(value) {
+	const root = asRecord(value) ?? {};
+	return asRecord(root.data) ?? root;
+}
+function paginationValue(result, ...keys) {
+	const payload = resultPayload(result);
+	const pagination = asRecord(payload.pagination) ?? asRecord(payload.pageInfo) ?? {};
+	for (const key of keys) {
+		const value = payload[key] ?? pagination[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+}
 function resultToSourceItems(input) {
 	return normalizeConnectedSourceResult(input).slice(0, 200).map((entity) => {
 		const record = entity.value;
 		const people = personEntities(record);
 		const ownerIdentities = connectionIdentities(input.connectionIdentity);
 		const observedIdentities = peopleSignals(people).map((value) => value.toLowerCase());
-		const actorAttributed = entity.metadata.actorAttributed === true || input.toolkit === "gmail" || input.toolkit === "googlecalendar" || input.toolkit === "github" && observedIdentities.some((value) => ownerIdentities.includes(value));
-		const serialized = JSON.stringify(sanitizeSourceValue(entity.value), null, 2);
+		const actorAttributed = entity.metadata.actorAttributed === true || input.toolkit === "gmail" || input.toolkit === "googlecalendar" || ["github", "googledrive"].includes(input.toolkit) && observedIdentities.some((value) => ownerIdentities.includes(value));
+		const serialized = JSON.stringify(sanitizeConnectedSourceValue(entity.value), null, 2);
 		const normalizedText = serialized.length > 8e3 ? `${serialized.slice(0, 8e3)}\n…` : serialized;
 		const contentHash = createHash("sha256").update(serialized).digest("hex");
 		return {
@@ -144,7 +135,8 @@ function resultToSourceItems(input) {
 			sensitivity: "personal",
 			retentionClass: "bounded",
 			synthesisPipeline: "connected_knowledge",
-			synthesisStatus: input.streamKind === "inventory" ? "ignored" : entity.synthesisStatus
+			synthesisStatus: input.streamKind === "inventory" ? "ignored" : entity.synthesisStatus,
+			deletedAt: entity.deletedAt
 		};
 	});
 }
@@ -154,7 +146,10 @@ var ComposioKnowledgeSourceAdapter = class {
 		this.input = input;
 	}
 	async pull(pull) {
-		const execution = await this.input.adapter.executeWithPolicy({
+		const cursor = decodeConnectedSourceCursor(pull.cursor);
+		const scanStartedAt = cursor?.scanStartedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+		const warnings = pull.cursor && !cursor ? ["Stored connected-source cursor was invalid and was replaced."] : [];
+		const execute = (cursorValue) => this.input.adapter.executeWithPolicy({
 			context: {
 				principalId: "local-owner",
 				toolkits: [this.input.toolkit]
@@ -174,26 +169,52 @@ var ComposioKnowledgeSourceAdapter = class {
 				curated: true,
 				cachedAt: (/* @__PURE__ */ new Date()).toISOString()
 			},
-			args: this.input.buildArguments?.(pull) ?? this.input.arguments,
+			args: this.input.buildArguments?.({
+				...pull,
+				cursor: cursorValue
+			}) ?? this.input.arguments,
 			confirmed: true
 		});
+		let execution;
+		try {
+			execution = await execute(pull.cursor);
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			if (this.input.toolkit !== "googlecalendar" || !cursor?.syncToken || !/\b410\b|sync.?token/i.test(message)) throw error;
+			warnings.push("Expired Google Calendar sync token was replaced with a full sync.");
+			execution = await execute(void 0);
+		}
 		if (execution.decision !== "allowed") throw new Error(execution.reason);
+		const items = resultToSourceItems({
+			result: execution.result,
+			sourceInstanceId: pull.instanceId,
+			collectionScope: pull.collectionScope,
+			streamKind: this.input.streamKind,
+			connectorId: this.input.connectorId,
+			connectionId: this.input.connection.id,
+			actionId: this.input.actionId,
+			toolkit: this.input.toolkit,
+			agentId: this.input.agentId,
+			workspaceId: this.input.workspaceId,
+			connectionIdentity: this.input.connection.identity
+		});
+		const nextPageToken = paginationValue(execution.result, "nextPageToken", "next_page_token");
+		const nextSyncToken = paginationValue(execution.result, "nextSyncToken", "next_sync_token");
+		const completeSnapshot = this.input.streamKind === "inventory" && (this.input.toolkit === "github" || !cursor?.checkpoint);
 		return {
-			items: resultToSourceItems({
-				result: execution.result,
-				sourceInstanceId: pull.instanceId,
-				collectionScope: pull.collectionScope,
-				streamKind: this.input.streamKind,
-				connectorId: this.input.connectorId,
-				connectionId: this.input.connection.id,
-				actionId: this.input.actionId,
-				toolkit: this.input.toolkit,
-				agentId: this.input.agentId,
-				workspaceId: this.input.workspaceId,
-				connectionIdentity: this.input.connection.identity
+			items,
+			nextCursor: nextPageToken ? encodeConnectedSourceCursor({
+				checkpoint: cursor?.checkpoint,
+				scanStartedAt,
+				pageToken: nextPageToken,
+				syncToken: cursor?.syncToken
+			}) : encodeConnectedSourceCursor({
+				checkpoint: scanStartedAt,
+				...nextSyncToken ? { syncToken: nextSyncToken } : {}
 			}),
-			nextCursor: (/* @__PURE__ */ new Date()).toISOString(),
-			warnings: []
+			hasMore: Boolean(nextPageToken),
+			warnings,
+			...completeSnapshot ? { snapshotExternalIds: items.map((item) => item.externalId) } : {}
 		};
 	}
 };

package/dist/src/connectors/connected-source-normalizers.d.ts

@@ -4,6 +4,7 @@ export type ConnectedSourceEntity = {
     itemType: string;
     occurredAt?: string;
     sourceUpdatedAt?: string;
+    deletedAt?: string;
     value: Record<string, unknown>;
     metadata: Record<string, unknown>;
     synthesisStatus: KnowledgeSynthesisStatus;

package/dist/src/connectors/connected-source-normalizers.js

@@ -65,11 +65,13 @@ function normalizeCalendar(result) {
 		const externalId = text(event, "id", "eventId", "event_id");
 		if (!externalId) return [];
 		const occurredAt = time(event.start ?? event.startTime ?? event.start_time);
+		const sourceUpdatedAt = time(event.updated ?? event.updatedAt ?? event.updated_at);
 		return [{
 			externalId,
 			itemType: "calendar_event",
 			occurredAt,
-			sourceUpdatedAt: time(event.updated ?? event.updatedAt ?? event.updated_at),
+			sourceUpdatedAt,
+			...event.status === "cancelled" ? { deletedAt: sourceUpdatedAt ?? (/* @__PURE__ */ new Date()).toISOString() } : {},
 			value: compact({
 				id: externalId,
 				title: text(event, "summary", "title", "name"),
@@ -88,6 +90,34 @@ function normalizeCalendar(result) {
 		}];
 	});
 }
+function normalizeGoogleDrive(result) {
+	return rows(payload(result), "files", "items").flatMap((file) => {
+		const externalId = text(file, "id", "fileId", "file_id");
+		const name = text(file, "name", "title");
+		if (!externalId || !name || file.trashed === true) return [];
+		const modifiedAt = time(file.modifiedTime ?? file.modified_time ?? file.modifiedDate);
+		return [{
+			externalId,
+			itemType: "cloud_document",
+			occurredAt: modifiedAt ?? time(file.createdTime ?? file.created_time ?? file.createdDate),
+			sourceUpdatedAt: modifiedAt,
+			value: compact({
+				id: externalId,
+				title: name,
+				mimeType: text(file, "mimeType", "mime_type"),
+				modifiedAt,
+				owners: file.owners,
+				webViewLink: text(file, "webViewLink", "web_view_link")
+			}),
+			metadata: {
+				observationKind: "document_metadata",
+				logicalEventKey: `googledrive:file:${externalId}`,
+				mimeType: text(file, "mimeType", "mime_type")
+			},
+			synthesisStatus: "ignored"
+		}];
+	});
+}
 function repositoryName(row) {
 	const explicit = text(record(row.repository) ?? record(row.repo), "full_name", "name") ?? text(row, "repository_full_name", "full_name", "repo");
 	if (explicit) return explicit;
@@ -178,8 +208,9 @@ function normalizeExternalTasks(result, toolkit) {
 function normalizeConnectedSourceResult(input) {
 	if (input.toolkit === "gmail") return normalizeGmail(input.result);
 	if (input.toolkit === "googlecalendar") return normalizeCalendar(input.result);
+	if (input.toolkit === "googledrive") return normalizeGoogleDrive(input.result);
 	if (input.toolkit === "github") return normalizeGitHub(input.result, input.actionId);
-	if (input.toolkit === "linear" || input.toolkit === "jira") return normalizeExternalTasks(input.result, input.toolkit);
+	if (input.toolkit === "linear") return normalizeExternalTasks(input.result, input.toolkit);
 	return [];
 }
 //#endregion

package/dist/src/connectors/connected-source-sanitization.d.ts

@@ -0,0 +1 @@
+export declare function sanitizeConnectedSourceValue(value: unknown, depth?: number): unknown;

package/dist/src/connectors/connected-source-sanitization.js

@@ -0,0 +1,21 @@
+//#region src/connectors/connected-source-sanitization.ts
+const SENSITIVE_KEY_RE = /(?:^|_)(?:api_?key|token|secret|password|passwd|authorization|credential|private_?key)(?:$|_)/i;
+const INLINE_SECRET_PATTERNS = [
+	/\bsk-[A-Za-z0-9_-]{8,}\b/g,
+	/\bghp_[A-Za-z0-9]{20,}\b/g,
+	/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g,
+	/-----BEGIN [^-]*PRIVATE KEY-----[\s\S]*?-----END [^-]*PRIVATE KEY-----/g
+];
+function record(value) {
+	return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+}
+function sanitizeConnectedSourceValue(value, depth = 0) {
+	if (depth > 12) return "[MaxDepth]";
+	if (typeof value === "string") return INLINE_SECRET_PATTERNS.reduce((text, pattern) => text.replace(pattern, "[REDACTED]"), value);
+	if (Array.isArray(value)) return value.map((item) => sanitizeConnectedSourceValue(item, depth + 1));
+	const source = record(value);
+	if (!source) return value;
+	return Object.fromEntries(Object.entries(source).map(([key, nested]) => [key, SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeConnectedSourceValue(nested, depth + 1)]));
+}
+//#endregion
+export { sanitizeConnectedSourceValue };

package/dist/src/connectors/connector-identity.js

@@ -23,6 +23,13 @@ function normalizeConnectorIdentity(toolkit, result) {
 		profileUrl: text(row, ["html_url"])
 	};
 	if (toolkit === "gmail") return { email: text(row, ["emailAddress", "email"]) };
+	if (toolkit === "googledrive") {
+		const user = object(row.user);
+		return {
+			email: text(user, ["emailAddress", "email"]) ?? text(row, ["emailAddress", "email"]),
+			displayName: text(user, ["displayName", "name"])
+		};
+	}
 	if (toolkit === "slack") return {
 		enterpriseId: text(row, ["enterprise_id", "enterpriseId"]),
 		workspaceId: text(row, ["team_id", "teamId"]),
@@ -62,9 +69,9 @@ function connectorIdentityKey(toolkit, identity) {
 		if (!workspaceId || !subjectId) return void 0;
 		return `slack:${text(identity, ["enterpriseId", "enterprise_id"]) ?? "-"}:${workspaceId}:${subjectId}`;
 	}
-	if (normalizedToolkit === "gmail") {
+	if (normalizedToolkit === "gmail" || normalizedToolkit === "googledrive") {
 		const email = text(identity, ["email"]);
-		return email ? `gmail:${email.toLowerCase()}` : void 0;
+		return email ? `${normalizedToolkit}:${email.toLowerCase()}` : void 0;
 	}
 	if (normalizedToolkit === "github") {
 		const username = text(identity, ["username"]);
@@ -75,6 +82,7 @@ function mergeConnectorIdentity(toolkit, current, incoming) {
 	const normalizedToolkit = toolkit.trim().toLowerCase();
 	if (![
 		"gmail",
+		"googledrive",
 		"github",
 		"slack"
 	].includes(normalizedToolkit)) return {

package/dist/src/connectors/content-enrichment.d.ts

@@ -0,0 +1,29 @@
+import { ComposioSessionsAdapter } from './composio-sessions.js';
+type ContentReadAdapter = Pick<ComposioSessionsAdapter, 'executeWithPolicy'>;
+export type ConnectedContentCandidate = {
+    sourceItemId: string;
+    sourceInstanceId: string;
+    toolkit: 'gmail' | 'googledrive';
+    title: string;
+    occurredAt?: string;
+    mimeType?: string;
+};
+export type ConnectedContentReadResult = {
+    requested: number;
+    completed: number;
+    failed: Array<{
+        sourceItemId: string;
+        error: string;
+    }>;
+    recordIds: string[];
+};
+export declare function listConnectedContentCandidates(options: {
+    agentId: string;
+    limit?: number;
+}): ConnectedContentCandidate[];
+export declare function readConnectedContent(input: {
+    sourceItemIds: string[];
+    agentId: string;
+    adapterFactory?: (downloadDirectory: string) => ContentReadAdapter;
+}): Promise<ConnectedContentReadResult>;
+export {};