@xopcai/xopc 0.0.170 → 0.0.172

package/dist/src/connectors/{connector-memory-sync.js → connected-source-ingestion.js}

@@ -9,9 +9,10 @@ import { KnowledgeIngestionService } from "../knowledge/ingestion-service.js";
 import { ConnectedKnowledgePipeline } from "../knowledge/connected-knowledge-pipeline.js";
 import { LocalFolderKnowledgeSourceAdapter } from "../knowledge/local-folder-adapter.js";
 import "../knowledge/index.js";
+import { normalizeConnectedSourceResult } from "./connected-source-normalizers.js";
 import { createHash } from "node:crypto";
-//#region src/connectors/connector-memory-sync.ts
-const COMPOSIO_MEMORY_SOURCE_TOOLKITS = new Set([
+//#region src/connectors/connected-source-ingestion.ts
+const COMPOSIO_CONNECTED_SOURCE_TOOLKITS = new Set([
 	"gmail",
 	"googlecalendar",
 	"googledrive",
@@ -27,34 +28,6 @@ const COMPOSIO_MEMORY_SOURCE_TOOLKITS = new Set([
 	"one_drive",
 	"excel"
 ]);
-const EXTERNAL_ID_KEYS = [
-	"id",
-	"messageId",
-	"message_id",
-	"threadId",
-	"thread_id",
-	"eventId",
-	"event_id",
-	"pageId",
-	"page_id",
-	"fileId",
-	"file_id"
-];
-const OCCURRED_AT_KEYS = [
-	"occurredAt",
-	"createdAt",
-	"created_at",
-	"date",
-	"timestamp",
-	"startTime",
-	"start_time"
-];
-const UPDATED_AT_KEYS = [
-	"updatedAt",
-	"updated_at",
-	"modifiedTime",
-	"modified_time"
-];
 const SENSITIVE_KEY_RE = /(?:^|_)(?:api_?key|token|secret|password|passwd|authorization|credential|private_?key)(?:$|_)/i;
 const INLINE_SECRET_PATTERNS = [
 	/\bsk-[A-Za-z0-9_-]{8,}\b/g,
@@ -65,34 +38,6 @@ const INLINE_SECRET_PATTERNS = [
 function asRecord(value) {
 	return value && typeof value === "object" && !Array.isArray(value) ? value : null;
 }
-function resultRows(value) {
-	if (Array.isArray(value)) return value;
-	const record = asRecord(value);
-	if (!record) return [value];
-	for (const key of [
-		"items",
-		"messages",
-		"results",
-		"events",
-		"files",
-		"pages",
-		"data"
-	]) if (Array.isArray(record[key])) return record[key];
-	return [value];
-}
-function stringField(record, keys) {
-	for (const key of keys) {
-		const value = record?.[key];
-		if (typeof value === "string" && value.trim()) return value.trim();
-		if (typeof value === "number" && Number.isFinite(value)) return String(value);
-	}
-}
-function timestampField(record, keys) {
-	const value = stringField(record, keys);
-	if (!value) return void 0;
-	const parsed = Date.parse(value);
-	return Number.isFinite(parsed) ? new Date(parsed).toISOString() : void 0;
-}
 function sanitizeSourceValue(value, depth = 0) {
 	if (depth > 12) return "[MaxDepth]";
 	if (typeof value === "string") return INLINE_SECRET_PATTERNS.reduce((text, pattern) => text.replace(pattern, "[REDACTED]"), value);
@@ -101,22 +46,6 @@ function sanitizeSourceValue(value, depth = 0) {
 	if (!record) return value;
 	return Object.fromEntries(Object.entries(record).map(([key, nested]) => [key, SENSITIVE_KEY_RE.test(key) ? "[REDACTED]" : sanitizeSourceValue(nested, depth + 1)]));
 }
-function itemTypeFor(toolkit, actionId) {
-	if (toolkit === "gmail" || toolkit === "outlook") return "email";
-	if (toolkit === "googlecalendar" || actionId.includes("CALENDAR")) return "calendar_event";
-	if (toolkit === "slack" || toolkit === "microsoft_teams") return "conversation_message";
-	if (toolkit === "github") return "development_activity";
-	if (toolkit === "linear" || toolkit === "jira") return "work_item";
-	if ([
-		"googledrive",
-		"googledocs",
-		"googlesheets",
-		"notion",
-		"one_drive",
-		"excel"
-	].includes(toolkit)) return "document";
-	return "connector_item";
-}
 function boundedPersonValue(value, maxLength = 320) {
 	return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
 }
@@ -176,21 +105,27 @@ function peopleSignals(entities) {
 	]).filter((value) => typeof value === "string").map((value) => value.trim()).filter(Boolean);
 	return [...new Set(values)].slice(0, 50);
 }
+function connectionIdentities(identity) {
+	return [...new Set(Object.values(identity).filter((value) => typeof value === "string").map((value) => value.trim().toLowerCase()).filter(Boolean))];
+}
 function resultToSourceItems(input) {
-	return resultRows(input.result).slice(0, 200).map((row, index) => {
-		const record = asRecord(row);
+	return normalizeConnectedSourceResult(input).slice(0, 200).map((entity) => {
+		const record = entity.value;
 		const people = personEntities(record);
-		const serialized = JSON.stringify(sanitizeSourceValue(row), null, 2) ?? String(row);
+		const ownerIdentities = connectionIdentities(input.connectionIdentity);
+		const observedIdentities = peopleSignals(people).map((value) => value.toLowerCase());
+		const actorAttributed = entity.metadata.actorAttributed === true || input.toolkit === "gmail" || input.toolkit === "googlecalendar" || input.toolkit === "github" && observedIdentities.some((value) => ownerIdentities.includes(value));
+		const serialized = JSON.stringify(sanitizeSourceValue(entity.value), null, 2);
 		const normalizedText = serialized.length > 8e3 ? `${serialized.slice(0, 8e3)}\n…` : serialized;
 		const contentHash = createHash("sha256").update(serialized).digest("hex");
-		const externalId = stringField(record, EXTERNAL_ID_KEYS) ?? `${input.actionId}:${index}`;
 		return {
 			sourceInstanceId: input.sourceInstanceId,
-			externalId,
-			itemType: itemTypeFor(input.toolkit, input.actionId),
+			collectionScope: input.collectionScope,
+			externalId: entity.externalId,
+			itemType: entity.itemType,
 			authorRole: "third_party",
-			occurredAt: timestampField(record, OCCURRED_AT_KEYS),
-			sourceUpdatedAt: timestampField(record, UPDATED_AT_KEYS),
+			occurredAt: entity.occurredAt,
+			sourceUpdatedAt: entity.sourceUpdatedAt,
 			contentHash,
 			normalizedText,
 			metadata: {
@@ -201,12 +136,15 @@ function resultToSourceItems(input) {
 				people: peopleSignals(people),
 				personEntities: people,
 				agentId: input.agentId,
-				workspaceId: input.workspaceId
+				workspaceId: input.workspaceId,
+				...entity.metadata,
+				ownerIdentities,
+				actorAttributed
 			},
 			sensitivity: "personal",
 			retentionClass: "bounded",
 			synthesisPipeline: "connected_knowledge",
-			synthesisStatus: "pending"
+			synthesisStatus: input.streamKind === "inventory" ? "ignored" : entity.synthesisStatus
 		};
 	});
 }
@@ -236,7 +174,7 @@ var ComposioKnowledgeSourceAdapter = class {
 				curated: true,
 				cachedAt: (/* @__PURE__ */ new Date()).toISOString()
 			},
-			args: this.input.arguments,
+			args: this.input.buildArguments?.(pull) ?? this.input.arguments,
 			confirmed: true
 		});
 		if (execution.decision !== "allowed") throw new Error(execution.reason);
@@ -244,25 +182,28 @@ var ComposioKnowledgeSourceAdapter = class {
 			items: resultToSourceItems({
 				result: execution.result,
 				sourceInstanceId: pull.instanceId,
+				collectionScope: pull.collectionScope,
+				streamKind: this.input.streamKind,
 				connectorId: this.input.connectorId,
 				connectionId: this.input.connection.id,
 				actionId: this.input.actionId,
 				toolkit: this.input.toolkit,
 				agentId: this.input.agentId,
-				workspaceId: this.input.workspaceId
+				workspaceId: this.input.workspaceId,
+				connectionIdentity: this.input.connection.identity
 			}),
 			nextCursor: (/* @__PURE__ */ new Date()).toISOString(),
 			warnings: []
 		};
 	}
 };
-async function syncComposioResultToMemory(input) {
+async function ingestComposioConnectedSource(input) {
 	const definition = getConnectorDefinition(input.connectorId);
-	if (definition?.runtime.type !== "composio" || definition.runtime.role !== "toolkit") throw new Error("Memory sync requires a Composio toolkit connector.");
+	if (definition?.runtime.type !== "composio" || definition.runtime.role !== "toolkit") throw new Error("Connected source ingestion requires a Composio toolkit connector.");
 	const toolkit = definition.runtime.toolkit;
-	if (!COMPOSIO_MEMORY_SOURCE_TOOLKITS.has(toolkit)) throw new Error(`Memory sync is not supported for the ${toolkit} connector.`);
+	if (!COMPOSIO_CONNECTED_SOURCE_TOOLKITS.has(toolkit)) throw new Error(`Connected source ingestion is not supported for the ${toolkit} connector.`);
 	const risk = scopeForComposioAction(input.actionId);
-	if (risk.toolkit !== toolkit || risk.scope !== "read" || !risk.curated) throw new Error("Memory sync only accepts a verified read action for the selected connector.");
+	if (risk.toolkit !== toolkit || risk.scope !== "read" || !risk.curated) throw new Error("Connected source ingestion only accepts a verified read action for the selected connector.");
 	const connectorId = definition.id;
 	const installation = getConnectorInstallation(`${connectorId}-local-owner`);
 	if (!installation?.enabled) throw new Error("Connector installation is not enabled.");
@@ -272,15 +213,18 @@ async function syncComposioResultToMemory(input) {
 		connectorId
 	}).filter((connection) => connection.status === "active");
 	const connection = input.connectionId ? activeConnections.find((candidate) => candidate.id === input.connectionId) : activeConnections.find((candidate) => candidate.isDefault) ?? activeConnections[0];
-	if (!connection) throw new Error("No active connector account is available for memory sync.");
+	if (!connection) throw new Error("No active connector account is available for connected source ingestion.");
 	const adapter = input.adapter ?? new ComposioSessionsAdapter();
 	const workspaceId = getWorkspacePath(input.config);
 	const sourceInstanceId = `composio:${connectorId}:${connection.id}`;
 	const sourceAdapter = new ComposioKnowledgeSourceAdapter({
 		connectorId,
+		collectionScope: input.collectionScope,
+		streamKind: input.streamKind,
 		connection,
 		actionId: input.actionId,
 		arguments: input.arguments ?? {},
+		buildArguments: input.buildArguments,
 		agentId: input.agentId,
 		toolkit,
 		workspaceId,
@@ -289,27 +233,30 @@ async function syncComposioResultToMemory(input) {
 	});
 	const syncRun = await new KnowledgeIngestionService(new Map([[sourceAdapter.kind, sourceAdapter]])).sync({
 		adapterKind: sourceAdapter.kind,
-		instanceId: sourceInstanceId
+		instanceId: sourceInstanceId,
+		collectionScope: input.collectionScope
 	});
-	if (syncRun.status === "failed" || syncRun.status === "cancelled") throw new Error(syncRun.error ?? `Connector memory sync ${syncRun.status}.`);
+	if (syncRun.status === "failed" || syncRun.status === "cancelled") throw new Error(syncRun.error ?? `Connected source ingestion ${syncRun.status}.`);
 	const pipeline = new ConnectedKnowledgePipeline({
 		agentId: input.agentId,
 		workspaceId
 	});
-	let recordId;
+	const recordIds = [];
 	for (let batch = 0; batch < 10; batch += 1) {
 		const synthesis = await pipeline.processPending(sourceInstanceId);
-		recordId ??= synthesis.recordIds[0];
+		recordIds.push(...synthesis.recordIds);
 		if (synthesis.claimed === 0) break;
 	}
-	if (!recordId) throw new Error("Connector result did not contain any synthesizable knowledge.");
 	return {
-		recordId,
 		connectorId,
-		actionId: input.actionId
+		actionId: input.actionId,
+		sourceInstanceId,
+		itemsSeen: syncRun.itemsSeen,
+		itemsIndexed: syncRun.itemsCreated + syncRun.itemsUpdated,
+		recordIds
 	};
 }
-async function syncLocalFolderToMemory(input) {
+async function ingestLocalFolderSource(input) {
 	const definition = getConnectorDefinition(input.connectorId);
 	if (definition?.runtime.type !== "memorySource" || definition.runtime.sourceKind !== "local-folder") throw new Error("Local folder sync requires a local-folder memory source connector.");
 	const instance = getConnectorInstance(input.config, definition.id);
@@ -320,7 +267,8 @@ async function syncLocalFolderToMemory(input) {
 	const adapter = new LocalFolderKnowledgeSourceAdapter(rootPath);
 	const syncRun = await new KnowledgeIngestionService(new Map([[adapter.kind, adapter]])).sync({
 		adapterKind: adapter.kind,
-		instanceId: sourceInstanceId
+		instanceId: sourceInstanceId,
+		collectionScope: "files"
 	});
 	if (syncRun.status === "failed" || syncRun.status === "cancelled") throw new Error(syncRun.error ?? `Local folder sync ${syncRun.status}.`);
 	const pipeline = new ConnectedKnowledgePipeline({
@@ -340,4 +288,4 @@ async function syncLocalFolderToMemory(input) {
 	};
 }
 //#endregion
-export { COMPOSIO_MEMORY_SOURCE_TOOLKITS, syncComposioResultToMemory, syncLocalFolderToMemory };
+export { COMPOSIO_CONNECTED_SOURCE_TOOLKITS, ingestComposioConnectedSource, ingestLocalFolderSource };

package/dist/src/connectors/connected-source-normalizers.d.ts

@@ -0,0 +1,15 @@
+import type { KnowledgeSynthesisStatus } from '../knowledge/types.js';
+export type ConnectedSourceEntity = {
+    externalId: string;
+    itemType: string;
+    occurredAt?: string;
+    sourceUpdatedAt?: string;
+    value: Record<string, unknown>;
+    metadata: Record<string, unknown>;
+    synthesisStatus: KnowledgeSynthesisStatus;
+};
+export declare function normalizeConnectedSourceResult(input: {
+    toolkit: string;
+    actionId: string;
+    result: unknown;
+}): ConnectedSourceEntity[];

package/dist/src/connectors/connected-source-normalizers.js

@@ -0,0 +1,186 @@
+//#region src/connectors/connected-source-normalizers.ts
+function record(value) {
+	return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function text(row, ...keys) {
+	for (const key of keys) {
+		const value = row?.[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+		if (typeof value === "number" && Number.isFinite(value)) return String(value);
+	}
+}
+function time(value) {
+	const nested = record(value);
+	const candidate = typeof value === "string" || typeof value === "number" ? value : text(nested, "dateTime", "date", "time", "timestamp");
+	if (candidate === void 0) return void 0;
+	const numeric = typeof candidate === "number" ? candidate : Number(candidate);
+	const parsed = Number.isFinite(numeric) && String(candidate).length >= 10 ? new Date(numeric < 1e10 ? numeric * 1e3 : numeric) : new Date(String(candidate));
+	return Number.isFinite(parsed.getTime()) ? parsed.toISOString() : void 0;
+}
+function payload(result) {
+	const envelope = record(result);
+	if (!envelope || envelope.error) return void 0;
+	return record(envelope.data) ?? envelope;
+}
+function rows(data, ...keys) {
+	for (const key of keys) {
+		const value = data?.[key];
+		if (Array.isArray(value)) return value.map(record).filter((item) => Boolean(item));
+	}
+	return [];
+}
+function compact(value) {
+	return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== void 0 && item !== null && item !== ""));
+}
+function normalizeGmail(result) {
+	return rows(payload(result), "messages").flatMap((message) => {
+		const externalId = text(message, "id", "messageId", "message_id");
+		if (!externalId) return [];
+		const occurredAt = time(message.internalDate ?? message.internal_date ?? message.date ?? message.timestamp);
+		return [{
+			externalId,
+			itemType: "email",
+			occurredAt,
+			sourceUpdatedAt: occurredAt,
+			value: compact({
+				id: externalId,
+				threadId: text(message, "threadId", "thread_id"),
+				subject: text(message, "subject"),
+				sender: message.sender ?? message.from,
+				recipients: message.recipients ?? message.to,
+				date: occurredAt,
+				labels: message.labelIds ?? message.label_ids ?? message.labels,
+				snippet: text(message, "snippet", "preview")
+			}),
+			metadata: {
+				observationKind: "message",
+				logicalEventKey: `gmail:message:${externalId}`
+			},
+			synthesisStatus: "pending"
+		}];
+	});
+}
+function normalizeCalendar(result) {
+	return rows(payload(result), "items", "events").flatMap((event) => {
+		const externalId = text(event, "id", "eventId", "event_id");
+		if (!externalId) return [];
+		const occurredAt = time(event.start ?? event.startTime ?? event.start_time);
+		return [{
+			externalId,
+			itemType: "calendar_event",
+			occurredAt,
+			sourceUpdatedAt: time(event.updated ?? event.updatedAt ?? event.updated_at),
+			value: compact({
+				id: externalId,
+				title: text(event, "summary", "title", "name"),
+				start: occurredAt,
+				end: time(event.end ?? event.endTime ?? event.end_time),
+				organizer: event.organizer,
+				attendees: event.attendees,
+				status: event.status,
+				recurrence: event.recurrence
+			}),
+			metadata: {
+				observationKind: "calendar_event",
+				logicalEventKey: `googlecalendar:event:${externalId}`
+			},
+			synthesisStatus: "pending"
+		}];
+	});
+}
+function repositoryName(row) {
+	const explicit = text(record(row.repository) ?? record(row.repo), "full_name", "name") ?? text(row, "repository_full_name", "full_name", "repo");
+	if (explicit) return explicit;
+	return (text(row, "repository_url", "html_url")?.match(/(?:repos\/|github\.com\/)([^/]+\/[^/#]+)(?:\/|$)/))?.[1];
+}
+function normalizeGitHub(result, actionId) {
+	const data = payload(result);
+	if (actionId === "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER") return rows(data, "repositories").flatMap((repository) => {
+		const externalId = text(repository, "id", "node_id", "full_name");
+		const fullName = text(repository, "full_name");
+		if (!externalId || !fullName) return [];
+		return [{
+			externalId,
+			itemType: "repository",
+			occurredAt: time(repository.created_at),
+			sourceUpdatedAt: time(repository.updated_at),
+			value: compact({
+				id: externalId,
+				fullName,
+				description: text(repository, "description"),
+				visibility: repository.visibility,
+				language: repository.language,
+				archived: repository.archived
+			}),
+			metadata: {
+				observationKind: "inventory",
+				logicalEventKey: `github:repository:${externalId}`,
+				subjectKey: fullName.toLowerCase(),
+				actorAttributed: false
+			},
+			synthesisStatus: "ignored"
+		}];
+	});
+	return rows(data, "commits", "pull_requests", "issues", "items").flatMap((activity) => {
+		const rawId = text(activity, "id", "node_id", "sha", "number");
+		if (!rawId) return [];
+		const subjectKey = repositoryName(activity);
+		const externalId = subjectKey ? `${subjectKey}:${rawId}` : rawId;
+		const occurredAt = time(activity.committed_at ?? activity.created_at ?? activity.updated_at);
+		return [{
+			externalId: `${actionId}:${externalId}`,
+			itemType: "development_activity",
+			occurredAt,
+			sourceUpdatedAt: time(activity.updated_at) ?? occurredAt,
+			value: compact({
+				id: externalId,
+				repository: subjectKey,
+				title: text(activity, "title", "message", "name"),
+				author: activity.author ?? activity.user ?? activity.actor,
+				state: activity.state,
+				occurredAt
+			}),
+			metadata: {
+				observationKind: "activity",
+				logicalEventKey: `github:${actionId.toLowerCase()}:${externalId}`,
+				subjectKey
+			},
+			synthesisStatus: "pending"
+		}];
+	});
+}
+function normalizeWorkItems(result, toolkit) {
+	return rows(payload(result), "issues", "items", "results").flatMap((item) => {
+		const externalId = text(item, "id", "identifier", "key");
+		if (!externalId) return [];
+		const occurredAt = time(item.createdAt ?? item.created_at);
+		return [{
+			externalId,
+			itemType: "work_item",
+			occurredAt,
+			sourceUpdatedAt: time(item.updatedAt ?? item.updated_at) ?? occurredAt,
+			value: compact({
+				id: externalId,
+				title: text(item, "title", "summary", "name"),
+				project: item.project ?? item.team,
+				assignee: item.assignee,
+				creator: item.creator ?? item.reporter,
+				state: item.state ?? item.status
+			}),
+			metadata: {
+				observationKind: "work_item",
+				logicalEventKey: `${toolkit}:work-item:${externalId}`
+			},
+			synthesisStatus: "pending"
+		}];
+	});
+}
+function normalizeConnectedSourceResult(input) {
+	if (input.toolkit === "gmail") return normalizeGmail(input.result);
+	if (input.toolkit === "googlecalendar") return normalizeCalendar(input.result);
+	if (input.toolkit === "github") return normalizeGitHub(input.result, input.actionId);
+	if (input.toolkit === "linear" || input.toolkit === "jira") return normalizeWorkItems(input.result, input.toolkit);
+	return [];
+}
+//#endregion
+export { normalizeConnectedSourceResult };

package/dist/src/connectors/connector-identity.d.ts

@@ -0,0 +1,2 @@
+import type { ConnectorConnection } from './types.js';
+export declare function normalizeConnectorIdentity(toolkit: string, result: unknown): ConnectorConnection['identity'];

package/dist/src/connectors/connector-identity.js

@@ -0,0 +1,29 @@
+//#region src/connectors/connector-identity.ts
+function object(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+	return value;
+}
+function payload(value) {
+	const root = object(value);
+	return object(root.data ?? root.result ?? root);
+}
+function text(row, keys) {
+	for (const key of keys) {
+		const value = row[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+}
+function normalizeConnectorIdentity(toolkit, result) {
+	const row = payload(result);
+	if (toolkit === "github") return {
+		username: text(row, ["login", "username"]),
+		displayName: text(row, ["name"]),
+		email: text(row, ["email"]),
+		avatarUrl: text(row, ["avatar_url"]),
+		profileUrl: text(row, ["html_url"])
+	};
+	if (toolkit === "gmail") return { email: text(row, ["emailAddress", "email"]) };
+	throw new Error(`Identity normalization is not defined for ${toolkit}.`);
+}
+//#endregion
+export { normalizeConnectorIdentity };

package/dist/src/connectors/learning-coordinator.d.ts

@@ -0,0 +1,25 @@
+import type { Config } from '../config/schema.js';
+import type { MemoryManager } from '../agent/memory/manager.js';
+import { type ConnectorLearningJob } from '../storage/sqlite/index.js';
+import { ComposioSessionsAdapter } from './composio-sessions.js';
+export type ConnectorLearningCoordinator = {
+    enqueueConnection(connectionId: string, request?: {
+        mode?: 'bootstrap' | 'incremental';
+        reason?: 'manual' | 'event' | 'schedule';
+        idempotencyKey?: string;
+        nextRunAt?: number;
+    }): ConnectorLearningJob | null;
+    enqueueToolkit(toolkit: string): ConnectorLearningJob[];
+    setPaused(connectionId: string, paused: boolean): number;
+    runNow(): Promise<void>;
+    stop(): void;
+};
+export declare function startConnectorLearningCoordinator(options: {
+    getConfig: () => Config;
+    resolveAgentId: () => string;
+    getMemoryManager: () => MemoryManager;
+    emit?: (type: string, payload: unknown) => void;
+    intervalMs?: number;
+    initialDelayMs?: number;
+    composioAdapter?: ComposioSessionsAdapter;
+}): ConnectorLearningCoordinator;