@xopcai/xopc 0.0.170 → 0.0.172

package/dist/src/connectors/learning-coordinator.js

@@ -0,0 +1,214 @@
+import { createLogger } from "../utils/logger/index.js";
+import { init_logger } from "../utils/logger.js";
+import { claimNextConnectorLearningJob, enqueueConnectorLearningJob, listConnectorLearningJobs, recoverStaleConnectorLearningJobs, setConnectorLearningPaused, updateConnectorLearningJob } from "../storage/sqlite/connector-learning-repository.js";
+import { pruneBoundedKnowledgeSourceItems } from "../storage/sqlite/knowledge-repository.js";
+import { getConnectorInstallation, listConnectorConnections, upsertConnectorConnection } from "../storage/sqlite/connector-repository.js";
+import "../storage/sqlite/index.js";
+import { ComposioSessionsAdapter } from "./composio-sessions.js";
+import { getConnectorDefinition } from "./catalog.js";
+import { ConnectedUnderstandingPipeline } from "../knowledge/connected-understanding-pipeline.js";
+import "../knowledge/index.js";
+import { ingestComposioConnectedSource } from "./connected-source-ingestion.js";
+import { normalizeConnectorIdentity } from "./connector-identity.js";
+import { buildConnectorLearningArguments, getConnectorLearningPlan } from "./learning-recipes.js";
+//#region src/connectors/learning-coordinator.ts
+init_logger();
+const log = createLogger("ConnectorLearning");
+const CONNECTED_ACCOUNT_UNAVAILABLE = "connected_account_unavailable";
+const CONNECTED_SOURCE_SYNC_FAILED = "connected_source_sync_failed";
+function learningFailureCode(error) {
+	const message = error instanceof Error ? error.message : String(error);
+	return message.includes("ToolRouterV2_InvalidConnectedAccountIds") || message.includes("Could not find connected account(s)") || message.includes("missing its provider identity") ? CONNECTED_ACCOUNT_UNAVAILABLE : CONNECTED_SOURCE_SYNC_FAILED;
+}
+function learningEnabled(config) {
+	return config.userContext.enabled && config.userContext.understanding.enabled && config.userContext.memory.mode !== "off" && config.userContext.memory.sources.includes("connectedSources");
+}
+function startConnectorLearningCoordinator(options) {
+	const intervalMs = Math.max(2e3, Math.min(options.intervalMs ?? 1e4, 6e4));
+	let timer = null;
+	let running = false;
+	let stopped = false;
+	const composioAdapter = options.composioAdapter ?? new ComposioSessionsAdapter();
+	const publish = (job) => options.emit?.("connector.learning.updated", job);
+	function enqueueConnection(connectionId, request = {}) {
+		if (!learningEnabled(options.getConfig())) return null;
+		const connection = listConnectorConnections().find((item) => item.id === connectionId && item.status === "active");
+		if (!connection) return null;
+		const definition = getConnectorDefinition(connection.connectorId);
+		if (definition?.runtime.type !== "composio" || definition.runtime.role !== "toolkit") return null;
+		if (!getConnectorLearningPlan(definition.runtime.toolkit)) return null;
+		const sourceInstanceId = `composio:${connection.connectorId}:${connection.id}`;
+		const existing = listConnectorLearningJobs({
+			connectionId: connection.id,
+			limit: 1
+		});
+		const mode = request.mode ?? (existing.length === 0 ? "bootstrap" : "incremental");
+		const bucket = Math.floor(Date.now() / 6e4);
+		const job = enqueueConnectorLearningJob({
+			connectorId: connection.connectorId,
+			connectionId: connection.id,
+			sourceInstanceId,
+			agentId: options.resolveAgentId(),
+			mode,
+			idempotencyKey: request.idempotencyKey ?? (mode === "bootstrap" ? `bootstrap:${connection.id}:v1` : `incremental:${connection.id}:${request.reason ?? "manual"}:${bucket}`),
+			nextRunAt: request.nextRunAt
+		});
+		publish(job);
+		queueMicrotask(() => void runNow());
+		return job;
+	}
+	function enqueueToolkit(toolkit) {
+		const normalized = toolkit.trim().toLowerCase();
+		return listConnectorConnections().filter((connection) => connection.status === "active").filter((connection) => {
+			const definition = getConnectorDefinition(connection.connectorId);
+			return definition?.runtime.type === "composio" && definition.runtime.role === "toolkit" && definition.runtime.toolkit === normalized;
+		}).map((connection) => enqueueConnection(connection.id, { reason: "event" })).filter((job) => Boolean(job));
+	}
+	async function execute(job) {
+		publish(job);
+		const definition = getConnectorDefinition(job.connectorId);
+		if (definition?.runtime.type !== "composio" || definition.runtime.role !== "toolkit") throw new Error(`Connector learning is unavailable for ${job.connectorId}.`);
+		const plan = getConnectorLearningPlan(definition.runtime.toolkit);
+		if (!plan) throw new Error(`Connector learning has no plan for ${definition.runtime.toolkit}.`);
+		let connection = listConnectorConnections().find((candidate) => candidate.id === job.connectionId);
+		if (!connection) throw new Error("Connector account no longer exists.");
+		const installation = getConnectorInstallation(`${job.connectorId}-local-owner`);
+		if (!installation?.enabled) throw new Error("Connector installation is not enabled.");
+		const hasIdentity = Object.values(connection.identity).some((value) => typeof value === "string" && value.trim());
+		if (plan.identityProbe && (job.mode === "bootstrap" || !hasIdentity)) {
+			const execution = await composioAdapter.executeWithPolicy({
+				context: {
+					principalId: connection.principalId,
+					toolkits: [plan.toolkit]
+				},
+				installation: {
+					...installation,
+					maxScope: "read",
+					confirmationPolicy: "never"
+				},
+				connection,
+				agentId: job.agentId,
+				action: {
+					connectorId: job.connectorId,
+					actionId: plan.identityProbe.actionId,
+					toolkit: plan.toolkit,
+					scope: "read",
+					curated: true,
+					cachedAt: (/* @__PURE__ */ new Date()).toISOString()
+				},
+				args: {},
+				confirmed: true
+			});
+			if (execution.decision !== "allowed") throw new Error(execution.reason);
+			connection = upsertConnectorConnection({
+				...connection,
+				identity: normalizeConnectorIdentity(plan.toolkit, execution.result)
+			});
+		}
+		let itemsSeen = 0;
+		let itemsIndexed = 0;
+		for (const stream of plan.streams) {
+			const synced = await ingestComposioConnectedSource({
+				config: options.getConfig(),
+				connectorId: job.connectorId,
+				connectionId: job.connectionId,
+				collectionScope: stream.scope,
+				streamKind: stream.kind,
+				actionId: stream.actionId,
+				arguments: stream.arguments,
+				buildArguments: (pull) => buildConnectorLearningArguments(plan, stream, pull, connection.identity),
+				agentId: job.agentId
+			});
+			itemsSeen += synced.itemsSeen;
+			itemsIndexed += synced.itemsIndexed;
+		}
+		if (listConnectorLearningJobs({
+			connectionId: job.connectionId,
+			limit: 100
+		}).find((candidate) => candidate.id === job.id)?.status === "paused") return;
+		publish(updateConnectorLearningJob(job.id, {
+			phase: "indexing",
+			itemsDiscovered: itemsSeen,
+			itemsIndexed
+		}));
+		publish(updateConnectorLearningJob(job.id, { phase: "deriving" }));
+		const understanding = await new ConnectedUnderstandingPipeline(options.getMemoryManager()).process(job.sourceInstanceId, job.connectorId, job.agentId);
+		publish(updateConnectorLearningJob(job.id, {
+			status: "completed",
+			phase: "completed",
+			candidatesCreated: understanding.created,
+			nextRunAt: null,
+			finished: true
+		}));
+		pruneBoundedKnowledgeSourceItems(job.sourceInstanceId, Date.now() - plan.bootstrapWindowDays * 24 * 60 * 6e4);
+		const nextRunAt = Date.now() + plan.intervalMinutes * 6e4;
+		enqueueConnection(job.connectionId, {
+			mode: "incremental",
+			reason: "schedule",
+			nextRunAt,
+			idempotencyKey: `scheduled:${job.connectionId}:${nextRunAt}`
+		});
+	}
+	async function runNow() {
+		if (running || stopped) return;
+		running = true;
+		try {
+			for (let count = 0; count < 10; count += 1) {
+				const job = claimNextConnectorLearningJob();
+				if (!job) break;
+				try {
+					await execute(job);
+				} catch (err) {
+					const errorMessage = err instanceof Error ? err.message : String(err);
+					const exhausted = job.attemptCount >= 5;
+					publish(updateConnectorLearningJob(job.id, {
+						status: exhausted ? "paused" : "failed",
+						error: learningFailureCode(err),
+						nextRunAt: exhausted ? null : Date.now() + Math.min(30 * 6e4, 3e4 * 2 ** Math.min(job.attemptCount, 6)),
+						finished: true
+					}));
+					log.warn({
+						err,
+						jobId: job.id,
+						connectorId: job.connectorId
+					}, `Connector learning failed: ${errorMessage}`);
+				}
+			}
+		} finally {
+			running = false;
+		}
+	}
+	recoverStaleConnectorLearningJobs(Date.now());
+	if (learningEnabled(options.getConfig())) {
+		for (const connection of listConnectorConnections().filter((item) => item.status === "active")) if (listConnectorLearningJobs({
+			connectionId: connection.id,
+			limit: 1
+		}).length === 0) enqueueConnection(connection.id, { reason: "schedule" });
+	}
+	timer = setTimeout(() => {
+		runNow();
+		timer = setInterval(() => void runNow(), intervalMs);
+		timer.unref?.();
+	}, Math.max(0, options.initialDelayMs ?? 2e3));
+	timer.unref?.();
+	return {
+		enqueueConnection,
+		enqueueToolkit,
+		setPaused(connectionId, paused) {
+			const changed = setConnectorLearningPaused(connectionId, paused);
+			if (!paused && changed > 0) queueMicrotask(() => void runNow());
+			return changed;
+		},
+		runNow,
+		stop() {
+			stopped = true;
+			if (timer) {
+				clearTimeout(timer);
+				clearInterval(timer);
+				timer = null;
+			}
+		}
+	};
+}
+//#endregion
+export { startConnectorLearningCoordinator };

package/dist/src/connectors/learning-recipes.d.ts

@@ -0,0 +1,22 @@
+export type ConnectorIdentityProbe = {
+    actionId: string;
+};
+export type ConnectorLearningStream = {
+    scope: string;
+    actionId: string;
+    arguments: Record<string, unknown>;
+    kind: 'activity' | 'inventory';
+};
+export type ConnectorLearningPlan = {
+    toolkit: string;
+    identityProbe?: ConnectorIdentityProbe;
+    streams: ConnectorLearningStream[];
+    bootstrapWindowDays: number;
+    intervalMinutes: number;
+};
+export declare function buildConnectorLearningArguments(plan: ConnectorLearningPlan, stream: ConnectorLearningStream, input: {
+    cursor?: string;
+    windowStart?: string;
+}, identity: Record<string, unknown>): Record<string, unknown>;
+export declare function getConnectorLearningPlan(toolkit: string): ConnectorLearningPlan | undefined;
+export declare function listConnectorLearningPlans(): ConnectorLearningPlan[];

package/dist/src/connectors/learning-recipes.js

@@ -0,0 +1,121 @@
+//#region src/connectors/learning-recipes.ts
+function buildConnectorLearningArguments(plan, stream, input, identity) {
+	if (plan.toolkit === "github" && stream.scope === "authored-work") {
+		const username = typeof identity.username === "string" ? identity.username.trim() : "";
+		if (!username) throw new Error("GitHub activity sync requires the authenticated username.");
+		return {
+			...stream.arguments,
+			q: `author:${username} sort:updated-desc`
+		};
+	}
+	if (!input.cursor) return { ...stream.arguments };
+	if (plan.toolkit === "gmail") {
+		const after = Math.floor(Date.parse(input.cursor) / 1e3);
+		return Number.isFinite(after) ? {
+			...stream.arguments,
+			query: `after:${after} -in:spam -in:trash`
+		} : { ...stream.arguments };
+	}
+	if (plan.toolkit === "googlecalendar") return {
+		...stream.arguments,
+		time_min: input.cursor
+	};
+	return { ...stream.arguments };
+}
+const PLANS = {
+	gmail: {
+		toolkit: "gmail",
+		identityProbe: { actionId: "GMAIL_GET_PROFILE" },
+		streams: [{
+			scope: "messages",
+			actionId: "GMAIL_FETCH_EMAILS",
+			kind: "activity",
+			arguments: {
+				max_results: 100,
+				query: "newer_than:30d -in:spam -in:trash"
+			}
+		}],
+		bootstrapWindowDays: 30,
+		intervalMinutes: 15
+	},
+	googlecalendar: {
+		toolkit: "googlecalendar",
+		streams: [{
+			scope: "events",
+			actionId: "GOOGLECALENDAR_LIST_EVENTS",
+			kind: "activity",
+			arguments: { max_results: 200 }
+		}],
+		bootstrapWindowDays: 90,
+		intervalMinutes: 15
+	},
+	googledrive: {
+		toolkit: "googledrive",
+		streams: [{
+			scope: "files",
+			actionId: "GOOGLEDRIVE_SEARCH_FILES",
+			kind: "inventory",
+			arguments: {}
+		}],
+		bootstrapWindowDays: 90,
+		intervalMinutes: 30
+	},
+	notion: {
+		toolkit: "notion",
+		streams: [{
+			scope: "pages",
+			actionId: "NOTION_SEARCH",
+			kind: "inventory",
+			arguments: {}
+		}],
+		bootstrapWindowDays: 90,
+		intervalMinutes: 30
+	},
+	slack: {
+		toolkit: "slack",
+		streams: [{
+			scope: "channels",
+			actionId: "SLACK_LIST_CHANNELS",
+			kind: "inventory",
+			arguments: {}
+		}],
+		bootstrapWindowDays: 30,
+		intervalMinutes: 15
+	},
+	github: {
+		toolkit: "github",
+		identityProbe: { actionId: "GITHUB_GET_THE_AUTHENTICATED_USER" },
+		streams: [{
+			scope: "repositories",
+			actionId: "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER",
+			kind: "inventory",
+			arguments: {}
+		}, {
+			scope: "authored-work",
+			actionId: "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS",
+			kind: "activity",
+			arguments: { per_page: 100 }
+		}],
+		bootstrapWindowDays: 90,
+		intervalMinutes: 30
+	},
+	linear: {
+		toolkit: "linear",
+		streams: [{
+			scope: "issues",
+			actionId: "LINEAR_LIST_ISSUES",
+			kind: "activity",
+			arguments: {}
+		}],
+		bootstrapWindowDays: 60,
+		intervalMinutes: 15
+	}
+};
+function getConnectorLearningPlan(toolkit) {
+	return PLANS[toolkit.trim().toLowerCase()];
+}
+function listConnectorLearningPlans() {
+	return Object.values(PLANS);
+}
+//#endregion
+export { buildConnectorLearningArguments, getConnectorLearningPlan, listConnectorLearningPlans };

package/dist/src/connectors/runtime-status.d.ts

@@ -1,9 +1,2 @@
 import type { ConnectorConnection, ConnectorInstance } from './types.js';
-export type ComposioRuntimeProbe = {
-    connectorId: string;
-    checkedAt: string;
-    toolCount?: number;
-    error?: string;
-    connectionError?: boolean;
-};
-export declare function projectComposioRuntimeStatus(instances: ConnectorInstance[], connections: ConnectorConnection[], probes: ComposioRuntimeProbe[]): ConnectorInstance[];
+export declare function projectComposioConnectionStatus(instances: ConnectorInstance[], connections: ConnectorConnection[]): ConnectorInstance[];

package/dist/src/connectors/runtime-status.js

@@ -1,43 +1,18 @@
 //#region src/connectors/runtime-status.ts
-function projectComposioRuntimeStatus(instances, connections, probes) {
-	const probeByConnectorId = new Map(probes.map((probe) => [probe.connectorId, probe]));
+function projectComposioConnectionStatus(instances, connections) {
 	return instances.map((instance) => {
 		if (instance.materialized.type !== "composio" || instance.materialized.role !== "toolkit" || !instance.enabled) return instance;
 		const relevant = connections.filter((connection) => connection.provider === "composio" && connection.connectorId === instance.connectorId && connection.status !== "revoked");
 		const active = relevant.filter((connection) => connection.status === "active");
-		const probe = probeByConnectorId.get(instance.connectorId);
 		const lastConnectedAt = active.flatMap((connection) => connection.connectedAt ? [connection.connectedAt] : []).sort().at(-1) ?? instance.lastConnectedAt;
-		if (active.length > 0 && probe && !probe.error && (probe.toolCount ?? 0) > 0) return {
+		if (active.length > 0) return {
 			...instance,
 			status: "connected",
 			connectionStatus: "connected",
 			authStatus: "connected",
 			lastConnectedAt,
-			lastError: void 0,
-			usage: {
-				...instance.usage,
-				lastHealthCheckAt: probe.checkedAt,
-				lastHealthStatus: "ok",
-				lastToolCount: probe.toolCount
-			}
+			lastError: void 0
 		};
-		if (active.length > 0 && probe) {
-			const error = probe.error ?? "The connection did not expose any usable tools.";
-			return {
-				...instance,
-				status: "degraded",
-				connectionStatus: probe.connectionError ? "error" : "connected",
-				authStatus: "connected",
-				lastConnectedAt,
-				lastError: error,
-				usage: {
-					...instance.usage,
-					lastHealthCheckAt: probe.checkedAt,
-					lastHealthStatus: probe.connectionError ? "network_failed" : "tools_list_failed",
-					lastToolCount: probe.toolCount ?? 0
-				}
-			};
-		}
 		if (relevant.some((connection) => connection.status === "pending")) return {
 			...instance,
 			status: "connecting",
@@ -65,4 +40,4 @@ function projectComposioRuntimeStatus(instances, connections, probes) {
 	});
 }
 //#endregion
-export { projectComposioRuntimeStatus };
+export { projectComposioConnectionStatus };

package/dist/src/gateway/hono/lib/config-payload.d.ts

@@ -307,6 +307,29 @@ export declare function buildSafeWebConfigPayload(service: GatewayService, optio
             maxEntries: number;
         };
     };
+    userContext: {
+        memory: {
+            retention: {
+                compaction: {
+                    enabled: boolean;
+                    triggerThreshold: number;
+                    reserveTokens: number;
+                    minMessagesBeforeCompact: number;
+                    keepRecentTokens: number;
+                    recentTurnsPreserve: number;
+                    summaryMaxTokens: number;
+                    summaryChunkTokens: number;
+                    summaryTimeoutMs: number;
+                    summaryRetries: number;
+                    qualityGuard: boolean;
+                    minToolResultKeepChars: number;
+                    maxActiveTranscriptBytes: number;
+                    postCompactionSections: string[];
+                    model?: string;
+                };
+            };
+        };
+    };
     tui: {
         defaultAgent: string;
     };

package/dist/src/gateway/hono/lib/config-payload.js

@@ -1,3 +1,4 @@
+import { ContextCompactionPolicySchema, init_config } from "../../../user-context/config.js";
 import { init_agent_scope, listAgentEntries, normalizeAgentId, resolveDefaultAgentId } from "../../../agent/agent-scope.js";
 import { resolveModelsJsonPath } from "../../../config/paths.js";
 import { buildChannelCatalogForConfig, buildChannelCatalogFromSnapshot, getChannelSetupStatus, init_channel_catalog_service } from "../../../channels/catalog/channel-catalog-service.js";
@@ -20,6 +21,7 @@ init_credentials();
 init_models_json();
 init_providers();
 init_plugin_registry();
+init_config();
 function readModelsJsonProviderApiKey(providerId) {
 	const { config } = loadModelsJson(resolveModelsJsonPath());
 	const key = (config.providers?.[providerId])?.apiKey;
@@ -175,6 +177,7 @@ async function buildSafeWebConfigPayload(service, options = {}) {
 		browser: buildSafeBrowserConfigForWeb(config.browser),
 		goals: resolveGoalsConfigForWeb(config),
 		session: resolveSessionConfigForWeb(config),
+		userContext: { memory: { retention: { compaction: ContextCompactionPolicySchema.parse(config.userContext?.memory.retention?.compaction) } } },
 		tui: { defaultAgent: config.tui?.defaultAgent ?? "coder" },
 		tunnel: {
 			enabled: config.tunnel?.enabled === true,

package/dist/src/gateway/hono/routes/config-patch/misc.d.ts

@@ -2,7 +2,7 @@
  * `PATCH /api/config` — tail sections that are each <50 lines:
  *   update / goals / session / gateway.{skillsMarketplaceProvider,
  *   skillsStoreBaseUrl} / providers / providersConfig / stt / tts / tools /
- *   tunnel / bindings / mcp.
+ *   userContext.memory.retention.compaction / tunnel / bindings / mcp.
  *
  * Most of these delegate to a `mergeXxxConfigPatch(config, body) → { ok,
  * message? }` helper that lives next to the schema, so this file is mostly

package/dist/src/gateway/hono/routes/config-patch/misc.js

@@ -1,3 +1,4 @@
+import { ContextCompactionPolicySchema, init_config } from "../../../../user-context/config.js";
 import { BindingsConfigSchema, BrowserConfigSchema, McpConfigSchema, VoiceConfigSchema, init_schema } from "../../../../config/schema.js";
 import { CredentialResolver, init_credentials } from "../../../../auth/credentials.js";
 import { canonicalizeConfiguredMcpServer } from "../../../../config/mcp-config-normalize.js";
@@ -13,6 +14,7 @@ import { PATCH_OK, patchError } from "./result.js";
 //#region src/gateway/hono/routes/config-patch/misc.ts
 init_schema();
 init_credentials();
+init_config();
 async function applyMiscPatch(config, body) {
 	if (body.update !== void 0 && typeof body.update === "object" && body.update !== null) {
 		const updateResult = mergeUpdateConfigPatch(config, body.update);
@@ -28,6 +30,32 @@ async function applyMiscPatch(config, body) {
 		const sessionResult = mergeSessionConfigPatch(config, body.session);
 		if (sessionResult.ok === false) return patchError(sessionResult.message);
 	}
+	if (body.userContext !== void 0) {
+		if (typeof body.userContext !== "object" || body.userContext === null || Array.isArray(body.userContext)) return patchError("userContext must be an object");
+		const userContextPatch = body.userContext;
+		if (userContextPatch.memory !== void 0) {
+			if (typeof userContextPatch.memory !== "object" || userContextPatch.memory === null || Array.isArray(userContextPatch.memory)) return patchError("userContext.memory must be an object");
+			const memoryPatch = userContextPatch.memory;
+			if (memoryPatch.retention !== void 0) {
+				if (typeof memoryPatch.retention !== "object" || memoryPatch.retention === null || Array.isArray(memoryPatch.retention)) return patchError("userContext.memory.retention must be an object");
+				const retentionPatch = memoryPatch.retention;
+				if (retentionPatch.compaction !== void 0) {
+					const parsed = ContextCompactionPolicySchema.safeParse(retentionPatch.compaction);
+					if (!parsed.success) return patchError(parsed.error.issues.map((issue) => issue.message).join("; "));
+					config.userContext = {
+						...config.userContext,
+						memory: {
+							...config.userContext?.memory,
+							retention: {
+								...config.userContext?.memory?.retention,
+								compaction: parsed.data
+							}
+						}
+					};
+				}
+			}
+		}
+	}
 	if (body.tui !== void 0) {
 		if (typeof body.tui !== "object" || body.tui === null || Array.isArray(body.tui)) return patchError("tui must be an object");
 		const tuiPatch = body.tui;