@xopcai/xopc 0.0.217 → 0.0.219

package/dist/src/config/models-json.d.ts

@@ -249,13 +249,13 @@ export declare const CustomModelSchema: z.ZodObject<{
     api: z.ZodOptional<z.ZodEnum<{
         "openai-completions": "openai-completions";
         "openai-responses": "openai-responses";
-        "anthropic-messages": "anthropic-messages";
-        "google-generative-ai": "google-generative-ai";
         "azure-openai-responses": "azure-openai-responses";
-        "bedrock-converse-stream": "bedrock-converse-stream";
         "openai-codex-responses": "openai-codex-responses";
-        "google-gemini-cli": "google-gemini-cli";
+        "anthropic-messages": "anthropic-messages";
+        "bedrock-converse-stream": "bedrock-converse-stream";
+        "google-generative-ai": "google-generative-ai";
         "google-vertex": "google-vertex";
+        "google-gemini-cli": "google-gemini-cli";
     }>>;
     reasoning: z.ZodOptional<z.ZodBoolean>;
     input: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -351,13 +351,13 @@ export declare const ProviderConfigSchema: z.ZodObject<{
     api: z.ZodOptional<z.ZodEnum<{
         "openai-completions": "openai-completions";
         "openai-responses": "openai-responses";
-        "anthropic-messages": "anthropic-messages";
-        "google-generative-ai": "google-generative-ai";
         "azure-openai-responses": "azure-openai-responses";
-        "bedrock-converse-stream": "bedrock-converse-stream";
         "openai-codex-responses": "openai-codex-responses";
-        "google-gemini-cli": "google-gemini-cli";
+        "anthropic-messages": "anthropic-messages";
+        "bedrock-converse-stream": "bedrock-converse-stream";
+        "google-generative-ai": "google-generative-ai";
         "google-vertex": "google-vertex";
+        "google-gemini-cli": "google-gemini-cli";
     }>>;
     headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     authHeader: z.ZodOptional<z.ZodBoolean>;
@@ -367,13 +367,13 @@ export declare const ProviderConfigSchema: z.ZodObject<{
         api: z.ZodOptional<z.ZodEnum<{
             "openai-completions": "openai-completions";
             "openai-responses": "openai-responses";
-            "anthropic-messages": "anthropic-messages";
-            "google-generative-ai": "google-generative-ai";
             "azure-openai-responses": "azure-openai-responses";
-            "bedrock-converse-stream": "bedrock-converse-stream";
             "openai-codex-responses": "openai-codex-responses";
-            "google-gemini-cli": "google-gemini-cli";
+            "anthropic-messages": "anthropic-messages";
+            "bedrock-converse-stream": "bedrock-converse-stream";
+            "google-generative-ai": "google-generative-ai";
             "google-vertex": "google-vertex";
+            "google-gemini-cli": "google-gemini-cli";
         }>>;
         reasoning: z.ZodOptional<z.ZodBoolean>;
         input: z.ZodOptional<z.ZodArray<z.ZodEnum<{
@@ -550,13 +550,13 @@ export declare const ModelsJsonSchema: z.ZodObject<{
         api: z.ZodOptional<z.ZodEnum<{
             "openai-completions": "openai-completions";
             "openai-responses": "openai-responses";
-            "anthropic-messages": "anthropic-messages";
-            "google-generative-ai": "google-generative-ai";
             "azure-openai-responses": "azure-openai-responses";
-            "bedrock-converse-stream": "bedrock-converse-stream";
             "openai-codex-responses": "openai-codex-responses";
-            "google-gemini-cli": "google-gemini-cli";
+            "anthropic-messages": "anthropic-messages";
+            "bedrock-converse-stream": "bedrock-converse-stream";
+            "google-generative-ai": "google-generative-ai";
             "google-vertex": "google-vertex";
+            "google-gemini-cli": "google-gemini-cli";
         }>>;
         headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         authHeader: z.ZodOptional<z.ZodBoolean>;
@@ -566,13 +566,13 @@ export declare const ModelsJsonSchema: z.ZodObject<{
             api: z.ZodOptional<z.ZodEnum<{
                 "openai-completions": "openai-completions";
                 "openai-responses": "openai-responses";
-                "anthropic-messages": "anthropic-messages";
-                "google-generative-ai": "google-generative-ai";
                 "azure-openai-responses": "azure-openai-responses";
-                "bedrock-converse-stream": "bedrock-converse-stream";
                 "openai-codex-responses": "openai-codex-responses";
-                "google-gemini-cli": "google-gemini-cli";
+                "anthropic-messages": "anthropic-messages";
+                "bedrock-converse-stream": "bedrock-converse-stream";
+                "google-generative-ai": "google-generative-ai";
                 "google-vertex": "google-vertex";
+                "google-gemini-cli": "google-gemini-cli";
             }>>;
             reasoning: z.ZodOptional<z.ZodBoolean>;
             input: z.ZodOptional<z.ZodArray<z.ZodEnum<{

package/dist/src/connectors/connected-source-ingestion.js

@@ -31,6 +31,9 @@ function asRecord(value) {
 function boundedPersonValue(value, maxLength = 320) {
 	return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
 }
+function normalizedEmail(value) {
+	return /<([^<>\s]+@[^<>\s]+)>/.exec(value)?.[1]?.toLowerCase() ?? value.toLowerCase();
+}
 function personEntities(record) {
 	if (!record) return [];
 	const extractPerson = (value, role) => {
@@ -39,7 +42,7 @@ function personEntities(record) {
 			if (!normalized) return [];
 			return normalized.includes("@") ? [{
 				role,
-				email: normalized
+				email: normalizedEmail(normalized)
 			}] : [{
 				role,
 				name: normalized
@@ -91,6 +94,20 @@ function peopleSignals(entities) {
 function connectionIdentities(identity) {
 	return [...new Set(Object.values(identity).filter((value) => typeof value === "string").map((value) => value.trim().toLowerCase()).filter(Boolean))];
 }
+function hasOwnerAttribution(toolkit, people, ownerIdentities) {
+	const attributableRoles = toolkit === "gmail" ? new Set(["sender", "from"]) : toolkit === "googlecalendar" ? new Set(["organizer"]) : new Set([
+		"author",
+		"user",
+		"username",
+		"owner",
+		"owners"
+	]);
+	return people.filter((person) => attributableRoles.has(person.role)).flatMap((person) => [
+		person.email,
+		person.username,
+		person.name
+	]).filter((value) => Boolean(value)).some((value) => ownerIdentities.includes(value.trim().toLowerCase()));
+}
 function resultPayload(value) {
 	const root = asRecord(value) ?? {};
 	return asRecord(root.data) ?? root;
@@ -119,10 +136,10 @@ function resultToSourceItems(input) {
 		const record = entity.value;
 		const people = personEntities(record);
 		const ownerIdentities = connectionIdentities(input.connectionIdentity);
-		const observedIdentities = peopleSignals(people).map((value) => value.toLowerCase());
-		const actorAttributed = entity.metadata.actorAttributed === true || input.toolkit === "gmail" || input.toolkit === "googlecalendar" || ["github", "googledrive"].includes(input.toolkit) && observedIdentities.some((value) => ownerIdentities.includes(value));
+		const actorAttributed = entity.metadata.actorAttributed === true || hasOwnerAttribution(input.toolkit, people, ownerIdentities);
 		const serialized = JSON.stringify(sanitizeConnectedSourceValue(entity.value), null, 2);
-		const normalizedText = serialized.length > 8e3 ? `${serialized.slice(0, 8e3)}\n…` : serialized;
+		const contentChars = typeof entity.value.content === "string" ? 24e3 : 8e3;
+		const normalizedText = serialized.length > contentChars ? `${serialized.slice(0, contentChars)}\n…` : serialized;
 		const contentHash = createHash("sha256").update(serialized).digest("hex");
 		return {
 			sourceInstanceId: input.sourceInstanceId,

package/dist/src/connectors/connected-source-normalizers.js

@@ -32,11 +32,57 @@ function rows(data, ...keys) {
 function compact(value) {
 	return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== void 0 && item !== null && item !== ""));
 }
+const MAX_EMAIL_CONTENT_CHARS = 24e3;
+function gmailHeaders(message) {
+	const messagePayload = record(message.payload);
+	const values = Array.isArray(messagePayload?.headers) ? messagePayload.headers : [];
+	const headers = /* @__PURE__ */ new Map();
+	for (const value of values) {
+		const header = record(value);
+		const name = text(header, "name")?.toLowerCase();
+		const headerValue = text(header, "value");
+		if (name && headerValue) headers.set(name, headerValue);
+	}
+	return headers;
+}
+function decodeGmailBody(value) {
+	if (typeof value !== "string" || !value.trim() || value.length > 256e3) return void 0;
+	try {
+		return Buffer.from(value.replace(/-/g, "+").replace(/_/g, "/"), "base64").toString("utf8").trim() || void 0;
+	} catch {
+		return;
+	}
+}
+function stripHtml(value) {
+	return value.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ").replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">").replace(/\s+/g, " ").trim();
+}
+function gmailContent(message) {
+	const direct = text(message, "plainText", "plain_text", "text", "content", "body");
+	if (direct) return direct.slice(0, MAX_EMAIL_CONTENT_CHARS);
+	const plain = [];
+	const html = [];
+	const visit = (value, depth = 0) => {
+		if (depth > 10 || plain.join("").length >= MAX_EMAIL_CONTENT_CHARS) return;
+		const part = record(value);
+		if (!part) return;
+		const mimeType = text(part, "mimeType", "mime_type")?.toLowerCase();
+		const decoded = decodeGmailBody(record(part.body)?.data);
+		if (decoded) {
+			if (mimeType === "text/html") html.push(stripHtml(decoded));
+			else if (!mimeType || mimeType === "text/plain") plain.push(decoded);
+		}
+		if (Array.isArray(part.parts)) for (const nested of part.parts) visit(nested, depth + 1);
+	};
+	visit(message.payload);
+	const content = (plain.length ? plain : html).filter(Boolean).join("\n\n").trim();
+	return content ? content.slice(0, MAX_EMAIL_CONTENT_CHARS) : void 0;
+}
 function normalizeGmail(result) {
 	return rows(payload(result), "messages").flatMap((message) => {
 		const externalId = text(message, "id", "messageId", "message_id");
 		if (!externalId) return [];
 		const occurredAt = time(message.internalDate ?? message.internal_date ?? message.date ?? message.timestamp);
+		const headers = gmailHeaders(message);
 		return [{
 			externalId,
 			itemType: "email",
@@ -45,12 +91,13 @@ function normalizeGmail(result) {
 			value: compact({
 				id: externalId,
 				threadId: text(message, "threadId", "thread_id"),
-				subject: text(message, "subject"),
-				sender: message.sender ?? message.from,
-				recipients: message.recipients ?? message.to,
+				subject: text(message, "subject") ?? headers.get("subject"),
+				sender: message.sender ?? message.from ?? headers.get("from"),
+				recipients: message.recipients ?? message.to ?? headers.get("to"),
 				date: occurredAt,
 				labels: message.labelIds ?? message.label_ids ?? message.labels,
-				snippet: text(message, "snippet", "preview")
+				snippet: text(message, "snippet", "preview"),
+				content: gmailContent(message)
 			}),
 			metadata: {
 				observationKind: "message",

package/dist/src/connectors/connected-source-understanding.d.ts

@@ -0,0 +1,20 @@
+import type { MemoryManager } from '../agent/memory/manager.js';
+import type { Config } from '../config/schema.js';
+import type { KnowledgeSourceItem } from '../knowledge/types.js';
+import type { UnderstandingSourceItem } from '../user-context/sources/types.js';
+import { analyzeUnderstandingSources } from '../work-discovery/analyzer.js';
+export declare function connectedItemsForUnderstanding(items: KnowledgeSourceItem[]): UnderstandingSourceItem[];
+export declare function deriveConnectedSourceUnderstanding(input: {
+    config: Config;
+    agentId: string;
+    sourceInstanceId: string;
+    sourceRunId: string;
+    processingPolicy: 'local_only' | 'remote_allowed';
+    memoryManager: MemoryManager;
+    analyze?: typeof analyzeUnderstandingSources;
+}): Promise<{
+    created: number;
+    focusCount: number;
+    status: 'completed' | 'partial' | 'failed';
+    error?: string;
+}>;

package/dist/src/connectors/connected-source-understanding.js

@@ -0,0 +1,180 @@
+import { finishContextExtractionRun } from "../storage/sqlite/context-extraction-repository.js";
+import { listKnowledgeSourceItems } from "../storage/sqlite/knowledge-repository.js";
+import "../storage/sqlite/index.js";
+import { upsertUserFocus } from "../user-context/sources/repository.js";
+import { claimRegisteredExtraction } from "../user-context/extraction/registry.js";
+import { allowsRemoteSourceProcessing } from "../user-context/sources/processing-policy.js";
+import { analyzeUnderstandingSources } from "../work-discovery/analyzer.js";
+import { createHash } from "node:crypto";
+//#region src/connectors/connected-source-understanding.ts
+const MAX_CONNECTED_ITEMS = 150;
+function normalizedValue(item) {
+	if (!item.normalizedText) return {};
+	try {
+		const value = JSON.parse(item.normalizedText);
+		return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+	} catch {
+		return {};
+	}
+}
+function text(value) {
+	return typeof value === "string" && value.trim() ? value.trim() : void 0;
+}
+function itemType(itemType) {
+	if (itemType === "email") return "mail";
+	if (itemType === "calendar_event") return "calendar_event";
+	if (itemType === "external_task") return "task";
+	if (itemType === "development_activity" || itemType === "repository") return "code_activity";
+	return "document";
+}
+function timestamp(value) {
+	if (!value) return void 0;
+	const parsed = Date.parse(value);
+	return Number.isFinite(parsed) ? parsed : void 0;
+}
+function contentPriority(item) {
+	const value = normalizedValue(item);
+	if (item.itemType === "connected_content") return 3;
+	if (text(value.content)) return 2;
+	return 1;
+}
+function connectedItemsForUnderstanding(items) {
+	return [...items].sort((left, right) => contentPriority(right) - contentPriority(left)).slice(0, MAX_CONNECTED_ITEMS).flatMap((item) => {
+		const value = normalizedValue(item);
+		const title = text(value.title) ?? text(value.subject) ?? text(value.fullName) ?? text(value.repository) ?? `${text(item.metadata.toolkit) ?? "Connected source"} ${item.itemType}`;
+		const normalizedText = item.normalizedText?.trim();
+		if (!title && !normalizedText) return [];
+		return [{
+			id: item.id,
+			sourceId: "connected-work",
+			type: itemType(item.itemType),
+			title,
+			...normalizedText ? { text: normalizedText.slice(0, 24e3) } : {},
+			...text(item.metadata.toolkit) ? { group: text(item.metadata.toolkit) } : {},
+			...timestamp(item.occurredAt) ? { occurredAt: timestamp(item.occurredAt) } : {},
+			...timestamp(item.sourceUpdatedAt) ? { modifiedAt: timestamp(item.sourceUpdatedAt) } : {},
+			ownerAttribution: item.metadata.actorAttributed === true ? "user" : "shared",
+			sensitivity: item.sensitivity,
+			evidenceRef: `knowledge-source://${item.id}`
+		}];
+	});
+}
+function understandingKind(category) {
+	if (category === "preference") return "preference";
+	if (category === "workflow") return "routine";
+	if (category === "focus") return "current_state";
+	return "project_context";
+}
+function understandingCandidate(candidate) {
+	const normalized = candidate.statement.toLocaleLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+	return {
+		kind: understandingKind(candidate.category),
+		content: candidate.statement,
+		canonicalKey: `connected-semantic:${candidate.category}:${createHash("sha256").update(normalized).digest("hex").slice(0, 20)}`,
+		confidence: candidate.confidence === "high" ? .9 : candidate.confidence === "medium" ? .72 : .55,
+		importance: candidate.category === "focus" ? .8 : .68,
+		explicitness: "inferred",
+		durability: candidate.category === "focus" ? "ephemeral" : candidate.category === "workflow" ? "recurring" : "durable",
+		sensitivity: "personal",
+		disclosurePolicy: "referenceable"
+	};
+}
+function hasLongVerbatimOverlap(value, sourceTexts, minimumLength = 32) {
+	const normalized = value.replace(/\s+/g, " ").trim();
+	if (normalized.length < minimumLength) return false;
+	return sourceTexts.some((source) => source.replace(/\s+/g, " ").includes(normalized));
+}
+async function deriveConnectedSourceUnderstanding(input) {
+	const knowledgeItems = listKnowledgeSourceItems({
+		agentId: input.agentId,
+		sourceInstanceId: input.sourceInstanceId,
+		includeDeleted: false,
+		limit: MAX_CONNECTED_ITEMS
+	});
+	const items = connectedItemsForUnderstanding(knowledgeItems);
+	if (!items.length) return {
+		created: 0,
+		focusCount: 0,
+		status: "completed"
+	};
+	const extraction = claimRegisteredExtraction({
+		extractorId: "connector-semantic",
+		sourceRef: `understanding-source-run:${input.sourceRunId}`,
+		contentForHash: knowledgeItems.map((item) => `${item.id}:${item.sourceUpdatedAt ?? ""}`).join("\n"),
+		processingPolicy: input.processingPolicy,
+		destination: "remote_model"
+	});
+	if (!allowsRemoteSourceProcessing([input.processingPolicy]) || !extraction.shouldExecute) return {
+		created: 0,
+		focusCount: 0,
+		status: "completed"
+	};
+	try {
+		const analysis = await (input.analyze ?? analyzeUnderstandingSources)({
+			config: input.config,
+			items
+		});
+		const rawTexts = items.flatMap((item) => item.text ? [item.text] : []);
+		const profileCandidates = analysis.profileCandidates.filter((candidate) => !hasLongVerbatimOverlap(candidate.statement, rawTexts));
+		const sourceItemIds = [...new Set([...profileCandidates.flatMap((candidate) => candidate.evidenceRefs ?? []), ...analysis.workThreadCandidates.flatMap((candidate) => candidate.evidenceRefs)].map((ref) => ref.startsWith("knowledge-source://") ? ref.slice(19) : ""))].filter(Boolean).slice(0, 20);
+		const reviewed = profileCandidates.length ? await input.memoryManager.applyUnderstandingCandidates(profileCandidates.map(understandingCandidate), {
+			agentId: input.agentId,
+			sourceItemIds,
+			sourceText: "Bounded semantic synthesis from explicitly connected work sources.",
+			source: {
+				provider: "connected-sources",
+				sourceInstanceId: input.sourceInstanceId
+			},
+			reviewSource: "background",
+			extractionRunId: extraction.run.id
+		}) : {
+			created: 0,
+			writeOutputs: []
+		};
+		const focuses = analysis.workThreadCandidates.filter((candidate) => !hasLongVerbatimOverlap(candidate.title, rawTexts) && !hasLongVerbatimOverlap(candidate.summary, rawTexts)).map((candidate) => upsertUserFocus({
+			canonicalKey: `connected-focus:${candidate.topicKey}`,
+			title: candidate.title,
+			summary: candidate.summary,
+			horizon: candidate.horizon,
+			status: "candidate",
+			confidence: candidate.confidence === "high" ? .9 : candidate.confidence === "medium" ? .72 : .55,
+			evidenceRefs: candidate.evidenceRefs,
+			sourceRunId: input.sourceRunId
+		}));
+		finishContextExtractionRun({
+			runId: extraction.run.id,
+			status: "completed",
+			outputs: [...(reviewed.writeOutputs ?? []).map((output) => ({
+				candidateKey: output.candidateKey,
+				...output.objectId ? {
+					objectType: "understanding",
+					objectId: output.objectId
+				} : {},
+				...output.versionId ? { versionId: output.versionId } : {},
+				outcome: output.outcome
+			})), ...focuses.map((focus) => ({
+				candidateKey: focus.canonicalKey,
+				objectType: "focus",
+				objectId: focus.id,
+				versionId: focus.versionId,
+				outcome: "created"
+			}))]
+		});
+		const sourceStatus = analysis.sourceStatuses.find((item) => item.sourceId === "connected-work");
+		return {
+			created: reviewed.created,
+			focusCount: focuses.length,
+			status: sourceStatus?.status ?? "failed",
+			...sourceStatus?.error ? { error: sourceStatus.error } : {}
+		};
+	} catch (error) {
+		finishContextExtractionRun({
+			runId: extraction.run.id,
+			status: "failed",
+			errorCode: "extractor_failed"
+		});
+		throw error;
+	}
+}
+//#endregion
+export { connectedItemsForUnderstanding, deriveConnectedSourceUnderstanding };

package/dist/src/connectors/content-enrichment.d.ts

@@ -19,6 +19,7 @@ export type ConnectedContentReadResult = {
 };
 export declare function listConnectedContentCandidates(options: {
     agentId: string;
+    sourceInstanceId?: string;
     limit?: number;
 }): ConnectedContentCandidate[];
 export declare function readConnectedContent(input: {

package/dist/src/connectors/content-enrichment.js

@@ -43,13 +43,16 @@ function candidateFor(item) {
 	const value = normalizedRecord(item);
 	const title = text(value.title) ?? text(value.subject);
 	if (!title) return void 0;
-	if (toolkit === "gmail" && item.itemType === "email") return {
-		sourceItemId: item.id,
-		sourceInstanceId: item.sourceInstanceId,
-		toolkit,
-		title,
-		occurredAt: item.occurredAt
-	};
+	if (toolkit === "gmail" && item.itemType === "email") {
+		if (text(value.content)) return void 0;
+		return {
+			sourceItemId: item.id,
+			sourceInstanceId: item.sourceInstanceId,
+			toolkit,
+			title,
+			occurredAt: item.occurredAt
+		};
+	}
 	const mimeType = text(item.metadata.mimeType) ?? text(value.mimeType);
 	if (toolkit === "googledrive" && item.itemType === "cloud_document" && mimeType && (mimeType === GOOGLE_DOCUMENT_MIME || DOWNLOADABLE_TEXT_MIMES.has(mimeType))) return {
 		sourceItemId: item.id,
@@ -64,6 +67,7 @@ function listConnectedContentCandidates(options) {
 	const limit = Math.max(1, Math.min(20, options.limit ?? 10));
 	const items = listKnowledgeSourceItems({
 		agentId: options.agentId,
+		...options.sourceInstanceId ? { sourceInstanceId: options.sourceInstanceId } : {},
 		includeDeleted: false,
 		limit: 500
 	});

package/dist/src/connectors/learning-coordinator.js

@@ -1,5 +1,6 @@
 import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
+import { finishContextExtractionRun } from "../storage/sqlite/context-extraction-repository.js";
 import { claimNextConnectorLearningJob, enqueueConnectorLearningJob, listConnectorLearningJobs, recoverStaleConnectorLearningJobs, setConnectorLearningPaused, updateConnectorLearningJob } from "../storage/sqlite/connector-learning-repository.js";
 import { pruneBoundedKnowledgeSourceItems } from "../storage/sqlite/knowledge-repository.js";
 import { getConnectorInstallation, listConnectorConnections, upsertConnectorConnection } from "../storage/sqlite/connector-repository.js";
@@ -10,15 +11,19 @@ import { connectorIdentityKey, normalizeConnectorIdentity } from "./connector-id
 import { ComposioSessionsAdapter } from "./composio-sessions.js";
 import { buildConnectorLearningArguments, getConnectorLearningPlan } from "./learning-recipes.js";
 import { getConnectorDefinition } from "./catalog.js";
-import { createUnderstandingSourceRun, listUnderstandingSourceRuns, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant, upsertUserFocus } from "../user-context/sources/repository.js";
+import { createUnderstandingSourceRun, getUnderstandingSourceGrant, listUnderstandingSourceRuns, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant, upsertUserFocus } from "../user-context/sources/repository.js";
+import { claimRegisteredExtraction } from "../user-context/extraction/registry.js";
 import { ConnectedUnderstandingPipeline } from "../knowledge/connected-understanding-pipeline.js";
 import "../knowledge/index.js";
 import { ingestComposioConnectedSource } from "./connected-source-ingestion.js";
+import { deriveConnectedSourceUnderstanding } from "./connected-source-understanding.js";
+import { listConnectedContentCandidates, readConnectedContent } from "./content-enrichment.js";
 //#region src/connectors/learning-coordinator.ts
 init_logger();
 const log = createLogger("ConnectorLearning");
 const CONNECTED_ACCOUNT_UNAVAILABLE = "connected_account_unavailable";
 const CONNECTED_SOURCE_SYNC_FAILED = "connected_source_sync_failed";
+const CONNECTED_SOURCE_ANALYSIS_FAILED = "connected_source_analysis_failed";
 function learningFailureCode(error) {
 	const message = error instanceof Error ? error.message : String(error);
 	return message.includes("ToolRouterV2_InvalidConnectedAccountIds") || message.includes("Could not find connected account(s)") || message.includes("missing its provider identity") ? CONNECTED_ACCOUNT_UNAVAILABLE : CONNECTED_SOURCE_SYNC_FAILED;
@@ -178,6 +183,22 @@ function startConnectorLearningCoordinator(options) {
 			itemsSeen += synced.itemsSeen;
 			itemsIndexed += synced.itemsIndexed;
 		}
+		const enrichmentErrors = [];
+		if (plan.toolkit === "googledrive") {
+			const candidates = listConnectedContentCandidates({
+				agentId: job.agentId,
+				sourceInstanceId: job.sourceInstanceId,
+				limit: 5
+			});
+			if (candidates.length) {
+				const enriched = await readConnectedContent({
+					agentId: job.agentId,
+					sourceItemIds: candidates.map((candidate) => candidate.sourceItemId)
+				});
+				enrichmentErrors.push(...enriched.failed.map((failure) => failure.error));
+				itemsIndexed += enriched.completed;
+			}
+		}
 		if (listConnectorLearningJobs({
 			accountId: job.accountId,
 			limit: 100
@@ -188,33 +209,109 @@ function startConnectorLearningCoordinator(options) {
 			itemsIndexed
 		}));
 		publish(updateConnectorLearningJob(job.id, { phase: "deriving" }));
-		const understanding = await new ConnectedUnderstandingPipeline(options.getMemoryManager()).process(job.agentId);
-		for (const record of understanding.createdRecords.filter((item) => item.kind === "project_context")) upsertUserFocus({
-			canonicalKey: `connector-focus:${job.accountId}:${record.id}`,
-			title: record.content.slice(0, 120),
-			summary: record.content,
-			horizon: "ongoing",
-			status: "candidate",
-			confidence: .72,
-			evidenceRefs: [`connector-account:${job.accountId}`],
-			sourceRunId: sourceRun.id
+		const structuralExtraction = claimRegisteredExtraction({
+			extractorId: "connector-structural",
+			sourceRef: `understanding-source-run:${sourceRun.id}`,
+			contentForHash: `${job.sourceInstanceId}:${itemsSeen}:${itemsIndexed}`,
+			processingPolicy: "local_only",
+			destination: "deterministic"
 		});
+		let understanding = {
+			proposed: 0,
+			created: 0,
+			deduplicated: 0,
+			rejected: 0,
+			createdRecords: [],
+			writeOutputs: []
+		};
+		if (structuralExtraction.shouldExecute) try {
+			understanding = await new ConnectedUnderstandingPipeline(options.getMemoryManager()).process(job.agentId, structuralExtraction.run.id);
+			const structuralFocuses = understanding.createdRecords.filter((item) => item.kind === "project_context").map((record) => upsertUserFocus({
+				canonicalKey: `connector-focus:${job.accountId}:${record.id}`,
+				title: record.content.slice(0, 120),
+				summary: record.content,
+				horizon: "ongoing",
+				status: "candidate",
+				confidence: .72,
+				evidenceRefs: [`connector-account:${job.accountId}`],
+				sourceRunId: sourceRun.id
+			}));
+			finishContextExtractionRun({
+				runId: structuralExtraction.run.id,
+				status: "completed",
+				outputs: [...(understanding.writeOutputs ?? []).map((output) => ({
+					candidateKey: output.candidateKey,
+					...output.objectId ? {
+						objectType: "understanding",
+						objectId: output.objectId
+					} : {},
+					...output.versionId ? { versionId: output.versionId } : {},
+					outcome: output.outcome
+				})), ...structuralFocuses.map((focus) => ({
+					candidateKey: focus.canonicalKey,
+					objectType: "focus",
+					objectId: focus.id,
+					versionId: focus.versionId,
+					outcome: "created"
+				}))]
+			});
+		} catch (error) {
+			finishContextExtractionRun({
+				runId: structuralExtraction.run.id,
+				status: "failed",
+				errorCode: "extractor_failed"
+			});
+			throw error;
+		}
+		let semantic = {
+			created: 0,
+			focusCount: 0,
+			status: "completed"
+		};
+		try {
+			semantic = await deriveConnectedSourceUnderstanding({
+				config: options.getConfig(),
+				agentId: job.agentId,
+				sourceInstanceId: job.sourceInstanceId,
+				sourceRunId: sourceRun.id,
+				processingPolicy: getUnderstandingSourceGrant(sourceRun.grantId)?.processingPolicy ?? "local_only",
+				memoryManager: options.getMemoryManager()
+			});
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			semantic = {
+				created: 0,
+				focusCount: 0,
+				status: "failed",
+				error: CONNECTED_SOURCE_ANALYSIS_FAILED
+			};
+			log.warn({
+				err: error,
+				jobId: job.id,
+				connectorId: job.connectorId
+			}, `Connected source semantic analysis failed: ${errorMessage}`);
+		}
+		const candidatesCreated = understanding.created + semantic.created + semantic.focusCount;
+		const incomplete = enrichmentErrors.length > 0 || semantic.status !== "completed";
 		const completed = updateConnectorLearningJob(job.id, {
 			status: "completed",
 			phase: "completed",
-			candidatesCreated: understanding.created,
+			candidatesCreated,
 			nextRunAt: null,
 			finished: true
 		});
 		updateUnderstandingSourceRun(sourceRun.id, {
-			status: "completed",
+			status: incomplete ? "partial" : "completed",
 			itemsSeen,
 			cursorAfter: (/* @__PURE__ */ new Date()).toISOString(),
 			completed: true,
 			metadata: {
 				...sourceRun.metadata,
 				itemsIndexed,
-				candidatesCreated: understanding.created
+				candidatesCreated,
+				semanticStatus: semantic.status,
+				...semantic.error ? { semanticError: semantic.error } : {},
+				...enrichmentErrors.length ? { enrichmentFailureCount: enrichmentErrors.length } : {}
 			}
 		});
 		updateUnderstandingSourceGrantCheckpoint(sourceRun.grantId, {
@@ -258,7 +355,7 @@ function startConnectorLearningCoordinator(options) {
 					const definition = getConnectorDefinition(job.connectorId);
 					if (definition?.runtime.type === "composio" && definition.runtime.role === "toolkit") updateUnderstandingSourceRun(ensureUnderstandingSourceRun(job, definition.runtime.toolkit, definition.displayName).id, {
 						status: "failed",
-						errorMessage,
+						errorMessage: learningFailureCode(err),
 						completed: true
 					});
 					log.warn({

package/dist/src/connectors/learning-recipes.js

@@ -43,8 +43,8 @@ const PLANS = {
 			actionId: "GMAIL_FETCH_EMAILS",
 			kind: "activity",
 			arguments: {
-				max_results: 100,
-				include_payload: false,
+				max_results: 30,
+				include_payload: true,
 				verbose: false,
 				query: "newer_than:30d -in:spam -in:trash"
 			}

package/dist/src/document-understanding/extract.d.ts

@@ -1,7 +1,7 @@
 export type DocumentExtractionResult = {
     ok: true;
     text: string;
-    kind: 'plain' | 'pdf';
+    kind: 'plain' | 'pdf' | 'office';
 } | {
     ok: false;
     reason: string;