@xopcai/xopc 0.0.218 → 0.0.220

package/dist/src/user-context/consolidation.js

@@ -1,9 +1,67 @@
 import { finishContextConsolidationRun, recordContextConsolidationDecision, startContextConsolidationRun } from "../storage/sqlite/context-consolidation-repository.js";
-import { listUnderstandingEvidence, listUnderstandings, setUnderstandingStatus } from "../storage/sqlite/user-context-repository.js";
+import { createUnderstanding, linkUnderstandingEvidence, listUnderstandingEvidence, listUnderstandings, setUnderstandingStatus } from "../storage/sqlite/user-context-repository.js";
 import "../storage/sqlite/index.js";
 //#region src/user-context/consolidation.ts
 const USER_CONTEXT_CONSOLIDATION_TOKEN = "__xopc_user_context_consolidation__";
 const USER_CONTEXT_CONSOLIDATION_AUTOMATION_ID = "system-user-context-consolidation";
+const GLOBAL_CANDIDATE_KINDS = new Set(["preference", "routine"]);
+function stricterSensitivity(items) {
+	const order = [
+		"normal",
+		"personal",
+		"secret",
+		"regulated"
+	];
+	return order[Math.max(...items.map((item) => order.indexOf(item.sensitivity)))] ?? "personal";
+}
+function stricterDisclosurePolicy(items) {
+	if (items.some((item) => item.disclosurePolicy === "silent")) return "silent";
+	if (items.some((item) => item.disclosurePolicy === "ask_before_reference")) return "ask_before_reference";
+	return "referenceable";
+}
+function createCrossProjectGlobalCandidates(minProjects) {
+	const all = listUnderstandings();
+	const existingGlobalKeys = new Set(all.filter((item) => item.scope.type === "global" && item.status !== "archived" && item.status !== "rejected").map((item) => item.canonicalKey));
+	const groups = /* @__PURE__ */ new Map();
+	for (const item of all) {
+		if (item.status !== "active" || item.scope.type !== "project" || !item.scope.id || !GLOBAL_CANDIDATE_KINDS.has(item.kind)) continue;
+		const normalizedStatement = item.statement.normalize("NFKC").toLocaleLowerCase().replace(/\s+/g, " ").trim();
+		const groupKey = `${item.canonicalKey}\u0000${normalizedStatement}`;
+		const group = groups.get(groupKey) ?? [];
+		group.push(item);
+		groups.set(groupKey, group);
+	}
+	let created = 0;
+	for (const items of groups.values()) {
+		const canonicalKey = items[0].canonicalKey;
+		const projectCount = new Set(items.map((item) => item.scope.id)).size;
+		if (projectCount < Math.max(2, minProjects) || existingGlobalKeys.has(canonicalKey)) continue;
+		const exemplar = [...items].sort((left, right) => right.confidence - left.confidence)[0];
+		const global = createUnderstanding({
+			kind: exemplar.kind,
+			canonicalKey,
+			status: "candidate",
+			scope: { type: "global" },
+			explicitness: "inferred",
+			durability: exemplar.durability,
+			sensitivity: stricterSensitivity(items),
+			disclosurePolicy: stricterDisclosurePolicy(items),
+			confidence: items.reduce((sum, item) => sum + item.confidence, 0) / items.length,
+			statement: exemplar.statement,
+			createdBy: "consolidation",
+			changeReason: `Repeated across ${projectCount} projects`
+		});
+		const linked = /* @__PURE__ */ new Set();
+		for (const item of items) for (const evidence of listUnderstandingEvidence(item.id, "supports")) {
+			if (linked.has(evidence.id)) continue;
+			linkUnderstandingEvidence(global.versionId, evidence.id, "supports", item.confidence);
+			linked.add(evidence.id);
+		}
+		existingGlobalKeys.add(canonicalKey);
+		created += 1;
+	}
+	return created;
+}
 function distinctEvidenceCount(understandingId, relation) {
 	return new Set(listUnderstandingEvidence(understandingId, relation).map((evidence) => `${evidence.sourceType}:${evidence.sourceInstanceId ?? ""}:${evidence.sourceRef}`)).size;
 }
@@ -29,7 +87,8 @@ async function runContextConsolidation(input) {
 	const metrics = {
 		scanned: 0,
 		needsReview: 0,
-		stale: 0
+		stale: 0,
+		globalCandidates: 0
 	};
 	try {
 		const understandings = listUnderstandings(["candidate", "active"]).slice(0, resolved.limit);
@@ -89,6 +148,7 @@ async function runContextConsolidation(input) {
 				metrics.needsReview += 1;
 			}
 		}
+		metrics.globalCandidates = createCrossProjectGlobalCandidates(resolved.minEvidenceSources);
 		return {
 			run: finishContextConsolidationRun({
 				runId: run.runId,

package/dist/src/user-context/context-objects.d.ts

@@ -0,0 +1,17 @@
+import type { ContextEvidence, UserContextScope } from './domain.js';
+export type ContextObjectView = 'current' | 'review' | 'history';
+export type ContextObjectOrigin = 'told_by_user' | 'observed' | 'inferred' | 'connected_source';
+export type ContextObject = {
+    objectType: 'profile' | 'rule' | 'focus' | 'understanding';
+    objectId: string;
+    status: string;
+    content: string;
+    scope: UserContextScope;
+    origin: ContextObjectOrigin;
+    confidence: number;
+    evidence: ContextEvidence[] | Array<{
+        sourceRef: string;
+    }>;
+    updatedAt: number;
+};
+export declare function listContextObjects(view: ContextObjectView): ContextObject[];

package/dist/src/user-context/context-objects.js

@@ -0,0 +1,99 @@
+import { getUserProfile, listCollaborationRules, listUnderstandingEvidence, listUnderstandings } from "../storage/sqlite/user-context-repository.js";
+import "../storage/sqlite/index.js";
+import { getUnderstandingSourceGrant, getUnderstandingSourceRun, listUserFocusEvidence, listUserFocuses } from "./sources/repository.js";
+//#region src/user-context/context-objects.ts
+function understandingOrigin(item, evidence) {
+	if (item.explicitness === "explicit") return "told_by_user";
+	if (evidence.some((entry) => entry.sourceType === "connector")) return "connected_source";
+	return item.explicitness === "observed" ? "observed" : "inferred";
+}
+function includesStatus(view, objectType, status) {
+	if (view === "current") return status === "active";
+	if (view === "review") return objectType === "understanding" ? [
+		"candidate",
+		"needs_review",
+		"stale"
+	].includes(status) : objectType === "focus" && status === "candidate";
+	return objectType === "understanding" ? ["archived", "rejected"].includes(status) : objectType === "focus" ? [
+		"paused",
+		"completed",
+		"rejected"
+	].includes(status) : objectType === "rule" && ["disabled", "archived"].includes(status);
+}
+function listContextObjects(view) {
+	const objects = [];
+	if (view === "current") {
+		const profile = getUserProfile();
+		const fields = [
+			["callName", profile.callName],
+			["role", profile.role],
+			["primaryGoal", profile.primaryGoal],
+			["pronouns", profile.pronouns],
+			["timezone", profile.timezone],
+			["locale", profile.locale]
+		];
+		for (const [field, value] of fields) {
+			if (!value) continue;
+			objects.push({
+				objectType: "profile",
+				objectId: field,
+				status: "active",
+				content: value,
+				scope: { type: "global" },
+				origin: "told_by_user",
+				confidence: 1,
+				evidence: [{ sourceRef: `user-profile://${field}` }],
+				updatedAt: profile.updatedAt
+			});
+		}
+	}
+	for (const rule of listCollaborationRules()) {
+		if (!includesStatus(view, "rule", rule.status)) continue;
+		objects.push({
+			objectType: "rule",
+			objectId: rule.id,
+			status: rule.status,
+			content: rule.statement,
+			scope: rule.scope,
+			origin: "told_by_user",
+			confidence: 1,
+			evidence: [{ sourceRef: `collaboration-rule://${rule.revisionId}` }],
+			updatedAt: rule.updatedAt
+		});
+	}
+	for (const focus of listUserFocuses()) {
+		if (!includesStatus(view, "focus", focus.status)) continue;
+		const sourceRun = focus.sourceRunId ? getUnderstandingSourceRun(focus.sourceRunId) : null;
+		const sourceGrant = sourceRun ? getUnderstandingSourceGrant(sourceRun.grantId) : null;
+		const linkedEvidence = listUserFocusEvidence(focus.id);
+		objects.push({
+			objectType: "focus",
+			objectId: focus.id,
+			status: focus.status,
+			content: `${focus.title}: ${focus.summary}`,
+			scope: focus.scope,
+			origin: focus.explicitness === "explicit" ? "told_by_user" : sourceGrant?.adapterId.startsWith("connector:") ? "connected_source" : focus.explicitness === "observed" ? "observed" : "inferred",
+			confidence: focus.confidence,
+			evidence: linkedEvidence.length ? linkedEvidence : focus.evidenceRefs.map((sourceRef) => ({ sourceRef })),
+			updatedAt: focus.updatedAt
+		});
+	}
+	for (const understanding of listUnderstandings()) {
+		if (!includesStatus(view, "understanding", understanding.status)) continue;
+		const evidence = listUnderstandingEvidence(understanding.id);
+		objects.push({
+			objectType: "understanding",
+			objectId: understanding.id,
+			status: understanding.status,
+			content: understanding.statement,
+			scope: understanding.scope,
+			origin: understandingOrigin(understanding, evidence),
+			confidence: understanding.confidence,
+			evidence,
+			updatedAt: understanding.updatedAt
+		});
+	}
+	return objects.sort((left, right) => right.updatedAt - left.updatedAt);
+}
+//#endregion
+export { listContextObjects };

package/dist/src/user-context/domain.d.ts

@@ -53,13 +53,14 @@ export type CollaborationRule = {
     updatedAt: number;
 };
 export type PersonalizationItem = {
-    objectType: 'profile' | 'rule' | 'understanding';
+    objectType: 'profile' | 'rule' | 'focus' | 'understanding';
     objectId: string;
     versionId?: string;
     decision: 'selected' | 'irrelevant' | 'expired' | 'scope_mismatch' | 'sensitive' | 'needs_consent' | 'budget_exceeded' | 'conflicted' | 'disabled';
     reason: string;
     content: string;
     sourceLabel: string;
+    origin: 'told_by_user' | 'observed' | 'inferred' | 'connected_source';
     rank?: number;
     score?: number;
     injectedChars: number;
@@ -69,8 +70,19 @@ export type ContextEvidence = {
     sourceType: 'conversation' | 'connector' | 'user' | 'runtime';
     sourceInstanceId?: string;
     sourceRef: string;
+    sourceRunId?: string;
+    sourceItemId?: string;
+    sessionId?: string;
+    turnId?: string;
+    messageId?: string;
+    contentHash?: string;
+    retentionPolicy?: 'metadata_only' | 'derived_only' | 'bounded_raw';
+    processingPolicy?: 'local_only' | 'remote_allowed';
+    extractorId?: string;
+    extractorVersion?: string;
     redactedExcerpt?: string;
     trustLevel: 'owner' | 'trusted' | 'untrusted';
     observedAt: number;
+    ingestedAt: number;
     createdAt: number;
 };

package/dist/src/user-context/evaluation/harness.d.ts

@@ -0,0 +1,13 @@
+export type RetrievalEvaluationCase = {
+    id: string;
+    expectedIds: string[];
+    shouldAbstain?: boolean;
+    retrieve: () => string[];
+};
+export type RetrievalEvaluationMetrics = {
+    cases: number;
+    recallAtK: number;
+    precisionAtK: number;
+    abstentionAccuracy: number;
+};
+export declare function evaluateRetrievalCases(cases: RetrievalEvaluationCase[], k?: number): RetrievalEvaluationMetrics;

package/dist/src/user-context/evaluation/harness.js

@@ -0,0 +1,31 @@
+//#region src/user-context/evaluation/harness.ts
+function evaluateRetrievalCases(cases, k = 3) {
+	let expected = 0;
+	let recalled = 0;
+	let returned = 0;
+	let relevantReturned = 0;
+	let abstentionCases = 0;
+	let correctAbstentions = 0;
+	for (const item of cases) {
+		const results = [...new Set(item.retrieve())].slice(0, Math.max(1, k));
+		const expectedIds = new Set(item.expectedIds);
+		if (item.shouldAbstain) {
+			abstentionCases += 1;
+			if (results.length === 0) correctAbstentions += 1;
+			continue;
+		}
+		expected += expectedIds.size;
+		returned += results.length;
+		const matches = results.filter((id) => expectedIds.has(id)).length;
+		recalled += matches;
+		relevantReturned += matches;
+	}
+	return {
+		cases: cases.length,
+		recallAtK: expected ? recalled / expected : 1,
+		precisionAtK: returned ? relevantReturned / returned : 1,
+		abstentionAccuracy: abstentionCases ? correctAbstentions / abstentionCases : 1
+	};
+}
+//#endregion
+export { evaluateRetrievalCases };

package/dist/src/user-context/extraction/registry.d.ts

@@ -0,0 +1,26 @@
+import { type ContextExtractionRun } from '../../storage/sqlite/index.js';
+import type { UnderstandingKind } from '../domain.js';
+export type ExtractorId = 'explicit-command' | 'deterministic-signal' | 'transcript-synthesis' | 'connector-structural' | 'connector-semantic';
+export type ExtractorDefinition = {
+    id: ExtractorId;
+    version: string;
+    inputKinds: Array<'conversation' | 'connector' | 'file' | 'task'>;
+    requiredProcessingPolicy: 'local_only' | 'remote_allowed';
+    authorityCeiling: 'user_explicit' | 'user_observed' | 'system_inferred' | 'external_untrusted';
+    candidateKinds: UnderstandingKind[];
+    maxAutomaticStatus: 'active' | 'candidate';
+    timeoutMs: number;
+};
+export declare function getExtractorDefinition(id: ExtractorId): ExtractorDefinition;
+export declare function getRegisteredExtractorDefinition(id: string): ExtractorDefinition | undefined;
+export declare function extractionInputHash(input: string): string;
+export declare function claimRegisteredExtraction(input: {
+    extractorId: ExtractorId;
+    sourceRef: string;
+    contentForHash: string;
+    processingPolicy: 'local_only' | 'remote_allowed';
+    destination: ContextExtractionRun['destination'];
+}): {
+    run: ContextExtractionRun;
+    shouldExecute: boolean;
+};

package/dist/src/user-context/extraction/registry.js

@@ -0,0 +1,113 @@
+import { claimContextExtractionRun, finishContextExtractionRun } from "../../storage/sqlite/context-extraction-repository.js";
+import "../../storage/sqlite/index.js";
+import { createHash } from "node:crypto";
+//#region src/user-context/extraction/registry.ts
+const UNDERSTANDING_KINDS = [
+	"preference",
+	"boundary",
+	"relationship",
+	"routine",
+	"current_state",
+	"long_term_goal",
+	"project_context",
+	"task_lesson",
+	"derived_insight"
+];
+const DEFINITIONS = {
+	"explicit-command": {
+		id: "explicit-command",
+		version: "1",
+		inputKinds: ["conversation"],
+		requiredProcessingPolicy: "local_only",
+		authorityCeiling: "user_explicit",
+		candidateKinds: UNDERSTANDING_KINDS,
+		maxAutomaticStatus: "active",
+		timeoutMs: 500
+	},
+	"deterministic-signal": {
+		id: "deterministic-signal",
+		version: "1",
+		inputKinds: ["conversation", "task"],
+		requiredProcessingPolicy: "local_only",
+		authorityCeiling: "user_observed",
+		candidateKinds: UNDERSTANDING_KINDS,
+		maxAutomaticStatus: "candidate",
+		timeoutMs: 500
+	},
+	"transcript-synthesis": {
+		id: "transcript-synthesis",
+		version: "1",
+		inputKinds: ["conversation"],
+		requiredProcessingPolicy: "remote_allowed",
+		authorityCeiling: "system_inferred",
+		candidateKinds: UNDERSTANDING_KINDS,
+		maxAutomaticStatus: "candidate",
+		timeoutMs: 3e4
+	},
+	"connector-structural": {
+		id: "connector-structural",
+		version: "1",
+		inputKinds: ["connector"],
+		requiredProcessingPolicy: "local_only",
+		authorityCeiling: "external_untrusted",
+		candidateKinds: [
+			"project_context",
+			"current_state",
+			"routine"
+		],
+		maxAutomaticStatus: "candidate",
+		timeoutMs: 1e4
+	},
+	"connector-semantic": {
+		id: "connector-semantic",
+		version: "1",
+		inputKinds: ["connector", "file"],
+		requiredProcessingPolicy: "remote_allowed",
+		authorityCeiling: "external_untrusted",
+		candidateKinds: [
+			"preference",
+			"routine",
+			"current_state",
+			"project_context"
+		],
+		maxAutomaticStatus: "candidate",
+		timeoutMs: 3e4
+	}
+};
+function getExtractorDefinition(id) {
+	return DEFINITIONS[id];
+}
+function getRegisteredExtractorDefinition(id) {
+	return DEFINITIONS[id];
+}
+function extractionInputHash(input) {
+	return createHash("sha256").update(input).digest("hex");
+}
+function claimRegisteredExtraction(input) {
+	const definition = getExtractorDefinition(input.extractorId);
+	const claimed = claimContextExtractionRun({
+		sourceRef: input.sourceRef,
+		extractorId: definition.id,
+		extractorVersion: definition.version,
+		processingPolicy: input.processingPolicy,
+		destination: input.destination,
+		inputHash: extractionInputHash(input.contentForHash)
+	});
+	if (definition.requiredProcessingPolicy === "remote_allowed" && input.processingPolicy !== "remote_allowed") {
+		if (claimed.run.status === "running") finishContextExtractionRun({
+			runId: claimed.run.id,
+			status: "skipped",
+			errorCode: "processing_policy"
+		});
+		return {
+			run: {
+				...claimed.run,
+				status: "skipped"
+			},
+			shouldExecute: false
+		};
+	}
+	return claimed;
+}
+//#endregion
+export { claimRegisteredExtraction, extractionInputHash, getExtractorDefinition, getRegisteredExtractorDefinition };

package/dist/src/user-context/extraction/repair.d.ts

@@ -0,0 +1,19 @@
+export type ContextRepairFilter = {
+    runId?: string;
+    sourceRef?: string;
+    extractorId?: string;
+    extractorVersion?: string;
+    objectVersionId?: string;
+};
+export declare function repairExtractedContext(filter: ContextRepairFilter): {
+    runIds: string[];
+    repaired: Array<{
+        objectType: 'focus' | 'understanding';
+        objectId: string;
+    }>;
+    retained: Array<{
+        objectType: 'focus' | 'understanding';
+        objectId: string;
+        reason: string;
+    }>;
+};

package/dist/src/user-context/extraction/repair.js

@@ -0,0 +1,68 @@
+import { hasIndependentExtractionOutput, listContextExtractionOutputs, listContextExtractionRuns } from "../../storage/sqlite/context-extraction-repository.js";
+import { getUnderstanding, setUnderstandingStatus } from "../../storage/sqlite/user-context-repository.js";
+import "../../storage/sqlite/index.js";
+import { listUserFocuses, updateUserFocus } from "../sources/repository.js";
+//#region src/user-context/extraction/repair.ts
+function repairExtractedContext(filter) {
+	const runs = listContextExtractionRuns({
+		...filter.sourceRef ? { sourceRef: filter.sourceRef } : {},
+		...filter.extractorId ? { extractorId: filter.extractorId } : {},
+		...filter.extractorVersion ? { extractorVersion: filter.extractorVersion } : {},
+		limit: 500
+	}).filter((run) => !filter.runId || run.id === filter.runId);
+	const repaired = [];
+	const retained = [];
+	const seen = /* @__PURE__ */ new Set();
+	const targetRunIds = runs.map((run) => run.id);
+	for (const run of runs) for (const output of listContextExtractionOutputs(run.id)) {
+		if (output.objectType !== "focus" && output.objectType !== "understanding" || !output.objectId) continue;
+		if (filter.objectVersionId && output.versionId !== filter.objectVersionId) continue;
+		const key = `${output.objectType}:${output.objectId}`;
+		if (seen.has(key)) continue;
+		seen.add(key);
+		if (hasIndependentExtractionOutput(targetRunIds, output.objectType, output.objectId)) {
+			retained.push({
+				objectType: output.objectType,
+				objectId: output.objectId,
+				reason: "independent_extraction"
+			});
+			continue;
+		}
+		if (output.objectType === "understanding") {
+			const understanding = getUnderstanding(output.objectId);
+			if (!understanding) continue;
+			if (understanding.explicitness === "explicit") {
+				retained.push({
+					objectType: "understanding",
+					objectId: output.objectId,
+					reason: "user_explicit"
+				});
+				continue;
+			}
+			setUnderstandingStatus(output.objectId, "archived");
+		} else {
+			const focus = listUserFocuses().find((item) => item.id === output.objectId);
+			if (!focus) continue;
+			if (focus.explicitness === "explicit") {
+				retained.push({
+					objectType: "focus",
+					objectId: output.objectId,
+					reason: "user_explicit"
+				});
+				continue;
+			}
+			updateUserFocus(output.objectId, { status: "paused" });
+		}
+		repaired.push({
+			objectType: output.objectType,
+			objectId: output.objectId
+		});
+	}
+	return {
+		runIds: runs.map((run) => run.id),
+		repaired,
+		retained
+	};
+}
+//#endregion
+export { repairExtractedContext };

package/dist/src/user-context/focus-lifecycle.d.ts

@@ -0,0 +1,2 @@
+import type { UserFocus } from './sources/types.js';
+export declare function focusLifecycle(horizon: UserFocus['horizon'], now?: number): Pick<UserFocus, 'validFrom' | 'validTo' | 'reviewAt'>;

package/dist/src/user-context/focus-lifecycle.js

@@ -0,0 +1,26 @@
+//#region src/user-context/focus-lifecycle.ts
+const DAY_MS = 1440 * 60 * 1e3;
+const HORIZON_DAYS = {
+	current: {
+		review: 14,
+		valid: 30
+	},
+	ongoing: {
+		review: 30,
+		valid: 120
+	},
+	long_term: {
+		review: 90,
+		valid: 365
+	}
+};
+function focusLifecycle(horizon, now = Date.now()) {
+	const days = HORIZON_DAYS[horizon];
+	return {
+		validFrom: now,
+		reviewAt: now + days.review * DAY_MS,
+		validTo: now + days.valid * DAY_MS
+	};
+}
+//#endregion
+export { focusLifecycle };