@xopcai/xopc 0.0.315 → 0.0.317

package/dist/src/storage/sqlite/migrations/201_connector_extraction_audit.sql

@@ -0,0 +1,127 @@
+ALTER TABLE understanding_source_runs ADD COLUMN connector_learning_job_id TEXT;
+
+UPDATE understanding_source_runs
+SET connector_learning_job_id = json_extract(metadata_json, '$.connectorLearningJobId')
+WHERE json_type(metadata_json, '$.connectorLearningJobId') = 'text';
+
+CREATE TEMP TABLE duplicate_connector_source_runs AS
+WITH ranked AS (
+  SELECT
+    run_id AS duplicate_id,
+    FIRST_VALUE(run_id) OVER (
+      PARTITION BY connector_learning_job_id
+      ORDER BY
+        CASE
+          WHEN json_extract(metadata_json, '$.semanticStatus') = 'completed' THEN 0
+          WHEN status = 'completed' THEN 1
+          WHEN status = 'partial' THEN 2
+          ELSE 3
+        END,
+        COALESCE(completed_at, started_at) DESC,
+        rowid DESC
+    ) AS keep_id,
+    ROW_NUMBER() OVER (
+      PARTITION BY connector_learning_job_id
+      ORDER BY
+        CASE
+          WHEN json_extract(metadata_json, '$.semanticStatus') = 'completed' THEN 0
+          WHEN status = 'completed' THEN 1
+          WHEN status = 'partial' THEN 2
+          ELSE 3
+        END,
+        COALESCE(completed_at, started_at) DESC,
+        rowid DESC
+    ) AS rank
+  FROM understanding_source_runs
+  WHERE connector_learning_job_id IS NOT NULL
+)
+SELECT duplicate_id, keep_id FROM ranked WHERE rank > 1;
+
+UPDATE context_evidence
+SET source_run_id = (
+  SELECT keep_id FROM duplicate_connector_source_runs
+  WHERE duplicate_id = context_evidence.source_run_id
+)
+WHERE source_run_id IN (SELECT duplicate_id FROM duplicate_connector_source_runs);
+
+DELETE FROM context_extraction_runs
+WHERE source_ref IN (
+  SELECT 'understanding-source-run:' || duplicate_id FROM duplicate_connector_source_runs
+)
+AND EXISTS (
+  SELECT 1
+  FROM duplicate_connector_source_runs duplicates
+  JOIN context_extraction_runs kept
+    ON kept.source_ref = 'understanding-source-run:' || duplicates.keep_id
+    AND kept.principal_id = context_extraction_runs.principal_id
+    AND kept.extractor_id = context_extraction_runs.extractor_id
+    AND kept.extractor_version = context_extraction_runs.extractor_version
+  WHERE context_extraction_runs.source_ref = 'understanding-source-run:' || duplicates.duplicate_id
+);
+
+UPDATE context_extraction_runs
+SET source_ref = 'understanding-source-run:' || (
+  SELECT keep_id FROM duplicate_connector_source_runs
+  WHERE 'understanding-source-run:' || duplicate_id = context_extraction_runs.source_ref
+)
+WHERE source_ref IN (
+  SELECT 'understanding-source-run:' || duplicate_id FROM duplicate_connector_source_runs
+);
+
+UPDATE knowledge_items
+SET source_json = json_set(
+  source_json,
+  '$.sourceRunId',
+  (
+    SELECT keep_id FROM duplicate_connector_source_runs
+    WHERE duplicate_id = json_extract(knowledge_items.source_json, '$.sourceRunId')
+  )
+)
+WHERE json_extract(source_json, '$.sourceRunId') IN (
+  SELECT duplicate_id FROM duplicate_connector_source_runs
+);
+
+DELETE FROM understanding_source_runs
+WHERE run_id IN (SELECT duplicate_id FROM duplicate_connector_source_runs);
+
+DROP TABLE duplicate_connector_source_runs;
+
+CREATE UNIQUE INDEX idx_understanding_source_runs_connector_job
+  ON understanding_source_runs(connector_learning_job_id)
+  WHERE connector_learning_job_id IS NOT NULL;
+
+ALTER TABLE context_extraction_outputs RENAME TO context_extraction_outputs_v200;
+
+CREATE TABLE context_extraction_outputs (
+  output_id TEXT PRIMARY KEY,
+  extraction_run_id TEXT NOT NULL REFERENCES context_extraction_runs(extraction_run_id) ON DELETE CASCADE,
+  ordinal INTEGER NOT NULL,
+  candidate_key TEXT NOT NULL,
+  object_type TEXT CHECK (object_type IN ('assertion', 'rule', 'goal', 'knowledge')),
+  object_id TEXT,
+  version_id TEXT,
+  outcome TEXT NOT NULL CHECK (outcome IN ('created', 'deduplicated', 'rejected')),
+  created_at INTEGER NOT NULL,
+  UNIQUE (extraction_run_id, ordinal)
+);
+
+INSERT INTO context_extraction_outputs (
+  output_id, extraction_run_id, ordinal, candidate_key, object_type,
+  object_id, version_id, outcome, created_at
+)
+SELECT
+  output_id,
+  extraction_run_id,
+  ordinal,
+  candidate_key,
+  CASE object_type WHEN 'rule' THEN 'rule' ELSE NULL END,
+  CASE object_type WHEN 'rule' THEN object_id ELSE NULL END,
+  CASE object_type WHEN 'rule' THEN version_id ELSE NULL END,
+  outcome,
+  created_at
+FROM context_extraction_outputs_v200;
+
+DROP TABLE context_extraction_outputs_v200;
+
+CREATE INDEX idx_context_extraction_outputs_object
+  ON context_extraction_outputs(object_type, object_id);

package/dist/src/storage/sqlite/migrations/runner.d.ts

@@ -3,7 +3,7 @@ import type { ApplyMigrationsOptions } from './types.js';
 /** Baseline schema version applied from schema.sql on first open. */
 export declare const XOPC_DB_BASELINE_SCHEMA_VERSION = 165;
 /** Latest schema version this release supports (increment when adding migrations). */
-export declare const XOPC_DB_SCHEMA_VERSION = 199;
+export declare const XOPC_DB_SCHEMA_VERSION = 201;
 export declare function resolveMigrationsDir(override?: string): string;
 /**
  * Apply sequential SQL migrations from (currentVersion + 1) through targetVersion.

package/dist/src/storage/sqlite/migrations/runner.js

@@ -26,7 +26,7 @@ function applySingleMigration(db, migration, discardOldScenes = false) {
 		targetVersion: migration.targetVersion,
 		file: migration.filename
 	}, "Applying SQLite migration");
-	const requiresForeignKeyPause = discardOldScenes || migration.targetVersion === 179 || migration.targetVersion === 182 || migration.targetVersion === 188;
+	const requiresForeignKeyPause = discardOldScenes || migration.targetVersion === 179 || migration.targetVersion === 182 || migration.targetVersion === 188 || migration.targetVersion === 200 || migration.targetVersion === 201;
 	const foreignKeysEnabled = Number(db.prepare("PRAGMA foreign_keys").get()?.foreign_keys) === 1;
 	if (requiresForeignKeyPause) db.exec("PRAGMA foreign_keys = OFF");
 	let transactionStarted = false;
@@ -57,7 +57,7 @@ function applySingleMigration(db, migration, discardOldScenes = false) {
 */
 function applyPendingMigrations(db, options = {}) {
 	db.function("xopc_operation_id", () => currentOperationId() ?? null);
-	const targetVersion = options.targetVersion ?? 199;
+	const targetVersion = options.targetVersion ?? 201;
 	let currentVersion = readSchemaVersion(db);
 	if (currentVersion > targetVersion) throw new DatabaseSchemaTooNewError(currentVersion, targetVersion);
 	if (!options.migrationsDir && currentVersion < 165) throw new DatabaseSchemaTooOldError(currentVersion, 165);
@@ -95,7 +95,7 @@ function applyPendingMigrations(db, options = {}) {
 }
 /** Inspect schema version without mutating the database. */
 function inspectSchemaMigrationStatus(db, options = {}) {
-	const appVersion = options.targetVersion ?? 199;
+	const appVersion = options.targetVersion ?? 201;
 	const dbVersion = readSchemaVersion(db);
 	if (dbVersion > appVersion) return {
 		dbVersion,
@@ -137,7 +137,7 @@ var init_runner = __esmMin((() => {
 	init_errors();
 	log = createLogger("Sqlite:Migrations");
 	XOPC_DB_BASELINE_SCHEMA_VERSION = 165;
-	XOPC_DB_SCHEMA_VERSION = 199;
+	XOPC_DB_SCHEMA_VERSION = 201;
 }));
 //#endregion
 init_runner();

package/dist/src/storage/sqlite/schema.js

@@ -28,7 +28,7 @@ function ensureXopcDatabaseSchema(db, options = {}) {
 		currentVersion = 165;
 	}
 	applyPendingMigrations(db, {
-		targetVersion: 199,
+		targetVersion: 201,
 		databasePath: fresh ? void 0 : options.databasePath
 	});
 }

package/dist/src/storage/sqlite/schema.sql

@@ -2275,7 +2275,7 @@ CREATE TABLE knowledge_items (
   knowledge_id TEXT PRIMARY KEY,
   principal_id TEXT NOT NULL,
   kind TEXT NOT NULL CHECK(kind IN (
-    'project_fact', 'workspace_fact', 'decision', 'task_lesson',
+    'work_thread', 'project_fact', 'workspace_fact', 'decision', 'task_lesson',
     'commitment', 'open_question', 'episode', 'note'
   )),
   scope_type TEXT NOT NULL CHECK(scope_type IN ('global', 'agent', 'workspace', 'project', 'session')),

package/dist/src/user-context/domain.d.ts

@@ -52,19 +52,6 @@ export type CollaborationRule = {
     createdAt: number;
     updatedAt: number;
 };
-export type PersonalizationItem = {
-    objectType: 'profile' | 'rule' | 'focus' | 'understanding';
-    objectId: string;
-    versionId?: string;
-    decision: 'selected' | 'irrelevant' | 'expired' | 'scope_mismatch' | 'sensitive' | 'needs_consent' | 'budget_exceeded' | 'conflicted' | 'disabled';
-    reason: string;
-    content: string;
-    sourceLabel: string;
-    origin: 'told_by_user' | 'observed' | 'inferred' | 'connected_source';
-    rank?: number;
-    score?: number;
-    injectedChars: number;
-};
 export type ContextEvidence = {
     id: string;
     sourceType: 'conversation' | 'connector' | 'user' | 'runtime';

package/dist/src/user-context/sources/index.js

@@ -1,4 +1,4 @@
-import { createUnderstandingSourceRun, getUnderstandingSourceGrant, getUnderstandingSourceRun, listUnderstandingSourceGrants, listUnderstandingSourceRuns, revokeUnderstandingSourceGrant, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceGrantPolicies, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant } from "./repository.js";
+import { createUnderstandingSourceRun, getConnectorUnderstandingSourceRun, getOrCreateConnectorUnderstandingSourceRun, getUnderstandingSourceGrant, getUnderstandingSourceRun, listUnderstandingSourceGrants, listUnderstandingSourceRuns, revokeUnderstandingSourceGrant, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceGrantPolicies, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant } from "./repository.js";
 import { clusterUnderstandingSignals, tokenizeUnderstandingSignal } from "./signal-clustering.js";
 import { listUnderstandingSourceDefinitions } from "./catalog.js";
-export { clusterUnderstandingSignals, createUnderstandingSourceRun, getUnderstandingSourceGrant, getUnderstandingSourceRun, listUnderstandingSourceDefinitions, listUnderstandingSourceGrants, listUnderstandingSourceRuns, revokeUnderstandingSourceGrant, tokenizeUnderstandingSignal, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceGrantPolicies, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant };
+export { clusterUnderstandingSignals, createUnderstandingSourceRun, getConnectorUnderstandingSourceRun, getOrCreateConnectorUnderstandingSourceRun, getUnderstandingSourceGrant, getUnderstandingSourceRun, listUnderstandingSourceDefinitions, listUnderstandingSourceGrants, listUnderstandingSourceRuns, revokeUnderstandingSourceGrant, tokenizeUnderstandingSignal, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceGrantPolicies, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant };

package/dist/src/user-context/sources/repository.d.ts

@@ -24,10 +24,20 @@ export declare function createUnderstandingSourceRun(input: {
     grantId: string;
     kind: UnderstandingSourceRun['kind'];
     status?: UnderstandingSourceRun['status'];
+    connectorLearningJobId?: string;
     cursorBefore?: string;
     metadata?: Record<string, unknown>;
     nowMs?: number;
 }): UnderstandingSourceRun;
+export declare function getConnectorUnderstandingSourceRun(connectorLearningJobId: string): UnderstandingSourceRun | null;
+export declare function getOrCreateConnectorUnderstandingSourceRun(input: {
+    grantId: string;
+    connectorLearningJobId: string;
+    kind: UnderstandingSourceRun['kind'];
+    status?: UnderstandingSourceRun['status'];
+    metadata?: Record<string, unknown>;
+    nowMs?: number;
+}): UnderstandingSourceRun;
 export declare function getUnderstandingSourceRun(id: string): UnderstandingSourceRun | null;
 export declare function listUnderstandingSourceRuns(grantId: string, limit?: number): UnderstandingSourceRun[];
 export declare function updateUnderstandingSourceRun(id: string, patch: {

package/dist/src/user-context/sources/repository.js

@@ -35,6 +35,7 @@ function runFromRow(row) {
 		id: row.run_id,
 		grantId: row.grant_id,
 		kind: row.kind,
+		...row.connector_learning_job_id ? { connectorLearningJobId: row.connector_learning_job_id } : {},
 		status: row.status,
 		...row.cursor_before ? { cursorBefore: row.cursor_before } : {},
 		...row.cursor_after ? { cursorAfter: row.cursor_after } : {},
@@ -104,11 +105,25 @@ function createUnderstandingSourceRun(input) {
 	const now = input.nowMs ?? Date.now();
 	runSqliteWriteTransaction((db) => db.prepare(`
     INSERT INTO understanding_source_runs (
-      run_id, grant_id, kind, status, cursor_before, metadata_json, started_at
-    ) VALUES (?, ?, ?, ?, ?, ?, ?)
-  `).run(id, input.grantId, input.kind, input.status ?? "running", input.cursorBefore ?? null, JSON.stringify(input.metadata ?? {}), now));
+      run_id, grant_id, connector_learning_job_id, kind, status, cursor_before, metadata_json, started_at
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(id, input.grantId, input.connectorLearningJobId ?? null, input.kind, input.status ?? "running", input.cursorBefore ?? null, JSON.stringify(input.metadata ?? {}), now));
 	return getUnderstandingSourceRun(id);
 }
+function getConnectorUnderstandingSourceRun(connectorLearningJobId) {
+	const row = getSqliteDatabase().prepare("SELECT * FROM understanding_source_runs WHERE connector_learning_job_id = ?").get(connectorLearningJobId);
+	return row ? runFromRow(row) : null;
+}
+function getOrCreateConnectorUnderstandingSourceRun(input) {
+	const id = randomUUID();
+	const now = input.nowMs ?? Date.now();
+	runSqliteWriteTransaction((db) => db.prepare(`
+    INSERT OR IGNORE INTO understanding_source_runs (
+      run_id, grant_id, connector_learning_job_id, kind, status, metadata_json, started_at
+    ) VALUES (?, ?, ?, ?, ?, ?, ?)
+  `).run(id, input.grantId, input.connectorLearningJobId, input.kind, input.status ?? "running", JSON.stringify(input.metadata ?? {}), now));
+	return getConnectorUnderstandingSourceRun(input.connectorLearningJobId);
+}
 function getUnderstandingSourceRun(id) {
 	const row = getSqliteDatabase().prepare("SELECT * FROM understanding_source_runs WHERE run_id = ?").get(id);
 	return row ? runFromRow(row) : null;
@@ -127,4 +142,4 @@ function updateUnderstandingSourceRun(id, patch) {
 	return getUnderstandingSourceRun(id);
 }
 //#endregion
-export { createUnderstandingSourceRun, getUnderstandingSourceGrant, getUnderstandingSourceRun, listUnderstandingSourceGrants, listUnderstandingSourceRuns, revokeUnderstandingSourceGrant, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceGrantPolicies, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant };
+export { createUnderstandingSourceRun, getConnectorUnderstandingSourceRun, getOrCreateConnectorUnderstandingSourceRun, getUnderstandingSourceGrant, getUnderstandingSourceRun, listUnderstandingSourceGrants, listUnderstandingSourceRuns, revokeUnderstandingSourceGrant, updateUnderstandingSourceGrantCheckpoint, updateUnderstandingSourceGrantPolicies, updateUnderstandingSourceRun, upsertUnderstandingSourceGrant };

package/dist/src/user-context/sources/types.d.ts

@@ -63,6 +63,7 @@ export interface UnderstandingSourceCollectionResult {
 export interface UnderstandingSourceRun {
     id: string;
     grantId: string;
+    connectorLearningJobId?: string;
     kind: 'preview' | 'bootstrap' | 'incremental' | 'fingerprint';
     status: 'queued' | 'running' | 'completed' | 'partial' | 'failed' | 'canceled';
     cursorBefore?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xopcai/xopc",
-  "version": "0.0.315",
+  "version": "0.0.317",
   "description": "Local-first personal AI for the one-person company, with durable context, tasks, workflows, automations, and local or cloud models.",
   "type": "module",
   "main": "dist/src/index.js",
@@ -66,10 +66,10 @@
   },
   "dependencies": {
     "@composio/core": "0.18.1",
-    "@earendil-works/pi-agent-core": "0.85.1",
-    "@earendil-works/pi-ai": "0.85.1",
-    "@earendil-works/pi-coding-agent": "0.85.1",
-    "@earendil-works/pi-tui": "0.85.1",
+    "@earendil-works/pi-agent-core": "0.87.0",
+    "@earendil-works/pi-ai": "0.87.0",
+    "@earendil-works/pi-coding-agent": "0.87.0",
+    "@earendil-works/pi-tui": "0.87.0",
     "@grammyjs/runner": "^2.0.3",
     "@hono/node-server": "^2.0.4",
     "@inquirer/prompts": "^8.5.0",
@@ -171,11 +171,11 @@
     "vue": "^3.5.35",
     "@xopcai/browser-control-contract": "0.1.0",
     "@xopcai/computer-control-contract": "0.1.0",
-    "@xopcai/endpoint-tools-client": "0.0.0",
     "@xopcai/endpoint-tools-protocol": "0.0.0",
     "@xopcai/gateway-contract": "0.0.0",
-    "@xopcai/realtime-protocol": "0.0.0",
-    "@xopcai/realtime-client": "0.0.0"
+    "@xopcai/endpoint-tools-client": "0.0.0",
+    "@xopcai/realtime-client": "0.0.0",
+    "@xopcai/realtime-protocol": "0.0.0"
   },
   "scripts": {
     "assets:brand": "node scripts/generate-brand-assets.mjs",