@signetai/connector-base 0.216.7 → 0.217.1

package/dist/index.js

@@ -16198,6 +16198,241 @@ function up145(db) {
 			ON dreaming_evidence_reviews (agent_id, reviewed_at DESC);
 	`);
 }
+function up146(db) {
+  db.exec(`
+		CREATE TABLE IF NOT EXISTS source_import_jobs (
+			id TEXT PRIMARY KEY, kind TEXT NOT NULL CHECK (kind = 'import'), agent_id TEXT NOT NULL,
+			schema_id TEXT NOT NULL, adapter_version INTEGER NOT NULL CHECK (adapter_version = 1),
+			state TEXT NOT NULL CHECK (state IN ('staging','inventorying','queued','running','paused','completed','completed_with_rejections','cancelled','failed')),
+			control_request TEXT CHECK (control_request IN ('pause','resume','retry','cancel') OR control_request IS NULL),
+			generation INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 0, imported INTEGER NOT NULL DEFAULT 0,
+			duplicate INTEGER NOT NULL DEFAULT 0, rejected INTEGER NOT NULL DEFAULT 0, pending INTEGER NOT NULL DEFAULT 0,
+			lease_token TEXT, lease_expires_at TEXT, error TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')),
+			updated_at TEXT NOT NULL DEFAULT (datetime('now')), started_at TEXT, completed_at TEXT, reconciled_at TEXT
+		);
+		CREATE INDEX IF NOT EXISTS idx_source_import_jobs_agent_state ON source_import_jobs(agent_id, state);
+
+		CREATE TABLE IF NOT EXISTS source_import_files (
+			id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES source_import_jobs(id) ON DELETE CASCADE,
+			source_id TEXT NOT NULL, agent_id TEXT NOT NULL, ordinal INTEGER NOT NULL, name TEXT NOT NULL,
+			managed_path TEXT NOT NULL, size_bytes INTEGER NOT NULL DEFAULT 0, content_hash TEXT,
+			state TEXT NOT NULL CHECK (state IN ('staging','ready','inventorying','completed','failed')),
+			record_count INTEGER NOT NULL DEFAULT 0, blank_count INTEGER NOT NULL DEFAULT 0, malformed_count INTEGER NOT NULL DEFAULT 0,
+			inventory_version INTEGER NOT NULL DEFAULT 1, checkpoint_ordinal INTEGER NOT NULL DEFAULT 0,
+			checkpoint_byte_offset INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+			UNIQUE(job_id, ordinal)
+		);
+		CREATE INDEX IF NOT EXISTS idx_source_import_files_job_state ON source_import_files(job_id, state);
+
+		CREATE TABLE IF NOT EXISTS source_import_records (
+			id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES source_import_jobs(id) ON DELETE CASCADE,
+			file_id TEXT NOT NULL REFERENCES source_import_files(id) ON DELETE CASCADE, source_id TEXT NOT NULL, agent_id TEXT NOT NULL,
+			ordinal INTEGER NOT NULL, line_number INTEGER NOT NULL, byte_offset INTEGER NOT NULL, byte_length INTEGER NOT NULL,
+			raw_hash TEXT NOT NULL, external_identity TEXT, conversation_fingerprint TEXT, status TEXT NOT NULL CHECK (status IN ('pending','imported','duplicate','rejected','cancelled')),
+			canonical_id TEXT, canonical_key TEXT, rejection_code TEXT, attempt_count INTEGER NOT NULL DEFAULT 0,
+			created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+			UNIQUE(file_id, ordinal), UNIQUE(file_id, byte_offset)
+		);
+		CREATE INDEX IF NOT EXISTS idx_source_import_records_job_status ON source_import_records(job_id, status);
+		CREATE INDEX IF NOT EXISTS idx_source_import_records_source_status ON source_import_records(agent_id, source_id, status);
+
+		CREATE TABLE IF NOT EXISTS transcript_import_conversations (
+			agent_id TEXT NOT NULL, external_identity TEXT NOT NULL, canonical_key TEXT NOT NULL,
+			conversation_fingerprint TEXT NOT NULL, canonical_id TEXT NOT NULL, owner_source_id TEXT NOT NULL,
+			owner_record_id TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('committing','committed','removed')),
+			content_hash TEXT NOT NULL, harness TEXT NOT NULL, timestamp TEXT NOT NULL,
+			created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+			PRIMARY KEY (agent_id, external_identity), UNIQUE(agent_id, canonical_key)
+		);
+		CREATE INDEX IF NOT EXISTS idx_transcript_import_conversations_source ON transcript_import_conversations(agent_id, owner_source_id);
+
+		CREATE TABLE IF NOT EXISTS source_import_record_attempts (
+			id INTEGER PRIMARY KEY AUTOINCREMENT, agent_id TEXT NOT NULL, job_id TEXT NOT NULL, file_id TEXT NOT NULL,
+			record_id TEXT NOT NULL, generation INTEGER NOT NULL, outcome TEXT NOT NULL, error_code TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now'))
+		);
+		CREATE INDEX IF NOT EXISTS idx_source_import_attempts_record ON source_import_record_attempts(agent_id, record_id, created_at);
+	`);
+  for (const column of ["source_id", "source_record_id", "source_meta_json"]) {
+    const exists = db.prepare("SELECT 1 AS found FROM pragma_table_info('session_transcripts') WHERE name = ?").get(column);
+    if (exists == null)
+      db.exec(`ALTER TABLE session_transcripts ADD COLUMN ${column} TEXT`);
+  }
+  db.exec("CREATE INDEX IF NOT EXISTS idx_session_transcripts_agent_source ON session_transcripts(agent_id, source_id)");
+}
+function up147(db) {
+  db.exec("CREATE INDEX IF NOT EXISTS idx_source_import_files_job_state ON source_import_files(job_id, state)");
+}
+function up148(db) {
+  const columns = db.prepare("PRAGMA table_info(source_import_record_attempts)").all();
+  if (!columns.some((column) => column.name === "source_id")) {
+    db.exec("ALTER TABLE source_import_record_attempts ADD COLUMN source_id TEXT");
+  }
+}
+function up149(db) {
+  const addColumn = (table, column, definition) => {
+    const exists = db.prepare("SELECT 1 AS found FROM pragma_table_info(?) WHERE name = ?").get(table, column);
+    if (exists == null)
+      db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+  };
+  addColumn("source_import_jobs", "duplicate_mode", "TEXT NOT NULL DEFAULT 'skip' CHECK (duplicate_mode IN ('skip','replace','reimport'))");
+  addColumn("source_import_jobs", "next_attempt_at", "TEXT");
+  addColumn("source_import_files", "error", "TEXT");
+}
+function addColumnIfMissing28(db, table, column, definition) {
+  const columns = db.prepare(`PRAGMA table_info(${table})`).all();
+  if (!columns.some((row) => row.name === column))
+    db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+}
+function up150(db) {
+  addColumnIfMissing28(db, "memory_md_heads", "is_current", "INTEGER NOT NULL DEFAULT 0");
+  addColumnIfMissing28(db, "dreaming_passes", "head_base_revision", "INTEGER");
+  db.exec("CREATE INDEX IF NOT EXISTS idx_memory_head_revisions_content_hash ON memory_head_revisions(content_hash)");
+  db.exec("CREATE INDEX IF NOT EXISTS idx_memory_md_heads_content_hash ON memory_md_heads(content_hash)");
+  db.exec(`
+		CREATE TRIGGER IF NOT EXISTS dreaming_passes_head_fence_ai
+		AFTER INSERT ON dreaming_passes
+		WHEN NEW.mode = 'incremental-content'
+		BEGIN
+			INSERT OR IGNORE INTO memory_md_heads (agent_id, content, content_hash, revision, updated_at, is_current)
+			VALUES (NEW.agent_id, '', '', 0, datetime('now'), 0);
+			UPDATE dreaming_passes
+			SET head_base_revision = (SELECT revision FROM memory_md_heads WHERE agent_id = NEW.agent_id)
+			WHERE id = NEW.id;
+		END;
+
+		CREATE TRIGGER IF NOT EXISTS memories_head_freshness_au
+		AFTER UPDATE OF content, superseded_by, stale_at, is_deleted, agent_id, visibility, scope, memory_kind ON memories
+		WHEN OLD.content IS NOT NEW.content OR OLD.superseded_by IS NOT NEW.superseded_by
+		  OR OLD.stale_at IS NOT NEW.stale_at OR OLD.is_deleted IS NOT NEW.is_deleted
+		  OR OLD.agent_id IS NOT NEW.agent_id OR OLD.visibility IS NOT NEW.visibility OR OLD.scope IS NOT NEW.scope OR OLD.memory_kind IS NOT NEW.memory_kind
+		BEGIN
+			UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+			WHERE agent_id IN (OLD.agent_id, NEW.agent_id)
+			   OR (COALESCE(OLD.visibility, 'global') = 'global' OR COALESCE(NEW.visibility, 'global') = 'global')
+			      AND (agent_id IN (SELECT id FROM agents WHERE read_policy = 'shared')
+			        OR agent_id IN (SELECT id FROM agents WHERE read_policy = 'group' AND policy_group IN
+			          (SELECT policy_group FROM agents WHERE id IN (OLD.agent_id, NEW.agent_id))));
+		END;
+
+		CREATE TRIGGER IF NOT EXISTS memories_head_freshness_ad
+		AFTER DELETE ON memories
+		BEGIN
+			UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+			WHERE agent_id = OLD.agent_id
+			   OR (COALESCE(OLD.visibility, 'global') = 'global'
+			      AND (agent_id IN (SELECT id FROM agents WHERE read_policy = 'shared')
+			        OR agent_id IN (SELECT id FROM agents WHERE read_policy = 'group' AND policy_group IN
+			          (SELECT policy_group FROM agents WHERE id = OLD.agent_id))));
+		END;
+
+  CREATE TRIGGER IF NOT EXISTS agents_head_freshness_ad
+  AFTER DELETE ON agents
+  BEGIN
+   UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+   WHERE agent_id = OLD.id OR agent_id IN (SELECT id FROM agents WHERE read_policy='group' AND policy_group=OLD.policy_group);
+  END;
+
+		CREATE TRIGGER IF NOT EXISTS agents_head_freshness_au
+		AFTER UPDATE OF read_policy, policy_group ON agents
+		WHEN OLD.read_policy IS NOT NEW.read_policy OR OLD.policy_group IS NOT NEW.policy_group
+		BEGIN
+			UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+			WHERE agent_id IN (OLD.id, NEW.id)
+			   OR agent_id IN (SELECT id FROM agents WHERE read_policy = 'shared')
+			   OR agent_id IN (SELECT id FROM agents WHERE read_policy = 'group' AND policy_group IN (OLD.policy_group, NEW.policy_group));
+		END;
+	`);
+  for (const table of ["memory_artifacts", "session_transcripts"]) {
+    const exists = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(table);
+    if (!exists)
+      continue;
+    const hasIsDeleted = db.prepare(`PRAGMA table_info(${table})`).all().some((row) => row.name === "is_deleted");
+    const updateColumns = table === "session_transcripts" ? "content, agent_id, completed_at" : hasIsDeleted ? "content, agent_id, is_deleted" : "content, agent_id";
+    const contentChanged = table === "session_transcripts" ? "OLD.completed_at IS NOT NULL AND (OLD.content IS NOT NEW.content OR NEW.completed_at IS NULL)" : `OLD.content IS NOT NEW.content${hasIsDeleted ? " OR OLD.is_deleted IS NOT NEW.is_deleted" : ""}`;
+    db.exec(`
+			CREATE TRIGGER IF NOT EXISTS ${table}_head_freshness_au
+			AFTER UPDATE OF ${updateColumns} ON ${table}
+			WHEN OLD.agent_id IS NOT NEW.agent_id OR ${contentChanged}
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+				WHERE agent_id IN (OLD.agent_id, NEW.agent_id);
+			END;
+			CREATE TRIGGER IF NOT EXISTS ${table}_head_freshness_ad
+			AFTER DELETE ON ${table}
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0 WHERE agent_id = OLD.agent_id;
+			END;
+		`);
+  }
+  if (db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='session_summaries'").get()) {
+    db.exec(`
+		CREATE TRIGGER IF NOT EXISTS session_summaries_head_freshness_au
+		AFTER UPDATE OF content, source_ref, source_type, depth, agent_id ON session_summaries
+		WHEN OLD.depth = 0 AND (OLD.content IS NOT NEW.content OR OLD.source_ref IS NOT NEW.source_ref OR OLD.source_type IS NOT NEW.source_type OR OLD.depth IS NOT NEW.depth OR OLD.agent_id IS NOT NEW.agent_id)
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0 WHERE agent_id IN (OLD.agent_id, NEW.agent_id);
+			END;
+			CREATE TRIGGER IF NOT EXISTS session_summaries_head_freshness_ad
+			AFTER DELETE ON session_summaries
+			WHEN OLD.depth = 0
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0 WHERE agent_id = OLD.agent_id;
+			END;
+		`);
+  }
+  const safetyExists = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory_content_safety'").get();
+  if (safetyExists) {
+    db.exec(`
+		CREATE TRIGGER IF NOT EXISTS memory_content_safety_head_freshness_ai
+		AFTER INSERT ON memory_content_safety
+		WHEN NEW.status IS NOT 'clean' OR NEW.context_eligible IS NOT 1
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+				WHERE agent_id IN (NEW.agent_id)
+				   OR agent_id IN (SELECT id FROM agents WHERE read_policy IN ('shared', 'group'));
+			END;
+
+		CREATE TRIGGER IF NOT EXISTS memory_content_safety_head_freshness_au
+		AFTER UPDATE OF context_eligible, agent_id, status ON memory_content_safety
+		WHEN (OLD.context_eligible IS NOT NEW.context_eligible OR OLD.agent_id IS NOT NEW.agent_id OR OLD.status IS NOT NEW.status)
+		  AND (OLD.status IS NOT 'clean' OR NEW.status IS NOT 'clean' OR OLD.context_eligible IS NOT 1 OR NEW.context_eligible IS NOT 1)
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+				WHERE agent_id IN (OLD.agent_id, NEW.agent_id)
+				   OR agent_id IN (SELECT id FROM agents WHERE read_policy = 'shared')
+				   OR agent_id IN (SELECT id FROM agents WHERE read_policy = 'group');
+			END;
+		`);
+  }
+  for (const table of ["memory_artifact_tombstones", "imported_source_lifecycle"]) {
+    const exists = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(table);
+    if (!exists)
+      continue;
+    const lifecycleClause = table === "imported_source_lifecycle" ? " WHEN NEW.status = 'unsupported'" : "";
+    const updateColumns = table === "imported_source_lifecycle" ? "agent_id, status" : "agent_id";
+    const lifecycleUpdateClause = table === "imported_source_lifecycle" ? " AND NEW.status = 'unsupported'" : "";
+    const lifecycleWhen = table === "imported_source_lifecycle" ? `(OLD.agent_id IS NOT NEW.agent_id OR OLD.status IS NOT NEW.status)${lifecycleUpdateClause}` : "OLD.agent_id IS NOT NEW.agent_id";
+    db.exec(`
+			CREATE TRIGGER IF NOT EXISTS ${table}_head_freshness_ai
+			AFTER INSERT ON ${table}${lifecycleClause}
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0 WHERE agent_id = NEW.agent_id;
+			END;
+			CREATE TRIGGER IF NOT EXISTS ${table}_head_freshness_au
+			AFTER UPDATE OF ${updateColumns} ON ${table}
+			WHEN ${lifecycleWhen}
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0
+				WHERE agent_id IN (OLD.agent_id, NEW.agent_id);
+			END;
+			CREATE TRIGGER IF NOT EXISTS ${table}_head_freshness_ad
+			AFTER DELETE ON ${table}
+			BEGIN
+				UPDATE memory_md_heads SET revision = revision + 1, is_current = 0 WHERE agent_id = OLD.agent_id;
+			END;
+		`);
+  }
+}
 var MIGRATIONS = [
   {
     version: 1,
@@ -17347,6 +17582,60 @@ var MIGRATIONS = [
     name: "dreaming-evidence-reviews",
     up: up145,
     artifacts: { tables: ["dreaming_evidence_reviews"] }
+  },
+  {
+    version: 146,
+    name: "source-transcript-import",
+    up: up146,
+    artifacts: {
+      tables: [
+        "source_import_jobs",
+        "source_import_files",
+        "source_import_records",
+        "transcript_import_conversations",
+        "source_import_record_attempts"
+      ],
+      columns: [
+        { table: "session_transcripts", column: "source_id" },
+        { table: "session_transcripts", column: "source_record_id" },
+        { table: "session_transcripts", column: "source_meta_json" }
+      ]
+    }
+  },
+  {
+    version: 147,
+    name: "source-import-replay-file-slots",
+    up: up147,
+    artifacts: { tables: ["source_import_files"] }
+  },
+  {
+    version: 148,
+    name: "source-import-attempt-provenance",
+    up: up148,
+    artifacts: { columns: [{ table: "source_import_record_attempts", column: "source_id" }] }
+  },
+  {
+    version: 149,
+    name: "transcript-import-state-machine",
+    up: up149,
+    artifacts: {
+      columns: [
+        { table: "source_import_jobs", column: "duplicate_mode" },
+        { table: "source_import_jobs", column: "next_attempt_at" },
+        { table: "source_import_files", column: "error" }
+      ]
+    }
+  },
+  {
+    version: 150,
+    name: "memory-head-freshness",
+    up: up150,
+    artifacts: {
+      columns: [
+        { table: "memory_md_heads", column: "is_current" },
+        { table: "dreaming_passes", column: "head_base_revision" }
+      ]
+    }
   }
 ];
 var LATEST_SCHEMA_VERSION = MIGRATIONS[MIGRATIONS.length - 1]?.version ?? 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@signetai/connector-base",
-  "version": "0.216.7",
+  "version": "0.217.1",
   "description": "Base class for Signet harness connectors",
   "type": "module",
   "main": "./dist/index.js",
@@ -26,7 +26,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@signetai/core": "0.216.7",
+    "@signetai/core": "0.217.1",
     "json5": "^2.2.3",
     "jsonc-parser": "3.3.1"
   },