@mndrk/agx 2.0.26 → 2.0.33

package/cloud-runtime/standalone/lib/queue/sqlite-adapter.ts

@@ -6,8 +6,15 @@ import os from "os";
 import { QueueAdapter, QueueOptions, WorkerOptions, Job } from "./adapter";
 import { validateSQLiteEnvironment, REQUIRED_PRAGMAS } from "../startup";
 import { WriteRateMonitor, WRITE_RATE_SAMPLE_WINDOW_MS } from "../limits";
+import { writeDebugLog } from "../debug-log";
 
 const AGX_DATA_DIR = process.env.AGX_DATA_DIR || path.join(os.homedir(), ".agx");
+const DEFAULT_STALE_JOB_TIMEOUT_MS = 10 * 60 * 1000;
+
+function getStaleJobTimeoutMs(): number {
+  const raw = Number(process.env.AGX_QUEUE_STALE_MS);
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_STALE_JOB_TIMEOUT_MS;
+}
 
 interface SQLiteJobRow {
   id: string;
@@ -120,6 +127,14 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     );
 
     this.writeMonitor.record();
+    writeDebugLog("queue.send", {
+      queue,
+      jobId: id,
+      startAfter,
+      retryLimit: options?.retryLimit ?? 3,
+      priority: options?.priority ?? 0,
+      data,
+    });
     return id;
   }
 
@@ -128,6 +143,12 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     handler: (jobs: Job<T>[]) => Promise<void>,
     options?: WorkerOptions
   ): Promise<void> {
+    writeDebugLog("queue.worker.register", {
+      queue,
+      batchSize: options?.batchSize ?? 1,
+      pollInterval: options?.pollInterval ?? null,
+      workerId: this.workerId,
+    });
     this.workers.set(queue, {
       handler,
       options: options || {},
@@ -163,9 +184,20 @@ export class SQLiteQueueAdapter implements QueueAdapter {
   private processQueue(queue: string, worker: { handler: Function; options: WorkerOptions }) {
     const batchSize = worker.options.batchSize || 1;
     const now = Date.now();
+    const staleBefore = now - getStaleJobTimeoutMs();
 
     // Transactional claim
     const jobs = this.db.transaction(() => {
+      const reapedJobs = this.reapStaleJobs(queue, now, staleBefore);
+      if (reapedJobs.length > 0) {
+        writeDebugLog("queue.reap_stale", {
+          queue,
+          workerId: this.workerId,
+          staleBefore,
+          jobs: reapedJobs,
+        });
+      }
+
       // 1. Find candidates
       const candidates = this.db.prepare(`
         SELECT id, data FROM agx_jobs
@@ -198,6 +230,11 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     })();
 
     if (jobs.length > 0) {
+      writeDebugLog("queue.claim", {
+        queue,
+        workerId: this.workerId,
+        jobIds: jobs.map((job) => job.id),
+      });
       // Execute handler
       // Note: We don't await here to avoid blocking the poll loop, 
       // but in single-threaded Node, we are effectively blocking if the handler is sync-heavy.
@@ -214,9 +251,16 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       await handler(jobs);
       // Success
       this.completeJobs(jobs.map(j => j.id));
+      writeDebugLog("queue.complete", {
+        jobIds: jobs.map((job) => job.id),
+      });
     } catch (err: any) {
       // Failure (batch failure — all jobs in the batch fail together)
       this.failJobs(jobs.map(j => j.id), err.message || String(err));
+      writeDebugLog("queue.fail", {
+        jobIds: jobs.map((job) => job.id),
+        error: err,
+      });
     }
   }
 
@@ -224,7 +268,7 @@ export class SQLiteQueueAdapter implements QueueAdapter {
     const now = Date.now();
     const stmt = this.db.prepare(`
       UPDATE agx_jobs
-      SET status = 'completed', completed_at = ?
+      SET status = 'completed', completed_at = ?, worker_id = NULL
       WHERE id = ?
     `);
     this.db.transaction(() => {
@@ -243,13 +287,15 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       UPDATE agx_jobs
       SET status = 'retry', retry_count = retry_count + 1, 
           start_after = ? + (retry_count * 1000 * 2), -- Exponential backoff (kinda)
+          started_at = NULL,
+          worker_id = NULL,
           error = ?
       WHERE id = ?
     `);
 
     const updateFailed = this.db.prepare(`
       UPDATE agx_jobs
-      SET status = 'failed', failed_at = ?, error = ?
+      SET status = 'failed', failed_at = ?, started_at = NULL, worker_id = NULL, error = ?
       WHERE id = ?
     `);
 
@@ -268,4 +314,73 @@ export class SQLiteQueueAdapter implements QueueAdapter {
       }
     })();
   }
+
+  private reapStaleJobs(queue: string, now: number, staleBefore: number): Array<{
+    id: string;
+    fromStatus: "active";
+    toStatus: "retry" | "failed";
+    retryCount: number;
+  }> {
+    const staleJobs = this.db.prepare(`
+      SELECT id, retry_count, retry_limit
+      FROM agx_jobs
+      WHERE queue = ?
+        AND status = 'active'
+        AND started_at IS NOT NULL
+        AND started_at <= ?
+      ORDER BY started_at ASC
+    `).all(queue, staleBefore) as Array<Pick<SQLiteJobRow, "id" | "retry_count" | "retry_limit">>;
+
+    if (staleJobs.length === 0) return [];
+
+    const updateRetry = this.db.prepare(`
+      UPDATE agx_jobs
+      SET status = 'retry',
+          retry_count = retry_count + 1,
+          start_after = ?,
+          started_at = NULL,
+          worker_id = NULL,
+          error = ?
+      WHERE id = ?
+    `);
+    const updateFailed = this.db.prepare(`
+      UPDATE agx_jobs
+      SET status = 'failed',
+          failed_at = ?,
+          started_at = NULL,
+          worker_id = NULL,
+          error = ?
+      WHERE id = ?
+    `);
+
+    const reaped: Array<{
+      id: string;
+      fromStatus: "active";
+      toStatus: "retry" | "failed";
+      retryCount: number;
+    }> = [];
+
+    for (const job of staleJobs) {
+      const error = `Job exceeded active timeout of ${getStaleJobTimeoutMs()}ms`;
+      if (job.retry_count < job.retry_limit) {
+        updateRetry.run(now, error, job.id);
+        reaped.push({
+          id: job.id,
+          fromStatus: "active",
+          toStatus: "retry",
+          retryCount: job.retry_count + 1,
+        });
+      } else {
+        updateFailed.run(now, error, job.id);
+        reaped.push({
+          id: job.id,
+          fromStatus: "active",
+          toStatus: "failed",
+          retryCount: job.retry_count,
+        });
+      }
+    }
+
+    return reaped;
+  }
 }

package/cloud-runtime/standalone/lib/repo-knowledge.ts

@@ -1,4 +1,5 @@
 import { getSQLiteDb } from "./sqlite-query-adapter";
+import { getKnowledgeNote } from "./knowledge-notes";
 
 export type RepoKnowledgeProducer = "human" | "system";
 
@@ -39,6 +40,19 @@ export function listRepoKnowledge(
   const db = getSQLiteDb();
   const producer = options?.producer;
 
+  if (producer === "system") {
+    const note = getKnowledgeNote("repo", repoId);
+    return note
+      ? [{
+        id: note.id,
+        repo_id: repoId,
+        content: note.content,
+        producer: "system",
+        created_at: note.updatedAt,
+      }]
+      : [];
+  }
+
   const rows = producer
     ? (db
         .prepare(
@@ -57,7 +71,7 @@ export function listRepoKnowledge(
         )
         .all(repoId) as RepoKnowledgeRow[]);
 
-  if (options?.includeLegacyHumanFallback === false || producer === "system") {
+  if (options?.includeLegacyHumanFallback === false) {
     return rows;
   }
 
@@ -139,7 +153,10 @@ export function listResolvedRepoKnowledge(repos: Array<{ id?: string; name: stri
       continue;
     }
 
-    const rows = listRepoKnowledge(repo.id);
+    const rows = [
+      ...listRepoKnowledge(repo.id, { producer: "human" }),
+      ...listRepoKnowledge(repo.id, { producer: "system", includeLegacyHumanFallback: false }),
+    ];
     for (const row of rows) {
       const content = row.content.trim();
       if (!content) continue;

package/cloud-runtime/standalone/lib/sqlite-query-adapter.ts

@@ -219,6 +219,84 @@ function runMigrations(db: Database.Database): void {
     CREATE INDEX IF NOT EXISTS idx_repo_knowledge_producer ON repo_knowledge (producer);
   `);
 
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS knowledge_entries (
+      id TEXT NOT NULL PRIMARY KEY,
+      scope TEXT NOT NULL CHECK(scope IN ('agent', 'repo', 'project')),
+      subject_id TEXT NOT NULL,
+      source_type TEXT NOT NULL CHECK(source_type IN ('reflection', 'thread_transition', 'task_completion')),
+      source_id TEXT NOT NULL,
+      kind TEXT NOT NULL CHECK(kind IN ('outcome', 'decision', 'pattern', 'gotcha', 'preference', 'constraint', 'convention', 'lesson')),
+      title TEXT NOT NULL,
+      body TEXT NOT NULL,
+      confidence REAL,
+      durability REAL,
+      tags JSON NOT NULL DEFAULT '[]',
+      evidence JSON NOT NULL DEFAULT '[]',
+      metadata JSON NOT NULL DEFAULT '{}',
+      content_hash TEXT NOT NULL,
+      created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+      UNIQUE(scope, subject_id, content_hash)
+    );
+    CREATE INDEX IF NOT EXISTS idx_knowledge_entries_scope_subject ON knowledge_entries (scope, subject_id, updated_at);
+    CREATE INDEX IF NOT EXISTS idx_knowledge_entries_source ON knowledge_entries (source_type, source_id);
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS knowledge_notes (
+      id TEXT NOT NULL PRIMARY KEY,
+      scope TEXT NOT NULL CHECK(scope IN ('agent', 'repo', 'project')),
+      subject_id TEXT NOT NULL,
+      content TEXT NOT NULL DEFAULT '',
+      change_summary TEXT,
+      source_type TEXT NOT NULL CHECK(source_type IN ('reflection', 'thread_transition', 'task_completion')),
+      source_id TEXT NOT NULL,
+      metadata JSON NOT NULL DEFAULT '{}',
+      version INTEGER NOT NULL DEFAULT 1,
+      created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+      UNIQUE(scope, subject_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_knowledge_notes_scope_subject ON knowledge_notes (scope, subject_id);
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS knowledge_note_versions (
+      id TEXT NOT NULL PRIMARY KEY,
+      note_id TEXT NOT NULL REFERENCES knowledge_notes(id) ON DELETE CASCADE,
+      version INTEGER NOT NULL,
+      content TEXT NOT NULL,
+      change_summary TEXT,
+      source_type TEXT NOT NULL CHECK(source_type IN ('reflection', 'thread_transition', 'task_completion')),
+      source_id TEXT NOT NULL,
+      metadata JSON NOT NULL DEFAULT '{}',
+      created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+    );
+    CREATE INDEX IF NOT EXISTS idx_knowledge_note_versions_note_version
+      ON knowledge_note_versions (note_id, version DESC);
+  `);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS thread_knowledge_runs (
+      id TEXT NOT NULL PRIMARY KEY,
+      thread_id TEXT NOT NULL,
+      root_message_id TEXT NOT NULL,
+      status TEXT NOT NULL CHECK(status IN ('running', 'completed', 'failed')),
+      requested_scopes JSON NOT NULL DEFAULT '[]',
+      repo_inserted_count INTEGER NOT NULL DEFAULT 0,
+      project_inserted_count INTEGER NOT NULL DEFAULT 0,
+      error TEXT,
+      created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
+      completed_at TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_thread_knowledge_runs_root_created
+      ON thread_knowledge_runs (root_message_id, created_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_thread_knowledge_runs_thread_created
+      ON thread_knowledge_runs (thread_id, created_at DESC);
+  `);
+
   // Create project_threads table (replaces team_workspaces)
   db.exec(`
     CREATE TABLE IF NOT EXISTS project_threads (

package/cloud-runtime/standalone/lib/stream-multiplexer.ts

@@ -6,18 +6,33 @@ import { saveMessages, setReaction, saveLogs } from "./history-store";
 import { parseReactionSignals } from "./reaction-protocol";
 import { getAttachmentMeta } from "./attachment-store";
 import { getSQLiteDb } from "./sqlite-query-adapter";
-import { readSelf, buildReflectionPrompt, writeSelf } from "./mesh-core/self";
-import { readJournal, appendJournal } from "./mesh-core/journal";
+import {
+  readSelf,
+  writeSelf,
+  readReflectionState,
+  writeReflectionState,
+  buildStructuredReflectionPrompt,
+  type AgentReflectionMemoryDraft,
+} from "./mesh-core/self";
+import { readJournal, readJournalSince, appendJournal } from "./mesh-core/journal";
 import { readIdentity, getTeamSelves, listAgents, ensureAgent } from "./mesh-core/agent";
 import { appendReaction } from "./mesh-core/reactions";
 import { logActivity } from "./mesh-core/activity";
 import type { MeshReactionType } from "./mesh-core/types";
+import { listKnowledgeEntries, storeKnowledgeEntries, type KnowledgeEvidence } from "./knowledge-store";
+import { randomUUID, createHash } from "crypto";
 
 /** Per-agent message counter for reflection cadence */
 const agentMessageCounts = new Map<string, number>();
 const reflectionQueues = new Map<string, Promise<void>>();
 const REFLECTION_CADENCE = 10;
 
+function isKnowledgeEvidence(
+  value: KnowledgeEvidence | null,
+): value is KnowledgeEvidence {
+  return value !== null;
+}
+
 function seedFromIdentityText(identity: string | undefined, name: string): string {
   const raw = String(identity || "").trim();
   if (!raw) return `I am ${name}. I evolve through experience and collaboration.`;
@@ -124,41 +139,113 @@ async function runReflection(p: Participant): Promise<void> {
   if (!identity) return;
 
   const currentSelf = readSelf(p.id);
-  const recentEntries = readJournal(p.id, 10);
+  const reflectionState = readReflectionState(p.id);
+  const recentEntries = readJournalSince(p.id, reflectionState?.lastProcessedJournalId)
+    .filter((entry) => entry.type === "post");
+  if (recentEntries.length === 0) return;
   const teamSelves = getTeamSelves(p.id);
   const newVersion = (currentSelf?.version ?? 0) + 1;
-
-  const reflectionPrompt = buildReflectionPrompt(
+  const existingMemories = listKnowledgeEntries({ scope: "agent", subjectId: p.id, limit: 50 });
+  const reflectionPrompt = buildStructuredReflectionPrompt(
     identity,
     currentSelf,
     recentEntries,
-    teamSelves
+    teamSelves,
+    existingMemories,
   );
 
-  // Call the LLM to produce a new self-model snapshot
-  let newSelfContent = "";
+  let reflectionRaw = "";
   await runCliResponse({
     provider: p.provider,
     model: p.model,
-    systemContext: `You are performing a self-modeling exercise. Output ONLY the updated self-model text — no greetings, no preamble, no markdown fences, no commentary. Write in first person as ${p.name}. Focus on specialization, comparative fit, strengths, gaps, and next direction. Keep it concise (under 200 words).`,
+    systemContext: `You are performing a self-modeling exercise. Output ONLY raw JSON matching the requested schema. No markdown fences, no commentary.`,
     prompt: reflectionPrompt,
     signal: undefined,
-    onDelta: (chunk) => { newSelfContent += chunk; },
+    onDelta: (chunk) => { reflectionRaw += chunk; },
   });
 
-  newSelfContent = newSelfContent.trim();
+  const cleaned = reflectionRaw.trim()
+    .replace(/^```(?:json)?\s*/i, "")
+    .replace(/\s*```\s*$/, "")
+    .trim();
+  let parsed: { self_model?: string; memories?: AgentReflectionMemoryDraft[] } | null = null;
+  try {
+    parsed = JSON.parse(cleaned) as { self_model?: string; memories?: AgentReflectionMemoryDraft[] };
+  } catch {
+    parsed = null;
+  }
+
+  const newSelfContent = parsed?.self_model?.trim() || "";
   if (!newSelfContent) return;
 
-  // Write the updated self-model
+  const memoryDrafts = Array.isArray(parsed?.memories)
+    ? parsed!.memories
+        .slice(0, 3)
+        .map((memory) => ({
+          scope: "agent" as const,
+          subjectId: p.id,
+          sourceType: "reflection" as const,
+          sourceId: recentEntries[recentEntries.length - 1]?.id || `reflection:${p.id}:${newVersion}`,
+          kind: memory.kind,
+          title: String(memory.title ?? "").trim(),
+          body: String(memory.body ?? "").trim(),
+          confidence: memory.confidence,
+          durability: memory.durability,
+          tags: memory.tags,
+          evidence: Array.isArray(memory.evidence)
+            ? memory.evidence
+                .map((item) => {
+                  const note = String(item?.note ?? "").trim();
+                  const id = typeof item?.id === "string" ? item.id.trim() : "";
+                  return note ? (id ? { id, note } : { note }) : null;
+                })
+                .filter(isKnowledgeEvidence)
+            : [],
+          metadata: {
+            reflection_window_start: recentEntries[0]?.id ?? null,
+            reflection_window_end: recentEntries[recentEntries.length - 1]?.id ?? null,
+            self_version: newVersion,
+          },
+        }))
+        .filter((memory) => memory.title && memory.body)
+    : [];
+
+  const insertedKnowledge = storeKnowledgeEntries(memoryDrafts);
   writeSelf(p.id, newSelfContent, newVersion);
+  writeReflectionState(p.id, {
+    lastProcessedJournalId: recentEntries[recentEntries.length - 1]?.id ?? reflectionState?.lastProcessedJournalId ?? null,
+    updatedAt: new Date().toISOString(),
+  });
+
+  if (insertedKnowledge > 0) {
+    const db = getSQLiteDb();
+    const stmt = db.prepare(
+      `INSERT OR IGNORE INTO agent_memory (id, agent_id, task_id, memory_type, content, content_hash, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`
+    );
+    const reflectionTaskId = `reflection:${p.id}:${recentEntries[recentEntries.length - 1]?.id ?? newVersion}`;
+    const now = Date.now();
+    for (const memory of memoryDrafts) {
+      const memoryType =
+        memory.kind === "decision"
+          ? "decision"
+          : memory.kind === "gotcha" || memory.kind === "constraint"
+            ? "gotcha"
+            : memory.kind === "outcome"
+              ? "outcome"
+              : "pattern";
+      const contentHash = createHash("sha256").update(memory.body.trim()).digest("hex");
+      stmt.run(randomUUID(), p.id, reflectionTaskId, memoryType, memory.body.trim(), contentHash, now);
+    }
+  }
 
   // Append reflection journal entry
   appendJournal(p.id, {
     t: new Date().toISOString(),
     type: "reflection",
     observation: `Reflected after ${REFLECTION_CADENCE} messages`,
-    judgement: "Updated self-model from recent evidence and team comparison",
-    delta: `self-model updated to v${newVersion}`,
+    judgement: "Updated self-model from incremental journal evidence and deduped agent memories",
+    delta: `self-model updated to v${newVersion}${insertedKnowledge > 0 ? `; stored ${insertedKnowledge} agent memories` : ""}`,
     threads: [],
     selfVersion: newVersion,
     body: newSelfContent,

package/cloud-runtime/standalone/lib/task-context.ts

@@ -2,6 +2,7 @@ import { createHash } from "crypto";
 import { createAdminDbClient } from "./db-adapter";
 import { getSQLiteDb } from "./sqlite-query-adapter";
 import { listResolvedRepoKnowledge } from "./repo-knowledge";
+import { getKnowledgeNote } from "./knowledge-notes";
 import { resolveMemoryAgentId } from "./memory-extractor";
 import {
   Learning,
@@ -315,7 +316,8 @@ async function resolveProjectContext(task: Task, _projectLearnings: Learning[]):
     return null;
   }
 
-  const projectKnowledge = await getProjectMemory(projectWithRepos.id);
+  const projectKnowledge = await getProjectMemory(projectWithRepos.id, "human");
+  const projectSystemNote = getKnowledgeNote("project", projectWithRepos.id);
 
   if (projectWithRepos.repos.length > 0) {
     const repoKnowledge = listResolvedRepoKnowledge(projectWithRepos.repos);
@@ -345,9 +347,10 @@ async function resolveProjectContext(task: Task, _projectLearnings: Learning[]):
   return {
     project: projectWithRepos,
     repos: projectWithRepos.repos ?? [],
-    learnings: projectKnowledge.map((entry) =>
-      entry.producer === "system" ? `[System-generated] ${entry.content}` : entry.content
-    ),
+    learnings: [
+      ...projectKnowledge.map((entry) => entry.content),
+      ...(projectSystemNote?.content ? [`[System-generated] ${projectSystemNote.content}`] : []),
+    ],
   };
 }