@askexenow/exe-os 0.9.65 → 0.9.67

package/dist/bin/postgres-agentic-reflection-backfill.js

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+
+// src/bin/postgres-agentic-reflection-backfill.ts
+import { Client } from "pg";
+
+// src/lib/agentic-ontology.ts
+import { createHash } from "crypto";
+function stableId(...parts) {
+  return createHash("sha256").update(parts.map((p) => String(p ?? "")).join("::")).digest("hex").slice(0, 32);
+}
+function clean(text, max = 240) {
+  return text.replace(/\u0000/g, "").replace(/```[\s\S]*?```/g, " ").replace(/\s+/g, " ").trim().slice(0, max);
+}
+
+// src/lib/reflection-checkpoints.ts
+function parsePayload(payload) {
+  if (!payload) return {};
+  try {
+    return JSON.parse(payload);
+  } catch {
+    return {};
+  }
+}
+function semanticArray(event, key) {
+  const payload = parsePayload(event.payload);
+  const semantic = payload.semantic;
+  const value = semantic?.[key];
+  return Array.isArray(value) ? value.map(String).filter(Boolean) : [];
+}
+function buildReflectionCheckpoint(events) {
+  if (!events.length) return null;
+  const sorted = [...events].sort((a, b) => a.occurred_at.localeCompare(b.occurred_at));
+  const first = sorted[0];
+  const last = sorted[sorted.length - 1];
+  const goals = /* @__PURE__ */ new Set();
+  const successes = [];
+  const failures = [];
+  const risks = [];
+  const nextActions = [];
+  const decisions = [];
+  for (const event of sorted) {
+    for (const goal of semanticArray(event, "goals")) goals.add(goal);
+    for (const item of semanticArray(event, "successSignals")) successes.push(item);
+    for (const item of semanticArray(event, "failureSignals")) failures.push(item);
+    for (const item of semanticArray(event, "nextActions")) nextActions.push(item);
+    for (const item of semanticArray(event, "decisions")) decisions.push(item);
+    if (event.impact === "negative" || event.outcome === "failure_signal" || event.event_type === "problem" || event.event_type === "error") {
+      failures.push(event.outcome || event.intention || event.event_type);
+    }
+    if (event.outcome === "risk_signal") risks.push(event.intention || event.outcome);
+    if (event.impact === "positive" || event.outcome === "success_signal" || event.event_type === "milestone") {
+      successes.push(event.outcome || event.intention || event.event_type);
+    }
+  }
+  const uniqueGoals = [...goals].slice(0, 6);
+  const uniqueSuccesses = [...new Set(successes.map((x) => clean(x, 180)).filter(Boolean))].slice(0, 6);
+  const uniqueFailures = [...new Set(failures.map((x) => clean(x, 180)).filter(Boolean))].slice(0, 6);
+  const uniqueRisks = [...new Set(risks.map((x) => clean(x, 180)).filter(Boolean))].slice(0, 6);
+  const uniqueNext = [...new Set(nextActions.map((x) => clean(x, 180)).filter(Boolean))].slice(0, 6);
+  const uniqueDecisions = [...new Set(decisions.map((x) => clean(x, 180)).filter(Boolean))].slice(0, 6);
+  const learnings = [
+    uniqueSuccesses.length ? `Working: ${uniqueSuccesses.slice(0, 3).join("; ")}` : "Working: no strong success signal yet",
+    uniqueFailures.length ? `Improve: ${uniqueFailures.slice(0, 3).join("; ")}` : "Improve: no strong failure signal in this window",
+    uniqueDecisions.length ? `Decision context: ${uniqueDecisions.slice(0, 2).join("; ")}` : "Decision context: no explicit decision signal"
+  ];
+  const summary = clean([
+    `Window ${first.occurred_at} \u2192 ${last.occurred_at}`,
+    `${sorted.length} events`,
+    uniqueGoals.length ? `goals: ${uniqueGoals.join("; ")}` : "goals: inferred from event trajectory",
+    uniqueSuccesses.length ? `successes: ${uniqueSuccesses.slice(0, 2).join("; ")}` : "successes: none explicit",
+    uniqueFailures.length ? `failures: ${uniqueFailures.slice(0, 2).join("; ")}` : "failures: none explicit"
+  ].join(". "), 1200);
+  return {
+    id: stableId("reflection", first.project_name ?? "", first.session_id ?? "", first.occurred_at, last.occurred_at, sorted.length),
+    projectName: first.project_name ?? null,
+    sessionId: first.session_id ?? null,
+    windowStartAt: first.occurred_at,
+    windowEndAt: last.occurred_at,
+    eventCount: sorted.length,
+    goalCount: uniqueGoals.length,
+    successCount: uniqueSuccesses.length,
+    failureCount: uniqueFailures.length,
+    riskCount: uniqueRisks.length,
+    summary,
+    learnings,
+    nextActions: uniqueNext.length ? uniqueNext : ["Continue from latest successful event and resolve highest-impact failure/risk first."],
+    evidenceEventIds: sorted.slice(-25).map((e) => e.id),
+    confidence: Math.min(0.95, 0.5 + Math.min(sorted.length, 100) / 250 + uniqueGoals.length * 0.03)
+  };
+}
+
+// src/bin/postgres-agentic-reflection-backfill.ts
+function arg(name) {
+  const i = process.argv.indexOf(name);
+  return i >= 0 ? process.argv[i + 1] : void 0;
+}
+async function ensureSchema(client) {
+  await client.query(`CREATE TABLE IF NOT EXISTS memory.agent_reflection_checkpoints (
+    id text PRIMARY KEY,
+    project_name text,
+    session_id text,
+    window_start_at timestamp(3) NOT NULL,
+    window_end_at timestamp(3) NOT NULL,
+    event_count integer NOT NULL DEFAULT 0,
+    goal_count integer NOT NULL DEFAULT 0,
+    success_count integer NOT NULL DEFAULT 0,
+    failure_count integer NOT NULL DEFAULT 0,
+    risk_count integer NOT NULL DEFAULT 0,
+    summary text NOT NULL,
+    learnings jsonb NOT NULL DEFAULT '[]'::jsonb,
+    next_actions jsonb NOT NULL DEFAULT '[]'::jsonb,
+    evidence_event_ids jsonb NOT NULL DEFAULT '[]'::jsonb,
+    confidence double precision NOT NULL DEFAULT 0,
+    created_at timestamp(3) NOT NULL
+  )`);
+  await client.query(`CREATE INDEX IF NOT EXISTS idx_agent_reflection_project_time ON memory.agent_reflection_checkpoints (project_name, window_end_at)`);
+  await client.query(`CREATE INDEX IF NOT EXISTS idx_agent_reflection_session_time ON memory.agent_reflection_checkpoints (session_id, window_end_at)`);
+}
+async function main() {
+  const url = process.env.DATABASE_URL || process.env.EXED_DATABASE_URL;
+  if (!url) throw new Error("DATABASE_URL or EXED_DATABASE_URL is required");
+  const limit = Number(arg("--limit") ?? "20000");
+  const windowSize = Number(arg("--window") ?? "100");
+  const minEvents = Number(arg("--min-events") ?? "20");
+  const project = arg("--project");
+  const client = new Client({ connectionString: url });
+  await client.connect();
+  try {
+    await ensureSchema(client);
+    const where = project ? "WHERE project_name = $2" : "";
+    const params = project ? [limit, project] : [limit];
+    const result = await client.query(`SELECT id, event_type, occurred_at::text, actor_agent_id, project_name, session_id,
+        intention, outcome, impact, evidence_memory_id, payload::text
+      FROM memory.agent_events ${where}
+      ORDER BY project_name, session_id, occurred_at, sequence_index
+      LIMIT $1`, params);
+    const groups = /* @__PURE__ */ new Map();
+    for (const event of result.rows) {
+      const key = `${event.project_name ?? ""}::${event.session_id ?? ""}`;
+      const bucket = groups.get(key) ?? [];
+      bucket.push(event);
+      groups.set(key, bucket);
+    }
+    let inserted = 0;
+    for (const events of groups.values()) {
+      for (let i = 0; i < events.length; i += windowSize) {
+        const window = events.slice(i, i + windowSize);
+        if (window.length < minEvents) continue;
+        const checkpoint = buildReflectionCheckpoint(window);
+        if (!checkpoint) continue;
+        await client.query(`INSERT INTO memory.agent_reflection_checkpoints
+          (id, project_name, session_id, window_start_at, window_end_at, event_count, goal_count,
+           success_count, failure_count, risk_count, summary, learnings, next_actions, evidence_event_ids,
+           confidence, created_at)
+          VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::jsonb,$13::jsonb,$14::jsonb,$15,now())
+          ON CONFLICT (id) DO UPDATE SET summary = excluded.summary, learnings = excluded.learnings,
+            next_actions = excluded.next_actions, confidence = excluded.confidence`, [
+          checkpoint.id,
+          checkpoint.projectName,
+          checkpoint.sessionId,
+          checkpoint.windowStartAt,
+          checkpoint.windowEndAt,
+          checkpoint.eventCount,
+          checkpoint.goalCount,
+          checkpoint.successCount,
+          checkpoint.failureCount,
+          checkpoint.riskCount,
+          checkpoint.summary,
+          JSON.stringify(checkpoint.learnings),
+          JSON.stringify(checkpoint.nextActions),
+          JSON.stringify(checkpoint.evidenceEventIds),
+          checkpoint.confidence
+        ]);
+        inserted++;
+      }
+    }
+    process.stderr.write(`[postgres-agentic-reflection-backfill] Inserted/updated ${inserted} reflection checkpoints.
+`);
+  } finally {
+    await client.end();
+  }
+}
+main().catch((err) => {
+  process.stderr.write(`[postgres-agentic-reflection-backfill] FATAL: ${err instanceof Error ? err.message : String(err)}
+`);
+  process.exit(1);
+});

package/dist/bin/postgres-agentic-semantic-backfill.js

@@ -0,0 +1,237 @@
+#!/usr/bin/env node
+
+// src/bin/postgres-agentic-semantic-backfill.ts
+import { Client } from "pg";
+
+// src/lib/agentic-ontology.ts
+import { createHash } from "crypto";
+function stableId(...parts) {
+  return createHash("sha256").update(parts.map((p) => String(p ?? "")).join("::")).digest("hex").slice(0, 32);
+}
+function clean(text, max = 240) {
+  return text.replace(/\u0000/g, "").replace(/```[\s\S]*?```/g, " ").replace(/\s+/g, " ").trim().slice(0, max);
+}
+function inferOntologyEventType(row) {
+  const lower = row.raw_text.toLowerCase();
+  if (row.has_error) return "error";
+  if (/\b(done|complete|completed|fixed|resolved|shipped|deployed|pushed|published)\b/.test(lower)) return "milestone";
+  if (/\b(blocked|failed|error|bug|regression|broken)\b/.test(lower)) return "problem";
+  if (/\b(decided|decision|adr|we chose|approved|rejected)\b/.test(lower)) return "decision";
+  if (/\b(goal|need to|we need|want to|trying to|objective)\b/.test(lower)) return "goal_signal";
+  if (["Bash", "Read", "Edit", "Write", "Grep", "Glob"].includes(row.tool_name)) return "tool_action";
+  if (row.tool_name.startsWith("memory_card")) return "memory_card";
+  return "memory_observation";
+}
+function inferIntention(row) {
+  if (row.intent) return clean(row.intent, 220);
+  const text = clean(row.raw_text, 1e3);
+  const patterns = [
+    /(?:we need to|need to|let'?s|i want to|we should|goal is to|objective is to|trying to)\s+([^.!?\n]{8,220})/i,
+    /(?:so that|in order to)\s+([^.!?\n]{8,220})/i,
+    /(?:task|plan):\s*([^.!?\n]{8,220})/i
+  ];
+  for (const p of patterns) {
+    const m = text.match(p);
+    if (m?.[1]) return clean(m[1], 220);
+  }
+  if (["Bash", "Read", "Edit", "Write", "Grep", "Glob"].includes(row.tool_name)) {
+    return `${row.tool_name} during ${row.project_name}`;
+  }
+  return null;
+}
+function inferOutcome(row) {
+  if (row.outcome) return clean(row.outcome, 220);
+  if (row.has_error) return "error";
+  const lower = row.raw_text.toLowerCase();
+  if (/\b(done|complete|completed|fixed|resolved|shipped|deployed|pushed|published|passed)\b/.test(lower)) return "success_signal";
+  if (/\b(blocked|failed|error|regression|broken|not working|could not)\b/.test(lower)) return "failure_signal";
+  if (/\b(warning|risk|concern|caveat)\b/.test(lower)) return "risk_signal";
+  return null;
+}
+function extractGoalCandidates(row) {
+  const text = clean(row.raw_text, 1600);
+  const patterns = [
+    /(?:we need to|need to|i want to|we should|goal is to|objective is to|trying to|let'?s)\s+([^.!?\n]{12,220})/gi,
+    /(?:success means|success criteria|so that)\s+([^.!?\n]{12,220})/gi
+  ];
+  const out = [];
+  for (const pattern of patterns) {
+    for (const m of text.matchAll(pattern)) {
+      const candidate = clean(m[1] ?? "", 220);
+      if (candidate.length >= 12 && !out.some((x) => x.toLowerCase() === candidate.toLowerCase())) out.push(candidate);
+      if (out.length >= 3) return out;
+    }
+  }
+  return out;
+}
+function uniq(values, max = 6) {
+  const out = [];
+  for (const value of values.map((v) => clean(v, 220)).filter(Boolean)) {
+    if (!out.some((x) => x.toLowerCase() === value.toLowerCase())) out.push(value);
+    if (out.length >= max) break;
+  }
+  return out;
+}
+function extractMatches(text, patterns, max = 5) {
+  const out = [];
+  for (const pattern of patterns) {
+    for (const match of text.matchAll(pattern)) {
+      const value = match[1] ?? match[0];
+      if (value) out.push(value);
+      if (out.length >= max) return uniq(out, max);
+    }
+  }
+  return uniq(out, max);
+}
+function inferSemanticLabel(row) {
+  const text = clean(row.raw_text, 2400);
+  const eventType = inferOntologyEventType(row);
+  const intention = inferIntention(row);
+  const outcome = inferOutcome(row);
+  const goals = extractGoalCandidates(row);
+  const milestones = extractMatches(text, [
+    /\b(?:completed|finished|fixed|resolved|shipped|deployed|published|pushed|passed)\b([^.!?\n]{0,180})/gi,
+    /(?:milestone|done):\s*([^.!?\n]{8,220})/gi
+  ]);
+  const problems = extractMatches(text, [
+    /\b(?:blocked by|failed because|bug|regression|broken|not working|error)\b([^.!?\n]{0,180})/gi,
+    /(?:problem|issue|risk):\s*([^.!?\n]{8,220})/gi
+  ]);
+  const decisions = extractMatches(text, [
+    /(?:decided|decision|adr|we chose|approved|rejected)\s+([^.!?\n]{8,220})/gi
+  ]);
+  const temporalAnchors = extractMatches(text, [
+    /\b(\d{4}-\d{2}-\d{2}(?:[T ][0-9:.+-Z]+)?)\b/g,
+    /\b(today|yesterday|tomorrow|this week|next week|last week|morning|afternoon|tonight)\b/gi
+  ], 8);
+  const nextActions = extractMatches(text, [
+    /(?:next|todo|follow[- ]?up|remaining|need to)\s*:?\s*([^.!?\n]{8,220})/gi
+  ]);
+  const actors = uniq([
+    row.agent_id,
+    ...extractMatches(text, [/\b(?:agent|employee|owner|assignee)[:= ]+([a-zA-Z][a-zA-Z0-9_-]{1,40})/gi], 5)
+  ], 6);
+  const successSignals = milestones.length ? milestones : outcome === "success_signal" ? [clean(text, 180)] : [];
+  const failureSignals = problems.length ? problems : outcome === "failure_signal" || row.has_error ? [clean(text, 180)] : [];
+  const impact = successSignals.length && failureSignals.length ? "mixed" : failureSignals.length ? "negative" : successSignals.length ? "positive" : "neutral";
+  const signalCount = goals.length + milestones.length + problems.length + decisions.length + nextActions.length;
+  return {
+    labeler: "deterministic",
+    schemaVersion: 1,
+    eventType,
+    intention,
+    outcome,
+    impact,
+    confidence: Math.min(0.95, 0.45 + signalCount * 0.08 + (intention ? 0.1 : 0) + (outcome ? 0.1 : 0)),
+    goals,
+    milestones,
+    problems,
+    decisions,
+    actors,
+    temporalAnchors,
+    successSignals,
+    failureSignals,
+    nextActions,
+    summary: clean(text, 280)
+  };
+}
+
+// src/bin/postgres-agentic-semantic-backfill.ts
+function arg(name) {
+  const i = process.argv.indexOf(name);
+  return i >= 0 ? process.argv[i + 1] : void 0;
+}
+async function ensureSchema(client) {
+  await client.query(`CREATE TABLE IF NOT EXISTS memory.agent_semantic_labels (
+    id text PRIMARY KEY,
+    source_memory_id text NOT NULL,
+    event_id text,
+    labeler text NOT NULL,
+    schema_version integer NOT NULL DEFAULT 1,
+    confidence double precision NOT NULL DEFAULT 0,
+    labels jsonb NOT NULL,
+    created_at timestamp(3) NOT NULL,
+    updated_at timestamp(3) NOT NULL
+  )`);
+  await client.query(`CREATE INDEX IF NOT EXISTS idx_agent_semantic_labels_memory ON memory.agent_semantic_labels (source_memory_id, labeler)`);
+  await client.query(`CREATE INDEX IF NOT EXISTS idx_agent_semantic_labels_event ON memory.agent_semantic_labels (event_id)`);
+}
+function scrubJson(value) {
+  return JSON.stringify(value).replace(/\u0000/g, "").replace(/\\u0000/gi, "").replace(/\\\\u0000/gi, "").replace(/\\u00[0-1][0-9a-f]/gi, " ");
+}
+async function main() {
+  const url = process.env.DATABASE_URL || process.env.EXED_DATABASE_URL;
+  if (!url) throw new Error("DATABASE_URL or EXED_DATABASE_URL is required");
+  const limit = Number(arg("--limit") ?? "20000");
+  const project = arg("--project");
+  const client = new Client({ connectionString: url });
+  await client.connect();
+  try {
+    await ensureSchema(client);
+    const params = project ? [limit, project] : [limit];
+    const where = project ? "WHERE project_name = $2" : "";
+    const result = await client.query(`SELECT id, agent_id, agent_role, session_id, timestamp::text, tool_name,
+        project_name, has_error, raw_text, version, task_id, intent, null::text as outcome, domain, null::text as trajectory
+      FROM memory.memory_records ${where}
+      ORDER BY timestamp DESC LIMIT $1`, params);
+    let count = 0;
+    for (const row of result.rows) {
+      const label = inferSemanticLabel({ ...row, has_error: row.has_error ? 1 : 0 });
+      const eventId = stableId(`event:${row.id}`);
+      const labelJson = scrubJson(label);
+      try {
+        await client.query(`INSERT INTO memory.agent_semantic_labels
+          (id, source_memory_id, event_id, labeler, schema_version, confidence, labels, created_at, updated_at)
+          VALUES ($1,$2,$3,$4,$5,$6,to_jsonb($7::text),now(),now())
+          ON CONFLICT (id) DO UPDATE SET confidence = excluded.confidence, labels = excluded.labels,
+            updated_at = excluded.updated_at`, [
+          stableId(`semantic:${row.id}:${label.labeler}:${label.schemaVersion}`),
+          row.id,
+          eventId,
+          label.labeler,
+          label.schemaVersion,
+          label.confidence,
+          labelJson
+        ]);
+      } catch {
+        const minimal = scrubJson({
+          labeler: label.labeler,
+          schemaVersion: label.schemaVersion,
+          eventType: label.eventType,
+          intention: label.intention,
+          outcome: label.outcome,
+          impact: label.impact,
+          confidence: label.confidence,
+          goals: label.goals,
+          summary: label.summary,
+          sanitized: true
+        });
+        await client.query(`INSERT INTO memory.agent_semantic_labels
+          (id, source_memory_id, event_id, labeler, schema_version, confidence, labels, created_at, updated_at)
+          VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,now(),now())
+          ON CONFLICT (id) DO UPDATE SET confidence = excluded.confidence, labels = excluded.labels,
+            updated_at = excluded.updated_at`, [
+          stableId(`semantic:${row.id}:${label.labeler}:${label.schemaVersion}`),
+          row.id,
+          eventId,
+          label.labeler,
+          label.schemaVersion,
+          label.confidence,
+          minimal
+        ]);
+      }
+      count++;
+      if (count % 5e3 === 0) process.stderr.write(`[postgres-agentic-semantic-backfill] labeled ${count}
+`);
+    }
+    process.stderr.write(`[postgres-agentic-semantic-backfill] Complete: ${count} semantic labels.
+`);
+  } finally {
+    await client.end();
+  }
+}
+main().catch((err) => {
+  process.stderr.write(`[postgres-agentic-semantic-backfill] FATAL: ${err instanceof Error ? err.message : String(err)}
+`);
+  process.exit(1);
+});