@joshuaswarren/openclaw-engram 8.3.57 → 8.3.59

package/dist/index.js

@@ -377,6 +377,8 @@ function parseConfig(raw) {
     proactiveExtractionEnabled: cfg.proactiveExtractionEnabled === true,
     contextCompressionActionsEnabled: cfg.contextCompressionActionsEnabled === true,
     compressionGuidelineLearningEnabled: cfg.compressionGuidelineLearningEnabled === true,
+    compressionGuidelineSemanticRefinementEnabled: cfg.compressionGuidelineSemanticRefinementEnabled === true,
+    compressionGuidelineSemanticTimeoutMs: typeof cfg.compressionGuidelineSemanticTimeoutMs === "number" ? Math.max(1, Math.floor(cfg.compressionGuidelineSemanticTimeoutMs)) : 2500,
     maxProactiveQuestionsPerExtraction: typeof cfg.maxProactiveQuestionsPerExtraction === "number" ? Math.max(0, Math.floor(cfg.maxProactiveQuestionsPerExtraction)) : 2,
     maxCompressionTokensPerHour: typeof cfg.maxCompressionTokensPerHour === "number" ? Math.max(0, Math.floor(cfg.maxCompressionTokensPerHour)) : 1500,
     // v8.0 phase 1
@@ -9470,6 +9472,207 @@ function buildRecallQueryPolicy(prompt, sessionKey, cfg) {
   };
 }
 
+// src/compression-optimizer.ts
+var MAX_DELTA = 0.15;
+var SPARSE_SAMPLE = 5;
+function clamp(value, min, max) {
+  return Math.min(max, Math.max(min, value));
+}
+function parseRecallQuality(reason) {
+  if (!reason) return "unknown";
+  const text = reason.toLowerCase();
+  if (/(recall[_\s-]?good|quality[:=]\s*(good|high)|improv(ed|e)|resolved)/i.test(text)) {
+    return "good";
+  }
+  if (/(recall[_\s-]?poor|quality[:=]\s*(poor|low)|degrad(ed|e)|miss(ed|ing)|irrelevant)/i.test(text)) {
+    return "poor";
+  }
+  return "unknown";
+}
+function nextGuidelineVersion(previousState) {
+  if (!previousState) return 1;
+  return Math.max(1, previousState.guidelineVersion + 1);
+}
+function nextOptimizerVersion(previousState) {
+  if (!previousState) return 1;
+  return Math.max(1, previousState.version + 1);
+}
+function roundDelta(value) {
+  return Math.round(value * 1e3) / 1e3;
+}
+function confidenceForDelta(delta) {
+  const magnitude = Math.abs(delta);
+  return magnitude >= 0.09 ? "high" : magnitude >= 0.04 ? "medium" : "low";
+}
+function directionForDelta(delta) {
+  return delta > 0 ? "increase" : delta < 0 ? "decrease" : "hold";
+}
+function computeCompressionGuidelineCandidate(events, options = {}) {
+  const generatedAt = options.generatedAtIso ?? (/* @__PURE__ */ new Date()).toISOString();
+  const previousState = options.previousState ?? null;
+  const totalCounts = {
+    total: events.length,
+    applied: 0,
+    skipped: 0,
+    failed: 0
+  };
+  const actionMap = /* @__PURE__ */ new Map();
+  let windowFrom = events[0]?.timestamp ?? generatedAt;
+  let windowTo = events[0]?.timestamp ?? generatedAt;
+  for (const event of events) {
+    if (event.timestamp < windowFrom) windowFrom = event.timestamp;
+    if (event.timestamp > windowTo) windowTo = event.timestamp;
+    totalCounts[event.outcome] += 1;
+    let summary = actionMap.get(event.action);
+    if (!summary) {
+      summary = {
+        action: event.action,
+        total: 0,
+        outcomes: { applied: 0, skipped: 0, failed: 0 },
+        quality: { good: 0, poor: 0, unknown: 0 }
+      };
+      actionMap.set(event.action, summary);
+    }
+    summary.total += 1;
+    summary.outcomes[event.outcome] += 1;
+    const quality = parseRecallQuality(event.reason);
+    summary.quality[quality] += 1;
+  }
+  const actionSummaries = [...actionMap.values()].sort((a, b) => {
+    if (b.total !== a.total) return b.total - a.total;
+    return a.action.localeCompare(b.action);
+  });
+  const ruleUpdates = actionSummaries.map((summary) => {
+    const notes = [];
+    if (summary.total < SPARSE_SAMPLE) {
+      notes.push("Sparse sample size; holding baseline policy.");
+      return {
+        action: summary.action,
+        delta: 0,
+        direction: "hold",
+        confidence: "low",
+        notes
+      };
+    }
+    const successRate = summary.outcomes.applied / summary.total;
+    const failureRate = summary.outcomes.failed / summary.total;
+    const qualitySeen = summary.quality.good + summary.quality.poor;
+    const qualitySignal = qualitySeen > 0 ? (summary.quality.good - summary.quality.poor) / qualitySeen : 0;
+    const rawDelta = clamp((successRate - failureRate) * 0.12 + qualitySignal * 0.06, -MAX_DELTA, MAX_DELTA);
+    const delta = roundDelta(rawDelta);
+    const direction = directionForDelta(delta);
+    if (direction === "decrease" && summary.outcomes.failed > summary.outcomes.applied) {
+      notes.push("Failures exceed applied outcomes; conservative down-adjustment.");
+    } else if (direction === "increase" && summary.quality.good > summary.quality.poor) {
+      notes.push("Good recall quality markers support this action.");
+    } else if (direction === "decrease" && summary.quality.poor > summary.quality.good) {
+      notes.push("Poor recall quality markers exceed good markers.");
+    } else {
+      notes.push("Outcomes are stable; keep bounded adjustments.");
+    }
+    const confidence = confidenceForDelta(delta);
+    return {
+      action: summary.action,
+      delta,
+      direction,
+      confidence,
+      notes
+    };
+  });
+  return {
+    generatedAt,
+    sourceWindow: {
+      from: events.length > 0 ? windowFrom : generatedAt,
+      to: events.length > 0 ? windowTo : generatedAt
+    },
+    eventCounts: totalCounts,
+    actionSummaries,
+    ruleUpdates,
+    guidelineVersion: nextGuidelineVersion(previousState),
+    optimizerVersion: nextOptimizerVersion(previousState)
+  };
+}
+async function refineCompressionGuidelineCandidateSemantically(baseline, options) {
+  if (!options.enabled) return baseline;
+  if (typeof options.runRefinement !== "function") return baseline;
+  const timeoutMs = Math.max(1, Math.floor(options.timeoutMs));
+  let timeoutId = null;
+  const timeout = new Promise((resolve) => {
+    timeoutId = setTimeout(() => resolve(null), timeoutMs);
+  });
+  const refinementPromise = options.runRefinement(baseline).catch(() => null);
+  let refinement = null;
+  try {
+    refinement = await Promise.race([refinementPromise, timeout]);
+  } catch {
+    if (timeoutId) clearTimeout(timeoutId);
+    return baseline;
+  }
+  if (timeoutId) clearTimeout(timeoutId);
+  if (!refinement || !Array.isArray(refinement.updates) || refinement.updates.length === 0) {
+    return baseline;
+  }
+  const updatesByAction = /* @__PURE__ */ new Map();
+  for (const update of refinement.updates) {
+    if (!update || typeof update.action !== "string") continue;
+    updatesByAction.set(update.action, update);
+  }
+  let changed = false;
+  const ruleUpdates = baseline.ruleUpdates.map((rule) => {
+    const patch = updatesByAction.get(rule.action);
+    if (!patch) return rule;
+    const nextDelta = typeof patch.delta === "number" && Number.isFinite(patch.delta) ? roundDelta(clamp(patch.delta, -MAX_DELTA, MAX_DELTA)) : rule.delta;
+    const nextConfidence = patch.confidence ?? confidenceForDelta(nextDelta);
+    const nextDirection = directionForDelta(nextDelta);
+    const nextNotes = typeof patch.note === "string" && patch.note.trim().length > 0 ? [patch.note.trim()] : rule.notes;
+    if (nextDelta !== rule.delta || nextDirection !== rule.direction || nextConfidence !== rule.confidence || nextNotes.join("\n") !== rule.notes.join("\n")) {
+      changed = true;
+    }
+    return {
+      ...rule,
+      delta: nextDelta,
+      direction: nextDirection,
+      confidence: nextConfidence,
+      notes: nextNotes
+    };
+  });
+  if (!changed) return baseline;
+  return {
+    ...baseline,
+    ruleUpdates
+  };
+}
+function renderCompressionGuidelinesMarkdown(candidate) {
+  const actionLines = candidate.actionSummaries.length === 0 ? ["- (none)"] : candidate.actionSummaries.map((item) => `- ${item.action}: ${item.total}`);
+  const outcomeLines = [
+    `- applied: ${candidate.eventCounts.applied}`,
+    `- skipped: ${candidate.eventCounts.skipped}`,
+    `- failed: ${candidate.eventCounts.failed}`
+  ];
+  const updateLines = candidate.ruleUpdates.length === 0 ? ["- No telemetry events available yet. Keep defaults conservative and gather action data first."] : candidate.ruleUpdates.map((update) => {
+    const sign = update.delta > 0 ? "+" : "";
+    return `- ${update.action}: ${update.direction} (${sign}${update.delta.toFixed(3)}, confidence=${update.confidence}) \u2014 ${update.notes.join(" ")}`;
+  });
+  return [
+    "# Compression Guidelines",
+    "",
+    `Generated: ${candidate.generatedAt}`,
+    `Source events analyzed: ${candidate.eventCounts.total}`,
+    `Source window: ${candidate.sourceWindow.from} -> ${candidate.sourceWindow.to}`,
+    `Guideline version: ${candidate.guidelineVersion}`,
+    "",
+    "## Action Distribution",
+    ...actionLines,
+    "",
+    "## Outcome Distribution",
+    ...outcomeLines,
+    "",
+    "## Suggested Guidelines",
+    ...updateLines,
+    ""
+  ].join("\n");
+}
+
 // src/boxes.ts
 import { mkdir as mkdir11, writeFile as writeFile11, readFile as readFile12, readdir as readdir6 } from "fs/promises";
 import path14 from "path";
@@ -12211,59 +12414,6 @@ function deriveTopicsFromExtraction(result) {
   }
   return [...topics].slice(0, 16);
 }
-function buildCompressionGuidelinesMarkdown(events, generatedAtIso = (/* @__PURE__ */ new Date()).toISOString()) {
-  const byAction = /* @__PURE__ */ new Map();
-  const byOutcome = /* @__PURE__ */ new Map();
-  for (const event of events) {
-    byAction.set(event.action, (byAction.get(event.action) ?? 0) + 1);
-    byOutcome.set(event.outcome, (byOutcome.get(event.outcome) ?? 0) + 1);
-  }
-  const actionLines = byAction.size === 0 ? ["- (none)"] : [...byAction.entries()].sort((a, b) => b[1] - a[1]).map(([action, count]) => `- ${action}: ${count}`);
-  const outcomeLines = byOutcome.size === 0 ? ["- (none)"] : [...byOutcome.entries()].sort((a, b) => b[1] - a[1]).map(([outcome, count]) => `- ${outcome}: ${count}`);
-  const failed = byOutcome.get("failed") ?? 0;
-  const skipped = byOutcome.get("skipped") ?? 0;
-  const applied = byOutcome.get("applied") ?? 0;
-  const topAction = [...byAction.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
-  const suggested = [];
-  if (events.length === 0) {
-    suggested.push(
-      "- No telemetry events available yet. Keep defaults conservative and gather action data first."
-    );
-  } else {
-    if (typeof topAction === "string") {
-      suggested.push(`- Prefer \`${topAction}\` when handling similar future compression decisions.`);
-    }
-    if (failed > 0) {
-      suggested.push(
-        "- Failure events detected. Favor smaller, reversible compression steps and keep fail-open behavior."
-      );
-    }
-    if (skipped > applied) {
-      suggested.push(
-        "- Skipped actions outnumber applied actions. Revisit gating/thresholds before tightening policies."
-      );
-    }
-    if (failed === 0 && skipped <= applied) {
-      suggested.push("- Current action outcomes are stable. Keep policy conservative and continue monitoring.");
-    }
-  }
-  return [
-    "# Compression Guidelines",
-    "",
-    `Generated: ${generatedAtIso}`,
-    `Source events analyzed: ${events.length}`,
-    "",
-    "## Action Distribution",
-    ...actionLines,
-    "",
-    "## Outcome Distribution",
-    ...outcomeLines,
-    "",
-    "## Suggested Guidelines",
-    ...suggested,
-    ""
-  ].join("\n");
-}
 function filterRecallCandidates(candidates, options) {
   const scopedByNamespace = options.namespacesEnabled ? candidates.filter((r) => options.recallNamespaces.includes(options.resolveNamespace(r.path))) : candidates;
   return scopedByNamespace.filter((r) => !isArtifactMemoryPath(r.path)).slice(0, Math.max(0, options.limit));
@@ -15016,14 +15166,89 @@ ${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
   async runCompressionGuidelineLearningPass() {
     if (!this.config.compressionGuidelineLearningEnabled) return;
     try {
+      const previousState = await this.storage.readCompressionGuidelineOptimizerState();
       const events = await this.storage.readMemoryActionEvents(500);
-      const content = buildCompressionGuidelinesMarkdown(events);
+      const generatedAt = (/* @__PURE__ */ new Date()).toISOString();
+      const candidate = computeCompressionGuidelineCandidate(events, {
+        generatedAtIso: generatedAt,
+        previousState
+      });
+      const refinedCandidate = await refineCompressionGuidelineCandidateSemantically(candidate, {
+        enabled: this.config.compressionGuidelineSemanticRefinementEnabled,
+        timeoutMs: this.config.compressionGuidelineSemanticTimeoutMs,
+        runRefinement: async (baseline) => {
+          const prompt = [
+            "You refine compression policy suggestions conservatively.",
+            "Return JSON only in this shape:",
+            '{"updates":[{"action":"summarize_node","delta":0.02,"confidence":"medium","note":"..."}]}',
+            "Constraints:",
+            "- Keep updates sparse and conservative.",
+            "- delta must stay between -0.15 and 0.15.",
+            "- Only include actions present in the input.",
+            "Input candidate:",
+            JSON.stringify(baseline)
+          ].join("\n");
+          const response = await this.localLlm.chatCompletion(
+            [
+              { role: "system", content: "Respond with strict JSON only. No markdown." },
+              { role: "user", content: prompt }
+            ],
+            {
+              temperature: 0.1,
+              maxTokens: 400,
+              timeoutMs: this.config.compressionGuidelineSemanticTimeoutMs,
+              operation: "compression_guideline_semantic_refinement"
+            }
+          );
+          return this.parseCompressionSemanticRefinement(response?.content ?? "");
+        }
+      });
+      const content = renderCompressionGuidelinesMarkdown(refinedCandidate);
       await this.storage.writeCompressionGuidelines(content);
+      await this.storage.writeCompressionGuidelineOptimizerState({
+        version: refinedCandidate.optimizerVersion,
+        updatedAt: refinedCandidate.generatedAt,
+        sourceWindow: refinedCandidate.sourceWindow,
+        eventCounts: refinedCandidate.eventCounts,
+        guidelineVersion: refinedCandidate.guidelineVersion
+      });
       log.info(`compression guideline learning updated (${events.length} events)`);
     } catch (err) {
       log.warn(`compression guideline learning failed (ignored): ${err}`);
     }
   }
+  parseCompressionSemanticRefinement(raw) {
+    if (typeof raw !== "string" || raw.trim().length === 0) return null;
+    const trimmed = raw.trim();
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start === -1 || end === -1 || end <= start) return null;
+    try {
+      const parsed = JSON.parse(trimmed.slice(start, end + 1));
+      if (!Array.isArray(parsed?.updates)) return null;
+      const validActions = /* @__PURE__ */ new Set([
+        "store_episode",
+        "store_note",
+        "update_note",
+        "create_artifact",
+        "summarize_node",
+        "discard",
+        "link_graph"
+      ]);
+      const updates = parsed.updates.filter((item) => item && typeof item.action === "string" && validActions.has(item.action)).map((item) => {
+        const confidence = item.confidence === "low" || item.confidence === "medium" || item.confidence === "high" ? item.confidence : void 0;
+        return {
+          action: item.action,
+          delta: typeof item.delta === "number" && Number.isFinite(item.delta) ? item.delta : void 0,
+          confidence,
+          note: typeof item.note === "string" ? item.note : void 0
+        };
+      });
+      return { updates };
+    } catch {
+      return null;
+    }
+  }
   async runLifecyclePolicyPass(allMemories) {
     const now = /* @__PURE__ */ new Date();
     const nowIso = now.toISOString();