metame-cli 1.4.17 → 1.4.18

package/scripts/signal-capture.js

@@ -13,6 +13,14 @@ const path = require('path');
 const os = require('os');
 
 const BUFFER_FILE = path.join(os.homedir(), '.metame', 'raw_signals.jsonl');
+const OVERFLOW_FILE = path.join(os.homedir(), '.metame', 'raw_signals.overflow.jsonl');
+const LOCK_FILE = path.join(os.homedir(), '.metame', 'raw_signals.lock');
+const LOCK_RETRY_MAX = 80;
+const LOCK_RETRY_MS = 8;
+const LOCK_STALE_MS = 30 * 1000;
+const MAX_BUFFER_LINES = 300;
+const MAX_CAPTURE_CHARS = 1600;
+const ABSOLUTE_MAX_CAPTURE_CHARS = 6000;
 
 // === CONFIDENCE PATTERNS ===
 
@@ -33,6 +41,90 @@ const CORRECTION_EN = /(no,? I meant|that's not what I|you misunderstood|wrong.+
 const META_ZH = /我(发现|意识到|觉得|反思|总结|复盘)|想错了|换个(思路|方向|方案)|回头(想想|看看)|之前的(方案|思路|方向).*(不行|不对|有问题)|我的(问题|毛病|习惯)是|下次(应该|要|得)/;
 const META_EN = /(I realize|looking back|on reflection|my (mistake|problem|habit) is|let me rethink|wrong approach|next time I should)/i;
 
+// Internal/system prompts must never enter cognition signal buffer.
+const INTERNAL_PROMPT_PATTERNS = [
+  /You are a MetaMe cognitive profile distiller/i,
+  /You are a metacognition pattern detector/i,
+  /你是精准的知识提取引擎/,
+  /RECALLED LONG-TERM FACTS \(context only/i,
+  /\[System hints - DO NOT mention these to user:/i,
+  /\[Mac automation policy - do NOT expose this block:/i,
+  /MANDATORY FIRST ACTION: The user has not been calibrated yet/i,
+  /<\!--\s*FACTS:START\s*-->/i,
+  /<\!--\s*MEMORY:START\s*-->/i,
+  /\[Task notification\]/i,
+  /<task-notification\b/i,
+];
+
+function sanitizeMetaMePrompt(text) {
+  let prompt = String(text || '');
+  if (!prompt) return '';
+
+  // Remove daemon-injected RAG blocks
+  prompt = prompt.replace(/<!--\s*FACTS:START\s*-->[\s\S]*?<!--\s*FACTS:END\s*-->/gi, ' ');
+  prompt = prompt.replace(/<!--\s*MEMORY:START\s*-->[\s\S]*?<!--\s*MEMORY:END\s*-->/gi, ' ');
+
+  // Remove daemon/system internal hint blocks
+  prompt = prompt.replace(/\[System hints - DO NOT mention these to user:[\s\S]*?\]/gi, ' ');
+  prompt = prompt.replace(/\[Mac automation policy - do NOT expose this block:[\s\S]*?\]/gi, ' ');
+  prompt = prompt.replace(/\[Task notification\][\s\S]*?(?=\n{2,}|$)/gi, ' ');
+  prompt = prompt.replace(/<task-notification\b[\s\S]*?<\/task-notification>/gi, ' ');
+  prompt = prompt.replace(/<task-notification\b[\s\S]*$/gi, ' ');
+
+  return prompt.trim();
+}
+
+function isInternalPrompt(text) {
+  const prompt = String(text || '');
+  if (!prompt) return false;
+  return INTERNAL_PROMPT_PATTERNS.some((re) => re.test(prompt));
+}
+
+function sleep(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+function withBufferLock(fn) {
+  const dir = path.dirname(BUFFER_FILE);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+
+  let acquired = false;
+  for (let i = 0; i < LOCK_RETRY_MAX; i++) {
+    try {
+      const fd = fs.openSync(LOCK_FILE, 'wx');
+      fs.writeSync(fd, process.pid.toString());
+      fs.closeSync(fd);
+      acquired = true;
+      break;
+    } catch (e) {
+      if (e.code !== 'EEXIST') throw e;
+      try {
+        const age = Date.now() - fs.statSync(LOCK_FILE).mtimeMs;
+        if (age > LOCK_STALE_MS) {
+          fs.unlinkSync(LOCK_FILE);
+          continue;
+        }
+      } catch { /* lock released by another process */ }
+      sleep(LOCK_RETRY_MS);
+    }
+  }
+
+  if (!acquired) return false;
+  try {
+    fn();
+    return true;
+  } finally {
+    try { fs.unlinkSync(LOCK_FILE); } catch { /* non-fatal */ }
+  }
+}
+
+function writeBufferAtomically(lines) {
+  const tmp = BUFFER_FILE + `.tmp.${process.pid}`;
+  const content = lines.length > 0 ? (lines.join('\n') + '\n') : '';
+  fs.writeFileSync(tmp, content, 'utf8');
+  fs.renameSync(tmp, BUFFER_FILE);
+}
+
 // Read JSON from stdin
 let input = '';
 process.stdin.setEncoding('utf8');
@@ -40,7 +132,23 @@ process.stdin.on('data', (chunk) => { input += chunk; });
 process.stdin.on('end', () => {
   try {
     const data = JSON.parse(input);
-    const prompt = (data.prompt || '').trim();
+    let prompt = (data.prompt || '').trim();
+
+    // Internal Claude subprocesses (distill/memory extract/skill evolution) set this flag.
+    if (process.env.METAME_INTERNAL_PROMPT === '1') {
+      process.exit(0);
+    }
+
+    // Strip daemon/system injected wrappers first; keep user payload if present.
+    prompt = sanitizeMetaMePrompt(prompt);
+    if (!prompt) {
+      process.exit(0);
+    }
+
+    // Belt-and-suspenders: filter known internal prompt templates.
+    if (isInternalPrompt(prompt)) {
+      process.exit(0);
+    }
 
     // === LAYER 0: Metacognitive bypass — always capture self-reflection ===
     const isMeta = META_ZH.test(prompt) || META_EN.test(prompt);
@@ -55,6 +163,14 @@ process.stdin.on('end', () => {
       process.exit(0);
     }
 
+    // Hard cap to prevent giant prompt pastes from poisoning distill budget.
+    if (prompt.length > ABSOLUTE_MAX_CAPTURE_CHARS) {
+      process.exit(0);
+    }
+    if (prompt.length > MAX_CAPTURE_CHARS) {
+      prompt = prompt.slice(0, MAX_CAPTURE_CHARS);
+    }
+
     // Skip messages that are purely code or file paths
     if (!isMeta && /^(```|\/[\w/]+\.\w+$)/.test(prompt)) {
       process.exit(0);
@@ -113,18 +229,62 @@ process.stdin.on('end', () => {
       cwd: data.cwd || null
     };
 
-    // Append to buffer, drop oldest if over cap
-    let existingLines = [];
-    try {
-      existingLines = fs.readFileSync(BUFFER_FILE, 'utf8')
-        .split('\n').filter(l => l.trim());
-    } catch {
-      // File doesn't exist yet, that's fine
-    }
+    // Append/update with process-level lock to avoid concurrent read-modify-write loss.
+    const locked = withBufferLock(() => {
+      let existingLines = [];
+      try {
+        existingLines = fs.readFileSync(BUFFER_FILE, 'utf8').split('\n').filter(l => l.trim());
+      } catch {
+        // File doesn't exist yet, that's fine
+      }
+
+      // Drain overflow written during prior lock-contention periods.
+      // unlink AFTER writeBufferAtomically succeeds — crash-safe ordering.
+      let overflowDrained = false;
+      try {
+        const overflowLines = fs.readFileSync(OVERFLOW_FILE, 'utf8').split('\n').filter(Boolean);
+        if (overflowLines.length > 0) {
+          existingLines = existingLines.concat(overflowLines);
+          overflowDrained = true;
+        }
+      } catch { /* no overflow file — normal case */ }
 
-    existingLines.push(JSON.stringify(entry));
+      // Opportunistic hygiene: remove old internal/system lines if any slipped in historically.
+      existingLines = existingLines.filter((line) => {
+        try {
+          const parsed = JSON.parse(line);
+          const p = String(parsed && parsed.prompt ? parsed.prompt : '');
+          if (!p) return false;
+          if (p.length > ABSOLUTE_MAX_CAPTURE_CHARS) return false;
+          if (isInternalPrompt(p)) return false;
+          return true;
+        } catch {
+          return false;
+        }
+      });
 
-    fs.writeFileSync(BUFFER_FILE, existingLines.join('\n') + '\n');
+      existingLines.push(JSON.stringify(entry));
+      if (existingLines.length > MAX_BUFFER_LINES) {
+        existingLines = existingLines.slice(-MAX_BUFFER_LINES);
+      }
+      writeBufferAtomically(existingLines);
+      // Unlink overflow only after main file is safely written.
+      if (overflowDrained) {
+        try { fs.unlinkSync(OVERFLOW_FILE); } catch { /* already gone */ }
+      }
+    });
+
+    if (!locked) {
+      // Last-resort fallback: write to overflow side-file so the next lock-holder
+      // can drain, clean, and cap it — never bypassing buffer rules.
+      // Guard against unbounded growth: drop entry if overflow already at cap.
+      fs.mkdirSync(path.dirname(OVERFLOW_FILE), { recursive: true });
+      try {
+        const ofLines = fs.readFileSync(OVERFLOW_FILE, 'utf8').split('\n').filter(Boolean);
+        if (ofLines.length >= MAX_BUFFER_LINES) return; // shed load
+      } catch { /* overflow file doesn't exist yet */ }
+      fs.appendFileSync(OVERFLOW_FILE, JSON.stringify(entry) + '\n', 'utf8');
+    }
 
   } catch {
     // Silently ignore parse errors — never block the user's workflow

package/scripts/skill-evolution.js

@@ -35,6 +35,8 @@ const os = require('os');
 const HOME = os.homedir();
 const METAME_DIR = path.join(HOME, '.metame');
 const SKILL_SIGNAL_FILE = path.join(METAME_DIR, 'skill_signals.jsonl');
+const SKILL_SIGNAL_OVERFLOW_FILE = path.join(METAME_DIR, 'skill_signals.overflow.jsonl');
+const SKILL_SIGNAL_LOCK_FILE = path.join(METAME_DIR, 'skill_signals.lock');
 const EVOLUTION_QUEUE_FILE = path.join(METAME_DIR, 'evolution_queue.yaml');
 const EVOLUTION_POLICY_FILE = path.join(METAME_DIR, 'evolution_policy.yaml');
 const BRAIN_FILE = path.join(HOME, '.claude_profile.yaml');
@@ -113,6 +115,54 @@ Respond with ONLY a JSON code block:
 If no actionable insights, respond with exactly: NO_EVOLUTION`,
 };
 
+function clampInt(value, fallback, min, max) {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return fallback;
+  const i = Math.floor(n);
+  return Math.max(min, Math.min(max, i));
+}
+
+function sanitizePatternList(value, fallback) {
+  const list = Array.isArray(value) ? value : fallback;
+  const clean = [];
+  for (const v of list) {
+    if (typeof v !== 'string') continue;
+    const p = v.trim();
+    if (!p || p.length > 300) continue;
+    try {
+      // Validate regex compileability once so hot path won't crash at runtime.
+      // eslint-disable-next-line no-new
+      new RegExp(p, 'i');
+      clean.push(p);
+    } catch { /* invalid pattern */ }
+  }
+  return clean.length > 0 ? clean : fallback.slice();
+}
+
+function sanitizePolicy(input) {
+  const merged = { ...DEFAULT_POLICY, ...(input && typeof input === 'object' ? input : {}) };
+  const policy = {
+    ...merged,
+    version: clampInt(merged.version, DEFAULT_POLICY.version, 1, 1000000),
+    hot_failure_threshold: clampInt(merged.hot_failure_threshold, DEFAULT_POLICY.hot_failure_threshold, 1, 50),
+    hot_failure_window_minutes: clampInt(merged.hot_failure_window_minutes, DEFAULT_POLICY.hot_failure_window_minutes, 1, 24 * 60),
+    min_signals_for_distill: clampInt(merged.min_signals_for_distill, DEFAULT_POLICY.min_signals_for_distill, 1, 5000),
+    max_signals_buffer: clampInt(merged.max_signals_buffer, DEFAULT_POLICY.max_signals_buffer, 20, 20000),
+    min_evidence_for_update: clampInt(merged.min_evidence_for_update, DEFAULT_POLICY.min_evidence_for_update, 1, 20),
+    min_evidence_for_gap: clampInt(merged.min_evidence_for_gap, DEFAULT_POLICY.min_evidence_for_gap, 1, 20),
+    max_updates_per_analysis: clampInt(merged.max_updates_per_analysis, DEFAULT_POLICY.max_updates_per_analysis, 1, 20),
+    max_gaps_per_analysis: clampInt(merged.max_gaps_per_analysis, DEFAULT_POLICY.max_gaps_per_analysis, 1, 20),
+    self_eval_interval: clampInt(merged.self_eval_interval, DEFAULT_POLICY.self_eval_interval, 1, 1000),
+    cold_path_run_count: clampInt(merged.cold_path_run_count, DEFAULT_POLICY.cold_path_run_count, 0, 1000000),
+    complaint_patterns: sanitizePatternList(merged.complaint_patterns, DEFAULT_POLICY.complaint_patterns),
+    missing_skill_patterns: sanitizePatternList(merged.missing_skill_patterns, DEFAULT_POLICY.missing_skill_patterns),
+    prompt_template: (typeof merged.prompt_template === 'string' && merged.prompt_template.trim())
+      ? merged.prompt_template
+      : DEFAULT_POLICY.prompt_template,
+  };
+  return policy;
+}
+
 function loadPolicy() {
   let yaml;
   try { yaml = require('js-yaml'); } catch { return { ...DEFAULT_POLICY }; }
@@ -125,8 +175,7 @@ function loadPolicy() {
     }
     const content = fs.readFileSync(EVOLUTION_POLICY_FILE, 'utf8');
     const loaded = yaml.load(content) || {};
-    // Merge with defaults (new fields auto-added on upgrade)
-    return { ...DEFAULT_POLICY, ...loaded };
+    return sanitizePolicy(loaded);
   } catch {
     return { ...DEFAULT_POLICY };
   }
@@ -135,7 +184,7 @@ function loadPolicy() {
 function savePolicy(yaml, policy) {
   try {
     if (!fs.existsSync(METAME_DIR)) fs.mkdirSync(METAME_DIR, { mode: 0o700, recursive: true });
-    fs.writeFileSync(EVOLUTION_POLICY_FILE, yaml.dump(policy, { lineWidth: 120 }), 'utf8');
+    fs.writeFileSync(EVOLUTION_POLICY_FILE, yaml.dump(sanitizePolicy(policy), { lineWidth: 120 }), 'utf8');
   } catch {}
 }
 
@@ -197,20 +246,92 @@ function extractSkillSignal(prompt, output, error, files, cwd, toolUsageLog) {
 /**
  * Append a skill signal to the JSONL buffer.
  */
+function withSkillSignalLock(fn) {
+  if (!fs.existsSync(METAME_DIR)) fs.mkdirSync(METAME_DIR, { mode: 0o700, recursive: true });
+  const maxRetry = 80;
+  const retryMs = 8;
+  const staleMs = 30 * 1000;
+
+  let acquired = false;
+  for (let i = 0; i < maxRetry; i++) {
+    try {
+      const fd = fs.openSync(SKILL_SIGNAL_LOCK_FILE, 'wx');
+      fs.writeSync(fd, String(process.pid));
+      fs.closeSync(fd);
+      acquired = true;
+      break;
+    } catch (e) {
+      if (e.code !== 'EEXIST') throw e;
+      try {
+        const age = Date.now() - fs.statSync(SKILL_SIGNAL_LOCK_FILE).mtimeMs;
+        if (age > staleMs) {
+          fs.unlinkSync(SKILL_SIGNAL_LOCK_FILE);
+          continue;
+        }
+      } catch { /* lock released elsewhere */ }
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, retryMs);
+    }
+  }
+
+  if (!acquired) return false;
+  try {
+    fn();
+    return true;
+  } finally {
+    try { fs.unlinkSync(SKILL_SIGNAL_LOCK_FILE); } catch {}
+  }
+}
+
+function writeSkillSignalLines(lines) {
+  const tmp = SKILL_SIGNAL_FILE + `.tmp.${process.pid}`;
+  const content = Array.isArray(lines) && lines.length > 0 ? lines.join('\n') + '\n' : '';
+  fs.writeFileSync(tmp, content, 'utf8');
+  fs.renameSync(tmp, SKILL_SIGNAL_FILE);
+}
+
 function appendSkillSignal(signal) {
   if (!signal) return;
   try {
-    if (!fs.existsSync(METAME_DIR)) fs.mkdirSync(METAME_DIR, { mode: 0o700, recursive: true });
+    const payload = JSON.stringify(signal);
+    const policy = loadPolicy();
 
-    // Append
-    fs.appendFileSync(SKILL_SIGNAL_FILE, JSON.stringify(signal) + '\n', 'utf8');
+    const locked = withSkillSignalLock(() => {
+      let lines = [];
+      try {
+        lines = fs.readFileSync(SKILL_SIGNAL_FILE, 'utf8').split('\n').filter(Boolean);
+      } catch { /* first write */ }
+
+      // Drain overflow written during prior lock-contention periods.
+      // unlink AFTER writeSkillSignalLines succeeds — crash-safe ordering.
+      let overflowDrained = false;
+      try {
+        const overflowLines = fs.readFileSync(SKILL_SIGNAL_OVERFLOW_FILE, 'utf8').split('\n').filter(Boolean);
+        if (overflowLines.length > 0) {
+          lines = lines.concat(overflowLines);
+          overflowDrained = true;
+        }
+      } catch { /* no overflow file — normal case */ }
+
+      lines.push(payload);
+      if (lines.length > policy.max_signals_buffer) {
+        lines = lines.slice(-policy.max_signals_buffer);
+      }
+      writeSkillSignalLines(lines);
+      // Unlink overflow only after main file is safely written.
+      if (overflowDrained) {
+        try { fs.unlinkSync(SKILL_SIGNAL_OVERFLOW_FILE); } catch { /* already gone */ }
+      }
+    });
 
-    // Truncate if over limit (keep newest)
-    const content = fs.readFileSync(SKILL_SIGNAL_FILE, 'utf8').trim();
-    const lines = content.split('\n');
-    const policy = loadPolicy();
-    if (lines.length > policy.max_signals_buffer) {
-      fs.writeFileSync(SKILL_SIGNAL_FILE, lines.slice(-policy.max_signals_buffer).join('\n') + '\n', 'utf8');
+    if (!locked) {
+      // Last-resort fallback: write to overflow side-file so the next lock-holder
+      // can drain and apply max_signals_buffer cap — never bypassing buffer rules.
+      // Guard against unbounded growth: drop entry if overflow already at cap.
+      try {
+        const ofLines = fs.readFileSync(SKILL_SIGNAL_OVERFLOW_FILE, 'utf8').split('\n').filter(Boolean);
+        if (ofLines.length >= policy.max_signals_buffer) return;
+      } catch { /* overflow file doesn't exist yet */ }
+      fs.appendFileSync(SKILL_SIGNAL_OVERFLOW_FILE, payload + '\n', 'utf8');
     }
   } catch {
     // Non-fatal
@@ -263,7 +384,12 @@ function checkHotEvolution(signal) {
   }
 
   // Rule 2: User complaints about a skill
-  const complaintRe = new RegExp(policy.complaint_patterns.join('|'), 'i');
+  let complaintRe;
+  try {
+    complaintRe = new RegExp(policy.complaint_patterns.join('|'), 'i');
+  } catch {
+    complaintRe = new RegExp(DEFAULT_POLICY.complaint_patterns.join('|'), 'i');
+  }
   if (signal.prompt && complaintRe.test(signal.prompt) && signal.skills_invoked.length > 0) {
     for (const sk of signal.skills_invoked) {
       addToQueue(queue, {
@@ -276,7 +402,12 @@ function checkHotEvolution(signal) {
   }
 
   // Rule 3: Missing skill detection
-  const missingRe = new RegExp(policy.missing_skill_patterns.join('|'), 'i');
+  let missingRe;
+  try {
+    missingRe = new RegExp(policy.missing_skill_patterns.join('|'), 'i');
+  } catch {
+    missingRe = new RegExp(DEFAULT_POLICY.missing_skill_patterns.join('|'), 'i');
+  }
   const missText = [(signal.error || ''), (signal.prompt || ''), (signal.output_excerpt || '')].join('\n');
   const hasMissingPattern = missingRe.test(missText);
   const hasExplicitSkillMiss = signal.has_tool_failure &&
@@ -414,7 +545,6 @@ async function distillSkills() {
   // Get installed skills list
   const installedSkills = listInstalledSkills();
   if (installedSkills.length === 0) {
-    clearSignals();
     return null;
   }
 
@@ -466,7 +596,6 @@ async function distillSkills() {
 
     const jsonMatch = result.match(/```json\s*([\s\S]*?)```/);
     if (!jsonMatch) {
-      clearSignals();
       return null;
     }
 
@@ -624,8 +753,15 @@ RULES:
     const patchData = yaml.load(yamlMatch[1]);
     if (!patchData || typeof patchData !== 'object') return;
 
-    // Apply patch (merge, don't overwrite entirely)
-    const newPolicy = { ...policy, ...patchData };
+    // Apply patch with schema/type sanitization.
+    const newPolicy = sanitizePolicy({ ...policy, ...patchData });
+    if (newPolicy.version <= policy.version) {
+      newPolicy.version = policy.version + 1;
+    }
+    if (JSON.stringify(newPolicy) === JSON.stringify(policy)) {
+      console.log('🧬 Policy self-eval: patch produced no effective change.');
+      return;
+    }
     savePolicy(yaml, newPolicy);
     console.log(`🧬 Policy self-evolved: v${policy.version} → v${newPolicy.version}`);
 
@@ -896,7 +1032,10 @@ function readRecentSignals() {
 }
 
 function clearSignals() {
-  try { fs.writeFileSync(SKILL_SIGNAL_FILE, '', 'utf8'); } catch {}
+  try {
+    const locked = withSkillSignalLock(() => writeSkillSignalLines([]));
+    if (!locked) fs.writeFileSync(SKILL_SIGNAL_FILE, '', 'utf8');
+  } catch {}
 }
 
 function listInstalledSkills() {