pan-wizard 3.5.2 → 3.7.10

package/pan-wizard-core/bin/lib/optimize.cjs

@@ -48,7 +48,53 @@ function generateSessionId() {
   return 'sess_' + now.toISOString().replace(/[-:.Z]/g, '').slice(0, 15);
 }
 
+// P-1404 helper: try to reuse a recent existing session.
+// Returns { session_id, started_at, directory, reused: true } on success,
+// null if no recent session exists or reuse failed.
+function tryReuseSession(cwd, opts) {
+  try {
+    const optimizeDir = getOptimizeDir(cwd);
+    const currentSessionPath = path.join(optimizeDir, CURRENT_SESSION_FILE);
+    const existingId = fs.readFileSync(currentSessionPath, 'utf-8').trim();
+    if (!existingId) return null;
+
+    const sessionDir = path.join(getTracesDir(cwd), existingId);
+    const sessionMetaPath = path.join(sessionDir, OPT_SESSION_FILE);
+    const meta = JSON.parse(fs.readFileSync(sessionMetaPath, 'utf-8'));
+
+    // Don't reuse if session is already explicitly ended
+    if (meta.ended_at) return null;
+
+    // Don't reuse if session started more than REUSE_WINDOW_MS ago
+    const startedAt = new Date(meta.started_at).getTime();
+    if (Date.now() - startedAt > SESSION_REUSE_WINDOW_MS) return null;
+
+    return {
+      session_id: existingId,
+      started_at: meta.started_at,
+      directory: sessionDir,
+      reused: true,
+    };
+  } catch {
+    return null;
+  }
+}
+
+// P-1404 fix (v3.7.3): a recent existing session (within REUSE_WINDOW_MS) is
+// reused instead of creating a new one. Without this, every /pan:exec-phase /
+// /pan:plan-phase / etc. creates its own session, fragmenting trace data
+// across many sub-sessions and making /pan:learn analysis incomplete.
+// Fragmentation surfaced by panloop run: 14 events scattered across 4 sessions.
+const SESSION_REUSE_WINDOW_MS = 60 * 60 * 1000; // 1 hour
+
 function initTraceSession(cwd, opts = {}) {
+  // Reuse logic: if no explicit sessionId requested AND a current-session
+  // file exists pointing at a recent session, reuse it.
+  if (!opts.sessionId && !opts.forceNew) {
+    const reused = tryReuseSession(cwd, opts);
+    if (reused) return reused;
+  }
+
   const sessionId = opts.sessionId || generateSessionId();
   const sessionDir = path.join(getTracesDir(cwd), sessionId);
 
@@ -73,7 +119,7 @@ function initTraceSession(cwd, opts = {}) {
     fs.mkdirSync(optimizeDir, { recursive: true });
     fs.writeFileSync(path.join(optimizeDir, CURRENT_SESSION_FILE), sessionId + '\n');
 
-    return { session_id: sessionId, started_at: meta.started_at, directory: sessionDir };
+    return { session_id: sessionId, started_at: meta.started_at, directory: sessionDir, reused: false };
   } catch (e) {
     return { error: e.message };
   }
@@ -299,6 +345,30 @@ function analyzeEvents(events, sessionMeta) {
 
   timing.token_data_available = events.some(e => e.context && (e.context.input_tokens || 0) > 0);
 
+  // P-1403 (v3.7.3): autonomous-overhead metrics. Useful as a trend signal —
+  // are autonomous runs getting cheaper/faster as patterns saturate? Caller
+  // can pass commitCount + costUsd via sessionMeta.commit_count /
+  // sessionMeta.cost_usd (read from harvest.json + claude-cli result JSON).
+  // When unavailable, fields are null (don't lie about absent data).
+  const overhead = {};
+  const commitCount = sessionMeta && typeof sessionMeta.commit_count === 'number'
+    ? sessionMeta.commit_count
+    : null;
+  const costUsd = sessionMeta && typeof sessionMeta.cost_usd === 'number'
+    ? sessionMeta.cost_usd
+    : null;
+  const durationMs = timing.session_duration_ms || null;
+
+  if (commitCount != null && durationMs) {
+    overhead.commits_per_minute = Math.round((commitCount / (durationMs / 60000)) * 100) / 100;
+    overhead.minutes_per_commit = Math.round((durationMs / 60000 / commitCount) * 100) / 100;
+  }
+  if (costUsd != null && commitCount != null && commitCount > 0) {
+    overhead.cost_usd_per_commit = Math.round((costUsd / commitCount) * 100) / 100;
+  }
+  if (costUsd != null) overhead.total_cost_usd = costUsd;
+  if (commitCount != null) overhead.commit_count = commitCount;
+
   return {
     summary: {
       total_events: events.length,
@@ -313,6 +383,7 @@ function analyzeEvents(events, sessionMeta) {
       memory_primed_count: memoryPrimed.length,
     },
     timing,
+    overhead,
     error_patterns: frequencyMap(errors),
     gap_patterns: frequencyMap(gaps),
     memory_miss_patterns: frequencyMap(memoryMisses),
@@ -612,6 +683,402 @@ function cmdOptimizeList(cwd, raw) {
 
 // ─── Exports ─────────────────────────────────────────────────────────────────
 
+// ─── W4: Promote (self-improvement loop) ──────────────────────────────────────
+//
+// Spec: docs/specs/self_improvement_loop_featureai.md §3.2 W4
+//
+// Promote a finding from a harvested experiment into the shipped behavioral
+// surface. Two scopes:
+//   - universal: ships to all 5 runtime installs (consumed by user-project workflows)
+//   - internal:  source-only (consumed when working on PAN itself)
+//
+// Each topic file is markdown with YAML frontmatter listing its pattern IDs.
+// The promote step is manual — the human running pan-tools picks scope/topic.
+// Auto-promote (rules-based, AI-confidence threshold) is deferred to v3.8+.
+
+const VALID_SCOPES = ['universal', 'internal'];
+// Topic name: lowercase, digits, hyphens; max 40 chars; no leading/trailing hyphen.
+// Same rules as experiment slug (intentional — symmetric naming).
+const TOPIC_RE = /^[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?$/;
+
+function getLearningsDir(sourceRoot, scope) {
+  return path.join(sourceRoot, 'pan-wizard-core', 'learnings', scope);
+}
+
+function getTopicFilePath(sourceRoot, scope, topic) {
+  return path.join(getLearningsDir(sourceRoot, scope), `${topic}.md`);
+}
+
+function validatePromoteInputs(pattern, opts) {
+  if (!opts || typeof opts !== 'object') return 'opts is required';
+  if (!VALID_SCOPES.includes(opts.scope)) {
+    return `scope must be one of: ${VALID_SCOPES.join(', ')}, got "${opts.scope}"`;
+  }
+  if (typeof opts.topic !== 'string' || !TOPIC_RE.test(opts.topic)) {
+    return 'topic invalid: must be lowercase letters, digits, hyphens (no path separators or leading/trailing hyphen)';
+  }
+  if (!opts.sourceRoot) return 'sourceRoot is required';
+  if (!pattern || typeof pattern !== 'object') return 'pattern is required';
+  if (!pattern.id) return 'pattern.id is required';
+  if (!pattern.summary) return 'pattern.summary is required';
+  if (!pattern.rule) return 'pattern.rule is required';
+  return null;
+}
+
+/**
+ * Classify whether a pattern looks STRUCTURAL (generalizes across models /
+ * languages / runtimes) or PROMPT-FRAGMENT (specific phrasing that doesn't
+ * generalize). Per P-RES-007 (Sakana DGM, 2025): structural changes
+ * transferred across models in self-improvement loops; prompt-fragment
+ * tweaks did not. Universal scope should be reserved for structural
+ * patterns; prompt fragments belong in internal scope at most.
+ *
+ * This is HEURISTIC. Returns { kind: 'structural'|'prompt-fragment'|'unclear',
+ * reasons: [...] } — never definitive. Used to surface a WARNING on
+ * `learn promote --scope universal` so the human gate has a signal.
+ *
+ * @param {object} pattern - { rule, summary, ... }
+ * @returns {object} { kind, reasons: string[] }
+ */
+function classifyPatternKind(pattern) {
+  const rule = String(pattern.rule || '').toLowerCase();
+  const summary = String(pattern.summary || '').toLowerCase();
+  const combined = rule + ' ' + summary;
+  const reasons = [];
+
+  // Strong structural markers — describe SHAPES, contracts, file/module patterns
+  const STRUCTURAL_RE = [
+    /\b(pattern|structure|architecture|module|interface|contract|api|signature|schema|invariant)\b/,
+    /\b(wrap|factor|compose|extract|encapsulate|inject)\b/,
+    /\b(closure|callback|generator|stream|state\s+machine)\b/,
+    /file:\/\/|\.md\b|\.cjs\b|\.js\b|\.ts\b/,
+    /\b(workflow|step|phase|gate|hook)\b/,
+  ];
+  let structuralHits = 0;
+  for (const re of STRUCTURAL_RE) {
+    if (re.test(combined)) structuralHits++;
+  }
+
+  // Prompt-fragment markers — specific phrasing, "always say X", quoted strings.
+  // Anchored carefully: "write" alone is too broad (matches "write to file"),
+  // so we require co-occurrence with "the words"/"exact"/quoted text.
+  const PROMPT_RE = [
+    /\b(say|write)\s+(the\s+exact|the\s+words?|"|')/,
+    /\buse\s+the\s+(exact|words?)\b/,
+    /\b(prepend|prefix\s+with)\b/,
+    /\balways\s+include\b/,
+    /\bnever\s+say\b/,
+    /\bphras(e|ing)\b/,
+  ];
+  let promptHits = 0;
+  for (const re of PROMPT_RE) {
+    if (re.test(combined)) promptHits++;
+  }
+
+  // Length heuristic — structural patterns need elaboration; very short rules are
+  // either trivial or prompt fragments.
+  const ruleLen = (pattern.rule || '').length;
+
+  if (structuralHits >= 2 && promptHits === 0) {
+    reasons.push(`structural markers: ${structuralHits} hit(s)`);
+    return { kind: 'structural', reasons };
+  }
+  if (promptHits >= 1 && structuralHits < 2) {
+    reasons.push(`prompt-fragment markers: ${promptHits} hit(s)`);
+    if (ruleLen < 200) reasons.push(`short rule (${ruleLen} chars) — typical of prompt tweaks`);
+    return { kind: 'prompt-fragment', reasons };
+  }
+  if (ruleLen < 100) {
+    reasons.push(`very short rule (${ruleLen} chars) — likely too narrow to generalize`);
+    return { kind: 'prompt-fragment', reasons };
+  }
+  reasons.push('no clear signal either way');
+  return { kind: 'unclear', reasons };
+}
+
+/**
+ * Parse a topic file: returns { frontmatter, body, patterns }.
+ * frontmatter is the parsed YAML-ish (we use a minimal parser since files are
+ * always written by us — no general YAML dependency needed).
+ */
+function readTopicFile(filePath) {
+  const content = fs.readFileSync(filePath, 'utf-8');
+  const fmMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!fmMatch) {
+    return { frontmatter: { topic: '', patterns: [] }, body: content, _raw: content };
+  }
+  const fmText = fmMatch[1];
+  const body = fmMatch[2];
+  const fm = parseSimpleFrontmatter(fmText);
+  return { frontmatter: fm, body, _raw: content };
+}
+
+/**
+ * Parse our own structured frontmatter shape:
+ *   topic: <name>
+ *   last_updated: <ISO>
+ *   patterns:
+ *     - id: P-001
+ *       summary: ...
+ *       promoted_at: ...
+ *       source_experiments: [a, b]
+ */
+function parseSimpleFrontmatter(text) {
+  const out = { topic: '', last_updated: '', patterns: [] };
+  const lines = text.split('\n');
+  let inPatterns = false;
+  let current = null;
+  for (const line of lines) {
+    if (line === 'patterns:') { inPatterns = true; continue; }
+    if (!inPatterns) {
+      const m = line.match(/^([a-z_]+):\s*(.*)$/);
+      if (m) out[m[1]] = m[2].trim();
+      continue;
+    }
+    // Inside patterns
+    if (line.startsWith('  - id:')) {
+      if (current) out.patterns.push(current);
+      current = { id: line.replace(/^\s*- id:\s*/, '').trim() };
+    } else if (current) {
+      const m = line.match(/^\s+([a-z_]+):\s*(.*)$/);
+      if (m) {
+        const key = m[1];
+        let val = m[2].trim();
+        if (val.startsWith('[') && val.endsWith(']')) {
+          val = val.slice(1, -1).split(',').map(s => s.trim()).filter(Boolean);
+        }
+        current[key] = val;
+      }
+    }
+  }
+  if (current) out.patterns.push(current);
+  return out;
+}
+
+function serializeTopicFile(topic, patterns, body) {
+  const ts = new Date().toISOString();
+  let fm = `topic: ${topic}\nlast_updated: ${ts}\n`;
+  fm += `patterns:\n`;
+  for (const p of patterns) {
+    fm += `  - id: ${p.id}\n`;
+    fm += `    summary: ${(p.summary || '').replace(/\n/g, ' ')}\n`;
+    fm += `    promoted_at: ${p.promoted_at || ts}\n`;
+    const srcExps = Array.isArray(p.source_experiments) ? p.source_experiments : [];
+    fm += `    source_experiments: [${srcExps.join(', ')}]\n`;
+  }
+  return `---\n${fm}---\n${body}`;
+}
+
+function buildPatternBody(pattern) {
+  const applies = pattern.applies_in || '';
+  return [
+    ``,
+    `## ${pattern.id} — ${pattern.summary}`,
+    ``,
+    `**Evidence:** ${pattern.evidence || '(no evidence captured)'}`,
+    ``,
+    `**Rule:** ${pattern.rule}`,
+    ``,
+    applies ? `**Applies in:** ${applies}\n` : '',
+  ].join('\n');
+}
+
+/**
+ * Promote a pattern into a topic file under learnings/{scope}/{topic}.md.
+ *
+ * @param {object} pattern - { id, summary, evidence, rule, applies_in?, source_experiments? }
+ * @param {object} opts
+ * @param {string} opts.scope - 'universal' | 'internal'
+ * @param {string} opts.topic - topic file name (no .md extension)
+ * @param {string} opts.sourceRoot - PAN source repo root
+ * @returns {object} { promoted_to, pattern_id, scope, topic } or { error }
+ */
+function promotePattern(pattern, opts) {
+  const validationError = validatePromoteInputs(pattern, opts);
+  if (validationError) return { error: validationError };
+
+  const { scope, topic, sourceRoot } = opts;
+  const learningsDir = getLearningsDir(sourceRoot, scope);
+
+  // P-RES-007 gate: warn (don't block) when a pattern that looks like a
+  // prompt fragment is being promoted to UNIVERSAL scope. Prompt fragments
+  // don't generalize across models/runtimes per Sakana DGM (2025); they
+  // should stay in internal scope. The check is HEURISTIC — final call is
+  // still the human's. We attach the warning to the result object.
+  let scopeWarning = null;
+  if (scope === 'universal') {
+    const classification = classifyPatternKind(pattern);
+    if (classification.kind === 'prompt-fragment') {
+      scopeWarning = {
+        code: 'P-RES-007',
+        kind: classification.kind,
+        message: `Pattern looks like a prompt-fragment (specific phrasing) rather than a structural pattern. Per P-RES-007 (Sakana DGM, 2025), prompt tweaks don't generalize across models — universal scope should be reserved for structural changes. Consider --scope internal instead, or reword the rule to describe the SHAPE, not the WORDS.`,
+        reasons: classification.reasons,
+      };
+    }
+  }
+
+  try {
+    fs.mkdirSync(learningsDir, { recursive: true });
+  } catch (err) {
+    return { error: `failed to ensure learnings dir: ${err.message}` };
+  }
+
+  const filePath = getTopicFilePath(sourceRoot, scope, topic);
+  const fileExists = fs.existsSync(filePath);
+
+  let frontmatter, body;
+  if (fileExists) {
+    const parsed = readTopicFile(filePath);
+    frontmatter = parsed.frontmatter;
+    body = parsed.body;
+
+    // Refuse duplicate pattern id
+    if (frontmatter.patterns.some(p => p.id === pattern.id)) {
+      return { error: `pattern "${pattern.id}" is already promoted in topic "${topic}"` };
+    }
+  } else {
+    frontmatter = { topic, last_updated: '', patterns: [] };
+    body = `\n# ${capitalize(topic.replace(/-/g, ' '))} (AI-derived)\n\n` +
+           `> Auto-maintained by \`pan-tools learn promote\`. Each pattern was extracted ` +
+           `from one or more experiment runs (see source_experiments). Patterns are ` +
+           `**advisory** — orchestrators should weight them against current context.\n`;
+  }
+
+  // Append pattern to body
+  body += buildPatternBody(pattern);
+
+  // Append to frontmatter pattern list
+  const promotedAt = new Date().toISOString();
+  frontmatter.patterns.push({
+    id: pattern.id,
+    summary: pattern.summary,
+    promoted_at: promotedAt,
+    source_experiments: pattern.source_experiments || [],
+  });
+
+  const serialized = serializeTopicFile(topic, frontmatter.patterns, body);
+
+  try {
+    fs.writeFileSync(filePath, serialized);
+  } catch (err) {
+    return { error: `failed to write topic file: ${err.message}` };
+  }
+
+  const result = {
+    promoted_to: filePath,
+    pattern_id: pattern.id,
+    scope,
+    topic,
+    promoted_at: promotedAt,
+  };
+  if (scopeWarning) result.warning = scopeWarning;
+  return result;
+}
+
+function capitalize(s) {
+  return s.split(' ').map(w => w[0] ? w[0].toUpperCase() + w.slice(1) : w).join(' ');
+}
+
+/**
+ * Walk both learnings tiers and return an inventory of all promoted patterns.
+ *
+ * @param {object} opts
+ * @param {string} opts.sourceRoot
+ * @returns {object} { universal: [...], internal: [...], total }
+ */
+function listPromotedPatterns(opts = {}) {
+  const sourceRoot = opts.sourceRoot;
+  if (!sourceRoot) return { error: 'sourceRoot is required' };
+
+  const result = { universal: [], internal: [], total: 0 };
+  for (const scope of VALID_SCOPES) {
+    const dir = getLearningsDir(sourceRoot, scope);
+    if (!fs.existsSync(dir)) continue;
+    const files = fs.readdirSync(dir).filter(f => f.endsWith('.md') && f !== 'README.md');
+    for (const file of files) {
+      const filePath = path.join(dir, file);
+      try {
+        const parsed = readTopicFile(filePath);
+        const topicName = file.replace(/\.md$/, '');
+        for (const p of parsed.frontmatter.patterns) {
+          result[scope].push({
+            id: p.id,
+            summary: p.summary,
+            scope,
+            topic: topicName,
+            promoted_at: p.promoted_at,
+            source_experiments: p.source_experiments || [],
+          });
+        }
+      } catch {
+        // Skip malformed files
+      }
+    }
+  }
+  result.total = result.universal.length + result.internal.length;
+  return result;
+}
+
+/**
+ * Remove a previously-promoted pattern from a topic file. If the topic file
+ * has no patterns left after removal, the file is deleted entirely.
+ *
+ * @param {string} patternId
+ * @param {object} opts - { scope, topic, sourceRoot }
+ */
+function unpromotePattern(patternId, opts) {
+  if (!opts || !VALID_SCOPES.includes(opts.scope)) {
+    return { error: `scope must be one of: ${VALID_SCOPES.join(', ')}` };
+  }
+  if (!opts.topic || !TOPIC_RE.test(opts.topic)) {
+    return { error: 'topic invalid' };
+  }
+  if (!opts.sourceRoot) return { error: 'sourceRoot is required' };
+  if (!patternId) return { error: 'patternId is required' };
+
+  const filePath = getTopicFilePath(opts.sourceRoot, opts.scope, opts.topic);
+  if (!fs.existsSync(filePath)) {
+    return { error: `topic file not found: ${opts.topic}` };
+  }
+
+  const parsed = readTopicFile(filePath);
+  const before = parsed.frontmatter.patterns.length;
+  parsed.frontmatter.patterns = parsed.frontmatter.patterns.filter(p => p.id !== patternId);
+  if (parsed.frontmatter.patterns.length === before) {
+    return { error: `pattern "${patternId}" not found in topic "${opts.topic}"` };
+  }
+
+  // Strip the pattern's body section. Pattern body is a `## P-<id> — ...` heading
+  // followed by content until the next `## ` or end-of-file.
+  const headingRe = new RegExp(
+    `\\n## ${patternId.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}\\b[^\\n]*[\\s\\S]*?(?=\\n## |$)`,
+    ''
+  );
+  const newBody = parsed.body.replace(headingRe, '');
+
+  // If no patterns left, remove the topic file entirely
+  if (parsed.frontmatter.patterns.length === 0) {
+    try {
+      fs.unlinkSync(filePath);
+    } catch (err) {
+      return { error: `failed to delete empty topic file: ${err.message}` };
+    }
+    return { removed: patternId, file_deleted: true };
+  }
+
+  const serialized = serializeTopicFile(opts.topic, parsed.frontmatter.patterns, newBody);
+  try {
+    fs.writeFileSync(filePath, serialized);
+  } catch (err) {
+    return { error: `failed to write topic file: ${err.message}` };
+  }
+
+  return { removed: patternId, file_deleted: false };
+}
+
+// ─── End W4 promote ──────────────────────────────────────────────────────────
+
 module.exports = {
   // Session management
   initTraceSession,
@@ -640,6 +1107,11 @@ module.exports = {
   getOptimizeDir,
   getTracesDir,
   getReportsDir,
+  // W4: Self-improvement loop promote
+  promotePattern,
+  listPromotedPatterns,
+  unpromotePattern,
+  classifyPatternKind,  // P-RES-007 (v3.7.10)
   // Constants (exported for hook + tests)
   OPTIMIZE_DIR,
   TRACES_DIR,
@@ -650,4 +1122,5 @@ module.exports = {
   CURRENT_SESSION_FILE,
   EVENT_TYPES,
   IMPACT_LEVELS,
+  VALID_SCOPES,
 };