@codeyam/codeyam-cli 0.1.0-staging.415103 → 0.1.0-staging.435c1ad

package/codeyam-cli/templates/skills/codeyam-memory/scripts/lib/read-json-field.mjs

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+// Lightweight jq replacement for simple field lookups.
+// Usage: node read-json-field.mjs <file> '<expression>'
+// Supports: .field.subfield, .field | length
+
+import { readFile } from 'node:fs/promises';
+
+const [, , filePath, expression] = process.argv;
+
+if (!filePath || !expression) {
+  console.error('Usage: node read-json-field.mjs <file> <expression>');
+  process.exit(1);
+}
+
+try {
+  const raw = await readFile(filePath, 'utf-8');
+  const data = JSON.parse(raw);
+  const result = evaluate(data, expression);
+  console.log(typeof result === 'string' ? result : JSON.stringify(result));
+} catch (err) {
+  console.error(err.message);
+  process.exit(1);
+}
+
+function evaluate(data, expr) {
+  const trimmed = expr.trim();
+
+  // Handle pipe: ".field | length"
+  const pipeIdx = trimmed.indexOf('|');
+  if (pipeIdx !== -1) {
+    const left = trimmed.slice(0, pipeIdx).trim();
+    const right = trimmed.slice(pipeIdx + 1).trim();
+    const intermediate = resolvePath(data, left);
+
+    if (right === 'length') {
+      if (Array.isArray(intermediate)) return intermediate.length;
+      if (typeof intermediate === 'object' && intermediate !== null) return Object.keys(intermediate).length;
+      if (typeof intermediate === 'string') return intermediate.length;
+      throw new Error(`Cannot get length of ${typeof intermediate}`);
+    }
+    throw new Error(`Unsupported pipe function: ${right}`);
+  }
+
+  return resolvePath(data, trimmed);
+}
+
+function resolvePath(data, pathExpr) {
+  // Strip leading dot: ".foo.bar" -> "foo.bar"
+  const cleaned = pathExpr.startsWith('.') ? pathExpr.slice(1) : pathExpr;
+  if (cleaned === '') return data;
+
+  const keys = cleaned.split('.');
+  let current = data;
+  for (const key of keys) {
+    if (current == null || typeof current !== 'object') {
+      throw new Error(`Cannot access '${key}' on ${JSON.stringify(current)}`);
+    }
+    current = current[key];
+  }
+  return current;
+}

package/codeyam-cli/templates/skills/codeyam-memory/scripts/lib/ripgrep-fallback.mjs

@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+// Shared ripgrep wrapper with pure-Node fallback when rg is not installed.
+
+import { execFile } from 'node:child_process';
+import { readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+/**
+ * @param {string} pattern - Regex pattern to search for
+ * @param {object} [opts]
+ * @param {string[]} [opts.types] - File type filters, e.g. ['ts', 'js']
+ * @param {string[]} [opts.globs] - Glob filters, e.g. ['!node_modules']
+ * @param {number} [opts.context] - Lines of context around matches (-C)
+ * @param {boolean} [opts.lineNumbers] - Include line numbers (-n)
+ * @param {string} [opts.cwd] - Working directory (defaults to process.cwd())
+ * @returns {Promise<string>} ripgrep-compatible output: "file:line:text\n" lines
+ */
+export async function ripgrepSearch(pattern, opts = {}) {
+  const { types = [], globs = [], context, lineNumbers = true, cwd = process.cwd() } = opts;
+
+  try {
+    return await runRipgrep(pattern, { types, globs, context, lineNumbers, cwd });
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      // rg not installed — use Node fallback
+      return nodeFallback(pattern, { types, globs, context, lineNumbers, cwd });
+    }
+    throw err;
+  }
+}
+
+function runRipgrep(pattern, { types, globs, context, lineNumbers, cwd }) {
+  return new Promise((resolve, reject) => {
+    const args = [];
+    if (lineNumbers) args.push('-n');
+    if (context != null) args.push('-C', String(context));
+    for (const t of types) args.push('--type', t);
+    for (const g of globs) args.push('--glob', g);
+    args.push(pattern, '.'); // Explicit path prevents rg from reading stdin
+
+    execFile('rg', args, { cwd, maxBuffer: 50 * 1024 * 1024 }, (err, stdout) => {
+      if (err) {
+        // Exit code 1 = no matches, 2 = error (e.g. no files searched)
+        if (err.code === 1 || err.code === 2) return resolve('');
+        // ENOENT = rg binary not found
+        if (err.code === 'ENOENT') {
+          const e = new Error('rg not found');
+          e.code = 'ENOENT';
+          return reject(e);
+        }
+        return reject(err);
+      }
+      resolve(stdout);
+    });
+  });
+}
+
+const TYPE_EXTENSIONS = {
+  ts: ['.ts', '.tsx', '.mts', '.cts'],
+  js: ['.js', '.jsx', '.mjs', '.cjs'],
+};
+
+const DEFAULT_IGNORE = ['node_modules', 'dist', 'build', '.next', '.git'];
+
+async function nodeFallback(pattern, { types = [], globs = [], context, lineNumbers = true, cwd = process.cwd() } = {}) {
+  // Build file extension filter from types
+  const extensions = new Set();
+  for (const t of types) {
+    const exts = TYPE_EXTENSIONS[t];
+    if (exts) exts.forEach((e) => extensions.add(e));
+  }
+
+  // Parse globs for negative patterns (simple heuristic: "!dirname")
+  const ignoreDirs = new Set(DEFAULT_IGNORE);
+  for (const g of globs) {
+    if (g.startsWith('!')) ignoreDirs.add(g.slice(1));
+  }
+  // Also handle glob patterns like "!*.d.ts" and "!*.map"
+  const ignoreExtensions = new Set();
+  for (const g of globs) {
+    const m = g.match(/^!\*(\.\w+(?:\.\w+)*)$/);
+    if (m) ignoreExtensions.add(m[1]);
+  }
+
+  const regex = new RegExp(pattern);
+  const contextLines = context ?? 0;
+  const results = [];
+
+  let entries;
+  try {
+    entries = await readdir(cwd, { recursive: true });
+  } catch {
+    return '';
+  }
+
+  for (const relPath of entries) {
+    // Skip ignored directories
+    const parts = relPath.split('/');
+    if (parts.some((p) => ignoreDirs.has(p))) continue;
+
+    // Check extension filters
+    if (extensions.size > 0) {
+      const hasMatchingExt = [...extensions].some((ext) => relPath.endsWith(ext));
+      if (!hasMatchingExt) continue;
+    }
+
+    // Check ignore extensions
+    if ([...ignoreExtensions].some((ext) => relPath.endsWith(ext))) continue;
+
+    const fullPath = join(cwd, relPath);
+    let content;
+    try {
+      content = await readFile(fullPath, 'utf-8');
+    } catch {
+      continue;
+    }
+
+    const lines = content.split('\n');
+    const matchIndices = [];
+    for (let i = 0; i < lines.length; i++) {
+      if (regex.test(lines[i])) matchIndices.push(i);
+    }
+
+    if (matchIndices.length === 0) continue;
+
+    if (contextLines === 0) {
+      for (const idx of matchIndices) {
+        const lineNum = idx + 1;
+        const prefix = lineNumbers ? `${relPath}:${lineNum}:` : `${relPath}:`;
+        results.push(`${prefix}${lines[idx]}`);
+      }
+    } else {
+      // With context: output blocks separated by "--"
+      const emitted = new Set();
+      for (const idx of matchIndices) {
+        const start = Math.max(0, idx - contextLines);
+        const end = Math.min(lines.length - 1, idx + contextLines);
+        for (let i = start; i <= end; i++) {
+          if (!emitted.has(i)) {
+            emitted.add(i);
+            const lineNum = i + 1;
+            const sep = i === idx ? ':' : '-';
+            const prefix = lineNumbers ? `${relPath}${sep}${lineNum}${sep}` : `${relPath}${sep}`;
+            results.push(`${prefix}${lines[i]}`);
+          }
+        }
+      }
+    }
+  }
+
+  return results.join('\n') + (results.length > 0 ? '\n' : '');
+}
+
+// Exported for testing — not part of the public API.
+export { nodeFallback as _nodeFallback };

package/codeyam-cli/templates/skills/codeyam-memory/scripts/session-mining/analyze-prompt.md

@@ -0,0 +1,46 @@
+# Session Log Analysis Prompt
+
+You are analyzing a filtered Claude Code session transcript to find instances where Claude got confused about **this specific codebase**. The transcript has been preprocessed — each line is a JSON object with `type` ("user" or "assistant"), `ts` (timestamp), and `content`.
+
+## Your Task
+
+Read the session file provided and identify moments where Claude demonstrated confusion that a `.claude/rules/` rule could have prevented.
+
+## Signal Types to Detect
+
+1. **USER_CORRECTION** — The user corrects Claude's approach, code, or assumption. Look for: "no, that's not how...", "actually...", "you need to use X not Y", "that's wrong", explicit disagreement.
+
+2. **RE_EDIT** — Claude edits the same file multiple times in quick succession, indicating the first attempt was wrong. Look for: multiple Edit/Write tool calls targeting the same file path with different content.
+
+3. **FAILED_PIVOT** — A tool call fails (test failure, bash error, type error) and Claude switches to a different approach. Look for: error in tool result followed by a new strategy in the next assistant message.
+
+4. **WRONG_ASSUMPTION** — Claude's thinking or text reveals an incorrect assumption about the codebase. Look for: statements that contradict later evidence, "I assumed...", reasoning that turns out wrong.
+
+5. **TRIBAL_KNOWLEDGE** — The user provides context Claude didn't have but needed. Look for: the user explaining how something works, why something is the way it is, historical context, non-obvious conventions.
+
+6. **APPROACH_PIVOT** — Claude abandons a call chain mid-stream and tries something completely different (without an explicit error triggering it). Look for: sudden topic/strategy shifts in assistant messages.
+
+## Rules for Reporting
+
+- **Only report codebase-specific confusion.** Generic programming mistakes (typos, syntax errors, forgetting imports for standard libraries) are NOT worth reporting.
+- **Only report findings where a `.claude/rules/` rule would prevent the confusion.** If the confusion is situational or one-off, skip it.
+- **Maximum 5 findings per session.** Prioritize the strongest, most rule-worthy signals.
+- **Be precise about file paths.** If the confusion relates to specific files, include them.
+
+## Output Format
+
+Return a JSON array (no markdown fences, no explanation outside the array). Each element:
+
+```json
+{
+  "signal": "USER_CORRECTION | RE_EDIT | FAILED_PIVOT | WRONG_ASSUMPTION | TRIBAL_KNOWLEDGE | APPROACH_PIVOT",
+  "summary": "One sentence describing what went wrong",
+  "evidence": "Brief quote or paraphrase from the session proving this happened",
+  "file_path": "path/to/relevant/file.ts or null if not file-specific",
+  "topic": "kebab-case-topic-label",
+  "rule_worthy": true,
+  "rule_worthy_reason": "Why a rule would prevent this from recurring"
+}
+```
+
+If the session contains no codebase-specific confusion worth documenting, return an empty array: `[]`

package/codeyam-cli/templates/skills/codeyam-memory/scripts/session-mining/cleanup.mjs

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+// Removes cached preprocessed session files.
+
+import { rm } from 'node:fs/promises';
+
+const OUTPUT_DIR = '/tmp/cc-session-analysis';
+
+try {
+  await rm(OUTPUT_DIR, { recursive: true, force: true });
+  console.log(`Cleaned up ${OUTPUT_DIR}`);
+} catch {
+  console.log(`Nothing to clean up (${OUTPUT_DIR} does not exist)`);
+}

package/codeyam-cli/templates/skills/codeyam-memory/scripts/session-mining/filter-session.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+// Session log filter — replaces filter.jq.
+// Strips progress/system/queue records, compresses tool inputs to 300 chars,
+// tool results to 500 chars, thinking to 1000 chars. Output is compact JSONL.
+
+import { createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+
+/**
+ * Filter a single parsed session log object.
+ * Returns the filtered object, or null if it should be skipped.
+ */
+export function filterSessionLine(obj) {
+  if (obj.type === 'assistant') {
+    const contentItems = obj.message?.content;
+    if (!Array.isArray(contentItems)) return null;
+
+    const filtered = [];
+    for (const item of contentItems) {
+      if (item.type === 'text') {
+        filtered.push({ t: 'text', text: item.text });
+      } else if (item.type === 'thinking') {
+        const thinking = item.thinking ?? '';
+        filtered.push({
+          t: 'think',
+          thinking: thinking.length > 1000 ? thinking.slice(0, 1000) + '...[truncated]' : thinking,
+        });
+      } else if (item.type === 'tool_use') {
+        const inputStr = typeof item.input === 'string' ? item.input : JSON.stringify(item.input);
+        filtered.push({
+          t: 'tool',
+          name: item.name,
+          input: inputStr.length > 300 ? inputStr.slice(0, 300) + '...' : inputStr,
+        });
+      }
+    }
+
+    return { type: 'assistant', ts: obj.timestamp, content: filtered };
+  }
+
+  if (obj.type === 'user') {
+    const msgContent = obj.message?.content;
+
+    if (typeof msgContent === 'string') {
+      return { type: 'user', ts: obj.timestamp, content: msgContent };
+    }
+
+    if (Array.isArray(msgContent)) {
+      const items = [];
+      for (const item of msgContent) {
+        if (item.type === 'tool_result') {
+          const contentStr = typeof item.content === 'string' ? item.content : JSON.stringify(item.content);
+          items.push({
+            t: 'result',
+            id: item.tool_use_id,
+            err: item.is_error ?? false,
+            content: contentStr.length > 500 ? contentStr.slice(0, 500) + '...[truncated]' : contentStr,
+          });
+        } else {
+          const text = typeof item === 'string' ? item : JSON.stringify(item);
+          items.push({
+            t: 'msg',
+            text: text.length > 500 ? text.slice(0, 500) + '...[truncated]' : text,
+          });
+        }
+      }
+      return { type: 'user', ts: obj.timestamp, content: items };
+    }
+
+    // Fallback: stringify whatever content is
+    const fallback = typeof msgContent === 'string' ? msgContent : JSON.stringify(msgContent);
+    return {
+      type: 'user',
+      ts: obj.timestamp,
+      content: fallback.length > 500 ? fallback.slice(0, 500) + '...[truncated]' : fallback,
+    };
+  }
+
+  return null;
+}
+
+// Standalone mode: node filter-session.mjs input.jsonl → filtered JSONL to stdout
+const inputFile = process.argv[2];
+if (inputFile) {
+  const rl = createInterface({ input: createReadStream(inputFile) });
+  for await (const line of rl) {
+    try {
+      const obj = JSON.parse(line);
+      const result = filterSessionLine(obj);
+      if (result) console.log(JSON.stringify(result));
+    } catch {
+      // Skip malformed lines (matching jq's silent-skip behavior)
+    }
+  }
+}

package/codeyam-cli/templates/skills/codeyam-memory/scripts/session-mining/preprocess.mjs

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+// Preprocesses Claude Code session logs for the codeyam-memory skill.
+// Replaces preprocess.sh — no jq or platform-specific stat required.
+//
+// - Finds JSONL session files (>=10KB, last 30 days)
+// - Excludes the current active session
+// - Runs JS filter for 5-50x compression
+// - Caches results in /tmp/cc-session-analysis/
+// - Prints filtered file paths to stdout (one per line)
+
+import { readdir, stat, mkdir, readFile, rm } from 'node:fs/promises';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { join, basename } from 'node:path';
+import { homedir } from 'node:os';
+import { filterSessionLine } from './filter-session.mjs';
+
+const OUTPUT_DIR = '/tmp/cc-session-analysis';
+const MIN_SIZE = 10240; // 10KB
+const MAX_AGE_DAYS = 30;
+const MAX_SESSIONS = 30;
+const FILTER_TIMEOUT = 30_000; // 30s per file
+
+// Compute Claude project directory (mirrors Claude Code's path hashing)
+const projectDir = process.cwd();
+const projectHash = projectDir.replace(/[/.]/g, '-');
+const sessionDir = join(homedir(), '.claude', 'projects', projectHash);
+
+// Check session directory exists
+try {
+  await stat(sessionDir);
+} catch {
+  process.stderr.write(`No Claude session directory found at ${sessionDir}\n`);
+  process.stderr.write('This project may not have any Claude Code session history.\n');
+  process.exit(0);
+}
+
+// Find eligible session files
+await mkdir(OUTPUT_DIR, { recursive: true });
+
+const entries = await readdir(sessionDir);
+const now = Date.now();
+const maxAgeMs = MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
+
+const sessionFiles = [];
+for (const name of entries) {
+  if (!name.endsWith('.jsonl')) continue;
+  const fullPath = join(sessionDir, name);
+  try {
+    const st = await stat(fullPath);
+    if (!st.isFile()) continue;
+    if (st.size < MIN_SIZE) continue;
+    if (now - st.mtimeMs > maxAgeMs) continue;
+    sessionFiles.push({ path: fullPath, mtimeMs: st.mtimeMs });
+  } catch {
+    continue;
+  }
+}
+
+if (sessionFiles.length === 0) process.exit(0);
+
+// Sort newest first, cap to MAX_SESSIONS
+sessionFiles.sort((a, b) => b.mtimeMs - a.mtimeMs);
+if (sessionFiles.length > MAX_SESSIONS) {
+  process.stderr.write(`Note: ${sessionFiles.length} sessions found, capping to ${MAX_SESSIONS} most recent\n`);
+}
+const sessions = sessionFiles.slice(0, MAX_SESSIONS);
+
+// Determine active session to exclude
+let activeSession = '';
+const claudeSessionId = process.env.CLAUDE_SESSION_ID;
+if (claudeSessionId) {
+  const match = sessions.find((s) => basename(s.path, '.jsonl') === claudeSessionId);
+  if (match) activeSession = match.path;
+}
+if (!activeSession && sessions.length > 0) {
+  activeSession = sessions[0].path; // newest = likely active
+}
+
+// Process each session
+for (const session of sessions) {
+  if (session.path === activeSession) continue;
+
+  const uuid = basename(session.path, '.jsonl');
+  const filtered = join(OUTPUT_DIR, `${uuid}.filtered.jsonl`);
+
+  // Check cache: skip if filtered file has a valid sentinel
+  try {
+    const firstLine = (await readFile(filtered, 'utf-8')).split('\n')[0];
+    const sentinel = JSON.parse(firstLine);
+    if (sentinel.processed_at) {
+      console.log(filtered);
+      continue;
+    }
+  } catch {
+    // No cache or invalid — reprocess
+  }
+
+  // Filter with timeout
+  try {
+    await filterFile(session.path, filtered);
+    console.log(filtered);
+  } catch (err) {
+    if (err.name === 'AbortError') {
+      process.stderr.write(`Warning: filter timed out after ${FILTER_TIMEOUT / 1000}s on ${basename(session.path)} — skipping\n`);
+      await rm(filtered, { force: true });
+    } else {
+      process.stderr.write(`Warning: error processing ${basename(session.path)}: ${err.message}\n`);
+      await rm(filtered, { force: true });
+    }
+  }
+}
+
+async function filterFile(inputPath, outputPath) {
+  const ac = new AbortController();
+  const timer = setTimeout(() => ac.abort(), FILTER_TIMEOUT);
+
+  try {
+    const st = await stat(inputPath);
+    const sentinel = JSON.stringify({
+      processed_at: new Date().toISOString(),
+      source_size: st.size,
+    });
+
+    const output = createWriteStream(outputPath);
+    output.write(sentinel + '\n');
+
+    const rl = createInterface({
+      input: createReadStream(inputPath),
+      signal: ac.signal,
+    });
+
+    let lineCount = 0;
+    for await (const line of rl) {
+      try {
+        const obj = JSON.parse(line);
+        const result = filterSessionLine(obj);
+        if (result) {
+          output.write(JSON.stringify(result) + '\n');
+          lineCount++;
+        }
+      } catch {
+        // Skip malformed lines
+      }
+    }
+
+    output.end();
+    await new Promise((resolve, reject) => {
+      output.on('finish', resolve);
+      output.on('error', reject);
+    });
+
+    // Remove files with no content beyond the sentinel
+    if (lineCount === 0) {
+      await rm(outputPath, { force: true });
+    }
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/package.json

@@ -1,30 +1,30 @@
 {
   "name": "@codeyam/codeyam-cli",
-  "version": "0.1.0-staging.415103",
+  "version": "0.1.0-staging.435c1ad",
   "description": "Local development CLI for CodeYam analysis",
   "type": "module",
   "bin": {
     "codeyam": "./codeyam-cli/src/codeyam-cli.js"
   },
   "scripts": {
-    "postinstall": "node ./scripts/finalize-analyzer.cjs && chmod +x ./node_modules/node-pty/prebuilds/darwin-arm64/spawn-helper 2>/dev/null; chmod +x ./node_modules/node-pty/prebuilds/darwin-x64/spawn-helper 2>/dev/null; true"
+    "postinstall": "node ./scripts/npm-post-install.cjs"
   },
   "dependencies": {
     "@anthropic-ai/claude-code": "^2.1.7",
-    "@anthropic-ai/sdk": "^0.74.0",
+    "@anthropic-ai/sdk": "^0.78.0",
     "@aws-sdk/client-dynamodb": "^3.990.0",
     "@aws-sdk/s3-request-presigner": "^3.990.0",
     "@aws-sdk/util-dynamodb": "^3.990.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
-    "@octokit/auth-app": "^8.1.0",
+    "@octokit/auth-app": "^8.2.0",
     "@octokit/request": "^10.0.3",
     "@octokit/rest": "^22.0.0",
     "@react-router/express": "^7.13.0",
-    "@react-router/node": "^7.11.0",
+    "@react-router/node": "^7.13.1",
     "react-router": "^7.13.0",
     "@sendgrid/mail": "^8.1.4",
     "@supabase/supabase-js": "2.88.0",
-    "better-sqlite3": "^12.4.1",
+    "better-sqlite3": "^12.6.2",
     "chalk": "^5.6.2",
     "chokidar": "^4.0.3",
     "cli-spinners": "^3.4.0",
@@ -41,11 +41,12 @@
     "kysely": "^0.28.11",
     "lru-cache": "^11.2.5",
     "lucide-react": "^0.556.0",
-    "minimatch": "^10.0.1",
+    "minimatch": "^10.2.3",
     "openai": "^6.16.0",
-    "p-queue": "^8.1.0",
+    "p-queue": "^9.1.0",
     "p-retry": "^7.1.1",
-    "pg": "^8.16.3",
+    "pg": "^8.19.0",
+    "posthog-node": "^4.0.0",
     "piscina": "^5.1.4",
     "pixelmatch": "^5.3.0",
     "playwright": "1.58.0",
@@ -61,7 +62,12 @@
     "node-pty": "^1.0.0",
     "sharp": "^0.34.5",
     "ws": "^8.18.3",
-    "simple-git": "^3.28.0",
+    "@xterm/xterm": "^6.0.0",
+    "@xterm/addon-fit": "^0.11.0",
+    "@xterm/addon-web-links": "^0.11.0",
+    "@xterm/addon-webgl": "^0.18.0",
+    "@xterm/addon-canvas": "^0.7.0",
+    "simple-git": "^3.32.2",
     "typescript": "^5.9.3",
     "uuid": "^11.1.0",
     "yargs": "^18.0.0",