@leing2021/super-pi 0.15.0 → 0.16.0

package/extensions/ce-core/index.ts

@@ -14,6 +14,8 @@ import { createPlanDiffTool } from "./tools/plan-diff"
 import { createSessionHistoryTool } from "./tools/session-history"
 import { createPatternExtractorTool } from "./tools/pattern-extractor"
 import { filterBashOutput } from "./tools/bash-output-filter"
+import { filterReadOutput } from "./tools/read-output-filter"
+import { COMPACTION_FOCUS_INSTRUCTIONS } from "./tools/compaction-optimizer"
 
 const artifactHelperParams = Type.Object({
   repoRoot: Type.String({ description: "Repository root where workflow artifacts should be created" }),
@@ -543,6 +545,50 @@ export default function ceCoreExtension(pi: ExtensionAPI) {
       },
     }
   })
+
+  // Read output smart filter — reduces context waste from large file reads
+  pi.on("tool_result", async (event, _ctx) => {
+    if (event.toolName !== "read") return undefined
+
+    // Extract path from input
+    const path = (event.input as any)?.path ?? ""
+    if (!path) return undefined
+
+    // Extract text content from tool result
+    const textBlocks = (event.content as Array<any>)?.filter((b: any) => b.type === "text") ?? []
+    if (textBlocks.length === 0) return undefined
+
+    const output = textBlocks.map((b: any) => b.text).join("")
+    const isImage = (event.content as Array<any>)?.some((b: any) => b.type === "image") ?? false
+
+    const result = filterReadOutput({
+      path,
+      output,
+      isError: event.isError ?? false,
+      isImage,
+    })
+
+    if (!result.filtered) return undefined
+
+    return {
+      content: [{ type: "text", text: result.output }],
+      details: {
+        ...event.details,
+        readFilter: {
+          strategy: result.strategy,
+          originalBytes: result.originalBytes,
+          filteredBytes: result.filteredBytes,
+        },
+      },
+    }
+  })
+
+  // Compaction prompt optimizer — makes summaries more focused and useful
+  pi.on("session_before_compact", async (_event, _ctx) => {
+    return {
+      customInstructions: COMPACTION_FOCUS_INSTRUCTIONS,
+    }
+  })
 }
 
 
@@ -567,3 +613,5 @@ export {
 } from "./utils/artifact-paths"
 export { normalizeSlug } from "./utils/name-utils"
 export { filterBashOutput } from "./tools/bash-output-filter"
+export { filterReadOutput } from "./tools/read-output-filter"
+export { COMPACTION_FOCUS_INSTRUCTIONS, TURN_PREFIX_FOCUS_INSTRUCTIONS } from "./tools/compaction-optimizer"

package/extensions/ce-core/tools/compaction-optimizer.ts

@@ -0,0 +1,38 @@
+// ============================================================================
+// Compaction Prompt Optimizer
+// ============================================================================
+//
+// Hooks into `session_before_compact` to inject custom instructions that make
+// compaction summaries more focused and useful for coding agent context.
+//
+// This is a "prompt-only" optimization — it doesn't replace pi's compaction
+// flow, just adds focus instructions to the summarization prompt.
+
+/**
+ * Custom instructions appended to compaction summarization prompts.
+ *
+ * Goals:
+ * 1. Preserve exact technical identifiers (paths, names, error messages)
+ * 2. Be terse on reasoning process, verbose on concrete state changes
+ * 3. Summarize file reads by purpose rather than including code snippets
+ * 4. Keep Critical Context section detailed for continuation
+ */
+export const COMPACTION_FOCUS_INSTRUCTIONS = `Additional focus for this summary:
+
+1. Preserve EXACT file paths, function names, class names, variable names, and error messages — never paraphrase these
+2. For each code change, note: file path, function/class, what changed, and why
+3. Summarize file reads by their purpose (e.g., "read auth.ts to understand JWT middleware flow") rather than including code snippets
+4. Be concise on the agent's reasoning process; be verbose on concrete state changes and decisions
+5. Keep the "Critical Context" section detailed — this is what the agent needs to continue working
+6. If any tests were run, summarize results by: file, pass/fail count, and specific failure messages
+7. Note any blocked items and their exact error state`
+
+/**
+ * Additional instructions for turn-prefix summaries (split turns).
+ * These are more concise since the turn suffix is retained.
+ */
+export const TURN_PREFIX_FOCUS_INSTRUCTIONS = `Focus the turn-prefix summary on:
+- What the user originally asked for
+- Key decisions made in the prefix
+- Exact file paths and identifiers needed to understand the retained suffix
+Skip reasoning details — only keep actionable context.`

package/extensions/ce-core/tools/read-output-filter.ts

@@ -0,0 +1,506 @@
+// ============================================================================
+// Read Output Filter
+// ============================================================================
+//
+// Filters `read` tool output at the source (via tool_result hook) to reduce
+// context waste from large file reads. Symmetric with bash-output-filter.ts.
+//
+// Strategies:
+// 1. Lock / generated files → extreme compression (summary only)
+// 2. package.json → keep scripts + dep summary, drop full version lists
+// 3. Large code files → keep signatures/structure, collapse bodies
+// 4. Large markdown → keep headings + first paragraph per section
+// 5. Generic → head/tail truncation for very large files
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface ReadOutputFilterInput {
+  /** File path that was read */
+  path: string
+  /** The raw output text */
+  output: string
+  /** Whether the read resulted in an error */
+  isError?: boolean
+  /** Whether the file is an image (skip filtering) */
+  isImage?: boolean
+}
+
+export interface ReadOutputFilterResult {
+  /** Filtered output text */
+  output: string
+  /** Whether filtering was applied */
+  filtered: boolean
+  /** Original size in bytes */
+  originalBytes: number
+  /** Filtered size in bytes */
+  filteredBytes: number
+  /** The filter strategy that was applied */
+  strategy: string
+}
+
+// ============================================================================
+// File Classification
+// ============================================================================
+
+type FileCategory =
+  | "lock-file"
+  | "generated-file"
+  | "minified-file"
+  | "package-json"
+  | "code"
+  | "markdown"
+  | "config"
+  | "unknown"
+
+interface FilePattern {
+  test: (path: string) => boolean
+  category: FileCategory
+}
+
+const FILE_PATTERNS: FilePattern[] = [
+  // Lock files — extreme compression
+  { test: (p) => /(^|\/)(package-lock\.json|yarn\.lock|bun\.lock|pnpm-lock\.yaml|Gemfile\.lock|Podfile\.lock|composer\.lock|Cargo\.lock)$/.test(p), category: "lock-file" },
+  // Minified / bundle files
+  { test: (p) => /\.(min\.js|min\.css|bundle\.js|chunk\.js|vendor\.js)$/i.test(p), category: "minified-file" },
+  // Generated files
+  { test: (p) => /\.(generated\.\w+|auto\.\w+|pb\.ts)$/i.test(p) || /(?:^|\/)(generated|auto-generated|__generated__)\//i.test(p), category: "generated-file" },
+  // package.json — smart filter
+  { test: (p) => /(^|\/)package\.json$/.test(p), category: "package-json" },
+  // Markdown
+  { test: (p) => /\.(md|mdx)$/i.test(p), category: "markdown" },
+  // Config files (JSON/YAML/TOML without lock)
+  { test: (p) => /\.(jsonc?|ya?ml|toml|ini|conf|rc)$/i.test(p) && !/lock/i.test(p), category: "config" },
+  // Code files
+  { test: (p) => /\.(ts|tsx|js|jsx|py|rs|go|java|kt|rb|c|cpp|h|hpp|cs|swift|zig|scala|clj)$/i.test(p), category: "code" },
+  // Code files (alternate extensions)
+  { test: (p) => /\.(vue|svelte|astro|sol|dart|lua|r|pl|pm|sh|bash|zsh|fish|ps1)$/i.test(p), category: "code" },
+]
+
+function classifyFile(path: string): FileCategory {
+  const normalized = path.replace(/\\/g, "/")
+  for (const pattern of FILE_PATTERNS) {
+    if (pattern.test(normalized)) return pattern.category
+  }
+  return "unknown"
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** Minimum output size (bytes) to trigger any filtering */
+const MIN_FILTER_THRESHOLD = 2048 // 2KB
+
+/** Maximum size for structural code compression trigger */
+const CODE_COMPRESS_THRESHOLD = 8192 // 8KB
+
+/** Maximum filtered output size (bytes) */
+const MAX_FILTERED_SIZE = 30720 // 30KB
+
+// ============================================================================
+// Filter: Lock Files
+// ============================================================================
+
+function filterLockFile(output: string, path: string): string {
+  const bytes = Buffer.byteLength(output, "utf-8")
+  const lines = output.split("\n").length
+
+  // Try to extract name and lockfileVersion from JSON lock files
+  let summary = `Lock file: ${path.split("/").pop()}`
+
+  try {
+    const data = JSON.parse(output)
+    if (data.name) summary += `\nProject: ${data.name}`
+    if (data.lockfileVersion) summary += `\nLockfile version: ${data.lockfileVersion}`
+    if (data.packages) {
+      const pkgCount = Object.keys(data.packages).length
+      summary += `\nPackages: ${pkgCount}`
+    }
+  } catch {
+    // Not JSON (yarn.lock, etc.) — count lines
+    summary += `\nLines: ${lines}`
+  }
+
+  summary += `\nOriginal size: ${formatBytes(bytes)}`
+  return summary
+}
+
+// ============================================================================
+// Filter: Minified / Generated Files
+// ============================================================================
+
+function filterMinifiedFile(output: string, path: string): string {
+  const bytes = Buffer.byteLength(output, "utf-8")
+  const ext = path.split(".").pop() ?? ""
+  return [
+    `Minified ${ext} file: ${path.split("/").pop()}`,
+    `Size: ${formatBytes(bytes)} (${output.split("\n").length} lines)`,
+    `First 500 chars:`,
+    output.slice(0, 500),
+    "",
+    `[... ${formatBytes(bytes - 500)} omitted]`,
+  ].join("\n")
+}
+
+function filterGeneratedFile(output: string, path: string): string {
+  const bytes = Buffer.byteLength(output, "utf-8")
+  const lines = output.split("\n")
+
+  // Keep first 30 lines + last 10 lines + count
+  if (lines.length <= 50) return output
+
+  const head = lines.slice(0, 30)
+  const tail = lines.slice(-10)
+  const omitted = lines.length - 40
+
+  return [
+    ...head,
+    "",
+    `[... ${omitted} lines omitted (generated file, ${formatBytes(bytes)})]`,
+    "",
+    ...tail,
+  ].join("\n")
+}
+
+// ============================================================================
+// Filter: package.json
+// ============================================================================
+
+function filterPackageJson(output: string): string {
+  try {
+    const pkg = JSON.parse(output)
+    const sections: string[] = []
+
+    sections.push(`Name: ${pkg.name ?? "unknown"}`)
+    if (pkg.version) sections.push(`Version: ${pkg.version}`)
+    if (pkg.description) sections.push(`Description: ${pkg.description}`)
+
+    if (pkg.scripts && Object.keys(pkg.scripts).length > 0) {
+      sections.push("")
+      sections.push("Scripts:")
+      for (const [name, cmd] of Object.entries(pkg.scripts)) {
+        sections.push(`  ${name}: ${cmd}`)
+      }
+    }
+
+    if (pkg.dependencies) {
+      const depCount = Object.keys(pkg.dependencies).length
+      sections.push("")
+      sections.push(`Dependencies (${depCount}):`)
+      if (depCount <= 15) {
+        for (const [name, ver] of Object.entries(pkg.dependencies)) {
+          sections.push(`  ${name}: ${ver}`)
+        }
+      } else {
+        // Show first 10 + summary
+        const entries = Object.entries(pkg.dependencies)
+        for (const [name, ver] of entries.slice(0, 10)) {
+          sections.push(`  ${name}: ${ver}`)
+        }
+        sections.push(`  ... and ${depCount - 10} more`)
+      }
+    }
+
+    if (pkg.devDependencies) {
+      const devCount = Object.keys(pkg.devDependencies).length
+      sections.push("")
+      sections.push(`DevDependencies (${devCount}):`)
+      if (devCount <= 10) {
+        for (const [name, ver] of Object.entries(pkg.devDependencies)) {
+          sections.push(`  ${name}: ${ver}`)
+        }
+      } else {
+        const entries = Object.entries(pkg.devDependencies)
+        for (const [name, ver] of entries.slice(0, 5)) {
+          sections.push(`  ${name}: ${ver}`)
+        }
+        sections.push(`  ... and ${devCount - 5} more`)
+      }
+    }
+
+    return sections.join("\n")
+  } catch {
+    // Not valid JSON — generic truncation
+    return truncateGeneric(output)
+  }
+}
+
+// ============================================================================
+// Filter: Large Code Files — Structural Compression
+// ============================================================================
+
+/**
+ * Structural compression for code files.
+ * Keeps: imports, exports, type definitions, function/class signatures, comments with TODO/FIXME/HACK
+ * Collapses: function bodies > N lines, consecutive blank lines
+ */
+function filterCodeFile(output: string): string {
+  const lines = output.split("\n")
+  const kept: string[] = []
+  let bodyDepth = 0
+  let bodyLines = 0
+  let skippedInBody = 0
+  const MAX_BODY_LINES = 8 // keep first N lines of a body
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]
+    const trimmed = line.trim()
+
+    // Track brace depth for body detection
+    const opens = (line.match(/\{/g) || []).length
+    const closes = (line.match(/\}/g) || []).length
+
+    // Always keep: empty lines (structure), imports, exports, type/interface/class declarations
+    // comments with TODO/FIXME/HACK, decorators
+    if (
+      trimmed === "" ||
+      /^(import |export |from )/.test(trimmed) ||
+      /^(export )?(default |abstract |async |const |let |var |function |class |interface |type |enum )/.test(trimmed) ||
+      /^(\/\/|#|\/\*|\*)/.test(trimmed) ||
+      /\b(TODO|FIXME|HACK|XXX|WARN|BUG|NOTE)\b/i.test(trimmed) ||
+      /^@/.test(trimmed) || // decorators
+      /^}\s*$/.test(trimmed) // closing brace
+    ) {
+      // If we were skipping body lines, add omission marker
+      if (skippedInBody > 0) {
+        kept.push(`  [... ${skippedInBody} lines omitted]`)
+        skippedInBody = 0
+      }
+      kept.push(line)
+      bodyDepth += opens - closes
+      bodyLines = 0
+      continue
+    }
+
+    // Inside a body (after opening brace)
+    bodyDepth += opens - closes
+
+    if (bodyDepth > 1 || (bodyDepth === 1 && bodyLines < MAX_BODY_LINES)) {
+      kept.push(line)
+      bodyLines++
+    } else if (bodyDepth === 1 && bodyLines >= MAX_BODY_LINES) {
+      // Skip body lines but track count
+      skippedInBody++
+      bodyLines++
+    } else {
+      // Top-level or unknown — keep it
+      kept.push(line)
+    }
+  }
+
+  // Final omission marker
+  if (skippedInBody > 0) {
+    kept.push(`  [... ${skippedInBody} lines omitted]`)
+  }
+
+  return deduplicateEmptyLines(kept.join("\n"))
+}
+
+// ============================================================================
+// Filter: Markdown
+// ============================================================================
+
+function filterMarkdown(output: string): string {
+  const lines = output.split("\n")
+  const kept: string[] = []
+  let inParagraph = false
+  let paragraphLineCount = 0
+  let inCodeBlock = false
+
+  for (const line of lines) {
+    const trimmed = line.trim()
+
+    // Track code block state
+    if (trimmed.startsWith("```")) {
+      inCodeBlock = !inCodeBlock
+      kept.push(line)
+      continue
+    }
+
+    // Keep all lines inside code blocks
+    if (inCodeBlock) {
+      kept.push(line)
+      continue
+    }
+
+    // Always keep headings
+    if (/^#{1,6}\s/.test(trimmed)) {
+      kept.push(line)
+      inParagraph = false
+      paragraphLineCount = 0
+      continue
+    }
+
+    // Empty lines
+    if (trimmed === "") {
+      kept.push(line)
+      inParagraph = false
+      paragraphLineCount = 0
+      continue
+    }
+
+    // Keep first paragraph line after heading, skip the rest
+    if (!inParagraph) {
+      kept.push(line)
+      inParagraph = true
+      paragraphLineCount = 1
+    } else {
+      paragraphLineCount++
+      // Skip subsequent paragraph lines
+      if (paragraphLineCount === 2) {
+        kept.push(`  [... additional paragraph content omitted]`)
+      }
+    }
+  }
+
+  return deduplicateEmptyLines(kept.join("\n"))
+}
+
+// ============================================================================
+// Filter: Generic
+// ============================================================================
+
+function truncateGeneric(output: string): string {
+  const lines = output.split("\n")
+  const bytes = Buffer.byteLength(output, "utf-8")
+
+  if (lines.length <= 100) return output
+
+  const head = lines.slice(0, 50)
+  const tail = lines.slice(-10)
+  const omitted = lines.length - 60
+
+  return [
+    ...head,
+    "",
+    `[... ${omitted} lines omitted (${formatBytes(bytes)})]`,
+    "",
+    ...tail,
+  ].join("\n")
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function deduplicateEmptyLines(text: string): string {
+  return text.replace(/\n{3,}/g, "\n\n")
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`
+}
+
+function appendFilterNotice(
+  output: string,
+  originalBytes: number,
+  filteredBytes: number,
+  strategy: string,
+): string {
+  const saved = formatBytes(originalBytes - filteredBytes)
+  return `${output}\n\n[Read output filtered: ${formatBytes(originalBytes)} → ${formatBytes(filteredBytes)} (${saved} saved, strategy: ${strategy})]`
+}
+
+// ============================================================================
+// Main Filter Function
+// ============================================================================
+
+export function filterReadOutput(input: ReadOutputFilterInput): ReadOutputFilterResult {
+  const { path, output, isError, isImage } = input
+  const originalBytes = Buffer.byteLength(output, "utf-8")
+
+  // Don't filter images
+  if (isImage) {
+    return { output, filtered: false, originalBytes, filteredBytes: originalBytes, strategy: "none (image)" }
+  }
+
+  // Don't filter error outputs
+  if (isError) {
+    return { output, filtered: false, originalBytes, filteredBytes: originalBytes, strategy: "none (error)" }
+  }
+
+  // Don't filter small outputs
+  if (originalBytes < MIN_FILTER_THRESHOLD) {
+    return { output, filtered: false, originalBytes, filteredBytes: originalBytes, strategy: "none (below threshold)" }
+  }
+
+  const category = classifyFile(path)
+
+  let filtered: string
+  let strategyName: string
+
+  switch (category) {
+    case "lock-file":
+      filtered = filterLockFile(output, path)
+      strategyName = "lock-file (summary only)"
+      break
+
+    case "minified-file":
+      filtered = filterMinifiedFile(output, path)
+      strategyName = "minified (head only)"
+      break
+
+    case "generated-file":
+      filtered = filterGeneratedFile(output, path)
+      strategyName = "generated (head + tail)"
+      break
+
+    case "package-json":
+      filtered = filterPackageJson(output)
+      strategyName = "package.json (scripts + dep summary)"
+      break
+
+    case "code":
+      if (originalBytes >= CODE_COMPRESS_THRESHOLD) {
+        filtered = filterCodeFile(output)
+        strategyName = "code (structural compression)"
+      } else {
+        return { output, filtered: false, originalBytes, filteredBytes: originalBytes, strategy: "none (code file below threshold)" }
+      }
+      break
+
+    case "markdown":
+      filtered = filterMarkdown(output)
+      strategyName = "markdown (headings + first paragraph)"
+      break
+
+    case "config":
+      // Config files: only filter if very large
+      if (originalBytes > MAX_FILTERED_SIZE) {
+        filtered = truncateGeneric(output)
+        strategyName = "config (generic truncation)"
+      } else {
+        return { output, filtered: false, originalBytes, filteredBytes: originalBytes, strategy: "none (config, within limit)" }
+      }
+      break
+
+    default:
+      // Unknown files: only filter if very large
+      if (originalBytes > MAX_FILTERED_SIZE) {
+        filtered = truncateGeneric(output)
+        strategyName = "generic (large unknown file)"
+      } else {
+        return { output, filtered: false, originalBytes, filteredBytes: originalBytes, strategy: "none (unknown, within limit)" }
+      }
+      break
+  }
+
+  const filteredBytes = Buffer.byteLength(filtered, "utf-8")
+
+  // If filtering didn't help much (< 20% reduction), keep original
+  if (filteredBytes > originalBytes * 0.8) {
+    return { output, filtered: false, originalBytes, filteredBytes, strategy: `none (${category}: filtering insufficient, <20% reduction)` }
+  }
+
+  return {
+    output: appendFilterNotice(filtered, originalBytes, filteredBytes, strategyName),
+    filtered: true,
+    originalBytes,
+    filteredBytes,
+    strategy: strategyName,
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leing2021/super-pi",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "private": false,
   "type": "module",
   "description": "Pi-native Compound Engineering package for iterative development workflows",