@goondocks/myco 0.6.3 → 0.6.5

package/dist/{chunk-PQWQC3RF.js → chunk-TK7A4RX7.js}

@@ -2,6 +2,7 @@ import { createRequire as __cr } from 'node:module'; const require = __cr(import
 import {
   ARTIFACT_TYPES,
   CONVERSATION_HEADING,
+  TURN_HEADING_PREFIX,
   VaultWriter,
   bareSessionId,
   buildClassificationPrompt,
@@ -23,39 +24,32 @@ import {
   stripReasoningTokens,
   supersedeSpore,
   supersededIdsSchema
-} from "./chunk-ALBVNGCF.js";
+} from "./chunk-V6BJVYNH.js";
 import {
   generateEmbedding
 } from "./chunk-RGVBGTD6.js";
 import {
   stripFrontmatter
-} from "./chunk-RY76WEN3.js";
+} from "./chunk-GENQ5QGP.js";
 import {
   initFts
 } from "./chunk-6FQISQNA.js";
 import {
   external_exports,
   require_dist
-} from "./chunk-JSK7L46L.js";
+} from "./chunk-ERG2IEWX.js";
 import {
-  AgentRegistry,
-  claudeCodeAdapter,
-  createPerProjectAdapter
-} from "./chunk-RNWALAFP.js";
-import {
-  AI_RESPONSE_PREVIEW_CHARS,
   CHARS_PER_TOKEN,
-  COMMAND_PREVIEW_CHARS,
   CURATION_CLUSTER_SIMILARITY,
   DIGEST_LLM_REQUEST_TIMEOUT_MS,
   DIGEST_SUBSTRATE_TYPE_WEIGHTS,
+  DIGEST_TIERS,
   DIGEST_TIER_MIN_CONTEXT,
   EMBEDDING_INPUT_LIMIT,
   LLM_REASONING_MODE,
-  PROMPT_PREVIEW_CHARS,
   SUPERSESSION_MAX_TOKENS,
   estimateTokens
-} from "./chunk-WBLTISAK.js";
+} from "./chunk-DYDBF5W6.js";
 import {
   __toESM
 } from "./chunk-PZUWP5VK.js";
@@ -69,7 +63,7 @@ import crypto from "crypto";
 // src/daemon/trace.ts
 import fs from "fs";
 import path from "path";
-function readLastTimestamp(filePath) {
+function readLastRecord(filePath) {
   let content;
   try {
     content = fs.readFileSync(filePath, "utf-8").trim();
@@ -80,12 +74,14 @@ function readLastTimestamp(filePath) {
   const lines = content.split("\n");
   const lastLine = lines[lines.length - 1];
   try {
-    const record = JSON.parse(lastLine);
-    return record.timestamp ?? null;
+    return JSON.parse(lastLine);
   } catch {
     return null;
   }
 }
+function readLastTimestamp(filePath) {
+  return readLastRecord(filePath)?.timestamp ?? null;
+}
 function appendTraceRecord(filePath, record) {
   fs.mkdirSync(path.dirname(filePath), { recursive: true });
   fs.appendFileSync(filePath, JSON.stringify(record) + "\n", "utf-8");
@@ -103,6 +99,10 @@ var DigestEngine = class {
   log;
   lastCycleTimestampCache = void 0;
   cycleInProgress = false;
+  /** Whether a digest cycle is currently running. */
+  get isCycleInProgress() {
+    return this.cycleInProgress;
+  }
   /** Hooks that run before each digest cycle (e.g., consolidation). */
   prePassHooks = [];
   /** Hooks that run after each successful digest cycle. */
@@ -149,7 +149,7 @@ var DigestEngine = class {
    */
   getEligibleTiers() {
     const contextWindow = this.config.digest.intelligence.context_window;
-    return this.config.digest.tiers.filter((tier) => {
+    return DIGEST_TIERS.filter((tier) => {
       const minContext = DIGEST_TIER_MIN_CONTEXT[tier];
       return minContext !== void 0 && minContext <= contextWindow;
     });
@@ -189,7 +189,7 @@ ${note.content}`;
    * Write a digest extract to the vault with YAML frontmatter.
    * Uses atomic write pattern (temp file + rename).
    */
-  writeExtract(tier, body, cycleId, model, substrateCount) {
+  writeExtract(tier, body, cycleId, model, substrateCount, substrateNotes, tokensUsed) {
     const digestDir = path2.join(this.vaultDir, "digest");
     fs2.mkdirSync(digestDir, { recursive: true });
     const frontmatter = {
@@ -200,6 +200,8 @@ ${note.content}`;
       substrate_count: substrateCount,
       model
     };
+    if (substrateNotes && substrateNotes.length > 0) frontmatter.substrate_notes = substrateNotes;
+    if (tokensUsed !== void 0) frontmatter.tokens_used = tokensUsed;
     const fmYaml = import_yaml.default.stringify(frontmatter, {
       defaultStringType: "QUOTE_DOUBLE",
       defaultKeyType: "PLAIN"
@@ -299,8 +301,16 @@ ${body}
         substrateIndex[key].push(note.id);
       }
     }
+    this.log("debug", "Substrate breakdown", {
+      sessions: substrateIndex.sessions.length,
+      spores: substrateIndex.spores.length,
+      plans: substrateIndex.plans.length,
+      artifacts: substrateIndex.artifacts.length,
+      team: substrateIndex.team.length
+    });
     const cycleTimestamp = (/* @__PURE__ */ new Date()).toISOString();
     const systemPrompt = loadPrompt("digest-system");
+    const allSubstrateIds = substrate.map((note) => note.id);
     for (const tier of eligibleTiers) {
       const tierPrompt = loadPrompt(`digest-${tier}`);
       const previousExtract = opts?.cleanSlate ? null : this.readPreviousExtract(tier);
@@ -343,12 +353,34 @@ ${body}
         const responseTokens = estimateTokens(extractText);
         totalTokensUsed += promptTokens + responseTokens;
         this.log("info", `Tier ${tier}: completed`, { durationMs: tierDuration, responseTokens, model: response.model });
-        this.writeExtract(tier, extractText, cycleId, response.model, substrate.length);
+        this.writeExtract(tier, extractText, cycleId, response.model, substrate.length, allSubstrateIds, promptTokens + responseTokens);
         tiersGenerated.push(tier);
       } catch (err) {
         this.log("warn", `Tier ${tier}: failed`, { error: err.message });
       }
     }
+    if (tiersGenerated.length > 0) {
+      const digestDir = path2.join(this.vaultDir, "digest");
+      for (const tier of tiersGenerated) {
+        const extractPath = path2.join(digestDir, `extract-${tier}.md`);
+        try {
+          const content = fs2.readFileSync(extractPath, "utf-8");
+          const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
+          if (fmMatch) {
+            const parsed = import_yaml.default.parse(fmMatch[1]);
+            parsed.tiers_generated = tiersGenerated;
+            const fmYaml = import_yaml.default.stringify(parsed, { defaultStringType: "QUOTE_DOUBLE", defaultKeyType: "PLAIN" }).trim();
+            const extractBody = content.slice(fmMatch[0].length);
+            const tmpPath = `${extractPath}.tmp`;
+            fs2.writeFileSync(tmpPath, `---
+${fmYaml}
+---${extractBody}`, "utf-8");
+            fs2.renameSync(tmpPath, extractPath);
+          }
+        } catch {
+        }
+      }
+    }
     const result = {
       cycleId,
       timestamp: cycleTimestamp,
@@ -450,6 +482,7 @@ var Metabolism = class {
 };
 
 // src/daemon/processor.ts
+var EXTRACTION_PROMPT_OVERHEAD_TOKENS = 500;
 var SUMMARIZATION_FAILED_MARKER = "summarization failed";
 var ClassificationResponseSchema = external_exports.object({
   artifacts: external_exports.array(external_exports.object({
@@ -479,11 +512,26 @@ var BufferProcessor = class {
     const charBudget = available * CHARS_PER_TOKEN;
     return data.slice(0, charBudget);
   }
-  async process(events, sessionId) {
-    const rawPrompt = this.buildPromptForExtraction(events, sessionId);
-    const prompt = this.truncateForContext(rawPrompt, this.extractionMaxTokens);
+  async process(conversationMarkdown, sessionId) {
+    if (!conversationMarkdown.trim()) {
+      return { summary: "", observations: [], degraded: false };
+    }
+    const availableTokens = this.contextWindow - EXTRACTION_PROMPT_OVERHEAD_TOKENS - this.extractionMaxTokens;
+    const availableChars = availableTokens * CHARS_PER_TOKEN;
+    let truncated = conversationMarkdown;
+    if (conversationMarkdown.length > availableChars) {
+      truncated = conversationMarkdown.slice(-availableChars);
+      const turnBoundary = truncated.indexOf(TURN_HEADING_PREFIX);
+      if (turnBoundary > 0) {
+        truncated = truncated.slice(turnBoundary);
+      }
+    }
+    const prompt = buildExtractionPrompt(sessionId, truncated, this.extractionMaxTokens);
     try {
-      const response = await this.backend.summarize(prompt, { maxTokens: this.extractionMaxTokens, reasoning: LLM_REASONING_MODE });
+      const response = await this.backend.summarize(prompt, {
+        maxTokens: this.extractionMaxTokens,
+        reasoning: LLM_REASONING_MODE
+      });
       const parsed = extractJson(response.text);
       return {
         summary: parsed.summary,
@@ -492,16 +540,12 @@ var BufferProcessor = class {
       };
     } catch (error) {
       return {
-        summary: `LLM processing failed for session ${sessionId}. ${events.length} events captured. Error: ${error.message}`,
+        summary: `LLM processing failed for session ${sessionId}. Error: ${error.message}`,
         observations: [],
         degraded: true
       };
     }
   }
-  buildPromptForExtraction(events, sessionId) {
-    const toolSummary = this.summarizeEvents(events);
-    return buildExtractionPrompt(sessionId, events.length, toolSummary, this.extractionMaxTokens);
-  }
   async summarizeSession(conversationMarkdown, sessionId, user) {
     const truncatedContent = this.truncateForContext(conversationMarkdown, this.summaryMaxTokens);
     const summaryPrompt = buildSummaryPrompt(sessionId, user ?? "unknown", truncatedContent, this.summaryMaxTokens);
@@ -533,102 +577,7 @@ var BufferProcessor = class {
   buildPromptForClassification(candidates, sessionId) {
     return buildClassificationPrompt(sessionId, candidates, this.classificationMaxTokens);
   }
-  summarizeEvents(events) {
-    const toolCounts = /* @__PURE__ */ new Map();
-    const filesAccessed = /* @__PURE__ */ new Set();
-    const prompts = [];
-    const aiResponses = [];
-    for (const event of events) {
-      if (event.type === "user_prompt") {
-        const prompt = String(event.prompt ?? "");
-        if (prompt) prompts.push(prompt.slice(0, PROMPT_PREVIEW_CHARS));
-        continue;
-      }
-      if (event.type === "ai_response") {
-        const content = String(event.content ?? "");
-        if (content) aiResponses.push(content.slice(0, AI_RESPONSE_PREVIEW_CHARS));
-        continue;
-      }
-      const tool = String(event.tool_name ?? event.tool ?? "unknown");
-      toolCounts.set(tool, (toolCounts.get(tool) ?? 0) + 1);
-      const input = event.tool_input ?? event.input;
-      if (input?.path) filesAccessed.add(String(input.path));
-      if (input?.file_path) filesAccessed.add(String(input.file_path));
-      if (input?.command) filesAccessed.add(`[cmd] ${String(input.command).slice(0, COMMAND_PREVIEW_CHARS)}`);
-    }
-    const lines = [];
-    if (prompts.length > 0) {
-      lines.push("### User Prompts");
-      for (const p of prompts) {
-        lines.push(`- "${p}"`);
-      }
-    }
-    lines.push("\n### Tool Usage");
-    for (const [tool, count] of toolCounts) {
-      lines.push(`- ${tool}: ${count} calls`);
-    }
-    if (filesAccessed.size > 0) {
-      lines.push("\n### Files Accessed");
-      for (const file of filesAccessed) {
-        lines.push(`- ${file}`);
-      }
-    }
-    if (aiResponses.length > 0) {
-      lines.push("\n### AI Responses");
-      for (const r of aiResponses) {
-        lines.push(`- "${r}"`);
-      }
-    }
-    return lines.join("\n");
-  }
-};
-
-// src/capture/transcript-miner.ts
-var TranscriptMiner = class {
-  registry;
-  constructor(config) {
-    this.registry = new AgentRegistry(config?.additionalAdapters);
-  }
-  /**
-   * Extract all conversation turns for a session.
-   * Convenience wrapper — delegates to getAllTurnsWithSource.
-   */
-  getAllTurns(sessionId) {
-    return this.getAllTurnsWithSource(sessionId).turns;
-  }
-  /**
-   * Extract turns using the hook-provided transcript path first (fast, no scanning),
-   * then fall back to adapter registry scanning if the path isn't provided.
-   */
-  getAllTurnsWithSource(sessionId, transcriptPath) {
-    if (transcriptPath) {
-      const result2 = this.registry.parseTurnsFromPath(transcriptPath);
-      if (result2) return result2;
-    }
-    const result = this.registry.getTranscriptTurns(sessionId);
-    if (result) return result;
-    return { turns: [], source: "none" };
-  }
 };
-function extractTurnsFromBuffer(events) {
-  const turns = [];
-  let current = null;
-  for (const event of events) {
-    const type = event.type;
-    if (type === "user_prompt") {
-      if (current) turns.push(current);
-      current = {
-        prompt: String(event.prompt ?? "").slice(0, PROMPT_PREVIEW_CHARS),
-        toolCount: 0,
-        timestamp: String(event.timestamp ?? (/* @__PURE__ */ new Date()).toISOString())
-      };
-    } else if (type === "tool_use") {
-      if (current) current.toolCount++;
-    }
-  }
-  if (current) turns.push(current);
-  return turns;
-}
 
 // src/vault/observations.ts
 function writeObservationNotes(observations, sessionId, writer, index, vaultDir) {
@@ -722,6 +671,13 @@ ${note.content}`.slice(0, EMBEDDING_INPUT_LIMIT);
       onProgress
     }
   );
+  if (ctx.pipeline) {
+    for (const note of activeNotes) {
+      ctx.pipeline.register(note.id, note.type, note.path);
+      ctx.pipeline.advance(note.id, note.type, "capture", "succeeded");
+      ctx.pipeline.advance(note.id, note.type, "embedding", "succeeded");
+    }
+  }
   return {
     ftsCount,
     embeddedCount: result.succeeded,
@@ -733,6 +689,16 @@ async function runDigest(ctx, llmProvider, options) {
   const { config, vaultDir, index } = ctx;
   const log = ctx.log ? (level, message, data) => ctx.log(level, message, data) : () => {
   };
+  if (ctx.pipeline && options?.full) {
+    const items = ctx.pipeline.listItems({ stage: "digest", status: "succeeded" });
+    let reset = 0;
+    for (const item of items.items) {
+      ctx.pipeline.advance(item.id, item.item_type, "digest", "pending");
+      reset++;
+    }
+    log("info", `Reset ${reset} item(s) to digest:pending for full reprocessing`);
+    return null;
+  }
   const engine = new DigestEngine({
     vaultDir,
     index,
@@ -806,6 +772,17 @@ async function runCuration(deps, dryRun) {
   });
   const allSpores = index.query({ type: "spore" });
   const activeSpores = allSpores.filter((n) => isActiveSpore(n.frontmatter));
+  if (deps.pipeline && !dryRun) {
+    let enqueued = 0;
+    for (const spore of activeSpores) {
+      deps.pipeline.register(spore.id, "spore", spore.path);
+      deps.pipeline.advance(spore.id, "spore", "capture", "succeeded");
+      deps.pipeline.advance(spore.id, "spore", "consolidation", "pending");
+      enqueued++;
+    }
+    log("info", `Enqueued ${enqueued} spore(s) for pipeline consolidation`);
+    return { scanned: activeSpores.length, clustersEvaluated: 0, superseded: 0, enqueued: true };
+  }
   if (activeSpores.length === 0) {
     return { scanned: 0, clustersEvaluated: 0, superseded: 0 };
   }
@@ -914,7 +891,14 @@ async function runCuration(deps, dryRun) {
 function updateTitleAndSummary(body, newTitle, newNarrative) {
   let updated = body.replace(/^# .*/m, `# ${newTitle}`);
   const summaryCallout = callout("abstract", "Summary", newNarrative);
-  updated = updated.replace(/> \[!abstract\] Summary\n(?:> .*\n?)*/m, summaryCallout + "\n");
+  const hasExistingCallout = /> \[!abstract\] Summary/.test(updated);
+  if (hasExistingCallout) {
+    updated = updated.replace(/> \[!abstract\] Summary\n(?:> .*\n?)*/m, summaryCallout + "\n");
+  } else {
+    updated = updated.replace(/^(# .*\n)/m, `$1
+${summaryCallout}
+`);
+  }
   return updated;
 }
 async function runReprocess(ctx, llmProvider, embeddingProvider, options, onProgress) {
@@ -925,14 +909,6 @@ async function runReprocess(ctx, llmProvider, embeddingProvider, options, onProg
   const dateFilter = options?.date;
   const failedOnly = options?.failed ?? false;
   const skipLlm = options?.indexOnly ?? false;
-  const effectiveLlm = skipLlm ? null : llmProvider;
-  const processor = effectiveLlm ? new BufferProcessor(effectiveLlm, config.intelligence.llm.context_window, config.capture) : null;
-  const writer = new VaultWriter(vaultDir);
-  const miner = new TranscriptMiner({
-    additionalAdapters: config.capture.transcript_paths.map(
-      (p) => createPerProjectAdapter(p, claudeCodeAdapter.parseTurns)
-    )
-  });
   const sessionsDir = path3.join(vaultDir, "sessions");
   if (!fs3.existsSync(sessionsDir)) {
     return { sessionsFound: 0, sessionsProcessed: 0, observationsExtracted: 0, summariesRegenerated: 0, embeddingsQueued: 0 };
@@ -946,12 +922,42 @@ async function runReprocess(ctx, llmProvider, embeddingProvider, options, onProg
       if (!file.startsWith("session-") || !file.endsWith(".md")) continue;
       const sessionId = file.replace("session-", "").replace(".md", "");
       if (sessionFilter && !sessionId.includes(sessionFilter)) continue;
-      sessionFiles.push({ relativePath: path3.join("sessions", dateDir, file), sessionId });
+      sessionFiles.push({ relativePath: path3.join("sessions", dateDir, file), sessionId, dateDir });
     }
   }
   if (sessionFiles.length === 0) {
     return { sessionsFound: 0, sessionsProcessed: 0, observationsExtracted: 0, summariesRegenerated: 0, embeddingsQueued: 0 };
   }
+  if (ctx.pipeline && !skipLlm) {
+    let enqueued = 0;
+    let eligibleFiles = sessionFiles;
+    if (failedOnly) {
+      eligibleFiles = sessionFiles.filter(({ relativePath }) => {
+        const rawContent = fs3.readFileSync(path3.join(vaultDir, relativePath), "utf-8");
+        return rawContent.includes(SUMMARIZATION_FAILED_MARKER);
+      });
+    }
+    for (const { relativePath, sessionId } of eligibleFiles) {
+      ctx.pipeline.register(sessionId, "session", relativePath);
+      ctx.pipeline.advance(sessionId, "session", "capture", "succeeded");
+      ctx.pipeline.advance(sessionId, "session", "extraction", "pending");
+      enqueued++;
+    }
+    log("info", `Enqueued ${enqueued} session(s) for pipeline reprocessing`, {
+      filters: { session: sessionFilter, date: dateFilter, failed: failedOnly }
+    });
+    return {
+      sessionsFound: sessionFiles.length,
+      sessionsProcessed: enqueued,
+      observationsExtracted: 0,
+      summariesRegenerated: 0,
+      embeddingsQueued: 0,
+      enqueued: true
+    };
+  }
+  const effectiveLlm = skipLlm ? null : llmProvider;
+  const processor = effectiveLlm ? new BufferProcessor(effectiveLlm, config.intelligence.llm.context_window, config.capture) : null;
+  const writer = new VaultWriter(vaultDir);
   const tasks = [];
   for (const { relativePath, sessionId } of sessionFiles) {
     const rawContent = fs3.readFileSync(path3.join(vaultDir, relativePath), "utf-8");
@@ -959,18 +965,10 @@ async function runReprocess(ctx, llmProvider, embeddingProvider, options, onProg
     if (failedOnly && !hasFailed) continue;
     const { data: frontmatter, content: body } = (0, import_gray_matter.default)(rawContent);
     const bare = bareSessionId(sessionId);
-    const turnsResult = miner.getAllTurnsWithSource(bare);
     const conversationSection = extractSection(body, CONVERSATION_HEADING);
     const fmEnd = rawContent.indexOf("---", 4);
     const frontmatterBlock = rawContent.slice(0, fmEnd + 3);
-    const batchEvents = turnsResult && turnsResult.turns.length > 0 ? turnsResult.turns.map((t) => ({
-      type: "turn",
-      prompt: t.prompt,
-      tool_count: t.toolCount,
-      response: t.aiResponse ?? "",
-      timestamp: t.timestamp
-    })) : null;
-    tasks.push({ relativePath, sessionId, bare, frontmatter, frontmatterBlock, body, conversationSection, batchEvents, turnCount: turnsResult?.turns.length ?? 0, hasFailed });
+    tasks.push({ relativePath, sessionId, bare, frontmatter, frontmatterBlock, body, conversationSection, hasFailed });
   }
   if (tasks.length === 0) {
     return { sessionsFound: sessionFiles.length, sessionsProcessed: 0, observationsExtracted: 0, summariesRegenerated: 0, embeddingsQueued: 0 };
@@ -994,8 +992,8 @@ async function runReprocess(ctx, llmProvider, embeddingProvider, options, onProg
     tasks,
     async (task) => {
       let obs = 0;
-      if (processor && task.batchEvents) {
-        const result = await processor.process(task.batchEvents, task.bare);
+      if (processor && task.conversationSection.trim()) {
+        const result = await processor.process(task.conversationSection, task.bare);
         if (result.observations.length > 0) {
           writeObservationNotes(result.observations, task.bare, writer, index, vaultDir);
           obs = result.observations.length;
@@ -1070,18 +1068,18 @@ ${task.frontmatter.summary ?? ""}`.slice(0, EMBEDDING_INPUT_LIMIT);
 }
 
 export {
+  readLastRecord,
   readLastTimestamp,
   appendTraceRecord,
   DigestEngine,
   Metabolism,
   SUMMARIZATION_FAILED_MARKER,
   BufferProcessor,
-  TranscriptMiner,
-  extractTurnsFromBuffer,
   writeObservationNotes,
   runRebuild,
   runDigest,
   runCuration,
+  updateTitleAndSummary,
   runReprocess
 };
-//# sourceMappingURL=chunk-PQWQC3RF.js.map
+//# sourceMappingURL=chunk-TK7A4RX7.js.map