@goondocks/myco 0.6.0 → 0.6.2

package/dist/{chunk-2ZBB3MQT.js → chunk-PQWQC3RF.js}

@@ -1,43 +1,61 @@
 import { createRequire as __cr } from 'node:module'; const require = __cr(import.meta.url);
 import {
-  EMBEDDING_BATCH_CONCURRENCY,
-  batchExecute
-} from "./chunk-3JCXYLHD.js";
-import {
-  isActiveSpore,
-  supersedeSpore,
-  supersededIdsSchema
-} from "./chunk-UVGAVYWZ.js";
-import {
+  ARTIFACT_TYPES,
+  CONVERSATION_HEADING,
+  VaultWriter,
+  bareSessionId,
+  buildClassificationPrompt,
+  buildExtractionPrompt,
+  buildSummaryPrompt,
+  buildTitlePrompt,
+  callout,
+  extractJson,
+  extractSection,
   formatNoteForPrompt,
   formatNotesForPrompt,
+  formatSporeBody,
+  indexNote,
+  isActiveSpore,
   loadPrompt,
   rebuildIndex,
-  stripReasoningTokens
-} from "./chunk-24DOZEUJ.js";
+  require_gray_matter,
+  sessionNoteId,
+  stripReasoningTokens,
+  supersedeSpore,
+  supersededIdsSchema
+} from "./chunk-ALBVNGCF.js";
 import {
   generateEmbedding
 } from "./chunk-RGVBGTD6.js";
 import {
   stripFrontmatter
-} from "./chunk-GNR3QAER.js";
+} from "./chunk-RY76WEN3.js";
 import {
   initFts
 } from "./chunk-6FQISQNA.js";
 import {
+  external_exports,
   require_dist
-} from "./chunk-MQSYSQ6T.js";
+} from "./chunk-JSK7L46L.js";
+import {
+  AgentRegistry,
+  claudeCodeAdapter,
+  createPerProjectAdapter
+} from "./chunk-RNWALAFP.js";
 import {
+  AI_RESPONSE_PREVIEW_CHARS,
   CHARS_PER_TOKEN,
+  COMMAND_PREVIEW_CHARS,
   CURATION_CLUSTER_SIMILARITY,
   DIGEST_LLM_REQUEST_TIMEOUT_MS,
   DIGEST_SUBSTRATE_TYPE_WEIGHTS,
   DIGEST_TIER_MIN_CONTEXT,
   EMBEDDING_INPUT_LIMIT,
   LLM_REASONING_MODE,
+  PROMPT_PREVIEW_CHARS,
   SUPERSESSION_MAX_TOKENS,
   estimateTokens
-} from "./chunk-6BSDCZ5Q.js";
+} from "./chunk-WBLTISAK.js";
 import {
   __toESM
 } from "./chunk-PZUWP5VK.js";
@@ -226,6 +244,11 @@ ${body}
     }
     this.cycleInProgress = true;
     try {
+      if (this.llm.ensureLoaded) {
+        const { context_window: contextWindow, gpu_kv_cache: gpuKvCache } = this.config.digest.intelligence;
+        this.log("debug", "Verifying digest model", { contextWindow, gpuKvCache });
+        await this.llm.ensureLoaded(contextWindow, gpuKvCache);
+      }
       for (const hook of this.prePassHooks) {
         try {
           await hook.fn();
@@ -239,11 +262,6 @@ ${body}
     }
   }
   async runCycleInternal(opts) {
-    if (this.llm.ensureLoaded) {
-      const { context_window: contextWindow, gpu_kv_cache: gpuKvCache } = this.config.digest.intelligence;
-      this.log("debug", "Verifying digest model", { contextWindow, gpuKvCache });
-      await this.llm.ensureLoaded(contextWindow, gpuKvCache);
-    }
     const startTime = Date.now();
     const fullReprocess = opts?.fullReprocess ?? false;
     const lastTimestamp = fullReprocess ? null : this.getLastCycleTimestamp();
@@ -431,7 +449,249 @@ var Metabolism = class {
   }
 };
 
+// src/daemon/processor.ts
+var SUMMARIZATION_FAILED_MARKER = "summarization failed";
+var ClassificationResponseSchema = external_exports.object({
+  artifacts: external_exports.array(external_exports.object({
+    source_path: external_exports.string(),
+    artifact_type: external_exports.enum(ARTIFACT_TYPES),
+    title: external_exports.string(),
+    tags: external_exports.array(external_exports.string()).default([])
+  })).default([])
+});
+var BufferProcessor = class {
+  constructor(backend, contextWindow = 8192, captureConfig) {
+    this.backend = backend;
+    this.contextWindow = contextWindow;
+    this.extractionMaxTokens = captureConfig?.extraction_max_tokens ?? 2048;
+    this.summaryMaxTokens = captureConfig?.summary_max_tokens ?? 512;
+    this.titleMaxTokens = captureConfig?.title_max_tokens ?? 32;
+    this.classificationMaxTokens = captureConfig?.classification_max_tokens ?? 1024;
+  }
+  extractionMaxTokens;
+  summaryMaxTokens;
+  titleMaxTokens;
+  classificationMaxTokens;
+  truncateForContext(data, maxTokens) {
+    const available = this.contextWindow - maxTokens;
+    const dataTokens = estimateTokens(data);
+    if (dataTokens <= available) return data;
+    const charBudget = available * CHARS_PER_TOKEN;
+    return data.slice(0, charBudget);
+  }
+  async process(events, sessionId) {
+    const rawPrompt = this.buildPromptForExtraction(events, sessionId);
+    const prompt = this.truncateForContext(rawPrompt, this.extractionMaxTokens);
+    try {
+      const response = await this.backend.summarize(prompt, { maxTokens: this.extractionMaxTokens, reasoning: LLM_REASONING_MODE });
+      const parsed = extractJson(response.text);
+      return {
+        summary: parsed.summary,
+        observations: parsed.observations ?? [],
+        degraded: false
+      };
+    } catch (error) {
+      return {
+        summary: `LLM processing failed for session ${sessionId}. ${events.length} events captured. Error: ${error.message}`,
+        observations: [],
+        degraded: true
+      };
+    }
+  }
+  buildPromptForExtraction(events, sessionId) {
+    const toolSummary = this.summarizeEvents(events);
+    return buildExtractionPrompt(sessionId, events.length, toolSummary, this.extractionMaxTokens);
+  }
+  async summarizeSession(conversationMarkdown, sessionId, user) {
+    const truncatedContent = this.truncateForContext(conversationMarkdown, this.summaryMaxTokens);
+    const summaryPrompt = buildSummaryPrompt(sessionId, user ?? "unknown", truncatedContent, this.summaryMaxTokens);
+    let summaryText;
+    try {
+      const response = await this.backend.summarize(summaryPrompt, { maxTokens: this.summaryMaxTokens, reasoning: LLM_REASONING_MODE });
+      summaryText = stripReasoningTokens(response.text);
+    } catch (error) {
+      summaryText = `Session ${sessionId} \u2014 ${SUMMARIZATION_FAILED_MARKER}: ${error.message}`;
+    }
+    const titlePrompt = buildTitlePrompt(summaryText, sessionId);
+    let title;
+    try {
+      const response = await this.backend.summarize(titlePrompt, { maxTokens: this.titleMaxTokens, reasoning: LLM_REASONING_MODE });
+      title = stripReasoningTokens(response.text).trim();
+    } catch {
+      title = `Session ${sessionId}`;
+    }
+    return { summary: summaryText, title };
+  }
+  async classifyArtifacts(candidates, sessionId) {
+    if (candidates.length === 0) return [];
+    const prompt = this.buildPromptForClassification(candidates, sessionId);
+    const response = await this.backend.summarize(prompt, { maxTokens: this.classificationMaxTokens, reasoning: LLM_REASONING_MODE });
+    const raw = extractJson(response.text);
+    const parsed = ClassificationResponseSchema.parse(raw);
+    return parsed.artifacts;
+  }
+  buildPromptForClassification(candidates, sessionId) {
+    return buildClassificationPrompt(sessionId, candidates, this.classificationMaxTokens);
+  }
+  summarizeEvents(events) {
+    const toolCounts = /* @__PURE__ */ new Map();
+    const filesAccessed = /* @__PURE__ */ new Set();
+    const prompts = [];
+    const aiResponses = [];
+    for (const event of events) {
+      if (event.type === "user_prompt") {
+        const prompt = String(event.prompt ?? "");
+        if (prompt) prompts.push(prompt.slice(0, PROMPT_PREVIEW_CHARS));
+        continue;
+      }
+      if (event.type === "ai_response") {
+        const content = String(event.content ?? "");
+        if (content) aiResponses.push(content.slice(0, AI_RESPONSE_PREVIEW_CHARS));
+        continue;
+      }
+      const tool = String(event.tool_name ?? event.tool ?? "unknown");
+      toolCounts.set(tool, (toolCounts.get(tool) ?? 0) + 1);
+      const input = event.tool_input ?? event.input;
+      if (input?.path) filesAccessed.add(String(input.path));
+      if (input?.file_path) filesAccessed.add(String(input.file_path));
+      if (input?.command) filesAccessed.add(`[cmd] ${String(input.command).slice(0, COMMAND_PREVIEW_CHARS)}`);
+    }
+    const lines = [];
+    if (prompts.length > 0) {
+      lines.push("### User Prompts");
+      for (const p of prompts) {
+        lines.push(`- "${p}"`);
+      }
+    }
+    lines.push("\n### Tool Usage");
+    for (const [tool, count] of toolCounts) {
+      lines.push(`- ${tool}: ${count} calls`);
+    }
+    if (filesAccessed.size > 0) {
+      lines.push("\n### Files Accessed");
+      for (const file of filesAccessed) {
+        lines.push(`- ${file}`);
+      }
+    }
+    if (aiResponses.length > 0) {
+      lines.push("\n### AI Responses");
+      for (const r of aiResponses) {
+        lines.push(`- "${r}"`);
+      }
+    }
+    return lines.join("\n");
+  }
+};
+
+// src/capture/transcript-miner.ts
+var TranscriptMiner = class {
+  registry;
+  constructor(config) {
+    this.registry = new AgentRegistry(config?.additionalAdapters);
+  }
+  /**
+   * Extract all conversation turns for a session.
+   * Convenience wrapper — delegates to getAllTurnsWithSource.
+   */
+  getAllTurns(sessionId) {
+    return this.getAllTurnsWithSource(sessionId).turns;
+  }
+  /**
+   * Extract turns using the hook-provided transcript path first (fast, no scanning),
+   * then fall back to adapter registry scanning if the path isn't provided.
+   */
+  getAllTurnsWithSource(sessionId, transcriptPath) {
+    if (transcriptPath) {
+      const result2 = this.registry.parseTurnsFromPath(transcriptPath);
+      if (result2) return result2;
+    }
+    const result = this.registry.getTranscriptTurns(sessionId);
+    if (result) return result;
+    return { turns: [], source: "none" };
+  }
+};
+function extractTurnsFromBuffer(events) {
+  const turns = [];
+  let current = null;
+  for (const event of events) {
+    const type = event.type;
+    if (type === "user_prompt") {
+      if (current) turns.push(current);
+      current = {
+        prompt: String(event.prompt ?? "").slice(0, PROMPT_PREVIEW_CHARS),
+        toolCount: 0,
+        timestamp: String(event.timestamp ?? (/* @__PURE__ */ new Date()).toISOString())
+      };
+    } else if (type === "tool_use") {
+      if (current) current.toolCount++;
+    }
+  }
+  if (current) turns.push(current);
+  return turns;
+}
+
+// src/vault/observations.ts
+function writeObservationNotes(observations, sessionId, writer, index, vaultDir) {
+  const results = [];
+  for (const obs of observations) {
+    const obsId = `${obs.type}-${sessionId.slice(-6)}-${Date.now()}`;
+    const body = formatSporeBody({
+      title: obs.title,
+      observationType: obs.type,
+      content: obs.content,
+      sessionId,
+      root_cause: obs.root_cause,
+      fix: obs.fix,
+      rationale: obs.rationale,
+      alternatives_rejected: obs.alternatives_rejected,
+      gained: obs.gained,
+      sacrificed: obs.sacrificed,
+      tags: obs.tags
+    });
+    const relativePath = writer.writeSpore({
+      id: obsId,
+      observation_type: obs.type,
+      session: sessionNoteId(sessionId),
+      tags: obs.tags,
+      content: body
+    });
+    indexNote(index, vaultDir, relativePath);
+    results.push({ id: obsId, path: relativePath, observation: obs });
+  }
+  return results;
+}
+
+// src/services/vault-ops.ts
+import fs3 from "fs";
+import path3 from "path";
+
+// src/intelligence/batch.ts
+var LLM_BATCH_CONCURRENCY = 3;
+var EMBEDDING_BATCH_CONCURRENCY = 4;
+async function batchExecute(items, fn, options) {
+  const { concurrency, onProgress } = options;
+  let succeeded = 0;
+  let failed = 0;
+  const results = [];
+  for (let i = 0; i < items.length; i += concurrency) {
+    const batch = items.slice(i, i + concurrency);
+    const settled = await Promise.allSettled(batch.map(fn));
+    for (const result of settled) {
+      if (result.status === "fulfilled") {
+        succeeded++;
+        results.push({ status: "fulfilled", value: result.value });
+      } else {
+        failed++;
+        results.push({ status: "rejected", reason: result.reason?.message ?? String(result.reason) });
+      }
+    }
+    onProgress?.(succeeded + failed, items.length);
+  }
+  return { succeeded, failed, results };
+}
+
 // src/services/vault-ops.ts
+var import_gray_matter = __toESM(require_gray_matter(), 1);
 async function runRebuild(ctx, embeddingProvider, onProgress) {
   const { index, vaultDir } = ctx;
   initFts(index);
@@ -651,14 +911,177 @@ async function runCuration(deps, dryRun) {
     superseded: totalSuperseded
   };
 }
+function updateTitleAndSummary(body, newTitle, newNarrative) {
+  let updated = body.replace(/^# .*/m, `# ${newTitle}`);
+  const summaryCallout = callout("abstract", "Summary", newNarrative);
+  updated = updated.replace(/> \[!abstract\] Summary\n(?:> .*\n?)*/m, summaryCallout + "\n");
+  return updated;
+}
+async function runReprocess(ctx, llmProvider, embeddingProvider, options, onProgress) {
+  const { vaultDir, config, index } = ctx;
+  const log = ctx.log ?? (() => {
+  });
+  const sessionFilter = options?.session;
+  const dateFilter = options?.date;
+  const failedOnly = options?.failed ?? false;
+  const skipLlm = options?.indexOnly ?? false;
+  const effectiveLlm = skipLlm ? null : llmProvider;
+  const processor = effectiveLlm ? new BufferProcessor(effectiveLlm, config.intelligence.llm.context_window, config.capture) : null;
+  const writer = new VaultWriter(vaultDir);
+  const miner = new TranscriptMiner({
+    additionalAdapters: config.capture.transcript_paths.map(
+      (p) => createPerProjectAdapter(p, claudeCodeAdapter.parseTurns)
+    )
+  });
+  const sessionsDir = path3.join(vaultDir, "sessions");
+  if (!fs3.existsSync(sessionsDir)) {
+    return { sessionsFound: 0, sessionsProcessed: 0, observationsExtracted: 0, summariesRegenerated: 0, embeddingsQueued: 0 };
+  }
+  const sessionFiles = [];
+  for (const dateDir of fs3.readdirSync(sessionsDir)) {
+    if (dateFilter && dateDir !== dateFilter) continue;
+    const datePath = path3.join(sessionsDir, dateDir);
+    if (!fs3.statSync(datePath).isDirectory()) continue;
+    for (const file of fs3.readdirSync(datePath)) {
+      if (!file.startsWith("session-") || !file.endsWith(".md")) continue;
+      const sessionId = file.replace("session-", "").replace(".md", "");
+      if (sessionFilter && !sessionId.includes(sessionFilter)) continue;
+      sessionFiles.push({ relativePath: path3.join("sessions", dateDir, file), sessionId });
+    }
+  }
+  if (sessionFiles.length === 0) {
+    return { sessionsFound: 0, sessionsProcessed: 0, observationsExtracted: 0, summariesRegenerated: 0, embeddingsQueued: 0 };
+  }
+  const tasks = [];
+  for (const { relativePath, sessionId } of sessionFiles) {
+    const rawContent = fs3.readFileSync(path3.join(vaultDir, relativePath), "utf-8");
+    const hasFailed = rawContent.includes(SUMMARIZATION_FAILED_MARKER);
+    if (failedOnly && !hasFailed) continue;
+    const { data: frontmatter, content: body } = (0, import_gray_matter.default)(rawContent);
+    const bare = bareSessionId(sessionId);
+    const turnsResult = miner.getAllTurnsWithSource(bare);
+    const conversationSection = extractSection(body, CONVERSATION_HEADING);
+    const fmEnd = rawContent.indexOf("---", 4);
+    const frontmatterBlock = rawContent.slice(0, fmEnd + 3);
+    const batchEvents = turnsResult && turnsResult.turns.length > 0 ? turnsResult.turns.map((t) => ({
+      type: "turn",
+      prompt: t.prompt,
+      tool_count: t.toolCount,
+      response: t.aiResponse ?? "",
+      timestamp: t.timestamp
+    })) : null;
+    tasks.push({ relativePath, sessionId, bare, frontmatter, frontmatterBlock, body, conversationSection, batchEvents, turnCount: turnsResult?.turns.length ?? 0, hasFailed });
+  }
+  if (tasks.length === 0) {
+    return { sessionsFound: sessionFiles.length, sessionsProcessed: 0, observationsExtracted: 0, summariesRegenerated: 0, embeddingsQueued: 0 };
+  }
+  log("info", `Reprocessing ${tasks.length} session(s)`, { filters: { session: sessionFilter, date: dateFilter, failed: failedOnly, indexOnly: skipLlm } });
+  let embeddingsQueued = 0;
+  const embedPending = [];
+  const fireEmbed = (id, text, metadata) => {
+    if (!ctx.vectorIndex) return;
+    embeddingsQueued++;
+    const vec = ctx.vectorIndex;
+    const p = generateEmbedding(embeddingProvider, text).then((emb) => {
+      vec.upsert(id, emb.embedding, metadata);
+    }).catch((err) => {
+      log("warn", `Embedding failed for ${id}`, { error: err.message });
+    });
+    embedPending.push(p);
+  };
+  let totalObservations = 0;
+  const extractionResult = await batchExecute(
+    tasks,
+    async (task) => {
+      let obs = 0;
+      if (processor && task.batchEvents) {
+        const result = await processor.process(task.batchEvents, task.bare);
+        if (result.observations.length > 0) {
+          writeObservationNotes(result.observations, task.bare, writer, index, vaultDir);
+          obs = result.observations.length;
+          for (const o of result.observations) {
+            fireEmbed(
+              `${o.type}-${task.bare.slice(-6)}-${Date.now()}`,
+              `${o.title}
+${o.content}`.slice(0, EMBEDDING_INPUT_LIMIT),
+              { type: "spore", session_id: task.bare }
+            );
+          }
+        }
+      }
+      indexNote(index, vaultDir, task.relativePath);
+      const embText = `${task.frontmatter.title ?? ""}
+${task.frontmatter.summary ?? ""}`.slice(0, EMBEDDING_INPUT_LIMIT);
+      if (embText.trim()) {
+        fireEmbed(sessionNoteId(task.bare), embText, { type: "session", session_id: task.bare });
+      }
+      return obs;
+    },
+    {
+      concurrency: LLM_BATCH_CONCURRENCY,
+      onProgress: (done, total) => onProgress?.("extraction", done, total)
+    }
+  );
+  for (const r of extractionResult.results) {
+    if (r.status === "fulfilled") totalObservations += r.value;
+  }
+  let summarized = 0;
+  if (processor) {
+    const summarizableTasks = tasks.filter((t) => t.conversationSection);
+    if (summarizableTasks.length > 0) {
+      const summaryResult = await batchExecute(
+        summarizableTasks,
+        async (task) => {
+          const user = typeof task.frontmatter.user === "string" ? task.frontmatter.user : void 0;
+          const result = await processor.summarizeSession(task.conversationSection, task.bare, user);
+          if (result.summary.includes(SUMMARIZATION_FAILED_MARKER)) {
+            log("warn", `Summarization failed for ${task.sessionId.slice(0, 12)}`);
+            return false;
+          }
+          const updatedBody = updateTitleAndSummary(task.body, result.title, result.summary);
+          fs3.writeFileSync(path3.join(vaultDir, task.relativePath), task.frontmatterBlock + updatedBody);
+          indexNote(index, vaultDir, task.relativePath);
+          return true;
+        },
+        {
+          concurrency: LLM_BATCH_CONCURRENCY,
+          onProgress: (done, total) => onProgress?.("summarization", done, total)
+        }
+      );
+      for (const r of summaryResult.results) {
+        if (r.status === "fulfilled" && r.value) summarized++;
+      }
+    }
+  }
+  await Promise.allSettled(embedPending);
+  log("info", "Reprocess completed", {
+    sessions: tasks.length,
+    observations: totalObservations,
+    summaries: summarized,
+    embeddings: embeddingsQueued
+  });
+  return {
+    sessionsFound: sessionFiles.length,
+    sessionsProcessed: tasks.length,
+    observationsExtracted: totalObservations,
+    summariesRegenerated: summarized,
+    embeddingsQueued
+  };
+}
 
 export {
   readLastTimestamp,
   appendTraceRecord,
   DigestEngine,
   Metabolism,
+  SUMMARIZATION_FAILED_MARKER,
+  BufferProcessor,
+  TranscriptMiner,
+  extractTurnsFromBuffer,
+  writeObservationNotes,
   runRebuild,
   runDigest,
-  runCuration
+  runCuration,
+  runReprocess
 };
-//# sourceMappingURL=chunk-2ZBB3MQT.js.map
+//# sourceMappingURL=chunk-PQWQC3RF.js.map