@townco/agent 0.1.138 → 0.1.140

package/dist/acp-server/adapter.js

@@ -1736,55 +1736,89 @@ export class AgentAcpAdapter {
                                 finalTokens: compactionMeta.finalTokens,
                                 hasOriginalContent: !!compactionMeta.originalContent,
                             });
+                            // Save pre-extracted sources before removing metadata
+                            const preExtractedSources = compactionMeta.preExtractedSources;
                             // Remove the metadata from rawOutput to keep it clean
                             const { _compactionMeta: _, ...cleanOutput } = rawOutput;
                             rawOutput = cleanOutput;
-                        }
-                        if (rawOutput && !this.noSession) {
-                            // IMPORTANT: Extract citations BEFORE hook execution
-                            // Hooks may compact/modify rawOutput, losing citation data
-                            // We extract from the original output to preserve citation sources
-                            const rawOutputForCitations = rawOutput;
-                            logger.debug("Extracting citations BEFORE compaction hooks", {
-                                toolCallId: outputMsg.toolCallId,
-                                toolName: toolCallBlock.title,
-                                rawOutputType: typeof rawOutputForCitations,
-                            });
-                            // Handle subagent tool outputs first
-                            const subagentResult = this.extractSubagentSources(rawOutput, outputMsg.toolCallId);
-                            if (subagentResult && subagentResult.sources.length > 0) {
-                                session.sources.push(...subagentResult.sources);
+                            // If we have pre-extracted sources from the runner (before compaction),
+                            // use them directly instead of trying to extract from compacted output
+                            if (preExtractedSources &&
+                                preExtractedSources.length > 0 &&
+                                !this.noSession) {
+                                const sourcesForSession = preExtractedSources.map((s) => ({
+                                    id: s.id,
+                                    url: s.url,
+                                    title: s.title,
+                                    toolCallId: outputMsg.toolCallId,
+                                    ...(s.snippet && { snippet: s.snippet }),
+                                    ...(s.favicon && { favicon: s.favicon }),
+                                    ...(s.sourceName && { sourceName: s.sourceName }),
+                                }));
+                                session.sources.push(...sourcesForSession);
                                 this.connection.sessionUpdate({
                                     sessionId: params.sessionId,
                                     update: {
                                         sessionUpdate: "sources",
-                                        sources: subagentResult.sources,
+                                        sources: sourcesForSession,
                                     },
                                 });
-                                logger.info("Extracted citation sources from subagent (pre-compaction)", {
+                                logger.info("Using pre-extracted sources from runner (preserved before compaction)", {
                                     toolCallId: outputMsg.toolCallId,
-                                    sourcesCount: subagentResult.sources.length,
+                                    toolName: toolCallBlock.title,
+                                    sourcesCount: sourcesForSession.length,
                                 });
                             }
-                            // Extract from regular tool outputs (WebSearch, WebFetch, MCP tools)
-                            const subagentHadSources = subagentResult && subagentResult.sources.length > 0;
-                            if (!subagentHadSources) {
-                                const extractedSources = this.extractSourcesFromToolOutput(toolCallBlock.title, rawOutput, outputMsg.toolCallId, session);
-                                if (extractedSources.length > 0) {
-                                    session.sources.push(...extractedSources);
+                        }
+                        if (rawOutput && !this.noSession) {
+                            // Check if we already have sources from pre-extraction
+                            const alreadyHasSources = session.sources.some((s) => s.toolCallId === outputMsg.toolCallId);
+                            if (!alreadyHasSources) {
+                                // IMPORTANT: Extract citations BEFORE hook execution
+                                // Hooks may compact/modify rawOutput, losing citation data
+                                // We extract from the original output to preserve citation sources
+                                const rawOutputForCitations = rawOutput;
+                                logger.debug("Extracting citations BEFORE compaction hooks", {
+                                    toolCallId: outputMsg.toolCallId,
+                                    toolName: toolCallBlock.title,
+                                    rawOutputType: typeof rawOutputForCitations,
+                                });
+                                // Handle subagent tool outputs first
+                                const subagentResult = this.extractSubagentSources(rawOutput, outputMsg.toolCallId);
+                                if (subagentResult && subagentResult.sources.length > 0) {
+                                    session.sources.push(...subagentResult.sources);
                                     this.connection.sessionUpdate({
                                         sessionId: params.sessionId,
                                         update: {
                                             sessionUpdate: "sources",
-                                            sources: extractedSources,
+                                            sources: subagentResult.sources,
                                         },
                                     });
-                                    logger.info("Extracted citation sources from tool output (pre-compaction)", {
+                                    logger.info("Extracted citation sources from subagent (pre-compaction)", {
                                         toolCallId: outputMsg.toolCallId,
-                                        toolName: toolCallBlock.title,
-                                        sourcesCount: extractedSources.length,
+                                        sourcesCount: subagentResult.sources.length,
                                     });
                                 }
+                                // Extract from regular tool outputs (WebSearch, WebFetch, MCP tools)
+                                const subagentHadSources = subagentResult && subagentResult.sources.length > 0;
+                                if (!subagentHadSources) {
+                                    const extractedSources = this.extractSourcesFromToolOutput(toolCallBlock.title, rawOutput, outputMsg.toolCallId, session);
+                                    if (extractedSources.length > 0) {
+                                        session.sources.push(...extractedSources);
+                                        this.connection.sessionUpdate({
+                                            sessionId: params.sessionId,
+                                            update: {
+                                                sessionUpdate: "sources",
+                                                sources: extractedSources,
+                                            },
+                                        });
+                                        logger.info("Extracted citation sources from tool output (pre-compaction)", {
+                                            toolCallId: outputMsg.toolCallId,
+                                            toolName: toolCallBlock.title,
+                                            sourcesCount: extractedSources.length,
+                                        });
+                                    }
+                                }
                             }
                             // Execute tool_response hooks if configured
                             const hooks = this.agent.definition.hooks ?? [];

package/dist/acp-server/session-storage.js

@@ -1,6 +1,8 @@
-import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync, } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
 import { join } from "node:path";
 import { z } from "zod";
+import { createLogger } from "../logger.js";
+const logger = createLogger("session-storage");
 /**
  * Zod schema for validating session files
  */
@@ -273,12 +275,8 @@ export class SessionStorage {
         try {
             // Write to temp file
             writeFileSync(tempPath, JSON.stringify(session, null, 2), "utf-8");
-            // Atomic rename
-            if (existsSync(sessionPath)) {
-                unlinkSync(sessionPath);
-            }
-            writeFileSync(sessionPath, readFileSync(tempPath, "utf-8"), "utf-8");
-            unlinkSync(tempPath);
+            // Atomic rename - renameSync is atomic on POSIX systems
+            renameSync(tempPath, sessionPath);
         }
         catch (error) {
             // Clean up temp file on error
@@ -424,8 +422,12 @@ export class SessionStorage {
                     sessions.push(entry);
                 }
             }
-            catch {
-                // Skip invalid sessions
+            catch (error) {
+                // Log and skip invalid sessions so they don't break the session list
+                logger.warn("Failed to load session, skipping from list", {
+                    sessionId,
+                    error: error instanceof Error ? error.message : String(error),
+                });
             }
         }
         // Sort by updatedAt, most recent first
@@ -506,11 +508,8 @@ export class SessionStorage {
         const tempPath = `${sessionPath}.tmp`;
         try {
             writeFileSync(tempPath, JSON.stringify(updatedSession, null, 2), "utf-8");
-            if (existsSync(sessionPath)) {
-                unlinkSync(sessionPath);
-            }
-            writeFileSync(sessionPath, readFileSync(tempPath, "utf-8"), "utf-8");
-            unlinkSync(tempPath);
+            // Atomic rename - renameSync is atomic on POSIX systems
+            renameSync(tempPath, sessionPath);
         }
         catch (error) {
             if (existsSync(tempPath)) {

package/dist/runner/langchain/index.js

@@ -39,6 +39,171 @@ getWeather.icon = "Cloud";
 function isPlainRecord(v) {
     return !!v && typeof v === "object" && !Array.isArray(v);
 }
+/**
+ * Extract source name from URL domain
+ */
+function getSourceNameFromUrl(url) {
+    try {
+        const domain = new URL(url).hostname.replace(/^www\./, "");
+        return domain;
+    }
+    catch {
+        return "Web";
+    }
+}
+/**
+ * Extract favicon URL from domain
+ */
+function getFaviconFromUrl(url) {
+    try {
+        const domain = new URL(url).hostname;
+        return `https://www.google.com/s2/favicons?domain=${domain}&sz=32`;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Extract citation sources from raw tool output BEFORE compaction.
+ * This preserves URLs that may be removed during LLM compaction.
+ */
+function extractSourcesBeforeCompaction(toolName, rawOutput) {
+    const sources = [];
+    let sourceCounter = 0;
+    // Parse content if it's a JSON string (common for MCP tools)
+    let actualOutput = rawOutput;
+    if (typeof rawOutput.content === "string") {
+        try {
+            const parsed = JSON.parse(rawOutput.content);
+            if (typeof parsed === "object" && parsed !== null) {
+                actualOutput = parsed;
+            }
+        }
+        catch {
+            // Not valid JSON, use rawOutput as-is
+        }
+    }
+    // Handle WebSearch (Exa) results
+    if (toolName === "WebSearch" ||
+        toolName === "web_search" ||
+        toolName === "town_web_search") {
+        const formattedResults = actualOutput.formattedForCitation;
+        if (Array.isArray(formattedResults)) {
+            for (const result of formattedResults) {
+                if (result &&
+                    typeof result === "object" &&
+                    "url" in result &&
+                    typeof result.url === "string") {
+                    sourceCounter++;
+                    const url = result.url;
+                    sources.push({
+                        id: typeof result.citationId === "number"
+                            ? String(result.citationId)
+                            : String(sourceCounter),
+                        url,
+                        title: typeof result.title === "string" ? result.title : "Untitled",
+                        snippet: typeof result.text === "string"
+                            ? result.text.slice(0, 200)
+                            : undefined,
+                        favicon: getFaviconFromUrl(url),
+                        sourceName: getSourceNameFromUrl(url),
+                    });
+                }
+            }
+        }
+        else {
+            // Fallback to raw results
+            const results = actualOutput.results;
+            if (Array.isArray(results)) {
+                for (const result of results) {
+                    if (result &&
+                        typeof result === "object" &&
+                        "url" in result &&
+                        typeof result.url === "string") {
+                        sourceCounter++;
+                        const url = result.url;
+                        sources.push({
+                            id: String(sourceCounter),
+                            url,
+                            title: typeof result.title === "string" ? result.title : "Untitled",
+                            snippet: typeof result.text === "string"
+                                ? result.text.slice(0, 200)
+                                : undefined,
+                            favicon: getFaviconFromUrl(url),
+                            sourceName: getSourceNameFromUrl(url),
+                        });
+                    }
+                }
+            }
+        }
+    }
+    // Handle library/document retrieval tools
+    const isLibraryTool = toolName.startsWith("library__") ||
+        toolName.includes("get_document") ||
+        toolName.includes("retrieve_document") ||
+        toolName.includes("bibliotecha");
+    if (isLibraryTool) {
+        const extractDocSource = (doc) => {
+            const docUrl = typeof doc.document_url === "string"
+                ? doc.document_url
+                : typeof doc.url === "string"
+                    ? doc.url
+                    : typeof doc.source_url === "string"
+                        ? doc.source_url
+                        : null;
+            const docTitle = typeof doc.title === "string" ? doc.title : null;
+            const docId = typeof doc.document_id === "number"
+                ? String(doc.document_id)
+                : typeof doc.document_id === "string"
+                    ? doc.document_id
+                    : null;
+            if (!docUrl && !docTitle)
+                return null;
+            sourceCounter++;
+            // Extract source name from source field or URL
+            let sourceName;
+            if (typeof doc.source === "string") {
+                sourceName = doc.source
+                    .split("_")
+                    .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+                    .join(" ");
+            }
+            else if (docUrl) {
+                sourceName = getSourceNameFromUrl(docUrl);
+            }
+            return {
+                id: docId || String(sourceCounter),
+                url: docUrl || "",
+                title: docTitle || docUrl || "Document",
+                snippet: typeof doc.summary === "string"
+                    ? doc.summary.slice(0, 200)
+                    : typeof doc.snippet === "string"
+                        ? doc.snippet.slice(0, 200)
+                        : undefined,
+                favicon: docUrl ? getFaviconFromUrl(docUrl) : undefined,
+                sourceName,
+            };
+        };
+        // Check for results array (library__search_keyword, library__semantic_search)
+        const results = actualOutput.results ?? actualOutput.documents;
+        if (Array.isArray(results)) {
+            for (const result of results) {
+                if (result && typeof result === "object") {
+                    const source = extractDocSource(result);
+                    if (source)
+                        sources.push(source);
+                }
+            }
+        }
+        else {
+            // Handle single document
+            const source = extractDocSource(actualOutput);
+            if (source)
+                sources.push(source);
+        }
+    }
+    return sources;
+}
 function stableStringify(value) {
     const seen = new WeakSet();
     const _stringify = (v) => {
@@ -589,6 +754,9 @@ export class LangchainAgent {
                             ? { content: result }
                             : { content: JSON.stringify(result) };
                     const outputTokens = await countToolResultTokens(rawOutput);
+                    // Extract citation sources BEFORE compaction to preserve URLs
+                    // Compaction LLM may remove URLs as "unnecessary" during summarization
+                    const preExtractedSources = extractSourcesBeforeCompaction(originalTool.name, rawOutput);
                     // Include current prompt as the last user message for better context.
                     const nowIso = new Date().toISOString();
                     const promptBlocksForHooks = req.prompt.map((block) => {
@@ -673,8 +841,20 @@ export class LangchainAgent {
                         if (typeof originalContentPath === "string") {
                             meta.originalContentPath = originalContentPath;
                         }
+                        // Include pre-extracted sources (extracted before compaction)
+                        if (preExtractedSources.length > 0) {
+                            meta.preExtractedSources = preExtractedSources;
+                        }
                         inflightCompactionMetaByToolCallId.set(toolCallId, meta);
                     }
+                    else if (preExtractedSources.length > 0) {
+                        // Even without compaction, pass pre-extracted sources to adapter
+                        // This ensures sources are always available for citation extraction
+                        inflightCompactionMetaByToolCallId.set(toolCallId, {
+                            action: "compacted", // Minimal action to indicate metadata presence
+                            preExtractedSources,
+                        });
+                    }
                     // Return compacted output to LangChain (model-visible), without metadata.
                     if (typeof result === "string") {
                         if (typeof finalOutput.content === "string") {

package/dist/runner/langchain/model-factory.js

@@ -30,6 +30,11 @@ export async function createModelFromString(modelString) {
         if (!shedAuth) {
             throw new Error("Not logged in. Run 'town login' or set SHED_API_KEY.");
         }
+        console.log("[model-factory] town- model, sending to shed:", {
+            shedUrl: shedAuth.shedUrl,
+            tokenPrefix: shedAuth.accessToken?.slice(0, 20),
+            tokenLength: shedAuth.accessToken?.length,
+        });
         return new ChatAnthropic({
             model: actualModel,
             anthropicApiUrl: `${shedAuth.shedUrl}/api/anthropic`,