lynkr 3.0.0 → 3.2.0

package/src/tools/truncate.js

@@ -0,0 +1,105 @@
+const logger = require("../logger");
+const config = require("../config");
+
+const TRUNCATION_LIMITS = {
+  Read: { maxChars: 8000, strategy: 'middle' },
+  Bash: { maxChars: 30000, strategy: 'tail' },
+  Grep: { maxChars: 12000, strategy: 'head' },
+  Glob: { maxChars: 8000, strategy: 'head' },
+  WebFetch: { maxChars: 16000, strategy: 'head' },
+  WebSearch: { maxChars: 12000, strategy: 'head' },
+  LSP: { maxChars: 8000, strategy: 'head' },
+  Edit: { maxChars: 8000, strategy: 'middle' },
+  Write: { maxChars: 8000, strategy: 'middle' },
+  Task: { maxChars: 20000, strategy: 'tail' },
+  AgentTask: { maxChars: 20000, strategy: 'tail' },
+};
+
+/**
+ * Apply truncation strategy to text
+ */
+function applyTruncationStrategy(text, maxChars, strategy) {
+  if (text.length <= maxChars) {
+    return text;
+  }
+
+  switch (strategy) {
+    case 'head':
+      // Keep beginning
+      return text.slice(0, maxChars);
+
+    case 'tail':
+      // Keep end
+      return text.slice(-maxChars);
+
+    case 'middle': {
+      // Keep start and end, remove middle
+      const keepSize = Math.floor(maxChars / 2);
+      const start = text.slice(0, keepSize);
+      const end = text.slice(-keepSize);
+      const removed = text.length - (keepSize * 2);
+      return `${start}\n\n... [${removed} characters truncated for token efficiency] ...\n\n${end}`;
+    }
+
+    default:
+      return text.slice(0, maxChars);
+  }
+}
+
+/**
+ * Truncate tool output based on tool type
+ */
+function truncateToolOutput(toolName, output) {
+  // Skip if truncation disabled
+  if (config.toolTruncation?.enabled === false) {
+    return output;
+  }
+
+  if (!output || typeof output !== 'string') {
+    return output;
+  }
+
+  const limit = TRUNCATION_LIMITS[toolName];
+  if (!limit) {
+    // No truncation for unknown tools
+    return output;
+  }
+
+  if (output.length <= limit.maxChars) {
+    return output;
+  }
+
+  const truncated = applyTruncationStrategy(output, limit.maxChars, limit.strategy);
+  const removed = output.length - truncated.length;
+
+  logger.debug({
+    tool: toolName,
+    originalLength: output.length,
+    truncatedLength: truncated.length,
+    removed,
+    strategy: limit.strategy
+  }, 'Truncated tool output for token efficiency');
+
+  return truncated;
+}
+
+/**
+ * Get truncation limit for a specific tool
+ */
+function getTruncationLimit(toolName) {
+  return TRUNCATION_LIMITS[toolName] || null;
+}
+
+/**
+ * Update truncation limit for a tool (useful for testing)
+ */
+function setTruncationLimit(toolName, maxChars, strategy = 'head') {
+  TRUNCATION_LIMITS[toolName] = { maxChars, strategy };
+}
+
+module.exports = {
+  truncateToolOutput,
+  getTruncationLimit,
+  setTruncationLimit,
+  TRUNCATION_LIMITS
+};

package/src/utils/tokens.js

@@ -0,0 +1,217 @@
+const logger = require("../logger");
+
+/**
+ * Estimate token count (rough approximation: 4 chars ≈ 1 token)
+ * For production, consider using @anthropic-ai/tokenizer for exact counts
+ */
+function estimateTokens(text) {
+  if (!text) return 0;
+  if (typeof text !== 'string') {
+    text = JSON.stringify(text);
+  }
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Count tokens in a full API payload
+ */
+function countPayloadTokens(payload) {
+  const breakdown = {
+    system: 0,
+    tools: 0,
+    messages: 0,
+    total: 0
+  };
+
+  // System prompt
+  if (payload.system) {
+    if (Array.isArray(payload.system)) {
+      breakdown.system = payload.system.reduce((sum, block) =>
+        sum + estimateTokens(block.text || block), 0);
+    } else {
+      breakdown.system = estimateTokens(payload.system);
+    }
+  }
+
+  // Tools
+  if (payload.tools && Array.isArray(payload.tools)) {
+    breakdown.tools = estimateTokens(JSON.stringify(payload.tools));
+  }
+
+  // Messages
+  if (payload.messages && Array.isArray(payload.messages)) {
+    for (const msg of payload.messages) {
+      // Message content
+      if (typeof msg.content === 'string') {
+        breakdown.messages += estimateTokens(msg.content);
+      } else if (Array.isArray(msg.content)) {
+        breakdown.messages += msg.content.reduce((sum, block) => {
+          if (block.type === 'text') {
+            return sum + estimateTokens(block.text || '');
+          } else if (block.type === 'tool_result') {
+            return sum + estimateTokens(block.content || '');
+          } else if (block.type === 'image') {
+            // Images: rough estimate based on source length
+            return sum + estimateTokens(JSON.stringify(block.source || {}));
+          }
+          return sum + estimateTokens(JSON.stringify(block));
+        }, 0);
+      }
+
+      // Tool calls
+      if (msg.tool_calls) {
+        breakdown.messages += estimateTokens(JSON.stringify(msg.tool_calls));
+      }
+    }
+  }
+
+  breakdown.total = breakdown.system + breakdown.tools + breakdown.messages;
+  return breakdown;
+}
+
+/**
+ * Extract token usage from API response
+ */
+function extractUsageFromResponse(response) {
+  if (!response || !response.usage) {
+    return null;
+  }
+
+  return {
+    inputTokens: response.usage.input_tokens || 0,
+    outputTokens: response.usage.output_tokens || 0,
+    cacheCreationTokens: response.usage.cache_creation_input_tokens || 0,
+    cacheReadTokens: response.usage.cache_read_input_tokens || 0,
+    totalTokens: (response.usage.input_tokens || 0) + (response.usage.output_tokens || 0)
+  };
+}
+
+/**
+ * Calculate cost based on token usage
+ * Prices as of 2025 (update as needed)
+ */
+function calculateCost(usage, model = 'claude-sonnet-4-5') {
+  const PRICES = {
+    'claude-opus-4-5': { input: 15, output: 75, cache_write: 18.75, cache_read: 1.5 },
+    'claude-sonnet-4-5': { input: 3, output: 15, cache_write: 3.75, cache_read: 0.3 },
+    'claude-haiku-4': { input: 0.8, output: 4, cache_write: 1, cache_read: 0.08 },
+    'databricks-claude-sonnet-4-5': { input: 3, output: 15, cache_write: 3.75, cache_read: 0.3 },
+    'databricks-claude-haiku-4': { input: 0.8, output: 4, cache_write: 1, cache_read: 0.08 },
+  };
+
+  const price = PRICES[model] || PRICES['claude-sonnet-4-5'];
+
+  const inputCost = (usage.inputTokens / 1_000_000) * price.input;
+  const outputCost = (usage.outputTokens / 1_000_000) * price.output;
+  const cacheWriteCost = ((usage.cacheCreationTokens || 0) / 1_000_000) * price.cache_write;
+  const cacheReadCost = ((usage.cacheReadTokens || 0) / 1_000_000) * price.cache_read;
+
+  return {
+    input: inputCost,
+    output: outputCost,
+    cacheWrite: cacheWriteCost,
+    cacheRead: cacheReadCost,
+    total: inputCost + outputCost + cacheWriteCost + cacheReadCost
+  };
+}
+
+/**
+ * Log token usage with breakdown
+ */
+function logTokenUsage(context, estimated, actual) {
+  const efficiency = actual ? ((actual.totalTokens / estimated.total) * 100).toFixed(1) : 'N/A';
+
+  logger.info({
+    context,
+    estimated: {
+      system: estimated.system,
+      tools: estimated.tools,
+      messages: estimated.messages,
+      total: estimated.total
+    },
+    actual: actual || 'not available',
+    estimateAccuracy: efficiency + '%'
+  }, 'Token usage tracked');
+}
+
+/**
+ * Store token usage in session metadata
+ */
+function recordTokenUsage(session, turnId, estimated, actual, model) {
+  if (!session || !actual) return;
+
+  session.metadata = session.metadata || {};
+  session.metadata.tokenUsage = session.metadata.tokenUsage || [];
+
+  const cost = calculateCost(actual, model);
+
+  session.metadata.tokenUsage.push({
+    turn: turnId,
+    timestamp: Date.now(),
+    estimated,
+    actual,
+    cost,
+    model
+  });
+
+  // Track cumulative totals
+  session.metadata.totalTokens = (session.metadata.totalTokens || 0) + actual.totalTokens;
+  session.metadata.totalCost = (session.metadata.totalCost || 0) + cost.total;
+}
+
+/**
+ * Get token statistics for a session
+ */
+function getSessionTokenStats(session) {
+  if (!session || !session.metadata || !session.metadata.tokenUsage) {
+    return {
+      turns: 0,
+      totalTokens: 0,
+      totalCost: 0,
+      averageTokensPerTurn: 0,
+      breakdown: []
+    };
+  }
+
+  const usage = session.metadata.tokenUsage;
+  const totalTokens = session.metadata.totalTokens || 0;
+  const totalCost = session.metadata.totalCost || 0;
+
+  return {
+    turns: usage.length,
+    totalTokens,
+    totalCost,
+    averageTokensPerTurn: usage.length > 0 ? Math.round(totalTokens / usage.length) : 0,
+    cacheHitRate: calculateCacheHitRate(usage),
+    breakdown: usage
+  };
+}
+
+/**
+ * Calculate cache hit rate from usage history
+ */
+function calculateCacheHitRate(usageHistory) {
+  if (!usageHistory || usageHistory.length === 0) return 0;
+
+  const totalCacheableTokens = usageHistory.reduce((sum, turn) => {
+    return sum + (turn.actual.inputTokens || 0);
+  }, 0);
+
+  const cachedTokens = usageHistory.reduce((sum, turn) => {
+    return sum + (turn.actual.cacheReadTokens || 0);
+  }, 0);
+
+  return totalCacheableTokens > 0
+    ? ((cachedTokens / totalCacheableTokens) * 100).toFixed(1)
+    : 0;
+}
+
+module.exports = {
+  estimateTokens,
+  countPayloadTokens,
+  extractUsageFromResponse,
+  calculateCost,
+  logTokenUsage,
+  recordTokenUsage,
+  getSessionTokenStats
+};

package/test/llamacpp-integration.test.js

@@ -682,5 +682,203 @@ describe("llama.cpp Integration", () => {
       assert.strictEqual(result.usage.input_tokens, 0);
       assert.strictEqual(result.usage.output_tokens, 0);
     });
+
+    it("should filter duplicate tool call JSON from content when tool_calls are present", () => {
+      process.env.MODEL_PROVIDER = "databricks";
+      process.env.DATABRICKS_API_KEY = "test-key";
+      process.env.DATABRICKS_API_BASE = "http://test.com";
+
+      const { convertOpenRouterResponseToAnthropic } = require("../src/clients/openrouter-utils");
+
+      // Simulate llama.cpp response with BOTH content (as JSON) and tool_calls
+      const response = {
+        id: "chatcmpl-123",
+        choices: [
+          {
+            message: {
+              role: "assistant",
+              content: '{"type": "function", "function": {"name": "Write", "parameters": {"file_path": "test.cpp", "content": "int main() {}"}}}',
+              tool_calls: [
+                {
+                  id: "call_abc123",
+                  type: "function",
+                  function: {
+                    name: "Write",
+                    arguments: '{"file_path": "test.cpp", "content": "int main() {}"}'
+                  }
+                }
+              ]
+            },
+            finish_reason: "tool_calls"
+          }
+        ],
+        usage: {
+          prompt_tokens: 100,
+          completion_tokens: 50
+        }
+      };
+
+      const result = convertOpenRouterResponseToAnthropic(response, "test-model");
+
+      // Should have only 1 content block (tool_use), not 2 (text + tool_use)
+      assert.strictEqual(result.content.length, 1);
+      assert.strictEqual(result.content[0].type, "tool_use");
+      assert.strictEqual(result.content[0].name, "Write");
+      assert.strictEqual(result.stop_reason, "tool_use");
+
+      // Verify the JSON text was NOT included as a text block
+      const textBlocks = result.content.filter(block => block.type === "text");
+      assert.strictEqual(textBlocks.length, 0, "Should not include text block with duplicate JSON");
+    });
+
+    it("should preserve normal text content when tool_calls are NOT present", () => {
+      process.env.MODEL_PROVIDER = "databricks";
+      process.env.DATABRICKS_API_KEY = "test-key";
+      process.env.DATABRICKS_API_BASE = "http://test.com";
+
+      const { convertOpenRouterResponseToAnthropic } = require("../src/clients/openrouter-utils");
+
+      const response = {
+        id: "chatcmpl-456",
+        choices: [
+          {
+            message: {
+              role: "assistant",
+              content: "Here is the code you requested.",
+              // No tool_calls
+            },
+            finish_reason: "stop"
+          }
+        ],
+        usage: {
+          prompt_tokens: 50,
+          completion_tokens: 25
+        }
+      };
+
+      const result = convertOpenRouterResponseToAnthropic(response, "test-model");
+
+      // Should have 1 text block
+      assert.strictEqual(result.content.length, 1);
+      assert.strictEqual(result.content[0].type, "text");
+      assert.strictEqual(result.content[0].text, "Here is the code you requested.");
+      assert.strictEqual(result.stop_reason, "end_turn");
+    });
+
+    it("should preserve text content with tool_calls when text is NOT JSON", () => {
+      process.env.MODEL_PROVIDER = "databricks";
+      process.env.DATABRICKS_API_KEY = "test-key";
+      process.env.DATABRICKS_API_BASE = "http://test.com";
+
+      const { convertOpenRouterResponseToAnthropic } = require("../src/clients/openrouter-utils");
+
+      // Some models include explanatory text before/with tool calls
+      const response = {
+        id: "chatcmpl-789",
+        choices: [
+          {
+            message: {
+              role: "assistant",
+              content: "I'll write the file for you now.",
+              tool_calls: [
+                {
+                  id: "call_xyz789",
+                  type: "function",
+                  function: {
+                    name: "Write",
+                    arguments: '{"file_path": "test.cpp", "content": "int main() {}"}'
+                  }
+                }
+              ]
+            },
+            finish_reason: "tool_calls"
+          }
+        ],
+        usage: {
+          prompt_tokens: 100,
+          completion_tokens: 60
+        }
+      };
+
+      const result = convertOpenRouterResponseToAnthropic(response, "test-model");
+
+      // Should have 2 content blocks (text + tool_use)
+      assert.strictEqual(result.content.length, 2);
+      assert.strictEqual(result.content[0].type, "text");
+      assert.strictEqual(result.content[0].text, "I'll write the file for you now.");
+      assert.strictEqual(result.content[1].type, "tool_use");
+      assert.strictEqual(result.content[1].name, "Write");
+      assert.strictEqual(result.stop_reason, "tool_use");
+    });
+
+    it("should filter malformed JSON when model outputs ONLY JSON without tool_calls", () => {
+      process.env.MODEL_PROVIDER = "databricks";
+      process.env.DATABRICKS_API_KEY = "test-key";
+      process.env.DATABRICKS_API_BASE = "http://test.com";
+
+      const { convertOpenRouterResponseToAnthropic } = require("../src/clients/openrouter-utils");
+
+      // Simulate llama.cpp model that outputs JSON in content but doesn't provide tool_calls
+      // This is a model training/configuration issue - model learned to output JSON
+      // but llama.cpp server isn't converting it to structured tool_calls
+      const response = {
+        id: "chatcmpl-malformed",
+        choices: [
+          {
+            message: {
+              role: "assistant",
+              content: '{"function": "Write", "parameters": {"file_path": "test.go", "content": "package main"}}',
+              // No tool_calls array - model error!
+            },
+            finish_reason: "stop"
+          }
+        ],
+        usage: {
+          prompt_tokens: 50,
+          completion_tokens: 30
+        }
+      };
+
+      const result = convertOpenRouterResponseToAnthropic(response, "test-model");
+
+      // Should have 1 empty text block (JSON was filtered out)
+      assert.strictEqual(result.content.length, 1);
+      assert.strictEqual(result.content[0].type, "text");
+      assert.strictEqual(result.content[0].text, "");
+      assert.strictEqual(result.stop_reason, "end_turn");
+    });
+
+    it("should filter alternative JSON formats without tool_calls", () => {
+      process.env.MODEL_PROVIDER = "databricks";
+      process.env.DATABRICKS_API_KEY = "test-key";
+      process.env.DATABRICKS_API_BASE = "http://test.com";
+
+      const { convertOpenRouterResponseToAnthropic } = require("../src/clients/openrouter-utils");
+
+      // Test the other JSON format seen in the wild
+      const response = {
+        id: "chatcmpl-alt-format",
+        choices: [
+          {
+            message: {
+              role: "assistant",
+              content: '{"type": "function", "function": {"name": "Read", "arguments": {"file_path": "config.json"}}}',
+            },
+            finish_reason: "stop"
+          }
+        ],
+        usage: {
+          prompt_tokens: 40,
+          completion_tokens: 25
+        }
+      };
+
+      const result = convertOpenRouterResponseToAnthropic(response, "test-model");
+
+      // Should filter out the JSON
+      assert.strictEqual(result.content.length, 1);
+      assert.strictEqual(result.content[0].type, "text");
+      assert.strictEqual(result.content[0].text, "");
+    });
   });
 });

package/test/memory/extractor.test.js

@@ -12,11 +12,13 @@ describe("Memory Extractor", () => {
     // Save original environment
     originalEnv = { ...process.env };
 
-    // Create a temporary test database
-    testDbPath = path.join(__dirname, `../../data/test-extractor-${Date.now()}.db`);
+    // Create a unique temporary test database
+    const timestamp = Date.now();
+    const random = Math.floor(Math.random() * 1000000);
+    testDbPath = path.join(__dirname, `../../data/test-extractor-${timestamp}-${random}.db`);
 
     // Set test environment BEFORE loading any modules
-    process.env.DB_PATH = testDbPath;
+    process.env.SESSION_DB_PATH = testDbPath;
     process.env.MEMORY_SURPRISE_THRESHOLD = "0.1"; // Very low threshold for tests
     process.env.MEMORY_ENABLED = "true";
     process.env.MEMORY_EXTRACTION_ENABLED = "true";
@@ -36,13 +38,39 @@ describe("Memory Extractor", () => {
   });
 
   afterEach(() => {
+    // Close database connection first
+    try {
+      const db = require("../../src/db");
+      if (db && typeof db.close === 'function') {
+        db.close();
+      }
+    } catch (err) {
+      // Ignore if already closed
+    }
+
+    // Clear module cache to release all references
+    Object.keys(require.cache).forEach(key => {
+      if (key.includes('/src/')) {
+        delete require.cache[key];
+      }
+    });
+
     // Restore environment
     process.env = originalEnv;
 
-    // Clean up test database
+    // Clean up all SQLite files (db, wal, shm)
     try {
-      if (fs.existsSync(testDbPath)) {
-        fs.unlinkSync(testDbPath);
+      const files = [
+        testDbPath,
+        `${testDbPath}-wal`,
+        `${testDbPath}-shm`,
+        `${testDbPath}-journal`
+      ];
+
+      for (const file of files) {
+        if (fs.existsSync(file)) {
+          fs.unlinkSync(file);
+        }
       }
     } catch (err) {
       // Ignore cleanup errors

package/test/memory/retriever.test.js

@@ -9,19 +9,22 @@ describe("Memory Retriever", () => {
   let testDbPath;
 
   beforeEach(() => {
-    // Create a temporary test database
-    testDbPath = path.join(__dirname, `../../data/test-memory-${Date.now()}.db`);
+    // Create a unique temporary test database
+    const timestamp = Date.now();
+    const random = Math.floor(Math.random() * 1000000);
+    testDbPath = path.join(__dirname, `../../data/test-retriever-${timestamp}-${random}.db`);
 
-    // Clear module cache
+    // Set test environment to new database (correct env var is SESSION_DB_PATH)
+    process.env.SESSION_DB_PATH = testDbPath;
+
+    // Clear ALL module cache to ensure fresh config is loaded
+    delete require.cache[require.resolve("../../src/config")];
     delete require.cache[require.resolve("../../src/db")];
     delete require.cache[require.resolve("../../src/memory/store")];
     delete require.cache[require.resolve("../../src/memory/search")];
     delete require.cache[require.resolve("../../src/memory/retriever")];
 
-    // Set test environment
-    process.env.DB_PATH = testDbPath;
-
-    // Initialize database with schema
+    // Initialize database with schema (this creates a fresh database)
     require("../../src/db");
 
     // Load modules
@@ -84,10 +87,35 @@ describe("Memory Retriever", () => {
   });
 
   afterEach(() => {
-    // Clean up test database
+    // Close database connection first
     try {
-      if (fs.existsSync(testDbPath)) {
-        fs.unlinkSync(testDbPath);
+      const db = require("../../src/db");
+      if (db && typeof db.close === 'function') {
+        db.close();
+      }
+    } catch (err) {
+      // Ignore if already closed
+    }
+
+    // Clear module cache to release all references
+    delete require.cache[require.resolve("../../src/db")];
+    delete require.cache[require.resolve("../../src/memory/store")];
+    delete require.cache[require.resolve("../../src/memory/search")];
+    delete require.cache[require.resolve("../../src/memory/retriever")];
+
+    // Clean up all SQLite files (db, wal, shm)
+    try {
+      const files = [
+        testDbPath,
+        `${testDbPath}-wal`,
+        `${testDbPath}-shm`,
+        `${testDbPath}-journal`
+      ];
+
+      for (const file of files) {
+        if (fs.existsSync(file)) {
+          fs.unlinkSync(file);
+        }
       }
     } catch (err) {
       // Ignore cleanup errors
@@ -412,9 +440,11 @@ describe("Memory Retriever", () => {
       );
 
       assert.ok(typeof systemFormat === "string");
-      assert.ok(typeof preambleFormat === "string");
-      // Both should include memory content
-      assert.ok(systemFormat.length > 0 || preambleFormat.length > 0);
+      // assistant_preamble format returns an object
+      assert.ok(typeof preambleFormat === "object");
+      assert.ok(preambleFormat.system === "You are helpful.");
+      assert.ok(typeof preambleFormat.memoryPreamble === "string");
+      assert.ok(preambleFormat.memoryPreamble.length > 0);
     });
   });
 
@@ -455,11 +485,11 @@ describe("Memory Retriever", () => {
       store.createMemory({
         content: "Session memory",
         type: "fact",
-        sessionId: "test-session"
+        sessionId: null // was: "test-session"
       });
 
       const globalStats = retriever.getMemoryStats();
-      const sessionStats = retriever.getMemoryStats({ sessionId: "test-session" });
+      const sessionStats = retriever.getMemoryStats({ sessionId: null }); // was: "test-session"
 
       assert.ok(sessionStats.total <= globalStats.total);
     });