lynkr 8.0.0 → 9.0.1

package/src/tools/tinyfish.js

@@ -0,0 +1,358 @@
+const { URL } = require("url");
+const { Agent } = require("undici");
+const config = require("../config");
+const logger = require("../logger");
+const { registerTool } = require(".");
+
+/**
+ * Dedicated HTTP agent for TinyFish SSE streams.
+ * The default webAgent in web-client.js has a 30s bodyTimeout which is too
+ * short for browser-automation tasks that can take up to 120s.
+ */
+const sseAgent = new Agent({
+  connections: 10,
+  pipelining: 1,
+  keepAliveTimeout: 60000,
+  connectTimeout: 15000,
+  bodyTimeout: 0, // no body timeout — we manage timeout via AbortController
+  headersTimeout: 15000,
+  maxRedirections: 3,
+  strictContentLength: false,
+});
+
+// ---------------------------------------------------------------------------
+// Argument normalisers
+// ---------------------------------------------------------------------------
+
+function normalizeUrl(args) {
+  const raw = args.url ?? args.uri ?? args.href ?? args.target_url;
+  if (typeof raw !== "string" || raw.trim().length === 0) {
+    throw new Error("web_agent requires a non-empty url string.");
+  }
+  // Validate URL
+  try {
+    new URL(raw.trim());
+  } catch {
+    throw new Error(`web_agent received an invalid URL: ${raw}`);
+  }
+  return raw.trim();
+}
+
+function normalizeGoal(args) {
+  const goal = args.goal ?? args.task ?? args.prompt ?? args.instruction;
+  if (typeof goal !== "string" || goal.trim().length === 0) {
+    throw new Error("web_agent requires a non-empty goal string.");
+  }
+  return goal.trim();
+}
+
+function resolveBrowserProfile(args) {
+  const profile = args.browser_profile ?? args.browserProfile ?? config.tinyfish.browserProfile;
+  if (profile === "stealth") return "stealth";
+  return "lite";
+}
+
+// ---------------------------------------------------------------------------
+// SSE stream consumer
+// ---------------------------------------------------------------------------
+
+async function consumeSSEStream(response, timeoutMs) {
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  const startTime = Date.now();
+
+  try {
+    while (true) {
+      if (Date.now() - startTime > timeoutMs) {
+        const err = new Error(`TinyFish SSE stream timed out after ${timeoutMs}ms`);
+        err.code = "ETIMEDOUT";
+        err.status = 504;
+        throw err;
+      }
+
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      buffer += decoder.decode(value, { stream: true });
+
+      // SSE events are separated by double newlines
+      const parts = buffer.split("\n\n");
+      // Keep the last (possibly incomplete) chunk in the buffer
+      buffer = parts.pop() || "";
+
+      for (const part of parts) {
+        // Extract the data: line(s)
+        const lines = part.split("\n");
+        let dataStr = "";
+        for (const line of lines) {
+          if (line.startsWith("data: ")) {
+            dataStr += line.slice(6);
+          } else if (line.startsWith("data:")) {
+            dataStr += line.slice(5);
+          }
+        }
+
+        if (!dataStr) continue;
+
+        let event;
+        try {
+          event = JSON.parse(dataStr);
+        } catch {
+          // Not valid JSON — skip this SSE frame
+          logger.debug({ raw: dataStr.slice(0, 200) }, "TinyFish: non-JSON SSE frame, skipping");
+          continue;
+        }
+
+        logger.debug(
+          { type: event.type, status: event.status },
+          "TinyFish SSE event"
+        );
+
+        if (event.type === "COMPLETE" || event.type === "complete") {
+          const status = (event.status ?? "").toUpperCase();
+          if (status === "COMPLETED" || status === "SUCCESS") {
+            return event.resultJson ?? event.result ?? event.data ?? event;
+          }
+          // Task failed
+          const errMsg = event.error ?? event.message ?? "TinyFish task failed";
+          const err = new Error(typeof errMsg === "string" ? errMsg : JSON.stringify(errMsg));
+          err.code = "TINYFISH_TASK_FAILED";
+          err.status = 502;
+          throw err;
+        }
+      }
+    }
+
+    // Stream ended without a COMPLETE event
+    const err = new Error("TinyFish SSE stream ended without a COMPLETE event");
+    err.code = "TINYFISH_INCOMPLETE";
+    err.status = 502;
+    throw err;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Core API call
+// ---------------------------------------------------------------------------
+
+async function callTinyFishAPI({ url, goal, browserProfile, proxyConfig, timeoutMs }) {
+  const endpoint = config.tinyfish.endpoint;
+  const apiKey = config.tinyfish.apiKey;
+
+  if (!apiKey) {
+    return {
+      ok: false,
+      status: 503,
+      content: JSON.stringify({
+        error: "tinyfish_not_configured",
+        message:
+          "TinyFish API key is not configured. Set TINYFISH_API_KEY in your .env file. Get a key from https://tinyfish.ai",
+      }, null, 2),
+    };
+  }
+
+  const body = {
+    url,
+    goal,
+    browserProfile,
+  };
+
+  if (proxyConfig) {
+    body.proxy = proxyConfig;
+  }
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "X-API-Key": apiKey,
+        Accept: "text/event-stream",
+      },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+      dispatcher: sseAgent,
+    });
+
+    // Handle non-2xx responses before attempting SSE parse
+    if (!response.ok) {
+      const text = await response.text().catch(() => "");
+      const shouldRetry = response.status === 429 || response.status >= 500;
+
+      if (shouldRetry) {
+        // Retry once with 2s backoff
+        logger.warn(
+          { status: response.status, body: text.slice(0, 200) },
+          "TinyFish API error, retrying once"
+        );
+        await new Promise((r) => setTimeout(r, 2000));
+
+        const retryController = new AbortController();
+        const retryTimeout = setTimeout(() => retryController.abort(), timeoutMs);
+        try {
+          const retryResponse = await fetch(endpoint, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              "X-API-Key": apiKey,
+              Accept: "text/event-stream",
+            },
+            body: JSON.stringify(body),
+            signal: retryController.signal,
+            dispatcher: sseAgent,
+          });
+
+          if (!retryResponse.ok) {
+            const retryText = await retryResponse.text().catch(() => "");
+            const err = new Error(
+              `TinyFish API error (${retryResponse.status}): ${retryResponse.statusText}`
+            );
+            err.status = retryResponse.status;
+            err.body = retryText;
+            throw err;
+          }
+
+          const result = await consumeSSEStream(retryResponse, timeoutMs);
+          return {
+            ok: true,
+            status: 200,
+            result,
+          };
+        } finally {
+          clearTimeout(retryTimeout);
+        }
+      }
+
+      const err = new Error(
+        `TinyFish API error (${response.status}): ${response.statusText}`
+      );
+      err.status = response.status;
+      err.body = text;
+      throw err;
+    }
+
+    const result = await consumeSSEStream(response, timeoutMs);
+    return {
+      ok: true,
+      status: 200,
+      result,
+    };
+  } catch (error) {
+    if (error.name === "AbortError") {
+      const err = new Error(`TinyFish request timed out after ${timeoutMs}ms`);
+      err.code = "ETIMEDOUT";
+      err.status = 504;
+      throw err;
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Tool registration
+// ---------------------------------------------------------------------------
+
+function registerTinyFishTool() {
+  registerTool(
+    "web_agent",
+    async ({ args = {} }) => {
+      const url = normalizeUrl(args);
+      const goal = normalizeGoal(args);
+      const browserProfile = resolveBrowserProfile(args);
+      const timeoutMs = config.tinyfish.timeoutMs;
+
+      // Build proxy config if enabled
+      let proxyConfig = null;
+      if (config.tinyfish.proxyEnabled) {
+        proxyConfig = {
+          enabled: true,
+          country: config.tinyfish.proxyCountry,
+        };
+      }
+
+      try {
+        const response = await callTinyFishAPI({
+          url,
+          goal,
+          browserProfile,
+          proxyConfig,
+          timeoutMs,
+        });
+
+        // Guard clause: not configured
+        if (!response.ok && response.status === 503) {
+          return response;
+        }
+
+        const resultStr =
+          typeof response.result === "string"
+            ? response.result
+            : JSON.stringify(response.result, null, 2);
+
+        logger.debug(
+          {
+            url,
+            goal: goal.slice(0, 100),
+            browserProfile,
+            resultLength: resultStr.length,
+          },
+          "TinyFish web_agent completed"
+        );
+
+        return {
+          ok: true,
+          status: 200,
+          content: resultStr,
+          metadata: {
+            url,
+            goal,
+            browserProfile,
+            resultLength: resultStr.length,
+          },
+        };
+      } catch (err) {
+        logger.error(
+          { err, url, goal: goal.slice(0, 100) },
+          "web_agent request failed"
+        );
+        return {
+          ok: false,
+          status: err.status ?? 500,
+          content: JSON.stringify(
+            {
+              error: err.code ?? "web_agent_failed",
+              message: err.message,
+              url,
+              ...(err.status ? { http_status: err.status } : {}),
+            },
+            null,
+            2
+          ),
+          metadata: {
+            url,
+            goal,
+            error_code: err.code,
+            ...(err.status ? { http_status: err.status } : {}),
+          },
+        };
+      }
+    },
+    { category: "tinyfish" }
+  );
+}
+
+function registerTinyFishTools() {
+  registerTinyFishTool();
+}
+
+module.exports = {
+  registerTinyFishTools,
+};

package/src/tools/truncate.js

@@ -8,6 +8,7 @@ const TRUNCATION_LIMITS = {
   Glob: { maxChars: 8000, strategy: 'head' },
   WebFetch: { maxChars: 16000, strategy: 'head' },
   WebSearch: { maxChars: 12000, strategy: 'head' },
+  WebAgent: { maxChars: 16000, strategy: 'head' },
   LSP: { maxChars: 8000, strategy: 'head' },
   Edit: { maxChars: 8000, strategy: 'middle' },
   Write: { maxChars: 8000, strategy: 'middle' },

package/src/utils/payload.js

@@ -0,0 +1,206 @@
+/**
+ * Smart Payload Cloning & Size Estimation
+ *
+ * Optimizes deep-cloning of LLM request payloads to avoid
+ * wasting memory on large base64 media blocks that will be
+ * discarded by flattenBlocks() for most providers.
+ *
+ * @module utils/payload
+ */
+
+const logger = require('../logger');
+
+/**
+ * Estimate the byte size of message content without full serialization.
+ * Scans for base64 image/audio data blocks and text blocks.
+ *
+ * @param {Object} payload - Request payload
+ * @returns {number} Estimated size in bytes
+ */
+function estimateContentSize(payload) {
+  if (!payload || !Array.isArray(payload.messages)) return 0;
+
+  let size = 0;
+  for (const msg of payload.messages) {
+    if (!msg) continue;
+
+    if (typeof msg.content === 'string') {
+      size += msg.content.length;
+      continue;
+    }
+
+    if (!Array.isArray(msg.content)) continue;
+
+    for (const block of msg.content) {
+      if (!block || typeof block !== 'object') continue;
+
+      if (block.text) {
+        size += block.text.length;
+      }
+      // Anthropic image format
+      if (block.source?.data) {
+        size += block.source.data.length;
+      }
+      // OpenAI image_url format (inline base64)
+      if (block.image_url?.url && block.image_url.url.startsWith('data:')) {
+        size += block.image_url.url.length;
+      }
+      // tool_result content
+      if (block.type === 'tool_result' && typeof block.content === 'string') {
+        size += block.content.length;
+      }
+    }
+  }
+
+  return size;
+}
+
+/**
+ * Check if any message content block has base64 media data exceeding threshold.
+ *
+ * @param {Object} payload - Request payload
+ * @param {number} threshold - Size threshold in bytes (default 1MB)
+ * @returns {boolean}
+ */
+function hasLargeMedia(payload, threshold = 1_048_576) {
+  if (!payload || !Array.isArray(payload.messages)) return false;
+
+  for (const msg of payload.messages) {
+    if (!msg || !Array.isArray(msg.content)) continue;
+
+    for (const block of msg.content) {
+      if (!block || typeof block !== 'object') continue;
+
+      // Anthropic base64 image
+      if (block.source?.data && block.source.data.length > threshold) {
+        return true;
+      }
+      // OpenAI inline base64 image
+      if (block.image_url?.url && block.image_url.url.length > threshold) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+// Block types that flattenBlocks() discards (returns empty string)
+const HEAVY_BLOCK_TYPES = new Set(['image', 'audio', 'image_url', 'video']);
+
+/**
+ * Check if a content block is a heavy media block that flattenBlocks() discards.
+ * @param {Object} block
+ * @returns {boolean}
+ */
+function isHeavyMediaBlock(block) {
+  if (!block || typeof block !== 'object') return false;
+  if (HEAVY_BLOCK_TYPES.has(block.type)) return true;
+  if (block.source?.type === 'base64') return true;
+  return false;
+}
+
+/**
+ * Clone payload optimized for providers that will flatten content.
+ * Skips cloning heavy media blocks since flattenBlocks() discards them.
+ *
+ * @param {Object} payload
+ * @returns {Object} Cloned payload with media placeholders
+ */
+function cloneWithFlattenAwareness(payload) {
+  const clean = { ...payload };
+
+  // Deep-clone messages array but skip heavy media blocks
+  if (Array.isArray(payload.messages)) {
+    clean.messages = payload.messages.map(msg => {
+      if (!msg) return msg;
+      const cloned = { ...msg };
+
+      if (Array.isArray(msg.content)) {
+        cloned.content = msg.content.map(block => {
+          if (!block || typeof block !== 'object') return block;
+
+          // Skip heavy media blocks — flattenBlocks() produces "" for these
+          if (isHeavyMediaBlock(block)) {
+            return { type: block.type, _skipped: true };
+          }
+
+          // Shallow clone small blocks (text, tool_result, tool_use)
+          if (block.type === 'tool_result' && typeof block.content === 'object') {
+            return { ...block, content: JSON.parse(JSON.stringify(block.content)) };
+          }
+          return { ...block };
+        });
+      } else if (typeof msg.content === 'object' && msg.content !== null) {
+        cloned.content = { ...msg.content };
+      }
+
+      // Clone tool_calls if present
+      if (Array.isArray(msg.tool_calls)) {
+        cloned.tool_calls = JSON.parse(JSON.stringify(msg.tool_calls));
+      }
+
+      return cloned;
+    });
+  }
+
+  // Deep-clone small arrays that get mutated
+  if (Array.isArray(payload.tools)) {
+    clean.tools = JSON.parse(JSON.stringify(payload.tools));
+  }
+  if (Array.isArray(payload.system)) {
+    clean.system = JSON.parse(JSON.stringify(payload.system));
+  } else if (typeof payload.system === 'string') {
+    clean.system = payload.system;
+  }
+
+  return clean;
+}
+
+/**
+ * Smart deep-clone a request payload.
+ *
+ * - If willFlatten is true: skips cloning heavy media blocks (they'll be discarded)
+ * - If willFlatten is false: uses structuredClone (faster than JSON round-trip)
+ * - Falls back to JSON.parse(JSON.stringify()) for compatibility
+ *
+ * @param {Object} payload - Request payload to clone
+ * @param {Object} options
+ * @param {boolean} options.willFlatten - Whether flattenBlocks() will run (true for most providers)
+ * @returns {Object} Cloned payload
+ */
+function clonePayloadSmart(payload, options = {}) {
+  if (!payload) return {};
+
+  const { willFlatten = false } = options;
+
+  // Fast path: provider will flatten content — skip cloning media blocks
+  if (willFlatten && Array.isArray(payload.messages)) {
+    const hasMedia = payload.messages.some(msg =>
+      Array.isArray(msg?.content) && msg.content.some(isHeavyMediaBlock)
+    );
+    if (hasMedia) {
+      logger.debug('[payload] Using flatten-aware clone (skipping media blocks)');
+      return cloneWithFlattenAwareness(payload);
+    }
+  }
+
+  // Medium path: structuredClone (faster, no string intermediate)
+  if (typeof structuredClone === 'function') {
+    try {
+      return structuredClone(payload);
+    } catch {
+      // structuredClone can fail on functions, symbols, etc.
+    }
+  }
+
+  // Slow path: JSON round-trip (original behavior)
+  return JSON.parse(JSON.stringify(payload));
+}
+
+module.exports = {
+  estimateContentSize,
+  hasLargeMedia,
+  clonePayloadSmart,
+  isHeavyMediaBlock,
+};

package/src/utils/perf-timer.js

@@ -0,0 +1,80 @@
+/**
+ * Request Performance Timer
+ *
+ * Lightweight timing instrumentation for the request hot path.
+ * Enable with LOG_LEVEL=debug or PERF_TIMER=true to see per-request
+ * breakdown of where time is spent.
+ *
+ * Usage:
+ *   const timer = createTimer('processMessage');
+ *   timer.mark('sanitizePayload');
+ *   // ... do work ...
+ *   timer.mark('cacheCheck');
+ *   // ... do work ...
+ *   timer.done(); // logs full breakdown
+ *
+ * @module utils/perf-timer
+ */
+
+const { performance } = require('perf_hooks');
+const logger = require('../logger');
+
+const ENABLED = process.env.PERF_TIMER === 'true';
+
+/**
+ * Create a performance timer for a named operation.
+ * @param {string} name - Timer name (e.g., 'processMessage', 'invokeModel')
+ * @returns {{ mark: (label: string) => void, done: () => Object }}
+ */
+function createTimer(name) {
+  if (!ENABLED) {
+    // No-op when disabled — zero overhead
+    return {
+      mark() {},
+      done() { return null; },
+    };
+  }
+
+  const start = performance.now();
+  const marks = [];
+  let lastMark = start;
+
+  return {
+    /**
+     * Record a checkpoint.
+     * @param {string} label - What just completed
+     */
+    mark(label) {
+      const now = performance.now();
+      marks.push({
+        label,
+        elapsed: now - lastMark,
+        cumulative: now - start,
+      });
+      lastMark = now;
+    },
+
+    /**
+     * Finish timing and log the breakdown.
+     * @returns {Object} Timing breakdown
+     */
+    done() {
+      const total = performance.now() - start;
+      const breakdown = {};
+
+      for (const m of marks) {
+        breakdown[m.label] = `${m.elapsed.toFixed(2)}ms`;
+      }
+
+      logger.info({
+        timer: name,
+        totalMs: total.toFixed(2),
+        breakdown,
+      }, `[perf] ${name}: ${total.toFixed(1)}ms`);
+
+      return { name, totalMs: total, marks, breakdown };
+    },
+  };
+}
+
+module.exports = { createTimer };

package/.github/FUNDING.yml

@@ -1,15 +0,0 @@
-# These are supported funding model platforms
-
-github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-polar: # Replace with a single Polar username
-buy_me_a_coffee: srinivasveera
-thanks_dev: # Replace with a single thanks.dev username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']