lynkr 7.2.5 → 8.0.1

package/src/tools/tinyfish.js

@@ -0,0 +1,358 @@
+const { URL } = require("url");
+const { Agent } = require("undici");
+const config = require("../config");
+const logger = require("../logger");
+const { registerTool } = require(".");
+
+/**
+ * Dedicated HTTP agent for TinyFish SSE streams.
+ * The default webAgent in web-client.js has a 30s bodyTimeout which is too
+ * short for browser-automation tasks that can take up to 120s.
+ */
+const sseAgent = new Agent({
+  connections: 10,
+  pipelining: 1,
+  keepAliveTimeout: 60000,
+  connectTimeout: 15000,
+  bodyTimeout: 0, // no body timeout — we manage timeout via AbortController
+  headersTimeout: 15000,
+  maxRedirections: 3,
+  strictContentLength: false,
+});
+
+// ---------------------------------------------------------------------------
+// Argument normalisers
+// ---------------------------------------------------------------------------
+
+function normalizeUrl(args) {
+  const raw = args.url ?? args.uri ?? args.href ?? args.target_url;
+  if (typeof raw !== "string" || raw.trim().length === 0) {
+    throw new Error("web_agent requires a non-empty url string.");
+  }
+  // Validate URL
+  try {
+    new URL(raw.trim());
+  } catch {
+    throw new Error(`web_agent received an invalid URL: ${raw}`);
+  }
+  return raw.trim();
+}
+
+function normalizeGoal(args) {
+  const goal = args.goal ?? args.task ?? args.prompt ?? args.instruction;
+  if (typeof goal !== "string" || goal.trim().length === 0) {
+    throw new Error("web_agent requires a non-empty goal string.");
+  }
+  return goal.trim();
+}
+
+function resolveBrowserProfile(args) {
+  const profile = args.browser_profile ?? args.browserProfile ?? config.tinyfish.browserProfile;
+  if (profile === "stealth") return "stealth";
+  return "lite";
+}
+
+// ---------------------------------------------------------------------------
+// SSE stream consumer
+// ---------------------------------------------------------------------------
+
+async function consumeSSEStream(response, timeoutMs) {
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  const startTime = Date.now();
+
+  try {
+    while (true) {
+      if (Date.now() - startTime > timeoutMs) {
+        const err = new Error(`TinyFish SSE stream timed out after ${timeoutMs}ms`);
+        err.code = "ETIMEDOUT";
+        err.status = 504;
+        throw err;
+      }
+
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      buffer += decoder.decode(value, { stream: true });
+
+      // SSE events are separated by double newlines
+      const parts = buffer.split("\n\n");
+      // Keep the last (possibly incomplete) chunk in the buffer
+      buffer = parts.pop() || "";
+
+      for (const part of parts) {
+        // Extract the data: line(s)
+        const lines = part.split("\n");
+        let dataStr = "";
+        for (const line of lines) {
+          if (line.startsWith("data: ")) {
+            dataStr += line.slice(6);
+          } else if (line.startsWith("data:")) {
+            dataStr += line.slice(5);
+          }
+        }
+
+        if (!dataStr) continue;
+
+        let event;
+        try {
+          event = JSON.parse(dataStr);
+        } catch {
+          // Not valid JSON — skip this SSE frame
+          logger.debug({ raw: dataStr.slice(0, 200) }, "TinyFish: non-JSON SSE frame, skipping");
+          continue;
+        }
+
+        logger.debug(
+          { type: event.type, status: event.status },
+          "TinyFish SSE event"
+        );
+
+        if (event.type === "COMPLETE" || event.type === "complete") {
+          const status = (event.status ?? "").toUpperCase();
+          if (status === "COMPLETED" || status === "SUCCESS") {
+            return event.resultJson ?? event.result ?? event.data ?? event;
+          }
+          // Task failed
+          const errMsg = event.error ?? event.message ?? "TinyFish task failed";
+          const err = new Error(typeof errMsg === "string" ? errMsg : JSON.stringify(errMsg));
+          err.code = "TINYFISH_TASK_FAILED";
+          err.status = 502;
+          throw err;
+        }
+      }
+    }
+
+    // Stream ended without a COMPLETE event
+    const err = new Error("TinyFish SSE stream ended without a COMPLETE event");
+    err.code = "TINYFISH_INCOMPLETE";
+    err.status = 502;
+    throw err;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Core API call
+// ---------------------------------------------------------------------------
+
+async function callTinyFishAPI({ url, goal, browserProfile, proxyConfig, timeoutMs }) {
+  const endpoint = config.tinyfish.endpoint;
+  const apiKey = config.tinyfish.apiKey;
+
+  if (!apiKey) {
+    return {
+      ok: false,
+      status: 503,
+      content: JSON.stringify({
+        error: "tinyfish_not_configured",
+        message:
+          "TinyFish API key is not configured. Set TINYFISH_API_KEY in your .env file. Get a key from https://tinyfish.ai",
+      }, null, 2),
+    };
+  }
+
+  const body = {
+    url,
+    goal,
+    browserProfile,
+  };
+
+  if (proxyConfig) {
+    body.proxy = proxyConfig;
+  }
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "X-API-Key": apiKey,
+        Accept: "text/event-stream",
+      },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+      dispatcher: sseAgent,
+    });
+
+    // Handle non-2xx responses before attempting SSE parse
+    if (!response.ok) {
+      const text = await response.text().catch(() => "");
+      const shouldRetry = response.status === 429 || response.status >= 500;
+
+      if (shouldRetry) {
+        // Retry once with 2s backoff
+        logger.warn(
+          { status: response.status, body: text.slice(0, 200) },
+          "TinyFish API error, retrying once"
+        );
+        await new Promise((r) => setTimeout(r, 2000));
+
+        const retryController = new AbortController();
+        const retryTimeout = setTimeout(() => retryController.abort(), timeoutMs);
+        try {
+          const retryResponse = await fetch(endpoint, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              "X-API-Key": apiKey,
+              Accept: "text/event-stream",
+            },
+            body: JSON.stringify(body),
+            signal: retryController.signal,
+            dispatcher: sseAgent,
+          });
+
+          if (!retryResponse.ok) {
+            const retryText = await retryResponse.text().catch(() => "");
+            const err = new Error(
+              `TinyFish API error (${retryResponse.status}): ${retryResponse.statusText}`
+            );
+            err.status = retryResponse.status;
+            err.body = retryText;
+            throw err;
+          }
+
+          const result = await consumeSSEStream(retryResponse, timeoutMs);
+          return {
+            ok: true,
+            status: 200,
+            result,
+          };
+        } finally {
+          clearTimeout(retryTimeout);
+        }
+      }
+
+      const err = new Error(
+        `TinyFish API error (${response.status}): ${response.statusText}`
+      );
+      err.status = response.status;
+      err.body = text;
+      throw err;
+    }
+
+    const result = await consumeSSEStream(response, timeoutMs);
+    return {
+      ok: true,
+      status: 200,
+      result,
+    };
+  } catch (error) {
+    if (error.name === "AbortError") {
+      const err = new Error(`TinyFish request timed out after ${timeoutMs}ms`);
+      err.code = "ETIMEDOUT";
+      err.status = 504;
+      throw err;
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Tool registration
+// ---------------------------------------------------------------------------
+
+function registerTinyFishTool() {
+  registerTool(
+    "web_agent",
+    async ({ args = {} }) => {
+      const url = normalizeUrl(args);
+      const goal = normalizeGoal(args);
+      const browserProfile = resolveBrowserProfile(args);
+      const timeoutMs = config.tinyfish.timeoutMs;
+
+      // Build proxy config if enabled
+      let proxyConfig = null;
+      if (config.tinyfish.proxyEnabled) {
+        proxyConfig = {
+          enabled: true,
+          country: config.tinyfish.proxyCountry,
+        };
+      }
+
+      try {
+        const response = await callTinyFishAPI({
+          url,
+          goal,
+          browserProfile,
+          proxyConfig,
+          timeoutMs,
+        });
+
+        // Guard clause: not configured
+        if (!response.ok && response.status === 503) {
+          return response;
+        }
+
+        const resultStr =
+          typeof response.result === "string"
+            ? response.result
+            : JSON.stringify(response.result, null, 2);
+
+        logger.debug(
+          {
+            url,
+            goal: goal.slice(0, 100),
+            browserProfile,
+            resultLength: resultStr.length,
+          },
+          "TinyFish web_agent completed"
+        );
+
+        return {
+          ok: true,
+          status: 200,
+          content: resultStr,
+          metadata: {
+            url,
+            goal,
+            browserProfile,
+            resultLength: resultStr.length,
+          },
+        };
+      } catch (err) {
+        logger.error(
+          { err, url, goal: goal.slice(0, 100) },
+          "web_agent request failed"
+        );
+        return {
+          ok: false,
+          status: err.status ?? 500,
+          content: JSON.stringify(
+            {
+              error: err.code ?? "web_agent_failed",
+              message: err.message,
+              url,
+              ...(err.status ? { http_status: err.status } : {}),
+            },
+            null,
+            2
+          ),
+          metadata: {
+            url,
+            goal,
+            error_code: err.code,
+            ...(err.status ? { http_status: err.status } : {}),
+          },
+        };
+      }
+    },
+    { category: "tinyfish" }
+  );
+}
+
+function registerTinyFishTools() {
+  registerTinyFishTool();
+}
+
+module.exports = {
+  registerTinyFishTools,
+};

package/src/tools/truncate.js

@@ -8,6 +8,7 @@ const TRUNCATION_LIMITS = {
   Glob: { maxChars: 8000, strategy: 'head' },
   WebFetch: { maxChars: 16000, strategy: 'head' },
   WebSearch: { maxChars: 12000, strategy: 'head' },
+  WebAgent: { maxChars: 16000, strategy: 'head' },
   LSP: { maxChars: 8000, strategy: 'head' },
   Edit: { maxChars: 8000, strategy: 'middle' },
   Write: { maxChars: 8000, strategy: 'middle' },

package/.github/FUNDING.yml

@@ -1,15 +0,0 @@
-# These are supported funding model platforms
-
-github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-polar: # Replace with a single Polar username
-buy_me_a_coffee: srinivasveera
-thanks_dev: # Replace with a single thanks.dev username
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

package/.github/workflows/README.md

@@ -1,215 +0,0 @@
-# GitHub Actions Workflows
-
-This directory contains GitHub Actions workflows for automated testing and CI/CD.
-
-## Available Workflows
-
-### 1. CI Tests (`ci.yml`)
-
-**Purpose:** Run comprehensive test suite on every push and pull request
-
-**Triggers:**
-- Push to `main` or `develop` branches
-- Pull requests to `main` or `develop` branches
-
-**What it does:**
-- Tests on Node.js 20.x and 22.x
-- Runs linter (`npm run lint`)
-- Runs unit tests (`npm run test:unit`)
-- Runs performance tests (`npm run test:performance`)
-- Uses npm cache for faster builds
-
-**Environment Variables:**
-- `DATABRICKS_API_KEY=test-key` (mock value for tests)
-- `DATABRICKS_API_BASE=http://test.com` (mock value for tests)
-
-**Status:** Runs on every push/PR, fails if unit tests fail
-
----
-
-### 2. Web Tools Tests (`web-tools-tests.yml`)
-
-**Purpose:** Run web search tool tests when related files change
-
-**Triggers:**
-- Changes to web tools source files:
-  - `src/tools/web.js`
-  - `src/tools/web-client.js`
-  - `src/clients/retry.js`
-  - `src/config/index.js`
-  - `test/web-tools.test.js`
-
-**What it does:**
-- Runs only the web tools test suite
-- Generates test summary in GitHub Actions UI
-- Faster feedback for web tools changes
-
-**Test Coverage:**
-- HTML extraction (9 tests)
-- HTTP keep-alive agent (2 tests)
-- Retry logic with exponential backoff (2 tests)
-- Configuration management (3 tests)
-- Error handling (1 test)
-- Performance validation (1 test)
-- Body preview configuration (1 test)
-
-**Total:** 19 tests
-
----
-
-### 3. NPM Publish (`npm-publish.yml`)
-
-**Purpose:** Automatically publish package to npm registry
-
-**Triggers:**
-- Git tags starting with `v` (e.g., `v0.1.5`)
-- GitHub Releases created
-
-**What it does:**
-- Runs full test suite before publishing
-- Checks if version already exists on npm
-- Publishes package to npm registry (if tests pass)
-- Prevents duplicate publishes
-- Creates publish summary
-
-**Requirements:**
-- `NPM_TOKEN` secret must be configured
-- Tests must pass
-- Version must be new
-
-**Status:** Only publishes on successful builds
-
----
-
-### 4. Version Bump (`version-bump.yml`)
-
-**Purpose:** Manual workflow to bump version and create releases
-
-**Triggers:**
-- Manual workflow dispatch (button in Actions tab)
-
-**What it does:**
-- Prompts for version type (patch/minor/major)
-- Runs tests before version bump
-- Updates package.json version
-- Creates git commit and tag
-- Pushes changes to repository
-- Creates GitHub Release with changelog
-- Triggers npm-publish workflow automatically
-
-**Options:**
-- `patch` - Bug fixes (0.1.4 → 0.1.5)
-- `minor` - New features (0.1.4 → 0.2.0)
-- `major` - Breaking changes (0.1.4 → 1.0.0)
-
----
-
-### 5. IndexNow Notification (`index.yml`)
-
-**Purpose:** Notify search engines when documentation is updated
-
-**Triggers:**
-- Push to `main` branch
-- Changes in `docs/**` directory
-
-**What it does:**
-- Notifies Bing IndexNow about updated documentation
-- Helps with SEO and documentation discoverability
-
----
-
-## Adding Status Badges
-
-Add these badges to your README.md:
-
-```markdown
-![CI Tests](https://github.com/vishalveerareddy123/Lynkr/actions/workflows/ci.yml/badge.svg)
-![Web Tools Tests](https://github.com/vishalveerareddy123/Lynkr/actions/workflows/web-tools-tests.yml/badge.svg)
-![npm version](https://img.shields.io/npm/v/lynkr.svg)
-![npm downloads](https://img.shields.io/npm/dt/lynkr.svg)
-```
-
-## Running Tests Locally
-
-Before pushing, run tests locally:
-
-```bash
-# Run all unit tests
-npm run test:unit
-
-# Run only web tools tests
-DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com \
-  node --test test/web-tools.test.js
-
-# Run quick tests (routing only)
-npm run test:quick
-
-# Run all tests including performance
-npm test
-```
-
-## Workflow Configuration
-
-### Required Secrets
-
-**For npm publishing workflows:**
-- `NPM_TOKEN` - Your npm automation token (required to publish)
-  - Get from: https://www.npmjs.com/settings/YOUR_USERNAME/tokens
-  - Type: "Automation" token
-  - Add to: Settings → Secrets → Actions → New repository secret
-
-**For test workflows:**
-- No secrets required (uses mock credentials)
-
-**For IndexNow workflow:**
-- `INDEX_NOW` - Your IndexNow API key (optional, only for docs)
-
-### Matrix Strategy
-
-The CI workflow uses a matrix strategy to test on multiple Node.js versions:
-- Node.js 20.x (LTS)
-- Node.js 22.x (Current)
-
-This ensures compatibility across different Node versions.
-
-## Troubleshooting
-
-### Tests fail locally but pass in CI
-- Check Node.js version (`node --version`)
-- Ensure `npm ci` is used (not `npm install`)
-- Check for platform-specific issues (macOS vs Linux)
-
-### Tests pass locally but fail in CI
-- Environment variables might be missing
-- Dependencies might need updating
-- Check GitHub Actions logs for details
-
-### Workflow doesn't trigger
-- Verify file paths in `on.push.paths`
-- Check branch names match
-- Ensure workflow file is in `.github/workflows/`
-
-## Modifying Workflows
-
-When making changes:
-
-1. Test YAML syntax (use a YAML validator)
-2. Test locally first with same commands
-3. Create a PR to test in CI before merging
-4. Check GitHub Actions tab for results
-
-## Performance Considerations
-
-- **npm cache:** Workflows cache `node_modules` for faster builds
-- **Parallel jobs:** Tests run on multiple Node versions in parallel
-- **Path filtering:** Web tools workflow only runs when relevant files change
-- **continue-on-error:** Performance tests won't fail the build
-
-## Future Improvements
-
-Potential additions:
-- Code coverage reporting
-- Docker container testing
-- E2E integration tests
-- Deploy previews for PRs
-- Automated dependency updates (Dependabot)

package/.github/workflows/ci.yml

@@ -1,69 +0,0 @@
-name: CI Tests
-
-on:
-  push:
-    branches:
-      - main
-      - develop
-  pull_request:
-    branches:
-      - main
-      - develop
-
-jobs:
-  test:
-    name: Run Tests
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        node-version: [20.x, 22.x, 23.x, 24.x]
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Setup Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: 'npm'
-
-      - name: Install dependencies (C++20 for native addons)
-        env:
-          CXXFLAGS: "-std=gnu++20"
-          npm_config_build_from_source: true
-        run: npm ci
-
-      - name: Run linter
-        run: npm run lint
-        continue-on-error: true
-
-      - name: Run unit tests
-        run: npm run test:unit
-        env:
-          DATABRICKS_API_KEY: test-key
-          DATABRICKS_API_BASE: http://test.com
-
-      - name: Run performance tests
-        run: npm run test:performance
-        env:
-          DATABRICKS_API_KEY: test-key
-          DATABRICKS_API_BASE: http://test.com
-        continue-on-error: true
-
-  test-summary:
-    name: Test Summary
-    runs-on: ubuntu-latest
-    needs: test
-    if: always()
-
-    steps:
-      - name: Check test results
-        run: |
-          echo "Tests completed"
-          if [ "${{ needs.test.result }}" == "failure" ]; then
-            echo "Tests failed!"
-            exit 1
-          fi
-          echo "All tests passed!"