neoagent 3.2.1-beta.2 → 3.2.1-beta.4

package/server/services/social_reach/channels/social_video.js

@@ -3,6 +3,7 @@
 const { getPlatformDefinition } = require('../platforms');
 const { SocialReachChannel } = require('./base');
 const { normalizeAndDetectPlatform } = require('../../social_video/url');
+const { createAbortError } = require('../../../utils/abort');
 
 const PLATFORM_LABELS = Object.freeze({
   youtube: 'YouTube',
@@ -26,7 +27,7 @@ class SocialVideoReachChannel extends SocialReachChannel {
     }
   }
 
-  async check() {
+  async check({ signal } = {}) {
     if (!this.socialVideoService || typeof this.socialVideoService.getHealthStatus !== 'function') {
       return {
         ...(await super.check()),
@@ -37,11 +38,14 @@ class SocialVideoReachChannel extends SocialReachChannel {
       };
     }
 
-    const health = await this.socialVideoService.getHealthStatus().catch((error) => ({
-      ready: false,
-      dependencies: [],
-      error: error.message || String(error),
-    }));
+    const health = await this.socialVideoService.getHealthStatus({ signal }).catch((error) => {
+      if (signal?.aborted) throw createAbortError(signal);
+      return {
+        ready: false,
+        dependencies: [],
+        error: error.message || String(error),
+      };
+    });
     const missing = (health.dependencies || [])
       .filter((item) => !item.available)
       .map((item) => item.name)
@@ -59,7 +63,7 @@ class SocialVideoReachChannel extends SocialReachChannel {
     };
   }
 
-  async read({ userId, url, include_frame: includeFrame, force_stt: forceStt, agentId }) {
+  async read({ userId, url, include_frame: includeFrame, force_stt: forceStt, agentId, signal }) {
     if (!this.socialVideoService || typeof this.socialVideoService.extractFromUrl !== 'function') {
       const error = new Error('Social video extraction is not connected right now.');
       error.status = 503;
@@ -70,6 +74,7 @@ class SocialVideoReachChannel extends SocialReachChannel {
       includeFrame: includeFrame !== false,
       forceStt: forceStt === true,
       agentId: agentId || null,
+      signal,
     });
     return {
       ...result,

package/server/services/social_reach/channels/v2ex.js

@@ -3,6 +3,7 @@
 const { SocialReachChannel } = require('./base');
 const { getPlatformDefinition } = require('../platforms');
 const { assertHttpUrl, fetchJson, normalizeLimit } = require('../utils');
+const { createAbortError } = require('../../../utils/abort');
 
 function shapeTopic(item = {}) {
   const node = item.node || {};
@@ -33,17 +34,26 @@ class V2exChannel extends SocialReachChannel {
     };
   }
 
-  async read({ url, limit }) {
+  async read({ url, limit, signal }) {
     const parsed = assertHttpUrl(url);
     const match = parsed.pathname.match(/^\/t\/(\d+)/);
     if (!match) {
-      return this.search({ query: parsed.searchParams.get('q') || 'hot', limit });
+      return this.search({ query: parsed.searchParams.get('q') || 'hot', limit, signal });
     }
     const topicId = match[1];
-    const topicData = await fetchJson(`https://www.v2ex.com/api/topics/show.json?id=${encodeURIComponent(topicId)}`);
+    const topicData = await fetchJson(
+      `https://www.v2ex.com/api/topics/show.json?id=${encodeURIComponent(topicId)}`,
+      { signal },
+    );
     const topic = Array.isArray(topicData) ? topicData[0] || {} : topicData || {};
-    const replies = await fetchJson(`https://www.v2ex.com/api/replies/show.json?topic_id=${encodeURIComponent(topicId)}&page=1`)
-      .catch(() => []);
+    const replies = await fetchJson(
+      `https://www.v2ex.com/api/replies/show.json?topic_id=${encodeURIComponent(topicId)}&page=1`,
+      { signal },
+    )
+      .catch((error) => {
+        if (signal?.aborted) throw createAbortError(signal);
+        return [];
+      });
     return {
       platform: this.id,
       ...shapeTopic(topic),
@@ -58,11 +68,14 @@ class V2exChannel extends SocialReachChannel {
     };
   }
 
-  async search({ query, limit }) {
+  async search({ query, limit, signal }) {
     const normalized = String(query || '').trim().toLowerCase();
     const capped = normalizeLimit(limit, 20, 100);
     if (normalized && normalized !== 'hot') {
-      const data = await fetchJson(`https://www.v2ex.com/api/topics/show.json?node_name=${encodeURIComponent(normalized)}&page=1`);
+      const data = await fetchJson(
+        `https://www.v2ex.com/api/topics/show.json?node_name=${encodeURIComponent(normalized)}&page=1`,
+        { signal },
+      );
       return {
         platform: this.id,
         query: normalized,
@@ -70,7 +83,7 @@ class V2exChannel extends SocialReachChannel {
         source: 'v2ex_public_api',
       };
     }
-    const data = await fetchJson('https://www.v2ex.com/api/topics/hot.json');
+    const data = await fetchJson('https://www.v2ex.com/api/topics/hot.json', { signal });
     return {
       platform: this.id,
       query: 'hot',

package/server/services/social_reach/channels/x.js

@@ -43,7 +43,7 @@ class XChannel extends SocialReachChannel {
     };
   }
 
-  async read({ userId, url }) {
+  async read({ userId, url, signal }) {
     const id = extractStatusId(url);
     if (!id) {
       const error = new Error('A public X post URL is required.');
@@ -54,7 +54,7 @@ class XChannel extends SocialReachChannel {
     const source = cookieHeaderForPlatform(userId, this.id) ? 'x_syndication_cookies' : 'x_syndication_public';
     const data = await fetchJson(
       `https://cdn.syndication.twimg.com/tweet-result?id=${encodeURIComponent(id)}&lang=en`,
-      { headers: this.#headers(userId) },
+      { headers: this.#headers(userId), signal },
     );
     return {
       platform: this.id,

package/server/services/social_reach/channels/xueqiu.js

@@ -37,7 +37,7 @@ class XueqiuChannel extends SocialReachChannel {
     };
   }
 
-  async search({ userId, query, limit }) {
+  async search({ userId, query, limit, signal }) {
     const q = String(query || '').trim();
     if (!q) {
       const error = new Error('query is required.');
@@ -45,7 +45,7 @@ class XueqiuChannel extends SocialReachChannel {
       throw error;
     }
     const url = `https://xueqiu.com/stock/search.json?code=${encodeURIComponent(q)}&size=${normalizeLimit(limit, 10, 50)}`;
-    const data = await fetchJson(url, { headers: this.#headers(userId) });
+    const data = await fetchJson(url, { headers: this.#headers(userId), signal });
     return {
       platform: this.id,
       query: q,
@@ -58,12 +58,12 @@ class XueqiuChannel extends SocialReachChannel {
     };
   }
 
-  async read({ userId, symbol, limit }) {
+  async read({ userId, symbol, limit, signal }) {
     const raw = String(symbol || '').trim().toUpperCase();
     if (raw) {
       const data = await fetchJson(
         `https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=${encodeURIComponent(raw)}`,
-        { headers: this.#headers(userId) },
+        { headers: this.#headers(userId), signal },
       );
       const quote = data.data?.items?.[0]?.quote || {};
       return {
@@ -76,7 +76,7 @@ class XueqiuChannel extends SocialReachChannel {
     }
     const data = await fetchJson(
       'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=20&category=-1',
-      { headers: this.#headers(userId) },
+      { headers: this.#headers(userId), signal },
     );
     return {
       platform: this.id,

package/server/services/social_reach/service.js

@@ -5,6 +5,7 @@ const { createChannels } = require('./channels');
 const { deleteCookieBundle, getCookieSummary, writeCookieBundle } = require('./store');
 const { domainsForPlatform, getPlatformDefinition, normalizePlatformId } = require('./platforms');
 const { assertHttpUrl } = require('./utils');
+const { createAbortError } = require('../../utils/abort');
 
 const COOKIE_IMPORT_PLATFORMS = new Set(['xueqiu', 'x']);
 const MAX_IMPORTED_COOKIES = 80;
@@ -59,16 +60,17 @@ class SocialReachService {
     return this.channels.find((channel) => channel.canHandleUrl(parsed.toString())) || null;
   }
 
-  async getStatus(userId) {
+  async getStatus(userId, options = {}) {
     const statuses = [];
     for (const channel of this.channels) {
       try {
-        const status = await channel.check({ userId });
+        const status = await channel.check({ userId, signal: options.signal });
         if (!status.cookie && channel.setupKind === 'cookies') {
           status.cookie = getCookieSummary(userId, channel.id);
         }
         statuses.push(status);
       } catch (error) {
+        if (options.signal?.aborted) throw createAbortError(options.signal);
         statuses.push({
           platform: channel.id,
           label: channel.label,
@@ -87,7 +89,7 @@ class SocialReachService {
     };
   }
 
-  async read(userId, args = {}) {
+  async read(userId, args = {}, options = {}) {
     const platform = normalizePlatformId(args.platform || '');
     const channel = platform ? this.getChannel(platform) : this.detectChannelForUrl(args.url);
     if (!channel) {
@@ -97,10 +99,10 @@ class SocialReachService {
       error.status = 400;
       throw error;
     }
-    return channel.read({ ...args, userId });
+    return channel.read({ ...args, userId, signal: options.signal });
   }
 
-  async search(userId, args = {}) {
+  async search(userId, args = {}, options = {}) {
     const platform = normalizePlatformId(args.platform || '');
     const channel = this.getChannel(platform);
     if (!channel) {
@@ -108,7 +110,7 @@ class SocialReachService {
       error.status = 400;
       throw error;
     }
-    return channel.search({ ...args, userId });
+    return channel.search({ ...args, userId, signal: options.signal });
   }
 
   async importCookiesFromExtension(userId, platform, options = {}) {
@@ -132,6 +134,7 @@ class SocialReachService {
     }, {
       tokenId: options.tokenId || null,
       timeoutMs: 15000,
+      signal: options.signal,
     });
     const cookies = sanitizeCookies(response?.cookies || [], domains);
     if (cookies.length === 0) {

package/server/services/social_reach/utils.js

@@ -1,9 +1,13 @@
 'use strict';
 
 const cheerio = require('cheerio');
+const { fetchResponseText } = require('../network/http');
+const { executeSafeHttpRequest } = require('../network/safe_request');
 
 const DEFAULT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
   + '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+const DEFAULT_TIMEOUT_MS = 20000;
+const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
 
 function normalizeLimit(value, fallback = 10, max = 50) {
   const n = Number(value);
@@ -31,19 +35,60 @@ function assertHttpUrl(value) {
 }
 
 async function fetchText(url, options = {}) {
-  const response = await fetch(url, {
-    redirect: 'follow',
-    ...options,
-    headers: {
-      'user-agent': DEFAULT_UA,
-      accept: 'text/plain,text/html,application/xml,application/rss+xml,application/atom+xml,*/*',
-      ...(options.headers || {}),
-    },
-  });
-  const text = await response.text();
-  if (!response.ok) {
-    const error = new Error(`Request failed with HTTP ${response.status}`);
-    error.status = response.status;
+  const {
+    lookup,
+    maxResponseBytes = MAX_RESPONSE_BYTES,
+    publicOnly = false,
+    requestImpl,
+    signal,
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+    ...requestOptions
+  } = options;
+  const headers = {
+    'user-agent': DEFAULT_UA,
+    accept: 'text/plain,text/html,application/xml,application/rss+xml,application/atom+xml,*/*',
+    ...(options.headers || {}),
+  };
+  let status;
+  let text;
+  if (publicOnly) {
+    const result = await executeSafeHttpRequest({
+      url,
+      method: requestOptions.method || 'GET',
+      headers,
+      body: requestOptions.body,
+      timeout_ms: timeoutMs,
+    }, {
+      signal,
+      lookup,
+      requestImpl,
+      maxResponseBytes,
+    });
+    if (result.truncated) {
+      const error = new Error('Response exceeded the Social Reach safety limit.');
+      error.code = 'SOCIAL_REACH_RESPONSE_TOO_LARGE';
+      error.status = 502;
+      throw error;
+    }
+    status = result.status;
+    text = result.body;
+  } else {
+    const result = await fetchResponseText(url, {
+      ...requestOptions,
+      headers,
+      signal,
+      timeoutMs,
+      maxResponseBytes,
+      serviceName: 'Social Reach',
+      timeoutCode: 'SOCIAL_REACH_TIMEOUT',
+      tooLargeCode: 'SOCIAL_REACH_RESPONSE_TOO_LARGE',
+    });
+    status = result.response.status;
+    text = result.text;
+  }
+  if (status < 200 || status >= 300) {
+    const error = new Error(`Request failed with HTTP ${status}`);
+    error.status = status;
     error.body = text.slice(0, 500);
     throw error;
   }
@@ -58,7 +103,13 @@ async function fetchJson(url, options = {}) {
       ...(options.headers || {}),
     },
   });
-  return JSON.parse(text);
+  try {
+    return JSON.parse(text);
+  } catch (cause) {
+    const error = new Error('Social Reach returned malformed JSON.', { cause });
+    error.status = 502;
+    throw error;
+  }
 }
 
 function htmlToText(html, maxChars = 20000) {