neoagent 3.2.1-beta.1 → 3.2.1-beta.11

package/server/services/social_reach/utils.js

@@ -1,9 +1,13 @@
 'use strict';
 
 const cheerio = require('cheerio');
+const { fetchResponseText } = require('../network/http');
+const { executeSafeHttpRequest } = require('../network/safe_request');
 
 const DEFAULT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
   + '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+const DEFAULT_TIMEOUT_MS = 20000;
+const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
 
 function normalizeLimit(value, fallback = 10, max = 50) {
   const n = Number(value);
@@ -31,19 +35,60 @@ function assertHttpUrl(value) {
 }
 
 async function fetchText(url, options = {}) {
-  const response = await fetch(url, {
-    redirect: 'follow',
-    ...options,
-    headers: {
-      'user-agent': DEFAULT_UA,
-      accept: 'text/plain,text/html,application/xml,application/rss+xml,application/atom+xml,*/*',
-      ...(options.headers || {}),
-    },
-  });
-  const text = await response.text();
-  if (!response.ok) {
-    const error = new Error(`Request failed with HTTP ${response.status}`);
-    error.status = response.status;
+  const {
+    lookup,
+    maxResponseBytes = MAX_RESPONSE_BYTES,
+    publicOnly = false,
+    requestImpl,
+    signal,
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+    ...requestOptions
+  } = options;
+  const headers = {
+    'user-agent': DEFAULT_UA,
+    accept: 'text/plain,text/html,application/xml,application/rss+xml,application/atom+xml,*/*',
+    ...(options.headers || {}),
+  };
+  let status;
+  let text;
+  if (publicOnly) {
+    const result = await executeSafeHttpRequest({
+      url,
+      method: requestOptions.method || 'GET',
+      headers,
+      body: requestOptions.body,
+      timeout_ms: timeoutMs,
+    }, {
+      signal,
+      lookup,
+      requestImpl,
+      maxResponseBytes,
+    });
+    if (result.truncated) {
+      const error = new Error('Response exceeded the Social Reach safety limit.');
+      error.code = 'SOCIAL_REACH_RESPONSE_TOO_LARGE';
+      error.status = 502;
+      throw error;
+    }
+    status = result.status;
+    text = result.body;
+  } else {
+    const result = await fetchResponseText(url, {
+      ...requestOptions,
+      headers,
+      signal,
+      timeoutMs,
+      maxResponseBytes,
+      serviceName: 'Social Reach',
+      timeoutCode: 'SOCIAL_REACH_TIMEOUT',
+      tooLargeCode: 'SOCIAL_REACH_RESPONSE_TOO_LARGE',
+    });
+    status = result.response.status;
+    text = result.text;
+  }
+  if (status < 200 || status >= 300) {
+    const error = new Error(`Request failed with HTTP ${status}`);
+    error.status = status;
     error.body = text.slice(0, 500);
     throw error;
   }
@@ -58,7 +103,13 @@ async function fetchJson(url, options = {}) {
       ...(options.headers || {}),
     },
   });
-  return JSON.parse(text);
+  try {
+    return JSON.parse(text);
+  } catch (cause) {
+    const error = new Error('Social Reach returned malformed JSON.', { cause });
+    error.status = 502;
+    throw error;
+  }
 }
 
 function htmlToText(html, maxChars = 20000) {

package/server/services/social_video/captions.js

@@ -27,7 +27,7 @@ function pickCaptionTrack(captionGroups = {}, preferredLanguages = []) {
 
   const preferred = preferredLanguages.map(normalizeLanguageCode).filter(Boolean);
   const preferredOrder = preferred.length > 0 ? preferred : ['en', 'en-us', 'en-gb'];
-  const preferredExt = ['vtt', 'webvtt', 'srt', 'json3'];
+  const preferredExt = ['vtt', 'webvtt', 'srt', 'json3', 'ttml', 'xml', 'json'];
 
   const scoreTrack = (track) => {
     const langIdx = preferredOrder.findIndex((code) => track.language === code || track.language.startsWith(`${code}-`));
@@ -148,7 +148,7 @@ function parseCaptionText(raw, extension = '') {
   if (ext === 'vtt' || ext === 'webvtt') {
     return parseWebVttToText(raw);
   }
-  if (ext === 'ttml') {
+  if (ext === 'ttml' || ext === 'xml') {
     return parseTtmlToText(raw);
   }
   if (ext === 'srt') {