neoagent 3.1.1-beta.2 → 3.1.1-beta.4

package/server/services/social_reach/channels/xueqiu.js

@@ -0,0 +1,101 @@
+'use strict';
+
+const { SocialReachChannel } = require('./base');
+const { getPlatformDefinition } = require('../platforms');
+const { cookieHeaderForPlatform, getCookieSummary } = require('../store');
+const { DEFAULT_UA, fetchJson, normalizeLimit, parseMaybeJson } = require('../utils');
+
+function stripHtml(value) {
+  return String(value || '').replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
+}
+
+class XueqiuChannel extends SocialReachChannel {
+  constructor() {
+    super(getPlatformDefinition('xueqiu'));
+  }
+
+  async check({ userId } = {}) {
+    const cookies = getCookieSummary(userId, this.id);
+    return {
+      ...(await super.check()),
+      ready: cookies.configured,
+      status: cookies.configured ? 'ok' : 'warn',
+      activeBackend: cookies.configured ? 'xueqiu_http_cookies' : null,
+      cookie: cookies,
+      message: cookies.configured
+        ? 'Cookie-backed Xueqiu APIs are configured.'
+        : 'Import Xueqiu cookies from the Chrome extension before using stock and community APIs.',
+    };
+  }
+
+  #headers(userId) {
+    const cookie = cookieHeaderForPlatform(userId, this.id);
+    return {
+      'user-agent': DEFAULT_UA,
+      referer: 'https://xueqiu.com/',
+      ...(cookie ? { cookie } : {}),
+    };
+  }
+
+  async search({ userId, query, limit }) {
+    const q = String(query || '').trim();
+    if (!q) {
+      const error = new Error('query is required.');
+      error.status = 400;
+      throw error;
+    }
+    const url = `https://xueqiu.com/stock/search.json?code=${encodeURIComponent(q)}&size=${normalizeLimit(limit, 10, 50)}`;
+    const data = await fetchJson(url, { headers: this.#headers(userId) });
+    return {
+      platform: this.id,
+      query: q,
+      results: (data.stocks || []).map((item) => ({
+        symbol: item.code || '',
+        name: item.name || '',
+        exchange: item.exchange || '',
+      })),
+      source: 'xueqiu_http_cookies',
+    };
+  }
+
+  async read({ userId, symbol, limit }) {
+    const raw = String(symbol || '').trim().toUpperCase();
+    if (raw) {
+      const data = await fetchJson(
+        `https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=${encodeURIComponent(raw)}`,
+        { headers: this.#headers(userId) },
+      );
+      const quote = data.data?.items?.[0]?.quote || {};
+      return {
+        platform: this.id,
+        symbol: quote.symbol || raw,
+        name: quote.name || '',
+        quote,
+        source: 'xueqiu_http_cookies',
+      };
+    }
+    const data = await fetchJson(
+      'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=20&category=-1',
+      { headers: this.#headers(userId) },
+    );
+    return {
+      platform: this.id,
+      results: (data.list || []).slice(0, normalizeLimit(limit, 20, 50)).map((item) => {
+        const post = parseMaybeJson(item.data, {});
+        return {
+          id: post.id || item.id || null,
+          title: post.title || '',
+          text: stripHtml(post.text || post.description || ''),
+          author: post.user?.screen_name || '',
+          likes: post.like_count || null,
+          url: post.target ? `https://xueqiu.com${post.target}` : '',
+        };
+      }),
+      source: 'xueqiu_http_cookies',
+    };
+  }
+}
+
+module.exports = {
+  XueqiuChannel,
+};

package/server/services/social_reach/channels/youtube.js

@@ -0,0 +1,42 @@
+'use strict';
+
+const { SocialReachChannel } = require('./base');
+const { getPlatformDefinition } = require('../platforms');
+const { assertHttpUrl, compactText, fetchJson, fetchText } = require('../utils');
+
+class YoutubeChannel extends SocialReachChannel {
+  constructor() {
+    super(getPlatformDefinition('youtube'));
+  }
+
+  async check() {
+    return {
+      ...(await super.check()),
+      activeBackend: 'youtube_oembed_jina',
+      message: 'Node-only mode supports public metadata and readable page text. Caption extraction is not available without non-Node tooling.',
+    };
+  }
+
+  async read({ url }) {
+    const parsed = assertHttpUrl(url);
+    const [oembed, page] = await Promise.all([
+      fetchJson(`https://www.youtube.com/oembed?url=${encodeURIComponent(parsed.toString())}&format=json`).catch(() => null),
+      fetchText(`https://r.jina.ai/${parsed.toString()}`, { headers: { accept: 'text/plain' } }).catch(() => ''),
+    ]);
+    return {
+      platform: this.id,
+      url: parsed.toString(),
+      title: oembed?.title || null,
+      author: oembed?.author_name || null,
+      thumbnailUrl: oembed?.thumbnail_url || null,
+      content: compactText(page, 12000),
+      transcript: null,
+      source: 'youtube_oembed_jina',
+      limitation: 'Transcript extraction is unavailable in Node-only mode.',
+    };
+  }
+}
+
+module.exports = {
+  YoutubeChannel,
+};

package/server/services/social_reach/index.js

@@ -0,0 +1,7 @@
+'use strict';
+
+const { SocialReachService } = require('./service');
+
+module.exports = {
+  SocialReachService,
+};

package/server/services/social_reach/platforms.js

@@ -0,0 +1,162 @@
+'use strict';
+
+const PLATFORM_DEFINITIONS = Object.freeze({
+  web: {
+    id: 'web',
+    label: 'Web',
+    tier: 0,
+    setupKind: 'none',
+    hosts: [],
+    domains: [],
+  },
+  rss: {
+    id: 'rss',
+    label: 'RSS / Atom',
+    tier: 0,
+    setupKind: 'none',
+    hosts: [],
+    domains: [],
+  },
+  v2ex: {
+    id: 'v2ex',
+    label: 'V2EX',
+    tier: 0,
+    setupKind: 'none',
+    hosts: ['v2ex.com', 'www.v2ex.com'],
+    domains: ['v2ex.com'],
+  },
+  github: {
+    id: 'github',
+    label: 'GitHub',
+    tier: 0,
+    setupKind: 'optional_auth',
+    hosts: ['github.com', 'www.github.com'],
+    domains: ['github.com'],
+  },
+  youtube: {
+    id: 'youtube',
+    label: 'YouTube',
+    tier: 0,
+    setupKind: 'none',
+    hosts: ['youtube.com', 'www.youtube.com', 'youtu.be'],
+    domains: ['youtube.com', 'youtu.be'],
+  },
+  linkedin: {
+    id: 'linkedin',
+    label: 'LinkedIn',
+    tier: 2,
+    setupKind: 'public_read_only',
+    hosts: ['linkedin.com', 'www.linkedin.com'],
+    domains: ['linkedin.com'],
+  },
+  xueqiu: {
+    id: 'xueqiu',
+    label: 'Xueqiu',
+    tier: 1,
+    setupKind: 'cookies',
+    hosts: ['xueqiu.com', 'www.xueqiu.com', 'stock.xueqiu.com'],
+    domains: ['xueqiu.com'],
+  },
+  twitter: {
+    id: 'twitter',
+    label: 'X / Twitter',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['x.com', 'twitter.com', 'www.x.com', 'www.twitter.com'],
+    domains: ['x.com', 'twitter.com'],
+  },
+  reddit: {
+    id: 'reddit',
+    label: 'Reddit',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['reddit.com', 'www.reddit.com', 'old.reddit.com'],
+    domains: ['reddit.com'],
+  },
+  facebook: {
+    id: 'facebook',
+    label: 'Facebook',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['facebook.com', 'www.facebook.com'],
+    domains: ['facebook.com'],
+  },
+  instagram: {
+    id: 'instagram',
+    label: 'Instagram',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['instagram.com', 'www.instagram.com'],
+    domains: ['instagram.com'],
+  },
+  bilibili: {
+    id: 'bilibili',
+    label: 'Bilibili',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['bilibili.com', 'www.bilibili.com', 'b23.tv'],
+    domains: ['bilibili.com', 'b23.tv'],
+  },
+  xiaohongshu: {
+    id: 'xiaohongshu',
+    label: 'Xiaohongshu',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['xiaohongshu.com', 'www.xiaohongshu.com', 'xhslink.com'],
+    domains: ['xiaohongshu.com', 'xhslink.com'],
+  },
+  xiaoyuzhou: {
+    id: 'xiaoyuzhou',
+    label: 'Xiaoyuzhou',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: ['xiaoyuzhoufm.com', 'www.xiaoyuzhoufm.com'],
+    domains: ['xiaoyuzhoufm.com'],
+  },
+  exa_search: {
+    id: 'exa_search',
+    label: 'Exa Search',
+    tier: 2,
+    setupKind: 'unsupported_node_only',
+    hosts: [],
+    domains: [],
+  },
+});
+
+function normalizePlatformId(value) {
+  const id = String(value || '').trim().toLowerCase().replace(/[^a-z0-9_]+/g, '_');
+  if (id === 'x') return 'twitter';
+  return id;
+}
+
+function getPlatformDefinition(platform) {
+  return PLATFORM_DEFINITIONS[normalizePlatformId(platform)] || null;
+}
+
+function listPlatformDefinitions() {
+  return Object.values(PLATFORM_DEFINITIONS);
+}
+
+function hostMatchesPlatform(hostname, platform) {
+  const definition = getPlatformDefinition(platform);
+  if (!definition) return false;
+  const host = String(hostname || '').toLowerCase().replace(/^www\./, '');
+  return definition.hosts.some((candidate) => {
+    const normalized = candidate.toLowerCase().replace(/^www\./, '');
+    return host === normalized || host.endsWith(`.${normalized}`);
+  });
+}
+
+function domainsForPlatform(platform) {
+  const definition = getPlatformDefinition(platform);
+  return definition ? [...definition.domains] : [];
+}
+
+module.exports = {
+  PLATFORM_DEFINITIONS,
+  domainsForPlatform,
+  getPlatformDefinition,
+  hostMatchesPlatform,
+  listPlatformDefinitions,
+  normalizePlatformId,
+};

package/server/services/social_reach/service.js

@@ -0,0 +1,163 @@
+'use strict';
+
+const { EXTENSION_COMMANDS } = require('../browser/extension/protocol');
+const { createChannels } = require('./channels');
+const { deleteCookieBundle, getCookieSummary, writeCookieBundle } = require('./store');
+const { domainsForPlatform, getPlatformDefinition, normalizePlatformId } = require('./platforms');
+const { assertHttpUrl } = require('./utils');
+
+const COOKIE_IMPORT_PLATFORMS = new Set(['xueqiu']);
+const MAX_IMPORTED_COOKIES = 80;
+const MAX_COOKIE_NAME_CHARS = 256;
+const MAX_COOKIE_VALUE_CHARS = 4096;
+
+function shapeError(error) {
+  return {
+    error: error?.message || String(error),
+    status: error?.status || 500,
+  };
+}
+
+function sanitizeCookies(cookies = [], allowedDomains = []) {
+  const allowed = allowedDomains.map((domain) => String(domain || '').toLowerCase()).filter(Boolean);
+  return (Array.isArray(cookies) ? cookies : [])
+    .filter((cookie) => cookie && cookie.name && cookie.value != null)
+    .filter((cookie) => String(cookie.name).length <= MAX_COOKIE_NAME_CHARS)
+    .filter((cookie) => String(cookie.value).length <= MAX_COOKIE_VALUE_CHARS)
+    .filter((cookie) => {
+      const domain = String(cookie.domain || '').replace(/^\./, '').toLowerCase();
+      return allowed.some((item) => domain === item || domain.endsWith(`.${item}`));
+    })
+    .slice(0, MAX_IMPORTED_COOKIES)
+    .map((cookie) => ({
+      name: String(cookie.name),
+      value: String(cookie.value),
+      domain: String(cookie.domain || ''),
+      path: String(cookie.path || '/'),
+      secure: cookie.secure === true,
+      httpOnly: cookie.httpOnly === true,
+      sameSite: cookie.sameSite || null,
+      expirationDate: cookie.expirationDate || cookie.expires || null,
+    }));
+}
+
+class SocialReachService {
+  constructor(options = {}) {
+    this.browserExtensionRegistry = options.browserExtensionRegistry || null;
+    this.channels = createChannels();
+    this.channelById = new Map(this.channels.map((channel) => [channel.id, channel]));
+  }
+
+  getChannel(platform) {
+    const id = normalizePlatformId(platform);
+    return this.channelById.get(id) || null;
+  }
+
+  detectChannelForUrl(url) {
+    const parsed = assertHttpUrl(url);
+    return this.channels.find((channel) => channel.id !== 'web' && channel.canHandleUrl(parsed.toString()))
+      || this.getChannel('web');
+  }
+
+  async getStatus(userId) {
+    const statuses = [];
+    for (const channel of this.channels) {
+      try {
+        const status = await channel.check({ userId });
+        if (!status.cookie && channel.setupKind === 'cookies') {
+          status.cookie = getCookieSummary(userId, channel.id);
+        }
+        statuses.push(status);
+      } catch (error) {
+        statuses.push({
+          platform: channel.id,
+          label: channel.label,
+          ready: false,
+          status: 'error',
+          activeBackend: null,
+          tier: channel.tier,
+          setupKind: channel.setupKind,
+          message: error?.message || String(error),
+        });
+      }
+    }
+    return {
+      platforms: statuses,
+      generatedAt: new Date().toISOString(),
+    };
+  }
+
+  async read(userId, args = {}) {
+    const platform = normalizePlatformId(args.platform || '');
+    const channel = platform ? this.getChannel(platform) : this.detectChannelForUrl(args.url);
+    if (!channel) {
+      const error = new Error(`Unsupported social reach platform: ${platform || 'unknown'}`);
+      error.status = 400;
+      throw error;
+    }
+    return channel.read({ ...args, userId });
+  }
+
+  async search(userId, args = {}) {
+    const platform = normalizePlatformId(args.platform || '');
+    const channel = this.getChannel(platform);
+    if (!channel) {
+      const error = new Error('platform is required and must be supported.');
+      error.status = 400;
+      throw error;
+    }
+    return channel.search({ ...args, userId });
+  }
+
+  async importCookiesFromExtension(userId, platform, options = {}) {
+    const id = normalizePlatformId(platform);
+    const definition = getPlatformDefinition(id);
+    if (!definition || !COOKIE_IMPORT_PLATFORMS.has(id)) {
+      const error = new Error(`${definition?.label || id || 'Platform'} does not support cookie import yet.`);
+      error.status = 400;
+      throw error;
+    }
+    if (!this.browserExtensionRegistry || typeof this.browserExtensionRegistry.dispatch !== 'function') {
+      const error = new Error('Chrome extension registry is unavailable.');
+      error.status = 503;
+      throw error;
+    }
+    const domains = domainsForPlatform(id);
+    const response = await this.browserExtensionRegistry.dispatch(userId, EXTENSION_COMMANDS.GET_COOKIES, {
+      platform: id,
+      domains,
+      tokenId: options.tokenId || null,
+    }, {
+      tokenId: options.tokenId || null,
+      timeoutMs: 15000,
+    });
+    const cookies = sanitizeCookies(response?.cookies || [], domains);
+    if (cookies.length === 0) {
+      const error = new Error(`No ${definition.label} cookies were found in the connected Chrome profile. Log in first, then import again.`);
+      error.status = 422;
+      throw error;
+    }
+    const bundle = writeCookieBundle(userId, id, { cookies });
+    return {
+      platform: id,
+      configured: true,
+      count: cookies.length,
+      importedAt: bundle.importedAt,
+    };
+  }
+
+  clearCookies(userId, platform) {
+    const id = normalizePlatformId(platform);
+    deleteCookieBundle(userId, id);
+    return { platform: id, configured: false, count: 0 };
+  }
+
+  shapeError(error) {
+    return shapeError(error);
+  }
+}
+
+module.exports = {
+  SocialReachService,
+  sanitizeCookies,
+};

package/server/services/social_reach/store.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const db = require('../../db/database');
+const { decryptValue, encryptValue } = require('../integrations/secrets');
+const { normalizePlatformId } = require('./platforms');
+
+function cookieSettingKey(platform) {
+  return `social_reach_cookies_${normalizePlatformId(platform)}`;
+}
+
+function readRawSetting(userId, key) {
+  if (!userId || !key) return null;
+  return db.prepare('SELECT value FROM user_settings WHERE user_id = ? AND key = ?').get(userId, key)?.value || null;
+}
+
+function writeRawSetting(userId, key, value) {
+  db.prepare(
+    `INSERT INTO user_settings (user_id, key, value)
+     VALUES (?, ?, ?)
+     ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`,
+  ).run(userId, key, value);
+}
+
+function deleteRawSetting(userId, key) {
+  db.prepare('DELETE FROM user_settings WHERE user_id = ? AND key = ?').run(userId, key);
+}
+
+function readJsonSetting(userId, key, fallback = null) {
+  const raw = readRawSetting(userId, key);
+  if (!raw) return fallback;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return fallback;
+  }
+}
+
+function readCookieBundle(userId, platform) {
+  const raw = readRawSetting(userId, cookieSettingKey(platform));
+  if (!raw) return null;
+  try {
+    return JSON.parse(decryptValue(raw));
+  } catch {
+    return null;
+  }
+}
+
+function writeCookieBundle(userId, platform, bundle) {
+  const payload = {
+    platform: normalizePlatformId(platform),
+    importedAt: new Date().toISOString(),
+    cookies: Array.isArray(bundle?.cookies) ? bundle.cookies : [],
+  };
+  writeRawSetting(userId, cookieSettingKey(platform), encryptValue(JSON.stringify(payload)));
+  return payload;
+}
+
+function deleteCookieBundle(userId, platform) {
+  deleteRawSetting(userId, cookieSettingKey(platform));
+}
+
+function getCookieSummary(userId, platform) {
+  const bundle = readCookieBundle(userId, platform);
+  const cookies = Array.isArray(bundle?.cookies) ? bundle.cookies : [];
+  return {
+    configured: cookies.length > 0,
+    count: cookies.length,
+    importedAt: bundle?.importedAt || null,
+  };
+}
+
+function cookieHeaderForPlatform(userId, platform) {
+  const bundle = readCookieBundle(userId, platform);
+  const cookies = Array.isArray(bundle?.cookies) ? bundle.cookies : [];
+  return cookies
+    .filter((cookie) => cookie && cookie.name && cookie.value != null)
+    .map((cookie) => `${String(cookie.name).trim()}=${String(cookie.value)}`)
+    .join('; ');
+}
+
+module.exports = {
+  cookieHeaderForPlatform,
+  cookieSettingKey,
+  deleteCookieBundle,
+  getCookieSummary,
+  readCookieBundle,
+  readJsonSetting,
+  writeCookieBundle,
+};

package/server/services/social_reach/utils.js

@@ -0,0 +1,95 @@
+'use strict';
+
+const cheerio = require('cheerio');
+
+const DEFAULT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
+  + '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+
+function normalizeLimit(value, fallback = 10, max = 50) {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n <= 0) return fallback;
+  return Math.max(1, Math.min(Math.floor(n), max));
+}
+
+function assertHttpUrl(value) {
+  const raw = String(value || '').trim();
+  const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
+  let parsed;
+  try {
+    parsed = new URL(withScheme);
+  } catch {
+    const error = new Error('A valid http or https URL is required.');
+    error.status = 400;
+    throw error;
+  }
+  if (!['http:', 'https:'].includes(parsed.protocol)) {
+    const error = new Error('Only http and https URLs are supported.');
+    error.status = 400;
+    throw error;
+  }
+  return parsed;
+}
+
+async function fetchText(url, options = {}) {
+  const response = await fetch(url, {
+    redirect: 'follow',
+    ...options,
+    headers: {
+      'user-agent': DEFAULT_UA,
+      accept: 'text/plain,text/html,application/xml,application/rss+xml,application/atom+xml,*/*',
+      ...(options.headers || {}),
+    },
+  });
+  const text = await response.text();
+  if (!response.ok) {
+    const error = new Error(`Request failed with HTTP ${response.status}`);
+    error.status = response.status;
+    error.body = text.slice(0, 500);
+    throw error;
+  }
+  return text;
+}
+
+async function fetchJson(url, options = {}) {
+  const text = await fetchText(url, {
+    ...options,
+    headers: {
+      accept: 'application/json,*/*',
+      ...(options.headers || {}),
+    },
+  });
+  return JSON.parse(text);
+}
+
+function htmlToText(html, maxChars = 20000) {
+  const $ = cheerio.load(String(html || ''));
+  $('script, style, noscript, svg').remove();
+  return $('body').text().replace(/\s+/g, ' ').trim().slice(0, maxChars);
+}
+
+function parseMaybeJson(value, fallback = null) {
+  if (value == null) return fallback;
+  if (typeof value === 'object') return value;
+  try {
+    return JSON.parse(String(value));
+  } catch {
+    return fallback;
+  }
+}
+
+function compactText(value, maxChars = 4000) {
+  const text = String(value || '').replace(/\s+/g, ' ').trim();
+  if (text.length <= maxChars) return text;
+  return `${text.slice(0, maxChars).trim()}...`;
+}
+
+module.exports = {
+  DEFAULT_UA,
+  assertHttpUrl,
+  compactText,
+  fetchJson,
+  fetchText,
+  htmlToText,
+  normalizeLimit,
+  parseMaybeJson,
+};