neoagent 3.0.1-beta.9 → 3.1.0

package/server/services/runtime/guest_image.js

@@ -0,0 +1,188 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { spawnSync } = require('child_process');
+const { DATA_DIR } = require('../../../runtime/paths');
+const { stageGuestPayload, normalizeRuntimeProfile } = require('./guest_bootstrap');
+
+// Where build contexts are staged. One directory per profile.
+const BUILD_ROOT = path.join(DATA_DIR, 'runtime-vms', 'guest-image');
+// Slim official Node base — Playwright browsers + OS deps are added at build time
+// via `playwright install --with-deps`, so the image is self-contained and the
+// browser version always matches the pinned `playwright-chromium` dependency.
+const BASE_IMAGE = String(process.env.NEOAGENT_GUEST_BASE_IMAGE || 'node:20-bookworm-slim').trim();
+const IMAGE_REPO = 'neoagent-guest-agent';
+const BUILD_TIMEOUT_MS = Number(process.env.NEOAGENT_GUEST_IMAGE_BUILD_TIMEOUT_MS || 20 * 60 * 1000);
+
+// The build context is the staged guest payload (package.json + runtime/ + server/).
+// Dependencies and browsers are installed once, at image build time — never per
+// container start. Copy package.json first so the dependency layer stays cached
+// across source-only changes.
+function dockerfileFor(profile) {
+  if (normalizeRuntimeProfile(profile) === 'android') {
+    return [
+      `FROM ${BASE_IMAGE}`,
+      'ENV NODE_ENV=production',
+      'ENV NEOAGENT_GUEST_PROFILE=android',
+      'WORKDIR /opt/neoagent',
+      'COPY package.json ./',
+      'RUN npm install --omit=dev --no-audit --no-fund && npm cache clean --force',
+      'COPY runtime ./runtime',
+      'COPY server ./server',
+      // World-writable runtime dir so the agent works whether the container runs
+      // as root (macOS) or as the host uid:gid (Linux, for shared file ownership).
+      'RUN mkdir -p /opt/neoagent/.runtime && chmod -R 0777 /opt/neoagent/.runtime',
+      'CMD ["node", "server/guest_agent.js"]',
+      '',
+    ].join('\n');
+  }
+  return [
+    `FROM ${BASE_IMAGE}`,
+    'ENV NODE_ENV=production',
+    'ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright',
+    'ENV NEOAGENT_GUEST_PROFILE=browser_cli',
+    'WORKDIR /opt/neoagent',
+    'COPY package.json ./',
+    // Install deps without running package postinstall scripts, then fetch the
+    // matching Chromium and its OS dependencies explicitly. This mirrors the
+    // proven QEMU guest bootstrap and keeps the browser revision deterministic.
+    'RUN npm install --omit=dev --ignore-scripts --no-audit --no-fund \\',
+    ' && npx playwright install --with-deps chromium \\',
+    ' && npm cache clean --force',
+    'COPY runtime ./runtime',
+    'COPY server ./server',
+    // World-writable runtime dir + browser cache so the agent works whether the
+    // container runs as root (macOS) or as the host uid:gid (Linux, for shared
+    // file ownership). Chromium needs PLAYWRIGHT_BROWSERS_PATH readable by all.
+    'RUN mkdir -p /opt/neoagent/.runtime && chmod -R 0777 /opt/neoagent/.runtime && chmod -R a+rX /ms-playwright',
+    // Chromium is baked in at build time, so the runtime-ready marker the browser
+    // controller polls for (BROWSER_READY_MARKER in server/services/browser/controller.js)
+    // is already satisfied. Create it here so ensureBrowser() launches immediately
+    // instead of waiting out its full 10-minute timeout for a marker that only the
+    // legacy QEMU per-boot bootstrap ever writes.
+    'RUN mkdir -p /var/lib/neoagent && touch /var/lib/neoagent/browser-runtime-ready',
+    'CMD ["node", "server/guest_agent.js"]',
+    '',
+  ].join('\n');
+}
+
+// Stable content hash over every staged file plus the Dockerfile, so the image
+// tag changes whenever the guest source, its dependencies, or the base image
+// change. A new tag forces a rebuild; an unchanged tag reuses the cached image.
+function hashBuildContext(contextDir, dockerfile) {
+  const hash = crypto.createHash('sha256');
+  hash.update(dockerfile);
+  const walk = (dir) => {
+    for (const name of fs.readdirSync(dir).sort()) {
+      const full = path.join(dir, name);
+      const stat = fs.statSync(full);
+      if (stat.isDirectory()) {
+        walk(full);
+      } else {
+        hash.update(path.relative(contextDir, full));
+        hash.update(fs.readFileSync(full));
+      }
+    }
+  };
+  walk(contextDir);
+  return hash.digest('hex').slice(0, 12);
+}
+
+function docker(args, opts = {}) {
+  return spawnSync('docker', args, {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    timeout: opts.timeout || 30000,
+    ...opts,
+  });
+}
+
+function dockerAvailable() {
+  const result = docker(['info'], { timeout: 5000 });
+  return !result.error && result.status === 0;
+}
+
+// Builds and caches the per-profile guest-agent Docker image. The image bakes in
+// the guest agent, its dependencies, and (for browser_cli) the Chromium browser,
+// so containers start the agent directly with no runtime installation step.
+class GuestImageBuilder {
+  #buildPromise = null;
+  #cachedTag = null;
+  #cachedBuiltAt = 0;
+
+  constructor(options = {}) {
+    this.profile = normalizeRuntimeProfile(options.runtimeProfile || 'browser_cli');
+    this.contextDir = path.join(BUILD_ROOT, this.profile);
+  }
+
+  // Stage the build context on disk and return its content-addressed image tag.
+  prepareContext() {
+    const dockerfile = dockerfileFor(this.profile);
+    stageGuestPayload(this.contextDir, this.profile);
+    fs.writeFileSync(path.join(this.contextDir, 'Dockerfile'), dockerfile);
+    fs.writeFileSync(path.join(this.contextDir, '.dockerignore'), 'node_modules\n');
+    const tag = `${IMAGE_REPO}:${this.profile}-${hashBuildContext(this.contextDir, dockerfile)}`;
+    this.#cachedTag = tag;
+    return tag;
+  }
+
+  imageExists(tag) {
+    const result = docker(['image', 'inspect', tag], { timeout: 10000 });
+    return !result.error && result.status === 0;
+  }
+
+  // Returns { dockerAvailable, imageBuilt, image } without triggering a build.
+  // Cheap enough to call from readiness polling (only stages the context once).
+  getState() {
+    if (!dockerAvailable()) {
+      return { dockerAvailable: false, imageBuilt: false, image: this.#cachedTag };
+    }
+    let tag = this.#cachedTag;
+    try {
+      if (!tag) tag = this.prepareContext();
+    } catch (err) {
+      console.warn(`[GuestImage:${this.profile}] Failed to stage build context: ${err.message}`);
+      return { dockerAvailable: true, imageBuilt: false, image: null };
+    }
+    return { dockerAvailable: true, imageBuilt: this.imageExists(tag), image: tag };
+  }
+
+  // Ensure the image exists, building it if necessary. Concurrent callers share
+  // the same in-flight build. Returns the resolved image tag.
+  async ensure() {
+    const tag = this.prepareContext();
+    if (this.imageExists(tag)) {
+      this.#cachedBuiltAt = Date.now();
+      return tag;
+    }
+    if (this.#buildPromise) return this.#buildPromise;
+    this.#buildPromise = this.#build(tag).finally(() => { this.#buildPromise = null; });
+    return this.#buildPromise;
+  }
+
+  async #build(tag) {
+    console.log(`[GuestImage:${this.profile}] Building guest image ${tag} (one-time; downloads browser + deps)…`);
+    const started = Date.now();
+    const result = docker(['build', '-t', tag, this.contextDir], {
+      timeout: BUILD_TIMEOUT_MS,
+      stdio: ['ignore', 'inherit', 'inherit'],
+    });
+    if (result.error) {
+      throw new Error(`Guest image build failed to start: ${result.error.message}`);
+    }
+    if (result.status !== 0) {
+      throw new Error(`Guest image build for ${tag} exited with status ${result.status}`);
+    }
+    this.#cachedBuiltAt = Date.now();
+    console.log(`[GuestImage:${this.profile}] Built ${tag} in ${Math.round((Date.now() - started) / 1000)}s`);
+    return tag;
+  }
+}
+
+module.exports = {
+  GuestImageBuilder,
+  dockerAvailable,
+  BASE_IMAGE,
+};

package/server/services/runtime/manager.js

@@ -19,7 +19,6 @@ class RuntimeManager {
 
     const browserVmManager = options.browserVmManager || new DockerVMManager({
       runtimeProfile: 'browser_cli',
-      image: 'mcr.microsoft.com/playwright:v1.44.0-focal',
       memoryMb: DEFAULT_VM_MEMORY_MB,
       cpus: DEFAULT_VM_CPUS,
     });

package/server/services/runtime/validation.js

@@ -11,14 +11,9 @@ function getRuntimeValidation(runtimeManager) {
 
   if (policy.profile === 'prod' || nodeEnvIsProd) {
     if (!browserVmReadiness) {
-      issues.push('prod profile requires a working local VM runtime for browser/CLI.');
-    } else if (!browserVmReadiness.ready) {
-      if (!browserVmReadiness.qemuAvailable) {
-        issues.push(`prod profile requires QEMU (${browserVmReadiness.qemuBinary}) to be installed for browser/CLI.`);
-      }
-      if (!browserVmReadiness.baseImageExists && !browserVmReadiness.downloadConfigured) {
-        issues.push('prod profile requires a VM base image or a downloadable base image URL for browser/CLI.');
-      }
+      issues.push('prod profile requires a working container runtime for browser/CLI.');
+    } else if (!browserVmReadiness.dockerAvailable) {
+      issues.push('prod profile requires Docker to be installed and running for the browser/CLI runtime.');
     }
   }
 

package/server/services/social_video/service.js

@@ -22,6 +22,41 @@ fs.mkdirSync(SOCIAL_VIDEO_TMP_DIR, { recursive: true });
 
 const HEALTH_CACHE_TTL_MS = 5 * 60 * 1000;
 
+// A realistic desktop browser UA. Social platforms frequently return 403/bot
+// challenge pages to the default Node/yt-dlp user agents, so we present a
+// browser-like identity for every outbound request.
+const BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+  + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+
+// Network hardening flags applied to every yt-dlp invocation. Transient
+// failures (rate limits, dropped fragments, slow extractors) are the most
+// common cause of unreliable extraction, so we retry aggressively and cap
+// socket waits instead of failing on the first hiccup.
+const YT_DLP_NETWORK_FLAGS = [
+  '--retries', '5',
+  '--fragment-retries', '5',
+  '--extractor-retries', '3',
+  '--socket-timeout', '30',
+  '--user-agent', BROWSER_USER_AGENT,
+];
+
+// Platforms that gate most video content behind authentication or aggressive
+// bot detection. For these we always try to attach the user's browser cookies.
+const COOKIE_ASSISTED_PLATFORMS = new Set(['instagram', 'youtube', 'tiktok', 'x']);
+
+async function fetchWithBrowserHeaders(url, options = {}) {
+  return fetch(url, {
+    redirect: 'follow',
+    ...options,
+    headers: {
+      'user-agent': BROWSER_USER_AGENT,
+      'accept-language': 'en-US,en;q=0.9',
+      accept: '*/*',
+      ...(options.headers || {}),
+    },
+  });
+}
+
 function shellEscape(value) {
   const text = String(value ?? '');
   if (!text.length) return process.platform === 'win32' ? '""' : "''";
@@ -178,12 +213,21 @@ function classifyExtractionError(error) {
   if (/unsupported social video url|unsupported url/.test(normalized)) {
     return { code: 'unsupported_url', message };
   }
-  if (/private|login required|sign in to confirm/.test(normalized)) {
+  if (/private|login required|sign in to confirm|requested content is not available|account is private/.test(normalized)) {
     return { code: 'private_or_auth_required', message };
   }
-  if (/403|forbidden|blocked/.test(normalized)) {
+  if (/\b429\b|rate.?limit|too many requests/.test(normalized)) {
+    return { code: 'rate_limited', message };
+  }
+  if (/\b403\b|forbidden|blocked|not a bot/.test(normalized)) {
     return { code: 'blocked_or_unavailable', message };
   }
+  if (/timed out|timeout|etimedout|socket hang up|network is unreachable/.test(normalized)) {
+    return { code: 'network_error', message };
+  }
+  if (/this video is unavailable|video unavailable|removed|404|not found/.test(normalized)) {
+    return { code: 'content_unavailable', message };
+  }
   return { code: 'social_video_extract_failed', message };
 }
 
@@ -358,6 +402,10 @@ class SocialVideoService {
     }
   }
 
+  #networkFlags() {
+    return YT_DLP_NETWORK_FLAGS.map(shellEscape).join(' ');
+  }
+
   async #runCommand(command, options = {}) {
     const result = await this.cliExecutor.execute(command, {
       cwd: options.cwd || process.cwd(),
@@ -423,7 +471,7 @@ class SocialVideoService {
       return browserMetadata;
     }
 
-    const response = await fetch(normalizedUrl, { redirect: 'follow' });
+    const response = await fetchWithBrowserHeaders(normalizedUrl);
     const html = await response.text();
     const metadata = extractPublicMetadataFromHtml(html, response.url || normalizedUrl);
     return {
@@ -473,7 +521,7 @@ class SocialVideoService {
     const infoTemplate = path.join(jobDir, 'media.%(ext)s');
     const infoPath = path.join(jobDir, 'media.info.json');
     const cookieArg = cookieFilePath ? ` --cookies ${shellEscape(cookieFilePath)}` : '';
-    const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist --skip-download --write-info-json --no-clean-infojson${cookieArg} -o ${shellEscape(infoTemplate)} -- ${shellEscape(normalizedUrl)}`;
+    const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist ${this.#networkFlags()} --skip-download --write-info-json --no-clean-infojson${cookieArg} -o ${shellEscape(infoTemplate)} -- ${shellEscape(normalizedUrl)}`;
     await this.#runCommand(command, { cwd: jobDir, timeout: 4 * 60 * 1000 });
     if (!fileExists(infoPath)) {
       throw new Error('yt-dlp did not produce an info JSON artifact.');
@@ -514,7 +562,7 @@ class SocialVideoService {
   }
 
   async #readTranscriptFromCaption(captionTrack) {
-    const response = await fetch(captionTrack.url, { redirect: 'follow' });
+    const response = await fetchWithBrowserHeaders(captionTrack.url);
     if (!response.ok) {
       throw new Error(`Caption request failed (${response.status}).`);
     }
@@ -525,7 +573,7 @@ class SocialVideoService {
   async #transcribeViaStt(context) {
     const template = path.join(context.jobDir, 'audio.%(ext)s');
     const cookieArg = context.cookieFilePath ? ` --cookies ${shellEscape(context.cookieFilePath)}` : '';
-    const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist${cookieArg} -o ${shellEscape(template)} -f bestaudio -- ${shellEscape(context.sourceUrl)}`;
+    const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist ${this.#networkFlags()}${cookieArg} -o ${shellEscape(template)} -f bestaudio/best -- ${shellEscape(context.sourceUrl)}`;
     await this.#runCommand(command, { cwd: context.jobDir, timeout: 10 * 60 * 1000 });
 
     const audioPath = firstFileMatching(context.jobDir, 'audio.');
@@ -561,7 +609,7 @@ class SocialVideoService {
   }
 
   async #resolveCookieFile(context) {
-    if (context.platform !== 'instagram') {
+    if (!COOKIE_ASSISTED_PLATFORMS.has(context.platform)) {
       return null;
     }
     if (!this.runtimeManager || typeof this.runtimeManager.getBrowserProviderForUser !== 'function') {
@@ -581,7 +629,9 @@ class SocialVideoService {
     });
     const cookies = Array.isArray(payload?.cookies) ? payload.cookies : [];
     if (cookies.length === 0) {
-      context.warnings.push('Browser cookie export returned no cookies for Instagram.');
+      // Not fatal: many videos are public. yt-dlp will simply attempt the
+      // extraction without an authenticated session.
+      context.warnings.push(`Browser cookie export returned no cookies for ${context.platform}.`);
       return null;
     }
 
@@ -610,7 +660,7 @@ class SocialVideoService {
   async #extractFrameFromVideo(context) {
     const template = path.join(context.jobDir, 'video.%(ext)s');
     const cookieArg = context.cookieFilePath ? ` --cookies ${shellEscape(context.cookieFilePath)}` : '';
-    const downloadCommand = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist${cookieArg} -o ${shellEscape(template)} -f "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/best" --merge-output-format mp4 -- ${shellEscape(context.sourceUrl)}`;
+    const downloadCommand = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist ${this.#networkFlags()}${cookieArg} -o ${shellEscape(template)} -f "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/best" --merge-output-format mp4 -- ${shellEscape(context.sourceUrl)}`;
     await this.#runCommand(downloadCommand, { cwd: context.jobDir, timeout: 14 * 60 * 1000 });
 
     const videoPath = firstFileMatching(context.jobDir, 'video.');
@@ -630,7 +680,7 @@ class SocialVideoService {
   }
 
   async #downloadThumbnailArtifact(userId, thumbnailUrl) {
-    const response = await fetch(thumbnailUrl, { redirect: 'follow' });
+    const response = await fetchWithBrowserHeaders(thumbnailUrl);
     if (!response.ok) {
       throw new Error(`Thumbnail request failed (${response.status}).`);
     }