neoagent 3.2.1-beta.2 → 3.2.1-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/chrome-browser/background.mjs +318 -88
- package/extensions/chrome-browser/http.mjs +136 -0
- package/extensions/chrome-browser/protocol.mjs +511 -89
- package/flutter_app/lib/main_chat.dart +118 -739
- package/flutter_app/lib/main_controller.dart +29 -20
- package/flutter_app/lib/main_models.dart +3 -0
- package/flutter_app/lib/main_operations.dart +334 -321
- package/flutter_app/lib/main_settings.dart +4 -3
- package/flutter_app/lib/main_shared.dart +14 -11
- package/flutter_app/lib/src/desktop_companion_actions.dart +185 -31
- package/flutter_app/lib/src/desktop_companion_io.dart +319 -86
- package/flutter_app/windows/runner/flutter_window.cpp +143 -32
- package/lib/manager.js +106 -89
- package/lib/schema_migrations.js +67 -13
- package/package.json +20 -13
- package/runtime/paths.js +49 -5
- package/server/guest_agent.js +52 -30
- package/server/http/middleware.js +24 -0
- package/server/http/routes.js +4 -6
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +30803 -30805
- package/server/routes/admin.js +1 -1
- package/server/routes/android.js +30 -34
- package/server/routes/browser.js +23 -15
- package/server/routes/desktop.js +18 -1
- package/server/routes/integrations.js +5 -1
- package/server/routes/memory.js +1 -0
- package/server/routes/settings.js +16 -5
- package/server/routes/social_reach.js +12 -3
- package/server/routes/social_video.js +4 -0
- package/server/services/ai/compaction.js +7 -2
- package/server/services/ai/history.js +44 -5
- package/server/services/ai/integrated_tools/http_request.js +8 -0
- package/server/services/ai/loop/agent_engine_core.js +347 -162
- package/server/services/ai/loop/callbacks.js +1 -0
- package/server/services/ai/loop/completion_judge.js +12 -0
- package/server/services/ai/loop/conversation_loop.js +348 -242
- package/server/services/ai/loop/messaging_delivery.js +129 -57
- package/server/services/ai/loop/model_call_guard.js +91 -0
- package/server/services/ai/loop/model_io.js +8 -41
- package/server/services/ai/loop/tool_dispatch.js +19 -8
- package/server/services/ai/loopPolicy.js +24 -19
- package/server/services/ai/model_discovery.js +227 -0
- package/server/services/ai/model_identity.js +71 -0
- package/server/services/ai/models.js +67 -162
- package/server/services/ai/providerRetry.js +17 -59
- package/server/services/ai/provider_selector.js +111 -0
- package/server/services/ai/providers/anthropic.js +2 -2
- package/server/services/ai/providers/claudeCode.js +15 -28
- package/server/services/ai/providers/githubCopilot.js +36 -16
- package/server/services/ai/providers/google.js +23 -5
- package/server/services/ai/providers/grok.js +4 -3
- package/server/services/ai/providers/grokOauth.js +13 -22
- package/server/services/ai/providers/nvidia.js +10 -5
- package/server/services/ai/providers/ollama.js +102 -82
- package/server/services/ai/providers/ollama_stream.js +142 -0
- package/server/services/ai/providers/openai.js +39 -5
- package/server/services/ai/providers/openaiCodex.js +11 -4
- package/server/services/ai/providers/openrouter.js +29 -7
- package/server/services/ai/providers/provider_error.js +36 -0
- package/server/services/ai/settings.js +26 -2
- package/server/services/ai/taskAnalysis.js +5 -1
- package/server/services/ai/terminal_reply.js +45 -0
- package/server/services/ai/toolEvidence.js +58 -29
- package/server/services/ai/tools.js +124 -111
- package/server/services/android/controller.js +770 -237
- package/server/services/android/process.js +140 -0
- package/server/services/android/sdk_download.js +143 -0
- package/server/services/android/uia.js +6 -5
- package/server/services/artifacts/store.js +24 -0
- package/server/services/browser/controller.js +736 -385
- package/server/services/browser/extension/gateway.js +40 -16
- package/server/services/browser/extension/protocol.js +12 -1
- package/server/services/browser/extension/provider.js +59 -47
- package/server/services/browser/extension/registry.js +155 -34
- package/server/services/cli/executor.js +62 -9
- package/server/services/desktop/gateway.js +41 -4
- package/server/services/desktop/protocol.js +3 -0
- package/server/services/desktop/provider.js +39 -42
- package/server/services/desktop/registry.js +137 -52
- package/server/services/integrations/figma/provider.js +78 -12
- package/server/services/integrations/github/common.js +11 -6
- package/server/services/integrations/github/provider.js +52 -53
- package/server/services/integrations/google/provider.js +55 -19
- package/server/services/integrations/home_assistant/network.js +17 -20
- package/server/services/integrations/home_assistant/provider.js +7 -5
- package/server/services/integrations/home_assistant/tools.js +17 -5
- package/server/services/integrations/http.js +51 -0
- package/server/services/integrations/manager.js +158 -53
- package/server/services/integrations/microsoft/provider.js +80 -13
- package/server/services/integrations/neoarchive/provider.js +55 -29
- package/server/services/integrations/neorecall/client.js +17 -10
- package/server/services/integrations/neorecall/provider.js +20 -11
- package/server/services/integrations/notion/provider.js +16 -13
- package/server/services/integrations/oauth_provider.js +115 -51
- package/server/services/integrations/slack/provider.js +98 -9
- package/server/services/integrations/spotify/provider.js +67 -71
- package/server/services/integrations/trello/provider.js +21 -7
- package/server/services/integrations/weather/provider.js +18 -12
- package/server/services/integrations/whatsapp/provider.js +76 -16
- package/server/services/manager.js +87 -1
- package/server/services/memory/embedding_index.js +20 -8
- package/server/services/memory/embeddings.js +151 -90
- package/server/services/memory/ingestion.js +50 -9
- package/server/services/memory/ingestion_documents.js +13 -3
- package/server/services/memory/manager.js +52 -19
- package/server/services/messaging/automation.js +84 -9
- package/server/services/messaging/http_platforms.js +33 -13
- package/server/services/messaging/inbound_queue.js +78 -24
- package/server/services/messaging/inbound_store.js +224 -0
- package/server/services/messaging/manager.js +326 -51
- package/server/services/messaging/typing_keepalive.js +5 -2
- package/server/services/messaging/whatsapp.js +22 -14
- package/server/services/network/http.js +210 -0
- package/server/services/network/safe_request.js +307 -0
- package/server/services/runtime/backends/local-vm.js +214 -66
- package/server/services/runtime/manager.js +17 -12
- package/server/services/social_reach/channels/github.js +10 -4
- package/server/services/social_reach/channels/reddit.js +4 -4
- package/server/services/social_reach/channels/rss.js +2 -2
- package/server/services/social_reach/channels/social_video.js +12 -7
- package/server/services/social_reach/channels/v2ex.js +21 -8
- package/server/services/social_reach/channels/x.js +2 -2
- package/server/services/social_reach/channels/xueqiu.js +5 -5
- package/server/services/social_reach/service.js +9 -6
- package/server/services/social_reach/utils.js +65 -14
- package/server/services/social_video/service.js +160 -50
- package/server/services/tasks/integration_runtime.js +18 -8
- package/server/services/tasks/runtime.js +39 -4
- package/server/services/voice/agentBridge.js +17 -4
- package/server/services/voice/bufferedLiveRelayAdapter.js +5 -0
- package/server/services/voice/liveSession.js +31 -0
- package/server/services/voice/openaiSpeech.js +33 -8
- package/server/services/voice/providers.js +233 -151
- package/server/services/voice/runtimeManager.js +118 -20
- package/server/services/voice/turnRunner.js +6 -0
- package/server/services/wearable/firmware_manifest.js +51 -13
- package/server/services/wearable/service.js +1 -0
- package/server/utils/abort.js +96 -0
- package/server/utils/cloud-security.js +110 -3
- package/server/utils/files.js +31 -0
- package/server/utils/image_payload.js +95 -0
- package/server/utils/retry.js +107 -0
|
@@ -8,19 +8,28 @@ const { randomUUID } = require('crypto');
|
|
|
8
8
|
const db = require('../../db/database');
|
|
9
9
|
const { DATA_DIR } = require('../../../runtime/paths');
|
|
10
10
|
const { CLIExecutor } = require('../cli/executor');
|
|
11
|
+
const { executeSafeHttpRequest } = require('../network/safe_request');
|
|
11
12
|
const { getAdapterForPlatform } = require('./adapters');
|
|
12
|
-
const {
|
|
13
|
+
const {
|
|
14
|
+
MAX_VTT_BYTES,
|
|
15
|
+
decideTranscriptPath,
|
|
16
|
+
parseCaptionText,
|
|
17
|
+
pickCaptionTrack,
|
|
18
|
+
} = require('./captions');
|
|
13
19
|
const { inferImageContentType, pickDeterministicFrameSecond } = require('./frame');
|
|
14
20
|
const { extractPublicMetadataFromHtml } = require('./metadata');
|
|
15
21
|
const { shapeSocialVideoResult } = require('./result');
|
|
16
22
|
const { normalizeAndDetectPlatform } = require('./url');
|
|
17
23
|
const { isMainAgent } = require('../agents/manager');
|
|
18
24
|
const { resolveSttModel, transcribeVoiceInput } = require('../voice/providers');
|
|
25
|
+
const { createAbortError, isAbortError, throwIfAborted } = require('../../utils/abort');
|
|
19
26
|
|
|
20
27
|
const SOCIAL_VIDEO_TMP_DIR = path.join(DATA_DIR, 'social-video-temp');
|
|
21
28
|
fs.mkdirSync(SOCIAL_VIDEO_TMP_DIR, { recursive: true });
|
|
22
29
|
|
|
23
30
|
const HEALTH_CACHE_TTL_MS = 5 * 60 * 1000;
|
|
31
|
+
const MAX_PAGE_HTML_BYTES = 2 * 1024 * 1024;
|
|
32
|
+
const MAX_THUMBNAIL_BYTES = 10 * 1024 * 1024;
|
|
24
33
|
|
|
25
34
|
// A realistic desktop browser UA. Social platforms frequently return 403/bot
|
|
26
35
|
// challenge pages to the default Node/yt-dlp user agents, so we present a
|
|
@@ -44,17 +53,38 @@ const YT_DLP_NETWORK_FLAGS = [
|
|
|
44
53
|
// bot detection. For these we always try to attach the user's browser cookies.
|
|
45
54
|
const COOKIE_ASSISTED_PLATFORMS = new Set(['instagram', 'youtube', 'tiktok', 'x']);
|
|
46
55
|
|
|
47
|
-
async function
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
56
|
+
async function fetchPublicResource(url, options = {}) {
|
|
57
|
+
const result = await executeSafeHttpRequest({
|
|
58
|
+
url,
|
|
59
|
+
method: 'GET',
|
|
60
|
+
timeout_ms: options.timeoutMs || 30000,
|
|
51
61
|
headers: {
|
|
52
62
|
'user-agent': BROWSER_USER_AGENT,
|
|
53
63
|
'accept-language': 'en-US,en;q=0.9',
|
|
54
|
-
accept: '*/*',
|
|
64
|
+
accept: options.accept || '*/*',
|
|
55
65
|
...(options.headers || {}),
|
|
56
66
|
},
|
|
67
|
+
}, {
|
|
68
|
+
signal: options.signal,
|
|
69
|
+
maxResponseBytes: options.maxResponseBytes,
|
|
70
|
+
responseType: options.responseType,
|
|
57
71
|
});
|
|
72
|
+
if (result.truncated) {
|
|
73
|
+
const error = new Error('Social video response exceeded its safety limit.');
|
|
74
|
+
error.code = 'SOCIAL_VIDEO_RESPONSE_TOO_LARGE';
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
if (result.status < 200 || result.status >= 300) {
|
|
78
|
+
const error = new Error(`Social video request failed (${result.status}).`);
|
|
79
|
+
error.status = result.status;
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function rethrowCancellation(error, signal) {
|
|
86
|
+
if (signal?.aborted) throw createAbortError(signal);
|
|
87
|
+
if (isAbortError(error)) throw error;
|
|
58
88
|
}
|
|
59
89
|
|
|
60
90
|
function shellEscape(value) {
|
|
@@ -256,6 +286,7 @@ class SocialVideoService {
|
|
|
256
286
|
}
|
|
257
287
|
|
|
258
288
|
async getHealthStatus(options = {}) {
|
|
289
|
+
throwIfAborted(options.signal, 'Social video health check aborted.');
|
|
259
290
|
const forceRefresh = options.forceRefresh === true;
|
|
260
291
|
const now = Date.now();
|
|
261
292
|
if (!forceRefresh && this._healthCache.value && (now - this._healthCache.ts) < HEALTH_CACHE_TTL_MS) {
|
|
@@ -263,8 +294,8 @@ class SocialVideoService {
|
|
|
263
294
|
}
|
|
264
295
|
|
|
265
296
|
const [ytDlp, ffmpeg] = await Promise.all([
|
|
266
|
-
this.#probeBinary(this.ytDlpBin, '--version'),
|
|
267
|
-
this.#probeBinary(this.ffmpegBin, '-version'),
|
|
297
|
+
this.#probeBinary(this.ytDlpBin, '--version', options.signal),
|
|
298
|
+
this.#probeBinary(this.ffmpegBin, '-version', options.signal),
|
|
268
299
|
]);
|
|
269
300
|
|
|
270
301
|
const health = {
|
|
@@ -291,7 +322,8 @@ class SocialVideoService {
|
|
|
291
322
|
let jobDir = null;
|
|
292
323
|
|
|
293
324
|
try {
|
|
294
|
-
|
|
325
|
+
throwIfAborted(options.signal, 'Social video extraction aborted.');
|
|
326
|
+
const health = await this.getHealthStatus({ signal: options.signal });
|
|
295
327
|
if (!health.ready) {
|
|
296
328
|
const missing = health.dependencies.filter((item) => !item.available).map((item) => item.name);
|
|
297
329
|
throw new Error(`Missing required dependency: ${missing.join(', ')}`);
|
|
@@ -303,16 +335,28 @@ class SocialVideoService {
|
|
|
303
335
|
throw new Error(`No adapter registered for platform: ${platform}`);
|
|
304
336
|
}
|
|
305
337
|
|
|
306
|
-
const pageMetadata = await this.#resolvePageMetadata(
|
|
338
|
+
const pageMetadata = await this.#resolvePageMetadata(
|
|
339
|
+
userId,
|
|
340
|
+
normalizedUrl,
|
|
341
|
+
warnings,
|
|
342
|
+
options.signal,
|
|
343
|
+
);
|
|
344
|
+
throwIfAborted(options.signal, 'Social video extraction aborted.');
|
|
307
345
|
jobDir = await fsp.mkdtemp(path.join(SOCIAL_VIDEO_TMP_DIR, `${platform}-${Date.now()}-`));
|
|
308
346
|
const cookieFilePath = await this.#resolveCookieFile({
|
|
309
347
|
userId,
|
|
310
348
|
platform,
|
|
311
349
|
jobDir,
|
|
312
350
|
warnings,
|
|
351
|
+
signal: options.signal,
|
|
313
352
|
});
|
|
314
353
|
|
|
315
|
-
const mediaInfo = await this.#readMediaInfo(
|
|
354
|
+
const mediaInfo = await this.#readMediaInfo(
|
|
355
|
+
normalizedUrl,
|
|
356
|
+
jobDir,
|
|
357
|
+
cookieFilePath,
|
|
358
|
+
options.signal,
|
|
359
|
+
);
|
|
316
360
|
const baseTitle = String(pageMetadata.title || mediaInfo.title || '').trim();
|
|
317
361
|
const baseDescription = String(pageMetadata.description || mediaInfo.description || '').trim();
|
|
318
362
|
const resolvedUrl = String(pageMetadata.resolvedUrl || mediaInfo.webpage_url || normalizedUrl).trim();
|
|
@@ -339,6 +383,7 @@ class SocialVideoService {
|
|
|
339
383
|
userId,
|
|
340
384
|
agentId,
|
|
341
385
|
warnings,
|
|
386
|
+
signal: options.signal,
|
|
342
387
|
});
|
|
343
388
|
|
|
344
389
|
const frameImage = options.includeFrame === false
|
|
@@ -350,6 +395,7 @@ class SocialVideoService {
|
|
|
350
395
|
jobDir,
|
|
351
396
|
cookieFilePath,
|
|
352
397
|
warnings,
|
|
398
|
+
signal: options.signal,
|
|
353
399
|
});
|
|
354
400
|
|
|
355
401
|
return shapeSocialVideoResult({
|
|
@@ -372,7 +418,8 @@ class SocialVideoService {
|
|
|
372
418
|
errors,
|
|
373
419
|
});
|
|
374
420
|
} catch (error) {
|
|
375
|
-
|
|
421
|
+
rethrowCancellation(error, options.signal);
|
|
422
|
+
const health = await this.getHealthStatus({ signal: options.signal }).catch(() => null);
|
|
376
423
|
errors.push(classifyExtractionError(error));
|
|
377
424
|
return shapeSocialVideoResult({
|
|
378
425
|
sourceUrl: source,
|
|
@@ -403,14 +450,16 @@ class SocialVideoService {
|
|
|
403
450
|
cwd: options.cwd || process.cwd(),
|
|
404
451
|
timeout: options.timeout || 10 * 60 * 1000,
|
|
405
452
|
env: options.env,
|
|
453
|
+
signal: options.signal,
|
|
406
454
|
});
|
|
455
|
+
throwIfAborted(options.signal, 'Social video command aborted.');
|
|
407
456
|
if (result.exitCode !== 0) {
|
|
408
457
|
throw new Error(result.stderr || result.stdout || `Command failed: ${command}`);
|
|
409
458
|
}
|
|
410
459
|
return result;
|
|
411
460
|
}
|
|
412
461
|
|
|
413
|
-
async #probeBinary(binary, versionFlag) {
|
|
462
|
+
async #probeBinary(binary, versionFlag, signal = null) {
|
|
414
463
|
const name = String(binary || '').trim();
|
|
415
464
|
const fallback = {
|
|
416
465
|
name,
|
|
@@ -430,7 +479,9 @@ class SocialVideoService {
|
|
|
430
479
|
const command = `${shellEscape(name)} ${versionFlag}`;
|
|
431
480
|
const result = await this.cliExecutor.execute(command, {
|
|
432
481
|
timeout: 8 * 1000,
|
|
482
|
+
signal,
|
|
433
483
|
});
|
|
484
|
+
throwIfAborted(signal, 'Social video dependency probe aborted.');
|
|
434
485
|
if (result.exitCode !== 0) {
|
|
435
486
|
return {
|
|
436
487
|
...fallback,
|
|
@@ -447,6 +498,7 @@ class SocialVideoService {
|
|
|
447
498
|
error: null,
|
|
448
499
|
};
|
|
449
500
|
} catch (error) {
|
|
501
|
+
rethrowCancellation(error, signal);
|
|
450
502
|
return {
|
|
451
503
|
...fallback,
|
|
452
504
|
error: error.message || String(error),
|
|
@@ -454,8 +506,13 @@ class SocialVideoService {
|
|
|
454
506
|
}
|
|
455
507
|
}
|
|
456
508
|
|
|
457
|
-
async #resolvePageMetadata(userId, normalizedUrl, warnings) {
|
|
458
|
-
const browserMetadata = await this.#resolvePageMetadataViaBrowser(
|
|
509
|
+
async #resolvePageMetadata(userId, normalizedUrl, warnings, signal = null) {
|
|
510
|
+
const browserMetadata = await this.#resolvePageMetadataViaBrowser(
|
|
511
|
+
userId,
|
|
512
|
+
normalizedUrl,
|
|
513
|
+
signal,
|
|
514
|
+
).catch((error) => {
|
|
515
|
+
rethrowCancellation(error, signal);
|
|
459
516
|
warnings.push(`Browser metadata resolve failed: ${error.message}`);
|
|
460
517
|
return null;
|
|
461
518
|
});
|
|
@@ -463,21 +520,24 @@ class SocialVideoService {
|
|
|
463
520
|
return browserMetadata;
|
|
464
521
|
}
|
|
465
522
|
|
|
466
|
-
const response = await
|
|
467
|
-
|
|
468
|
-
|
|
523
|
+
const response = await fetchPublicResource(normalizedUrl, {
|
|
524
|
+
signal,
|
|
525
|
+
maxResponseBytes: MAX_PAGE_HTML_BYTES,
|
|
526
|
+
accept: 'text/html,*/*',
|
|
527
|
+
});
|
|
528
|
+
const metadata = extractPublicMetadataFromHtml(response.body, response.finalUrl || normalizedUrl);
|
|
469
529
|
return {
|
|
470
530
|
...metadata,
|
|
471
|
-
resolvedUrl: String(response.
|
|
531
|
+
resolvedUrl: String(response.finalUrl || normalizedUrl),
|
|
472
532
|
};
|
|
473
533
|
}
|
|
474
534
|
|
|
475
|
-
async #resolvePageMetadataViaBrowser(userId, normalizedUrl) {
|
|
535
|
+
async #resolvePageMetadataViaBrowser(userId, normalizedUrl, signal = null) {
|
|
476
536
|
if (!this.runtimeManager || typeof this.runtimeManager.getBrowserProviderForUser !== 'function') {
|
|
477
537
|
throw new Error('Runtime browser provider is unavailable.');
|
|
478
538
|
}
|
|
479
539
|
|
|
480
|
-
const browser = await this.runtimeManager.getBrowserProviderForUser(userId);
|
|
540
|
+
const browser = await this.runtimeManager.getBrowserProviderForUser(userId, { signal });
|
|
481
541
|
if (!browser || typeof browser.navigate !== 'function' || typeof browser.extract !== 'function') {
|
|
482
542
|
throw new Error('Runtime browser provider does not support metadata extraction.');
|
|
483
543
|
}
|
|
@@ -485,16 +545,29 @@ class SocialVideoService {
|
|
|
485
545
|
const nav = await browser.navigate(normalizedUrl, {
|
|
486
546
|
screenshot: false,
|
|
487
547
|
waitUntil: 'domcontentloaded',
|
|
548
|
+
signal,
|
|
488
549
|
});
|
|
489
550
|
if (nav?.error) {
|
|
490
551
|
throw new Error(nav.error);
|
|
491
552
|
}
|
|
492
553
|
|
|
493
554
|
const [canonicalRaw, descriptionRaw, ogDescriptionRaw, titleTagRaw] = await Promise.all([
|
|
494
|
-
browser.extract('link[rel="canonical"]', 'href', false).catch(() =>
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
555
|
+
browser.extract('link[rel="canonical"]', 'href', false, { signal }).catch((error) => {
|
|
556
|
+
rethrowCancellation(error, signal);
|
|
557
|
+
return '';
|
|
558
|
+
}),
|
|
559
|
+
browser.extract('meta[name="description"]', 'content', false, { signal }).catch((error) => {
|
|
560
|
+
rethrowCancellation(error, signal);
|
|
561
|
+
return '';
|
|
562
|
+
}),
|
|
563
|
+
browser.extract('meta[property="og:description"]', 'content', false, { signal }).catch((error) => {
|
|
564
|
+
rethrowCancellation(error, signal);
|
|
565
|
+
return '';
|
|
566
|
+
}),
|
|
567
|
+
browser.extract('meta[property="og:title"]', 'content', false, { signal }).catch((error) => {
|
|
568
|
+
rethrowCancellation(error, signal);
|
|
569
|
+
return '';
|
|
570
|
+
}),
|
|
498
571
|
]);
|
|
499
572
|
const canonical = unwrapBrowserExtractValue(canonicalRaw);
|
|
500
573
|
const description = unwrapBrowserExtractValue(descriptionRaw);
|
|
@@ -509,16 +582,17 @@ class SocialVideoService {
|
|
|
509
582
|
};
|
|
510
583
|
}
|
|
511
584
|
|
|
512
|
-
async #readMediaInfo(normalizedUrl, jobDir, cookieFilePath = null) {
|
|
585
|
+
async #readMediaInfo(normalizedUrl, jobDir, cookieFilePath = null, signal = null) {
|
|
513
586
|
const infoTemplate = path.join(jobDir, 'media.%(ext)s');
|
|
514
587
|
const infoPath = path.join(jobDir, 'media.info.json');
|
|
515
588
|
const cookieArg = cookieFilePath ? ` --cookies ${shellEscape(cookieFilePath)}` : '';
|
|
516
589
|
const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist ${this.#networkFlags()} --skip-download --write-info-json --no-clean-infojson${cookieArg} -o ${shellEscape(infoTemplate)} -- ${shellEscape(normalizedUrl)}`;
|
|
517
|
-
await this.#runCommand(command, { cwd: jobDir, timeout: 4 * 60 * 1000 });
|
|
590
|
+
await this.#runCommand(command, { cwd: jobDir, timeout: 4 * 60 * 1000, signal });
|
|
518
591
|
if (!fileExists(infoPath)) {
|
|
519
592
|
throw new Error('yt-dlp did not produce an info JSON artifact.');
|
|
520
593
|
}
|
|
521
594
|
const raw = String(await fsp.readFile(infoPath, 'utf8')).trim();
|
|
595
|
+
throwIfAborted(signal, 'Social video extraction aborted.');
|
|
522
596
|
let parsed;
|
|
523
597
|
try {
|
|
524
598
|
parsed = JSON.parse(raw);
|
|
@@ -530,7 +604,11 @@ class SocialVideoService {
|
|
|
530
604
|
|
|
531
605
|
async #resolveTranscript(context) {
|
|
532
606
|
if (context.transcriptDecision.mode === 'captions' && context.captionTrack) {
|
|
533
|
-
const captionText = await this.#readTranscriptFromCaption(
|
|
607
|
+
const captionText = await this.#readTranscriptFromCaption(
|
|
608
|
+
context.captionTrack,
|
|
609
|
+
context.signal,
|
|
610
|
+
).catch((error) => {
|
|
611
|
+
rethrowCancellation(error, context.signal);
|
|
534
612
|
context.warnings.push(`Caption transcript failed: ${error.message}`);
|
|
535
613
|
return '';
|
|
536
614
|
});
|
|
@@ -544,6 +622,7 @@ class SocialVideoService {
|
|
|
544
622
|
}
|
|
545
623
|
|
|
546
624
|
const transcript = await this.#transcribeViaStt(context).catch((error) => {
|
|
625
|
+
rethrowCancellation(error, context.signal);
|
|
547
626
|
context.warnings.push(`Speech-to-text fallback failed: ${error.message}`);
|
|
548
627
|
return '';
|
|
549
628
|
});
|
|
@@ -553,20 +632,24 @@ class SocialVideoService {
|
|
|
553
632
|
};
|
|
554
633
|
}
|
|
555
634
|
|
|
556
|
-
async #readTranscriptFromCaption(captionTrack) {
|
|
557
|
-
const response = await
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
return parseCaptionText(
|
|
635
|
+
async #readTranscriptFromCaption(captionTrack, signal = null) {
|
|
636
|
+
const response = await fetchPublicResource(captionTrack.url, {
|
|
637
|
+
signal,
|
|
638
|
+
maxResponseBytes: MAX_VTT_BYTES,
|
|
639
|
+
accept: 'text/vtt,text/plain,application/json,application/xml,*/*',
|
|
640
|
+
});
|
|
641
|
+
return parseCaptionText(response.body, captionTrack.ext);
|
|
563
642
|
}
|
|
564
643
|
|
|
565
644
|
async #transcribeViaStt(context) {
|
|
566
645
|
const template = path.join(context.jobDir, 'audio.%(ext)s');
|
|
567
646
|
const cookieArg = context.cookieFilePath ? ` --cookies ${shellEscape(context.cookieFilePath)}` : '';
|
|
568
647
|
const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist ${this.#networkFlags()}${cookieArg} -o ${shellEscape(template)} -f bestaudio/best -- ${shellEscape(context.sourceUrl)}`;
|
|
569
|
-
await this.#runCommand(command, {
|
|
648
|
+
await this.#runCommand(command, {
|
|
649
|
+
cwd: context.jobDir,
|
|
650
|
+
timeout: 10 * 60 * 1000,
|
|
651
|
+
signal: context.signal,
|
|
652
|
+
});
|
|
570
653
|
|
|
571
654
|
const audioPath = firstFileMatching(context.jobDir, 'audio.');
|
|
572
655
|
if (!audioPath || !fileExists(audioPath)) {
|
|
@@ -576,10 +659,12 @@ class SocialVideoService {
|
|
|
576
659
|
const sttConfig = await Promise.resolve(
|
|
577
660
|
this.voiceSettingsResolver(context.userId, context.agentId),
|
|
578
661
|
);
|
|
662
|
+
throwIfAborted(context.signal, 'Social video transcription aborted.');
|
|
579
663
|
return this.voiceTranscriber(audioPath, {
|
|
580
664
|
provider: sttConfig?.provider || 'openai',
|
|
581
665
|
model: sttConfig?.model || '',
|
|
582
666
|
mimeType: detectMimeFromFile(audioPath),
|
|
667
|
+
signal: context.signal,
|
|
583
668
|
});
|
|
584
669
|
}
|
|
585
670
|
|
|
@@ -599,13 +684,19 @@ class SocialVideoService {
|
|
|
599
684
|
}
|
|
600
685
|
|
|
601
686
|
const browser = await Promise.resolve(
|
|
602
|
-
this.runtimeManager.getBrowserProviderForUser(context.userId
|
|
603
|
-
|
|
687
|
+
this.runtimeManager.getBrowserProviderForUser(context.userId, {
|
|
688
|
+
signal: context.signal,
|
|
689
|
+
}),
|
|
690
|
+
).catch((error) => {
|
|
691
|
+
rethrowCancellation(error, context.signal);
|
|
692
|
+
return null;
|
|
693
|
+
});
|
|
604
694
|
if (!browser || typeof browser.getCookies !== 'function') {
|
|
605
695
|
return null;
|
|
606
696
|
}
|
|
607
697
|
|
|
608
|
-
const payload = await browser.getCookies().catch((error) => {
|
|
698
|
+
const payload = await browser.getCookies({ signal: context.signal }).catch((error) => {
|
|
699
|
+
rethrowCancellation(error, context.signal);
|
|
609
700
|
context.warnings.push(`Browser cookie export failed: ${error.message}`);
|
|
610
701
|
return null;
|
|
611
702
|
});
|
|
@@ -619,11 +710,13 @@ class SocialVideoService {
|
|
|
619
710
|
|
|
620
711
|
const cookieFilePath = path.join(context.jobDir, 'browser.cookies.txt');
|
|
621
712
|
await fsp.writeFile(cookieFilePath, serializeCookiesForNetscapeJar(cookies), 'utf8');
|
|
713
|
+
throwIfAborted(context.signal, 'Social video cookie export aborted.');
|
|
622
714
|
return cookieFilePath;
|
|
623
715
|
}
|
|
624
716
|
|
|
625
717
|
async #resolveFrameImage(context) {
|
|
626
718
|
const downloadedFrame = await this.#extractFrameFromVideo(context).catch((error) => {
|
|
719
|
+
rethrowCancellation(error, context.signal);
|
|
627
720
|
context.warnings.push(`Frame extraction failed: ${error.message}`);
|
|
628
721
|
return null;
|
|
629
722
|
});
|
|
@@ -636,14 +729,22 @@ class SocialVideoService {
|
|
|
636
729
|
context.warnings.push('No thumbnail fallback was available after frame extraction failed.');
|
|
637
730
|
return null;
|
|
638
731
|
}
|
|
639
|
-
return this.#downloadThumbnailArtifact(
|
|
732
|
+
return this.#downloadThumbnailArtifact(
|
|
733
|
+
context.userId,
|
|
734
|
+
thumbnail.url,
|
|
735
|
+
context.signal,
|
|
736
|
+
);
|
|
640
737
|
}
|
|
641
738
|
|
|
642
739
|
async #extractFrameFromVideo(context) {
|
|
643
740
|
const template = path.join(context.jobDir, 'video.%(ext)s');
|
|
644
741
|
const cookieArg = context.cookieFilePath ? ` --cookies ${shellEscape(context.cookieFilePath)}` : '';
|
|
645
742
|
const downloadCommand = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist ${this.#networkFlags()}${cookieArg} -o ${shellEscape(template)} -f "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/best" --merge-output-format mp4 -- ${shellEscape(context.sourceUrl)}`;
|
|
646
|
-
await this.#runCommand(downloadCommand, {
|
|
743
|
+
await this.#runCommand(downloadCommand, {
|
|
744
|
+
cwd: context.jobDir,
|
|
745
|
+
timeout: 14 * 60 * 1000,
|
|
746
|
+
signal: context.signal,
|
|
747
|
+
});
|
|
647
748
|
|
|
648
749
|
const videoPath = firstFileMatching(context.jobDir, 'video.');
|
|
649
750
|
if (!videoPath || !fileExists(videoPath)) {
|
|
@@ -653,7 +754,11 @@ class SocialVideoService {
|
|
|
653
754
|
const framePath = path.join(context.jobDir, 'frame.jpg');
|
|
654
755
|
const frameSecond = pickDeterministicFrameSecond(context.mediaInfo.duration);
|
|
655
756
|
const frameCommand = `${shellEscape(this.ffmpegBin)} -hwaccel none -y -hide_banner -loglevel error -ss ${frameSecond} -i ${shellEscape(videoPath)} -frames:v 1 -q:v 2 ${shellEscape(framePath)}`;
|
|
656
|
-
await this.#runCommand(frameCommand, {
|
|
757
|
+
await this.#runCommand(frameCommand, {
|
|
758
|
+
cwd: context.jobDir,
|
|
759
|
+
timeout: 2 * 60 * 1000,
|
|
760
|
+
signal: context.signal,
|
|
761
|
+
});
|
|
657
762
|
|
|
658
763
|
if (!fileExists(framePath)) {
|
|
659
764
|
throw new Error('ffmpeg did not produce a frame image.');
|
|
@@ -661,14 +766,19 @@ class SocialVideoService {
|
|
|
661
766
|
return this.#saveImageArtifact(context.userId, framePath, 'frame');
|
|
662
767
|
}
|
|
663
768
|
|
|
664
|
-
async #downloadThumbnailArtifact(userId, thumbnailUrl) {
|
|
665
|
-
const response = await
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
const
|
|
769
|
+
async #downloadThumbnailArtifact(userId, thumbnailUrl, signal = null) {
|
|
770
|
+
const response = await fetchPublicResource(thumbnailUrl, {
|
|
771
|
+
signal,
|
|
772
|
+
maxResponseBytes: MAX_THUMBNAIL_BYTES,
|
|
773
|
+
responseType: 'buffer',
|
|
774
|
+
accept: 'image/*,*/*',
|
|
775
|
+
});
|
|
776
|
+
const buffer = response.body;
|
|
777
|
+
const guessedExtension = path.extname(
|
|
778
|
+
new URL(response.finalUrl || thumbnailUrl).pathname,
|
|
779
|
+
).replace('.', '') || 'jpg';
|
|
780
|
+
const mimeType = String(response.headers['content-type'] || '').trim()
|
|
781
|
+
|| `image/${guessedExtension}`;
|
|
672
782
|
if (!this.artifactStore || userId == null) {
|
|
673
783
|
return {
|
|
674
784
|
url: null,
|
|
@@ -16,7 +16,14 @@ function sortByTimestamp(left, right) {
|
|
|
16
16
|
return String(left.timestamp).localeCompare(String(right.timestamp));
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
async function fetchTriggerRows({
|
|
19
|
+
async function fetchTriggerRows({
|
|
20
|
+
integrationManager,
|
|
21
|
+
userId,
|
|
22
|
+
agentId,
|
|
23
|
+
triggerType,
|
|
24
|
+
config,
|
|
25
|
+
signal = null,
|
|
26
|
+
}) {
|
|
20
27
|
if (!integrationManager) return [];
|
|
21
28
|
const scopedAgentId = resolveAgentId(userId, agentId);
|
|
22
29
|
const connectionArg = {
|
|
@@ -36,7 +43,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
36
43
|
maxResults: 20,
|
|
37
44
|
q: queryParts.join(' ').trim() || undefined,
|
|
38
45
|
},
|
|
39
|
-
}, scopedAgentId);
|
|
46
|
+
}, scopedAgentId, { signal });
|
|
40
47
|
const messages = Array.isArray(result?.messages) ? result.messages : [];
|
|
41
48
|
return messages
|
|
42
49
|
.map((item) => ({
|
|
@@ -62,7 +69,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
62
69
|
...(filters.length ? { '$filter': filters.join(' and ') } : {}),
|
|
63
70
|
...(escapedQuery ? { '$search': `"${escapedQuery}"` } : {}),
|
|
64
71
|
},
|
|
65
|
-
}, scopedAgentId);
|
|
72
|
+
}, scopedAgentId, { signal });
|
|
66
73
|
const messages = Array.isArray(result?.value) ? result.value : [];
|
|
67
74
|
return messages
|
|
68
75
|
.map((item) => ({
|
|
@@ -85,7 +92,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
85
92
|
...connectionArg,
|
|
86
93
|
channel: config.channel,
|
|
87
94
|
limit: 20,
|
|
88
|
-
}, scopedAgentId);
|
|
95
|
+
}, scopedAgentId, { signal });
|
|
89
96
|
const messages = Array.isArray(result?.result?.messages)
|
|
90
97
|
? result.result.messages
|
|
91
98
|
: Array.isArray(result?.messages)
|
|
@@ -115,7 +122,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
115
122
|
method: 'GET',
|
|
116
123
|
path: `/v1.0/me/chats/${encodeURIComponent(config.chatId)}/messages`,
|
|
117
124
|
query: { '$top': 20 },
|
|
118
|
-
}, scopedAgentId);
|
|
125
|
+
}, scopedAgentId, { signal });
|
|
119
126
|
const messages = Array.isArray(result?.value) ? result.value : [];
|
|
120
127
|
return messages
|
|
121
128
|
.filter((item) => {
|
|
@@ -143,7 +150,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
143
150
|
...connectionArg,
|
|
144
151
|
chat_id: config.chatId,
|
|
145
152
|
limit: 25,
|
|
146
|
-
}, scopedAgentId);
|
|
153
|
+
}, scopedAgentId, { signal });
|
|
147
154
|
const messages = Array.isArray(result?.messages) ? result.messages : [];
|
|
148
155
|
return messages
|
|
149
156
|
.filter((item) => item && item.fromMe !== true)
|
|
@@ -172,7 +179,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
172
179
|
...connectionArg,
|
|
173
180
|
...(config.location ? { location: config.location } : {}),
|
|
174
181
|
forecast_hours: Math.max(1, Math.min(Number(config.horizonHours) || 12, 48)),
|
|
175
|
-
}, scopedAgentId);
|
|
182
|
+
}, scopedAgentId, { signal });
|
|
176
183
|
const hourly = Array.isArray(forecast?.hourly) ? forecast.hourly : [];
|
|
177
184
|
const eventTypes = Array.isArray(config.eventTypes) ? config.eventTypes : [];
|
|
178
185
|
const rows = [];
|
|
@@ -254,7 +261,7 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
|
|
|
254
261
|
return [];
|
|
255
262
|
}
|
|
256
263
|
|
|
257
|
-
async function pollIntegrationTask(runtime, task) {
|
|
264
|
+
async function pollIntegrationTask(runtime, task, options = {}) {
|
|
258
265
|
const config = normalizeJsonObject(task.trigger_config);
|
|
259
266
|
const rows = await fetchTriggerRows({
|
|
260
267
|
integrationManager: runtime.integrationManager,
|
|
@@ -262,6 +269,7 @@ async function pollIntegrationTask(runtime, task) {
|
|
|
262
269
|
agentId: task.agent_id,
|
|
263
270
|
triggerType: task.trigger_type,
|
|
264
271
|
config,
|
|
272
|
+
signal: options.signal,
|
|
265
273
|
});
|
|
266
274
|
if (!rows.length) return;
|
|
267
275
|
|
|
@@ -314,8 +322,10 @@ function attachIntegrationEventSources(runtime) {
|
|
|
314
322
|
const provider = runtime.integrationManager?.getProvider?.('whatsapp_personal');
|
|
315
323
|
if (provider && typeof provider.on === 'function') {
|
|
316
324
|
const listener = async (event) => {
|
|
325
|
+
if (runtime.stopping) return;
|
|
317
326
|
const tasks = runtime.taskRepository.listEnabledWhatsappEventTasks(event.userId, event.agentId);
|
|
318
327
|
for (const task of tasks) {
|
|
328
|
+
if (runtime.stopping) break;
|
|
319
329
|
if (!matchesWhatsappTaskEvent(task, event)) continue;
|
|
320
330
|
await runtime.fireTaskFromTrigger(task.id, task.user_id, createWhatsappTriggerPayload(event)).catch((error) => {
|
|
321
331
|
const logger = runtime.logger?.error || console.error;
|