@juspay/neurolink 13.8.17 → 13.8.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -2
- package/README.md +1 -1
- package/dist/action/actionExecutor.d.ts +33 -3
- package/dist/action/actionExecutor.js +71 -15
- package/dist/action/actionInputs.js +4 -1
- package/dist/action/cliVersion.d.ts +25 -0
- package/dist/action/cliVersion.js +26 -0
- package/dist/action/index.d.ts +2 -1
- package/dist/action/index.js +2 -1
- package/dist/adapters/video/directorPipeline.js +46 -36
- package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
- package/dist/adapters/video/ffmpegAdapter.js +32 -5
- package/dist/adapters/video/frameExtractor.js +1 -1
- package/dist/adapters/video/videoMerger.js +5 -2
- package/dist/agent/arithmeticEvaluator.d.ts +32 -0
- package/dist/agent/arithmeticEvaluator.js +349 -0
- package/dist/agent/directTools.d.ts +6 -18
- package/dist/agent/directTools.js +17 -56
- package/dist/artifacts/artifactStore.d.ts +22 -1
- package/dist/artifacts/artifactStore.js +66 -6
- package/dist/auth/anthropicOAuth.d.ts +26 -1
- package/dist/auth/anthropicOAuth.js +81 -5
- package/dist/auth/secureFiles.d.ts +37 -0
- package/dist/auth/secureFiles.js +95 -0
- package/dist/auth/tokenStore.d.ts +43 -6
- package/dist/auth/tokenStore.js +260 -79
- package/dist/avatar/providers/DIDAvatar.js +49 -45
- package/dist/avatar/providers/ReplicateAvatar.js +33 -40
- package/dist/browser/neurolink.min.js +485 -558
- package/dist/cli/commands/auth.js +112 -70
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +128 -13
- package/dist/cli/commands/serve.d.ts +10 -1
- package/dist/cli/commands/serve.js +98 -17
- package/dist/cli/commands/server.js +17 -5
- package/dist/cli/index.js +146 -5
- package/dist/cli/loop/optionsSchema.d.ts +1 -1
- package/dist/cli/utils/envManager.js +17 -3
- package/dist/cli/utils/serverUtils.d.ts +39 -3
- package/dist/cli/utils/serverUtils.js +63 -6
- package/dist/core/baseProvider.js +7 -1
- package/dist/core/modules/MessageBuilder.js +5 -0
- package/dist/core/modules/ToolsManager.d.ts +46 -1
- package/dist/core/modules/ToolsManager.js +190 -43
- package/dist/files/fileReferenceRegistry.d.ts +2 -23
- package/dist/files/fileReferenceRegistry.js +55 -5
- package/dist/hitl/hitlManager.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -1
- package/dist/mcp/auth/tokenStorage.d.ts +7 -0
- package/dist/mcp/auth/tokenStorage.js +23 -4
- package/dist/mcp/externalServerManager.d.ts +13 -3
- package/dist/mcp/externalServerManager.js +154 -136
- package/dist/mcp/index.d.ts +2 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/mcpClientFactory.d.ts +14 -0
- package/dist/mcp/mcpClientFactory.js +70 -7
- package/dist/mcp/projectConfigTrust.d.ts +72 -0
- package/dist/mcp/projectConfigTrust.js +237 -0
- package/dist/mcp/transportUrlPolicy.d.ts +39 -0
- package/dist/mcp/transportUrlPolicy.js +172 -0
- package/dist/music/providers/BeatovenMusic.js +35 -46
- package/dist/music/providers/ReplicateMusic.js +33 -40
- package/dist/neurolink.d.ts +16 -1
- package/dist/neurolink.js +64 -11
- package/dist/processors/archive/ArchiveProcessor.js +15 -1
- package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
- package/dist/processors/archive/zipEntryIndex.js +33 -0
- package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
- package/dist/processors/base/BaseFileProcessor.js +40 -111
- package/dist/processors/data/YamlProcessor.d.ts +13 -1
- package/dist/processors/data/YamlProcessor.js +93 -2
- package/dist/processors/document/OpenDocumentProcessor.js +7 -2
- package/dist/processors/document/PptxProcessor.js +12 -7
- package/dist/processors/media/VideoProcessor.js +93 -15
- package/dist/providers/openaiChatCompletionsClient.js +12 -3
- package/dist/proxy/accountLedger.js +6 -8
- package/dist/proxy/geminiFormat.d.ts +3 -11
- package/dist/proxy/geminiFormat.js +11 -2
- package/dist/proxy/inboundAuth.d.ts +89 -0
- package/dist/proxy/inboundAuth.js +219 -0
- package/dist/proxy/openaiFormat.d.ts +3 -9
- package/dist/proxy/openaiFormat.js +21 -15
- package/dist/proxy/proxyTranslationEngine.d.ts +2 -6
- package/dist/proxy/proxyTranslationEngine.js +25 -14
- package/dist/proxy/routingPolicy.d.ts +3 -1
- package/dist/proxy/routingPolicy.js +7 -8
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +13 -1
- package/dist/proxy/updateState.js +4 -4
- package/dist/rag/document/loaders.d.ts +0 -6
- package/dist/rag/document/loaders.js +16 -10
- package/dist/rag/ragIntegration.d.ts +2 -2
- package/dist/rag/ragIntegration.js +35 -10
- package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
- package/dist/rag/retrieval/vectorQueryTool.js +54 -17
- package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
- package/dist/server/abstract/baseServerAdapter.js +119 -6
- package/dist/server/adapters/expressAdapter.d.ts +3 -0
- package/dist/server/adapters/expressAdapter.js +78 -6
- package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
- package/dist/server/adapters/fastifyAdapter.js +165 -70
- package/dist/server/adapters/honoAdapter.d.ts +9 -1
- package/dist/server/adapters/honoAdapter.js +99 -21
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +153 -74
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +4 -0
- package/dist/server/middleware/auth.js +6 -2
- package/dist/server/middleware/cache.js +6 -1
- package/dist/server/middleware/rateLimit.js +29 -4
- package/dist/server/routes/agentRoutes.js +24 -11
- package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/server/routes/claudeProxyRoutes.js +273 -84
- package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
- package/dist/server/routes/codexProxyRoutes.js +125 -40
- package/dist/server/routes/index.d.ts +4 -0
- package/dist/server/routes/index.js +35 -4
- package/dist/server/routes/mcpRoutes.d.ts +6 -2
- package/dist/server/routes/mcpRoutes.js +21 -7
- package/dist/server/routes/memoryRoutes.js +7 -6
- package/dist/server/routes/toolRoutes.d.ts +12 -2
- package/dist/server/routes/toolRoutes.js +33 -3
- package/dist/server/routes/transcribeRoutes.js +60 -13
- package/dist/server/utils/clientAddress.d.ts +23 -0
- package/dist/server/utils/clientAddress.js +65 -0
- package/dist/server/utils/clientError.d.ts +35 -0
- package/dist/server/utils/clientError.js +129 -0
- package/dist/server/utils/pathMatch.d.ts +21 -0
- package/dist/server/utils/pathMatch.js +41 -0
- package/dist/server/utils/validation.d.ts +9 -0
- package/dist/server/utils/validation.js +9 -0
- package/dist/server/websocket/WebSocketHandler.js +22 -3
- package/dist/server/websocket/frameQueue.d.ts +19 -0
- package/dist/server/websocket/frameQueue.js +41 -3
- package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
- package/dist/server/websocket/transcribeWebSocket.js +223 -34
- package/dist/skills/skillStores.js +5 -0
- package/dist/tasks/store/fileTaskStore.d.ts +2 -0
- package/dist/tasks/store/fileTaskStore.js +11 -4
- package/dist/types/attachments.d.ts +117 -0
- package/dist/types/attachments.js +15 -0
- package/dist/types/auth.d.ts +6 -0
- package/dist/types/cli.d.ts +4 -0
- package/dist/types/config.d.ts +56 -0
- package/dist/types/file.d.ts +8 -0
- package/dist/types/hitl.d.ts +8 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/mcp.d.ts +98 -0
- package/dist/types/multimodal.js +0 -41
- package/dist/types/openaiCompatible.d.ts +1 -1
- package/dist/types/processor.d.ts +14 -1
- package/dist/types/proxy.d.ts +71 -16
- package/dist/types/proxyClient.d.ts +2 -0
- package/dist/types/rag.d.ts +9 -0
- package/dist/types/safeFetch.d.ts +40 -0
- package/dist/types/server.d.ts +206 -10
- package/dist/types/tools.d.ts +57 -0
- package/dist/types/transcribe.d.ts +6 -0
- package/dist/utils/attachmentPolicy.d.ts +89 -0
- package/dist/utils/attachmentPolicy.js +327 -0
- package/dist/utils/dotenvBootstrap.d.ts +48 -19
- package/dist/utils/dotenvBootstrap.js +172 -31
- package/dist/utils/fileDetector.js +115 -65
- package/dist/utils/json/coerce.js +11 -1
- package/dist/utils/logSanitize.d.ts +5 -0
- package/dist/utils/logSanitize.js +8 -0
- package/dist/utils/logger.js +73 -1
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +118 -105
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +2 -0
- package/dist/utils/redis.d.ts +12 -0
- package/dist/utils/redis.js +72 -0
- package/dist/utils/retryAfter.d.ts +32 -0
- package/dist/utils/retryAfter.js +135 -11
- package/dist/utils/safeFetch.d.ts +22 -1
- package/dist/utils/safeFetch.js +132 -38
- package/dist/utils/safeStoreId.d.ts +19 -0
- package/dist/utils/safeStoreId.js +29 -0
- package/dist/utils/schemaConversion.d.ts +30 -3
- package/dist/utils/schemaConversion.js +457 -84
- package/dist/utils/sizeGuard.js +2 -2
- package/dist/utils/ssrfGuard.d.ts +3 -3
- package/dist/utils/ssrfGuard.js +71 -39
- package/dist/voice/eval/sttEvalStore.d.ts +26 -3
- package/dist/voice/eval/sttEvalStore.js +77 -12
- package/dist/voice/livekit/agentDefinition.d.ts +18 -0
- package/dist/voice/livekit/agentDefinition.js +22 -0
- package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
- package/dist/voice/livekit/realtimeMcpTools.js +8 -7
- package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
- package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
- package/dist/voice/livekit/voiceAgent.d.ts +2 -2
- package/dist/voice/livekit/voiceAgent.js +2 -1
- package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
- package/dist/voice/livekit/xaiSessionWatch.js +30 -0
- package/dist/voice/providers/GeminiLive.js +21 -6
- package/dist/voice/providers/GoogleSTT.js +8 -5
- package/dist/voice/transcribe.d.ts +2 -2
- package/dist/voice/transcribe.js +34 -18
- package/dist/voice/whistle/audio.js +11 -5
- package/dist/workflow/workflows/adaptiveWorkflow.d.ts +2 -2
- package/dist/workflow/workflows/adaptiveWorkflow.js +10 -10
- package/dist/workflow/workflows/consensusWorkflow.d.ts +4 -4
- package/dist/workflow/workflows/consensusWorkflow.js +12 -12
- package/dist/workflow/workflows/fallbackWorkflow.d.ts +3 -3
- package/dist/workflow/workflows/fallbackWorkflow.js +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.d.ts +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.js +31 -31
- package/docs-site/static/search-index.json +74 -52
- package/package.json +173 -156
- package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
|
@@ -47,6 +47,7 @@ import { promisify } from "util";
|
|
|
47
47
|
import { gunzip } from "zlib";
|
|
48
48
|
import { SIZE_LIMITS } from "../config/index.js";
|
|
49
49
|
import { isAbortError } from "../../utils/errorHandling.js";
|
|
50
|
+
import { downloadAttachment } from "../../utils/attachmentPolicy.js";
|
|
50
51
|
import { withSpan } from "../../telemetry/withSpan.js";
|
|
51
52
|
import { tracers } from "../../telemetry/tracers.js";
|
|
52
53
|
// Starts tracing at the first span, as this module always did: the shared
|
|
@@ -425,126 +426,54 @@ export class BaseFileProcessor {
|
|
|
425
426
|
/**
|
|
426
427
|
* Download file from URL with authentication.
|
|
427
428
|
*
|
|
429
|
+
* Goes through `downloadAttachment`, which applies the attachment policy
|
|
430
|
+
* (HTTPS and public hosts unless the instance says otherwise), pins the
|
|
431
|
+
* connection to the vetted addresses, re-validates every redirect hop and
|
|
432
|
+
* bounds the body while it streams. `authHeaders` ride on the first request
|
|
433
|
+
* only — they are never replayed to a redirect target.
|
|
434
|
+
*
|
|
428
435
|
* @param url - URL to download from
|
|
429
436
|
* @param authHeaders - Optional authentication headers
|
|
430
437
|
* @param timeout - Optional timeout override
|
|
438
|
+
* @param attachmentPolicy - Policy for the fetch; the environment default when omitted
|
|
431
439
|
* @returns Downloaded file content as Buffer
|
|
432
440
|
* @throws Error if download fails
|
|
433
441
|
*/
|
|
434
|
-
async downloadFile(url, authHeaders, timeout) {
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
};
|
|
448
|
-
const response = await fetch(url, {
|
|
449
|
-
headers,
|
|
450
|
-
signal: controller.signal,
|
|
451
|
-
});
|
|
452
|
-
if (!response.ok) {
|
|
453
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
454
|
-
}
|
|
455
|
-
// Reject HTML responses - likely an error page or redirect
|
|
456
|
-
const contentType = response.headers.get("Content-Type");
|
|
457
|
-
if (contentType && contentType.includes("text/html")) {
|
|
458
|
-
throw new Error(`Received HTML response instead of file content (Content-Type: ${contentType}). This usually means the download URL returned an error page.`);
|
|
459
|
-
}
|
|
460
|
-
const maxBytes = this.config.maxSizeMB * 1024 * 1024;
|
|
461
|
-
let buffer = await this.readBodyWithinLimit(response, maxBytes);
|
|
462
|
-
// Check for gzip encoding and decompress if needed
|
|
463
|
-
// Only decompress if the data actually starts with gzip magic bytes (0x1f 0x8b)
|
|
464
|
-
const contentEncoding = response.headers.get("Content-Encoding");
|
|
465
|
-
const isActuallyGzipped = buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
|
|
466
|
-
if (contentEncoding?.toLowerCase().includes("gzip") &&
|
|
467
|
-
isActuallyGzipped) {
|
|
468
|
-
try {
|
|
469
|
-
// Bounded at the decoder. The size check that runs after the download
|
|
470
|
-
// only ever sees what already fits in memory, so a few KB of gzip
|
|
471
|
-
// declaring gigabytes was inflated in full before anything objected —
|
|
472
|
-
// the response passed the "is it small" test on its way in.
|
|
473
|
-
buffer = Buffer.from(await gunzipAsync(buffer, { maxOutputLength: maxBytes }));
|
|
474
|
-
}
|
|
475
|
-
catch (gzipError) {
|
|
476
|
-
if (isBufferTooLargeError(gzipError)) {
|
|
477
|
-
throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
|
|
478
|
-
}
|
|
479
|
-
throw new Error(`Failed to decompress gzip response: ${gzipError instanceof Error ? gzipError.message : String(gzipError)}`, { cause: gzipError });
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
return buffer;
|
|
483
|
-
}
|
|
484
|
-
finally {
|
|
485
|
-
clearTimeout(timeoutId);
|
|
486
|
-
// Release the connection on every early exit. Rejecting a response for
|
|
487
|
-
// its status or its Content-Type leaves the body unread, which keeps the
|
|
488
|
-
// socket out of the pool until the response is garbage-collected. After
|
|
489
|
-
// a successful read there is nothing left in flight, so this is a no-op.
|
|
490
|
-
controller.abort();
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
/**
|
|
494
|
-
* Read a response body, refusing it the moment it passes `maxBytes`.
|
|
495
|
-
*
|
|
496
|
-
* `response.arrayBuffer()` buffers whatever the server sends before anyone
|
|
497
|
-
* can object, so the size check further up only ever ran against a body that
|
|
498
|
-
* had already been paid for in full. Metering the stream makes the ceiling
|
|
499
|
-
* the configured limit instead of the server's choice, and cancelling the
|
|
500
|
-
* reader stops the transfer rather than merely stopping our interest in it.
|
|
501
|
-
*
|
|
502
|
-
* Content-Length is deliberately not trusted as the bound — it is a hint that
|
|
503
|
-
* can be absent or a lie — but it is worth honouring when present, because it
|
|
504
|
-
* refuses the honest oversized case without reading a byte.
|
|
505
|
-
*
|
|
506
|
-
* @param response - Fetch response whose body should be read
|
|
507
|
-
* @param maxBytes - Ceiling for the body, in bytes
|
|
508
|
-
* @returns The body as a Buffer
|
|
509
|
-
* @throws When the body exceeds `maxBytes`
|
|
510
|
-
*/
|
|
511
|
-
async readBodyWithinLimit(response, maxBytes) {
|
|
512
|
-
const declared = Number(response.headers.get("Content-Length"));
|
|
513
|
-
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
514
|
-
// Release the socket. An unread body keeps the connection out of the
|
|
515
|
-
// pool until the response is collected, so a run of oversized downloads
|
|
516
|
-
// would hold one socket open each.
|
|
517
|
-
await response.body?.cancel();
|
|
518
|
-
throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
|
|
519
|
-
}
|
|
520
|
-
// Some responses carry no readable stream (mocked fetch, polyfills). The
|
|
521
|
-
// buffered read is the fallback, still checked — just after the fact.
|
|
522
|
-
if (!response.body) {
|
|
523
|
-
const buffered = Buffer.from(await response.arrayBuffer());
|
|
524
|
-
if (buffered.length > maxBytes) {
|
|
525
|
-
throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
|
|
526
|
-
}
|
|
527
|
-
return buffered;
|
|
442
|
+
async downloadFile(url, authHeaders, timeout, attachmentPolicy) {
|
|
443
|
+
const maxBytes = this.config.maxSizeMB * 1024 * 1024;
|
|
444
|
+
const downloaded = await downloadAttachment(url, {
|
|
445
|
+
policy: attachmentPolicy,
|
|
446
|
+
maxBytes,
|
|
447
|
+
label: `${this.config.fileTypeName} download`,
|
|
448
|
+
headers: authHeaders,
|
|
449
|
+
timeoutMs: timeout ?? this.config.timeoutMs,
|
|
450
|
+
});
|
|
451
|
+
// Reject HTML responses - likely an error page or redirect
|
|
452
|
+
const contentType = downloaded.contentType;
|
|
453
|
+
if (contentType && contentType.includes("text/html")) {
|
|
454
|
+
throw new Error(`Received HTML response instead of file content (Content-Type: ${contentType}). This usually means the download URL returned an error page.`);
|
|
528
455
|
}
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
456
|
+
let buffer = downloaded.buffer;
|
|
457
|
+
// Check for gzip encoding and decompress if needed
|
|
458
|
+
// Only decompress if the data actually starts with gzip magic bytes (0x1f 0x8b)
|
|
459
|
+
const contentEncoding = downloaded.contentEncoding;
|
|
460
|
+
const isActuallyGzipped = buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
|
|
461
|
+
if (contentEncoding.toLowerCase().includes("gzip") && isActuallyGzipped) {
|
|
462
|
+
try {
|
|
463
|
+
// Bounded at the decoder. The size check that runs after the download
|
|
464
|
+
// only ever sees what already fits in memory, so a few KB of gzip
|
|
465
|
+
// declaring gigabytes was inflated in full before anything objected —
|
|
466
|
+
// the response passed the "is it small" test on its way in.
|
|
467
|
+
buffer = Buffer.from(await gunzipAsync(buffer, { maxOutputLength: maxBytes }));
|
|
539
468
|
}
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
469
|
+
catch (gzipError) {
|
|
470
|
+
if (isBufferTooLargeError(gzipError)) {
|
|
471
|
+
throw downloadTooLargeError(this.config.maxSizeMB, this.config.fileTypeName);
|
|
472
|
+
}
|
|
473
|
+
throw new Error(`Failed to decompress gzip response: ${gzipError instanceof Error ? gzipError.message : String(gzipError)}`, { cause: gzipError });
|
|
544
474
|
}
|
|
545
|
-
chunks.push(Buffer.from(value));
|
|
546
475
|
}
|
|
547
|
-
return
|
|
476
|
+
return buffer;
|
|
548
477
|
}
|
|
549
478
|
/**
|
|
550
479
|
* Download file with retry logic for transient failures.
|
|
@@ -567,7 +496,7 @@ export class BaseFileProcessor {
|
|
|
567
496
|
let lastError;
|
|
568
497
|
for (let attempt = 0; attempt <= retryConfig.maxRetries; attempt++) {
|
|
569
498
|
try {
|
|
570
|
-
const buffer = await this.downloadFile(url, options?.authHeaders, options?.timeout);
|
|
499
|
+
const buffer = await this.downloadFile(url, options?.authHeaders, options?.timeout, options?.attachmentPolicy);
|
|
571
500
|
return { success: true, data: buffer };
|
|
572
501
|
}
|
|
573
502
|
catch (error) {
|
|
@@ -15,7 +15,10 @@
|
|
|
15
15
|
* 2. **Billion Laughs Attack (Entity Expansion)**: YAML supports anchors (&) and aliases (*)
|
|
16
16
|
* for referencing content. Malicious YAML can use nested aliases to create exponential
|
|
17
17
|
* expansion (e.g., 10 levels of 10x expansion = 10^10 entities from a small file).
|
|
18
|
-
*
|
|
18
|
+
* js-yaml parses aliases as SHARED references, so the parsed tree stays small — the
|
|
19
|
+
* expansion happens at serialization. `asJson` is therefore produced by a bounded
|
|
20
|
+
* serializer that stops at the processor's byte limit and reports `null` instead of
|
|
21
|
+
* materialising the expanded document.
|
|
19
22
|
*
|
|
20
23
|
* 3. **Denial of Service**: Large or deeply nested YAML files can exhaust memory/CPU.
|
|
21
24
|
* Size limits are enforced by the base processor's maxSizeMB configuration.
|
|
@@ -109,6 +112,15 @@ export declare class YamlProcessor extends BaseFileProcessor<ProcessedYaml> {
|
|
|
109
112
|
*/
|
|
110
113
|
protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedYaml;
|
|
111
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* `JSON.stringify(value, null, 2)` that stops once the output would pass
|
|
117
|
+
* `maxChars`, returning `null` instead. Walks the value iteratively enough to
|
|
118
|
+
* visit shared references once per path (which is what expands an alias
|
|
119
|
+
* bomb) but counts output as it goes, so the work is bounded by the limit
|
|
120
|
+
* rather than by the document's expanded size. Cycles — which YAML anchors
|
|
121
|
+
* can also express — serialize as `"[Circular]"`.
|
|
122
|
+
*/
|
|
123
|
+
export declare function boundedJsonStringify(value: unknown, maxChars: number): string | null;
|
|
112
124
|
/** Singleton YAML processor instance */
|
|
113
125
|
export declare const yamlProcessor: YamlProcessor;
|
|
114
126
|
/**
|
|
@@ -15,7 +15,10 @@
|
|
|
15
15
|
* 2. **Billion Laughs Attack (Entity Expansion)**: YAML supports anchors (&) and aliases (*)
|
|
16
16
|
* for referencing content. Malicious YAML can use nested aliases to create exponential
|
|
17
17
|
* expansion (e.g., 10 levels of 10x expansion = 10^10 entities from a small file).
|
|
18
|
-
*
|
|
18
|
+
* js-yaml parses aliases as SHARED references, so the parsed tree stays small — the
|
|
19
|
+
* expansion happens at serialization. `asJson` is therefore produced by a bounded
|
|
20
|
+
* serializer that stops at the processor's byte limit and reports `null` instead of
|
|
21
|
+
* materialising the expanded document.
|
|
19
22
|
*
|
|
20
23
|
* 3. **Denial of Service**: Large or deeply nested YAML files can exhaust memory/CPU.
|
|
21
24
|
* Size limits are enforced by the base processor's maxSizeMB configuration.
|
|
@@ -45,6 +48,7 @@ import { createRequire } from "node:module";
|
|
|
45
48
|
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
|
|
46
49
|
import { SIZE_LIMITS_MB } from "../config/index.js";
|
|
47
50
|
import { createFileError, FileErrorCode } from "../errors/index.js";
|
|
51
|
+
import { logger } from "../../utils/logger.js";
|
|
48
52
|
// Re-export for consumers who import from this module
|
|
49
53
|
// Import for local use
|
|
50
54
|
const require = createRequire(import.meta.url);
|
|
@@ -211,7 +215,14 @@ export class YamlProcessor extends BaseFileProcessor {
|
|
|
211
215
|
try {
|
|
212
216
|
// Use secure parsing - validation already passed, but maintain consistent security
|
|
213
217
|
parsed = this.parseYamlSecurely(content);
|
|
214
|
-
|
|
218
|
+
// Bounded: a 30-level alias chain is a few hundred bytes of YAML and a
|
|
219
|
+
// small shared-reference graph in memory, but 2^30 nodes once
|
|
220
|
+
// stringified. The serializer gives up at the processor's own byte
|
|
221
|
+
// limit rather than letting `JSON.stringify` expand it.
|
|
222
|
+
asJson = boundedJsonStringify(parsed, this.config.maxSizeMB * 1024 * 1024);
|
|
223
|
+
if (asJson === null) {
|
|
224
|
+
logger.warn(`[YamlProcessor] "${this.getFilename(fileInfo)}" expands past the ${this.config.maxSizeMB}MB limit when serialized (alias expansion); asJson omitted`);
|
|
225
|
+
}
|
|
215
226
|
}
|
|
216
227
|
catch (error) {
|
|
217
228
|
// This shouldn't happen since we validate, but handle gracefully
|
|
@@ -232,6 +243,86 @@ export class YamlProcessor extends BaseFileProcessor {
|
|
|
232
243
|
}
|
|
233
244
|
}
|
|
234
245
|
// =============================================================================
|
|
246
|
+
// BOUNDED SERIALIZATION
|
|
247
|
+
// =============================================================================
|
|
248
|
+
/**
|
|
249
|
+
* `JSON.stringify(value, null, 2)` that stops once the output would pass
|
|
250
|
+
* `maxChars`, returning `null` instead. Walks the value iteratively enough to
|
|
251
|
+
* visit shared references once per path (which is what expands an alias
|
|
252
|
+
* bomb) but counts output as it goes, so the work is bounded by the limit
|
|
253
|
+
* rather than by the document's expanded size. Cycles — which YAML anchors
|
|
254
|
+
* can also express — serialize as `"[Circular]"`.
|
|
255
|
+
*/
|
|
256
|
+
export function boundedJsonStringify(value, maxChars) {
|
|
257
|
+
const parts = [];
|
|
258
|
+
let length = 0;
|
|
259
|
+
const ancestors = [];
|
|
260
|
+
const push = (text) => {
|
|
261
|
+
length += text.length;
|
|
262
|
+
if (length > maxChars) {
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
parts.push(text);
|
|
266
|
+
return true;
|
|
267
|
+
};
|
|
268
|
+
const walk = (node, indent) => {
|
|
269
|
+
if (node === null || typeof node !== "object") {
|
|
270
|
+
if (node === undefined || typeof node === "function") {
|
|
271
|
+
return push("null");
|
|
272
|
+
}
|
|
273
|
+
if (typeof node === "number" && !Number.isFinite(node)) {
|
|
274
|
+
return push("null");
|
|
275
|
+
}
|
|
276
|
+
return push(JSON.stringify(node));
|
|
277
|
+
}
|
|
278
|
+
if (ancestors.includes(node)) {
|
|
279
|
+
return push('"[Circular]"');
|
|
280
|
+
}
|
|
281
|
+
ancestors.push(node);
|
|
282
|
+
try {
|
|
283
|
+
const inner = `${indent} `;
|
|
284
|
+
if (Array.isArray(node)) {
|
|
285
|
+
if (node.length === 0) {
|
|
286
|
+
return push("[]");
|
|
287
|
+
}
|
|
288
|
+
if (!push("[\n")) {
|
|
289
|
+
return false;
|
|
290
|
+
}
|
|
291
|
+
for (let i = 0; i < node.length; i++) {
|
|
292
|
+
if (!push(inner) || !walk(node[i], inner)) {
|
|
293
|
+
return false;
|
|
294
|
+
}
|
|
295
|
+
if (!push(i < node.length - 1 ? ",\n" : "\n")) {
|
|
296
|
+
return false;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return push(`${indent}]`);
|
|
300
|
+
}
|
|
301
|
+
const entries = Object.entries(node).filter(([, v]) => v !== undefined && typeof v !== "function");
|
|
302
|
+
if (entries.length === 0) {
|
|
303
|
+
return push("{}");
|
|
304
|
+
}
|
|
305
|
+
if (!push("{\n")) {
|
|
306
|
+
return false;
|
|
307
|
+
}
|
|
308
|
+
for (let i = 0; i < entries.length; i++) {
|
|
309
|
+
const [key, child] = entries[i];
|
|
310
|
+
if (!push(`${inner}${JSON.stringify(key)}: `) || !walk(child, inner)) {
|
|
311
|
+
return false;
|
|
312
|
+
}
|
|
313
|
+
if (!push(i < entries.length - 1 ? ",\n" : "\n")) {
|
|
314
|
+
return false;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return push(`${indent}}`);
|
|
318
|
+
}
|
|
319
|
+
finally {
|
|
320
|
+
ancestors.pop();
|
|
321
|
+
}
|
|
322
|
+
};
|
|
323
|
+
return walk(value, "") ? parts.join("") : null;
|
|
324
|
+
}
|
|
325
|
+
// =============================================================================
|
|
235
326
|
// SINGLETON INSTANCE
|
|
236
327
|
// =============================================================================
|
|
237
328
|
/** Singleton YAML processor instance */
|
|
@@ -10,6 +10,7 @@ import { createRequire } from "node:module";
|
|
|
10
10
|
import * as zlib from "node:zlib";
|
|
11
11
|
import { BaseFileProcessor, contentTooLargeError, isContentTooLarge, } from "../base/BaseFileProcessor.js";
|
|
12
12
|
import { readZipEntryWithinLimit } from "../archive/zipEntryReader.js";
|
|
13
|
+
import { indexZipEntriesByName } from "../archive/zipEntryIndex.js";
|
|
13
14
|
import { SIZE_LIMITS } from "../config/index.js";
|
|
14
15
|
const require = createRequire(import.meta.url);
|
|
15
16
|
// Re-import for local use within this file
|
|
@@ -63,8 +64,12 @@ export class OpenDocumentProcessor extends BaseFileProcessor {
|
|
|
63
64
|
// Dynamically import adm-zip to avoid issues if not available
|
|
64
65
|
const AdmZip = require("adm-zip");
|
|
65
66
|
const zip = new AdmZip(buffer);
|
|
66
|
-
// Try to get content.xml
|
|
67
|
-
|
|
67
|
+
// Try to get content.xml — the FIRST member of that name, which is the
|
|
68
|
+
// one a listing shows; `getEntry` would resolve the last duplicate.
|
|
69
|
+
// `require("adm-zip")` is untyped; name the entry shape the bounded
|
|
70
|
+
// reader needs so the index is typed rather than `any`.
|
|
71
|
+
const zipEntries = zip.getEntries();
|
|
72
|
+
const contentEntry = indexZipEntriesByName(zipEntries).byName.get("content.xml");
|
|
68
73
|
if (contentEntry) {
|
|
69
74
|
// Bounded rather than `getData()`: an ODF file is a ZIP, so its
|
|
70
75
|
// content.xml can declare any uncompressed size it likes, and
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
import AdmZip from "adm-zip";
|
|
42
42
|
import * as zlib from "node:zlib";
|
|
43
43
|
import { readZipEntryWithinLimit } from "../archive/zipEntryReader.js";
|
|
44
|
+
import { indexZipEntriesByName } from "../archive/zipEntryIndex.js";
|
|
44
45
|
import { SIZE_LIMITS } from "../config/index.js";
|
|
45
46
|
/**
|
|
46
47
|
* Total budget for everything read out of one presentation.
|
|
@@ -182,9 +183,12 @@ export class PptxProcessor {
|
|
|
182
183
|
const zip = new AdmZip(content);
|
|
183
184
|
const entries = zip.getEntries();
|
|
184
185
|
const readEntry = createBoundedEntryReader();
|
|
186
|
+
// First occurrence per name: a deck with two `slide1.xml` members shows
|
|
187
|
+
// the slide the listing found first, and only once.
|
|
188
|
+
const { byName } = indexZipEntriesByName(entries);
|
|
185
189
|
// Collect slide entries with their slide numbers for sorting
|
|
186
190
|
const slides = [];
|
|
187
|
-
for (const entry of
|
|
191
|
+
for (const entry of byName.values()) {
|
|
188
192
|
const match = entry.entryName.match(SLIDE_ENTRY_REGEX);
|
|
189
193
|
if (match) {
|
|
190
194
|
const slideNumber = parseInt(match[1], 10);
|
|
@@ -203,7 +207,7 @@ export class PptxProcessor {
|
|
|
203
207
|
parts.push(`Presentation: ${slides.length} slide(s)\n`);
|
|
204
208
|
for (const slide of slides) {
|
|
205
209
|
const texts = PptxProcessor.extractTextFromXml(slide.xml);
|
|
206
|
-
const notes = PptxProcessor.extractNotesForSlide(
|
|
210
|
+
const notes = PptxProcessor.extractNotesForSlide(byName, slide.slideNumber, readEntry);
|
|
207
211
|
// Emit a slide section when it has either body text or speaker notes.
|
|
208
212
|
if (texts.length > 0 || notes) {
|
|
209
213
|
parts.push(`### Slide ${slide.slideNumber}`);
|
|
@@ -232,8 +236,8 @@ export class PptxProcessor {
|
|
|
232
236
|
* @param slideNumber - 1-indexed slide number
|
|
233
237
|
* @returns The notes text (runs joined by a space), or null when absent
|
|
234
238
|
*/
|
|
235
|
-
static extractNotesForSlide(
|
|
236
|
-
const relsXml = readEntry(
|
|
239
|
+
static extractNotesForSlide(byName, slideNumber, readEntry) {
|
|
240
|
+
const relsXml = readEntry(byName.get(SLIDE_RELS_NAME(slideNumber)));
|
|
237
241
|
if (relsXml === null) {
|
|
238
242
|
return null;
|
|
239
243
|
}
|
|
@@ -254,7 +258,7 @@ export class PptxProcessor {
|
|
|
254
258
|
const normalized = notesTarget
|
|
255
259
|
.replace(/^\.\.\//, "ppt/")
|
|
256
260
|
.replace(/^\/+/, "");
|
|
257
|
-
const notesXml = readEntry(
|
|
261
|
+
const notesXml = readEntry(byName.get(normalized));
|
|
258
262
|
if (notesXml === null) {
|
|
259
263
|
return null;
|
|
260
264
|
}
|
|
@@ -352,9 +356,10 @@ export class PptxProcessor {
|
|
|
352
356
|
const zip = new AdmZip(content);
|
|
353
357
|
const entries = zip.getEntries();
|
|
354
358
|
const readEntry = createBoundedEntryReader();
|
|
359
|
+
const { byName } = indexZipEntriesByName(entries);
|
|
355
360
|
// Collect all slides
|
|
356
361
|
const slides = [];
|
|
357
|
-
for (const entry of
|
|
362
|
+
for (const entry of byName.values()) {
|
|
358
363
|
const match = entry.entryName.match(SLIDE_ENTRY_REGEX);
|
|
359
364
|
if (match) {
|
|
360
365
|
const slideNumber = parseInt(match[1], 10);
|
|
@@ -380,7 +385,7 @@ export class PptxProcessor {
|
|
|
380
385
|
const parts = [];
|
|
381
386
|
for (const slide of slides) {
|
|
382
387
|
const texts = PptxProcessor.extractTextFromXml(slide.xml);
|
|
383
|
-
const notes = PptxProcessor.extractNotesForSlide(
|
|
388
|
+
const notes = PptxProcessor.extractNotesForSlide(byName, slide.slideNumber, readEntry);
|
|
384
389
|
parts.push(`### Slide ${slide.slideNumber}`);
|
|
385
390
|
if (texts.length > 0) {
|
|
386
391
|
parts.push(texts.join("\n"));
|
|
@@ -43,7 +43,6 @@
|
|
|
43
43
|
* }
|
|
44
44
|
* ```
|
|
45
45
|
*/
|
|
46
|
-
import { randomUUID } from "crypto";
|
|
47
46
|
import { createWriteStream, existsSync, promises as fs } from "fs";
|
|
48
47
|
import { tmpdir } from "os";
|
|
49
48
|
import { join } from "path";
|
|
@@ -135,6 +134,44 @@ async function loadSharp() {
|
|
|
135
134
|
// =============================================================================
|
|
136
135
|
// FFMPEG PATH INITIALIZATION
|
|
137
136
|
// =============================================================================
|
|
137
|
+
/**
|
|
138
|
+
* Extensions a caller-supplied filename may contribute to a temp file name.
|
|
139
|
+
* Anything else is replaced by the MIME-derived or `.mp4` default, so the
|
|
140
|
+
* suffix ffmpeg sees as a format hint is always one of ours.
|
|
141
|
+
*/
|
|
142
|
+
const ALLOWED_TEMP_VIDEO_EXTENSIONS = new Set([
|
|
143
|
+
".mp4",
|
|
144
|
+
".m4v",
|
|
145
|
+
".mkv",
|
|
146
|
+
".mov",
|
|
147
|
+
".webm",
|
|
148
|
+
".avi",
|
|
149
|
+
".wmv",
|
|
150
|
+
".flv",
|
|
151
|
+
".3gp",
|
|
152
|
+
".3g2",
|
|
153
|
+
".ts",
|
|
154
|
+
".mts",
|
|
155
|
+
".m2ts",
|
|
156
|
+
".ogv",
|
|
157
|
+
".ogg",
|
|
158
|
+
".mpg",
|
|
159
|
+
".mpeg",
|
|
160
|
+
]);
|
|
161
|
+
/**
|
|
162
|
+
* Input-side ffmpeg/ffprobe options applied to every invocation here.
|
|
163
|
+
*
|
|
164
|
+
* `-protocol_whitelist file,crypto,data` pins the demuxer to local files: a
|
|
165
|
+
* playlist or container that references `http://`, `concat:` or `subfile:`
|
|
166
|
+
* cannot make ffmpeg fetch or read anything else, whatever the installed
|
|
167
|
+
* build's defaults are. `-nostdin` keeps a child from ever waiting on a
|
|
168
|
+
* terminal.
|
|
169
|
+
*/
|
|
170
|
+
const FFMPEG_INPUT_HARDENING = [
|
|
171
|
+
"-nostdin",
|
|
172
|
+
"-protocol_whitelist",
|
|
173
|
+
"file,crypto,data",
|
|
174
|
+
];
|
|
138
175
|
/**
|
|
139
176
|
* Whether ffmpeg/ffprobe paths have been initialized.
|
|
140
177
|
* We only attempt path resolution once to avoid repeated dynamic import overhead.
|
|
@@ -361,8 +398,9 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
361
398
|
logger.info(`[NEUROLINK] Video processing started: ${filename} (${(sizeBytes / (1024 * 1024)).toFixed(2)} MB, ${fileInfo.mimetype || "video/mp4"})`);
|
|
362
399
|
// Ensure ffmpeg paths are initialized before any processing
|
|
363
400
|
await initFfmpegPaths();
|
|
364
|
-
// Temp directory for this processing run
|
|
365
|
-
|
|
401
|
+
// Temp directory for this processing run — minted with mkdtemp
|
|
402
|
+
// (0o700) once there are bytes to write, see Step 3.
|
|
403
|
+
let tempDir = "";
|
|
366
404
|
let tempCreated = false;
|
|
367
405
|
try {
|
|
368
406
|
// Step 1: Validate file type and size
|
|
@@ -430,7 +468,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
430
468
|
};
|
|
431
469
|
}
|
|
432
470
|
// Step 3: Write buffer to temp file (ffmpeg needs a file path)
|
|
433
|
-
await fs.
|
|
471
|
+
tempDir = await fs.mkdtemp(join(tmpdir(), "neurolink-video-"));
|
|
434
472
|
tempCreated = true;
|
|
435
473
|
const extension = this.getExtensionFromFileInfo(fileInfo);
|
|
436
474
|
const tempVideoPath = join(tempDir, `input${extension}`);
|
|
@@ -589,7 +627,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
589
627
|
error: `ffprobe timed out after ${VIDEO_CONFIG.FFPROBE_TIMEOUT_MS}ms`,
|
|
590
628
|
});
|
|
591
629
|
}, VIDEO_CONFIG.FFPROBE_TIMEOUT_MS);
|
|
592
|
-
ffmpeg.ffprobe(filePath, (err, data) => {
|
|
630
|
+
ffmpeg.ffprobe(filePath, ["-protocol_whitelist", "file,crypto,data"], (err, data) => {
|
|
593
631
|
clearTimeout(timeoutId);
|
|
594
632
|
if (err) {
|
|
595
633
|
resolve({
|
|
@@ -671,7 +709,17 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
671
709
|
let report;
|
|
672
710
|
let exitFailure;
|
|
673
711
|
try {
|
|
674
|
-
const { stderr } = await runFfmpeg([
|
|
712
|
+
const { stderr } = await runFfmpeg([
|
|
713
|
+
...FFMPEG_INPUT_HARDENING,
|
|
714
|
+
"-hide_banner",
|
|
715
|
+
"-i",
|
|
716
|
+
filePath,
|
|
717
|
+
"-t",
|
|
718
|
+
"0",
|
|
719
|
+
"-f",
|
|
720
|
+
"null",
|
|
721
|
+
"-",
|
|
722
|
+
], { timeoutMs: VIDEO_CONFIG.FFPROBE_TIMEOUT_MS });
|
|
675
723
|
report = stderr;
|
|
676
724
|
}
|
|
677
725
|
catch (error) {
|
|
@@ -923,6 +971,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
923
971
|
const audioPath = join(tempDir, `audio.${audioFormat}`);
|
|
924
972
|
try {
|
|
925
973
|
await runFfmpeg([
|
|
974
|
+
...FFMPEG_INPUT_HARDENING,
|
|
926
975
|
"-y",
|
|
927
976
|
"-v",
|
|
928
977
|
"error",
|
|
@@ -1045,7 +1094,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1045
1094
|
}
|
|
1046
1095
|
// Extract frames using ffmpeg
|
|
1047
1096
|
const framesDir = join(tempDir, "frames");
|
|
1048
|
-
await fs.mkdir(framesDir, { recursive: true });
|
|
1097
|
+
await fs.mkdir(framesDir, { recursive: true, mode: 0o700 });
|
|
1049
1098
|
// The requested `timestamps` are an idealized, evenly-spaced schedule.
|
|
1050
1099
|
// ffmpeg's `-vf select` only guarantees a MINIMUM gap since the last
|
|
1051
1100
|
// frame it actually selected — on a source whose real frame spacing
|
|
@@ -1131,10 +1180,20 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1131
1180
|
const selectExpr = `isnan(prev_selected_t)+gte(t-prev_selected_t,${intervalSec}-0.001)`;
|
|
1132
1181
|
const PTS_TIME_RE = /pts_time:\s*(-?\d+(?:\.\d+)?)/;
|
|
1133
1182
|
const realTimestamps = [];
|
|
1183
|
+
// Keep the command so the timeout can kill the child rather than only
|
|
1184
|
+
// abandon it: a rejected promise used to leave ffmpeg running to
|
|
1185
|
+
// completion on a file nobody would read.
|
|
1186
|
+
const command = ff(videoPath).inputOptions([...FFMPEG_INPUT_HARDENING]);
|
|
1134
1187
|
const timeoutId = setTimeout(() => {
|
|
1188
|
+
try {
|
|
1189
|
+
command.kill("SIGKILL");
|
|
1190
|
+
}
|
|
1191
|
+
catch {
|
|
1192
|
+
// Already exited.
|
|
1193
|
+
}
|
|
1135
1194
|
reject(new Error(`ffmpeg frame extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
|
|
1136
1195
|
}, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
|
|
1137
|
-
|
|
1196
|
+
command
|
|
1138
1197
|
.outputOptions([
|
|
1139
1198
|
"-vf",
|
|
1140
1199
|
`select='${selectExpr}',showinfo,scale='min(${VIDEO_CONFIG.FRAME_MAX_DIMENSION}\\,iw):-2'`,
|
|
@@ -1205,10 +1264,19 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1205
1264
|
const subtitlePath = join(tempDir, "subtitles.srt");
|
|
1206
1265
|
const ffSub = await loadFluentFfmpeg();
|
|
1207
1266
|
await new Promise((resolve, reject) => {
|
|
1267
|
+
const command = ffSub(videoPath).inputOptions([
|
|
1268
|
+
...FFMPEG_INPUT_HARDENING,
|
|
1269
|
+
]);
|
|
1208
1270
|
const timeoutId = setTimeout(() => {
|
|
1271
|
+
try {
|
|
1272
|
+
command.kill("SIGKILL");
|
|
1273
|
+
}
|
|
1274
|
+
catch {
|
|
1275
|
+
// Already exited.
|
|
1276
|
+
}
|
|
1209
1277
|
reject(new Error(`ffmpeg subtitle extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
|
|
1210
1278
|
}, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
|
|
1211
|
-
|
|
1279
|
+
command
|
|
1212
1280
|
.outputOptions(["-map", "0:s:0", "-c:s", "srt"])
|
|
1213
1281
|
.output(subtitlePath)
|
|
1214
1282
|
.on("end", () => {
|
|
@@ -1383,7 +1451,13 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1383
1451
|
const name = fileInfo.name || "";
|
|
1384
1452
|
const dotIndex = name.lastIndexOf(".");
|
|
1385
1453
|
if (dotIndex >= 0) {
|
|
1386
|
-
|
|
1454
|
+
const fromName = name.substring(dotIndex).toLowerCase();
|
|
1455
|
+
// The tail of a caller-supplied name becomes a temp file suffix that
|
|
1456
|
+
// ffmpeg reads as a format hint; only extensions this processor knows
|
|
1457
|
+
// are allowed through, anything else falls back to the MIME type.
|
|
1458
|
+
if (ALLOWED_TEMP_VIDEO_EXTENSIONS.has(fromName)) {
|
|
1459
|
+
return fromName;
|
|
1460
|
+
}
|
|
1387
1461
|
}
|
|
1388
1462
|
// Fallback: derive from MIME type
|
|
1389
1463
|
const mimeExtMap = {
|
|
@@ -1409,7 +1483,9 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1409
1483
|
*/
|
|
1410
1484
|
async writeBufferToFile(buffer, filePath) {
|
|
1411
1485
|
const readable = Readable.from(buffer);
|
|
1412
|
-
|
|
1486
|
+
// Owner-only, and never over an existing file: the directory is fresh
|
|
1487
|
+
// from mkdtemp, so a path that already exists is something planted.
|
|
1488
|
+
const writable = createWriteStream(filePath, { mode: 0o600, flags: "wx" });
|
|
1413
1489
|
await pipeline(readable, writable);
|
|
1414
1490
|
}
|
|
1415
1491
|
// ===========================================================================
|
|
@@ -1431,9 +1507,8 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1431
1507
|
*/
|
|
1432
1508
|
async extractFrameRange(buffer, filename, startSec, endSec, frameCount = 5) {
|
|
1433
1509
|
await initFfmpegPaths();
|
|
1434
|
-
const tempDir = join(tmpdir(),
|
|
1510
|
+
const tempDir = await fs.mkdtemp(join(tmpdir(), "neurolink-video-extract-"));
|
|
1435
1511
|
try {
|
|
1436
|
-
await fs.mkdir(tempDir, { recursive: true });
|
|
1437
1512
|
// Write buffer to temp file
|
|
1438
1513
|
const ext = this.guessExtensionFromName(filename);
|
|
1439
1514
|
const tempVideoPath = join(tempDir, `input${ext}`);
|
|
@@ -1457,7 +1532,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1457
1532
|
}
|
|
1458
1533
|
// Extract frames
|
|
1459
1534
|
const framesDir = join(tempDir, "frames");
|
|
1460
|
-
await fs.mkdir(framesDir, { recursive: true });
|
|
1535
|
+
await fs.mkdir(framesDir, { recursive: true, mode: 0o700 });
|
|
1461
1536
|
await this.runFfmpegFrameExtraction(tempVideoPath, framesDir, timestamps, interval);
|
|
1462
1537
|
// Read and resize frames
|
|
1463
1538
|
const keyframes = [];
|
|
@@ -1494,7 +1569,10 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
1494
1569
|
guessExtensionFromName(filename) {
|
|
1495
1570
|
const dotIndex = filename.lastIndexOf(".");
|
|
1496
1571
|
if (dotIndex >= 0) {
|
|
1497
|
-
|
|
1572
|
+
const ext = filename.substring(dotIndex).toLowerCase();
|
|
1573
|
+
if (ALLOWED_TEMP_VIDEO_EXTENSIONS.has(ext)) {
|
|
1574
|
+
return ext;
|
|
1575
|
+
}
|
|
1498
1576
|
}
|
|
1499
1577
|
return ".mp4";
|
|
1500
1578
|
}
|