@juspay/neurolink 13.8.17 → 13.8.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -2
- package/README.md +1 -1
- package/dist/action/actionExecutor.d.ts +33 -3
- package/dist/action/actionExecutor.js +71 -15
- package/dist/action/actionInputs.js +4 -1
- package/dist/action/cliVersion.d.ts +25 -0
- package/dist/action/cliVersion.js +26 -0
- package/dist/action/index.d.ts +2 -1
- package/dist/action/index.js +2 -1
- package/dist/adapters/video/directorPipeline.js +46 -36
- package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
- package/dist/adapters/video/ffmpegAdapter.js +32 -5
- package/dist/adapters/video/frameExtractor.js +1 -1
- package/dist/adapters/video/videoMerger.js +5 -2
- package/dist/agent/arithmeticEvaluator.d.ts +32 -0
- package/dist/agent/arithmeticEvaluator.js +349 -0
- package/dist/agent/directTools.d.ts +6 -18
- package/dist/agent/directTools.js +17 -56
- package/dist/artifacts/artifactStore.d.ts +22 -1
- package/dist/artifacts/artifactStore.js +66 -6
- package/dist/auth/anthropicOAuth.d.ts +26 -1
- package/dist/auth/anthropicOAuth.js +81 -5
- package/dist/auth/secureFiles.d.ts +37 -0
- package/dist/auth/secureFiles.js +95 -0
- package/dist/auth/tokenStore.d.ts +43 -6
- package/dist/auth/tokenStore.js +260 -79
- package/dist/avatar/providers/DIDAvatar.js +49 -45
- package/dist/avatar/providers/ReplicateAvatar.js +33 -40
- package/dist/browser/neurolink.min.js +485 -558
- package/dist/cli/commands/auth.js +112 -70
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +128 -13
- package/dist/cli/commands/serve.d.ts +10 -1
- package/dist/cli/commands/serve.js +98 -17
- package/dist/cli/commands/server.js +17 -5
- package/dist/cli/index.js +146 -5
- package/dist/cli/loop/optionsSchema.d.ts +1 -1
- package/dist/cli/utils/envManager.js +17 -3
- package/dist/cli/utils/serverUtils.d.ts +39 -3
- package/dist/cli/utils/serverUtils.js +63 -6
- package/dist/core/baseProvider.js +7 -1
- package/dist/core/modules/MessageBuilder.js +5 -0
- package/dist/core/modules/ToolsManager.d.ts +46 -1
- package/dist/core/modules/ToolsManager.js +190 -43
- package/dist/files/fileReferenceRegistry.d.ts +2 -23
- package/dist/files/fileReferenceRegistry.js +55 -5
- package/dist/hitl/hitlManager.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -1
- package/dist/mcp/auth/tokenStorage.d.ts +7 -0
- package/dist/mcp/auth/tokenStorage.js +23 -4
- package/dist/mcp/externalServerManager.d.ts +13 -3
- package/dist/mcp/externalServerManager.js +154 -136
- package/dist/mcp/index.d.ts +2 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/mcpClientFactory.d.ts +14 -0
- package/dist/mcp/mcpClientFactory.js +70 -7
- package/dist/mcp/projectConfigTrust.d.ts +72 -0
- package/dist/mcp/projectConfigTrust.js +237 -0
- package/dist/mcp/transportUrlPolicy.d.ts +39 -0
- package/dist/mcp/transportUrlPolicy.js +172 -0
- package/dist/music/providers/BeatovenMusic.js +35 -46
- package/dist/music/providers/ReplicateMusic.js +33 -40
- package/dist/neurolink.d.ts +16 -1
- package/dist/neurolink.js +64 -11
- package/dist/processors/archive/ArchiveProcessor.js +15 -1
- package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
- package/dist/processors/archive/zipEntryIndex.js +33 -0
- package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
- package/dist/processors/base/BaseFileProcessor.js +40 -111
- package/dist/processors/data/YamlProcessor.d.ts +13 -1
- package/dist/processors/data/YamlProcessor.js +93 -2
- package/dist/processors/document/OpenDocumentProcessor.js +7 -2
- package/dist/processors/document/PptxProcessor.js +12 -7
- package/dist/processors/media/VideoProcessor.js +93 -15
- package/dist/providers/openaiChatCompletionsClient.js +12 -3
- package/dist/proxy/accountLedger.js +6 -8
- package/dist/proxy/geminiFormat.d.ts +3 -11
- package/dist/proxy/geminiFormat.js +11 -2
- package/dist/proxy/inboundAuth.d.ts +89 -0
- package/dist/proxy/inboundAuth.js +219 -0
- package/dist/proxy/openaiFormat.d.ts +3 -9
- package/dist/proxy/openaiFormat.js +21 -15
- package/dist/proxy/proxyTranslationEngine.d.ts +2 -6
- package/dist/proxy/proxyTranslationEngine.js +25 -14
- package/dist/proxy/routingPolicy.d.ts +3 -1
- package/dist/proxy/routingPolicy.js +7 -8
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +13 -1
- package/dist/proxy/updateState.js +4 -4
- package/dist/rag/document/loaders.d.ts +0 -6
- package/dist/rag/document/loaders.js +16 -10
- package/dist/rag/ragIntegration.d.ts +2 -2
- package/dist/rag/ragIntegration.js +35 -10
- package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
- package/dist/rag/retrieval/vectorQueryTool.js +54 -17
- package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
- package/dist/server/abstract/baseServerAdapter.js +119 -6
- package/dist/server/adapters/expressAdapter.d.ts +3 -0
- package/dist/server/adapters/expressAdapter.js +78 -6
- package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
- package/dist/server/adapters/fastifyAdapter.js +165 -70
- package/dist/server/adapters/honoAdapter.d.ts +9 -1
- package/dist/server/adapters/honoAdapter.js +99 -21
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +153 -74
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +4 -0
- package/dist/server/middleware/auth.js +6 -2
- package/dist/server/middleware/cache.js +6 -1
- package/dist/server/middleware/rateLimit.js +29 -4
- package/dist/server/routes/agentRoutes.js +24 -11
- package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/server/routes/claudeProxyRoutes.js +273 -84
- package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
- package/dist/server/routes/codexProxyRoutes.js +125 -40
- package/dist/server/routes/index.d.ts +4 -0
- package/dist/server/routes/index.js +35 -4
- package/dist/server/routes/mcpRoutes.d.ts +6 -2
- package/dist/server/routes/mcpRoutes.js +21 -7
- package/dist/server/routes/memoryRoutes.js +7 -6
- package/dist/server/routes/toolRoutes.d.ts +12 -2
- package/dist/server/routes/toolRoutes.js +33 -3
- package/dist/server/routes/transcribeRoutes.js +60 -13
- package/dist/server/utils/clientAddress.d.ts +23 -0
- package/dist/server/utils/clientAddress.js +65 -0
- package/dist/server/utils/clientError.d.ts +35 -0
- package/dist/server/utils/clientError.js +129 -0
- package/dist/server/utils/pathMatch.d.ts +21 -0
- package/dist/server/utils/pathMatch.js +41 -0
- package/dist/server/utils/validation.d.ts +9 -0
- package/dist/server/utils/validation.js +9 -0
- package/dist/server/websocket/WebSocketHandler.js +22 -3
- package/dist/server/websocket/frameQueue.d.ts +19 -0
- package/dist/server/websocket/frameQueue.js +41 -3
- package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
- package/dist/server/websocket/transcribeWebSocket.js +223 -34
- package/dist/skills/skillStores.js +5 -0
- package/dist/tasks/store/fileTaskStore.d.ts +2 -0
- package/dist/tasks/store/fileTaskStore.js +11 -4
- package/dist/types/attachments.d.ts +117 -0
- package/dist/types/attachments.js +15 -0
- package/dist/types/auth.d.ts +6 -0
- package/dist/types/cli.d.ts +4 -0
- package/dist/types/config.d.ts +56 -0
- package/dist/types/file.d.ts +8 -0
- package/dist/types/hitl.d.ts +8 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/mcp.d.ts +98 -0
- package/dist/types/multimodal.js +0 -41
- package/dist/types/openaiCompatible.d.ts +1 -1
- package/dist/types/processor.d.ts +14 -1
- package/dist/types/proxy.d.ts +71 -16
- package/dist/types/proxyClient.d.ts +2 -0
- package/dist/types/rag.d.ts +9 -0
- package/dist/types/safeFetch.d.ts +40 -0
- package/dist/types/server.d.ts +206 -10
- package/dist/types/tools.d.ts +57 -0
- package/dist/types/transcribe.d.ts +6 -0
- package/dist/utils/attachmentPolicy.d.ts +89 -0
- package/dist/utils/attachmentPolicy.js +327 -0
- package/dist/utils/dotenvBootstrap.d.ts +48 -19
- package/dist/utils/dotenvBootstrap.js +172 -31
- package/dist/utils/fileDetector.js +115 -65
- package/dist/utils/json/coerce.js +11 -1
- package/dist/utils/logSanitize.d.ts +5 -0
- package/dist/utils/logSanitize.js +8 -0
- package/dist/utils/logger.js +73 -1
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +118 -105
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +2 -0
- package/dist/utils/redis.d.ts +12 -0
- package/dist/utils/redis.js +72 -0
- package/dist/utils/retryAfter.d.ts +32 -0
- package/dist/utils/retryAfter.js +135 -11
- package/dist/utils/safeFetch.d.ts +22 -1
- package/dist/utils/safeFetch.js +132 -38
- package/dist/utils/safeStoreId.d.ts +19 -0
- package/dist/utils/safeStoreId.js +29 -0
- package/dist/utils/schemaConversion.d.ts +30 -3
- package/dist/utils/schemaConversion.js +457 -84
- package/dist/utils/sizeGuard.js +2 -2
- package/dist/utils/ssrfGuard.d.ts +3 -3
- package/dist/utils/ssrfGuard.js +71 -39
- package/dist/voice/eval/sttEvalStore.d.ts +26 -3
- package/dist/voice/eval/sttEvalStore.js +77 -12
- package/dist/voice/livekit/agentDefinition.d.ts +18 -0
- package/dist/voice/livekit/agentDefinition.js +22 -0
- package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
- package/dist/voice/livekit/realtimeMcpTools.js +8 -7
- package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
- package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
- package/dist/voice/livekit/voiceAgent.d.ts +2 -2
- package/dist/voice/livekit/voiceAgent.js +2 -1
- package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
- package/dist/voice/livekit/xaiSessionWatch.js +30 -0
- package/dist/voice/providers/GeminiLive.js +21 -6
- package/dist/voice/providers/GoogleSTT.js +8 -5
- package/dist/voice/transcribe.d.ts +2 -2
- package/dist/voice/transcribe.js +34 -18
- package/dist/voice/whistle/audio.js +11 -5
- package/dist/workflow/workflows/adaptiveWorkflow.d.ts +2 -2
- package/dist/workflow/workflows/adaptiveWorkflow.js +10 -10
- package/dist/workflow/workflows/consensusWorkflow.d.ts +4 -4
- package/dist/workflow/workflows/consensusWorkflow.js +12 -12
- package/dist/workflow/workflows/fallbackWorkflow.d.ts +3 -3
- package/dist/workflow/workflows/fallbackWorkflow.js +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.d.ts +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.js +31 -31
- package/docs-site/static/search-index.json +74 -52
- package/package.json +173 -156
- package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
package/dist/auth/tokenStore.js
CHANGED
|
@@ -15,13 +15,33 @@
|
|
|
15
15
|
import { promises as fs } from "fs";
|
|
16
16
|
import { join } from "path";
|
|
17
17
|
import { homedir } from "os";
|
|
18
|
-
import { createHash } from "crypto";
|
|
18
|
+
import { createHash, randomUUID } from "crypto";
|
|
19
19
|
import { logger } from "../utils/logger.js";
|
|
20
20
|
import { TokenStoreError } from "../types/index.js";
|
|
21
21
|
import { AsyncMutex } from "../utils/asyncMutex.js";
|
|
22
22
|
import { withSpan } from "../telemetry/withSpan.js";
|
|
23
23
|
import { tracers } from "../telemetry/tracers.js";
|
|
24
|
-
|
|
24
|
+
import { ensurePrivateDirectory, writePrivateFileAtomically, } from "./secureFiles.js";
|
|
25
|
+
const { readFile, unlink, access, rename, open, rm, stat } = fs;
|
|
26
|
+
/** Marks a store written with the UTF-8-safe codec; older files carry no prefix. */
|
|
27
|
+
const OBFUSCATION_V2_PREFIX = "nlts2:";
|
|
28
|
+
/** Cross-process lock: how long a writer waits for another process's turn. */
|
|
29
|
+
const FILE_LOCK_TIMEOUT_MS = 5_000;
|
|
30
|
+
/** A lock older than this whose owner is gone is abandoned and removed. */
|
|
31
|
+
const FILE_LOCK_STALE_MS = 30_000;
|
|
32
|
+
const FILE_LOCK_RETRY_MS = 25;
|
|
33
|
+
function processIsRunning(pid) {
|
|
34
|
+
try {
|
|
35
|
+
process.kill(pid, 0);
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
return error.code === "EPERM";
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
async function sleep(ms) {
|
|
43
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
44
|
+
}
|
|
25
45
|
// =============================================================================
|
|
26
46
|
// TOKEN STORE CLASS
|
|
27
47
|
// =============================================================================
|
|
@@ -62,12 +82,12 @@ export class TokenStore {
|
|
|
62
82
|
static NEUROLINK_DIR = ".neurolink";
|
|
63
83
|
static TOKEN_FILE = "tokens.json";
|
|
64
84
|
static FILE_PERMISSIONS = 0o600;
|
|
65
|
-
static DIR_PERMISSIONS = 0o700;
|
|
66
85
|
/** Default expiration buffer: 1 hour (proactive refresh before actual expiry) */
|
|
67
86
|
static DEFAULT_EXPIRY_BUFFER_MS = 60 * 60 * 1000;
|
|
68
87
|
storagePath;
|
|
69
88
|
encryptionEnabled;
|
|
70
89
|
encryptionKey;
|
|
90
|
+
encryptionKeyBytes;
|
|
71
91
|
tokenRefreshers = new Map();
|
|
72
92
|
inFlightRefreshes = new Map();
|
|
73
93
|
_mutex = new AsyncMutex();
|
|
@@ -83,6 +103,7 @@ export class TokenStore {
|
|
|
83
103
|
const { encryptionEnabled = true, customStoragePath } = options;
|
|
84
104
|
this.encryptionEnabled = encryptionEnabled;
|
|
85
105
|
this.encryptionKey = this.deriveEncryptionKey();
|
|
106
|
+
this.encryptionKeyBytes = this.deriveEncryptionKeyBytes();
|
|
86
107
|
if (customStoragePath) {
|
|
87
108
|
this.storagePath = customStoragePath;
|
|
88
109
|
}
|
|
@@ -117,7 +138,7 @@ export class TokenStore {
|
|
|
117
138
|
"auth.has_refresh_token": Boolean(tokens.refreshToken),
|
|
118
139
|
"auth.token_type": tokens.tokenType,
|
|
119
140
|
},
|
|
120
|
-
}, async () => this.
|
|
141
|
+
}, async () => this.runExclusiveLocked(async () => {
|
|
121
142
|
await this._saveTokensInternal(provider, tokens);
|
|
122
143
|
}));
|
|
123
144
|
}
|
|
@@ -127,8 +148,7 @@ export class TokenStore {
|
|
|
127
148
|
async _saveTokensInternal(provider, tokens) {
|
|
128
149
|
// Validate tokens before saving
|
|
129
150
|
this.validateTokens(tokens);
|
|
130
|
-
|
|
131
|
-
await this.ensureDirectory(storageDir);
|
|
151
|
+
await this.ensureDirectory(join(this.storagePath, ".."));
|
|
132
152
|
// Load existing data or create new
|
|
133
153
|
let storageData;
|
|
134
154
|
try {
|
|
@@ -165,17 +185,7 @@ export class TokenStore {
|
|
|
165
185
|
};
|
|
166
186
|
storageData.lastModified = now;
|
|
167
187
|
try {
|
|
168
|
-
|
|
169
|
-
? this.obfuscate(JSON.stringify(storageData))
|
|
170
|
-
: JSON.stringify(storageData, null, 2);
|
|
171
|
-
// Write to temporary file first for atomic operation
|
|
172
|
-
// Use PID-scoped temp file to avoid cross-process race conditions
|
|
173
|
-
const tempPath = `${this.storagePath}.tmp.${process.pid}`;
|
|
174
|
-
await writeFile(tempPath, content, "utf-8");
|
|
175
|
-
// Set restrictive permissions before moving to final location
|
|
176
|
-
await chmod(tempPath, TokenStore.FILE_PERMISSIONS);
|
|
177
|
-
// Rename to final location (atomic on most filesystems)
|
|
178
|
-
await fs.rename(tempPath, this.storagePath);
|
|
188
|
+
await this.writeStorageFile(storageData);
|
|
179
189
|
logger.debug("OAuth tokens saved successfully", {
|
|
180
190
|
provider,
|
|
181
191
|
path: this.storagePath,
|
|
@@ -269,9 +279,10 @@ export class TokenStore {
|
|
|
269
279
|
logger.debug("No stored tokens found for provider", { provider });
|
|
270
280
|
return null;
|
|
271
281
|
}
|
|
272
|
-
//
|
|
282
|
+
// Access time is bookkeeping, not state: rewriting the whole store on
|
|
283
|
+
// every read raced other processes' writes (a stale snapshot written late
|
|
284
|
+
// reverted a rotated refresh token or a disable flag).
|
|
273
285
|
providerData.lastAccessed = Date.now();
|
|
274
|
-
await this.saveStorageData(storageData);
|
|
275
286
|
logger.debug("OAuth tokens retrieved successfully", {
|
|
276
287
|
provider,
|
|
277
288
|
expiresAt: new Date(providerData.tokens.expiresAt).toISOString(),
|
|
@@ -300,7 +311,7 @@ export class TokenStore {
|
|
|
300
311
|
}, async () => this._clearTokensImpl(provider));
|
|
301
312
|
}
|
|
302
313
|
async _clearTokensImpl(provider) {
|
|
303
|
-
return this.
|
|
314
|
+
return this.runExclusiveLocked(async () => {
|
|
304
315
|
// Clear in-memory refresh state so re-adding an account starts fresh
|
|
305
316
|
this.inFlightRefreshes.delete(provider);
|
|
306
317
|
this.tokenRefreshers.delete(provider);
|
|
@@ -430,7 +441,7 @@ export class TokenStore {
|
|
|
430
441
|
// Also guard against resurrecting cleared/disabled entries:
|
|
431
442
|
// if the entry was removed or disabled during the in-flight refresh,
|
|
432
443
|
// skip the save to avoid re-creating a dead entry.
|
|
433
|
-
const persistedTokens = await this.
|
|
444
|
+
const persistedTokens = await this.runExclusiveLocked(async () => {
|
|
434
445
|
// Guard: do not resurrect cleared entries
|
|
435
446
|
const current = await this._loadTokensInternal(provider);
|
|
436
447
|
if (!current) {
|
|
@@ -552,7 +563,7 @@ export class TokenStore {
|
|
|
552
563
|
* @param reason - Optional human-readable reason (e.g., "refresh_failed")
|
|
553
564
|
*/
|
|
554
565
|
async markDisabled(provider, reason) {
|
|
555
|
-
return this.
|
|
566
|
+
return this.runExclusiveLocked(async () => {
|
|
556
567
|
let storageData;
|
|
557
568
|
try {
|
|
558
569
|
storageData = await this.loadStorageData();
|
|
@@ -589,7 +600,7 @@ export class TokenStore {
|
|
|
589
600
|
* request that observed the authentication failure.
|
|
590
601
|
*/
|
|
591
602
|
async markDisabledIfCurrent(provider, expectedTokens, reason) {
|
|
592
|
-
return this.
|
|
603
|
+
return this.runExclusiveLocked(async () => {
|
|
593
604
|
let storageData;
|
|
594
605
|
try {
|
|
595
606
|
storageData = await this.loadStorageData();
|
|
@@ -631,7 +642,7 @@ export class TokenStore {
|
|
|
631
642
|
* @param provider - The provider key (e.g., "anthropic:user@example.com")
|
|
632
643
|
*/
|
|
633
644
|
async markEnabled(provider) {
|
|
634
|
-
return this.
|
|
645
|
+
return this.runExclusiveLocked(async () => {
|
|
635
646
|
let storageData;
|
|
636
647
|
try {
|
|
637
648
|
storageData = await this.loadStorageData();
|
|
@@ -738,7 +749,7 @@ export class TokenStore {
|
|
|
738
749
|
* @returns Array of provider keys that were removed
|
|
739
750
|
*/
|
|
740
751
|
async pruneExpired(bufferMs = 0) {
|
|
741
|
-
return this.
|
|
752
|
+
return this.runExclusiveLocked(async () => {
|
|
742
753
|
let storageData;
|
|
743
754
|
try {
|
|
744
755
|
storageData = await this.loadStorageData();
|
|
@@ -783,7 +794,7 @@ export class TokenStore {
|
|
|
783
794
|
* @throws TokenStoreError if deletion fails
|
|
784
795
|
*/
|
|
785
796
|
async clearAllTokens() {
|
|
786
|
-
return this.
|
|
797
|
+
return this.runExclusiveLocked(async () => {
|
|
787
798
|
// Clear all in-memory refresh state so re-adding accounts starts fresh
|
|
788
799
|
this.inFlightRefreshes.clear();
|
|
789
800
|
this.tokenRefreshers.clear();
|
|
@@ -805,50 +816,222 @@ export class TokenStore {
|
|
|
805
816
|
// File doesn't exist, return empty storage
|
|
806
817
|
throw new TokenStoreError("Token storage file not found", "NOT_FOUND");
|
|
807
818
|
}
|
|
819
|
+
let content;
|
|
808
820
|
try {
|
|
809
|
-
|
|
810
|
-
let storageData;
|
|
811
|
-
if (this.encryptionEnabled) {
|
|
812
|
-
const decrypted = this.deobfuscate(content);
|
|
813
|
-
storageData = JSON.parse(decrypted);
|
|
814
|
-
}
|
|
815
|
-
else {
|
|
816
|
-
storageData = JSON.parse(content);
|
|
817
|
-
}
|
|
818
|
-
// Validate storage data structure
|
|
819
|
-
if (!storageData.providers || !storageData.version) {
|
|
820
|
-
throw new TokenStoreError("Invalid token storage format", "VALIDATION_ERROR");
|
|
821
|
-
}
|
|
822
|
-
return storageData;
|
|
821
|
+
content = await readFile(this.storagePath, "utf-8");
|
|
823
822
|
}
|
|
824
823
|
catch (error) {
|
|
825
|
-
if (error instanceof TokenStoreError) {
|
|
826
|
-
throw error;
|
|
827
|
-
}
|
|
828
824
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
829
825
|
logger.error("Failed to read token storage", { error: errorMessage });
|
|
830
826
|
throw new TokenStoreError(`Failed to read token storage: ${errorMessage}`, "STORAGE_ERROR");
|
|
831
827
|
}
|
|
828
|
+
const storageData = this.encryptionEnabled
|
|
829
|
+
? this.parseObfuscatedStorage(content)
|
|
830
|
+
: this.parseStorageJson(content);
|
|
831
|
+
if (storageData) {
|
|
832
|
+
return storageData;
|
|
833
|
+
}
|
|
834
|
+
// An unreadable store used to throw from every caller, which took the whole
|
|
835
|
+
// pool offline with no repair path short of deleting the file by hand. Set
|
|
836
|
+
// it aside under a dated name instead: the proxy starts with an empty pool,
|
|
837
|
+
// `auth login` can repopulate it, and the bytes are still there to inspect.
|
|
838
|
+
await this.quarantineStorageFile();
|
|
839
|
+
throw new TokenStoreError("Token storage file not found", "NOT_FOUND");
|
|
840
|
+
}
|
|
841
|
+
/** Parse plain-JSON storage; null when it is not a usable store. */
|
|
842
|
+
parseStorageJson(content) {
|
|
843
|
+
try {
|
|
844
|
+
const parsed = JSON.parse(content);
|
|
845
|
+
if (!parsed ||
|
|
846
|
+
typeof parsed !== "object" ||
|
|
847
|
+
!parsed.providers ||
|
|
848
|
+
typeof parsed.providers !== "object" ||
|
|
849
|
+
!parsed.version) {
|
|
850
|
+
return null;
|
|
851
|
+
}
|
|
852
|
+
return parsed;
|
|
853
|
+
}
|
|
854
|
+
catch {
|
|
855
|
+
return null;
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
/**
|
|
859
|
+
* Decode a store written by either codec.
|
|
860
|
+
*
|
|
861
|
+
* The current codec is prefixed and XORs UTF-8 bytes. The older one XORed
|
|
862
|
+
* UTF-16 code units and then truncated each to a byte, so a store holding a
|
|
863
|
+
* single non-Latin-1 character (an emoji in a label, a smart quote in a
|
|
864
|
+
* disable reason) could never be read back. Older files carry no prefix and
|
|
865
|
+
* are read the old way; the next save rewrites them with the new codec.
|
|
866
|
+
*/
|
|
867
|
+
parseObfuscatedStorage(content) {
|
|
868
|
+
const trimmed = content.trim();
|
|
869
|
+
if (trimmed.startsWith(OBFUSCATION_V2_PREFIX)) {
|
|
870
|
+
try {
|
|
871
|
+
return this.parseStorageJson(this.deobfuscateV2(trimmed));
|
|
872
|
+
}
|
|
873
|
+
catch {
|
|
874
|
+
return null;
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
try {
|
|
878
|
+
const legacy = this.parseStorageJson(this.deobfuscateLegacy(trimmed));
|
|
879
|
+
if (legacy) {
|
|
880
|
+
return legacy;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
catch {
|
|
884
|
+
// Fall through to the other codec and the plain-JSON reading.
|
|
885
|
+
}
|
|
886
|
+
try {
|
|
887
|
+
const current = this.parseStorageJson(this.deobfuscateV2(trimmed));
|
|
888
|
+
if (current) {
|
|
889
|
+
return current;
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
catch {
|
|
893
|
+
// Not the current codec either.
|
|
894
|
+
}
|
|
895
|
+
return this.parseStorageJson(trimmed);
|
|
896
|
+
}
|
|
897
|
+
async quarantineStorageFile() {
|
|
898
|
+
const quarantinePath = `${this.storagePath}.corrupt-${Date.now()}`;
|
|
899
|
+
try {
|
|
900
|
+
await rename(this.storagePath, quarantinePath);
|
|
901
|
+
logger.error("Token storage could not be parsed and was quarantined; the pool starts empty", { path: this.storagePath, quarantinedTo: quarantinePath });
|
|
902
|
+
}
|
|
903
|
+
catch (error) {
|
|
904
|
+
logger.error("Token storage could not be parsed or quarantined", {
|
|
905
|
+
path: this.storagePath,
|
|
906
|
+
error: error instanceof Error ? error.message : String(error),
|
|
907
|
+
});
|
|
908
|
+
}
|
|
832
909
|
}
|
|
833
910
|
/**
|
|
834
911
|
* Saves storage data to file
|
|
835
912
|
*/
|
|
836
913
|
async saveStorageData(data) {
|
|
837
914
|
try {
|
|
838
|
-
|
|
839
|
-
? this.obfuscate(JSON.stringify(data))
|
|
840
|
-
: JSON.stringify(data, null, 2);
|
|
841
|
-
// Use PID-scoped temp file to avoid cross-process race conditions
|
|
842
|
-
const tmpPath = `${this.storagePath}.tmp.${process.pid}`;
|
|
843
|
-
await writeFile(tmpPath, content, "utf-8");
|
|
844
|
-
await chmod(tmpPath, TokenStore.FILE_PERMISSIONS);
|
|
845
|
-
await rename(tmpPath, this.storagePath);
|
|
915
|
+
await this.writeStorageFile(data);
|
|
846
916
|
}
|
|
847
917
|
catch (error) {
|
|
848
918
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
849
919
|
throw new TokenStoreError(`Failed to save token storage: ${errorMessage}`, "STORAGE_ERROR");
|
|
850
920
|
}
|
|
851
921
|
}
|
|
922
|
+
/**
|
|
923
|
+
* Serialize and write the store: owner-only from creation, under an
|
|
924
|
+
* unpredictable temp name, renamed into place. See `secureFiles`.
|
|
925
|
+
*/
|
|
926
|
+
async writeStorageFile(data) {
|
|
927
|
+
const content = this.encryptionEnabled
|
|
928
|
+
? this.obfuscate(JSON.stringify(data))
|
|
929
|
+
: JSON.stringify(data, null, 2);
|
|
930
|
+
await writePrivateFileAtomically(this.storagePath, content);
|
|
931
|
+
}
|
|
932
|
+
/**
|
|
933
|
+
* Take the in-process mutex, then the cross-process file lock, around a
|
|
934
|
+
* read-modify-write. The CLI and a running proxy share one file; without
|
|
935
|
+
* the second lock a stale snapshot written late reverted the other's
|
|
936
|
+
* rotated refresh token or disable flag.
|
|
937
|
+
*/
|
|
938
|
+
runExclusiveLocked(fn) {
|
|
939
|
+
return this._mutex.runExclusive(async () => {
|
|
940
|
+
const release = await this.acquireFileLock();
|
|
941
|
+
try {
|
|
942
|
+
return await fn();
|
|
943
|
+
}
|
|
944
|
+
finally {
|
|
945
|
+
await release();
|
|
946
|
+
}
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
get lockPath() {
|
|
950
|
+
return `${this.storagePath}.lock`;
|
|
951
|
+
}
|
|
952
|
+
async readLockOwner() {
|
|
953
|
+
try {
|
|
954
|
+
const parsed = JSON.parse(await readFile(this.lockPath, "utf8"));
|
|
955
|
+
if (typeof parsed.pid !== "number" ||
|
|
956
|
+
typeof parsed.token !== "string" ||
|
|
957
|
+
typeof parsed.acquiredAt !== "number") {
|
|
958
|
+
return null;
|
|
959
|
+
}
|
|
960
|
+
return {
|
|
961
|
+
pid: parsed.pid,
|
|
962
|
+
token: parsed.token,
|
|
963
|
+
acquiredAt: parsed.acquiredAt,
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
catch {
|
|
967
|
+
return null;
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
async removeAbandonedLock() {
|
|
971
|
+
const owner = await this.readLockOwner();
|
|
972
|
+
try {
|
|
973
|
+
const lockStat = await stat(this.lockPath);
|
|
974
|
+
const oldEnough = Date.now() - lockStat.mtimeMs >= FILE_LOCK_STALE_MS;
|
|
975
|
+
const ownerRunning = owner ? processIsRunning(owner.pid) : false;
|
|
976
|
+
if ((owner && !ownerRunning) || oldEnough) {
|
|
977
|
+
await rm(this.lockPath, { force: true });
|
|
978
|
+
return true;
|
|
979
|
+
}
|
|
980
|
+
return false;
|
|
981
|
+
}
|
|
982
|
+
catch (error) {
|
|
983
|
+
return error.code === "ENOENT";
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
/**
|
|
987
|
+
* Best-effort cross-process lock. When another process holds it past the
|
|
988
|
+
* timeout the write proceeds anyway with a warning: a stuck lock must not
|
|
989
|
+
* take the pool offline, which is worse than the race it guards against.
|
|
990
|
+
*/
|
|
991
|
+
async acquireFileLock() {
|
|
992
|
+
await ensurePrivateDirectory(join(this.storagePath, ".."));
|
|
993
|
+
const deadline = Date.now() + FILE_LOCK_TIMEOUT_MS;
|
|
994
|
+
for (;;) {
|
|
995
|
+
const owner = {
|
|
996
|
+
pid: process.pid,
|
|
997
|
+
token: randomUUID(),
|
|
998
|
+
acquiredAt: Date.now(),
|
|
999
|
+
};
|
|
1000
|
+
let handle;
|
|
1001
|
+
try {
|
|
1002
|
+
handle = await open(this.lockPath, "wx", TokenStore.FILE_PERMISSIONS);
|
|
1003
|
+
await handle.writeFile(JSON.stringify(owner));
|
|
1004
|
+
const acquired = handle;
|
|
1005
|
+
return async () => {
|
|
1006
|
+
await acquired.close().catch(() => undefined);
|
|
1007
|
+
const current = await this.readLockOwner();
|
|
1008
|
+
if (current?.token === owner.token) {
|
|
1009
|
+
await rm(this.lockPath, { force: true }).catch(() => undefined);
|
|
1010
|
+
}
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
catch (error) {
|
|
1014
|
+
if (handle) {
|
|
1015
|
+
await handle.close().catch(() => undefined);
|
|
1016
|
+
await rm(this.lockPath, { force: true }).catch(() => undefined);
|
|
1017
|
+
}
|
|
1018
|
+
if (error.code !== "EEXIST") {
|
|
1019
|
+
logger.debug("Token store lock unavailable; proceeding unlocked", {
|
|
1020
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1021
|
+
});
|
|
1022
|
+
return async () => undefined;
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
if (await this.removeAbandonedLock()) {
|
|
1026
|
+
continue;
|
|
1027
|
+
}
|
|
1028
|
+
if (Date.now() >= deadline) {
|
|
1029
|
+
logger.warn("Token store lock held by another process past the timeout; proceeding unlocked", { lockPath: this.lockPath });
|
|
1030
|
+
return async () => undefined;
|
|
1031
|
+
}
|
|
1032
|
+
await sleep(FILE_LOCK_RETRY_MS);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
852
1035
|
/**
|
|
853
1036
|
* Deletes the storage file
|
|
854
1037
|
*/
|
|
@@ -887,21 +1070,9 @@ export class TokenStore {
|
|
|
887
1070
|
* Ensures the storage directory exists with proper permissions
|
|
888
1071
|
*/
|
|
889
1072
|
async ensureDirectory(dirPath) {
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
mode: TokenStore.DIR_PERMISSIONS,
|
|
894
|
-
});
|
|
895
|
-
}
|
|
896
|
-
catch {
|
|
897
|
-
// Directory might already exist, try to set permissions
|
|
898
|
-
try {
|
|
899
|
-
await chmod(dirPath, TokenStore.DIR_PERMISSIONS);
|
|
900
|
-
}
|
|
901
|
-
catch {
|
|
902
|
-
// Ignore permission errors on existing directories
|
|
903
|
-
}
|
|
904
|
-
}
|
|
1073
|
+
// `mkdir({recursive})` never throws for an existing directory, so the mode
|
|
1074
|
+
// must be re-asserted every time rather than only on first creation.
|
|
1075
|
+
await ensurePrivateDirectory(dirPath);
|
|
905
1076
|
}
|
|
906
1077
|
/**
|
|
907
1078
|
* Derives an encoding key from machine-specific information
|
|
@@ -912,27 +1083,37 @@ export class TokenStore {
|
|
|
912
1083
|
const machineInfo = `${homedir()}-neurolink-token-store`;
|
|
913
1084
|
return createHash("sha256").update(machineInfo).digest("hex");
|
|
914
1085
|
}
|
|
1086
|
+
deriveEncryptionKeyBytes() {
|
|
1087
|
+
const machineInfo = `${homedir()}-neurolink-token-store`;
|
|
1088
|
+
return createHash("sha256").update(machineInfo).digest();
|
|
1089
|
+
}
|
|
915
1090
|
/**
|
|
916
|
-
*
|
|
917
|
-
*
|
|
918
|
-
*
|
|
1091
|
+
* XOR obfuscation over the UTF-8 bytes of the document, base64-encoded and
|
|
1092
|
+
* prefixed so a reader can tell it from the older code-unit codec.
|
|
1093
|
+
* Note: this is NOT cryptographically secure, just basic obfuscation.
|
|
919
1094
|
*/
|
|
920
1095
|
obfuscate(data) {
|
|
921
|
-
const key = this.
|
|
922
|
-
|
|
923
|
-
for (let i = 0; i <
|
|
924
|
-
|
|
925
|
-
|
|
1096
|
+
const key = this.encryptionKeyBytes;
|
|
1097
|
+
const bytes = Buffer.from(data, "utf8");
|
|
1098
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
1099
|
+
bytes[i] ^= key[i % key.length];
|
|
1100
|
+
}
|
|
1101
|
+
return `${OBFUSCATION_V2_PREFIX}${bytes.toString("base64")}`;
|
|
1102
|
+
}
|
|
1103
|
+
deobfuscateV2(data) {
|
|
1104
|
+
const key = this.encryptionKeyBytes;
|
|
1105
|
+
const bytes = Buffer.from(data.slice(OBFUSCATION_V2_PREFIX.length), "base64");
|
|
1106
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
1107
|
+
bytes[i] ^= key[i % key.length];
|
|
926
1108
|
}
|
|
927
|
-
|
|
928
|
-
return Buffer.from(result, "binary").toString("base64");
|
|
1109
|
+
return bytes.toString("utf8");
|
|
929
1110
|
}
|
|
930
1111
|
/**
|
|
931
|
-
*
|
|
1112
|
+
* The pre-UTF-8 codec, kept only to read stores written before it existed.
|
|
1113
|
+
* It cannot round-trip any character above U+00FF.
|
|
932
1114
|
*/
|
|
933
|
-
|
|
1115
|
+
deobfuscateLegacy(data) {
|
|
934
1116
|
const key = this.encryptionKey;
|
|
935
|
-
// Decode from base64
|
|
936
1117
|
const decoded = Buffer.from(data, "base64").toString("binary");
|
|
937
1118
|
let result = "";
|
|
938
1119
|
for (let i = 0; i < decoded.length; i++) {
|
|
@@ -13,7 +13,8 @@ import { logger } from "../../utils/logger.js";
|
|
|
13
13
|
import { sanitizeForLog } from "../../utils/logSanitize.js";
|
|
14
14
|
import { AVATAR_ERROR_CODES, AvatarError, } from "../../utils/avatarProcessor.js";
|
|
15
15
|
import { assertSafeUrl } from "../../utils/ssrfGuard.js";
|
|
16
|
-
import {
|
|
16
|
+
import { safeDownload } from "../../utils/safeFetch.js";
|
|
17
|
+
import { MAX_AUDIO_BYTES, MAX_IMAGE_BYTES, MAX_VIDEO_BYTES, } from "../../utils/sizeGuard.js";
|
|
17
18
|
const DEFAULT_BASE_URL = "https://api.d-id.com";
|
|
18
19
|
const REQUEST_TIMEOUT_MS = 30_000;
|
|
19
20
|
const POLL_INTERVAL_MS = 3_000;
|
|
@@ -281,27 +282,26 @@ export class DIDAvatar {
|
|
|
281
282
|
});
|
|
282
283
|
}
|
|
283
284
|
async downloadResult(url) {
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
message: `D-ID result download failed: ${response.status}`,
|
|
289
|
-
category: ErrorCategory.NETWORK,
|
|
290
|
-
severity: ErrorSeverity.MEDIUM,
|
|
291
|
-
retriable: response.status >= 500,
|
|
292
|
-
context: { status: response.status, url },
|
|
293
|
-
});
|
|
294
|
-
}
|
|
285
|
+
// `safeDownload` re-validates the provider-returned URL, pins the
|
|
286
|
+
// connection to the vetted addresses, refuses redirects and bounds the
|
|
287
|
+
// body while it streams. The URL pre-check the caller already ran covered
|
|
288
|
+
// the URL text only, not the connection it produced.
|
|
295
289
|
try {
|
|
296
|
-
return await
|
|
290
|
+
return await safeDownload(url, {
|
|
291
|
+
maxBytes: MAX_VIDEO_BYTES,
|
|
292
|
+
label: "D-ID result",
|
|
293
|
+
timeoutMs: REQUEST_TIMEOUT_MS,
|
|
294
|
+
});
|
|
297
295
|
}
|
|
298
296
|
catch (err) {
|
|
297
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
298
|
+
const status = /HTTP (\d{3})/.exec(message)?.[1];
|
|
299
299
|
throw new AvatarError({
|
|
300
300
|
code: AVATAR_ERROR_CODES.GENERATION_FAILED,
|
|
301
|
-
message: `D-ID result download
|
|
301
|
+
message: `D-ID result download failed: ${message}`,
|
|
302
302
|
category: ErrorCategory.NETWORK,
|
|
303
|
-
severity: ErrorSeverity.HIGH,
|
|
304
|
-
retriable:
|
|
303
|
+
severity: status !== undefined ? ErrorSeverity.MEDIUM : ErrorSeverity.HIGH,
|
|
304
|
+
retriable: status !== undefined && Number(status) >= 500,
|
|
305
305
|
context: { url },
|
|
306
306
|
originalError: err instanceof Error ? err : undefined,
|
|
307
307
|
});
|
|
@@ -321,45 +321,49 @@ export class DIDAvatar {
|
|
|
321
321
|
retriable: false,
|
|
322
322
|
});
|
|
323
323
|
}
|
|
324
|
-
try {
|
|
325
|
-
await assertSafeUrl(input);
|
|
326
|
-
}
|
|
327
|
-
catch (err) {
|
|
328
|
-
throw new AvatarError({
|
|
329
|
-
code: AVATAR_ERROR_CODES.INVALID_INPUT,
|
|
330
|
-
message: `Unsafe URL rejected: ${err instanceof Error ? err.message : String(err)}`,
|
|
331
|
-
category: ErrorCategory.VALIDATION,
|
|
332
|
-
severity: ErrorSeverity.HIGH,
|
|
333
|
-
retriable: false,
|
|
334
|
-
context: { url: input },
|
|
335
|
-
});
|
|
336
|
-
}
|
|
337
|
-
const response = await this.fetchWithTimeout(input, { method: "GET" });
|
|
338
|
-
if (!response.ok) {
|
|
339
|
-
throw new AvatarError({
|
|
340
|
-
code: AVATAR_ERROR_CODES.INVALID_INPUT,
|
|
341
|
-
message: `Failed to fetch input from ${input}: ${response.status}`,
|
|
342
|
-
category: ErrorCategory.NETWORK,
|
|
343
|
-
severity: ErrorSeverity.MEDIUM,
|
|
344
|
-
retriable: response.status >= 500,
|
|
345
|
-
context: { url: input, status: response.status },
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
324
|
// Use the larger of the two input caps (audio 50 MiB > image 25 MiB) so
|
|
349
325
|
// both audio and image URLs are bounded without falsely rejecting valid audio.
|
|
350
326
|
const inputCap = Math.max(MAX_AUDIO_BYTES, MAX_IMAGE_BYTES);
|
|
327
|
+
// `safeDownload`: URL vetted (HTTPS, public host), connection pinned to
|
|
328
|
+
// the vetted addresses, redirects refused, body bounded while streaming.
|
|
351
329
|
try {
|
|
352
|
-
return await
|
|
330
|
+
return await safeDownload(input, {
|
|
331
|
+
maxBytes: inputCap,
|
|
332
|
+
label: "D-ID input",
|
|
333
|
+
timeoutMs: REQUEST_TIMEOUT_MS,
|
|
334
|
+
});
|
|
353
335
|
}
|
|
354
336
|
catch (err) {
|
|
337
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
338
|
+
if (/rejected|permitted|blocked range|redirect/i.test(message)) {
|
|
339
|
+
throw new AvatarError({
|
|
340
|
+
code: AVATAR_ERROR_CODES.INVALID_INPUT,
|
|
341
|
+
message: `Unsafe URL rejected: ${message}`,
|
|
342
|
+
category: ErrorCategory.VALIDATION,
|
|
343
|
+
severity: ErrorSeverity.HIGH,
|
|
344
|
+
retriable: false,
|
|
345
|
+
context: { url: input },
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
if (/too large|size cap/i.test(message)) {
|
|
349
|
+
throw new AvatarError({
|
|
350
|
+
code: AVATAR_ERROR_CODES.INVALID_INPUT,
|
|
351
|
+
message: `D-ID input download rejected: ${message}`,
|
|
352
|
+
category: ErrorCategory.NETWORK,
|
|
353
|
+
severity: ErrorSeverity.HIGH,
|
|
354
|
+
retriable: false,
|
|
355
|
+
context: { url: input },
|
|
356
|
+
originalError: err instanceof Error ? err : undefined,
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
const status = /HTTP (\d{3})/.exec(message)?.[1];
|
|
355
360
|
throw new AvatarError({
|
|
356
361
|
code: AVATAR_ERROR_CODES.INVALID_INPUT,
|
|
357
|
-
message: `
|
|
362
|
+
message: `Failed to fetch input from ${input}: ${message}`,
|
|
358
363
|
category: ErrorCategory.NETWORK,
|
|
359
|
-
severity: ErrorSeverity.
|
|
360
|
-
retriable:
|
|
364
|
+
severity: ErrorSeverity.MEDIUM,
|
|
365
|
+
retriable: status === undefined || Number(status) >= 500,
|
|
361
366
|
context: { url: input },
|
|
362
|
-
originalError: err instanceof Error ? err : undefined,
|
|
363
367
|
});
|
|
364
368
|
}
|
|
365
369
|
}
|