@juspay/neurolink 10.11.2 → 10.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/adapters/audioFormatSupport.d.ts +53 -0
- package/dist/adapters/audioFormatSupport.js +200 -0
- package/dist/browser/neurolink.min.js +399 -398
- package/dist/cli/commands/auth.d.ts +8 -1
- package/dist/cli/commands/auth.js +185 -6
- package/dist/cli/factories/authCommandFactory.js +7 -1
- package/dist/lib/adapters/audioFormatSupport.d.ts +53 -0
- package/dist/lib/adapters/audioFormatSupport.js +201 -0
- package/dist/lib/processors/archive/ArchiveProcessor.d.ts +37 -0
- package/dist/lib/processors/archive/ArchiveProcessor.js +347 -32
- package/dist/lib/providers/googleAiStudio/client.d.ts +17 -0
- package/dist/lib/providers/googleAiStudio/client.js +45 -19
- package/dist/lib/providers/googleNativeGemini3/utils.d.ts +22 -1
- package/dist/lib/providers/googleNativeGemini3/utils.js +54 -0
- package/dist/lib/providers/googleVertex/client.js +3 -0
- package/dist/lib/proxy/accountQuota.d.ts +6 -0
- package/dist/lib/proxy/accountQuota.js +19 -2
- package/dist/lib/proxy/accountUsage.d.ts +45 -0
- package/dist/lib/proxy/accountUsage.js +289 -0
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +15 -1
- package/dist/lib/server/routes/claudeProxyRoutes.js +166 -0
- package/dist/lib/types/cli.d.ts +12 -0
- package/dist/lib/types/file.d.ts +41 -0
- package/dist/lib/types/generate.d.ts +12 -1
- package/dist/lib/types/processor.d.ts +20 -1
- package/dist/lib/types/providers.d.ts +7 -0
- package/dist/lib/types/proxy.d.ts +101 -0
- package/dist/lib/utils/fileDetector.d.ts +27 -0
- package/dist/lib/utils/fileDetector.js +130 -7
- package/dist/lib/utils/imageProcessor.js +31 -0
- package/dist/lib/utils/messageBuilder.d.ts +0 -9
- package/dist/lib/utils/messageBuilder.js +380 -56
- package/dist/processors/archive/ArchiveProcessor.d.ts +37 -0
- package/dist/processors/archive/ArchiveProcessor.js +347 -32
- package/dist/providers/googleAiStudio/client.d.ts +17 -0
- package/dist/providers/googleAiStudio/client.js +45 -19
- package/dist/providers/googleNativeGemini3/utils.d.ts +22 -1
- package/dist/providers/googleNativeGemini3/utils.js +54 -0
- package/dist/providers/googleVertex/client.js +3 -0
- package/dist/proxy/accountQuota.d.ts +6 -0
- package/dist/proxy/accountQuota.js +19 -2
- package/dist/proxy/accountUsage.d.ts +45 -0
- package/dist/proxy/accountUsage.js +288 -0
- package/dist/server/routes/claudeProxyRoutes.d.ts +15 -1
- package/dist/server/routes/claudeProxyRoutes.js +166 -0
- package/dist/types/cli.d.ts +12 -0
- package/dist/types/file.d.ts +41 -0
- package/dist/types/generate.d.ts +12 -1
- package/dist/types/processor.d.ts +20 -1
- package/dist/types/providers.d.ts +7 -0
- package/dist/types/proxy.d.ts +101 -0
- package/dist/utils/fileDetector.d.ts +27 -0
- package/dist/utils/fileDetector.js +130 -7
- package/dist/utils/imageProcessor.js +31 -0
- package/dist/utils/messageBuilder.d.ts +0 -9
- package/dist/utils/messageBuilder.js +380 -56
- package/package.json +3 -2
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* Currently supports:
|
|
18
18
|
* - Anthropic (API key + OAuth)
|
|
19
19
|
*/
|
|
20
|
-
import type { AuthCommandArgs } from "../../lib/types/index.js";
|
|
20
|
+
import type { AccountQuota, AuthCommandArgs } from "../../lib/types/index.js";
|
|
21
21
|
/**
|
|
22
22
|
* Handle the login subcommand
|
|
23
23
|
* `neurolink auth login <provider>`
|
|
@@ -26,6 +26,13 @@ import type { AuthCommandArgs } from "../../lib/types/index.js";
|
|
|
26
26
|
* (e.g., "anthropic:alice") to support multi-account pools.
|
|
27
27
|
*/
|
|
28
28
|
export declare function handleLogin(argv: AuthCommandArgs): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Format the dynamic per-plan limit windows (model-scoped weeklies such as
|
|
31
|
+
* Fable, plus any future kinds) as extra display lines. `session` and
|
|
32
|
+
* `weekly_all` are omitted — they already render as the SESSION / WEEKLY
|
|
33
|
+
* columns. Exported for the continuous test suite.
|
|
34
|
+
*/
|
|
35
|
+
export declare function formatQuotaWindowRows(quota: AccountQuota): string[];
|
|
29
36
|
/**
|
|
30
37
|
* Handle the list subcommand
|
|
31
38
|
* `neurolink auth list`
|
|
@@ -27,7 +27,8 @@ import ora from "ora";
|
|
|
27
27
|
import { logger } from "../../lib/utils/logger.js";
|
|
28
28
|
import { defaultTokenStore } from "../../lib/auth/tokenStore.js";
|
|
29
29
|
import { CLAUDE_CODE_CLIENT_ID, ANTHROPIC_AUTH_URL, ANTHROPIC_TOKEN_URL, ANTHROPIC_REDIRECT_URI, CLAUDE_CLI_USER_AGENT, OAUTH_BETA_HEADERS, } from "../../lib/auth/anthropicOAuth.js";
|
|
30
|
-
import { loadAccountQuotas } from "../../lib/proxy/accountQuota.js";
|
|
30
|
+
import { flushAccountQuotas, loadAccountQuotas, saveAccountQuota, } from "../../lib/proxy/accountQuota.js";
|
|
31
|
+
import { fetchAccountUsage, listAnthropicAccountsForUsage, usageToQuota, } from "../../lib/proxy/accountUsage.js";
|
|
31
32
|
// =============================================================================
|
|
32
33
|
// CONSTANTS
|
|
33
34
|
// =============================================================================
|
|
@@ -161,6 +162,137 @@ function formatQuotaColumns(quota) {
|
|
|
161
162
|
: "",
|
|
162
163
|
};
|
|
163
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* Format the dynamic per-plan limit windows (model-scoped weeklies such as
|
|
167
|
+
* Fable, plus any future kinds) as extra display lines. `session` and
|
|
168
|
+
* `weekly_all` are omitted — they already render as the SESSION / WEEKLY
|
|
169
|
+
* columns. Exported for the continuous test suite.
|
|
170
|
+
*/
|
|
171
|
+
export function formatQuotaWindowRows(quota) {
|
|
172
|
+
if (!quota.windows?.length) {
|
|
173
|
+
return [];
|
|
174
|
+
}
|
|
175
|
+
const colorize = (pct, text) => {
|
|
176
|
+
if (pct <= 10) {
|
|
177
|
+
return chalk.red(text);
|
|
178
|
+
}
|
|
179
|
+
if (pct <= 30) {
|
|
180
|
+
return chalk.yellow(text);
|
|
181
|
+
}
|
|
182
|
+
return chalk.green(text);
|
|
183
|
+
};
|
|
184
|
+
const rows = [];
|
|
185
|
+
for (const window of quota.windows) {
|
|
186
|
+
if (window.kind === "session" || window.kind === "weekly_all") {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const remaining = Math.round((1 - window.used) * 100);
|
|
190
|
+
const label = window.scopeModel
|
|
191
|
+
? `${window.group ?? window.kind} (${window.scopeModel})`
|
|
192
|
+
: window.kind;
|
|
193
|
+
const reset = window.resetsAt > 0
|
|
194
|
+
? chalk.gray(` resets ${formatTimeUntil(window.resetsAt)}`)
|
|
195
|
+
: "";
|
|
196
|
+
rows.push(`${chalk.gray(`${label}:`)} ${colorize(remaining, `${remaining}% left`)}${reset}`);
|
|
197
|
+
}
|
|
198
|
+
return rows;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Fetch fresh limits for `auth list --refresh`.
|
|
202
|
+
*
|
|
203
|
+
* Prefers the running proxy's GET /limits endpoint so the proxy's in-memory
|
|
204
|
+
* routing state is refreshed as a side effect; falls back to fetching the
|
|
205
|
+
* usage endpoint directly from this process (persisting through the same
|
|
206
|
+
* quota store) when no proxy is running or the call fails.
|
|
207
|
+
*/
|
|
208
|
+
async function refreshAccountLimitsForList() {
|
|
209
|
+
const errors = [];
|
|
210
|
+
const proxyState = detectRunningProxyState();
|
|
211
|
+
if (proxyState?.port) {
|
|
212
|
+
const host = proxyState.host && proxyState.host !== "0.0.0.0"
|
|
213
|
+
? proxyState.host
|
|
214
|
+
: "127.0.0.1";
|
|
215
|
+
try {
|
|
216
|
+
const response = await fetch(`http://${host}:${proxyState.port}/limits`, {
|
|
217
|
+
signal: AbortSignal.timeout(45_000),
|
|
218
|
+
});
|
|
219
|
+
if (response.ok) {
|
|
220
|
+
const payload = (await response.json());
|
|
221
|
+
const quotas = {};
|
|
222
|
+
for (const result of payload.results) {
|
|
223
|
+
if (result.quota) {
|
|
224
|
+
quotas[result.account] = result.quota;
|
|
225
|
+
}
|
|
226
|
+
if (result.status === "error" && result.error) {
|
|
227
|
+
errors.push(`${result.account}: ${result.error}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return { via: "proxy", quotas, errors };
|
|
231
|
+
}
|
|
232
|
+
errors.push(`running proxy /limits returned HTTP ${response.status}; fetching directly`);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
// Keep the message generic: a raw fetch error can echo the requested
|
|
236
|
+
// URL, and this string reaches the text and JSON CLI output.
|
|
237
|
+
errors.push("running proxy /limits unreachable; fetching directly");
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
try {
|
|
241
|
+
const accounts = await listAnthropicAccountsForUsage();
|
|
242
|
+
const prior = await loadAccountQuotas().catch(() => ({}));
|
|
243
|
+
const quotas = {};
|
|
244
|
+
const CONCURRENCY = 3;
|
|
245
|
+
let nextIndex = 0;
|
|
246
|
+
const worker = async () => {
|
|
247
|
+
for (;;) {
|
|
248
|
+
const index = nextIndex++;
|
|
249
|
+
if (index >= accounts.length) {
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
const account = accounts[index];
|
|
253
|
+
if (account.type !== "oauth") {
|
|
254
|
+
continue; // api_key accounts have no subscription windows
|
|
255
|
+
}
|
|
256
|
+
// Isolate failures per account: one rejection must not abort the
|
|
257
|
+
// Promise.all sweep or discard the other accounts' refreshed quotas.
|
|
258
|
+
try {
|
|
259
|
+
const result = await fetchAccountUsage(account);
|
|
260
|
+
if (!result.ok) {
|
|
261
|
+
errors.push(`${account.label}: ${result.error}`);
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
const quota = usageToQuota(result.usage, {
|
|
265
|
+
now: Date.now(),
|
|
266
|
+
prior: prior[account.label] ?? null,
|
|
267
|
+
});
|
|
268
|
+
if (!quota) {
|
|
269
|
+
errors.push(`${account.label}: usage payload had no recognizable limit windows`);
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
await saveAccountQuota(account.label, quota);
|
|
273
|
+
quotas[account.label] = quota;
|
|
274
|
+
}
|
|
275
|
+
catch (err) {
|
|
276
|
+
errors.push(`${account.label}: ${err instanceof Error ? err.message : String(err)}`);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
};
|
|
280
|
+
try {
|
|
281
|
+
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, accounts.length || 1) }, () => worker()));
|
|
282
|
+
}
|
|
283
|
+
finally {
|
|
284
|
+
// The quota store's debounced flush timer is unref()'d and this process
|
|
285
|
+
// is short-lived — flush now (even on a partial sweep) or the completed
|
|
286
|
+
// saves never reach disk.
|
|
287
|
+
await flushAccountQuotas().catch(() => undefined);
|
|
288
|
+
}
|
|
289
|
+
return { via: "direct", quotas, errors };
|
|
290
|
+
}
|
|
291
|
+
catch (err) {
|
|
292
|
+
errors.push(`direct limit fetch failed (${err instanceof Error ? err.message : String(err)})`);
|
|
293
|
+
return { via: "none", quotas: null, errors };
|
|
294
|
+
}
|
|
295
|
+
}
|
|
164
296
|
/**
|
|
165
297
|
* Handle the list subcommand
|
|
166
298
|
* `neurolink auth list`
|
|
@@ -189,6 +321,7 @@ export async function handleList(argv) {
|
|
|
189
321
|
let email;
|
|
190
322
|
let tokenStatus = "unknown";
|
|
191
323
|
let expiresAt;
|
|
324
|
+
let tokenType;
|
|
192
325
|
// Derive email from the compound key label when it looks like an email.
|
|
193
326
|
// The credentials file is a shared singleton that gets overwritten on
|
|
194
327
|
// every login — reading email from it would show the LATEST login's
|
|
@@ -220,6 +353,7 @@ export async function handleList(argv) {
|
|
|
220
353
|
const tokens = await defaultTokenStore.loadTokens(key);
|
|
221
354
|
if (tokens) {
|
|
222
355
|
expiresAt = tokens.expiresAt;
|
|
356
|
+
tokenType = tokens.tokenType;
|
|
223
357
|
const isExpired = defaultTokenStore.isTokenExpired(tokens, 0);
|
|
224
358
|
tokenStatus = isExpired ? "expired" : "valid";
|
|
225
359
|
// Extract per-account metadata from scope (e.g. "tier:pro email:user@example.com")
|
|
@@ -242,9 +376,24 @@ export async function handleList(argv) {
|
|
|
242
376
|
catch {
|
|
243
377
|
// Token load failed — show as unknown
|
|
244
378
|
}
|
|
245
|
-
return {
|
|
379
|
+
return {
|
|
380
|
+
key,
|
|
381
|
+
provider,
|
|
382
|
+
label,
|
|
383
|
+
email,
|
|
384
|
+
tier,
|
|
385
|
+
tokenStatus,
|
|
386
|
+
expiresAt,
|
|
387
|
+
tokenType,
|
|
388
|
+
};
|
|
246
389
|
}));
|
|
247
|
-
//
|
|
390
|
+
// Optionally fetch FRESH limits from Anthropic before rendering.
|
|
391
|
+
let refreshOutcome;
|
|
392
|
+
if (argv.refresh) {
|
|
393
|
+
refreshOutcome = await refreshAccountLimitsForList();
|
|
394
|
+
}
|
|
395
|
+
// Load persisted quota data (captured from proxy responses), then overlay
|
|
396
|
+
// anything just refreshed — freshly fetched values win over the snapshot.
|
|
248
397
|
let quotas = {};
|
|
249
398
|
try {
|
|
250
399
|
quotas = await loadAccountQuotas();
|
|
@@ -252,6 +401,9 @@ export async function handleList(argv) {
|
|
|
252
401
|
catch {
|
|
253
402
|
// Non-fatal — quota display is best-effort
|
|
254
403
|
}
|
|
404
|
+
if (refreshOutcome?.quotas) {
|
|
405
|
+
quotas = { ...quotas, ...refreshOutcome.quotas };
|
|
406
|
+
}
|
|
255
407
|
if (argv.format === "json") {
|
|
256
408
|
// Merge quota data into each account object for JSON output
|
|
257
409
|
const withQuota = enrichedAccounts.map((acct) => {
|
|
@@ -259,9 +411,29 @@ export async function handleList(argv) {
|
|
|
259
411
|
const quota = quotas[quotaKey] ?? null;
|
|
260
412
|
return { ...acct, quota };
|
|
261
413
|
});
|
|
262
|
-
|
|
414
|
+
if (refreshOutcome) {
|
|
415
|
+
// --refresh envelopes the array so the fetch outcome travels with it.
|
|
416
|
+
logger.always(JSON.stringify({
|
|
417
|
+
refresh: {
|
|
418
|
+
via: refreshOutcome.via,
|
|
419
|
+
errors: refreshOutcome.errors,
|
|
420
|
+
},
|
|
421
|
+
accounts: withQuota,
|
|
422
|
+
}, null, 2));
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
logger.always(JSON.stringify(withQuota, null, 2));
|
|
426
|
+
}
|
|
263
427
|
}
|
|
264
428
|
else {
|
|
429
|
+
if (refreshOutcome) {
|
|
430
|
+
if (refreshOutcome.via !== "none") {
|
|
431
|
+
logger.always(chalk.gray(`\nFetched fresh limits from Anthropic (${refreshOutcome.via === "proxy" ? "via running proxy" : "direct"}).`));
|
|
432
|
+
}
|
|
433
|
+
for (const refreshError of refreshOutcome.errors) {
|
|
434
|
+
logger.always(chalk.yellow(`⚠ ${refreshError}`));
|
|
435
|
+
}
|
|
436
|
+
}
|
|
265
437
|
logger.always(chalk.bold("\nAuthenticated Accounts:\n"));
|
|
266
438
|
// Check if any account has quota data to decide column layout
|
|
267
439
|
const hasQuota = enrichedAccounts.some((acct) => {
|
|
@@ -296,14 +468,21 @@ export async function handleList(argv) {
|
|
|
296
468
|
if (hasQuota && quota) {
|
|
297
469
|
const qc = formatQuotaColumns(quota);
|
|
298
470
|
logger.always(` ${chalk.cyan(displayLabel)} ${displayProvider} ${displayEmail} ${statusText} ${qc.sessionText.padEnd(10)} ${qc.weeklyText.padEnd(10)}`);
|
|
471
|
+
const indent = " ".repeat(2 + 20 + 1 + 12 + 1 + 28 + 1 + 14 + 1);
|
|
299
472
|
// Second line: reset times (indented under session/weekly columns)
|
|
300
473
|
if (qc.sessionReset || qc.weeklyReset) {
|
|
301
|
-
const indent = " ".repeat(2 + 20 + 1 + 12 + 1 + 28 + 1 + 14 + 1);
|
|
302
474
|
logger.always(`${indent}${(qc.sessionReset || "").padEnd(10)} ${qc.weeklyReset || ""}`);
|
|
303
475
|
}
|
|
476
|
+
// Dynamic per-plan windows (e.g. the Fable-only weekly limit)
|
|
477
|
+
for (const windowRow of formatQuotaWindowRows(quota)) {
|
|
478
|
+
logger.always(`${indent}${windowRow}`);
|
|
479
|
+
}
|
|
304
480
|
}
|
|
305
481
|
else {
|
|
306
|
-
|
|
482
|
+
const apiKeyNote = refreshOutcome && acct.tokenType && acct.tokenType !== "Bearer"
|
|
483
|
+
? chalk.gray(" (api key — not refreshed)")
|
|
484
|
+
: "";
|
|
485
|
+
logger.always(` ${chalk.cyan(displayLabel)} ${displayProvider} ${displayEmail} ${statusText}${hasQuota ? " - -" : ""}${apiKeyNote}`);
|
|
307
486
|
}
|
|
308
487
|
}
|
|
309
488
|
logger.always("");
|
|
@@ -214,8 +214,14 @@ export class AuthCommandFactory {
|
|
|
214
214
|
*/
|
|
215
215
|
static buildListOptions(yargs) {
|
|
216
216
|
return yargs
|
|
217
|
+
.option("refresh", {
|
|
218
|
+
type: "boolean",
|
|
219
|
+
default: false,
|
|
220
|
+
description: "Fetch fresh limits from Anthropic for all OAuth accounts before listing (via the running proxy when available)",
|
|
221
|
+
})
|
|
217
222
|
.example("$0 auth list", "List all authenticated accounts")
|
|
218
|
-
.example("$0 auth list --format json", "List accounts in JSON format")
|
|
223
|
+
.example("$0 auth list --format json", "List accounts in JSON format")
|
|
224
|
+
.example("$0 auth list --refresh", "Fetch fresh session/weekly/model-scoped limits before listing");
|
|
219
225
|
}
|
|
220
226
|
/**
|
|
221
227
|
* Build options for remove subcommand
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native audio delivery to providers that can listen.
|
|
3
|
+
*
|
|
4
|
+
* ## The gap this closes
|
|
5
|
+
*
|
|
6
|
+
* Until this module existed, attaching an audio file produced a message
|
|
7
|
+
* containing only a metadata block:
|
|
8
|
+
*
|
|
9
|
+
* ## Audio File: "recording.mp3"
|
|
10
|
+
* Duration: 19s | Codec: MPEG 2 Layer 3 | Bitrate: 32 kbps |
|
|
11
|
+
* Sample Rate: 22050 Hz | Channels: 1 (Mono)
|
|
12
|
+
*
|
|
13
|
+
* No audio bytes were ever handed to the provider. Every question about what
|
|
14
|
+
* the recording *says* — transcribe this, who is speaking, what was agreed —
|
|
15
|
+
* was answered from a description of the file, and Gemini has accepted inline
|
|
16
|
+
* audio the whole time.
|
|
17
|
+
*
|
|
18
|
+
* The failure was invisible for an instructive reason: that metadata block
|
|
19
|
+
* answers precisely the questions a test is most tempted to ask. "How long is
|
|
20
|
+
* this audio?" and "what sample rate is it?" both succeed with no audio
|
|
21
|
+
* attached, so a suite built on them reports working audio support. It took an
|
|
22
|
+
* end-to-end test asking for a spoken word to expose it.
|
|
23
|
+
*
|
|
24
|
+
* ## Provider scope
|
|
25
|
+
*
|
|
26
|
+
* Deliberately a capability map rather than "send audio to everyone". A
|
|
27
|
+
* provider that cannot accept an audio part responds with an opaque HTTP 400,
|
|
28
|
+
* which is worse than the metadata summary it would otherwise have received —
|
|
29
|
+
* so an unlisted provider keeps the existing text-only behaviour and loses
|
|
30
|
+
* nothing.
|
|
31
|
+
*
|
|
32
|
+
* @module adapters/audioFormatSupport
|
|
33
|
+
*/
|
|
34
|
+
import type { AudioConversionResult } from "../types/index.js";
|
|
35
|
+
/** Whether `provider` can be handed raw audio bytes. */
|
|
36
|
+
export declare function supportsNativeAudio(provider: string): boolean;
|
|
37
|
+
/** Whether `mimeType` must be re-encoded before a native provider will read it. */
|
|
38
|
+
export declare function needsAudioTranscode(mimeType: string): boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Return audio bytes a native provider can read, transcoding when the source
|
|
41
|
+
* container is one it does not accept.
|
|
42
|
+
*
|
|
43
|
+
* Never throws for audio reasons. When conversion is impossible — no ffmpeg, an
|
|
44
|
+
* unreadable stream — the original bytes and MIME type come back with
|
|
45
|
+
* `converted: false`, and the caller falls back to the metadata summary. That
|
|
46
|
+
* keeps this from turning a previously-working (if limited) request into a
|
|
47
|
+
* failure.
|
|
48
|
+
*
|
|
49
|
+
* @param buffer - Raw audio bytes.
|
|
50
|
+
* @param mimeType - Detected MIME type of `buffer`.
|
|
51
|
+
* @param extension - Source extension, used so ffmpeg picks the right demuxer.
|
|
52
|
+
*/
|
|
53
|
+
export declare function toProviderCompatibleAudio(buffer: Buffer, mimeType: string, extension: string): Promise<AudioConversionResult>;
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native audio delivery to providers that can listen.
|
|
3
|
+
*
|
|
4
|
+
* ## The gap this closes
|
|
5
|
+
*
|
|
6
|
+
* Until this module existed, attaching an audio file produced a message
|
|
7
|
+
* containing only a metadata block:
|
|
8
|
+
*
|
|
9
|
+
* ## Audio File: "recording.mp3"
|
|
10
|
+
* Duration: 19s | Codec: MPEG 2 Layer 3 | Bitrate: 32 kbps |
|
|
11
|
+
* Sample Rate: 22050 Hz | Channels: 1 (Mono)
|
|
12
|
+
*
|
|
13
|
+
* No audio bytes were ever handed to the provider. Every question about what
|
|
14
|
+
* the recording *says* — transcribe this, who is speaking, what was agreed —
|
|
15
|
+
* was answered from a description of the file, and Gemini has accepted inline
|
|
16
|
+
* audio the whole time.
|
|
17
|
+
*
|
|
18
|
+
* The failure was invisible for an instructive reason: that metadata block
|
|
19
|
+
* answers precisely the questions a test is most tempted to ask. "How long is
|
|
20
|
+
* this audio?" and "what sample rate is it?" both succeed with no audio
|
|
21
|
+
* attached, so a suite built on them reports working audio support. It took an
|
|
22
|
+
* end-to-end test asking for a spoken word to expose it.
|
|
23
|
+
*
|
|
24
|
+
* ## Provider scope
|
|
25
|
+
*
|
|
26
|
+
* Deliberately a capability map rather than "send audio to everyone". A
|
|
27
|
+
* provider that cannot accept an audio part responds with an opaque HTTP 400,
|
|
28
|
+
* which is worse than the metadata summary it would otherwise have received —
|
|
29
|
+
* so an unlisted provider keeps the existing text-only behaviour and loses
|
|
30
|
+
* nothing.
|
|
31
|
+
*
|
|
32
|
+
* @module adapters/audioFormatSupport
|
|
33
|
+
*/
|
|
34
|
+
import { withTimeout } from "../utils/errorHandling.js";
|
|
35
|
+
import { logger } from "../utils/logger.js";
|
|
36
|
+
import { getFfmpegPath, runFfmpeg } from "./video/ffmpegAdapter.js";
|
|
37
|
+
/**
|
|
38
|
+
* Ceiling for one audio conversion.
|
|
39
|
+
*
|
|
40
|
+
* Longer than the image equivalent because a lossless hour-long WAV is a
|
|
41
|
+
* legitimate input and re-encoding it is not instant, but still bounded so a
|
|
42
|
+
* wedged decoder cannot hold a generation request open indefinitely.
|
|
43
|
+
*/
|
|
44
|
+
const AUDIO_TRANSCODE_TIMEOUT_MS = 120_000;
|
|
45
|
+
/**
|
|
46
|
+
* Providers that accept inline audio parts.
|
|
47
|
+
*
|
|
48
|
+
* Google's Gemini models (both Vertex and AI Studio) take audio as `inlineData`
|
|
49
|
+
* alongside text. Other providers are omitted rather than assumed: OpenAI's
|
|
50
|
+
* audio models use a different request shape than the chat-completions path
|
|
51
|
+
* NeuroLink builds here, and sending an audio part to a provider that does not
|
|
52
|
+
* expect one converts a working (if limited) response into a hard failure.
|
|
53
|
+
*/
|
|
54
|
+
const NATIVE_AUDIO_PROVIDERS = new Set([
|
|
55
|
+
"vertex",
|
|
56
|
+
"google-vertex",
|
|
57
|
+
"googlevertex",
|
|
58
|
+
"google-ai-studio",
|
|
59
|
+
"googleaistudio",
|
|
60
|
+
"google-ai",
|
|
61
|
+
"googleai",
|
|
62
|
+
"gemini",
|
|
63
|
+
]);
|
|
64
|
+
/**
|
|
65
|
+
* Audio MIME types the native providers accept as-is.
|
|
66
|
+
*
|
|
67
|
+
* Gemini's documented set. Anything outside it is transcoded rather than
|
|
68
|
+
* rejected, because the container a user happens to have — a voice memo in
|
|
69
|
+
* CAF, a Windows recording in WMA — says nothing about whether the audio
|
|
70
|
+
* inside is useful.
|
|
71
|
+
*/
|
|
72
|
+
const NATIVE_AUDIO_MIME_TYPES = new Set([
|
|
73
|
+
"audio/wav",
|
|
74
|
+
"audio/x-wav",
|
|
75
|
+
"audio/mpeg",
|
|
76
|
+
"audio/mp3",
|
|
77
|
+
"audio/aiff",
|
|
78
|
+
"audio/x-aiff",
|
|
79
|
+
"audio/aac",
|
|
80
|
+
"audio/ogg",
|
|
81
|
+
"audio/flac",
|
|
82
|
+
"audio/x-flac",
|
|
83
|
+
]);
|
|
84
|
+
/** MIME type every transcode targets. Universally accepted and compact. */
|
|
85
|
+
const TRANSCODE_TARGET_MIME = "audio/mpeg";
|
|
86
|
+
/** Whether `provider` can be handed raw audio bytes. */
|
|
87
|
+
export function supportsNativeAudio(provider) {
|
|
88
|
+
return NATIVE_AUDIO_PROVIDERS.has(provider.toLowerCase().trim());
|
|
89
|
+
}
|
|
90
|
+
/** Whether `mimeType` must be re-encoded before a native provider will read it. */
|
|
91
|
+
export function needsAudioTranscode(mimeType) {
|
|
92
|
+
return !NATIVE_AUDIO_MIME_TYPES.has(normalizeAudioMime(mimeType));
|
|
93
|
+
}
|
|
94
|
+
function normalizeAudioMime(mimeType) {
|
|
95
|
+
return mimeType.split(";")[0].trim().toLowerCase();
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Re-encode audio to MP3 with ffmpeg.
|
|
99
|
+
*
|
|
100
|
+
* Temp files rather than stdin: several of the containers that need converting
|
|
101
|
+
* (CAF, WavPack, AU) carry their metadata in a trailer or require seeking, and
|
|
102
|
+
* a piped stream leaves ffmpeg unable to find it. The directory is removed in
|
|
103
|
+
* `finally` whether or not the conversion succeeded.
|
|
104
|
+
*
|
|
105
|
+
* Node builtins are imported dynamically because the browser bundle stubs
|
|
106
|
+
* `node:fs/promises` without `mkdtemp`; nothing in a browser spawns ffmpeg, so
|
|
107
|
+
* the import belongs at the point of use.
|
|
108
|
+
*/
|
|
109
|
+
async function transcodeToMp3(buffer, extension) {
|
|
110
|
+
const [{ randomUUID }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join },] = await Promise.all([
|
|
111
|
+
import("node:crypto"),
|
|
112
|
+
import("node:fs/promises"),
|
|
113
|
+
import("node:os"),
|
|
114
|
+
import("node:path"),
|
|
115
|
+
]);
|
|
116
|
+
const workDir = await mkdtemp(join(tmpdir(), "neurolink-audio-"));
|
|
117
|
+
const inputPath = join(workDir, `${randomUUID()}${extension}`);
|
|
118
|
+
const outputPath = join(workDir, `${randomUUID()}.mp3`);
|
|
119
|
+
try {
|
|
120
|
+
await writeFile(inputPath, buffer);
|
|
121
|
+
await runFfmpeg([
|
|
122
|
+
"-y",
|
|
123
|
+
"-v",
|
|
124
|
+
"error",
|
|
125
|
+
"-i",
|
|
126
|
+
inputPath,
|
|
127
|
+
// Downmix and cap the rate: speech is the point, and a 48 kHz stereo
|
|
128
|
+
// re-encode of a mono voice memo triples the payload for nothing.
|
|
129
|
+
"-ac",
|
|
130
|
+
"1",
|
|
131
|
+
"-ar",
|
|
132
|
+
"16000",
|
|
133
|
+
"-c:a",
|
|
134
|
+
"libmp3lame",
|
|
135
|
+
"-q:a",
|
|
136
|
+
"4",
|
|
137
|
+
outputPath,
|
|
138
|
+
],
|
|
139
|
+
// Without this the call inherits runFfmpeg's frame-extraction default of
|
|
140
|
+
// 30s, which is sized for pulling a single video frame — so the 120s
|
|
141
|
+
// ceiling above, chosen precisely because re-encoding a lossless
|
|
142
|
+
// hour-long WAV is not instant, could never be reached. ffmpeg killed the
|
|
143
|
+
// transcode at 30s and the outer race never got to run.
|
|
144
|
+
{ timeoutMs: AUDIO_TRANSCODE_TIMEOUT_MS });
|
|
145
|
+
return await readFile(outputPath);
|
|
146
|
+
}
|
|
147
|
+
finally {
|
|
148
|
+
await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Return audio bytes a native provider can read, transcoding when the source
|
|
153
|
+
* container is one it does not accept.
|
|
154
|
+
*
|
|
155
|
+
* Never throws for audio reasons. When conversion is impossible — no ffmpeg, an
|
|
156
|
+
* unreadable stream — the original bytes and MIME type come back with
|
|
157
|
+
* `converted: false`, and the caller falls back to the metadata summary. That
|
|
158
|
+
* keeps this from turning a previously-working (if limited) request into a
|
|
159
|
+
* failure.
|
|
160
|
+
*
|
|
161
|
+
* @param buffer - Raw audio bytes.
|
|
162
|
+
* @param mimeType - Detected MIME type of `buffer`.
|
|
163
|
+
* @param extension - Source extension, used so ffmpeg picks the right demuxer.
|
|
164
|
+
*/
|
|
165
|
+
export async function toProviderCompatibleAudio(buffer, mimeType, extension) {
|
|
166
|
+
const normalized = normalizeAudioMime(mimeType);
|
|
167
|
+
if (!needsAudioTranscode(normalized)) {
|
|
168
|
+
return { buffer, mimeType: normalized, converted: false };
|
|
169
|
+
}
|
|
170
|
+
// Resolving the binary first turns "ffmpeg is not installed" into one clear
|
|
171
|
+
// warning rather than a spawn error surfacing from inside the conversion.
|
|
172
|
+
const ffmpegAvailable = await getFfmpegPath()
|
|
173
|
+
.then(() => true)
|
|
174
|
+
.catch(() => false);
|
|
175
|
+
if (!ffmpegAvailable) {
|
|
176
|
+
logger.warn(`[audioFormatSupport] ${normalized} needs conversion before a provider can ` +
|
|
177
|
+
`read it, but ffmpeg is unavailable — falling back to a metadata-only ` +
|
|
178
|
+
`summary. Install ffmpeg (or set FFMPEG_PATH) to enable this format.`);
|
|
179
|
+
return { buffer, mimeType: normalized, converted: false };
|
|
180
|
+
}
|
|
181
|
+
try {
|
|
182
|
+
const converted = await withTimeout(transcodeToMp3(buffer, extension), AUDIO_TRANSCODE_TIMEOUT_MS, new Error(`audio transcode exceeded ${AUDIO_TRANSCODE_TIMEOUT_MS}ms`));
|
|
183
|
+
if (converted.length === 0) {
|
|
184
|
+
throw new Error("produced an empty audio stream");
|
|
185
|
+
}
|
|
186
|
+
logger.debug(`[audioFormatSupport] Transcoded ${normalized} → ${TRANSCODE_TARGET_MIME} ` +
|
|
187
|
+
`(${buffer.length} → ${converted.length} bytes) for native delivery`);
|
|
188
|
+
return {
|
|
189
|
+
buffer: converted,
|
|
190
|
+
mimeType: TRANSCODE_TARGET_MIME,
|
|
191
|
+
converted: true,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
logger.warn(`[audioFormatSupport] Could not convert ${normalized} for native delivery ` +
|
|
196
|
+
`— falling back to a metadata-only summary: ` +
|
|
197
|
+
`${error instanceof Error ? error.message.split("\n")[0] : String(error)}`);
|
|
198
|
+
return { buffer, mimeType: normalized, converted: false };
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=audioFormatSupport.js.map
|
|
@@ -193,6 +193,43 @@ export declare class ArchiveProcessor extends BaseFileProcessor<ProcessedArchive
|
|
|
193
193
|
* @param entries - Previously extracted entry metadata
|
|
194
194
|
* @returns Map of entry name to extracted text content
|
|
195
195
|
*/
|
|
196
|
+
/**
|
|
197
|
+
* Whether an entry name looks like something worth inlining as text.
|
|
198
|
+
*
|
|
199
|
+
* Shared by the ZIP and TAR paths so the two cannot drift into disagreeing
|
|
200
|
+
* about which members are worth reading — they did, because only ZIP had the
|
|
201
|
+
* rule at all.
|
|
202
|
+
*/
|
|
203
|
+
private isExtractableEntryName;
|
|
204
|
+
/**
|
|
205
|
+
* Decode bytes to text, or null when they are not text.
|
|
206
|
+
*
|
|
207
|
+
* A NUL byte in the first 512 bytes, or a high proportion of replacement
|
|
208
|
+
* characters after decoding, means binary — inlining that would spend the
|
|
209
|
+
* extraction budget on mojibake.
|
|
210
|
+
*/
|
|
211
|
+
private decodeEntryText;
|
|
212
|
+
/**
|
|
213
|
+
* Decompress a single-stream archive (.bz2, .xz, .zst).
|
|
214
|
+
*
|
|
215
|
+
* Node ships zstd from v22.15/23, so that one needs no help. bzip2 and xz
|
|
216
|
+
* have no Node binding, and adding a native module for them would make an
|
|
217
|
+
* optional format a build-time dependency for every consumer — so the system
|
|
218
|
+
* tools are used when present, the same soft-dependency arrangement this
|
|
219
|
+
* codebase already has with ffmpeg. Absent tooling returns null and the
|
|
220
|
+
* caller reports the format as unsupported *on this machine* rather than
|
|
221
|
+
* unsupported in principle.
|
|
222
|
+
*/
|
|
223
|
+
private decompressSingleStream;
|
|
224
|
+
/**
|
|
225
|
+
* Extract a single-stream archive: decompress, then treat the result as a
|
|
226
|
+
* TAR when it is one and as a lone file otherwise.
|
|
227
|
+
*
|
|
228
|
+
* The tar check matters because `.tar.xz` and `.tar.zst` are how these
|
|
229
|
+
* formats are usually met — reporting one opaque "decompressed-content" blob
|
|
230
|
+
* for an archive of forty files would be technically true and useless.
|
|
231
|
+
*/
|
|
232
|
+
private extractSingleStreamEntries;
|
|
196
233
|
private extractEntryContents;
|
|
197
234
|
/**
|
|
198
235
|
* Build a structured text description of the archive for LLM consumption.
|