@mengruo/dsh-vision-toolkit 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +2 -2
- package/README.md +17 -42
- package/README.zh.md +16 -41
- package/assets/logo_eapi_dark.png +0 -0
- package/docs/aihubmix-gemini-vision.i18n.yaml +2 -2
- package/docs/aihubmix-gemini-vision.md +2 -2
- package/docs/aihubmix-gemini-vision.zh.md +2 -2
- package/lib/client.js +246 -68
- package/lib/client.js.map +1 -1
- package/lib/config.js +143 -24
- package/lib/config.js.map +1 -1
- package/lib/evidence-cache.js +14 -0
- package/lib/evidence-cache.js.map +1 -1
- package/lib/image-input-variants.js +39 -27
- package/lib/image-input-variants.js.map +1 -1
- package/lib/runtime-install.js +204 -19
- package/lib/runtime-install.js.map +1 -1
- package/lib/runtime.js +310 -154
- package/lib/runtime.js.map +1 -1
- package/lib/tools.js +1 -0
- package/lib/tools.js.map +1 -1
- package/lib/types/client/index.d.ts +45 -3
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/config.d.ts +53 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/evidence-cache.d.ts.map +1 -1
- package/lib/types/image-input-variants.d.ts +1 -6
- package/lib/types/image-input-variants.d.ts.map +1 -1
- package/lib/types/runtime-install.d.ts +21 -0
- package/lib/types/runtime-install.d.ts.map +1 -1
- package/lib/types/runtime.d.ts +45 -10
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/tools.d.ts.map +1 -1
- package/lib/types/upstream.d.ts +3 -0
- package/lib/types/upstream.d.ts.map +1 -1
- package/lib/types/web.d.ts +8 -0
- package/lib/types/web.d.ts.map +1 -1
- package/lib/upstream.js +11 -6
- package/lib/upstream.js.map +1 -1
- package/lib/web.js +50 -7
- package/lib/web.js.map +1 -1
- package/package.json +1 -1
- package/src/client/index.tsx +331 -88
- package/src/config.ts +210 -28
- package/src/evidence-cache.ts +14 -0
- package/src/image-input-variants.ts +39 -27
- package/src/runtime-install.ts +231 -20
- package/src/runtime.ts +333 -180
- package/src/tools.ts +1 -0
- package/src/upstream.ts +15 -8
- package/src/web.ts +67 -8
package/lib/runtime.js
CHANGED
|
@@ -149,6 +149,16 @@ export class Semaphore {
|
|
|
149
149
|
next.resolve();
|
|
150
150
|
}
|
|
151
151
|
}
|
|
152
|
+
/** Non-blocking acquisition: claim a free slot immediately, else return false. */
|
|
153
|
+
tryAcquire(permits = 1) {
|
|
154
|
+
if (!Number.isInteger(permits) || permits < 1 || permits > this.limit)
|
|
155
|
+
return false;
|
|
156
|
+
if (this.waiters.length === 0 && this.active + permits <= this.limit) {
|
|
157
|
+
this.active += permits;
|
|
158
|
+
return true;
|
|
159
|
+
}
|
|
160
|
+
return false;
|
|
161
|
+
}
|
|
152
162
|
}
|
|
153
163
|
const REGION_PATTERN = /^\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*$/;
|
|
154
164
|
const MAX_TIMEOUT_MS = 600_000;
|
|
@@ -301,6 +311,7 @@ export class VisionToolkitRuntime {
|
|
|
301
311
|
config;
|
|
302
312
|
semaphores = new Map();
|
|
303
313
|
glanceCache = new WeakMap();
|
|
314
|
+
providerGates = new Map();
|
|
304
315
|
adapter;
|
|
305
316
|
constructor(ctx, config, adapter) {
|
|
306
317
|
this.ctx = ctx;
|
|
@@ -317,20 +328,37 @@ export class VisionToolkitRuntime {
|
|
|
317
328
|
}
|
|
318
329
|
/** Capture the credential and provider identity used by one evidence conversion. */
|
|
319
330
|
async captureEvidenceRuntime() {
|
|
320
|
-
|
|
321
|
-
|
|
331
|
+
// The primary provider's key hash sharpens cache invalidation, but a
|
|
332
|
+
// missing primary credential must not block evidence conversion when a
|
|
333
|
+
// later provider in the failover pool is available.
|
|
334
|
+
let credentialSha256;
|
|
335
|
+
let sslVerify;
|
|
336
|
+
try {
|
|
337
|
+
const env = await this.resolveVisionEnv();
|
|
338
|
+
credentialSha256 = createHash('sha256').update(env.VISION_API_KEY).digest('hex');
|
|
339
|
+
sslVerify = env.VISION_SSL_VERIFY;
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
credentialSha256 = undefined;
|
|
343
|
+
}
|
|
344
|
+
const evidenceFingerprint = evidenceRuntimeFingerprint(this.config, credentialSha256, sslVerify);
|
|
322
345
|
return Object.freeze({
|
|
323
346
|
evidenceFingerprint,
|
|
324
|
-
glance: (request, options) => this.glanceWithEnv(request, options
|
|
347
|
+
glance: (request, options) => this.glanceWithEnv(request, options),
|
|
325
348
|
});
|
|
326
349
|
}
|
|
327
350
|
timeout(options) {
|
|
328
|
-
const value = options.timeoutMs ?? this.
|
|
351
|
+
const value = options.timeoutMs ?? this.operationTimeoutMs();
|
|
329
352
|
if (!Number.isInteger(value) || value < 1000 || value > MAX_TIMEOUT_MS) {
|
|
330
353
|
throw new VisionToolkitError('input', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`);
|
|
331
354
|
}
|
|
332
355
|
return value;
|
|
333
356
|
}
|
|
357
|
+
/** Overall operation budget: the slowest enabled provider's timeout, else the global default. */
|
|
358
|
+
operationTimeoutMs() {
|
|
359
|
+
const providers = this.config.providers.filter(provider => provider.enabled);
|
|
360
|
+
return providers.length === 0 ? this.config.timeoutMs : Math.max(...providers.map(provider => provider.timeoutMs));
|
|
361
|
+
}
|
|
334
362
|
operationError(tool, error, deadline, phase = 'execution') {
|
|
335
363
|
if (deadline.cancelled) {
|
|
336
364
|
return new VisionToolkitError('cancelled', phase === 'queue' ? `${tool}: cancelled while waiting for a concurrency slot` : `${tool}: cancelled`);
|
|
@@ -408,29 +436,69 @@ export class VisionToolkitRuntime {
|
|
|
408
436
|
this.semaphores.delete(semaphore.key);
|
|
409
437
|
}
|
|
410
438
|
}
|
|
411
|
-
/**
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
415
|
-
: await this.ctx.credentials.resolve(this.config.provider.credential);
|
|
416
|
-
if (resolved === undefined) {
|
|
417
|
-
throw new VisionToolkitError('config', `credential ${this.config.provider.credential} is not configured; set it through DSH credentials`);
|
|
418
|
-
}
|
|
419
|
-
return this.visionEnv(resolved);
|
|
439
|
+
/** Highest-priority enabled provider, falling back to the first entry. */
|
|
440
|
+
get primaryProvider() {
|
|
441
|
+
return this.config.providers.find(provider => provider.enabled) ?? this.config.providers[0];
|
|
420
442
|
}
|
|
421
|
-
|
|
443
|
+
/** Build the upstream environment for one resolved provider. */
|
|
444
|
+
providerEnv(provider, resolved) {
|
|
422
445
|
const sslVerify = process.env.VISION_SSL_VERIFY?.trim();
|
|
423
446
|
return {
|
|
424
447
|
VISION_API_KEY: resolved.value,
|
|
425
|
-
VISION_BASE_URL:
|
|
426
|
-
VISION_MODEL:
|
|
427
|
-
VISION_API_PROTOCOL:
|
|
428
|
-
VISION_ANTHROPIC_THINKING:
|
|
448
|
+
VISION_BASE_URL: provider.baseUrl,
|
|
449
|
+
VISION_MODEL: provider.model,
|
|
450
|
+
VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
|
|
451
|
+
VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
|
|
429
452
|
...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
|
|
430
|
-
VISION_USER_AGENT:
|
|
453
|
+
VISION_USER_AGENT: provider.userAgent,
|
|
431
454
|
LANG: this.config.language,
|
|
432
455
|
};
|
|
433
456
|
}
|
|
457
|
+
/** Resolve one provider's credential into its environment, or undefined when unavailable. */
|
|
458
|
+
async resolveProviderEnv(provider) {
|
|
459
|
+
let resolved;
|
|
460
|
+
try {
|
|
461
|
+
resolved = isBuiltInFreeVisionProvider({
|
|
462
|
+
baseUrl: provider.baseUrl,
|
|
463
|
+
credential: provider.credential,
|
|
464
|
+
model: provider.model,
|
|
465
|
+
protocol: provider.protocol,
|
|
466
|
+
anthropicThinking: provider.anthropicThinking,
|
|
467
|
+
userAgent: provider.userAgent,
|
|
468
|
+
})
|
|
469
|
+
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
470
|
+
: await this.ctx.credentials.resolve(provider.credential);
|
|
471
|
+
}
|
|
472
|
+
catch {
|
|
473
|
+
resolved = undefined;
|
|
474
|
+
}
|
|
475
|
+
if (resolved === undefined)
|
|
476
|
+
return undefined;
|
|
477
|
+
return { provider, env: this.providerEnv(provider, resolved) };
|
|
478
|
+
}
|
|
479
|
+
/** Resolve every enabled provider in priority order, skipping unreadable credentials. */
|
|
480
|
+
async resolveProviderPool() {
|
|
481
|
+
const pool = [];
|
|
482
|
+
for (const provider of this.config.providers) {
|
|
483
|
+
if (!provider.enabled)
|
|
484
|
+
continue;
|
|
485
|
+
const entry = await this.resolveProviderEnv(provider);
|
|
486
|
+
if (entry === undefined) {
|
|
487
|
+
this.ctx.logger.warn('dsh-vision-toolkit provider=%s credential=%s unavailable; skipped from the failover pool', provider.name, String(provider.credential));
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
pool.push(entry);
|
|
491
|
+
}
|
|
492
|
+
return pool;
|
|
493
|
+
}
|
|
494
|
+
/** Resolve the primary provider's credential at the remote-operation boundary. */
|
|
495
|
+
async resolveVisionEnv() {
|
|
496
|
+
const entry = await this.resolveProviderEnv(this.primaryProvider);
|
|
497
|
+
if (entry === undefined) {
|
|
498
|
+
throw new VisionToolkitError('config', `credential ${String(this.primaryProvider.credential)} is not configured; set it through DSH credentials`);
|
|
499
|
+
}
|
|
500
|
+
return entry.env;
|
|
501
|
+
}
|
|
434
502
|
pathPolicy(workspace) {
|
|
435
503
|
return createPathPolicy(workspace, this.config.allowedDirs);
|
|
436
504
|
}
|
|
@@ -503,7 +571,7 @@ export class VisionToolkitRuntime {
|
|
|
503
571
|
|| probed.width * probed.height > maxPixels) {
|
|
504
572
|
return undefined;
|
|
505
573
|
}
|
|
506
|
-
return { path: real, bytes: bytes.length, width: probed.width, height: probed.height, format: probed.format };
|
|
574
|
+
return { path: real, bytes: bytes.length, width: probed.width, height: probed.height, format: probed.format, hasAlpha: probed.hasAlpha };
|
|
507
575
|
}
|
|
508
576
|
cacheEntryOutDigest(entry, prefix) {
|
|
509
577
|
const tail = entry.slice(prefix.length + 1);
|
|
@@ -560,7 +628,7 @@ export class VisionToolkitRuntime {
|
|
|
560
628
|
}
|
|
561
629
|
await Promise.all([...stalePartials, ...remove].map(name => rm(join(root, name), { force: true }).catch(() => { })));
|
|
562
630
|
}
|
|
563
|
-
async autoCompressImage(image, policy, operation) {
|
|
631
|
+
async autoCompressImage(image, policy, operation, maxBytes, maxPixels) {
|
|
564
632
|
let bytes;
|
|
565
633
|
try {
|
|
566
634
|
bytes = await readFile(image.path, { signal: operation.signal });
|
|
@@ -574,7 +642,7 @@ export class VisionToolkitRuntime {
|
|
|
574
642
|
const digest = createHash('sha256').update(bytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH);
|
|
575
643
|
const root = await this.compressedImageRoot(policy);
|
|
576
644
|
await this.pruneCompressedCache(root);
|
|
577
|
-
const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${
|
|
645
|
+
const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${maxBytes}-p${maxPixels}`;
|
|
578
646
|
for (const entry of await readdir(root)) {
|
|
579
647
|
if (!entry.startsWith(`${prefix}-`) || entry.startsWith('.'))
|
|
580
648
|
continue;
|
|
@@ -583,7 +651,7 @@ export class VisionToolkitRuntime {
|
|
|
583
651
|
await rm(join(root, entry), { force: true }).catch(() => { });
|
|
584
652
|
continue;
|
|
585
653
|
}
|
|
586
|
-
const cached = await this.readCacheCandidate(root, entry, outDigestPrefix,
|
|
654
|
+
const cached = await this.readCacheCandidate(root, entry, outDigestPrefix, maxBytes, maxPixels, operation);
|
|
587
655
|
if (cached !== undefined) {
|
|
588
656
|
return { ...cached, originalPath: image.path };
|
|
589
657
|
}
|
|
@@ -592,7 +660,7 @@ export class VisionToolkitRuntime {
|
|
|
592
660
|
const staged = join(root, `.${prefix}-${randomUUID()}.partial`);
|
|
593
661
|
let compressed;
|
|
594
662
|
try {
|
|
595
|
-
compressed = await this.adapter.compressImage(image.path, staged,
|
|
663
|
+
compressed = await this.adapter.compressImage(image.path, staged, maxBytes, maxPixels, { signal: operation.signal });
|
|
596
664
|
}
|
|
597
665
|
catch (error) {
|
|
598
666
|
await rm(staged, { force: true }).catch(() => { });
|
|
@@ -603,7 +671,7 @@ export class VisionToolkitRuntime {
|
|
|
603
671
|
const outDigest = createHash('sha256').update(stagedBytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH);
|
|
604
672
|
const finalName = `${prefix}-${outDigest}-${compressed.width}x${compressed.height}.${extension}`;
|
|
605
673
|
const finalPath = join(root, finalName);
|
|
606
|
-
const existing = await this.readCacheCandidate(root, finalName, outDigest,
|
|
674
|
+
const existing = await this.readCacheCandidate(root, finalName, outDigest, maxBytes, maxPixels, operation);
|
|
607
675
|
if (existing !== undefined) {
|
|
608
676
|
await rm(staged, { force: true }).catch(() => { });
|
|
609
677
|
return { ...existing, originalPath: image.path };
|
|
@@ -623,9 +691,11 @@ export class VisionToolkitRuntime {
|
|
|
623
691
|
width: compressed.width,
|
|
624
692
|
height: compressed.height,
|
|
625
693
|
format: compressed.format,
|
|
694
|
+
hasAlpha: compressed.hasAlpha,
|
|
626
695
|
originalPath: image.path,
|
|
627
696
|
};
|
|
628
697
|
}
|
|
698
|
+
/** Validate one image against the configured global limits (used by local tools). */
|
|
629
699
|
async validateImage(raw, policy, operation) {
|
|
630
700
|
const image = await resolveInputFile(raw, policy);
|
|
631
701
|
const decoded = await this.adapter.probeImageSize(image.path, { signal: operation.signal });
|
|
@@ -639,16 +709,106 @@ export class VisionToolkitRuntime {
|
|
|
639
709
|
throw new VisionToolkitError('input', `image content is ${decoded.format}, but the filename uses ${extension}`);
|
|
640
710
|
}
|
|
641
711
|
if (image.bytes <= this.config.maxImageBytes && pixels <= this.config.maxImagePixels) {
|
|
642
|
-
return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, originalPath: image.path };
|
|
712
|
+
return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, hasAlpha: decoded.hasAlpha, originalPath: image.path };
|
|
643
713
|
}
|
|
644
|
-
return this.autoCompressImage(image, policy, operation);
|
|
714
|
+
return this.autoCompressImage(image, policy, operation, this.config.maxImageBytes, this.config.maxImagePixels);
|
|
645
715
|
}
|
|
646
716
|
accountImage(image, operation) {
|
|
647
717
|
operation.metrics.imageCount += 1;
|
|
648
718
|
operation.metrics.imageBytes += image.bytes;
|
|
649
719
|
operation.metrics.imagePixels += image.width * image.height;
|
|
650
720
|
}
|
|
651
|
-
|
|
721
|
+
/** Stable gate key for one provider's in-flight request cap. */
|
|
722
|
+
providerGate(provider) {
|
|
723
|
+
const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`;
|
|
724
|
+
let gate = this.providerGates.get(key);
|
|
725
|
+
if (gate === undefined) {
|
|
726
|
+
gate = new Semaphore(provider.concurrency);
|
|
727
|
+
this.providerGates.set(key, gate);
|
|
728
|
+
}
|
|
729
|
+
return gate;
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* Prepare one image for an online vision request against the enabled
|
|
733
|
+
* provider pool. The raw image is kept when at least one enabled provider
|
|
734
|
+
* accepts it; otherwise it is compressed once to the first (highest
|
|
735
|
+
* priority) provider's limits so the priority route can proceed.
|
|
736
|
+
*/
|
|
737
|
+
async prepareVisionImage(raw, providers, policy, operation) {
|
|
738
|
+
const image = await resolveInputFile(raw, policy);
|
|
739
|
+
const decoded = await this.adapter.probeImageSize(image.path, { signal: operation.signal });
|
|
740
|
+
const pixels = decoded.width * decoded.height;
|
|
741
|
+
if (!Number.isSafeInteger(pixels) || pixels < 1) {
|
|
742
|
+
throw new VisionToolkitError('input', `image dimensions are invalid: ${decoded.width}x${decoded.height}`);
|
|
743
|
+
}
|
|
744
|
+
const extension = extname(image.path).toLowerCase();
|
|
745
|
+
const expected = FORMAT_BY_EXTENSION.get(extension);
|
|
746
|
+
if (expected !== decoded.format) {
|
|
747
|
+
throw new VisionToolkitError('input', `image content is ${decoded.format}, but the filename uses ${extension}`);
|
|
748
|
+
}
|
|
749
|
+
const fits = providers.some(provider => image.bytes <= provider.maxImageBytes && pixels <= provider.maxImagePixels);
|
|
750
|
+
if (fits) {
|
|
751
|
+
return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, hasAlpha: decoded.hasAlpha, originalPath: image.path };
|
|
752
|
+
}
|
|
753
|
+
const first = providers[0];
|
|
754
|
+
if (first === undefined) {
|
|
755
|
+
throw new VisionToolkitError('config', 'no enabled vision provider is available for this image');
|
|
756
|
+
}
|
|
757
|
+
return this.autoCompressImage(image, policy, operation, first.maxImageBytes, first.maxImagePixels);
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* Run one online-vision upstream command across the enabled provider pool in
|
|
761
|
+
* priority order. A provider is skipped when its size limits or concurrency
|
|
762
|
+
* are exhausted, retried up to its attempt count, and the next provider
|
|
763
|
+
* takes over on failure. Throws once every provider has failed.
|
|
764
|
+
*/
|
|
765
|
+
async runVisionWithFailover(tool, args, images, operation, pool) {
|
|
766
|
+
if (pool.length === 0) {
|
|
767
|
+
throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
|
|
768
|
+
}
|
|
769
|
+
let lastError;
|
|
770
|
+
let attempted = false;
|
|
771
|
+
for (const { provider, env } of pool) {
|
|
772
|
+
const fits = images.every(image => image.bytes <= provider.maxImageBytes && image.width * image.height <= provider.maxImagePixels);
|
|
773
|
+
if (!fits)
|
|
774
|
+
continue;
|
|
775
|
+
const gate = this.providerGate(provider);
|
|
776
|
+
if (!gate.tryAcquire())
|
|
777
|
+
continue;
|
|
778
|
+
attempted = true;
|
|
779
|
+
try {
|
|
780
|
+
for (let attempt = 1; attempt <= provider.attempts; attempt++) {
|
|
781
|
+
const attemptDeadline = createDeadline(operation.signal, provider.timeoutMs);
|
|
782
|
+
try {
|
|
783
|
+
return await this.runUpstream(tool, args, { signal: attemptDeadline.signal, metrics: operation.metrics }, env);
|
|
784
|
+
}
|
|
785
|
+
catch (error) {
|
|
786
|
+
if (operation.signal.aborted)
|
|
787
|
+
throw error;
|
|
788
|
+
const classified = error instanceof VisionToolkitError
|
|
789
|
+
? error
|
|
790
|
+
: new VisionToolkitError('service', `${tool}: request failed`, { cause: error });
|
|
791
|
+
lastError = attemptDeadline.timedOut && classified.code === 'cancelled'
|
|
792
|
+
? new VisionToolkitError('timeout', `${tool}: ${provider.name} request timed out after ${provider.timeoutMs}ms`, { cause: classified })
|
|
793
|
+
: classified;
|
|
794
|
+
}
|
|
795
|
+
finally {
|
|
796
|
+
attemptDeadline.cleanup();
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
finally {
|
|
801
|
+
gate.release();
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
if (!attempted) {
|
|
805
|
+
throw new VisionToolkitError('capacity', `${tool}: no enabled vision provider accepts the image size or has a free concurrency slot`);
|
|
806
|
+
}
|
|
807
|
+
if (lastError instanceof VisionToolkitError)
|
|
808
|
+
throw lastError;
|
|
809
|
+
throw new VisionToolkitError('service', `${tool}: all vision providers failed`, { cause: lastError });
|
|
810
|
+
}
|
|
811
|
+
async glanceCacheKey(request, images, pool, signal) {
|
|
652
812
|
const imageFingerprints = await Promise.all(images.map(async (image) => {
|
|
653
813
|
let bytes;
|
|
654
814
|
try {
|
|
@@ -670,16 +830,20 @@ export class VisionToolkitRuntime {
|
|
|
670
830
|
query: request.query ?? null,
|
|
671
831
|
ocr: request.ocr === true,
|
|
672
832
|
region: request.region ?? null,
|
|
673
|
-
|
|
833
|
+
language: this.config.language,
|
|
834
|
+
providers: pool.map(({ provider, env }) => ({
|
|
835
|
+
name: provider.name,
|
|
674
836
|
baseUrl: env.VISION_BASE_URL,
|
|
675
837
|
model: env.VISION_MODEL,
|
|
676
838
|
protocol: env.VISION_API_PROTOCOL,
|
|
677
839
|
anthropicThinking: env.VISION_ANTHROPIC_THINKING,
|
|
678
840
|
sslVerify: env.VISION_SSL_VERIFY ?? null,
|
|
679
841
|
userAgent: env.VISION_USER_AGENT,
|
|
680
|
-
language: env.LANG,
|
|
681
842
|
credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
|
|
682
|
-
|
|
843
|
+
maxImageBytes: provider.maxImageBytes,
|
|
844
|
+
maxImagePixels: provider.maxImagePixels,
|
|
845
|
+
attempts: provider.attempts,
|
|
846
|
+
})),
|
|
683
847
|
});
|
|
684
848
|
}
|
|
685
849
|
async runUpstream(tool, args, operation, env) {
|
|
@@ -746,7 +910,7 @@ export class VisionToolkitRuntime {
|
|
|
746
910
|
async glance(request, options) {
|
|
747
911
|
return this.glanceWithEnv(request, options);
|
|
748
912
|
}
|
|
749
|
-
async glanceWithEnv(request, options
|
|
913
|
+
async glanceWithEnv(request, options) {
|
|
750
914
|
return this.runOperation('vision_glance', options, async (operation) => {
|
|
751
915
|
if (request.images.length === 0)
|
|
752
916
|
throw new VisionToolkitError('input', 'glance requires at least one image');
|
|
@@ -759,10 +923,15 @@ export class VisionToolkitRuntime {
|
|
|
759
923
|
if (request.region !== undefined)
|
|
760
924
|
parseRegion(request.region);
|
|
761
925
|
const policy = await this.pathPolicy(options.workspace);
|
|
926
|
+
const pool = await this.resolveProviderPool();
|
|
927
|
+
if (pool.length === 0) {
|
|
928
|
+
throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
|
|
929
|
+
}
|
|
930
|
+
const providers = pool.map(entry => entry.provider);
|
|
762
931
|
const images = [];
|
|
763
932
|
const seen = new Set();
|
|
764
933
|
for (const raw of request.images) {
|
|
765
|
-
const image = await this.
|
|
934
|
+
const image = await this.prepareVisionImage(raw, providers, policy, operation);
|
|
766
935
|
if (seen.has(image.path)) {
|
|
767
936
|
operation.metrics.cacheHits += 1;
|
|
768
937
|
continue;
|
|
@@ -771,10 +940,9 @@ export class VisionToolkitRuntime {
|
|
|
771
940
|
this.accountImage(image, operation);
|
|
772
941
|
images.push(image);
|
|
773
942
|
}
|
|
774
|
-
const env = capturedEnv ?? await this.resolveVisionEnv();
|
|
775
943
|
const cacheKey = options.sessionScope === undefined
|
|
776
944
|
? undefined
|
|
777
|
-
: await this.glanceCacheKey(request, images,
|
|
945
|
+
: await this.glanceCacheKey(request, images, pool, operation.signal);
|
|
778
946
|
if (options.sessionScope !== undefined && cacheKey !== undefined) {
|
|
779
947
|
const cached = this.glanceCache.get(options.sessionScope);
|
|
780
948
|
if (cached?.key === cacheKey) {
|
|
@@ -782,12 +950,12 @@ export class VisionToolkitRuntime {
|
|
|
782
950
|
return cached.result;
|
|
783
951
|
}
|
|
784
952
|
}
|
|
785
|
-
const result = await this.
|
|
953
|
+
const result = await this.runVisionWithFailover('glance', [
|
|
786
954
|
...images.map(image => image.path),
|
|
787
955
|
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
788
956
|
...(request.ocr === true ? ['--ocr'] : []),
|
|
789
957
|
...(request.query !== undefined ? ['-q', request.query] : []),
|
|
790
|
-
], operation,
|
|
958
|
+
], images, operation, pool);
|
|
791
959
|
const answer = result.stdout.trim();
|
|
792
960
|
if (answer.length === 0)
|
|
793
961
|
throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
|
|
@@ -823,14 +991,17 @@ export class VisionToolkitRuntime {
|
|
|
823
991
|
if (request.region !== undefined)
|
|
824
992
|
parseRegion(request.region);
|
|
825
993
|
const policy = await this.pathPolicy(options.workspace);
|
|
826
|
-
const
|
|
994
|
+
const pool = await this.resolveProviderPool();
|
|
995
|
+
if (pool.length === 0) {
|
|
996
|
+
throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
|
|
997
|
+
}
|
|
998
|
+
const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation);
|
|
827
999
|
this.accountImage(image, operation);
|
|
828
|
-
const
|
|
829
|
-
const result = await this.runUpstream(tool, [
|
|
1000
|
+
const result = await this.runVisionWithFailover(tool, [
|
|
830
1001
|
image.path,
|
|
831
1002
|
request.target,
|
|
832
1003
|
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
833
|
-
], operation,
|
|
1004
|
+
], [image], operation, pool);
|
|
834
1005
|
const elements = parseLocationOutput(result.stdout);
|
|
835
1006
|
this.validateLocations(elements, image.width, image.height);
|
|
836
1007
|
return { image, elements };
|
|
@@ -1142,7 +1313,13 @@ export class VisionToolkitRuntime {
|
|
|
1142
1313
|
throw new VisionToolkitError('input', 'long_screenshot_ocr.prompt must not be empty when provided');
|
|
1143
1314
|
}
|
|
1144
1315
|
const policy = await this.pathPolicy(options.workspace);
|
|
1145
|
-
const
|
|
1316
|
+
const pool = splitOnly ? [] : await this.resolveProviderPool();
|
|
1317
|
+
if (!splitOnly && pool.length === 0) {
|
|
1318
|
+
throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
|
|
1319
|
+
}
|
|
1320
|
+
const image = splitOnly
|
|
1321
|
+
? await this.validateImage(request.image, policy, operation)
|
|
1322
|
+
: await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation);
|
|
1146
1323
|
this.accountImage(image, operation);
|
|
1147
1324
|
const stem = basename(image.originalPath, extname(image.originalPath));
|
|
1148
1325
|
const finalDirectory = resolveOutputDirectory(request.runName, policy, `${stem}.long-ocr`);
|
|
@@ -1158,7 +1335,7 @@ export class VisionToolkitRuntime {
|
|
|
1158
1335
|
const finalOutput = join(finalDirectory, basename(stagedOutput));
|
|
1159
1336
|
const stagedChunks = join(stagedDirectory, 'chunks');
|
|
1160
1337
|
const stagedManifest = join(stagedChunks, 'manifest.json');
|
|
1161
|
-
const
|
|
1338
|
+
const ocrArgs = [
|
|
1162
1339
|
image.path,
|
|
1163
1340
|
'--mode',
|
|
1164
1341
|
mode,
|
|
@@ -1177,7 +1354,10 @@ export class VisionToolkitRuntime {
|
|
|
1177
1354
|
String(chunkTimeoutSeconds),
|
|
1178
1355
|
...(splitOnly ? ['--split-only'] : []),
|
|
1179
1356
|
...(request.resume === true ? ['--resume'] : []),
|
|
1180
|
-
]
|
|
1357
|
+
];
|
|
1358
|
+
const result = splitOnly
|
|
1359
|
+
? await this.runUpstream('long_screenshot_ocr', ocrArgs, operation)
|
|
1360
|
+
: await this.runVisionWithFailover('long_screenshot_ocr', ocrArgs, [image], operation, pool);
|
|
1181
1361
|
const reported = result.stdout.trim();
|
|
1182
1362
|
const expectedReported = splitOnly ? stagedManifest : stagedOutput;
|
|
1183
1363
|
if (reported !== expectedReported) {
|
|
@@ -1509,20 +1689,8 @@ export class VisionToolkitRuntime {
|
|
|
1509
1689
|
}
|
|
1510
1690
|
});
|
|
1511
1691
|
}
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
try {
|
|
1515
|
-
await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' });
|
|
1516
|
-
await rm(probe, { force: true });
|
|
1517
|
-
return { status: 'ok', detail: `${label} is writable: ${path}` };
|
|
1518
|
-
}
|
|
1519
|
-
catch {
|
|
1520
|
-
await rm(probe, { force: true }).catch(() => { });
|
|
1521
|
-
return { status: 'error', detail: `${label} is not writable: ${path}` };
|
|
1522
|
-
}
|
|
1523
|
-
}
|
|
1524
|
-
/** Health: inspect local readiness, optionally probe `/models`, and explicitly test one real multimodal request. */
|
|
1525
|
-
async health(testConnection, options, testModel = false) {
|
|
1692
|
+
/** Health: inspect local readiness, and optionally probe one provider's `/models` plus one real multimodal request. */
|
|
1693
|
+
async health(testConnection, options, testModel = false, provider) {
|
|
1526
1694
|
return this.runOperation('vision_toolkit_health', options, async (operation) => {
|
|
1527
1695
|
const info = this.upstreamVersion;
|
|
1528
1696
|
const python = { status: 'ok', detail: `${info.pythonVersion} via ${info.python}` };
|
|
@@ -1544,117 +1712,104 @@ export class VisionToolkitRuntime {
|
|
|
1544
1712
|
throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled');
|
|
1545
1713
|
chrome = { status: 'error', detail: 'Chrome availability probe failed' };
|
|
1546
1714
|
}
|
|
1547
|
-
let resolvedCredential;
|
|
1548
|
-
let credential;
|
|
1549
|
-
try {
|
|
1550
|
-
resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
|
|
1551
|
-
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
1552
|
-
: await this.ctx.credentials.resolve(this.config.provider.credential);
|
|
1553
|
-
credential = resolvedCredential === undefined
|
|
1554
|
-
? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
|
|
1555
|
-
: { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` };
|
|
1556
|
-
}
|
|
1557
|
-
catch {
|
|
1558
|
-
credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` };
|
|
1559
|
-
}
|
|
1560
|
-
let artifactDirectory;
|
|
1561
|
-
try {
|
|
1562
|
-
// allowedDirs are session input roots; they do not affect output readiness.
|
|
1563
|
-
const policy = await createPathPolicy(options.workspace, []);
|
|
1564
|
-
artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory');
|
|
1565
|
-
}
|
|
1566
|
-
catch {
|
|
1567
|
-
artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' };
|
|
1568
|
-
}
|
|
1569
|
-
const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory');
|
|
1570
1715
|
let service = {
|
|
1571
1716
|
status: 'not_tested',
|
|
1572
|
-
detail: 'Connection was not tested;
|
|
1717
|
+
detail: 'Connection was not tested; use the per-provider API test',
|
|
1573
1718
|
};
|
|
1574
1719
|
let model = {
|
|
1575
1720
|
status: 'not_tested',
|
|
1576
|
-
detail: 'Vision model was not tested;
|
|
1721
|
+
detail: 'Vision model was not tested; use the per-provider model test',
|
|
1577
1722
|
};
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
const endpoint = `${this.config.provider.baseUrl}/models`;
|
|
1585
|
-
try {
|
|
1586
|
-
const started = Date.now();
|
|
1587
|
-
const headers = {
|
|
1588
|
-
Accept: 'application/json',
|
|
1589
|
-
'User-Agent': this.config.provider.userAgent,
|
|
1590
|
-
};
|
|
1591
|
-
if (this.config.provider.protocol === 'anthropic') {
|
|
1592
|
-
headers['x-api-key'] = resolvedCredential.value;
|
|
1593
|
-
headers['anthropic-version'] = '2023-06-01';
|
|
1594
|
-
}
|
|
1595
|
-
else {
|
|
1596
|
-
headers.Authorization = `Bearer ${resolvedCredential.value}`;
|
|
1597
|
-
}
|
|
1598
|
-
const response = await fetch(endpoint, {
|
|
1599
|
-
method: 'GET',
|
|
1600
|
-
headers,
|
|
1601
|
-
signal: operation.signal,
|
|
1602
|
-
});
|
|
1603
|
-
operation.metrics.upstreamMs += Date.now() - started;
|
|
1604
|
-
await response.body?.cancel().catch(() => { });
|
|
1605
|
-
if (response.ok) {
|
|
1606
|
-
service = { status: 'ok', detail: `Service responded at ${endpoint} (HTTP ${response.status})` };
|
|
1607
|
-
}
|
|
1608
|
-
else if (response.status === 401) {
|
|
1609
|
-
service = { status: 'error', detail: `Service rejected the configured credential (HTTP ${response.status})` };
|
|
1610
|
-
}
|
|
1611
|
-
else if (response.status === 403) {
|
|
1612
|
-
// Some providers (e.g. Groq preview/account restrictions) block GET /models
|
|
1613
|
-
// while real multimodal requests still work. Treat 403 as a warning so the
|
|
1614
|
-
// explicit vision-model test, not the model list endpoint, decides access.
|
|
1615
|
-
service = { status: 'warning', detail: `Service is reachable but restricted GET /models (HTTP 403); the credential may still be valid for real vision requests` };
|
|
1616
|
-
}
|
|
1617
|
-
else if (response.status === 404 || response.status === 405) {
|
|
1618
|
-
service = { status: 'warning', detail: `Service is reachable but does not expose GET /models (HTTP ${response.status})` };
|
|
1619
|
-
}
|
|
1620
|
-
else if (response.status === 429) {
|
|
1621
|
-
service = { status: 'warning', detail: 'Service is reachable but rate-limited the connection test (HTTP 429)' };
|
|
1622
|
-
}
|
|
1623
|
-
else {
|
|
1624
|
-
service = { status: 'error', detail: `Service connection test failed with HTTP ${response.status}` };
|
|
1625
|
-
}
|
|
1723
|
+
const target = provider ?? (testConnection || testModel ? this.primaryProvider : undefined);
|
|
1724
|
+
if (target !== undefined && (testConnection || testModel)) {
|
|
1725
|
+
const entry = await this.resolveProviderEnv(target);
|
|
1726
|
+
if (entry === undefined) {
|
|
1727
|
+
if (testConnection) {
|
|
1728
|
+
service = { status: 'error', detail: `Connection test skipped because credential ${String(target.credential)} is unavailable` };
|
|
1626
1729
|
}
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
throw new VisionToolkitError('cancelled', 'vision_toolkit_health: connection test cancelled');
|
|
1630
|
-
service = { status: 'error', detail: `Service could not be reached at ${endpoint}` };
|
|
1730
|
+
if (testModel) {
|
|
1731
|
+
model = { status: 'error', detail: `Vision model test skipped because credential ${String(target.credential)} is unavailable` };
|
|
1631
1732
|
}
|
|
1632
1733
|
}
|
|
1633
|
-
}
|
|
1634
|
-
if (testModel) {
|
|
1635
|
-
if (resolvedCredential === undefined) {
|
|
1636
|
-
model = { status: 'error', detail: 'Vision model test skipped because the configured credential is unavailable' };
|
|
1637
|
-
}
|
|
1638
1734
|
else {
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1735
|
+
if (testConnection) {
|
|
1736
|
+
operation.metrics.usedVisionService = true;
|
|
1737
|
+
const endpoint = `${target.baseUrl}/models`;
|
|
1738
|
+
try {
|
|
1739
|
+
const started = Date.now();
|
|
1740
|
+
const headers = {
|
|
1741
|
+
Accept: 'application/json',
|
|
1742
|
+
'User-Agent': target.userAgent,
|
|
1743
|
+
};
|
|
1744
|
+
if (target.protocol === 'anthropic') {
|
|
1745
|
+
headers['x-api-key'] = entry.env.VISION_API_KEY;
|
|
1746
|
+
headers['anthropic-version'] = '2023-06-01';
|
|
1747
|
+
}
|
|
1748
|
+
else {
|
|
1749
|
+
headers.Authorization = `Bearer ${entry.env.VISION_API_KEY}`;
|
|
1750
|
+
}
|
|
1751
|
+
const response = await fetch(endpoint, {
|
|
1752
|
+
method: 'GET',
|
|
1753
|
+
headers,
|
|
1754
|
+
signal: operation.signal,
|
|
1755
|
+
});
|
|
1756
|
+
operation.metrics.upstreamMs += Date.now() - started;
|
|
1757
|
+
await response.body?.cancel().catch(() => { });
|
|
1758
|
+
if (response.ok) {
|
|
1759
|
+
service = { status: 'ok', detail: `Service responded at ${endpoint} (HTTP ${response.status})` };
|
|
1760
|
+
}
|
|
1761
|
+
else if (response.status === 401) {
|
|
1762
|
+
service = { status: 'error', detail: `Service rejected the configured credential (HTTP ${response.status})` };
|
|
1763
|
+
}
|
|
1764
|
+
else if (response.status === 403) {
|
|
1765
|
+
// Some providers (e.g. Groq preview/account restrictions) block GET /models
|
|
1766
|
+
// while real multimodal requests still work. Treat 403 as a warning so the
|
|
1767
|
+
// explicit vision-model test, not the model list endpoint, decides access.
|
|
1768
|
+
service = { status: 'warning', detail: `Service is reachable but restricted GET /models (HTTP 403); the credential may still be valid for real vision requests` };
|
|
1769
|
+
}
|
|
1770
|
+
else if (response.status === 404 || response.status === 405) {
|
|
1771
|
+
service = { status: 'warning', detail: `Service is reachable but does not expose GET /models (HTTP ${response.status})` };
|
|
1772
|
+
}
|
|
1773
|
+
else if (response.status === 429) {
|
|
1774
|
+
service = { status: 'warning', detail: 'Service is reachable but rate-limited the connection test (HTTP 429)' };
|
|
1775
|
+
}
|
|
1776
|
+
else {
|
|
1777
|
+
service = { status: 'error', detail: `Service connection test failed with HTTP ${response.status}` };
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
catch {
|
|
1781
|
+
if (operation.signal.aborted)
|
|
1782
|
+
throw new VisionToolkitError('cancelled', 'vision_toolkit_health: connection test cancelled');
|
|
1783
|
+
service = { status: 'error', detail: `Service could not be reached at ${endpoint}` };
|
|
1643
1784
|
}
|
|
1644
|
-
model = {
|
|
1645
|
-
status: 'ok',
|
|
1646
|
-
detail: `Vision model ${this.config.provider.model} completed a multimodal request`,
|
|
1647
|
-
};
|
|
1648
1785
|
}
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1786
|
+
if (testModel) {
|
|
1787
|
+
try {
|
|
1788
|
+
const attemptDeadline = createDeadline(operation.signal, target.timeoutMs);
|
|
1789
|
+
try {
|
|
1790
|
+
const result = await this.runUpstream('glance', [VISION_MODEL_TEST_IMAGE, '-q', VISION_MODEL_TEST_PROMPT], { signal: attemptDeadline.signal, metrics: operation.metrics }, entry.env);
|
|
1791
|
+
if (result.stdout.trim().length === 0) {
|
|
1792
|
+
throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
|
|
1793
|
+
}
|
|
1794
|
+
model = {
|
|
1795
|
+
status: 'ok',
|
|
1796
|
+
detail: `Vision model ${target.model} completed a multimodal request`,
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
finally {
|
|
1800
|
+
attemptDeadline.cleanup();
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
catch (error) {
|
|
1804
|
+
if (operation.signal.aborted)
|
|
1805
|
+
throw error;
|
|
1806
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1807
|
+
model = { status: 'error', detail: `Vision model test failed: ${detail.slice(0, 600)}` };
|
|
1808
|
+
}
|
|
1654
1809
|
}
|
|
1655
1810
|
}
|
|
1656
1811
|
}
|
|
1657
|
-
const checks = { python, dependencies, chrome,
|
|
1812
|
+
const checks = { python, dependencies, chrome, service, model };
|
|
1658
1813
|
const healthy = Object.values(checks).every(check => check.status !== 'error');
|
|
1659
1814
|
return {
|
|
1660
1815
|
pluginVersion: PLUGIN_VERSION,
|
|
@@ -1663,6 +1818,7 @@ export class VisionToolkitRuntime {
|
|
|
1663
1818
|
healthy,
|
|
1664
1819
|
connectionTested: testConnection,
|
|
1665
1820
|
modelTested: testModel,
|
|
1821
|
+
...(provider === undefined ? {} : { providerName: provider.name }),
|
|
1666
1822
|
};
|
|
1667
1823
|
});
|
|
1668
1824
|
}
|