@mengruo/dsh-vision-toolkit 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.i18n.yaml +2 -2
  2. package/README.md +17 -42
  3. package/README.zh.md +16 -41
  4. package/assets/logo_eapi_dark.png +0 -0
  5. package/docs/aihubmix-gemini-vision.i18n.yaml +2 -2
  6. package/docs/aihubmix-gemini-vision.md +2 -2
  7. package/docs/aihubmix-gemini-vision.zh.md +2 -2
  8. package/lib/client.js +246 -68
  9. package/lib/client.js.map +1 -1
  10. package/lib/config.js +143 -24
  11. package/lib/config.js.map +1 -1
  12. package/lib/evidence-cache.js +14 -0
  13. package/lib/evidence-cache.js.map +1 -1
  14. package/lib/image-input-variants.js +39 -27
  15. package/lib/image-input-variants.js.map +1 -1
  16. package/lib/runtime-install.js +204 -19
  17. package/lib/runtime-install.js.map +1 -1
  18. package/lib/runtime.js +310 -154
  19. package/lib/runtime.js.map +1 -1
  20. package/lib/tools.js +1 -0
  21. package/lib/tools.js.map +1 -1
  22. package/lib/types/client/index.d.ts +45 -3
  23. package/lib/types/client/index.d.ts.map +1 -1
  24. package/lib/types/config.d.ts +53 -0
  25. package/lib/types/config.d.ts.map +1 -1
  26. package/lib/types/evidence-cache.d.ts.map +1 -1
  27. package/lib/types/image-input-variants.d.ts +1 -6
  28. package/lib/types/image-input-variants.d.ts.map +1 -1
  29. package/lib/types/runtime-install.d.ts +21 -0
  30. package/lib/types/runtime-install.d.ts.map +1 -1
  31. package/lib/types/runtime.d.ts +45 -10
  32. package/lib/types/runtime.d.ts.map +1 -1
  33. package/lib/types/tools.d.ts.map +1 -1
  34. package/lib/types/upstream.d.ts +3 -0
  35. package/lib/types/upstream.d.ts.map +1 -1
  36. package/lib/types/web.d.ts +8 -0
  37. package/lib/types/web.d.ts.map +1 -1
  38. package/lib/upstream.js +11 -6
  39. package/lib/upstream.js.map +1 -1
  40. package/lib/web.js +50 -7
  41. package/lib/web.js.map +1 -1
  42. package/package.json +1 -1
  43. package/src/client/index.tsx +331 -88
  44. package/src/config.ts +210 -28
  45. package/src/evidence-cache.ts +14 -0
  46. package/src/image-input-variants.ts +39 -27
  47. package/src/runtime-install.ts +231 -20
  48. package/src/runtime.ts +333 -180
  49. package/src/tools.ts +1 -0
  50. package/src/upstream.ts +15 -8
  51. package/src/web.ts +67 -8
package/lib/runtime.js CHANGED
@@ -149,6 +149,16 @@ export class Semaphore {
149
149
  next.resolve();
150
150
  }
151
151
  }
152
+ /** Non-blocking acquisition: claim a free slot immediately, else return false. */
153
+ tryAcquire(permits = 1) {
154
+ if (!Number.isInteger(permits) || permits < 1 || permits > this.limit)
155
+ return false;
156
+ if (this.waiters.length === 0 && this.active + permits <= this.limit) {
157
+ this.active += permits;
158
+ return true;
159
+ }
160
+ return false;
161
+ }
152
162
  }
153
163
  const REGION_PATTERN = /^\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*,\s*(-?\d+)\s*$/;
154
164
  const MAX_TIMEOUT_MS = 600_000;
@@ -301,6 +311,7 @@ export class VisionToolkitRuntime {
301
311
  config;
302
312
  semaphores = new Map();
303
313
  glanceCache = new WeakMap();
314
+ providerGates = new Map();
304
315
  adapter;
305
316
  constructor(ctx, config, adapter) {
306
317
  this.ctx = ctx;
@@ -317,20 +328,37 @@ export class VisionToolkitRuntime {
317
328
  }
318
329
  /** Capture the credential and provider identity used by one evidence conversion. */
319
330
  async captureEvidenceRuntime() {
320
- const env = await this.resolveVisionEnv();
321
- const evidenceFingerprint = evidenceRuntimeFingerprint(this.config, createHash('sha256').update(env.VISION_API_KEY).digest('hex'), env.VISION_SSL_VERIFY);
331
+ // The primary provider's key hash sharpens cache invalidation, but a
332
+ // missing primary credential must not block evidence conversion when a
333
+ // later provider in the failover pool is available.
334
+ let credentialSha256;
335
+ let sslVerify;
336
+ try {
337
+ const env = await this.resolveVisionEnv();
338
+ credentialSha256 = createHash('sha256').update(env.VISION_API_KEY).digest('hex');
339
+ sslVerify = env.VISION_SSL_VERIFY;
340
+ }
341
+ catch {
342
+ credentialSha256 = undefined;
343
+ }
344
+ const evidenceFingerprint = evidenceRuntimeFingerprint(this.config, credentialSha256, sslVerify);
322
345
  return Object.freeze({
323
346
  evidenceFingerprint,
324
- glance: (request, options) => this.glanceWithEnv(request, options, env),
347
+ glance: (request, options) => this.glanceWithEnv(request, options),
325
348
  });
326
349
  }
327
350
  timeout(options) {
328
- const value = options.timeoutMs ?? this.config.timeoutMs;
351
+ const value = options.timeoutMs ?? this.operationTimeoutMs();
329
352
  if (!Number.isInteger(value) || value < 1000 || value > MAX_TIMEOUT_MS) {
330
353
  throw new VisionToolkitError('input', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`);
331
354
  }
332
355
  return value;
333
356
  }
357
+ /** Overall operation budget: the slowest enabled provider's timeout, else the global default. */
358
+ operationTimeoutMs() {
359
+ const providers = this.config.providers.filter(provider => provider.enabled);
360
+ return providers.length === 0 ? this.config.timeoutMs : Math.max(...providers.map(provider => provider.timeoutMs));
361
+ }
334
362
  operationError(tool, error, deadline, phase = 'execution') {
335
363
  if (deadline.cancelled) {
336
364
  return new VisionToolkitError('cancelled', phase === 'queue' ? `${tool}: cancelled while waiting for a concurrency slot` : `${tool}: cancelled`);
@@ -408,29 +436,69 @@ export class VisionToolkitRuntime {
408
436
  this.semaphores.delete(semaphore.key);
409
437
  }
410
438
  }
411
- /** Resolve the configured credential at the remote-operation boundary. */
412
- async resolveVisionEnv() {
413
- const resolved = isBuiltInFreeVisionProvider(this.config.provider)
414
- ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
415
- : await this.ctx.credentials.resolve(this.config.provider.credential);
416
- if (resolved === undefined) {
417
- throw new VisionToolkitError('config', `credential ${this.config.provider.credential} is not configured; set it through DSH credentials`);
418
- }
419
- return this.visionEnv(resolved);
439
+ /** Highest-priority enabled provider, falling back to the first entry. */
440
+ get primaryProvider() {
441
+ return this.config.providers.find(provider => provider.enabled) ?? this.config.providers[0];
420
442
  }
421
- visionEnv(resolved) {
443
+ /** Build the upstream environment for one resolved provider. */
444
+ providerEnv(provider, resolved) {
422
445
  const sslVerify = process.env.VISION_SSL_VERIFY?.trim();
423
446
  return {
424
447
  VISION_API_KEY: resolved.value,
425
- VISION_BASE_URL: this.config.provider.baseUrl,
426
- VISION_MODEL: this.config.provider.model,
427
- VISION_API_PROTOCOL: this.config.provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
428
- VISION_ANTHROPIC_THINKING: this.config.provider.anthropicThinking,
448
+ VISION_BASE_URL: provider.baseUrl,
449
+ VISION_MODEL: provider.model,
450
+ VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
451
+ VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
429
452
  ...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
430
- VISION_USER_AGENT: this.config.provider.userAgent,
453
+ VISION_USER_AGENT: provider.userAgent,
431
454
  LANG: this.config.language,
432
455
  };
433
456
  }
457
+ /** Resolve one provider's credential into its environment, or undefined when unavailable. */
458
+ async resolveProviderEnv(provider) {
459
+ let resolved;
460
+ try {
461
+ resolved = isBuiltInFreeVisionProvider({
462
+ baseUrl: provider.baseUrl,
463
+ credential: provider.credential,
464
+ model: provider.model,
465
+ protocol: provider.protocol,
466
+ anthropicThinking: provider.anthropicThinking,
467
+ userAgent: provider.userAgent,
468
+ })
469
+ ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
470
+ : await this.ctx.credentials.resolve(provider.credential);
471
+ }
472
+ catch {
473
+ resolved = undefined;
474
+ }
475
+ if (resolved === undefined)
476
+ return undefined;
477
+ return { provider, env: this.providerEnv(provider, resolved) };
478
+ }
479
+ /** Resolve every enabled provider in priority order, skipping unreadable credentials. */
480
+ async resolveProviderPool() {
481
+ const pool = [];
482
+ for (const provider of this.config.providers) {
483
+ if (!provider.enabled)
484
+ continue;
485
+ const entry = await this.resolveProviderEnv(provider);
486
+ if (entry === undefined) {
487
+ this.ctx.logger.warn('dsh-vision-toolkit provider=%s credential=%s unavailable; skipped from the failover pool', provider.name, String(provider.credential));
488
+ continue;
489
+ }
490
+ pool.push(entry);
491
+ }
492
+ return pool;
493
+ }
494
+ /** Resolve the primary provider's credential at the remote-operation boundary. */
495
+ async resolveVisionEnv() {
496
+ const entry = await this.resolveProviderEnv(this.primaryProvider);
497
+ if (entry === undefined) {
498
+ throw new VisionToolkitError('config', `credential ${String(this.primaryProvider.credential)} is not configured; set it through DSH credentials`);
499
+ }
500
+ return entry.env;
501
+ }
434
502
  pathPolicy(workspace) {
435
503
  return createPathPolicy(workspace, this.config.allowedDirs);
436
504
  }
@@ -503,7 +571,7 @@ export class VisionToolkitRuntime {
503
571
  || probed.width * probed.height > maxPixels) {
504
572
  return undefined;
505
573
  }
506
- return { path: real, bytes: bytes.length, width: probed.width, height: probed.height, format: probed.format };
574
+ return { path: real, bytes: bytes.length, width: probed.width, height: probed.height, format: probed.format, hasAlpha: probed.hasAlpha };
507
575
  }
508
576
  cacheEntryOutDigest(entry, prefix) {
509
577
  const tail = entry.slice(prefix.length + 1);
@@ -560,7 +628,7 @@ export class VisionToolkitRuntime {
560
628
  }
561
629
  await Promise.all([...stalePartials, ...remove].map(name => rm(join(root, name), { force: true }).catch(() => { })));
562
630
  }
563
- async autoCompressImage(image, policy, operation) {
631
+ async autoCompressImage(image, policy, operation, maxBytes, maxPixels) {
564
632
  let bytes;
565
633
  try {
566
634
  bytes = await readFile(image.path, { signal: operation.signal });
@@ -574,7 +642,7 @@ export class VisionToolkitRuntime {
574
642
  const digest = createHash('sha256').update(bytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH);
575
643
  const root = await this.compressedImageRoot(policy);
576
644
  await this.pruneCompressedCache(root);
577
- const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${this.config.maxImageBytes}-p${this.config.maxImagePixels}`;
645
+ const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${maxBytes}-p${maxPixels}`;
578
646
  for (const entry of await readdir(root)) {
579
647
  if (!entry.startsWith(`${prefix}-`) || entry.startsWith('.'))
580
648
  continue;
@@ -583,7 +651,7 @@ export class VisionToolkitRuntime {
583
651
  await rm(join(root, entry), { force: true }).catch(() => { });
584
652
  continue;
585
653
  }
586
- const cached = await this.readCacheCandidate(root, entry, outDigestPrefix, this.config.maxImageBytes, this.config.maxImagePixels, operation);
654
+ const cached = await this.readCacheCandidate(root, entry, outDigestPrefix, maxBytes, maxPixels, operation);
587
655
  if (cached !== undefined) {
588
656
  return { ...cached, originalPath: image.path };
589
657
  }
@@ -592,7 +660,7 @@ export class VisionToolkitRuntime {
592
660
  const staged = join(root, `.${prefix}-${randomUUID()}.partial`);
593
661
  let compressed;
594
662
  try {
595
- compressed = await this.adapter.compressImage(image.path, staged, this.config.maxImageBytes, this.config.maxImagePixels, { signal: operation.signal });
663
+ compressed = await this.adapter.compressImage(image.path, staged, maxBytes, maxPixels, { signal: operation.signal });
596
664
  }
597
665
  catch (error) {
598
666
  await rm(staged, { force: true }).catch(() => { });
@@ -603,7 +671,7 @@ export class VisionToolkitRuntime {
603
671
  const outDigest = createHash('sha256').update(stagedBytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH);
604
672
  const finalName = `${prefix}-${outDigest}-${compressed.width}x${compressed.height}.${extension}`;
605
673
  const finalPath = join(root, finalName);
606
- const existing = await this.readCacheCandidate(root, finalName, outDigest, this.config.maxImageBytes, this.config.maxImagePixels, operation);
674
+ const existing = await this.readCacheCandidate(root, finalName, outDigest, maxBytes, maxPixels, operation);
607
675
  if (existing !== undefined) {
608
676
  await rm(staged, { force: true }).catch(() => { });
609
677
  return { ...existing, originalPath: image.path };
@@ -623,9 +691,11 @@ export class VisionToolkitRuntime {
623
691
  width: compressed.width,
624
692
  height: compressed.height,
625
693
  format: compressed.format,
694
+ hasAlpha: compressed.hasAlpha,
626
695
  originalPath: image.path,
627
696
  };
628
697
  }
698
+ /** Validate one image against the configured global limits (used by local tools). */
629
699
  async validateImage(raw, policy, operation) {
630
700
  const image = await resolveInputFile(raw, policy);
631
701
  const decoded = await this.adapter.probeImageSize(image.path, { signal: operation.signal });
@@ -639,16 +709,106 @@ export class VisionToolkitRuntime {
639
709
  throw new VisionToolkitError('input', `image content is ${decoded.format}, but the filename uses ${extension}`);
640
710
  }
641
711
  if (image.bytes <= this.config.maxImageBytes && pixels <= this.config.maxImagePixels) {
642
- return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, originalPath: image.path };
712
+ return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, hasAlpha: decoded.hasAlpha, originalPath: image.path };
643
713
  }
644
- return this.autoCompressImage(image, policy, operation);
714
+ return this.autoCompressImage(image, policy, operation, this.config.maxImageBytes, this.config.maxImagePixels);
645
715
  }
646
716
  accountImage(image, operation) {
647
717
  operation.metrics.imageCount += 1;
648
718
  operation.metrics.imageBytes += image.bytes;
649
719
  operation.metrics.imagePixels += image.width * image.height;
650
720
  }
651
- async glanceCacheKey(request, images, env, signal) {
721
+ /** Stable gate key for one provider's in-flight request cap. */
722
+ providerGate(provider) {
723
+ const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`;
724
+ let gate = this.providerGates.get(key);
725
+ if (gate === undefined) {
726
+ gate = new Semaphore(provider.concurrency);
727
+ this.providerGates.set(key, gate);
728
+ }
729
+ return gate;
730
+ }
731
+ /**
732
+ * Prepare one image for an online vision request against the enabled
733
+ * provider pool. The raw image is kept when at least one enabled provider
734
+ * accepts it; otherwise it is compressed once to the first (highest
735
+ * priority) provider's limits so the priority route can proceed.
736
+ */
737
+ async prepareVisionImage(raw, providers, policy, operation) {
738
+ const image = await resolveInputFile(raw, policy);
739
+ const decoded = await this.adapter.probeImageSize(image.path, { signal: operation.signal });
740
+ const pixels = decoded.width * decoded.height;
741
+ if (!Number.isSafeInteger(pixels) || pixels < 1) {
742
+ throw new VisionToolkitError('input', `image dimensions are invalid: ${decoded.width}x${decoded.height}`);
743
+ }
744
+ const extension = extname(image.path).toLowerCase();
745
+ const expected = FORMAT_BY_EXTENSION.get(extension);
746
+ if (expected !== decoded.format) {
747
+ throw new VisionToolkitError('input', `image content is ${decoded.format}, but the filename uses ${extension}`);
748
+ }
749
+ const fits = providers.some(provider => image.bytes <= provider.maxImageBytes && pixels <= provider.maxImagePixels);
750
+ if (fits) {
751
+ return { ...image, width: decoded.width, height: decoded.height, format: decoded.format, hasAlpha: decoded.hasAlpha, originalPath: image.path };
752
+ }
753
+ const first = providers[0];
754
+ if (first === undefined) {
755
+ throw new VisionToolkitError('config', 'no enabled vision provider is available for this image');
756
+ }
757
+ return this.autoCompressImage(image, policy, operation, first.maxImageBytes, first.maxImagePixels);
758
+ }
759
+ /**
760
+ * Run one online-vision upstream command across the enabled provider pool in
761
+ * priority order. A provider is skipped when its size limits or concurrency
762
+ * are exhausted, retried up to its attempt count, and the next provider
763
+ * takes over on failure. Throws once every provider has failed.
764
+ */
765
+ async runVisionWithFailover(tool, args, images, operation, pool) {
766
+ if (pool.length === 0) {
767
+ throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
768
+ }
769
+ let lastError;
770
+ let attempted = false;
771
+ for (const { provider, env } of pool) {
772
+ const fits = images.every(image => image.bytes <= provider.maxImageBytes && image.width * image.height <= provider.maxImagePixels);
773
+ if (!fits)
774
+ continue;
775
+ const gate = this.providerGate(provider);
776
+ if (!gate.tryAcquire())
777
+ continue;
778
+ attempted = true;
779
+ try {
780
+ for (let attempt = 1; attempt <= provider.attempts; attempt++) {
781
+ const attemptDeadline = createDeadline(operation.signal, provider.timeoutMs);
782
+ try {
783
+ return await this.runUpstream(tool, args, { signal: attemptDeadline.signal, metrics: operation.metrics }, env);
784
+ }
785
+ catch (error) {
786
+ if (operation.signal.aborted)
787
+ throw error;
788
+ const classified = error instanceof VisionToolkitError
789
+ ? error
790
+ : new VisionToolkitError('service', `${tool}: request failed`, { cause: error });
791
+ lastError = attemptDeadline.timedOut && classified.code === 'cancelled'
792
+ ? new VisionToolkitError('timeout', `${tool}: ${provider.name} request timed out after ${provider.timeoutMs}ms`, { cause: classified })
793
+ : classified;
794
+ }
795
+ finally {
796
+ attemptDeadline.cleanup();
797
+ }
798
+ }
799
+ }
800
+ finally {
801
+ gate.release();
802
+ }
803
+ }
804
+ if (!attempted) {
805
+ throw new VisionToolkitError('capacity', `${tool}: no enabled vision provider accepts the image size or has a free concurrency slot`);
806
+ }
807
+ if (lastError instanceof VisionToolkitError)
808
+ throw lastError;
809
+ throw new VisionToolkitError('service', `${tool}: all vision providers failed`, { cause: lastError });
810
+ }
811
+ async glanceCacheKey(request, images, pool, signal) {
652
812
  const imageFingerprints = await Promise.all(images.map(async (image) => {
653
813
  let bytes;
654
814
  try {
@@ -670,16 +830,20 @@ export class VisionToolkitRuntime {
670
830
  query: request.query ?? null,
671
831
  ocr: request.ocr === true,
672
832
  region: request.region ?? null,
673
- provider: {
833
+ language: this.config.language,
834
+ providers: pool.map(({ provider, env }) => ({
835
+ name: provider.name,
674
836
  baseUrl: env.VISION_BASE_URL,
675
837
  model: env.VISION_MODEL,
676
838
  protocol: env.VISION_API_PROTOCOL,
677
839
  anthropicThinking: env.VISION_ANTHROPIC_THINKING,
678
840
  sslVerify: env.VISION_SSL_VERIFY ?? null,
679
841
  userAgent: env.VISION_USER_AGENT,
680
- language: env.LANG,
681
842
  credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
682
- },
843
+ maxImageBytes: provider.maxImageBytes,
844
+ maxImagePixels: provider.maxImagePixels,
845
+ attempts: provider.attempts,
846
+ })),
683
847
  });
684
848
  }
685
849
  async runUpstream(tool, args, operation, env) {
@@ -746,7 +910,7 @@ export class VisionToolkitRuntime {
746
910
  async glance(request, options) {
747
911
  return this.glanceWithEnv(request, options);
748
912
  }
749
- async glanceWithEnv(request, options, capturedEnv) {
913
+ async glanceWithEnv(request, options) {
750
914
  return this.runOperation('vision_glance', options, async (operation) => {
751
915
  if (request.images.length === 0)
752
916
  throw new VisionToolkitError('input', 'glance requires at least one image');
@@ -759,10 +923,15 @@ export class VisionToolkitRuntime {
759
923
  if (request.region !== undefined)
760
924
  parseRegion(request.region);
761
925
  const policy = await this.pathPolicy(options.workspace);
926
+ const pool = await this.resolveProviderPool();
927
+ if (pool.length === 0) {
928
+ throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
929
+ }
930
+ const providers = pool.map(entry => entry.provider);
762
931
  const images = [];
763
932
  const seen = new Set();
764
933
  for (const raw of request.images) {
765
- const image = await this.validateImage(raw, policy, operation);
934
+ const image = await this.prepareVisionImage(raw, providers, policy, operation);
766
935
  if (seen.has(image.path)) {
767
936
  operation.metrics.cacheHits += 1;
768
937
  continue;
@@ -771,10 +940,9 @@ export class VisionToolkitRuntime {
771
940
  this.accountImage(image, operation);
772
941
  images.push(image);
773
942
  }
774
- const env = capturedEnv ?? await this.resolveVisionEnv();
775
943
  const cacheKey = options.sessionScope === undefined
776
944
  ? undefined
777
- : await this.glanceCacheKey(request, images, env, operation.signal);
945
+ : await this.glanceCacheKey(request, images, pool, operation.signal);
778
946
  if (options.sessionScope !== undefined && cacheKey !== undefined) {
779
947
  const cached = this.glanceCache.get(options.sessionScope);
780
948
  if (cached?.key === cacheKey) {
@@ -782,12 +950,12 @@ export class VisionToolkitRuntime {
782
950
  return cached.result;
783
951
  }
784
952
  }
785
- const result = await this.runUpstream('glance', [
953
+ const result = await this.runVisionWithFailover('glance', [
786
954
  ...images.map(image => image.path),
787
955
  ...(request.region !== undefined ? ['--region', request.region] : []),
788
956
  ...(request.ocr === true ? ['--ocr'] : []),
789
957
  ...(request.query !== undefined ? ['-q', request.query] : []),
790
- ], operation, env);
958
+ ], images, operation, pool);
791
959
  const answer = result.stdout.trim();
792
960
  if (answer.length === 0)
793
961
  throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
@@ -823,14 +991,17 @@ export class VisionToolkitRuntime {
823
991
  if (request.region !== undefined)
824
992
  parseRegion(request.region);
825
993
  const policy = await this.pathPolicy(options.workspace);
826
- const image = await this.validateImage(request.image, policy, operation);
994
+ const pool = await this.resolveProviderPool();
995
+ if (pool.length === 0) {
996
+ throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
997
+ }
998
+ const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation);
827
999
  this.accountImage(image, operation);
828
- const env = await this.resolveVisionEnv();
829
- const result = await this.runUpstream(tool, [
1000
+ const result = await this.runVisionWithFailover(tool, [
830
1001
  image.path,
831
1002
  request.target,
832
1003
  ...(request.region !== undefined ? ['--region', request.region] : []),
833
- ], operation, env);
1004
+ ], [image], operation, pool);
834
1005
  const elements = parseLocationOutput(result.stdout);
835
1006
  this.validateLocations(elements, image.width, image.height);
836
1007
  return { image, elements };
@@ -1142,7 +1313,13 @@ export class VisionToolkitRuntime {
1142
1313
  throw new VisionToolkitError('input', 'long_screenshot_ocr.prompt must not be empty when provided');
1143
1314
  }
1144
1315
  const policy = await this.pathPolicy(options.workspace);
1145
- const image = await this.validateImage(request.image, policy, operation);
1316
+ const pool = splitOnly ? [] : await this.resolveProviderPool();
1317
+ if (!splitOnly && pool.length === 0) {
1318
+ throw new VisionToolkitError('config', 'no enabled vision provider has a resolvable credential');
1319
+ }
1320
+ const image = splitOnly
1321
+ ? await this.validateImage(request.image, policy, operation)
1322
+ : await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation);
1146
1323
  this.accountImage(image, operation);
1147
1324
  const stem = basename(image.originalPath, extname(image.originalPath));
1148
1325
  const finalDirectory = resolveOutputDirectory(request.runName, policy, `${stem}.long-ocr`);
@@ -1158,7 +1335,7 @@ export class VisionToolkitRuntime {
1158
1335
  const finalOutput = join(finalDirectory, basename(stagedOutput));
1159
1336
  const stagedChunks = join(stagedDirectory, 'chunks');
1160
1337
  const stagedManifest = join(stagedChunks, 'manifest.json');
1161
- const result = await this.runUpstream('long_screenshot_ocr', [
1338
+ const ocrArgs = [
1162
1339
  image.path,
1163
1340
  '--mode',
1164
1341
  mode,
@@ -1177,7 +1354,10 @@ export class VisionToolkitRuntime {
1177
1354
  String(chunkTimeoutSeconds),
1178
1355
  ...(splitOnly ? ['--split-only'] : []),
1179
1356
  ...(request.resume === true ? ['--resume'] : []),
1180
- ], operation, splitOnly ? undefined : await this.resolveVisionEnv());
1357
+ ];
1358
+ const result = splitOnly
1359
+ ? await this.runUpstream('long_screenshot_ocr', ocrArgs, operation)
1360
+ : await this.runVisionWithFailover('long_screenshot_ocr', ocrArgs, [image], operation, pool);
1181
1361
  const reported = result.stdout.trim();
1182
1362
  const expectedReported = splitOnly ? stagedManifest : stagedOutput;
1183
1363
  if (reported !== expectedReported) {
@@ -1509,20 +1689,8 @@ export class VisionToolkitRuntime {
1509
1689
  }
1510
1690
  });
1511
1691
  }
1512
- async writableDirectoryCheck(path, label) {
1513
- const probe = join(path, `.vision-toolkit-health-${randomUUID()}`);
1514
- try {
1515
- await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' });
1516
- await rm(probe, { force: true });
1517
- return { status: 'ok', detail: `${label} is writable: ${path}` };
1518
- }
1519
- catch {
1520
- await rm(probe, { force: true }).catch(() => { });
1521
- return { status: 'error', detail: `${label} is not writable: ${path}` };
1522
- }
1523
- }
1524
- /** Health: inspect local readiness, optionally probe `/models`, and explicitly test one real multimodal request. */
1525
- async health(testConnection, options, testModel = false) {
1692
+ /** Health: inspect local readiness, and optionally probe one provider's `/models` plus one real multimodal request. */
1693
+ async health(testConnection, options, testModel = false, provider) {
1526
1694
  return this.runOperation('vision_toolkit_health', options, async (operation) => {
1527
1695
  const info = this.upstreamVersion;
1528
1696
  const python = { status: 'ok', detail: `${info.pythonVersion} via ${info.python}` };
@@ -1544,117 +1712,104 @@ export class VisionToolkitRuntime {
1544
1712
  throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled');
1545
1713
  chrome = { status: 'error', detail: 'Chrome availability probe failed' };
1546
1714
  }
1547
- let resolvedCredential;
1548
- let credential;
1549
- try {
1550
- resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
1551
- ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
1552
- : await this.ctx.credentials.resolve(this.config.provider.credential);
1553
- credential = resolvedCredential === undefined
1554
- ? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
1555
- : { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` };
1556
- }
1557
- catch {
1558
- credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` };
1559
- }
1560
- let artifactDirectory;
1561
- try {
1562
- // allowedDirs are session input roots; they do not affect output readiness.
1563
- const policy = await createPathPolicy(options.workspace, []);
1564
- artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory');
1565
- }
1566
- catch {
1567
- artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' };
1568
- }
1569
- const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory');
1570
1715
  let service = {
1571
1716
  status: 'not_tested',
1572
- detail: 'Connection was not tested; pass testConnection=true to query the configured /models endpoint',
1717
+ detail: 'Connection was not tested; use the per-provider API test',
1573
1718
  };
1574
1719
  let model = {
1575
1720
  status: 'not_tested',
1576
- detail: 'Vision model was not tested; run an explicit model test to send the bundled diagnostic image',
1721
+ detail: 'Vision model was not tested; use the per-provider model test',
1577
1722
  };
1578
- if (testConnection) {
1579
- if (resolvedCredential === undefined) {
1580
- service = { status: 'error', detail: 'Connection test skipped because the configured credential is unavailable' };
1581
- }
1582
- else {
1583
- operation.metrics.usedVisionService = true;
1584
- const endpoint = `${this.config.provider.baseUrl}/models`;
1585
- try {
1586
- const started = Date.now();
1587
- const headers = {
1588
- Accept: 'application/json',
1589
- 'User-Agent': this.config.provider.userAgent,
1590
- };
1591
- if (this.config.provider.protocol === 'anthropic') {
1592
- headers['x-api-key'] = resolvedCredential.value;
1593
- headers['anthropic-version'] = '2023-06-01';
1594
- }
1595
- else {
1596
- headers.Authorization = `Bearer ${resolvedCredential.value}`;
1597
- }
1598
- const response = await fetch(endpoint, {
1599
- method: 'GET',
1600
- headers,
1601
- signal: operation.signal,
1602
- });
1603
- operation.metrics.upstreamMs += Date.now() - started;
1604
- await response.body?.cancel().catch(() => { });
1605
- if (response.ok) {
1606
- service = { status: 'ok', detail: `Service responded at ${endpoint} (HTTP ${response.status})` };
1607
- }
1608
- else if (response.status === 401) {
1609
- service = { status: 'error', detail: `Service rejected the configured credential (HTTP ${response.status})` };
1610
- }
1611
- else if (response.status === 403) {
1612
- // Some providers (e.g. Groq preview/account restrictions) block GET /models
1613
- // while real multimodal requests still work. Treat 403 as a warning so the
1614
- // explicit vision-model test, not the model list endpoint, decides access.
1615
- service = { status: 'warning', detail: `Service is reachable but restricted GET /models (HTTP 403); the credential may still be valid for real vision requests` };
1616
- }
1617
- else if (response.status === 404 || response.status === 405) {
1618
- service = { status: 'warning', detail: `Service is reachable but does not expose GET /models (HTTP ${response.status})` };
1619
- }
1620
- else if (response.status === 429) {
1621
- service = { status: 'warning', detail: 'Service is reachable but rate-limited the connection test (HTTP 429)' };
1622
- }
1623
- else {
1624
- service = { status: 'error', detail: `Service connection test failed with HTTP ${response.status}` };
1625
- }
1723
+ const target = provider ?? (testConnection || testModel ? this.primaryProvider : undefined);
1724
+ if (target !== undefined && (testConnection || testModel)) {
1725
+ const entry = await this.resolveProviderEnv(target);
1726
+ if (entry === undefined) {
1727
+ if (testConnection) {
1728
+ service = { status: 'error', detail: `Connection test skipped because credential ${String(target.credential)} is unavailable` };
1626
1729
  }
1627
- catch {
1628
- if (operation.signal.aborted)
1629
- throw new VisionToolkitError('cancelled', 'vision_toolkit_health: connection test cancelled');
1630
- service = { status: 'error', detail: `Service could not be reached at ${endpoint}` };
1730
+ if (testModel) {
1731
+ model = { status: 'error', detail: `Vision model test skipped because credential ${String(target.credential)} is unavailable` };
1631
1732
  }
1632
1733
  }
1633
- }
1634
- if (testModel) {
1635
- if (resolvedCredential === undefined) {
1636
- model = { status: 'error', detail: 'Vision model test skipped because the configured credential is unavailable' };
1637
- }
1638
1734
  else {
1639
- try {
1640
- const result = await this.runUpstream('glance', [VISION_MODEL_TEST_IMAGE, '-q', VISION_MODEL_TEST_PROMPT], operation, this.visionEnv(resolvedCredential));
1641
- if (result.stdout.trim().length === 0) {
1642
- throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
1735
+ if (testConnection) {
1736
+ operation.metrics.usedVisionService = true;
1737
+ const endpoint = `${target.baseUrl}/models`;
1738
+ try {
1739
+ const started = Date.now();
1740
+ const headers = {
1741
+ Accept: 'application/json',
1742
+ 'User-Agent': target.userAgent,
1743
+ };
1744
+ if (target.protocol === 'anthropic') {
1745
+ headers['x-api-key'] = entry.env.VISION_API_KEY;
1746
+ headers['anthropic-version'] = '2023-06-01';
1747
+ }
1748
+ else {
1749
+ headers.Authorization = `Bearer ${entry.env.VISION_API_KEY}`;
1750
+ }
1751
+ const response = await fetch(endpoint, {
1752
+ method: 'GET',
1753
+ headers,
1754
+ signal: operation.signal,
1755
+ });
1756
+ operation.metrics.upstreamMs += Date.now() - started;
1757
+ await response.body?.cancel().catch(() => { });
1758
+ if (response.ok) {
1759
+ service = { status: 'ok', detail: `Service responded at ${endpoint} (HTTP ${response.status})` };
1760
+ }
1761
+ else if (response.status === 401) {
1762
+ service = { status: 'error', detail: `Service rejected the configured credential (HTTP ${response.status})` };
1763
+ }
1764
+ else if (response.status === 403) {
1765
+ // Some providers (e.g. Groq preview/account restrictions) block GET /models
1766
+ // while real multimodal requests still work. Treat 403 as a warning so the
1767
+ // explicit vision-model test, not the model list endpoint, decides access.
1768
+ service = { status: 'warning', detail: `Service is reachable but restricted GET /models (HTTP 403); the credential may still be valid for real vision requests` };
1769
+ }
1770
+ else if (response.status === 404 || response.status === 405) {
1771
+ service = { status: 'warning', detail: `Service is reachable but does not expose GET /models (HTTP ${response.status})` };
1772
+ }
1773
+ else if (response.status === 429) {
1774
+ service = { status: 'warning', detail: 'Service is reachable but rate-limited the connection test (HTTP 429)' };
1775
+ }
1776
+ else {
1777
+ service = { status: 'error', detail: `Service connection test failed with HTTP ${response.status}` };
1778
+ }
1779
+ }
1780
+ catch {
1781
+ if (operation.signal.aborted)
1782
+ throw new VisionToolkitError('cancelled', 'vision_toolkit_health: connection test cancelled');
1783
+ service = { status: 'error', detail: `Service could not be reached at ${endpoint}` };
1643
1784
  }
1644
- model = {
1645
- status: 'ok',
1646
- detail: `Vision model ${this.config.provider.model} completed a multimodal request`,
1647
- };
1648
1785
  }
1649
- catch (error) {
1650
- if (operation.signal.aborted)
1651
- throw error;
1652
- const detail = error instanceof Error ? error.message : String(error);
1653
- model = { status: 'error', detail: `Vision model test failed: ${detail.slice(0, 600)}` };
1786
+ if (testModel) {
1787
+ try {
1788
+ const attemptDeadline = createDeadline(operation.signal, target.timeoutMs);
1789
+ try {
1790
+ const result = await this.runUpstream('glance', [VISION_MODEL_TEST_IMAGE, '-q', VISION_MODEL_TEST_PROMPT], { signal: attemptDeadline.signal, metrics: operation.metrics }, entry.env);
1791
+ if (result.stdout.trim().length === 0) {
1792
+ throw new VisionToolkitError('output', 'glance: vision API returned an empty description');
1793
+ }
1794
+ model = {
1795
+ status: 'ok',
1796
+ detail: `Vision model ${target.model} completed a multimodal request`,
1797
+ };
1798
+ }
1799
+ finally {
1800
+ attemptDeadline.cleanup();
1801
+ }
1802
+ }
1803
+ catch (error) {
1804
+ if (operation.signal.aborted)
1805
+ throw error;
1806
+ const detail = error instanceof Error ? error.message : String(error);
1807
+ model = { status: 'error', detail: `Vision model test failed: ${detail.slice(0, 600)}` };
1808
+ }
1654
1809
  }
1655
1810
  }
1656
1811
  }
1657
- const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model };
1812
+ const checks = { python, dependencies, chrome, service, model };
1658
1813
  const healthy = Object.values(checks).every(check => check.status !== 'error');
1659
1814
  return {
1660
1815
  pluginVersion: PLUGIN_VERSION,
@@ -1663,6 +1818,7 @@ export class VisionToolkitRuntime {
1663
1818
  healthy,
1664
1819
  connectionTested: testConnection,
1665
1820
  modelTested: testModel,
1821
+ ...(provider === undefined ? {} : { providerName: provider.name }),
1666
1822
  };
1667
1823
  });
1668
1824
  }