@mengruo/dsh-vision-toolkit 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.i18n.yaml +2 -2
  2. package/README.md +15 -70
  3. package/README.zh.md +15 -69
  4. package/docs/requirements-traceability/README.i18n.yaml +2 -2
  5. package/docs/requirements-traceability/README.md +1 -1
  6. package/docs/requirements-traceability/README.zh.md +1 -1
  7. package/lib/artifact-access.js +20 -2
  8. package/lib/artifact-access.js.map +1 -1
  9. package/lib/client.js +128 -12
  10. package/lib/client.js.map +1 -1
  11. package/lib/config.js +69 -0
  12. package/lib/config.js.map +1 -1
  13. package/lib/errors.js +25 -2
  14. package/lib/errors.js.map +1 -1
  15. package/lib/evidence-cache.js +2 -1
  16. package/lib/evidence-cache.js.map +1 -1
  17. package/lib/exposure.js +14 -1
  18. package/lib/exposure.js.map +1 -1
  19. package/lib/image-input-variants.js +22 -12
  20. package/lib/image-input-variants.js.map +1 -1
  21. package/lib/index.js +53 -6
  22. package/lib/index.js.map +1 -1
  23. package/lib/object-storage.js +141 -0
  24. package/lib/object-storage.js.map +1 -0
  25. package/lib/paste-images.js +67 -19
  26. package/lib/paste-images.js.map +1 -1
  27. package/lib/paths.js +214 -28
  28. package/lib/paths.js.map +1 -1
  29. package/lib/runtime-manager.js +76 -10
  30. package/lib/runtime-manager.js.map +1 -1
  31. package/lib/runtime.js +219 -38
  32. package/lib/runtime.js.map +1 -1
  33. package/lib/storage-history.js +154 -0
  34. package/lib/storage-history.js.map +1 -0
  35. package/lib/types/artifact-access.d.ts.map +1 -1
  36. package/lib/types/client/index.d.ts +45 -2
  37. package/lib/types/client/index.d.ts.map +1 -1
  38. package/lib/types/client/paste-images.d.ts +2 -0
  39. package/lib/types/client/paste-images.d.ts.map +1 -1
  40. package/lib/types/config.d.ts +55 -0
  41. package/lib/types/config.d.ts.map +1 -1
  42. package/lib/types/errors.d.ts +18 -2
  43. package/lib/types/errors.d.ts.map +1 -1
  44. package/lib/types/evidence-cache.d.ts +1 -1
  45. package/lib/types/evidence-cache.d.ts.map +1 -1
  46. package/lib/types/exposure.d.ts.map +1 -1
  47. package/lib/types/image-input-variants.d.ts +5 -3
  48. package/lib/types/image-input-variants.d.ts.map +1 -1
  49. package/lib/types/index.d.ts.map +1 -1
  50. package/lib/types/object-storage.d.ts +54 -0
  51. package/lib/types/object-storage.d.ts.map +1 -0
  52. package/lib/types/paste-images.d.ts +12 -4
  53. package/lib/types/paste-images.d.ts.map +1 -1
  54. package/lib/types/paths.d.ts +31 -5
  55. package/lib/types/paths.d.ts.map +1 -1
  56. package/lib/types/runtime-manager.d.ts +28 -4
  57. package/lib/types/runtime-manager.d.ts.map +1 -1
  58. package/lib/types/runtime.d.ts +31 -1
  59. package/lib/types/runtime.d.ts.map +1 -1
  60. package/lib/types/storage-history.d.ts +63 -0
  61. package/lib/types/storage-history.d.ts.map +1 -0
  62. package/lib/types/upstream.d.ts +1 -0
  63. package/lib/types/upstream.d.ts.map +1 -1
  64. package/lib/types/web.d.ts +7 -0
  65. package/lib/types/web.d.ts.map +1 -1
  66. package/lib/upstream.js +34 -8
  67. package/lib/upstream.js.map +1 -1
  68. package/lib/web.js +44 -9
  69. package/lib/web.js.map +1 -1
  70. package/package.json +3 -1
  71. package/src/artifact-access.ts +22 -2
  72. package/src/client/index.tsx +169 -9
  73. package/src/client/paste-images.tsx +14 -4
  74. package/src/config.ts +128 -0
  75. package/src/errors.ts +25 -2
  76. package/src/evidence-cache.ts +2 -1
  77. package/src/exposure.ts +16 -2
  78. package/src/image-input-variants.ts +21 -6
  79. package/src/index.ts +65 -6
  80. package/src/object-storage.ts +174 -0
  81. package/src/paste-images.ts +81 -19
  82. package/src/paths.ts +249 -28
  83. package/src/runtime-manager.ts +93 -10
  84. package/src/runtime.ts +215 -36
  85. package/src/storage-history.ts +172 -0
  86. package/src/upstream.ts +36 -7
  87. package/src/web.ts +54 -9
  88. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
  89. package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
  90. package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
  91. package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
  92. package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
  93. package/vendor/agent-vision-toolkit/bin/glance +8 -1
  94. package/vendor/agent-vision-toolkit/detect.py +13 -7
  95. package/vendor/agent-vision-toolkit/ground.py +43 -18
  96. package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
  97. package/vendor/agent-vision-toolkit/vision_client.py +84 -6
  98. package/assets/community-group-qr.png +0 -0
  99. package/assets/logo_aihubmix.png +0 -0
  100. package/assets/logo_eapi_dark.png +0 -0
  101. package/assets/wechat-reward.png +0 -0
@@ -7,6 +7,7 @@
7
7
 
8
8
  import type { Context } from '@deepseek-ai/cordis'
9
9
  import { resolveConfig, type ResolvedVisionToolkitConfig, type VisionToolkitConfig } from './config.ts'
10
+ import { preflightSharedStorageBase } from './paths.ts'
10
11
  import { VisionToolkitRuntime } from './runtime.ts'
11
12
  import { UpstreamAdapter, type UpstreamVersionInfo } from './upstream.ts'
12
13
 
@@ -26,16 +27,30 @@ export interface RuntimeManagerStatus {
26
27
  lastError?: string
27
28
  }
28
29
 
30
+ /** Storage selection paired to the active or validated startup generation. */
31
+ export interface RuntimeStorageGeneration {
32
+ generation: number
33
+ storageDir?: string
34
+ }
35
+
29
36
  /** Test seam for preparing one generation. */
30
37
  export type RuntimeGenerationFactory = (
31
38
  ctx: Context,
32
39
  config: ResolvedVisionToolkitConfig,
40
+ readableStorageDirs: readonly string[],
33
41
  ) => Promise<VisionToolkitRuntime>
34
42
 
35
- async function defaultFactory(ctx: Context, config: ResolvedVisionToolkitConfig): Promise<VisionToolkitRuntime> {
43
+ /** Async commit prerequisite run after preparation and before a generation becomes active. */
44
+ export type RuntimeGenerationBeforePublish = (candidate: PreparedRuntimeGeneration) => Promise<void>
45
+
46
+ async function defaultFactory(
47
+ ctx: Context,
48
+ config: ResolvedVisionToolkitConfig,
49
+ readableStorageDirs: readonly string[],
50
+ ): Promise<VisionToolkitRuntime> {
36
51
  const adapter = new UpstreamAdapter(ctx, config)
37
52
  await adapter.prepare()
38
- return new VisionToolkitRuntime(ctx, config, adapter)
53
+ return new VisionToolkitRuntime(ctx, config, adapter, readableStorageDirs)
39
54
  }
40
55
 
41
56
  function fingerprint(config: ResolvedVisionToolkitConfig): string {
@@ -57,6 +72,8 @@ export class VisionToolkitRuntimeManager {
57
72
  private generation = 0
58
73
  private reconfigureTicket = 0
59
74
  private lastError: string | undefined
75
+ private validatedStartupStorageDir: string | null | undefined
76
+ private readonly readableStorageDirs = new Set<string>()
60
77
 
61
78
  constructor(
62
79
  private readonly ctx: Context,
@@ -69,22 +86,69 @@ export class VisionToolkitRuntimeManager {
69
86
  return this.active.runtime
70
87
  }
71
88
 
89
+ /** Configuration belonging to the currently serving runtime generation. */
90
+ currentConfig(): ResolvedVisionToolkitConfig {
91
+ if (this.active === undefined) throw new Error('dsh-vision-toolkit runtime is not ready')
92
+ return this.active.config
93
+ }
94
+
72
95
  /** Whether at least one generation is available. */
73
96
  get ready(): boolean {
74
97
  return this.active !== undefined
75
98
  }
76
99
 
77
- /** Resolve and fully prepare a candidate without changing the active runtime. */
78
- async prepareCandidate(raw: VisionToolkitConfig): Promise<PreparedRuntimeGeneration> {
79
- const config = resolveConfig(raw)
100
+ /** Storage config safe for paste writes even when initial runtime preparation failed. */
101
+ storageGeneration(): RuntimeStorageGeneration {
102
+ if (this.active !== undefined) {
103
+ return {
104
+ generation: this.generation,
105
+ ...(this.active.config.storageDir === undefined ? {} : { storageDir: this.active.config.storageDir }),
106
+ }
107
+ }
108
+ if (this.validatedStartupStorageDir === undefined) {
109
+ throw new Error('dsh-vision-toolkit storage configuration is not ready')
110
+ }
111
+ return {
112
+ generation: this.generation,
113
+ ...(this.validatedStartupStorageDir === null ? {} : { storageDir: this.validatedStartupStorageDir }),
114
+ }
115
+ }
116
+
117
+ /** Validated storage for best-effort consumers; undefined when startup preflight failed. */
118
+ validatedStorageDirectory(): string | undefined {
119
+ if (this.active !== undefined) return this.active.config.storageDir
120
+ return this.validatedStartupStorageDir ?? undefined
121
+ }
122
+
123
+ private async prepareResolvedCandidate(config: ResolvedVisionToolkitConfig): Promise<PreparedRuntimeGeneration> {
80
124
  const resolvedFingerprint = fingerprint(config)
81
125
  if (this.active?.fingerprint === resolvedFingerprint) {
82
126
  return { ...this.active, config }
83
127
  }
84
- const runtime = await this.factory(this.ctx, config)
128
+ const runtime = await this.factory(
129
+ this.ctx,
130
+ config,
131
+ [...new Set([...this.readableStorageDirs, ...config.storageHistory])]
132
+ .filter(storageDir => storageDir !== config.storageDir),
133
+ )
85
134
  return { config, fingerprint: resolvedFingerprint, runtime }
86
135
  }
87
136
 
137
+ private rememberStorageDirectory(storageDir: string | undefined): void {
138
+ if (storageDir !== undefined) this.readableStorageDirs.add(storageDir)
139
+ }
140
+
141
+ private rememberStorageDirectories(storageDirs: readonly string[]): void {
142
+ for (const storageDir of storageDirs) this.rememberStorageDirectory(storageDir)
143
+ }
144
+
145
+ /** Resolve and fully prepare a candidate without changing the active runtime. */
146
+ async prepareCandidate(raw: VisionToolkitConfig): Promise<PreparedRuntimeGeneration> {
147
+ const config = resolveConfig(raw)
148
+ if (config.storageDir !== undefined) await preflightSharedStorageBase(config.storageDir)
149
+ return this.prepareResolvedCandidate(config)
150
+ }
151
+
88
152
  /**
89
153
  * Publish one already-prepared generation atomically.
90
154
  * @param candidate - generation returned by {@link prepareCandidate}.
@@ -97,6 +161,8 @@ export class VisionToolkitRuntimeManager {
97
161
  }
98
162
  this.reconfigureTicket += 1
99
163
  this.active = candidate
164
+ this.rememberStorageDirectories(candidate.config.storageHistory)
165
+ this.rememberStorageDirectory(candidate.config.storageDir)
100
166
  this.generation += 1
101
167
  this.lastError = undefined
102
168
  this.ctx.logger.info(
@@ -108,10 +174,21 @@ export class VisionToolkitRuntimeManager {
108
174
  )
109
175
  }
110
176
 
111
- /** Prepare and publish the initial or explicitly validated generation. */
112
- async initialize(raw: VisionToolkitConfig): Promise<void> {
177
+ /**
178
+ * Prepare and publish the initial or explicitly validated generation.
179
+ * @param raw - untrusted Settings generation to resolve and prepare.
180
+ * @param beforePublish - optional durable prerequisite run after preparation.
181
+ */
182
+ async initialize(raw: VisionToolkitConfig, beforePublish?: RuntimeGenerationBeforePublish): Promise<void> {
113
183
  try {
114
- this.activateCandidate(await this.prepareCandidate(raw))
184
+ const config = resolveConfig(raw)
185
+ if (config.storageDir !== undefined) await preflightSharedStorageBase(config.storageDir)
186
+ this.validatedStartupStorageDir = config.storageDir ?? null
187
+ this.rememberStorageDirectories(config.storageHistory)
188
+ this.rememberStorageDirectory(config.storageDir)
189
+ const candidate = await this.prepareResolvedCandidate(config)
190
+ await beforePublish?.(candidate)
191
+ this.activateCandidate(candidate)
115
192
  } catch (error) {
116
193
  this.lastError = messageOf(error)
117
194
  throw error
@@ -121,13 +198,17 @@ export class VisionToolkitRuntimeManager {
121
198
  /**
122
199
  * Apply an externally committed Settings generation. Concurrent edits are
123
200
  * last-write-wins; a slower obsolete prepare can never overwrite a newer one.
201
+ * @param raw - externally committed Settings generation.
202
+ * @param beforePublish - optional durable prerequisite run after preparation.
124
203
  * @returns whether this call published a new active generation.
125
204
  */
126
- async reconfigure(raw: VisionToolkitConfig): Promise<boolean> {
205
+ async reconfigure(raw: VisionToolkitConfig, beforePublish?: RuntimeGenerationBeforePublish): Promise<boolean> {
127
206
  const ticket = ++this.reconfigureTicket
128
207
  let candidate: PreparedRuntimeGeneration
129
208
  try {
130
209
  candidate = await this.prepareCandidate(raw)
210
+ if (ticket !== this.reconfigureTicket) return false
211
+ await beforePublish?.(candidate)
131
212
  } catch (error) {
132
213
  if (ticket === this.reconfigureTicket) this.lastError = messageOf(error)
133
214
  throw error
@@ -135,6 +216,8 @@ export class VisionToolkitRuntimeManager {
135
216
  if (ticket !== this.reconfigureTicket) return false
136
217
  const changed = this.active?.fingerprint !== candidate.fingerprint
137
218
  this.active = candidate
219
+ this.rememberStorageDirectories(candidate.config.storageHistory)
220
+ this.rememberStorageDirectory(candidate.config.storageDir)
138
221
  if (changed) {
139
222
  this.generation += 1
140
223
  this.ctx.logger.info(
package/src/runtime.ts CHANGED
@@ -18,6 +18,7 @@ import { isBuiltInFreeVisionProvider, type ResolvedProvider, type ResolvedVision
18
18
  import { BUILT_IN_FREE_VISION_KEY } from './defaults.ts'
19
19
  import { evidenceRuntimeFingerprint } from './evidence-cache.ts'
20
20
  import { VisionToolkitError, type VisionToolkitErrorCode } from './errors.ts'
21
+ import { ObjectStorageClient, splitObjectStorageCredential, type ObjectStorageSettings } from './object-storage.ts'
21
22
  import {
22
23
  assertDistinctOutput,
23
24
  commitStagedDirectory,
@@ -475,6 +476,9 @@ export interface VisionToolkitHealthResult {
475
476
  python: HealthCheck
476
477
  dependencies: HealthCheck
477
478
  chrome: HealthCheck
479
+ credential: HealthCheck
480
+ artifactDirectory: HealthCheck
481
+ tempDirectory: HealthCheck
478
482
  service: HealthCheck
479
483
  model: HealthCheck
480
484
  }
@@ -537,8 +541,26 @@ const FORMAT_BY_EXTENSION = new Map([
537
541
  ])
538
542
  const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/
539
543
 
540
- /** Error codes a provider retries within its attempt budget (429 is handled separately). */
541
- const RETRYABLE_CODES: ReadonlySet<VisionToolkitErrorCode> = new Set(['service', 'timeout'])
544
+ /** MIME type for one analyzed image format, used when uploading to object storage. */
545
+ function imageMimeType(format: string): string {
546
+ switch (format) {
547
+ case 'png': return 'image/png'
548
+ case 'jpeg': return 'image/jpeg'
549
+ case 'gif': return 'image/gif'
550
+ case 'webp': return 'image/webp'
551
+ default: return 'application/octet-stream'
552
+ }
553
+ }
554
+
555
+ /**
556
+ * Error codes a provider retries against the SAME provider within its
557
+ * `attempts` budget. Only transient failures are worth re-requesting: a
558
+ * timeout may clear on the next attempt and a 5xx / network drop is usually
559
+ * ephemeral. Deterministic failures (auth, quota, rate_limit, invalid_request,
560
+ * region, tos) must fail over to the next provider immediately instead of
561
+ * re-requesting a backend that cannot succeed with the same input.
562
+ */
563
+ const RETRYABLE_CODES: ReadonlySet<VisionToolkitErrorCode> = new Set(['timeout', 'server', 'network'])
542
564
 
543
565
  /** Resolve as soon as `signal` aborts (or immediately when already aborted). */
544
566
  function untilAbort(signal: AbortSignal): Promise<void> {
@@ -771,6 +793,7 @@ export class VisionToolkitRuntime {
771
793
  private readonly ctx: Context,
772
794
  private readonly config: ResolvedVisionToolkitConfig,
773
795
  adapter?: UpstreamAdapter,
796
+ private readonly readableStorageDirs: readonly string[] = [],
774
797
  ) {
775
798
  this.adapter = adapter ?? new UpstreamAdapter(ctx, config)
776
799
  }
@@ -785,6 +808,11 @@ export class VisionToolkitRuntime {
785
808
  return this.config.sessionMaxConcurrency
786
809
  }
787
810
 
811
+ /** Shared storage root belonging to this immutable runtime generation. */
812
+ get storageDirectory(): string | undefined {
813
+ return this.config.storageDir
814
+ }
815
+
788
816
  /** Stable identity for persisted image descriptions produced by this runtime. */
789
817
  get evidenceFingerprint(): string {
790
818
  return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim())
@@ -952,6 +980,7 @@ export class VisionToolkitRuntime {
952
980
  VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
953
981
  VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
954
982
  ...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
983
+ ...(provider.stream ? { VISION_STREAM: '1' } : {}),
955
984
  VISION_USER_AGENT: provider.userAgent,
956
985
  LANG: this.config.language,
957
986
  }
@@ -968,6 +997,8 @@ export class VisionToolkitRuntime {
968
997
  protocol: provider.protocol,
969
998
  anthropicThinking: provider.anthropicThinking,
970
999
  userAgent: provider.userAgent,
1000
+ stream: provider.stream,
1001
+ uploadViaUrl: provider.uploadViaUrl,
971
1002
  })
972
1003
  ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
973
1004
  : await this.ctx.credentials.resolve(provider.credential)
@@ -1010,13 +1041,13 @@ export class VisionToolkitRuntime {
1010
1041
  }
1011
1042
 
1012
1043
  private pathPolicy(workspace: string): Promise<PathPolicy> {
1013
- return createPathPolicy(workspace, this.config.allowedDirs)
1044
+ return createPathPolicy(workspace, this.config.allowedDirs, this.config.storageDir, this.readableStorageDirs)
1014
1045
  }
1015
1046
 
1016
1047
  private async compressedImageRoot(policy: PathPolicy): Promise<string> {
1017
- const root = join(policy.workspace, '.dsh-vision-toolkit', 'tmp', 'compressed-images')
1018
- let current = policy.workspace
1019
- for (const segment of ['.dsh-vision-toolkit', 'tmp', 'compressed-images']) {
1048
+ const root = join(policy.storageRoot, 'tmp', 'compressed-images')
1049
+ let current = policy.storageRoot
1050
+ for (const segment of ['tmp', 'compressed-images']) {
1020
1051
  current = join(current, segment)
1021
1052
  try {
1022
1053
  await mkdir(current, { mode: 0o700 })
@@ -1027,13 +1058,13 @@ export class VisionToolkitRuntime {
1027
1058
  if (info.isSymbolicLink() || !info.isDirectory()) {
1028
1059
  throw new VisionToolkitError('path', `compressed-image cache path is not a real directory: ${current}`)
1029
1060
  }
1030
- if (!isWithin(policy.workspace, current)) {
1031
- throw new VisionToolkitError('path', `compressed-image cache path escaped the workspace: ${current}`)
1061
+ if (!isWithin(policy.storageRoot, current)) {
1062
+ throw new VisionToolkitError('path', `compressed-image cache path escaped plugin storage: ${current}`)
1032
1063
  }
1033
1064
  }
1034
1065
  const canonical = await realpath(root)
1035
- if (!isWithin(policy.workspace, canonical)) {
1036
- throw new VisionToolkitError('path', 'compressed-image cache resolved outside the workspace')
1066
+ if (!isWithin(policy.storageRoot, canonical)) {
1067
+ throw new VisionToolkitError('path', 'compressed-image cache resolved outside plugin storage')
1037
1068
  }
1038
1069
  return canonical
1039
1070
  }
@@ -1237,6 +1268,84 @@ export class VisionToolkitRuntime {
1237
1268
  operation.metrics.imagePixels += image.width * image.height
1238
1269
  }
1239
1270
 
1271
+ /** Resolve the configured object storage into a usable client, or undefined. */
1272
+ private async resolveObjectStorageClient(): Promise<ObjectStorageClient | undefined> {
1273
+ const objectStorage = this.config.objectStorage
1274
+ if (objectStorage.credential === undefined || objectStorage.endpoint.length === 0 || objectStorage.bucket.length === 0) {
1275
+ return undefined
1276
+ }
1277
+ let resolved: ResolvedCredential | undefined
1278
+ try {
1279
+ resolved = await this.ctx.credentials.resolve(objectStorage.credential)
1280
+ } catch {
1281
+ resolved = undefined
1282
+ }
1283
+ if (resolved === undefined) return undefined
1284
+ let accessKeyId: string
1285
+ let secretAccessKey: string
1286
+ try {
1287
+ const split = splitObjectStorageCredential(resolved.value)
1288
+ accessKeyId = split.accessKeyId
1289
+ secretAccessKey = split.secretAccessKey
1290
+ } catch (error) {
1291
+ throw new VisionToolkitError('config', 'object storage credential is malformed', { cause: error })
1292
+ }
1293
+ const settings: ObjectStorageSettings = {
1294
+ endpoint: objectStorage.endpoint,
1295
+ bucket: objectStorage.bucket,
1296
+ accessKeyId,
1297
+ secretAccessKey,
1298
+ ...(objectStorage.publicBase === undefined ? {} : { publicBase: objectStorage.publicBase }),
1299
+ }
1300
+ return new ObjectStorageClient(settings)
1301
+ }
1302
+
1303
+ /**
1304
+ * Upload the prepared images to object storage and return their URLs plus a
1305
+ * cleanup callback, when the primary provider opts into URL transfer and
1306
+ * object storage is configured. Returns undefined otherwise (base64 path).
1307
+ */
1308
+ private async maybeTransferImages(
1309
+ pool: readonly ResolvedProviderEnv[],
1310
+ images: readonly ImageInfo[],
1311
+ operation: OperationContext,
1312
+ ): Promise<{ urls: string[]; cleanup: () => Promise<void> } | undefined> {
1313
+ const primary = pool[0]
1314
+ if (primary === undefined || primary.provider.uploadViaUrl !== true) return undefined
1315
+ const client = await this.resolveObjectStorageClient()
1316
+ if (client === undefined) {
1317
+ throw new VisionToolkitError('config', 'uploadViaUrl is enabled but object storage is not configured')
1318
+ }
1319
+ const keys: string[] = []
1320
+ const urls: string[] = []
1321
+ try {
1322
+ for (const image of images) {
1323
+ if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision image upload cancelled')
1324
+ const uploaded = await client.uploadImage(image.path, imageMimeType(image.format))
1325
+ keys.push(uploaded.key)
1326
+ urls.push(uploaded.url)
1327
+ }
1328
+ } catch (error) {
1329
+ await Promise.allSettled(keys.map(key => client.deleteObject(key)))
1330
+ throw error
1331
+ }
1332
+ return {
1333
+ urls,
1334
+ cleanup: async () => {
1335
+ await Promise.allSettled(keys.map(key => client.deleteObject(key)))
1336
+ },
1337
+ }
1338
+ }
1339
+
1340
+ /** Settings "test storage" probe: upload → head → delete a tiny marker object. */
1341
+ async testObjectStorage(): Promise<{ detail: string }> {
1342
+ const client = await this.resolveObjectStorageClient()
1343
+ if (client === undefined) {
1344
+ throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
1345
+ }
1346
+ return client.test()
1347
+ }
1348
+
1240
1349
  /** Stable gate key for one provider's in-flight request cap. */
1241
1350
  private providerGate(provider: ResolvedProvider): Semaphore {
1242
1351
  const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`
@@ -1383,6 +1492,22 @@ export class VisionToolkitRuntime {
1383
1492
  return Math.max(0, Math.min(t2Remaining, globalRemaining))
1384
1493
  }
1385
1494
 
1495
+ /**
1496
+ * Advance to the next provider after one provider reached a terminal
1497
+ * failure. This is what makes failover work for FAST failures too: the
1498
+ * hedge timer only launches the next provider when the current one is SLOW
1499
+ * (crosses t1), so a quick auth/5xx/network failure must explicitly launch
1500
+ * the successor. Never advances when a higher-priority provider superseded
1501
+ * this task, when the whole operation was cancelled, or when the global
1502
+ * deadline has too little room left for another request. `launch` is
1503
+ * idempotent, so an earlier hedge timer cannot cause a double launch.
1504
+ */
1505
+ private advanceAfterFailure(task: ProviderTask, operation: OperationContext, launchNext: () => void): void {
1506
+ if (task.abort.signal.aborted || operation.signal.aborted) return
1507
+ if (operation.deadlineAt - Date.now() < this.config.minAvailableSeconds * 1000) return
1508
+ launchNext()
1509
+ }
1510
+
1386
1511
  /**
1387
1512
  * Run one provider to a terminal state: retryable errors retry within
1388
1513
  * `attempts`, a single request crossing t1 hedges the next provider, and the
@@ -1415,6 +1540,7 @@ export class VisionToolkitRuntime {
1415
1540
  if (budget < minAvailableMs) {
1416
1541
  task.status = 'failed'
1417
1542
  task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} has insufficient remaining time`)
1543
+ this.advanceAfterFailure(task, operation, launchNext)
1418
1544
  return
1419
1545
  }
1420
1546
  const reqDeadline = createDeadline(AbortSignal.any([operation.signal, task.abort.signal]), budget)
@@ -1453,6 +1579,7 @@ export class VisionToolkitRuntime {
1453
1579
  if (reqDeadline.timedOut) {
1454
1580
  task.status = 'failed'
1455
1581
  task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} exhausted its t2 budget`)
1582
+ this.advanceAfterFailure(task, operation, launchNext)
1456
1583
  return
1457
1584
  }
1458
1585
  if (classified.code === 'rate_limit') {
@@ -1467,6 +1594,7 @@ export class VisionToolkitRuntime {
1467
1594
  }
1468
1595
  task.status = 'failed'
1469
1596
  task.error = classified
1597
+ this.advanceAfterFailure(task, operation, launchNext)
1470
1598
  return
1471
1599
  } finally {
1472
1600
  reqDeadline.cleanup()
@@ -1568,6 +1696,8 @@ export class VisionToolkitRuntime {
1568
1696
  protocol: env.VISION_API_PROTOCOL,
1569
1697
  anthropicThinking: env.VISION_ANTHROPIC_THINKING,
1570
1698
  sslVerify: env.VISION_SSL_VERIFY ?? null,
1699
+ stream: env.VISION_STREAM === '1',
1700
+ uploadViaUrl: provider.uploadViaUrl,
1571
1701
  userAgent: env.VISION_USER_AGENT,
1572
1702
  credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
1573
1703
  maxImageBytes: provider.maxImageBytes,
@@ -1698,24 +1828,31 @@ export class VisionToolkitRuntime {
1698
1828
  return cached.result
1699
1829
  }
1700
1830
  }
1701
- const result = await this.runVisionHedge('glance', [
1702
- ...images.map(image => image.path),
1703
- ...(request.region !== undefined ? ['--region', request.region] : []),
1704
- ...(request.ocr === true ? ['--ocr'] : []),
1705
- ...(request.query !== undefined ? ['-q', request.query] : []),
1706
- ], images, operation, pool)
1707
- const answer = result.stdout.trim()
1708
- if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
1709
- const value: GlanceResult = {
1710
- images,
1711
- mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
1712
- answer,
1713
- truncated: false,
1714
- }
1715
- if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
1716
- this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
1831
+ const transfer = request.region === undefined
1832
+ ? await this.maybeTransferImages(pool, images, operation)
1833
+ : undefined
1834
+ try {
1835
+ const result = await this.runVisionHedge('glance', [
1836
+ ...(transfer !== undefined ? transfer.urls : images.map(image => image.path)),
1837
+ ...(transfer === undefined && request.region !== undefined ? ['--region', request.region] : []),
1838
+ ...(request.ocr === true ? ['--ocr'] : []),
1839
+ ...(request.query !== undefined ? ['-q', request.query] : []),
1840
+ ], images, operation, pool)
1841
+ const answer = result.stdout.trim()
1842
+ if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
1843
+ const value: GlanceResult = {
1844
+ images,
1845
+ mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
1846
+ answer,
1847
+ truncated: false,
1848
+ }
1849
+ if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
1850
+ this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
1851
+ }
1852
+ return value
1853
+ } finally {
1854
+ if (transfer !== undefined) await transfer.cleanup()
1717
1855
  }
1718
- return value
1719
1856
  })
1720
1857
  }
1721
1858
 
@@ -1751,14 +1888,23 @@ export class VisionToolkitRuntime {
1751
1888
  }
1752
1889
  const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation)
1753
1890
  this.accountImage(image, operation)
1754
- const result = await this.runVisionHedge(tool, [
1755
- image.path,
1756
- request.target,
1757
- ...(request.region !== undefined ? ['--region', request.region] : []),
1758
- ], [image], operation, pool)
1759
- const elements = parseLocationOutput(result.stdout)
1760
- this.validateLocations(elements, image.width, image.height)
1761
- return { image, elements }
1891
+ const transfer = request.region === undefined
1892
+ ? await this.maybeTransferImages(pool, [image], operation)
1893
+ : undefined
1894
+ try {
1895
+ const result = await this.runVisionHedge(tool, transfer !== undefined
1896
+ ? [transfer.urls[0]!, request.target, '--size', `${image.width}x${image.height}`]
1897
+ : [
1898
+ image.path,
1899
+ request.target,
1900
+ ...(request.region !== undefined ? ['--region', request.region] : []),
1901
+ ], [image], operation, pool)
1902
+ const elements = parseLocationOutput(result.stdout)
1903
+ this.validateLocations(elements, image.width, image.height)
1904
+ return { image, elements }
1905
+ } finally {
1906
+ if (transfer !== undefined) await transfer.cleanup()
1907
+ }
1762
1908
  }
1763
1909
 
1764
1910
  /** ground: locate one named target and return pixel boxes. */
@@ -2453,6 +2599,18 @@ export class VisionToolkitRuntime {
2453
2599
  })
2454
2600
  }
2455
2601
 
2602
+ private async writableDirectoryCheck(path: string, label: string): Promise<HealthCheck> {
2603
+ const probe = join(path, `.vision-toolkit-health-${randomUUID()}`)
2604
+ try {
2605
+ await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' })
2606
+ await rm(probe, { force: true })
2607
+ return { status: 'ok', detail: `${label} is writable: ${path}` }
2608
+ } catch {
2609
+ await rm(probe, { force: true }).catch(() => {})
2610
+ return { status: 'error', detail: `${label} is not writable: ${path}` }
2611
+ }
2612
+ }
2613
+
2456
2614
  /** Health: inspect local readiness, and optionally probe one provider's `/models` plus one real multimodal request. */
2457
2615
  async health(testConnection: boolean, options: ToolCallOptions, testModel = false, provider?: ResolvedProvider): Promise<VisionToolkitHealthResult> {
2458
2616
  return this.runOperation('vision_toolkit_health', options, async (operation) => {
@@ -2474,6 +2632,27 @@ export class VisionToolkitRuntime {
2474
2632
  if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled')
2475
2633
  chrome = { status: 'error', detail: 'Chrome availability probe failed' }
2476
2634
  }
2635
+ let resolvedCredential: ResolvedCredential | undefined
2636
+ let credential: HealthCheck
2637
+ try {
2638
+ resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
2639
+ ? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
2640
+ : await this.ctx.credentials.resolve(this.config.provider.credential)
2641
+ credential = resolvedCredential === undefined
2642
+ ? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
2643
+ : { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` }
2644
+ } catch {
2645
+ credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` }
2646
+ }
2647
+ let artifactDirectory: HealthCheck
2648
+ try {
2649
+ // allowedDirs are session input roots; they do not affect output readiness.
2650
+ const policy = await createPathPolicy(options.workspace, [], this.config.storageDir)
2651
+ artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory')
2652
+ } catch {
2653
+ artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' }
2654
+ }
2655
+ const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory')
2477
2656
  let service: HealthCheck = {
2478
2657
  status: 'not_tested',
2479
2658
  detail: 'Connection was not tested; use the per-provider API test',
@@ -2564,7 +2743,7 @@ export class VisionToolkitRuntime {
2564
2743
  }
2565
2744
  }
2566
2745
  }
2567
- const checks = { python, dependencies, chrome, service, model }
2746
+ const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model }
2568
2747
  const healthy = Object.values(checks).every(check => check.status !== 'error')
2569
2748
  return {
2570
2749
  pluginVersion: PLUGIN_VERSION,