@mengruo/dsh-vision-toolkit 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +2 -2
- package/README.md +15 -70
- package/README.zh.md +15 -69
- package/docs/requirements-traceability/README.i18n.yaml +2 -2
- package/docs/requirements-traceability/README.md +1 -1
- package/docs/requirements-traceability/README.zh.md +1 -1
- package/lib/artifact-access.js +20 -2
- package/lib/artifact-access.js.map +1 -1
- package/lib/client.js +128 -12
- package/lib/client.js.map +1 -1
- package/lib/config.js +69 -0
- package/lib/config.js.map +1 -1
- package/lib/errors.js +25 -2
- package/lib/errors.js.map +1 -1
- package/lib/evidence-cache.js +2 -1
- package/lib/evidence-cache.js.map +1 -1
- package/lib/exposure.js +14 -1
- package/lib/exposure.js.map +1 -1
- package/lib/image-input-variants.js +22 -12
- package/lib/image-input-variants.js.map +1 -1
- package/lib/index.js +53 -6
- package/lib/index.js.map +1 -1
- package/lib/object-storage.js +141 -0
- package/lib/object-storage.js.map +1 -0
- package/lib/paste-images.js +67 -19
- package/lib/paste-images.js.map +1 -1
- package/lib/paths.js +214 -28
- package/lib/paths.js.map +1 -1
- package/lib/runtime-manager.js +76 -10
- package/lib/runtime-manager.js.map +1 -1
- package/lib/runtime.js +219 -38
- package/lib/runtime.js.map +1 -1
- package/lib/storage-history.js +154 -0
- package/lib/storage-history.js.map +1 -0
- package/lib/types/artifact-access.d.ts.map +1 -1
- package/lib/types/client/index.d.ts +45 -2
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/client/paste-images.d.ts +2 -0
- package/lib/types/client/paste-images.d.ts.map +1 -1
- package/lib/types/config.d.ts +55 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/errors.d.ts +18 -2
- package/lib/types/errors.d.ts.map +1 -1
- package/lib/types/evidence-cache.d.ts +1 -1
- package/lib/types/evidence-cache.d.ts.map +1 -1
- package/lib/types/exposure.d.ts.map +1 -1
- package/lib/types/image-input-variants.d.ts +5 -3
- package/lib/types/image-input-variants.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/object-storage.d.ts +54 -0
- package/lib/types/object-storage.d.ts.map +1 -0
- package/lib/types/paste-images.d.ts +12 -4
- package/lib/types/paste-images.d.ts.map +1 -1
- package/lib/types/paths.d.ts +31 -5
- package/lib/types/paths.d.ts.map +1 -1
- package/lib/types/runtime-manager.d.ts +28 -4
- package/lib/types/runtime-manager.d.ts.map +1 -1
- package/lib/types/runtime.d.ts +31 -1
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/storage-history.d.ts +63 -0
- package/lib/types/storage-history.d.ts.map +1 -0
- package/lib/types/upstream.d.ts +1 -0
- package/lib/types/upstream.d.ts.map +1 -1
- package/lib/types/web.d.ts +7 -0
- package/lib/types/web.d.ts.map +1 -1
- package/lib/upstream.js +34 -8
- package/lib/upstream.js.map +1 -1
- package/lib/web.js +44 -9
- package/lib/web.js.map +1 -1
- package/package.json +3 -1
- package/src/artifact-access.ts +22 -2
- package/src/client/index.tsx +169 -9
- package/src/client/paste-images.tsx +14 -4
- package/src/config.ts +128 -0
- package/src/errors.ts +25 -2
- package/src/evidence-cache.ts +2 -1
- package/src/exposure.ts +16 -2
- package/src/image-input-variants.ts +21 -6
- package/src/index.ts +65 -6
- package/src/object-storage.ts +174 -0
- package/src/paste-images.ts +81 -19
- package/src/paths.ts +249 -28
- package/src/runtime-manager.ts +93 -10
- package/src/runtime.ts +215 -36
- package/src/storage-history.ts +172 -0
- package/src/upstream.ts +36 -7
- package/src/web.ts +54 -9
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
- package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/glance +8 -1
- package/vendor/agent-vision-toolkit/detect.py +13 -7
- package/vendor/agent-vision-toolkit/ground.py +43 -18
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
- package/vendor/agent-vision-toolkit/vision_client.py +84 -6
- package/assets/community-group-qr.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/logo_eapi_dark.png +0 -0
- package/assets/wechat-reward.png +0 -0
package/src/runtime-manager.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { Context } from '@deepseek-ai/cordis'
|
|
9
9
|
import { resolveConfig, type ResolvedVisionToolkitConfig, type VisionToolkitConfig } from './config.ts'
|
|
10
|
+
import { preflightSharedStorageBase } from './paths.ts'
|
|
10
11
|
import { VisionToolkitRuntime } from './runtime.ts'
|
|
11
12
|
import { UpstreamAdapter, type UpstreamVersionInfo } from './upstream.ts'
|
|
12
13
|
|
|
@@ -26,16 +27,30 @@ export interface RuntimeManagerStatus {
|
|
|
26
27
|
lastError?: string
|
|
27
28
|
}
|
|
28
29
|
|
|
30
|
+
/** Storage selection paired to the active or validated startup generation. */
|
|
31
|
+
export interface RuntimeStorageGeneration {
|
|
32
|
+
generation: number
|
|
33
|
+
storageDir?: string
|
|
34
|
+
}
|
|
35
|
+
|
|
29
36
|
/** Test seam for preparing one generation. */
|
|
30
37
|
export type RuntimeGenerationFactory = (
|
|
31
38
|
ctx: Context,
|
|
32
39
|
config: ResolvedVisionToolkitConfig,
|
|
40
|
+
readableStorageDirs: readonly string[],
|
|
33
41
|
) => Promise<VisionToolkitRuntime>
|
|
34
42
|
|
|
35
|
-
|
|
43
|
+
/** Async commit prerequisite run after preparation and before a generation becomes active. */
|
|
44
|
+
export type RuntimeGenerationBeforePublish = (candidate: PreparedRuntimeGeneration) => Promise<void>
|
|
45
|
+
|
|
46
|
+
async function defaultFactory(
|
|
47
|
+
ctx: Context,
|
|
48
|
+
config: ResolvedVisionToolkitConfig,
|
|
49
|
+
readableStorageDirs: readonly string[],
|
|
50
|
+
): Promise<VisionToolkitRuntime> {
|
|
36
51
|
const adapter = new UpstreamAdapter(ctx, config)
|
|
37
52
|
await adapter.prepare()
|
|
38
|
-
return new VisionToolkitRuntime(ctx, config, adapter)
|
|
53
|
+
return new VisionToolkitRuntime(ctx, config, adapter, readableStorageDirs)
|
|
39
54
|
}
|
|
40
55
|
|
|
41
56
|
function fingerprint(config: ResolvedVisionToolkitConfig): string {
|
|
@@ -57,6 +72,8 @@ export class VisionToolkitRuntimeManager {
|
|
|
57
72
|
private generation = 0
|
|
58
73
|
private reconfigureTicket = 0
|
|
59
74
|
private lastError: string | undefined
|
|
75
|
+
private validatedStartupStorageDir: string | null | undefined
|
|
76
|
+
private readonly readableStorageDirs = new Set<string>()
|
|
60
77
|
|
|
61
78
|
constructor(
|
|
62
79
|
private readonly ctx: Context,
|
|
@@ -69,22 +86,69 @@ export class VisionToolkitRuntimeManager {
|
|
|
69
86
|
return this.active.runtime
|
|
70
87
|
}
|
|
71
88
|
|
|
89
|
+
/** Configuration belonging to the currently serving runtime generation. */
|
|
90
|
+
currentConfig(): ResolvedVisionToolkitConfig {
|
|
91
|
+
if (this.active === undefined) throw new Error('dsh-vision-toolkit runtime is not ready')
|
|
92
|
+
return this.active.config
|
|
93
|
+
}
|
|
94
|
+
|
|
72
95
|
/** Whether at least one generation is available. */
|
|
73
96
|
get ready(): boolean {
|
|
74
97
|
return this.active !== undefined
|
|
75
98
|
}
|
|
76
99
|
|
|
77
|
-
/**
|
|
78
|
-
|
|
79
|
-
|
|
100
|
+
/** Storage config safe for paste writes even when initial runtime preparation failed. */
|
|
101
|
+
storageGeneration(): RuntimeStorageGeneration {
|
|
102
|
+
if (this.active !== undefined) {
|
|
103
|
+
return {
|
|
104
|
+
generation: this.generation,
|
|
105
|
+
...(this.active.config.storageDir === undefined ? {} : { storageDir: this.active.config.storageDir }),
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (this.validatedStartupStorageDir === undefined) {
|
|
109
|
+
throw new Error('dsh-vision-toolkit storage configuration is not ready')
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
generation: this.generation,
|
|
113
|
+
...(this.validatedStartupStorageDir === null ? {} : { storageDir: this.validatedStartupStorageDir }),
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Validated storage for best-effort consumers; undefined when startup preflight failed. */
|
|
118
|
+
validatedStorageDirectory(): string | undefined {
|
|
119
|
+
if (this.active !== undefined) return this.active.config.storageDir
|
|
120
|
+
return this.validatedStartupStorageDir ?? undefined
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
private async prepareResolvedCandidate(config: ResolvedVisionToolkitConfig): Promise<PreparedRuntimeGeneration> {
|
|
80
124
|
const resolvedFingerprint = fingerprint(config)
|
|
81
125
|
if (this.active?.fingerprint === resolvedFingerprint) {
|
|
82
126
|
return { ...this.active, config }
|
|
83
127
|
}
|
|
84
|
-
const runtime = await this.factory(
|
|
128
|
+
const runtime = await this.factory(
|
|
129
|
+
this.ctx,
|
|
130
|
+
config,
|
|
131
|
+
[...new Set([...this.readableStorageDirs, ...config.storageHistory])]
|
|
132
|
+
.filter(storageDir => storageDir !== config.storageDir),
|
|
133
|
+
)
|
|
85
134
|
return { config, fingerprint: resolvedFingerprint, runtime }
|
|
86
135
|
}
|
|
87
136
|
|
|
137
|
+
private rememberStorageDirectory(storageDir: string | undefined): void {
|
|
138
|
+
if (storageDir !== undefined) this.readableStorageDirs.add(storageDir)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private rememberStorageDirectories(storageDirs: readonly string[]): void {
|
|
142
|
+
for (const storageDir of storageDirs) this.rememberStorageDirectory(storageDir)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Resolve and fully prepare a candidate without changing the active runtime. */
|
|
146
|
+
async prepareCandidate(raw: VisionToolkitConfig): Promise<PreparedRuntimeGeneration> {
|
|
147
|
+
const config = resolveConfig(raw)
|
|
148
|
+
if (config.storageDir !== undefined) await preflightSharedStorageBase(config.storageDir)
|
|
149
|
+
return this.prepareResolvedCandidate(config)
|
|
150
|
+
}
|
|
151
|
+
|
|
88
152
|
/**
|
|
89
153
|
* Publish one already-prepared generation atomically.
|
|
90
154
|
* @param candidate - generation returned by {@link prepareCandidate}.
|
|
@@ -97,6 +161,8 @@ export class VisionToolkitRuntimeManager {
|
|
|
97
161
|
}
|
|
98
162
|
this.reconfigureTicket += 1
|
|
99
163
|
this.active = candidate
|
|
164
|
+
this.rememberStorageDirectories(candidate.config.storageHistory)
|
|
165
|
+
this.rememberStorageDirectory(candidate.config.storageDir)
|
|
100
166
|
this.generation += 1
|
|
101
167
|
this.lastError = undefined
|
|
102
168
|
this.ctx.logger.info(
|
|
@@ -108,10 +174,21 @@ export class VisionToolkitRuntimeManager {
|
|
|
108
174
|
)
|
|
109
175
|
}
|
|
110
176
|
|
|
111
|
-
/**
|
|
112
|
-
|
|
177
|
+
/**
|
|
178
|
+
* Prepare and publish the initial or explicitly validated generation.
|
|
179
|
+
* @param raw - untrusted Settings generation to resolve and prepare.
|
|
180
|
+
* @param beforePublish - optional durable prerequisite run after preparation.
|
|
181
|
+
*/
|
|
182
|
+
async initialize(raw: VisionToolkitConfig, beforePublish?: RuntimeGenerationBeforePublish): Promise<void> {
|
|
113
183
|
try {
|
|
114
|
-
|
|
184
|
+
const config = resolveConfig(raw)
|
|
185
|
+
if (config.storageDir !== undefined) await preflightSharedStorageBase(config.storageDir)
|
|
186
|
+
this.validatedStartupStorageDir = config.storageDir ?? null
|
|
187
|
+
this.rememberStorageDirectories(config.storageHistory)
|
|
188
|
+
this.rememberStorageDirectory(config.storageDir)
|
|
189
|
+
const candidate = await this.prepareResolvedCandidate(config)
|
|
190
|
+
await beforePublish?.(candidate)
|
|
191
|
+
this.activateCandidate(candidate)
|
|
115
192
|
} catch (error) {
|
|
116
193
|
this.lastError = messageOf(error)
|
|
117
194
|
throw error
|
|
@@ -121,13 +198,17 @@ export class VisionToolkitRuntimeManager {
|
|
|
121
198
|
/**
|
|
122
199
|
* Apply an externally committed Settings generation. Concurrent edits are
|
|
123
200
|
* last-write-wins; a slower obsolete prepare can never overwrite a newer one.
|
|
201
|
+
* @param raw - externally committed Settings generation.
|
|
202
|
+
* @param beforePublish - optional durable prerequisite run after preparation.
|
|
124
203
|
* @returns whether this call published a new active generation.
|
|
125
204
|
*/
|
|
126
|
-
async reconfigure(raw: VisionToolkitConfig): Promise<boolean> {
|
|
205
|
+
async reconfigure(raw: VisionToolkitConfig, beforePublish?: RuntimeGenerationBeforePublish): Promise<boolean> {
|
|
127
206
|
const ticket = ++this.reconfigureTicket
|
|
128
207
|
let candidate: PreparedRuntimeGeneration
|
|
129
208
|
try {
|
|
130
209
|
candidate = await this.prepareCandidate(raw)
|
|
210
|
+
if (ticket !== this.reconfigureTicket) return false
|
|
211
|
+
await beforePublish?.(candidate)
|
|
131
212
|
} catch (error) {
|
|
132
213
|
if (ticket === this.reconfigureTicket) this.lastError = messageOf(error)
|
|
133
214
|
throw error
|
|
@@ -135,6 +216,8 @@ export class VisionToolkitRuntimeManager {
|
|
|
135
216
|
if (ticket !== this.reconfigureTicket) return false
|
|
136
217
|
const changed = this.active?.fingerprint !== candidate.fingerprint
|
|
137
218
|
this.active = candidate
|
|
219
|
+
this.rememberStorageDirectories(candidate.config.storageHistory)
|
|
220
|
+
this.rememberStorageDirectory(candidate.config.storageDir)
|
|
138
221
|
if (changed) {
|
|
139
222
|
this.generation += 1
|
|
140
223
|
this.ctx.logger.info(
|
package/src/runtime.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { isBuiltInFreeVisionProvider, type ResolvedProvider, type ResolvedVision
|
|
|
18
18
|
import { BUILT_IN_FREE_VISION_KEY } from './defaults.ts'
|
|
19
19
|
import { evidenceRuntimeFingerprint } from './evidence-cache.ts'
|
|
20
20
|
import { VisionToolkitError, type VisionToolkitErrorCode } from './errors.ts'
|
|
21
|
+
import { ObjectStorageClient, splitObjectStorageCredential, type ObjectStorageSettings } from './object-storage.ts'
|
|
21
22
|
import {
|
|
22
23
|
assertDistinctOutput,
|
|
23
24
|
commitStagedDirectory,
|
|
@@ -475,6 +476,9 @@ export interface VisionToolkitHealthResult {
|
|
|
475
476
|
python: HealthCheck
|
|
476
477
|
dependencies: HealthCheck
|
|
477
478
|
chrome: HealthCheck
|
|
479
|
+
credential: HealthCheck
|
|
480
|
+
artifactDirectory: HealthCheck
|
|
481
|
+
tempDirectory: HealthCheck
|
|
478
482
|
service: HealthCheck
|
|
479
483
|
model: HealthCheck
|
|
480
484
|
}
|
|
@@ -537,8 +541,26 @@ const FORMAT_BY_EXTENSION = new Map([
|
|
|
537
541
|
])
|
|
538
542
|
const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/
|
|
539
543
|
|
|
540
|
-
/**
|
|
541
|
-
|
|
544
|
+
/** MIME type for one analyzed image format, used when uploading to object storage. */
|
|
545
|
+
function imageMimeType(format: string): string {
|
|
546
|
+
switch (format) {
|
|
547
|
+
case 'png': return 'image/png'
|
|
548
|
+
case 'jpeg': return 'image/jpeg'
|
|
549
|
+
case 'gif': return 'image/gif'
|
|
550
|
+
case 'webp': return 'image/webp'
|
|
551
|
+
default: return 'application/octet-stream'
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/**
|
|
556
|
+
* Error codes a provider retries against the SAME provider within its
|
|
557
|
+
* `attempts` budget. Only transient failures are worth re-requesting: a
|
|
558
|
+
* timeout may clear on the next attempt and a 5xx / network drop is usually
|
|
559
|
+
* ephemeral. Deterministic failures (auth, quota, rate_limit, invalid_request,
|
|
560
|
+
* region, tos) must fail over to the next provider immediately instead of
|
|
561
|
+
* re-requesting a backend that cannot succeed with the same input.
|
|
562
|
+
*/
|
|
563
|
+
const RETRYABLE_CODES: ReadonlySet<VisionToolkitErrorCode> = new Set(['timeout', 'server', 'network'])
|
|
542
564
|
|
|
543
565
|
/** Resolve as soon as `signal` aborts (or immediately when already aborted). */
|
|
544
566
|
function untilAbort(signal: AbortSignal): Promise<void> {
|
|
@@ -771,6 +793,7 @@ export class VisionToolkitRuntime {
|
|
|
771
793
|
private readonly ctx: Context,
|
|
772
794
|
private readonly config: ResolvedVisionToolkitConfig,
|
|
773
795
|
adapter?: UpstreamAdapter,
|
|
796
|
+
private readonly readableStorageDirs: readonly string[] = [],
|
|
774
797
|
) {
|
|
775
798
|
this.adapter = adapter ?? new UpstreamAdapter(ctx, config)
|
|
776
799
|
}
|
|
@@ -785,6 +808,11 @@ export class VisionToolkitRuntime {
|
|
|
785
808
|
return this.config.sessionMaxConcurrency
|
|
786
809
|
}
|
|
787
810
|
|
|
811
|
+
/** Shared storage root belonging to this immutable runtime generation. */
|
|
812
|
+
get storageDirectory(): string | undefined {
|
|
813
|
+
return this.config.storageDir
|
|
814
|
+
}
|
|
815
|
+
|
|
788
816
|
/** Stable identity for persisted image descriptions produced by this runtime. */
|
|
789
817
|
get evidenceFingerprint(): string {
|
|
790
818
|
return evidenceRuntimeFingerprint(this.config, undefined, process.env.VISION_SSL_VERIFY?.trim())
|
|
@@ -952,6 +980,7 @@ export class VisionToolkitRuntime {
|
|
|
952
980
|
VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
|
|
953
981
|
VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
|
|
954
982
|
...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
|
|
983
|
+
...(provider.stream ? { VISION_STREAM: '1' } : {}),
|
|
955
984
|
VISION_USER_AGENT: provider.userAgent,
|
|
956
985
|
LANG: this.config.language,
|
|
957
986
|
}
|
|
@@ -968,6 +997,8 @@ export class VisionToolkitRuntime {
|
|
|
968
997
|
protocol: provider.protocol,
|
|
969
998
|
anthropicThinking: provider.anthropicThinking,
|
|
970
999
|
userAgent: provider.userAgent,
|
|
1000
|
+
stream: provider.stream,
|
|
1001
|
+
uploadViaUrl: provider.uploadViaUrl,
|
|
971
1002
|
})
|
|
972
1003
|
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
973
1004
|
: await this.ctx.credentials.resolve(provider.credential)
|
|
@@ -1010,13 +1041,13 @@ export class VisionToolkitRuntime {
|
|
|
1010
1041
|
}
|
|
1011
1042
|
|
|
1012
1043
|
private pathPolicy(workspace: string): Promise<PathPolicy> {
|
|
1013
|
-
return createPathPolicy(workspace, this.config.allowedDirs)
|
|
1044
|
+
return createPathPolicy(workspace, this.config.allowedDirs, this.config.storageDir, this.readableStorageDirs)
|
|
1014
1045
|
}
|
|
1015
1046
|
|
|
1016
1047
|
private async compressedImageRoot(policy: PathPolicy): Promise<string> {
|
|
1017
|
-
const root = join(policy.
|
|
1018
|
-
let current = policy.
|
|
1019
|
-
for (const segment of ['
|
|
1048
|
+
const root = join(policy.storageRoot, 'tmp', 'compressed-images')
|
|
1049
|
+
let current = policy.storageRoot
|
|
1050
|
+
for (const segment of ['tmp', 'compressed-images']) {
|
|
1020
1051
|
current = join(current, segment)
|
|
1021
1052
|
try {
|
|
1022
1053
|
await mkdir(current, { mode: 0o700 })
|
|
@@ -1027,13 +1058,13 @@ export class VisionToolkitRuntime {
|
|
|
1027
1058
|
if (info.isSymbolicLink() || !info.isDirectory()) {
|
|
1028
1059
|
throw new VisionToolkitError('path', `compressed-image cache path is not a real directory: ${current}`)
|
|
1029
1060
|
}
|
|
1030
|
-
if (!isWithin(policy.
|
|
1031
|
-
throw new VisionToolkitError('path', `compressed-image cache path escaped
|
|
1061
|
+
if (!isWithin(policy.storageRoot, current)) {
|
|
1062
|
+
throw new VisionToolkitError('path', `compressed-image cache path escaped plugin storage: ${current}`)
|
|
1032
1063
|
}
|
|
1033
1064
|
}
|
|
1034
1065
|
const canonical = await realpath(root)
|
|
1035
|
-
if (!isWithin(policy.
|
|
1036
|
-
throw new VisionToolkitError('path', 'compressed-image cache resolved outside
|
|
1066
|
+
if (!isWithin(policy.storageRoot, canonical)) {
|
|
1067
|
+
throw new VisionToolkitError('path', 'compressed-image cache resolved outside plugin storage')
|
|
1037
1068
|
}
|
|
1038
1069
|
return canonical
|
|
1039
1070
|
}
|
|
@@ -1237,6 +1268,84 @@ export class VisionToolkitRuntime {
|
|
|
1237
1268
|
operation.metrics.imagePixels += image.width * image.height
|
|
1238
1269
|
}
|
|
1239
1270
|
|
|
1271
|
+
/** Resolve the configured object storage into a usable client, or undefined. */
|
|
1272
|
+
private async resolveObjectStorageClient(): Promise<ObjectStorageClient | undefined> {
|
|
1273
|
+
const objectStorage = this.config.objectStorage
|
|
1274
|
+
if (objectStorage.credential === undefined || objectStorage.endpoint.length === 0 || objectStorage.bucket.length === 0) {
|
|
1275
|
+
return undefined
|
|
1276
|
+
}
|
|
1277
|
+
let resolved: ResolvedCredential | undefined
|
|
1278
|
+
try {
|
|
1279
|
+
resolved = await this.ctx.credentials.resolve(objectStorage.credential)
|
|
1280
|
+
} catch {
|
|
1281
|
+
resolved = undefined
|
|
1282
|
+
}
|
|
1283
|
+
if (resolved === undefined) return undefined
|
|
1284
|
+
let accessKeyId: string
|
|
1285
|
+
let secretAccessKey: string
|
|
1286
|
+
try {
|
|
1287
|
+
const split = splitObjectStorageCredential(resolved.value)
|
|
1288
|
+
accessKeyId = split.accessKeyId
|
|
1289
|
+
secretAccessKey = split.secretAccessKey
|
|
1290
|
+
} catch (error) {
|
|
1291
|
+
throw new VisionToolkitError('config', 'object storage credential is malformed', { cause: error })
|
|
1292
|
+
}
|
|
1293
|
+
const settings: ObjectStorageSettings = {
|
|
1294
|
+
endpoint: objectStorage.endpoint,
|
|
1295
|
+
bucket: objectStorage.bucket,
|
|
1296
|
+
accessKeyId,
|
|
1297
|
+
secretAccessKey,
|
|
1298
|
+
...(objectStorage.publicBase === undefined ? {} : { publicBase: objectStorage.publicBase }),
|
|
1299
|
+
}
|
|
1300
|
+
return new ObjectStorageClient(settings)
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
/**
|
|
1304
|
+
* Upload the prepared images to object storage and return their URLs plus a
|
|
1305
|
+
* cleanup callback, when the primary provider opts into URL transfer and
|
|
1306
|
+
* object storage is configured. Returns undefined otherwise (base64 path).
|
|
1307
|
+
*/
|
|
1308
|
+
private async maybeTransferImages(
|
|
1309
|
+
pool: readonly ResolvedProviderEnv[],
|
|
1310
|
+
images: readonly ImageInfo[],
|
|
1311
|
+
operation: OperationContext,
|
|
1312
|
+
): Promise<{ urls: string[]; cleanup: () => Promise<void> } | undefined> {
|
|
1313
|
+
const primary = pool[0]
|
|
1314
|
+
if (primary === undefined || primary.provider.uploadViaUrl !== true) return undefined
|
|
1315
|
+
const client = await this.resolveObjectStorageClient()
|
|
1316
|
+
if (client === undefined) {
|
|
1317
|
+
throw new VisionToolkitError('config', 'uploadViaUrl is enabled but object storage is not configured')
|
|
1318
|
+
}
|
|
1319
|
+
const keys: string[] = []
|
|
1320
|
+
const urls: string[] = []
|
|
1321
|
+
try {
|
|
1322
|
+
for (const image of images) {
|
|
1323
|
+
if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision image upload cancelled')
|
|
1324
|
+
const uploaded = await client.uploadImage(image.path, imageMimeType(image.format))
|
|
1325
|
+
keys.push(uploaded.key)
|
|
1326
|
+
urls.push(uploaded.url)
|
|
1327
|
+
}
|
|
1328
|
+
} catch (error) {
|
|
1329
|
+
await Promise.allSettled(keys.map(key => client.deleteObject(key)))
|
|
1330
|
+
throw error
|
|
1331
|
+
}
|
|
1332
|
+
return {
|
|
1333
|
+
urls,
|
|
1334
|
+
cleanup: async () => {
|
|
1335
|
+
await Promise.allSettled(keys.map(key => client.deleteObject(key)))
|
|
1336
|
+
},
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
/** Settings "test storage" probe: upload → head → delete a tiny marker object. */
|
|
1341
|
+
async testObjectStorage(): Promise<{ detail: string }> {
|
|
1342
|
+
const client = await this.resolveObjectStorageClient()
|
|
1343
|
+
if (client === undefined) {
|
|
1344
|
+
throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
|
|
1345
|
+
}
|
|
1346
|
+
return client.test()
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1240
1349
|
/** Stable gate key for one provider's in-flight request cap. */
|
|
1241
1350
|
private providerGate(provider: ResolvedProvider): Semaphore {
|
|
1242
1351
|
const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`
|
|
@@ -1383,6 +1492,22 @@ export class VisionToolkitRuntime {
|
|
|
1383
1492
|
return Math.max(0, Math.min(t2Remaining, globalRemaining))
|
|
1384
1493
|
}
|
|
1385
1494
|
|
|
1495
|
+
/**
|
|
1496
|
+
* Advance to the next provider after one provider reached a terminal
|
|
1497
|
+
* failure. This is what makes failover work for FAST failures too: the
|
|
1498
|
+
* hedge timer only launches the next provider when the current one is SLOW
|
|
1499
|
+
* (crosses t1), so a quick auth/5xx/network failure must explicitly launch
|
|
1500
|
+
* the successor. Never advances when a higher-priority provider superseded
|
|
1501
|
+
* this task, when the whole operation was cancelled, or when the global
|
|
1502
|
+
* deadline has too little room left for another request. `launch` is
|
|
1503
|
+
* idempotent, so an earlier hedge timer cannot cause a double launch.
|
|
1504
|
+
*/
|
|
1505
|
+
private advanceAfterFailure(task: ProviderTask, operation: OperationContext, launchNext: () => void): void {
|
|
1506
|
+
if (task.abort.signal.aborted || operation.signal.aborted) return
|
|
1507
|
+
if (operation.deadlineAt - Date.now() < this.config.minAvailableSeconds * 1000) return
|
|
1508
|
+
launchNext()
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1386
1511
|
/**
|
|
1387
1512
|
* Run one provider to a terminal state: retryable errors retry within
|
|
1388
1513
|
* `attempts`, a single request crossing t1 hedges the next provider, and the
|
|
@@ -1415,6 +1540,7 @@ export class VisionToolkitRuntime {
|
|
|
1415
1540
|
if (budget < minAvailableMs) {
|
|
1416
1541
|
task.status = 'failed'
|
|
1417
1542
|
task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} has insufficient remaining time`)
|
|
1543
|
+
this.advanceAfterFailure(task, operation, launchNext)
|
|
1418
1544
|
return
|
|
1419
1545
|
}
|
|
1420
1546
|
const reqDeadline = createDeadline(AbortSignal.any([operation.signal, task.abort.signal]), budget)
|
|
@@ -1453,6 +1579,7 @@ export class VisionToolkitRuntime {
|
|
|
1453
1579
|
if (reqDeadline.timedOut) {
|
|
1454
1580
|
task.status = 'failed'
|
|
1455
1581
|
task.error = new VisionToolkitError('timeout', `${tool}: ${provider.name} exhausted its t2 budget`)
|
|
1582
|
+
this.advanceAfterFailure(task, operation, launchNext)
|
|
1456
1583
|
return
|
|
1457
1584
|
}
|
|
1458
1585
|
if (classified.code === 'rate_limit') {
|
|
@@ -1467,6 +1594,7 @@ export class VisionToolkitRuntime {
|
|
|
1467
1594
|
}
|
|
1468
1595
|
task.status = 'failed'
|
|
1469
1596
|
task.error = classified
|
|
1597
|
+
this.advanceAfterFailure(task, operation, launchNext)
|
|
1470
1598
|
return
|
|
1471
1599
|
} finally {
|
|
1472
1600
|
reqDeadline.cleanup()
|
|
@@ -1568,6 +1696,8 @@ export class VisionToolkitRuntime {
|
|
|
1568
1696
|
protocol: env.VISION_API_PROTOCOL,
|
|
1569
1697
|
anthropicThinking: env.VISION_ANTHROPIC_THINKING,
|
|
1570
1698
|
sslVerify: env.VISION_SSL_VERIFY ?? null,
|
|
1699
|
+
stream: env.VISION_STREAM === '1',
|
|
1700
|
+
uploadViaUrl: provider.uploadViaUrl,
|
|
1571
1701
|
userAgent: env.VISION_USER_AGENT,
|
|
1572
1702
|
credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
|
|
1573
1703
|
maxImageBytes: provider.maxImageBytes,
|
|
@@ -1698,24 +1828,31 @@ export class VisionToolkitRuntime {
|
|
|
1698
1828
|
return cached.result
|
|
1699
1829
|
}
|
|
1700
1830
|
}
|
|
1701
|
-
const
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
images,
|
|
1711
|
-
|
|
1712
|
-
answer,
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1831
|
+
const transfer = request.region === undefined
|
|
1832
|
+
? await this.maybeTransferImages(pool, images, operation)
|
|
1833
|
+
: undefined
|
|
1834
|
+
try {
|
|
1835
|
+
const result = await this.runVisionHedge('glance', [
|
|
1836
|
+
...(transfer !== undefined ? transfer.urls : images.map(image => image.path)),
|
|
1837
|
+
...(transfer === undefined && request.region !== undefined ? ['--region', request.region] : []),
|
|
1838
|
+
...(request.ocr === true ? ['--ocr'] : []),
|
|
1839
|
+
...(request.query !== undefined ? ['-q', request.query] : []),
|
|
1840
|
+
], images, operation, pool)
|
|
1841
|
+
const answer = result.stdout.trim()
|
|
1842
|
+
if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
|
|
1843
|
+
const value: GlanceResult = {
|
|
1844
|
+
images,
|
|
1845
|
+
mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
|
|
1846
|
+
answer,
|
|
1847
|
+
truncated: false,
|
|
1848
|
+
}
|
|
1849
|
+
if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
|
|
1850
|
+
this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
|
|
1851
|
+
}
|
|
1852
|
+
return value
|
|
1853
|
+
} finally {
|
|
1854
|
+
if (transfer !== undefined) await transfer.cleanup()
|
|
1717
1855
|
}
|
|
1718
|
-
return value
|
|
1719
1856
|
})
|
|
1720
1857
|
}
|
|
1721
1858
|
|
|
@@ -1751,14 +1888,23 @@ export class VisionToolkitRuntime {
|
|
|
1751
1888
|
}
|
|
1752
1889
|
const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation)
|
|
1753
1890
|
this.accountImage(image, operation)
|
|
1754
|
-
const
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1891
|
+
const transfer = request.region === undefined
|
|
1892
|
+
? await this.maybeTransferImages(pool, [image], operation)
|
|
1893
|
+
: undefined
|
|
1894
|
+
try {
|
|
1895
|
+
const result = await this.runVisionHedge(tool, transfer !== undefined
|
|
1896
|
+
? [transfer.urls[0]!, request.target, '--size', `${image.width}x${image.height}`]
|
|
1897
|
+
: [
|
|
1898
|
+
image.path,
|
|
1899
|
+
request.target,
|
|
1900
|
+
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
1901
|
+
], [image], operation, pool)
|
|
1902
|
+
const elements = parseLocationOutput(result.stdout)
|
|
1903
|
+
this.validateLocations(elements, image.width, image.height)
|
|
1904
|
+
return { image, elements }
|
|
1905
|
+
} finally {
|
|
1906
|
+
if (transfer !== undefined) await transfer.cleanup()
|
|
1907
|
+
}
|
|
1762
1908
|
}
|
|
1763
1909
|
|
|
1764
1910
|
/** ground: locate one named target and return pixel boxes. */
|
|
@@ -2453,6 +2599,18 @@ export class VisionToolkitRuntime {
|
|
|
2453
2599
|
})
|
|
2454
2600
|
}
|
|
2455
2601
|
|
|
2602
|
+
private async writableDirectoryCheck(path: string, label: string): Promise<HealthCheck> {
|
|
2603
|
+
const probe = join(path, `.vision-toolkit-health-${randomUUID()}`)
|
|
2604
|
+
try {
|
|
2605
|
+
await writeFile(probe, 'ok\n', { encoding: 'utf8', flag: 'wx' })
|
|
2606
|
+
await rm(probe, { force: true })
|
|
2607
|
+
return { status: 'ok', detail: `${label} is writable: ${path}` }
|
|
2608
|
+
} catch {
|
|
2609
|
+
await rm(probe, { force: true }).catch(() => {})
|
|
2610
|
+
return { status: 'error', detail: `${label} is not writable: ${path}` }
|
|
2611
|
+
}
|
|
2612
|
+
}
|
|
2613
|
+
|
|
2456
2614
|
/** Health: inspect local readiness, and optionally probe one provider's `/models` plus one real multimodal request. */
|
|
2457
2615
|
async health(testConnection: boolean, options: ToolCallOptions, testModel = false, provider?: ResolvedProvider): Promise<VisionToolkitHealthResult> {
|
|
2458
2616
|
return this.runOperation('vision_toolkit_health', options, async (operation) => {
|
|
@@ -2474,6 +2632,27 @@ export class VisionToolkitRuntime {
|
|
|
2474
2632
|
if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision_toolkit_health: cancelled')
|
|
2475
2633
|
chrome = { status: 'error', detail: 'Chrome availability probe failed' }
|
|
2476
2634
|
}
|
|
2635
|
+
let resolvedCredential: ResolvedCredential | undefined
|
|
2636
|
+
let credential: HealthCheck
|
|
2637
|
+
try {
|
|
2638
|
+
resolvedCredential = isBuiltInFreeVisionProvider(this.config.provider)
|
|
2639
|
+
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
2640
|
+
: await this.ctx.credentials.resolve(this.config.provider.credential)
|
|
2641
|
+
credential = resolvedCredential === undefined
|
|
2642
|
+
? { status: 'error', detail: `credential ${this.config.provider.credential} is not configured` }
|
|
2643
|
+
: { status: 'ok', detail: `credential ${this.config.provider.credential} is resolvable` }
|
|
2644
|
+
} catch {
|
|
2645
|
+
credential = { status: 'error', detail: `credential ${this.config.provider.credential} could not be resolved` }
|
|
2646
|
+
}
|
|
2647
|
+
let artifactDirectory: HealthCheck
|
|
2648
|
+
try {
|
|
2649
|
+
// allowedDirs are session input roots; they do not affect output readiness.
|
|
2650
|
+
const policy = await createPathPolicy(options.workspace, [], this.config.storageDir)
|
|
2651
|
+
artifactDirectory = await this.writableDirectoryCheck(policy.outputDir, 'Artifact directory')
|
|
2652
|
+
} catch {
|
|
2653
|
+
artifactDirectory = { status: 'error', detail: 'Artifact directory could not be prepared' }
|
|
2654
|
+
}
|
|
2655
|
+
const tempDirectory = await this.writableDirectoryCheck(info.runtimeHome, 'Runtime temp directory')
|
|
2477
2656
|
let service: HealthCheck = {
|
|
2478
2657
|
status: 'not_tested',
|
|
2479
2658
|
detail: 'Connection was not tested; use the per-provider API test',
|
|
@@ -2564,7 +2743,7 @@ export class VisionToolkitRuntime {
|
|
|
2564
2743
|
}
|
|
2565
2744
|
}
|
|
2566
2745
|
}
|
|
2567
|
-
const checks = { python, dependencies, chrome, service, model }
|
|
2746
|
+
const checks = { python, dependencies, chrome, credential, artifactDirectory, tempDirectory, service, model }
|
|
2568
2747
|
const healthy = Object.values(checks).every(check => check.status !== 'error')
|
|
2569
2748
|
return {
|
|
2570
2749
|
pluginVersion: PLUGIN_VERSION,
|