@mengruo/dsh-vision-toolkit 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -70
- package/README.zh.md +11 -69
- package/lib/client.js +107 -6
- package/lib/client.js.map +1 -1
- package/lib/config.js +35 -0
- package/lib/config.js.map +1 -1
- package/lib/object-storage.js +141 -0
- package/lib/object-storage.js.map +1 -0
- package/lib/runtime.js +139 -26
- package/lib/runtime.js.map +1 -1
- package/lib/types/client/index.d.ts +39 -1
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/config.d.ts +33 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/object-storage.d.ts +54 -0
- package/lib/types/object-storage.d.ts.map +1 -0
- package/lib/types/runtime.d.ts +12 -0
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/upstream.d.ts +1 -0
- package/lib/types/upstream.d.ts.map +1 -1
- package/lib/types/web.d.ts +7 -0
- package/lib/types/web.d.ts.map +1 -1
- package/lib/upstream.js +3 -0
- package/lib/upstream.js.map +1 -1
- package/lib/web.js +35 -6
- package/lib/web.js.map +1 -1
- package/package.json +3 -1
- package/src/client/index.tsx +151 -6
- package/src/config.ts +67 -0
- package/src/object-storage.ts +174 -0
- package/src/runtime.ts +136 -25
- package/src/upstream.ts +4 -0
- package/src/web.ts +45 -7
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
- package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/glance +8 -1
- package/vendor/agent-vision-toolkit/detect.py +13 -7
- package/vendor/agent-vision-toolkit/ground.py +43 -18
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
- package/vendor/agent-vision-toolkit/vision_client.py +84 -6
- package/assets/community-group-qr.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/logo_eapi_dark.png +0 -0
- package/assets/wechat-reward.png +0 -0
package/src/config.ts
CHANGED
|
@@ -77,6 +77,10 @@ export interface VisionProviderConfig {
|
|
|
77
77
|
anthropicThinking?: 'omit' | 'disabled' | 'adaptive'
|
|
78
78
|
/** Outbound User-Agent for provider requests and connection tests. */
|
|
79
79
|
userAgent?: string
|
|
80
|
+
/** Whether to request a streamed (SSE) completion instead of one JSON response (default false). */
|
|
81
|
+
stream?: boolean
|
|
82
|
+
/** Whether to upload images to object storage and send the model a URL instead of base64 (default false). */
|
|
83
|
+
uploadViaUrl?: boolean
|
|
80
84
|
/** t1: per-request hedge threshold in seconds. A single request exceeding t1 keeps running while the next provider starts in parallel. */
|
|
81
85
|
t1Seconds?: number
|
|
82
86
|
/** t2: per-provider cumulative cutoff in seconds. Total accumulated request time reaching t2 terminates the provider. */
|
|
@@ -106,6 +110,10 @@ export interface VisionToolkitConfig {
|
|
|
106
110
|
anthropicThinking?: 'omit' | 'disabled' | 'adaptive'
|
|
107
111
|
/** Outbound User-Agent for provider requests and connection tests. */
|
|
108
112
|
userAgent?: string
|
|
113
|
+
/** Whether to request a streamed (SSE) completion instead of one JSON response (default false). */
|
|
114
|
+
stream?: boolean
|
|
115
|
+
/** Whether to upload images to object storage and send the model a URL instead of base64 (default false). */
|
|
116
|
+
uploadViaUrl?: boolean
|
|
109
117
|
}
|
|
110
118
|
/** Ordered online vision providers; array order is the failover priority. */
|
|
111
119
|
providers?: VisionProviderConfig[]
|
|
@@ -123,6 +131,21 @@ export interface VisionToolkitConfig {
|
|
|
123
131
|
maxImagePixels?: number
|
|
124
132
|
/** Default per-model in-flight request cap inherited by providers that do not set their own. */
|
|
125
133
|
concurrency?: number
|
|
134
|
+
/**
|
|
135
|
+
* Optional S3-compatible object storage used by the URL image-transfer path.
|
|
136
|
+
* `endpoint`, `bucket`, and `credential` are required to enable URL transfer;
|
|
137
|
+
* `publicBase` is optional and falls back to presigned URLs when unset.
|
|
138
|
+
*/
|
|
139
|
+
objectStorage?: {
|
|
140
|
+
/** S3-compatible API endpoint (e.g. R2, MinIO, Tencent COS). */
|
|
141
|
+
endpoint?: string
|
|
142
|
+
/** Bucket name. */
|
|
143
|
+
bucket?: string
|
|
144
|
+
/** DSH Credential reference holding "accessKeyId:secretAccessKey". */
|
|
145
|
+
credential?: string
|
|
146
|
+
/** Public base URL (custom domain / r2.dev); when unset, presigned URLs are used. */
|
|
147
|
+
publicBase?: string
|
|
148
|
+
}
|
|
126
149
|
runtime?: {
|
|
127
150
|
/** `managed` uses the packaged snapshot and isolated venv; `external` uses a clean pinned checkout. */
|
|
128
151
|
mode?: 'managed' | 'external'
|
|
@@ -182,6 +205,7 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
182
205
|
protocol: z.union(['openai', 'anthropic'] as const).default('openai'),
|
|
183
206
|
anthropicThinking: z.union(['omit', 'disabled', 'adaptive'] as const).default('omit'),
|
|
184
207
|
userAgent: z.string().default(DEFAULT_VISION_USER_AGENT),
|
|
208
|
+
stream: z.boolean().default(false),
|
|
185
209
|
}),
|
|
186
210
|
providers: z.array(z.object({
|
|
187
211
|
name: z.string(),
|
|
@@ -193,6 +217,8 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
193
217
|
protocol: z.union(['openai', 'anthropic'] as const).default('openai'),
|
|
194
218
|
anthropicThinking: z.union(['omit', 'disabled', 'adaptive'] as const).default('omit'),
|
|
195
219
|
userAgent: z.string(),
|
|
220
|
+
stream: z.boolean().default(false),
|
|
221
|
+
uploadViaUrl: z.boolean().default(false),
|
|
196
222
|
t1Seconds: z.number(),
|
|
197
223
|
t2Seconds: z.number(),
|
|
198
224
|
maxImageBytes: z.number(),
|
|
@@ -207,6 +233,12 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
207
233
|
maxImageBytes: z.number().default(4194304),
|
|
208
234
|
maxImagePixels: z.number().default(20000000),
|
|
209
235
|
concurrency: z.number().default(4),
|
|
236
|
+
objectStorage: z.object({
|
|
237
|
+
endpoint: z.string().default(''),
|
|
238
|
+
bucket: z.string().default(''),
|
|
239
|
+
credential: z.string().default(''),
|
|
240
|
+
publicBase: z.string().default(''),
|
|
241
|
+
}),
|
|
210
242
|
runtime: z.object({
|
|
211
243
|
mode: z.union(['managed', 'external'] as const).default('managed'),
|
|
212
244
|
agentVisionToolkitPath: z.string(),
|
|
@@ -235,6 +267,8 @@ export interface ResolvedProvider {
|
|
|
235
267
|
protocol: 'openai' | 'anthropic'
|
|
236
268
|
anthropicThinking: 'omit' | 'disabled' | 'adaptive'
|
|
237
269
|
userAgent: string
|
|
270
|
+
stream: boolean
|
|
271
|
+
uploadViaUrl: boolean
|
|
238
272
|
t1Seconds: number
|
|
239
273
|
t2Seconds: number
|
|
240
274
|
maxImageBytes: number
|
|
@@ -252,6 +286,8 @@ export interface ResolvedVisionToolkitConfig {
|
|
|
252
286
|
protocol: 'openai' | 'anthropic'
|
|
253
287
|
anthropicThinking: 'omit' | 'disabled' | 'adaptive'
|
|
254
288
|
userAgent: string
|
|
289
|
+
stream: boolean
|
|
290
|
+
uploadViaUrl: boolean
|
|
255
291
|
}
|
|
256
292
|
/** Ordered failover pool; array order is the priority, highest first. */
|
|
257
293
|
providers: ResolvedProvider[]
|
|
@@ -262,6 +298,12 @@ export interface ResolvedVisionToolkitConfig {
|
|
|
262
298
|
maxImageBytes: number
|
|
263
299
|
maxImagePixels: number
|
|
264
300
|
concurrency: number
|
|
301
|
+
objectStorage: {
|
|
302
|
+
endpoint: string
|
|
303
|
+
bucket: string
|
|
304
|
+
credential?: CredentialRef
|
|
305
|
+
publicBase?: string
|
|
306
|
+
}
|
|
265
307
|
runtime: {
|
|
266
308
|
mode: 'managed' | 'external'
|
|
267
309
|
agentVisionToolkitPath?: string
|
|
@@ -366,6 +408,8 @@ function resolveProvider(
|
|
|
366
408
|
if (userAgent.length === 0) {
|
|
367
409
|
throw new VisionToolkitError('config', `${label}.userAgent must not be empty`)
|
|
368
410
|
}
|
|
411
|
+
const stream = input.stream === true
|
|
412
|
+
const uploadViaUrl = input.uploadViaUrl === true
|
|
369
413
|
const t1Seconds = input.t1Seconds ?? 90
|
|
370
414
|
if (!Number.isInteger(t1Seconds) || t1Seconds < 1 || t1Seconds > MAX_TIMEOUT_SECONDS) {
|
|
371
415
|
throw new VisionToolkitError('config', `${label}.t1Seconds must be an integer between 1 and ${MAX_TIMEOUT_SECONDS}`)
|
|
@@ -402,6 +446,8 @@ function resolveProvider(
|
|
|
402
446
|
protocol,
|
|
403
447
|
anthropicThinking,
|
|
404
448
|
userAgent,
|
|
449
|
+
stream,
|
|
450
|
+
uploadViaUrl,
|
|
405
451
|
t1Seconds,
|
|
406
452
|
t2Seconds,
|
|
407
453
|
maxImageBytes,
|
|
@@ -472,6 +518,19 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
472
518
|
.map(dir => dir.trim())
|
|
473
519
|
.filter(dir => dir.length > 0 && dir !== storageDir))]
|
|
474
520
|
const allowedDirs = (config.allowedDirs ?? []).map(dir => dir.trim()).filter(dir => dir.length > 0)
|
|
521
|
+
const objectStorageInput = config.objectStorage ?? {}
|
|
522
|
+
const objectStorageEndpoint = objectStorageInput.endpoint?.trim() ?? ''
|
|
523
|
+
const objectStorageBucket = objectStorageInput.bucket?.trim() ?? ''
|
|
524
|
+
const objectStoragePublicBase = objectStorageInput.publicBase?.trim()
|
|
525
|
+
let objectStorageCredential: CredentialRef | undefined
|
|
526
|
+
const objectStorageCredentialSource = objectStorageInput.credential?.trim()
|
|
527
|
+
if (objectStorageCredentialSource !== undefined && objectStorageCredentialSource.length > 0) {
|
|
528
|
+
try {
|
|
529
|
+
objectStorageCredential = credentialRef(objectStorageCredentialSource)
|
|
530
|
+
} catch (error) {
|
|
531
|
+
throw new VisionToolkitError('config', `objectStorage.credential "${objectStorageCredentialSource}" is not a valid credential reference`, { cause: error })
|
|
532
|
+
}
|
|
533
|
+
}
|
|
475
534
|
const imageInputVariants = config.imageInputVariants ?? {}
|
|
476
535
|
const variantProviders = (imageInputVariants.providers ?? [])
|
|
477
536
|
.map(provider => provider.trim())
|
|
@@ -494,6 +553,8 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
494
553
|
protocol: primary.protocol,
|
|
495
554
|
anthropicThinking: primary.anthropicThinking,
|
|
496
555
|
userAgent: primary.userAgent,
|
|
556
|
+
stream: primary.stream,
|
|
557
|
+
uploadViaUrl: primary.uploadViaUrl,
|
|
497
558
|
},
|
|
498
559
|
providers,
|
|
499
560
|
language,
|
|
@@ -503,6 +564,12 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
503
564
|
maxImageBytes,
|
|
504
565
|
maxImagePixels,
|
|
505
566
|
concurrency,
|
|
567
|
+
objectStorage: {
|
|
568
|
+
endpoint: objectStorageEndpoint,
|
|
569
|
+
bucket: objectStorageBucket,
|
|
570
|
+
...(objectStorageCredential === undefined ? {} : { credential: objectStorageCredential }),
|
|
571
|
+
...(objectStoragePublicBase === undefined || objectStoragePublicBase.length === 0 ? {} : { publicBase: objectStoragePublicBase }),
|
|
572
|
+
},
|
|
506
573
|
runtime: {
|
|
507
574
|
mode,
|
|
508
575
|
...(toolkitPath !== undefined ? { agentVisionToolkitPath: toolkitPath } : {}),
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal S3-compatible object storage bridge used by the URL image-transfer
|
|
3
|
+
* path. It uploads one image, resolves a model-reachable URL (a configured
|
|
4
|
+
* public base URL or a temporary presigned URL), and deletes the object after
|
|
5
|
+
* the vision operation settles. It also provides the Settings "test storage"
|
|
6
|
+
* probe (upload → head → delete).
|
|
7
|
+
* @module dsh-vision-toolkit/object-storage
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
11
|
+
import { readFile } from 'node:fs/promises'
|
|
12
|
+
import { basename } from 'node:path'
|
|
13
|
+
import {
|
|
14
|
+
DeleteObjectCommand,
|
|
15
|
+
GetObjectCommand,
|
|
16
|
+
HeadObjectCommand,
|
|
17
|
+
PutObjectCommand,
|
|
18
|
+
S3Client,
|
|
19
|
+
type S3ClientConfig,
|
|
20
|
+
} from '@aws-sdk/client-s3'
|
|
21
|
+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
|
|
22
|
+
import { VisionToolkitError } from './errors.ts'
|
|
23
|
+
|
|
24
|
+
/** Fully resolved object-storage connection settings (secrets already filled). */
|
|
25
|
+
export interface ObjectStorageSettings {
|
|
26
|
+
endpoint: string
|
|
27
|
+
bucket: string
|
|
28
|
+
accessKeyId: string
|
|
29
|
+
secretAccessKey: string
|
|
30
|
+
publicBase?: string
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Whether the required connection fields are present enough to attempt a request. */
|
|
34
|
+
export function isObjectStorageConfigured(settings: ObjectStorageSettings): boolean {
|
|
35
|
+
return settings.endpoint.length > 0
|
|
36
|
+
&& settings.bucket.length > 0
|
|
37
|
+
&& settings.accessKeyId.length > 0
|
|
38
|
+
&& settings.secretAccessKey.length > 0
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Stable object-key prefix so every upload lives under one deletable namespace. */
|
|
42
|
+
const OBJECT_KEY_PREFIX = 'dsh-vision-toolkit'
|
|
43
|
+
|
|
44
|
+
function encodeKey(key: string): string {
|
|
45
|
+
return key.split('/').map(encodeURIComponent).join('/')
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function clientFor(settings: ObjectStorageSettings): S3Client {
|
|
49
|
+
const config: S3ClientConfig = {
|
|
50
|
+
region: 'auto',
|
|
51
|
+
forcePathStyle: true,
|
|
52
|
+
credentials: {
|
|
53
|
+
accessKeyId: settings.accessKeyId,
|
|
54
|
+
secretAccessKey: settings.secretAccessKey,
|
|
55
|
+
},
|
|
56
|
+
}
|
|
57
|
+
if (settings.endpoint.length > 0) config.endpoint = settings.endpoint
|
|
58
|
+
return new S3Client(config)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function publicError(error: unknown): string {
|
|
62
|
+
if (error instanceof Error) return error.message
|
|
63
|
+
return String(error)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* One upload's worth of bookkeeping: the object key and the URL handed to the
|
|
68
|
+
* model. The key is returned to the runtime so it can delete the object after
|
|
69
|
+
* the operation settles.
|
|
70
|
+
*/
|
|
71
|
+
export interface UploadedObject {
|
|
72
|
+
key: string
|
|
73
|
+
url: string
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** A small S3-compatible object store bound to one bucket and credential. */
|
|
77
|
+
export class ObjectStorageClient {
|
|
78
|
+
private client?: S3Client
|
|
79
|
+
|
|
80
|
+
constructor(private readonly settings: ObjectStorageSettings) {}
|
|
81
|
+
|
|
82
|
+
private requireClient(): S3Client {
|
|
83
|
+
if (this.client === undefined) this.client = clientFor(this.settings)
|
|
84
|
+
return this.client
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Upload one local image file and resolve its model-reachable URL. */
|
|
88
|
+
async uploadImage(localPath: string, contentType: string): Promise<UploadedObject> {
|
|
89
|
+
const body = await readFile(localPath)
|
|
90
|
+
const digest = createHash('sha256').update(body).digest('hex').slice(0, 12)
|
|
91
|
+
const name = basename(localPath).replace(/[^A-Za-z0-9._-]/g, '_')
|
|
92
|
+
const key = `${OBJECT_KEY_PREFIX}/${randomUUID()}-${digest}-${name}`
|
|
93
|
+
try {
|
|
94
|
+
await this.requireClient().send(new PutObjectCommand({
|
|
95
|
+
Bucket: this.settings.bucket,
|
|
96
|
+
Key: key,
|
|
97
|
+
Body: body,
|
|
98
|
+
ContentType: contentType,
|
|
99
|
+
}))
|
|
100
|
+
} catch (error) {
|
|
101
|
+
throw new VisionToolkitError('service', `object storage upload failed: ${publicError(error)}`, { cause: error })
|
|
102
|
+
}
|
|
103
|
+
return { key, url: await this.urlFor(key) }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Resolve the model-reachable URL: public base URL when set, else presigned. */
|
|
107
|
+
async urlFor(key: string): Promise<string> {
|
|
108
|
+
if (this.settings.publicBase !== undefined && this.settings.publicBase.length > 0) {
|
|
109
|
+
return `${this.settings.publicBase}/${encodeKey(key)}`
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
return await getSignedUrl(
|
|
113
|
+
this.requireClient(),
|
|
114
|
+
new GetObjectCommand({ Bucket: this.settings.bucket, Key: key }),
|
|
115
|
+
{ expiresIn: 3600 },
|
|
116
|
+
)
|
|
117
|
+
} catch (error) {
|
|
118
|
+
throw new VisionToolkitError('service', `object storage presign failed: ${publicError(error)}`, { cause: error })
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Delete one uploaded object; failures are logged, never fatal to the call. */
|
|
123
|
+
async deleteObject(key: string): Promise<void> {
|
|
124
|
+
try {
|
|
125
|
+
await this.requireClient().send(new DeleteObjectCommand({
|
|
126
|
+
Bucket: this.settings.bucket,
|
|
127
|
+
Key: key,
|
|
128
|
+
}))
|
|
129
|
+
} catch {
|
|
130
|
+
// Best-effort cleanup: a failed delete must not mask the vision result.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Settings "test storage" probe: upload a tiny object, head it, then delete it. */
|
|
135
|
+
async test(): Promise<{ detail: string }> {
|
|
136
|
+
if (!isObjectStorageConfigured(this.settings)) {
|
|
137
|
+
throw new VisionToolkitError('config', 'object storage is not fully configured (endpoint, bucket, access key id, and secret access key are required)')
|
|
138
|
+
}
|
|
139
|
+
const key = `${OBJECT_KEY_PREFIX}/.connection-test-${randomUUID()}`
|
|
140
|
+
const marker = `dsh-vision-toolkit object storage test ${Date.now()}`
|
|
141
|
+
try {
|
|
142
|
+
await this.requireClient().send(new PutObjectCommand({
|
|
143
|
+
Bucket: this.settings.bucket,
|
|
144
|
+
Key: key,
|
|
145
|
+
Body: marker,
|
|
146
|
+
ContentType: 'text/plain',
|
|
147
|
+
}))
|
|
148
|
+
await this.requireClient().send(new HeadObjectCommand({ Bucket: this.settings.bucket, Key: key }))
|
|
149
|
+
await this.requireClient().send(new DeleteObjectCommand({ Bucket: this.settings.bucket, Key: key }))
|
|
150
|
+
} catch (error) {
|
|
151
|
+
throw new VisionToolkitError('service', `object storage test failed: ${publicError(error)}`, { cause: error })
|
|
152
|
+
}
|
|
153
|
+
const urlMode = this.settings.publicBase !== undefined && this.settings.publicBase.length > 0
|
|
154
|
+
? `public base ${this.settings.publicBase}`
|
|
155
|
+
: 'presigned URL'
|
|
156
|
+
return { detail: `bucket ${this.settings.bucket} reachable; model URLs will use ${urlMode}` }
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Split a credential value of the form `accessKeyId:secretAccessKey` into its
|
|
162
|
+
* two parts. The access key id never contains a colon, so splitting on the
|
|
163
|
+
* first colon is safe.
|
|
164
|
+
*/
|
|
165
|
+
export function splitObjectStorageCredential(value: string): { accessKeyId: string; secretAccessKey: string } {
|
|
166
|
+
const index = value.indexOf(':')
|
|
167
|
+
if (index <= 0) {
|
|
168
|
+
throw new VisionToolkitError('config', 'object storage credential must be "accessKeyId:secretAccessKey"')
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
accessKeyId: value.slice(0, index),
|
|
172
|
+
secretAccessKey: value.slice(index + 1),
|
|
173
|
+
}
|
|
174
|
+
}
|
package/src/runtime.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { isBuiltInFreeVisionProvider, type ResolvedProvider, type ResolvedVision
|
|
|
18
18
|
import { BUILT_IN_FREE_VISION_KEY } from './defaults.ts'
|
|
19
19
|
import { evidenceRuntimeFingerprint } from './evidence-cache.ts'
|
|
20
20
|
import { VisionToolkitError, type VisionToolkitErrorCode } from './errors.ts'
|
|
21
|
+
import { ObjectStorageClient, splitObjectStorageCredential, type ObjectStorageSettings } from './object-storage.ts'
|
|
21
22
|
import {
|
|
22
23
|
assertDistinctOutput,
|
|
23
24
|
commitStagedDirectory,
|
|
@@ -540,6 +541,17 @@ const FORMAT_BY_EXTENSION = new Map([
|
|
|
540
541
|
])
|
|
541
542
|
const HEX_COLOR_PATTERN = /^#[0-9A-F]{6}$/
|
|
542
543
|
|
|
544
|
+
/** MIME type for one analyzed image format, used when uploading to object storage. */
|
|
545
|
+
function imageMimeType(format: string): string {
|
|
546
|
+
switch (format) {
|
|
547
|
+
case 'png': return 'image/png'
|
|
548
|
+
case 'jpeg': return 'image/jpeg'
|
|
549
|
+
case 'gif': return 'image/gif'
|
|
550
|
+
case 'webp': return 'image/webp'
|
|
551
|
+
default: return 'application/octet-stream'
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
543
555
|
/**
|
|
544
556
|
* Error codes a provider retries against the SAME provider within its
|
|
545
557
|
* `attempts` budget. Only transient failures are worth re-requesting: a
|
|
@@ -968,6 +980,7 @@ export class VisionToolkitRuntime {
|
|
|
968
980
|
VISION_API_PROTOCOL: provider.protocol === 'anthropic' ? 'anthropic' : 'chat_completions',
|
|
969
981
|
VISION_ANTHROPIC_THINKING: provider.anthropicThinking,
|
|
970
982
|
...(sslVerify === undefined ? {} : { VISION_SSL_VERIFY: sslVerify }),
|
|
983
|
+
...(provider.stream ? { VISION_STREAM: '1' } : {}),
|
|
971
984
|
VISION_USER_AGENT: provider.userAgent,
|
|
972
985
|
LANG: this.config.language,
|
|
973
986
|
}
|
|
@@ -984,6 +997,8 @@ export class VisionToolkitRuntime {
|
|
|
984
997
|
protocol: provider.protocol,
|
|
985
998
|
anthropicThinking: provider.anthropicThinking,
|
|
986
999
|
userAgent: provider.userAgent,
|
|
1000
|
+
stream: provider.stream,
|
|
1001
|
+
uploadViaUrl: provider.uploadViaUrl,
|
|
987
1002
|
})
|
|
988
1003
|
? { value: BUILT_IN_FREE_VISION_KEY, source: 'built-in' }
|
|
989
1004
|
: await this.ctx.credentials.resolve(provider.credential)
|
|
@@ -1253,6 +1268,84 @@ export class VisionToolkitRuntime {
|
|
|
1253
1268
|
operation.metrics.imagePixels += image.width * image.height
|
|
1254
1269
|
}
|
|
1255
1270
|
|
|
1271
|
+
/** Resolve the configured object storage into a usable client, or undefined. */
|
|
1272
|
+
private async resolveObjectStorageClient(): Promise<ObjectStorageClient | undefined> {
|
|
1273
|
+
const objectStorage = this.config.objectStorage
|
|
1274
|
+
if (objectStorage.credential === undefined || objectStorage.endpoint.length === 0 || objectStorage.bucket.length === 0) {
|
|
1275
|
+
return undefined
|
|
1276
|
+
}
|
|
1277
|
+
let resolved: ResolvedCredential | undefined
|
|
1278
|
+
try {
|
|
1279
|
+
resolved = await this.ctx.credentials.resolve(objectStorage.credential)
|
|
1280
|
+
} catch {
|
|
1281
|
+
resolved = undefined
|
|
1282
|
+
}
|
|
1283
|
+
if (resolved === undefined) return undefined
|
|
1284
|
+
let accessKeyId: string
|
|
1285
|
+
let secretAccessKey: string
|
|
1286
|
+
try {
|
|
1287
|
+
const split = splitObjectStorageCredential(resolved.value)
|
|
1288
|
+
accessKeyId = split.accessKeyId
|
|
1289
|
+
secretAccessKey = split.secretAccessKey
|
|
1290
|
+
} catch (error) {
|
|
1291
|
+
throw new VisionToolkitError('config', 'object storage credential is malformed', { cause: error })
|
|
1292
|
+
}
|
|
1293
|
+
const settings: ObjectStorageSettings = {
|
|
1294
|
+
endpoint: objectStorage.endpoint,
|
|
1295
|
+
bucket: objectStorage.bucket,
|
|
1296
|
+
accessKeyId,
|
|
1297
|
+
secretAccessKey,
|
|
1298
|
+
...(objectStorage.publicBase === undefined ? {} : { publicBase: objectStorage.publicBase }),
|
|
1299
|
+
}
|
|
1300
|
+
return new ObjectStorageClient(settings)
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
/**
|
|
1304
|
+
* Upload the prepared images to object storage and return their URLs plus a
|
|
1305
|
+
* cleanup callback, when the primary provider opts into URL transfer and
|
|
1306
|
+
* object storage is configured. Returns undefined otherwise (base64 path).
|
|
1307
|
+
*/
|
|
1308
|
+
private async maybeTransferImages(
|
|
1309
|
+
pool: readonly ResolvedProviderEnv[],
|
|
1310
|
+
images: readonly ImageInfo[],
|
|
1311
|
+
operation: OperationContext,
|
|
1312
|
+
): Promise<{ urls: string[]; cleanup: () => Promise<void> } | undefined> {
|
|
1313
|
+
const primary = pool[0]
|
|
1314
|
+
if (primary === undefined || primary.provider.uploadViaUrl !== true) return undefined
|
|
1315
|
+
const client = await this.resolveObjectStorageClient()
|
|
1316
|
+
if (client === undefined) {
|
|
1317
|
+
throw new VisionToolkitError('config', 'uploadViaUrl is enabled but object storage is not configured')
|
|
1318
|
+
}
|
|
1319
|
+
const keys: string[] = []
|
|
1320
|
+
const urls: string[] = []
|
|
1321
|
+
try {
|
|
1322
|
+
for (const image of images) {
|
|
1323
|
+
if (operation.signal.aborted) throw new VisionToolkitError('cancelled', 'vision image upload cancelled')
|
|
1324
|
+
const uploaded = await client.uploadImage(image.path, imageMimeType(image.format))
|
|
1325
|
+
keys.push(uploaded.key)
|
|
1326
|
+
urls.push(uploaded.url)
|
|
1327
|
+
}
|
|
1328
|
+
} catch (error) {
|
|
1329
|
+
await Promise.allSettled(keys.map(key => client.deleteObject(key)))
|
|
1330
|
+
throw error
|
|
1331
|
+
}
|
|
1332
|
+
return {
|
|
1333
|
+
urls,
|
|
1334
|
+
cleanup: async () => {
|
|
1335
|
+
await Promise.allSettled(keys.map(key => client.deleteObject(key)))
|
|
1336
|
+
},
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
/** Settings "test storage" probe: upload → head → delete a tiny marker object. */
|
|
1341
|
+
async testObjectStorage(): Promise<{ detail: string }> {
|
|
1342
|
+
const client = await this.resolveObjectStorageClient()
|
|
1343
|
+
if (client === undefined) {
|
|
1344
|
+
throw new VisionToolkitError('config', 'object storage is not configured (endpoint, bucket, and credential are required)')
|
|
1345
|
+
}
|
|
1346
|
+
return client.test()
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1256
1349
|
/** Stable gate key for one provider's in-flight request cap. */
|
|
1257
1350
|
private providerGate(provider: ResolvedProvider): Semaphore {
|
|
1258
1351
|
const key = `${provider.baseUrl}\u0000${provider.model}\u0000${String(provider.credential)}`
|
|
@@ -1603,6 +1696,8 @@ export class VisionToolkitRuntime {
|
|
|
1603
1696
|
protocol: env.VISION_API_PROTOCOL,
|
|
1604
1697
|
anthropicThinking: env.VISION_ANTHROPIC_THINKING,
|
|
1605
1698
|
sslVerify: env.VISION_SSL_VERIFY ?? null,
|
|
1699
|
+
stream: env.VISION_STREAM === '1',
|
|
1700
|
+
uploadViaUrl: provider.uploadViaUrl,
|
|
1606
1701
|
userAgent: env.VISION_USER_AGENT,
|
|
1607
1702
|
credentialSha256: createHash('sha256').update(env.VISION_API_KEY).digest('hex'),
|
|
1608
1703
|
maxImageBytes: provider.maxImageBytes,
|
|
@@ -1733,24 +1828,31 @@ export class VisionToolkitRuntime {
|
|
|
1733
1828
|
return cached.result
|
|
1734
1829
|
}
|
|
1735
1830
|
}
|
|
1736
|
-
const
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
images,
|
|
1746
|
-
|
|
1747
|
-
answer,
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1831
|
+
const transfer = request.region === undefined
|
|
1832
|
+
? await this.maybeTransferImages(pool, images, operation)
|
|
1833
|
+
: undefined
|
|
1834
|
+
try {
|
|
1835
|
+
const result = await this.runVisionHedge('glance', [
|
|
1836
|
+
...(transfer !== undefined ? transfer.urls : images.map(image => image.path)),
|
|
1837
|
+
...(transfer === undefined && request.region !== undefined ? ['--region', request.region] : []),
|
|
1838
|
+
...(request.ocr === true ? ['--ocr'] : []),
|
|
1839
|
+
...(request.query !== undefined ? ['-q', request.query] : []),
|
|
1840
|
+
], images, operation, pool)
|
|
1841
|
+
const answer = result.stdout.trim()
|
|
1842
|
+
if (answer.length === 0) throw new VisionToolkitError('output', 'glance: vision API returned an empty description')
|
|
1843
|
+
const value: GlanceResult = {
|
|
1844
|
+
images,
|
|
1845
|
+
mode: request.ocr === true ? 'ocr' : request.query !== undefined ? 'qa' : 'describe',
|
|
1846
|
+
answer,
|
|
1847
|
+
truncated: false,
|
|
1848
|
+
}
|
|
1849
|
+
if (options.sessionScope !== undefined && cacheKey !== undefined && !operation.signal.aborted) {
|
|
1850
|
+
this.glanceCache.set(options.sessionScope, { key: cacheKey, result: value })
|
|
1851
|
+
}
|
|
1852
|
+
return value
|
|
1853
|
+
} finally {
|
|
1854
|
+
if (transfer !== undefined) await transfer.cleanup()
|
|
1752
1855
|
}
|
|
1753
|
-
return value
|
|
1754
1856
|
})
|
|
1755
1857
|
}
|
|
1756
1858
|
|
|
@@ -1786,14 +1888,23 @@ export class VisionToolkitRuntime {
|
|
|
1786
1888
|
}
|
|
1787
1889
|
const image = await this.prepareVisionImage(request.image, pool.map(entry => entry.provider), policy, operation)
|
|
1788
1890
|
this.accountImage(image, operation)
|
|
1789
|
-
const
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1891
|
+
const transfer = request.region === undefined
|
|
1892
|
+
? await this.maybeTransferImages(pool, [image], operation)
|
|
1893
|
+
: undefined
|
|
1894
|
+
try {
|
|
1895
|
+
const result = await this.runVisionHedge(tool, transfer !== undefined
|
|
1896
|
+
? [transfer.urls[0]!, request.target, '--size', `${image.width}x${image.height}`]
|
|
1897
|
+
: [
|
|
1898
|
+
image.path,
|
|
1899
|
+
request.target,
|
|
1900
|
+
...(request.region !== undefined ? ['--region', request.region] : []),
|
|
1901
|
+
], [image], operation, pool)
|
|
1902
|
+
const elements = parseLocationOutput(result.stdout)
|
|
1903
|
+
this.validateLocations(elements, image.width, image.height)
|
|
1904
|
+
return { image, elements }
|
|
1905
|
+
} finally {
|
|
1906
|
+
if (transfer !== undefined) await transfer.cleanup()
|
|
1907
|
+
}
|
|
1797
1908
|
}
|
|
1798
1909
|
|
|
1799
1910
|
/** ground: locate one named target and return pixel boxes. */
|
package/src/upstream.ts
CHANGED
|
@@ -41,6 +41,7 @@ export interface UpstreamEnvironment {
|
|
|
41
41
|
VISION_API_PROTOCOL: 'chat_completions' | 'anthropic'
|
|
42
42
|
VISION_ANTHROPIC_THINKING: 'omit' | 'disabled' | 'adaptive'
|
|
43
43
|
VISION_SSL_VERIFY?: string
|
|
44
|
+
VISION_STREAM?: string
|
|
44
45
|
VISION_USER_AGENT: string
|
|
45
46
|
LANG: 'zh' | 'en'
|
|
46
47
|
}
|
|
@@ -761,6 +762,9 @@ export class UpstreamAdapter {
|
|
|
761
762
|
...(options.env.VISION_SSL_VERIFY === undefined
|
|
762
763
|
? {}
|
|
763
764
|
: { VISION_SSL_VERIFY: options.env.VISION_SSL_VERIFY }),
|
|
765
|
+
...(options.env.VISION_STREAM === undefined
|
|
766
|
+
? {}
|
|
767
|
+
: { VISION_STREAM: options.env.VISION_STREAM }),
|
|
764
768
|
VISION_USER_AGENT: options.env.VISION_USER_AGENT,
|
|
765
769
|
LANG: options.env.LANG,
|
|
766
770
|
VISION_ENV_FILE: join(prepared.cleanHome, 'vision.env'),
|