@mengruo/dsh-vision-toolkit 0.1.4 → 0.1.6-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -70
- package/README.zh.md +11 -69
- package/assets/1.mp4 +0 -0
- package/assets/skill/SKILL.md +37 -5
- package/docs/plan-per-tool-visibility.md +82 -0
- package/lib/client.js +157 -6
- package/lib/client.js.map +1 -1
- package/lib/config.js +52 -0
- package/lib/config.js.map +1 -1
- package/lib/exposure.js +35 -8
- package/lib/exposure.js.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/object-storage.js +141 -0
- package/lib/object-storage.js.map +1 -0
- package/lib/paths.js +9 -0
- package/lib/paths.js.map +1 -1
- package/lib/runtime.js +314 -27
- package/lib/runtime.js.map +1 -1
- package/lib/tools.js +116 -4
- package/lib/tools.js.map +1 -1
- package/lib/types/client/index.d.ts +62 -1
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/config.d.ts +59 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/exposure.d.ts +14 -2
- package/lib/types/exposure.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/object-storage.d.ts +54 -0
- package/lib/types/object-storage.d.ts.map +1 -0
- package/lib/types/paths.d.ts +7 -0
- package/lib/types/paths.d.ts.map +1 -1
- package/lib/types/runtime.d.ts +50 -0
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/tools.d.ts +22 -1
- package/lib/types/tools.d.ts.map +1 -1
- package/lib/types/upstream.d.ts +1 -0
- package/lib/types/upstream.d.ts.map +1 -1
- package/lib/types/video.d.ts +78 -0
- package/lib/types/video.d.ts.map +1 -0
- package/lib/types/web.d.ts +8 -0
- package/lib/types/web.d.ts.map +1 -1
- package/lib/upstream.js +3 -0
- package/lib/upstream.js.map +1 -1
- package/lib/video.js +169 -0
- package/lib/video.js.map +1 -0
- package/lib/web.js +78 -6
- package/lib/web.js.map +1 -1
- package/package.json +4 -1
- package/src/client/index.tsx +222 -6
- package/src/config.ts +110 -0
- package/src/exposure.ts +34 -9
- package/src/index.ts +10 -5
- package/src/object-storage.ts +174 -0
- package/src/paths.ts +11 -0
- package/src/runtime.ts +332 -25
- package/src/tools.ts +144 -3
- package/src/upstream.ts +4 -0
- package/src/video.ts +222 -0
- package/src/web.ts +93 -7
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
- package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/glance +8 -1
- package/vendor/agent-vision-toolkit/detect.py +13 -7
- package/vendor/agent-vision-toolkit/ground.py +43 -18
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
- package/vendor/agent-vision-toolkit/vision_client.py +84 -6
- package/assets/community-group-qr.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/logo_eapi_dark.png +0 -0
- package/assets/wechat-reward.png +0 -0
package/src/config.ts
CHANGED
|
@@ -77,6 +77,12 @@ export interface VisionProviderConfig {
|
|
|
77
77
|
anthropicThinking?: 'omit' | 'disabled' | 'adaptive'
|
|
78
78
|
/** Outbound User-Agent for provider requests and connection tests. */
|
|
79
79
|
userAgent?: string
|
|
80
|
+
/** Whether to request a streamed (SSE) completion instead of one JSON response (default false). */
|
|
81
|
+
stream?: boolean
|
|
82
|
+
/** Whether to upload images to object storage and send the model a URL instead of base64 (default false). */
|
|
83
|
+
uploadViaUrl?: boolean
|
|
84
|
+
/** OpenAI-compatible video understanding (default false); Anthropic providers ignore this placeholder flag. */
|
|
85
|
+
videoSupport?: boolean
|
|
80
86
|
/** t1: per-request hedge threshold in seconds. A single request exceeding t1 keeps running while the next provider starts in parallel. */
|
|
81
87
|
t1Seconds?: number
|
|
82
88
|
/** t2: per-provider cumulative cutoff in seconds. Total accumulated request time reaching t2 terminates the provider. */
|
|
@@ -106,6 +112,12 @@ export interface VisionToolkitConfig {
|
|
|
106
112
|
anthropicThinking?: 'omit' | 'disabled' | 'adaptive'
|
|
107
113
|
/** Outbound User-Agent for provider requests and connection tests. */
|
|
108
114
|
userAgent?: string
|
|
115
|
+
/** Whether to request a streamed (SSE) completion instead of one JSON response (default false). */
|
|
116
|
+
stream?: boolean
|
|
117
|
+
/** Whether to upload images to object storage and send the model a URL instead of base64 (default false). */
|
|
118
|
+
uploadViaUrl?: boolean
|
|
119
|
+
/** OpenAI-compatible video understanding (default false); Anthropic providers ignore this placeholder flag. */
|
|
120
|
+
videoSupport?: boolean
|
|
109
121
|
}
|
|
110
122
|
/** Ordered online vision providers; array order is the failover priority. */
|
|
111
123
|
providers?: VisionProviderConfig[]
|
|
@@ -123,6 +135,21 @@ export interface VisionToolkitConfig {
|
|
|
123
135
|
maxImagePixels?: number
|
|
124
136
|
/** Default per-model in-flight request cap inherited by providers that do not set their own. */
|
|
125
137
|
concurrency?: number
|
|
138
|
+
/**
|
|
139
|
+
* Optional S3-compatible object storage used by the URL image-transfer path.
|
|
140
|
+
* `endpoint`, `bucket`, and `credential` are required to enable URL transfer;
|
|
141
|
+
* `publicBase` is optional and falls back to presigned URLs when unset.
|
|
142
|
+
*/
|
|
143
|
+
objectStorage?: {
|
|
144
|
+
/** S3-compatible API endpoint (e.g. R2, MinIO, Tencent COS). */
|
|
145
|
+
endpoint?: string
|
|
146
|
+
/** Bucket name. */
|
|
147
|
+
bucket?: string
|
|
148
|
+
/** DSH Credential reference holding "accessKeyId:secretAccessKey". */
|
|
149
|
+
credential?: string
|
|
150
|
+
/** Public base URL (custom domain / r2.dev); when unset, presigned URLs are used. */
|
|
151
|
+
publicBase?: string
|
|
152
|
+
}
|
|
126
153
|
runtime?: {
|
|
127
154
|
/** `managed` uses the packaged snapshot and isolated venv; `external` uses a clean pinned checkout. */
|
|
128
155
|
mode?: 'managed' | 'external'
|
|
@@ -171,6 +198,21 @@ export interface VisionToolkitConfig {
|
|
|
171
198
|
*/
|
|
172
199
|
hidden?: boolean
|
|
173
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* Per-group tool visibility, applied when each Agent's visual tool set is
|
|
203
|
+
* materialized (a session-head snapshot: an already-active Agent keeps its
|
|
204
|
+
* activation-time set; a change only affects the next Agent's tool set).
|
|
205
|
+
* Tools are grouped into three buckets; a bucket that is off contributes none
|
|
206
|
+
* of its tools to an Agent's visible surface.
|
|
207
|
+
*/
|
|
208
|
+
toolVisibility?: {
|
|
209
|
+
/** Local-processing tools (no on-line fan-out, no concurrency charge). Default true. */
|
|
210
|
+
local?: boolean
|
|
211
|
+
/** On-line image tools plus the concurrency/status probe. Default true. */
|
|
212
|
+
online?: boolean
|
|
213
|
+
/** Video-understanding tool (experimental). Default false. */
|
|
214
|
+
video?: boolean
|
|
215
|
+
}
|
|
174
216
|
}
|
|
175
217
|
|
|
176
218
|
/** Configuration schema with the documented P0 defaults. */
|
|
@@ -182,6 +224,9 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
182
224
|
protocol: z.union(['openai', 'anthropic'] as const).default('openai'),
|
|
183
225
|
anthropicThinking: z.union(['omit', 'disabled', 'adaptive'] as const).default('omit'),
|
|
184
226
|
userAgent: z.string().default(DEFAULT_VISION_USER_AGENT),
|
|
227
|
+
stream: z.boolean().default(false),
|
|
228
|
+
uploadViaUrl: z.boolean().default(false),
|
|
229
|
+
videoSupport: z.boolean().default(false),
|
|
185
230
|
}),
|
|
186
231
|
providers: z.array(z.object({
|
|
187
232
|
name: z.string(),
|
|
@@ -193,6 +238,9 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
193
238
|
protocol: z.union(['openai', 'anthropic'] as const).default('openai'),
|
|
194
239
|
anthropicThinking: z.union(['omit', 'disabled', 'adaptive'] as const).default('omit'),
|
|
195
240
|
userAgent: z.string(),
|
|
241
|
+
stream: z.boolean().default(false),
|
|
242
|
+
uploadViaUrl: z.boolean().default(false),
|
|
243
|
+
videoSupport: z.boolean().default(false),
|
|
196
244
|
t1Seconds: z.number(),
|
|
197
245
|
t2Seconds: z.number(),
|
|
198
246
|
maxImageBytes: z.number(),
|
|
@@ -207,6 +255,12 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
207
255
|
maxImageBytes: z.number().default(4194304),
|
|
208
256
|
maxImagePixels: z.number().default(20000000),
|
|
209
257
|
concurrency: z.number().default(4),
|
|
258
|
+
objectStorage: z.object({
|
|
259
|
+
endpoint: z.string().default(''),
|
|
260
|
+
bucket: z.string().default(''),
|
|
261
|
+
credential: z.string().default(''),
|
|
262
|
+
publicBase: z.string().default(''),
|
|
263
|
+
}),
|
|
210
264
|
runtime: z.object({
|
|
211
265
|
mode: z.union(['managed', 'external'] as const).default('managed'),
|
|
212
266
|
agentVisionToolkitPath: z.string(),
|
|
@@ -221,6 +275,11 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
|
221
275
|
autoSwitch: z.boolean().default(true),
|
|
222
276
|
hidden: z.boolean().default(true),
|
|
223
277
|
}),
|
|
278
|
+
toolVisibility: z.object({
|
|
279
|
+
local: z.boolean().default(true),
|
|
280
|
+
online: z.boolean().default(true),
|
|
281
|
+
video: z.boolean().default(false),
|
|
282
|
+
}),
|
|
224
283
|
})
|
|
225
284
|
|
|
226
285
|
/** One resolved online vision provider, with every default materialized. */
|
|
@@ -235,6 +294,9 @@ export interface ResolvedProvider {
|
|
|
235
294
|
protocol: 'openai' | 'anthropic'
|
|
236
295
|
anthropicThinking: 'omit' | 'disabled' | 'adaptive'
|
|
237
296
|
userAgent: string
|
|
297
|
+
stream: boolean
|
|
298
|
+
uploadViaUrl: boolean
|
|
299
|
+
videoSupport: boolean
|
|
238
300
|
t1Seconds: number
|
|
239
301
|
t2Seconds: number
|
|
240
302
|
maxImageBytes: number
|
|
@@ -252,6 +314,9 @@ export interface ResolvedVisionToolkitConfig {
|
|
|
252
314
|
protocol: 'openai' | 'anthropic'
|
|
253
315
|
anthropicThinking: 'omit' | 'disabled' | 'adaptive'
|
|
254
316
|
userAgent: string
|
|
317
|
+
stream: boolean
|
|
318
|
+
uploadViaUrl: boolean
|
|
319
|
+
videoSupport: boolean
|
|
255
320
|
}
|
|
256
321
|
/** Ordered failover pool; array order is the priority, highest first. */
|
|
257
322
|
providers: ResolvedProvider[]
|
|
@@ -262,6 +327,12 @@ export interface ResolvedVisionToolkitConfig {
|
|
|
262
327
|
maxImageBytes: number
|
|
263
328
|
maxImagePixels: number
|
|
264
329
|
concurrency: number
|
|
330
|
+
objectStorage: {
|
|
331
|
+
endpoint: string
|
|
332
|
+
bucket: string
|
|
333
|
+
credential?: CredentialRef
|
|
334
|
+
publicBase?: string
|
|
335
|
+
}
|
|
265
336
|
runtime: {
|
|
266
337
|
mode: 'managed' | 'external'
|
|
267
338
|
agentVisionToolkitPath?: string
|
|
@@ -276,6 +347,11 @@ export interface ResolvedVisionToolkitConfig {
|
|
|
276
347
|
autoSwitch: boolean
|
|
277
348
|
hidden: boolean
|
|
278
349
|
}
|
|
350
|
+
toolVisibility: {
|
|
351
|
+
local: boolean
|
|
352
|
+
online: boolean
|
|
353
|
+
video: boolean
|
|
354
|
+
}
|
|
279
355
|
}
|
|
280
356
|
|
|
281
357
|
const MAX_TIMEOUT_SECONDS = 600
|
|
@@ -366,6 +442,9 @@ function resolveProvider(
|
|
|
366
442
|
if (userAgent.length === 0) {
|
|
367
443
|
throw new VisionToolkitError('config', `${label}.userAgent must not be empty`)
|
|
368
444
|
}
|
|
445
|
+
const stream = input.stream === true
|
|
446
|
+
const uploadViaUrl = input.uploadViaUrl === true
|
|
447
|
+
const videoSupport = input.videoSupport === true
|
|
369
448
|
const t1Seconds = input.t1Seconds ?? 90
|
|
370
449
|
if (!Number.isInteger(t1Seconds) || t1Seconds < 1 || t1Seconds > MAX_TIMEOUT_SECONDS) {
|
|
371
450
|
throw new VisionToolkitError('config', `${label}.t1Seconds must be an integer between 1 and ${MAX_TIMEOUT_SECONDS}`)
|
|
@@ -402,6 +481,9 @@ function resolveProvider(
|
|
|
402
481
|
protocol,
|
|
403
482
|
anthropicThinking,
|
|
404
483
|
userAgent,
|
|
484
|
+
stream,
|
|
485
|
+
uploadViaUrl,
|
|
486
|
+
videoSupport,
|
|
405
487
|
t1Seconds,
|
|
406
488
|
t2Seconds,
|
|
407
489
|
maxImageBytes,
|
|
@@ -472,10 +554,24 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
472
554
|
.map(dir => dir.trim())
|
|
473
555
|
.filter(dir => dir.length > 0 && dir !== storageDir))]
|
|
474
556
|
const allowedDirs = (config.allowedDirs ?? []).map(dir => dir.trim()).filter(dir => dir.length > 0)
|
|
557
|
+
const objectStorageInput = config.objectStorage ?? {}
|
|
558
|
+
const objectStorageEndpoint = objectStorageInput.endpoint?.trim() ?? ''
|
|
559
|
+
const objectStorageBucket = objectStorageInput.bucket?.trim() ?? ''
|
|
560
|
+
const objectStoragePublicBase = objectStorageInput.publicBase?.trim()
|
|
561
|
+
let objectStorageCredential: CredentialRef | undefined
|
|
562
|
+
const objectStorageCredentialSource = objectStorageInput.credential?.trim()
|
|
563
|
+
if (objectStorageCredentialSource !== undefined && objectStorageCredentialSource.length > 0) {
|
|
564
|
+
try {
|
|
565
|
+
objectStorageCredential = credentialRef(objectStorageCredentialSource)
|
|
566
|
+
} catch (error) {
|
|
567
|
+
throw new VisionToolkitError('config', `objectStorage.credential "${objectStorageCredentialSource}" is not a valid credential reference`, { cause: error })
|
|
568
|
+
}
|
|
569
|
+
}
|
|
475
570
|
const imageInputVariants = config.imageInputVariants ?? {}
|
|
476
571
|
const variantProviders = (imageInputVariants.providers ?? [])
|
|
477
572
|
.map(provider => provider.trim())
|
|
478
573
|
.filter(provider => provider.length > 0)
|
|
574
|
+
const toolVisibility = config.toolVisibility ?? {}
|
|
479
575
|
const providerDefaults: ProviderDefaults = { maxImageBytes, maxImagePixels, concurrency }
|
|
480
576
|
const configuredProviders = config.providers ?? []
|
|
481
577
|
if (configuredProviders.length > MAX_PROVIDERS) {
|
|
@@ -494,6 +590,9 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
494
590
|
protocol: primary.protocol,
|
|
495
591
|
anthropicThinking: primary.anthropicThinking,
|
|
496
592
|
userAgent: primary.userAgent,
|
|
593
|
+
stream: primary.stream,
|
|
594
|
+
uploadViaUrl: primary.uploadViaUrl,
|
|
595
|
+
videoSupport: primary.videoSupport,
|
|
497
596
|
},
|
|
498
597
|
providers,
|
|
499
598
|
language,
|
|
@@ -503,6 +602,12 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
503
602
|
maxImageBytes,
|
|
504
603
|
maxImagePixels,
|
|
505
604
|
concurrency,
|
|
605
|
+
objectStorage: {
|
|
606
|
+
endpoint: objectStorageEndpoint,
|
|
607
|
+
bucket: objectStorageBucket,
|
|
608
|
+
...(objectStorageCredential === undefined ? {} : { credential: objectStorageCredential }),
|
|
609
|
+
...(objectStoragePublicBase === undefined || objectStoragePublicBase.length === 0 ? {} : { publicBase: objectStoragePublicBase }),
|
|
610
|
+
},
|
|
506
611
|
runtime: {
|
|
507
612
|
mode,
|
|
508
613
|
...(toolkitPath !== undefined ? { agentVisionToolkitPath: toolkitPath } : {}),
|
|
@@ -517,6 +622,11 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
|
|
|
517
622
|
autoSwitch: imageInputVariants.autoSwitch ?? true,
|
|
518
623
|
hidden: imageInputVariants.hidden ?? true,
|
|
519
624
|
},
|
|
625
|
+
toolVisibility: {
|
|
626
|
+
local: toolVisibility.local ?? true,
|
|
627
|
+
online: toolVisibility.online ?? true,
|
|
628
|
+
video: toolVisibility.video ?? false,
|
|
629
|
+
},
|
|
520
630
|
}
|
|
521
631
|
}
|
|
522
632
|
|
package/src/exposure.ts
CHANGED
|
@@ -12,7 +12,7 @@ import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
|
|
|
12
12
|
import { defineTool, type ToolDefinition } from '@deepseek-ai/dsh-tools'
|
|
13
13
|
import type { Context } from '@deepseek-ai/cordis'
|
|
14
14
|
import { VISION_SKILLS_CONTENT, VISION_SKILLS_NAME } from './skill.ts'
|
|
15
|
-
import { VISION_TOOL_NAMES } from './tools.ts'
|
|
15
|
+
import { VISION_TOOL_NAMES, type ToolVisibility } from './tools.ts'
|
|
16
16
|
|
|
17
17
|
/** Small bootstrap tool retained only until the current Agent gains visual tools. */
|
|
18
18
|
export const VISION_TOOLKIT_ACTIVATE = 'vision_toolkit_activate'
|
|
@@ -139,17 +139,21 @@ export class VisionToolExposure {
|
|
|
139
139
|
|
|
140
140
|
/**
|
|
141
141
|
* @param ctx - Plugin context with Tool and Agent registries.
|
|
142
|
-
* @param createTools - Fresh definitions bound to the current runtime
|
|
142
|
+
* @param createTools - Fresh definitions bound to the current runtime
|
|
143
|
+
* generation, filtered by a tool-visibility snapshot.
|
|
144
|
+
* @param resolveVisibility - Live resolver for the current tool-visibility
|
|
145
|
+
* snapshot; read once per activation (a session-head snapshot).
|
|
143
146
|
*/
|
|
144
147
|
constructor(
|
|
145
148
|
private readonly ctx: Context,
|
|
146
|
-
private readonly createTools: () => ToolDefinition[],
|
|
149
|
+
private readonly createTools: (snapshot: ToolVisibility) => ToolDefinition[],
|
|
150
|
+
private readonly resolveVisibility: () => ToolVisibility = () => ({ local: true, online: true, video: false }),
|
|
147
151
|
) {
|
|
148
152
|
this.activationTool = defineTool({
|
|
149
153
|
name: VISION_TOOLKIT_ACTIVATE,
|
|
150
|
-
description: `
|
|
151
|
-
+ `
|
|
152
|
-
+ 'It
|
|
154
|
+
description: `Report and (re)mount the Vision Toolkit execution tools for this Agent: the currently visible subset of ${Object.values(VISION_TOOL_NAMES).join(', ')} plus optional video understanding. `
|
|
155
|
+
+ `The visual tool set is normally mounted automatically when the ${VISION_SKILLS_NAME} Skill loads; call this ONLY when the user explicitly asks you to refresh or reload the current session's vision tools, or to list which vision tools are currently available. `
|
|
156
|
+
+ 'It restores the tool set to the latest Settings snapshot and returns the tool names actually mounted.',
|
|
153
157
|
parameters: {},
|
|
154
158
|
output: {
|
|
155
159
|
schema: {
|
|
@@ -166,9 +170,12 @@ export class VisionToolExposure {
|
|
|
166
170
|
if (exec.agent === undefined) {
|
|
167
171
|
throw new Error(`${VISION_TOOLKIT_ACTIVATE}: an Agent Session is required`)
|
|
168
172
|
}
|
|
169
|
-
|
|
173
|
+
// An already-active Agent is reloaded (re-reads the latest Settings
|
|
174
|
+
// snapshot) rather than returning the stale set, so the tool genuinely
|
|
175
|
+
// refreshes the visible tool list on an explicit user request.
|
|
176
|
+
return Promise.resolve(this.reload(exec.agent))
|
|
170
177
|
},
|
|
171
|
-
presentCall: () => ({ card: 'generic', title: '
|
|
178
|
+
presentCall: () => ({ card: 'generic', title: 'Refresh or list vision tools', kind: 'execute' }),
|
|
172
179
|
})
|
|
173
180
|
}
|
|
174
181
|
|
|
@@ -222,7 +229,7 @@ export class VisionToolExposure {
|
|
|
222
229
|
if (state === undefined) throw new Error(`dsh-vision-toolkit: Agent ${String(agent.id)} has no exposure state`)
|
|
223
230
|
if (state.active) return { activated: false, tools: [...state.toolNames] }
|
|
224
231
|
|
|
225
|
-
const definitions = this.createTools()
|
|
232
|
+
const definitions = this.createTools(this.resolveVisibility())
|
|
226
233
|
const toolDisposers: Array<() => void> = []
|
|
227
234
|
try {
|
|
228
235
|
for (const definition of definitions) toolDisposers.push(agent.ctx.tools.register(definition))
|
|
@@ -241,6 +248,24 @@ export class VisionToolExposure {
|
|
|
241
248
|
return { activated: true, tools: [...state.toolNames] }
|
|
242
249
|
}
|
|
243
250
|
|
|
251
|
+
/**
|
|
252
|
+
* Explicit refresh/reload from the bootstrap tool. An inactive Agent is
|
|
253
|
+
* mounted; an active Agent is torn down and re-mounted from the *current*
|
|
254
|
+
* Settings snapshot, so the returned tool list reflects the latest
|
|
255
|
+
* `toolVisibility` instead of the stale activation-time set.
|
|
256
|
+
*/
|
|
257
|
+
private reload(agent: Agent): VisionToolkitActivationResult {
|
|
258
|
+
this.attach(agent)
|
|
259
|
+
const state = this.states.get(agent)
|
|
260
|
+
if (state === undefined) return { activated: false, tools: [] }
|
|
261
|
+
if (!state.active) return this.activate(agent)
|
|
262
|
+
// Dispose the current tool generation (including the hide-restriction on
|
|
263
|
+
// the bootstrap gauntlet) and rebuild from the fresh snapshot.
|
|
264
|
+
this.disposeState(state)
|
|
265
|
+
this.states.set(agent, { active: false, toolDisposers: [], toolNames: [] })
|
|
266
|
+
return this.activate(agent)
|
|
267
|
+
}
|
|
268
|
+
|
|
244
269
|
/** Whether the session is attached to the live SessionStore (production). */
|
|
245
270
|
private isLiveSession(session: Session): boolean {
|
|
246
271
|
return this.ctx.sessions.get(session.id) === session
|
package/src/index.ts
CHANGED
|
@@ -83,11 +83,16 @@ export async function apply(ctx: Context, config: VisionToolkitConfig = {}): Pro
|
|
|
83
83
|
|
|
84
84
|
const ensureOperational = (): void => {
|
|
85
85
|
if (!manager.ready || operationalDisposers !== undefined) return
|
|
86
|
-
const exposure = new VisionToolExposure(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
86
|
+
const exposure = new VisionToolExposure(
|
|
87
|
+
ctx,
|
|
88
|
+
(visibility) => createVisionTools(
|
|
89
|
+
() => manager.current(),
|
|
90
|
+
value => artifacts.presentationMeta(value),
|
|
91
|
+
lifecycle.signal,
|
|
92
|
+
visibility,
|
|
93
|
+
),
|
|
94
|
+
() => manager.currentConfig().toolVisibility,
|
|
95
|
+
)
|
|
91
96
|
let activationTool: (() => void) | undefined
|
|
92
97
|
let exposureDisposer: (() => void) | undefined
|
|
93
98
|
let skill: (() => void) | undefined
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal S3-compatible object storage bridge used by the URL image-transfer
|
|
3
|
+
* path. It uploads one image, resolves a model-reachable URL (a configured
|
|
4
|
+
* public base URL or a temporary presigned URL), and deletes the object after
|
|
5
|
+
* the vision operation settles. It also provides the Settings "test storage"
|
|
6
|
+
* probe (upload → head → delete).
|
|
7
|
+
* @module dsh-vision-toolkit/object-storage
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
11
|
+
import { readFile } from 'node:fs/promises'
|
|
12
|
+
import { basename } from 'node:path'
|
|
13
|
+
import {
|
|
14
|
+
DeleteObjectCommand,
|
|
15
|
+
GetObjectCommand,
|
|
16
|
+
HeadObjectCommand,
|
|
17
|
+
PutObjectCommand,
|
|
18
|
+
S3Client,
|
|
19
|
+
type S3ClientConfig,
|
|
20
|
+
} from '@aws-sdk/client-s3'
|
|
21
|
+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
|
|
22
|
+
import { VisionToolkitError } from './errors.ts'
|
|
23
|
+
|
|
24
|
+
/** Fully resolved object-storage connection settings (secrets already filled). */
|
|
25
|
+
export interface ObjectStorageSettings {
|
|
26
|
+
endpoint: string
|
|
27
|
+
bucket: string
|
|
28
|
+
accessKeyId: string
|
|
29
|
+
secretAccessKey: string
|
|
30
|
+
publicBase?: string
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Whether the required connection fields are present enough to attempt a request. */
|
|
34
|
+
export function isObjectStorageConfigured(settings: ObjectStorageSettings): boolean {
|
|
35
|
+
return settings.endpoint.length > 0
|
|
36
|
+
&& settings.bucket.length > 0
|
|
37
|
+
&& settings.accessKeyId.length > 0
|
|
38
|
+
&& settings.secretAccessKey.length > 0
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Stable object-key prefix so every upload lives under one deletable namespace. */
|
|
42
|
+
const OBJECT_KEY_PREFIX = 'dsh-vision-toolkit'
|
|
43
|
+
|
|
44
|
+
function encodeKey(key: string): string {
|
|
45
|
+
return key.split('/').map(encodeURIComponent).join('/')
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function clientFor(settings: ObjectStorageSettings): S3Client {
|
|
49
|
+
const config: S3ClientConfig = {
|
|
50
|
+
region: 'auto',
|
|
51
|
+
forcePathStyle: true,
|
|
52
|
+
credentials: {
|
|
53
|
+
accessKeyId: settings.accessKeyId,
|
|
54
|
+
secretAccessKey: settings.secretAccessKey,
|
|
55
|
+
},
|
|
56
|
+
}
|
|
57
|
+
if (settings.endpoint.length > 0) config.endpoint = settings.endpoint
|
|
58
|
+
return new S3Client(config)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function publicError(error: unknown): string {
|
|
62
|
+
if (error instanceof Error) return error.message
|
|
63
|
+
return String(error)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* One upload's worth of bookkeeping: the object key and the URL handed to the
|
|
68
|
+
* model. The key is returned to the runtime so it can delete the object after
|
|
69
|
+
* the operation settles.
|
|
70
|
+
*/
|
|
71
|
+
export interface UploadedObject {
|
|
72
|
+
key: string
|
|
73
|
+
url: string
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** A small S3-compatible object store bound to one bucket and credential. */
|
|
77
|
+
export class ObjectStorageClient {
|
|
78
|
+
private client?: S3Client
|
|
79
|
+
|
|
80
|
+
constructor(private readonly settings: ObjectStorageSettings) {}
|
|
81
|
+
|
|
82
|
+
private requireClient(): S3Client {
|
|
83
|
+
if (this.client === undefined) this.client = clientFor(this.settings)
|
|
84
|
+
return this.client
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Upload one local image file and resolve its model-reachable URL. */
|
|
88
|
+
async uploadImage(localPath: string, contentType: string): Promise<UploadedObject> {
|
|
89
|
+
const body = await readFile(localPath)
|
|
90
|
+
const digest = createHash('sha256').update(body).digest('hex').slice(0, 12)
|
|
91
|
+
const name = basename(localPath).replace(/[^A-Za-z0-9._-]/g, '_')
|
|
92
|
+
const key = `${OBJECT_KEY_PREFIX}/${randomUUID()}-${digest}-${name}`
|
|
93
|
+
try {
|
|
94
|
+
await this.requireClient().send(new PutObjectCommand({
|
|
95
|
+
Bucket: this.settings.bucket,
|
|
96
|
+
Key: key,
|
|
97
|
+
Body: body,
|
|
98
|
+
ContentType: contentType,
|
|
99
|
+
}))
|
|
100
|
+
} catch (error) {
|
|
101
|
+
throw new VisionToolkitError('service', `object storage upload failed: ${publicError(error)}`, { cause: error })
|
|
102
|
+
}
|
|
103
|
+
return { key, url: await this.urlFor(key) }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Resolve the model-reachable URL: public base URL when set, else presigned. */
|
|
107
|
+
async urlFor(key: string): Promise<string> {
|
|
108
|
+
if (this.settings.publicBase !== undefined && this.settings.publicBase.length > 0) {
|
|
109
|
+
return `${this.settings.publicBase}/${encodeKey(key)}`
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
return await getSignedUrl(
|
|
113
|
+
this.requireClient(),
|
|
114
|
+
new GetObjectCommand({ Bucket: this.settings.bucket, Key: key }),
|
|
115
|
+
{ expiresIn: 3600 },
|
|
116
|
+
)
|
|
117
|
+
} catch (error) {
|
|
118
|
+
throw new VisionToolkitError('service', `object storage presign failed: ${publicError(error)}`, { cause: error })
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Delete one uploaded object; failures are logged, never fatal to the call. */
|
|
123
|
+
async deleteObject(key: string): Promise<void> {
|
|
124
|
+
try {
|
|
125
|
+
await this.requireClient().send(new DeleteObjectCommand({
|
|
126
|
+
Bucket: this.settings.bucket,
|
|
127
|
+
Key: key,
|
|
128
|
+
}))
|
|
129
|
+
} catch {
|
|
130
|
+
// Best-effort cleanup: a failed delete must not mask the vision result.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Settings "test storage" probe: upload a tiny object, head it, then delete it. */
|
|
135
|
+
async test(): Promise<{ detail: string }> {
|
|
136
|
+
if (!isObjectStorageConfigured(this.settings)) {
|
|
137
|
+
throw new VisionToolkitError('config', 'object storage is not fully configured (endpoint, bucket, access key id, and secret access key are required)')
|
|
138
|
+
}
|
|
139
|
+
const key = `${OBJECT_KEY_PREFIX}/.connection-test-${randomUUID()}`
|
|
140
|
+
const marker = `dsh-vision-toolkit object storage test ${Date.now()}`
|
|
141
|
+
try {
|
|
142
|
+
await this.requireClient().send(new PutObjectCommand({
|
|
143
|
+
Bucket: this.settings.bucket,
|
|
144
|
+
Key: key,
|
|
145
|
+
Body: marker,
|
|
146
|
+
ContentType: 'text/plain',
|
|
147
|
+
}))
|
|
148
|
+
await this.requireClient().send(new HeadObjectCommand({ Bucket: this.settings.bucket, Key: key }))
|
|
149
|
+
await this.requireClient().send(new DeleteObjectCommand({ Bucket: this.settings.bucket, Key: key }))
|
|
150
|
+
} catch (error) {
|
|
151
|
+
throw new VisionToolkitError('service', `object storage test failed: ${publicError(error)}`, { cause: error })
|
|
152
|
+
}
|
|
153
|
+
const urlMode = this.settings.publicBase !== undefined && this.settings.publicBase.length > 0
|
|
154
|
+
? `public base ${this.settings.publicBase}`
|
|
155
|
+
: 'presigned URL'
|
|
156
|
+
return { detail: `bucket ${this.settings.bucket} reachable; model URLs will use ${urlMode}` }
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Split a credential value of the form `accessKeyId:secretAccessKey` into its
|
|
162
|
+
* two parts. The access key id never contains a colon, so splitting on the
|
|
163
|
+
* first colon is safe.
|
|
164
|
+
*/
|
|
165
|
+
export function splitObjectStorageCredential(value: string): { accessKeyId: string; secretAccessKey: string } {
|
|
166
|
+
const index = value.indexOf(':')
|
|
167
|
+
if (index <= 0) {
|
|
168
|
+
throw new VisionToolkitError('config', 'object storage credential must be "accessKeyId:secretAccessKey"')
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
accessKeyId: value.slice(0, index),
|
|
172
|
+
secretAccessKey: value.slice(index + 1),
|
|
173
|
+
}
|
|
174
|
+
}
|
package/src/paths.ts
CHANGED
|
@@ -16,6 +16,12 @@ import { VisionToolkitError } from './errors.ts'
|
|
|
16
16
|
/** Supported input image extensions (the upstream client's allowlist). */
|
|
17
17
|
export const SUPPORTED_IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp'] as const
|
|
18
18
|
|
|
19
|
+
/** Supported input video extensions probed with the bundled ffprobe binary. */
|
|
20
|
+
export const SUPPORTED_VIDEO_EXTENSIONS = [
|
|
21
|
+
'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v',
|
|
22
|
+
'.mpg', '.mpeg', '.wmv', '.flv', '.ts', '.m2ts', '.3gp',
|
|
23
|
+
] as const
|
|
24
|
+
|
|
19
25
|
/** Resolved path policy for one tool invocation. */
|
|
20
26
|
export interface PathPolicy {
|
|
21
27
|
/** Real workspace root. */
|
|
@@ -399,6 +405,11 @@ export function resolveHtmlFile(raw: string, policy: PathPolicy): Promise<{ path
|
|
|
399
405
|
return resolveAuthorizedFile(raw, policy, ['.html', '.htm'], 'HTML source')
|
|
400
406
|
}
|
|
401
407
|
|
|
408
|
+
/** Validate one input video path against the video extension allowlist. */
|
|
409
|
+
export function resolveInputVideo(raw: string, policy: PathPolicy): Promise<{ path: string; bytes: number }> {
|
|
410
|
+
return resolveAuthorizedFile(raw, policy, SUPPORTED_VIDEO_EXTENSIONS, 'video')
|
|
411
|
+
}
|
|
412
|
+
|
|
402
413
|
/**
|
|
403
414
|
* Resolve an optional user-supplied output filename inside the plugin output
|
|
404
415
|
* directory. Absolute paths, `..` segments, and wrong extensions are rejected.
|