@mengruo/dsh-vision-toolkit 0.1.4 → 0.1.6-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +11 -70
  2. package/README.zh.md +11 -69
  3. package/assets/1.mp4 +0 -0
  4. package/assets/skill/SKILL.md +37 -5
  5. package/docs/plan-per-tool-visibility.md +82 -0
  6. package/lib/client.js +157 -6
  7. package/lib/client.js.map +1 -1
  8. package/lib/config.js +52 -0
  9. package/lib/config.js.map +1 -1
  10. package/lib/exposure.js +35 -8
  11. package/lib/exposure.js.map +1 -1
  12. package/lib/index.js +1 -1
  13. package/lib/index.js.map +1 -1
  14. package/lib/object-storage.js +141 -0
  15. package/lib/object-storage.js.map +1 -0
  16. package/lib/paths.js +9 -0
  17. package/lib/paths.js.map +1 -1
  18. package/lib/runtime.js +314 -27
  19. package/lib/runtime.js.map +1 -1
  20. package/lib/tools.js +116 -4
  21. package/lib/tools.js.map +1 -1
  22. package/lib/types/client/index.d.ts +62 -1
  23. package/lib/types/client/index.d.ts.map +1 -1
  24. package/lib/types/config.d.ts +59 -0
  25. package/lib/types/config.d.ts.map +1 -1
  26. package/lib/types/exposure.d.ts +14 -2
  27. package/lib/types/exposure.d.ts.map +1 -1
  28. package/lib/types/index.d.ts.map +1 -1
  29. package/lib/types/object-storage.d.ts +54 -0
  30. package/lib/types/object-storage.d.ts.map +1 -0
  31. package/lib/types/paths.d.ts +7 -0
  32. package/lib/types/paths.d.ts.map +1 -1
  33. package/lib/types/runtime.d.ts +50 -0
  34. package/lib/types/runtime.d.ts.map +1 -1
  35. package/lib/types/tools.d.ts +22 -1
  36. package/lib/types/tools.d.ts.map +1 -1
  37. package/lib/types/upstream.d.ts +1 -0
  38. package/lib/types/upstream.d.ts.map +1 -1
  39. package/lib/types/video.d.ts +78 -0
  40. package/lib/types/video.d.ts.map +1 -0
  41. package/lib/types/web.d.ts +8 -0
  42. package/lib/types/web.d.ts.map +1 -1
  43. package/lib/upstream.js +3 -0
  44. package/lib/upstream.js.map +1 -1
  45. package/lib/video.js +169 -0
  46. package/lib/video.js.map +1 -0
  47. package/lib/web.js +78 -6
  48. package/lib/web.js.map +1 -1
  49. package/package.json +4 -1
  50. package/src/client/index.tsx +222 -6
  51. package/src/config.ts +110 -0
  52. package/src/exposure.ts +34 -9
  53. package/src/index.ts +10 -5
  54. package/src/object-storage.ts +174 -0
  55. package/src/paths.ts +11 -0
  56. package/src/runtime.ts +332 -25
  57. package/src/tools.ts +144 -3
  58. package/src/upstream.ts +4 -0
  59. package/src/video.ts +222 -0
  60. package/src/web.ts +93 -7
  61. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
  62. package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
  63. package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
  64. package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
  65. package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
  66. package/vendor/agent-vision-toolkit/bin/glance +8 -1
  67. package/vendor/agent-vision-toolkit/detect.py +13 -7
  68. package/vendor/agent-vision-toolkit/ground.py +43 -18
  69. package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
  70. package/vendor/agent-vision-toolkit/vision_client.py +84 -6
  71. package/assets/community-group-qr.png +0 -0
  72. package/assets/logo_aihubmix.png +0 -0
  73. package/assets/logo_eapi_dark.png +0 -0
  74. package/assets/wechat-reward.png +0 -0
package/src/config.ts CHANGED
@@ -77,6 +77,12 @@ export interface VisionProviderConfig {
77
77
  anthropicThinking?: 'omit' | 'disabled' | 'adaptive'
78
78
  /** Outbound User-Agent for provider requests and connection tests. */
79
79
  userAgent?: string
80
+ /** Whether to request a streamed (SSE) completion instead of one JSON response (default false). */
81
+ stream?: boolean
82
+ /** Whether to upload images to object storage and send the model a URL instead of base64 (default false). */
83
+ uploadViaUrl?: boolean
84
+ /** OpenAI-compatible video understanding (default false); Anthropic providers ignore this placeholder flag. */
85
+ videoSupport?: boolean
80
86
  /** t1: per-request hedge threshold in seconds. A single request exceeding t1 keeps running while the next provider starts in parallel. */
81
87
  t1Seconds?: number
82
88
  /** t2: per-provider cumulative cutoff in seconds. Total accumulated request time reaching t2 terminates the provider. */
@@ -106,6 +112,12 @@ export interface VisionToolkitConfig {
106
112
  anthropicThinking?: 'omit' | 'disabled' | 'adaptive'
107
113
  /** Outbound User-Agent for provider requests and connection tests. */
108
114
  userAgent?: string
115
+ /** Whether to request a streamed (SSE) completion instead of one JSON response (default false). */
116
+ stream?: boolean
117
+ /** Whether to upload images to object storage and send the model a URL instead of base64 (default false). */
118
+ uploadViaUrl?: boolean
119
+ /** OpenAI-compatible video understanding (default false); Anthropic providers ignore this placeholder flag. */
120
+ videoSupport?: boolean
109
121
  }
110
122
  /** Ordered online vision providers; array order is the failover priority. */
111
123
  providers?: VisionProviderConfig[]
@@ -123,6 +135,21 @@ export interface VisionToolkitConfig {
123
135
  maxImagePixels?: number
124
136
  /** Default per-model in-flight request cap inherited by providers that do not set their own. */
125
137
  concurrency?: number
138
+ /**
139
+ * Optional S3-compatible object storage used by the URL image-transfer path.
140
+ * `endpoint`, `bucket`, and `credential` are required to enable URL transfer;
141
+ * `publicBase` is optional and falls back to presigned URLs when unset.
142
+ */
143
+ objectStorage?: {
144
+ /** S3-compatible API endpoint (e.g. R2, MinIO, Tencent COS). */
145
+ endpoint?: string
146
+ /** Bucket name. */
147
+ bucket?: string
148
+ /** DSH Credential reference holding "accessKeyId:secretAccessKey". */
149
+ credential?: string
150
+ /** Public base URL (custom domain / r2.dev); when unset, presigned URLs are used. */
151
+ publicBase?: string
152
+ }
126
153
  runtime?: {
127
154
  /** `managed` uses the packaged snapshot and isolated venv; `external` uses a clean pinned checkout. */
128
155
  mode?: 'managed' | 'external'
@@ -171,6 +198,21 @@ export interface VisionToolkitConfig {
171
198
  */
172
199
  hidden?: boolean
173
200
  }
201
+ /**
202
+ * Per-group tool visibility, applied when each Agent's visual tool set is
203
+ * materialized (a session-head snapshot: an already-active Agent keeps its
204
+ * activation-time set; a change only affects the next Agent's tool set).
205
+ * Tools are grouped into three buckets; a bucket that is off contributes none
206
+ * of its tools to an Agent's visible surface.
207
+ */
208
+ toolVisibility?: {
209
+ /** Local-processing tools (no on-line fan-out, no concurrency charge). Default true. */
210
+ local?: boolean
211
+ /** On-line image tools plus the concurrency/status probe. Default true. */
212
+ online?: boolean
213
+ /** Video-understanding tool (experimental). Default false. */
214
+ video?: boolean
215
+ }
174
216
  }
175
217
 
176
218
  /** Configuration schema with the documented P0 defaults. */
@@ -182,6 +224,9 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
182
224
  protocol: z.union(['openai', 'anthropic'] as const).default('openai'),
183
225
  anthropicThinking: z.union(['omit', 'disabled', 'adaptive'] as const).default('omit'),
184
226
  userAgent: z.string().default(DEFAULT_VISION_USER_AGENT),
227
+ stream: z.boolean().default(false),
228
+ uploadViaUrl: z.boolean().default(false),
229
+ videoSupport: z.boolean().default(false),
185
230
  }),
186
231
  providers: z.array(z.object({
187
232
  name: z.string(),
@@ -193,6 +238,9 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
193
238
  protocol: z.union(['openai', 'anthropic'] as const).default('openai'),
194
239
  anthropicThinking: z.union(['omit', 'disabled', 'adaptive'] as const).default('omit'),
195
240
  userAgent: z.string(),
241
+ stream: z.boolean().default(false),
242
+ uploadViaUrl: z.boolean().default(false),
243
+ videoSupport: z.boolean().default(false),
196
244
  t1Seconds: z.number(),
197
245
  t2Seconds: z.number(),
198
246
  maxImageBytes: z.number(),
@@ -207,6 +255,12 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
207
255
  maxImageBytes: z.number().default(4194304),
208
256
  maxImagePixels: z.number().default(20000000),
209
257
  concurrency: z.number().default(4),
258
+ objectStorage: z.object({
259
+ endpoint: z.string().default(''),
260
+ bucket: z.string().default(''),
261
+ credential: z.string().default(''),
262
+ publicBase: z.string().default(''),
263
+ }),
210
264
  runtime: z.object({
211
265
  mode: z.union(['managed', 'external'] as const).default('managed'),
212
266
  agentVisionToolkitPath: z.string(),
@@ -221,6 +275,11 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
221
275
  autoSwitch: z.boolean().default(true),
222
276
  hidden: z.boolean().default(true),
223
277
  }),
278
+ toolVisibility: z.object({
279
+ local: z.boolean().default(true),
280
+ online: z.boolean().default(true),
281
+ video: z.boolean().default(false),
282
+ }),
224
283
  })
225
284
 
226
285
  /** One resolved online vision provider, with every default materialized. */
@@ -235,6 +294,9 @@ export interface ResolvedProvider {
235
294
  protocol: 'openai' | 'anthropic'
236
295
  anthropicThinking: 'omit' | 'disabled' | 'adaptive'
237
296
  userAgent: string
297
+ stream: boolean
298
+ uploadViaUrl: boolean
299
+ videoSupport: boolean
238
300
  t1Seconds: number
239
301
  t2Seconds: number
240
302
  maxImageBytes: number
@@ -252,6 +314,9 @@ export interface ResolvedVisionToolkitConfig {
252
314
  protocol: 'openai' | 'anthropic'
253
315
  anthropicThinking: 'omit' | 'disabled' | 'adaptive'
254
316
  userAgent: string
317
+ stream: boolean
318
+ uploadViaUrl: boolean
319
+ videoSupport: boolean
255
320
  }
256
321
  /** Ordered failover pool; array order is the priority, highest first. */
257
322
  providers: ResolvedProvider[]
@@ -262,6 +327,12 @@ export interface ResolvedVisionToolkitConfig {
262
327
  maxImageBytes: number
263
328
  maxImagePixels: number
264
329
  concurrency: number
330
+ objectStorage: {
331
+ endpoint: string
332
+ bucket: string
333
+ credential?: CredentialRef
334
+ publicBase?: string
335
+ }
265
336
  runtime: {
266
337
  mode: 'managed' | 'external'
267
338
  agentVisionToolkitPath?: string
@@ -276,6 +347,11 @@ export interface ResolvedVisionToolkitConfig {
276
347
  autoSwitch: boolean
277
348
  hidden: boolean
278
349
  }
350
+ toolVisibility: {
351
+ local: boolean
352
+ online: boolean
353
+ video: boolean
354
+ }
279
355
  }
280
356
 
281
357
  const MAX_TIMEOUT_SECONDS = 600
@@ -366,6 +442,9 @@ function resolveProvider(
366
442
  if (userAgent.length === 0) {
367
443
  throw new VisionToolkitError('config', `${label}.userAgent must not be empty`)
368
444
  }
445
+ const stream = input.stream === true
446
+ const uploadViaUrl = input.uploadViaUrl === true
447
+ const videoSupport = input.videoSupport === true
369
448
  const t1Seconds = input.t1Seconds ?? 90
370
449
  if (!Number.isInteger(t1Seconds) || t1Seconds < 1 || t1Seconds > MAX_TIMEOUT_SECONDS) {
371
450
  throw new VisionToolkitError('config', `${label}.t1Seconds must be an integer between 1 and ${MAX_TIMEOUT_SECONDS}`)
@@ -402,6 +481,9 @@ function resolveProvider(
402
481
  protocol,
403
482
  anthropicThinking,
404
483
  userAgent,
484
+ stream,
485
+ uploadViaUrl,
486
+ videoSupport,
405
487
  t1Seconds,
406
488
  t2Seconds,
407
489
  maxImageBytes,
@@ -472,10 +554,24 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
472
554
  .map(dir => dir.trim())
473
555
  .filter(dir => dir.length > 0 && dir !== storageDir))]
474
556
  const allowedDirs = (config.allowedDirs ?? []).map(dir => dir.trim()).filter(dir => dir.length > 0)
557
+ const objectStorageInput = config.objectStorage ?? {}
558
+ const objectStorageEndpoint = objectStorageInput.endpoint?.trim() ?? ''
559
+ const objectStorageBucket = objectStorageInput.bucket?.trim() ?? ''
560
+ const objectStoragePublicBase = objectStorageInput.publicBase?.trim()
561
+ let objectStorageCredential: CredentialRef | undefined
562
+ const objectStorageCredentialSource = objectStorageInput.credential?.trim()
563
+ if (objectStorageCredentialSource !== undefined && objectStorageCredentialSource.length > 0) {
564
+ try {
565
+ objectStorageCredential = credentialRef(objectStorageCredentialSource)
566
+ } catch (error) {
567
+ throw new VisionToolkitError('config', `objectStorage.credential "${objectStorageCredentialSource}" is not a valid credential reference`, { cause: error })
568
+ }
569
+ }
475
570
  const imageInputVariants = config.imageInputVariants ?? {}
476
571
  const variantProviders = (imageInputVariants.providers ?? [])
477
572
  .map(provider => provider.trim())
478
573
  .filter(provider => provider.length > 0)
574
+ const toolVisibility = config.toolVisibility ?? {}
479
575
  const providerDefaults: ProviderDefaults = { maxImageBytes, maxImagePixels, concurrency }
480
576
  const configuredProviders = config.providers ?? []
481
577
  if (configuredProviders.length > MAX_PROVIDERS) {
@@ -494,6 +590,9 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
494
590
  protocol: primary.protocol,
495
591
  anthropicThinking: primary.anthropicThinking,
496
592
  userAgent: primary.userAgent,
593
+ stream: primary.stream,
594
+ uploadViaUrl: primary.uploadViaUrl,
595
+ videoSupport: primary.videoSupport,
497
596
  },
498
597
  providers,
499
598
  language,
@@ -503,6 +602,12 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
503
602
  maxImageBytes,
504
603
  maxImagePixels,
505
604
  concurrency,
605
+ objectStorage: {
606
+ endpoint: objectStorageEndpoint,
607
+ bucket: objectStorageBucket,
608
+ ...(objectStorageCredential === undefined ? {} : { credential: objectStorageCredential }),
609
+ ...(objectStoragePublicBase === undefined || objectStoragePublicBase.length === 0 ? {} : { publicBase: objectStoragePublicBase }),
610
+ },
506
611
  runtime: {
507
612
  mode,
508
613
  ...(toolkitPath !== undefined ? { agentVisionToolkitPath: toolkitPath } : {}),
@@ -517,6 +622,11 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
517
622
  autoSwitch: imageInputVariants.autoSwitch ?? true,
518
623
  hidden: imageInputVariants.hidden ?? true,
519
624
  },
625
+ toolVisibility: {
626
+ local: toolVisibility.local ?? true,
627
+ online: toolVisibility.online ?? true,
628
+ video: toolVisibility.video ?? false,
629
+ },
520
630
  }
521
631
  }
522
632
 
package/src/exposure.ts CHANGED
@@ -12,7 +12,7 @@ import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
12
12
  import { defineTool, type ToolDefinition } from '@deepseek-ai/dsh-tools'
13
13
  import type { Context } from '@deepseek-ai/cordis'
14
14
  import { VISION_SKILLS_CONTENT, VISION_SKILLS_NAME } from './skill.ts'
15
- import { VISION_TOOL_NAMES } from './tools.ts'
15
+ import { VISION_TOOL_NAMES, type ToolVisibility } from './tools.ts'
16
16
 
17
17
  /** Small bootstrap tool retained only until the current Agent gains visual tools. */
18
18
  export const VISION_TOOLKIT_ACTIVATE = 'vision_toolkit_activate'
@@ -139,17 +139,21 @@ export class VisionToolExposure {
139
139
 
140
140
  /**
141
141
  * @param ctx - Plugin context with Tool and Agent registries.
142
- * @param createTools - Fresh definitions bound to the current runtime generation.
142
+ * @param createTools - Fresh definitions bound to the current runtime
143
+ * generation, filtered by a tool-visibility snapshot.
144
+ * @param resolveVisibility - Live resolver for the current tool-visibility
145
+ * snapshot; read once per activation (a session-head snapshot).
143
146
  */
144
147
  constructor(
145
148
  private readonly ctx: Context,
146
- private readonly createTools: () => ToolDefinition[],
149
+ private readonly createTools: (snapshot: ToolVisibility) => ToolDefinition[],
150
+ private readonly resolveVisibility: () => ToolVisibility = () => ({ local: true, online: true, video: false }),
147
151
  ) {
148
152
  this.activationTool = defineTool({
149
153
  name: VISION_TOOLKIT_ACTIVATE,
150
- description: `Activate the independent Vision Toolkit execution tools for this Agent: ${Object.values(VISION_TOOL_NAMES).join(', ')}. `
151
- + `Loading the ${VISION_SKILLS_NAME} Skill normally activates them automatically; call this once when the visual tools are still absent, then use them for image understanding, OCR, UI detection, and related tasks. `
152
- + 'It is safe to call before the Skill is loaded, and this activation tool disappears after success.',
154
+ description: `Report and (re)mount the Vision Toolkit execution tools for this Agent: the currently visible subset of ${Object.values(VISION_TOOL_NAMES).join(', ')} plus optional video understanding. `
155
+ + `The visual tool set is normally mounted automatically when the ${VISION_SKILLS_NAME} Skill loads; call this ONLY when the user explicitly asks you to refresh or reload the current session's vision tools, or to list which vision tools are currently available. `
156
+ + 'It restores the tool set to the latest Settings snapshot and returns the tool names actually mounted.',
153
157
  parameters: {},
154
158
  output: {
155
159
  schema: {
@@ -166,9 +170,12 @@ export class VisionToolExposure {
166
170
  if (exec.agent === undefined) {
167
171
  throw new Error(`${VISION_TOOLKIT_ACTIVATE}: an Agent Session is required`)
168
172
  }
169
- return Promise.resolve(this.activate(exec.agent))
173
+ // An already-active Agent is reloaded (re-reads the latest Settings
174
+ // snapshot) rather than returning the stale set, so the tool genuinely
175
+ // refreshes the visible tool list on an explicit user request.
176
+ return Promise.resolve(this.reload(exec.agent))
170
177
  },
171
- presentCall: () => ({ card: 'generic', title: 'Activate vision tools', kind: 'execute' }),
178
+ presentCall: () => ({ card: 'generic', title: 'Refresh or list vision tools', kind: 'execute' }),
172
179
  })
173
180
  }
174
181
 
@@ -222,7 +229,7 @@ export class VisionToolExposure {
222
229
  if (state === undefined) throw new Error(`dsh-vision-toolkit: Agent ${String(agent.id)} has no exposure state`)
223
230
  if (state.active) return { activated: false, tools: [...state.toolNames] }
224
231
 
225
- const definitions = this.createTools()
232
+ const definitions = this.createTools(this.resolveVisibility())
226
233
  const toolDisposers: Array<() => void> = []
227
234
  try {
228
235
  for (const definition of definitions) toolDisposers.push(agent.ctx.tools.register(definition))
@@ -241,6 +248,24 @@ export class VisionToolExposure {
241
248
  return { activated: true, tools: [...state.toolNames] }
242
249
  }
243
250
 
251
+ /**
252
+ * Explicit refresh/reload from the bootstrap tool. An inactive Agent is
253
+ * mounted; an active Agent is torn down and re-mounted from the *current*
254
+ * Settings snapshot, so the returned tool list reflects the latest
255
+ * `toolVisibility` instead of the stale activation-time set.
256
+ */
257
+ private reload(agent: Agent): VisionToolkitActivationResult {
258
+ this.attach(agent)
259
+ const state = this.states.get(agent)
260
+ if (state === undefined) return { activated: false, tools: [] }
261
+ if (!state.active) return this.activate(agent)
262
+ // Dispose the current tool generation (including the hide-restriction on
263
+ // the bootstrap gauntlet) and rebuild from the fresh snapshot.
264
+ this.disposeState(state)
265
+ this.states.set(agent, { active: false, toolDisposers: [], toolNames: [] })
266
+ return this.activate(agent)
267
+ }
268
+
244
269
  /** Whether the session is attached to the live SessionStore (production). */
245
270
  private isLiveSession(session: Session): boolean {
246
271
  return this.ctx.sessions.get(session.id) === session
package/src/index.ts CHANGED
@@ -83,11 +83,16 @@ export async function apply(ctx: Context, config: VisionToolkitConfig = {}): Pro
83
83
 
84
84
  const ensureOperational = (): void => {
85
85
  if (!manager.ready || operationalDisposers !== undefined) return
86
- const exposure = new VisionToolExposure(ctx, () => createVisionTools(
87
- () => manager.current(),
88
- value => artifacts.presentationMeta(value),
89
- lifecycle.signal,
90
- ))
86
+ const exposure = new VisionToolExposure(
87
+ ctx,
88
+ (visibility) => createVisionTools(
89
+ () => manager.current(),
90
+ value => artifacts.presentationMeta(value),
91
+ lifecycle.signal,
92
+ visibility,
93
+ ),
94
+ () => manager.currentConfig().toolVisibility,
95
+ )
91
96
  let activationTool: (() => void) | undefined
92
97
  let exposureDisposer: (() => void) | undefined
93
98
  let skill: (() => void) | undefined
@@ -0,0 +1,174 @@
1
+ /**
2
+ * Minimal S3-compatible object storage bridge used by the URL image-transfer
3
+ * path. It uploads one image, resolves a model-reachable URL (a configured
4
+ * public base URL or a temporary presigned URL), and deletes the object after
5
+ * the vision operation settles. It also provides the Settings "test storage"
6
+ * probe (upload → head → delete).
7
+ * @module dsh-vision-toolkit/object-storage
8
+ */
9
+
10
+ import { createHash, randomUUID } from 'node:crypto'
11
+ import { readFile } from 'node:fs/promises'
12
+ import { basename } from 'node:path'
13
+ import {
14
+ DeleteObjectCommand,
15
+ GetObjectCommand,
16
+ HeadObjectCommand,
17
+ PutObjectCommand,
18
+ S3Client,
19
+ type S3ClientConfig,
20
+ } from '@aws-sdk/client-s3'
21
+ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
22
+ import { VisionToolkitError } from './errors.ts'
23
+
24
+ /** Fully resolved object-storage connection settings (secrets already filled). */
25
+ export interface ObjectStorageSettings {
26
+ endpoint: string
27
+ bucket: string
28
+ accessKeyId: string
29
+ secretAccessKey: string
30
+ publicBase?: string
31
+ }
32
+
33
+ /** Whether the required connection fields are present enough to attempt a request. */
34
+ export function isObjectStorageConfigured(settings: ObjectStorageSettings): boolean {
35
+ return settings.endpoint.length > 0
36
+ && settings.bucket.length > 0
37
+ && settings.accessKeyId.length > 0
38
+ && settings.secretAccessKey.length > 0
39
+ }
40
+
41
+ /** Stable object-key prefix so every upload lives under one deletable namespace. */
42
+ const OBJECT_KEY_PREFIX = 'dsh-vision-toolkit'
43
+
44
+ function encodeKey(key: string): string {
45
+ return key.split('/').map(encodeURIComponent).join('/')
46
+ }
47
+
48
+ function clientFor(settings: ObjectStorageSettings): S3Client {
49
+ const config: S3ClientConfig = {
50
+ region: 'auto',
51
+ forcePathStyle: true,
52
+ credentials: {
53
+ accessKeyId: settings.accessKeyId,
54
+ secretAccessKey: settings.secretAccessKey,
55
+ },
56
+ }
57
+ if (settings.endpoint.length > 0) config.endpoint = settings.endpoint
58
+ return new S3Client(config)
59
+ }
60
+
61
+ function publicError(error: unknown): string {
62
+ if (error instanceof Error) return error.message
63
+ return String(error)
64
+ }
65
+
66
+ /**
67
+ * One upload's worth of bookkeeping: the object key and the URL handed to the
68
+ * model. The key is returned to the runtime so it can delete the object after
69
+ * the operation settles.
70
+ */
71
+ export interface UploadedObject {
72
+ key: string
73
+ url: string
74
+ }
75
+
76
+ /** A small S3-compatible object store bound to one bucket and credential. */
77
+ export class ObjectStorageClient {
78
+ private client?: S3Client
79
+
80
+ constructor(private readonly settings: ObjectStorageSettings) {}
81
+
82
+ private requireClient(): S3Client {
83
+ if (this.client === undefined) this.client = clientFor(this.settings)
84
+ return this.client
85
+ }
86
+
87
+ /** Upload one local image file and resolve its model-reachable URL. */
88
+ async uploadImage(localPath: string, contentType: string): Promise<UploadedObject> {
89
+ const body = await readFile(localPath)
90
+ const digest = createHash('sha256').update(body).digest('hex').slice(0, 12)
91
+ const name = basename(localPath).replace(/[^A-Za-z0-9._-]/g, '_')
92
+ const key = `${OBJECT_KEY_PREFIX}/${randomUUID()}-${digest}-${name}`
93
+ try {
94
+ await this.requireClient().send(new PutObjectCommand({
95
+ Bucket: this.settings.bucket,
96
+ Key: key,
97
+ Body: body,
98
+ ContentType: contentType,
99
+ }))
100
+ } catch (error) {
101
+ throw new VisionToolkitError('service', `object storage upload failed: ${publicError(error)}`, { cause: error })
102
+ }
103
+ return { key, url: await this.urlFor(key) }
104
+ }
105
+
106
+ /** Resolve the model-reachable URL: public base URL when set, else presigned. */
107
+ async urlFor(key: string): Promise<string> {
108
+ if (this.settings.publicBase !== undefined && this.settings.publicBase.length > 0) {
109
+ return `${this.settings.publicBase}/${encodeKey(key)}`
110
+ }
111
+ try {
112
+ return await getSignedUrl(
113
+ this.requireClient(),
114
+ new GetObjectCommand({ Bucket: this.settings.bucket, Key: key }),
115
+ { expiresIn: 3600 },
116
+ )
117
+ } catch (error) {
118
+ throw new VisionToolkitError('service', `object storage presign failed: ${publicError(error)}`, { cause: error })
119
+ }
120
+ }
121
+
122
+ /** Delete one uploaded object; failures are logged, never fatal to the call. */
123
+ async deleteObject(key: string): Promise<void> {
124
+ try {
125
+ await this.requireClient().send(new DeleteObjectCommand({
126
+ Bucket: this.settings.bucket,
127
+ Key: key,
128
+ }))
129
+ } catch {
130
+ // Best-effort cleanup: a failed delete must not mask the vision result.
131
+ }
132
+ }
133
+
134
+ /** Settings "test storage" probe: upload a tiny object, head it, then delete it. */
135
+ async test(): Promise<{ detail: string }> {
136
+ if (!isObjectStorageConfigured(this.settings)) {
137
+ throw new VisionToolkitError('config', 'object storage is not fully configured (endpoint, bucket, access key id, and secret access key are required)')
138
+ }
139
+ const key = `${OBJECT_KEY_PREFIX}/.connection-test-${randomUUID()}`
140
+ const marker = `dsh-vision-toolkit object storage test ${Date.now()}`
141
+ try {
142
+ await this.requireClient().send(new PutObjectCommand({
143
+ Bucket: this.settings.bucket,
144
+ Key: key,
145
+ Body: marker,
146
+ ContentType: 'text/plain',
147
+ }))
148
+ await this.requireClient().send(new HeadObjectCommand({ Bucket: this.settings.bucket, Key: key }))
149
+ await this.requireClient().send(new DeleteObjectCommand({ Bucket: this.settings.bucket, Key: key }))
150
+ } catch (error) {
151
+ throw new VisionToolkitError('service', `object storage test failed: ${publicError(error)}`, { cause: error })
152
+ }
153
+ const urlMode = this.settings.publicBase !== undefined && this.settings.publicBase.length > 0
154
+ ? `public base ${this.settings.publicBase}`
155
+ : 'presigned URL'
156
+ return { detail: `bucket ${this.settings.bucket} reachable; model URLs will use ${urlMode}` }
157
+ }
158
+ }
159
+
160
+ /**
161
+ * Split a credential value of the form `accessKeyId:secretAccessKey` into its
162
+ * two parts. The access key id never contains a colon, so splitting on the
163
+ * first colon is safe.
164
+ */
165
+ export function splitObjectStorageCredential(value: string): { accessKeyId: string; secretAccessKey: string } {
166
+ const index = value.indexOf(':')
167
+ if (index <= 0) {
168
+ throw new VisionToolkitError('config', 'object storage credential must be "accessKeyId:secretAccessKey"')
169
+ }
170
+ return {
171
+ accessKeyId: value.slice(0, index),
172
+ secretAccessKey: value.slice(index + 1),
173
+ }
174
+ }
package/src/paths.ts CHANGED
@@ -16,6 +16,12 @@ import { VisionToolkitError } from './errors.ts'
16
16
  /** Supported input image extensions (the upstream client's allowlist). */
17
17
  export const SUPPORTED_IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp'] as const
18
18
 
19
+ /** Supported input video extensions probed with the bundled ffprobe binary. */
20
+ export const SUPPORTED_VIDEO_EXTENSIONS = [
21
+ '.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v',
22
+ '.mpg', '.mpeg', '.wmv', '.flv', '.ts', '.m2ts', '.3gp',
23
+ ] as const
24
+
19
25
  /** Resolved path policy for one tool invocation. */
20
26
  export interface PathPolicy {
21
27
  /** Real workspace root. */
@@ -399,6 +405,11 @@ export function resolveHtmlFile(raw: string, policy: PathPolicy): Promise<{ path
399
405
  return resolveAuthorizedFile(raw, policy, ['.html', '.htm'], 'HTML source')
400
406
  }
401
407
 
408
+ /** Validate one input video path against the video extension allowlist. */
409
+ export function resolveInputVideo(raw: string, policy: PathPolicy): Promise<{ path: string; bytes: number }> {
410
+ return resolveAuthorizedFile(raw, policy, SUPPORTED_VIDEO_EXTENSIONS, 'video')
411
+ }
412
+
402
413
  /**
403
414
  * Resolve an optional user-supplied output filename inside the plugin output
404
415
  * directory. Absolute paths, `..` segments, and wrong extensions are rejected.