@mengruo/dsh-vision-toolkit 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.i18n.yaml +2 -2
  2. package/README.md +15 -70
  3. package/README.zh.md +15 -69
  4. package/docs/requirements-traceability/README.i18n.yaml +2 -2
  5. package/docs/requirements-traceability/README.md +1 -1
  6. package/docs/requirements-traceability/README.zh.md +1 -1
  7. package/lib/artifact-access.js +20 -2
  8. package/lib/artifact-access.js.map +1 -1
  9. package/lib/client.js +128 -12
  10. package/lib/client.js.map +1 -1
  11. package/lib/config.js +69 -0
  12. package/lib/config.js.map +1 -1
  13. package/lib/errors.js +25 -2
  14. package/lib/errors.js.map +1 -1
  15. package/lib/evidence-cache.js +2 -1
  16. package/lib/evidence-cache.js.map +1 -1
  17. package/lib/exposure.js +14 -1
  18. package/lib/exposure.js.map +1 -1
  19. package/lib/image-input-variants.js +22 -12
  20. package/lib/image-input-variants.js.map +1 -1
  21. package/lib/index.js +53 -6
  22. package/lib/index.js.map +1 -1
  23. package/lib/object-storage.js +141 -0
  24. package/lib/object-storage.js.map +1 -0
  25. package/lib/paste-images.js +67 -19
  26. package/lib/paste-images.js.map +1 -1
  27. package/lib/paths.js +214 -28
  28. package/lib/paths.js.map +1 -1
  29. package/lib/runtime-manager.js +76 -10
  30. package/lib/runtime-manager.js.map +1 -1
  31. package/lib/runtime.js +219 -38
  32. package/lib/runtime.js.map +1 -1
  33. package/lib/storage-history.js +154 -0
  34. package/lib/storage-history.js.map +1 -0
  35. package/lib/types/artifact-access.d.ts.map +1 -1
  36. package/lib/types/client/index.d.ts +45 -2
  37. package/lib/types/client/index.d.ts.map +1 -1
  38. package/lib/types/client/paste-images.d.ts +2 -0
  39. package/lib/types/client/paste-images.d.ts.map +1 -1
  40. package/lib/types/config.d.ts +55 -0
  41. package/lib/types/config.d.ts.map +1 -1
  42. package/lib/types/errors.d.ts +18 -2
  43. package/lib/types/errors.d.ts.map +1 -1
  44. package/lib/types/evidence-cache.d.ts +1 -1
  45. package/lib/types/evidence-cache.d.ts.map +1 -1
  46. package/lib/types/exposure.d.ts.map +1 -1
  47. package/lib/types/image-input-variants.d.ts +5 -3
  48. package/lib/types/image-input-variants.d.ts.map +1 -1
  49. package/lib/types/index.d.ts.map +1 -1
  50. package/lib/types/object-storage.d.ts +54 -0
  51. package/lib/types/object-storage.d.ts.map +1 -0
  52. package/lib/types/paste-images.d.ts +12 -4
  53. package/lib/types/paste-images.d.ts.map +1 -1
  54. package/lib/types/paths.d.ts +31 -5
  55. package/lib/types/paths.d.ts.map +1 -1
  56. package/lib/types/runtime-manager.d.ts +28 -4
  57. package/lib/types/runtime-manager.d.ts.map +1 -1
  58. package/lib/types/runtime.d.ts +31 -1
  59. package/lib/types/runtime.d.ts.map +1 -1
  60. package/lib/types/storage-history.d.ts +63 -0
  61. package/lib/types/storage-history.d.ts.map +1 -0
  62. package/lib/types/upstream.d.ts +1 -0
  63. package/lib/types/upstream.d.ts.map +1 -1
  64. package/lib/types/web.d.ts +7 -0
  65. package/lib/types/web.d.ts.map +1 -1
  66. package/lib/upstream.js +34 -8
  67. package/lib/upstream.js.map +1 -1
  68. package/lib/web.js +44 -9
  69. package/lib/web.js.map +1 -1
  70. package/package.json +3 -1
  71. package/src/artifact-access.ts +22 -2
  72. package/src/client/index.tsx +169 -9
  73. package/src/client/paste-images.tsx +14 -4
  74. package/src/config.ts +128 -0
  75. package/src/errors.ts +25 -2
  76. package/src/evidence-cache.ts +2 -1
  77. package/src/exposure.ts +16 -2
  78. package/src/image-input-variants.ts +21 -6
  79. package/src/index.ts +65 -6
  80. package/src/object-storage.ts +174 -0
  81. package/src/paste-images.ts +81 -19
  82. package/src/paths.ts +249 -28
  83. package/src/runtime-manager.ts +93 -10
  84. package/src/runtime.ts +215 -36
  85. package/src/storage-history.ts +172 -0
  86. package/src/upstream.ts +36 -7
  87. package/src/web.ts +54 -9
  88. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
  89. package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
  90. package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
  91. package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
  92. package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
  93. package/vendor/agent-vision-toolkit/bin/glance +8 -1
  94. package/vendor/agent-vision-toolkit/detect.py +13 -7
  95. package/vendor/agent-vision-toolkit/ground.py +43 -18
  96. package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
  97. package/vendor/agent-vision-toolkit/vision_client.py +84 -6
  98. package/assets/community-group-qr.png +0 -0
  99. package/assets/logo_aihubmix.png +0 -0
  100. package/assets/logo_eapi_dark.png +0 -0
  101. package/assets/wechat-reward.png +0 -0
@@ -0,0 +1,172 @@
1
+ /** Durable configured-storage history used to authorize persisted image paths after Profile restarts. */
2
+
3
+ import type { Context, Fiber } from '@deepseek-ai/cordis'
4
+ import { defineDomain } from '@deepseek-ai/dsh-storage-domain'
5
+ import type { DomainGlobal } from '@deepseek-ai/dsh-storage-domain'
6
+ import { z } from 'zod'
7
+ import { resolveConfig, type VisionToolkitConfig } from './config.ts'
8
+
9
+ const storageHistoryStateSchema = z.object({
10
+ roots: z.array(z.string().min(1)),
11
+ })
12
+
13
+ type StorageHistoryState = z.infer<typeof storageHistoryStateSchema>
14
+
15
+ /** Plugin-owned storage roots that survive Settings-provider and Profile restarts. */
16
+ export const storageHistoryDomainSpec = defineDomain({
17
+ name: 'vision_toolkit_storage',
18
+ version: 0,
19
+ global: {
20
+ schema: storageHistoryStateSchema,
21
+ initial: { roots: [] },
22
+ },
23
+ tables: {},
24
+ })
25
+
26
+ interface StorageBinding {
27
+ accepting: boolean
28
+ global: DomainGlobal<StorageHistoryState>
29
+ }
30
+
31
+ function sameRoots(left: readonly string[], right: readonly string[]): boolean {
32
+ return left.length === right.length && left.every((root, index) => root === right[index])
33
+ }
34
+
35
+ /**
36
+ * Return every configured root that must remain readable, including the active root.
37
+ * @param config - Settings generation to summarize.
38
+ * @returns normalized configured roots in retention order.
39
+ */
40
+ export function configuredStorageRoots(config: VisionToolkitConfig): string[] {
41
+ const resolved = resolveConfig(config)
42
+ return [...new Set([
43
+ ...resolved.storageHistory,
44
+ ...(resolved.storageDir === undefined ? [] : [resolved.storageDir]),
45
+ ])]
46
+ }
47
+
48
+ /**
49
+ * Merge plugin-owned roots into a Settings generation without retaining its active root as history.
50
+ * @param config - Settings generation being restored.
51
+ * @param durableRoots - roots loaded from the plugin-owned sidecar.
52
+ * @returns the original generation when unchanged, otherwise a generation with restored history.
53
+ */
54
+ export function restoreDurableStorageHistory(
55
+ config: VisionToolkitConfig,
56
+ durableRoots: readonly string[],
57
+ ): VisionToolkitConfig {
58
+ const resolved = resolveConfig(config)
59
+ const storageHistory = [...new Set([...resolved.storageHistory, ...durableRoots])]
60
+ .filter(root => root !== resolved.storageDir)
61
+ if (sameRoots(storageHistory, resolved.storageHistory)) return config
62
+ return { ...config, storageHistory }
63
+ }
64
+
65
+ /** Optional storage-domain sidecar for storage roots that Settings cannot persist itself. */
66
+ export class StorageHistoryStore {
67
+ private storage: StorageBinding | undefined
68
+ private storageFiber: (Fiber & PromiseLike<Fiber>) | undefined
69
+ private storageReady: Promise<void> | undefined
70
+ private mutationTail: Promise<void> = Promise.resolve()
71
+ private desiredRoots: readonly string[] | undefined
72
+ private persistenceTicket = 0
73
+ private warned = false
74
+
75
+ constructor(private readonly ctx: Context) {
76
+ if (typeof ctx.inject !== 'function') return
77
+ this.storageFiber = ctx.inject(['storageDomain'], async (storageCtx: Context) => {
78
+ const domain = await storageCtx.storageDomain.open(storageHistoryDomainSpec)
79
+ const binding: StorageBinding = { accepting: true, global: domain.global }
80
+ this.storage = binding
81
+ try {
82
+ if (this.desiredRoots !== undefined) {
83
+ await this.write(binding, this.desiredRoots, this.persistenceTicket)
84
+ }
85
+ } catch (error) {
86
+ this.storage = undefined
87
+ await domain.close()
88
+ throw error
89
+ }
90
+ return async () => {
91
+ binding.accepting = false
92
+ if (this.storage === binding) this.storage = undefined
93
+ await this.mutationTail
94
+ await domain.close()
95
+ }
96
+ })
97
+ this.storageReady = Promise.resolve(this.storageFiber).then(
98
+ () => undefined,
99
+ (error: unknown) => { this.warnOnce(error) },
100
+ )
101
+ }
102
+
103
+ /**
104
+ * Restore durable roots into one Settings generation before runtime preparation.
105
+ * @param config - Settings generation to restore.
106
+ * @returns the generation with available durable roots merged into its history.
107
+ */
108
+ async restore(config: VisionToolkitConfig): Promise<VisionToolkitConfig> {
109
+ const binding = await this.prepareStorage()
110
+ return restoreDurableStorageHistory(config, binding?.global.get().roots ?? [])
111
+ }
112
+
113
+ /**
114
+ * Persist the active and historical configured roots.
115
+ * @param config - validated generation whose roots must survive restart.
116
+ * @returns false when no storage-domain is available; true after persistence or when there are no roots.
117
+ */
118
+ async persist(config: VisionToolkitConfig): Promise<boolean> {
119
+ const roots = configuredStorageRoots(config)
120
+ const ticket = ++this.persistenceTicket
121
+ this.desiredRoots = roots
122
+ if (roots.length === 0) return true
123
+ const binding = await this.prepareStorage()
124
+ if (binding === undefined) return false
125
+ await this.write(binding, roots, ticket)
126
+ return true
127
+ }
128
+
129
+ /** Release the optional storage-domain binding with the plugin lifecycle. */
130
+ dispose(): void {
131
+ const fiber = this.storageFiber
132
+ this.storageFiber = undefined
133
+ this.storageReady = undefined
134
+ if (fiber !== undefined) void fiber.dispose().catch(error => { this.warnOnce(error) })
135
+ }
136
+
137
+ private async prepareStorage(): Promise<StorageBinding | undefined> {
138
+ const current = this.activeStorage()
139
+ if (current !== undefined) return current
140
+ if (this.ctx.get('storageDomain') === undefined) return undefined
141
+ await this.storageReady
142
+ return this.activeStorage()
143
+ }
144
+
145
+ private activeStorage(): StorageBinding | undefined {
146
+ return this.storage?.accepting === true ? this.storage : undefined
147
+ }
148
+
149
+ private write(binding: StorageBinding, roots: readonly string[], ticket: number): Promise<void> {
150
+ return this.enqueueMutation(async () => {
151
+ if (!binding.accepting) throw new Error('the storage-domain provider changed while storage history was pending')
152
+ if (ticket !== this.persistenceTicket) return
153
+ if (sameRoots(binding.global.get().roots, roots)) return
154
+ await binding.global.set({ roots: [...roots] })
155
+ })
156
+ }
157
+
158
+ private enqueueMutation(operation: () => Promise<void>): Promise<void> {
159
+ const result = this.mutationTail.then(operation)
160
+ this.mutationTail = result.then(() => undefined, () => undefined)
161
+ return result
162
+ }
163
+
164
+ private warnOnce(error: unknown): void {
165
+ if (this.warned) return
166
+ this.warned = true
167
+ this.ctx.logger?.warn(
168
+ 'dsh-vision-toolkit: configured storage history sidecar is unavailable. %s',
169
+ (error instanceof Error ? error.message : String(error)).slice(0, 500),
170
+ )
171
+ }
172
+ }
package/src/upstream.ts CHANGED
@@ -41,6 +41,7 @@ export interface UpstreamEnvironment {
41
41
  VISION_API_PROTOCOL: 'chat_completions' | 'anthropic'
42
42
  VISION_ANTHROPIC_THINKING: 'omit' | 'disabled' | 'adaptive'
43
43
  VISION_SSL_VERIFY?: string
44
+ VISION_STREAM?: string
44
45
  VISION_USER_AGENT: string
45
46
  LANG: 'zh' | 'en'
46
47
  }
@@ -761,6 +762,9 @@ export class UpstreamAdapter {
761
762
  ...(options.env.VISION_SSL_VERIFY === undefined
762
763
  ? {}
763
764
  : { VISION_SSL_VERIFY: options.env.VISION_SSL_VERIFY }),
765
+ ...(options.env.VISION_STREAM === undefined
766
+ ? {}
767
+ : { VISION_STREAM: options.env.VISION_STREAM }),
764
768
  VISION_USER_AGENT: options.env.VISION_USER_AGENT,
765
769
  LANG: options.env.LANG,
766
770
  VISION_ENV_FILE: join(prepared.cleanHome, 'vision.env'),
@@ -1059,22 +1063,47 @@ export class UpstreamAdapter {
1059
1063
  return new VisionToolkitError('output', `${tool}: upstream output exceeded the capture limit`)
1060
1064
  }
1061
1065
  const message = upstreamFailureMessage(tool, result.stderr, options.secrets ?? [])
1062
- if (/HTTP 401|\b401\b|Unauthorized|authentication/i.test(result.stderr)) {
1063
- return new VisionToolkitError('service', `${message}; verify the configured credential`)
1066
+ const stderr = result.stderr
1067
+
1068
+ // Remote provider failures, classified into a machine-routable taxonomy so
1069
+ // the failover loop retries transient errors and fails over immediately on
1070
+ // deterministic ones. Order matters: auth/quota/rate-limit/region/tos are
1071
+ // checked before the broader server/network/request-shape patterns.
1072
+ if (/HTTP 401|\b401\b|HTTP 403|\b403\b|Unauthorized|Forbidden|authentication|Invalid API-key|invalid api key|check the api key/i.test(stderr)) {
1073
+ return new VisionToolkitError('auth', `${message}; verify the configured credential`)
1074
+ }
1075
+ if (/HTTP 402|\b402\b|Payment Required|insufficient balance|insufficient quota|insufficient credits|billing|out of credits|out of quota/i.test(stderr)) {
1076
+ return new VisionToolkitError('quota', `${message}; the provider account is out of quota or unpaid`)
1064
1077
  }
1065
- if (/HTTP 429|\b429\b|rate limit|quota/i.test(result.stderr)) {
1078
+ if (/HTTP 429|\b429\b|rate ?limit|too many requests/i.test(stderr)) {
1066
1079
  return new VisionToolkitError('rate_limit', `${message}; retry later or reduce concurrency`)
1067
1080
  }
1068
- if (/Missing config VISION_/i.test(result.stderr)) {
1081
+ if (/not available in your region|prohibited region|unsupported region|region is not supported/i.test(stderr)) {
1082
+ return new VisionToolkitError('region', `${message}; the provider is not available in this region`)
1083
+ }
1084
+ if (/terms of service|\btos\b|content policy|safety system/i.test(stderr)) {
1085
+ return new VisionToolkitError('tos', `${message}; the request was rejected by the provider content policy`)
1086
+ }
1087
+ if (/HTTP 5\d\d|\b500\b|\b502\b|\b503\b|\b504\b|bad gateway|service unavailable|internal server error/i.test(stderr)) {
1088
+ return new VisionToolkitError('server', `${message}; the provider returned a server error`)
1089
+ }
1090
+ if (/connection refused|connection reset|ECONN|ENOTFOUND|EAI_AGAIN|getaddrinfo|fetch failed|name resolution|network error|network is unreachable/i.test(stderr)) {
1091
+ return new VisionToolkitError('network', `${message}; network failure reaching the provider`)
1092
+ }
1093
+ if (/HTTP 400|\b400\b|HTTP 404|\b404\b|HTTP 422|\b422\b|invalid request|invalid model|no such model|model not exist|unknown model/i.test(stderr)) {
1094
+ return new VisionToolkitError('invalid_request', `${message}; the request or model is not supported by this provider`)
1095
+ }
1096
+
1097
+ if (/Missing config VISION_/i.test(stderr)) {
1069
1098
  return new VisionToolkitError('config', message)
1070
1099
  }
1071
- if (/maxImagePixels|exceed(?:s|ing).*pixels/i.test(result.stderr)) {
1100
+ if (/maxImagePixels|exceed(?:s|ing).*pixels/i.test(stderr)) {
1072
1101
  return new VisionToolkitError('capacity', message)
1073
1102
  }
1074
- if (/not found|only PNG|unsupported|cannot open|empty region|must be|expects|invalid colour|needs at least/i.test(result.stderr)) {
1103
+ if (/not found|only PNG|unsupported|cannot open|empty region|must be|expects|invalid colour|needs at least/i.test(stderr)) {
1075
1104
  return new VisionToolkitError('input', message)
1076
1105
  }
1077
- if (/requires Pillow|requires numpy|requires vtracer|no Chrome|capture failed/i.test(result.stderr)) {
1106
+ if (/requires Pillow|requires numpy|requires vtracer|no Chrome|capture failed/i.test(stderr)) {
1078
1107
  return new VisionToolkitError('runtime', message)
1079
1108
  }
1080
1109
  return new VisionToolkitError(
package/src/web.ts CHANGED
@@ -22,6 +22,7 @@ import {
22
22
  } from './paste-images.ts'
23
23
  import {
24
24
  resolveConfig,
25
+ retainedStorageHistory,
25
26
  isBuiltInFreeVisionProvider,
26
27
  VISION_TOOLKIT_SETTINGS_NAMESPACE,
27
28
  type ResolvedProvider,
@@ -74,6 +75,12 @@ export interface VisionToolkitSettingsSnapshot {
74
75
  source?: string
75
76
  writable: boolean
76
77
  }>
78
+ /** Object-storage credential state; `ref` is empty when object storage is unset. */
79
+ objectStorageCredential: {
80
+ ref: string
81
+ configured: boolean
82
+ writable: boolean
83
+ }
77
84
  runtime: RuntimeManagerStatus
78
85
  release: {
79
86
  pluginVersion: string
@@ -120,7 +127,11 @@ interface ApplyUpdateRequest {
120
127
  expectedVersion: string
121
128
  }
122
129
 
123
- type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest
130
+ interface StorageTestRequest {
131
+ action: 'test-storage'
132
+ }
133
+
134
+ type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest | StorageTestRequest
124
135
 
125
136
  interface JsonError {
126
137
  ok: false
@@ -260,6 +271,7 @@ function parseRequest(value: unknown): SettingsRequest {
260
271
  }
261
272
  return { action: 'apply-update', expectedVersion: value.expectedVersion.trim() }
262
273
  }
274
+ if (value.action === 'test-storage') return { action: 'test-storage' }
263
275
  throw new TypeError(`unsupported action: ${value.action}`)
264
276
  }
265
277
 
@@ -314,6 +326,14 @@ export class VisionToolkitWebBackend {
314
326
  writable: info.writable,
315
327
  }
316
328
  }))
329
+ const objectStorageRef = resolved.objectStorage.credential === undefined ? '' : String(resolved.objectStorage.credential)
330
+ const objectStorageCredential = objectStorageRef === ''
331
+ ? { ref: '', configured: false, writable: this.ctx.settings.writable }
332
+ : await this.ctx.credentials.describe(credentialRef(objectStorageRef)).then(info => ({
333
+ ref: objectStorageRef,
334
+ configured: info.configured,
335
+ writable: info.writable,
336
+ }))
317
337
  const update = await this.updater.capability()
318
338
  return {
319
339
  schemaVersion: 1,
@@ -332,6 +352,7 @@ export class VisionToolkitWebBackend {
332
352
  writable: credential.writable,
333
353
  },
334
354
  credentials,
355
+ objectStorageCredential,
335
356
  runtime: this.manager.status(),
336
357
  release: {
337
358
  pluginVersion: PLUGIN_VERSION,
@@ -346,16 +367,22 @@ export class VisionToolkitWebBackend {
346
367
 
347
368
  private async save(request: SaveRequest): Promise<VisionToolkitSettingsSnapshot> {
348
369
  if (!this.ctx.settings.writable) throw new Error('settings provider is read-only')
370
+ const current = resolveConfig(descriptorOf(this.ctx).value as VisionToolkitConfig)
371
+ const storageHistory = retainedStorageHistory(request.value, current)
372
+ const value: VisionToolkitConfig = {
373
+ ...request.value,
374
+ ...(storageHistory.length === 0 ? {} : { storageHistory }),
375
+ }
349
376
  let candidate: PreparedRuntimeGeneration
350
377
  try {
351
- candidate = await this.manager.prepareCandidate(request.value)
378
+ candidate = await this.manager.prepareCandidate(value)
352
379
  } catch (error) {
353
380
  this.manager.recordFailure(error)
354
381
  throw error
355
382
  }
356
383
  await this.ctx.settings.replace(
357
384
  VISION_TOOLKIT_SETTINGS_NAMESPACE,
358
- request.value as object,
385
+ value as object,
359
386
  request.expectedRevision,
360
387
  )
361
388
  this.manager.activateCandidate(candidate)
@@ -373,10 +400,18 @@ export class VisionToolkitWebBackend {
373
400
  )
374
401
  }
375
402
  const resolved = resolveConfig(descriptor.value as VisionToolkitConfig)
376
- const provider = resolved.providers.find(entry => String(entry.credential) === String(request.ref))
403
+ const ref = String(request.ref)
404
+ if (resolved.objectStorage.credential !== undefined && String(resolved.objectStorage.credential) === ref) {
405
+ if (!request.value.includes(':')) {
406
+ throw new Error('object storage credential must be "accessKeyId:secretAccessKey"')
407
+ }
408
+ await this.ctx.credentials.set(request.ref, request.value)
409
+ return this.snapshot()
410
+ }
411
+ const provider = resolved.providers.find(entry => String(entry.credential) === ref)
377
412
  if (provider === undefined) {
378
413
  throw new CredentialReferenceConflictError(
379
- `credential reference "${String(request.ref)}" does not match any configured vision provider; reload Settings and try again`,
414
+ `credential reference "${ref}" does not match any configured vision provider; reload Settings and try again`,
380
415
  )
381
416
  }
382
417
  if (isBuiltInFreeVisionProvider(provider)) {
@@ -419,6 +454,11 @@ export class VisionToolkitWebBackend {
419
454
  }
420
455
  }
421
456
 
457
+ private async testStorage(): Promise<{ detail: string }> {
458
+ if (!this.manager.ready) throw new Error('runtime is not ready; fix Settings and save a valid configuration first')
459
+ return this.manager.current().testObjectStorage()
460
+ }
461
+
422
462
  /** Handle the exact Settings route. */
423
463
  async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
424
464
  if (req.method === 'GET') {
@@ -451,6 +491,9 @@ export class VisionToolkitWebBackend {
451
491
  case 'health':
452
492
  responseJson(res, 200, { ok: true, value: await this.health(parsed, req) })
453
493
  break
494
+ case 'test-storage':
495
+ responseJson(res, 200, { ok: true, value: await this.testStorage() })
496
+ break
454
497
  case 'save':
455
498
  responseJson(res, 200, { ok: true, value: await this.save(parsed) })
456
499
  break
@@ -479,14 +522,16 @@ export class VisionToolkitWebBackend {
479
522
  ? error.code
480
523
  : parsed.action === 'health'
481
524
  ? 'health-failed'
482
- : parsed.action === 'credential'
483
- ? 'credential-rejected'
484
- : 'settings-rejected'
525
+ : parsed.action === 'test-storage'
526
+ ? 'storage-test-failed'
527
+ : parsed.action === 'credential'
528
+ ? 'credential-rejected'
529
+ : 'settings-rejected'
485
530
  const updateConflict = updateError && ['update-in-progress', 'update-stale', 'update-unavailable', 'already-current'].includes(error.code)
486
531
  const updateGateway = updateError && error.code === 'update-check-failed'
487
532
  const status = settingsConflict || credentialConflict || updateConflict
488
533
  ? 409
489
- : parsed.action === 'health'
534
+ : parsed.action === 'health' || parsed.action === 'test-storage'
490
535
  ? 503
491
536
  : updateGateway
492
537
  ? 502
@@ -3,7 +3,7 @@
3
3
  "repository": "https://github.com/Anionex/agent-vision-toolkit",
4
4
  "version": "v0.1.0+snapshot.bc9803d",
5
5
  "commit": "bc9803d7d6300c864d17460ecbb33540b26638e0",
6
- "contentSha256": "0eaa22a0d1d0dd6d6523a0ddd3fa8197246005042bc164a703fb05102b287a38",
6
+ "contentSha256": "09bedf007e1469a28981227def5aa33e03667ea21fe2972937f0359898e2928e",
7
7
  "files": [
8
8
  {
9
9
  "path": "CHANGELOG.md",
@@ -32,8 +32,8 @@
32
32
  },
33
33
  {
34
34
  "path": "bin/glance",
35
- "bytes": 3742,
36
- "sha256": "fa4ba52e8e180475b948daec817151d893d957f973f7d9df1b8bbf201051cefc"
35
+ "bytes": 3941,
36
+ "sha256": "7552bef0ba396162f64e8fc4c4a53404dfe2d3149ca3d1c5c8bd717facf82af6"
37
37
  },
38
38
  {
39
39
  "path": "bin/ground",
@@ -47,13 +47,13 @@
47
47
  },
48
48
  {
49
49
  "path": "detect.py",
50
- "bytes": 2218,
51
- "sha256": "48a7070084f5b23b1477fa9a690ef1e679da8a03228e64a1ac583de633040bfc"
50
+ "bytes": 2532,
51
+ "sha256": "cac2135f9d835215f3ba90bfd70fd84dd02b0e88299f2b60e109e006b4f3a4d9"
52
52
  },
53
53
  {
54
54
  "path": "ground.py",
55
- "bytes": 10117,
56
- "sha256": "845e56dbdf92f2c79495170f5d215de49985b3099012b4d398231d3c78bd0090"
55
+ "bytes": 11289,
56
+ "sha256": "6d3175d3f5a7d561d55b2a967c5ca3d49940e80014de658ed9a3e5d45a233a2f"
57
57
  },
58
58
  {
59
59
  "path": "skills/vision-tools/scripts/dominant_colors.py",
@@ -82,13 +82,13 @@
82
82
  },
83
83
  {
84
84
  "path": "tests/test_vision_client.py",
85
- "bytes": 20481,
86
- "sha256": "b019f0da28b7567226292044f791059ee34d7f0088bd9ceb328cfa1bab250a48"
85
+ "bytes": 24882,
86
+ "sha256": "87e39429766b38caed6a4ccb1ac8b193094c69858a0f7541940a005f66421e45"
87
87
  },
88
88
  {
89
89
  "path": "vision_client.py",
90
- "bytes": 11838,
91
- "sha256": "c6a48048c864e99513faf90ba17c4ac0d9462bb718683682ed8eda12c0e2cfe0"
90
+ "bytes": 15053,
91
+ "sha256": "6800843b99df500a476b1dd6ff5a4c05aa8122ed990f6f91b16a11ee07bdebe6"
92
92
  }
93
93
  ]
94
94
  }
@@ -40,6 +40,13 @@ def region_data_url(path, region):
40
40
  return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
41
41
 
42
42
 
43
+ def image_url(path):
44
+ """Pass http(s) URLs through unchanged; encode local files as data URLs."""
45
+ if path.startswith(("http://", "https://")):
46
+ return path
47
+ return image_path_to_data_url(path)
48
+
49
+
43
50
  def build_prompt(args, count):
44
51
  if args.ocr is not None:
45
52
  extra = f" Additional requirements: {args.ocr}" if args.ocr else ""
@@ -77,7 +84,7 @@ def main():
77
84
  if args.region and len(args.images) > 1:
78
85
  raise VisionError("--region works with exactly one image")
79
86
  urls = ([region_data_url(args.images[0], args.region)] if args.region
80
- else [image_path_to_data_url(path) for path in args.images])
87
+ else [image_url(path) for path in args.images])
81
88
  answer = describe_image(
82
89
  urls,
83
90
  build_prompt(args, len(urls)),
@@ -8,7 +8,7 @@ try:
8
8
  except ImportError:
9
9
  Image = None
10
10
 
11
- from ground import GroundError, _position, locate
11
+ from ground import GroundError, _parse_size, _position, locate
12
12
  from vision_client import VisionError
13
13
 
14
14
  DEFAULT_CATEGORY = ("UI element (buttons, links, inputs, icons, labels, "
@@ -38,18 +38,24 @@ def main() -> None:
38
38
  prog="detect",
39
39
  description="Inventory the elements in an image (or a region) with pixel bounding boxes",
40
40
  )
41
- parser.add_argument("image", type=Path, help="path to the image")
41
+ parser.add_argument("image", help="path or http(s) URL to the image")
42
42
  parser.add_argument("category", nargs="?",
43
43
  help='restrict to a category, e.g. "buttons" or "icons" (default: all UI elements)')
44
44
  parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
45
45
  help="inventory only this pixel box; output stays in original-image coordinates")
46
+ parser.add_argument("--size", metavar="WxH",
47
+ help="analyzed image dimensions when the image is an http(s) URL")
46
48
  args = parser.parse_args()
47
49
  try:
48
- matches = locate(args.image.expanduser(), build_target(args.category), region=args.region)
49
- if Image is None:
50
- raise GroundError("detect requires Pillow; install the optional dependency pillow first")
51
- with Image.open(args.image.expanduser()) as image:
52
- width, height = image.size
50
+ size = _parse_size(args.size) if args.size else None
51
+ matches = locate(args.image, build_target(args.category), region=args.region, size=size)
52
+ if size is not None:
53
+ width, height = size
54
+ else:
55
+ if Image is None:
56
+ raise GroundError("detect requires Pillow; install the optional dependency pillow first")
57
+ with Image.open(Path(args.image).expanduser()) as image:
58
+ width, height = image.size
53
59
  except (GroundError, VisionError) as exc:
54
60
  parser.exit(1, f"detect: {exc}\n")
55
61
  for line in format_inventory(matches, width, height):
@@ -184,23 +184,32 @@ def _parse_region(region: str, width: int, height: int) -> tuple[int, int, int,
184
184
  return box
185
185
 
186
186
 
187
- def locate(image_path: Path, target: str, region: str | None = None) -> list[Match]:
187
+ def locate(image_path: str, target: str, region: str | None = None,
188
+ size: tuple[int, int] | None = None) -> list[Match]:
188
189
  if Image is None:
189
190
  raise GroundError("ground requires Pillow; install the optional dependency pillow first")
190
191
  load_default_env()
191
192
  box = None
192
- try:
193
- with Image.open(image_path) as image:
194
- width, height = image.size
195
- if region:
196
- box = _parse_region(region, width, height)
197
- buffer = io.BytesIO()
198
- image.crop(box).save(buffer, format="PNG")
199
- url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
200
- except (OSError, ValueError) as exc:
201
- raise GroundError(f"Cannot read image: {image_path}") from exc
193
+ if size is not None:
194
+ # URL transfer: the image was uploaded to object storage and this caller
195
+ # supplies the analyzed dimensions, so no local file access is needed.
196
+ width, height = size
197
+ url = image_path if image_path.startswith(("http://", "https://")) else image_path_to_data_url(image_path)
198
+ else:
199
+ local = Path(image_path).expanduser()
200
+ try:
201
+ with Image.open(local) as image:
202
+ width, height = image.size
203
+ if region:
204
+ box = _parse_region(region, width, height)
205
+ buffer = io.BytesIO()
206
+ image.crop(box).save(buffer, format="PNG")
207
+ url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
208
+ except (OSError, ValueError) as exc:
209
+ raise GroundError(f"Cannot read image: {image_path}") from exc
210
+ if box is None:
211
+ url = image_path_to_data_url(local)
202
212
  if box is None:
203
- url = image_path_to_data_url(image_path)
204
213
  width_used, height_used = width, height
205
214
  else:
206
215
  width_used, height_used = box[2] - box[0], box[3] - box[1]
@@ -216,6 +225,16 @@ def locate(image_path: Path, target: str, region: str | None = None) -> list[Mat
216
225
  m.bbox[2] + box[0], m.bbox[3] + box[1])) for m in matches]
217
226
 
218
227
 
228
+ def _parse_size(value: str) -> tuple[int, int]:
229
+ try:
230
+ width, height = (int(part) for part in value.lower().split("x"))
231
+ except (ValueError, AttributeError):
232
+ raise GroundError("--size expects WIDTHxHEIGHT (e.g. 1024x768)") from None
233
+ if width <= 0 or height <= 0:
234
+ raise GroundError("--size expects positive WIDTHxHEIGHT")
235
+ return width, height
236
+
237
+
219
238
  def _position(box: tuple[int, int, int, int], width: int, height: int) -> str:
220
239
  x1, y1, x2, y2 = box
221
240
  x = (x1 + x2) / 2
@@ -246,17 +265,23 @@ def main() -> None:
246
265
  prog="ground",
247
266
  description="Locate targets in an image with natural language and output pixel coordinates",
248
267
  )
249
- parser.add_argument("image", type=Path, help="path to the image")
268
+ parser.add_argument("image", help="path or http(s) URL to the image")
250
269
  parser.add_argument("target", help="target object or region to locate")
251
270
  parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
252
271
  help="search only this pixel box; output stays in original-image coordinates")
272
+ parser.add_argument("--size", metavar="WxH",
273
+ help="analyzed image dimensions when the image is an http(s) URL")
253
274
  args = parser.parse_args()
254
275
  try:
255
- matches = locate(args.image.expanduser(), args.target, region=args.region)
256
- if Image is None:
257
- raise GroundError("ground requires Pillow; install the optional dependency pillow first")
258
- with Image.open(args.image.expanduser()) as image:
259
- width, height = image.size
276
+ size = _parse_size(args.size) if args.size else None
277
+ matches = locate(args.image, args.target, region=args.region, size=size)
278
+ if size is not None:
279
+ width, height = size
280
+ else:
281
+ if Image is None:
282
+ raise GroundError("ground requires Pillow; install the optional dependency pillow first")
283
+ with Image.open(Path(args.image).expanduser()) as image:
284
+ width, height = image.size
260
285
  except (GroundError, VisionError) as exc:
261
286
  parser.exit(1, f"ground: {exc}\n")
262
287
  for line in format_matches(matches, width, height):