@mengruo/dsh-vision-toolkit 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +2 -2
- package/README.md +15 -70
- package/README.zh.md +15 -69
- package/docs/requirements-traceability/README.i18n.yaml +2 -2
- package/docs/requirements-traceability/README.md +1 -1
- package/docs/requirements-traceability/README.zh.md +1 -1
- package/lib/artifact-access.js +20 -2
- package/lib/artifact-access.js.map +1 -1
- package/lib/client.js +128 -12
- package/lib/client.js.map +1 -1
- package/lib/config.js +69 -0
- package/lib/config.js.map +1 -1
- package/lib/errors.js +25 -2
- package/lib/errors.js.map +1 -1
- package/lib/evidence-cache.js +2 -1
- package/lib/evidence-cache.js.map +1 -1
- package/lib/exposure.js +14 -1
- package/lib/exposure.js.map +1 -1
- package/lib/image-input-variants.js +22 -12
- package/lib/image-input-variants.js.map +1 -1
- package/lib/index.js +53 -6
- package/lib/index.js.map +1 -1
- package/lib/object-storage.js +141 -0
- package/lib/object-storage.js.map +1 -0
- package/lib/paste-images.js +67 -19
- package/lib/paste-images.js.map +1 -1
- package/lib/paths.js +214 -28
- package/lib/paths.js.map +1 -1
- package/lib/runtime-manager.js +76 -10
- package/lib/runtime-manager.js.map +1 -1
- package/lib/runtime.js +219 -38
- package/lib/runtime.js.map +1 -1
- package/lib/storage-history.js +154 -0
- package/lib/storage-history.js.map +1 -0
- package/lib/types/artifact-access.d.ts.map +1 -1
- package/lib/types/client/index.d.ts +45 -2
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/client/paste-images.d.ts +2 -0
- package/lib/types/client/paste-images.d.ts.map +1 -1
- package/lib/types/config.d.ts +55 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/errors.d.ts +18 -2
- package/lib/types/errors.d.ts.map +1 -1
- package/lib/types/evidence-cache.d.ts +1 -1
- package/lib/types/evidence-cache.d.ts.map +1 -1
- package/lib/types/exposure.d.ts.map +1 -1
- package/lib/types/image-input-variants.d.ts +5 -3
- package/lib/types/image-input-variants.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/object-storage.d.ts +54 -0
- package/lib/types/object-storage.d.ts.map +1 -0
- package/lib/types/paste-images.d.ts +12 -4
- package/lib/types/paste-images.d.ts.map +1 -1
- package/lib/types/paths.d.ts +31 -5
- package/lib/types/paths.d.ts.map +1 -1
- package/lib/types/runtime-manager.d.ts +28 -4
- package/lib/types/runtime-manager.d.ts.map +1 -1
- package/lib/types/runtime.d.ts +31 -1
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/storage-history.d.ts +63 -0
- package/lib/types/storage-history.d.ts.map +1 -0
- package/lib/types/upstream.d.ts +1 -0
- package/lib/types/upstream.d.ts.map +1 -1
- package/lib/types/web.d.ts +7 -0
- package/lib/types/web.d.ts.map +1 -1
- package/lib/upstream.js +34 -8
- package/lib/upstream.js.map +1 -1
- package/lib/web.js +44 -9
- package/lib/web.js.map +1 -1
- package/package.json +3 -1
- package/src/artifact-access.ts +22 -2
- package/src/client/index.tsx +169 -9
- package/src/client/paste-images.tsx +14 -4
- package/src/config.ts +128 -0
- package/src/errors.ts +25 -2
- package/src/evidence-cache.ts +2 -1
- package/src/exposure.ts +16 -2
- package/src/image-input-variants.ts +21 -6
- package/src/index.ts +65 -6
- package/src/object-storage.ts +174 -0
- package/src/paste-images.ts +81 -19
- package/src/paths.ts +249 -28
- package/src/runtime-manager.ts +93 -10
- package/src/runtime.ts +215 -36
- package/src/storage-history.ts +172 -0
- package/src/upstream.ts +36 -7
- package/src/web.ts +54 -9
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +11 -11
- package/vendor/agent-vision-toolkit/__pycache__/detect.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/ground.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/__pycache__/vision_client.cpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/__pycache__/glancecpython-314.pyc +0 -0
- package/vendor/agent-vision-toolkit/bin/glance +8 -1
- package/vendor/agent-vision-toolkit/detect.py +13 -7
- package/vendor/agent-vision-toolkit/ground.py +43 -18
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +88 -0
- package/vendor/agent-vision-toolkit/vision_client.py +84 -6
- package/assets/community-group-qr.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/logo_eapi_dark.png +0 -0
- package/assets/wechat-reward.png +0 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/** Durable configured-storage history used to authorize persisted image paths after Profile restarts. */
|
|
2
|
+
|
|
3
|
+
import type { Context, Fiber } from '@deepseek-ai/cordis'
|
|
4
|
+
import { defineDomain } from '@deepseek-ai/dsh-storage-domain'
|
|
5
|
+
import type { DomainGlobal } from '@deepseek-ai/dsh-storage-domain'
|
|
6
|
+
import { z } from 'zod'
|
|
7
|
+
import { resolveConfig, type VisionToolkitConfig } from './config.ts'
|
|
8
|
+
|
|
9
|
+
const storageHistoryStateSchema = z.object({
|
|
10
|
+
roots: z.array(z.string().min(1)),
|
|
11
|
+
})
|
|
12
|
+
|
|
13
|
+
type StorageHistoryState = z.infer<typeof storageHistoryStateSchema>
|
|
14
|
+
|
|
15
|
+
/** Plugin-owned storage roots that survive Settings-provider and Profile restarts. */
|
|
16
|
+
export const storageHistoryDomainSpec = defineDomain({
|
|
17
|
+
name: 'vision_toolkit_storage',
|
|
18
|
+
version: 0,
|
|
19
|
+
global: {
|
|
20
|
+
schema: storageHistoryStateSchema,
|
|
21
|
+
initial: { roots: [] },
|
|
22
|
+
},
|
|
23
|
+
tables: {},
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
interface StorageBinding {
|
|
27
|
+
accepting: boolean
|
|
28
|
+
global: DomainGlobal<StorageHistoryState>
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function sameRoots(left: readonly string[], right: readonly string[]): boolean {
|
|
32
|
+
return left.length === right.length && left.every((root, index) => root === right[index])
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Return every configured root that must remain readable, including the active root.
|
|
37
|
+
* @param config - Settings generation to summarize.
|
|
38
|
+
* @returns normalized configured roots in retention order.
|
|
39
|
+
*/
|
|
40
|
+
export function configuredStorageRoots(config: VisionToolkitConfig): string[] {
|
|
41
|
+
const resolved = resolveConfig(config)
|
|
42
|
+
return [...new Set([
|
|
43
|
+
...resolved.storageHistory,
|
|
44
|
+
...(resolved.storageDir === undefined ? [] : [resolved.storageDir]),
|
|
45
|
+
])]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Merge plugin-owned roots into a Settings generation without retaining its active root as history.
|
|
50
|
+
* @param config - Settings generation being restored.
|
|
51
|
+
* @param durableRoots - roots loaded from the plugin-owned sidecar.
|
|
52
|
+
* @returns the original generation when unchanged, otherwise a generation with restored history.
|
|
53
|
+
*/
|
|
54
|
+
export function restoreDurableStorageHistory(
|
|
55
|
+
config: VisionToolkitConfig,
|
|
56
|
+
durableRoots: readonly string[],
|
|
57
|
+
): VisionToolkitConfig {
|
|
58
|
+
const resolved = resolveConfig(config)
|
|
59
|
+
const storageHistory = [...new Set([...resolved.storageHistory, ...durableRoots])]
|
|
60
|
+
.filter(root => root !== resolved.storageDir)
|
|
61
|
+
if (sameRoots(storageHistory, resolved.storageHistory)) return config
|
|
62
|
+
return { ...config, storageHistory }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Optional storage-domain sidecar for storage roots that Settings cannot persist itself. */
|
|
66
|
+
export class StorageHistoryStore {
|
|
67
|
+
private storage: StorageBinding | undefined
|
|
68
|
+
private storageFiber: (Fiber & PromiseLike<Fiber>) | undefined
|
|
69
|
+
private storageReady: Promise<void> | undefined
|
|
70
|
+
private mutationTail: Promise<void> = Promise.resolve()
|
|
71
|
+
private desiredRoots: readonly string[] | undefined
|
|
72
|
+
private persistenceTicket = 0
|
|
73
|
+
private warned = false
|
|
74
|
+
|
|
75
|
+
constructor(private readonly ctx: Context) {
|
|
76
|
+
if (typeof ctx.inject !== 'function') return
|
|
77
|
+
this.storageFiber = ctx.inject(['storageDomain'], async (storageCtx: Context) => {
|
|
78
|
+
const domain = await storageCtx.storageDomain.open(storageHistoryDomainSpec)
|
|
79
|
+
const binding: StorageBinding = { accepting: true, global: domain.global }
|
|
80
|
+
this.storage = binding
|
|
81
|
+
try {
|
|
82
|
+
if (this.desiredRoots !== undefined) {
|
|
83
|
+
await this.write(binding, this.desiredRoots, this.persistenceTicket)
|
|
84
|
+
}
|
|
85
|
+
} catch (error) {
|
|
86
|
+
this.storage = undefined
|
|
87
|
+
await domain.close()
|
|
88
|
+
throw error
|
|
89
|
+
}
|
|
90
|
+
return async () => {
|
|
91
|
+
binding.accepting = false
|
|
92
|
+
if (this.storage === binding) this.storage = undefined
|
|
93
|
+
await this.mutationTail
|
|
94
|
+
await domain.close()
|
|
95
|
+
}
|
|
96
|
+
})
|
|
97
|
+
this.storageReady = Promise.resolve(this.storageFiber).then(
|
|
98
|
+
() => undefined,
|
|
99
|
+
(error: unknown) => { this.warnOnce(error) },
|
|
100
|
+
)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Restore durable roots into one Settings generation before runtime preparation.
|
|
105
|
+
* @param config - Settings generation to restore.
|
|
106
|
+
* @returns the generation with available durable roots merged into its history.
|
|
107
|
+
*/
|
|
108
|
+
async restore(config: VisionToolkitConfig): Promise<VisionToolkitConfig> {
|
|
109
|
+
const binding = await this.prepareStorage()
|
|
110
|
+
return restoreDurableStorageHistory(config, binding?.global.get().roots ?? [])
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Persist the active and historical configured roots.
|
|
115
|
+
* @param config - validated generation whose roots must survive restart.
|
|
116
|
+
* @returns false when no storage-domain is available; true after persistence or when there are no roots.
|
|
117
|
+
*/
|
|
118
|
+
async persist(config: VisionToolkitConfig): Promise<boolean> {
|
|
119
|
+
const roots = configuredStorageRoots(config)
|
|
120
|
+
const ticket = ++this.persistenceTicket
|
|
121
|
+
this.desiredRoots = roots
|
|
122
|
+
if (roots.length === 0) return true
|
|
123
|
+
const binding = await this.prepareStorage()
|
|
124
|
+
if (binding === undefined) return false
|
|
125
|
+
await this.write(binding, roots, ticket)
|
|
126
|
+
return true
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Release the optional storage-domain binding with the plugin lifecycle. */
|
|
130
|
+
dispose(): void {
|
|
131
|
+
const fiber = this.storageFiber
|
|
132
|
+
this.storageFiber = undefined
|
|
133
|
+
this.storageReady = undefined
|
|
134
|
+
if (fiber !== undefined) void fiber.dispose().catch(error => { this.warnOnce(error) })
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private async prepareStorage(): Promise<StorageBinding | undefined> {
|
|
138
|
+
const current = this.activeStorage()
|
|
139
|
+
if (current !== undefined) return current
|
|
140
|
+
if (this.ctx.get('storageDomain') === undefined) return undefined
|
|
141
|
+
await this.storageReady
|
|
142
|
+
return this.activeStorage()
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private activeStorage(): StorageBinding | undefined {
|
|
146
|
+
return this.storage?.accepting === true ? this.storage : undefined
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private write(binding: StorageBinding, roots: readonly string[], ticket: number): Promise<void> {
|
|
150
|
+
return this.enqueueMutation(async () => {
|
|
151
|
+
if (!binding.accepting) throw new Error('the storage-domain provider changed while storage history was pending')
|
|
152
|
+
if (ticket !== this.persistenceTicket) return
|
|
153
|
+
if (sameRoots(binding.global.get().roots, roots)) return
|
|
154
|
+
await binding.global.set({ roots: [...roots] })
|
|
155
|
+
})
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private enqueueMutation(operation: () => Promise<void>): Promise<void> {
|
|
159
|
+
const result = this.mutationTail.then(operation)
|
|
160
|
+
this.mutationTail = result.then(() => undefined, () => undefined)
|
|
161
|
+
return result
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
private warnOnce(error: unknown): void {
|
|
165
|
+
if (this.warned) return
|
|
166
|
+
this.warned = true
|
|
167
|
+
this.ctx.logger?.warn(
|
|
168
|
+
'dsh-vision-toolkit: configured storage history sidecar is unavailable. %s',
|
|
169
|
+
(error instanceof Error ? error.message : String(error)).slice(0, 500),
|
|
170
|
+
)
|
|
171
|
+
}
|
|
172
|
+
}
|
package/src/upstream.ts
CHANGED
|
@@ -41,6 +41,7 @@ export interface UpstreamEnvironment {
|
|
|
41
41
|
VISION_API_PROTOCOL: 'chat_completions' | 'anthropic'
|
|
42
42
|
VISION_ANTHROPIC_THINKING: 'omit' | 'disabled' | 'adaptive'
|
|
43
43
|
VISION_SSL_VERIFY?: string
|
|
44
|
+
VISION_STREAM?: string
|
|
44
45
|
VISION_USER_AGENT: string
|
|
45
46
|
LANG: 'zh' | 'en'
|
|
46
47
|
}
|
|
@@ -761,6 +762,9 @@ export class UpstreamAdapter {
|
|
|
761
762
|
...(options.env.VISION_SSL_VERIFY === undefined
|
|
762
763
|
? {}
|
|
763
764
|
: { VISION_SSL_VERIFY: options.env.VISION_SSL_VERIFY }),
|
|
765
|
+
...(options.env.VISION_STREAM === undefined
|
|
766
|
+
? {}
|
|
767
|
+
: { VISION_STREAM: options.env.VISION_STREAM }),
|
|
764
768
|
VISION_USER_AGENT: options.env.VISION_USER_AGENT,
|
|
765
769
|
LANG: options.env.LANG,
|
|
766
770
|
VISION_ENV_FILE: join(prepared.cleanHome, 'vision.env'),
|
|
@@ -1059,22 +1063,47 @@ export class UpstreamAdapter {
|
|
|
1059
1063
|
return new VisionToolkitError('output', `${tool}: upstream output exceeded the capture limit`)
|
|
1060
1064
|
}
|
|
1061
1065
|
const message = upstreamFailureMessage(tool, result.stderr, options.secrets ?? [])
|
|
1062
|
-
|
|
1063
|
-
|
|
1066
|
+
const stderr = result.stderr
|
|
1067
|
+
|
|
1068
|
+
// Remote provider failures, classified into a machine-routable taxonomy so
|
|
1069
|
+
// the failover loop retries transient errors and fails over immediately on
|
|
1070
|
+
// deterministic ones. Order matters: auth/quota/rate-limit/region/tos are
|
|
1071
|
+
// checked before the broader server/network/request-shape patterns.
|
|
1072
|
+
if (/HTTP 401|\b401\b|HTTP 403|\b403\b|Unauthorized|Forbidden|authentication|Invalid API-key|invalid api key|check the api key/i.test(stderr)) {
|
|
1073
|
+
return new VisionToolkitError('auth', `${message}; verify the configured credential`)
|
|
1074
|
+
}
|
|
1075
|
+
if (/HTTP 402|\b402\b|Payment Required|insufficient balance|insufficient quota|insufficient credits|billing|out of credits|out of quota/i.test(stderr)) {
|
|
1076
|
+
return new VisionToolkitError('quota', `${message}; the provider account is out of quota or unpaid`)
|
|
1064
1077
|
}
|
|
1065
|
-
if (/HTTP 429|\b429\b|rate limit|
|
|
1078
|
+
if (/HTTP 429|\b429\b|rate ?limit|too many requests/i.test(stderr)) {
|
|
1066
1079
|
return new VisionToolkitError('rate_limit', `${message}; retry later or reduce concurrency`)
|
|
1067
1080
|
}
|
|
1068
|
-
if (/
|
|
1081
|
+
if (/not available in your region|prohibited region|unsupported region|region is not supported/i.test(stderr)) {
|
|
1082
|
+
return new VisionToolkitError('region', `${message}; the provider is not available in this region`)
|
|
1083
|
+
}
|
|
1084
|
+
if (/terms of service|\btos\b|content policy|safety system/i.test(stderr)) {
|
|
1085
|
+
return new VisionToolkitError('tos', `${message}; the request was rejected by the provider content policy`)
|
|
1086
|
+
}
|
|
1087
|
+
if (/HTTP 5\d\d|\b500\b|\b502\b|\b503\b|\b504\b|bad gateway|service unavailable|internal server error/i.test(stderr)) {
|
|
1088
|
+
return new VisionToolkitError('server', `${message}; the provider returned a server error`)
|
|
1089
|
+
}
|
|
1090
|
+
if (/connection refused|connection reset|ECONN|ENOTFOUND|EAI_AGAIN|getaddrinfo|fetch failed|name resolution|network error|network is unreachable/i.test(stderr)) {
|
|
1091
|
+
return new VisionToolkitError('network', `${message}; network failure reaching the provider`)
|
|
1092
|
+
}
|
|
1093
|
+
if (/HTTP 400|\b400\b|HTTP 404|\b404\b|HTTP 422|\b422\b|invalid request|invalid model|no such model|model not exist|unknown model/i.test(stderr)) {
|
|
1094
|
+
return new VisionToolkitError('invalid_request', `${message}; the request or model is not supported by this provider`)
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
if (/Missing config VISION_/i.test(stderr)) {
|
|
1069
1098
|
return new VisionToolkitError('config', message)
|
|
1070
1099
|
}
|
|
1071
|
-
if (/maxImagePixels|exceed(?:s|ing).*pixels/i.test(
|
|
1100
|
+
if (/maxImagePixels|exceed(?:s|ing).*pixels/i.test(stderr)) {
|
|
1072
1101
|
return new VisionToolkitError('capacity', message)
|
|
1073
1102
|
}
|
|
1074
|
-
if (/not found|only PNG|unsupported|cannot open|empty region|must be|expects|invalid colour|needs at least/i.test(
|
|
1103
|
+
if (/not found|only PNG|unsupported|cannot open|empty region|must be|expects|invalid colour|needs at least/i.test(stderr)) {
|
|
1075
1104
|
return new VisionToolkitError('input', message)
|
|
1076
1105
|
}
|
|
1077
|
-
if (/requires Pillow|requires numpy|requires vtracer|no Chrome|capture failed/i.test(
|
|
1106
|
+
if (/requires Pillow|requires numpy|requires vtracer|no Chrome|capture failed/i.test(stderr)) {
|
|
1078
1107
|
return new VisionToolkitError('runtime', message)
|
|
1079
1108
|
}
|
|
1080
1109
|
return new VisionToolkitError(
|
package/src/web.ts
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
} from './paste-images.ts'
|
|
23
23
|
import {
|
|
24
24
|
resolveConfig,
|
|
25
|
+
retainedStorageHistory,
|
|
25
26
|
isBuiltInFreeVisionProvider,
|
|
26
27
|
VISION_TOOLKIT_SETTINGS_NAMESPACE,
|
|
27
28
|
type ResolvedProvider,
|
|
@@ -74,6 +75,12 @@ export interface VisionToolkitSettingsSnapshot {
|
|
|
74
75
|
source?: string
|
|
75
76
|
writable: boolean
|
|
76
77
|
}>
|
|
78
|
+
/** Object-storage credential state; `ref` is empty when object storage is unset. */
|
|
79
|
+
objectStorageCredential: {
|
|
80
|
+
ref: string
|
|
81
|
+
configured: boolean
|
|
82
|
+
writable: boolean
|
|
83
|
+
}
|
|
77
84
|
runtime: RuntimeManagerStatus
|
|
78
85
|
release: {
|
|
79
86
|
pluginVersion: string
|
|
@@ -120,7 +127,11 @@ interface ApplyUpdateRequest {
|
|
|
120
127
|
expectedVersion: string
|
|
121
128
|
}
|
|
122
129
|
|
|
123
|
-
|
|
130
|
+
interface StorageTestRequest {
|
|
131
|
+
action: 'test-storage'
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest | StorageTestRequest
|
|
124
135
|
|
|
125
136
|
interface JsonError {
|
|
126
137
|
ok: false
|
|
@@ -260,6 +271,7 @@ function parseRequest(value: unknown): SettingsRequest {
|
|
|
260
271
|
}
|
|
261
272
|
return { action: 'apply-update', expectedVersion: value.expectedVersion.trim() }
|
|
262
273
|
}
|
|
274
|
+
if (value.action === 'test-storage') return { action: 'test-storage' }
|
|
263
275
|
throw new TypeError(`unsupported action: ${value.action}`)
|
|
264
276
|
}
|
|
265
277
|
|
|
@@ -314,6 +326,14 @@ export class VisionToolkitWebBackend {
|
|
|
314
326
|
writable: info.writable,
|
|
315
327
|
}
|
|
316
328
|
}))
|
|
329
|
+
const objectStorageRef = resolved.objectStorage.credential === undefined ? '' : String(resolved.objectStorage.credential)
|
|
330
|
+
const objectStorageCredential = objectStorageRef === ''
|
|
331
|
+
? { ref: '', configured: false, writable: this.ctx.settings.writable }
|
|
332
|
+
: await this.ctx.credentials.describe(credentialRef(objectStorageRef)).then(info => ({
|
|
333
|
+
ref: objectStorageRef,
|
|
334
|
+
configured: info.configured,
|
|
335
|
+
writable: info.writable,
|
|
336
|
+
}))
|
|
317
337
|
const update = await this.updater.capability()
|
|
318
338
|
return {
|
|
319
339
|
schemaVersion: 1,
|
|
@@ -332,6 +352,7 @@ export class VisionToolkitWebBackend {
|
|
|
332
352
|
writable: credential.writable,
|
|
333
353
|
},
|
|
334
354
|
credentials,
|
|
355
|
+
objectStorageCredential,
|
|
335
356
|
runtime: this.manager.status(),
|
|
336
357
|
release: {
|
|
337
358
|
pluginVersion: PLUGIN_VERSION,
|
|
@@ -346,16 +367,22 @@ export class VisionToolkitWebBackend {
|
|
|
346
367
|
|
|
347
368
|
private async save(request: SaveRequest): Promise<VisionToolkitSettingsSnapshot> {
|
|
348
369
|
if (!this.ctx.settings.writable) throw new Error('settings provider is read-only')
|
|
370
|
+
const current = resolveConfig(descriptorOf(this.ctx).value as VisionToolkitConfig)
|
|
371
|
+
const storageHistory = retainedStorageHistory(request.value, current)
|
|
372
|
+
const value: VisionToolkitConfig = {
|
|
373
|
+
...request.value,
|
|
374
|
+
...(storageHistory.length === 0 ? {} : { storageHistory }),
|
|
375
|
+
}
|
|
349
376
|
let candidate: PreparedRuntimeGeneration
|
|
350
377
|
try {
|
|
351
|
-
candidate = await this.manager.prepareCandidate(
|
|
378
|
+
candidate = await this.manager.prepareCandidate(value)
|
|
352
379
|
} catch (error) {
|
|
353
380
|
this.manager.recordFailure(error)
|
|
354
381
|
throw error
|
|
355
382
|
}
|
|
356
383
|
await this.ctx.settings.replace(
|
|
357
384
|
VISION_TOOLKIT_SETTINGS_NAMESPACE,
|
|
358
|
-
|
|
385
|
+
value as object,
|
|
359
386
|
request.expectedRevision,
|
|
360
387
|
)
|
|
361
388
|
this.manager.activateCandidate(candidate)
|
|
@@ -373,10 +400,18 @@ export class VisionToolkitWebBackend {
|
|
|
373
400
|
)
|
|
374
401
|
}
|
|
375
402
|
const resolved = resolveConfig(descriptor.value as VisionToolkitConfig)
|
|
376
|
-
const
|
|
403
|
+
const ref = String(request.ref)
|
|
404
|
+
if (resolved.objectStorage.credential !== undefined && String(resolved.objectStorage.credential) === ref) {
|
|
405
|
+
if (!request.value.includes(':')) {
|
|
406
|
+
throw new Error('object storage credential must be "accessKeyId:secretAccessKey"')
|
|
407
|
+
}
|
|
408
|
+
await this.ctx.credentials.set(request.ref, request.value)
|
|
409
|
+
return this.snapshot()
|
|
410
|
+
}
|
|
411
|
+
const provider = resolved.providers.find(entry => String(entry.credential) === ref)
|
|
377
412
|
if (provider === undefined) {
|
|
378
413
|
throw new CredentialReferenceConflictError(
|
|
379
|
-
`credential reference "${
|
|
414
|
+
`credential reference "${ref}" does not match any configured vision provider; reload Settings and try again`,
|
|
380
415
|
)
|
|
381
416
|
}
|
|
382
417
|
if (isBuiltInFreeVisionProvider(provider)) {
|
|
@@ -419,6 +454,11 @@ export class VisionToolkitWebBackend {
|
|
|
419
454
|
}
|
|
420
455
|
}
|
|
421
456
|
|
|
457
|
+
private async testStorage(): Promise<{ detail: string }> {
|
|
458
|
+
if (!this.manager.ready) throw new Error('runtime is not ready; fix Settings and save a valid configuration first')
|
|
459
|
+
return this.manager.current().testObjectStorage()
|
|
460
|
+
}
|
|
461
|
+
|
|
422
462
|
/** Handle the exact Settings route. */
|
|
423
463
|
async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
424
464
|
if (req.method === 'GET') {
|
|
@@ -451,6 +491,9 @@ export class VisionToolkitWebBackend {
|
|
|
451
491
|
case 'health':
|
|
452
492
|
responseJson(res, 200, { ok: true, value: await this.health(parsed, req) })
|
|
453
493
|
break
|
|
494
|
+
case 'test-storage':
|
|
495
|
+
responseJson(res, 200, { ok: true, value: await this.testStorage() })
|
|
496
|
+
break
|
|
454
497
|
case 'save':
|
|
455
498
|
responseJson(res, 200, { ok: true, value: await this.save(parsed) })
|
|
456
499
|
break
|
|
@@ -479,14 +522,16 @@ export class VisionToolkitWebBackend {
|
|
|
479
522
|
? error.code
|
|
480
523
|
: parsed.action === 'health'
|
|
481
524
|
? 'health-failed'
|
|
482
|
-
: parsed.action === '
|
|
483
|
-
? '
|
|
484
|
-
: '
|
|
525
|
+
: parsed.action === 'test-storage'
|
|
526
|
+
? 'storage-test-failed'
|
|
527
|
+
: parsed.action === 'credential'
|
|
528
|
+
? 'credential-rejected'
|
|
529
|
+
: 'settings-rejected'
|
|
485
530
|
const updateConflict = updateError && ['update-in-progress', 'update-stale', 'update-unavailable', 'already-current'].includes(error.code)
|
|
486
531
|
const updateGateway = updateError && error.code === 'update-check-failed'
|
|
487
532
|
const status = settingsConflict || credentialConflict || updateConflict
|
|
488
533
|
? 409
|
|
489
|
-
: parsed.action === 'health'
|
|
534
|
+
: parsed.action === 'health' || parsed.action === 'test-storage'
|
|
490
535
|
? 503
|
|
491
536
|
: updateGateway
|
|
492
537
|
? 502
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"repository": "https://github.com/Anionex/agent-vision-toolkit",
|
|
4
4
|
"version": "v0.1.0+snapshot.bc9803d",
|
|
5
5
|
"commit": "bc9803d7d6300c864d17460ecbb33540b26638e0",
|
|
6
|
-
"contentSha256": "
|
|
6
|
+
"contentSha256": "09bedf007e1469a28981227def5aa33e03667ea21fe2972937f0359898e2928e",
|
|
7
7
|
"files": [
|
|
8
8
|
{
|
|
9
9
|
"path": "CHANGELOG.md",
|
|
@@ -32,8 +32,8 @@
|
|
|
32
32
|
},
|
|
33
33
|
{
|
|
34
34
|
"path": "bin/glance",
|
|
35
|
-
"bytes":
|
|
36
|
-
"sha256": "
|
|
35
|
+
"bytes": 3941,
|
|
36
|
+
"sha256": "7552bef0ba396162f64e8fc4c4a53404dfe2d3149ca3d1c5c8bd717facf82af6"
|
|
37
37
|
},
|
|
38
38
|
{
|
|
39
39
|
"path": "bin/ground",
|
|
@@ -47,13 +47,13 @@
|
|
|
47
47
|
},
|
|
48
48
|
{
|
|
49
49
|
"path": "detect.py",
|
|
50
|
-
"bytes":
|
|
51
|
-
"sha256": "
|
|
50
|
+
"bytes": 2532,
|
|
51
|
+
"sha256": "cac2135f9d835215f3ba90bfd70fd84dd02b0e88299f2b60e109e006b4f3a4d9"
|
|
52
52
|
},
|
|
53
53
|
{
|
|
54
54
|
"path": "ground.py",
|
|
55
|
-
"bytes":
|
|
56
|
-
"sha256": "
|
|
55
|
+
"bytes": 11289,
|
|
56
|
+
"sha256": "6d3175d3f5a7d561d55b2a967c5ca3d49940e80014de658ed9a3e5d45a233a2f"
|
|
57
57
|
},
|
|
58
58
|
{
|
|
59
59
|
"path": "skills/vision-tools/scripts/dominant_colors.py",
|
|
@@ -82,13 +82,13 @@
|
|
|
82
82
|
},
|
|
83
83
|
{
|
|
84
84
|
"path": "tests/test_vision_client.py",
|
|
85
|
-
"bytes":
|
|
86
|
-
"sha256": "
|
|
85
|
+
"bytes": 24882,
|
|
86
|
+
"sha256": "87e39429766b38caed6a4ccb1ac8b193094c69858a0f7541940a005f66421e45"
|
|
87
87
|
},
|
|
88
88
|
{
|
|
89
89
|
"path": "vision_client.py",
|
|
90
|
-
"bytes":
|
|
91
|
-
"sha256": "
|
|
90
|
+
"bytes": 15053,
|
|
91
|
+
"sha256": "6800843b99df500a476b1dd6ff5a4c05aa8122ed990f6f91b16a11ee07bdebe6"
|
|
92
92
|
}
|
|
93
93
|
]
|
|
94
94
|
}
|
|
Binary file
|
|
Binary file
|
|
@@ -40,6 +40,13 @@ def region_data_url(path, region):
|
|
|
40
40
|
return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
def image_url(path):
|
|
44
|
+
"""Pass http(s) URLs through unchanged; encode local files as data URLs."""
|
|
45
|
+
if path.startswith(("http://", "https://")):
|
|
46
|
+
return path
|
|
47
|
+
return image_path_to_data_url(path)
|
|
48
|
+
|
|
49
|
+
|
|
43
50
|
def build_prompt(args, count):
|
|
44
51
|
if args.ocr is not None:
|
|
45
52
|
extra = f" Additional requirements: {args.ocr}" if args.ocr else ""
|
|
@@ -77,7 +84,7 @@ def main():
|
|
|
77
84
|
if args.region and len(args.images) > 1:
|
|
78
85
|
raise VisionError("--region works with exactly one image")
|
|
79
86
|
urls = ([region_data_url(args.images[0], args.region)] if args.region
|
|
80
|
-
else [
|
|
87
|
+
else [image_url(path) for path in args.images])
|
|
81
88
|
answer = describe_image(
|
|
82
89
|
urls,
|
|
83
90
|
build_prompt(args, len(urls)),
|
|
@@ -8,7 +8,7 @@ try:
|
|
|
8
8
|
except ImportError:
|
|
9
9
|
Image = None
|
|
10
10
|
|
|
11
|
-
from ground import GroundError, _position, locate
|
|
11
|
+
from ground import GroundError, _parse_size, _position, locate
|
|
12
12
|
from vision_client import VisionError
|
|
13
13
|
|
|
14
14
|
DEFAULT_CATEGORY = ("UI element (buttons, links, inputs, icons, labels, "
|
|
@@ -38,18 +38,24 @@ def main() -> None:
|
|
|
38
38
|
prog="detect",
|
|
39
39
|
description="Inventory the elements in an image (or a region) with pixel bounding boxes",
|
|
40
40
|
)
|
|
41
|
-
parser.add_argument("image",
|
|
41
|
+
parser.add_argument("image", help="path or http(s) URL to the image")
|
|
42
42
|
parser.add_argument("category", nargs="?",
|
|
43
43
|
help='restrict to a category, e.g. "buttons" or "icons" (default: all UI elements)')
|
|
44
44
|
parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
|
|
45
45
|
help="inventory only this pixel box; output stays in original-image coordinates")
|
|
46
|
+
parser.add_argument("--size", metavar="WxH",
|
|
47
|
+
help="analyzed image dimensions when the image is an http(s) URL")
|
|
46
48
|
args = parser.parse_args()
|
|
47
49
|
try:
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
size = _parse_size(args.size) if args.size else None
|
|
51
|
+
matches = locate(args.image, build_target(args.category), region=args.region, size=size)
|
|
52
|
+
if size is not None:
|
|
53
|
+
width, height = size
|
|
54
|
+
else:
|
|
55
|
+
if Image is None:
|
|
56
|
+
raise GroundError("detect requires Pillow; install the optional dependency pillow first")
|
|
57
|
+
with Image.open(Path(args.image).expanduser()) as image:
|
|
58
|
+
width, height = image.size
|
|
53
59
|
except (GroundError, VisionError) as exc:
|
|
54
60
|
parser.exit(1, f"detect: {exc}\n")
|
|
55
61
|
for line in format_inventory(matches, width, height):
|
|
@@ -184,23 +184,32 @@ def _parse_region(region: str, width: int, height: int) -> tuple[int, int, int,
|
|
|
184
184
|
return box
|
|
185
185
|
|
|
186
186
|
|
|
187
|
-
def locate(image_path:
|
|
187
|
+
def locate(image_path: str, target: str, region: str | None = None,
|
|
188
|
+
size: tuple[int, int] | None = None) -> list[Match]:
|
|
188
189
|
if Image is None:
|
|
189
190
|
raise GroundError("ground requires Pillow; install the optional dependency pillow first")
|
|
190
191
|
load_default_env()
|
|
191
192
|
box = None
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
193
|
+
if size is not None:
|
|
194
|
+
# URL transfer: the image was uploaded to object storage and this caller
|
|
195
|
+
# supplies the analyzed dimensions, so no local file access is needed.
|
|
196
|
+
width, height = size
|
|
197
|
+
url = image_path if image_path.startswith(("http://", "https://")) else image_path_to_data_url(image_path)
|
|
198
|
+
else:
|
|
199
|
+
local = Path(image_path).expanduser()
|
|
200
|
+
try:
|
|
201
|
+
with Image.open(local) as image:
|
|
202
|
+
width, height = image.size
|
|
203
|
+
if region:
|
|
204
|
+
box = _parse_region(region, width, height)
|
|
205
|
+
buffer = io.BytesIO()
|
|
206
|
+
image.crop(box).save(buffer, format="PNG")
|
|
207
|
+
url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
|
208
|
+
except (OSError, ValueError) as exc:
|
|
209
|
+
raise GroundError(f"Cannot read image: {image_path}") from exc
|
|
210
|
+
if box is None:
|
|
211
|
+
url = image_path_to_data_url(local)
|
|
202
212
|
if box is None:
|
|
203
|
-
url = image_path_to_data_url(image_path)
|
|
204
213
|
width_used, height_used = width, height
|
|
205
214
|
else:
|
|
206
215
|
width_used, height_used = box[2] - box[0], box[3] - box[1]
|
|
@@ -216,6 +225,16 @@ def locate(image_path: Path, target: str, region: str | None = None) -> list[Mat
|
|
|
216
225
|
m.bbox[2] + box[0], m.bbox[3] + box[1])) for m in matches]
|
|
217
226
|
|
|
218
227
|
|
|
228
|
+
def _parse_size(value: str) -> tuple[int, int]:
|
|
229
|
+
try:
|
|
230
|
+
width, height = (int(part) for part in value.lower().split("x"))
|
|
231
|
+
except (ValueError, AttributeError):
|
|
232
|
+
raise GroundError("--size expects WIDTHxHEIGHT (e.g. 1024x768)") from None
|
|
233
|
+
if width <= 0 or height <= 0:
|
|
234
|
+
raise GroundError("--size expects positive WIDTHxHEIGHT")
|
|
235
|
+
return width, height
|
|
236
|
+
|
|
237
|
+
|
|
219
238
|
def _position(box: tuple[int, int, int, int], width: int, height: int) -> str:
|
|
220
239
|
x1, y1, x2, y2 = box
|
|
221
240
|
x = (x1 + x2) / 2
|
|
@@ -246,17 +265,23 @@ def main() -> None:
|
|
|
246
265
|
prog="ground",
|
|
247
266
|
description="Locate targets in an image with natural language and output pixel coordinates",
|
|
248
267
|
)
|
|
249
|
-
parser.add_argument("image",
|
|
268
|
+
parser.add_argument("image", help="path or http(s) URL to the image")
|
|
250
269
|
parser.add_argument("target", help="target object or region to locate")
|
|
251
270
|
parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
|
|
252
271
|
help="search only this pixel box; output stays in original-image coordinates")
|
|
272
|
+
parser.add_argument("--size", metavar="WxH",
|
|
273
|
+
help="analyzed image dimensions when the image is an http(s) URL")
|
|
253
274
|
args = parser.parse_args()
|
|
254
275
|
try:
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
276
|
+
size = _parse_size(args.size) if args.size else None
|
|
277
|
+
matches = locate(args.image, args.target, region=args.region, size=size)
|
|
278
|
+
if size is not None:
|
|
279
|
+
width, height = size
|
|
280
|
+
else:
|
|
281
|
+
if Image is None:
|
|
282
|
+
raise GroundError("ground requires Pillow; install the optional dependency pillow first")
|
|
283
|
+
with Image.open(Path(args.image).expanduser()) as image:
|
|
284
|
+
width, height = image.size
|
|
260
285
|
except (GroundError, VisionError) as exc:
|
|
261
286
|
parser.exit(1, f"ground: {exc}\n")
|
|
262
287
|
for line in format_matches(matches, width, height):
|