@anionex/dsh-vision-toolkit 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -20
- package/README.zh.md +16 -14
- package/assets/skill/SKILL.md +2 -2
- package/assets/skill/UPSTREAM.json +5 -5
- package/assets/skill/references/restore-ui.md +6 -4
- package/docs/requirements-traceability/README.md +1 -1
- package/docs/requirements-traceability/README.zh.md +1 -1
- package/lib/exposure.js +18 -7
- package/lib/exposure.js.map +1 -1
- package/lib/image-input-variants.js +72 -22
- package/lib/image-input-variants.js.map +1 -1
- package/lib/index.js +4 -4
- package/lib/index.js.map +1 -1
- package/lib/plugin-update.js +8 -1
- package/lib/plugin-update.js.map +1 -1
- package/lib/runtime.js +12 -9
- package/lib/runtime.js.map +1 -1
- package/lib/skill.js +7 -7
- package/lib/skill.js.map +1 -1
- package/lib/types/exposure.d.ts +4 -0
- package/lib/types/exposure.d.ts.map +1 -1
- package/lib/types/image-input-variants.d.ts.map +1 -1
- package/lib/types/index.d.ts +1 -1
- package/lib/types/plugin-update.d.ts.map +1 -1
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/skill.d.ts +4 -4
- package/lib/types/skill.d.ts.map +1 -1
- package/package.json +2 -1
- package/patches/vision-tools-dsh.patch +29 -13
- package/src/exposure.ts +22 -7
- package/src/image-input-variants.ts +74 -21
- package/src/index.ts +4 -4
- package/src/plugin-update.ts +8 -1
- package/src/runtime.ts +12 -9
- package/src/skill.ts +7 -7
|
@@ -16,7 +16,8 @@ index 8f42552..f2ee3f2 100644
|
|
|
16
16
|
- image yourself when a description you were given lacks a detail.
|
|
17
17
|
----
|
|
18
18
|
-
|
|
19
|
-
|
|
19
|
+
-# vision-tools
|
|
20
|
+
+# vision-skills
|
|
20
21
|
|
|
21
22
|
-Five local CLIs that give a text-only agent eyes. They read one shared
|
|
22
23
|
-vision config (`VISION_API_KEY` / `VISION_BASE_URL` / `VISION_MODEL` /
|
|
@@ -29,7 +30,7 @@ index 8f42552..f2ee3f2 100644
|
|
|
29
30
|
+
|
|
30
31
|
+The visual execution schemas are mounted only for the current Agent after this
|
|
31
32
|
+Skill is loaded. A normal `skill` call activates them for the next model step.
|
|
32
|
-
+If this content arrived through a direct `/vision-
|
|
33
|
+
+If this content arrived through a direct `/vision-skills` invocation and the
|
|
33
34
|
+visual tools are still absent, call `vision_toolkit_activate` once. Do not call
|
|
34
35
|
+that bootstrap when the visual tools are already present.
|
|
35
36
|
|
|
@@ -802,10 +803,10 @@ index 3261fcf..68025be 100644
|
|
|
802
803
|
nodes, labels, groups, edges, directions, and edge labels.
|
|
803
804
|
6. Generate the Mermaid, Graphviz, or requested structure from that inventory.
|
|
804
805
|
diff --git a/references/restore-ui.md b/references/restore-ui.md
|
|
805
|
-
index f9ec11a..
|
|
806
|
+
index f9ec11a..69627895161fc1b57c6241665255fdf86ae1c1e5 100644
|
|
806
807
|
--- a/references/restore-ui.md
|
|
807
808
|
+++ b/references/restore-ui.md
|
|
808
|
-
@@ -28,
|
|
809
|
+
@@ -28,17 +28,18 @@
|
|
809
810
|
1. Inspect the existing frontend stack, component library, icon set, source
|
|
810
811
|
assets, and design tokens before writing code, but stop searching as soon as
|
|
811
812
|
a usable local primitive is found.
|
|
@@ -824,12 +825,27 @@ index f9ec11a..de97a02 100644
|
|
|
824
825
|
starting the next call. Normally one or two rounds are enough.
|
|
825
826
|
-4. Do not use `trace`, foreground extraction, repeated color sampling, or
|
|
826
827
|
- iterative `pixel_diff` work in fast mode. Those are fidelity tools and will
|
|
828
|
+
- consume the delivery window.
|
|
827
829
|
+4. Do not use `vision_trace`, foreground extraction, repeated color sampling,
|
|
828
|
-
+
|
|
829
|
-
|
|
830
|
+
+ iterative `vision_pixel_diff` work, or hand-written SVG in fast mode. Those
|
|
831
|
+
+ are fidelity tools and will consume the delivery window. Icons and decorative
|
|
832
|
+
+ marks stay library-based or screenshot-backed; never hand-write SVG code.
|
|
830
833
|
|
|
831
834
|
### Build the approximation
|
|
832
|
-
|
|
835
|
+
|
|
836
|
+
@@ -47,8 +48,9 @@
|
|
837
|
+
when zoomed in.
|
|
838
|
+
2. Reuse the project's existing components and CSS tokens. If its frontend or
|
|
839
|
+
icon library contains a reasonably similar component or icon, use it
|
|
840
|
+
- directly instead of recreating the reference. Prefer an approximate library
|
|
841
|
+
- icon over cropping, tracing, or hand-drawing a new one.
|
|
842
|
+
+ directly instead of recreating the reference. Never hand-write SVG in fast
|
|
843
|
+
+ mode: use an approximate library icon, or extract the original pixels as a
|
|
844
|
+
+ screenshot-backed asset.
|
|
845
|
+
3. Use nearby existing palette tokens or visually similar CSS values. Exact
|
|
846
|
+
sampled hex values, gradients, subtle borders, and shadow opacity are out of
|
|
847
|
+
scope unless one of them defines the whole composition.
|
|
848
|
+
@@ -57,7 +59,7 @@
|
|
833
849
|
|
|
834
850
|
### Render once, fix once, deliver
|
|
835
851
|
|
|
@@ -838,7 +854,7 @@ index f9ec11a..de97a02 100644
|
|
|
838
854
|
existing browser setup.
|
|
839
855
|
2. Inspect the screenshot once. If there is an obvious structural failure such
|
|
840
856
|
as a missing major region, broken wrapping, or a wildly wrong scale, make
|
|
841
|
-
@@ -66,7 +
|
|
857
|
+
@@ -66,7 +68,7 @@
|
|
842
858
|
small color, icon, font, shadow, radius, or spacing differences.
|
|
843
859
|
|
|
844
860
|
A practical time box is roughly 30 seconds for project inspection plus
|
|
@@ -847,7 +863,7 @@ index f9ec11a..de97a02 100644
|
|
|
847
863
|
rendering, one correction, and screenshot delivery. Dependency installation or
|
|
848
864
|
a project that does not already run may extend that target; do not compensate
|
|
849
865
|
by silently switching back to a long precision loop.
|
|
850
|
-
@@ -112,18 +
|
|
866
|
+
@@ -112,18 +114,19 @@
|
|
851
867
|
### 2. Establish the coordinate system
|
|
852
868
|
|
|
853
869
|
Record the reference image dimensions and the target viewport. Screenshots may
|
|
@@ -875,7 +891,7 @@ index f9ec11a..de97a02 100644
|
|
|
875
891
|
|
|
876
892
|
Treat model boxes as approximate handles. Use them to organize the page, but
|
|
877
893
|
do not mistake their final few pixels for measured boundaries.
|
|
878
|
-
@@ -141,15 +
|
|
894
|
+
@@ -141,15 +144,11 @@
|
|
879
895
|
extraction, tighten the final region to the visual's own ink; adjacent text or
|
|
880
896
|
rules inside the region can become foreground components too.
|
|
881
897
|
|
|
@@ -896,7 +912,7 @@ index f9ec11a..de97a02 100644
|
|
|
896
912
|
```
|
|
897
913
|
|
|
898
914
|
Inspect the transparent result before use. Confirm that the whole visual is
|
|
899
|
-
@@ -162,8 +
|
|
915
|
+
@@ -162,8 +161,8 @@
|
|
900
916
|
- Follow the project's existing framework and component patterns.
|
|
901
917
|
- Build content, interaction, responsive layout, borders, shadows, and simple
|
|
902
918
|
shapes natively.
|
|
@@ -907,7 +923,7 @@ index f9ec11a..de97a02 100644
|
|
|
907
923
|
- Place extracted assets with explicit logical dimensions. Preserve their
|
|
908
924
|
aspect ratio and avoid baking surrounding whitespace into the asset.
|
|
909
925
|
- Match structure and proportions before tuning small spacing.
|
|
910
|
-
@@ -172,10 +
|
|
926
|
+
@@ -172,10 +171,10 @@
|
|
911
927
|
|
|
912
928
|
For an existing implementation, start here.
|
|
913
929
|
|
|
@@ -920,7 +936,7 @@ index f9ec11a..de97a02 100644
|
|
|
920
936
|
spot or explain.
|
|
921
937
|
3. Fix material discrepancies: missing or wrong content, incorrect hierarchy,
|
|
922
938
|
broken wrapping, visibly wrong alignment or scale, wrong component state,
|
|
923
|
-
@@ -185,7 +
|
|
939
|
+
@@ -185,7 +184,7 @@
|
|
924
940
|
rasterization, antialiasing, subpixel placement, or another imperceptible
|
|
925
941
|
rendering detail.
|
|
926
942
|
|
package/src/exposure.ts
CHANGED
|
@@ -11,12 +11,18 @@ import type { ContentBlock } from '@deepseek-ai/dsh-llm'
|
|
|
11
11
|
import type { Session } from '@deepseek-ai/dsh-session'
|
|
12
12
|
import { defineTool, type ToolDefinition } from '@deepseek-ai/dsh-tools'
|
|
13
13
|
import type { Context } from '@deepseek-ai/cordis'
|
|
14
|
-
import {
|
|
14
|
+
import { VISION_SKILLS_CONTENT, VISION_SKILLS_NAME } from './skill.ts'
|
|
15
15
|
import { VISION_TOOL_NAMES } from './tools.ts'
|
|
16
16
|
|
|
17
17
|
/** Small bootstrap tool retained only until the current Agent gains visual tools. */
|
|
18
18
|
export const VISION_TOOLKIT_ACTIVATE = 'vision_toolkit_activate'
|
|
19
19
|
|
|
20
|
+
/** Skill name used by releases before the rename to vision-skills. */
|
|
21
|
+
export const LEGACY_VISION_TOOLS_SKILL_NAME = 'vision-tools'
|
|
22
|
+
|
|
23
|
+
/** Unique pre-rename line in bundled instructions, kept for Session restore. */
|
|
24
|
+
export const LEGACY_VISION_TOOLS_SKILL_MARKER = 'If this content arrived through a direct `/vision-tools` invocation and the'
|
|
25
|
+
|
|
20
26
|
interface AgentExposure {
|
|
21
27
|
active: boolean
|
|
22
28
|
liftRestriction?: () => void
|
|
@@ -38,8 +44,16 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
38
44
|
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
39
45
|
}
|
|
40
46
|
|
|
47
|
+
function isBundledSkillName(name: unknown): boolean {
|
|
48
|
+
return name === VISION_SKILLS_NAME || name === LEGACY_VISION_TOOLS_SKILL_NAME
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function isBundledSkillContent(text: string): boolean {
|
|
52
|
+
return text.includes(VISION_SKILLS_CONTENT) || text.includes(LEGACY_VISION_TOOLS_SKILL_MARKER)
|
|
53
|
+
}
|
|
54
|
+
|
|
41
55
|
function isVisionSkillArguments(value: unknown): boolean {
|
|
42
|
-
return isRecord(value) && value.name
|
|
56
|
+
return isRecord(value) && isBundledSkillName(value.name)
|
|
43
57
|
}
|
|
44
58
|
|
|
45
59
|
function nativeSkillCall(raw: string): boolean {
|
|
@@ -54,13 +68,14 @@ function containsBundledSkillContent(blocks: readonly unknown[]): boolean {
|
|
|
54
68
|
return blocks.some(block => isRecord(block)
|
|
55
69
|
&& block.type === 'text'
|
|
56
70
|
&& typeof block.text === 'string'
|
|
57
|
-
&& block.text
|
|
71
|
+
&& isBundledSkillContent(block.text))
|
|
58
72
|
}
|
|
59
73
|
|
|
60
74
|
function isBundledSkillResult(value: unknown): boolean {
|
|
61
75
|
return isRecord(value)
|
|
62
|
-
&& value.name
|
|
63
|
-
&& value.content ===
|
|
76
|
+
&& isBundledSkillName(value.name)
|
|
77
|
+
&& typeof value.content === 'string'
|
|
78
|
+
&& isBundledSkillContent(value.content)
|
|
64
79
|
}
|
|
65
80
|
|
|
66
81
|
/** Whether durable history proves that this Session loaded the bundled Skill. */
|
|
@@ -70,7 +85,7 @@ function hasLoadedVisionSkill(session: Session): boolean {
|
|
|
70
85
|
if (event.type === 'user/message') {
|
|
71
86
|
const source = event.data.source
|
|
72
87
|
if (source.kind === 'skill-invocation'
|
|
73
|
-
&& source.name
|
|
88
|
+
&& isBundledSkillName(source.name)
|
|
74
89
|
&& containsBundledSkillContent(event.data.content)) return true
|
|
75
90
|
continue
|
|
76
91
|
}
|
|
@@ -119,7 +134,7 @@ export class VisionToolExposure {
|
|
|
119
134
|
this.activationTool = defineTool({
|
|
120
135
|
name: VISION_TOOLKIT_ACTIVATE,
|
|
121
136
|
description: `Activate the independent Vision Toolkit execution tools for this Agent: ${Object.values(VISION_TOOL_NAMES).join(', ')}. `
|
|
122
|
-
+ `Loading the ${
|
|
137
|
+
+ `Loading the ${VISION_SKILLS_NAME} Skill normally activates them automatically; call this once when the visual tools are still absent, then use them for image understanding, OCR, UI detection, and related tasks. `
|
|
123
138
|
+ 'It is safe to call before the Skill is loaded, and this activation tool disappears after success.',
|
|
124
139
|
parameters: {},
|
|
125
140
|
output: {
|
|
@@ -41,6 +41,12 @@ export const VARIANT_SUFFIX = ' (Vision Toolkit)'
|
|
|
41
41
|
/** Promise-cache bound for image descriptions, so a long-lived Web profile cannot hoard evidence text. */
|
|
42
42
|
const EVIDENCE_CACHE_LIMIT = 64
|
|
43
43
|
|
|
44
|
+
/** Periodic self-heal sweep for image-input variant routes. */
|
|
45
|
+
const VARIANT_SWEEP_INTERVAL_MS = 10_000
|
|
46
|
+
|
|
47
|
+
/** How long an upstream must stay absent before its wrapper is released. */
|
|
48
|
+
const VARIANT_RELEASE_GRACE_MS = 10_000
|
|
49
|
+
|
|
44
50
|
/**
|
|
45
51
|
* Media types the Vision Toolkit glance pipeline accepts, by declared media
|
|
46
52
|
* type. Narrower than the paste-to-workspace route (which stores any image
|
|
@@ -835,6 +841,12 @@ export function installImageInputVariants(
|
|
|
835
841
|
getRuntime: () => VisionToolkitRuntime | undefined,
|
|
836
842
|
): { dispose: () => void; reconcile: () => void } {
|
|
837
843
|
const registrations = new Map<string, () => void>()
|
|
844
|
+
// Upstream ids observed missing and the timestamp of the first missing
|
|
845
|
+
// observation. A wrapper is only released after its upstream stays absent
|
|
846
|
+
// for the full grace period, so a transient registry gap (adapter
|
|
847
|
+
// re-registration, HMR reload) cannot drop a wrapper that a live session is
|
|
848
|
+
// still pointed at, no matter how many sweeps fire in between.
|
|
849
|
+
const stale = new Map<string, number>()
|
|
838
850
|
let disposed = false
|
|
839
851
|
// Serialize sweeps: a registration itself announces llm/adapters-updated,
|
|
840
852
|
// and two interleaved sweeps must never probe the same route concurrently.
|
|
@@ -843,13 +855,29 @@ export function installImageInputVariants(
|
|
|
843
855
|
// not one per notification (a single registration emits a notification).
|
|
844
856
|
let sweepQueued = false
|
|
845
857
|
|
|
858
|
+
const release = (upstream: string): void => {
|
|
859
|
+
const dispose = registrations.get(upstream)
|
|
860
|
+
if (dispose === undefined) return
|
|
861
|
+
registrations.delete(upstream)
|
|
862
|
+
stale.delete(upstream)
|
|
863
|
+
try {
|
|
864
|
+
dispose()
|
|
865
|
+
} catch (error) {
|
|
866
|
+
ctx.logger.warn(
|
|
867
|
+
'dsh-vision-toolkit: image-input variant release failed for "%s": %s',
|
|
868
|
+
upstream,
|
|
869
|
+
messageOf(error),
|
|
870
|
+
)
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
|
|
846
874
|
const releaseAll = (): void => {
|
|
847
|
-
|
|
848
|
-
registrations.
|
|
875
|
+
stale.clear()
|
|
876
|
+
for (const upstream of [...registrations.keys()]) release(upstream)
|
|
849
877
|
}
|
|
850
878
|
|
|
851
879
|
const sweep = (): void => {
|
|
852
|
-
if (sweepQueued) return
|
|
880
|
+
if (disposed || sweepQueued) return
|
|
853
881
|
sweepQueued = true
|
|
854
882
|
queueMicrotask(() => {
|
|
855
883
|
sweepQueued = false
|
|
@@ -874,12 +902,33 @@ export function installImageInputVariants(
|
|
|
874
902
|
} catch {
|
|
875
903
|
return
|
|
876
904
|
}
|
|
905
|
+
const live = new Set(providers.map(provider => provider.id))
|
|
906
|
+
// Release restrict-narrowed routes immediately (a configuration change is
|
|
907
|
+
// authoritative). For routes merely missing from the live registry, keep
|
|
908
|
+
// the wrapper until the absence outlives the grace period.
|
|
909
|
+
for (const upstream of [...registrations.keys()]) {
|
|
910
|
+
if (restrict.size > 0 && !restrict.has(upstream)) {
|
|
911
|
+
release(upstream)
|
|
912
|
+
continue
|
|
913
|
+
}
|
|
914
|
+
if (live.has(upstream)) {
|
|
915
|
+
stale.delete(upstream)
|
|
916
|
+
continue
|
|
917
|
+
}
|
|
918
|
+
const missedAt = stale.get(upstream)
|
|
919
|
+
if (missedAt !== undefined && Date.now() - missedAt >= VARIANT_RELEASE_GRACE_MS) {
|
|
920
|
+
release(upstream)
|
|
921
|
+
} else if (missedAt === undefined) {
|
|
922
|
+
stale.set(upstream, Date.now())
|
|
923
|
+
}
|
|
924
|
+
}
|
|
877
925
|
for (const provider of providers) {
|
|
878
926
|
const upstream = provider.id
|
|
879
927
|
if (restrict.size > 0 && !restrict.has(upstream)) continue
|
|
880
928
|
// Our own variant routes declare image input and are never wrapped;
|
|
881
929
|
// probing them would just re-probe their upstream route.
|
|
882
930
|
if (upstream.startsWith(VARIANT_PROVIDER_PREFIX)) continue
|
|
931
|
+
const variantId = variantProviderId(upstream)
|
|
883
932
|
let models: readonly LlmModelInfo[]
|
|
884
933
|
try {
|
|
885
934
|
models = await llm.listModels(upstream)
|
|
@@ -888,18 +937,26 @@ export function installImageInputVariants(
|
|
|
888
937
|
}
|
|
889
938
|
const eligible = models.some(shouldWrapModel)
|
|
890
939
|
const registered = registrations.has(upstream)
|
|
891
|
-
if (
|
|
892
|
-
//
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
940
|
+
if (registered && live.has(variantId)) {
|
|
941
|
+
// Re-read the live registry after the await: a rebuild inside the
|
|
942
|
+
// probe window can drop our wrapper without touching our map.
|
|
943
|
+
const liveNow = new Set(llm.listProviders().map(provider => provider.id))
|
|
944
|
+
if (liveNow.has(variantId)) {
|
|
945
|
+
// Healthy handle: only react when the route lost every eligible model.
|
|
946
|
+
if (!eligible) release(upstream)
|
|
947
|
+
continue
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
if (registered) {
|
|
951
|
+
// Dead handle from a registry rebuild/reset: drop it and register a
|
|
952
|
+
// fresh wrapper below.
|
|
953
|
+
release(upstream)
|
|
897
954
|
}
|
|
898
|
-
if (!eligible
|
|
955
|
+
if (!eligible) continue
|
|
899
956
|
if (disposed) return
|
|
900
957
|
try {
|
|
901
958
|
const dispose = llm.registerAdapter(
|
|
902
|
-
[
|
|
959
|
+
[variantId],
|
|
903
960
|
new ImageInputVariantAdapter(
|
|
904
961
|
ctx,
|
|
905
962
|
llm,
|
|
@@ -910,6 +967,7 @@ export function installImageInputVariants(
|
|
|
910
967
|
),
|
|
911
968
|
)
|
|
912
969
|
registrations.set(upstream, dispose)
|
|
970
|
+
stale.delete(upstream)
|
|
913
971
|
} catch (error) {
|
|
914
972
|
ctx.logger.warn(
|
|
915
973
|
'dsh-vision-toolkit: image-input variant registration skipped for "%s": %s',
|
|
@@ -918,16 +976,6 @@ export function installImageInputVariants(
|
|
|
918
976
|
)
|
|
919
977
|
}
|
|
920
978
|
}
|
|
921
|
-
const live = new Set(providers.map(provider => provider.id))
|
|
922
|
-
for (const [upstream, dispose] of [...registrations]) {
|
|
923
|
-
// A wrapper is released when its upstream route vanished OR the
|
|
924
|
-
// current configuration no longer allows that route (restrict
|
|
925
|
-
// narrowing must not leave stale variants behind).
|
|
926
|
-
if (!live.has(upstream) || (restrict.size > 0 && !restrict.has(upstream))) {
|
|
927
|
-
dispose()
|
|
928
|
-
registrations.delete(upstream)
|
|
929
|
-
}
|
|
930
|
-
}
|
|
931
979
|
} catch (error) {
|
|
932
980
|
ctx.logger.warn('dsh-vision-toolkit: image-input variant sweep failed: %s', messageOf(error))
|
|
933
981
|
}
|
|
@@ -937,9 +985,14 @@ export function installImageInputVariants(
|
|
|
937
985
|
ctx.on('llm/adapters-updated', () => { sweep() })
|
|
938
986
|
}
|
|
939
987
|
sweep()
|
|
988
|
+
// Self-heal safety net: re-verify periodically so a wrapper dropped by an
|
|
989
|
+
// unobserved cause (registry rebuild without an event we can see) is restored
|
|
990
|
+
// within one interval.
|
|
991
|
+
const timer = setInterval(() => { sweep() }, VARIANT_SWEEP_INTERVAL_MS)
|
|
940
992
|
return {
|
|
941
993
|
dispose: () => {
|
|
942
994
|
disposed = true
|
|
995
|
+
clearInterval(timer)
|
|
943
996
|
releaseAll()
|
|
944
997
|
},
|
|
945
998
|
reconcile: () => { sweep() },
|
package/src/index.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* @anionex/dsh-vision-toolkit — DSH Vision Toolkit profile bundle.
|
|
3
3
|
*
|
|
4
4
|
* Plugin lifecycle follows the documented readiness chain: verify the pinned
|
|
5
|
-
* upstream checkout, publish the vision-
|
|
5
|
+
* upstream checkout, publish the vision-skills Skill and its one-shot bootstrap,
|
|
6
6
|
* then mount the execution tools only in Agents that load that Skill or invoke
|
|
7
7
|
* the bootstrap. Any
|
|
8
8
|
* failure leaves no model capability behind, and disposal unregisters every
|
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
import { VisionToolExposure } from './exposure.ts'
|
|
25
25
|
import { createPasteTakeoverResolver, installImageInputVariants } from './image-input-variants.ts'
|
|
26
26
|
import { VisionToolkitRuntimeManager } from './runtime-manager.ts'
|
|
27
|
-
import {
|
|
27
|
+
import { VISION_SKILLS_SKILL } from './skill.ts'
|
|
28
28
|
import { createVisionTools } from './tools.ts'
|
|
29
29
|
import { PLUGIN_VERSION } from './version.ts'
|
|
30
30
|
import { installVisionToolkitWeb, VisionToolkitWebBackend } from './web.ts'
|
|
@@ -65,7 +65,7 @@ export async function apply(ctx: Context, config: VisionToolkitConfig = {}): Pro
|
|
|
65
65
|
let skill: (() => void) | undefined
|
|
66
66
|
try {
|
|
67
67
|
activationTool = ctx.tools.register(exposure.activationTool)
|
|
68
|
-
skill = ctx.skills.register(
|
|
68
|
+
skill = ctx.skills.register(VISION_SKILLS_SKILL)
|
|
69
69
|
exposureDisposer = exposure.install()
|
|
70
70
|
operationalDisposers = { activationTool, exposure: exposureDisposer, skill }
|
|
71
71
|
const info = manager.current().upstreamVersion
|
|
@@ -90,7 +90,7 @@ export async function apply(ctx: Context, config: VisionToolkitConfig = {}): Pro
|
|
|
90
90
|
} catch (error) {
|
|
91
91
|
const message = error instanceof Error ? error.message : String(error)
|
|
92
92
|
ctx.logger.error(
|
|
93
|
-
'dsh-vision-toolkit %s: runtime not ready; the vision-
|
|
93
|
+
'dsh-vision-toolkit %s: runtime not ready; the vision-skills skill, activation bootstrap, and Agent-scoped visual tools are NOT registered. Settings remain available for repair. %s',
|
|
94
94
|
PLUGIN_VERSION,
|
|
95
95
|
message,
|
|
96
96
|
)
|
package/src/plugin-update.ts
CHANGED
|
@@ -822,6 +822,13 @@ export class VisionToolkitPluginUpdateService {
|
|
|
822
822
|
profile: ProfileInstall,
|
|
823
823
|
pnpmPath: string,
|
|
824
824
|
): Promise<CommandResult> {
|
|
825
|
+
// Windows batch shims cannot be spawned directly; route them through cmd.exe.
|
|
826
|
+
const program = this.platform === 'win32' && /\.(?:cmd|bat)$/i.test(pnpmPath)
|
|
827
|
+
? process.env.COMSPEC ?? 'cmd.exe'
|
|
828
|
+
: pnpmPath
|
|
829
|
+
const argv = program === pnpmPath
|
|
830
|
+
? [pnpmPath, ...args]
|
|
831
|
+
: [program, '/d', '/s', '/c', pnpmPath, ...args]
|
|
825
832
|
const controller = new AbortController()
|
|
826
833
|
let timedOut = false
|
|
827
834
|
const timeout = setTimeout(() => {
|
|
@@ -830,7 +837,7 @@ export class VisionToolkitPluginUpdateService {
|
|
|
830
837
|
}, timeoutMs)
|
|
831
838
|
try {
|
|
832
839
|
const handle = this.ctx.subprocess.spawn({
|
|
833
|
-
argv
|
|
840
|
+
argv,
|
|
834
841
|
cwd: profile.profileDir,
|
|
835
842
|
stdio: {
|
|
836
843
|
stdin: 'ignore',
|
package/src/runtime.ts
CHANGED
|
@@ -56,7 +56,9 @@ const VISION_MODEL_TEST_IMAGE = fileURLToPath(new URL('../assets/vision-model-te
|
|
|
56
56
|
const VISION_MODEL_TEST_PROMPT = 'This is an explicit service readiness test. Reply with one short sentence confirming that you received the image.'
|
|
57
57
|
|
|
58
58
|
/** Bump when the Pillow compression ladder changes so stale cache entries are ignored. */
|
|
59
|
-
const COMPRESSED_IMAGE_CACHE_VERSION = '
|
|
59
|
+
const COMPRESSED_IMAGE_CACHE_VERSION = 'v2'
|
|
60
|
+
/** Cache keys carry 64-bit digests so Windows paths stay below MAX_PATH; the full file sha256 is computed on read and compared against this prefix. */
|
|
61
|
+
const COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH = 16
|
|
60
62
|
const COMPRESSED_IMAGE_CACHE_MAX_ENTRIES = 200
|
|
61
63
|
const COMPRESSED_IMAGE_CACHE_MAX_BYTES = 512 * 1024 * 1024
|
|
62
64
|
const COMPRESSED_IMAGE_CACHE_STALE_PARTIAL_MS = 60 * 60 * 1000
|
|
@@ -883,7 +885,7 @@ export class VisionToolkitRuntime {
|
|
|
883
885
|
private async readCacheCandidate(
|
|
884
886
|
root: string,
|
|
885
887
|
name: string,
|
|
886
|
-
|
|
888
|
+
expectedOutDigestPrefix: string,
|
|
887
889
|
maxBytes: number,
|
|
888
890
|
maxPixels: number,
|
|
889
891
|
operation: OperationContext,
|
|
@@ -909,7 +911,8 @@ export class VisionToolkitRuntime {
|
|
|
909
911
|
} catch {
|
|
910
912
|
return undefined
|
|
911
913
|
}
|
|
912
|
-
|
|
914
|
+
const digest = createHash('sha256').update(bytes).digest('hex')
|
|
915
|
+
if (bytes.length !== info.size || !digest.startsWith(expectedOutDigestPrefix)) return undefined
|
|
913
916
|
let probed: { width: number; height: number; format: string } | undefined
|
|
914
917
|
try {
|
|
915
918
|
probed = await this.adapter.probeImageSize(real, { signal: operation.signal })
|
|
@@ -929,7 +932,7 @@ export class VisionToolkitRuntime {
|
|
|
929
932
|
|
|
930
933
|
private cacheEntryOutDigest(entry: string, prefix: string): string | undefined {
|
|
931
934
|
const tail = entry.slice(prefix.length + 1)
|
|
932
|
-
return /^[0-9a-f]{
|
|
935
|
+
return /^[0-9a-f]{16}-/u.test(tail) ? tail.slice(0, 16) : undefined
|
|
933
936
|
}
|
|
934
937
|
|
|
935
938
|
private async pruneCompressedCache(root: string): Promise<void> {
|
|
@@ -997,21 +1000,21 @@ export class VisionToolkitRuntime {
|
|
|
997
1000
|
if (bytes.length !== image.bytes) {
|
|
998
1001
|
throw new VisionToolkitError('input', `image changed while preparing the vision request: ${image.path}`)
|
|
999
1002
|
}
|
|
1000
|
-
const digest = createHash('sha256').update(bytes).digest('hex')
|
|
1003
|
+
const digest = createHash('sha256').update(bytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH)
|
|
1001
1004
|
const root = await this.compressedImageRoot(policy)
|
|
1002
1005
|
await this.pruneCompressedCache(root)
|
|
1003
1006
|
const prefix = `${COMPRESSED_IMAGE_CACHE_VERSION}-${digest}-b${this.config.maxImageBytes}-p${this.config.maxImagePixels}`
|
|
1004
1007
|
for (const entry of await readdir(root)) {
|
|
1005
1008
|
if (!entry.startsWith(`${prefix}-`) || entry.startsWith('.')) continue
|
|
1006
|
-
const
|
|
1007
|
-
if (
|
|
1009
|
+
const outDigestPrefix = this.cacheEntryOutDigest(entry, prefix)
|
|
1010
|
+
if (outDigestPrefix === undefined) {
|
|
1008
1011
|
await rm(join(root, entry), { force: true }).catch(() => {})
|
|
1009
1012
|
continue
|
|
1010
1013
|
}
|
|
1011
1014
|
const cached = await this.readCacheCandidate(
|
|
1012
1015
|
root,
|
|
1013
1016
|
entry,
|
|
1014
|
-
|
|
1017
|
+
outDigestPrefix,
|
|
1015
1018
|
this.config.maxImageBytes,
|
|
1016
1019
|
this.config.maxImagePixels,
|
|
1017
1020
|
operation,
|
|
@@ -1037,7 +1040,7 @@ export class VisionToolkitRuntime {
|
|
|
1037
1040
|
}
|
|
1038
1041
|
const extension = compressed.format === 'jpeg' ? 'jpg' : compressed.format
|
|
1039
1042
|
const stagedBytes = await readFile(staged, { signal: operation.signal })
|
|
1040
|
-
const outDigest = createHash('sha256').update(stagedBytes).digest('hex')
|
|
1043
|
+
const outDigest = createHash('sha256').update(stagedBytes).digest('hex').slice(0, COMPRESSED_IMAGE_CACHE_KEY_DIGEST_LENGTH)
|
|
1041
1044
|
const finalName = `${prefix}-${outDigest}-${compressed.width}x${compressed.height}.${extension}`
|
|
1042
1045
|
const finalPath = join(root, finalName)
|
|
1043
1046
|
const existing = await this.readCacheCandidate(
|
package/src/skill.ts
CHANGED
|
@@ -8,28 +8,28 @@ import { fileURLToPath } from 'node:url'
|
|
|
8
8
|
import type { SkillRegistration } from '@deepseek-ai/dsh-skill'
|
|
9
9
|
|
|
10
10
|
/** Stable catalog/invocation name shared with progressive tool exposure. */
|
|
11
|
-
export const
|
|
11
|
+
export const VISION_SKILLS_NAME = 'vision-skills'
|
|
12
12
|
|
|
13
13
|
/** Packaged resource root for the adapted upstream playbooks. */
|
|
14
|
-
export const
|
|
14
|
+
export const VISION_SKILLS_RESOURCE_BASE = fileURLToPath(
|
|
15
15
|
new URL('../assets/skill/', import.meta.url),
|
|
16
16
|
)
|
|
17
17
|
|
|
18
18
|
/** Exact bundled instructions used as the progressive-exposure evidence marker. */
|
|
19
|
-
export const
|
|
19
|
+
export const VISION_SKILLS_CONTENT = readFileSync(
|
|
20
20
|
new URL('../assets/skill/SKILL.md', import.meta.url),
|
|
21
21
|
'utf8',
|
|
22
22
|
)
|
|
23
23
|
|
|
24
24
|
/** Runtime skill registration mounted only after every native tool is ready. */
|
|
25
|
-
export const
|
|
26
|
-
name:
|
|
25
|
+
export const VISION_SKILLS_SKILL: SkillRegistration = {
|
|
26
|
+
name: VISION_SKILLS_NAME,
|
|
27
27
|
description: '把截图或设计图还原为 UI(网页、应用界面或组件),生成 HTML/CSS 或项目现有前端代码;也支持图片问答、长截图/聊天记录 OCR、元素定位与盘点、裁剪、前景提取、像素对比、取色、SVG 描摹和 HTML 转截图。当任务涉及图片理解、UI 还原/重建(rebuild or restore a UI from a screenshot)、视觉回归、像素坐标、基于截图的 GUI 操作、可复用图片/SVG 素材、图表还原或长截图 OCR 时使用。',
|
|
28
28
|
whenToUse: '任务依赖图片文字/内容、像素坐标、截图转 UI 重建、视觉回归、可复用图片/SVG 素材、图表还原、基于截图的 GUI 操作或长截图 OCR 时使用。',
|
|
29
29
|
source: 'runtime',
|
|
30
30
|
resourceBase: {
|
|
31
31
|
kind: 'directory',
|
|
32
|
-
path:
|
|
32
|
+
path: VISION_SKILLS_RESOURCE_BASE,
|
|
33
33
|
},
|
|
34
|
-
content:
|
|
34
|
+
content: VISION_SKILLS_CONTENT,
|
|
35
35
|
}
|