@liustack/modlens 3.14.0 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +49 -2
- package/README.zh-CN.md +68 -17
- package/dist/main.js +55 -11
- package/docs/cli.md +2 -0
- package/docs/cli.zh-CN.md +98 -0
- package/docs/harness-setup.md +18 -4
- package/docs/harness-setup.zh-CN.md +67 -0
- package/docs/output-schema.md +2 -0
- package/docs/output-schema.zh-CN.md +74 -0
- package/docs/security.md +2 -0
- package/docs/security.zh-CN.md +43 -0
- package/docs/troubleshooting.md +2 -0
- package/docs/troubleshooting.zh-CN.md +210 -0
- package/dsh/client.js +87 -19
- package/dsh/index.js +329 -80
- package/package.json +1 -1
- package/skills/modlens/SKILL.md +5 -5
- package/skills/modlens/references/configure.md +4 -2
- package/skills/modlens/references/configure.zh-CN.md +177 -0
- package/skills/modlens/references/runtime.md +1 -1
- package/skills/modlens/scripts/run.ps1 +1 -1
- package/skills/modlens/scripts/run.sh +1 -1
package/dsh/index.js
CHANGED
|
@@ -15,9 +15,7 @@ import { fileURLToPath } from 'node:url'
|
|
|
15
15
|
const CLI_PATH = fileURLToPath(new URL('../dist/main.js', import.meta.url))
|
|
16
16
|
// Kept in lockstep with src/schema.ts by a repo test; the plugin file cannot
|
|
17
17
|
// import the TS source and stays fully dependency-free (node builtins only).
|
|
18
|
-
const OUTPUT_SCHEMA = JSON.parse(
|
|
19
|
-
readFileSync(new URL('./vision-schema.json', import.meta.url), 'utf8'),
|
|
20
|
-
)
|
|
18
|
+
const OUTPUT_SCHEMA = JSON.parse(readFileSync(new URL('./vision-schema.json', import.meta.url), 'utf8'))
|
|
21
19
|
|
|
22
20
|
const CLI_TIMEOUT_MS = 180_000
|
|
23
21
|
|
|
@@ -55,7 +53,9 @@ export function apply(ctx, config = {}) {
|
|
|
55
53
|
if (config.pasteToPath !== false && typeof ctx.inject === 'function') {
|
|
56
54
|
ctx.inject(['webServer'], (scope) => {
|
|
57
55
|
try {
|
|
58
|
-
|
|
56
|
+
// scope carries webServer; the plugin's own ctx carries llm for the
|
|
57
|
+
// takeover verdicts.
|
|
58
|
+
registerPasteRoute(scope, ctx)
|
|
59
59
|
} catch (error) {
|
|
60
60
|
console.error(`[modlens] paste-to-path route skipped: ${error}`)
|
|
61
61
|
}
|
|
@@ -117,9 +117,7 @@ export function apply(ctx, config = {}) {
|
|
|
117
117
|
}
|
|
118
118
|
const { stdout, stderr, code } = await run(process.execPath, cliArgs, exec.signal)
|
|
119
119
|
if (code !== 0) {
|
|
120
|
-
throw new Error(
|
|
121
|
-
`modlens failed (exit ${code}): ${(stderr || stdout).trim().slice(0, 500)}`,
|
|
122
|
-
)
|
|
120
|
+
throw new Error(`modlens failed (exit ${code}): ${(stderr || stdout).trim().slice(0, 500)}`)
|
|
123
121
|
}
|
|
124
122
|
let parsed
|
|
125
123
|
try {
|
|
@@ -140,9 +138,7 @@ export function apply(ctx, config = {}) {
|
|
|
140
138
|
if (preferred !== fallback && /already|duplicate/i.test(String(error))) {
|
|
141
139
|
try {
|
|
142
140
|
ctx.tools.register(readImageTool(fallback))
|
|
143
|
-
console.error(
|
|
144
|
-
`[modlens] tool name "${preferred}" is taken by the host; registered as "${fallback}" instead`,
|
|
145
|
-
)
|
|
141
|
+
console.error(`[modlens] tool name "${preferred}" is taken by the host; registered as "${fallback}" instead`)
|
|
146
142
|
} catch (retryError) {
|
|
147
143
|
console.error(`[modlens] read_image registration skipped: ${retryError}`)
|
|
148
144
|
}
|
|
@@ -153,27 +149,188 @@ export function apply(ctx, config = {}) {
|
|
|
153
149
|
}
|
|
154
150
|
|
|
155
151
|
// Image magic bytes for the paste route: refuse anything that is not a real
|
|
156
|
-
// image before a byte touches disk. Mirrors the CLI's sniffing table
|
|
152
|
+
// image before a byte touches disk. Mirrors the CLI's sniffing table
|
|
153
|
+
// (src/imageInput.ts SNIFFERS) signature for signature: full PNG magic, both
|
|
154
|
+
// GIF variants, and ftyp only with a known heic/heif brand — a generic BMFF
|
|
155
|
+
// (`ftypmp42`, plain video) must not be saved as an image.
|
|
157
156
|
const PASTE_SNIFFS = [
|
|
158
|
-
{
|
|
157
|
+
{
|
|
158
|
+
ext: '.png',
|
|
159
|
+
test: (b) =>
|
|
160
|
+
b.length >= 8 &&
|
|
161
|
+
b[0] === 0x89 &&
|
|
162
|
+
b[1] === 0x50 &&
|
|
163
|
+
b[2] === 0x4e &&
|
|
164
|
+
b[3] === 0x47 &&
|
|
165
|
+
b[4] === 0x0d &&
|
|
166
|
+
b[5] === 0x0a &&
|
|
167
|
+
b[6] === 0x1a &&
|
|
168
|
+
b[7] === 0x0a,
|
|
169
|
+
},
|
|
159
170
|
{ ext: '.jpg', test: (b) => b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
|
|
160
|
-
{
|
|
161
|
-
|
|
162
|
-
|
|
171
|
+
{
|
|
172
|
+
ext: '.gif',
|
|
173
|
+
test: (b) => b.length >= 6 && ['GIF87a', 'GIF89a'].includes(b.toString('ascii', 0, 6)),
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
ext: '.webp',
|
|
177
|
+
test: (b) => b.length >= 12 && b.toString('ascii', 0, 4) === 'RIFF' && b.toString('ascii', 8, 12) === 'WEBP',
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
ext: '.heic',
|
|
181
|
+
test: (b) =>
|
|
182
|
+
b.length >= 12 &&
|
|
183
|
+
b.toString('ascii', 4, 8) === 'ftyp' &&
|
|
184
|
+
['heic', 'heix', 'hevc', 'hevx'].includes(b.toString('ascii', 8, 12)),
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
ext: '.heif',
|
|
188
|
+
test: (b) =>
|
|
189
|
+
b.length >= 12 &&
|
|
190
|
+
b.toString('ascii', 4, 8) === 'ftyp' &&
|
|
191
|
+
['mif1', 'msf1', 'heif'].includes(b.toString('ascii', 8, 12)),
|
|
192
|
+
},
|
|
163
193
|
]
|
|
164
194
|
const PASTE_MAX_BYTES = 25 * 1024 * 1024
|
|
165
195
|
|
|
166
196
|
/**
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
197
|
+
* Should the browser take a paste over for the model behind this selector
|
|
198
|
+
* label? Decided here, not in the browser, because only the host holds the
|
|
199
|
+
* structured model metadata: a name regex in the client called every vision
|
|
200
|
+
* model it did not recognize text-only and hijacked its native paste.
|
|
201
|
+
*
|
|
202
|
+
* The label carries no provider id, only prose plus a display name, so the
|
|
203
|
+
* host cannot know WHICH matching model is selected: a longest-match pick
|
|
204
|
+
* was still hijackable (a text route named "Current Pro" outscored a selected
|
|
205
|
+
* vision model named "Pro", because the label's own "current" prose completed
|
|
206
|
+
* the longer name). So no picking at all: the answer is true only when EVERY
|
|
207
|
+
* model whose name or id appears in the label is positively confirmed
|
|
208
|
+
* text-only. One image-capable match anywhere vetoes; a model with no
|
|
209
|
+
* declared inputModalities is UNKNOWN, not text-only; and a provider whose
|
|
210
|
+
* catalog cannot be read is unknown too, a veto rather than a shrug, since the
|
|
211
|
+
* unreadable route is exactly where the vision twin could live. Anything
|
|
212
|
+
* unresolvable answers false: the native path is the safe default, and a
|
|
213
|
+
* text-only model merely keeps its old error message.
|
|
170
214
|
*/
|
|
171
|
-
function
|
|
215
|
+
async function pasteTakeoverVerdict(host, label) {
|
|
216
|
+
if (typeof label !== 'string' || label.trim() === '') return false
|
|
217
|
+
// Our own wrappers convert pastes at request time with the thumbnail
|
|
218
|
+
// preserved; taking their paste over would defeat the better path.
|
|
219
|
+
if (/\(modlens vision\)/i.test(label)) return false
|
|
220
|
+
const llm = host.llm
|
|
221
|
+
if (!llm || typeof llm.listProviders !== 'function' || typeof llm.listModels !== 'function') {
|
|
222
|
+
return false
|
|
223
|
+
}
|
|
224
|
+
const lowered = label.toLowerCase()
|
|
225
|
+
let matchedAny = false
|
|
226
|
+
for (const info of llm.listProviders()) {
|
|
227
|
+
const providerId = info?.id
|
|
228
|
+
if (!providerId) continue
|
|
229
|
+
let models = []
|
|
230
|
+
try {
|
|
231
|
+
models = await llm.listModels(providerId)
|
|
232
|
+
} catch {
|
|
233
|
+
return false
|
|
234
|
+
}
|
|
235
|
+
for (const model of models) {
|
|
236
|
+
for (const candidate of [model?.name, model?.id]) {
|
|
237
|
+
if (typeof candidate !== 'string' || candidate.length === 0) continue
|
|
238
|
+
if (!lowered.includes(candidate.toLowerCase())) continue
|
|
239
|
+
// The veto has no length floor: a vision model named "AI" appears in
|
|
240
|
+
// the label just as legitimately as a long name does, and skipping
|
|
241
|
+
// short names let a longer text-only name confirm the takeover alone.
|
|
242
|
+
const modalities = model?.inputModalities
|
|
243
|
+
if (!Array.isArray(modalities) || modalities.includes('image')) {
|
|
244
|
+
return false
|
|
245
|
+
}
|
|
246
|
+
// Positive confirmation does have a floor: one- and two-character
|
|
247
|
+
// text-only names match label prose far too easily to identify the
|
|
248
|
+
// selected model.
|
|
249
|
+
if (candidate.length >= 3) {
|
|
250
|
+
matchedAny = true
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return matchedAny
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Verdicts are stable for the lifetime of a model route but the inventory can
|
|
259
|
+
// grow (llm-pi-ai mounts after settings load), so cache briefly, not forever.
|
|
260
|
+
const PASTE_VERDICT_TTL_MS = 15_000
|
|
261
|
+
const PASTE_VERDICT_CAP = 32
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* The paste route. POST /modlens/paste: image bytes in, `{ path }` out; the
|
|
265
|
+
* file is private (0600) in a fresh unpredictable temp dir, magic-byte
|
|
266
|
+
* checked and size-capped. GET /modlens/paste?model=<selector label>:
|
|
267
|
+
* `{ takeover }`: the browser half asks before ever touching a paste, so a
|
|
268
|
+
* disabled route (pasteToPath: false, or no web profile) means the client
|
|
269
|
+
* stands down instead of swallowing pastes into a 404. Bound to the dsh web
|
|
270
|
+
* server, which listens on loopback by default.
|
|
271
|
+
*/
|
|
272
|
+
function registerPasteRoute(ctx, host) {
|
|
273
|
+
const verdicts = new Map()
|
|
274
|
+
// The cache key is only the selector label, which cannot tell two
|
|
275
|
+
// same-named models on different routes apart. A route mounting mid-TTL
|
|
276
|
+
// (llm-pi-ai lands after settings load) could therefore serve a stale
|
|
277
|
+
// verdict computed before its vision twin existed, so every topology
|
|
278
|
+
// change empties the cache at exactly the boundary that invalidates it.
|
|
279
|
+
// The epoch guards the async gap the clear cannot reach: a verdict whose
|
|
280
|
+
// computation STARTED before the event describes a registry that no longer
|
|
281
|
+
// exists, and without the counter it was written back into the just-
|
|
282
|
+
// emptied cache and served for a full TTL.
|
|
283
|
+
let topologyEpoch = 0
|
|
284
|
+
if (typeof host.on === 'function') {
|
|
285
|
+
host.on('llm/adapters-updated', () => {
|
|
286
|
+
topologyEpoch += 1
|
|
287
|
+
verdicts.clear()
|
|
288
|
+
})
|
|
289
|
+
}
|
|
172
290
|
ctx.webServer.register({
|
|
173
291
|
name: 'modlens-paste',
|
|
174
292
|
kind: 'exact',
|
|
175
293
|
path: '/modlens/paste',
|
|
176
294
|
handler: async (req, res) => {
|
|
295
|
+
if (req.method === 'GET') {
|
|
296
|
+
try {
|
|
297
|
+
const label = new URL(req.url, 'http://localhost').searchParams.get('model') ?? ''
|
|
298
|
+
const cached = verdicts.get(label)
|
|
299
|
+
let takeover
|
|
300
|
+
if (cached && Date.now() - cached.at < PASTE_VERDICT_TTL_MS) {
|
|
301
|
+
takeover = cached.takeover
|
|
302
|
+
} else {
|
|
303
|
+
// Recompute while the topology moves under the computation: an
|
|
304
|
+
// answer read from a pre-event registry snapshot must be neither
|
|
305
|
+
// cached nor served. Bounded, and the give-up answer is the
|
|
306
|
+
// conservative one.
|
|
307
|
+
let attempts = 0
|
|
308
|
+
for (;;) {
|
|
309
|
+
const startedEpoch = topologyEpoch
|
|
310
|
+
takeover = await pasteTakeoverVerdict(host, label)
|
|
311
|
+
if (topologyEpoch === startedEpoch) {
|
|
312
|
+
verdicts.delete(label)
|
|
313
|
+
verdicts.set(label, { takeover, at: Date.now() })
|
|
314
|
+
if (verdicts.size > PASTE_VERDICT_CAP) {
|
|
315
|
+
verdicts.delete(verdicts.keys().next().value)
|
|
316
|
+
}
|
|
317
|
+
break
|
|
318
|
+
}
|
|
319
|
+
attempts += 1
|
|
320
|
+
if (attempts >= 3) {
|
|
321
|
+
takeover = false
|
|
322
|
+
break
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
res.writeHead(200, { 'content-type': 'application/json' })
|
|
327
|
+
res.end(JSON.stringify({ takeover }))
|
|
328
|
+
} catch (error) {
|
|
329
|
+
res.writeHead(500, { 'content-type': 'application/json' })
|
|
330
|
+
res.end(JSON.stringify({ error: String(error?.message ?? error) }))
|
|
331
|
+
}
|
|
332
|
+
return
|
|
333
|
+
}
|
|
177
334
|
if (req.method !== 'POST') {
|
|
178
335
|
res.writeHead(405).end()
|
|
179
336
|
return
|
|
@@ -208,7 +365,7 @@ function registerPasteRoute(ctx) {
|
|
|
208
365
|
res.end(JSON.stringify({ path: file }))
|
|
209
366
|
} catch (error) {
|
|
210
367
|
res.writeHead(500, { 'content-type': 'application/json' })
|
|
211
|
-
res.end(JSON.stringify({ error: String(error
|
|
368
|
+
res.end(JSON.stringify({ error: String(error?.message ? error.message : error) }))
|
|
212
369
|
}
|
|
213
370
|
},
|
|
214
371
|
})
|
|
@@ -220,15 +377,25 @@ function registerPasteRoute(ctx) {
|
|
|
220
377
|
* text-only, so pastes are refused before any plugin hook runs. This wrapper
|
|
221
378
|
* registers a NEW provider whose model metadata declares image input and
|
|
222
379
|
* whose stream() is a one-line delegation back to the real route. Pick the
|
|
223
|
-
* wrapped model in the model selector, paste, and the
|
|
380
|
+
* wrapped model in the model selector, paste, and the request-time rewrite
|
|
224
381
|
* turns the image into evidence text before the delegated request goes out;
|
|
225
382
|
* the upstream serializer's own image rejection stays as the fail-closed
|
|
226
383
|
* backstop. Guarded feature-detection: if the llm registration surface moved
|
|
227
384
|
* (developer preview), the plugin quietly stays a read_image-only tool.
|
|
385
|
+
*
|
|
386
|
+
* Two modes (issue #29, design contributed by @zlycode01):
|
|
387
|
+
* - `config.upstream` set: wrap exactly that one route, legacy behavior.
|
|
388
|
+
* - unset: auto-discovery — every registered provider route carrying
|
|
389
|
+
* wrappable text-only family models gets its own `modlens-<provider>`
|
|
390
|
+
* wrapper, so a machine with several subscription packages (opencode-go,
|
|
391
|
+
* zai, ...) wraps them all instead of hand-picking one. A `discover` array
|
|
392
|
+
* of provider ids narrows the set. Routes that register late (llm-pi-ai
|
|
393
|
+
* mounts its routes after settings load) are picked up by re-sweeping on
|
|
394
|
+
* the registry's own `llm/adapters-updated` notification, no polling. The
|
|
395
|
+
* deepseek-official wrap keeps its historical `deepseek-modlens` id, so a
|
|
396
|
+
* selector remembering that provider survives the upgrade.
|
|
228
397
|
*/
|
|
229
398
|
function registerVisionProvider(ctx, config) {
|
|
230
|
-
const upstream = config.upstream || 'deepseek-official'
|
|
231
|
-
const providerId = config.providerId || 'deepseek-modlens'
|
|
232
399
|
// Wrap only the text-only members of these families. Their own vision
|
|
233
400
|
// models (present or future: deepseek-vl/ocr/janus, glm-4.5v, glm-5v-...)
|
|
234
401
|
// need no bridge and are excluded by name and by declared modality.
|
|
@@ -244,58 +411,144 @@ function registerVisionProvider(ctx, config) {
|
|
|
244
411
|
if (typeof ctx.llm?.registerAdapter !== 'function' || typeof ctx.llm?.stream !== 'function') {
|
|
245
412
|
return
|
|
246
413
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
414
|
+
|
|
415
|
+
const registerWrapper = (upstream, providerId, displayName) => {
|
|
416
|
+
const withVision = (info) => ({
|
|
417
|
+
...info,
|
|
418
|
+
provider: providerId,
|
|
419
|
+
inputModalities: ['text', 'image'],
|
|
420
|
+
})
|
|
421
|
+
try {
|
|
422
|
+
ctx.llm.registerAdapter([providerId], {
|
|
423
|
+
// Duck-typing LlmAdapter: providerInfo/providerRetryPolicy are
|
|
424
|
+
// base-class defaults a plain object must supply itself (their
|
|
425
|
+
// absence is exactly the silent registration failure this catch
|
|
426
|
+
// used to swallow).
|
|
427
|
+
providerInfo(provider) {
|
|
428
|
+
return { id: provider, name: displayName }
|
|
429
|
+
},
|
|
430
|
+
providerRetryPolicy() {
|
|
431
|
+
return undefined
|
|
432
|
+
},
|
|
433
|
+
async listModels(_provider, signal) {
|
|
434
|
+
try {
|
|
435
|
+
const models = await ctx.llm.listModels(upstream, signal)
|
|
436
|
+
return models.filter(shouldWrap).map((model) => ({
|
|
437
|
+
...withVision(model),
|
|
438
|
+
name: `${model.name ?? model.id} (modlens vision)`,
|
|
439
|
+
}))
|
|
440
|
+
} catch {
|
|
441
|
+
return []
|
|
442
|
+
}
|
|
443
|
+
},
|
|
444
|
+
async resolveModel(_provider, model, signal) {
|
|
445
|
+
const info = await ctx.llm.resolveModelInfo(upstream, model, signal)
|
|
446
|
+
if (!shouldWrap(info)) {
|
|
447
|
+
throw new Error(`model "${model}" is outside the modlens vision wrap scope`)
|
|
448
|
+
}
|
|
449
|
+
return { ...withVision(info), id: model }
|
|
450
|
+
},
|
|
451
|
+
stream(options) {
|
|
452
|
+
// Convert at request time, not at log time: the durable session
|
|
453
|
+
// log keeps the real image blocks (so the UI shows the paste
|
|
454
|
+
// natively), and only the wire messages carry evidence text.
|
|
455
|
+
// Cached per attachment, since the same history rides every step.
|
|
456
|
+
const self = this
|
|
457
|
+
return (async function* () {
|
|
458
|
+
const messages = await convertImagesToEvidence(ctx, options.messages, options.signal, self)
|
|
459
|
+
yield* ctx.llm.stream({ ...options, provider: upstream, messages })
|
|
460
|
+
})()
|
|
461
|
+
},
|
|
462
|
+
evidenceCache: new Map(),
|
|
463
|
+
})
|
|
464
|
+
return true
|
|
465
|
+
} catch (error) {
|
|
466
|
+
// A duplicate means a concurrent or earlier registration already won:
|
|
467
|
+
// that is success for the claim, not a reason to retry forever.
|
|
468
|
+
if (/already|duplicate/i.test(String(error))) {
|
|
469
|
+
console.error(`[modlens] vision provider ${providerId} already registered, keeping the existing one`)
|
|
470
|
+
return true
|
|
471
|
+
}
|
|
472
|
+
// A preview-era surface change: degrade to the read_image-only plugin,
|
|
473
|
+
// but say so in the harness log instead of vanishing (a swallowed
|
|
474
|
+
// TypeError here once hid a missing base method).
|
|
475
|
+
console.error(`[modlens] vision provider registration skipped (${providerId}): ${error}`)
|
|
476
|
+
return false
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
if (config.upstream) {
|
|
481
|
+
registerWrapper(config.upstream, config.providerId || 'deepseek-modlens', 'DeepSeek (modlens vision)')
|
|
482
|
+
return
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// Auto-discovery. `wrapped` guards duplicates across sweeps and the
|
|
486
|
+
// self-nesting case (our own wrappers appear in listProviders too). Two
|
|
487
|
+
// re-entrancy rules matter because registerAdapter itself broadcasts
|
|
488
|
+
// llm/adapters-updated, so every successful wrap re-triggers a sweep:
|
|
489
|
+
// an id is claimed in `wrapped` BEFORE any await (a concurrent sweep must
|
|
490
|
+
// skip it while this one is still probing), and sweeps are serialized on
|
|
491
|
+
// one promise chain so two can never interleave their probes at all.
|
|
492
|
+
const discover = Array.isArray(config.discover) ? new Set(config.discover) : null
|
|
493
|
+
const wrapped = new Set(['deepseek-modlens'])
|
|
494
|
+
const sweepOnce = async () => {
|
|
495
|
+
try {
|
|
496
|
+
await sweepBody()
|
|
497
|
+
} catch (error) {
|
|
498
|
+
// A sweep failure must never become an unhandled rejection inside the
|
|
499
|
+
// host process; the next topology notification simply tries again.
|
|
500
|
+
console.error(`[modlens] vision provider discovery sweep failed: ${error}`)
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
const sweepBody = async () => {
|
|
504
|
+
if (typeof ctx.llm.listProviders !== 'function') {
|
|
505
|
+
// Older registry surface: fall back to the single legacy wrap once.
|
|
506
|
+
if (!wrapped.has('__legacy_fallback__')) {
|
|
507
|
+
wrapped.add('__legacy_fallback__')
|
|
508
|
+
registerWrapper('deepseek-official', 'deepseek-modlens', 'DeepSeek (modlens vision)')
|
|
509
|
+
}
|
|
510
|
+
return
|
|
511
|
+
}
|
|
512
|
+
for (const info of ctx.llm.listProviders()) {
|
|
513
|
+
const id = info?.id
|
|
514
|
+
if (!id || wrapped.has(id) || String(id).startsWith('modlens-')) continue
|
|
515
|
+
if (discover && !discover.has(id)) continue
|
|
516
|
+
// Claim before the await: the probe may suspend, and the sweep a
|
|
517
|
+
// registration triggers must not probe the same id concurrently.
|
|
518
|
+
wrapped.add(id)
|
|
519
|
+
let models = []
|
|
520
|
+
try {
|
|
521
|
+
models = await ctx.llm.listModels(id)
|
|
522
|
+
} catch {
|
|
523
|
+
// Unreachable route today; release the claim so a later topology
|
|
524
|
+
// change retries it.
|
|
525
|
+
wrapped.delete(id)
|
|
526
|
+
continue
|
|
527
|
+
}
|
|
528
|
+
if (!models.some(shouldWrap)) {
|
|
529
|
+
// No eligible models yet: release, the route may gain some later.
|
|
530
|
+
wrapped.delete(id)
|
|
531
|
+
continue
|
|
532
|
+
}
|
|
533
|
+
const providerId = id === 'deepseek-official' ? 'deepseek-modlens' : `modlens-${id}`
|
|
534
|
+
const base = info.name ?? id
|
|
535
|
+
if (!registerWrapper(id, providerId, `${base} (modlens vision)`)) {
|
|
536
|
+
wrapped.delete(id)
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
// Serialize: a sweep triggered mid-sweep runs after, never interleaved.
|
|
541
|
+
// The first sweep is invoked directly so its synchronous prefix (the
|
|
542
|
+
// legacy fallback, the pre-await claims) completes during apply().
|
|
543
|
+
let sweeping = sweepOnce()
|
|
544
|
+
const sweep = () => {
|
|
545
|
+
sweeping = sweeping.then(sweepOnce, sweepOnce)
|
|
546
|
+
return sweeping
|
|
547
|
+
}
|
|
548
|
+
if (typeof ctx.on === 'function') {
|
|
549
|
+
ctx.on('llm/adapters-updated', () => {
|
|
550
|
+
void sweep()
|
|
293
551
|
})
|
|
294
|
-
} catch (error) {
|
|
295
|
-
// DUPLICATE_ADAPTER or a preview-era surface change: degrade to the
|
|
296
|
-
// read_image-only plugin, but say so in the harness log instead of
|
|
297
|
-
// vanishing (a swallowed TypeError here once hid a missing base method).
|
|
298
|
-
console.error(`[modlens] vision provider registration skipped: ${error}`)
|
|
299
552
|
}
|
|
300
553
|
}
|
|
301
554
|
|
|
@@ -388,9 +641,7 @@ function abortableWait(promise, signal) {
|
|
|
388
641
|
function contentHasImage(blocks) {
|
|
389
642
|
return (
|
|
390
643
|
Array.isArray(blocks) &&
|
|
391
|
-
blocks.some(
|
|
392
|
-
(b) => b?.type === 'image' || (b?.type === 'tool-result' && contentHasImage(b.content)),
|
|
393
|
-
)
|
|
644
|
+
blocks.some((b) => b?.type === 'image' || (b?.type === 'tool-result' && contentHasImage(b.content)))
|
|
394
645
|
)
|
|
395
646
|
}
|
|
396
647
|
|
|
@@ -474,9 +725,7 @@ async function readImageBlock(ctx, block, signal) {
|
|
|
474
725
|
if (!stored?.data) {
|
|
475
726
|
// Named failure instead of Buffer.from(undefined)'s bare TypeError the
|
|
476
727
|
// next time a developer-preview release moves the field (issue #17).
|
|
477
|
-
throw new Error(
|
|
478
|
-
"attachments.readImage returned no 'data' bytes; the dsh attachment shape may have changed",
|
|
479
|
-
)
|
|
728
|
+
throw new Error("attachments.readImage returned no 'data' bytes; the dsh attachment shape may have changed")
|
|
480
729
|
}
|
|
481
730
|
const mediaType = stored.ref?.mediaType ?? block.attachment?.mediaType
|
|
482
731
|
const ext = MEDIA_EXT[mediaType]
|
package/package.json
CHANGED
package/skills/modlens/SKILL.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: modlens
|
|
3
3
|
description: "Plug-in vision for text-only models. Hard rule: when a file path or URL with an image extension (.png, .jpg, .jpeg, .webp, .gif, .heic, .heif) appears anywhere in the conversation (typed by the user, injected as a `[Image: source: <path>]` line, or inside a tag) and you cannot see that image's content, run this skill on it before any other approach: no self-built OCR, no PIL, no tesseract. Also triggers on pasted-image placeholders such as `[Image #1]` and `[Unsupported Image]`. If you can actually see the image, do not use this skill. When unsure, run `modlens guard` before the first read of a session: a deny verdict means the active model has native vision and must read the image itself. Runs the modlens CLI to convert the image into structured JSON evidence: every word transcribed, layout regions, semantics, visual clues. Also use when the user asks how to install, configure, or switch modlens providers (Gemini API key, OpenAI-compatible endpoints, Claude API or Claude Code CLI)."
|
|
4
|
-
compatibility: Requires network access and one of node 22+/npx, bun/bunx, or a preinstalled modlens binary on PATH.
|
|
4
|
+
compatibility: Requires network access and one of node 22.19+/npx, bun/bunx, or a preinstalled modlens binary on PATH.
|
|
5
5
|
allowed-tools: Bash
|
|
6
6
|
---
|
|
7
7
|
|
|
@@ -20,11 +20,11 @@ powershell -ExecutionPolicy Bypass -File <skill-dir>\scripts\run.ps1 <args>
|
|
|
20
20
|
|
|
21
21
|
It resolves a working runtime (PATH `modlens`, then `npx`, then `bunx`) and forwards your arguments unchanged. Exit 78 means no runtime: relay the `nextSteps` from its stderr JSON instead of retrying.
|
|
22
22
|
|
|
23
|
-
If your harness forbids running scripts, reason through the same order by hand and run the first line that works (the pinned version is 3.
|
|
23
|
+
If your harness forbids running scripts, reason through the same order by hand and run the first line that works (the pinned version is 3.16.0):
|
|
24
24
|
|
|
25
|
-
1. A `modlens` on `PATH` whose major version is 3 and is at least 3.
|
|
26
|
-
2. Otherwise, if `npx` exists: `npx --yes --package @liustack/modlens@3.
|
|
27
|
-
3. Otherwise, if `bunx` exists: `bunx --bun @liustack/modlens@3.
|
|
25
|
+
1. A `modlens` on `PATH` whose major version is 3 and is at least 3.16.0: `modlens <args>`.
|
|
26
|
+
2. Otherwise, if `npx` exists: `npx --yes --package @liustack/modlens@3.16.0 modlens <args>`.
|
|
27
|
+
3. Otherwise, if `bunx` exists: `bunx --bun @liustack/modlens@3.16.0 <args>`.
|
|
28
28
|
4. Otherwise tell the user no JavaScript runtime was found and that installing Node 22.19+ (https://nodejs.org) or Bun (https://bun.sh) is the next step. Do not claim modlens itself failed.
|
|
29
29
|
|
|
30
30
|
`references/runtime.md` documents the pin and the diagnostic fields.
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
# Configuring ModLens
|
|
2
2
|
|
|
3
|
+
English | [中文](configure.zh-CN.md)
|
|
4
|
+
|
|
3
5
|
Read this when the user asks how to set up, configure, or switch ModLens providers. Prefer running the commands for the user over explaining them.
|
|
4
6
|
|
|
5
7
|
## Where config lives
|
|
6
8
|
|
|
7
|
-
`~/.modlens/config.json`, managed by the CLI. Precedence: CLI flags > environment variables > config file > built-in defaults.
|
|
9
|
+
`~/.modlens/config.json`, managed by the CLI. Precedence: CLI flags > environment variables > config file > built-in defaults. With no `provider` set, runs walk the failover chain in order (an available `gemini-api` key is tried before the agent CLIs); a machine with nothing configured at all ends up on `antigravity-cli`.
|
|
8
10
|
|
|
9
11
|
```bash
|
|
10
12
|
modlens config init # write a starter config (refuses to overwrite; --force to redo)
|
|
@@ -54,7 +56,7 @@ Everything lives under four top-level keys, all optional. This example shows eve
|
|
|
54
56
|
|
|
55
57
|
Field semantics:
|
|
56
58
|
|
|
57
|
-
- `provider`: which provider runs when `-p` is not given. Canonical names or aliases both work (`agy`/`antigravity` for `antigravity-cli`, `gemini` for `gemini-api`, `openai-compat` for `openai`, `claude` for `anthropic`, `claude-code` for `claude-cli`). Empty or absent
|
|
59
|
+
- `provider`: which provider runs when `-p` is not given. Canonical names or aliases both work (`agy`/`antigravity` for `antigravity-cli`, `gemini` for `gemini-api`, `openai-compat` for `openai`, `claude` for `anthropic`, `claude-code` for `claude-cli`). Empty or absent pins nothing: the failover chain decides, trying configured API providers before the agent CLIs.
|
|
58
60
|
- `providers.<name>.<field>`: four fields exist, `apiKey`, `baseUrl`, `model`, and `extraBody`. Every provider entry is optional, and every field inside it is optional. Alias keys are read too (settings saved under `gemini` are found when `gemini-api` resolves), with the canonical key winning on conflict.
|
|
59
61
|
- `providers.<name>.extraBody`: a JSON object merged into the request body of the API providers (`gemini-api`, `openai`, `anthropic`), for whatever knobs that vendor has and modlens has no flag for. Turning thinking off is the usual reason, see the section below. Nested objects merge key by key, so adding one knob leaves the rest of that block alone. The fields carrying the image, the prompt, and the schema enforcement are refused with an error naming the field. The two CLI providers take no request body, so a run on `antigravity-cli` or `claude-cli` ignores it and says so in `meta.warnings`.
|
|
60
62
|
- `guards`: the invocation guard, for people who run both text-only and vision-capable models through the same client. Both lists hold glob patterns (`*` and `?`, case-insensitive, matched against the model name and `provider/model`), set with `modlens config set guards.denyModels '["gemini-3*"]'` or `guards.allowModels` (a JSON array or a comma-separated list, empty clears). Two ways to express the same intent, pick the shorter list:
|