museav-cli 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +17 -2
- package/CHANGELOG.md +31 -0
- package/README.md +52 -4
- package/dist/client.d.ts +51 -0
- package/dist/client.js +39 -0
- package/dist/commands/gen.d.ts +1 -0
- package/dist/commands/gen.js +12 -0
- package/dist/commands/img-tools.d.ts +16 -0
- package/dist/commands/img-tools.js +81 -0
- package/dist/commands/jobs.d.ts +1 -0
- package/dist/commands/jobs.js +7 -1
- package/dist/commands/projects.d.ts +21 -0
- package/dist/commands/projects.js +69 -0
- package/dist/commands/reverse.d.ts +7 -2
- package/dist/commands/reverse.js +28 -2
- package/dist/commands/templates.js +2 -1
- package/dist/commands/video-templates.d.ts +2 -0
- package/dist/commands/video-templates.js +14 -5
- package/dist/index.js +85 -7
- package/dist/local-bg.d.ts +17 -0
- package/dist/local-bg.js +143 -0
- package/dist/local-vision.d.ts +14 -0
- package/dist/local-vision.js +134 -0
- package/package.json +3 -2
- package/src/client.ts +94 -13
- package/src/commands/gen.ts +14 -0
- package/src/commands/img-tools.ts +95 -0
- package/src/commands/jobs.ts +8 -2
- package/src/commands/projects.ts +80 -0
- package/src/commands/reverse.ts +36 -4
- package/src/commands/templates.ts +2 -1
- package/src/commands/video-templates.ts +15 -4
- package/src/index.ts +93 -7
- package/src/local-bg.ts +142 -0
- package/src/local-vision.ts +150 -0
package/src/index.ts
CHANGED
|
@@ -15,6 +15,8 @@ import { bindFeishu } from './commands/bind-feishu.js'
|
|
|
15
15
|
import { printWelcome } from './commands/welcome.js'
|
|
16
16
|
import { gen } from './commands/gen.js'
|
|
17
17
|
import { reverse } from './commands/reverse.js'
|
|
18
|
+
import { compressCmd, removeBgCmd } from './commands/img-tools.js'
|
|
19
|
+
import { projects, createProject, listAssets, addAsset, removeAsset, resolveWorkspace } from './commands/projects.js'
|
|
18
20
|
import { imageToTemplate } from './commands/image-to-template.js'
|
|
19
21
|
import { upload } from './commands/upload.js'
|
|
20
22
|
import { models } from './commands/models.js'
|
|
@@ -93,6 +95,34 @@ function withClient<T extends (...args: any[]) => Promise<any>>(fn: T) {
|
|
|
93
95
|
}
|
|
94
96
|
}
|
|
95
97
|
|
|
98
|
+
// reverse 的本地路(Ollama)不需要中台凭证,client 懒构造:只有真回落 API 才建,
|
|
99
|
+
// 未登录的报错也只在那时候出现
|
|
100
|
+
function withLazyClient(fn: (getClient: () => StudioClient, ...args: any[]) => Promise<any>) {
|
|
101
|
+
return async (...args: any[]) => {
|
|
102
|
+
try {
|
|
103
|
+
const opts = args[args.length - 2]
|
|
104
|
+
const positional = args.slice(0, -2)
|
|
105
|
+
const getClient = () => new StudioClient(loadConfig())
|
|
106
|
+
await fn(getClient, ...positional, opts)
|
|
107
|
+
} catch (e) {
|
|
108
|
+
process.stderr.write(`❌ ${(e as Error).message}\n`)
|
|
109
|
+
process.exit(1)
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// 本地命令(compress / remove-bg 等):不碰中台、不需要任何凭证,只包一层统一的错误出口
|
|
115
|
+
function asyncRun(fn: (...args: any[]) => Promise<any>) {
|
|
116
|
+
return async (...args: any[]) => {
|
|
117
|
+
try {
|
|
118
|
+
await fn(...args)
|
|
119
|
+
} catch (e) {
|
|
120
|
+
process.stderr.write(`❌ ${(e as Error).message}\n`)
|
|
121
|
+
process.exit(1)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
96
126
|
// products / assets 查的是租户自己后台的数据,不是 Studio 中台的,走独立的 TenantClient
|
|
97
127
|
// (见 tenant-client.ts 顶部注释),只支持租户 apiKey 身份,不支持个人 login token。
|
|
98
128
|
function withTenantClient<T extends (...args: any[]) => Promise<any>>(fn: T) {
|
|
@@ -127,7 +157,7 @@ program
|
|
|
127
157
|
// 不设默认值:--skill / --template 场景下要让技能/模板自己的比例生效,
|
|
128
158
|
// CLI 强填默认值会把它们覆盖掉(服务端在纯 --prompt 场景已有 3:4 兜底,这里不用重复兜底)
|
|
129
159
|
.option('-r, --ratio <ratio>', '宽高比: 3:4 / 9:16 / 1:1 / 4:3 / 16:9(不指定则用技能/模板自己的比例,纯 prompt 模式兜底 3:4)')
|
|
130
|
-
.option('-m, --model <name>', '指定模型,如 gpt-image-2 /
|
|
160
|
+
.option('-m, --model <name>', '指定模型,如 gpt-image-2 / artsdance-2-0-pro-260801(视频不传则走 auto 路由)')
|
|
131
161
|
.option('-q, --quality <level>', '质量: low / medium / high(仅 gpt-image)')
|
|
132
162
|
// 可重复:--ref 正面.jpg --ref 背景.jpg。顺序即语义——提示词里写「参考图片1的排版、
|
|
133
163
|
// 用图片2作为背景」时,图片N 对应第 N 个 --ref。commander 的 collect 保证顺序。
|
|
@@ -136,15 +166,69 @@ program
|
|
|
136
166
|
// 透明背景是上游的 background 参数,不是提示词能表达的东西——提示词里写
|
|
137
167
|
// "transparent background" 只是在描述构图,模型照样铺一层白底。这个开关才是抠图开关。
|
|
138
168
|
.option('--transparent', '透明背景 PNG(抠掉背景,带 alpha 通道)。仅部分上游支持,不支持时中台明确报错、不会悄悄给白底图;服务端自动强制 PNG 输出(JPEG 没有 alpha 通道)')
|
|
139
|
-
.option('--video', '生成视频(走 /api/videos
|
|
169
|
+
.option('--video', '生成视频(走 /api/videos 链路;模型档次如 artsdance-2-0-pro-260801,不传 --model 走 auto 路由)')
|
|
140
170
|
.option('--duration <sec>', '视频时长(秒,仅 --video;由模型与上游支持范围决定)', (v) => Number(v))
|
|
141
171
|
.option('--image <file>', '图生视频首帧图(仅 --video,自动上传)')
|
|
172
|
+
.option('--project <id|名>', '归档进该工作区(museav projects 查;账户身份才生效)')
|
|
142
173
|
.action(withClient((client: StudioClient, opts: any) => gen(client, opts)))
|
|
143
174
|
|
|
175
|
+
program
|
|
176
|
+
.command('compress <file>')
|
|
177
|
+
.description('本地压缩图片(sharp,免登录):默认同目录 <名>-min.<格式>,不覆写原文件')
|
|
178
|
+
.option('--out <path>', '输出路径(默认 <名>-min.<格式>)')
|
|
179
|
+
.option('--max-edge <px>', '最长边缩到该像素(等比,inside)')
|
|
180
|
+
.option('--quality <1-100>', 'jpg/webp 质量,默认 82')
|
|
181
|
+
.option('--format <fmt>', '输出格式 jpg / png / webp(默认跟随原格式)')
|
|
182
|
+
.option('--overwrite', '允许覆盖已存在的输出文件')
|
|
183
|
+
.action(asyncRun((input: string, opts: any) => compressCmd(input, opts)))
|
|
184
|
+
|
|
185
|
+
program
|
|
186
|
+
.command('remove-bg <file>')
|
|
187
|
+
.description('本地抠图去背景(ISNet/U2Net + onnxruntime,免登录):输出带 alpha 的 PNG。首次使用自动下载模型(~170MB,缓存 ~/.museav-models)')
|
|
188
|
+
.option('--out <path>', '输出路径(默认 <名>-nobg.png)')
|
|
189
|
+
.option('--model <name>', 'isnet(默认,质量优先)/ u2net')
|
|
190
|
+
.option('--overwrite', '允许覆盖已存在的输出文件')
|
|
191
|
+
.action(asyncRun((input: string, opts: any) => removeBgCmd(input, opts)))
|
|
192
|
+
|
|
193
|
+
// 工作区(项目)与项目素材库:平台 → 账户 → 工作区三层归属,素材挂工作区
|
|
194
|
+
const projectsCmd = program
|
|
195
|
+
.command('projects')
|
|
196
|
+
.description('工作区(项目)管理:一个账户多个工作区,每个工作区有自己的素材库(人像库/产品库各管各的业务)')
|
|
197
|
+
.action(withClient((client: StudioClient) => projects(client)))
|
|
198
|
+
|
|
199
|
+
projectsCmd
|
|
200
|
+
.command('create')
|
|
201
|
+
.description('新建工作区(每账户最多 5 个)')
|
|
202
|
+
.requiredOption('--name <name>', '工作区名称(最多 20 字)')
|
|
203
|
+
.action(withClient((client: StudioClient, opts: any) => createProject(client, opts)))
|
|
204
|
+
|
|
205
|
+
const assetsCmd = projectsCmd
|
|
206
|
+
.command('assets')
|
|
207
|
+
.description('项目素材库:列出 / 上传 / 删除该工作区的素材(垫图母版,不压缩)')
|
|
208
|
+
|
|
209
|
+
assetsCmd
|
|
210
|
+
.description('列工作区素材库')
|
|
211
|
+
.option('--project <id|名>', '工作区 id 或名称(必填,不传会明确报错)')
|
|
212
|
+
.action(withClient((client: StudioClient, opts: any) => listAssets(client, opts)))
|
|
213
|
+
|
|
214
|
+
assetsCmd
|
|
215
|
+
.command('add <file>')
|
|
216
|
+
.description('上传素材进工作区素材库(图片/音频/视频,按字节判型;母版不压缩)')
|
|
217
|
+
.requiredOption('--project <id|名>', '工作区 id 或名称')
|
|
218
|
+
.option('--name <name>', '素材名,如「白T正面」')
|
|
219
|
+
.option('--tag <tag>', '标签,可重复(产品 / 人像 / 场景…)', (v: string, acc: string[]) => [...acc, v], [] as string[])
|
|
220
|
+
.action(withClient((client: StudioClient, file: string, opts: any) => addAsset(client, file, opts)))
|
|
221
|
+
|
|
222
|
+
assetsCmd
|
|
223
|
+
.command('rm <id>')
|
|
224
|
+
.description('删除素材(硬删:R2 对象 + 记录)')
|
|
225
|
+
.action(withClient((client: StudioClient, id: string) => removeAsset(client, { id })))
|
|
226
|
+
|
|
144
227
|
program
|
|
145
228
|
.command('reverse <input>')
|
|
146
|
-
.description('
|
|
147
|
-
.
|
|
229
|
+
.description('读图:反推 SCULPT prompt,stdout 输出英文 prompt。主路本地 Ollama(qwen3-vl,快,无需登录);本地不可用回落中台 API(会提示较慢)。只读图;要做成模板用 image-to-template')
|
|
230
|
+
.option('--api', '跳过本地 Ollama,强制走中台 API(慢,需登录)')
|
|
231
|
+
.action(withLazyClient((getClient: () => StudioClient, input: string, opts: any) => reverse(getClient, input, opts)))
|
|
148
232
|
|
|
149
233
|
program
|
|
150
234
|
.command('image-to-template <input>')
|
|
@@ -194,10 +278,11 @@ videoTemplatesCmd
|
|
|
194
278
|
)
|
|
195
279
|
.requiredOption('--name <zh_name>', '模板中文名')
|
|
196
280
|
.requiredOption('--prompt <template>', '提示词模板,占位符用 {key} 形式,如 "{product} 在 {scene} 中展示"')
|
|
281
|
+
.option('--slug <slug>', '对外调用标识(全局唯一,视频模板硬必填);不传自动生成 vt- 前缀短标识')
|
|
197
282
|
.option('--category <name>', '分类,默认「其他」')
|
|
198
283
|
.option('--description <text>', '模板说明')
|
|
199
|
-
.option('--model <name>', '
|
|
200
|
-
.option('--duration <sec>', '视频时长(秒,4-15,可选)')
|
|
284
|
+
.option('--model <name>', '视频模型档次,默认 auto(交给中台路由);锁死可选 artsdance-2-0-pro-260801(Seedance 2.0)/ artsdance-2-0-fast-260801 / artsdance-2-0-mini-260801 / artsdance-2-5-pro-260801(Seedance 2.5)')
|
|
285
|
+
.option('--duration <sec>', '视频时长(秒,4-30:Seedance 2.0 系上限 15、2.5 到 30,可选)')
|
|
201
286
|
.option('--ratio <ratio>', '画面比例: 9:16 / 16:9 / 1:1 / 3:4(可选)')
|
|
202
287
|
.option('--sample-video <url>', '参考视频 URL(可选,展示给用户的示例片)')
|
|
203
288
|
.option('--sample-cover <url>', '封面图 URL(可选)')
|
|
@@ -237,9 +322,10 @@ program
|
|
|
237
322
|
|
|
238
323
|
program
|
|
239
324
|
.command('jobs')
|
|
240
|
-
.description('查自己名下的出图工作流(个人 login 看自己的;租户 apiKey 看业务下全部)——服务端固定返回最近 50 条,limit/status 是本地过滤')
|
|
325
|
+
.description('查自己名下的出图工作流(个人 login 看自己的;租户 apiKey 看业务下全部)——服务端固定返回最近 50 条,limit/status/project 是本地过滤')
|
|
241
326
|
.option('--limit <n>', '最多显示几条(在最近 50 条以内截取),默认 20', '20')
|
|
242
327
|
.option('--status <status>', '按状态过滤: pending / processing / done / failed(本地过滤,不是服务端查询)')
|
|
328
|
+
.option('--project <id|名>', '只看归档进该工作区的任务(本地过滤)')
|
|
243
329
|
.action(withClient((client: StudioClient, opts: any) => jobs(client, opts)))
|
|
244
330
|
|
|
245
331
|
program
|
package/src/local-bg.ts
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 本地抠图(去背景)—— remove-bg 的核心实现。
|
|
3
|
+
* 模型走 ONNX(ISNet / U2Net,均 Apache-2.0),推理走 onnxruntime-node(MIT),
|
|
4
|
+
* 前后处理走 sharp —— 整条链路许可证干净(imgly 那个 npm 包是 AGPL,不进依赖),
|
|
5
|
+
* 且三个依赖在 macOS / Windows / Linux 都有预编译,无平台特化代码。
|
|
6
|
+
* 模型文件首次使用时下载到 ~/.museav-models/ 缓存(一次性 ~170MB)。
|
|
7
|
+
*/
|
|
8
|
+
import { mkdir, writeFile, stat } from 'node:fs/promises'
|
|
9
|
+
import { join } from 'node:path'
|
|
10
|
+
import { homedir } from 'node:os'
|
|
11
|
+
|
|
12
|
+
export const BG_MODELS = {
|
|
13
|
+
isnet: {
|
|
14
|
+
file: 'isnet-general-use.onnx',
|
|
15
|
+
// rembg 官方 release 托管的同一份模型(Apache-2.0,源自 xuebinqin/DIS)
|
|
16
|
+
url: 'https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx',
|
|
17
|
+
label: 'ISNet(通用,质量优先)',
|
|
18
|
+
},
|
|
19
|
+
u2net: {
|
|
20
|
+
file: 'u2net.onnx',
|
|
21
|
+
url: 'https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net.onnx',
|
|
22
|
+
label: 'U2Net(经典通用)',
|
|
23
|
+
},
|
|
24
|
+
} as const
|
|
25
|
+
|
|
26
|
+
export type BgModelKey = keyof typeof BG_MODELS
|
|
27
|
+
|
|
28
|
+
const MODEL_DIR = join(homedir(), '.museav-models')
|
|
29
|
+
const INPUT_EDGE = 1024
|
|
30
|
+
|
|
31
|
+
function modelPath(key: BgModelKey): string {
|
|
32
|
+
return join(MODEL_DIR, BG_MODELS[key].file)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** 模型在位返回路径;不在则下载(流式,进度打 stderr)。下载失败抛 Error */
|
|
36
|
+
export async function ensureBgModel(key: BgModelKey): Promise<string> {
|
|
37
|
+
const dest = modelPath(key)
|
|
38
|
+
try {
|
|
39
|
+
const s = await stat(dest)
|
|
40
|
+
if (s.size > 10_000_000) return dest // 正常模型都是百 MB 级;太小的文件视为残缺重下
|
|
41
|
+
} catch {
|
|
42
|
+
// 不存在,走下载
|
|
43
|
+
}
|
|
44
|
+
await mkdir(MODEL_DIR, { recursive: true })
|
|
45
|
+
const def = BG_MODELS[key]
|
|
46
|
+
process.stderr.write(`↓ 首次使用,下载 ${def.label}(~170MB,一次性,缓存到 ${MODEL_DIR})...\n`)
|
|
47
|
+
const resp = await fetch(def.url)
|
|
48
|
+
if (!resp.ok || !resp.body) throw new Error(`模型下载失败 HTTP ${resp.status}:${def.url}`)
|
|
49
|
+
const total = Number(resp.headers.get('content-length') || 0)
|
|
50
|
+
const chunks: Buffer[] = []
|
|
51
|
+
let got = 0
|
|
52
|
+
const reader = resp.body.getReader()
|
|
53
|
+
for (;;) {
|
|
54
|
+
const { done, value } = await reader.read()
|
|
55
|
+
if (done) break
|
|
56
|
+
chunks.push(Buffer.from(value))
|
|
57
|
+
got += value.length
|
|
58
|
+
if (total) process.stderr.write(` ${((got / total) * 100).toFixed(0)}%\r`)
|
|
59
|
+
}
|
|
60
|
+
process.stderr.write('\n')
|
|
61
|
+
const buf = Buffer.concat(chunks)
|
|
62
|
+
if (buf.length < 10_000_000) throw new Error('模型下载不完整,请重试')
|
|
63
|
+
await writeFile(dest, buf)
|
|
64
|
+
return dest
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** 抠图主流程:输入图片路径 → 输出带 alpha 的 PNG Buffer */
|
|
68
|
+
export async function removeBackgroundLocal(inputPath: string, modelKey: BgModelKey): Promise<Buffer> {
|
|
69
|
+
// 动态加载:onnxruntime-node 是 optionalDependency,缺失时给安装指引而不是崩
|
|
70
|
+
let ort: typeof import('onnxruntime-node')
|
|
71
|
+
try {
|
|
72
|
+
ort = await import('onnxruntime-node')
|
|
73
|
+
} catch {
|
|
74
|
+
throw new Error('onnxruntime-node 不可用。重装 CLI 即可补上:npm install -g museav-cli')
|
|
75
|
+
}
|
|
76
|
+
const sharp = await loadSharpOrThrow()
|
|
77
|
+
const modelFile = await ensureBgModel(modelKey)
|
|
78
|
+
const session = await ort.InferenceSession.create(modelFile)
|
|
79
|
+
|
|
80
|
+
// ── 预处理:EXIF 转正、去 alpha、RGB raw ──
|
|
81
|
+
const { data: rgb, info } = await sharp(inputPath).rotate().removeAlpha().raw().toBuffer({ resolveWithObject: true })
|
|
82
|
+
if (info.channels !== 3) throw new Error(`预处理得到 ${info.channels} 通道(预期 3)`)
|
|
83
|
+
|
|
84
|
+
// ── 模型输入:拉伸到 1024×1024,(x/255 - 0.5)/0.5 归一化,HWC → CHW ──
|
|
85
|
+
const small = await sharp(rgb, { raw: { width: info.width, height: info.height, channels: 3 } })
|
|
86
|
+
.resize(INPUT_EDGE, INPUT_EDGE, { fit: 'fill' })
|
|
87
|
+
.raw()
|
|
88
|
+
.toBuffer()
|
|
89
|
+
const f32 = new Float32Array(3 * INPUT_EDGE * INPUT_EDGE)
|
|
90
|
+
const N = INPUT_EDGE * INPUT_EDGE
|
|
91
|
+
for (let i = 0; i < N; i++) {
|
|
92
|
+
f32[i] = (small[i * 3] / 255 - 0.5) / 0.5
|
|
93
|
+
f32[N + i] = (small[i * 3 + 1] / 255 - 0.5) / 0.5
|
|
94
|
+
f32[2 * N + i] = (small[i * 3 + 2] / 255 - 0.5) / 0.5
|
|
95
|
+
}
|
|
96
|
+
const feeds: Record<string, import('onnxruntime-node').Tensor> = {}
|
|
97
|
+
feeds[session.inputNames[0]] = new ort.Tensor('float32', f32, [1, 3, INPUT_EDGE, INPUT_EDGE])
|
|
98
|
+
const results = await session.run(feeds)
|
|
99
|
+
const out = results[session.outputNames[0]]
|
|
100
|
+
const maskFlat = out.data as Float32Array
|
|
101
|
+
if (maskFlat.length < N) throw new Error(`模型输出尺寸异常(${maskFlat.length})`)
|
|
102
|
+
|
|
103
|
+
// ── 后处理:min-max 归一化到 0-255,再缩回原图尺寸 ──
|
|
104
|
+
let lo = Infinity
|
|
105
|
+
let hi = -Infinity
|
|
106
|
+
for (let i = 0; i < N; i++) {
|
|
107
|
+
const v = maskFlat[i]
|
|
108
|
+
if (v < lo) lo = v
|
|
109
|
+
if (v > hi) hi = v
|
|
110
|
+
}
|
|
111
|
+
const range = hi - lo || 1
|
|
112
|
+
const mask8 = Buffer.alloc(N)
|
|
113
|
+
for (let i = 0; i < N; i++) mask8[i] = Math.round(((maskFlat[i] - lo) / range) * 255)
|
|
114
|
+
const maskFull = await sharp(mask8, { raw: { width: INPUT_EDGE, height: INPUT_EDGE, channels: 1 } })
|
|
115
|
+
.resize(info.width, info.height, { fit: 'fill' })
|
|
116
|
+
.raw()
|
|
117
|
+
.toBuffer()
|
|
118
|
+
|
|
119
|
+
// ── alpha 合成:直接构造 RGBA(alpha = mask),不依赖 composite 的混合语义 ──
|
|
120
|
+
const w = info.width
|
|
121
|
+
const h = info.height
|
|
122
|
+
const rgba = Buffer.alloc(w * h * 4)
|
|
123
|
+
for (let i = 0; i < w * h; i++) {
|
|
124
|
+
rgba[i * 4] = rgb[i * 3]
|
|
125
|
+
rgba[i * 4 + 1] = rgb[i * 3 + 1]
|
|
126
|
+
rgba[i * 4 + 2] = rgb[i * 3 + 2]
|
|
127
|
+
rgba[i * 4 + 3] = maskFull[i]
|
|
128
|
+
}
|
|
129
|
+
return sharp(rgba, { raw: { width: w, height: h, channels: 4 } }).png().toBuffer()
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async function loadSharpOrThrow(): Promise<any> {
|
|
133
|
+
try {
|
|
134
|
+
const m = await import('sharp')
|
|
135
|
+
const sharp = (m as any).default ?? m
|
|
136
|
+
// 造 1px 图跑通全链路:native binding 坏了在第一次真用时才炸,这里提前暴露
|
|
137
|
+
await sharp({ create: { width: 1, height: 1, channels: 3, background: '#000' } }).raw().toBuffer()
|
|
138
|
+
return sharp
|
|
139
|
+
} catch {
|
|
140
|
+
throw new Error('sharp 不可用。重装 CLI 即可补上:npm install -g museav-cli')
|
|
141
|
+
}
|
|
142
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 本地视觉模型(Ollama + qwen3-vl)—— reverse 的主路。
|
|
3
|
+
* 中台 API 逆向一次要十几秒到几十秒,本地 8b 量化模型在 Apple Silicon 上更快且零成本;
|
|
4
|
+
* API 降级为回落路(commands/reverse.ts 负责切换与提示)。
|
|
5
|
+
* 提示词与返回结构从中台 _reverse-core.js / reverse-template.js 移植,保证两条路产出同构。
|
|
6
|
+
*/
|
|
7
|
+
import { readFile } from 'node:fs/promises'
|
|
8
|
+
import { compressForVision } from './compress.js'
|
|
9
|
+
import type { ReverseResult } from './client.js'
|
|
10
|
+
|
|
11
|
+
/** 本地读图模型。换档位用 MUSEAV_LOCAL_VLM 环境变量,不用改代码 */
|
|
12
|
+
export const LOCAL_VLM_MODEL = process.env.MUSEAV_LOCAL_VLM || 'qwen3-vl:8b'
|
|
13
|
+
|
|
14
|
+
const ALLOWED_RATIOS = ['3:4', '9:16', '1:1', '4:3', '16:9']
|
|
15
|
+
|
|
16
|
+
// OLLAMA_HOST 生态里带不带 scheme、带不带尾斜杠的写法都有
|
|
17
|
+
function ollamaHost(): string {
|
|
18
|
+
let host = process.env.OLLAMA_HOST || 'http://localhost:11434'
|
|
19
|
+
if (!/^https?:\/\//.test(host)) host = `http://${host}`
|
|
20
|
+
return host.replace(/\/+$/, '')
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface LocalVlmStatus {
|
|
24
|
+
running: boolean
|
|
25
|
+
modelPresent: boolean
|
|
26
|
+
host: string
|
|
27
|
+
/** running=false 时的原因(给用户看的行动指引) */
|
|
28
|
+
reason?: string
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// 各系统启动 Ollama 的正确姿势不同,提示语跟着平台走(Windows 没有 brew)
|
|
32
|
+
function ollamaStartHint(): string {
|
|
33
|
+
if (process.platform === 'win32') return '启动 Ollama 应用(开始菜单 / Ollama.exe),或命令行运行 ollama serve'
|
|
34
|
+
if (process.platform === 'darwin') return 'brew services start ollama,或 ollama serve'
|
|
35
|
+
return 'systemctl --user start ollama,或 ollama serve'
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** 探活 + 模型在位检查。3 秒探不通就是没起服务,不等推理超时才发现 */
|
|
39
|
+
export async function checkLocalVlm(): Promise<LocalVlmStatus> {
|
|
40
|
+
const host = ollamaHost()
|
|
41
|
+
try {
|
|
42
|
+
const resp = await fetch(`${host}/api/tags`, { signal: AbortSignal.timeout(3000) })
|
|
43
|
+
if (!resp.ok) {
|
|
44
|
+
return { running: false, modelPresent: false, host, reason: `Ollama 探活返回 HTTP ${resp.status}` }
|
|
45
|
+
}
|
|
46
|
+
const tags = (await resp.json()) as { models?: Array<{ name?: string }> }
|
|
47
|
+
const names = (tags.models || []).map((m) => m.name || '')
|
|
48
|
+
if (!names.includes(LOCAL_VLM_MODEL)) {
|
|
49
|
+
return { running: true, modelPresent: false, host, reason: `模型未拉取,执行: ollama pull ${LOCAL_VLM_MODEL}` }
|
|
50
|
+
}
|
|
51
|
+
return { running: true, modelPresent: true, host }
|
|
52
|
+
} catch {
|
|
53
|
+
return { running: false, modelPresent: false, host, reason: `Ollama 未运行(${host}),${ollamaStartHint()}` }
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** SCULPT 系统提示词 —— 从中台 reverse-template.js 移植。本地路只做纯读图,
|
|
58
|
+
* 中台提示词里的 genre / body_md(给 image-to-template 用的)在 ReverseResult 里
|
|
59
|
+
* 根本不消费,本地砍掉这两项省几百个输出 token——输出长度直接决定本地推理耗时 */
|
|
60
|
+
function sculptSystemPrompt(): string {
|
|
61
|
+
return (
|
|
62
|
+
`你是一位专业的 AI 图像逆向工程师。请分析这张图片,用 SCULPT 六要素框架逆推生成该图片所需的 prompt。` +
|
|
63
|
+
`严格输出 JSON,不要输出任何其他文字:\n` +
|
|
64
|
+
`{\n` +
|
|
65
|
+
` "sculpt": {\n` +
|
|
66
|
+
` "subject": "主体描述 — 画面中的人物/物体/场景,包括外貌、姿态、服饰",\n` +
|
|
67
|
+
` "composition": "构图描述 — 视角、布局、留白、视觉引导线",\n` +
|
|
68
|
+
` "universe": "世界观 — 时代背景、艺术风格、整体氛围",\n` +
|
|
69
|
+
` "light": "光影描述 — 光源方向、色温、明暗对比、光影效果",\n` +
|
|
70
|
+
` "print": "输出特性 — 比例、色调倾向、对比度、饱和度",\n` +
|
|
71
|
+
` "texture": "质感描述 — 材质、表面纹理、细节精度"\n` +
|
|
72
|
+
` },\n` +
|
|
73
|
+
` "prompt": "整合 SCULPT 六要素后的完整英文 prompt(适合 AI 图像生成模型)",\n` +
|
|
74
|
+
` "prompt_cn": "对应中文 prompt",\n` +
|
|
75
|
+
` "style_tags": ["2-4 个关键风格标签"],\n` +
|
|
76
|
+
` "aspect_ratio": "推荐比例,从 3:4|9:16|1:1|4:3|16:9 中按图片比例选一个",\n` +
|
|
77
|
+
` "zh_name": "4-8 字风格名(供技能命名)",\n` +
|
|
78
|
+
` "description": "一句话描述该风格"` +
|
|
79
|
+
`\n}\n要求:prompt 必须是英文,详细且精确,覆盖全部六个维度;prompt_cn 为对应中文;只输出 JSON。`
|
|
80
|
+
)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** 本地逆向一张图。任何失败都抛 Error,由调用方决定回落 */
|
|
84
|
+
export async function reverseLocally(filePath: string): Promise<ReverseResult> {
|
|
85
|
+
// 复用上传同款压缩:图小不仅传得快,本地 VLM 推理也快
|
|
86
|
+
const { buffer, note } = await compressForVision(filePath)
|
|
87
|
+
if (note) process.stderr.write(` ${note}\n`)
|
|
88
|
+
const bytes = buffer ?? (await readFile(filePath))
|
|
89
|
+
const b64 = Buffer.from(bytes).toString('base64')
|
|
90
|
+
|
|
91
|
+
const payload = {
|
|
92
|
+
model: LOCAL_VLM_MODEL,
|
|
93
|
+
messages: [
|
|
94
|
+
{ role: 'system', content: sculptSystemPrompt() },
|
|
95
|
+
{ role: 'user', content: '用 SCULPT 六要素分析这张图,逆推出图 prompt', images: [b64] },
|
|
96
|
+
],
|
|
97
|
+
stream: false,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// 8b 视觉推理单张图几十秒量级,给足余量
|
|
101
|
+
const resp = await fetch(`${ollamaHost()}/api/chat`, {
|
|
102
|
+
method: 'POST',
|
|
103
|
+
headers: { 'Content-Type': 'application/json' },
|
|
104
|
+
body: JSON.stringify(payload),
|
|
105
|
+
signal: AbortSignal.timeout(5 * 60 * 1000),
|
|
106
|
+
})
|
|
107
|
+
if (!resp.ok) {
|
|
108
|
+
throw new Error(`Ollama 返回 HTTP ${resp.status}: ${(await resp.text()).slice(0, 200)}`)
|
|
109
|
+
}
|
|
110
|
+
const out = (await resp.json()) as { message?: { content?: string } }
|
|
111
|
+
const content = out.message?.content || ''
|
|
112
|
+
if (!content.trim()) throw new Error('本地模型返回空内容')
|
|
113
|
+
|
|
114
|
+
return normalizeSculpt(parseJsonLoose(content))
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** 视觉模型「只输出 JSON」的承诺不可信:剥 ```json 围栏、截首尾大括号 */
|
|
118
|
+
function parseJsonLoose(text: string): Record<string, unknown> {
|
|
119
|
+
let t = text.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '')
|
|
120
|
+
const start = t.indexOf('{')
|
|
121
|
+
const end = t.lastIndexOf('}')
|
|
122
|
+
if (start >= 0 && end > start) t = t.slice(start, end + 1)
|
|
123
|
+
return JSON.parse(t) as Record<string, unknown>
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** 与中台 basePayload 同构的归一化:超长截断、非法比例兜底 3:4 */
|
|
127
|
+
function normalizeSculpt(parsed: Record<string, unknown>): ReverseResult {
|
|
128
|
+
const sculptIn = (parsed.sculpt || {}) as Record<string, unknown>
|
|
129
|
+
const sculpt: Record<string, string> = {}
|
|
130
|
+
for (const key of ['subject', 'composition', 'universe', 'light', 'print', 'texture']) {
|
|
131
|
+
sculpt[key] = String(sculptIn[key] || '').slice(0, 500)
|
|
132
|
+
}
|
|
133
|
+
const ratio = ALLOWED_RATIOS.includes(parsed.aspect_ratio as string)
|
|
134
|
+
? (parsed.aspect_ratio as string)
|
|
135
|
+
: ALLOWED_RATIOS.includes(parsed.ratio as string)
|
|
136
|
+
? (parsed.ratio as string)
|
|
137
|
+
: '3:4'
|
|
138
|
+
return {
|
|
139
|
+
ok: true,
|
|
140
|
+
sculpt,
|
|
141
|
+
prompt: String(parsed.prompt || '').slice(0, 2000),
|
|
142
|
+
prompt_cn: String(parsed.prompt_cn || '').slice(0, 2000),
|
|
143
|
+
style_tags: Array.isArray(parsed.style_tags)
|
|
144
|
+
? (parsed.style_tags as unknown[]).slice(0, 6).map((t) => String(t).slice(0, 30))
|
|
145
|
+
: [],
|
|
146
|
+
aspect_ratio: ratio,
|
|
147
|
+
zh_name: String(parsed.zh_name || '裂变风格').slice(0, 24),
|
|
148
|
+
description: String(parsed.description || '').slice(0, 200),
|
|
149
|
+
}
|
|
150
|
+
}
|