@yangzhenghan/dsh-bluewhale 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +47 -3
- package/assets/silero-vad.onnx +0 -0
- package/cordis.patch.yml +8 -0
- package/icon.png +0 -0
- package/index.js +121 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +16 -4
- package/src/clean-text.js +60 -0
- package/src/cookie-cleaner.js +92 -0
- package/src/dsh.js +102 -0
- package/src/followups.js +68 -0
- package/src/helpers.js +36 -0
- package/src/idle-motion.js +13 -0
- package/src/outfit.js +32 -0
- package/src/pet-download.js +153 -0
- package/src/pet-rules.js +24 -0
- package/src/pet.js +48 -0
- package/src/route.js +43 -0
- package/src/server.js +207 -0
- package/src/sessions.js +50 -0
- package/src/state.js +43 -0
- package/src/vad-mirror.js +97 -0
- package/src/vision.js +82 -0
- package//346/241/214/345/256/240/350/247/204/345/210/231.md +4 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// 国内能用的语音检测模型
|
|
2
|
+
// 桌宠听你说话前,要先用 Silero 小模型判断「有没有人在说话」,这个模型原本从 huggingface.co 下载,国内连不上,
|
|
3
|
+
// 听力就会卡在「Loading…」或报「Failed to fetch」「Audio detector fell behind」。
|
|
4
|
+
// 这里做两件事:
|
|
5
|
+
// 1. 插件自带模型文件(assets/silero-vad.onnx),在本机 127.0.0.1:18101 提供下载;其它文件转回 huggingface.co
|
|
6
|
+
// 2. 把桌宠程序里的下载地址 https://huggingface.co/ 换成 http://127.0.0.1:18101/(两者都是 23 个字符,原位替换)
|
|
7
|
+
// 改回去:把 app.asar 里的 http://127.0.0.1:18101/ 换回 https://huggingface.co/ 即可
|
|
8
|
+
import fs from 'node:fs'
|
|
9
|
+
import http from 'node:http'
|
|
10
|
+
import path from 'node:path'
|
|
11
|
+
import { fileURLToPath } from 'node:url'
|
|
12
|
+
import { log } from './state.js'
|
|
13
|
+
|
|
14
|
+
export const VAD_PORT = 18101
|
|
15
|
+
const OLD = Buffer.from('remoteHost: "https://huggingface.co/"')
|
|
16
|
+
const NEW = Buffer.from(`remoteHost: "http://127.0.0.1:${VAD_PORT}/"`)
|
|
17
|
+
const MODEL_FILE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'assets', 'silero-vad.onnx')
|
|
18
|
+
const MODEL_PATH = '/onnx-community/silero-vad/resolve/main/onnx/model.onnx'
|
|
19
|
+
|
|
20
|
+
if (OLD.length !== NEW.length) throw new Error('vad-mirror:替换前后长度不一致')
|
|
21
|
+
|
|
22
|
+
const CORS = {
|
|
23
|
+
'Access-Control-Allow-Origin': '*',
|
|
24
|
+
'Access-Control-Allow-Headers': '*',
|
|
25
|
+
'Access-Control-Allow-Private-Network': 'true',
|
|
26
|
+
'Access-Control-Expose-Headers': 'Content-Length, ETag',
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// 本机的小下载站:只认语音检测模型,其它请求转回 huggingface.co(和原来一样)
|
|
30
|
+
export function startModelServer() {
|
|
31
|
+
const server = http.createServer((req, res) => {
|
|
32
|
+
const url = new URL(req.url, 'http://127.0.0.1')
|
|
33
|
+
if (req.method === 'OPTIONS') {
|
|
34
|
+
res.writeHead(204, CORS)
|
|
35
|
+
return res.end()
|
|
36
|
+
}
|
|
37
|
+
if (url.pathname === MODEL_PATH && fs.existsSync(MODEL_FILE)) {
|
|
38
|
+
const size = fs.statSync(MODEL_FILE).size
|
|
39
|
+
res.writeHead(200, { ...CORS, 'Content-Type': 'application/octet-stream', 'Content-Length': size, ETag: '"silero-vad-bundled"' })
|
|
40
|
+
if (req.method === 'HEAD') return res.end()
|
|
41
|
+
return fs.createReadStream(MODEL_FILE).pipe(res)
|
|
42
|
+
}
|
|
43
|
+
res.writeHead(302, { ...CORS, Location: 'https://huggingface.co' + url.pathname + url.search })
|
|
44
|
+
res.end()
|
|
45
|
+
})
|
|
46
|
+
server.on('error', (e) => log('✗ 语音检测模型服务没启动(端口 ' + VAD_PORT + '):', e.message))
|
|
47
|
+
server.listen(VAD_PORT, '127.0.0.1', () => log('语音检测模型服务已启动:http://127.0.0.1:' + VAD_PORT))
|
|
48
|
+
return {
|
|
49
|
+
stop: () => new Promise((resolve) => server.close(() => resolve())),
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// 在 app.asar 里找出所有旧地址(分块读,不把 500MB 一次读进内存)
|
|
54
|
+
function findAll(fd, size, needle) {
|
|
55
|
+
const CHUNK = 32 * 1024 * 1024
|
|
56
|
+
const buf = Buffer.alloc(CHUNK + needle.length)
|
|
57
|
+
const hits = []
|
|
58
|
+
for (let pos = 0; pos < size; pos += CHUNK) {
|
|
59
|
+
const n = fs.readSync(fd, buf, 0, Math.min(CHUNK + needle.length - 1, size - pos), pos)
|
|
60
|
+
const view = buf.subarray(0, n)
|
|
61
|
+
for (let i = view.indexOf(needle); i !== -1; i = view.indexOf(needle, i + 1)) {
|
|
62
|
+
if (i < CHUNK) hits.push(pos + i)
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return hits
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// 把桌宠程序里的模型下载地址换成本机的;已经换过就什么都不做
|
|
69
|
+
export function patchPetForChina(airiExe) {
|
|
70
|
+
const asar = path.join(path.dirname(airiExe), 'resources', 'app.asar')
|
|
71
|
+
if (!fs.existsSync(asar)) return log('找不到桌宠的 app.asar,跳过语音检测模型修补:', asar)
|
|
72
|
+
// 检查过的 app.asar 记下大小和修改时间,没变就不再扫一遍 500MB
|
|
73
|
+
const mark = asar + '.vad-checked'
|
|
74
|
+
const stamp = (st) => `${st.size}:${Math.floor(st.mtimeMs)}`
|
|
75
|
+
try {
|
|
76
|
+
if (fs.readFileSync(mark, 'utf8') === stamp(fs.statSync(asar))) return
|
|
77
|
+
} catch {}
|
|
78
|
+
let fd
|
|
79
|
+
try {
|
|
80
|
+
fd = fs.openSync(asar, 'r+')
|
|
81
|
+
const size = fs.fstatSync(fd).size
|
|
82
|
+
const hits = findAll(fd, size, OLD)
|
|
83
|
+
if (hits.length) {
|
|
84
|
+
for (const at of hits) fs.writeSync(fd, NEW, 0, NEW.length, at)
|
|
85
|
+
log(`已把桌宠的语音检测模型地址换成本机(${hits.length} 处)`)
|
|
86
|
+
} else {
|
|
87
|
+
log(findAll(fd, size, NEW).length ? '桌宠的语音检测模型地址已经是本机的了' : '桌宠程序里没找到模型下载地址,跳过修补')
|
|
88
|
+
}
|
|
89
|
+
fs.closeSync(fd)
|
|
90
|
+
fd = undefined
|
|
91
|
+
fs.writeFileSync(mark, stamp(fs.statSync(asar)))
|
|
92
|
+
} catch (e) {
|
|
93
|
+
log('✗ 修补桌宠的语音检测模型地址失败(桌宠开着时改不了,关掉桌宠后重启 DSH 会再试):', e.message)
|
|
94
|
+
} finally {
|
|
95
|
+
if (fd !== undefined) fs.closeSync(fd)
|
|
96
|
+
}
|
|
97
|
+
}
|
package/src/vision.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// 桌宠的「视觉」:AIRI 截屏或收到图片时,请视觉模型写一段文字描述,再把描述放进聊天给角色参考
|
|
2
|
+
// 这里用 DSH 自己的模型看图(ctx.llm),不经过角色的会话,所以截图不会进聊天记录
|
|
3
|
+
// 图片先交给 ctx.attachments 存成附件,再作为 image 块发给模型
|
|
4
|
+
import { log } from './state.js'
|
|
5
|
+
|
|
6
|
+
// OpenAI 格式里带 image_url 的就是看图请求
|
|
7
|
+
export function isVisionRequest(body) {
|
|
8
|
+
return (body?.messages || []).some((m) => Array.isArray(m.content) && m.content.some((p) => p?.type === 'image_url'))
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function textOf(content) {
|
|
12
|
+
if (typeof content === 'string') return content
|
|
13
|
+
return (content || []).filter((p) => p?.type === 'text').map((p) => p.text).join('\n')
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// data:image/png;base64,xxxx → { mediaType, data }
|
|
17
|
+
function parseDataUrl(url) {
|
|
18
|
+
const m = /^data:([^;,]+)(;base64)?,(.*)$/s.exec(String(url || ''))
|
|
19
|
+
if (!m) return null
|
|
20
|
+
const data = m[2] ? m[3] : Buffer.from(decodeURIComponent(m[3])).toString('base64')
|
|
21
|
+
return { mediaType: m[1] === 'image/jpg' ? 'image/jpeg' : m[1], data }
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// 要试的模型路由:填了 visionProvider 就只用它;没填就按 DSH 的顺序挨个试,上次成功的排最前
|
|
25
|
+
let lastGood = ''
|
|
26
|
+
function providersToTry(llm, cfg) {
|
|
27
|
+
if (cfg.visionProvider) return [cfg.visionProvider]
|
|
28
|
+
const ids = (llm.listProviders?.() || []).map((p) => (typeof p === 'string' ? p : p?.id)).filter(Boolean)
|
|
29
|
+
return lastGood && ids.includes(lastGood) ? [lastGood, ...ids.filter((id) => id !== lastGood)] : ids
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// 跑一次模型,返回文字;没有文字时把 finish 分片里的原因记下来
|
|
33
|
+
async function runOnce(llm, provider, model, content) {
|
|
34
|
+
let text = ''
|
|
35
|
+
let finish
|
|
36
|
+
for await (const chunk of llm.stream({ provider, model, messages: [{ role: 'user', content }] }, AbortSignal.timeout(60000))) {
|
|
37
|
+
if (chunk.type === 'text-delta') text += chunk.text
|
|
38
|
+
else if (chunk.type === 'finish' || chunk.type === 'error') finish = chunk
|
|
39
|
+
}
|
|
40
|
+
if (!text.trim()) throw new Error(`${provider} 没有返回文字:${JSON.stringify(finish ?? null).slice(0, 300)}`)
|
|
41
|
+
return text.trim()
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function describeImages(ctx, cfg, body) {
|
|
45
|
+
const llm = ctx.get('llm')
|
|
46
|
+
const attachments = ctx.get('attachments')
|
|
47
|
+
if (!llm || !attachments) throw new Error('DSH 里找不到 llm 或 attachments 服务')
|
|
48
|
+
|
|
49
|
+
// system 提示词和用户文字放在最前面,图片按原顺序跟在后面
|
|
50
|
+
const parts = []
|
|
51
|
+
const system = (body.messages || []).filter((m) => m.role === 'system').map((m) => textOf(m.content)).join('\n').trim()
|
|
52
|
+
if (system) parts.push({ type: 'text', text: system })
|
|
53
|
+
for (const m of body.messages || []) {
|
|
54
|
+
if (m.role !== 'user') continue
|
|
55
|
+
if (typeof m.content === 'string') {
|
|
56
|
+
if (m.content.trim()) parts.push({ type: 'text', text: m.content })
|
|
57
|
+
continue
|
|
58
|
+
}
|
|
59
|
+
for (const p of m.content || []) {
|
|
60
|
+
if (p?.type === 'text' && p.text?.trim()) parts.push({ type: 'text', text: p.text })
|
|
61
|
+
else if (p?.type === 'image_url') {
|
|
62
|
+
const image = parseDataUrl(p.image_url?.url ?? p.image_url)
|
|
63
|
+
if (image) parts.push({ type: 'image', ...image })
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (!parts.some((p) => p.type === 'image')) throw new Error('看图请求里没有能用的图片(只支持 data: 开头的图片)')
|
|
68
|
+
|
|
69
|
+
const content = await attachments.admitPromptContent(parts)
|
|
70
|
+
const errors = []
|
|
71
|
+
for (const provider of providersToTry(llm, cfg)) {
|
|
72
|
+
try {
|
|
73
|
+
const text = await runOnce(llm, provider, cfg.visionModel, content)
|
|
74
|
+
if (lastGood !== provider) log(`看图用的模型路由:${provider}`)
|
|
75
|
+
lastGood = provider
|
|
76
|
+
return text
|
|
77
|
+
} catch (e) {
|
|
78
|
+
errors.push(`${provider}:${e.message}`)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
throw new Error('所有模型路由都看不了图\n' + errors.join('\n'))
|
|
82
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
这句话是对方在桌宠面前当面说的,你的回复会被念出来:用口语短句,不用 markdown、emoji 和网址。
|
|
2
|
+
把这条回复最鲜明的情绪写成一个动作,放在那句话前面;最多一个,平淡时不写。写法 <|ACT {"motion":"Animation/情绪-开心.motion3.json"}|>,「开心」可换成:害羞、心动、难过、生气、惊讶、尴尬、调皮、温柔。
|
|
3
|
+
对方让你换装才写 <|ACT {"expression":{"name":"长裙"}}|>,名字可选:大号、中号、小号、长裙、短裙、脱鞋、穿鞋、尾巴开关、放出尾巴、胸口衣服、换回胸口衣服;全部换回写 <|ACT {"expression":null}|>。
|
|
4
|
+
标记照抄,不解释、不提标记。
|