@aiti-ai/aiti-ai-input 0.0.0-stage → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +28 -0
- package/LICENSE +21 -0
- package/README.md +190 -2
- package/asr-providers.test.mjs +133 -0
- package/asr-providers.ts +575 -0
- package/asr-usage.test.mjs +102 -0
- package/asr-usage.ts +186 -0
- package/check-client.mjs +273 -0
- package/client/src/asr-panel.tsx +935 -0
- package/client/src/components.tsx +500 -0
- package/client/src/entry.tsx +171 -0
- package/client/src/host-shim.ts +97 -0
- package/client/src/it-terms.js +107 -0
- package/client/src/jsx-runtime.ts +2 -0
- package/client/src/logic.ts +1367 -0
- package/client/src/usage-report.tsx +276 -0
- package/client/src/voice-l2.ts +891 -0
- package/deploy-commit.mjs +52 -0
- package/dev-pack.mjs +246 -0
- package/dist/aiti-ai-input-client.mjs +14 -0
- package/dist/asr-panel-BBYVQ9DA.js +985 -0
- package/dist/entry-DEHZ7Pxd.js +2222 -0
- package/dist/ext-float-dock-Cc85nz4q.js +515 -0
- package/dist/usage-report-CxUjd6Gz.js +265 -0
- package/dist/voice-l2-CbtgVsNv.js +628 -0
- package/ext-icons-deploy.mjs +24 -0
- package/icon.svg +1 -0
- package/icons/mic.svg +1 -0
- package/index.ts +1534 -0
- package/package.json +56 -4
- package/reload-instances.mjs +60 -0
- package/server/asr_common.py +576 -0
- package/server/glm/glm_asr.py +80 -0
- package/server/glm/glm_asr_service.py +356 -0
- package/server/glm/glm_server.py +138 -0
- package/server/glm/setup-glm-venv.sh +38 -0
- package/server/requirements.txt +9 -0
- package/server/start-stream.ps1 +127 -0
- package/server/start-stream.sh +167 -0
- package/server/test-voice.html +74 -0
- package/server/test_asr_stream.py +62 -0
- package/server/test_ws_proxy.py +56 -0
- package/server/tests/report-17b.md +17 -0
- package/server/tests/report-17b.md.json +91 -0
- package/server/tests/report-full.md +277 -0
- package/server/tests/report-full.md.json +842 -0
- package/server/tests/report-permodel.md.nano.json +39 -0
- package/server/tests/report-permodel.md.nano.json.json +912 -0
- package/server/tests/report-permodel.md.qwen3-0.6b.json +39 -0
- package/server/tests/report-permodel.md.qwen3-0.6b.json.json +912 -0
- package/server/tests/report-permodel.md.qwen3-1.7b.json +39 -0
- package/server/tests/report-permodel.md.qwen3-1.7b.json.json +912 -0
- package/server/tests/report-permodel.md.sensevoice.json +39 -0
- package/server/tests/report-permodel.md.sensevoice.json.json +912 -0
- package/server/tests/report-smoke.md +11 -0
- package/server/tests/report-smoke.md.json +52 -0
- package/server/tests/run_model_matrix.py +143 -0
- package/server/tests/test_asr_models.py +285 -0
- package/server/tests/test_vad_ws_smoke.py +200 -0
- package/server/voice-asr-stream.py +893 -0
- package/skills/asr-gpu/SKILL.md +184 -0
- package/skills/asr-server/SKILL.md +92 -0
- package/skills/index.json +4 -0
- package/tools/build-client.mjs +28 -0
- package/tools/deploy.mjs +125 -0
- package/vite.config.ts +43 -0
- package/xfyun-bridge.ts +292 -0
package/AGENTS.md
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# aiti-ai-input 扩展(部署目录直改指南)
|
|
2
|
+
|
|
3
|
+
## 结构
|
|
4
|
+
- 运行时文件平铺在本目录根(index.ts / core.cjs …,宿主 loader 按相对路径加载——**禁手改运行时平铺文件**,改 src/ 再 build)
|
|
5
|
+
- `src/` 源码(client/src + 服务端 ts/cjs)· `dist/` 构建产物(禁手改)· `tools/` 构建/部署脚本
|
|
6
|
+
- npm 包形态的扩展:本目录 package.json 含 version 与 aitiConfigs 声明
|
|
7
|
+
|
|
8
|
+
## 小改现场流程(改文案/样式/小逻辑)
|
|
9
|
+
1. 改 `src/` 下对应源码
|
|
10
|
+
2. `node tools/build.mjs`(client 有构建产物的扩展;纯服务端改动跳过)
|
|
11
|
+
3. `curl -X POST http://127.0.0.1:3001/api/extensions/reload`(dev;prod 3002;外部经 nginx 带 basePath:/dev、/mac、/mdev、/win、/wdev → `/xxx/api/extensions/reload`)
|
|
12
|
+
4. 浏览器刷新页面验证
|
|
13
|
+
|
|
14
|
+
## 大改回源码仓
|
|
15
|
+
- 源码仓:`extensions/aiti-ai-input/`(git 管理,deploy 同步回本目录)
|
|
16
|
+
- 改完在源码仓跑 `node tools/deploy.mjs`(或仓库根 `node scripts/deploy-all.mjs --only aiti-ai-input`)
|
|
17
|
+
|
|
18
|
+
## 数据目录(数据分层:config / cache)
|
|
19
|
+
- `ctx.configDir` 根级 = 配置与关键小状态(aiti-configs 备份范围;小、珍贵、人可读)
|
|
20
|
+
- `ctx.cacheDir`(= `<configDir>/cache/`)= 可删缓存:db/venv/日志/大文件;rm -rf 后扩展下次访问自动重建
|
|
21
|
+
- `ctx.dataDir` 已移除(访问即抛)——新代码禁用
|
|
22
|
+
- 本扩展白名单(永不为 cache 的配置文件)见 package.json `aitiConfigs.configFiles`
|
|
23
|
+
|
|
24
|
+
## 红线
|
|
25
|
+
- 禁轮询(setInterval 拉取);需要等待用事件/回调/SSE(桥的 SSE、aiti-event-bus topics)
|
|
26
|
+
- 禁重启生产服务(pi-web 3002 等 restart 须用户明确同意)
|
|
27
|
+
- 禁手改 dist/ 与运行时平铺文件;跨扩展通信走 ctx.services(禁 globalThis.__ext 直穿)
|
|
28
|
+
- 服务端:SQLite 一律 WAL + 分批小事务(批间 setImmediate 让步);禁 execFileSync 调 >1s 外部程序;>8MB 文件禁同步读
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 aiti-ai contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,191 @@
|
|
|
1
|
-
#
|
|
1
|
+
# AI input(aiti-ai-input)
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## 来源 · Origin
|
|
4
|
+
|
|
5
|
+
本包是 **aiti** 生态的组成部分——以 [pi](https://pi.dev) 为引擎的个人 AI 开发工作台与本地调试注入生态:**aiti-agent**(自托管 Web 工作台,基于开源项目 [pi-web](https://github.com/aiti-ai/pi-web) 构建,浏览器为主操作面)+ 各开发工具内的调试注入(aiti-devtools)+ 丰富的扩展体系。
|
|
6
|
+
|
|
7
|
+
This package is part of the **aiti** ecosystem — a personal AI development workbench powered by [pi](https://pi.dev): **aiti-agent** (a self-hosted web workbench built on the open-source [pi-web](https://github.com/aiti-ai/pi-web)) plus debug injection into mainstream dev tools (aiti-devtools) and a rich extension system.
|
|
8
|
+
|
|
9
|
+
**English**: Input enhancement for the aiti-agent chat box: L0/L1 autocomplete (local prefix match + AI next sentence), favorites suggest with one-click run, streaming ASR voice input with pluggable engines / hotwords / 2-pass refinement, and dictation-detection repair.
|
|
10
|
+
|
|
11
|
+
- 工作台 / workbench: <https://github.com/aiti-ai/pi-web>
|
|
12
|
+
- 生态仓库 / monorepo: <https://github.com/aiti-ai/aiti-ai>
|
|
13
|
+
- 官网 / site: <https://aiti.pw>
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
AI 聊天输入框的输入增强扩展(**长期保留,非临时测试**)。
|
|
17
|
+
|
|
18
|
+
## 当前能力(语音来源标记 + 自动补全 + 收藏联想 + L2 ASR + 热词系统)
|
|
19
|
+
|
|
20
|
+
### 自动补全(L0 本地秒出 → L1 AI)
|
|
21
|
+
|
|
22
|
+
打字静止 700ms 且光标在末尾 → 灰字显示下一句(Tab/→ 接受,Esc/继续打字取消)。**两层**:
|
|
23
|
+
|
|
24
|
+
- **L0 本地层(零模型秒出)**:输入静止后**先查本地**——收藏 prompt + 会话历史输入,服务端
|
|
25
|
+
`POST /complete` 的 `mode=local` 做**前缀匹配**(输入尾段最长后缀 vs 候选 name/text 前缀)。
|
|
26
|
+
命中即灰字直出 + **↦收藏/↦历史 角标**,**不请求 AI**——本地先上,不等 AI 结果一起。
|
|
27
|
+
- **L1 AI 层**:仅 L0 未命中才请求同款模型小请求(0.4–2s 后出灰字,在途时灰字先空着)。
|
|
28
|
+
- 治理走 extman 功能管理:`autocomplete.enabled` 总开关 · `autocomplete.localOnly`(只跑 L0,0 出网)·
|
|
29
|
+
`autocomplete.minChars` 最小字符数(默认 3)· ctx 三档上下文预算。
|
|
30
|
+
|
|
31
|
+
### 语音输入(系统输入法听写检测 + L2 ASR)
|
|
32
|
+
|
|
33
|
+
1. **检测语音输入与高速输入**(系统输入法听写 / 连续快速粘贴式整句上屏)——
|
|
34
|
+
纯浏览器事件特征(`beforeinput` inputType + `getTargetRanges` + 节奏),零服务端依赖;
|
|
35
|
+
2. 检测到 → 输入框上方(`chatInputTop`)显示「🎤 识别到快速大量输入 · 正在帮你检查修复转写错误」标记条,**用户可 ✕ 删除**;
|
|
36
|
+
3. 保留 → 发送/上屏时对输入做**拼写修复**(语音或高速输入产生的错字/音译词/重复词,AI 修复后仅当有歧义才给替换建议);
|
|
37
|
+
4. **L2 ASR 保底路径**:浏览器录音(webm/opus,≤60s)→ `/voice/transcribe` → 服务端转写 → 文本回填;
|
|
38
|
+
5. **实时流式识别 + 候选词 + 2-pass 精修(🎤 主路径,v0.4.0 起)**:按住 🎤 说话 → 浏览器实时采 PCM 经
|
|
39
|
+
WebSocket 发 `voice-asr-stream.py`(paraformer-zh-streaming 流式,600ms 级低延迟)。**实时出字默认关**
|
|
40
|
+
(`asr.streaming=false`——CPU 流式模型质量差)+ **2-pass 精修无延迟上屏**(`PASS2_DELAY_MS=0`):
|
|
41
|
+
录音结束即调离线 2-pass 模型(**Qwen3-ASR-0.6B / Fun-ASR-Nano / seaco** 可选,弹窗下拉切换)精修后
|
|
42
|
+
立即写入输入框,不等 0.6s;兑底链:WS 不可用 → 整段 webm 直发 8767 Qwen3-ASR(voice-asr2,同样 GPU 加速)。
|
|
43
|
+
- **上屏格式**:录入结束自动补句号(随界面语言——中文 `。` / 英文等 `.`,已带句末标点不重复);不自动换行,**在光标处插入**(有选区则替换选区),插入完光标落到新文本后面;未聚焦且无可靠光标位时退到文末追加。
|
|
44
|
+
- 治理:extman features `asr.streaming`(默认关)· `asr.pass2`(默认开)· `asr.model.stream` · `asr.model.offline`。
|
|
45
|
+
- **入口总闸(顶级,不分组)**:`voice.enabled`(🎤 语音输入按钮是否挂载,未测好默认关)·
|
|
46
|
+
`toolbar.enabled`(顶部工具条 🎙️ 管理弹窗入口,默认关;旧 `asr.hotwords.panel` 已改名并自动迁移)。
|
|
47
|
+
|
|
48
|
+
### 语音识别引擎可切换(v0.5.0+ · 自建 8767 只是其中一种)
|
|
49
|
+
|
|
50
|
+
引擎层 `asr-providers.ts`(`docs/aiti-ai-input-asr-provider-plan.md`)四引擎同为一等公民:
|
|
51
|
+
|
|
52
|
+
| 引擎 id | 说明 | 实测时延(4s 中文) | 状态 |
|
|
53
|
+
|---|---|---|---|
|
|
54
|
+
| `builtin` | 自建 8767(FunASR/Qwen3/GLM):真流式 + 模型级热词 + 2-pass | 本机 GPU 秒级 | ✅ 有本地算力时最优 |
|
|
55
|
+
| `xfyun` | 讯飞听写 iat —— **真流式**(客户端 WS → 本地桥 `xfyun-bridge.mjs`:8768 → 讯飞 WSS;松手走一次性提交兜底) | **0.3–2.6s**(流式 5.0s 出首句) | ✅ **当前默认** |
|
|
56
|
+
| `baidu` | 百度短语音识别 REST(token 30 天缓存) | 0.3–0.7s | ✅ 已可用(曾报 3305 缺额度,现已解除) |
|
|
57
|
+
| `siliconflow` | 硅基流动 SenseVoiceSmall | 27–82s | ❌ 排队太慢,仅兜底 |
|
|
58
|
+
| `openai` | 任意 OpenAI 兼容端点(自建/代理/其它云,填 `baseUrl`+`model`,自动补 `/audio/transcriptions`) | 取决于端点 | ✅ 已实施 |
|
|
59
|
+
|
|
60
|
+
云引擎下 WS/热词/启停自动门控(`/asr/control` 拒绝、热词面板显示「当前引擎不支持」),客户端链路不变
|
|
61
|
+
(本地 VAD 切句 → `/voice-quick` 逐句 → `/voice-asr2` 精修)。兑底音源已改 WAV(云引擎不吃 webm)。
|
|
62
|
+
|
|
63
|
+
- **语音流量统计报表**(`client/src/usage-report.tsx`,形态对齐 aiti-ai-usage:零依赖手写 SVG)
|
|
64
|
+
- 入口:面板用量卡里「📊 详细报表」按钮 · 左栏 dock 区块「语音流量」· 右栏 tab · extman 的 `aiti-asr-usage-report` 弹窗
|
|
65
|
+
- 内容:**KPI 卡**(今日 / 近 7 天 / 累计的 请求数·音频分钟·估算费用)· **近 30 天堆叠柱**(按引擎分色、图例可切显隐、悬浮显数值)·
|
|
66
|
+
**按引擎明细**(累计/今日/错误 + 单价元/分钟 + 估算费用)· **按链路模式**(quick 分句快识 / final 整段精修 / vad 真流式会话)
|
|
67
|
+
- 数据:`GET /asr/usage/series?days=30` 一次拉全(不分块多请求);**本地日**口径(与 aiti-ai-usage 一致),
|
|
68
|
+
只保留 90 天;存 `stateDir/asr-usage.json`(不可再生 → state 层)
|
|
69
|
+
- 单测:`asr-usage.test.mjs`(6 例)——本地日/模式聚合/旧格式迁移/90 天滚动/单价/reset
|
|
70
|
+
|
|
71
|
+
**接入方式**
|
|
72
|
+
- **Groq**:https://console.groq.com 注册 → API Keys → Create(`gsk_...`)→ 用 `openai` 引擎填
|
|
73
|
+
端点 `https://api.groq.com/openai/v1` + 该 key + 模型 `whisper-large-v3-turbo`(免费额度,无需改代码)。
|
|
74
|
+
- **Azure**:https://portal.azure.com 建 **Speech** 资源(Free F0,5 小时/月;区域 eastasia/southeastasia)
|
|
75
|
+
→ 「密钥和终结点」取 **密钥 1** 与 **区域** → 用 `azure` 引擎填密钥/区域/`zh-CN`。
|
|
76
|
+
⚠ 本机实测 `*.stc.speech.microsoft.com` **连不通**(curl 返回 000),需能直连 Azure 的网络环境。
|
|
77
|
+
|
|
78
|
+
> **e2e 实测**:五引擎逐个跑真音频(日常句 + 中英 IT 术语混说 × quick/asr2 两入口)**全部真识别出文本**,
|
|
79
|
+
> 逐条结果见 `docs/aiti-ai-input-asr-provider-plan.md` §十。
|
|
80
|
+
> **讯飞真流式**:宿主扩展不能自己终止 WebSocket(只有 middleware rewrite),所以扩展在**自己进程内**
|
|
81
|
+
> 起一个 WS 服务(`xfyun-bridge.ts`,`node:http` + upgrade,零依赖),监听 127.0.0.1:8768,
|
|
82
|
+
> middleware 把 `/aiti-ai-input/asr/ws/vad` rewrite 过去;桥只做 HMAC 签名 + PCM 转发,不做推理、不起子进程。
|
|
83
|
+
> `POST /asr/ensure` 幂等(已在跑就复用),server 挂 `globalThis.__aitiXfBridge` → 扩展 reload 后复用旧监听器;
|
|
84
|
+
> 凭据每条会话现读,改 key 即时生效。协议与自建 8767 的 /ws/vad 一致 → 客户端零改动。
|
|
85
|
+
> **端口随机**(`listen(0)`):每个 pi-web 实例各起各的桥,同机 dev/prod 不可能抢端口(早期固定 8768
|
|
86
|
+
> 导致第二个实例报「端口被占用」已修);实际端口可从 `/asr/ensure` 或 `/asr/info` 的 `bridgePort` 看到,
|
|
87
|
+
> 调试要固定端口就设 env `AITI_XF_BRIDGE_PORT`。
|
|
88
|
+
|
|
89
|
+
> builtin 必须走扩展自愈/`/asr/control` 启动(带 `AITI_ASR_VENV_DIR=cache/asr-venv`);
|
|
90
|
+
> 手工 `bash start-stream.sh` 会落到源码目录 `.venv`(缺 `qwen-asr` → 2-pass 加载失败)。
|
|
91
|
+
|
|
92
|
+
配置(M2 管理面板落地前走接口;写盘即时生效,无需重启):
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
# 读(key 脱敏:只回 configured + 尾 4 位)
|
|
96
|
+
curl -s http://127.0.0.1:3001/dev/api/extensions/aiti-ai-input/asr/config
|
|
97
|
+
# 写:配 key 并切引擎(空串 = 删除该项)
|
|
98
|
+
curl -s -X POST http://127.0.0.1:3001/dev/api/extensions/aiti-ai-input/asr/config \
|
|
99
|
+
-H 'content-type: application/json' \
|
|
100
|
+
-d '{"provider":"siliconflow","providers":{"siliconflow":{"apiKey":"sk-...","model":"FunAudioLLM/SenseVoiceSmall"}}}'
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### 热词系统(v0.4.2+)
|
|
104
|
+
|
|
105
|
+
- **自动热词**:session-manager 从 All Sessions 索引提取(关注路径中文/英文 + 全局中文/英文,**四档 ×
|
|
106
|
+
`asr.hotwords.count`(默认 100,每档数量,总上限约 400)**,按序去重)——**用户热词最优先** → 关注路径 →
|
|
107
|
+
全局;异步分块提取(每 4000 条让出事件循环,不阻塞请求);时间加权 + 10min 缓存。
|
|
108
|
+
- **workspace 粒度**:8767 服务按 **workspace 哈希 key**(cwd/关注路径 sha1 前 12 位)分套存储热词——
|
|
109
|
+
多 workspace 并行互不冲突;2-pass 模型级热词:**Qwen3 走 `context=`**、**Fun-ASR-Nano/seaco 走 `hotword=`**
|
|
110
|
+
(非文本级替换——后者有 pypinyin 空串 bug 已弃用);全链不见文件路径(cwd 走 header/body,日志干净)。
|
|
111
|
+
- **用户热词**:`GET/PUT /api/extensions/aiti-ai-input/asr/user-hotwords`(上限 500,用户词与自动词合并、最优先)。
|
|
112
|
+
- **管理弹窗「语音识别引擎管理」**(工具条 🎙️,顶级开关 `toolbar.enabled` 默认关——extman 打开后出现;
|
|
113
|
+
旧 id `asr.hotwords.panel` 已退役,startup 自动迁移旧设置)——**三段式**:
|
|
114
|
+
1. **引擎切换**:四引擎卡片(label + 能力标签「真流式/热词/2-pass」+ 是否已配置),点即切(`POST /asr/config {provider}`);
|
|
115
|
+
2. **引擎配置**:当前引擎的参数表单(字段由后端 `fields` 元信息驱动,secret 只回显尾 4 位、留空=不改)
|
|
116
|
+
+ **🩺 测试连接**(`POST /asr/test`——服务端发 1s 静音音频走一遍引擎,不说话也能验 key/网络/耗时);
|
|
117
|
+
builtin 时此处显示 8767 状态/启停(`POST /asr/control`)/模型下拉/热词统计/用户热词编辑·导入·导出
|
|
118
|
+
(`GET /asr/effective-hotwords`)——热词与用户热词卡在云引擎下标「不生效」;
|
|
119
|
+
3. **用量统计**:按天记录每引擎的 **请求数 / 音频时长 / 错误数 / 最近使用**(报表见下),可填「单价(元/分钟)」自动估算费用,
|
|
120
|
+
可清空(`GET|POST /asr/usage`;数据落 `stateDir/asr-usage.json`,属不可再生数据)。
|
|
121
|
+
口径:`voice-quick`/`voice-asr2` 每次请求计 1 次;真流式会话结束计 1 次;时长按 16k/16bit 折算(估算)
|
|
122
|
+
4. **通用配置**:与引擎无关的链路开关(`asr.streaming`/`asr.vad`/`asr.split`/`asr.pass2`)+ 环境变量入口。
|
|
123
|
+
弹窗按 aiti-float 契约(拖拽/缩放/持久化 + 小屏全屏 + 自动高度)。
|
|
124
|
+
- **AI 运维**:skills `asr-server`(分平台配置/运行/排障/一键修复,ai-helper 聚合)。
|
|
125
|
+
|
|
126
|
+
与 `[发自: pageId · url]` 来源标注并列、各自幂等清理,互不干扰。
|
|
127
|
+
|
|
128
|
+
## 设计 / 方案文档
|
|
129
|
+
|
|
130
|
+
- 语音完整方案(流式 + 2-pass + 自愈 + 热词 + 跨平台):`docs/aiti-voice-input-realtime-asr-plan.md`
|
|
131
|
+
- 热词提取与消费(session-manager + 8767 + 弹窗):`docs/session-manager-hotwords-api.md`
|
|
132
|
+
- 完整版设计(🎤 按钮 + Web Speech API / L2 ASR + 三级纠错管线 + diff 确认 UI):
|
|
133
|
+
`docs/aiti-voice-input-plan.md`
|
|
134
|
+
|
|
135
|
+
## 部署
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
node tools/deploy.mjs # 拷贝 index.ts + client + skills → ~/.pi/web{,-dev}/extensions/aiti-ai-input/
|
|
139
|
+
# 生效:POST /api/extensions/reload;client no-store 刷新页面即新
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### 实时流式 ASR 服务(8767)· 一键脚本 + 自愈(主路径)
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
cd pi-web-ext/aiti-ai-input/server
|
|
146
|
+
bash start-stream.sh # 默认 start:幂等(已在运行则跳过);Linux/macOS 通用(uname 分支:macOS 用 lsof)
|
|
147
|
+
powershell -File start-stream.ps1 # Windows 对应(status/stop/restart 同语义)
|
|
148
|
+
bash start-stream.sh status / stop / restart
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
环境变量:`ASR_STREAM_PORT`(8767) · `ASR_MAX_SECONDS`(60)。**模型不在环境变量/命令行**——统一走配置文件
|
|
152
|
+
`~/.pi/web/data/aiti-ai-input/asr-config.json`(ext dataDir):`{ stream_model, offline_model, max_seconds, model_hotwords_max }`,
|
|
153
|
+
服务启动按 `ASR_CONFIG_PATH` 读取(配置 > env > 默认),避免进程列表泄漏模型。**2-pass 在 `asr-venv`**
|
|
154
|
+
(独立 Python 环境,含 Qwen3-ASR + CPU torch + funasr)运行——`start-stream.sh` 优先用 `asr-venv`,无则回退 `.venv`。
|
|
155
|
+
|
|
156
|
+
**2-pass 模型(弹窗下拉可切,切换写配置+重启生效)**:
|
|
157
|
+
- `Qwen/Qwen3-ASR-0.6B`(默认 · 52 语种+22 中文方言 · 标点/术语大写 · `context=` 热词)
|
|
158
|
+
- `FunAudioLLM/Fun-ASR-Nano-2512`(中英日+方言 · 术语合成好 · `hotword=` · 更快)
|
|
159
|
+
- `iic/speech_seaco_paraformer_...`(旧 · 无标点 · 仅保留)
|
|
160
|
+
流式 1-pass 始终 paraformer-zh-streaming(实时出字不变;`asr.streaming` 默认关)。
|
|
161
|
+
|
|
162
|
+
扩展侧:`AITI_ASR_STREAM_BASE` · `AITI_ASR_SERVER_DIR` · `AITI_ASR_START_CMD`(按 `process.platform` 自动选 .sh/.ps1)。
|
|
163
|
+
|
|
164
|
+
**自愈(同进退模型)**:每次按 🎤 经 `/asr/ensure` 探测 8767,不通时经 aiti-cmd PTY 自动拉起
|
|
165
|
+
(日志在 ⚡命令 面板);停掉就停掉,下次语音自动再拉起;不做 systemd 托管。
|
|
166
|
+
管理端点:`POST /asr/control {action: start|stop|restart}`(stop 优先按平台脚本杀端口,兜底会话 kill;
|
|
167
|
+
覆盖 reload 后孤儿进程)。
|
|
168
|
+
|
|
169
|
+
验证:`bash -c 'asr-venv/bin/python test_asr_stream.py && asr-venv/bin/python test_ws_proxy.py'`
|
|
170
|
+
|
|
171
|
+
## 验证(手机浏览器)
|
|
172
|
+
|
|
173
|
+
1. 手机访问 pi.example.com:8443(或 /dev/)→ 打开任意会话;
|
|
174
|
+
2. 按住 🎤 说话 → 松手即精修(无延迟)+ 标点/术语大小写正确(Qwen3/Fun-ASR-Nano);
|
|
175
|
+
(🎤 按钮由顶级开关 `voice.enabled` 控制,未测好默认关——extman 打开后输入框上才出现按钮)
|
|
176
|
+
3. extman 打开「语音识别服务器管理」(顶级 `toolbar.enabled`)→ 工具条 🎙️ → 弹窗内
|
|
177
|
+
状态/启停/**模型下拉切换**/热词统计/用户热词增删改;启动后状态自动刷新到运行中;
|
|
178
|
+
4. 多 workspace 切换(如 pi-work ↔ pi-prod)→ 弹窗「最终生效(当前 workspace)」随当前会话变化。
|
|
179
|
+
|
|
180
|
+
## 手动测试(浏览器 console)
|
|
181
|
+
|
|
182
|
+
```js
|
|
183
|
+
fetch('/api/extensions/aiti-ai-input/voice-asr2', {method:'POST', body: new FormData().append('audio', file)}).then(r=>r.json()).then(console.log)
|
|
184
|
+
window.__aitiVoiceInputL2?.start() / .stop() / .recording()
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## 文档 · Documentation
|
|
188
|
+
|
|
189
|
+
- 中文完整文档(aiti-docs): <https://aiti.pw/extensions/aiti-ai-input.html>
|
|
190
|
+
- English docs: <https://aiti.pw/en/extensions/aiti-ai-input.html>
|
|
191
|
+
- 扩展注册表 / registry: <https://aiti.pw/extensions.html> · machine-readable <https://aiti.pw/extensions.json>
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* asr-providers 单测 —— ASR 引擎层的纯逻辑(配置解析 / 脱敏 / 白名单合并 / multipart 编码)。
|
|
3
|
+
*
|
|
4
|
+
* 覆盖 docs/aiti-ai-input-asr-provider-plan.md §3.2/§3.3:
|
|
5
|
+
* 引擎 id 解析与旧配置兼容、按引擎取配置、ready 判定、读回脱敏(key 绝不出现明文)、
|
|
6
|
+
* 写入白名单(只认 fields 声明的键、空串=删除、未知 provider 拒绝)、手写 multipart 报文。
|
|
7
|
+
*
|
|
8
|
+
* Run: node --experimental-strip-types --test pi-web-ext/aiti-ai-input/asr-providers.test.mjs
|
|
9
|
+
*/
|
|
10
|
+
import { test } from "node:test";
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import {
|
|
13
|
+
ASR_PROVIDERS,
|
|
14
|
+
asrProviderCfg,
|
|
15
|
+
asrProviderId,
|
|
16
|
+
asrProviderReady,
|
|
17
|
+
asrProviderSpec,
|
|
18
|
+
asrProviderView,
|
|
19
|
+
asrMergeProviderCfg,
|
|
20
|
+
encodeMultipart,
|
|
21
|
+
} from "./asr-providers.ts";
|
|
22
|
+
|
|
23
|
+
const KEY = "sk-abcdefghijklmnop1234";
|
|
24
|
+
|
|
25
|
+
test("注册表:六个引擎,id 唯一,builtin 居首(旧配置缺省落 builtin)", () => {
|
|
26
|
+
const ids = ASR_PROVIDERS.map((p) => p.id);
|
|
27
|
+
assert.deepEqual(ids, ["builtin", "xfyun", "baidu", "siliconflow", "openai", "azure"]);
|
|
28
|
+
assert.equal(new Set(ids).size, ids.length);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("asrProviderId:无 provider 节/非法值 → builtin(旧配置行为不变)", () => {
|
|
32
|
+
assert.equal(asrProviderId({}), "builtin");
|
|
33
|
+
assert.equal(asrProviderId({ stream_model: "x" }), "builtin");
|
|
34
|
+
assert.equal(asrProviderId({ provider: "nope" }), "builtin");
|
|
35
|
+
assert.equal(asrProviderId({ provider: "xfyun" }), "xfyun");
|
|
36
|
+
assert.equal(asrProviderId({ provider: " xfyun " }), "xfyun");
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("asrProviderCfg:取 providers[id],缺失/非对象 → {}", () => {
|
|
40
|
+
const cfg = { providers: { xfyun: { appId: "2061196f" }, baidu: "bad" } };
|
|
41
|
+
assert.deepEqual(asrProviderCfg(cfg, "xfyun"), { appId: "2061196f" });
|
|
42
|
+
assert.deepEqual(asrProviderCfg(cfg, "baidu"), {});
|
|
43
|
+
assert.deepEqual(asrProviderCfg(cfg, "openai"), {});
|
|
44
|
+
assert.deepEqual(asrProviderCfg({}, "xfyun"), {});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("asrProviderReady:各引擎按必填字段判定", () => {
|
|
48
|
+
assert.equal(asrProviderReady({}, "builtin"), true);
|
|
49
|
+
assert.equal(asrProviderReady({ providers: { siliconflow: { apiKey: KEY } } }, "siliconflow"), true);
|
|
50
|
+
assert.equal(asrProviderReady({ providers: { siliconflow: { apiKey: " " } } }, "siliconflow"), false);
|
|
51
|
+
assert.equal(asrProviderReady({ providers: { xfyun: { appId: "a", apiKey: "b" } } }, "xfyun"), false); // 缺 apiSecret
|
|
52
|
+
assert.equal(asrProviderReady({ providers: { xfyun: { appId: "a", apiKey: "b", apiSecret: "c" } } }, "xfyun"), true);
|
|
53
|
+
assert.equal(asrProviderReady({ providers: { baidu: { apiKey: "a", secretKey: "b" } } }, "baidu"), true);
|
|
54
|
+
assert.equal(asrProviderReady({ providers: { openai: { baseUrl: "http://x/v1" } } }, "openai"), true);
|
|
55
|
+
assert.equal(asrProviderReady({ providers: { openai: { apiKey: KEY } } }, "openai"), false); // 缺端点
|
|
56
|
+
assert.equal(asrProviderReady({ providers: { azure: { subscriptionKey: "k" } } }, "azure"), false); // 缺 region
|
|
57
|
+
assert.equal(asrProviderReady({ providers: { azure: { subscriptionKey: "k", region: "eastasia" } } }, "azure"), true);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("asrProviderView:key 只回 configured + 尾 4 位,全文序列化不含明文", () => {
|
|
61
|
+
const cfg = {
|
|
62
|
+
provider: "siliconflow",
|
|
63
|
+
providers: { siliconflow: { apiKey: KEY, model: "FunAudioLLM/SenseVoiceSmall" } },
|
|
64
|
+
};
|
|
65
|
+
const view = asrProviderView(cfg);
|
|
66
|
+
assert.equal(view.provider, "siliconflow");
|
|
67
|
+
assert.equal(view.providerReady, true);
|
|
68
|
+
const sf = view.providers.find((p) => p.id === "siliconflow");
|
|
69
|
+
assert.deepEqual(sf.params.apiKey, { configured: true, tail: "1234" });
|
|
70
|
+
assert.equal(sf.params.model.value, "FunAudioLLM/SenseVoiceSmall");
|
|
71
|
+
assert.equal(sf.fields.some((f) => f.key === "apiKey" && f.secret === true), true);
|
|
72
|
+
assert.equal(JSON.stringify(view).includes(KEY), false, "脱敏后不得出现 key 明文");
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("asrProviderView:未配置引擎 params 标 configured:false", () => {
|
|
76
|
+
const view = asrProviderView({ provider: "builtin" });
|
|
77
|
+
const xf = view.providers.find((p) => p.id === "xfyun");
|
|
78
|
+
assert.equal(xf.ready, false);
|
|
79
|
+
assert.deepEqual(xf.params.apiKey, { configured: false }); // secret:未配置不给 value
|
|
80
|
+
assert.deepEqual(xf.params.appId, { configured: false, value: "" }); // 非 secret:回空值供表单回填
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("asrMergeProviderCfg:切 provider + 白名单写入 + 空串删除 + 未知字段忽略", () => {
|
|
84
|
+
const base = { stream_model: "paraformer-zh-streaming", provider: "builtin" };
|
|
85
|
+
const r1 = asrMergeProviderCfg(base, {
|
|
86
|
+
provider: "xfyun",
|
|
87
|
+
providers: { xfyun: { appId: "2061196f", apiKey: "de33", apiSecret: "NTgx", evil: "x" } },
|
|
88
|
+
});
|
|
89
|
+
assert.equal(r1.error, undefined);
|
|
90
|
+
assert.equal(r1.cfg.provider, "xfyun");
|
|
91
|
+
assert.equal(r1.cfg.stream_model, "paraformer-zh-streaming", "已有字段必须保留");
|
|
92
|
+
assert.deepEqual(r1.cfg.providers.xfyun, { appId: "2061196f", apiKey: "de33", apiSecret: "NTgx" });
|
|
93
|
+
|
|
94
|
+
const r2 = asrMergeProviderCfg(r1.cfg, { providers: { xfyun: { apiKey: "" } } });
|
|
95
|
+
assert.equal("apiKey" in r2.cfg.providers.xfyun, false, "空串 = 删除该项");
|
|
96
|
+
assert.equal(r2.cfg.providers.xfyun.appId, "2061196f", "其它键不受影响");
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("asrMergeProviderCfg:未知 provider 拒绝(不写盘)", () => {
|
|
100
|
+
const r = asrMergeProviderCfg({ provider: "xfyun" }, { provider: "nope" });
|
|
101
|
+
assert.match(r.error || "", /provider 必须是/);
|
|
102
|
+
assert.equal(r.cfg.provider, "xfyun");
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("asrProviderSpec:未知 id 回落 builtin(保证分发不炸)", () => {
|
|
106
|
+
assert.equal(asrProviderSpec("nope").id, "builtin");
|
|
107
|
+
assert.equal(asrProviderSpec("xfyun").timeout.quick > 0, true);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("encodeMultipart:boundary/字段名/二进制长度/contentType 正确", () => {
|
|
111
|
+
const bin = new Uint8Array([0x52, 0x49, 0x46, 0x46, 0x00, 0x01]);
|
|
112
|
+
const mp = encodeMultipart([
|
|
113
|
+
{ name: "model", value: "FunAudioLLM/SenseVoiceSmall" },
|
|
114
|
+
{ name: "file", filename: "voice.wav", type: "audio/wav", value: bin },
|
|
115
|
+
]);
|
|
116
|
+
assert.match(mp.contentType, /^multipart\/form-data; boundary=----aitiAsrForm/);
|
|
117
|
+
const text = Buffer.from(mp.body).toString("latin1");
|
|
118
|
+
const boundary = mp.contentType.split("boundary=")[1];
|
|
119
|
+
assert.ok(text.includes(`Content-Disposition: form-data; name="model"`));
|
|
120
|
+
assert.ok(text.includes('Content-Disposition: form-data; name="file"; filename="voice.wav"'));
|
|
121
|
+
assert.ok(text.includes("Content-Type: audio/wav"));
|
|
122
|
+
assert.ok(text.includes("FunAudioLLM/SenseVoiceSmall"));
|
|
123
|
+
assert.ok(text.includes("RIFF"));
|
|
124
|
+
assert.ok(text.endsWith(`--${boundary}--\r\n`));
|
|
125
|
+
assert.equal(mp.body.length > bin.length, true);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("encodeMultipart:文件名含引号/换行被净化(防报文注入)", () => {
|
|
129
|
+
const mp = encodeMultipart([{ name: "file", filename: 'a"b\nc.wav', type: "audio/wav", value: new Uint8Array([1]) }]);
|
|
130
|
+
const text = Buffer.from(mp.body).toString("latin1");
|
|
131
|
+
assert.ok(text.includes('filename="a_b_c.wav"'));
|
|
132
|
+
assert.equal(text.includes('a"b'), false);
|
|
133
|
+
});
|