@honlnk/picsense 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +235 -0
- package/dist/core/image-loader.js +123 -0
- package/dist/core/image-loader.js.map +1 -0
- package/dist/core/session-manager.js +231 -0
- package/dist/core/session-manager.js.map +1 -0
- package/dist/core/video-loader.js +260 -0
- package/dist/core/video-loader.js.map +1 -0
- package/dist/index.js +177 -0
- package/dist/index.js.map +1 -0
- package/dist/providers/index.js +31 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/openai.js +185 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/registry.js +15 -0
- package/dist/providers/registry.js.map +1 -0
- package/dist/providers/retry.js +51 -0
- package/dist/providers/retry.js.map +1 -0
- package/dist/tools/analyze-document.js +130 -0
- package/dist/tools/analyze-document.js.map +1 -0
- package/dist/tools/analyze-images.js +37 -0
- package/dist/tools/analyze-images.js.map +1 -0
- package/dist/tools/analyze-video.js +46 -0
- package/dist/tools/analyze-video.js.map +1 -0
- package/dist/tools/list-sessions.js +11 -0
- package/dist/tools/list-sessions.js.map +1 -0
- package/dist/utils/config.js +88 -0
- package/dist/utils/config.js.map +1 -0
- package/package.json +66 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for describing the origin of the Work and
|
|
141
|
+
reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may accept a fee in
|
|
167
|
+
exchange for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2026 honlnk
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/README.md
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# picsense
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@honlnk/picsense)
|
|
4
|
+
[](https://github.com/honlnk/picsense/blob/main/LICENSE)
|
|
5
|
+
[](https://nodejs.org)
|
|
6
|
+
|
|
7
|
+
> 本地图片/视频识别 MCP,让单模态基座模型获得多模态视觉能力。
|
|
8
|
+
|
|
9
|
+
`picsense` 是一款**本地安装**的 [MCP](https://modelcontextprotocol.io/)(Model Context Protocol)服务。它通过调用多模态视觉模型 API,让任何**单模态**基座模型(如 GLM-5.2 这类无法直接处理图片/视频的模型)也能识别图片与视频内容。
|
|
10
|
+
|
|
11
|
+
核心差异化:**支持多轮迭代识别**——基座模型可在处理任务的过程中多次调用,边干边查,逐步精修对图片/视频的理解。
|
|
12
|
+
|
|
13
|
+
## 功能
|
|
14
|
+
|
|
15
|
+
四个工具,按输入形态划分(不按场景拆工具,把 prompt 控制权交给基座模型):
|
|
16
|
+
|
|
17
|
+
| 工具 | 输入 | 用途 |
|
|
18
|
+
|------|------|------|
|
|
19
|
+
| `analyze_images` | 图片数组 + prompt + 可选 `session_id` | 图片识别 + 多轮迭代(核心工具;传一张是单图,传多张是批量/对比) |
|
|
20
|
+
| `analyze_video` | 视频(URL / 本地路径)+ prompt + 可选 `session_id` | 视频识别(抽帧后送视觉模型)+ 多轮迭代 |
|
|
21
|
+
| `list_sessions` | 无 | 查看当前所有识别会话的列表与简介 |
|
|
22
|
+
| `analyze_document` | 文档(URL / HTML / markdown) | 解析文档,识别其中所有图片,返回标注了图片描述的完整文档 |
|
|
23
|
+
|
|
24
|
+
### 多轮迭代识别
|
|
25
|
+
|
|
26
|
+
传统图片识别 MCP 是一次性的:给一张图 + 一个 prompt,返回描述,结束。但一次性描述往往不够详细或不够准确。
|
|
27
|
+
|
|
28
|
+
`picsense` 通过 session 机制支持多轮:
|
|
29
|
+
|
|
30
|
+
1. **首次调用** `analyze_images`(不传 `session_id`)→ 创建 session,返回描述 + `session_id`
|
|
31
|
+
2. 基座模型判断描述是否满足需求,**不满足则再次调用**(传入 `session_id`)→ 在已有对话基础上追加提问
|
|
32
|
+
3. 重复直到满足,基座模型基于最终描述继续处理任务
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
第 1 轮:analyze_images(图 + 初始 prompt) → description A + session_id
|
|
36
|
+
第 2 轮:analyze_images(session_id + "重点描述导航栏样式") → description B
|
|
37
|
+
... 直到满足 ...
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
> 这是「边干边查」的能力——基座模型在写代码过程中发现细节不清,可以随时重新读取图片的某个局部。
|
|
41
|
+
|
|
42
|
+
## 安装
|
|
43
|
+
|
|
44
|
+
需要 Node.js ≥ 20。无需克隆仓库——直接用 `npx` 即可:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
npx @honlnk/picsense
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
或全局安装:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
npm install -g @honlnk/picsense
|
|
54
|
+
picsense
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
> **本地开发**:克隆仓库后 `pnpm install && pnpm build`,入口在 `dist/index.js`。
|
|
58
|
+
|
|
59
|
+
### 视频识别的 ffmpeg 依赖
|
|
60
|
+
|
|
61
|
+
视频识别(`analyze_video`)需要 ffmpeg。安装时会自动下载内置的 `ffmpeg-static` 二进制。若该二进制下载失败(如 `--ignore-scripts`、企业内网代理屏蔽 GitHub releases),picsense 会自动 fallback 到**系统的 ffmpeg**——因此只要系统装了 ffmpeg 即可:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# macOS
|
|
65
|
+
brew install ffmpeg
|
|
66
|
+
# Debian / Ubuntu
|
|
67
|
+
apt install ffmpeg
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
> **pnpm 用户**:pnpm 默认不运行第三方包的安装脚本,若用 pnpm 全局安装发现二进制未下载,确认 `package.json` 的 `pnpm.onlyBuiltDependencies` 已包含 `ffmpeg-static`,或直接装系统 ffmpeg 走 fallback。
|
|
71
|
+
|
|
72
|
+
## 配置
|
|
73
|
+
|
|
74
|
+
通过环境变量配置,代码内零硬编码。环境变量在 MCP 客户端的配置里通过 `env` 字段传入(见下方接入示例),本地开发时也可用 `.env` 文件。
|
|
75
|
+
|
|
76
|
+
### 环境变量
|
|
77
|
+
|
|
78
|
+
| 变量 | 必填 | 默认值 | 说明 |
|
|
79
|
+
|------|------|--------|------|
|
|
80
|
+
| `DEFAULT_PROVIDER` | 是 | `openai` | 默认 provider(`openai` / `qwen` / `kimi`) |
|
|
81
|
+
| `OPENAI_API_KEY` | 是* | — | OpenAI API Key(当 provider=openai 时必填) |
|
|
82
|
+
| `OPENAI_MODEL` | 是* | — | OpenAI 模型名(如 `gpt-5.6-sol`) |
|
|
83
|
+
| `OPENAI_BASE_URL` | 否 | `https://api.openai.com/v1` | 自定义 base URL(代理或兼容网关)。会自动规范化:不带 `/v1` 则补上 |
|
|
84
|
+
| `MAX_IMAGE_MB` | 否 | `5` | 单张图片大小上限(MB) |
|
|
85
|
+
| `MAX_VIDEO_MB` | 否 | `100` | 单个视频大小上限(MB) |
|
|
86
|
+
| `VIDEO_MAX_FRAMES` | 否 | `30` | 视频抽帧的最大帧数(覆盖大多数 30 秒以内的短视频) |
|
|
87
|
+
| `VIDEO_FPS` | 否 | `1` | 视频抽帧的采样率(每秒抽几帧) |
|
|
88
|
+
| `TIMEOUT_MS` | 否 | `300000` | 视觉模型请求超时(毫秒) |
|
|
89
|
+
|
|
90
|
+
\* 默认 provider 的 Key/Model 必填;其他 provider 仅在切换使用时才需要。
|
|
91
|
+
|
|
92
|
+
> **API 格式**:provider 使用 OpenAI **Responses API**(`/v1/responses` 原生格式),而非 Chat Completions。兼容任何实现了 Responses API 的网关。
|
|
93
|
+
|
|
94
|
+
### 多 provider 配置示例
|
|
95
|
+
|
|
96
|
+
**OpenAI(首版推荐):**
|
|
97
|
+
```bash
|
|
98
|
+
DEFAULT_PROVIDER=openai
|
|
99
|
+
OPENAI_API_KEY=sk-xxx
|
|
100
|
+
OPENAI_MODEL=gpt-5.6-sol
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
**Qwen(后续支持) / Kimi(后续支持):** 当前版本仅实现 OpenAI 适配器,Qwen 与 Kimi 适配器规划中。新增 provider 只需实现 `VisionProvider` 接口。
|
|
104
|
+
|
|
105
|
+
## 接入 AI 客户端
|
|
106
|
+
|
|
107
|
+
### ZCode
|
|
108
|
+
|
|
109
|
+
在 ZCode 的 MCP 配置中加入:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{
|
|
113
|
+
"mcpServers": {
|
|
114
|
+
"picsense": {
|
|
115
|
+
"command": "npx",
|
|
116
|
+
"args": ["-y", "@honlnk/picsense"],
|
|
117
|
+
"env": {
|
|
118
|
+
"DEFAULT_PROVIDER": "openai",
|
|
119
|
+
"OPENAI_API_KEY": "sk-xxx",
|
|
120
|
+
"OPENAI_MODEL": "gpt-5.6-sol"
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
> `-y` 让 npx 首次运行时自动确认安装,避免阻塞。
|
|
128
|
+
|
|
129
|
+
### Claude Desktop / Cursor / 其他 MCP 客户端
|
|
130
|
+
|
|
131
|
+
配置 JSON 结构相同(`command` + `args` + `env`),按各客户端的 MCP 配置位置填入即可。
|
|
132
|
+
|
|
133
|
+
### Windows
|
|
134
|
+
|
|
135
|
+
Windows 上 npx 需通过 `cmd` 包裹:
|
|
136
|
+
|
|
137
|
+
```json
|
|
138
|
+
{
|
|
139
|
+
"mcpServers": {
|
|
140
|
+
"picsense": {
|
|
141
|
+
"command": "cmd",
|
|
142
|
+
"args": ["/c", "npx", "-y", "@honlnk/picsense"],
|
|
143
|
+
"env": {
|
|
144
|
+
"DEFAULT_PROVIDER": "openai",
|
|
145
|
+
"OPENAI_API_KEY": "sk-xxx",
|
|
146
|
+
"OPENAI_MODEL": "gpt-5.6-sol"
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
> **单模态模型场景说明:** 在 ZCode + 单模态模型(如 GLM-5.2)下,用户粘贴的图片会被 ZCode 自动上传图床,以 **http URL** 形态到达 MCP 工具。`image_sources` 已设计为自动识别 URL / 本地路径 / base64,无需额外处理。
|
|
154
|
+
|
|
155
|
+
## 图片限制
|
|
156
|
+
|
|
157
|
+
- 格式:jpg / jpeg / png
|
|
158
|
+
- 单张大小:≤ 5MB
|
|
159
|
+
|
|
160
|
+
## 视频识别说明
|
|
161
|
+
|
|
162
|
+
由于默认 provider(OpenAI Responses API)原生不支持视频,`analyze_video` 采用**抽帧方案**:用 ffmpeg 把视频解码成 JPEG 帧序列,再作为多张图发送给视觉模型。ffmpeg 优先用内置的 `ffmpeg-static` 二进制,下载失败时自动 fallback 到系统 ffmpeg(见上方安装说明)。
|
|
163
|
+
|
|
164
|
+
- 默认每秒抽 1 帧、最多 30 帧(可通过 `VIDEO_FPS` / `VIDEO_MAX_FRAMES` 调整)
|
|
165
|
+
- 视频格式:mp4 / mov / m4v / avi / wmv / webm / mkv / flv / mpeg / mpg
|
|
166
|
+
- 单个视频大小:≤ 100MB(可通过 `MAX_VIDEO_MB` 调整)
|
|
167
|
+
- URL 视频会先下载到临时目录再抽帧,用完即清理
|
|
168
|
+
|
|
169
|
+
## 使用示例
|
|
170
|
+
|
|
171
|
+
**单图识别:**
|
|
172
|
+
```
|
|
173
|
+
analyze_images({
|
|
174
|
+
image_sources: ["https://example.com/screenshot.png"],
|
|
175
|
+
prompt: "描述这张 UI 截图的整体布局"
|
|
176
|
+
})
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
**多轮迭代——细化某个局部:**
|
|
180
|
+
```
|
|
181
|
+
// 第 2 轮(复用上一轮返回的 session_id)
|
|
182
|
+
analyze_images({
|
|
183
|
+
session_id: "<上一轮返回的 session_id>",
|
|
184
|
+
prompt: "重点描述导航栏的样式,包括颜色、间距、字体"
|
|
185
|
+
})
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
**多图对比:**
|
|
189
|
+
```
|
|
190
|
+
analyze_images({
|
|
191
|
+
image_sources: ["https://example.com/expected.png", "https://example.com/actual.png"],
|
|
192
|
+
prompt: "对比这两张图,找出差异"
|
|
193
|
+
})
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
**视频识别:**
|
|
197
|
+
```
|
|
198
|
+
analyze_video({
|
|
199
|
+
video_source: "https://example.com/demo.mp4",
|
|
200
|
+
prompt: "描述这段视频的内容和关键画面"
|
|
201
|
+
})
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
**文档图片标注:**
|
|
205
|
+
```
|
|
206
|
+
analyze_document({
|
|
207
|
+
document: "https://example.com/article-with-images"
|
|
208
|
+
})
|
|
209
|
+
// 返回标注了每张图片描述的完整文档
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## 技术栈
|
|
213
|
+
|
|
214
|
+
- TypeScript + Node.js(stdio 本地 MCP)
|
|
215
|
+
- `@modelcontextprotocol/sdk` 官方 SDK
|
|
216
|
+
- 多 provider 架构(`VisionProvider` 接口)
|
|
217
|
+
- 零第三方 HTTP 库(仅用内置 fetch)
|
|
218
|
+
|
|
219
|
+
## 开发
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
pnpm install
|
|
223
|
+
pnpm build # 编译
|
|
224
|
+
pnpm typecheck # 类型检查
|
|
225
|
+
pnpm dev # tsx watch 调试
|
|
226
|
+
pnpm smoke # image-loader 冒烟(无需 API Key)
|
|
227
|
+
pnpm smoke:session # session-manager 单元测试(mock provider)
|
|
228
|
+
pnpm smoke:tools # 四个工具逻辑测试(mock provider)
|
|
229
|
+
pnpm smoke:video # 视频抽帧 + 工具逻辑测试(真实 ffmpeg,无需 API Key)
|
|
230
|
+
pnpm e2e # 端到端 stdio 协议测试
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
## 许可证
|
|
234
|
+
|
|
235
|
+
Apache-2.0
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 图片加载:自动识别 http(s) URL / 本地文件路径 / base64,统一返回视觉模型可消费的形态。
|
|
3
|
+
*
|
|
4
|
+
* 设计依据:docs/picsense-design.md §5.3.5、§6.4。
|
|
5
|
+
* - URL(含 ZCode 图床预签名 URL):直传给视觉模型,不转 base64(避免无谓的带宽/内存开销)。
|
|
6
|
+
* - 本地路径:读文件转 base64 data URL。
|
|
7
|
+
* - base64 字符串:补全 data URL 前缀。
|
|
8
|
+
* - 格式白名单:jpg / jpeg / png;单张 ≤ maxImageBytes(默认 5MB)。
|
|
9
|
+
*/
|
|
10
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
/** 支持的图片 MIME 白名单(与设计文档 §6.4 一致)。 */
|
|
13
|
+
const SUPPORTED_MIME = new Set(['image/jpeg', 'image/jpg', 'image/png']);
|
|
14
|
+
/** 扩展名 → MIME 映射。 */
|
|
15
|
+
const EXT_TO_MIME = {
|
|
16
|
+
jpg: 'image/jpeg',
|
|
17
|
+
jpeg: 'image/jpeg',
|
|
18
|
+
png: 'image/png',
|
|
19
|
+
};
|
|
20
|
+
/** 常见 base64 data URL 前缀。 */
|
|
21
|
+
const DATA_URL_RE = /^data:([a-zA-Z]+\/[a-zA-Z0-9.+-]+);base64,(.+)$/;
|
|
22
|
+
export class ImageLoadError extends Error {
|
|
23
|
+
constructor(message) {
|
|
24
|
+
super(message);
|
|
25
|
+
this.name = 'ImageLoadError';
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function assertSupportedMime(mime, source) {
|
|
29
|
+
// 归一化:image/jpg 视作 image/jpeg 的别名。
|
|
30
|
+
const normalized = mime === 'image/jpg' ? 'image/jpeg' : mime;
|
|
31
|
+
if (!SUPPORTED_MIME.has(normalized)) {
|
|
32
|
+
throw new ImageLoadError(`Unsupported image type "${mime}" from "${source}". Only jpg/jpeg/png are supported.`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function assertSize(bytes, maxBytes, source) {
|
|
36
|
+
if (bytes > maxBytes) {
|
|
37
|
+
const mb = (bytes / 1024 / 1024).toFixed(1);
|
|
38
|
+
const maxMb = (maxBytes / 1024 / 1024).toFixed(0);
|
|
39
|
+
throw new ImageLoadError(`Image "${source}" is ${mb}MB, exceeds the ${maxMb}MB limit.`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function mimeFromExt(filePath) {
|
|
43
|
+
const ext = path.extname(filePath).slice(1).toLowerCase();
|
|
44
|
+
return EXT_TO_MIME[ext];
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* 判断来源形态并加载。
|
|
48
|
+
*
|
|
49
|
+
* @param source 原始图片引用(URL / 本地路径 / base64)。
|
|
50
|
+
* @param maxBytes 单张图片字节上限。
|
|
51
|
+
*/
|
|
52
|
+
export async function loadImage(source, maxBytes) {
|
|
53
|
+
const trimmed = source.trim();
|
|
54
|
+
if (!trimmed) {
|
|
55
|
+
throw new ImageLoadError('Empty image source.');
|
|
56
|
+
}
|
|
57
|
+
// 1. http(s) URL —— 直传,不下载。
|
|
58
|
+
if (/^https?:\/\//i.test(trimmed)) {
|
|
59
|
+
// URL 的大小/格式无法在本地可靠校验(图床预签名 URL 不带稳定后缀),
|
|
60
|
+
// 这里只做最低限度校验,真正的限制由视觉模型 API 侧把关。
|
|
61
|
+
return { url: trimmed, mimeType: inferMimeFromUrl(trimmed), sourceType: 'url' };
|
|
62
|
+
}
|
|
63
|
+
// 2. data: base64 —— 已编码,补全校验。
|
|
64
|
+
const dataUrlMatch = trimmed.match(DATA_URL_RE);
|
|
65
|
+
if (dataUrlMatch) {
|
|
66
|
+
const [, mime, b64] = dataUrlMatch;
|
|
67
|
+
assertSupportedMime(mime, trimmed.slice(0, 40));
|
|
68
|
+
// base64 编码后每 4 字符约 3 字节。
|
|
69
|
+
const approxBytes = Math.floor((b64.length * 3) / 4);
|
|
70
|
+
assertSize(approxBytes, maxBytes, 'base64 image');
|
|
71
|
+
return { url: trimmed, mimeType: normalizeMime(mime), sourceType: 'base64' };
|
|
72
|
+
}
|
|
73
|
+
// 3. 裸 base64 字符串(无 data: 前缀)—— 兜底,按 jpeg 推断(最常见),
|
|
74
|
+
// 无法可靠判断真实格式,仅在确实拿到裸串时使用。
|
|
75
|
+
if (/^[A-Za-z0-9+/=\s]+$/.test(trimmed) && trimmed.length > 64) {
|
|
76
|
+
const mime = 'image/jpeg';
|
|
77
|
+
const approxBytes = Math.floor((trimmed.length * 3) / 4);
|
|
78
|
+
assertSize(approxBytes, maxBytes, 'base64 image');
|
|
79
|
+
return {
|
|
80
|
+
url: `data:${mime};base64,${trimmed.replace(/\s/g, '')}`,
|
|
81
|
+
mimeType: mime,
|
|
82
|
+
sourceType: 'base64',
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
// 4. 本地文件路径。
|
|
86
|
+
return loadFromFile(trimmed, maxBytes);
|
|
87
|
+
}
|
|
88
|
+
async function loadFromFile(filePath, maxBytes) {
|
|
89
|
+
let stats;
|
|
90
|
+
try {
|
|
91
|
+
stats = await stat(filePath);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
throw new ImageLoadError(`Image file not found or unreadable: "${filePath}".`);
|
|
95
|
+
}
|
|
96
|
+
if (!stats.isFile()) {
|
|
97
|
+
throw new ImageLoadError(`Image path is not a file: "${filePath}".`);
|
|
98
|
+
}
|
|
99
|
+
assertSize(stats.size, maxBytes, filePath);
|
|
100
|
+
const mime = mimeFromExt(filePath);
|
|
101
|
+
if (!mime) {
|
|
102
|
+
throw new ImageLoadError(`Cannot determine image type from path "${filePath}". Only jpg/jpeg/png are supported.`);
|
|
103
|
+
}
|
|
104
|
+
assertSupportedMime(mime, filePath);
|
|
105
|
+
const buf = await readFile(filePath);
|
|
106
|
+
const b64 = buf.toString('base64');
|
|
107
|
+
return {
|
|
108
|
+
url: `data:${mime};base64,${b64}`,
|
|
109
|
+
mimeType: mime,
|
|
110
|
+
sourceType: 'file',
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/** 从 URL 路径推断 MIME,无法推断时返回 image/jpeg(视觉模型会自行解析)。 */
|
|
114
|
+
function inferMimeFromUrl(url) {
|
|
115
|
+
// 去掉 query/hash 后取扩展名。
|
|
116
|
+
const clean = url.split('?')[0]?.split('#')[0] ?? url;
|
|
117
|
+
const ext = path.extname(clean).slice(1).toLowerCase();
|
|
118
|
+
return EXT_TO_MIME[ext] ?? 'image/jpeg';
|
|
119
|
+
}
|
|
120
|
+
function normalizeMime(mime) {
|
|
121
|
+
return mime === 'image/jpg' ? 'image/jpeg' : mime;
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=image-loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"image-loader.js","sourceRoot":"","sources":["../../src/core/image-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAY7B,qCAAqC;AACrC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;AAEzE,qBAAqB;AACrB,MAAM,WAAW,GAA2B;IAC1C,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,WAAW;CACjB,CAAC;AAEF,6BAA6B;AAC7B,MAAM,WAAW,GAAG,iDAAiD,CAAC;AAEtE,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,MAAc;IACvD,mCAAmC;IACnC,MAAM,UAAU,GAAG,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,cAAc,CACtB,2BAA2B,IAAI,WAAW,MAAM,qCAAqC,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,QAAgB,EAAE,MAAc;IACjE,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,IAAI,cAAc,CACtB,UAAU,MAAM,QAAQ,EAAE,mBAAmB,KAAK,WAAW,CAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,QAAgB;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,QAAgB;IAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,cAAc,CAAC,qBAAqB,CAAC,CAAC;IAClD,CAAC;IAED,4BAA4B;IAC5B,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAClC,yCAAyC;QACzC,iCAAiC;QACjC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAClF,CAAC;IAED,+BAA+B;IAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAChD,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,YAAY,CAAC;QACnC,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAChD,0BAA0B;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrD,UAAU,CAAC,WAAW,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;IAC/E,CAAC;IAED,mDAAmD;IACnD,6BAA6B;IAC7B,IAAI,qBAAqB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QAC/D,MAAM,IAAI,GAAG,YAAY,CAAC;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzD,UAAU,CAAC,WAAW,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;QAClD,OAAO;YACL,GAAG,EAAE,QAAQ,IAAI,WAAW,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE;YACxD,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,QAAQ;SACrB,CAAC;IACJ,CAAC;IAED,aAAa;IACb,OAAO,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AACzC,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,QAAgB;IAC5D,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,cAAc,CAAC,wCAAwC,QAAQ,IAAI,CAAC,CAAC;IACjF,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,8BAA8B,QAAQ,IAAI,CAAC,CAAC;IACvE,CAAC;IACD,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,cAAc,CACtB,0CAA0C,QAAQ,qCAAqC,CACxF,CAAC;IACJ,CAAC;IACD,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAEpC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO;QACL,GAAG,EAAE,QAAQ,IAAI,WAAW,GAAG,EAAE;QACjC,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED,qDAAqD;AACrD,SAAS,gBAAgB,CAAC,GAAW;IACnC,uBAAuB;IACvB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACvD,OAAO,WAAW,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC"}
|