@lykling/pi-vision 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Pride Leong (lykling)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,73 @@
1
- # Temporary Holding Version
1
+ # @lykling/pi-vision
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Pi coding agent 的按需视觉工具:主会话继续用文本模型做编码与分析,
4
+ 需要识图时由 agent 调用 `see` 工具,把图片转发给可配置的视觉模型并返回文字回答。
5
+
6
+ ## 安装
7
+
8
+ ```bash
9
+ pi install npm:@lykling/pi-vision
10
+ ```
11
+
12
+ 然后 `/reload`(或重启 pi)。agent 的工具列表里会出现 `see`。
13
+
14
+ ## 工具
15
+
16
+ **see(image, prompt)** — 查看一张图片并回答问题。
17
+
18
+ - `image`:本地路径或 http(s) URL(png/jpg/webp/gif/bmp,base64 后 ≤5MB)
19
+ - `prompt`:想让视觉模型回答的问题
20
+
21
+ 触发逻辑是标准 LLM function calling:你让 agent "看看这张图/识别照片",
22
+ agent 根据工具描述发起调用,视觉模型的文字回答作为 tool result 回到对话。
23
+
24
+ ## 配置
25
+
26
+ 优先级从高到低(**配置文件为主,环境变量只是 fallback**):
27
+
28
+ | 优先级 | 来源 | 说明 |
29
+ |---|---|---|
30
+ | 1 | `<项目>/.pi/vision.json` | 仅受信项目加载,支持 JSONC 注释 |
31
+ | 2 | `~/.pi/agent/vision.json` | 全局,支持 JSONC 注释 |
32
+ | 3 | `VISION_MODEL` / `VISION_BASE_URL` / `VISION_API_KEY` | 环境变量 |
33
+ | 4 | 内置默认 | `glm-4.6v` @ `https://open.bigmodel.cn/api/coding/paas/v4`;key 复用 `~/.pi/agent/auth.json` 的 `zai-coding-cn.key` |
34
+
35
+ 字段:
36
+
37
+ ```jsonc
38
+ // ~/.pi/agent/vision.json 示例
39
+ {
40
+ "model": "glm-4.6v", // 任意 OpenAI 兼容视觉模型
41
+ "baseUrl": "https://open.bigmodel.cn/api/coding/paas/v4",
42
+ "apiKey": "sk-..." // 缺省时读 auth.json,一般不用填
43
+ }
44
+ ```
45
+
46
+ 换后端示例(通义千问 VL):
47
+
48
+ ```json
49
+ {
50
+ "model": "qwen-vl-max-latest",
51
+ "baseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1",
52
+ "apiKey": "sk-xxx"
53
+ }
54
+ ```
55
+
56
+ 配置在每次工具调用时解析,改完文件即生效,无需 `/reload`。
57
+ 每次调用的 details 里会标注实际命中的配置来源(project-config / global-config / env / default / auth.json)。
58
+
59
+ ## 开发
60
+
61
+ ```bash
62
+ npm install
63
+ npm run typecheck
64
+ npm test
65
+ ```
66
+
67
+ 发布走 GitHub Actions OIDC trusted publishing(见 `.github/workflows/publish.yml`):
68
+ npmjs.com 上为 `lykling/pi-vision` 配置 Trusted Publisher(workflow 文件名 `publish.yml`),
69
+ 然后打 tag `v*` 推送即可。
70
+
71
+ ## License
72
+
73
+ MIT
package/package.json CHANGED
@@ -1,6 +1,50 @@
1
1
  {
2
2
  "name": "@lykling/pi-vision",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.1.0",
4
+ "description": "On-demand vision tool (see) for the Pi coding agent: text models delegate image understanding to a configurable vision model.",
5
+ "keywords": [
6
+ "pi-package",
7
+ "pi",
8
+ "vision",
9
+ "glm",
10
+ "coding-agent"
11
+ ],
12
+ "license": "MIT",
13
+ "author": "lykling <lykling.lyk@gmail.com>",
14
+ "repository": {
15
+ "type": "git",
16
+ "url": "git+https://github.com/lykling/pi-vision.git"
17
+ },
18
+ "type": "module",
19
+ "main": "src/index.ts",
20
+ "scripts": {
21
+ "typecheck": "tsc --noEmit",
22
+ "test": "node --test \"test/*.test.ts\""
23
+ },
24
+ "engines": {
25
+ "node": ">=20"
26
+ },
27
+ "peerDependencies": {
28
+ "@earendil-works/pi-coding-agent": "*",
29
+ "typebox": "*"
30
+ },
31
+ "devDependencies": {
32
+ "@earendil-works/pi-coding-agent": "^1.5.2",
33
+ "typebox": "^1.4.7",
34
+ "typescript": "^5.9.3"
35
+ },
36
+ "pi": {
37
+ "extensions": [
38
+ "./src/index.ts"
39
+ ]
40
+ },
41
+ "files": [
42
+ "src",
43
+ "README.md",
44
+ "LICENSE"
45
+ ],
46
+ "publishConfig": {
47
+ "access": "public",
48
+ "provenance": true
49
+ }
50
+ }
package/src/config.ts ADDED
@@ -0,0 +1,192 @@
1
+ import { existsSync, readFileSync } from "node:fs";
2
+ import { homedir } from "node:os";
3
+ import { join } from "node:path";
4
+
5
+ /** Settings for the vision backend, resolved from config files, env, defaults. */
6
+ export interface VisionSettings {
7
+ model: string;
8
+ baseUrl: string;
9
+ apiKey: string | undefined;
10
+ /** Human-readable source of each resolved field, for tool result details. */
11
+ source: { model: string; baseUrl: string; apiKey: string };
12
+ }
13
+
14
+ /** Injection point for tests and project-local resolution. */
15
+ export interface ResolveOptions {
16
+ /** Project directory; project config is read from <cwd>/.pi/vision.json. */
17
+ cwd?: string;
18
+ /** Whether the project is trusted. Project config is only honored when true. */
19
+ trusted?: boolean;
20
+ /** Config dir name inside the project (pi convention: ".pi", rebrandable). */
21
+ configDirName?: string;
22
+ /** Override paths for tests. */
23
+ globalConfigPath?: string;
24
+ projectConfigPath?: string;
25
+ }
26
+
27
+ const DEFAULT_MODEL = "glm-4.6v";
28
+ const DEFAULT_BASE_URL = "https://open.bigmodel.cn/api/coding/paas/v4";
29
+
30
+ /**
31
+ * Strip line and block comments plus trailing commas so JSONC files parse.
32
+ * Same approach as @lykling/pi-supermemory config.
33
+ */
34
+ export function stripJsoncComments(content: string): string {
35
+ let result = "";
36
+ let i = 0;
37
+ let inString = false;
38
+ let inSingleLineComment = false;
39
+ let inMultiLineComment = false;
40
+ while (i < content.length) {
41
+ const char = content[i];
42
+ const nextChar = content[i + 1];
43
+ if (!inSingleLineComment && !inMultiLineComment) {
44
+ if (char === '"') {
45
+ let backslashCount = 0;
46
+ let j = i - 1;
47
+ while (j >= 0 && content[j] === "\\") {
48
+ backslashCount++;
49
+ j--;
50
+ }
51
+ if (backslashCount % 2 === 0) {
52
+ inString = !inString;
53
+ }
54
+ result += char;
55
+ i++;
56
+ continue;
57
+ }
58
+ }
59
+ if (inString) {
60
+ result += char;
61
+ i++;
62
+ continue;
63
+ }
64
+ if (!inSingleLineComment && !inMultiLineComment) {
65
+ if (char === "/" && nextChar === "/") {
66
+ inSingleLineComment = true;
67
+ i += 2;
68
+ continue;
69
+ }
70
+ if (char === "/" && nextChar === "*") {
71
+ inMultiLineComment = true;
72
+ i += 2;
73
+ continue;
74
+ }
75
+ }
76
+ if (inSingleLineComment) {
77
+ if (char === "\n") {
78
+ inSingleLineComment = false;
79
+ result += char;
80
+ }
81
+ i++;
82
+ continue;
83
+ }
84
+ if (inMultiLineComment) {
85
+ if (char === "*" && nextChar === "/") {
86
+ inMultiLineComment = false;
87
+ i += 2;
88
+ continue;
89
+ }
90
+ if (char === "\n") {
91
+ result += char;
92
+ }
93
+ i++;
94
+ continue;
95
+ }
96
+ result += char;
97
+ i++;
98
+ }
99
+ return result.replace(/,\s*([}\]])/g, "$1");
100
+ }
101
+
102
+ /** Read and parse a JSON or JSONC file, returning null on any failure. */
103
+ export function readJsonFile(path: string): Record<string, unknown> | null {
104
+ try {
105
+ if (!existsSync(path)) return null;
106
+ const raw = readFileSync(path, "utf-8");
107
+ return JSON.parse(stripJsoncComments(raw)) as Record<string, unknown>;
108
+ } catch {
109
+ return null;
110
+ }
111
+ }
112
+
113
+ /** Fallback API key: reuse the pi zai-coding-cn credential. */
114
+ function authJsonKey(): string | undefined {
115
+ try {
116
+ const auth = readJsonFile(join(homedir(), ".pi", "agent", "auth.json"));
117
+ const key = auth?.["zai-coding-cn"];
118
+ if (key && typeof key === "object") {
119
+ const v = (key as Record<string, unknown>).key;
120
+ return typeof v === "string" && v.trim() ? v.trim() : undefined;
121
+ }
122
+ return undefined;
123
+ } catch {
124
+ return undefined;
125
+ }
126
+ }
127
+
128
+ /**
129
+ * Resolve vision backend settings.
130
+ *
131
+ * Precedence (config files first, env as fallback, built-ins last):
132
+ * 1. <project>/.pi/vision.json — only when the project is trusted
133
+ * 2. ~/.pi/agent/vision.json — global (JSONC allowed)
134
+ * 3. VISION_MODEL / VISION_BASE_URL / VISION_API_KEY env vars
135
+ * 4. built-in defaults (glm-4.6v on the bigmodel coding endpoint;
136
+ * apiKey falls back to ~/.pi/agent/auth.json "zai-coding-cn".key)
137
+ */
138
+ export function resolveSettings(options: ResolveOptions = {}): VisionSettings {
139
+ const globalPath =
140
+ options.globalConfigPath ?? join(homedir(), ".pi", "agent", "vision.json");
141
+ const projectPath =
142
+ options.projectConfigPath ??
143
+ (options.cwd !== undefined
144
+ ? join(options.cwd, options.configDirName ?? ".pi", "vision.json")
145
+ : undefined);
146
+
147
+ const projectConfig =
148
+ options.trusted && projectPath ? readJsonFile(projectPath) : null;
149
+ const globalConfig = readJsonFile(globalPath);
150
+
151
+ const pick = (
152
+ projectValue: unknown,
153
+ globalValue: unknown,
154
+ envValue: string | undefined,
155
+ fallback: string | undefined,
156
+ ): { value: string | undefined; source: string } => {
157
+ const first = (v: unknown): string | undefined =>
158
+ typeof v === "string" && v.trim() ? v.trim() : undefined;
159
+ if (first(projectValue)) {
160
+ return { value: first(projectValue), source: "project-config" };
161
+ }
162
+ if (first(globalValue)) {
163
+ return { value: first(globalValue), source: "global-config" };
164
+ }
165
+ if (envValue && envValue.trim()) {
166
+ return { value: envValue.trim(), source: "env" };
167
+ }
168
+ return { value: fallback, source: "default" };
169
+ };
170
+
171
+ const model = pick(projectConfig?.model, globalConfig?.model, process.env.VISION_MODEL, DEFAULT_MODEL);
172
+ const baseUrl = pick(
173
+ projectConfig?.baseUrl,
174
+ globalConfig?.baseUrl,
175
+ process.env.VISION_BASE_URL,
176
+ DEFAULT_BASE_URL,
177
+ );
178
+ const apiKey = pick(
179
+ projectConfig?.apiKey,
180
+ globalConfig?.apiKey,
181
+ process.env.VISION_API_KEY,
182
+ authJsonKey(),
183
+ );
184
+ if (apiKey.source === "default" && apiKey.value) apiKey.source = "auth.json";
185
+
186
+ return {
187
+ model: model.value as string,
188
+ baseUrl: baseUrl.value as string,
189
+ apiKey: apiKey.value,
190
+ source: { model: model.source, baseUrl: baseUrl.source, apiKey: apiKey.source },
191
+ };
192
+ }
package/src/index.ts ADDED
@@ -0,0 +1,152 @@
1
+ /**
2
+ * @lykling/pi-vision — on-demand vision tool for the Pi coding agent.
3
+ *
4
+ * Keep the main session on a text coding/analysis model; when the user asks to
5
+ * look at an image, the agent calls the `see` tool, which forwards the image to
6
+ * a configurable vision model and returns its textual answer.
7
+ *
8
+ * Configuration (config files first, env as fallback):
9
+ * 1. <project>/.pi/vision.json (trusted projects only, JSONC ok)
10
+ * 2. ~/.pi/agent/vision.json (global, JSONC ok)
11
+ * 3. VISION_MODEL / VISION_BASE_URL / VISION_API_KEY
12
+ * 4. defaults: glm-4.6v @ bigmodel coding endpoint,
13
+ * key from ~/.pi/agent/auth.json "zai-coding-cn".key
14
+ */
15
+ import { CONFIG_DIR_NAME, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
16
+ import { Type } from "typebox";
17
+ import { readFileSync } from "node:fs";
18
+ import { resolveSettings } from "./config.ts";
19
+
20
+ const MIME: Record<string, string> = {
21
+ jpg: "image/jpeg",
22
+ jpeg: "image/jpeg",
23
+ png: "image/png",
24
+ gif: "image/gif",
25
+ webp: "image/webp",
26
+ bmp: "image/bmp",
27
+ };
28
+
29
+ function toImageUrl(path: string): string {
30
+ if (/^https?:\/\//.test(path)) return path;
31
+ const buf = readFileSync(path); // missing file throws, caught by caller
32
+ const ext = path.split(".").pop()?.toLowerCase() ?? "";
33
+ const mime = MIME[ext] ?? "image/jpeg";
34
+ const b64 = buf.toString("base64");
35
+ if (b64.length > 5 * 1024 * 1024) {
36
+ throw new Error(`image base64 exceeds 5MB (${path}), downscale it first`);
37
+ }
38
+ return `data:${mime};base64,${b64}`;
39
+ }
40
+
41
+ interface ChatCompletionResponse {
42
+ choices?: Array<{ message?: { content?: string } }>;
43
+ }
44
+
45
+ export default function (pi: ExtensionAPI) {
46
+ pi.registerTool({
47
+ name: "see",
48
+ label: "看图",
49
+ description:
50
+ "调用视觉模型查看一张图片并回答问题。当用户让你看照片/截图/识别图片内容时使用。" +
51
+ "image 传本地路径或 http(s) URL;prompt 传想让视觉模型回答的问题。",
52
+ parameters: Type.Object({
53
+ image: Type.String({ description: "图片的本地路径或 http(s) URL" }),
54
+ prompt: Type.String({
55
+ description: "想让视觉模型回答的问题,例如'这张图里是什么硬件?丝印写了什么?'",
56
+ }),
57
+ }),
58
+ async execute(_toolCallId, params, signal, _onUpdate, ctx) {
59
+ // Per-call resolution: config file edits take effect without /reload.
60
+ const settings = resolveSettings({
61
+ cwd: ctx.cwd,
62
+ trusted: ctx.isProjectTrusted(),
63
+ configDirName: CONFIG_DIR_NAME,
64
+ });
65
+
66
+ if (!settings.apiKey) {
67
+ return {
68
+ content: [
69
+ {
70
+ type: "text" as const,
71
+ text:
72
+ "错误:无可用 API key。配置方式任选其一:" +
73
+ "~/.pi/agent/vision.json 的 apiKey、项目 .pi/vision.json 的 apiKey、" +
74
+ "环境变量 VISION_API_KEY,或确保 ~/.pi/agent/auth.json 里有 zai-coding-cn.key",
75
+ },
76
+ ],
77
+ details: {},
78
+ };
79
+ }
80
+
81
+ let imageUrl: string;
82
+ try {
83
+ imageUrl = toImageUrl(params.image);
84
+ } catch (e) {
85
+ return {
86
+ content: [{ type: "text" as const, text: `读取图片失败:${e}` }],
87
+ details: {},
88
+ };
89
+ }
90
+
91
+ try {
92
+ const resp = await fetch(`${settings.baseUrl}/chat/completions`, {
93
+ method: "POST",
94
+ headers: {
95
+ "Content-Type": "application/json",
96
+ Authorization: `Bearer ${settings.apiKey}`,
97
+ },
98
+ body: JSON.stringify({
99
+ model: settings.model,
100
+ messages: [
101
+ {
102
+ role: "user",
103
+ content: [
104
+ { type: "image_url", image_url: { url: imageUrl } },
105
+ { type: "text", text: params.prompt },
106
+ ],
107
+ },
108
+ ],
109
+ max_tokens: 2048,
110
+ }),
111
+ signal,
112
+ });
113
+ if (!resp.ok) {
114
+ const t = await resp.text().catch(() => "");
115
+ return {
116
+ content: [
117
+ {
118
+ type: "text" as const,
119
+ text: `视觉模型 API 错误 HTTP ${resp.status}: ${t.slice(0, 300)}`,
120
+ },
121
+ ],
122
+ details: { model: settings.model, baseUrl: settings.baseUrl },
123
+ };
124
+ }
125
+ const j = (await resp.json()) as ChatCompletionResponse;
126
+ const text =
127
+ j?.choices?.[0]?.message?.content ??
128
+ `视觉模型返回了无法解析的内容:${JSON.stringify(j).slice(0, 400)}`;
129
+ return {
130
+ content: [{ type: "text" as const, text }],
131
+ details: {
132
+ model: settings.model,
133
+ baseUrl: settings.baseUrl,
134
+ configSource: settings.source,
135
+ image: params.image,
136
+ },
137
+ };
138
+ } catch (e) {
139
+ if ((e as Error).name === "AbortError") {
140
+ return {
141
+ content: [{ type: "text" as const, text: "视觉模型调用已取消" }],
142
+ details: {},
143
+ };
144
+ }
145
+ return {
146
+ content: [{ type: "text" as const, text: `视觉模型调用失败:${e}` }],
147
+ details: { model: settings.model, baseUrl: settings.baseUrl },
148
+ };
149
+ }
150
+ },
151
+ });
152
+ }