@stage-labs/metro 0.1.0-beta.227 → 0.1.0-beta.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/marketplace/plugin/.claude-plugin/plugin.json +1 -1
- package/runtime/marketplace/plugin/{orchestrator.md → METRO.md} +2 -2
- package/runtime/marketplace/plugin/STAGE.md +32 -0
- package/runtime/marketplace/plugin/bin/session-start.mjs +3 -3
- package/runtime/node_modules/@metro-labs/daemon/src/claude/setup.ts +63 -16
- package/runtime/node_modules/@metro-labs/daemon/src/gateway/codex-translate.ts +19 -5
- package/runtime/node_modules/@metro-labs/daemon/src/gateway/gemini-translate.ts +16 -5
- package/runtime/node_modules/@metro-labs/daemon/src/gateway/text.ts +25 -4
- package/runtime/runtime.json +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "metro",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.229",
|
|
4
4
|
"description": "Metro for Claude Code, installed by the metro daemon on this machine: every connector the agent holds as its own MCP server relayed through the daemon, the orchestrator-only guard on the main thread, and the standing rules loaded at each session start. No vendor credential ever sits in a config file.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Bonustrack",
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
---
|
|
2
|
-
name: metro
|
|
2
|
+
name: metro
|
|
3
3
|
description: How this box's agent works on metro. Acknowledge, delegate, relay, answer over chat, never block on the terminal. Loaded at every session start by the metro plugin; edit it here to change the rules on this box.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
You are the main thread of a Claude Code session that is connected to chat through metro, on a machine nobody watches. The main thread is an orchestrator: it may only delegate and talk. A guard in the metro plugin denies every other tool on this thread, so this is the shape of the work, not a preference.
|
|
7
7
|
|
|
8
|
-
**Inbound.** A message arrives as a `<channel source="metro" …>` block. Read `addressed` first: `direct`, `mention` or `reply` means it is for you; without it the message merely happened in a room you watch, so treat it as context and stay silent unless the person plainly wants you. For a message that is for you, the first tool call is `react` on it (👀, then 👍 or ✅ once done) so the person knows you saw it, and any reply goes on the same `line`, passed verbatim.
|
|
8
|
+
**Inbound.** A message arrives as a `<channel source="metro" …>` block. Read `addressed` first: `direct`, `mention` or `reply` means it is for you; without it the message merely happened in a room you watch, so treat it as context and stay silent unless the person plainly wants you. For a message that is for you, the first tool call is `react` on it (👀, then 👍 or ✅ once done) so the person knows you saw it, and any reply goes on the same `line`, passed verbatim. Exception: when the person asks you to close something ("mark as done"), skip the reaction and the acknowledgement: close it directly and confirm once when it is finished.
|
|
9
9
|
|
|
10
10
|
**Typing.** Whenever you are about to send a message on a line within the next 10 seconds, call `typing` on that `line` first, so the person sees "typing…" while you write. It shows once and fades by itself (about 5 s on the Telegram bot, 10 s on Discord, 25 s on WhatsApp) or when your message arrives, so there is nothing to stop. Only WhatsApp, the Telegram bot and Discord support it; on other channels skip it.
|
|
11
11
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: stage
|
|
3
|
+
description: How to use Stage channels (XMTP groups, https://stage.box) through metro. Create a channel, rename it, add or remove members, set labels and assignees, read its info, and send, reply, react, unreact, delete and read messages. Works only when this agent has an XMTP account in metro.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Stage (https://stage.box) is a messenger built on XMTP. A Stage channel is an XMTP group. You use it with the metro tools below (`mcp__metro__<name>`).
|
|
7
|
+
|
|
8
|
+
**XMTP only.** All of this needs an XMTP account connected to this agent in metro. Check with `list_accounts`: it must show an account under `xmtp`. Many agents have none. Then none of this works: say so plainly and do not try another channel. The owner can add an XMTP channel on https://metro.box.
|
|
9
|
+
|
|
10
|
+
**Lines and links.** A channel's line is `metro://xmtp/<account>/<channel id>`. Pass it verbatim as `line`. Its link is `https://stage.box/#/channel/<channel id>`, the last part of the line. Stage shows Markdown, so `[#Launch plan](https://stage.box/#/channel/<channel id>)` is a clickable link. A `message_id` comes from the inbound message or from `read`.
|
|
11
|
+
|
|
12
|
+
**Channels.**
|
|
13
|
+
- Create: `create_group {"station": "xmtp", "name": "Launch plan", "members": ["0x…"]}`. Members are 0x addresses or XMTP inbox ids. It returns the new `line` and `id`.
|
|
14
|
+
- Rename: `set_channel_metadata {"line": "…", "name": "New title"}`. No tool sets the description or the picture.
|
|
15
|
+
- Add or remove people: `add_members {"line": "…", "members": ["0x…"]}`. `remove_members` takes the same shape.
|
|
16
|
+
- Leave: `close_channel {"line": "…", "removeSelf": true}`.
|
|
17
|
+
- Read: `group_info {"line": "…"}` gives the name, `labels`, `assigned`, `appData` and the members (`inboxId`, `address`). `list_members {"line": "…"}` gives the members only.
|
|
18
|
+
- Direct chat: `dm {"address": "0x…"}` opens a 1:1 chat and returns its line.
|
|
19
|
+
- With several XMTP accounts, add `"account": "<id>"` to `create_group` and `dm`.
|
|
20
|
+
|
|
21
|
+
**Labels and assignees.** They live in the channel's metadata, and Stage shows them on the channel.
|
|
22
|
+
- Labels: `set_channel_metadata {"line": "…", "metadata": {"labels": ["🚧 In progress", "Metro"]}}`. At most 16 labels of 24 characters each.
|
|
23
|
+
- Assignees: `set_channel_metadata {"line": "…", "metadata": {"assigned": ["0x…"]}}`. Only 0x addresses of current members.
|
|
24
|
+
- Each key you send replaces that whole list. Keys you leave out stay as they are. To add one label or one assignee, read `group_info` first and send the old list plus the new one. `[]` clears a list. Read `group_info` again to check it saved.
|
|
25
|
+
|
|
26
|
+
**Messages.**
|
|
27
|
+
- Send: `send {"line": "…", "text": "Hello"}`. Reply with a quote: `reply {"line": "…", "message_id": "…", "text": "…"}`.
|
|
28
|
+
- React: `react {"line": "…", "message_id": "…", "emoji": "👍"}`. `unreact` takes the same shape.
|
|
29
|
+
- Delete one of your own messages: `delete {"line": "…", "message_id": "…"}`. XMTP has no edit and no typing.
|
|
30
|
+
- History: `read {"line": "…", "limit": 20}`.
|
|
31
|
+
- Poll: `ask {"line": "…", "question": "Ship today?", "options": ["Yes", "No"]}`.
|
|
32
|
+
- File: `create_upload {"name": "report.pdf"}`, run the `curl` line it returns in a shell, then `send {"line": "…", "attachments": [{"upload": "<upload_id>"}]}`. Files that are not images are refused over about 190 KiB.
|
|
@@ -6,8 +6,8 @@ import { fileURLToPath } from 'node:url';
|
|
|
6
6
|
|
|
7
7
|
const given = (value) => (typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined);
|
|
8
8
|
const claudeDir = given(process.env.CLAUDE_CONFIG_DIR) ?? join(given(process.env.HOME) ?? homedir(), '.claude');
|
|
9
|
-
const SKILL = join(claudeDir, 'skills', 'metro
|
|
10
|
-
const FALLBACK = join(dirname(dirname(fileURLToPath(import.meta.url))), '
|
|
9
|
+
const SKILL = join(claudeDir, 'skills', 'metro', 'SKILL.md');
|
|
10
|
+
const FALLBACK = join(dirname(dirname(fileURLToPath(import.meta.url))), 'METRO.md');
|
|
11
11
|
|
|
12
12
|
const body = (text) => text.replace(/^---[\s\S]*?\n---\n/, '').trim();
|
|
13
13
|
|
|
@@ -27,5 +27,5 @@ function guidance() {
|
|
|
27
27
|
const text = guidance();
|
|
28
28
|
if (text !== '')
|
|
29
29
|
process.stdout.write(
|
|
30
|
-
`${JSON.stringify({ hookSpecificOutput: { hookEventName: 'SessionStart', additionalContext: `This machine runs as a metro agent. Standing rules, kept in the metro
|
|
30
|
+
`${JSON.stringify({ hookSpecificOutput: { hookEventName: 'SessionStart', additionalContext: `This machine runs as a metro agent. Standing rules, kept in the metro skill:\n\n${text}` } })}\n`,
|
|
31
31
|
);
|
|
@@ -7,7 +7,7 @@ import { errMsg, log } from '@metro-labs/core/log';
|
|
|
7
7
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
8
8
|
import { readJson, writeAtomic, writeJson } from '@metro-labs/core/secure-fs';
|
|
9
9
|
import { agentsDir } from '../agents/files.js';
|
|
10
|
-
import { writeHomeText } from '../agent-user/home-fs.js';
|
|
10
|
+
import { moveHome, removeHome, writeHomeText } from '../agent-user/home-fs.js';
|
|
11
11
|
import { claudeDir } from './files.js';
|
|
12
12
|
import { stagedMarketplaceDir } from './plugin-install.js';
|
|
13
13
|
import { readModelConfig, routedConnection, type ModelConfig } from '../gateway/model-config.js';
|
|
@@ -19,9 +19,11 @@ export const PRIVACY_ENV: Record<string, string> = {
|
|
|
19
19
|
CLAUDE_CODE_GB_DISK_CACHE_WHEN_TELEMETRY_OFF: '1',
|
|
20
20
|
};
|
|
21
21
|
export const RETENTION_DAYS = 7;
|
|
22
|
-
const SKILL_NAME = 'metro-orchestrator';
|
|
23
22
|
const STATE_FILE = 'claude-setup.json';
|
|
24
|
-
const
|
|
23
|
+
const RULES_FILE = 'METRO.md';
|
|
24
|
+
const RULES_SKILL = 'metro';
|
|
25
|
+
const RENAMED_SKILL = 'metro-orchestrator';
|
|
26
|
+
const RENAMED_NAME = /^name:[ \t]*(["']?)metro-orchestrator\1[ \t]*$/m;
|
|
25
27
|
const SYSTEM_PROMPT_FILE = 'system-prompt.md';
|
|
26
28
|
export const SYSTEM_PROMPT_MAX = 64 * 1024;
|
|
27
29
|
|
|
@@ -45,7 +47,7 @@ Your final message is a report to the orchestrator, not a message to a human. It
|
|
|
45
47
|
export interface SetupDeps {
|
|
46
48
|
dir?: string;
|
|
47
49
|
agents?: string;
|
|
48
|
-
|
|
50
|
+
plugin?: string;
|
|
49
51
|
env?: NodeJS.ProcessEnv;
|
|
50
52
|
}
|
|
51
53
|
|
|
@@ -57,6 +59,7 @@ export interface SetupReport {
|
|
|
57
59
|
guard: 'plugin';
|
|
58
60
|
worker: Placed;
|
|
59
61
|
skill: Placed;
|
|
62
|
+
stage: Placed;
|
|
60
63
|
settings: SettingsOutcome;
|
|
61
64
|
}
|
|
62
65
|
|
|
@@ -68,6 +71,7 @@ export interface SetupStatus {
|
|
|
68
71
|
guard: 'plugin';
|
|
69
72
|
worker: boolean;
|
|
70
73
|
skill: boolean;
|
|
74
|
+
stage: boolean;
|
|
71
75
|
privacyApplied: boolean;
|
|
72
76
|
retentionDays: number | null;
|
|
73
77
|
}
|
|
@@ -127,22 +131,34 @@ export function setSystemPrompt(text: string, agents = agentsDir()): void {
|
|
|
127
131
|
writeAtomic(promptPath(agents), `${trimmed}\n`, 0o600);
|
|
128
132
|
}
|
|
129
133
|
|
|
130
|
-
function
|
|
134
|
+
function pluginDir(env: NodeJS.ProcessEnv = process.env): string {
|
|
131
135
|
const staged = stagedMarketplaceDir(env);
|
|
132
136
|
if (staged !== null) {
|
|
133
|
-
const
|
|
134
|
-
if (existsSync(
|
|
137
|
+
const dir = join(staged, 'plugin');
|
|
138
|
+
if (existsSync(join(dir, RULES_FILE))) return dir;
|
|
135
139
|
}
|
|
136
|
-
return fileURLToPath(new URL(
|
|
140
|
+
return fileURLToPath(new URL('../../../../plugin', import.meta.url));
|
|
137
141
|
}
|
|
138
142
|
|
|
139
143
|
const PRIOR_WORKER: ReadonlySet<string> = new Set(['08a0cd8710df285d6512245246bb8b658b7cfc7f3e89490514ee5376779eb2ef']);
|
|
140
|
-
const
|
|
144
|
+
const PRIOR_METRO: ReadonlySet<string> = new Set([
|
|
141
145
|
'0b3d122ed95beff09d579cf912cd4238e1db524c41fce4b314de57d6ff5908ad',
|
|
142
146
|
'36f4fb57f231a119d717b5e3d6ccb654f94960fcca708683f376679aead8df25',
|
|
143
147
|
'3840bc50253e51f431691376cf29a925b8245d87b77b8b20804a159e378a6e67',
|
|
144
148
|
'e6b59eba825b4568dfa9ace5049fd9568a2e10d731719976ad2657084fd4e5f9',
|
|
149
|
+
'5ce3f76d610aae91adc922d0edc6c8be1f27aa2fec4a5e3a6d6231a04fb9310b',
|
|
150
|
+
'b910597d24f31fa479b130a2545f94dd23684d802d99e36b637fc57c818a33be',
|
|
145
151
|
]);
|
|
152
|
+
const PRIOR_STAGE: ReadonlySet<string> = new Set();
|
|
153
|
+
|
|
154
|
+
interface ShippedSkill {
|
|
155
|
+
name: string;
|
|
156
|
+
file: string;
|
|
157
|
+
prior: ReadonlySet<string>;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const METRO_SKILL: ShippedSkill = { name: RULES_SKILL, file: RULES_FILE, prior: PRIOR_METRO };
|
|
161
|
+
const STAGE_SKILL: ShippedSkill = { name: 'stage', file: 'STAGE.md', prior: PRIOR_STAGE };
|
|
146
162
|
|
|
147
163
|
const digest = (text: string): string => createHash('sha256').update(text).digest('hex');
|
|
148
164
|
|
|
@@ -190,12 +206,39 @@ function applyPrivacy(dir: string, enabled: boolean): SettingsOutcome {
|
|
|
190
206
|
return mergeSettings(dir, (current) => withPrivacy(current, enabled));
|
|
191
207
|
}
|
|
192
208
|
|
|
193
|
-
const skillPath = (dir: string): string => join(dir, 'skills',
|
|
209
|
+
const skillPath = (dir: string, name: string): string => join(dir, 'skills', name, 'SKILL.md');
|
|
194
210
|
const workerPath = (dir: string): string => join(dir, 'agents', 'worker.md');
|
|
195
211
|
|
|
196
|
-
function placeSkill(dir: string,
|
|
197
|
-
|
|
198
|
-
|
|
212
|
+
function placeSkill(dir: string, plugin: string, skill: ShippedSkill): Placed {
|
|
213
|
+
const source = join(plugin, skill.file);
|
|
214
|
+
if (!existsSync(source)) return 'missing';
|
|
215
|
+
return placeFile(skillPath(dir, skill.name), readFileSync(source, 'utf8'), skill.prior);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function moveRenamedSkill(dir: string): void {
|
|
219
|
+
const from = join(dir, 'skills', RENAMED_SKILL);
|
|
220
|
+
const old = skillPath(dir, RENAMED_SKILL);
|
|
221
|
+
if (!existsSync(old)) return;
|
|
222
|
+
const shipped = PRIOR_METRO.has(digest(readFileSync(old, 'utf8')));
|
|
223
|
+
if (existsSync(join(dir, 'skills', RULES_SKILL))) {
|
|
224
|
+
if (shipped) removeHome(from, true);
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
moveHome(from, join(dir, 'skills', RULES_SKILL));
|
|
228
|
+
log.info({ from, shipped }, 'claude-setup: moved the metro-orchestrator skill to metro');
|
|
229
|
+
if (shipped) return;
|
|
230
|
+
const path = skillPath(dir, RULES_SKILL);
|
|
231
|
+
const text = readFileSync(path, 'utf8');
|
|
232
|
+
const renamed = text.replace(RENAMED_NAME, `name: ${RULES_SKILL}`);
|
|
233
|
+
if (renamed !== text) writeHomeText(path, renamed, statSync(path).mode & 0o777);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function tryMoveRenamedSkill(dir: string): void {
|
|
237
|
+
try {
|
|
238
|
+
moveRenamedSkill(dir);
|
|
239
|
+
} catch (err) {
|
|
240
|
+
log.warn({ err: errMsg(err) }, 'claude-setup: could not move the metro-orchestrator skill to metro');
|
|
241
|
+
}
|
|
199
242
|
}
|
|
200
243
|
|
|
201
244
|
const written = (placed: Placed): boolean => placed === 'written' || placed === 'updated';
|
|
@@ -204,15 +247,18 @@ export function ensureClaudeSetup(deps: SetupDeps = {}): SetupReport {
|
|
|
204
247
|
const dir = deps.dir ?? claudeDir();
|
|
205
248
|
const agents = deps.agents ?? agentsDir();
|
|
206
249
|
const privacy = privacyEnabled(agents);
|
|
250
|
+
const plugin = deps.plugin ?? pluginDir(deps.env);
|
|
251
|
+
tryMoveRenamedSkill(dir);
|
|
207
252
|
const report: SetupReport = {
|
|
208
253
|
privacy,
|
|
209
254
|
guard: 'plugin',
|
|
210
255
|
worker: placeFile(workerPath(dir), WORKER_AGENT, PRIOR_WORKER),
|
|
211
|
-
skill: placeSkill(dir,
|
|
256
|
+
skill: placeSkill(dir, plugin, METRO_SKILL),
|
|
257
|
+
stage: placeSkill(dir, plugin, STAGE_SKILL),
|
|
212
258
|
settings: applyPrivacy(dir, privacy),
|
|
213
259
|
};
|
|
214
260
|
syncAvailableModelsQuietly({ ...deps, dir, agents });
|
|
215
|
-
if (written(report.worker) || written(report.skill) || report.settings === 'written')
|
|
261
|
+
if (written(report.worker) || written(report.skill) || written(report.stage) || report.settings === 'written')
|
|
216
262
|
log.info(report, 'claude-setup: applied the Claude Code setup for a metro box');
|
|
217
263
|
if (report.settings === 'unreadable') log.warn({ path: join(dir, 'settings.json') }, 'claude-setup: settings.json is not valid JSON, so the privacy settings were not written');
|
|
218
264
|
return report;
|
|
@@ -231,7 +277,8 @@ export function claudeSetupStatus(deps: SetupDeps = {}): SetupStatus {
|
|
|
231
277
|
liveEvents: liveEvents(agents),
|
|
232
278
|
guard: 'plugin',
|
|
233
279
|
worker: existsSync(workerPath(dir)),
|
|
234
|
-
skill: existsSync(skillPath(dir)),
|
|
280
|
+
skill: existsSync(skillPath(dir, METRO_SKILL.name)),
|
|
281
|
+
stage: existsSync(skillPath(dir, STAGE_SKILL.name)),
|
|
235
282
|
privacyApplied: Object.keys(PRIVACY_ENV).every((key) => env[key] === '1'),
|
|
236
283
|
retentionDays: typeof days === 'number' ? days : null,
|
|
237
284
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { isRecord } from '@metro-labs/core/is-record';
|
|
3
|
-
import { resultText } from './text.js';
|
|
3
|
+
import { inlineImage, partText, resultImages, resultParts, resultText, TOOL_ERROR, type InlineImage } from './text.js';
|
|
4
4
|
import { stringOf } from '@metro-labs/http/api-http';
|
|
5
5
|
|
|
6
6
|
const SIGNATURE_PREFIX = 'metro-codex:';
|
|
@@ -82,11 +82,25 @@ export function decodeSignature(signature: unknown): Item | null {
|
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
const imagePart = (image: InlineImage): Item => ({ type: 'input_image', image_url: `data:${image.mediaType};base64,${image.data}` });
|
|
86
|
+
|
|
85
87
|
function userPart(block: Item): Item | null {
|
|
86
88
|
if (block.type === 'text') return { type: 'input_text', text: stringOf(block.text) };
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
const image = inlineImage(block);
|
|
90
|
+
return image === null ? null : imagePart(image);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function outputPart(part: Item): Item | null {
|
|
94
|
+
const image = inlineImage(part);
|
|
95
|
+
if (image !== null) return imagePart(image);
|
|
96
|
+
const text = partText(part);
|
|
97
|
+
return text === '' ? null : { type: 'input_text', text };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function toolOutput(block: Item): string | Item[] {
|
|
101
|
+
if (resultImages(block).length === 0) return resultText(block);
|
|
102
|
+
const parts = resultParts(block).map(outputPart).filter((part): part is Item => part !== null);
|
|
103
|
+
return block.is_error === true ? [{ type: 'input_text', text: TOOL_ERROR }, ...parts] : parts;
|
|
90
104
|
}
|
|
91
105
|
|
|
92
106
|
function userItems(content: unknown): Item[] {
|
|
@@ -101,7 +115,7 @@ function userItems(content: unknown): Item[] {
|
|
|
101
115
|
for (const block of content.filter(isRecord)) {
|
|
102
116
|
if (block.type === 'tool_result') {
|
|
103
117
|
flush();
|
|
104
|
-
out.push({ type: 'function_call_output', call_id: stringOf(block.tool_use_id), output:
|
|
118
|
+
out.push({ type: 'function_call_output', call_id: stringOf(block.tool_use_id), output: toolOutput(block) });
|
|
105
119
|
continue;
|
|
106
120
|
}
|
|
107
121
|
const part = userPart(block);
|
|
@@ -3,7 +3,7 @@ import { isRecord } from '@metro-labs/core/is-record';
|
|
|
3
3
|
import { ToolNames } from './codex-translate.js';
|
|
4
4
|
import { CLIENT_NAME, requestId, SYSTEM_PREFIX } from './gemini-client.js';
|
|
5
5
|
import { cappedEffort, effortToApply } from './effort.js';
|
|
6
|
-
import { resultText } from './text.js';
|
|
6
|
+
import { inlineImage, resultImages, resultText, type InlineImage } from './text.js';
|
|
7
7
|
import { stringOf } from '@metro-labs/http/api-http';
|
|
8
8
|
|
|
9
9
|
type Item = Record<string, unknown>;
|
|
@@ -63,11 +63,22 @@ export function systemText(system: unknown): string {
|
|
|
63
63
|
.trim();
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
const imagePart = (image: InlineImage): Item => ({ inlineData: { mimeType: image.mediaType, data: image.data } });
|
|
67
|
+
|
|
68
|
+
const GEMINI_IMAGES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/heic', 'image/heif']);
|
|
69
|
+
const geminiSees = (mediaType: string): boolean => GEMINI_IMAGES.has(mediaType);
|
|
70
|
+
|
|
71
|
+
function toolParts(block: Item, id: string, name: string): Item[] {
|
|
72
|
+
const images = resultImages(block, geminiSees);
|
|
73
|
+
const text = resultText(block, geminiSees);
|
|
74
|
+
const result = text === '' && images.length > 0 ? `Binary content provided (${String(images.length)} item(s)).` : text;
|
|
75
|
+
return [{ functionResponse: { id, name, response: { result } } }, ...images.map(imagePart)];
|
|
76
|
+
}
|
|
77
|
+
|
|
66
78
|
function userPart(block: Item): Item | null {
|
|
67
79
|
if (block.type === 'text') return { text: stringOf(block.text) };
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
return null;
|
|
80
|
+
const image = inlineImage(block);
|
|
81
|
+
return image === null ? null : imagePart(image);
|
|
71
82
|
}
|
|
72
83
|
|
|
73
84
|
function userParts(content: unknown, calls: Map<string, string>): Item[] {
|
|
@@ -77,7 +88,7 @@ function userParts(content: unknown, calls: Map<string, string>): Item[] {
|
|
|
77
88
|
for (const block of content.filter(isRecord)) {
|
|
78
89
|
if (block.type === 'tool_result') {
|
|
79
90
|
const id = stringOf(block.tool_use_id);
|
|
80
|
-
out.push(
|
|
91
|
+
out.push(...toolParts(block, id, calls.get(id) ?? 'tool'));
|
|
81
92
|
continue;
|
|
82
93
|
}
|
|
83
94
|
const part = userPart(block);
|
|
@@ -2,13 +2,34 @@ import { isRecord } from '@metro-labs/core/is-record';
|
|
|
2
2
|
import { stringOf } from '@metro-labs/http/api-http';
|
|
3
3
|
|
|
4
4
|
const IMAGE_NOTE = '[an image was attached here; this model cannot see it]';
|
|
5
|
+
export const TOOL_ERROR = '[tool error]';
|
|
5
6
|
|
|
6
7
|
export const nonEmpty = (value: unknown): string | null => (typeof value === 'string' && value !== '' ? value : null);
|
|
7
8
|
|
|
8
|
-
|
|
9
|
+
export interface InlineImage {
|
|
10
|
+
mediaType: string;
|
|
11
|
+
data: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type Part = Record<string, unknown>;
|
|
15
|
+
type Accepts = (mediaType: string) => boolean;
|
|
16
|
+
const anyImage: Accepts = () => true;
|
|
17
|
+
|
|
18
|
+
export function inlineImage(block: Part, accepts = anyImage): InlineImage | null {
|
|
19
|
+
if (block.type !== 'image' || !isRecord(block.source) || block.source.type !== 'base64') return null;
|
|
20
|
+
const image = { mediaType: stringOf(block.source.media_type), data: stringOf(block.source.data) };
|
|
21
|
+
return image.mediaType !== '' && image.data !== '' && accepts(image.mediaType) ? image : null;
|
|
22
|
+
}
|
|
9
23
|
|
|
10
|
-
export
|
|
24
|
+
export const resultParts = (block: Part): Part[] => (Array.isArray(block.content) ? block.content.filter(isRecord) : []);
|
|
25
|
+
|
|
26
|
+
export const partText = (part: Part, accepts = anyImage): string =>
|
|
27
|
+
part.type === 'text' ? stringOf(part.text) : part.type === 'image' && inlineImage(part, accepts) === null ? IMAGE_NOTE : '';
|
|
28
|
+
|
|
29
|
+
export function resultText(block: Part, accepts = anyImage): string {
|
|
11
30
|
const content = block.content;
|
|
12
|
-
const text = typeof content === 'string' ? content :
|
|
13
|
-
return block.is_error === true ?
|
|
31
|
+
const text = typeof content === 'string' ? content : resultParts(block).map((part) => partText(part, accepts)).filter((t) => t !== '').join('\n');
|
|
32
|
+
return block.is_error === true ? `${TOOL_ERROR} ${text}` : text;
|
|
14
33
|
}
|
|
34
|
+
|
|
35
|
+
export const resultImages = (block: Part, accepts = anyImage): InlineImage[] => resultParts(block).flatMap((part) => inlineImage(part, accepts) ?? []);
|
package/runtime/runtime.json
CHANGED