@ouro.bot/cli 0.1.0-alpha.289 → 0.1.0-alpha.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/changelog.json +7 -0
- package/dist/arc/packets.js +203 -0
- package/dist/heart/attachments/image-normalize.js +194 -0
- package/dist/heart/attachments/materialize.js +230 -0
- package/dist/heart/attachments/render.js +29 -0
- package/dist/heart/attachments/store.js +103 -0
- package/dist/heart/attachments/types.js +138 -0
- package/dist/heart/core.js +187 -162
- package/dist/heart/tool-description.js +3 -3
- package/dist/heart/tool-friction.js +55 -0
- package/dist/mind/prompt.js +27 -16
- package/dist/repertoire/tool-results.js +29 -0
- package/dist/repertoire/tools-attachments.js +316 -0
- package/dist/repertoire/tools-base.js +2 -0
- package/dist/repertoire/tools-bluebubbles.js +0 -107
- package/dist/repertoire/tools-flow.js +48 -5
- package/dist/repertoire/tools.js +2 -10
- package/dist/senses/attention-queue.js +20 -1
- package/dist/senses/bluebubbles/attachment-download.js +137 -0
- package/dist/senses/bluebubbles/media.js +36 -101
- package/dist/senses/bluebubbles/model.js +9 -11
- package/dist/senses/cli/image-paste.js +79 -3
- package/dist/senses/inner-dialog.js +9 -0
- package/dist/senses/shared-turn.js +3 -7
- package/package.json +1 -1
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.isBlueBubblesImageAttachment = isBlueBubblesImageAttachment;
|
|
37
|
+
exports.downloadBlueBubblesAttachment = downloadBlueBubblesAttachment;
|
|
38
|
+
const path = __importStar(require("node:path"));
|
|
39
|
+
const image_normalize_1 = require("../../heart/attachments/image-normalize");
|
|
40
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
41
|
+
const MAX_NON_IMAGE_ATTACHMENT_BYTES = 8 * 1024 * 1024;
|
|
42
|
+
const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".heif", ".tif", ".tiff", ".bmp"]);
|
|
43
|
+
function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
|
|
44
|
+
const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
|
|
45
|
+
const url = new URL(endpoint.replace(/^\//, ""), root);
|
|
46
|
+
url.searchParams.set("password", password);
|
|
47
|
+
return url.toString();
|
|
48
|
+
}
|
|
49
|
+
function inferContentType(attachment, responseType) {
|
|
50
|
+
const normalizedResponseType = responseType?.split(";")[0]?.trim().toLowerCase();
|
|
51
|
+
if (normalizedResponseType) {
|
|
52
|
+
return normalizedResponseType;
|
|
53
|
+
}
|
|
54
|
+
return attachment.mimeType?.trim().toLowerCase() || undefined;
|
|
55
|
+
}
|
|
56
|
+
function isBlueBubblesImageAttachment(attachment, contentType) {
|
|
57
|
+
if (contentType?.startsWith("image/"))
|
|
58
|
+
return true;
|
|
59
|
+
const normalizedMime = attachment.mimeType?.trim().toLowerCase();
|
|
60
|
+
if (normalizedMime?.startsWith("image/"))
|
|
61
|
+
return true;
|
|
62
|
+
const extension = path.extname(attachment.transferName ?? "").toLowerCase();
|
|
63
|
+
return IMAGE_EXTENSIONS.has(extension);
|
|
64
|
+
}
|
|
65
|
+
function maxDownloadBytesForAttachment(attachment, contentType) {
|
|
66
|
+
return isBlueBubblesImageAttachment(attachment, contentType)
|
|
67
|
+
? image_normalize_1.MAX_ATTACHMENT_DOWNLOAD_BYTES_IMAGE
|
|
68
|
+
: MAX_NON_IMAGE_ATTACHMENT_BYTES;
|
|
69
|
+
}
|
|
70
|
+
async function downloadBlueBubblesAttachment(attachment, config, channelConfig, fetchImpl = fetch) {
|
|
71
|
+
const guid = attachment.guid?.trim();
|
|
72
|
+
if (!guid) {
|
|
73
|
+
(0, runtime_1.emitNervesEvent)({
|
|
74
|
+
level: "warn",
|
|
75
|
+
component: "senses",
|
|
76
|
+
event: "senses.bluebubbles_attachment_download_error",
|
|
77
|
+
message: "bluebubbles attachment download failed",
|
|
78
|
+
meta: { reason: "missing_guid" },
|
|
79
|
+
});
|
|
80
|
+
throw new Error("attachment guid missing");
|
|
81
|
+
}
|
|
82
|
+
const advertisedLimit = maxDownloadBytesForAttachment(attachment);
|
|
83
|
+
if (typeof attachment.totalBytes === "number" && attachment.totalBytes > advertisedLimit) {
|
|
84
|
+
(0, runtime_1.emitNervesEvent)({
|
|
85
|
+
level: "warn",
|
|
86
|
+
component: "senses",
|
|
87
|
+
event: "senses.bluebubbles_attachment_download_error",
|
|
88
|
+
message: "bluebubbles attachment download failed",
|
|
89
|
+
meta: { attachmentGuid: guid, reason: "advertised_limit_exceeded", advertisedLimit, totalBytes: attachment.totalBytes },
|
|
90
|
+
});
|
|
91
|
+
throw new Error(`attachment exceeds ${advertisedLimit} byte limit`);
|
|
92
|
+
}
|
|
93
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/attachment/${encodeURIComponent(guid)}/download`, config.password);
|
|
94
|
+
(0, runtime_1.emitNervesEvent)({
|
|
95
|
+
component: "senses",
|
|
96
|
+
event: "senses.bluebubbles_attachment_download_start",
|
|
97
|
+
message: "bluebubbles attachment download started",
|
|
98
|
+
meta: { attachmentGuid: guid, advertisedLimit },
|
|
99
|
+
});
|
|
100
|
+
const response = await fetchImpl(url, {
|
|
101
|
+
method: "GET",
|
|
102
|
+
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
103
|
+
});
|
|
104
|
+
if (!response.ok) {
|
|
105
|
+
(0, runtime_1.emitNervesEvent)({
|
|
106
|
+
level: "warn",
|
|
107
|
+
component: "senses",
|
|
108
|
+
event: "senses.bluebubbles_attachment_download_error",
|
|
109
|
+
message: "bluebubbles attachment download failed",
|
|
110
|
+
meta: { attachmentGuid: guid, reason: "http_error", status: response.status },
|
|
111
|
+
});
|
|
112
|
+
throw new Error(`HTTP ${response.status}`);
|
|
113
|
+
}
|
|
114
|
+
const contentType = inferContentType(attachment, response.headers.get("content-type"));
|
|
115
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
116
|
+
const actualLimit = maxDownloadBytesForAttachment(attachment, contentType);
|
|
117
|
+
if (buffer.length > actualLimit) {
|
|
118
|
+
(0, runtime_1.emitNervesEvent)({
|
|
119
|
+
level: "warn",
|
|
120
|
+
component: "senses",
|
|
121
|
+
event: "senses.bluebubbles_attachment_download_error",
|
|
122
|
+
message: "bluebubbles attachment download failed",
|
|
123
|
+
meta: { attachmentGuid: guid, reason: "actual_limit_exceeded", actualLimit, byteCount: buffer.length, contentType: contentType ?? null },
|
|
124
|
+
});
|
|
125
|
+
throw new Error(`attachment exceeds ${actualLimit} byte limit`);
|
|
126
|
+
}
|
|
127
|
+
(0, runtime_1.emitNervesEvent)({
|
|
128
|
+
component: "senses",
|
|
129
|
+
event: "senses.bluebubbles_attachment_download_end",
|
|
130
|
+
message: "bluebubbles attachment download completed",
|
|
131
|
+
meta: { attachmentGuid: guid, byteCount: buffer.length, contentType: contentType ?? null },
|
|
132
|
+
});
|
|
133
|
+
return {
|
|
134
|
+
buffer,
|
|
135
|
+
contentType,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
@@ -33,8 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.VLM_TEXT_WRAPPERS = void 0;
|
|
37
|
-
exports.downloadBlueBubblesAttachment = downloadBlueBubblesAttachment;
|
|
36
|
+
exports.downloadBlueBubblesAttachment = exports.VLM_TEXT_WRAPPERS = void 0;
|
|
38
37
|
exports.hydrateBlueBubblesAttachments = hydrateBlueBubblesAttachments;
|
|
39
38
|
const node_child_process_1 = require("node:child_process");
|
|
40
39
|
const fs = __importStar(require("node:fs/promises"));
|
|
@@ -43,30 +42,23 @@ const path = __importStar(require("node:path"));
|
|
|
43
42
|
const runtime_1 = require("../../nerves/runtime");
|
|
44
43
|
const identity_1 = require("../../heart/identity");
|
|
45
44
|
const model_capabilities_1 = require("../../heart/model-capabilities");
|
|
46
|
-
const
|
|
45
|
+
const image_normalize_1 = require("../../heart/attachments/image-normalize");
|
|
46
|
+
const store_1 = require("../../heart/attachments/store");
|
|
47
|
+
const types_1 = require("../../heart/attachments/types");
|
|
48
|
+
const materialize_1 = require("../../heart/attachments/materialize");
|
|
49
|
+
const attachment_download_1 = require("./attachment-download");
|
|
47
50
|
// Pin the exact wrapper strings so tests and code read from one source.
|
|
48
51
|
exports.VLM_TEXT_WRAPPERS = {
|
|
49
52
|
description: (desc) => `[image description: ${desc}]`,
|
|
50
|
-
unsupported: (mimeType) => `[image attachment not shown: unsupported format ${mimeType}]`,
|
|
51
53
|
failure: (reason) => `[image description failed: ${reason}]`,
|
|
52
54
|
};
|
|
53
|
-
// VLM supported formats for the /v1/coding_plan/vlm endpoint. The client in
|
|
54
|
-
// minimax-vlm.ts enforces the same set — this helper lets the sense skip the
|
|
55
|
-
// call entirely and emit a clearer `vision_format_unsupported` event instead
|
|
56
|
-
// of letting the VLM client throw.
|
|
57
|
-
const VLM_SUPPORTED_MIME_TYPES = new Set(["image/png", "image/jpeg", "image/webp"]);
|
|
58
|
-
function isSupportedVlmFormat(contentType) {
|
|
59
|
-
return VLM_SUPPORTED_MIME_TYPES.has(contentType.toLowerCase());
|
|
60
|
-
}
|
|
61
55
|
// D3 prompt template. Included here (not templated out) because the contract
|
|
62
56
|
// is load-bearing — see planning doc section D3 and AX-4.
|
|
63
57
|
function buildVlmPrompt(userText) {
|
|
64
58
|
const body = userText ?? "";
|
|
65
59
|
return `User message: "${body}"\n\nDescribe this image in detail, focusing on anything relevant to what the user said above.\nInclude any text visible in the image verbatim.`;
|
|
66
60
|
}
|
|
67
|
-
const MAX_ATTACHMENT_BYTES = 8 * 1024 * 1024;
|
|
68
61
|
const AUDIO_EXTENSIONS = new Set([".mp3", ".wav", ".m4a", ".caf", ".ogg"]);
|
|
69
|
-
const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".heif"]);
|
|
70
62
|
const AUDIO_EXTENSION_BY_CONTENT_TYPE = {
|
|
71
63
|
"audio/wav": ".wav",
|
|
72
64
|
"audio/x-wav": ".wav",
|
|
@@ -100,28 +92,9 @@ function whisperCppPaths() {
|
|
|
100
92
|
const modelPath = path.join(modelsDir, WHISPER_CPP_MODEL_NAME);
|
|
101
93
|
return { toolsDir, modelsDir, modelPath };
|
|
102
94
|
}
|
|
103
|
-
function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
|
|
104
|
-
const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
|
|
105
|
-
const url = new URL(endpoint.replace(/^\//, ""), root);
|
|
106
|
-
url.searchParams.set("password", password);
|
|
107
|
-
return url.toString();
|
|
108
|
-
}
|
|
109
95
|
function describeAttachment(attachment) {
|
|
110
96
|
return attachment.transferName?.trim() || attachment.guid?.trim() || "attachment";
|
|
111
97
|
}
|
|
112
|
-
function inferContentType(attachment, responseType) {
|
|
113
|
-
const normalizedResponseType = responseType?.split(";")[0]?.trim().toLowerCase();
|
|
114
|
-
if (normalizedResponseType) {
|
|
115
|
-
return normalizedResponseType;
|
|
116
|
-
}
|
|
117
|
-
return attachment.mimeType?.trim().toLowerCase() || undefined;
|
|
118
|
-
}
|
|
119
|
-
function isImageAttachment(attachment, contentType) {
|
|
120
|
-
if (contentType?.startsWith("image/"))
|
|
121
|
-
return true;
|
|
122
|
-
const extension = path.extname(attachment.transferName ?? "").toLowerCase();
|
|
123
|
-
return IMAGE_EXTENSIONS.has(extension);
|
|
124
|
-
}
|
|
125
98
|
function isAudioAttachment(attachment, contentType) {
|
|
126
99
|
if (contentType?.startsWith("audio/"))
|
|
127
100
|
return true;
|
|
@@ -268,34 +241,8 @@ async function transcribeAudioWithWhisperCpp(params, modelFetchImpl = fetch) {
|
|
|
268
241
|
await fs.rm(workDir, { recursive: true, force: true }).catch(() => undefined);
|
|
269
242
|
}
|
|
270
243
|
}
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
}
|
|
274
|
-
async function downloadAttachment(attachment, config, channelConfig, fetchImpl) {
|
|
275
|
-
const guid = attachment.guid?.trim();
|
|
276
|
-
if (!guid) {
|
|
277
|
-
throw new Error("attachment guid missing");
|
|
278
|
-
}
|
|
279
|
-
if (typeof attachment.totalBytes === "number" && attachment.totalBytes > MAX_ATTACHMENT_BYTES) {
|
|
280
|
-
throw new Error(`attachment exceeds ${MAX_ATTACHMENT_BYTES} byte limit`);
|
|
281
|
-
}
|
|
282
|
-
const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/attachment/${encodeURIComponent(guid)}/download`, config.password);
|
|
283
|
-
const response = await fetchImpl(url, {
|
|
284
|
-
method: "GET",
|
|
285
|
-
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
286
|
-
});
|
|
287
|
-
if (!response.ok) {
|
|
288
|
-
throw new Error(`HTTP ${response.status}`);
|
|
289
|
-
}
|
|
290
|
-
const buffer = Buffer.from(await response.arrayBuffer());
|
|
291
|
-
if (buffer.length > MAX_ATTACHMENT_BYTES) {
|
|
292
|
-
throw new Error(`attachment exceeds ${MAX_ATTACHMENT_BYTES} byte limit`);
|
|
293
|
-
}
|
|
294
|
-
return {
|
|
295
|
-
buffer,
|
|
296
|
-
contentType: inferContentType(attachment, response.headers.get("content-type")),
|
|
297
|
-
};
|
|
298
|
-
}
|
|
244
|
+
var attachment_download_2 = require("./attachment-download");
|
|
245
|
+
Object.defineProperty(exports, "downloadBlueBubblesAttachment", { enumerable: true, get: function () { return attachment_download_2.downloadBlueBubblesAttachment; } });
|
|
299
246
|
async function hydrateBlueBubblesAttachments(attachments, config, channelConfig, deps = {}) {
|
|
300
247
|
const fetchImpl = deps.fetchImpl ?? fetch;
|
|
301
248
|
const modelFetchImpl = deps.modelFetchImpl ?? fetch;
|
|
@@ -303,19 +250,28 @@ async function hydrateBlueBubblesAttachments(attachments, config, channelConfig,
|
|
|
303
250
|
const preferAudioInput = deps.preferAudioInput ?? false;
|
|
304
251
|
const chatModel = deps.chatModel;
|
|
305
252
|
const visionCapable = chatModel ? (0, model_capabilities_1.getModelCapabilities)(chatModel).vision === true : true;
|
|
253
|
+
const agentName = (0, identity_1.getAgentName)();
|
|
254
|
+
const agentRoot = (0, identity_1.getAgentRoot)(agentName);
|
|
306
255
|
const inputParts = [];
|
|
307
256
|
const transcriptAdditions = [];
|
|
308
257
|
const notices = [];
|
|
309
258
|
for (const attachment of attachments) {
|
|
310
259
|
const name = describeAttachment(attachment);
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
260
|
+
const initialRecord = attachment.guid?.trim()
|
|
261
|
+
? (0, store_1.rememberRecentAttachment)(agentName, (0, types_1.buildBlueBubblesAttachmentRecord)(attachment), agentRoot)
|
|
262
|
+
: null;
|
|
314
263
|
try {
|
|
315
|
-
const downloaded = await
|
|
264
|
+
const downloaded = await (0, attachment_download_1.downloadBlueBubblesAttachment)(attachment, config, channelConfig, fetchImpl);
|
|
316
265
|
const base64 = downloaded.buffer.toString("base64");
|
|
317
266
|
const byteCount = downloaded.buffer.length;
|
|
318
|
-
|
|
267
|
+
const persistedRecord = initialRecord && initialRecord.source === "bluebubbles"
|
|
268
|
+
? await (0, materialize_1.persistBlueBubblesAttachmentSource)(agentName, initialRecord, {
|
|
269
|
+
buffer: downloaded.buffer,
|
|
270
|
+
mimeType: downloaded.contentType,
|
|
271
|
+
byteCount,
|
|
272
|
+
}, agentRoot)
|
|
273
|
+
: null;
|
|
274
|
+
if ((0, attachment_download_1.isBlueBubblesImageAttachment)(attachment, downloaded.contentType)) {
|
|
319
275
|
const mimeType = downloaded.contentType ?? "application/octet-stream";
|
|
320
276
|
if (visionCapable) {
|
|
321
277
|
inputParts.push({
|
|
@@ -339,48 +295,27 @@ async function hydrateBlueBubblesAttachments(attachments, config, channelConfig,
|
|
|
339
295
|
});
|
|
340
296
|
continue;
|
|
341
297
|
}
|
|
342
|
-
// Non-vision chat model — try the VLM describe fallback.
|
|
343
|
-
if (!isSupportedVlmFormat(mimeType)) {
|
|
344
|
-
(0, runtime_1.emitNervesEvent)({
|
|
345
|
-
level: "warn",
|
|
346
|
-
component: "senses",
|
|
347
|
-
event: "senses.bluebubbles_vision_format_unsupported",
|
|
348
|
-
message: "bluebubbles vision format unsupported",
|
|
349
|
-
meta: {
|
|
350
|
-
mimeType,
|
|
351
|
-
transferName: attachment.transferName,
|
|
352
|
-
attachmentGuid: attachment.guid,
|
|
353
|
-
chatModel,
|
|
354
|
-
},
|
|
355
|
-
});
|
|
356
|
-
inputParts.push({
|
|
357
|
-
type: "text",
|
|
358
|
-
text: exports.VLM_TEXT_WRAPPERS.unsupported(mimeType),
|
|
359
|
-
});
|
|
360
|
-
(0, runtime_1.emitNervesEvent)({
|
|
361
|
-
level: "warn",
|
|
362
|
-
component: "senses",
|
|
363
|
-
event: "senses.bluebubbles_media_hydrate",
|
|
364
|
-
message: "bluebubbles media hydrate",
|
|
365
|
-
meta: {
|
|
366
|
-
attachmentGuid: attachment.guid,
|
|
367
|
-
mimeType,
|
|
368
|
-
byteCount,
|
|
369
|
-
hydrationPath: "skip-unsupported",
|
|
370
|
-
},
|
|
371
|
-
});
|
|
372
|
-
continue;
|
|
373
|
-
}
|
|
374
|
-
const dataUrl = `data:${mimeType};base64,${base64}`;
|
|
375
298
|
try {
|
|
376
299
|
if (!deps.vlmDescribe) {
|
|
377
300
|
throw new Error("no VLM describer configured — wire a vlmDescribe dep or configure a vision-capable chat model");
|
|
378
301
|
}
|
|
302
|
+
if (!persistedRecord?.sourceData.localPath) {
|
|
303
|
+
throw new Error("image attachment could not be persisted for normalization");
|
|
304
|
+
}
|
|
305
|
+
const normalized = await (0, image_normalize_1.normalizeImageForVision)({
|
|
306
|
+
attachment: persistedRecord,
|
|
307
|
+
sourcePath: persistedRecord.sourceData.localPath,
|
|
308
|
+
agentName,
|
|
309
|
+
agentRoot,
|
|
310
|
+
});
|
|
311
|
+
const normalizedBuffer = await fs.readFile(normalized.path);
|
|
312
|
+
const normalizedMime = normalized.mimeType ?? persistedRecord.mimeType ?? "image/jpeg";
|
|
313
|
+
const dataUrl = `data:${normalizedMime};base64,${normalizedBuffer.toString("base64")}`;
|
|
379
314
|
const description = await deps.vlmDescribe({
|
|
380
315
|
prompt: buildVlmPrompt(deps.userText),
|
|
381
316
|
imageDataUrl: dataUrl,
|
|
382
317
|
attachmentGuid: attachment.guid,
|
|
383
|
-
mimeType,
|
|
318
|
+
mimeType: normalizedMime,
|
|
384
319
|
chatModel,
|
|
385
320
|
});
|
|
386
321
|
inputParts.push({
|
|
@@ -402,7 +337,7 @@ async function hydrateBlueBubblesAttachments(attachments, config, channelConfig,
|
|
|
402
337
|
message: "bluebubbles media hydrate",
|
|
403
338
|
meta: {
|
|
404
339
|
attachmentGuid: attachment.guid,
|
|
405
|
-
mimeType,
|
|
340
|
+
mimeType: persistedRecord?.mimeType ?? mimeType,
|
|
406
341
|
byteCount,
|
|
407
342
|
hydrationPath: "vlm-describe",
|
|
408
343
|
},
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.normalizeBlueBubblesEvent = normalizeBlueBubblesEvent;
|
|
4
4
|
const runtime_1 = require("../../nerves/runtime");
|
|
5
|
+
const render_1 = require("../../heart/attachments/render");
|
|
6
|
+
const types_1 = require("../../heart/attachments/types");
|
|
5
7
|
function asRecord(value) {
|
|
6
8
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
7
9
|
? value
|
|
@@ -107,18 +109,14 @@ function extractAttachments(data) {
|
|
|
107
109
|
function formatAttachmentText(attachments) {
|
|
108
110
|
if (attachments.length === 0)
|
|
109
111
|
return "";
|
|
112
|
+
const renderable = attachments
|
|
113
|
+
.filter((attachment) => typeof attachment.guid === "string" && attachment.guid.trim().length > 0)
|
|
114
|
+
.map((attachment) => (0, types_1.buildBlueBubblesAttachmentRecord)(attachment));
|
|
115
|
+
if (renderable.length > 0) {
|
|
116
|
+
return (0, render_1.renderAttachmentBlock)(renderable);
|
|
117
|
+
}
|
|
110
118
|
const [first] = attachments;
|
|
111
|
-
|
|
112
|
-
const label = mime.startsWith("image/")
|
|
113
|
-
? "image attachment"
|
|
114
|
-
: mime.startsWith("audio/")
|
|
115
|
-
? "audio attachment"
|
|
116
|
-
: "attachment";
|
|
117
|
-
const name = first.transferName ? `: ${first.transferName}` : "";
|
|
118
|
-
const dimensions = typeof first.width === "number" && typeof first.height === "number" && first.width > 0 && first.height > 0
|
|
119
|
-
? ` (${first.width}x${first.height})`
|
|
120
|
-
: "";
|
|
121
|
-
return `[${label}${name}${dimensions}]`;
|
|
119
|
+
return `[attachment: ${first.transferName?.trim() || "unknown"}]`;
|
|
122
120
|
}
|
|
123
121
|
function formatMessageText(data, attachments) {
|
|
124
122
|
const text = readString(data, "text")?.trim() ?? "";
|
|
@@ -47,12 +47,19 @@ exports.resolveImageContent = resolveImageContent;
|
|
|
47
47
|
* When a user drags an image file onto the terminal, macOS pastes the absolute
|
|
48
48
|
* path (often with backslash-escaped spaces). This module detects image paths
|
|
49
49
|
* in the input text, replaces them with `[Image #N]` references, reads the
|
|
50
|
-
* files, and produces OpenAI-compatible
|
|
50
|
+
* files, registers them as shared attachments, and produces OpenAI-compatible
|
|
51
|
+
* `image_url` content blocks.
|
|
51
52
|
*/
|
|
53
|
+
const node_crypto_1 = require("node:crypto");
|
|
52
54
|
const promises_1 = require("fs/promises");
|
|
53
55
|
const child_process_1 = require("child_process");
|
|
54
56
|
const path = __importStar(require("path"));
|
|
55
57
|
const runtime_1 = require("../../nerves/runtime");
|
|
58
|
+
const identity_1 = require("../../heart/identity");
|
|
59
|
+
const render_1 = require("../../heart/attachments/render");
|
|
60
|
+
const store_1 = require("../../heart/attachments/store");
|
|
61
|
+
const types_1 = require("../../heart/attachments/types");
|
|
62
|
+
const materialize_1 = require("../../heart/attachments/materialize");
|
|
56
63
|
/** Matches image file extensions (case-insensitive) */
|
|
57
64
|
exports.IMAGE_EXTENSION_REGEX = /\.(png|jpe?g|gif|webp)$/i;
|
|
58
65
|
/** Check whether a string (possibly backslash-escaped) ends with an image extension */
|
|
@@ -96,8 +103,10 @@ async function tryReadImage(absolutePath) {
|
|
|
96
103
|
if (buffer.length > 0) {
|
|
97
104
|
const ext = path.extname(absolutePath);
|
|
98
105
|
return {
|
|
106
|
+
buffer: Buffer.from(buffer),
|
|
99
107
|
base64: Buffer.from(buffer).toString("base64"),
|
|
100
108
|
mediaType: extensionToMediaType(ext),
|
|
109
|
+
fromClipboard: false,
|
|
101
110
|
};
|
|
102
111
|
}
|
|
103
112
|
}
|
|
@@ -142,8 +151,10 @@ async function getImageFromClipboard() {
|
|
|
142
151
|
if (buffer.length === 0)
|
|
143
152
|
return null;
|
|
144
153
|
return {
|
|
154
|
+
buffer: Buffer.from(buffer),
|
|
145
155
|
base64: Buffer.from(buffer).toString("base64"),
|
|
146
156
|
mediaType: "image/png",
|
|
157
|
+
fromClipboard: true,
|
|
147
158
|
};
|
|
148
159
|
}
|
|
149
160
|
catch {
|
|
@@ -179,6 +190,34 @@ function replacePathsWithRefs(input) {
|
|
|
179
190
|
function processSubmitInput(text) {
|
|
180
191
|
return replacePathsWithRefs(text);
|
|
181
192
|
}
|
|
193
|
+
function extensionForMediaType(mediaType) {
|
|
194
|
+
switch (mediaType) {
|
|
195
|
+
case "image/jpeg":
|
|
196
|
+
return ".jpg";
|
|
197
|
+
case "image/gif":
|
|
198
|
+
return ".gif";
|
|
199
|
+
case "image/webp":
|
|
200
|
+
return ".webp";
|
|
201
|
+
default:
|
|
202
|
+
return ".png";
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async function pathExists(targetPath) {
|
|
206
|
+
try {
|
|
207
|
+
await (0, promises_1.access)(targetPath);
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async function persistClipboardImage(agentRoot, originalPath, image) {
|
|
215
|
+
const hash = (0, node_crypto_1.createHash)("sha1").update(originalPath).digest("hex").slice(0, 16);
|
|
216
|
+
const targetPath = path.join(agentRoot, "state", "attachments", "imports", "cli", `${hash}${extensionForMediaType(image.mediaType)}`);
|
|
217
|
+
await (0, promises_1.mkdir)(path.dirname(targetPath), { recursive: true });
|
|
218
|
+
await (0, promises_1.writeFile)(targetPath, image.buffer);
|
|
219
|
+
return targetPath;
|
|
220
|
+
}
|
|
182
221
|
/**
|
|
183
222
|
* Build OpenAI content parts from text + image map.
|
|
184
223
|
* For each image in the map, reads the file and creates an `image_url` content part.
|
|
@@ -189,10 +228,45 @@ async function resolveImageContent(text, images) {
|
|
|
189
228
|
if (images.size === 0) {
|
|
190
229
|
return [{ type: "text", text }];
|
|
191
230
|
}
|
|
231
|
+
const agentName = (0, identity_1.getAgentName)();
|
|
232
|
+
const agentRoot = (0, identity_1.getAgentRoot)(agentName);
|
|
192
233
|
const parts = [];
|
|
234
|
+
const rememberedAttachments = [];
|
|
193
235
|
// Read all images in parallel
|
|
194
236
|
const entries = Array.from(images.entries());
|
|
195
|
-
const results = await Promise.all(entries.map(async ([, absolutePath]) =>
|
|
237
|
+
const results = await Promise.all(entries.map(async ([, absolutePath]) => {
|
|
238
|
+
const image = await tryReadImage(absolutePath);
|
|
239
|
+
if (!image)
|
|
240
|
+
return null;
|
|
241
|
+
const sourcePath = image.fromClipboard || !await pathExists(absolutePath)
|
|
242
|
+
? await persistClipboardImage(agentRoot, absolutePath, image)
|
|
243
|
+
: absolutePath;
|
|
244
|
+
const attachment = (0, store_1.rememberRecentAttachment)(agentName, (0, types_1.buildCliLocalFileAttachmentRecord)({
|
|
245
|
+
path: sourcePath,
|
|
246
|
+
mimeType: image.mediaType,
|
|
247
|
+
byteCount: image.buffer.length,
|
|
248
|
+
}), agentRoot);
|
|
249
|
+
rememberedAttachments.push(attachment);
|
|
250
|
+
try {
|
|
251
|
+
const materialized = await (0, materialize_1.materializeAttachment)(agentName, attachment.id, {
|
|
252
|
+
agentRoot,
|
|
253
|
+
variant: "vision_safe",
|
|
254
|
+
});
|
|
255
|
+
const normalizedBuffer = await (0, promises_1.readFile)(materialized.path);
|
|
256
|
+
return {
|
|
257
|
+
attachment,
|
|
258
|
+
mediaType: materialized.mimeType ?? image.mediaType,
|
|
259
|
+
base64: Buffer.from(normalizedBuffer).toString("base64"),
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
catch {
|
|
263
|
+
return {
|
|
264
|
+
attachment,
|
|
265
|
+
mediaType: image.mediaType,
|
|
266
|
+
base64: image.base64,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
}));
|
|
196
270
|
for (const result of results) {
|
|
197
271
|
if (result) {
|
|
198
272
|
parts.push({
|
|
@@ -202,7 +276,9 @@ async function resolveImageContent(text, images) {
|
|
|
202
276
|
}
|
|
203
277
|
}
|
|
204
278
|
// Always include the text part
|
|
205
|
-
|
|
279
|
+
const attachmentBlock = (0, render_1.renderAttachmentBlock)(rememberedAttachments);
|
|
280
|
+
const textWithAttachments = [text, attachmentBlock].filter(Boolean).join("\n");
|
|
281
|
+
parts.push({ type: "text", text: textWithAttachments || text });
|
|
206
282
|
// If no images resolved, return just text
|
|
207
283
|
if (parts.length === 1) {
|
|
208
284
|
return [{ type: "text", text }];
|
|
@@ -58,6 +58,7 @@ const bundle_manifest_1 = require("../mind/bundle-manifest");
|
|
|
58
58
|
const pending_1 = require("../mind/pending");
|
|
59
59
|
const obligations_1 = require("../arc/obligations");
|
|
60
60
|
const attention_queue_1 = require("./attention-queue");
|
|
61
|
+
const packets_1 = require("../arc/packets");
|
|
61
62
|
const channel_1 = require("../mind/friends/channel");
|
|
62
63
|
const trust_gate_1 = require("./trust-gate");
|
|
63
64
|
const tokens_1 = require("../mind/friends/tokens");
|
|
@@ -714,6 +715,14 @@ async function runInnerDialogTurn(options) {
|
|
|
714
715
|
return null;
|
|
715
716
|
}
|
|
716
717
|
},
|
|
718
|
+
packetResolver: (packetId) => {
|
|
719
|
+
try {
|
|
720
|
+
return (0, packets_1.readPonderPacket)((0, identity_1.getAgentRoot)(agentName), packetId);
|
|
721
|
+
}
|
|
722
|
+
catch {
|
|
723
|
+
return null;
|
|
724
|
+
}
|
|
725
|
+
},
|
|
717
726
|
});
|
|
718
727
|
const summary = (0, attention_queue_1.buildAttentionQueueSummary)(attentionQueue);
|
|
719
728
|
return summary ? [summary] : [];
|
|
@@ -127,7 +127,7 @@ async function runSenseTurn(options) {
|
|
|
127
127
|
};
|
|
128
128
|
/* v8 ignore stop */
|
|
129
129
|
// Run the pipeline
|
|
130
|
-
|
|
130
|
+
await (0, pipeline_1.handleInboundTurn)({
|
|
131
131
|
channel,
|
|
132
132
|
sessionKey,
|
|
133
133
|
capabilities,
|
|
@@ -159,14 +159,10 @@ async function runSenseTurn(options) {
|
|
|
159
159
|
/* v8 ignore stop */
|
|
160
160
|
accumulateFriendTokens: tokens_1.accumulateFriendTokens,
|
|
161
161
|
});
|
|
162
|
-
|
|
163
|
-
const ponderDeferred = result.turnOutcome === "pondered";
|
|
162
|
+
const ponderDeferred = false;
|
|
164
163
|
// Build response
|
|
165
164
|
let finalResponse;
|
|
166
|
-
if (
|
|
167
|
-
finalResponse = "the agent is pondering this — check back shortly via check_response.";
|
|
168
|
-
}
|
|
169
|
-
else if (responseText.length === 0) {
|
|
165
|
+
if (responseText.length === 0) {
|
|
170
166
|
// Agent settled but no text came through callbacks — check session transcript for the settle answer
|
|
171
167
|
const postTurnSession = (0, context_1.loadSession)(sessPath);
|
|
172
168
|
if (postTurnSession?.messages) {
|