@ouro.bot/cli 0.1.0-alpha.16 → 0.1.0-alpha.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/heart/core.js +1 -1
- package/dist/heart/daemon/specialist-prompt.js +1 -0
- package/dist/heart/daemon/specialist-tools.js +8 -28
- package/dist/heart/streaming.js +55 -1
- package/dist/mind/prompt.js +3 -3
- package/dist/repertoire/coding/feedback.js +134 -0
- package/dist/repertoire/coding/index.js +4 -1
- package/dist/repertoire/coding/manager.js +61 -2
- package/dist/repertoire/coding/spawner.js +3 -3
- package/dist/repertoire/coding/tools.js +41 -2
- package/dist/repertoire/tools-base.js +69 -5
- package/dist/repertoire/tools.js +32 -9
- package/dist/senses/bluebubbles-client.js +159 -5
- package/dist/senses/bluebubbles-media.js +244 -0
- package/dist/senses/bluebubbles.js +108 -19
- package/dist/senses/debug-activity.js +107 -0
- package/package.json +1 -1
- package/subagents/work-doer.md +26 -24
- package/subagents/work-merger.md +24 -30
- package/subagents/work-planner.md +34 -25
|
@@ -46,6 +46,25 @@ const tasks_1 = require("./tasks");
|
|
|
46
46
|
const tools_1 = require("./coding/tools");
|
|
47
47
|
const memory_1 = require("../mind/memory");
|
|
48
48
|
const postIt = (msg) => `post-it from past you:\n${msg}`;
|
|
49
|
+
function normalizeOptionalText(value) {
|
|
50
|
+
if (typeof value !== "string")
|
|
51
|
+
return null;
|
|
52
|
+
const trimmed = value.trim();
|
|
53
|
+
return trimmed.length > 0 ? trimmed : null;
|
|
54
|
+
}
|
|
55
|
+
function buildTaskCreateInput(args) {
|
|
56
|
+
return {
|
|
57
|
+
title: args.title,
|
|
58
|
+
type: args.type,
|
|
59
|
+
category: args.category,
|
|
60
|
+
body: args.body,
|
|
61
|
+
status: normalizeOptionalText(args.status) ?? undefined,
|
|
62
|
+
validator: normalizeOptionalText(args.validator),
|
|
63
|
+
requester: normalizeOptionalText(args.requester),
|
|
64
|
+
cadence: normalizeOptionalText(args.cadence),
|
|
65
|
+
scheduledAt: normalizeOptionalText(args.scheduledAt),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
49
68
|
exports.baseToolDefinitions = [
|
|
50
69
|
{
|
|
51
70
|
tool: {
|
|
@@ -75,7 +94,11 @@ exports.baseToolDefinitions = [
|
|
|
75
94
|
},
|
|
76
95
|
},
|
|
77
96
|
},
|
|
78
|
-
handler: (a) =>
|
|
97
|
+
handler: (a) => {
|
|
98
|
+
fs.mkdirSync(path.dirname(a.path), { recursive: true });
|
|
99
|
+
fs.writeFileSync(a.path, a.content, "utf-8");
|
|
100
|
+
return "ok";
|
|
101
|
+
},
|
|
79
102
|
},
|
|
80
103
|
{
|
|
81
104
|
tool: {
|
|
@@ -235,7 +258,7 @@ exports.baseToolDefinitions = [
|
|
|
235
258
|
},
|
|
236
259
|
handler: (a) => {
|
|
237
260
|
try {
|
|
238
|
-
const result = (0, child_process_1.spawnSync)("claude", ["-p", "--dangerously-skip-permissions", "--add-dir", "."], {
|
|
261
|
+
const result = (0, child_process_1.spawnSync)("claude", ["-p", "--no-session-persistence", "--dangerously-skip-permissions", "--add-dir", "."], {
|
|
239
262
|
input: a.prompt,
|
|
240
263
|
encoding: "utf-8",
|
|
241
264
|
timeout: 60000,
|
|
@@ -394,7 +417,7 @@ exports.baseToolDefinitions = [
|
|
|
394
417
|
type: "function",
|
|
395
418
|
function: {
|
|
396
419
|
name: "task_create",
|
|
397
|
-
description: "create a new task in the bundle task system",
|
|
420
|
+
description: "create a new task in the bundle task system. optionally set `scheduledAt` for a one-time reminder or `cadence` for recurring daemon-scheduled work.",
|
|
398
421
|
parameters: {
|
|
399
422
|
type: "object",
|
|
400
423
|
properties: {
|
|
@@ -402,18 +425,59 @@ exports.baseToolDefinitions = [
|
|
|
402
425
|
type: { type: "string", enum: ["one-shot", "ongoing", "habit"] },
|
|
403
426
|
category: { type: "string" },
|
|
404
427
|
body: { type: "string" },
|
|
428
|
+
status: { type: "string" },
|
|
429
|
+
validator: { type: "string" },
|
|
430
|
+
requester: { type: "string" },
|
|
431
|
+
scheduledAt: { type: "string", description: "ISO timestamp for a one-time scheduled run/reminder" },
|
|
432
|
+
cadence: { type: "string", description: "recurrence like 30m, 1h, 1d, or cron" },
|
|
405
433
|
},
|
|
406
434
|
required: ["title", "type", "category", "body"],
|
|
407
435
|
},
|
|
408
436
|
},
|
|
409
437
|
},
|
|
410
438
|
handler: (a) => {
|
|
439
|
+
try {
|
|
440
|
+
const created = (0, tasks_1.getTaskModule)().createTask(buildTaskCreateInput(a));
|
|
441
|
+
return `created: ${created}`;
|
|
442
|
+
}
|
|
443
|
+
catch (error) {
|
|
444
|
+
return `error: ${error instanceof Error ? error.message : String(error)}`;
|
|
445
|
+
}
|
|
446
|
+
},
|
|
447
|
+
},
|
|
448
|
+
{
|
|
449
|
+
tool: {
|
|
450
|
+
type: "function",
|
|
451
|
+
function: {
|
|
452
|
+
name: "schedule_reminder",
|
|
453
|
+
description: "create a scheduled reminder or recurring daemon job. use `scheduledAt` for one-time reminders and `cadence` for recurring reminders. this writes canonical task fields that the daemon reconciles into OS-level jobs.",
|
|
454
|
+
parameters: {
|
|
455
|
+
type: "object",
|
|
456
|
+
properties: {
|
|
457
|
+
title: { type: "string" },
|
|
458
|
+
body: { type: "string" },
|
|
459
|
+
category: { type: "string" },
|
|
460
|
+
scheduledAt: { type: "string", description: "ISO timestamp for a one-time reminder" },
|
|
461
|
+
cadence: { type: "string", description: "recurrence like 30m, 1h, 1d, or cron" },
|
|
462
|
+
},
|
|
463
|
+
required: ["title", "body"],
|
|
464
|
+
},
|
|
465
|
+
},
|
|
466
|
+
},
|
|
467
|
+
handler: (a) => {
|
|
468
|
+
const scheduledAt = normalizeOptionalText(a.scheduledAt);
|
|
469
|
+
const cadence = normalizeOptionalText(a.cadence);
|
|
470
|
+
if (!scheduledAt && !cadence) {
|
|
471
|
+
return "error: provide scheduledAt or cadence";
|
|
472
|
+
}
|
|
411
473
|
try {
|
|
412
474
|
const created = (0, tasks_1.getTaskModule)().createTask({
|
|
413
475
|
title: a.title,
|
|
414
|
-
type:
|
|
415
|
-
category: a.category,
|
|
476
|
+
type: cadence ? "habit" : "one-shot",
|
|
477
|
+
category: normalizeOptionalText(a.category) ?? "reminder",
|
|
416
478
|
body: a.body,
|
|
479
|
+
scheduledAt,
|
|
480
|
+
cadence,
|
|
417
481
|
});
|
|
418
482
|
return `created: ${created}`;
|
|
419
483
|
}
|
package/dist/repertoire/tools.js
CHANGED
|
@@ -19,9 +19,29 @@ Object.defineProperty(exports, "teamsTools", { enumerable: true, get: function (
|
|
|
19
19
|
// All tool definitions in a single registry
|
|
20
20
|
const allDefinitions = [...tools_base_1.baseToolDefinitions, ...tools_teams_1.teamsToolDefinitions, ...ado_semantic_1.adoSemanticToolDefinitions, ...tools_github_1.githubToolDefinitions];
|
|
21
21
|
const REMOTE_BLOCKED_LOCAL_TOOLS = new Set(["shell", "read_file", "write_file", "git_commit", "gh_cli"]);
|
|
22
|
-
function
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
function isRemoteChannel(capabilities) {
|
|
23
|
+
return capabilities?.channel === "teams" || capabilities?.channel === "bluebubbles";
|
|
24
|
+
}
|
|
25
|
+
function isSharedRemoteContext(friend) {
|
|
26
|
+
const externalIds = friend.externalIds ?? [];
|
|
27
|
+
return externalIds.some((externalId) => externalId.externalId.startsWith("group:") || externalId.provider === "teams-conversation");
|
|
28
|
+
}
|
|
29
|
+
function isTrustedRemoteContext(context) {
|
|
30
|
+
if (!context?.friend || !isRemoteChannel(context.channel))
|
|
31
|
+
return false;
|
|
32
|
+
const trustLevel = context.friend.trustLevel ?? "stranger";
|
|
33
|
+
return trustLevel !== "stranger" && !isSharedRemoteContext(context.friend);
|
|
34
|
+
}
|
|
35
|
+
function shouldBlockLocalTools(capabilities, context) {
|
|
36
|
+
if (!isRemoteChannel(capabilities))
|
|
37
|
+
return false;
|
|
38
|
+
return !isTrustedRemoteContext(context);
|
|
39
|
+
}
|
|
40
|
+
function blockedLocalToolMessage() {
|
|
41
|
+
return "I can't do that from here because I'm talking to multiple people in a shared remote channel, and local shell/file/git/gh operations could let conversations interfere with each other. Ask me for a remote-safe alternative (Graph/ADO/web), or run that operation from CLI.";
|
|
42
|
+
}
|
|
43
|
+
function baseToolsForCapabilities(capabilities, context) {
|
|
44
|
+
if (!shouldBlockLocalTools(capabilities, context))
|
|
25
45
|
return tools_base_1.tools;
|
|
26
46
|
return tools_base_1.tools.filter((tool) => !REMOTE_BLOCKED_LOCAL_TOOLS.has(tool.function.name));
|
|
27
47
|
}
|
|
@@ -39,8 +59,8 @@ function applyPreference(tool, pref) {
|
|
|
39
59
|
// Base tools (no integration) are always included.
|
|
40
60
|
// Teams/integration tools are included only if their integration is in availableIntegrations.
|
|
41
61
|
// When toolPreferences is provided, matching preferences are appended to tool descriptions.
|
|
42
|
-
function getToolsForChannel(capabilities, toolPreferences) {
|
|
43
|
-
const baseTools = baseToolsForCapabilities(capabilities);
|
|
62
|
+
function getToolsForChannel(capabilities, toolPreferences, context) {
|
|
63
|
+
const baseTools = baseToolsForCapabilities(capabilities, context);
|
|
44
64
|
if (!capabilities || capabilities.availableIntegrations.length === 0) {
|
|
45
65
|
return baseTools;
|
|
46
66
|
}
|
|
@@ -89,9 +109,8 @@ async function execTool(name, args, ctx) {
|
|
|
89
109
|
});
|
|
90
110
|
return `unknown: ${name}`;
|
|
91
111
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const message = "I can't do that from here because I'm talking to multiple people in a shared remote channel, and local shell/file/git/gh operations could let conversations interfere with each other. Ask me for a remote-safe alternative (Graph/ADO/web), or run that operation from CLI.";
|
|
112
|
+
if (shouldBlockLocalTools(ctx?.context?.channel, ctx?.context) && REMOTE_BLOCKED_LOCAL_TOOLS.has(name)) {
|
|
113
|
+
const message = blockedLocalToolMessage();
|
|
95
114
|
(0, runtime_1.emitNervesEvent)({
|
|
96
115
|
level: "warn",
|
|
97
116
|
event: "tool.error",
|
|
@@ -165,7 +184,9 @@ function summarizeArgs(name, args) {
|
|
|
165
184
|
if (name === "load_skill")
|
|
166
185
|
return summarizeKeyValues(args, ["name"]);
|
|
167
186
|
if (name === "task_create")
|
|
168
|
-
return summarizeKeyValues(args, ["title", "type", "category"]);
|
|
187
|
+
return summarizeKeyValues(args, ["title", "type", "category", "scheduledAt", "cadence"]);
|
|
188
|
+
if (name === "schedule_reminder")
|
|
189
|
+
return summarizeKeyValues(args, ["title", "scheduledAt", "cadence"]);
|
|
169
190
|
if (name === "task_update_status")
|
|
170
191
|
return summarizeKeyValues(args, ["name", "status"]);
|
|
171
192
|
if (name === "task_board_status")
|
|
@@ -178,6 +199,8 @@ function summarizeArgs(name, args) {
|
|
|
178
199
|
return summarizeKeyValues(args, ["runner", "workdir", "taskRef"]);
|
|
179
200
|
if (name === "coding_status")
|
|
180
201
|
return summarizeKeyValues(args, ["sessionId"]);
|
|
202
|
+
if (name === "coding_tail")
|
|
203
|
+
return summarizeKeyValues(args, ["sessionId"]);
|
|
181
204
|
if (name === "coding_send_input")
|
|
182
205
|
return summarizeKeyValues(args, ["sessionId", "input"]);
|
|
183
206
|
if (name === "coding_kill")
|
|
@@ -3,8 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.createBlueBubblesClient = createBlueBubblesClient;
|
|
4
4
|
const node_crypto_1 = require("node:crypto");
|
|
5
5
|
const config_1 = require("../heart/config");
|
|
6
|
+
const identity_1 = require("../heart/identity");
|
|
6
7
|
const runtime_1 = require("../nerves/runtime");
|
|
7
8
|
const bluebubbles_model_1 = require("./bluebubbles-model");
|
|
9
|
+
const bluebubbles_media_1 = require("./bluebubbles-media");
|
|
8
10
|
function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
|
|
9
11
|
const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
|
|
10
12
|
const url = new URL(endpoint.replace(/^\//, ""), root);
|
|
@@ -16,6 +18,10 @@ function asRecord(value) {
|
|
|
16
18
|
? value
|
|
17
19
|
: null;
|
|
18
20
|
}
|
|
21
|
+
function readString(record, key) {
|
|
22
|
+
const value = record[key];
|
|
23
|
+
return typeof value === "string" ? value : undefined;
|
|
24
|
+
}
|
|
19
25
|
function extractMessageGuid(payload) {
|
|
20
26
|
if (!payload || typeof payload !== "object")
|
|
21
27
|
return undefined;
|
|
@@ -55,6 +61,75 @@ function buildRepairUrl(baseUrl, messageGuid, password) {
|
|
|
55
61
|
parsed.searchParams.set("with", "attachments,payloadData,chats,messageSummaryInfo");
|
|
56
62
|
return parsed.toString();
|
|
57
63
|
}
|
|
64
|
+
function extractChatIdentifierFromGuid(chatGuid) {
|
|
65
|
+
const parts = chatGuid.split(";");
|
|
66
|
+
return parts.length >= 3 ? parts[2]?.trim() || undefined : undefined;
|
|
67
|
+
}
|
|
68
|
+
function extractChatGuid(value) {
|
|
69
|
+
const record = asRecord(value);
|
|
70
|
+
const candidates = [
|
|
71
|
+
record?.chatGuid,
|
|
72
|
+
record?.guid,
|
|
73
|
+
record?.chat_guid,
|
|
74
|
+
record?.identifier,
|
|
75
|
+
record?.chatIdentifier,
|
|
76
|
+
record?.chat_identifier,
|
|
77
|
+
];
|
|
78
|
+
for (const candidate of candidates) {
|
|
79
|
+
if (typeof candidate === "string" && candidate.trim()) {
|
|
80
|
+
return candidate.trim();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
function extractQueriedChatIdentifier(chat, chatGuid) {
|
|
86
|
+
const explicitIdentifier = readString(chat, "chatIdentifier")
|
|
87
|
+
?? readString(chat, "identifier")
|
|
88
|
+
?? readString(chat, "chat_identifier");
|
|
89
|
+
if (explicitIdentifier) {
|
|
90
|
+
return explicitIdentifier;
|
|
91
|
+
}
|
|
92
|
+
return extractChatIdentifierFromGuid(chatGuid);
|
|
93
|
+
}
|
|
94
|
+
function extractChatQueryRows(payload) {
|
|
95
|
+
const record = asRecord(payload);
|
|
96
|
+
const data = Array.isArray(record?.data) ? record.data : payload;
|
|
97
|
+
if (!Array.isArray(data)) {
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
return data.map((entry) => asRecord(entry)).filter((entry) => entry !== null);
|
|
101
|
+
}
|
|
102
|
+
async function resolveChatGuidForIdentifier(config, channelConfig, chatIdentifier) {
|
|
103
|
+
const trimmedIdentifier = chatIdentifier.trim();
|
|
104
|
+
if (!trimmedIdentifier)
|
|
105
|
+
return undefined;
|
|
106
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, "/api/v1/chat/query", config.password);
|
|
107
|
+
const response = await fetch(url, {
|
|
108
|
+
method: "POST",
|
|
109
|
+
headers: { "Content-Type": "application/json" },
|
|
110
|
+
body: JSON.stringify({
|
|
111
|
+
limit: 500,
|
|
112
|
+
offset: 0,
|
|
113
|
+
with: ["participants"],
|
|
114
|
+
}),
|
|
115
|
+
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
116
|
+
});
|
|
117
|
+
if (!response.ok) {
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
const payload = await parseJsonBody(response);
|
|
121
|
+
const rows = extractChatQueryRows(payload);
|
|
122
|
+
for (const row of rows) {
|
|
123
|
+
const guid = extractChatGuid(row);
|
|
124
|
+
if (!guid)
|
|
125
|
+
continue;
|
|
126
|
+
const identifier = extractQueriedChatIdentifier(row, guid);
|
|
127
|
+
if (identifier === trimmedIdentifier || guid === trimmedIdentifier) {
|
|
128
|
+
return guid;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return undefined;
|
|
132
|
+
}
|
|
58
133
|
function collectPreviewStrings(value, out, depth = 0) {
|
|
59
134
|
if (depth > 4 || out.length >= 4)
|
|
60
135
|
return;
|
|
@@ -120,6 +195,13 @@ function extractRepairData(payload) {
|
|
|
120
195
|
const record = asRecord(payload);
|
|
121
196
|
return asRecord(record?.data) ?? record;
|
|
122
197
|
}
|
|
198
|
+
function providerSupportsAudioInput(provider) {
|
|
199
|
+
return provider === "azure" || provider === "openai-codex";
|
|
200
|
+
}
|
|
201
|
+
async function resolveChatGuid(chat, config, channelConfig) {
|
|
202
|
+
return chat.chatGuid
|
|
203
|
+
?? await resolveChatGuidForIdentifier(config, channelConfig, chat.chatIdentifier ?? "");
|
|
204
|
+
}
|
|
123
205
|
function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(), channelConfig = (0, config_1.getBlueBubblesChannelConfig)()) {
|
|
124
206
|
return {
|
|
125
207
|
async sendText(params) {
|
|
@@ -127,12 +209,13 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
127
209
|
if (!trimmedText) {
|
|
128
210
|
throw new Error("BlueBubbles send requires non-empty text.");
|
|
129
211
|
}
|
|
130
|
-
|
|
212
|
+
const resolvedChatGuid = await resolveChatGuid(params.chat, config, channelConfig);
|
|
213
|
+
if (!resolvedChatGuid) {
|
|
131
214
|
throw new Error("BlueBubbles send currently requires chat.chatGuid from the inbound event.");
|
|
132
215
|
}
|
|
133
216
|
const url = buildBlueBubblesApiUrl(config.serverUrl, "/api/v1/message/text", config.password);
|
|
134
217
|
const body = {
|
|
135
|
-
chatGuid:
|
|
218
|
+
chatGuid: resolvedChatGuid,
|
|
136
219
|
tempGuid: (0, node_crypto_1.randomUUID)(),
|
|
137
220
|
message: trimmedText,
|
|
138
221
|
};
|
|
@@ -146,7 +229,7 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
146
229
|
event: "senses.bluebubbles_send_start",
|
|
147
230
|
message: "sending bluebubbles message",
|
|
148
231
|
meta: {
|
|
149
|
-
chatGuid:
|
|
232
|
+
chatGuid: resolvedChatGuid,
|
|
150
233
|
hasReplyTarget: Boolean(params.replyToMessageGuid?.trim()),
|
|
151
234
|
},
|
|
152
235
|
});
|
|
@@ -177,12 +260,68 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
177
260
|
event: "senses.bluebubbles_send_end",
|
|
178
261
|
message: "bluebubbles message sent",
|
|
179
262
|
meta: {
|
|
180
|
-
chatGuid:
|
|
263
|
+
chatGuid: resolvedChatGuid,
|
|
181
264
|
messageGuid: messageGuid ?? null,
|
|
182
265
|
},
|
|
183
266
|
});
|
|
184
267
|
return { messageGuid };
|
|
185
268
|
},
|
|
269
|
+
async editMessage(params) {
|
|
270
|
+
const messageGuid = params.messageGuid.trim();
|
|
271
|
+
const text = params.text.trim();
|
|
272
|
+
if (!messageGuid) {
|
|
273
|
+
throw new Error("BlueBubbles edit requires messageGuid.");
|
|
274
|
+
}
|
|
275
|
+
if (!text) {
|
|
276
|
+
throw new Error("BlueBubbles edit requires non-empty text.");
|
|
277
|
+
}
|
|
278
|
+
const editTimeoutMs = Math.max(channelConfig.requestTimeoutMs, 120000);
|
|
279
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/message/${encodeURIComponent(messageGuid)}/edit`, config.password);
|
|
280
|
+
const response = await fetch(url, {
|
|
281
|
+
method: "POST",
|
|
282
|
+
headers: { "Content-Type": "application/json" },
|
|
283
|
+
body: JSON.stringify({
|
|
284
|
+
editedMessage: text,
|
|
285
|
+
backwardsCompatibilityMessage: params.backwardsCompatibilityMessage ?? `Edited to: ${text}`,
|
|
286
|
+
partIndex: typeof params.partIndex === "number" ? params.partIndex : 0,
|
|
287
|
+
}),
|
|
288
|
+
signal: AbortSignal.timeout(editTimeoutMs),
|
|
289
|
+
});
|
|
290
|
+
if (!response.ok) {
|
|
291
|
+
const errorText = await response.text().catch(() => "");
|
|
292
|
+
throw new Error(`BlueBubbles edit failed (${response.status}): ${errorText || "unknown"}`);
|
|
293
|
+
}
|
|
294
|
+
},
|
|
295
|
+
async setTyping(chat, typing) {
|
|
296
|
+
const resolvedChatGuid = await resolveChatGuid(chat, config, channelConfig);
|
|
297
|
+
if (!resolvedChatGuid) {
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/chat/${encodeURIComponent(resolvedChatGuid)}/typing`, config.password);
|
|
301
|
+
const response = await fetch(url, {
|
|
302
|
+
method: typing ? "POST" : "DELETE",
|
|
303
|
+
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
304
|
+
});
|
|
305
|
+
if (!response.ok) {
|
|
306
|
+
const errorText = await response.text().catch(() => "");
|
|
307
|
+
throw new Error(`BlueBubbles typing failed (${response.status}): ${errorText || "unknown"}`);
|
|
308
|
+
}
|
|
309
|
+
},
|
|
310
|
+
async markChatRead(chat) {
|
|
311
|
+
const resolvedChatGuid = await resolveChatGuid(chat, config, channelConfig);
|
|
312
|
+
if (!resolvedChatGuid) {
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/chat/${encodeURIComponent(resolvedChatGuid)}/read`, config.password);
|
|
316
|
+
const response = await fetch(url, {
|
|
317
|
+
method: "POST",
|
|
318
|
+
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
319
|
+
});
|
|
320
|
+
if (!response.ok) {
|
|
321
|
+
const errorText = await response.text().catch(() => "");
|
|
322
|
+
throw new Error(`BlueBubbles read failed (${response.status}): ${errorText || "unknown"}`);
|
|
323
|
+
}
|
|
324
|
+
},
|
|
186
325
|
async repairEvent(event) {
|
|
187
326
|
if (!event.requiresRepair) {
|
|
188
327
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -247,7 +386,22 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
|
|
|
247
386
|
type: event.eventType,
|
|
248
387
|
data,
|
|
249
388
|
});
|
|
250
|
-
|
|
389
|
+
let hydrated = hydrateTextForAgent(normalized, data);
|
|
390
|
+
if (hydrated.kind === "message" &&
|
|
391
|
+
hydrated.balloonBundleId !== "com.apple.messages.URLBalloonProvider" &&
|
|
392
|
+
hydrated.attachments.length > 0) {
|
|
393
|
+
const media = await (0, bluebubbles_media_1.hydrateBlueBubblesAttachments)(hydrated.attachments, config, channelConfig, {
|
|
394
|
+
preferAudioInput: providerSupportsAudioInput((0, identity_1.loadAgentConfig)().provider),
|
|
395
|
+
});
|
|
396
|
+
const transcriptSuffix = media.transcriptAdditions.map((entry) => `[${entry}]`).join("\n");
|
|
397
|
+
const noticeSuffix = media.notices.map((entry) => `[${entry}]`).join("\n");
|
|
398
|
+
const combinedSuffix = [transcriptSuffix, noticeSuffix].filter(Boolean).join("\n");
|
|
399
|
+
hydrated = {
|
|
400
|
+
...hydrated,
|
|
401
|
+
inputPartsForAgent: media.inputParts.length > 0 ? media.inputParts : undefined,
|
|
402
|
+
textForAgent: [hydrated.textForAgent, combinedSuffix].filter(Boolean).join("\n"),
|
|
403
|
+
};
|
|
404
|
+
}
|
|
251
405
|
(0, runtime_1.emitNervesEvent)({
|
|
252
406
|
component: "senses",
|
|
253
407
|
event: "senses.bluebubbles_repair_end",
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.hydrateBlueBubblesAttachments = hydrateBlueBubblesAttachments;
|
|
37
|
+
const node_child_process_1 = require("node:child_process");
|
|
38
|
+
const fs = __importStar(require("node:fs/promises"));
|
|
39
|
+
const os = __importStar(require("node:os"));
|
|
40
|
+
const path = __importStar(require("node:path"));
|
|
41
|
+
const runtime_1 = require("../nerves/runtime");
|
|
42
|
+
const MAX_ATTACHMENT_BYTES = 8 * 1024 * 1024;
|
|
43
|
+
const AUDIO_EXTENSIONS = new Set([".mp3", ".wav", ".m4a", ".caf", ".ogg"]);
|
|
44
|
+
const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".heif"]);
|
|
45
|
+
const AUDIO_EXTENSION_BY_CONTENT_TYPE = {
|
|
46
|
+
"audio/wav": ".wav",
|
|
47
|
+
"audio/x-wav": ".wav",
|
|
48
|
+
"audio/mp3": ".mp3",
|
|
49
|
+
"audio/mpeg": ".mp3",
|
|
50
|
+
"audio/x-caf": ".caf",
|
|
51
|
+
"audio/caf": ".caf",
|
|
52
|
+
"audio/mp4": ".m4a",
|
|
53
|
+
"audio/x-m4a": ".m4a",
|
|
54
|
+
};
|
|
55
|
+
const AUDIO_INPUT_FORMAT_BY_CONTENT_TYPE = {
|
|
56
|
+
"audio/wav": "wav",
|
|
57
|
+
"audio/x-wav": "wav",
|
|
58
|
+
"audio/mp3": "mp3",
|
|
59
|
+
"audio/mpeg": "mp3",
|
|
60
|
+
};
|
|
61
|
+
const AUDIO_INPUT_FORMAT_BY_EXTENSION = {
|
|
62
|
+
".wav": "wav",
|
|
63
|
+
".mp3": "mp3",
|
|
64
|
+
};
|
|
65
|
+
function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
|
|
66
|
+
const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
|
|
67
|
+
const url = new URL(endpoint.replace(/^\//, ""), root);
|
|
68
|
+
url.searchParams.set("password", password);
|
|
69
|
+
return url.toString();
|
|
70
|
+
}
|
|
71
|
+
function describeAttachment(attachment) {
|
|
72
|
+
return attachment.transferName?.trim() || attachment.guid?.trim() || "attachment";
|
|
73
|
+
}
|
|
74
|
+
function inferContentType(attachment, responseType) {
|
|
75
|
+
const normalizedResponseType = responseType?.split(";")[0]?.trim().toLowerCase();
|
|
76
|
+
if (normalizedResponseType) {
|
|
77
|
+
return normalizedResponseType;
|
|
78
|
+
}
|
|
79
|
+
return attachment.mimeType?.trim().toLowerCase() || undefined;
|
|
80
|
+
}
|
|
81
|
+
function isImageAttachment(attachment, contentType) {
|
|
82
|
+
if (contentType?.startsWith("image/"))
|
|
83
|
+
return true;
|
|
84
|
+
const extension = path.extname(attachment.transferName ?? "").toLowerCase();
|
|
85
|
+
return IMAGE_EXTENSIONS.has(extension);
|
|
86
|
+
}
|
|
87
|
+
function isAudioAttachment(attachment, contentType) {
|
|
88
|
+
if (contentType?.startsWith("audio/"))
|
|
89
|
+
return true;
|
|
90
|
+
const extension = path.extname(attachment.transferName ?? "").toLowerCase();
|
|
91
|
+
return AUDIO_EXTENSIONS.has(extension);
|
|
92
|
+
}
|
|
93
|
+
function sanitizeFilename(name) {
|
|
94
|
+
return path.basename(name).replace(/[\r\n"\\]/g, "_");
|
|
95
|
+
}
|
|
96
|
+
function fileExtensionForAudio(attachment, contentType) {
|
|
97
|
+
const transferExt = path.extname(attachment.transferName ?? "").toLowerCase();
|
|
98
|
+
if (transferExt) {
|
|
99
|
+
return transferExt;
|
|
100
|
+
}
|
|
101
|
+
if (contentType && AUDIO_EXTENSION_BY_CONTENT_TYPE[contentType]) {
|
|
102
|
+
return AUDIO_EXTENSION_BY_CONTENT_TYPE[contentType];
|
|
103
|
+
}
|
|
104
|
+
return ".audio";
|
|
105
|
+
}
|
|
106
|
+
function audioFormatForInput(contentType, attachment) {
|
|
107
|
+
const extension = path.extname(attachment?.transferName ?? "").toLowerCase();
|
|
108
|
+
return AUDIO_INPUT_FORMAT_BY_CONTENT_TYPE[contentType ?? ""] ?? AUDIO_INPUT_FORMAT_BY_EXTENSION[extension];
|
|
109
|
+
}
|
|
110
|
+
async function transcribeAudioWithWhisper(params) {
|
|
111
|
+
const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "ouro-bb-audio-"));
|
|
112
|
+
const filename = sanitizeFilename(describeAttachment(params.attachment));
|
|
113
|
+
const extension = fileExtensionForAudio(params.attachment, params.contentType);
|
|
114
|
+
const audioPath = path.join(workDir, `${path.parse(filename).name}${extension}`);
|
|
115
|
+
try {
|
|
116
|
+
await fs.writeFile(audioPath, params.buffer);
|
|
117
|
+
await new Promise((resolve, reject) => {
|
|
118
|
+
(0, node_child_process_1.execFile)("whisper", [
|
|
119
|
+
audioPath,
|
|
120
|
+
"--model",
|
|
121
|
+
"turbo",
|
|
122
|
+
"--output_dir",
|
|
123
|
+
workDir,
|
|
124
|
+
"--output_format",
|
|
125
|
+
"json",
|
|
126
|
+
"--verbose",
|
|
127
|
+
"False",
|
|
128
|
+
], { timeout: Math.max(params.timeoutMs, 120000) }, (error) => {
|
|
129
|
+
if (error) {
|
|
130
|
+
reject(error);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
resolve();
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
const transcriptPath = path.join(workDir, `${path.parse(audioPath).name}.json`);
|
|
137
|
+
const raw = await fs.readFile(transcriptPath, "utf8");
|
|
138
|
+
const parsed = JSON.parse(raw);
|
|
139
|
+
return typeof parsed.text === "string" ? parsed.text.trim() : "";
|
|
140
|
+
}
|
|
141
|
+
finally {
|
|
142
|
+
await fs.rm(workDir, { recursive: true, force: true }).catch(() => undefined);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async function downloadAttachment(attachment, config, channelConfig, fetchImpl) {
|
|
146
|
+
const guid = attachment.guid?.trim();
|
|
147
|
+
if (!guid) {
|
|
148
|
+
throw new Error("attachment guid missing");
|
|
149
|
+
}
|
|
150
|
+
if (typeof attachment.totalBytes === "number" && attachment.totalBytes > MAX_ATTACHMENT_BYTES) {
|
|
151
|
+
throw new Error(`attachment exceeds ${MAX_ATTACHMENT_BYTES} byte limit`);
|
|
152
|
+
}
|
|
153
|
+
const url = buildBlueBubblesApiUrl(config.serverUrl, `/api/v1/attachment/${encodeURIComponent(guid)}/download`, config.password);
|
|
154
|
+
const response = await fetchImpl(url, {
|
|
155
|
+
method: "GET",
|
|
156
|
+
signal: AbortSignal.timeout(channelConfig.requestTimeoutMs),
|
|
157
|
+
});
|
|
158
|
+
if (!response.ok) {
|
|
159
|
+
throw new Error(`HTTP ${response.status}`);
|
|
160
|
+
}
|
|
161
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
162
|
+
if (buffer.length > MAX_ATTACHMENT_BYTES) {
|
|
163
|
+
throw new Error(`attachment exceeds ${MAX_ATTACHMENT_BYTES} byte limit`);
|
|
164
|
+
}
|
|
165
|
+
return {
|
|
166
|
+
buffer,
|
|
167
|
+
contentType: inferContentType(attachment, response.headers.get("content-type")),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
async function hydrateBlueBubblesAttachments(attachments, config, channelConfig, deps = {}) {
|
|
171
|
+
(0, runtime_1.emitNervesEvent)({
|
|
172
|
+
component: "senses",
|
|
173
|
+
event: "senses.bluebubbles_media_hydrate",
|
|
174
|
+
message: "hydrating bluebubbles attachments",
|
|
175
|
+
meta: {
|
|
176
|
+
attachmentCount: attachments.length,
|
|
177
|
+
preferAudioInput: deps.preferAudioInput ?? false,
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
const fetchImpl = deps.fetchImpl ?? fetch;
|
|
181
|
+
const transcribeAudio = deps.transcribeAudio ?? transcribeAudioWithWhisper;
|
|
182
|
+
const preferAudioInput = deps.preferAudioInput ?? false;
|
|
183
|
+
const inputParts = [];
|
|
184
|
+
const transcriptAdditions = [];
|
|
185
|
+
const notices = [];
|
|
186
|
+
for (const attachment of attachments) {
|
|
187
|
+
const name = describeAttachment(attachment);
|
|
188
|
+
try {
|
|
189
|
+
const downloaded = await downloadAttachment(attachment, config, channelConfig, fetchImpl);
|
|
190
|
+
const base64 = downloaded.buffer.toString("base64");
|
|
191
|
+
if (isImageAttachment(attachment, downloaded.contentType)) {
|
|
192
|
+
inputParts.push({
|
|
193
|
+
type: "image_url",
|
|
194
|
+
image_url: {
|
|
195
|
+
url: `data:${downloaded.contentType ?? "application/octet-stream"};base64,${base64}`,
|
|
196
|
+
detail: "auto",
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (isAudioAttachment(attachment, downloaded.contentType)) {
|
|
202
|
+
const audioFormat = audioFormatForInput(downloaded.contentType, attachment);
|
|
203
|
+
if (preferAudioInput && audioFormat) {
|
|
204
|
+
inputParts.push({
|
|
205
|
+
type: "input_audio",
|
|
206
|
+
input_audio: {
|
|
207
|
+
data: base64,
|
|
208
|
+
format: audioFormat,
|
|
209
|
+
},
|
|
210
|
+
});
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
const transcript = (await transcribeAudio({
|
|
214
|
+
attachment,
|
|
215
|
+
buffer: downloaded.buffer,
|
|
216
|
+
contentType: downloaded.contentType,
|
|
217
|
+
timeoutMs: channelConfig.requestTimeoutMs,
|
|
218
|
+
})).trim();
|
|
219
|
+
if (!transcript) {
|
|
220
|
+
notices.push(`attachment hydration failed for ${name}: empty audio transcript`);
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
transcriptAdditions.push(`voice note transcript: ${transcript}`);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
inputParts.push({
|
|
227
|
+
type: "file",
|
|
228
|
+
file: {
|
|
229
|
+
file_data: base64,
|
|
230
|
+
filename: sanitizeFilename(name),
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
catch (error) {
|
|
235
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
236
|
+
notices.push(`attachment hydration failed for ${name}: ${reason}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
inputParts,
|
|
241
|
+
transcriptAdditions,
|
|
242
|
+
notices,
|
|
243
|
+
};
|
|
244
|
+
}
|