@qearlyao/familiar 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -14
- package/config.example.toml +1 -1
- package/dist/added-models.js +6 -15
- package/dist/agent-events.js +1 -3
- package/dist/agent.js +3 -4
- package/dist/browser-tools.js +15 -11
- package/dist/chat-log.js +3 -2
- package/dist/cli.js +2 -2
- package/dist/config-overrides.js +5 -14
- package/dist/config-registry.js +1 -4
- package/dist/config.js +45 -113
- package/dist/contact-note.js +2 -12
- package/dist/data-retention.js +1 -3
- package/dist/discord.js +72 -19
- package/dist/generated-media.js +3 -2
- package/dist/hot-reload.js +1 -3
- package/dist/image-gen.js +12 -51
- package/dist/inbound-attachments.js +64 -23
- package/dist/memory/diary/ambient-injector.js +1 -3
- package/dist/memory/diary/ambient.js +1 -3
- package/dist/memory/diary/chunks.js +1 -3
- package/dist/memory/diary/indexer.js +1 -3
- package/dist/memory/doctor.js +3 -8
- package/dist/memory/index/chunk-indexer.js +27 -6
- package/dist/memory/index/retrieval.js +1 -3
- package/dist/memory/index/store.js +47 -19
- package/dist/memory/lcm/backfill.js +19 -16
- package/dist/memory/lcm/context-transformer.js +17 -29
- package/dist/memory/lcm/context.js +10 -4
- package/dist/memory/lcm/eviction-score.js +25 -13
- package/dist/memory/lcm/indexer.js +1 -5
- package/dist/memory/lcm/normalize.js +22 -1
- package/dist/memory/lcm/store.js +27 -24
- package/dist/memory/operator.js +3 -31
- package/dist/memory/service.js +1 -3
- package/dist/memory/tools.js +0 -4
- package/dist/memory/util.js +6 -0
- package/dist/models.js +3 -0
- package/dist/persona.js +3 -15
- package/dist/runtime.js +12 -23
- package/dist/scheduler.js +15 -49
- package/dist/service.js +39 -27
- package/dist/settings.js +7 -32
- package/dist/silent-marker.js +64 -0
- package/dist/tts.js +0 -6
- package/dist/util/fs.js +41 -0
- package/dist/util/guards.js +8 -0
- package/dist/util/image-mime.js +31 -0
- package/dist/util/time.js +29 -0
- package/dist/web-auth.js +4 -1
- package/dist/web-static.js +36 -1
- package/dist/web-tools.js +8 -5
- package/dist/web.js +253 -69
- package/npm-shrinkwrap.json +5139 -0
- package/package.json +5 -4
- package/web/dist/assets/index-B23WT77N.js +63 -0
- package/web/dist/assets/index-D3MotFzN.css +2 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BPZQbZh5.js +0 -61
- package/web/dist/assets/index-CcQ13VAY.css +0 -2
package/dist/discord.js
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { once } from "node:events";
|
|
3
3
|
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { extname } from "node:path";
|
|
4
5
|
import { ApplicationCommandOptionType, ApplicationCommandType, ApplicationIntegrationType, ChannelType, Client, Events, GatewayIntentBits, InteractionContextType, MessageFlags, Partials, } from "discord.js";
|
|
5
6
|
import { createAgentEventRecorder, storedAgentEventFromAgentEvent, thinkingDurationMs, updateAgentEventSummary, } from "./agent-events.js";
|
|
6
7
|
import { chatChannelKey, createChatLog } from "./chat-log.js";
|
|
7
8
|
import { materializeInboundAttachments, promptImagesFromAttachments } from "./inbound-attachments.js";
|
|
8
9
|
import { ConversationRuntime } from "./runtime.js";
|
|
9
|
-
import { appendSchedulerLog, buildCronInjectionText, buildHeartbeatInjectionText, dueCronSlot, isHeartbeatDue, loadSchedulerState, saveSchedulerState, } from "./scheduler.js";
|
|
10
|
+
import { appendSchedulerLog, buildCronInjectionText, buildHeartbeatInjectionText, dueCronSlot, formatIdleDuration, isHeartbeatDue, loadSchedulerState, saveSchedulerState, } from "./scheduler.js";
|
|
11
|
+
import { parseAgentReply as parseSilentMarker } from "./silent-marker.js";
|
|
10
12
|
const FAMILIAR_COMMAND_NAME = "familiar";
|
|
11
13
|
const THINKING_CHOICES = ["off", "minimal", "low", "medium", "high", "xhigh"];
|
|
12
14
|
const CHANNEL_TRIGGER_CHOICES = ["mention", "always"];
|
|
13
15
|
const EPHEMERAL_REPLY = MessageFlags.Ephemeral;
|
|
14
|
-
const SILENT_RESPONSE_MARKER = "[[FAMILIAR_SILENT]]";
|
|
15
16
|
const HEARTBEAT_SKIPPED = Symbol("heartbeat-skipped");
|
|
16
17
|
const CRON_SKIPPED = Symbol("cron-skipped");
|
|
18
|
+
const DISCORD_ATTACHMENT_SEND_TIMEOUT_MS = 20_000;
|
|
17
19
|
async function withReadyClient(token) {
|
|
18
20
|
const client = new Client({
|
|
19
21
|
intents: [
|
|
@@ -327,12 +329,19 @@ async function delayBetweenBurstChunks(config, channel) {
|
|
|
327
329
|
function normalizeOutboundText(text) {
|
|
328
330
|
return text.trim() || "(empty response)";
|
|
329
331
|
}
|
|
332
|
+
function fallbackMimeType(name) {
|
|
333
|
+
return extname(name).toLowerCase() === ".mp3" ? "audio/mpeg" : "application/octet-stream";
|
|
334
|
+
}
|
|
330
335
|
async function discordAttachmentPayload(attachment) {
|
|
331
336
|
if (!attachment.localPath)
|
|
332
337
|
return undefined;
|
|
338
|
+
const data = await readFile(attachment.localPath);
|
|
339
|
+
const bytes = new Uint8Array(data.byteLength);
|
|
340
|
+
bytes.set(data);
|
|
333
341
|
return {
|
|
334
|
-
|
|
342
|
+
bytes,
|
|
335
343
|
name: attachment.name,
|
|
344
|
+
mimeType: attachment.mimeType || fallbackMimeType(attachment.name),
|
|
336
345
|
};
|
|
337
346
|
}
|
|
338
347
|
async function discordAttachmentPayloads(attachments) {
|
|
@@ -344,19 +353,48 @@ async function discordAttachmentPayloads(attachments) {
|
|
|
344
353
|
}
|
|
345
354
|
return payloads;
|
|
346
355
|
}
|
|
356
|
+
async function postDiscordAttachments(config, channelId, attachments) {
|
|
357
|
+
const files = await discordAttachmentPayloads(attachments);
|
|
358
|
+
if (files.length === 0)
|
|
359
|
+
return [];
|
|
360
|
+
const form = new FormData();
|
|
361
|
+
form.set("payload_json", JSON.stringify({}));
|
|
362
|
+
for (const [index, file] of files.entries()) {
|
|
363
|
+
form.set(`files[${index}]`, new Blob([file.bytes], { type: file.mimeType }), file.name);
|
|
364
|
+
}
|
|
365
|
+
const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, {
|
|
366
|
+
method: "POST",
|
|
367
|
+
headers: { Authorization: `Bot ${config.discord.token}` },
|
|
368
|
+
body: form,
|
|
369
|
+
});
|
|
370
|
+
const data = (await response.json().catch(() => ({})));
|
|
371
|
+
if (!response.ok || !data.id)
|
|
372
|
+
throw new Error(data.message || `Discord attachment send failed (${response.status})`);
|
|
373
|
+
return [data.id];
|
|
374
|
+
}
|
|
375
|
+
async function withDiscordSendTimeout(operation, label, timeoutMs = DISCORD_ATTACHMENT_SEND_TIMEOUT_MS) {
|
|
376
|
+
let timeout;
|
|
377
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
378
|
+
timeout = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
379
|
+
});
|
|
380
|
+
try {
|
|
381
|
+
return await Promise.race([operation, timeoutPromise]);
|
|
382
|
+
}
|
|
383
|
+
finally {
|
|
384
|
+
if (timeout)
|
|
385
|
+
clearTimeout(timeout);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
347
388
|
export const __test = {
|
|
348
389
|
discordAttachmentPayloads,
|
|
390
|
+
postDiscordAttachments,
|
|
391
|
+
withDiscordSendTimeout,
|
|
349
392
|
};
|
|
350
393
|
function parseAgentReply(text) {
|
|
351
|
-
const
|
|
352
|
-
if (
|
|
353
|
-
return
|
|
354
|
-
}
|
|
355
|
-
if (normalized.startsWith(`${SILENT_RESPONSE_MARKER}\n`)) {
|
|
356
|
-
const reason = normalized.slice(SILENT_RESPONSE_MARKER.length).trim();
|
|
357
|
-
return { text: reason, silent: true };
|
|
358
|
-
}
|
|
359
|
-
return { text: normalizeOutboundText(text), silent: false };
|
|
394
|
+
const parsed = parseSilentMarker(text);
|
|
395
|
+
if (parsed.silent)
|
|
396
|
+
return parsed;
|
|
397
|
+
return { text: normalizeOutboundText(parsed.text), silent: false };
|
|
360
398
|
}
|
|
361
399
|
async function sendReply(config, message, text, replyToMessageId, attachments = []) {
|
|
362
400
|
const normalizedText = normalizeOutboundText(text);
|
|
@@ -365,7 +403,6 @@ async function sendReply(config, message, text, replyToMessageId, attachments =
|
|
|
365
403
|
for (const [index, chunk] of chunks.entries()) {
|
|
366
404
|
if (index > 0)
|
|
367
405
|
await delayBetweenBurstChunks(config, message.channel);
|
|
368
|
-
const files = index === 0 ? await discordAttachmentPayloads(attachments) : [];
|
|
369
406
|
let sent;
|
|
370
407
|
if (index === 0 && config.discord.replyMode === "reply") {
|
|
371
408
|
try {
|
|
@@ -373,7 +410,7 @@ async function sendReply(config, message, text, replyToMessageId, attachments =
|
|
|
373
410
|
if (!message.channel.isSendable()) {
|
|
374
411
|
throw new Error(`Discord channel is not sendable: ${message.channelId}`);
|
|
375
412
|
}
|
|
376
|
-
const options = { content: chunk, reply: { messageReference: replyTarget }
|
|
413
|
+
const options = { content: chunk, reply: { messageReference: replyTarget } };
|
|
377
414
|
sent = await message.channel.send(options);
|
|
378
415
|
sentIds.push(sent.id);
|
|
379
416
|
continue;
|
|
@@ -385,9 +422,10 @@ async function sendReply(config, message, text, replyToMessageId, attachments =
|
|
|
385
422
|
if (!message.channel.isSendable()) {
|
|
386
423
|
throw new Error(`Discord channel is not sendable: ${message.channelId}`);
|
|
387
424
|
}
|
|
388
|
-
sent = await message.channel.send(
|
|
425
|
+
sent = await message.channel.send(chunk);
|
|
389
426
|
sentIds.push(sent.id);
|
|
390
427
|
}
|
|
428
|
+
sendDiscordAttachmentsInBackground(config, message.channelId, attachments);
|
|
391
429
|
return sentIds;
|
|
392
430
|
}
|
|
393
431
|
async function sendChannelMessage(config, channel, text, attachments = []) {
|
|
@@ -400,12 +438,19 @@ async function sendChannelMessage(config, channel, text, attachments = []) {
|
|
|
400
438
|
for (const [index, chunk] of chunks.entries()) {
|
|
401
439
|
if (index > 0)
|
|
402
440
|
await delayBetweenBurstChunks(config, channel);
|
|
403
|
-
const
|
|
404
|
-
const sent = await channel.send(files.length > 0 ? { content: chunk, files } : chunk);
|
|
441
|
+
const sent = await channel.send(chunk);
|
|
405
442
|
sentIds.push(sent.id);
|
|
406
443
|
}
|
|
444
|
+
sendDiscordAttachmentsInBackground(config, channel.id, attachments);
|
|
407
445
|
return sentIds;
|
|
408
446
|
}
|
|
447
|
+
function sendDiscordAttachmentsInBackground(config, channelId, attachments) {
|
|
448
|
+
if (attachments.length === 0)
|
|
449
|
+
return;
|
|
450
|
+
void withDiscordSendTimeout(postDiscordAttachments(config, channelId, attachments), "Discord attachment send").catch((error) => {
|
|
451
|
+
console.error("Discord attachment send failed", error);
|
|
452
|
+
});
|
|
453
|
+
}
|
|
409
454
|
function buildChannelRef(channel, channelId) {
|
|
410
455
|
const scope = channel.type === ChannelType.DM ? "dm" : channel.isThread() ? "thread" : "channel";
|
|
411
456
|
const channelName = "name" in channel ? channel.name : undefined;
|
|
@@ -796,9 +841,13 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
|
|
|
796
841
|
await recorder.flush();
|
|
797
842
|
}
|
|
798
843
|
const parsedReply = parseAgentReply(reply.text);
|
|
844
|
+
const replyAnchor = await fetchMessageAnchor(message, dispatch.triggerMessageId);
|
|
799
845
|
const messageIds = parsedReply.silent
|
|
800
846
|
? []
|
|
801
|
-
: await sendReply(config,
|
|
847
|
+
: await sendReply(config, replyAnchor, parsedReply.text, dispatch.triggerMessageId, reply.attachments);
|
|
848
|
+
if (parsedReply.silent) {
|
|
849
|
+
sendDiscordAttachmentsInBackground(config, replyAnchor.channelId, reply.attachments);
|
|
850
|
+
}
|
|
802
851
|
await runtime.completeActiveJob({
|
|
803
852
|
text: parsedReply.text,
|
|
804
853
|
messageIds,
|
|
@@ -868,7 +917,7 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
|
|
|
868
917
|
schedulerState.heartbeat = { lastFiredAt: new Date(queuedNow).toISOString() };
|
|
869
918
|
await saveScheduler();
|
|
870
919
|
const text = buildHeartbeatInjectionText({ now: queuedNow, idleSince: latestUserInteractionAt });
|
|
871
|
-
await heartbeatRuntime.noteHeartbeat(`
|
|
920
|
+
await heartbeatRuntime.noteHeartbeat(`heartbeat stirred after ${formatIdleDuration(queuedNow - latestUserInteractionAt)}`);
|
|
872
921
|
return scheduledUserMessage(text, queuedNow);
|
|
873
922
|
}, async (event) => {
|
|
874
923
|
updateAgentEventSummary(summary, event);
|
|
@@ -888,6 +937,8 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
|
|
|
888
937
|
const messageIds = parsedReply.silent
|
|
889
938
|
? []
|
|
890
939
|
: await sendChannelMessage(config, channel, parsedReply.text, reply.attachments);
|
|
940
|
+
if (parsedReply.silent)
|
|
941
|
+
sendDiscordAttachmentsInBackground(config, channel.id, reply.attachments);
|
|
891
942
|
await heartbeatRuntime.noteOutbound({
|
|
892
943
|
text: parsedReply.text,
|
|
893
944
|
messageIds,
|
|
@@ -1000,6 +1051,8 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
|
|
|
1000
1051
|
const messageIds = parsedReply.silent
|
|
1001
1052
|
? []
|
|
1002
1053
|
: await sendChannelMessage(config, channel, parsedReply.text, reply.attachments);
|
|
1054
|
+
if (parsedReply.silent)
|
|
1055
|
+
sendDiscordAttachmentsInBackground(config, channel.id, reply.attachments);
|
|
1003
1056
|
await runtime.noteOutbound({
|
|
1004
1057
|
text: parsedReply.text,
|
|
1005
1058
|
messageIds,
|
package/dist/generated-media.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { lstat, mkdir, readdir, rm } from "node:fs/promises";
|
|
2
2
|
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
3
|
+
import { isEnoent } from "./util/fs.js";
|
|
3
4
|
export function createGeneratedMediaSink() {
|
|
4
5
|
const attachments = [];
|
|
5
6
|
return {
|
|
@@ -41,7 +42,7 @@ export async function cleanupGeneratedAttachments(config, now = Date.now()) {
|
|
|
41
42
|
entries = await readdir(dir);
|
|
42
43
|
}
|
|
43
44
|
catch (error) {
|
|
44
|
-
if (error
|
|
45
|
+
if (isEnoent(error))
|
|
45
46
|
return 0;
|
|
46
47
|
throw error;
|
|
47
48
|
}
|
|
@@ -52,7 +53,7 @@ export async function cleanupGeneratedAttachments(config, now = Date.now()) {
|
|
|
52
53
|
if (!fileStat?.isFile() || fileStat.mtimeMs > cutoff)
|
|
53
54
|
continue;
|
|
54
55
|
await rm(path).catch((error) => {
|
|
55
|
-
if (!(error
|
|
56
|
+
if (!isEnoent(error))
|
|
56
57
|
throw error;
|
|
57
58
|
});
|
|
58
59
|
removed++;
|
package/dist/hot-reload.js
CHANGED
|
@@ -2,6 +2,7 @@ import { watch } from "node:fs";
|
|
|
2
2
|
import { readdir } from "node:fs/promises";
|
|
3
3
|
import { basename, relative, resolve, sep } from "node:path";
|
|
4
4
|
import { refreshContactNote } from "./contact-note.js";
|
|
5
|
+
import { isEnoent } from "./util/fs.js";
|
|
5
6
|
const ROOT_FILES = new Set([
|
|
6
7
|
"config.toml",
|
|
7
8
|
".env",
|
|
@@ -13,9 +14,6 @@ const ROOT_FILES = new Set([
|
|
|
13
14
|
"HEARTBEAT.md",
|
|
14
15
|
]);
|
|
15
16
|
const SKILLS_DIR = "skills";
|
|
16
|
-
function isEnoent(error) {
|
|
17
|
-
return !!error && typeof error === "object" && error.code === "ENOENT";
|
|
18
|
-
}
|
|
19
17
|
function shouldReloadForPath(workspacePath, changedPath) {
|
|
20
18
|
const relativePath = relative(workspacePath, resolve(changedPath));
|
|
21
19
|
if (!relativePath || relativePath.startsWith("..") || relativePath.split(sep).includes(".."))
|
package/dist/image-gen.js
CHANGED
|
@@ -1,21 +1,15 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { lstat, writeFile } from "node:fs/promises";
|
|
3
|
-
import { basename,
|
|
3
|
+
import { basename, isAbsolute, relative, resolve } from "node:path";
|
|
4
4
|
import { findEnvKeys, generateImages, getEnvApiKey, getImageModels, getImageProviders, } from "@earendil-works/pi-ai";
|
|
5
5
|
import { Type } from "typebox";
|
|
6
6
|
import { ensureGeneratedAttachmentsDir } from "./generated-media.js";
|
|
7
7
|
import { ensureInlineImageDerivative } from "./image-derivatives.js";
|
|
8
8
|
import { promptImagesFromAttachments } from "./inbound-attachments.js";
|
|
9
9
|
import { parseModelRef } from "./models.js";
|
|
10
|
+
import { imageMimeTypeFromPath, sniffImageMimeType } from "./util/image-mime.js";
|
|
10
11
|
const IMAGE_GEN_NOTICE_PREFIX = "Generated image attachment:";
|
|
11
12
|
const OPENROUTER_IMAGE_BASE_URL = "https://openrouter.ai/api/v1";
|
|
12
|
-
const IMAGE_MIME_BY_EXTENSION = {
|
|
13
|
-
".jpg": "image/jpeg",
|
|
14
|
-
".jpeg": "image/jpeg",
|
|
15
|
-
".png": "image/png",
|
|
16
|
-
".gif": "image/gif",
|
|
17
|
-
".webp": "image/webp",
|
|
18
|
-
};
|
|
19
13
|
const imageGenSchema = Type.Object({
|
|
20
14
|
prompt: Type.String({ description: "Image generation prompt." }),
|
|
21
15
|
referenceImages: Type.Optional(Type.Array(Type.String(), {
|
|
@@ -107,20 +101,6 @@ function textOutput(result) {
|
|
|
107
101
|
.filter(Boolean)
|
|
108
102
|
.join("\n");
|
|
109
103
|
}
|
|
110
|
-
function imageMimeTypeFromBytes(buffer) {
|
|
111
|
-
if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
|
|
112
|
-
return "image/jpeg";
|
|
113
|
-
if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
|
114
|
-
return "image/png";
|
|
115
|
-
}
|
|
116
|
-
if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
|
|
117
|
-
return "image/gif";
|
|
118
|
-
}
|
|
119
|
-
if (buffer.subarray(0, 4).toString("ascii") === "RIFF" && buffer.subarray(8, 12).toString("ascii") === "WEBP") {
|
|
120
|
-
return "image/webp";
|
|
121
|
-
}
|
|
122
|
-
return undefined;
|
|
123
|
-
}
|
|
124
104
|
function recoveredImageFromBase64(value) {
|
|
125
105
|
const data = value.trim();
|
|
126
106
|
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(data) || data.length % 4 !== 0)
|
|
@@ -128,7 +108,7 @@ function recoveredImageFromBase64(value) {
|
|
|
128
108
|
const buffer = Buffer.from(data, "base64");
|
|
129
109
|
if (!buffer.length)
|
|
130
110
|
return undefined;
|
|
131
|
-
const detectedMimeType =
|
|
111
|
+
const detectedMimeType = sniffImageMimeType(buffer);
|
|
132
112
|
if (!detectedMimeType)
|
|
133
113
|
return undefined;
|
|
134
114
|
return {
|
|
@@ -167,9 +147,6 @@ function normalizeCompatibleImageText(result) {
|
|
|
167
147
|
return result;
|
|
168
148
|
return { ...result, output };
|
|
169
149
|
}
|
|
170
|
-
function mimeTypeFromPath(path) {
|
|
171
|
-
return IMAGE_MIME_BY_EXTENSION[extname(path).toLowerCase()];
|
|
172
|
-
}
|
|
173
150
|
function resolveWorkspaceReferencePath(config, rawRef) {
|
|
174
151
|
const path = isAbsolute(rawRef) ? resolve(rawRef) : resolve(config.workspacePath, rawRef);
|
|
175
152
|
const workspaceRelative = relative(config.workspacePath, path);
|
|
@@ -190,7 +167,7 @@ async function collectWorkspaceReferenceImages(config, rawRef) {
|
|
|
190
167
|
}
|
|
191
168
|
if (!pathStat.isFile())
|
|
192
169
|
throw new Error(`Reference image path is not a file or folder: ${rawRef}`);
|
|
193
|
-
const mimeType =
|
|
170
|
+
const mimeType = imageMimeTypeFromPath(path);
|
|
194
171
|
if (!mimeType)
|
|
195
172
|
throw new Error(`Reference image path is not a supported image: ${rawRef}`);
|
|
196
173
|
return [
|
|
@@ -338,11 +315,7 @@ async function tryGenerateImages(config, ref, prompt, references, workspaceRefs,
|
|
|
338
315
|
}
|
|
339
316
|
function attemptDetails(model, result) {
|
|
340
317
|
return {
|
|
341
|
-
|
|
342
|
-
model: model.id,
|
|
343
|
-
api: model.api,
|
|
344
|
-
baseUrl: model.baseUrl,
|
|
345
|
-
...(result.responseId ? { responseId: result.responseId } : {}),
|
|
318
|
+
model: `${model.provider}/${model.id}`,
|
|
346
319
|
stopReason: result.stopReason,
|
|
347
320
|
...(result.errorMessage ? { errorMessage: result.errorMessage } : {}),
|
|
348
321
|
};
|
|
@@ -378,18 +351,8 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
|
|
|
378
351
|
}
|
|
379
352
|
catch (error) {
|
|
380
353
|
const message = error instanceof Error ? error.message : String(error);
|
|
381
|
-
let baseUrl = "";
|
|
382
|
-
try {
|
|
383
|
-
baseUrl = resolveImageModel(config, ref).baseUrl;
|
|
384
|
-
}
|
|
385
|
-
catch {
|
|
386
|
-
baseUrl = "";
|
|
387
|
-
}
|
|
388
354
|
attempts.push({
|
|
389
|
-
|
|
390
|
-
model: ref.id,
|
|
391
|
-
api: config.imageGen.api,
|
|
392
|
-
baseUrl,
|
|
355
|
+
model: `${ref.provider}/${ref.id}`,
|
|
393
356
|
stopReason: "error",
|
|
394
357
|
errorMessage: message,
|
|
395
358
|
});
|
|
@@ -414,8 +377,10 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
|
|
|
414
377
|
if (!selected)
|
|
415
378
|
throw new Error(`Image generation failed: ${selectedError}`);
|
|
416
379
|
const attachments = await writeGeneratedImages(config, mediaSink, selected.result);
|
|
380
|
+
const primaryAttachment = attachments[0];
|
|
417
381
|
const notices = attachments.map((attachment) => formatImageGenNotice(attachment.name));
|
|
418
382
|
const sideText = textOutput(selected.result);
|
|
383
|
+
const selectedAttempt = attempts.at(-1);
|
|
419
384
|
return {
|
|
420
385
|
content: [
|
|
421
386
|
{
|
|
@@ -424,15 +389,11 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
|
|
|
424
389
|
},
|
|
425
390
|
],
|
|
426
391
|
details: {
|
|
427
|
-
|
|
428
|
-
model: selected.model.id,
|
|
429
|
-
api: selected.model.api,
|
|
430
|
-
baseUrl: selected.model.baseUrl,
|
|
431
|
-
prompt,
|
|
432
|
-
...(selected.result.responseId ? { responseId: selected.result.responseId } : {}),
|
|
392
|
+
model: `${selected.model.provider}/${selected.model.id}`,
|
|
433
393
|
...(sideText ? { textOutput: sideText } : {}),
|
|
434
|
-
|
|
435
|
-
|
|
394
|
+
...(primaryAttachment ? { id: primaryAttachment.id, localPath: primaryAttachment.localPath } : {}),
|
|
395
|
+
stopReason: selectedAttempt?.stopReason ?? selected.result.stopReason,
|
|
396
|
+
...(selectedAttempt?.errorMessage ? { errorMessage: selectedAttempt.errorMessage } : {}),
|
|
436
397
|
},
|
|
437
398
|
};
|
|
438
399
|
},
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdir, readdir, readFile, unlink, writeFile } from "node:fs/promises";
|
|
3
3
|
import { basename, extname, resolve } from "node:path";
|
|
4
4
|
import { attachmentsDir, publicAttachmentPath } from "./generated-media.js";
|
|
5
5
|
import { ensureInlineImageDerivative, MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
|
|
6
6
|
import { deriveInboundAttachmentText } from "./media-understanding.js";
|
|
7
|
+
import { IMAGE_EXTENSION_BY_MIME, sniffImageMimeType } from "./util/image-mime.js";
|
|
7
8
|
export { MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
|
|
8
9
|
export const MAX_INBOUND_ATTACHMENTS = 4;
|
|
9
10
|
export const MAX_INBOUND_ATTACHMENT_BYTES = 12 * 1024 * 1024;
|
|
10
11
|
export const MAX_INBOUND_TOTAL_BYTES = 24 * 1024 * 1024;
|
|
12
|
+
const TEXT_ATTACHMENT_PREVIEW_LINES = 2;
|
|
13
|
+
const TEXT_ATTACHMENT_PREVIEW_CHARS = 1000;
|
|
11
14
|
const ALLOWED_MIME_TYPES = new Set([
|
|
12
15
|
"image/jpeg",
|
|
13
16
|
"image/png",
|
|
@@ -23,10 +26,7 @@ const ALLOWED_MIME_TYPES = new Set([
|
|
|
23
26
|
"text/plain",
|
|
24
27
|
]);
|
|
25
28
|
const EXTENSIONS_BY_MIME = {
|
|
26
|
-
|
|
27
|
-
"image/png": ".png",
|
|
28
|
-
"image/gif": ".gif",
|
|
29
|
-
"image/webp": ".webp",
|
|
29
|
+
...IMAGE_EXTENSION_BY_MIME,
|
|
30
30
|
"audio/mpeg": ".mp3",
|
|
31
31
|
"audio/ogg": ".ogg",
|
|
32
32
|
"audio/wav": ".wav",
|
|
@@ -51,32 +51,42 @@ function kindFromMime(mimeType) {
|
|
|
51
51
|
return "video";
|
|
52
52
|
return "file";
|
|
53
53
|
}
|
|
54
|
+
function textAttachmentPreview(buffer, mimeType) {
|
|
55
|
+
if (mimeType !== "text/plain")
|
|
56
|
+
return undefined;
|
|
57
|
+
const decoded = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
|
|
58
|
+
const normalized = decoded.replace(/\r\n/g, "\n").replace(/\r/g, "\n").trim();
|
|
59
|
+
if (!normalized || normalized.includes("\uFFFD"))
|
|
60
|
+
return undefined;
|
|
61
|
+
const lines = normalized.split("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_LINES);
|
|
62
|
+
const preview = lines.join("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_CHARS).trim();
|
|
63
|
+
return preview || undefined;
|
|
64
|
+
}
|
|
54
65
|
function sniffText(buffer) {
|
|
55
66
|
if (buffer.length === 0)
|
|
56
67
|
return "text/plain";
|
|
57
68
|
const head = buffer.subarray(0, Math.min(buffer.length, 512));
|
|
58
69
|
if (head.includes(0))
|
|
59
70
|
return undefined;
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
:
|
|
63
|
-
}
|
|
64
|
-
function sniffMimeType(buffer, declared) {
|
|
65
|
-
let detected;
|
|
66
|
-
if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
|
|
67
|
-
detected = "image/jpeg";
|
|
68
|
-
else if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
|
69
|
-
detected = "image/png";
|
|
71
|
+
let decoded;
|
|
72
|
+
try {
|
|
73
|
+
decoded = new TextDecoder("utf-8", { fatal: true }).decode(head);
|
|
70
74
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
detected = "image/gif";
|
|
75
|
+
catch {
|
|
76
|
+
return undefined;
|
|
74
77
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
+
for (const char of decoded) {
|
|
79
|
+
const code = char.codePointAt(0) ?? 0;
|
|
80
|
+
if (code === 9 || code === 10 || code === 13)
|
|
81
|
+
continue;
|
|
82
|
+
if (code < 32 || code === 127)
|
|
83
|
+
return undefined;
|
|
78
84
|
}
|
|
79
|
-
|
|
85
|
+
return "text/plain";
|
|
86
|
+
}
|
|
87
|
+
function sniffMimeType(buffer, declared) {
|
|
88
|
+
let detected = sniffImageMimeType(buffer);
|
|
89
|
+
if (!detected && buffer.subarray(0, 4).toString("ascii") === "%PDF")
|
|
80
90
|
detected = "application/pdf";
|
|
81
91
|
else if (buffer.subarray(0, 3).toString("ascii") === "ID3" ||
|
|
82
92
|
buffer.subarray(0, 2).equals(Buffer.from([0xff, 0xfb]))) {
|
|
@@ -184,6 +194,7 @@ export async function materializeInboundAttachments(config, inputs) {
|
|
|
184
194
|
}
|
|
185
195
|
const stored = [];
|
|
186
196
|
const writtenPaths = [];
|
|
197
|
+
const existingDerivedPaths = await knownDerivedImagePaths(config);
|
|
187
198
|
try {
|
|
188
199
|
for (const attachment of prepared) {
|
|
189
200
|
const dir = resolve(attachmentsDir(config), "inbound", attachment.source);
|
|
@@ -203,8 +214,24 @@ export async function materializeInboundAttachments(config, inputs) {
|
|
|
203
214
|
source: attachment.source,
|
|
204
215
|
sha256: attachment.sha256,
|
|
205
216
|
};
|
|
217
|
+
const textPreview = textAttachmentPreview(attachment.buffer, attachment.mimeType);
|
|
218
|
+
if (textPreview) {
|
|
219
|
+
finalAttachment.derived = {
|
|
220
|
+
...finalAttachment.derived,
|
|
221
|
+
text: {
|
|
222
|
+
provider: "local",
|
|
223
|
+
model: "text-preview",
|
|
224
|
+
label: "preview",
|
|
225
|
+
text: textPreview,
|
|
226
|
+
},
|
|
227
|
+
};
|
|
228
|
+
}
|
|
206
229
|
const derivedImage = await ensureInlineImageDerivative(config, finalAttachment);
|
|
207
230
|
if (derivedImage) {
|
|
231
|
+
if (derivedImage.localPath && !existingDerivedPaths.has(derivedImage.localPath)) {
|
|
232
|
+
writtenPaths.push(derivedImage.localPath);
|
|
233
|
+
existingDerivedPaths.add(derivedImage.localPath);
|
|
234
|
+
}
|
|
208
235
|
finalAttachment.derived = {
|
|
209
236
|
...finalAttachment.derived,
|
|
210
237
|
image: derivedImage,
|
|
@@ -219,6 +246,11 @@ export async function materializeInboundAttachments(config, inputs) {
|
|
|
219
246
|
throw error;
|
|
220
247
|
}
|
|
221
248
|
}
|
|
249
|
+
async function knownDerivedImagePaths(config) {
|
|
250
|
+
const dir = resolve(attachmentsDir(config), "derived", "image");
|
|
251
|
+
const entries = await readdir(dir).catch(() => []);
|
|
252
|
+
return new Set(entries.map((entry) => resolve(dir, entry)));
|
|
253
|
+
}
|
|
222
254
|
export async function promptImagesFromAttachments(attachments) {
|
|
223
255
|
const images = [];
|
|
224
256
|
const notes = [];
|
|
@@ -250,7 +282,16 @@ export async function promptImagesFromAttachments(attachments) {
|
|
|
250
282
|
export function promptAttachmentNotes(attachments) {
|
|
251
283
|
return attachments
|
|
252
284
|
.map((attachment) => {
|
|
253
|
-
const attrs =
|
|
285
|
+
const attrs = [
|
|
286
|
+
`name="${attachment.name}"`,
|
|
287
|
+
`id="${attachment.id}"`,
|
|
288
|
+
`kind="${attachment.kind ?? "file"}"`,
|
|
289
|
+
`mime="${attachment.mimeType ?? "unknown"}"`,
|
|
290
|
+
`size="${attachment.size ?? "unknown"}"`,
|
|
291
|
+
attachment.localPath ? `path="${attachment.localPath}"` : undefined,
|
|
292
|
+
]
|
|
293
|
+
.filter(Boolean)
|
|
294
|
+
.join(" ");
|
|
254
295
|
const derivedText = attachment.derived?.text?.text;
|
|
255
296
|
if (derivedText) {
|
|
256
297
|
const label = attachment.derived?.text?.label || (attachment.kind === "audio" ? "transcription" : "summary");
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { positiveIntegerOrDefault } from "../util.js";
|
|
1
2
|
import { retrieveAmbientDiary } from "./ambient.js";
|
|
2
3
|
const INJECTED_MEMORY_OPEN = "<injected_memory>";
|
|
3
4
|
const INJECTED_MEMORY_CLOSE = "</injected_memory>";
|
|
@@ -64,9 +65,6 @@ export class AmbientDiaryInjector {
|
|
|
64
65
|
}
|
|
65
66
|
}
|
|
66
67
|
}
|
|
67
|
-
function positiveIntegerOrDefault(value, fallback) {
|
|
68
|
-
return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
|
|
69
|
-
}
|
|
70
68
|
function nonNegativeIntegerOrDefault(value, fallback) {
|
|
71
69
|
return value !== undefined && Number.isInteger(value) && value >= 0 ? value : fallback;
|
|
72
70
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { retrieveMemory, } from "../index/retrieval.js";
|
|
2
|
+
import { positiveIntegerOrDefault } from "../util.js";
|
|
2
3
|
import { DIARY_CHUNK_CORPUS } from "./chunks.js";
|
|
3
4
|
const DEFAULT_LIMIT = 4;
|
|
4
5
|
const DEFAULT_CANDIDATE_MULTIPLIER = 5;
|
|
@@ -119,6 +120,3 @@ function normalizeUnit(value) {
|
|
|
119
120
|
const absolute = Math.abs(value);
|
|
120
121
|
return Math.max(0, Math.min(1, absolute > 1 ? absolute / 10 : absolute));
|
|
121
122
|
}
|
|
122
|
-
function positiveIntegerOrDefault(value, fallback) {
|
|
123
|
-
return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
|
|
124
|
-
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { basename } from "node:path";
|
|
2
|
+
import { positiveIntegerOrDefault } from "../util.js";
|
|
2
3
|
export const DIARY_CHUNK_CORPUS = "diary_chunk";
|
|
3
4
|
const DEFAULT_MAX_CHARS = 2400;
|
|
4
5
|
const DIARY_DATE_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
|
|
@@ -226,6 +227,3 @@ function stripInlineMarkdown(value) {
|
|
|
226
227
|
function isMarkdownHeading(line) {
|
|
227
228
|
return /^#{1,6}\s+/.test(line);
|
|
228
229
|
}
|
|
229
|
-
function positiveIntegerOrDefault(value, fallback) {
|
|
230
|
-
return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
|
|
231
|
-
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
2
2
|
import { basename, isAbsolute, join, resolve } from "node:path";
|
|
3
|
+
import { isEnoent } from "../../util/fs.js";
|
|
3
4
|
import { DIARY_CHUNK_CORPUS, indexDiaryMarkdown } from "./chunks.js";
|
|
4
5
|
export const DIARY_INDEX_FILE_RE = /^\d{4}-\d{2}-\d{2}\.md$/;
|
|
5
6
|
export function isDatedDiaryMarkdownFile(path) {
|
|
@@ -22,9 +23,6 @@ export async function listDiaryMarkdownFiles(config) {
|
|
|
22
23
|
.map((entry) => join(config.memory.diariesDir, entry.name))
|
|
23
24
|
.sort();
|
|
24
25
|
}
|
|
25
|
-
function isEnoent(error) {
|
|
26
|
-
return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
|
27
|
-
}
|
|
28
26
|
export async function indexDiaryFile(options) {
|
|
29
27
|
const path = resolveDiaryPath(options.config, options.path);
|
|
30
28
|
const sourceId = basename(path);
|
package/dist/memory/doctor.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { runInTransaction } from "./util.js";
|
|
1
2
|
export function runDoctor(stores, opts = {}) {
|
|
2
3
|
void opts;
|
|
3
4
|
const findings = [];
|
|
@@ -32,10 +33,7 @@ export function applyDoctorFixes(stores, report) {
|
|
|
32
33
|
fixed += before;
|
|
33
34
|
}
|
|
34
35
|
};
|
|
35
|
-
|
|
36
|
-
runIndexFixes();
|
|
37
|
-
else
|
|
38
|
-
stores.index.db.transaction(runIndexFixes).immediate();
|
|
36
|
+
runInTransaction(stores.index.db, runIndexFixes);
|
|
39
37
|
const runLcmFixes = () => {
|
|
40
38
|
fixed += stores.lcm.db
|
|
41
39
|
.prepare(`DELETE FROM lcm_segments
|
|
@@ -63,10 +61,7 @@ export function applyDoctorFixes(stores, report) {
|
|
|
63
61
|
}
|
|
64
62
|
}
|
|
65
63
|
};
|
|
66
|
-
|
|
67
|
-
runLcmFixes();
|
|
68
|
-
else
|
|
69
|
-
stores.lcm.db.transaction(runLcmFixes).immediate();
|
|
64
|
+
runInTransaction(stores.lcm.db, runLcmFixes);
|
|
70
65
|
if (report.findings.some((finding) => finding.kind === "summary_fk_violation")) {
|
|
71
66
|
warnings.push("summary FK violations were not modified; inspect LCM summary lineage manually");
|
|
72
67
|
}
|