@qearlyao/familiar 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +6 -14
  2. package/config.example.toml +1 -1
  3. package/dist/added-models.js +6 -15
  4. package/dist/agent-events.js +1 -3
  5. package/dist/agent.js +3 -4
  6. package/dist/browser-tools.js +15 -11
  7. package/dist/chat-log.js +3 -2
  8. package/dist/cli.js +2 -2
  9. package/dist/config-overrides.js +5 -14
  10. package/dist/config-registry.js +1 -4
  11. package/dist/config.js +45 -113
  12. package/dist/contact-note.js +2 -12
  13. package/dist/data-retention.js +1 -3
  14. package/dist/discord.js +72 -19
  15. package/dist/generated-media.js +3 -2
  16. package/dist/hot-reload.js +1 -3
  17. package/dist/image-gen.js +12 -51
  18. package/dist/inbound-attachments.js +64 -23
  19. package/dist/memory/diary/ambient-injector.js +1 -3
  20. package/dist/memory/diary/ambient.js +1 -3
  21. package/dist/memory/diary/chunks.js +1 -3
  22. package/dist/memory/diary/indexer.js +1 -3
  23. package/dist/memory/doctor.js +3 -8
  24. package/dist/memory/index/chunk-indexer.js +27 -6
  25. package/dist/memory/index/retrieval.js +1 -3
  26. package/dist/memory/index/store.js +47 -19
  27. package/dist/memory/lcm/backfill.js +19 -16
  28. package/dist/memory/lcm/context-transformer.js +17 -29
  29. package/dist/memory/lcm/context.js +10 -4
  30. package/dist/memory/lcm/eviction-score.js +25 -13
  31. package/dist/memory/lcm/indexer.js +1 -5
  32. package/dist/memory/lcm/normalize.js +22 -1
  33. package/dist/memory/lcm/store.js +27 -24
  34. package/dist/memory/operator.js +3 -31
  35. package/dist/memory/service.js +1 -3
  36. package/dist/memory/tools.js +0 -4
  37. package/dist/memory/util.js +6 -0
  38. package/dist/models.js +3 -0
  39. package/dist/persona.js +3 -15
  40. package/dist/runtime.js +12 -23
  41. package/dist/scheduler.js +15 -49
  42. package/dist/service.js +39 -27
  43. package/dist/settings.js +7 -32
  44. package/dist/silent-marker.js +64 -0
  45. package/dist/tts.js +0 -6
  46. package/dist/util/fs.js +41 -0
  47. package/dist/util/guards.js +8 -0
  48. package/dist/util/image-mime.js +31 -0
  49. package/dist/util/time.js +29 -0
  50. package/dist/web-auth.js +4 -1
  51. package/dist/web-static.js +36 -1
  52. package/dist/web-tools.js +8 -5
  53. package/dist/web.js +253 -69
  54. package/npm-shrinkwrap.json +5139 -0
  55. package/package.json +5 -4
  56. package/web/dist/assets/index-B23WT77N.js +63 -0
  57. package/web/dist/assets/index-D3MotFzN.css +2 -0
  58. package/web/dist/index.html +2 -2
  59. package/web/dist/assets/index-BPZQbZh5.js +0 -61
  60. package/web/dist/assets/index-CcQ13VAY.css +0 -2
package/dist/discord.js CHANGED
@@ -1,19 +1,21 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { once } from "node:events";
3
3
  import { readFile } from "node:fs/promises";
4
+ import { extname } from "node:path";
4
5
  import { ApplicationCommandOptionType, ApplicationCommandType, ApplicationIntegrationType, ChannelType, Client, Events, GatewayIntentBits, InteractionContextType, MessageFlags, Partials, } from "discord.js";
5
6
  import { createAgentEventRecorder, storedAgentEventFromAgentEvent, thinkingDurationMs, updateAgentEventSummary, } from "./agent-events.js";
6
7
  import { chatChannelKey, createChatLog } from "./chat-log.js";
7
8
  import { materializeInboundAttachments, promptImagesFromAttachments } from "./inbound-attachments.js";
8
9
  import { ConversationRuntime } from "./runtime.js";
9
- import { appendSchedulerLog, buildCronInjectionText, buildHeartbeatInjectionText, dueCronSlot, isHeartbeatDue, loadSchedulerState, saveSchedulerState, } from "./scheduler.js";
10
+ import { appendSchedulerLog, buildCronInjectionText, buildHeartbeatInjectionText, dueCronSlot, formatIdleDuration, isHeartbeatDue, loadSchedulerState, saveSchedulerState, } from "./scheduler.js";
11
+ import { parseAgentReply as parseSilentMarker } from "./silent-marker.js";
10
12
  const FAMILIAR_COMMAND_NAME = "familiar";
11
13
  const THINKING_CHOICES = ["off", "minimal", "low", "medium", "high", "xhigh"];
12
14
  const CHANNEL_TRIGGER_CHOICES = ["mention", "always"];
13
15
  const EPHEMERAL_REPLY = MessageFlags.Ephemeral;
14
- const SILENT_RESPONSE_MARKER = "[[FAMILIAR_SILENT]]";
15
16
  const HEARTBEAT_SKIPPED = Symbol("heartbeat-skipped");
16
17
  const CRON_SKIPPED = Symbol("cron-skipped");
18
+ const DISCORD_ATTACHMENT_SEND_TIMEOUT_MS = 20_000;
17
19
  async function withReadyClient(token) {
18
20
  const client = new Client({
19
21
  intents: [
@@ -327,12 +329,19 @@ async function delayBetweenBurstChunks(config, channel) {
327
329
  function normalizeOutboundText(text) {
328
330
  return text.trim() || "(empty response)";
329
331
  }
332
+ function fallbackMimeType(name) {
333
+ return extname(name).toLowerCase() === ".mp3" ? "audio/mpeg" : "application/octet-stream";
334
+ }
330
335
  async function discordAttachmentPayload(attachment) {
331
336
  if (!attachment.localPath)
332
337
  return undefined;
338
+ const data = await readFile(attachment.localPath);
339
+ const bytes = new Uint8Array(data.byteLength);
340
+ bytes.set(data);
333
341
  return {
334
- attachment: await readFile(attachment.localPath),
342
+ bytes,
335
343
  name: attachment.name,
344
+ mimeType: attachment.mimeType || fallbackMimeType(attachment.name),
336
345
  };
337
346
  }
338
347
  async function discordAttachmentPayloads(attachments) {
@@ -344,19 +353,48 @@ async function discordAttachmentPayloads(attachments) {
344
353
  }
345
354
  return payloads;
346
355
  }
356
+ async function postDiscordAttachments(config, channelId, attachments) {
357
+ const files = await discordAttachmentPayloads(attachments);
358
+ if (files.length === 0)
359
+ return [];
360
+ const form = new FormData();
361
+ form.set("payload_json", JSON.stringify({}));
362
+ for (const [index, file] of files.entries()) {
363
+ form.set(`files[${index}]`, new Blob([file.bytes], { type: file.mimeType }), file.name);
364
+ }
365
+ const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, {
366
+ method: "POST",
367
+ headers: { Authorization: `Bot ${config.discord.token}` },
368
+ body: form,
369
+ });
370
+ const data = (await response.json().catch(() => ({})));
371
+ if (!response.ok || !data.id)
372
+ throw new Error(data.message || `Discord attachment send failed (${response.status})`);
373
+ return [data.id];
374
+ }
375
+ async function withDiscordSendTimeout(operation, label, timeoutMs = DISCORD_ATTACHMENT_SEND_TIMEOUT_MS) {
376
+ let timeout;
377
+ const timeoutPromise = new Promise((_, reject) => {
378
+ timeout = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
379
+ });
380
+ try {
381
+ return await Promise.race([operation, timeoutPromise]);
382
+ }
383
+ finally {
384
+ if (timeout)
385
+ clearTimeout(timeout);
386
+ }
387
+ }
347
388
  export const __test = {
348
389
  discordAttachmentPayloads,
390
+ postDiscordAttachments,
391
+ withDiscordSendTimeout,
349
392
  };
350
393
  function parseAgentReply(text) {
351
- const normalized = text.replace(/\r\n/g, "\n").trim();
352
- if (normalized === SILENT_RESPONSE_MARKER) {
353
- return { text: "", silent: true };
354
- }
355
- if (normalized.startsWith(`${SILENT_RESPONSE_MARKER}\n`)) {
356
- const reason = normalized.slice(SILENT_RESPONSE_MARKER.length).trim();
357
- return { text: reason, silent: true };
358
- }
359
- return { text: normalizeOutboundText(text), silent: false };
394
+ const parsed = parseSilentMarker(text);
395
+ if (parsed.silent)
396
+ return parsed;
397
+ return { text: normalizeOutboundText(parsed.text), silent: false };
360
398
  }
361
399
  async function sendReply(config, message, text, replyToMessageId, attachments = []) {
362
400
  const normalizedText = normalizeOutboundText(text);
@@ -365,7 +403,6 @@ async function sendReply(config, message, text, replyToMessageId, attachments =
365
403
  for (const [index, chunk] of chunks.entries()) {
366
404
  if (index > 0)
367
405
  await delayBetweenBurstChunks(config, message.channel);
368
- const files = index === 0 ? await discordAttachmentPayloads(attachments) : [];
369
406
  let sent;
370
407
  if (index === 0 && config.discord.replyMode === "reply") {
371
408
  try {
@@ -373,7 +410,7 @@ async function sendReply(config, message, text, replyToMessageId, attachments =
373
410
  if (!message.channel.isSendable()) {
374
411
  throw new Error(`Discord channel is not sendable: ${message.channelId}`);
375
412
  }
376
- const options = { content: chunk, reply: { messageReference: replyTarget }, files };
413
+ const options = { content: chunk, reply: { messageReference: replyTarget } };
377
414
  sent = await message.channel.send(options);
378
415
  sentIds.push(sent.id);
379
416
  continue;
@@ -385,9 +422,10 @@ async function sendReply(config, message, text, replyToMessageId, attachments =
385
422
  if (!message.channel.isSendable()) {
386
423
  throw new Error(`Discord channel is not sendable: ${message.channelId}`);
387
424
  }
388
- sent = await message.channel.send(files.length > 0 ? { content: chunk, files } : chunk);
425
+ sent = await message.channel.send(chunk);
389
426
  sentIds.push(sent.id);
390
427
  }
428
+ sendDiscordAttachmentsInBackground(config, message.channelId, attachments);
391
429
  return sentIds;
392
430
  }
393
431
  async function sendChannelMessage(config, channel, text, attachments = []) {
@@ -400,12 +438,19 @@ async function sendChannelMessage(config, channel, text, attachments = []) {
400
438
  for (const [index, chunk] of chunks.entries()) {
401
439
  if (index > 0)
402
440
  await delayBetweenBurstChunks(config, channel);
403
- const files = index === 0 ? await discordAttachmentPayloads(attachments) : [];
404
- const sent = await channel.send(files.length > 0 ? { content: chunk, files } : chunk);
441
+ const sent = await channel.send(chunk);
405
442
  sentIds.push(sent.id);
406
443
  }
444
+ sendDiscordAttachmentsInBackground(config, channel.id, attachments);
407
445
  return sentIds;
408
446
  }
447
+ function sendDiscordAttachmentsInBackground(config, channelId, attachments) {
448
+ if (attachments.length === 0)
449
+ return;
450
+ void withDiscordSendTimeout(postDiscordAttachments(config, channelId, attachments), "Discord attachment send").catch((error) => {
451
+ console.error("Discord attachment send failed", error);
452
+ });
453
+ }
409
454
  function buildChannelRef(channel, channelId) {
410
455
  const scope = channel.type === ChannelType.DM ? "dm" : channel.isThread() ? "thread" : "channel";
411
456
  const channelName = "name" in channel ? channel.name : undefined;
@@ -796,9 +841,13 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
796
841
  await recorder.flush();
797
842
  }
798
843
  const parsedReply = parseAgentReply(reply.text);
844
+ const replyAnchor = await fetchMessageAnchor(message, dispatch.triggerMessageId);
799
845
  const messageIds = parsedReply.silent
800
846
  ? []
801
- : await sendReply(config, await fetchMessageAnchor(message, dispatch.triggerMessageId), parsedReply.text, dispatch.triggerMessageId, reply.attachments);
847
+ : await sendReply(config, replyAnchor, parsedReply.text, dispatch.triggerMessageId, reply.attachments);
848
+ if (parsedReply.silent) {
849
+ sendDiscordAttachmentsInBackground(config, replyAnchor.channelId, reply.attachments);
850
+ }
802
851
  await runtime.completeActiveJob({
803
852
  text: parsedReply.text,
804
853
  messageIds,
@@ -868,7 +917,7 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
868
917
  schedulerState.heartbeat = { lastFiredAt: new Date(queuedNow).toISOString() };
869
918
  await saveScheduler();
870
919
  const text = buildHeartbeatInjectionText({ now: queuedNow, idleSince: latestUserInteractionAt });
871
- await heartbeatRuntime.noteHeartbeat(`started after ${Math.floor((queuedNow - latestUserInteractionAt) / 60_000)} idle minute(s)`);
920
+ await heartbeatRuntime.noteHeartbeat(`heartbeat stirred after ${formatIdleDuration(queuedNow - latestUserInteractionAt)}`);
872
921
  return scheduledUserMessage(text, queuedNow);
873
922
  }, async (event) => {
874
923
  updateAgentEventSummary(summary, event);
@@ -888,6 +937,8 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
888
937
  const messageIds = parsedReply.silent
889
938
  ? []
890
939
  : await sendChannelMessage(config, channel, parsedReply.text, reply.attachments);
940
+ if (parsedReply.silent)
941
+ sendDiscordAttachmentsInBackground(config, channel.id, reply.attachments);
891
942
  await heartbeatRuntime.noteOutbound({
892
943
  text: parsedReply.text,
893
944
  messageIds,
@@ -1000,6 +1051,8 @@ export async function startDiscordDaemon(config, familiarAgent, settings, memory
1000
1051
  const messageIds = parsedReply.silent
1001
1052
  ? []
1002
1053
  : await sendChannelMessage(config, channel, parsedReply.text, reply.attachments);
1054
+ if (parsedReply.silent)
1055
+ sendDiscordAttachmentsInBackground(config, channel.id, reply.attachments);
1003
1056
  await runtime.noteOutbound({
1004
1057
  text: parsedReply.text,
1005
1058
  messageIds,
@@ -1,5 +1,6 @@
1
1
  import { lstat, mkdir, readdir, rm } from "node:fs/promises";
2
2
  import { isAbsolute, join, relative, resolve } from "node:path";
3
+ import { isEnoent } from "./util/fs.js";
3
4
  export function createGeneratedMediaSink() {
4
5
  const attachments = [];
5
6
  return {
@@ -41,7 +42,7 @@ export async function cleanupGeneratedAttachments(config, now = Date.now()) {
41
42
  entries = await readdir(dir);
42
43
  }
43
44
  catch (error) {
44
- if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
45
+ if (isEnoent(error))
45
46
  return 0;
46
47
  throw error;
47
48
  }
@@ -52,7 +53,7 @@ export async function cleanupGeneratedAttachments(config, now = Date.now()) {
52
53
  if (!fileStat?.isFile() || fileStat.mtimeMs > cutoff)
53
54
  continue;
54
55
  await rm(path).catch((error) => {
55
- if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT"))
56
+ if (!isEnoent(error))
56
57
  throw error;
57
58
  });
58
59
  removed++;
@@ -2,6 +2,7 @@ import { watch } from "node:fs";
2
2
  import { readdir } from "node:fs/promises";
3
3
  import { basename, relative, resolve, sep } from "node:path";
4
4
  import { refreshContactNote } from "./contact-note.js";
5
+ import { isEnoent } from "./util/fs.js";
5
6
  const ROOT_FILES = new Set([
6
7
  "config.toml",
7
8
  ".env",
@@ -13,9 +14,6 @@ const ROOT_FILES = new Set([
13
14
  "HEARTBEAT.md",
14
15
  ]);
15
16
  const SKILLS_DIR = "skills";
16
- function isEnoent(error) {
17
- return !!error && typeof error === "object" && error.code === "ENOENT";
18
- }
19
17
  function shouldReloadForPath(workspacePath, changedPath) {
20
18
  const relativePath = relative(workspacePath, resolve(changedPath));
21
19
  if (!relativePath || relativePath.startsWith("..") || relativePath.split(sep).includes(".."))
package/dist/image-gen.js CHANGED
@@ -1,21 +1,15 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { lstat, writeFile } from "node:fs/promises";
3
- import { basename, extname, isAbsolute, relative, resolve } from "node:path";
3
+ import { basename, isAbsolute, relative, resolve } from "node:path";
4
4
  import { findEnvKeys, generateImages, getEnvApiKey, getImageModels, getImageProviders, } from "@earendil-works/pi-ai";
5
5
  import { Type } from "typebox";
6
6
  import { ensureGeneratedAttachmentsDir } from "./generated-media.js";
7
7
  import { ensureInlineImageDerivative } from "./image-derivatives.js";
8
8
  import { promptImagesFromAttachments } from "./inbound-attachments.js";
9
9
  import { parseModelRef } from "./models.js";
10
+ import { imageMimeTypeFromPath, sniffImageMimeType } from "./util/image-mime.js";
10
11
  const IMAGE_GEN_NOTICE_PREFIX = "Generated image attachment:";
11
12
  const OPENROUTER_IMAGE_BASE_URL = "https://openrouter.ai/api/v1";
12
- const IMAGE_MIME_BY_EXTENSION = {
13
- ".jpg": "image/jpeg",
14
- ".jpeg": "image/jpeg",
15
- ".png": "image/png",
16
- ".gif": "image/gif",
17
- ".webp": "image/webp",
18
- };
19
13
  const imageGenSchema = Type.Object({
20
14
  prompt: Type.String({ description: "Image generation prompt." }),
21
15
  referenceImages: Type.Optional(Type.Array(Type.String(), {
@@ -107,20 +101,6 @@ function textOutput(result) {
107
101
  .filter(Boolean)
108
102
  .join("\n");
109
103
  }
110
- function imageMimeTypeFromBytes(buffer) {
111
- if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
112
- return "image/jpeg";
113
- if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
114
- return "image/png";
115
- }
116
- if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
117
- return "image/gif";
118
- }
119
- if (buffer.subarray(0, 4).toString("ascii") === "RIFF" && buffer.subarray(8, 12).toString("ascii") === "WEBP") {
120
- return "image/webp";
121
- }
122
- return undefined;
123
- }
124
104
  function recoveredImageFromBase64(value) {
125
105
  const data = value.trim();
126
106
  if (!/^[A-Za-z0-9+/]+={0,2}$/.test(data) || data.length % 4 !== 0)
@@ -128,7 +108,7 @@ function recoveredImageFromBase64(value) {
128
108
  const buffer = Buffer.from(data, "base64");
129
109
  if (!buffer.length)
130
110
  return undefined;
131
- const detectedMimeType = imageMimeTypeFromBytes(buffer);
111
+ const detectedMimeType = sniffImageMimeType(buffer);
132
112
  if (!detectedMimeType)
133
113
  return undefined;
134
114
  return {
@@ -167,9 +147,6 @@ function normalizeCompatibleImageText(result) {
167
147
  return result;
168
148
  return { ...result, output };
169
149
  }
170
- function mimeTypeFromPath(path) {
171
- return IMAGE_MIME_BY_EXTENSION[extname(path).toLowerCase()];
172
- }
173
150
  function resolveWorkspaceReferencePath(config, rawRef) {
174
151
  const path = isAbsolute(rawRef) ? resolve(rawRef) : resolve(config.workspacePath, rawRef);
175
152
  const workspaceRelative = relative(config.workspacePath, path);
@@ -190,7 +167,7 @@ async function collectWorkspaceReferenceImages(config, rawRef) {
190
167
  }
191
168
  if (!pathStat.isFile())
192
169
  throw new Error(`Reference image path is not a file or folder: ${rawRef}`);
193
- const mimeType = mimeTypeFromPath(path);
170
+ const mimeType = imageMimeTypeFromPath(path);
194
171
  if (!mimeType)
195
172
  throw new Error(`Reference image path is not a supported image: ${rawRef}`);
196
173
  return [
@@ -338,11 +315,7 @@ async function tryGenerateImages(config, ref, prompt, references, workspaceRefs,
338
315
  }
339
316
  function attemptDetails(model, result) {
340
317
  return {
341
- provider: model.provider,
342
- model: model.id,
343
- api: model.api,
344
- baseUrl: model.baseUrl,
345
- ...(result.responseId ? { responseId: result.responseId } : {}),
318
+ model: `${model.provider}/${model.id}`,
346
319
  stopReason: result.stopReason,
347
320
  ...(result.errorMessage ? { errorMessage: result.errorMessage } : {}),
348
321
  };
@@ -378,18 +351,8 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
378
351
  }
379
352
  catch (error) {
380
353
  const message = error instanceof Error ? error.message : String(error);
381
- let baseUrl = "";
382
- try {
383
- baseUrl = resolveImageModel(config, ref).baseUrl;
384
- }
385
- catch {
386
- baseUrl = "";
387
- }
388
354
  attempts.push({
389
- provider: ref.provider,
390
- model: ref.id,
391
- api: config.imageGen.api,
392
- baseUrl,
355
+ model: `${ref.provider}/${ref.id}`,
393
356
  stopReason: "error",
394
357
  errorMessage: message,
395
358
  });
@@ -414,8 +377,10 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
414
377
  if (!selected)
415
378
  throw new Error(`Image generation failed: ${selectedError}`);
416
379
  const attachments = await writeGeneratedImages(config, mediaSink, selected.result);
380
+ const primaryAttachment = attachments[0];
417
381
  const notices = attachments.map((attachment) => formatImageGenNotice(attachment.name));
418
382
  const sideText = textOutput(selected.result);
383
+ const selectedAttempt = attempts.at(-1);
419
384
  return {
420
385
  content: [
421
386
  {
@@ -424,15 +389,11 @@ export function createImageGenTool(config, mediaSink, deps = {}) {
424
389
  },
425
390
  ],
426
391
  details: {
427
- provider: selected.model.provider,
428
- model: selected.model.id,
429
- api: selected.model.api,
430
- baseUrl: selected.model.baseUrl,
431
- prompt,
432
- ...(selected.result.responseId ? { responseId: selected.result.responseId } : {}),
392
+ model: `${selected.model.provider}/${selected.model.id}`,
433
393
  ...(sideText ? { textOutput: sideText } : {}),
434
- attachments,
435
- attempts,
394
+ ...(primaryAttachment ? { id: primaryAttachment.id, localPath: primaryAttachment.localPath } : {}),
395
+ stopReason: selectedAttempt?.stopReason ?? selected.result.stopReason,
396
+ ...(selectedAttempt?.errorMessage ? { errorMessage: selectedAttempt.errorMessage } : {}),
436
397
  },
437
398
  };
438
399
  },
@@ -1,13 +1,16 @@
1
1
  import { createHash, randomUUID } from "node:crypto";
2
- import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
2
+ import { mkdir, readdir, readFile, unlink, writeFile } from "node:fs/promises";
3
3
  import { basename, extname, resolve } from "node:path";
4
4
  import { attachmentsDir, publicAttachmentPath } from "./generated-media.js";
5
5
  import { ensureInlineImageDerivative, MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
6
6
  import { deriveInboundAttachmentText } from "./media-understanding.js";
7
+ import { IMAGE_EXTENSION_BY_MIME, sniffImageMimeType } from "./util/image-mime.js";
7
8
  export { MAX_INLINE_IMAGE_BASE64_BYTES } from "./image-derivatives.js";
8
9
  export const MAX_INBOUND_ATTACHMENTS = 4;
9
10
  export const MAX_INBOUND_ATTACHMENT_BYTES = 12 * 1024 * 1024;
10
11
  export const MAX_INBOUND_TOTAL_BYTES = 24 * 1024 * 1024;
12
+ const TEXT_ATTACHMENT_PREVIEW_LINES = 2;
13
+ const TEXT_ATTACHMENT_PREVIEW_CHARS = 1000;
11
14
  const ALLOWED_MIME_TYPES = new Set([
12
15
  "image/jpeg",
13
16
  "image/png",
@@ -23,10 +26,7 @@ const ALLOWED_MIME_TYPES = new Set([
23
26
  "text/plain",
24
27
  ]);
25
28
  const EXTENSIONS_BY_MIME = {
26
- "image/jpeg": ".jpg",
27
- "image/png": ".png",
28
- "image/gif": ".gif",
29
- "image/webp": ".webp",
29
+ ...IMAGE_EXTENSION_BY_MIME,
30
30
  "audio/mpeg": ".mp3",
31
31
  "audio/ogg": ".ogg",
32
32
  "audio/wav": ".wav",
@@ -51,32 +51,42 @@ function kindFromMime(mimeType) {
51
51
  return "video";
52
52
  return "file";
53
53
  }
54
+ function textAttachmentPreview(buffer, mimeType) {
55
+ if (mimeType !== "text/plain")
56
+ return undefined;
57
+ const decoded = new TextDecoder("utf-8", { fatal: false }).decode(buffer);
58
+ const normalized = decoded.replace(/\r\n/g, "\n").replace(/\r/g, "\n").trim();
59
+ if (!normalized || normalized.includes("\uFFFD"))
60
+ return undefined;
61
+ const lines = normalized.split("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_LINES);
62
+ const preview = lines.join("\n").slice(0, TEXT_ATTACHMENT_PREVIEW_CHARS).trim();
63
+ return preview || undefined;
64
+ }
54
65
  function sniffText(buffer) {
55
66
  if (buffer.length === 0)
56
67
  return "text/plain";
57
68
  const head = buffer.subarray(0, Math.min(buffer.length, 512));
58
69
  if (head.includes(0))
59
70
  return undefined;
60
- return head.every((byte) => byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte <= 126))
61
- ? "text/plain"
62
- : undefined;
63
- }
64
- function sniffMimeType(buffer, declared) {
65
- let detected;
66
- if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
67
- detected = "image/jpeg";
68
- else if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
69
- detected = "image/png";
71
+ let decoded;
72
+ try {
73
+ decoded = new TextDecoder("utf-8", { fatal: true }).decode(head);
70
74
  }
71
- else if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" ||
72
- buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
73
- detected = "image/gif";
75
+ catch {
76
+ return undefined;
74
77
  }
75
- else if (buffer.subarray(0, 4).toString("ascii") === "RIFF" &&
76
- buffer.subarray(8, 12).toString("ascii") === "WEBP") {
77
- detected = "image/webp";
78
+ for (const char of decoded) {
79
+ const code = char.codePointAt(0) ?? 0;
80
+ if (code === 9 || code === 10 || code === 13)
81
+ continue;
82
+ if (code < 32 || code === 127)
83
+ return undefined;
78
84
  }
79
- else if (buffer.subarray(0, 4).toString("ascii") === "%PDF")
85
+ return "text/plain";
86
+ }
87
+ function sniffMimeType(buffer, declared) {
88
+ let detected = sniffImageMimeType(buffer);
89
+ if (!detected && buffer.subarray(0, 4).toString("ascii") === "%PDF")
80
90
  detected = "application/pdf";
81
91
  else if (buffer.subarray(0, 3).toString("ascii") === "ID3" ||
82
92
  buffer.subarray(0, 2).equals(Buffer.from([0xff, 0xfb]))) {
@@ -184,6 +194,7 @@ export async function materializeInboundAttachments(config, inputs) {
184
194
  }
185
195
  const stored = [];
186
196
  const writtenPaths = [];
197
+ const existingDerivedPaths = await knownDerivedImagePaths(config);
187
198
  try {
188
199
  for (const attachment of prepared) {
189
200
  const dir = resolve(attachmentsDir(config), "inbound", attachment.source);
@@ -203,8 +214,24 @@ export async function materializeInboundAttachments(config, inputs) {
203
214
  source: attachment.source,
204
215
  sha256: attachment.sha256,
205
216
  };
217
+ const textPreview = textAttachmentPreview(attachment.buffer, attachment.mimeType);
218
+ if (textPreview) {
219
+ finalAttachment.derived = {
220
+ ...finalAttachment.derived,
221
+ text: {
222
+ provider: "local",
223
+ model: "text-preview",
224
+ label: "preview",
225
+ text: textPreview,
226
+ },
227
+ };
228
+ }
206
229
  const derivedImage = await ensureInlineImageDerivative(config, finalAttachment);
207
230
  if (derivedImage) {
231
+ if (derivedImage.localPath && !existingDerivedPaths.has(derivedImage.localPath)) {
232
+ writtenPaths.push(derivedImage.localPath);
233
+ existingDerivedPaths.add(derivedImage.localPath);
234
+ }
208
235
  finalAttachment.derived = {
209
236
  ...finalAttachment.derived,
210
237
  image: derivedImage,
@@ -219,6 +246,11 @@ export async function materializeInboundAttachments(config, inputs) {
219
246
  throw error;
220
247
  }
221
248
  }
249
+ async function knownDerivedImagePaths(config) {
250
+ const dir = resolve(attachmentsDir(config), "derived", "image");
251
+ const entries = await readdir(dir).catch(() => []);
252
+ return new Set(entries.map((entry) => resolve(dir, entry)));
253
+ }
222
254
  export async function promptImagesFromAttachments(attachments) {
223
255
  const images = [];
224
256
  const notes = [];
@@ -250,7 +282,16 @@ export async function promptImagesFromAttachments(attachments) {
250
282
  export function promptAttachmentNotes(attachments) {
251
283
  return attachments
252
284
  .map((attachment) => {
253
- const attrs = `name="${attachment.name}" id="${attachment.id}" kind="${attachment.kind ?? "file"}" mime="${attachment.mimeType ?? "unknown"}" size="${attachment.size ?? "unknown"}"`;
285
+ const attrs = [
286
+ `name="${attachment.name}"`,
287
+ `id="${attachment.id}"`,
288
+ `kind="${attachment.kind ?? "file"}"`,
289
+ `mime="${attachment.mimeType ?? "unknown"}"`,
290
+ `size="${attachment.size ?? "unknown"}"`,
291
+ attachment.localPath ? `path="${attachment.localPath}"` : undefined,
292
+ ]
293
+ .filter(Boolean)
294
+ .join(" ");
254
295
  const derivedText = attachment.derived?.text?.text;
255
296
  if (derivedText) {
256
297
  const label = attachment.derived?.text?.label || (attachment.kind === "audio" ? "transcription" : "summary");
@@ -1,3 +1,4 @@
1
+ import { positiveIntegerOrDefault } from "../util.js";
1
2
  import { retrieveAmbientDiary } from "./ambient.js";
2
3
  const INJECTED_MEMORY_OPEN = "<injected_memory>";
3
4
  const INJECTED_MEMORY_CLOSE = "</injected_memory>";
@@ -64,9 +65,6 @@ export class AmbientDiaryInjector {
64
65
  }
65
66
  }
66
67
  }
67
- function positiveIntegerOrDefault(value, fallback) {
68
- return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
69
- }
70
68
  function nonNegativeIntegerOrDefault(value, fallback) {
71
69
  return value !== undefined && Number.isInteger(value) && value >= 0 ? value : fallback;
72
70
  }
@@ -1,4 +1,5 @@
1
1
  import { retrieveMemory, } from "../index/retrieval.js";
2
+ import { positiveIntegerOrDefault } from "../util.js";
2
3
  import { DIARY_CHUNK_CORPUS } from "./chunks.js";
3
4
  const DEFAULT_LIMIT = 4;
4
5
  const DEFAULT_CANDIDATE_MULTIPLIER = 5;
@@ -119,6 +120,3 @@ function normalizeUnit(value) {
119
120
  const absolute = Math.abs(value);
120
121
  return Math.max(0, Math.min(1, absolute > 1 ? absolute / 10 : absolute));
121
122
  }
122
- function positiveIntegerOrDefault(value, fallback) {
123
- return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
124
- }
@@ -1,4 +1,5 @@
1
1
  import { basename } from "node:path";
2
+ import { positiveIntegerOrDefault } from "../util.js";
2
3
  export const DIARY_CHUNK_CORPUS = "diary_chunk";
3
4
  const DEFAULT_MAX_CHARS = 2400;
4
5
  const DIARY_DATE_RE = /^(\d{4}-\d{2}-\d{2})\.md$/;
@@ -226,6 +227,3 @@ function stripInlineMarkdown(value) {
226
227
  function isMarkdownHeading(line) {
227
228
  return /^#{1,6}\s+/.test(line);
228
229
  }
229
- function positiveIntegerOrDefault(value, fallback) {
230
- return value !== undefined && Number.isInteger(value) && value > 0 ? value : fallback;
231
- }
@@ -1,5 +1,6 @@
1
1
  import { readdir, readFile, stat } from "node:fs/promises";
2
2
  import { basename, isAbsolute, join, resolve } from "node:path";
3
+ import { isEnoent } from "../../util/fs.js";
3
4
  import { DIARY_CHUNK_CORPUS, indexDiaryMarkdown } from "./chunks.js";
4
5
  export const DIARY_INDEX_FILE_RE = /^\d{4}-\d{2}-\d{2}\.md$/;
5
6
  export function isDatedDiaryMarkdownFile(path) {
@@ -22,9 +23,6 @@ export async function listDiaryMarkdownFiles(config) {
22
23
  .map((entry) => join(config.memory.diariesDir, entry.name))
23
24
  .sort();
24
25
  }
25
- function isEnoent(error) {
26
- return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
27
- }
28
26
  export async function indexDiaryFile(options) {
29
27
  const path = resolveDiaryPath(options.config, options.path);
30
28
  const sourceId = basename(path);
@@ -1,3 +1,4 @@
1
+ import { runInTransaction } from "./util.js";
1
2
  export function runDoctor(stores, opts = {}) {
2
3
  void opts;
3
4
  const findings = [];
@@ -32,10 +33,7 @@ export function applyDoctorFixes(stores, report) {
32
33
  fixed += before;
33
34
  }
34
35
  };
35
- if (stores.index.db.inTransaction)
36
- runIndexFixes();
37
- else
38
- stores.index.db.transaction(runIndexFixes).immediate();
36
+ runInTransaction(stores.index.db, runIndexFixes);
39
37
  const runLcmFixes = () => {
40
38
  fixed += stores.lcm.db
41
39
  .prepare(`DELETE FROM lcm_segments
@@ -63,10 +61,7 @@ export function applyDoctorFixes(stores, report) {
63
61
  }
64
62
  }
65
63
  };
66
- if (stores.lcm.db.inTransaction)
67
- runLcmFixes();
68
- else
69
- stores.lcm.db.transaction(runLcmFixes).immediate();
64
+ runInTransaction(stores.lcm.db, runLcmFixes);
70
65
  if (report.findings.some((finding) => finding.kind === "summary_fk_violation")) {
71
66
  warnings.push("summary FK violations were not modified; inspect LCM summary lineage manually");
72
67
  }