@qearlyao/familiar 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +31 -0
- package/HEARTBEAT.md +23 -0
- package/LICENSE +21 -0
- package/MEMORY.md +1 -0
- package/README.md +245 -0
- package/SOUL.md +13 -0
- package/USER.md +13 -0
- package/config.example.toml +221 -0
- package/dist/agent-events.js +167 -0
- package/dist/agent.js +590 -0
- package/dist/browser-tools.js +638 -0
- package/dist/chat-log.js +130 -0
- package/dist/cli.js +168 -0
- package/dist/config.js +804 -0
- package/dist/data-retention.js +54 -0
- package/dist/discord.js +1203 -0
- package/dist/generated-media.js +86 -0
- package/dist/image-derivatives.js +102 -0
- package/dist/image-gen.js +440 -0
- package/dist/inbound-attachments.js +266 -0
- package/dist/index.js +10 -0
- package/dist/media-understanding.js +120 -0
- package/dist/memory/diary/ambient-injector.js +180 -0
- package/dist/memory/diary/ambient.js +124 -0
- package/dist/memory/diary/chunks.js +231 -0
- package/dist/memory/diary/index.js +3 -0
- package/dist/memory/diary/indexer.js +93 -0
- package/dist/memory/doctor.js +250 -0
- package/dist/memory/index/chunk-indexer.js +151 -0
- package/dist/memory/index/embedding-provider.js +119 -0
- package/dist/memory/index/fts-query.js +18 -0
- package/dist/memory/index/retrieval.js +246 -0
- package/dist/memory/index/schema.js +157 -0
- package/dist/memory/index/store.js +513 -0
- package/dist/memory/index/vec.js +72 -0
- package/dist/memory/index/vector-codec.js +27 -0
- package/dist/memory/lcm/backfill.js +247 -0
- package/dist/memory/lcm/condense.js +146 -0
- package/dist/memory/lcm/context-transformer.js +662 -0
- package/dist/memory/lcm/context.js +421 -0
- package/dist/memory/lcm/eviction-score.js +38 -0
- package/dist/memory/lcm/index.js +6 -0
- package/dist/memory/lcm/indexer.js +200 -0
- package/dist/memory/lcm/normalize.js +235 -0
- package/dist/memory/lcm/schema.js +188 -0
- package/dist/memory/lcm/segment-manager.js +136 -0
- package/dist/memory/lcm/store.js +722 -0
- package/dist/memory/lcm/summarizer.js +258 -0
- package/dist/memory/lcm/types.js +1 -0
- package/dist/memory/operator.js +477 -0
- package/dist/memory/service.js +202 -0
- package/dist/memory/tools.js +205 -0
- package/dist/models.js +165 -0
- package/dist/persona.js +54 -0
- package/dist/runtime.js +493 -0
- package/dist/scheduler.js +200 -0
- package/dist/settings.js +116 -0
- package/dist/skills.js +38 -0
- package/dist/tts.js +143 -0
- package/dist/web-auth.js +105 -0
- package/dist/web-events.js +114 -0
- package/dist/web-http.js +29 -0
- package/dist/web-static.js +106 -0
- package/dist/web-tools.js +940 -0
- package/dist/web-types.js +2 -0
- package/dist/web.js +844 -0
- package/package.json +60 -0
- package/web/dist/assets/index-ClgkMgaq.css +2 -0
- package/web/dist/assets/index-Cu2QquuR.js +59 -0
- package/web/dist/favicon.svg +1 -0
- package/web/dist/icons.svg +24 -0
- package/web/dist/index.html +20 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { lstat, mkdir, readdir, rm } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
3
|
+
export function createGeneratedMediaSink() {
|
|
4
|
+
const attachments = [];
|
|
5
|
+
return {
|
|
6
|
+
add(attachment) {
|
|
7
|
+
attachments.push(attachment);
|
|
8
|
+
},
|
|
9
|
+
drain() {
|
|
10
|
+
return attachments.splice(0);
|
|
11
|
+
},
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export function generatedAttachmentsDir(config) {
|
|
15
|
+
return resolve(config.workspace.dataDir, "attachments", "generated");
|
|
16
|
+
}
|
|
17
|
+
export function attachmentsDir(config) {
|
|
18
|
+
return resolve(config.workspace.dataDir, "attachments");
|
|
19
|
+
}
|
|
20
|
+
export function browserScreenshotsDir(config) {
|
|
21
|
+
return resolve(config.workspace.dataDir, "attachments", "screenshot");
|
|
22
|
+
}
|
|
23
|
+
export async function ensureGeneratedAttachmentsDir(config) {
|
|
24
|
+
const dir = generatedAttachmentsDir(config);
|
|
25
|
+
await mkdir(dir, { recursive: true });
|
|
26
|
+
return dir;
|
|
27
|
+
}
|
|
28
|
+
export async function ensureBrowserScreenshotsDir(config) {
|
|
29
|
+
const dir = browserScreenshotsDir(config);
|
|
30
|
+
await mkdir(dir, { recursive: true });
|
|
31
|
+
return dir;
|
|
32
|
+
}
|
|
33
|
+
export async function cleanupGeneratedAttachments(config, now = Date.now()) {
|
|
34
|
+
const retentionDays = config.media.generatedRetentionDays;
|
|
35
|
+
if (retentionDays <= 0)
|
|
36
|
+
return 0;
|
|
37
|
+
const dir = generatedAttachmentsDir(config);
|
|
38
|
+
const cutoff = now - retentionDays * 24 * 60 * 60 * 1000;
|
|
39
|
+
let entries;
|
|
40
|
+
try {
|
|
41
|
+
entries = await readdir(dir);
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
|
|
45
|
+
return 0;
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
let removed = 0;
|
|
49
|
+
for (const entry of entries) {
|
|
50
|
+
const path = join(dir, entry);
|
|
51
|
+
const fileStat = await lstat(path).catch(() => undefined);
|
|
52
|
+
if (!fileStat?.isFile() || fileStat.mtimeMs > cutoff)
|
|
53
|
+
continue;
|
|
54
|
+
await rm(path).catch((error) => {
|
|
55
|
+
if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT"))
|
|
56
|
+
throw error;
|
|
57
|
+
});
|
|
58
|
+
removed++;
|
|
59
|
+
}
|
|
60
|
+
return removed;
|
|
61
|
+
}
|
|
62
|
+
export function publicAttachmentPath(config, localPath) {
|
|
63
|
+
const absolutePath = resolve(localPath);
|
|
64
|
+
const generatedRelativePath = relative(generatedAttachmentsDir(config), absolutePath);
|
|
65
|
+
if (generatedRelativePath && !generatedRelativePath.startsWith("..") && !isAbsolute(generatedRelativePath)) {
|
|
66
|
+
return `/api/web/attachments/${generatedRelativePath
|
|
67
|
+
.split(/[\\/]+/)
|
|
68
|
+
.map(encodeURIComponent)
|
|
69
|
+
.join("/")}`;
|
|
70
|
+
}
|
|
71
|
+
const screenshotRelativePath = relative(browserScreenshotsDir(config), absolutePath);
|
|
72
|
+
if (screenshotRelativePath && !screenshotRelativePath.startsWith("..") && !isAbsolute(screenshotRelativePath)) {
|
|
73
|
+
return `/api/web/attachments/screenshot/${screenshotRelativePath
|
|
74
|
+
.split(/[\\/]+/)
|
|
75
|
+
.map(encodeURIComponent)
|
|
76
|
+
.join("/")}`;
|
|
77
|
+
}
|
|
78
|
+
const relativePath = relative(attachmentsDir(config), absolutePath);
|
|
79
|
+
if (relativePath.startsWith("..") || isAbsolute(relativePath) || relativePath === "") {
|
|
80
|
+
throw new Error(`Attachment path is outside generated attachments dir: ${localPath}`);
|
|
81
|
+
}
|
|
82
|
+
return `/api/web/attachments/${relativePath
|
|
83
|
+
.split(/[\\/]+/)
|
|
84
|
+
.map(encodeURIComponent)
|
|
85
|
+
.join("/")}`;
|
|
86
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { basename, dirname, extname, resolve } from "node:path";
|
|
4
|
+
import sharp from "sharp";
|
|
5
|
+
import { attachmentsDir } from "./generated-media.js";
|
|
6
|
+
export const MAX_INLINE_IMAGE_BASE64_BYTES = 4.5 * 1024 * 1024;
|
|
7
|
+
const DERIVED_IMAGE_MIME_TYPE = "image/webp";
|
|
8
|
+
const DERIVED_IMAGE_EXTENSION = ".webp";
|
|
9
|
+
const DERIVED_IMAGE_MAX_EDGE = 1600;
|
|
10
|
+
const DERIVED_IMAGE_QUALITY_STEPS = [82, 72, 62, 52];
|
|
11
|
+
const DERIVED_IMAGE_EDGE_STEPS = [1600, 1400, 1200, 1000, 800, 640];
|
|
12
|
+
function safeDerivedStem(name) {
|
|
13
|
+
const base = basename(name, extname(name))
|
|
14
|
+
.replace(/[^A-Za-z0-9._=-]+/g, "_")
|
|
15
|
+
.slice(0, 96);
|
|
16
|
+
return base || "image";
|
|
17
|
+
}
|
|
18
|
+
function derivedImagePath(config, source, fingerprint) {
|
|
19
|
+
return resolve(attachmentsDir(config), "derived", "image", `${safeDerivedStem(source.name)}-${fingerprint}${DERIVED_IMAGE_EXTENSION}`);
|
|
20
|
+
}
|
|
21
|
+
async function sourceFingerprint(source, localPath) {
|
|
22
|
+
if (source.sha256)
|
|
23
|
+
return source.sha256.slice(0, 16);
|
|
24
|
+
const buffer = await readFile(localPath);
|
|
25
|
+
return createHash("sha256").update(buffer).digest("hex").slice(0, 16);
|
|
26
|
+
}
|
|
27
|
+
function inlineBase64Size(bytes) {
|
|
28
|
+
return Math.ceil(bytes / 3) * 4;
|
|
29
|
+
}
|
|
30
|
+
async function outputWithinInlineLimit(path) {
|
|
31
|
+
const fileStat = await stat(path).catch(() => undefined);
|
|
32
|
+
return !!fileStat && inlineBase64Size(fileStat.size) <= MAX_INLINE_IMAGE_BASE64_BYTES;
|
|
33
|
+
}
|
|
34
|
+
function derivativeNote(width, height) {
|
|
35
|
+
const dimensions = width && height ? ` ${width}x${height}` : "";
|
|
36
|
+
return `[Resized image${dimensions} for inline model input.]`;
|
|
37
|
+
}
|
|
38
|
+
export async function ensureInlineImageDerivative(config, source) {
|
|
39
|
+
const localPath = source.localPath;
|
|
40
|
+
if (!localPath)
|
|
41
|
+
return undefined;
|
|
42
|
+
if (!source.mimeType?.startsWith("image/"))
|
|
43
|
+
return undefined;
|
|
44
|
+
if (source.derived?.image?.localPath && (await outputWithinInlineLimit(source.derived.image.localPath))) {
|
|
45
|
+
return source.derived.image;
|
|
46
|
+
}
|
|
47
|
+
const sourceStat = await stat(localPath).catch(() => undefined);
|
|
48
|
+
const sourceSize = source.size ?? sourceStat?.size;
|
|
49
|
+
if (sourceSize !== undefined && inlineBase64Size(sourceSize) <= MAX_INLINE_IMAGE_BASE64_BYTES)
|
|
50
|
+
return undefined;
|
|
51
|
+
const fingerprint = await sourceFingerprint(source, localPath);
|
|
52
|
+
const outputPath = derivedImagePath(config, source, fingerprint);
|
|
53
|
+
if (await outputWithinInlineLimit(outputPath)) {
|
|
54
|
+
const metadata = await sharp(outputPath).metadata();
|
|
55
|
+
return {
|
|
56
|
+
localPath: outputPath,
|
|
57
|
+
mimeType: DERIVED_IMAGE_MIME_TYPE,
|
|
58
|
+
size: (await stat(outputPath)).size,
|
|
59
|
+
width: metadata.width,
|
|
60
|
+
height: metadata.height,
|
|
61
|
+
note: derivativeNote(metadata.width, metadata.height),
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const original = sharp(localPath, { animated: false, limitInputPixels: 64_000_000 }).rotate();
|
|
65
|
+
let best;
|
|
66
|
+
for (const edge of DERIVED_IMAGE_EDGE_STEPS) {
|
|
67
|
+
if (edge > DERIVED_IMAGE_MAX_EDGE)
|
|
68
|
+
continue;
|
|
69
|
+
for (const quality of DERIVED_IMAGE_QUALITY_STEPS) {
|
|
70
|
+
const output = await original
|
|
71
|
+
.clone()
|
|
72
|
+
.resize({ width: edge, height: edge, fit: "inside", withoutEnlargement: true })
|
|
73
|
+
.webp({ quality })
|
|
74
|
+
.toBuffer({ resolveWithObject: true });
|
|
75
|
+
best = { buffer: output.data, width: output.info.width, height: output.info.height };
|
|
76
|
+
if (inlineBase64Size(output.data.length) <= MAX_INLINE_IMAGE_BASE64_BYTES) {
|
|
77
|
+
await mkdir(dirname(outputPath), { recursive: true });
|
|
78
|
+
await writeFile(outputPath, output.data);
|
|
79
|
+
return {
|
|
80
|
+
localPath: outputPath,
|
|
81
|
+
mimeType: DERIVED_IMAGE_MIME_TYPE,
|
|
82
|
+
size: output.data.length,
|
|
83
|
+
width: output.info.width,
|
|
84
|
+
height: output.info.height,
|
|
85
|
+
note: derivativeNote(output.info.width, output.info.height),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (!best)
|
|
91
|
+
return undefined;
|
|
92
|
+
await mkdir(dirname(outputPath), { recursive: true });
|
|
93
|
+
await writeFile(outputPath, best.buffer);
|
|
94
|
+
return {
|
|
95
|
+
localPath: outputPath,
|
|
96
|
+
mimeType: DERIVED_IMAGE_MIME_TYPE,
|
|
97
|
+
size: best.buffer.length,
|
|
98
|
+
width: best.width,
|
|
99
|
+
height: best.height,
|
|
100
|
+
note: derivativeNote(best.width, best.height),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { lstat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { basename, extname, isAbsolute, relative, resolve } from "node:path";
|
|
4
|
+
import { findEnvKeys, generateImages, getEnvApiKey, getImageModels, getImageProviders, } from "@earendil-works/pi-ai";
|
|
5
|
+
import { Type } from "typebox";
|
|
6
|
+
import { ensureGeneratedAttachmentsDir } from "./generated-media.js";
|
|
7
|
+
import { ensureInlineImageDerivative } from "./image-derivatives.js";
|
|
8
|
+
import { promptImagesFromAttachments } from "./inbound-attachments.js";
|
|
9
|
+
import { parseModelRef } from "./models.js";
|
|
10
|
+
const IMAGE_GEN_NOTICE_PREFIX = "Generated image attachment:";
|
|
11
|
+
const OPENROUTER_IMAGE_BASE_URL = "https://openrouter.ai/api/v1";
|
|
12
|
+
const IMAGE_MIME_BY_EXTENSION = {
|
|
13
|
+
".jpg": "image/jpeg",
|
|
14
|
+
".jpeg": "image/jpeg",
|
|
15
|
+
".png": "image/png",
|
|
16
|
+
".gif": "image/gif",
|
|
17
|
+
".webp": "image/webp",
|
|
18
|
+
};
|
|
19
|
+
const imageGenSchema = Type.Object({
|
|
20
|
+
prompt: Type.String({ description: "Image generation prompt." }),
|
|
21
|
+
referenceImages: Type.Optional(Type.Array(Type.String(), {
|
|
22
|
+
description: "Optional. Image attachment IDs or names, or workspace-relative image file paths, to use as visual references. Prefer IDs from the attachment tags when available.",
|
|
23
|
+
})),
|
|
24
|
+
}, { additionalProperties: false });
|
|
25
|
+
function formatImageGenNotice(name) {
|
|
26
|
+
return `${IMAGE_GEN_NOTICE_PREFIX} ${name}`;
|
|
27
|
+
}
|
|
28
|
+
export function imageExtension(mimeType) {
|
|
29
|
+
const normalized = mimeType.toLowerCase();
|
|
30
|
+
if (normalized === "image/jpeg" || normalized === "image/jpg")
|
|
31
|
+
return "jpg";
|
|
32
|
+
if (normalized === "image/webp")
|
|
33
|
+
return "webp";
|
|
34
|
+
if (normalized === "image/gif")
|
|
35
|
+
return "gif";
|
|
36
|
+
if (normalized === "image/svg+xml")
|
|
37
|
+
return "svg";
|
|
38
|
+
return "png";
|
|
39
|
+
}
|
|
40
|
+
function resolveConfiguredBaseUrl(config, ref, model) {
|
|
41
|
+
return (config.models.baseUrls[ref.key] ??
|
|
42
|
+
config.models.baseUrls[ref.provider] ??
|
|
43
|
+
model?.baseUrl ??
|
|
44
|
+
(ref.provider === "openrouter" ? OPENROUTER_IMAGE_BASE_URL : undefined));
|
|
45
|
+
}
|
|
46
|
+
function resolveConfiguredApiKeyEnv(config, model) {
|
|
47
|
+
return config.models.apiKeyEnvs[`${model.provider}/${model.id}`] ?? config.models.apiKeyEnvs[model.provider];
|
|
48
|
+
}
|
|
49
|
+
function findBuiltInImageModel(ref) {
|
|
50
|
+
if (!getImageProviders().includes(ref.provider))
|
|
51
|
+
return undefined;
|
|
52
|
+
return getImageModels(ref.provider).find((model) => model.id === ref.id);
|
|
53
|
+
}
|
|
54
|
+
export function resolveImageModel(config, ref) {
|
|
55
|
+
const builtIn = findBuiltInImageModel(ref);
|
|
56
|
+
const baseUrl = resolveConfiguredBaseUrl(config, ref, builtIn);
|
|
57
|
+
if (!baseUrl) {
|
|
58
|
+
throw new Error(`Missing image model base URL for ${ref.key}. Set models.base_urls.${ref.provider}.`);
|
|
59
|
+
}
|
|
60
|
+
const model = builtIn
|
|
61
|
+
? { ...builtIn, api: config.imageGen.api, baseUrl }
|
|
62
|
+
: {
|
|
63
|
+
id: ref.id,
|
|
64
|
+
name: ref.id,
|
|
65
|
+
api: config.imageGen.api,
|
|
66
|
+
provider: ref.provider,
|
|
67
|
+
baseUrl,
|
|
68
|
+
input: ["text", "image"],
|
|
69
|
+
output: ["image", "text"],
|
|
70
|
+
cost: {
|
|
71
|
+
input: 0,
|
|
72
|
+
output: 0,
|
|
73
|
+
cacheRead: 0,
|
|
74
|
+
cacheWrite: 0,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
return model;
|
|
78
|
+
}
|
|
79
|
+
function resolveImageModelApiKey(config, model) {
|
|
80
|
+
const configuredEnv = resolveConfiguredApiKeyEnv(config, model);
|
|
81
|
+
if (configuredEnv) {
|
|
82
|
+
const apiKey = process.env[configuredEnv];
|
|
83
|
+
if (!apiKey)
|
|
84
|
+
throw new Error(`Missing image generation API key env: ${configuredEnv}`);
|
|
85
|
+
return apiKey;
|
|
86
|
+
}
|
|
87
|
+
const apiKey = getEnvApiKey(model.provider);
|
|
88
|
+
if (apiKey)
|
|
89
|
+
return apiKey;
|
|
90
|
+
const envKeys = findEnvKeys(model.provider);
|
|
91
|
+
const hint = envKeys?.length ? envKeys.join(", ") : `models.api_key_envs.${model.provider}`;
|
|
92
|
+
throw new Error(`Missing image generation API key for ${model.provider}/${model.id}: ${hint}`);
|
|
93
|
+
}
|
|
94
|
+
function imageResultError(result) {
|
|
95
|
+
if (result.stopReason === "error")
|
|
96
|
+
return result.errorMessage ?? "image generation failed";
|
|
97
|
+
if (result.stopReason === "aborted")
|
|
98
|
+
return result.errorMessage ?? "image generation aborted";
|
|
99
|
+
if (!result.output.some((item) => item.type === "image"))
|
|
100
|
+
return "image generation returned no image output";
|
|
101
|
+
return undefined;
|
|
102
|
+
}
|
|
103
|
+
function textOutput(result) {
|
|
104
|
+
return result.output
|
|
105
|
+
.filter((item) => item.type === "text")
|
|
106
|
+
.map((item) => item.text.trim())
|
|
107
|
+
.filter(Boolean)
|
|
108
|
+
.join("\n");
|
|
109
|
+
}
|
|
110
|
+
function imageMimeTypeFromBytes(buffer) {
|
|
111
|
+
if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff])))
|
|
112
|
+
return "image/jpeg";
|
|
113
|
+
if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
|
114
|
+
return "image/png";
|
|
115
|
+
}
|
|
116
|
+
if (buffer.subarray(0, 6).toString("ascii") === "GIF87a" || buffer.subarray(0, 6).toString("ascii") === "GIF89a") {
|
|
117
|
+
return "image/gif";
|
|
118
|
+
}
|
|
119
|
+
if (buffer.subarray(0, 4).toString("ascii") === "RIFF" && buffer.subarray(8, 12).toString("ascii") === "WEBP") {
|
|
120
|
+
return "image/webp";
|
|
121
|
+
}
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
function recoveredImageFromBase64(value) {
|
|
125
|
+
const data = value.trim();
|
|
126
|
+
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(data) || data.length % 4 !== 0)
|
|
127
|
+
return undefined;
|
|
128
|
+
const buffer = Buffer.from(data, "base64");
|
|
129
|
+
if (!buffer.length)
|
|
130
|
+
return undefined;
|
|
131
|
+
const detectedMimeType = imageMimeTypeFromBytes(buffer);
|
|
132
|
+
if (!detectedMimeType)
|
|
133
|
+
return undefined;
|
|
134
|
+
return {
|
|
135
|
+
mimeType: detectedMimeType,
|
|
136
|
+
data,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function recoveredImageFromText(text) {
|
|
140
|
+
const trimmed = text.trim();
|
|
141
|
+
const dataUrlMatch = trimmed.match(/^data:(image\/[^;]+);base64,([A-Za-z0-9+/]+={0,2})$/);
|
|
142
|
+
if (dataUrlMatch)
|
|
143
|
+
return recoveredImageFromBase64(dataUrlMatch[2] ?? "");
|
|
144
|
+
const embeddedDataUrlMatch = text.match(/data:(image\/[^;)\]\s]+);base64,([A-Za-z0-9+/]+={0,2})/);
|
|
145
|
+
if (embeddedDataUrlMatch) {
|
|
146
|
+
return recoveredImageFromBase64(embeddedDataUrlMatch[2] ?? "");
|
|
147
|
+
}
|
|
148
|
+
return recoveredImageFromBase64(trimmed);
|
|
149
|
+
}
|
|
150
|
+
function normalizeCompatibleImageText(result) {
|
|
151
|
+
if (result.output.some((item) => item.type === "image"))
|
|
152
|
+
return result;
|
|
153
|
+
const output = [];
|
|
154
|
+
for (const item of result.output) {
|
|
155
|
+
if (item.type !== "text") {
|
|
156
|
+
output.push(item);
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const recovered = recoveredImageFromText(item.text);
|
|
160
|
+
if (!recovered) {
|
|
161
|
+
output.push(item);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
output.push({ type: "image", mimeType: recovered.mimeType, data: recovered.data });
|
|
165
|
+
}
|
|
166
|
+
if (!output.some((item) => item.type === "image"))
|
|
167
|
+
return result;
|
|
168
|
+
return { ...result, output };
|
|
169
|
+
}
|
|
170
|
+
function mimeTypeFromPath(path) {
|
|
171
|
+
return IMAGE_MIME_BY_EXTENSION[extname(path).toLowerCase()];
|
|
172
|
+
}
|
|
173
|
+
function resolveWorkspaceReferencePath(config, rawRef) {
|
|
174
|
+
const path = isAbsolute(rawRef) ? resolve(rawRef) : resolve(config.workspacePath, rawRef);
|
|
175
|
+
const workspaceRelative = relative(config.workspacePath, path);
|
|
176
|
+
if (!workspaceRelative || workspaceRelative.startsWith("..") || isAbsolute(workspaceRelative)) {
|
|
177
|
+
throw new Error(`Reference image path must be inside the workspace: ${rawRef}`);
|
|
178
|
+
}
|
|
179
|
+
return path;
|
|
180
|
+
}
|
|
181
|
+
async function collectWorkspaceReferenceImages(config, rawRef) {
|
|
182
|
+
const path = resolveWorkspaceReferencePath(config, rawRef);
|
|
183
|
+
const pathStat = await lstat(path).catch(() => undefined);
|
|
184
|
+
if (!pathStat)
|
|
185
|
+
throw new Error(`Reference image path not found: ${rawRef}`);
|
|
186
|
+
if (pathStat.isSymbolicLink())
|
|
187
|
+
throw new Error(`Reference image path cannot be a symlink: ${rawRef}`);
|
|
188
|
+
if (pathStat.isDirectory()) {
|
|
189
|
+
throw new Error(`Reference image path must be a file, not a folder: ${rawRef}`);
|
|
190
|
+
}
|
|
191
|
+
if (!pathStat.isFile())
|
|
192
|
+
throw new Error(`Reference image path is not a file or folder: ${rawRef}`);
|
|
193
|
+
const mimeType = mimeTypeFromPath(path);
|
|
194
|
+
if (!mimeType)
|
|
195
|
+
throw new Error(`Reference image path is not a supported image: ${rawRef}`);
|
|
196
|
+
return [
|
|
197
|
+
{
|
|
198
|
+
localPath: path,
|
|
199
|
+
name: basename(path),
|
|
200
|
+
mimeType,
|
|
201
|
+
size: pathStat.size,
|
|
202
|
+
},
|
|
203
|
+
];
|
|
204
|
+
}
|
|
205
|
+
function splitReferenceImages(attachments, references) {
|
|
206
|
+
if (!references?.length)
|
|
207
|
+
return { attachments: [], workspaceRefs: [] };
|
|
208
|
+
const imageAttachments = attachments.filter((attachment) => {
|
|
209
|
+
return (attachment.localPath &&
|
|
210
|
+
attachment.mimeType?.startsWith("image/") &&
|
|
211
|
+
(!attachment.kind || attachment.kind === "image"));
|
|
212
|
+
});
|
|
213
|
+
const selected = [];
|
|
214
|
+
const workspaceRefs = [];
|
|
215
|
+
const seenAttachments = new Set();
|
|
216
|
+
const seenWorkspaceRefs = new Set();
|
|
217
|
+
for (const rawRef of references) {
|
|
218
|
+
const ref = rawRef.trim();
|
|
219
|
+
if (!ref)
|
|
220
|
+
continue;
|
|
221
|
+
const attachment = imageAttachments.find((candidate) => candidate.id === ref || candidate.name === ref);
|
|
222
|
+
if (attachment) {
|
|
223
|
+
if (seenAttachments.has(attachment.id))
|
|
224
|
+
continue;
|
|
225
|
+
seenAttachments.add(attachment.id);
|
|
226
|
+
selected.push(attachment);
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const anyAttachment = attachments.find((candidate) => candidate.id === ref || candidate.name === ref);
|
|
230
|
+
if (anyAttachment) {
|
|
231
|
+
throw new Error(`Reference image is not an image attachment: ${ref}`);
|
|
232
|
+
}
|
|
233
|
+
if (seenWorkspaceRefs.has(ref))
|
|
234
|
+
continue;
|
|
235
|
+
seenWorkspaceRefs.add(ref);
|
|
236
|
+
workspaceRefs.push(ref);
|
|
237
|
+
}
|
|
238
|
+
return { attachments: selected, workspaceRefs };
|
|
239
|
+
}
|
|
240
|
+
async function workspaceReferenceAttachments(config, images) {
|
|
241
|
+
const attachments = [];
|
|
242
|
+
for (const image of images) {
|
|
243
|
+
const attachment = {
|
|
244
|
+
id: `workspace:${image.localPath}`,
|
|
245
|
+
name: image.name,
|
|
246
|
+
kind: "image",
|
|
247
|
+
mimeType: image.mimeType,
|
|
248
|
+
size: image.size,
|
|
249
|
+
localPath: image.localPath,
|
|
250
|
+
};
|
|
251
|
+
const derivedImage = await ensureInlineImageDerivative(config, attachment);
|
|
252
|
+
if (derivedImage) {
|
|
253
|
+
attachment.derived = {
|
|
254
|
+
...attachment.derived,
|
|
255
|
+
image: derivedImage,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
attachments.push(attachment);
|
|
259
|
+
}
|
|
260
|
+
return attachments;
|
|
261
|
+
}
|
|
262
|
+
async function buildImageContext(model, prompt, references, workspaceRefs, config) {
|
|
263
|
+
const input = [{ type: "text", text: prompt }];
|
|
264
|
+
const hasReferences = references.length > 0 || workspaceRefs.some((ref) => ref.trim().length > 0);
|
|
265
|
+
if (!hasReferences)
|
|
266
|
+
return { input };
|
|
267
|
+
if (!model.input.includes("image")) {
|
|
268
|
+
throw new Error(`Image model does not support reference images: ${model.provider}/${model.id}`);
|
|
269
|
+
}
|
|
270
|
+
const workspaceImages = [];
|
|
271
|
+
const seenWorkspaceImagePaths = new Set();
|
|
272
|
+
for (const rawRef of workspaceRefs) {
|
|
273
|
+
if (!rawRef.trim())
|
|
274
|
+
continue;
|
|
275
|
+
for (const image of await collectWorkspaceReferenceImages(config, rawRef)) {
|
|
276
|
+
if (seenWorkspaceImagePaths.has(image.localPath))
|
|
277
|
+
continue;
|
|
278
|
+
seenWorkspaceImagePaths.add(image.localPath);
|
|
279
|
+
workspaceImages.push(image);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const promptImages = await promptImagesFromAttachments([
|
|
283
|
+
...references,
|
|
284
|
+
...(await workspaceReferenceAttachments(config, workspaceImages)),
|
|
285
|
+
]);
|
|
286
|
+
if (promptImages.promptSuffix)
|
|
287
|
+
input.push({ type: "text", text: promptImages.promptSuffix });
|
|
288
|
+
input.push(...promptImages.images);
|
|
289
|
+
if (!promptImages.images.length)
|
|
290
|
+
throw new Error("No reference images could be inlined for image_gen.");
|
|
291
|
+
return { input };
|
|
292
|
+
}
|
|
293
|
+
async function writeGeneratedImages(config, mediaSink, result) {
|
|
294
|
+
const attachmentDir = await ensureGeneratedAttachmentsDir(config);
|
|
295
|
+
const attachments = [];
|
|
296
|
+
for (const item of result.output) {
|
|
297
|
+
if (item.type !== "image")
|
|
298
|
+
continue;
|
|
299
|
+
const buffer = Buffer.from(item.data, "base64");
|
|
300
|
+
const extension = imageExtension(item.mimeType);
|
|
301
|
+
const id = `image_gen_${randomUUID()}`;
|
|
302
|
+
const name = `${id}.${extension}`;
|
|
303
|
+
const localPath = resolve(attachmentDir, name);
|
|
304
|
+
await writeFile(localPath, buffer);
|
|
305
|
+
const attachment = {
|
|
306
|
+
id,
|
|
307
|
+
name,
|
|
308
|
+
kind: "image",
|
|
309
|
+
source: "generated",
|
|
310
|
+
mimeType: item.mimeType,
|
|
311
|
+
size: buffer.length,
|
|
312
|
+
localPath,
|
|
313
|
+
provider: result.provider,
|
|
314
|
+
toolName: "image_gen",
|
|
315
|
+
};
|
|
316
|
+
mediaSink.add(attachment);
|
|
317
|
+
attachments.push({
|
|
318
|
+
id,
|
|
319
|
+
name,
|
|
320
|
+
localPath,
|
|
321
|
+
mimeType: item.mimeType,
|
|
322
|
+
size: buffer.length,
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
return attachments;
|
|
326
|
+
}
|
|
327
|
+
async function tryGenerateImages(config, ref, prompt, references, workspaceRefs, signal, generate) {
|
|
328
|
+
const model = resolveImageModel(config, ref);
|
|
329
|
+
const context = await buildImageContext(model, prompt, references, workspaceRefs, config);
|
|
330
|
+
return {
|
|
331
|
+
model,
|
|
332
|
+
result: normalizeCompatibleImageText(await generate(model, context, {
|
|
333
|
+
apiKey: resolveImageModelApiKey(config, model),
|
|
334
|
+
signal,
|
|
335
|
+
timeoutMs: config.imageGen.timeoutMs,
|
|
336
|
+
})),
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
function attemptDetails(model, result) {
|
|
340
|
+
return {
|
|
341
|
+
provider: model.provider,
|
|
342
|
+
model: model.id,
|
|
343
|
+
api: model.api,
|
|
344
|
+
baseUrl: model.baseUrl,
|
|
345
|
+
...(result.responseId ? { responseId: result.responseId } : {}),
|
|
346
|
+
stopReason: result.stopReason,
|
|
347
|
+
...(result.errorMessage ? { errorMessage: result.errorMessage } : {}),
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
export function createImageGenTool(config, mediaSink, deps = {}) {
|
|
351
|
+
return {
|
|
352
|
+
name: "image_gen",
|
|
353
|
+
label: "Image Gen",
|
|
354
|
+
description: "make an image from a prompt. pass referenceImages to riff on existing pictures.",
|
|
355
|
+
parameters: imageGenSchema,
|
|
356
|
+
executionMode: "sequential",
|
|
357
|
+
async execute(_toolCallId, input, signal) {
|
|
358
|
+
const prompt = input.prompt.trim();
|
|
359
|
+
if (!prompt)
|
|
360
|
+
throw new Error("image_gen prompt is required.");
|
|
361
|
+
const primaryRef = parseModelRef(config.imageGen.model);
|
|
362
|
+
if (!primaryRef)
|
|
363
|
+
throw new Error(`Invalid image_gen.model: ${config.imageGen.model}`);
|
|
364
|
+
const fallbackRef = config.imageGen.fallbackModel ? parseModelRef(config.imageGen.fallbackModel) : undefined;
|
|
365
|
+
if (config.imageGen.fallbackModel && !fallbackRef) {
|
|
366
|
+
throw new Error(`Invalid image_gen.fallback_model: ${config.imageGen.fallbackModel}`);
|
|
367
|
+
}
|
|
368
|
+
const allAttachmentRefs = deps.referenceAttachments?.() ?? [];
|
|
369
|
+
const { attachments: attachmentReferences, workspaceRefs: workspaceReferences } = splitReferenceImages(allAttachmentRefs, input.referenceImages);
|
|
370
|
+
const generate = deps.generateImages ?? generateImages;
|
|
371
|
+
const attempts = [];
|
|
372
|
+
let selected;
|
|
373
|
+
let selectedError = "";
|
|
374
|
+
for (const ref of [primaryRef, fallbackRef].filter((ref) => !!ref)) {
|
|
375
|
+
let attempt;
|
|
376
|
+
try {
|
|
377
|
+
attempt = await tryGenerateImages(config, ref, prompt, attachmentReferences, workspaceReferences, signal, generate);
|
|
378
|
+
}
|
|
379
|
+
catch (error) {
|
|
380
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
381
|
+
let baseUrl = "";
|
|
382
|
+
try {
|
|
383
|
+
baseUrl = resolveImageModel(config, ref).baseUrl;
|
|
384
|
+
}
|
|
385
|
+
catch {
|
|
386
|
+
baseUrl = "";
|
|
387
|
+
}
|
|
388
|
+
attempts.push({
|
|
389
|
+
provider: ref.provider,
|
|
390
|
+
model: ref.id,
|
|
391
|
+
api: config.imageGen.api,
|
|
392
|
+
baseUrl,
|
|
393
|
+
stopReason: "error",
|
|
394
|
+
errorMessage: message,
|
|
395
|
+
});
|
|
396
|
+
if (message.includes("does not support reference images") && ref === primaryRef && fallbackRef) {
|
|
397
|
+
selectedError = message;
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
throw error;
|
|
401
|
+
}
|
|
402
|
+
if (!attempt)
|
|
403
|
+
continue;
|
|
404
|
+
attempts.push(attemptDetails(attempt.model, attempt.result));
|
|
405
|
+
const error = imageResultError(attempt.result);
|
|
406
|
+
if (!error) {
|
|
407
|
+
selected = attempt;
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
selectedError = error;
|
|
411
|
+
if (attempt.result.stopReason === "aborted")
|
|
412
|
+
break;
|
|
413
|
+
}
|
|
414
|
+
if (!selected)
|
|
415
|
+
throw new Error(`Image generation failed: ${selectedError}`);
|
|
416
|
+
const attachments = await writeGeneratedImages(config, mediaSink, selected.result);
|
|
417
|
+
const notices = attachments.map((attachment) => formatImageGenNotice(attachment.name));
|
|
418
|
+
const sideText = textOutput(selected.result);
|
|
419
|
+
return {
|
|
420
|
+
content: [
|
|
421
|
+
{
|
|
422
|
+
type: "text",
|
|
423
|
+
text: [sideText, ...notices].filter(Boolean).join("\n"),
|
|
424
|
+
},
|
|
425
|
+
],
|
|
426
|
+
details: {
|
|
427
|
+
provider: selected.model.provider,
|
|
428
|
+
model: selected.model.id,
|
|
429
|
+
api: selected.model.api,
|
|
430
|
+
baseUrl: selected.model.baseUrl,
|
|
431
|
+
prompt,
|
|
432
|
+
...(selected.result.responseId ? { responseId: selected.result.responseId } : {}),
|
|
433
|
+
...(sideText ? { textOutput: sideText } : {}),
|
|
434
|
+
attachments,
|
|
435
|
+
attempts,
|
|
436
|
+
},
|
|
437
|
+
};
|
|
438
|
+
},
|
|
439
|
+
};
|
|
440
|
+
}
|