@vellumai/assistant 0.11.9-staging.2 → 0.11.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +9 -0
- package/docker-bun-no-autoserve.js +38 -0
- package/docker-node-launcher.sh +37 -0
- package/node_modules/@vellumai/avatar-catalog/src/colors.ts +8 -0
- package/node_modules/@vellumai/avatar-manifest/package.json +2 -1
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/accent.test.ts +118 -0
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/manifest.test.ts +83 -4
- package/node_modules/@vellumai/avatar-manifest/src/accent.ts +150 -0
- package/node_modules/@vellumai/avatar-manifest/src/index.ts +7 -0
- package/node_modules/@vellumai/avatar-manifest/src/manifest.ts +55 -3
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +40 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +21 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +40 -1
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +21 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +40 -1
- package/node_modules/@vellumai/service-contracts/src/channels.ts +21 -0
- package/openapi.yaml +116 -1
- package/package.json +1 -1
- package/src/__tests__/bun-no-autoserve.test.ts +178 -0
- package/src/avatar/__tests__/avatar-manifest.test.ts +10 -0
- package/src/avatar/__tests__/avatar-store.test.ts +130 -8
- package/src/avatar/__tests__/ensure-raster.test.ts +15 -0
- package/src/avatar/avatar-accent.ts +57 -0
- package/src/avatar/avatar-manifest.ts +20 -2
- package/src/avatar/avatar-store.ts +111 -4
- package/src/config/bundled-skills/media-processing/SKILL.md +3 -3
- package/src/config/bundled-skills/media-processing/TOOLS.json +1 -1
- package/src/config/bundled-skills/schedule/SKILL.md +1 -1
- package/src/platform/sync-avatar.test.ts +3 -0
- package/src/runtime/routes/__tests__/avatar-state-routes.test.ts +147 -27
- package/src/runtime/routes/avatar-routes.ts +84 -28
- package/src/runtime/routes/settings-routes.ts +1 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +60 -1
- package/src/tools/terminal/safe-env.ts +70 -1
|
@@ -32,7 +32,13 @@ import { tmpdir } from "node:os";
|
|
|
32
32
|
import { join } from "node:path";
|
|
33
33
|
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
34
34
|
|
|
35
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
backfillAccent,
|
|
37
|
+
clearAvatar,
|
|
38
|
+
setAccent,
|
|
39
|
+
setCharacter,
|
|
40
|
+
setImage,
|
|
41
|
+
} from "../avatar-store.js";
|
|
36
42
|
|
|
37
43
|
// A valid trait triple drawn from the real component set.
|
|
38
44
|
const VALID_TRAITS = { bodyShape: "blob", eyeStyle: "curious", color: "green" };
|
|
@@ -43,11 +49,18 @@ const ASCII_FILENAME = "character-ascii.txt";
|
|
|
43
49
|
const MANIFEST_FILENAME = "avatar.json";
|
|
44
50
|
const NATIVE_RENDER_TEST_TIMEOUT_MS = 15_000;
|
|
45
51
|
|
|
52
|
+
/** A 4x4 PNG of one red (#c81e1e), so an accent can be read out of it. */
|
|
53
|
+
const RED_PNG = Buffer.from(
|
|
54
|
+
"iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEklEQVQImWM4ISf3HxkzkC4AAEG4IDHG8wOiAAAAAElFTkSuQmCC",
|
|
55
|
+
"base64",
|
|
56
|
+
);
|
|
57
|
+
|
|
46
58
|
interface ManifestShape {
|
|
47
59
|
kind: string;
|
|
48
60
|
traits: Record<string, unknown> | null;
|
|
49
61
|
source: string | null;
|
|
50
62
|
image: { updatedAt: string; etag: string } | null;
|
|
63
|
+
accent: { hex: string; source: string } | null;
|
|
51
64
|
}
|
|
52
65
|
|
|
53
66
|
describe("avatar-store", () => {
|
|
@@ -117,6 +130,8 @@ describe("avatar-store", () => {
|
|
|
117
130
|
expect(manifest!.source).toBe("builder");
|
|
118
131
|
expect(manifest!.image).not.toBeNull();
|
|
119
132
|
expect(manifest!.image!.etag).toMatch(/^[0-9a-f]{16}$/);
|
|
133
|
+
// The accent is the chosen palette colour.
|
|
134
|
+
expect(manifest!.accent).toEqual({ hex: "#4C9B50", source: "palette" });
|
|
120
135
|
},
|
|
121
136
|
NATIVE_RENDER_TEST_TIMEOUT_MS,
|
|
122
137
|
);
|
|
@@ -135,8 +150,8 @@ describe("avatar-store", () => {
|
|
|
135
150
|
});
|
|
136
151
|
|
|
137
152
|
describe("setImage", () => {
|
|
138
|
-
test("writes the PNG and an image manifest", () => {
|
|
139
|
-
setImage(Buffer.from("fake png bytes"), "upload");
|
|
153
|
+
test("writes the PNG and an image manifest", async () => {
|
|
154
|
+
await setImage(Buffer.from("fake png bytes"), "upload");
|
|
140
155
|
|
|
141
156
|
expect(existsSync(path(IMAGE_FILENAME))).toBe(true);
|
|
142
157
|
expect(readFileSync(path(IMAGE_FILENAME)).toString()).toBe(
|
|
@@ -153,14 +168,24 @@ describe("avatar-store", () => {
|
|
|
153
168
|
expect(manifest!.image!.updatedAt).toMatch(
|
|
154
169
|
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,
|
|
155
170
|
);
|
|
171
|
+
// Bytes that do not decode carry no colour, and that is not a failure.
|
|
172
|
+
expect(manifest!.accent).toBeNull();
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test("reads the accent out of the image", async () => {
|
|
176
|
+
await setImage(RED_PNG, "upload");
|
|
177
|
+
expect(readManifestFile()!.accent).toEqual({
|
|
178
|
+
hex: "#c81e1e",
|
|
179
|
+
source: "derived",
|
|
180
|
+
});
|
|
156
181
|
});
|
|
157
182
|
|
|
158
|
-
test("removes stale character sidecars on transition", () => {
|
|
183
|
+
test("removes stale character sidecars on transition", async () => {
|
|
159
184
|
// Seed legacy character artifacts.
|
|
160
185
|
writeFileSync(path(TRAITS_FILENAME), JSON.stringify(VALID_TRAITS));
|
|
161
186
|
writeFileSync(path(ASCII_FILENAME), "ascii art");
|
|
162
187
|
|
|
163
|
-
setImage(Buffer.from("png"), "ai");
|
|
188
|
+
await setImage(Buffer.from("png"), "ai");
|
|
164
189
|
|
|
165
190
|
expect(existsSync(path(TRAITS_FILENAME))).toBe(false);
|
|
166
191
|
expect(existsSync(path(ASCII_FILENAME))).toBe(false);
|
|
@@ -168,15 +193,112 @@ describe("avatar-store", () => {
|
|
|
168
193
|
expect(readManifestFile()!.kind).toBe("image");
|
|
169
194
|
});
|
|
170
195
|
|
|
171
|
-
test("is idempotent across repeated calls", () => {
|
|
172
|
-
setImage(Buffer.from("v1"), "upload");
|
|
173
|
-
setImage(Buffer.from("v2"), "upload");
|
|
196
|
+
test("is idempotent across repeated calls", async () => {
|
|
197
|
+
await setImage(Buffer.from("v1"), "upload");
|
|
198
|
+
await setImage(Buffer.from("v2"), "upload");
|
|
174
199
|
|
|
175
200
|
expect(readFileSync(path(IMAGE_FILENAME)).toString()).toBe("v2");
|
|
176
201
|
expect(readManifestFile()!.kind).toBe("image");
|
|
177
202
|
});
|
|
178
203
|
});
|
|
179
204
|
|
|
205
|
+
describe("setAccent", () => {
|
|
206
|
+
test("writes a custom accent over an image without touching the artifacts", async () => {
|
|
207
|
+
await setImage(RED_PNG, "upload");
|
|
208
|
+
const before = readManifestFile()!;
|
|
209
|
+
|
|
210
|
+
const state = await setAccent("#12ab34");
|
|
211
|
+
expect(state?.accent).toEqual({ hex: "#12ab34", source: "custom" });
|
|
212
|
+
const after = readManifestFile()!;
|
|
213
|
+
expect(after.accent).toEqual({ hex: "#12ab34", source: "custom" });
|
|
214
|
+
expect(after.image).toEqual(before.image);
|
|
215
|
+
expect(readFileSync(path(IMAGE_FILENAME))).toEqual(RED_PNG);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test("null returns an image to the colour read out of it", async () => {
|
|
219
|
+
await setImage(RED_PNG, "upload");
|
|
220
|
+
await setAccent("#12ab34");
|
|
221
|
+
|
|
222
|
+
const state = await setAccent(null);
|
|
223
|
+
expect(state?.accent).toEqual({ hex: "#c81e1e", source: "derived" });
|
|
224
|
+
expect(readManifestFile()!.accent).toEqual({
|
|
225
|
+
hex: "#c81e1e",
|
|
226
|
+
source: "derived",
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("null returns a character to its palette colour", async () => {
|
|
231
|
+
// Seeded as a manifest rather than through setCharacter, whose native
|
|
232
|
+
// renderer may be absent here.
|
|
233
|
+
writeFileSync(
|
|
234
|
+
path(MANIFEST_FILENAME),
|
|
235
|
+
JSON.stringify({
|
|
236
|
+
kind: "character",
|
|
237
|
+
traits: { ...VALID_TRAITS, color: "orange" },
|
|
238
|
+
source: "builder",
|
|
239
|
+
image: null,
|
|
240
|
+
accent: { hex: "#12ab34", source: "custom" },
|
|
241
|
+
}),
|
|
242
|
+
);
|
|
243
|
+
|
|
244
|
+
const state = await setAccent(null);
|
|
245
|
+
expect(state?.accent).toEqual({ hex: "#E9642F", source: "palette" });
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
test("refuses when there is no avatar, writing nothing", async () => {
|
|
249
|
+
expect(await setAccent("#12ab34")).toBeNull();
|
|
250
|
+
expect(readManifestFile()).toBeNull();
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
describe("backfillAccent", () => {
|
|
255
|
+
const imageState = (etag: string) => ({
|
|
256
|
+
kind: "image" as const,
|
|
257
|
+
traits: null,
|
|
258
|
+
source: "upload" as const,
|
|
259
|
+
image: { updatedAt: "2026-01-01T00:00:00.000Z", etag },
|
|
260
|
+
accent: null,
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
test("reads an image's accent out of the PNG on disk and persists it", async () => {
|
|
264
|
+
writeFileSync(path(IMAGE_FILENAME), RED_PNG);
|
|
265
|
+
const state = imageState("backfill-red");
|
|
266
|
+
writeFileSync(path(MANIFEST_FILENAME), JSON.stringify(state));
|
|
267
|
+
|
|
268
|
+
const result = await backfillAccent(state);
|
|
269
|
+
expect(result.accent).toEqual({ hex: "#c81e1e", source: "derived" });
|
|
270
|
+
expect(readManifestFile()!.accent).toEqual({
|
|
271
|
+
hex: "#c81e1e",
|
|
272
|
+
source: "derived",
|
|
273
|
+
});
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
test("leaves a state that already has an accent, or none to derive, alone", async () => {
|
|
277
|
+
const withAccent = {
|
|
278
|
+
...imageState("has-accent"),
|
|
279
|
+
accent: { hex: "#12ab34", source: "custom" as const },
|
|
280
|
+
};
|
|
281
|
+
expect(await backfillAccent(withAccent)).toBe(withAccent);
|
|
282
|
+
|
|
283
|
+
const none = {
|
|
284
|
+
kind: "none" as const,
|
|
285
|
+
traits: null,
|
|
286
|
+
source: null,
|
|
287
|
+
image: null,
|
|
288
|
+
accent: null,
|
|
289
|
+
};
|
|
290
|
+
expect(await backfillAccent(none)).toBe(none);
|
|
291
|
+
expect(readManifestFile()).toBeNull();
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
test("returns the state unchanged, and writes nothing, when the image cannot be read", async () => {
|
|
295
|
+
writeFileSync(path(IMAGE_FILENAME), Buffer.from("not a png"));
|
|
296
|
+
const state = imageState("backfill-bad");
|
|
297
|
+
expect(await backfillAccent(state)).toBe(state);
|
|
298
|
+
expect(readManifestFile()).toBeNull();
|
|
299
|
+
});
|
|
300
|
+
});
|
|
301
|
+
|
|
180
302
|
describe("clearAvatar", () => {
|
|
181
303
|
test("removes all artifacts AND the manifest (none == absence)", () => {
|
|
182
304
|
writeFileSync(path(IMAGE_FILENAME), Buffer.from("png"));
|
|
@@ -68,6 +68,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
68
68
|
traits: null,
|
|
69
69
|
source: null,
|
|
70
70
|
image: null,
|
|
71
|
+
accent: null,
|
|
71
72
|
});
|
|
72
73
|
expect(await ensureAvatarRaster()).toBeNull();
|
|
73
74
|
});
|
|
@@ -83,6 +84,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
83
84
|
traits: null,
|
|
84
85
|
source: "upload",
|
|
85
86
|
image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
|
|
87
|
+
accent: null,
|
|
86
88
|
});
|
|
87
89
|
const raster = await ensureAvatarRaster();
|
|
88
90
|
expect(raster).not.toBeNull();
|
|
@@ -96,6 +98,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
96
98
|
traits: VALID_TRAITS,
|
|
97
99
|
source: "builder",
|
|
98
100
|
image: null,
|
|
101
|
+
accent: null,
|
|
99
102
|
});
|
|
100
103
|
// Force the unavailable path: a re-render attempt would return null.
|
|
101
104
|
__setResvgCacheForTests({ available: false, error: new Error("nope") });
|
|
@@ -113,6 +116,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
113
116
|
traits: null,
|
|
114
117
|
source: "upload",
|
|
115
118
|
image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
|
|
119
|
+
accent: null,
|
|
116
120
|
});
|
|
117
121
|
expect(await ensureAvatarRasterPath()).toBe(
|
|
118
122
|
join(avatarDir, IMAGE_FILENAME),
|
|
@@ -130,6 +134,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
130
134
|
traits: null,
|
|
131
135
|
source: "upload",
|
|
132
136
|
image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
|
|
137
|
+
accent: null,
|
|
133
138
|
});
|
|
134
139
|
expect(await ensureAvatarRaster()).toBeNull();
|
|
135
140
|
rmSync(foreign, { recursive: true, force: true });
|
|
@@ -141,6 +146,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
141
146
|
traits: null,
|
|
142
147
|
source: "upload",
|
|
143
148
|
image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
|
|
149
|
+
accent: null,
|
|
144
150
|
});
|
|
145
151
|
expect(await ensureAvatarRaster()).toBeNull();
|
|
146
152
|
expect(existsSync(join(avatarDir, IMAGE_FILENAME))).toBe(false);
|
|
@@ -152,6 +158,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
152
158
|
traits: VALID_TRAITS,
|
|
153
159
|
source: "builder",
|
|
154
160
|
image: null,
|
|
161
|
+
accent: null,
|
|
155
162
|
});
|
|
156
163
|
__setResvgCacheForTests({ available: false, error: new Error("nope") });
|
|
157
164
|
expect(await ensureAvatarRaster()).toBeNull();
|
|
@@ -166,6 +173,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
166
173
|
traits: VALID_TRAITS,
|
|
167
174
|
source: "builder",
|
|
168
175
|
image: null,
|
|
176
|
+
accent: null,
|
|
169
177
|
});
|
|
170
178
|
const raster = await ensureAvatarRaster();
|
|
171
179
|
if (!isResvgAvailable()) {
|
|
@@ -192,6 +200,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
192
200
|
traits: null,
|
|
193
201
|
source: null,
|
|
194
202
|
image: null,
|
|
203
|
+
accent: null,
|
|
195
204
|
}),
|
|
196
205
|
).toBeNull();
|
|
197
206
|
});
|
|
@@ -204,6 +213,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
204
213
|
traits: null,
|
|
205
214
|
source: null,
|
|
206
215
|
image: null,
|
|
216
|
+
accent: null,
|
|
207
217
|
});
|
|
208
218
|
expect(await ensureAvatarRasterPath()).toBeNull();
|
|
209
219
|
});
|
|
@@ -218,6 +228,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
218
228
|
updatedAt: new Date().toISOString(),
|
|
219
229
|
etag: "0123456789abcdef",
|
|
220
230
|
},
|
|
231
|
+
accent: null,
|
|
221
232
|
});
|
|
222
233
|
expect(await ensureAvatarRasterPath()).toBe(
|
|
223
234
|
join(avatarDir, IMAGE_FILENAME),
|
|
@@ -236,6 +247,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
236
247
|
updatedAt: new Date().toISOString(),
|
|
237
248
|
etag: "0123456789abcdef",
|
|
238
249
|
},
|
|
250
|
+
accent: null,
|
|
239
251
|
});
|
|
240
252
|
expect(await ensureAvatarRasterPath()).toBe(
|
|
241
253
|
join(avatarDir, IMAGE_FILENAME),
|
|
@@ -252,6 +264,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
252
264
|
updatedAt: new Date().toISOString(),
|
|
253
265
|
etag: "0123456789abcdef",
|
|
254
266
|
},
|
|
267
|
+
accent: null,
|
|
255
268
|
});
|
|
256
269
|
expect(await ensureAvatarRasterPath()).toBeNull();
|
|
257
270
|
});
|
|
@@ -262,6 +275,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
262
275
|
traits: VALID_TRAITS,
|
|
263
276
|
source: "builder",
|
|
264
277
|
image: null,
|
|
278
|
+
accent: null,
|
|
265
279
|
});
|
|
266
280
|
__setResvgCacheForTests({ available: false, error: new Error("nope") });
|
|
267
281
|
expect(await ensureAvatarRasterPath()).toBeNull();
|
|
@@ -276,6 +290,7 @@ describe("ensureAvatarRaster", () => {
|
|
|
276
290
|
traits: VALID_TRAITS,
|
|
277
291
|
source: "builder",
|
|
278
292
|
image: null,
|
|
293
|
+
accent: null,
|
|
279
294
|
});
|
|
280
295
|
const path = await ensureAvatarRasterPath();
|
|
281
296
|
if (!isResvgAvailable()) {
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading an accent colour out of an avatar image.
|
|
3
|
+
*
|
|
4
|
+
* The colour maths is `dominantAccentHex` in `@vellumai/avatar-manifest`,
|
|
5
|
+
* shared with the web so both sides agree on what colour an image is. This
|
|
6
|
+
* module only decodes: sharp turns the upload into a small square of RGBA
|
|
7
|
+
* pixels, centre-cropped the way the avatar is drawn, so the sample comes from
|
|
8
|
+
* the pixels the user actually sees.
|
|
9
|
+
*/
|
|
10
|
+
import { accentHexForColorId } from "@vellumai/avatar-catalog/colors";
|
|
11
|
+
import {
|
|
12
|
+
type AvatarAccent,
|
|
13
|
+
dominantAccentHex,
|
|
14
|
+
} from "@vellumai/avatar-manifest";
|
|
15
|
+
|
|
16
|
+
import { getLogger } from "../util/logger.js";
|
|
17
|
+
|
|
18
|
+
const log = getLogger("avatar-accent");
|
|
19
|
+
|
|
20
|
+
/** Side of the square the image is sampled at; enough for a histogram, cheap to decode to. */
|
|
21
|
+
const SAMPLE_SIZE = 48;
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The dominant accent of an image, or null when the bytes cannot be decoded
|
|
25
|
+
* or hold no opaque pixel. Never throws: the accent decorates surfaces, and a
|
|
26
|
+
* decode failure must not fail the upload that carried the image.
|
|
27
|
+
*/
|
|
28
|
+
export async function deriveAccentHexFromImage(
|
|
29
|
+
bytes: Buffer,
|
|
30
|
+
): Promise<string | null> {
|
|
31
|
+
try {
|
|
32
|
+
const { default: sharp } = await import("sharp");
|
|
33
|
+
const { data } = await sharp(bytes, { failOn: "error" })
|
|
34
|
+
.resize(SAMPLE_SIZE, SAMPLE_SIZE, { fit: "cover" })
|
|
35
|
+
.ensureAlpha()
|
|
36
|
+
.raw()
|
|
37
|
+
.toBuffer({ resolveWithObject: true });
|
|
38
|
+
return dominantAccentHex(data);
|
|
39
|
+
} catch (err) {
|
|
40
|
+
log.warn(
|
|
41
|
+
{ err },
|
|
42
|
+
"Could not read an accent colour out of the avatar image",
|
|
43
|
+
);
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The accent a character wears: its palette colour. Null for an id the palette lacks. */
|
|
49
|
+
export function paletteAccent(colorId: string): AvatarAccent | null {
|
|
50
|
+
const hex = accentHexForColorId(colorId);
|
|
51
|
+
return hex ? { hex, source: "palette" } : null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The accent read out of an image's pixels, or null when there was none to read. */
|
|
55
|
+
export function derivedAccent(hex: string | null): AvatarAccent | null {
|
|
56
|
+
return hex ? { hex, source: "derived" } : null;
|
|
57
|
+
}
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
AVATAR_IMAGE_FILENAME,
|
|
22
22
|
AVATAR_MANIFEST_FILENAME,
|
|
23
23
|
AVATAR_TRAITS_FILENAME,
|
|
24
|
+
type AvatarAccent,
|
|
24
25
|
type AvatarImageMeta,
|
|
25
26
|
type AvatarKind,
|
|
26
27
|
type AvatarSource,
|
|
@@ -32,10 +33,17 @@ import {
|
|
|
32
33
|
|
|
33
34
|
import { getLogger } from "../util/logger.js";
|
|
34
35
|
import { getAvatarDir } from "../util/platform.js";
|
|
36
|
+
import { paletteAccent } from "./avatar-accent.js";
|
|
35
37
|
|
|
36
38
|
const log = getLogger("avatar-manifest");
|
|
37
39
|
|
|
38
|
-
export type {
|
|
40
|
+
export type {
|
|
41
|
+
AvatarAccent,
|
|
42
|
+
AvatarImageMeta,
|
|
43
|
+
AvatarKind,
|
|
44
|
+
AvatarSource,
|
|
45
|
+
AvatarState,
|
|
46
|
+
};
|
|
39
47
|
export type { CharacterTraits };
|
|
40
48
|
|
|
41
49
|
/**
|
|
@@ -134,16 +142,26 @@ export function deriveStateFromLegacyFiles(
|
|
|
134
142
|
traits: derived.traits,
|
|
135
143
|
source: null,
|
|
136
144
|
image: null,
|
|
145
|
+
accent: paletteAccent(derived.traits.color),
|
|
137
146
|
};
|
|
138
147
|
case "image":
|
|
148
|
+
// The image's accent is read out of its pixels by the store's backfill,
|
|
149
|
+
// which the state route runs; this derivation stays synchronous.
|
|
139
150
|
return {
|
|
140
151
|
kind: "image",
|
|
141
152
|
traits: null,
|
|
142
153
|
source: null,
|
|
143
154
|
image: computeImageMeta(imagePath),
|
|
155
|
+
accent: null,
|
|
144
156
|
};
|
|
145
157
|
case "none":
|
|
146
|
-
return {
|
|
158
|
+
return {
|
|
159
|
+
kind: "none",
|
|
160
|
+
traits: null,
|
|
161
|
+
source: null,
|
|
162
|
+
image: null,
|
|
163
|
+
accent: null,
|
|
164
|
+
};
|
|
147
165
|
}
|
|
148
166
|
}
|
|
149
167
|
|
|
@@ -19,15 +19,24 @@ import {
|
|
|
19
19
|
AVATAR_IMAGE_FILENAME,
|
|
20
20
|
AVATAR_MANIFEST_FILENAME,
|
|
21
21
|
AVATAR_TRAITS_FILENAME,
|
|
22
|
+
type AvatarAccent,
|
|
22
23
|
} from "@vellumai/avatar-manifest";
|
|
23
24
|
|
|
24
25
|
import { getLogger } from "../util/logger.js";
|
|
25
26
|
import { getAvatarDir, getAvatarImagePath } from "../util/platform.js";
|
|
27
|
+
import {
|
|
28
|
+
deriveAccentHexFromImage,
|
|
29
|
+
derivedAccent,
|
|
30
|
+
paletteAccent,
|
|
31
|
+
} from "./avatar-accent.js";
|
|
26
32
|
import {
|
|
27
33
|
type AvatarSource,
|
|
34
|
+
type AvatarState,
|
|
28
35
|
computeImageMeta,
|
|
36
|
+
readAvatarState,
|
|
29
37
|
writeManifest,
|
|
30
38
|
} from "./avatar-manifest.js";
|
|
39
|
+
import { readContainedAvatarRaster } from "./ensure-raster.js";
|
|
31
40
|
import {
|
|
32
41
|
ASCII_FILENAME,
|
|
33
42
|
type CharacterTraits,
|
|
@@ -58,6 +67,7 @@ export function setCharacter(traits: CharacterTraits): TraitsSyncResult {
|
|
|
58
67
|
traits,
|
|
59
68
|
source: "builder",
|
|
60
69
|
image: computeImageMeta(getAvatarImagePath()),
|
|
70
|
+
accent: paletteAccent(traits.color),
|
|
61
71
|
});
|
|
62
72
|
return result;
|
|
63
73
|
}
|
|
@@ -65,10 +75,15 @@ export function setCharacter(traits: CharacterTraits): TraitsSyncResult {
|
|
|
65
75
|
/**
|
|
66
76
|
* Sets the avatar to an uploaded/AI image: atomically writes the PNG, removes
|
|
67
77
|
* the now-stale character sidecars (traits + ASCII), then records an `image`
|
|
68
|
-
* manifest
|
|
69
|
-
*
|
|
78
|
+
* manifest carrying the accent read out of the image. The accent is derived
|
|
79
|
+
* before anything is written and the PNG is written before the manifest, so an
|
|
80
|
+
* interrupted call never leaves the manifest ahead of the artifact.
|
|
70
81
|
*/
|
|
71
|
-
export function setImage(
|
|
82
|
+
export async function setImage(
|
|
83
|
+
pngBuffer: Buffer,
|
|
84
|
+
source: AvatarSource,
|
|
85
|
+
): Promise<void> {
|
|
86
|
+
const accent = derivedAccent(await deriveAccentHexFromImage(pngBuffer));
|
|
72
87
|
const avatarDir = getAvatarDir();
|
|
73
88
|
mkdirSync(avatarDir, { recursive: true });
|
|
74
89
|
|
|
@@ -85,9 +100,101 @@ export function setImage(pngBuffer: Buffer, source: AvatarSource): void {
|
|
|
85
100
|
traits: null,
|
|
86
101
|
source,
|
|
87
102
|
image: computeImageMeta(pngPath),
|
|
103
|
+
accent,
|
|
88
104
|
});
|
|
89
105
|
|
|
90
|
-
log.info(
|
|
106
|
+
log.info(
|
|
107
|
+
{ source, accent: accent?.hex ?? null },
|
|
108
|
+
"Set avatar from image and removed character sidecars",
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* The accent the current avatar earns on its own: a character's palette
|
|
114
|
+
* colour, or the colour read out of the image on disk. Null for `none`, and
|
|
115
|
+
* for an image that cannot be read.
|
|
116
|
+
*/
|
|
117
|
+
async function automaticAccent(
|
|
118
|
+
state: AvatarState,
|
|
119
|
+
): Promise<AvatarAccent | null> {
|
|
120
|
+
if (state.kind === "character" && state.traits) {
|
|
121
|
+
return paletteAccent(state.traits.color);
|
|
122
|
+
}
|
|
123
|
+
if (state.kind === "image") {
|
|
124
|
+
const bytes = readContainedAvatarRaster(getAvatarImagePath());
|
|
125
|
+
return bytes ? derivedAccent(await deriveAccentHexFromImage(bytes)) : null;
|
|
126
|
+
}
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Sets the accent over the current avatar: a `#rrggbb` the user chose, or
|
|
132
|
+
* `null` to go back to the automatic one. Returns the state as written, or
|
|
133
|
+
* null when there is no avatar to colour. Only the manifest changes; the
|
|
134
|
+
* artifacts are untouched.
|
|
135
|
+
*/
|
|
136
|
+
export async function setAccent(
|
|
137
|
+
hex: string | null,
|
|
138
|
+
): Promise<AvatarState | null> {
|
|
139
|
+
const state = readAvatarState();
|
|
140
|
+
if (state.kind === "none") {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
const next: AvatarState = {
|
|
144
|
+
...state,
|
|
145
|
+
accent: hex ? { hex, source: "custom" } : await automaticAccent(state),
|
|
146
|
+
};
|
|
147
|
+
writeManifest(next);
|
|
148
|
+
return next;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Image etags whose accent derivation came back empty. Remembered so a read
|
|
153
|
+
* of an unreadable image does not decode it again on every request; a new
|
|
154
|
+
* upload carries a new etag and is tried afresh.
|
|
155
|
+
*/
|
|
156
|
+
const accentlessImageEtags = new Set<string>();
|
|
157
|
+
/** The one derivation in flight per etag, so concurrent reads share it. */
|
|
158
|
+
const pendingAccents = new Map<string, Promise<AvatarAccent | null>>();
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Fills in the accent of a state written before accents existed, persisting
|
|
162
|
+
* it so later reads are manifest-only. The one exception to the manifest
|
|
163
|
+
* being written only by the mutations above, for the same reason the read
|
|
164
|
+
* handlers self-heal a missing manifest: the accent is derivable from what is
|
|
165
|
+
* on disk, and deriving it once beats every client deriving it on every read.
|
|
166
|
+
* The persist is best-effort; a read-only workspace still gets the accent.
|
|
167
|
+
*/
|
|
168
|
+
export async function backfillAccent(state: AvatarState): Promise<AvatarState> {
|
|
169
|
+
if (state.kind === "none" || state.accent !== null) {
|
|
170
|
+
return state;
|
|
171
|
+
}
|
|
172
|
+
const etag = state.image?.etag ?? null;
|
|
173
|
+
if (etag && accentlessImageEtags.has(etag)) {
|
|
174
|
+
return state;
|
|
175
|
+
}
|
|
176
|
+
const key = etag ?? state.kind;
|
|
177
|
+
let pending = pendingAccents.get(key);
|
|
178
|
+
if (!pending) {
|
|
179
|
+
pending = automaticAccent(state).finally(() => {
|
|
180
|
+
pendingAccents.delete(key);
|
|
181
|
+
});
|
|
182
|
+
pendingAccents.set(key, pending);
|
|
183
|
+
}
|
|
184
|
+
const accent = await pending;
|
|
185
|
+
if (!accent) {
|
|
186
|
+
if (etag) {
|
|
187
|
+
accentlessImageEtags.add(etag);
|
|
188
|
+
}
|
|
189
|
+
return state;
|
|
190
|
+
}
|
|
191
|
+
const next: AvatarState = { ...state, accent };
|
|
192
|
+
try {
|
|
193
|
+
writeManifest(next);
|
|
194
|
+
} catch (err) {
|
|
195
|
+
log.warn({ err }, "Failed to persist the backfilled avatar accent");
|
|
196
|
+
}
|
|
197
|
+
return next;
|
|
91
198
|
}
|
|
92
199
|
|
|
93
200
|
/**
|
|
@@ -25,7 +25,7 @@ The processing pipeline follows a sequential 3-phase flow:
|
|
|
25
25
|
|
|
26
26
|
1. **Ingest** (`ingest_media`) - Register a media file, detect MIME type, extract duration, deduplicate by content hash.
|
|
27
27
|
2. **Preprocess** (`extract_keyframes`) - Detect dead time, segment the video into windows, extract downscaled keyframes, build a subject registry, and write a pipeline manifest.
|
|
28
|
-
3. **Map** (`analyze_keyframes`) - Send each segment's frames to Gemini
|
|
28
|
+
3. **Map** (`analyze_keyframes`) - Send each segment's frames to the configured Gemini vision model with assistant-provided extraction instructions and a JSON Schema for guaranteed structured output. Supports concurrency pooling, cost tracking, resumability, and automatic retries.
|
|
29
29
|
4. **Reduce / Query** (`query_media`) - Send all map output to Claude for intelligent analysis and Q&A. Supports arbitrary natural language queries about video content.
|
|
30
30
|
5. **Clip** (`generate_clip`) - Extract video clips around specific moments.
|
|
31
31
|
|
|
@@ -85,7 +85,7 @@ Parameters:
|
|
|
85
85
|
- `asset_id` (required) - ID of the media asset.
|
|
86
86
|
- `query` (required) - Natural language query about the video data.
|
|
87
87
|
- `system_prompt` - Optional system prompt for Claude.
|
|
88
|
-
- `model` - LLM model to use
|
|
88
|
+
- `model` - LLM model to use. Defaults to the assistant's active chat model.
|
|
89
89
|
|
|
90
90
|
### generate_clip
|
|
91
91
|
|
|
@@ -110,7 +110,7 @@ Handles dead-time detection, video segmentation, keyframe extraction, and subjec
|
|
|
110
110
|
|
|
111
111
|
### Gemini Map (services/gemini-map.ts)
|
|
112
112
|
|
|
113
|
-
Sends video segments to Gemini
|
|
113
|
+
Sends video segments to the configured Gemini vision model with structured output schemas. Handles concurrency pooling, cost tracking, resumability, and retries.
|
|
114
114
|
|
|
115
115
|
### Reduce (services/reduce.ts)
|
|
116
116
|
|
|
@@ -170,7 +170,7 @@
|
|
|
170
170
|
},
|
|
171
171
|
"model": {
|
|
172
172
|
"type": "string",
|
|
173
|
-
"description": "LLM model to use for analysis.
|
|
173
|
+
"description": "LLM model to use for analysis. Defaults to the assistant's active chat model."
|
|
174
174
|
}
|
|
175
175
|
},
|
|
176
176
|
"required": ["asset_id", "query"]
|
|
@@ -247,4 +247,4 @@ Choose the right delivery tool based on the content:
|
|
|
247
247
|
|
|
248
248
|
Example schedule message for a Slack digest:
|
|
249
249
|
|
|
250
|
-
> "Scan my Slack channels for the last 24 hours using the Slack Web API via bash (network_mode: proxied, credential_ids: ['slack_channel/bot_token']), then post the summary to
|
|
250
|
+
> "Scan my Slack channels for the last 24 hours using the Slack Web API via bash (network_mode: proxied, credential_ids: ['slack_channel/bot_token']), then post the summary to the channel the user named."
|
|
@@ -127,6 +127,7 @@ function imageState(etag: string): AvatarState {
|
|
|
127
127
|
traits: null,
|
|
128
128
|
source: "upload",
|
|
129
129
|
image: { updatedAt: "2026-01-01T00:00:00.000Z", etag },
|
|
130
|
+
accent: null,
|
|
130
131
|
};
|
|
131
132
|
}
|
|
132
133
|
|
|
@@ -135,6 +136,7 @@ const NONE: AvatarState = {
|
|
|
135
136
|
traits: null,
|
|
136
137
|
source: null,
|
|
137
138
|
image: null,
|
|
139
|
+
accent: null,
|
|
138
140
|
};
|
|
139
141
|
|
|
140
142
|
interface Patch {
|
|
@@ -275,6 +277,7 @@ describe("syncAvatarToPlatform", () => {
|
|
|
275
277
|
traits: { bodyShape: "blob", eyeStyle: "curious", color: "green" },
|
|
276
278
|
source: null,
|
|
277
279
|
image: null,
|
|
280
|
+
accent: null,
|
|
278
281
|
};
|
|
279
282
|
mockRasterPath = null;
|
|
280
283
|
syncAvatarToPlatform();
|