@vellumai/assistant 0.11.9-staging.2 → 0.11.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/Dockerfile +9 -0
  2. package/docker-bun-no-autoserve.js +38 -0
  3. package/docker-node-launcher.sh +37 -0
  4. package/node_modules/@vellumai/avatar-catalog/src/colors.ts +8 -0
  5. package/node_modules/@vellumai/avatar-manifest/package.json +2 -1
  6. package/node_modules/@vellumai/avatar-manifest/src/__tests__/accent.test.ts +118 -0
  7. package/node_modules/@vellumai/avatar-manifest/src/__tests__/manifest.test.ts +83 -4
  8. package/node_modules/@vellumai/avatar-manifest/src/accent.ts +150 -0
  9. package/node_modules/@vellumai/avatar-manifest/src/index.ts +7 -0
  10. package/node_modules/@vellumai/avatar-manifest/src/manifest.ts +55 -3
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +40 -1
  12. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +21 -0
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +40 -1
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +21 -0
  15. package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +40 -1
  16. package/node_modules/@vellumai/service-contracts/src/channels.ts +21 -0
  17. package/openapi.yaml +116 -1
  18. package/package.json +1 -1
  19. package/src/__tests__/bun-no-autoserve.test.ts +178 -0
  20. package/src/avatar/__tests__/avatar-manifest.test.ts +10 -0
  21. package/src/avatar/__tests__/avatar-store.test.ts +130 -8
  22. package/src/avatar/__tests__/ensure-raster.test.ts +15 -0
  23. package/src/avatar/avatar-accent.ts +57 -0
  24. package/src/avatar/avatar-manifest.ts +20 -2
  25. package/src/avatar/avatar-store.ts +111 -4
  26. package/src/config/bundled-skills/media-processing/SKILL.md +3 -3
  27. package/src/config/bundled-skills/media-processing/TOOLS.json +1 -1
  28. package/src/config/bundled-skills/schedule/SKILL.md +1 -1
  29. package/src/platform/sync-avatar.test.ts +3 -0
  30. package/src/runtime/routes/__tests__/avatar-state-routes.test.ts +147 -27
  31. package/src/runtime/routes/avatar-routes.ts +84 -28
  32. package/src/runtime/routes/settings-routes.ts +1 -1
  33. package/src/tools/terminal/__tests__/safe-env.test.ts +60 -1
  34. package/src/tools/terminal/safe-env.ts +70 -1
@@ -32,7 +32,13 @@ import { tmpdir } from "node:os";
32
32
  import { join } from "node:path";
33
33
  import { afterEach, beforeEach, describe, expect, test } from "bun:test";
34
34
 
35
- import { clearAvatar, setCharacter, setImage } from "../avatar-store.js";
35
+ import {
36
+ backfillAccent,
37
+ clearAvatar,
38
+ setAccent,
39
+ setCharacter,
40
+ setImage,
41
+ } from "../avatar-store.js";
36
42
 
37
43
  // A valid trait triple drawn from the real component set.
38
44
  const VALID_TRAITS = { bodyShape: "blob", eyeStyle: "curious", color: "green" };
@@ -43,11 +49,18 @@ const ASCII_FILENAME = "character-ascii.txt";
43
49
  const MANIFEST_FILENAME = "avatar.json";
44
50
  const NATIVE_RENDER_TEST_TIMEOUT_MS = 15_000;
45
51
 
52
+ /** A 4x4 PNG of one red (#c81e1e), so an accent can be read out of it. */
53
+ const RED_PNG = Buffer.from(
54
+ "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAYAAACp8Z5+AAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEklEQVQImWM4ISf3HxkzkC4AAEG4IDHG8wOiAAAAAElFTkSuQmCC",
55
+ "base64",
56
+ );
57
+
46
58
  interface ManifestShape {
47
59
  kind: string;
48
60
  traits: Record<string, unknown> | null;
49
61
  source: string | null;
50
62
  image: { updatedAt: string; etag: string } | null;
63
+ accent: { hex: string; source: string } | null;
51
64
  }
52
65
 
53
66
  describe("avatar-store", () => {
@@ -117,6 +130,8 @@ describe("avatar-store", () => {
117
130
  expect(manifest!.source).toBe("builder");
118
131
  expect(manifest!.image).not.toBeNull();
119
132
  expect(manifest!.image!.etag).toMatch(/^[0-9a-f]{16}$/);
133
+ // The accent is the chosen palette colour.
134
+ expect(manifest!.accent).toEqual({ hex: "#4C9B50", source: "palette" });
120
135
  },
121
136
  NATIVE_RENDER_TEST_TIMEOUT_MS,
122
137
  );
@@ -135,8 +150,8 @@ describe("avatar-store", () => {
135
150
  });
136
151
 
137
152
  describe("setImage", () => {
138
- test("writes the PNG and an image manifest", () => {
139
- setImage(Buffer.from("fake png bytes"), "upload");
153
+ test("writes the PNG and an image manifest", async () => {
154
+ await setImage(Buffer.from("fake png bytes"), "upload");
140
155
 
141
156
  expect(existsSync(path(IMAGE_FILENAME))).toBe(true);
142
157
  expect(readFileSync(path(IMAGE_FILENAME)).toString()).toBe(
@@ -153,14 +168,24 @@ describe("avatar-store", () => {
153
168
  expect(manifest!.image!.updatedAt).toMatch(
154
169
  /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,
155
170
  );
171
+ // Bytes that do not decode carry no colour, and that is not a failure.
172
+ expect(manifest!.accent).toBeNull();
173
+ });
174
+
175
+ test("reads the accent out of the image", async () => {
176
+ await setImage(RED_PNG, "upload");
177
+ expect(readManifestFile()!.accent).toEqual({
178
+ hex: "#c81e1e",
179
+ source: "derived",
180
+ });
156
181
  });
157
182
 
158
- test("removes stale character sidecars on transition", () => {
183
+ test("removes stale character sidecars on transition", async () => {
159
184
  // Seed legacy character artifacts.
160
185
  writeFileSync(path(TRAITS_FILENAME), JSON.stringify(VALID_TRAITS));
161
186
  writeFileSync(path(ASCII_FILENAME), "ascii art");
162
187
 
163
- setImage(Buffer.from("png"), "ai");
188
+ await setImage(Buffer.from("png"), "ai");
164
189
 
165
190
  expect(existsSync(path(TRAITS_FILENAME))).toBe(false);
166
191
  expect(existsSync(path(ASCII_FILENAME))).toBe(false);
@@ -168,15 +193,112 @@ describe("avatar-store", () => {
168
193
  expect(readManifestFile()!.kind).toBe("image");
169
194
  });
170
195
 
171
- test("is idempotent across repeated calls", () => {
172
- setImage(Buffer.from("v1"), "upload");
173
- setImage(Buffer.from("v2"), "upload");
196
+ test("is idempotent across repeated calls", async () => {
197
+ await setImage(Buffer.from("v1"), "upload");
198
+ await setImage(Buffer.from("v2"), "upload");
174
199
 
175
200
  expect(readFileSync(path(IMAGE_FILENAME)).toString()).toBe("v2");
176
201
  expect(readManifestFile()!.kind).toBe("image");
177
202
  });
178
203
  });
179
204
 
205
+ describe("setAccent", () => {
206
+ test("writes a custom accent over an image without touching the artifacts", async () => {
207
+ await setImage(RED_PNG, "upload");
208
+ const before = readManifestFile()!;
209
+
210
+ const state = await setAccent("#12ab34");
211
+ expect(state?.accent).toEqual({ hex: "#12ab34", source: "custom" });
212
+ const after = readManifestFile()!;
213
+ expect(after.accent).toEqual({ hex: "#12ab34", source: "custom" });
214
+ expect(after.image).toEqual(before.image);
215
+ expect(readFileSync(path(IMAGE_FILENAME))).toEqual(RED_PNG);
216
+ });
217
+
218
+ test("null returns an image to the colour read out of it", async () => {
219
+ await setImage(RED_PNG, "upload");
220
+ await setAccent("#12ab34");
221
+
222
+ const state = await setAccent(null);
223
+ expect(state?.accent).toEqual({ hex: "#c81e1e", source: "derived" });
224
+ expect(readManifestFile()!.accent).toEqual({
225
+ hex: "#c81e1e",
226
+ source: "derived",
227
+ });
228
+ });
229
+
230
+ test("null returns a character to its palette colour", async () => {
231
+ // Seeded as a manifest rather than through setCharacter, whose native
232
+ // renderer may be absent here.
233
+ writeFileSync(
234
+ path(MANIFEST_FILENAME),
235
+ JSON.stringify({
236
+ kind: "character",
237
+ traits: { ...VALID_TRAITS, color: "orange" },
238
+ source: "builder",
239
+ image: null,
240
+ accent: { hex: "#12ab34", source: "custom" },
241
+ }),
242
+ );
243
+
244
+ const state = await setAccent(null);
245
+ expect(state?.accent).toEqual({ hex: "#E9642F", source: "palette" });
246
+ });
247
+
248
+ test("refuses when there is no avatar, writing nothing", async () => {
249
+ expect(await setAccent("#12ab34")).toBeNull();
250
+ expect(readManifestFile()).toBeNull();
251
+ });
252
+ });
253
+
254
+ describe("backfillAccent", () => {
255
+ const imageState = (etag: string) => ({
256
+ kind: "image" as const,
257
+ traits: null,
258
+ source: "upload" as const,
259
+ image: { updatedAt: "2026-01-01T00:00:00.000Z", etag },
260
+ accent: null,
261
+ });
262
+
263
+ test("reads an image's accent out of the PNG on disk and persists it", async () => {
264
+ writeFileSync(path(IMAGE_FILENAME), RED_PNG);
265
+ const state = imageState("backfill-red");
266
+ writeFileSync(path(MANIFEST_FILENAME), JSON.stringify(state));
267
+
268
+ const result = await backfillAccent(state);
269
+ expect(result.accent).toEqual({ hex: "#c81e1e", source: "derived" });
270
+ expect(readManifestFile()!.accent).toEqual({
271
+ hex: "#c81e1e",
272
+ source: "derived",
273
+ });
274
+ });
275
+
276
+ test("leaves a state that already has an accent, or none to derive, alone", async () => {
277
+ const withAccent = {
278
+ ...imageState("has-accent"),
279
+ accent: { hex: "#12ab34", source: "custom" as const },
280
+ };
281
+ expect(await backfillAccent(withAccent)).toBe(withAccent);
282
+
283
+ const none = {
284
+ kind: "none" as const,
285
+ traits: null,
286
+ source: null,
287
+ image: null,
288
+ accent: null,
289
+ };
290
+ expect(await backfillAccent(none)).toBe(none);
291
+ expect(readManifestFile()).toBeNull();
292
+ });
293
+
294
+ test("returns the state unchanged, and writes nothing, when the image cannot be read", async () => {
295
+ writeFileSync(path(IMAGE_FILENAME), Buffer.from("not a png"));
296
+ const state = imageState("backfill-bad");
297
+ expect(await backfillAccent(state)).toBe(state);
298
+ expect(readManifestFile()).toBeNull();
299
+ });
300
+ });
301
+
180
302
  describe("clearAvatar", () => {
181
303
  test("removes all artifacts AND the manifest (none == absence)", () => {
182
304
  writeFileSync(path(IMAGE_FILENAME), Buffer.from("png"));
@@ -68,6 +68,7 @@ describe("ensureAvatarRaster", () => {
68
68
  traits: null,
69
69
  source: null,
70
70
  image: null,
71
+ accent: null,
71
72
  });
72
73
  expect(await ensureAvatarRaster()).toBeNull();
73
74
  });
@@ -83,6 +84,7 @@ describe("ensureAvatarRaster", () => {
83
84
  traits: null,
84
85
  source: "upload",
85
86
  image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
87
+ accent: null,
86
88
  });
87
89
  const raster = await ensureAvatarRaster();
88
90
  expect(raster).not.toBeNull();
@@ -96,6 +98,7 @@ describe("ensureAvatarRaster", () => {
96
98
  traits: VALID_TRAITS,
97
99
  source: "builder",
98
100
  image: null,
101
+ accent: null,
99
102
  });
100
103
  // Force the unavailable path: a re-render attempt would return null.
101
104
  __setResvgCacheForTests({ available: false, error: new Error("nope") });
@@ -113,6 +116,7 @@ describe("ensureAvatarRaster", () => {
113
116
  traits: null,
114
117
  source: "upload",
115
118
  image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
119
+ accent: null,
116
120
  });
117
121
  expect(await ensureAvatarRasterPath()).toBe(
118
122
  join(avatarDir, IMAGE_FILENAME),
@@ -130,6 +134,7 @@ describe("ensureAvatarRaster", () => {
130
134
  traits: null,
131
135
  source: "upload",
132
136
  image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
137
+ accent: null,
133
138
  });
134
139
  expect(await ensureAvatarRaster()).toBeNull();
135
140
  rmSync(foreign, { recursive: true, force: true });
@@ -141,6 +146,7 @@ describe("ensureAvatarRaster", () => {
141
146
  traits: null,
142
147
  source: "upload",
143
148
  image: { updatedAt: new Date().toISOString(), etag: "0123456789abcdef" },
149
+ accent: null,
144
150
  });
145
151
  expect(await ensureAvatarRaster()).toBeNull();
146
152
  expect(existsSync(join(avatarDir, IMAGE_FILENAME))).toBe(false);
@@ -152,6 +158,7 @@ describe("ensureAvatarRaster", () => {
152
158
  traits: VALID_TRAITS,
153
159
  source: "builder",
154
160
  image: null,
161
+ accent: null,
155
162
  });
156
163
  __setResvgCacheForTests({ available: false, error: new Error("nope") });
157
164
  expect(await ensureAvatarRaster()).toBeNull();
@@ -166,6 +173,7 @@ describe("ensureAvatarRaster", () => {
166
173
  traits: VALID_TRAITS,
167
174
  source: "builder",
168
175
  image: null,
176
+ accent: null,
169
177
  });
170
178
  const raster = await ensureAvatarRaster();
171
179
  if (!isResvgAvailable()) {
@@ -192,6 +200,7 @@ describe("ensureAvatarRaster", () => {
192
200
  traits: null,
193
201
  source: null,
194
202
  image: null,
203
+ accent: null,
195
204
  }),
196
205
  ).toBeNull();
197
206
  });
@@ -204,6 +213,7 @@ describe("ensureAvatarRaster", () => {
204
213
  traits: null,
205
214
  source: null,
206
215
  image: null,
216
+ accent: null,
207
217
  });
208
218
  expect(await ensureAvatarRasterPath()).toBeNull();
209
219
  });
@@ -218,6 +228,7 @@ describe("ensureAvatarRaster", () => {
218
228
  updatedAt: new Date().toISOString(),
219
229
  etag: "0123456789abcdef",
220
230
  },
231
+ accent: null,
221
232
  });
222
233
  expect(await ensureAvatarRasterPath()).toBe(
223
234
  join(avatarDir, IMAGE_FILENAME),
@@ -236,6 +247,7 @@ describe("ensureAvatarRaster", () => {
236
247
  updatedAt: new Date().toISOString(),
237
248
  etag: "0123456789abcdef",
238
249
  },
250
+ accent: null,
239
251
  });
240
252
  expect(await ensureAvatarRasterPath()).toBe(
241
253
  join(avatarDir, IMAGE_FILENAME),
@@ -252,6 +264,7 @@ describe("ensureAvatarRaster", () => {
252
264
  updatedAt: new Date().toISOString(),
253
265
  etag: "0123456789abcdef",
254
266
  },
267
+ accent: null,
255
268
  });
256
269
  expect(await ensureAvatarRasterPath()).toBeNull();
257
270
  });
@@ -262,6 +275,7 @@ describe("ensureAvatarRaster", () => {
262
275
  traits: VALID_TRAITS,
263
276
  source: "builder",
264
277
  image: null,
278
+ accent: null,
265
279
  });
266
280
  __setResvgCacheForTests({ available: false, error: new Error("nope") });
267
281
  expect(await ensureAvatarRasterPath()).toBeNull();
@@ -276,6 +290,7 @@ describe("ensureAvatarRaster", () => {
276
290
  traits: VALID_TRAITS,
277
291
  source: "builder",
278
292
  image: null,
293
+ accent: null,
279
294
  });
280
295
  const path = await ensureAvatarRasterPath();
281
296
  if (!isResvgAvailable()) {
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Reading an accent colour out of an avatar image.
3
+ *
4
+ * The colour maths is `dominantAccentHex` in `@vellumai/avatar-manifest`,
5
+ * shared with the web so both sides agree on what colour an image is. This
6
+ * module only decodes: sharp turns the upload into a small square of RGBA
7
+ * pixels, centre-cropped the way the avatar is drawn, so the sample comes from
8
+ * the pixels the user actually sees.
9
+ */
10
+ import { accentHexForColorId } from "@vellumai/avatar-catalog/colors";
11
+ import {
12
+ type AvatarAccent,
13
+ dominantAccentHex,
14
+ } from "@vellumai/avatar-manifest";
15
+
16
+ import { getLogger } from "../util/logger.js";
17
+
18
+ const log = getLogger("avatar-accent");
19
+
20
+ /** Side of the square the image is sampled at; enough for a histogram, cheap to decode to. */
21
+ const SAMPLE_SIZE = 48;
22
+
23
+ /**
24
+ * The dominant accent of an image, or null when the bytes cannot be decoded
25
+ * or hold no opaque pixel. Never throws: the accent decorates surfaces, and a
26
+ * decode failure must not fail the upload that carried the image.
27
+ */
28
+ export async function deriveAccentHexFromImage(
29
+ bytes: Buffer,
30
+ ): Promise<string | null> {
31
+ try {
32
+ const { default: sharp } = await import("sharp");
33
+ const { data } = await sharp(bytes, { failOn: "error" })
34
+ .resize(SAMPLE_SIZE, SAMPLE_SIZE, { fit: "cover" })
35
+ .ensureAlpha()
36
+ .raw()
37
+ .toBuffer({ resolveWithObject: true });
38
+ return dominantAccentHex(data);
39
+ } catch (err) {
40
+ log.warn(
41
+ { err },
42
+ "Could not read an accent colour out of the avatar image",
43
+ );
44
+ return null;
45
+ }
46
+ }
47
+
48
+ /** The accent a character wears: its palette colour. Null for an id the palette lacks. */
49
+ export function paletteAccent(colorId: string): AvatarAccent | null {
50
+ const hex = accentHexForColorId(colorId);
51
+ return hex ? { hex, source: "palette" } : null;
52
+ }
53
+
54
+ /** The accent read out of an image's pixels, or null when there was none to read. */
55
+ export function derivedAccent(hex: string | null): AvatarAccent | null {
56
+ return hex ? { hex, source: "derived" } : null;
57
+ }
@@ -21,6 +21,7 @@ import {
21
21
  AVATAR_IMAGE_FILENAME,
22
22
  AVATAR_MANIFEST_FILENAME,
23
23
  AVATAR_TRAITS_FILENAME,
24
+ type AvatarAccent,
24
25
  type AvatarImageMeta,
25
26
  type AvatarKind,
26
27
  type AvatarSource,
@@ -32,10 +33,17 @@ import {
32
33
 
33
34
  import { getLogger } from "../util/logger.js";
34
35
  import { getAvatarDir } from "../util/platform.js";
36
+ import { paletteAccent } from "./avatar-accent.js";
35
37
 
36
38
  const log = getLogger("avatar-manifest");
37
39
 
38
- export type { AvatarImageMeta, AvatarKind, AvatarSource, AvatarState };
40
+ export type {
41
+ AvatarAccent,
42
+ AvatarImageMeta,
43
+ AvatarKind,
44
+ AvatarSource,
45
+ AvatarState,
46
+ };
39
47
  export type { CharacterTraits };
40
48
 
41
49
  /**
@@ -134,16 +142,26 @@ export function deriveStateFromLegacyFiles(
134
142
  traits: derived.traits,
135
143
  source: null,
136
144
  image: null,
145
+ accent: paletteAccent(derived.traits.color),
137
146
  };
138
147
  case "image":
148
+ // The image's accent is read out of its pixels by the store's backfill,
149
+ // which the state route runs; this derivation stays synchronous.
139
150
  return {
140
151
  kind: "image",
141
152
  traits: null,
142
153
  source: null,
143
154
  image: computeImageMeta(imagePath),
155
+ accent: null,
144
156
  };
145
157
  case "none":
146
- return { kind: "none", traits: null, source: null, image: null };
158
+ return {
159
+ kind: "none",
160
+ traits: null,
161
+ source: null,
162
+ image: null,
163
+ accent: null,
164
+ };
147
165
  }
148
166
  }
149
167
 
@@ -19,15 +19,24 @@ import {
19
19
  AVATAR_IMAGE_FILENAME,
20
20
  AVATAR_MANIFEST_FILENAME,
21
21
  AVATAR_TRAITS_FILENAME,
22
+ type AvatarAccent,
22
23
  } from "@vellumai/avatar-manifest";
23
24
 
24
25
  import { getLogger } from "../util/logger.js";
25
26
  import { getAvatarDir, getAvatarImagePath } from "../util/platform.js";
27
+ import {
28
+ deriveAccentHexFromImage,
29
+ derivedAccent,
30
+ paletteAccent,
31
+ } from "./avatar-accent.js";
26
32
  import {
27
33
  type AvatarSource,
34
+ type AvatarState,
28
35
  computeImageMeta,
36
+ readAvatarState,
29
37
  writeManifest,
30
38
  } from "./avatar-manifest.js";
39
+ import { readContainedAvatarRaster } from "./ensure-raster.js";
31
40
  import {
32
41
  ASCII_FILENAME,
33
42
  type CharacterTraits,
@@ -58,6 +67,7 @@ export function setCharacter(traits: CharacterTraits): TraitsSyncResult {
58
67
  traits,
59
68
  source: "builder",
60
69
  image: computeImageMeta(getAvatarImagePath()),
70
+ accent: paletteAccent(traits.color),
61
71
  });
62
72
  return result;
63
73
  }
@@ -65,10 +75,15 @@ export function setCharacter(traits: CharacterTraits): TraitsSyncResult {
65
75
  /**
66
76
  * Sets the avatar to an uploaded/AI image: atomically writes the PNG, removes
67
77
  * the now-stale character sidecars (traits + ASCII), then records an `image`
68
- * manifest. The PNG is written before the manifest so an interrupted call never
69
- * leaves the manifest ahead of the artifact.
78
+ * manifest carrying the accent read out of the image. The accent is derived
79
+ * before anything is written and the PNG is written before the manifest, so an
80
+ * interrupted call never leaves the manifest ahead of the artifact.
70
81
  */
71
- export function setImage(pngBuffer: Buffer, source: AvatarSource): void {
82
+ export async function setImage(
83
+ pngBuffer: Buffer,
84
+ source: AvatarSource,
85
+ ): Promise<void> {
86
+ const accent = derivedAccent(await deriveAccentHexFromImage(pngBuffer));
72
87
  const avatarDir = getAvatarDir();
73
88
  mkdirSync(avatarDir, { recursive: true });
74
89
 
@@ -85,9 +100,101 @@ export function setImage(pngBuffer: Buffer, source: AvatarSource): void {
85
100
  traits: null,
86
101
  source,
87
102
  image: computeImageMeta(pngPath),
103
+ accent,
88
104
  });
89
105
 
90
- log.info({ source }, "Set avatar from image and removed character sidecars");
106
+ log.info(
107
+ { source, accent: accent?.hex ?? null },
108
+ "Set avatar from image and removed character sidecars",
109
+ );
110
+ }
111
+
112
+ /**
113
+ * The accent the current avatar earns on its own: a character's palette
114
+ * colour, or the colour read out of the image on disk. Null for `none`, and
115
+ * for an image that cannot be read.
116
+ */
117
+ async function automaticAccent(
118
+ state: AvatarState,
119
+ ): Promise<AvatarAccent | null> {
120
+ if (state.kind === "character" && state.traits) {
121
+ return paletteAccent(state.traits.color);
122
+ }
123
+ if (state.kind === "image") {
124
+ const bytes = readContainedAvatarRaster(getAvatarImagePath());
125
+ return bytes ? derivedAccent(await deriveAccentHexFromImage(bytes)) : null;
126
+ }
127
+ return null;
128
+ }
129
+
130
+ /**
131
+ * Sets the accent over the current avatar: a `#rrggbb` the user chose, or
132
+ * `null` to go back to the automatic one. Returns the state as written, or
133
+ * null when there is no avatar to colour. Only the manifest changes; the
134
+ * artifacts are untouched.
135
+ */
136
+ export async function setAccent(
137
+ hex: string | null,
138
+ ): Promise<AvatarState | null> {
139
+ const state = readAvatarState();
140
+ if (state.kind === "none") {
141
+ return null;
142
+ }
143
+ const next: AvatarState = {
144
+ ...state,
145
+ accent: hex ? { hex, source: "custom" } : await automaticAccent(state),
146
+ };
147
+ writeManifest(next);
148
+ return next;
149
+ }
150
+
151
+ /**
152
+ * Image etags whose accent derivation came back empty. Remembered so a read
153
+ * of an unreadable image does not decode it again on every request; a new
154
+ * upload carries a new etag and is tried afresh.
155
+ */
156
+ const accentlessImageEtags = new Set<string>();
157
+ /** The one derivation in flight per etag, so concurrent reads share it. */
158
+ const pendingAccents = new Map<string, Promise<AvatarAccent | null>>();
159
+
160
+ /**
161
+ * Fills in the accent of a state written before accents existed, persisting
162
+ * it so later reads are manifest-only. The one exception to the manifest
163
+ * being written only by the mutations above, for the same reason the read
164
+ * handlers self-heal a missing manifest: the accent is derivable from what is
165
+ * on disk, and deriving it once beats every client deriving it on every read.
166
+ * The persist is best-effort; a read-only workspace still gets the accent.
167
+ */
168
+ export async function backfillAccent(state: AvatarState): Promise<AvatarState> {
169
+ if (state.kind === "none" || state.accent !== null) {
170
+ return state;
171
+ }
172
+ const etag = state.image?.etag ?? null;
173
+ if (etag && accentlessImageEtags.has(etag)) {
174
+ return state;
175
+ }
176
+ const key = etag ?? state.kind;
177
+ let pending = pendingAccents.get(key);
178
+ if (!pending) {
179
+ pending = automaticAccent(state).finally(() => {
180
+ pendingAccents.delete(key);
181
+ });
182
+ pendingAccents.set(key, pending);
183
+ }
184
+ const accent = await pending;
185
+ if (!accent) {
186
+ if (etag) {
187
+ accentlessImageEtags.add(etag);
188
+ }
189
+ return state;
190
+ }
191
+ const next: AvatarState = { ...state, accent };
192
+ try {
193
+ writeManifest(next);
194
+ } catch (err) {
195
+ log.warn({ err }, "Failed to persist the backfilled avatar accent");
196
+ }
197
+ return next;
91
198
  }
92
199
 
93
200
  /**
@@ -25,7 +25,7 @@ The processing pipeline follows a sequential 3-phase flow:
25
25
 
26
26
  1. **Ingest** (`ingest_media`) - Register a media file, detect MIME type, extract duration, deduplicate by content hash.
27
27
  2. **Preprocess** (`extract_keyframes`) - Detect dead time, segment the video into windows, extract downscaled keyframes, build a subject registry, and write a pipeline manifest.
28
- 3. **Map** (`analyze_keyframes`) - Send each segment's frames to Gemini 2.5 Flash with assistant-provided extraction instructions and a JSON Schema for guaranteed structured output. Supports concurrency pooling, cost tracking, resumability, and automatic retries.
28
+ 3. **Map** (`analyze_keyframes`) - Send each segment's frames to the configured Gemini vision model with assistant-provided extraction instructions and a JSON Schema for guaranteed structured output. Supports concurrency pooling, cost tracking, resumability, and automatic retries.
29
29
  4. **Reduce / Query** (`query_media`) - Send all map output to Claude for intelligent analysis and Q&A. Supports arbitrary natural language queries about video content.
30
30
  5. **Clip** (`generate_clip`) - Extract video clips around specific moments.
31
31
 
@@ -85,7 +85,7 @@ Parameters:
85
85
  - `asset_id` (required) - ID of the media asset.
86
86
  - `query` (required) - Natural language query about the video data.
87
87
  - `system_prompt` - Optional system prompt for Claude.
88
- - `model` - LLM model to use (default: `claude-sonnet-4-6`).
88
+ - `model` - LLM model to use. Defaults to the assistant's active chat model.
89
89
 
90
90
  ### generate_clip
91
91
 
@@ -110,7 +110,7 @@ Handles dead-time detection, video segmentation, keyframe extraction, and subjec
110
110
 
111
111
  ### Gemini Map (services/gemini-map.ts)
112
112
 
113
- Sends video segments to Gemini 2.5 Flash with structured output schemas. Handles concurrency pooling, cost tracking, resumability, and retries.
113
+ Sends video segments to the configured Gemini vision model with structured output schemas. Handles concurrency pooling, cost tracking, resumability, and retries.
114
114
 
115
115
  ### Reduce (services/reduce.ts)
116
116
 
@@ -170,7 +170,7 @@
170
170
  },
171
171
  "model": {
172
172
  "type": "string",
173
- "description": "LLM model to use for analysis. Default: 'claude-sonnet-4-6'"
173
+ "description": "LLM model to use for analysis. Defaults to the assistant's active chat model."
174
174
  }
175
175
  },
176
176
  "required": ["asset_id", "query"]
@@ -247,4 +247,4 @@ Choose the right delivery tool based on the content:
247
247
 
248
248
  Example schedule message for a Slack digest:
249
249
 
250
- > "Scan my Slack channels for the last 24 hours using the Slack Web API via bash (network_mode: proxied, credential_ids: ['slack_channel/bot_token']), then post the summary to #alex-agent-messages (C0A7STRJ4G5)."
250
+ > "Scan my Slack channels for the last 24 hours using the Slack Web API via bash (network_mode: proxied, credential_ids: ['slack_channel/bot_token']), then post the summary to the channel the user named."
@@ -127,6 +127,7 @@ function imageState(etag: string): AvatarState {
127
127
  traits: null,
128
128
  source: "upload",
129
129
  image: { updatedAt: "2026-01-01T00:00:00.000Z", etag },
130
+ accent: null,
130
131
  };
131
132
  }
132
133
 
@@ -135,6 +136,7 @@ const NONE: AvatarState = {
135
136
  traits: null,
136
137
  source: null,
137
138
  image: null,
139
+ accent: null,
138
140
  };
139
141
 
140
142
  interface Patch {
@@ -275,6 +277,7 @@ describe("syncAvatarToPlatform", () => {
275
277
  traits: { bodyShape: "blob", eyeStyle: "curious", color: "green" },
276
278
  source: null,
277
279
  image: null,
280
+ accent: null,
278
281
  };
279
282
  mockRasterPath = null;
280
283
  syncAvatarToPlatform();