@avocadostudio-ai/orchestrator-core 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat/chat-pipeline-image.d.ts +169 -3
- package/dist/chat/chat-pipeline-image.js +397 -54
- package/dist/chat/chat-pipeline-shared.d.ts +26 -0
- package/dist/chat/chat-pipeline-shared.js +38 -0
- package/dist/chat/chat-pipeline.d.ts +14 -5
- package/dist/chat/chat-pipeline.js +176 -120
- package/dist/chat/plan-json-schema.d.ts +4 -0
- package/dist/chat/plan-json-schema.js +5 -1
- package/dist/chat/prompts.js +2 -0
- package/dist/chat/provider-routing.d.ts +14 -0
- package/dist/chat/provider-routing.js +20 -0
- package/dist/handler/create-orchestrator.js +4 -1
- package/dist/image/image-helpers.d.ts +24 -1
- package/dist/image/image-helpers.js +159 -5
- package/dist/nlp/deterministic-planner-patches.js +19 -1
- package/dist/nlp/plan-normalizer.js +44 -0
- package/dist/telemetry/chat-telemetry.d.ts +22 -0
- package/dist/telemetry/chat-telemetry.js +2 -0
- package/dist/tools/builtins/unsplash-search.js +12 -3
- package/dist/tools/types.d.ts +16 -0
- package/package.json +3 -3
|
@@ -1,18 +1,56 @@
|
|
|
1
|
-
import { getAllBlockMeta } from "@avocadostudio-ai/shared";
|
|
1
|
+
import { getAllBlockMeta, getImageFields, getImageFieldMap, getListImageFields, isChrome } from "@avocadostudio-ai/shared";
|
|
2
2
|
import { inferTranslationScopeFromMessage } from "./chat-pipeline-translation.js";
|
|
3
3
|
import { heroImageQueryFromContext, imageKeywordsFromQuery, generateVariationImageWithOpenAI, resolveUnsplashImage, resolveGdriveImage, isExplicitImageGenRequest, extractImagePromptFromMessage, estimatedImageGenMs, recordImageGenDuration } from "../image/image-helpers.js";
|
|
4
4
|
import { isGdriveConfigured } from "../image/gdrive-client.js";
|
|
5
5
|
import { firstUrlFromText, looksLikeUserInstruction, preferredImageAltText } from "./chat-pipeline-ui.js";
|
|
6
6
|
import { isDemoModeEnabled, isDemoImageGenDisabled } from "../demo-mode.js";
|
|
7
|
+
/**
|
|
8
|
+
* The Unsplash lookup, behind a seam.
|
|
9
|
+
*
|
|
10
|
+
* Every hermetic test runs without `UNSPLASH_ACCESS_KEY`, so this always
|
|
11
|
+
* answered null and the only image behaviour the suite could observe was the
|
|
12
|
+
* one that fires when there is no image to be had. Injecting a resolver lets a
|
|
13
|
+
* test assert what happens when there *is* one — which is where the pipeline
|
|
14
|
+
* actually decides which prop to write and where its alt text goes.
|
|
15
|
+
*/
|
|
16
|
+
let resolveUnsplashImageImpl = resolveUnsplashImage;
|
|
17
|
+
export function setChatImageResolverForTests(fn) {
|
|
18
|
+
resolveUnsplashImageImpl = fn ?? resolveUnsplashImage;
|
|
19
|
+
}
|
|
7
20
|
/** Is demo mode active AND image gen disabled for demo? Memoized per call. */
|
|
8
21
|
function isDemoImageGenActive() {
|
|
9
22
|
return isDemoModeEnabled() && isDemoImageGenDisabled();
|
|
10
23
|
}
|
|
11
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Which prop on this block an image belongs in, or null if none does.
|
|
26
|
+
*
|
|
27
|
+
* Used by the last-resort path: the message asked for a photo, the planner
|
|
28
|
+
* produced no image op, so find somewhere on the page to put one. It looked
|
|
29
|
+
* for a literal `imageUrl` prop, which meant that on a site whose hero image
|
|
30
|
+
* field is `backgroundImage` the fallback found nothing and the request
|
|
31
|
+
* quietly did nothing at all.
|
|
32
|
+
*
|
|
33
|
+
* Chrome blocks are excluded: the header and the footer own the logo, and
|
|
34
|
+
* "add a photo" has never meant "replace the logo".
|
|
35
|
+
*/
|
|
36
|
+
export function primaryImageKeyFor(block) {
|
|
12
37
|
if (!block)
|
|
13
|
-
return
|
|
38
|
+
return null;
|
|
14
39
|
const props = block.props;
|
|
15
|
-
|
|
40
|
+
if (typeof props !== "object" || props === null)
|
|
41
|
+
return null;
|
|
42
|
+
if (isChrome(block.type))
|
|
43
|
+
return null;
|
|
44
|
+
if (Object.prototype.hasOwnProperty.call(props, "imageUrl"))
|
|
45
|
+
return "imageUrl";
|
|
46
|
+
for (const key of getImageFields(block.type)) {
|
|
47
|
+
if (Object.prototype.hasOwnProperty.call(props, key))
|
|
48
|
+
return key;
|
|
49
|
+
}
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
export function blockHasImageUrlProp(block) {
|
|
53
|
+
return primaryImageKeyFor(block) !== null;
|
|
16
54
|
}
|
|
17
55
|
export function parsePath(path) {
|
|
18
56
|
const parts = [];
|
|
@@ -151,50 +189,239 @@ export function extractReferencedItemIndices(message) {
|
|
|
151
189
|
return { numeric, includesLast, hasConstraint: numeric.size > 0 || includesLast };
|
|
152
190
|
}
|
|
153
191
|
/**
|
|
154
|
-
* Check whether a block type's schema
|
|
192
|
+
* Check whether a block type's schema declares an image at a given path.
|
|
155
193
|
* E.g. "imageUrl" → true for Hero, "features[0].imageUrl" → false for FeatureGrid.
|
|
194
|
+
*
|
|
195
|
+
* The final key is read against the block's own field metadata rather than
|
|
196
|
+
* compared to the literal `imageUrl`, so a block whose image field is called
|
|
197
|
+
* `backgroundImage` or `photo` answers for it too.
|
|
156
198
|
*/
|
|
157
199
|
export function blockSupportsImageAtPath(blockType, imagePath) {
|
|
158
200
|
const meta = getAllBlockMeta()[blockType];
|
|
159
201
|
if (!meta)
|
|
160
202
|
return true; // unknown block type → optimistic
|
|
161
|
-
// Top-level
|
|
162
|
-
|
|
163
|
-
|
|
203
|
+
// Top-level (e.g. Hero's imageUrl)
|
|
204
|
+
const topLevel = imagePath.match(/^([a-zA-Z_][a-zA-Z0-9_]*)$/);
|
|
205
|
+
if (topLevel) {
|
|
206
|
+
const key = topLevel[1];
|
|
207
|
+
return key in meta.fields;
|
|
164
208
|
}
|
|
165
|
-
// Nested: e.g. "features[0].imageUrl" or "items[2].
|
|
166
|
-
const listMatch = imagePath.match(/^([a-zA-Z_]
|
|
209
|
+
// Nested: e.g. "features[0].imageUrl" or "items[2].photo"
|
|
210
|
+
const listMatch = imagePath.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\[\d+\]\.([a-zA-Z_][a-zA-Z0-9_]*)$/);
|
|
167
211
|
if (listMatch) {
|
|
168
|
-
const listName = listMatch
|
|
212
|
+
const [, listName, key] = listMatch;
|
|
169
213
|
const listMeta = meta.listFields?.[listName];
|
|
170
214
|
if (!listMeta)
|
|
171
215
|
return false;
|
|
172
|
-
|
|
216
|
+
if (key in listMeta.itemFields)
|
|
217
|
+
return true;
|
|
218
|
+
return Object.values(listMeta.itemFieldsByType ?? {}).some((branch) => key in branch);
|
|
173
219
|
}
|
|
174
220
|
// Deeper nesting we can't check — be optimistic
|
|
175
221
|
return true;
|
|
176
222
|
}
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
223
|
+
/**
|
|
224
|
+
* Every path inside `value` that holds an image URL.
|
|
225
|
+
*
|
|
226
|
+
* Pass `blockType` and the block's declared `kind: "image"` fields are what
|
|
227
|
+
* counts; without it — a patch whose target is unknown, or a block that ships
|
|
228
|
+
* no field metadata — only the built-in `imageUrl` name is recognised, which
|
|
229
|
+
* is what this did for every caller before the block type was threaded in.
|
|
230
|
+
*/
|
|
231
|
+
export function detectImagePaths(value, opts) {
|
|
232
|
+
const declared = opts?.blockType ? getImageFieldMap(opts.blockType).image : undefined;
|
|
233
|
+
const isImageKey = (key) => key === "imageUrl" || Boolean(declared?.has(key));
|
|
234
|
+
const acc = new Set();
|
|
235
|
+
const walk = (node, basePath) => {
|
|
236
|
+
if (Array.isArray(node)) {
|
|
237
|
+
node.forEach((entry, idx) => walk(entry, `${basePath}[${idx}]`));
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
if (!node || typeof node !== "object")
|
|
241
|
+
return;
|
|
242
|
+
for (const [key, child] of Object.entries(node)) {
|
|
243
|
+
const nextPath = basePath ? `${basePath}.${key}` : key;
|
|
244
|
+
if (isImageKey(key))
|
|
245
|
+
acc.add(nextPath);
|
|
246
|
+
walk(child, nextPath);
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
walk(value, "");
|
|
191
250
|
return acc;
|
|
192
251
|
}
|
|
252
|
+
/**
|
|
253
|
+
* Is the field the user has selected in the preview an image field?
|
|
254
|
+
*
|
|
255
|
+
* The overlay reports the path it marked — `imageUrl`, `cards[2].photo` — and
|
|
256
|
+
* the pipeline used to compare it to the string "imageUrl", so selecting the
|
|
257
|
+
* image on any block that names the field differently read as "the user is not
|
|
258
|
+
* looking at an image".
|
|
259
|
+
*/
|
|
260
|
+
export function isImageEditablePath(blockType, editablePath) {
|
|
261
|
+
if (!editablePath)
|
|
262
|
+
return false;
|
|
263
|
+
const lastKey = editablePath.split(".").pop() ?? editablePath;
|
|
264
|
+
if (lastKey === "imageUrl")
|
|
265
|
+
return true;
|
|
266
|
+
if (!blockType)
|
|
267
|
+
return false;
|
|
268
|
+
return getImageFieldMap(blockType).image.has(lastKey);
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Where the alt text for the image at `imagePath` belongs.
|
|
272
|
+
*
|
|
273
|
+
* Built-in blocks pair `imageUrl` with `imageAlt`, which is why the pairing
|
|
274
|
+
* used to be a string replace — but a Gallery item pairs `imageUrl` with
|
|
275
|
+
* `alt`, so that replace produced a key the Gallery does not have. The
|
|
276
|
+
* registry knows the pairing; ask it, and keep the old replace for blocks that
|
|
277
|
+
* declare nothing.
|
|
278
|
+
*/
|
|
279
|
+
export function imageAltPathFor(blockType, imagePath) {
|
|
280
|
+
const lastKey = imagePath.split(".").pop() ?? imagePath;
|
|
281
|
+
const declaredAlt = blockType ? getImageFieldMap(blockType).altFor.get(lastKey) : undefined;
|
|
282
|
+
if (!declaredAlt)
|
|
283
|
+
return imagePath.replace(/imageUrl$/, "imageAlt");
|
|
284
|
+
return imagePath.slice(0, imagePath.length - lastKey.length) + declaredAlt;
|
|
285
|
+
}
|
|
193
286
|
function isRemoteHttpUrl(value) {
|
|
194
287
|
return typeof value === "string" && /^https?:\/\//i.test(value.trim());
|
|
195
288
|
}
|
|
196
|
-
|
|
197
|
-
|
|
289
|
+
/** The patch an op carries, whether it is wrapped in `props` or not. */
|
|
290
|
+
function unwrapPatchProps(patch) {
|
|
291
|
+
if (!patch || typeof patch !== "object" || Array.isArray(patch))
|
|
292
|
+
return {};
|
|
293
|
+
const raw = patch;
|
|
294
|
+
const inner = raw.props;
|
|
295
|
+
return inner && typeof inner === "object" && !Array.isArray(inner) ? inner : raw;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* A props patch that writes `value` at `path` without collapsing the list the
|
|
299
|
+
* path runs through.
|
|
300
|
+
*
|
|
301
|
+
* `update_props` merges a list entry by entry and keeps the *patch's* length,
|
|
302
|
+
* so the obvious construction — `setValueAtPath({}, "cards[0].imageUrl", url)`
|
|
303
|
+
* → `{ cards: [{ imageUrl: url }] }` — takes a three-card grid down to one
|
|
304
|
+
* card. The list has to be rebuilt at the length it currently has, with every
|
|
305
|
+
* entry but the addressed one left empty for the merge to fill back in.
|
|
306
|
+
*/
|
|
307
|
+
export function buildImagePatchProps(blockProps, path, value) {
|
|
308
|
+
const listMatch = path.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\[(\d+)\]\.(.+)$/);
|
|
309
|
+
const patch = {};
|
|
310
|
+
if (!listMatch) {
|
|
311
|
+
setValueAtPath(patch, path, value);
|
|
312
|
+
return patch;
|
|
313
|
+
}
|
|
314
|
+
const [, listKey, rawIndex, itemPath] = listMatch;
|
|
315
|
+
const index = Number(rawIndex);
|
|
316
|
+
const currentList = blockProps[listKey];
|
|
317
|
+
const length = Math.max(Array.isArray(currentList) ? currentList.length : 0, index + 1);
|
|
318
|
+
patch[listKey] = Array.from({ length }, (_, i) => {
|
|
319
|
+
if (i !== index)
|
|
320
|
+
return {};
|
|
321
|
+
const entry = {};
|
|
322
|
+
setValueAtPath(entry, itemPath, value);
|
|
323
|
+
return entry;
|
|
324
|
+
});
|
|
325
|
+
return patch;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Write `value` at `path` in a patch that is being edited in place, without
|
|
329
|
+
* shortening the list the path runs through.
|
|
330
|
+
*
|
|
331
|
+
* `setValueAtPath` alone creates the list it needs and no more, so writing
|
|
332
|
+
* `cards[0].imageUrl` into a patch that says nothing about `cards` produces a
|
|
333
|
+
* one-element array — and update_props keeps the patch's length, so applying it
|
|
334
|
+
* deletes every other card. That is not hypothetical: a turn asking for the
|
|
335
|
+
* first card's image planned an unrelated `{ title }` patch, the image pass
|
|
336
|
+
* added `cards[0].imageUrl` to it, and a three-card grid came back with one
|
|
337
|
+
* card in it.
|
|
338
|
+
*
|
|
339
|
+
* A patch that already carries the list is left at the length it states — a
|
|
340
|
+
* plan replacing three cards with two means it.
|
|
341
|
+
*/
|
|
342
|
+
export function setImageInPatch(patch, blockProps, path, value) {
|
|
343
|
+
const listMatch = path.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\[(\d+)\]\./);
|
|
344
|
+
if (listMatch) {
|
|
345
|
+
const [, listKey, rawIndex] = listMatch;
|
|
346
|
+
if (!Array.isArray(patch[listKey])) {
|
|
347
|
+
const currentList = blockProps[listKey];
|
|
348
|
+
const length = Math.max(Array.isArray(currentList) ? currentList.length : 0, Number(rawIndex) + 1);
|
|
349
|
+
patch[listKey] = Array.from({ length }, () => ({}));
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
setValueAtPath(patch, path, value);
|
|
353
|
+
}
|
|
354
|
+
export function findImagePatchSites(op,
|
|
355
|
+
/** The page as it stands now — a lookup, so the caller needs no op narrowing of its own. */
|
|
356
|
+
pageAt) {
|
|
357
|
+
if (op.op !== "update_props" && op.op !== "update_item")
|
|
358
|
+
return [];
|
|
359
|
+
const block = pageAt(op.pageSlug)?.blocks.find((b) => b.id === op.blockId);
|
|
360
|
+
const blockType = block?.type;
|
|
361
|
+
const blockProps = block?.props ?? {};
|
|
362
|
+
const isItemOp = op.op === "update_item";
|
|
363
|
+
const patch = isItemOp ? op.patch : unwrapPatchProps(op.patch);
|
|
364
|
+
const sites = [];
|
|
365
|
+
for (const path of detectImagePaths(patch, { blockType })) {
|
|
366
|
+
const value = getValueAtPath(patch, path);
|
|
367
|
+
if (typeof value !== "string")
|
|
368
|
+
continue;
|
|
369
|
+
const altPath = imageAltPathFor(blockType, path);
|
|
370
|
+
const altValue = getValueAtPath(patch, altPath);
|
|
371
|
+
sites.push({
|
|
372
|
+
blockId: op.blockId,
|
|
373
|
+
pageSlug: op.pageSlug,
|
|
374
|
+
blockType,
|
|
375
|
+
path,
|
|
376
|
+
altPath,
|
|
377
|
+
value,
|
|
378
|
+
alt: typeof altValue === "string" ? altValue : undefined,
|
|
379
|
+
rewrite: (url, alt) => {
|
|
380
|
+
if (isItemOp && op.op === "update_item") {
|
|
381
|
+
const itemPatch = {};
|
|
382
|
+
setValueAtPath(itemPatch, path, url);
|
|
383
|
+
if (alt)
|
|
384
|
+
setValueAtPath(itemPatch, altPath, alt);
|
|
385
|
+
return {
|
|
386
|
+
op: "update_item",
|
|
387
|
+
pageSlug: op.pageSlug,
|
|
388
|
+
blockId: op.blockId,
|
|
389
|
+
listKey: op.listKey,
|
|
390
|
+
...(op.itemId ? { itemId: op.itemId } : {}),
|
|
391
|
+
...(typeof op.index === "number" ? { index: op.index } : {}),
|
|
392
|
+
patch: itemPatch
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
const props = buildImagePatchProps(blockProps, path, url);
|
|
396
|
+
if (alt)
|
|
397
|
+
setValueAtPath(props, altPath, alt);
|
|
398
|
+
return { op: "update_props", pageSlug: op.pageSlug, blockId: op.blockId, patch: { props } };
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
return sites;
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Do two ops address the same image? Block identity is enough for
|
|
406
|
+
* `update_props`; an item op also has to agree on which entry of which list,
|
|
407
|
+
* or a plan touching two cards would patch the wrong one.
|
|
408
|
+
*/
|
|
409
|
+
export function opsAddressSameTarget(a, b) {
|
|
410
|
+
if (a.op !== b.op)
|
|
411
|
+
return false;
|
|
412
|
+
if (a.op !== "update_props" && a.op !== "update_item")
|
|
413
|
+
return false;
|
|
414
|
+
if (a.blockId !== b.blockId)
|
|
415
|
+
return false;
|
|
416
|
+
if (a.pageSlug !== b.pageSlug)
|
|
417
|
+
return false;
|
|
418
|
+
if (a.op === "update_item" && b.op === "update_item") {
|
|
419
|
+
return a.listKey === b.listKey && a.itemId === b.itemId && a.index === b.index;
|
|
420
|
+
}
|
|
421
|
+
return true;
|
|
422
|
+
}
|
|
423
|
+
function patchContainsResolvedImageUrl(patch, blockType) {
|
|
424
|
+
const imagePaths = detectImagePaths(patch, { blockType });
|
|
198
425
|
for (const path of imagePaths) {
|
|
199
426
|
if (isRemoteHttpUrl(getValueAtPath(patch, path)))
|
|
200
427
|
return true;
|
|
@@ -247,17 +474,27 @@ export function findImageTargets(args) {
|
|
|
247
474
|
message: args.message,
|
|
248
475
|
currentPage: args.currentPage,
|
|
249
476
|
targetBlock: args.targetBlock,
|
|
250
|
-
patchCandidate: args.patchCandidate
|
|
477
|
+
patchCandidate: args.patchCandidate,
|
|
478
|
+
plannerQuery: args.plannerQuery
|
|
251
479
|
});
|
|
252
|
-
const
|
|
480
|
+
const blockType = args.targetBlock.type;
|
|
481
|
+
const imagePaths = detectImagePaths(mergedProps, { blockType });
|
|
253
482
|
if (shouldPopulateAllChildImages(args.message)) {
|
|
483
|
+
// "add photos to all the cards" reaches items that have no image key yet,
|
|
484
|
+
// so there is nothing in the props to detect — the key has to come from
|
|
485
|
+
// what the list declares. A list that declares none keeps `imageUrl`,
|
|
486
|
+
// which is the only name this could invent before.
|
|
487
|
+
const declaredListImages = getListImageFields(blockType);
|
|
254
488
|
for (const [key, value] of Object.entries(mergedProps)) {
|
|
255
489
|
if (!Array.isArray(value))
|
|
256
490
|
continue;
|
|
491
|
+
const itemKeys = declaredListImages.get(key);
|
|
492
|
+
const keysToFill = itemKeys && itemKeys.size > 0 ? [...itemKeys] : ["imageUrl"];
|
|
257
493
|
value.forEach((entry, idx) => {
|
|
258
494
|
if (!entry || typeof entry !== "object" || Array.isArray(entry))
|
|
259
495
|
return;
|
|
260
|
-
|
|
496
|
+
for (const itemKey of keysToFill)
|
|
497
|
+
imagePaths.add(`${key}[${idx}].${itemKey}`);
|
|
261
498
|
});
|
|
262
499
|
}
|
|
263
500
|
}
|
|
@@ -268,7 +505,7 @@ export function findImageTargets(args) {
|
|
|
268
505
|
.join(" ");
|
|
269
506
|
const targets = [];
|
|
270
507
|
for (const path of imagePaths) {
|
|
271
|
-
const itemMatch = path.match(/^(.*\[(\d+)\])\.
|
|
508
|
+
const itemMatch = path.match(/^(.*\[(\d+)\])\.[a-zA-Z_][a-zA-Z0-9_]*$/);
|
|
272
509
|
const index = itemMatch?.[2] ? Number(itemMatch[2]) : undefined;
|
|
273
510
|
const itemPath = itemMatch?.[1];
|
|
274
511
|
if (constrainedIndices.hasConstraint && itemPath && index !== undefined) {
|
|
@@ -291,10 +528,30 @@ export function findImageTargets(args) {
|
|
|
291
528
|
}
|
|
292
529
|
if (!query)
|
|
293
530
|
query = defaultQuery;
|
|
294
|
-
targets.push({ path, altPath:
|
|
531
|
+
targets.push({ path, altPath: imageAltPathFor(blockType, path), query });
|
|
295
532
|
}
|
|
296
533
|
return targets;
|
|
297
534
|
}
|
|
535
|
+
/**
|
|
536
|
+
* The image subject the planner stated anywhere in this plan.
|
|
537
|
+
*
|
|
538
|
+
* The two hero fallbacks below fire when no op yielded an image target — the
|
|
539
|
+
* plan named a block this page does not have, or patched nothing an image
|
|
540
|
+
* could hang on. The request was still about an image and the planner still
|
|
541
|
+
* said what it should show, so that answer beats re-deriving one from the
|
|
542
|
+
* sentence. Ops are checked in order and the first stated subject wins; a plan
|
|
543
|
+
* that resolves several images by this route is not a case worth guessing at.
|
|
544
|
+
*/
|
|
545
|
+
export function statedImageQuery(plan) {
|
|
546
|
+
for (const op of plan.ops) {
|
|
547
|
+
if (op.op !== "update_props")
|
|
548
|
+
continue;
|
|
549
|
+
const stated = op.imageQuery?.trim();
|
|
550
|
+
if (stated)
|
|
551
|
+
return stated;
|
|
552
|
+
}
|
|
553
|
+
return undefined;
|
|
554
|
+
}
|
|
298
555
|
export function rewriteAddBlockToChildImageUpdate(args) {
|
|
299
556
|
if (args.plan.intent !== "edit_plan")
|
|
300
557
|
return args.plan;
|
|
@@ -491,21 +748,22 @@ export async function withUnsplashHeroImage(args) {
|
|
|
491
748
|
const patchCandidate = rawPatch && typeof rawPatch.props === "object" && rawPatch.props !== null && !Array.isArray(rawPatch.props)
|
|
492
749
|
? rawPatch.props
|
|
493
750
|
: rawPatch;
|
|
494
|
-
const touchesImage = detectImagePaths(patchCandidate).size > 0 ||
|
|
495
|
-
args.activeEditablePath
|
|
751
|
+
const touchesImage = detectImagePaths(patchCandidate, { blockType: target.type }).size > 0 ||
|
|
752
|
+
isImageEditablePath(target.type, args.activeEditablePath) ||
|
|
496
753
|
/\b(images?|photos?|pictures?)\b/.test(lowerMessage);
|
|
497
754
|
const userProvidedExplicitUrl = Boolean(firstUrlFromText(args.message));
|
|
498
755
|
const targets = findImageTargets({
|
|
499
756
|
message: args.message,
|
|
500
757
|
currentPage: args.currentPage,
|
|
501
758
|
targetBlock: target,
|
|
502
|
-
patchCandidate
|
|
759
|
+
patchCandidate,
|
|
760
|
+
plannerQuery: op.imageQuery
|
|
503
761
|
});
|
|
504
|
-
// Filter out targets whose block schema doesn't
|
|
762
|
+
// Filter out targets whose block schema doesn't declare an image at that path
|
|
505
763
|
const supportedTargets = targets.filter((t) => blockSupportsImageAtPath(target.type, t.path));
|
|
506
764
|
const unresolvedTargets = supportedTargets.filter((t) => !isRemoteHttpUrl(getValueAtPath(patchCandidate, t.path)));
|
|
507
765
|
const hasAnyImageTarget = supportedTargets.length > 0;
|
|
508
|
-
const hasImageUrlInPatch = detectImagePaths(patchCandidate).size > 0;
|
|
766
|
+
const hasImageUrlInPatch = detectImagePaths(patchCandidate, { blockType: target.type }).size > 0;
|
|
509
767
|
const shouldReplace = !userProvidedExplicitUrl && touchesImage && hasAnyImageTarget && unresolvedTargets.length > 0 && (explicitUnsplashRequest || hasImageUrlInPatch || explicitImageGen);
|
|
510
768
|
if (!touchesImage || !shouldReplace || unresolvedTargets.length === 0)
|
|
511
769
|
continue;
|
|
@@ -568,7 +826,7 @@ export async function withUnsplashHeroImage(args) {
|
|
|
568
826
|
if (currentImageUrl)
|
|
569
827
|
globalUsedImageUrls.add(currentImageUrl);
|
|
570
828
|
const usedImageUrls = globalUsedImageUrls.size > 0 ? globalUsedImageUrls : undefined;
|
|
571
|
-
resolved = await
|
|
829
|
+
resolved = await resolveUnsplashImageImpl(imageQuery, { subjectKeywords: imageKeywordsFromQuery(imageQuery, 4), usedImageUrls }, { chatRequestId: args.chatRequestId, logger: args.log });
|
|
572
830
|
if (resolved)
|
|
573
831
|
imageSource = resolved.url.includes("unsplash") ? "unsplash" : "placeholder";
|
|
574
832
|
}
|
|
@@ -580,7 +838,7 @@ export async function withUnsplashHeroImage(args) {
|
|
|
580
838
|
skippedImageCount++;
|
|
581
839
|
continue;
|
|
582
840
|
}
|
|
583
|
-
|
|
841
|
+
setImageInPatch(patchCandidate, targetProps, targetImage.path, resolved.url);
|
|
584
842
|
const existingAlt = getValueAtPath(patchCandidate, targetImage.altPath);
|
|
585
843
|
const nextAlt = preferredImageAltText({
|
|
586
844
|
query: imageQuery,
|
|
@@ -588,7 +846,7 @@ export async function withUnsplashHeroImage(args) {
|
|
|
588
846
|
existingAlt: typeof existingAlt === "string" ? existingAlt : undefined
|
|
589
847
|
});
|
|
590
848
|
if (nextAlt.trim().length > 0)
|
|
591
|
-
|
|
849
|
+
setImageInPatch(patchCandidate, targetProps, targetImage.altPath, nextAlt);
|
|
592
850
|
sourceQuery = resolved.query;
|
|
593
851
|
args.log.info({
|
|
594
852
|
event: "image_rewrite_applied",
|
|
@@ -596,6 +854,12 @@ export async function withUnsplashHeroImage(args) {
|
|
|
596
854
|
slug: args.slug,
|
|
597
855
|
blockId: op.blockId,
|
|
598
856
|
query: imageQuery,
|
|
857
|
+
// What the planner said the picture should show, logged next to what
|
|
858
|
+
// was actually searched for. A trace where these differ, or where
|
|
859
|
+
// this is absent on an image request, is the whole failure mode:
|
|
860
|
+
// otherwise the only way to tell a stated subject from one
|
|
861
|
+
// reconstructed off the block's heading is to re-derive it by hand.
|
|
862
|
+
plannerQuery: op.imageQuery,
|
|
599
863
|
explicitUnsplashRequest,
|
|
600
864
|
path: targetImage.path,
|
|
601
865
|
nextImageUrl: resolved.url,
|
|
@@ -617,7 +881,7 @@ export async function withUnsplashHeroImage(args) {
|
|
|
617
881
|
const patchCandidate = patch && typeof patch.props === "object" && patch.props !== null && !Array.isArray(patch.props)
|
|
618
882
|
? patch.props
|
|
619
883
|
: patch;
|
|
620
|
-
return patchContainsResolvedImageUrl(patchCandidate);
|
|
884
|
+
return patchContainsResolvedImageUrl(patchCandidate, args.currentPage.blocks.find((block) => block.id === op.blockId)?.type);
|
|
621
885
|
});
|
|
622
886
|
if (!changed && !planAlreadyHasResolvedImage && (explicitUnsplashRequest || explicitImageGen) && /\b(images?|photos?|pictures?|hero)\b/.test(lowerMessage)) {
|
|
623
887
|
const selectedBlock = args.activeBlockId && args.currentPage.blocks.find((block) => block.id === args.activeBlockId)
|
|
@@ -630,7 +894,8 @@ export async function withUnsplashHeroImage(args) {
|
|
|
630
894
|
const query = heroImageQueryFromContext({
|
|
631
895
|
message: args.message,
|
|
632
896
|
currentPage: args.currentPage,
|
|
633
|
-
targetBlock: fallbackHero
|
|
897
|
+
targetBlock: fallbackHero,
|
|
898
|
+
plannerQuery: statedImageQuery(plan)
|
|
634
899
|
});
|
|
635
900
|
let resolved = null;
|
|
636
901
|
if (!explicitUnsplashRequest && process.env.OPENAI_API_KEY && (explicitImageGen || userImagePrompt)) {
|
|
@@ -681,9 +946,10 @@ export async function withUnsplashHeroImage(args) {
|
|
|
681
946
|
if (!resolved) {
|
|
682
947
|
args.onStatusUpdate?.("Finding a suitable image...");
|
|
683
948
|
const fbProps = fallbackHero.props;
|
|
684
|
-
const
|
|
949
|
+
const fbImageKey = primaryImageKeyFor(fallbackHero) ?? "imageUrl";
|
|
950
|
+
const fbCurrentUrl = typeof fbProps[fbImageKey] === "string" ? fbProps[fbImageKey] : "";
|
|
685
951
|
const fbUsedUrls = fbCurrentUrl ? new Set([fbCurrentUrl]) : undefined;
|
|
686
|
-
resolved = await
|
|
952
|
+
resolved = await resolveUnsplashImageImpl(query, { subjectKeywords: imageKeywordsFromQuery(query, 4), usedImageUrls: fbUsedUrls }, { chatRequestId: args.chatRequestId, logger: args.log });
|
|
687
953
|
if (resolved)
|
|
688
954
|
imageSource = resolved.url.includes("unsplash") ? "unsplash" : "placeholder";
|
|
689
955
|
}
|
|
@@ -692,11 +958,15 @@ export async function withUnsplashHeroImage(args) {
|
|
|
692
958
|
placeholderSkipped = true;
|
|
693
959
|
}
|
|
694
960
|
else {
|
|
961
|
+
const imageKey = primaryImageKeyFor(fallbackHero) ?? "imageUrl";
|
|
695
962
|
plan.ops.push({
|
|
696
963
|
op: "update_props",
|
|
697
964
|
pageSlug: args.slug,
|
|
698
965
|
blockId: fallbackHero.id,
|
|
699
|
-
patch: {
|
|
966
|
+
patch: {
|
|
967
|
+
[imageKey]: resolved.url,
|
|
968
|
+
[imageAltPathFor(fallbackHero.type, imageKey)]: preferredImageAltText({ query, resolvedAlt: resolved.alt })
|
|
969
|
+
}
|
|
700
970
|
});
|
|
701
971
|
sourceQuery = resolved.query;
|
|
702
972
|
changed = true;
|
|
@@ -792,12 +1062,75 @@ export async function resolveHeroImageForCreatePage(args) {
|
|
|
792
1062
|
}
|
|
793
1063
|
}
|
|
794
1064
|
args.onStatusUpdate?.("Finding image for new page...");
|
|
795
|
-
const resolved = await
|
|
1065
|
+
const resolved = await resolveUnsplashImageImpl(args.query, { subjectKeywords: imageKeywordsFromQuery(args.query, 4) }, { chatRequestId: args.chatRequestId, logger: args.log });
|
|
796
1066
|
if (resolved && resolved.url.includes("unsplash")) {
|
|
797
1067
|
return { url: resolved.url, alt: resolved.alt, source: "unsplash" };
|
|
798
1068
|
}
|
|
799
1069
|
return null;
|
|
800
1070
|
}
|
|
1071
|
+
/**
|
|
1072
|
+
* Is this patch all structure and no content — nothing but empty objects and
|
|
1073
|
+
* empty lists of them?
|
|
1074
|
+
*
|
|
1075
|
+
* `{}` is obviously nothing. `{ cards: [{}, {}, {}] }` is the same nothing
|
|
1076
|
+
* wearing the shape a list patch has to wear to keep its length, and the ops
|
|
1077
|
+
* engine skips it as an unchanged value. Callers that need to know whether
|
|
1078
|
+
* stripping a patch left anything behind have to ask the second question too.
|
|
1079
|
+
*/
|
|
1080
|
+
export function isVacuousPatch(value) {
|
|
1081
|
+
if (Array.isArray(value))
|
|
1082
|
+
return value.every(isVacuousPatch);
|
|
1083
|
+
if (value && typeof value === "object")
|
|
1084
|
+
return Object.values(value).every(isVacuousPatch);
|
|
1085
|
+
return false;
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* Rewrite an `update_item` that sets an image into the nested `update_props`
|
|
1089
|
+
* form, so the image passes can see it at all.
|
|
1090
|
+
*
|
|
1091
|
+
* `detectImageOps` and `withUnsplashHeroImage` both walk `update_props` ops and
|
|
1092
|
+
* read image paths out of the block's merged props — `cards[0].imageUrl`. An
|
|
1093
|
+
* item op carries exactly that edit in a shape neither reads. For a planner
|
|
1094
|
+
* with image tools to call that only costs a detour; for one without them —
|
|
1095
|
+
* OpenAI defers no image tools, and it is the default planner — those two
|
|
1096
|
+
* passes are the *only* route to a real photo, so the stand-in the prompt tells
|
|
1097
|
+
* the model to write was the final answer. A card kept `/hero-generated.svg`
|
|
1098
|
+
* under a summary saying a photo had been found.
|
|
1099
|
+
*
|
|
1100
|
+
* Only item ops that actually set an image are rewritten. An item op changing
|
|
1101
|
+
* text keeps its shape, which addresses the entry far more precisely.
|
|
1102
|
+
*/
|
|
1103
|
+
export function rewriteItemImageUpdateToProps(args) {
|
|
1104
|
+
if (args.plan.intent !== "edit_plan")
|
|
1105
|
+
return args.plan;
|
|
1106
|
+
let changed = false;
|
|
1107
|
+
const ops = args.plan.ops.map((op) => {
|
|
1108
|
+
if (op.op !== "update_item" || op.pageSlug !== args.slug)
|
|
1109
|
+
return op;
|
|
1110
|
+
const block = args.currentPage.blocks.find((b) => b.id === op.blockId);
|
|
1111
|
+
if (!block)
|
|
1112
|
+
return op;
|
|
1113
|
+
const patch = op.patch;
|
|
1114
|
+
if (detectImagePaths(patch, { blockType: block.type }).size === 0)
|
|
1115
|
+
return op;
|
|
1116
|
+
const list = block.props[op.listKey];
|
|
1117
|
+
if (!Array.isArray(list))
|
|
1118
|
+
return op;
|
|
1119
|
+
const index = op.itemId
|
|
1120
|
+
? list.findIndex((entry) => entry && typeof entry === "object" && entry.id === op.itemId)
|
|
1121
|
+
: typeof op.index === "number" ? op.index : -1;
|
|
1122
|
+
if (index < 0 || index >= list.length)
|
|
1123
|
+
return op;
|
|
1124
|
+
changed = true;
|
|
1125
|
+
return {
|
|
1126
|
+
op: "update_props",
|
|
1127
|
+
pageSlug: op.pageSlug,
|
|
1128
|
+
blockId: op.blockId,
|
|
1129
|
+
patch: { props: { [op.listKey]: list.map((_, i) => (i === index ? { ...patch } : {})) } }
|
|
1130
|
+
};
|
|
1131
|
+
});
|
|
1132
|
+
return changed ? { ...args.plan, ops } : args.plan;
|
|
1133
|
+
}
|
|
801
1134
|
// ---------------------------------------------------------------------------
|
|
802
1135
|
// Synchronous image-op detection (no API calls)
|
|
803
1136
|
// ---------------------------------------------------------------------------
|
|
@@ -827,17 +1160,18 @@ export function detectImageOps(args) {
|
|
|
827
1160
|
const patchCandidate = rawPatch && typeof rawPatch.props === "object" && rawPatch.props !== null && !Array.isArray(rawPatch.props)
|
|
828
1161
|
? rawPatch.props
|
|
829
1162
|
: rawPatch;
|
|
830
|
-
const touchesImage = detectImagePaths(patchCandidate).size > 0 ||
|
|
831
|
-
args.activeEditablePath
|
|
1163
|
+
const touchesImage = detectImagePaths(patchCandidate, { blockType: target.type }).size > 0 ||
|
|
1164
|
+
isImageEditablePath(target.type, args.activeEditablePath) ||
|
|
832
1165
|
/\b(images?|photos?|pictures?)\b/.test(lowerMessage);
|
|
833
1166
|
const userProvidedExplicitUrl = Boolean(firstUrlFromText(args.message));
|
|
834
1167
|
const targets = findImageTargets({
|
|
835
1168
|
message: args.message,
|
|
836
1169
|
currentPage: args.currentPage,
|
|
837
1170
|
targetBlock: target,
|
|
838
|
-
patchCandidate
|
|
1171
|
+
patchCandidate,
|
|
1172
|
+
plannerQuery: op.imageQuery
|
|
839
1173
|
});
|
|
840
|
-
const hasImageUrlInPatch = detectImagePaths(patchCandidate).size > 0;
|
|
1174
|
+
const hasImageUrlInPatch = detectImagePaths(patchCandidate, { blockType: target.type }).size > 0;
|
|
841
1175
|
const shouldReplace = !userProvidedExplicitUrl &&
|
|
842
1176
|
touchesImage &&
|
|
843
1177
|
targets.length > 0 &&
|
|
@@ -871,7 +1205,7 @@ export function detectImageOps(args) {
|
|
|
871
1205
|
const patchCandidate = patch && typeof patch.props === "object" && patch.props !== null && !Array.isArray(patch.props)
|
|
872
1206
|
? patch.props
|
|
873
1207
|
: patch;
|
|
874
|
-
return patchContainsResolvedImageUrl(patchCandidate);
|
|
1208
|
+
return patchContainsResolvedImageUrl(patchCandidate, args.currentPage.blocks.find((block) => block.id === op.blockId)?.type);
|
|
875
1209
|
});
|
|
876
1210
|
if (results.length === 0 && !planAlreadyHasResolvedImage && (explicitUnsplashRequest || explicitImageGen) && /\b(images?|photos?|pictures?|hero)\b/.test(lowerMessage)) {
|
|
877
1211
|
const selectedBlock = args.activeBlockId && args.currentPage.blocks.find((block) => block.id === args.activeBlockId)
|
|
@@ -884,13 +1218,22 @@ export function detectImageOps(args) {
|
|
|
884
1218
|
const query = heroImageQueryFromContext({
|
|
885
1219
|
message: args.message,
|
|
886
1220
|
currentPage: args.currentPage,
|
|
887
|
-
targetBlock: fallbackHero
|
|
1221
|
+
targetBlock: fallbackHero,
|
|
1222
|
+
plannerQuery: statedImageQuery(args.plan)
|
|
888
1223
|
});
|
|
889
1224
|
const provider = explicitGdriveRequest && isGdriveConfigured() ? "gdrive"
|
|
890
1225
|
: explicitUnsplashRequest ? "unsplash"
|
|
891
1226
|
: process.env.OPENAI_API_KEY ? "auto"
|
|
892
1227
|
: "unsplash";
|
|
893
|
-
|
|
1228
|
+
const path = primaryImageKeyFor(fallbackHero) ?? "imageUrl";
|
|
1229
|
+
results.push({
|
|
1230
|
+
blockId: fallbackHero.id,
|
|
1231
|
+
pageSlug: args.slug,
|
|
1232
|
+
query,
|
|
1233
|
+
provider,
|
|
1234
|
+
path,
|
|
1235
|
+
altPath: imageAltPathFor(fallbackHero.type, path)
|
|
1236
|
+
});
|
|
894
1237
|
}
|
|
895
1238
|
}
|
|
896
1239
|
return results;
|