@avocadostudio-ai/orchestrator-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/LICENSE +201 -0
  2. package/dist/agent/agent-context.d.ts +19 -0
  3. package/dist/agent/agent-context.js +67 -0
  4. package/dist/agent/agent-logger.d.ts +5 -0
  5. package/dist/agent/agent-logger.js +22 -0
  6. package/dist/agent/agent-loop-openai.d.ts +8 -0
  7. package/dist/agent/agent-loop-openai.js +172 -0
  8. package/dist/agent/agent-loop.d.ts +56 -0
  9. package/dist/agent/agent-loop.js +167 -0
  10. package/dist/agent/agent-provider.d.ts +28 -0
  11. package/dist/agent/agent-provider.js +63 -0
  12. package/dist/agent/agent-tools.d.ts +28 -0
  13. package/dist/agent/agent-tools.js +899 -0
  14. package/dist/agent/context/editing-guidelines.md +46 -0
  15. package/dist/agent/context/role.md +39 -0
  16. package/dist/agent/integration-prompt.d.ts +9 -0
  17. package/dist/agent/integration-prompt.js +154 -0
  18. package/dist/agent/sites-agent-context.d.ts +12 -0
  19. package/dist/agent/sites-agent-context.js +316 -0
  20. package/dist/agent/sites-agent-shared.d.ts +161 -0
  21. package/dist/agent/sites-agent-shared.js +1101 -0
  22. package/dist/agent/sites-agent-tools.d.ts +18 -0
  23. package/dist/agent/sites-agent-tools.js +1227 -0
  24. package/dist/chat/anthropic-cache.d.ts +20 -0
  25. package/dist/chat/anthropic-cache.js +54 -0
  26. package/dist/chat/anthropic-planner.d.ts +98 -0
  27. package/dist/chat/anthropic-planner.js +1012 -0
  28. package/dist/chat/changelog-coverage-validator.d.ts +37 -0
  29. package/dist/chat/changelog-coverage-validator.js +215 -0
  30. package/dist/chat/chat-pipeline-context.d.ts +211 -0
  31. package/dist/chat/chat-pipeline-context.js +249 -0
  32. package/dist/chat/chat-pipeline-deterministic.d.ts +61 -0
  33. package/dist/chat/chat-pipeline-deterministic.js +407 -0
  34. package/dist/chat/chat-pipeline-image.d.ts +86 -0
  35. package/dist/chat/chat-pipeline-image.js +897 -0
  36. package/dist/chat/chat-pipeline-shared.d.ts +69 -0
  37. package/dist/chat/chat-pipeline-shared.js +212 -0
  38. package/dist/chat/chat-pipeline-translation.d.ts +27 -0
  39. package/dist/chat/chat-pipeline-translation.js +417 -0
  40. package/dist/chat/chat-pipeline-ui.d.ts +14 -0
  41. package/dist/chat/chat-pipeline-ui.js +244 -0
  42. package/dist/chat/chat-pipeline.d.ts +99 -0
  43. package/dist/chat/chat-pipeline.js +3999 -0
  44. package/dist/chat/decomposer.d.ts +21 -0
  45. package/dist/chat/decomposer.js +65 -0
  46. package/dist/chat/gemini-planner.d.ts +70 -0
  47. package/dist/chat/gemini-planner.js +541 -0
  48. package/dist/chat/hallucination-validator.d.ts +36 -0
  49. package/dist/chat/hallucination-validator.js +110 -0
  50. package/dist/chat/locale-strings.d.ts +47 -0
  51. package/dist/chat/locale-strings.js +100 -0
  52. package/dist/chat/plan-json-schema.d.ts +133 -0
  53. package/dist/chat/plan-json-schema.js +112 -0
  54. package/dist/chat/planner-types.d.ts +120 -0
  55. package/dist/chat/planner-types.js +66 -0
  56. package/dist/chat/planner.d.ts +148 -0
  57. package/dist/chat/planner.js +1361 -0
  58. package/dist/chat/prompts.d.ts +67 -0
  59. package/dist/chat/prompts.js +356 -0
  60. package/dist/chat/provider-routing.d.ts +14 -0
  61. package/dist/chat/provider-routing.js +27 -0
  62. package/dist/chat/variation-pipeline.d.ts +135 -0
  63. package/dist/chat/variation-pipeline.js +837 -0
  64. package/dist/chat/vision-alt-generator.d.ts +35 -0
  65. package/dist/chat/vision-alt-generator.js +152 -0
  66. package/dist/cms/adapter.d.ts +62 -0
  67. package/dist/cms/adapter.js +1 -0
  68. package/dist/cms/bootstrap.d.ts +17 -0
  69. package/dist/cms/bootstrap.js +85 -0
  70. package/dist/cms/editor-api-adapter.d.ts +23 -0
  71. package/dist/cms/editor-api-adapter.js +71 -0
  72. package/dist/cms/index.d.ts +4 -0
  73. package/dist/cms/index.js +3 -0
  74. package/dist/cms/json-file-adapter.d.ts +11 -0
  75. package/dist/cms/json-file-adapter.js +62 -0
  76. package/dist/demo-mode.d.ts +59 -0
  77. package/dist/demo-mode.js +201 -0
  78. package/dist/errors.d.ts +67 -0
  79. package/dist/errors.js +129 -0
  80. package/dist/http/chat-stream-resumable.d.ts +108 -0
  81. package/dist/http/chat-stream-resumable.js +290 -0
  82. package/dist/http/chat-stream.d.ts +99 -0
  83. package/dist/http/chat-stream.js +92 -0
  84. package/dist/image/gdrive-client.d.ts +22 -0
  85. package/dist/image/gdrive-client.js +215 -0
  86. package/dist/image/image-helpers.d.ts +95 -0
  87. package/dist/image/image-helpers.js +488 -0
  88. package/dist/index.d.ts +1 -0
  89. package/dist/index.js +1 -0
  90. package/dist/jira/jira-approval.d.ts +22 -0
  91. package/dist/jira/jira-approval.js +51 -0
  92. package/dist/jira/jira-client.d.ts +44 -0
  93. package/dist/jira/jira-client.js +313 -0
  94. package/dist/jira/jira-poller.d.ts +46 -0
  95. package/dist/jira/jira-poller.js +184 -0
  96. package/dist/jira/jira-processor.d.ts +103 -0
  97. package/dist/jira/jira-processor.js +1085 -0
  98. package/dist/jira/jira-types.d.ts +117 -0
  99. package/dist/jira/jira-types.js +38 -0
  100. package/dist/logger.d.ts +12 -0
  101. package/dist/logger.js +28 -0
  102. package/dist/migration/mcp-server-stdio.d.ts +8 -0
  103. package/dist/migration/mcp-server-stdio.js +672 -0
  104. package/dist/migration/migration-prompt.d.ts +7 -0
  105. package/dist/migration/migration-prompt.js +197 -0
  106. package/dist/migration/migration-tools.d.ts +17 -0
  107. package/dist/migration/migration-tools.js +159 -0
  108. package/dist/migration/scrape-cache.d.ts +9 -0
  109. package/dist/migration/scrape-cache.js +19 -0
  110. package/dist/nlp/deterministic-planner-context.d.ts +141 -0
  111. package/dist/nlp/deterministic-planner-context.js +362 -0
  112. package/dist/nlp/deterministic-planner-pages.d.ts +26 -0
  113. package/dist/nlp/deterministic-planner-pages.js +170 -0
  114. package/dist/nlp/deterministic-planner-patches.d.ts +80 -0
  115. package/dist/nlp/deterministic-planner-patches.js +508 -0
  116. package/dist/nlp/deterministic-planner-refs.d.ts +33 -0
  117. package/dist/nlp/deterministic-planner-refs.js +164 -0
  118. package/dist/nlp/deterministic-planner-suggestions.d.ts +49 -0
  119. package/dist/nlp/deterministic-planner-suggestions.js +579 -0
  120. package/dist/nlp/deterministic-planner.d.ts +85 -0
  121. package/dist/nlp/deterministic-planner.js +1631 -0
  122. package/dist/nlp/intent-detection.d.ts +309 -0
  123. package/dist/nlp/intent-detection.js +730 -0
  124. package/dist/nlp/intent-helpers.d.ts +15 -0
  125. package/dist/nlp/intent-helpers.js +243 -0
  126. package/dist/nlp/intent-patterns.d.ts +40 -0
  127. package/dist/nlp/intent-patterns.js +223 -0
  128. package/dist/nlp/plan-normalizer.d.ts +41 -0
  129. package/dist/nlp/plan-normalizer.js +1537 -0
  130. package/dist/ops/destructive-action-gate.d.ts +44 -0
  131. package/dist/ops/destructive-action-gate.js +90 -0
  132. package/dist/ops/ops-engine.d.ts +151 -0
  133. package/dist/ops/ops-engine.js +1394 -0
  134. package/dist/publish/diff-engine.d.ts +18 -0
  135. package/dist/publish/diff-engine.js +305 -0
  136. package/dist/publish/publish-helpers.d.ts +87 -0
  137. package/dist/publish/publish-helpers.js +521 -0
  138. package/dist/publish/publish-target-registry.d.ts +7 -0
  139. package/dist/publish/publish-target-registry.js +61 -0
  140. package/dist/publish/publish-target.d.ts +81 -0
  141. package/dist/publish/publish-target.js +1 -0
  142. package/dist/publish/targets/deploy-hook.d.ts +13 -0
  143. package/dist/publish/targets/deploy-hook.js +123 -0
  144. package/dist/publish/targets/git.d.ts +13 -0
  145. package/dist/publish/targets/git.js +55 -0
  146. package/dist/publish/targets/site-contract.d.ts +19 -0
  147. package/dist/publish/targets/site-contract.js +124 -0
  148. package/dist/state/content-source.d.ts +17 -0
  149. package/dist/state/content-source.js +1 -0
  150. package/dist/state/in-memory-content-source.d.ts +27 -0
  151. package/dist/state/in-memory-content-source.js +51 -0
  152. package/dist/state/session-lock.d.ts +13 -0
  153. package/dist/state/session-lock.js +29 -0
  154. package/dist/state/session-state.d.ts +310 -0
  155. package/dist/state/session-state.js +1083 -0
  156. package/dist/state/sqlite-store-singleton.d.ts +31 -0
  157. package/dist/state/sqlite-store-singleton.js +170 -0
  158. package/dist/state/sqlite-store.d.ts +135 -0
  159. package/dist/state/sqlite-store.js +421 -0
  160. package/dist/telemetry/chat-telemetry.d.ts +105 -0
  161. package/dist/telemetry/chat-telemetry.js +247 -0
  162. package/dist/telemetry/eval-candidate-store.d.ts +50 -0
  163. package/dist/telemetry/eval-candidate-store.js +120 -0
  164. package/dist/telemetry/feedback-store.d.ts +34 -0
  165. package/dist/telemetry/feedback-store.js +76 -0
  166. package/dist/telemetry/jira-telemetry.d.ts +57 -0
  167. package/dist/telemetry/jira-telemetry.js +68 -0
  168. package/dist/telemetry/migration-telemetry.d.ts +35 -0
  169. package/dist/telemetry/migration-telemetry.js +40 -0
  170. package/dist/telemetry/usage.d.ts +24 -0
  171. package/dist/telemetry/usage.js +80 -0
  172. package/dist/tools/builtin-registrations.d.ts +12 -0
  173. package/dist/tools/builtin-registrations.js +33 -0
  174. package/dist/tools/builtins/gdrive-browse.d.ts +3 -0
  175. package/dist/tools/builtins/gdrive-browse.js +68 -0
  176. package/dist/tools/builtins/image-generate.d.ts +3 -0
  177. package/dist/tools/builtins/image-generate.js +211 -0
  178. package/dist/tools/builtins/unsplash-get-by-id.d.ts +23 -0
  179. package/dist/tools/builtins/unsplash-get-by-id.js +119 -0
  180. package/dist/tools/builtins/unsplash-search.d.ts +3 -0
  181. package/dist/tools/builtins/unsplash-search.js +74 -0
  182. package/dist/tools/executor.d.ts +23 -0
  183. package/dist/tools/executor.js +169 -0
  184. package/dist/tools/index.d.ts +5 -0
  185. package/dist/tools/index.js +5 -0
  186. package/dist/tools/registry.d.ts +21 -0
  187. package/dist/tools/registry.js +75 -0
  188. package/dist/tools/runtime.d.ts +27 -0
  189. package/dist/tools/runtime.js +48 -0
  190. package/dist/tools/schema-validator.d.ts +24 -0
  191. package/dist/tools/schema-validator.js +88 -0
  192. package/dist/tools/types.d.ts +86 -0
  193. package/dist/tools/types.js +1 -0
  194. package/dist/variation-images.d.ts +19 -0
  195. package/dist/variation-images.js +12 -0
  196. package/package.json +78 -0
@@ -0,0 +1,541 @@
1
+ import { z } from "zod";
2
+ import { allowedBlockTypes, editPlanSchema } from "@avocadostudio-ai/shared";
3
+ import { buildIntentParserSystemPrompt, buildPlannerSystemPrompt } from "./prompts.js";
4
+ import { extractAudienceTarget, fetchImageAsBase64, resolveAttachmentsForLlm, intentSchema } from "../nlp/deterministic-planner.js";
5
+ import { isBatchAddRequest, isBatchRemoveRequest, isBatchReorderRequest, isPageWideRewriteRequest, requestsPlanFirst } from "../nlp/intent-detection.js";
6
+ import { extractJsonObject, normalizePlanCandidate, repairAndParseJsonWithMeta } from "../nlp/plan-normalizer.js";
7
+ import { ZERO_USAGE } from "../telemetry/usage.js";
8
+ import { buildEditPlanJsonSchema, intentJsonSchema } from "./plan-json-schema.js";
9
+ import { executeToolCall } from "../tools/runtime.js";
10
+ import { PlannerError } from "../errors.js";
11
+ import { extractSummaryFromPlanBuffer, extractOpsFromPlanBuffer, extractUpdatePropsFieldDraftsFromPlanBuffer, isChatStrictPrimaryOpMode, isPageWideTranslationRequest, buildPlannerSchemaContext } from "./planner.js";
12
+ import { getGeminiClient } from "../image/image-helpers.js";
13
+ import { GENERATING_IMAGE_PLACEHOLDER, SEARCHING_IMAGE_PLACEHOLDER, DEFERRABLE_IMAGE_TOOLS } from "./chat-pipeline-shared.js";
14
+ const LLM_TIMEOUT_MS = 120_000;
15
+ function raceWithAbort(promise, signal) {
16
+ return new Promise((resolve, reject) => {
17
+ if (signal.aborted) {
18
+ reject(signal.reason);
19
+ return;
20
+ }
21
+ signal.addEventListener("abort", () => reject(signal.reason), { once: true });
22
+ promise.then(resolve, reject);
23
+ });
24
+ }
25
+ // ---------------------------------------------------------------------------
26
+ // Helpers
27
+ // ---------------------------------------------------------------------------
28
+ // Lenient: change_log can be string or array (Gemini returns string per schema,
29
+ // normalizePlanCandidate converts it downstream).
30
+ const rawPlanCandidateSchema = z.looseObject({
31
+ intent: z.enum(["edit_plan", "needs_clarification", "content_answer"]).optional().catch(undefined),
32
+ summary_for_user: z.string().optional(),
33
+ change_log: z.union([z.array(z.string()), z.string()]).optional(),
34
+ ops: z.array(z.record(z.string(), z.unknown())).optional(),
35
+ suggested_next_actions: z.union([z.array(z.string()), z.string()]).optional()
36
+ });
37
+ function toPlannerError(category, message, retryable = false) {
38
+ return new PlannerError(message, { reasonCategory: category, retryable });
39
+ }
40
+ function extractGeminiUsage(response) {
41
+ const meta = response?.usageMetadata;
42
+ if (!meta)
43
+ return { ...ZERO_USAGE };
44
+ const inputTokens = typeof meta.promptTokenCount === "number" ? meta.promptTokenCount : 0;
45
+ const outputTokens = typeof meta.candidatesTokenCount === "number" ? meta.candidatesTokenCount : 0;
46
+ return { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens };
47
+ }
48
+ function extractGeminiText(response) {
49
+ const candidates = response?.candidates;
50
+ if (!Array.isArray(candidates) || candidates.length === 0)
51
+ return "";
52
+ const parts = candidates[0]?.content?.parts;
53
+ if (!Array.isArray(parts))
54
+ return "";
55
+ return parts.map((p) => p.text ?? "").join("");
56
+ }
57
+ function extractGeminiFunctionCalls(response) {
58
+ const candidates = response?.candidates;
59
+ if (!Array.isArray(candidates) || candidates.length === 0)
60
+ return [];
61
+ const parts = candidates[0]?.content?.parts;
62
+ if (!Array.isArray(parts))
63
+ return [];
64
+ return parts
65
+ .filter((p) => !!p.functionCall)
66
+ .map((p) => ({ name: p.functionCall.name, args: p.functionCall.args }));
67
+ }
68
+ function toGeminiHistory(history) {
69
+ // Drop blank turns — an empty/whitespace text part is invalid and can fail the
70
+ // request, so one stray turn must not break the whole plan (#33, mirrors Anthropic).
71
+ return history
72
+ .filter((h) => typeof h.content === "string" && h.content.trim().length > 0)
73
+ .map((h) => ({
74
+ role: h.role === "assistant" ? "model" : "user",
75
+ parts: [{ text: h.content }]
76
+ }));
77
+ }
78
+ /** Relaxed safety settings — users are editing their own site content */
79
+ const PERMISSIVE_SAFETY_SETTINGS = [
80
+ { category: "HARM_CATEGORY_HARASSMENT", threshold: "BLOCK_ONLY_HIGH" },
81
+ { category: "HARM_CATEGORY_HATE_SPEECH", threshold: "BLOCK_ONLY_HIGH" },
82
+ { category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", threshold: "BLOCK_ONLY_HIGH" },
83
+ { category: "HARM_CATEGORY_DANGEROUS_CONTENT", threshold: "BLOCK_ONLY_HIGH" },
84
+ ];
85
+ // ---------------------------------------------------------------------------
86
+ // Intent parsing
87
+ // ---------------------------------------------------------------------------
88
+ export async function parseIntentWithGemini(args) {
89
+ const ai = await getGeminiClient();
90
+ const system = buildIntentParserSystemPrompt();
91
+ const user = {
92
+ request: args.message,
93
+ slug: args.slug,
94
+ activeBlockId: args.activeBlockId ?? null,
95
+ activeBlockType: args.activeBlockType ?? null,
96
+ activeEditablePath: args.activeEditablePath ?? null,
97
+ availableBlockTypes: allowedBlockTypes,
98
+ blocks: args.currentPage.blocks.map((b) => ({ id: b.id, type: b.type, props: Object.keys(b.props) }))
99
+ };
100
+ const response = await raceWithAbort(ai.models.generateContent({
101
+ model: args.model,
102
+ contents: JSON.stringify(user),
103
+ config: {
104
+ systemInstruction: system,
105
+ responseMimeType: "application/json",
106
+ responseJsonSchema: intentJsonSchema,
107
+ temperature: 0,
108
+ safetySettings: PERMISSIVE_SAFETY_SETTINGS,
109
+ }
110
+ }), AbortSignal.timeout(LLM_TIMEOUT_MS));
111
+ const raw = extractGeminiText(response);
112
+ if (!raw.trim())
113
+ throw new Error("Intent parser did not return JSON");
114
+ const normalized = JSON.parse(raw);
115
+ // Nulls from the schema become undefined for Zod optional fields
116
+ for (const key of Object.keys(normalized)) {
117
+ if (normalized[key] === null)
118
+ delete normalized[key];
119
+ }
120
+ const parsed = intentSchema.safeParse(normalized);
121
+ if (!parsed.success) {
122
+ const issue = parsed.error.issues[0];
123
+ const detail = issue?.message ?? "Invalid intent parser output";
124
+ const at = issue?.path?.length ? ` at ${issue.path.join(".")}` : "";
125
+ throw new Error(`${detail}${at}`);
126
+ }
127
+ return parsed.data;
128
+ }
129
+ // ---------------------------------------------------------------------------
130
+ // Plan generation
131
+ // ---------------------------------------------------------------------------
132
+ export async function generatePlanWithGemini(args) {
133
+ const ai = await getGeminiClient();
134
+ const effectiveSignal = args.signal
135
+ ? AbortSignal.any([args.signal, AbortSignal.timeout(LLM_TIMEOUT_MS)])
136
+ : AbortSignal.timeout(LLM_TIMEOUT_MS);
137
+ const effectiveBlockTypes = args.componentsManifest ? args.componentsManifest.blocks.map(c => c.type) : allowedBlockTypes;
138
+ const batchOverride = isBatchAddRequest(args.message) || isBatchRemoveRequest(args.message) || isBatchReorderRequest(args.message) || isPageWideRewriteRequest(args.message);
139
+ const pageWideRewrite = isPageWideRewriteRequest(args.message);
140
+ const planFirstMode = requestsPlanFirst(args.message);
141
+ // In plan-first mode the intent enum is narrowed so the model can't bail to
142
+ // needs_clarification — it must produce a reviewable plan (see buildEditPlanJsonSchema).
143
+ const requestEditPlanSchema = buildEditPlanJsonSchema({ planFirstMode });
144
+ const requestGeminiEditPlanSchema = {
145
+ ...requestEditPlanSchema,
146
+ propertyOrdering: ["intent", "summary_for_user", "change_log", "ops", "suggested_next_actions"]
147
+ };
148
+ const pageWideTranslation = isPageWideTranslationRequest(args.message);
149
+ const chatStrictPrimaryOpMode = isChatStrictPrimaryOpMode() && !batchOverride && !pageWideTranslation;
150
+ const selectedBlockId = String(args.contextPack.selected.blockId ?? "");
151
+ const audienceHint = extractAudienceTarget(args.message);
152
+ const explicitOtherReference = selectedBlockId.length > 0 &&
153
+ Array.isArray(args.contextPack.resolvedReferences.mentionedBlocks) &&
154
+ args.contextPack.resolvedReferences.mentionedBlocks.some((entry) => entry && typeof entry === "object" && "id" in entry && entry.id !== selectedBlockId);
155
+ const system = buildPlannerSystemPrompt({
156
+ provider: "gemini",
157
+ lightweight: !!args.lightweight,
158
+ selectedBlockId,
159
+ explicitOtherReference: !!explicitOtherReference,
160
+ chatStrictPrimaryOpMode,
161
+ pageWideTranslation,
162
+ pageWideRewrite,
163
+ planFirstMode,
164
+ effectiveBlockTypes,
165
+ siteContextBlock: args.siteContextBlock,
166
+ imageUrlForVision: args.contextPack.selected?.imageUrlForVision,
167
+ editablePath: args.contextPack.selected?.editablePath,
168
+ blockId: args.contextPack.selected?.blockId,
169
+ locale: args.locale,
170
+ imageSourceChoiceOpen: args.imageSourceChoiceOpen,
171
+ attachmentCount: args.contextPack.attachments?.length,
172
+ });
173
+ const includeContracts = !args.lightweight && (batchOverride ||
174
+ pageWideTranslation ||
175
+ /\b(create|add|insert|build|generate)\b/.test(args.message.toLowerCase()) ||
176
+ /\b(seo|meta|metadata|og\s*image|open\s*graph|description|structured\s*data|schema\.org)\b/.test(args.message.toLowerCase()) ||
177
+ /\d{2,3}\s*char/i.test(args.message));
178
+ const schemaContext = args.lightweight
179
+ ? {
180
+ payload: {},
181
+ meta: {
182
+ contractMode: "minimal",
183
+ contractBytes: 0,
184
+ contractBlockCount: 0,
185
+ targetBlockTypes: [],
186
+ strictJsonEnabled: false
187
+ }
188
+ }
189
+ : buildPlannerSchemaContext({
190
+ message: args.message,
191
+ contextPack: args.contextPack,
192
+ batchOverride,
193
+ pageWideTranslation,
194
+ legacyIncludeContracts: includeContracts,
195
+ forceFullContracts: args.forceFullSchemaContracts,
196
+ componentsManifest: args.componentsManifest
197
+ });
198
+ const userPayload = {
199
+ request: args.message,
200
+ audienceHint: audienceHint ?? null,
201
+ slug: args.slug,
202
+ contextPack: args.contextPack,
203
+ ...schemaContext.payload,
204
+ feedback: args.feedback ?? null
205
+ };
206
+ // Vision support
207
+ const imageUrlForVision = typeof args.contextPack.selected?.imageUrlForVision === "string"
208
+ ? args.contextPack.selected.imageUrlForVision
209
+ : null;
210
+ const imageBase64 = imageUrlForVision ? await fetchImageAsBase64(imageUrlForVision) : null;
211
+ // User-attached files (images + PDFs) as inline data — Gemini reads both natively.
212
+ const resolvedAttachments = await resolveAttachmentsForLlm(args.contextPack.attachments);
213
+ // Build user content parts
214
+ const userParts = [];
215
+ if (imageBase64) {
216
+ userParts.push({ inlineData: { mimeType: imageBase64.mediaType, data: imageBase64.base64 } });
217
+ }
218
+ for (const attachment of resolvedAttachments) {
219
+ userParts.push({ inlineData: { mimeType: attachment.mediaType, data: attachment.base64 } });
220
+ }
221
+ userParts.push({ text: JSON.stringify(userPayload) });
222
+ // Build contents array (history + current message)
223
+ const contents = [
224
+ ...toGeminiHistory(args.history ?? []),
225
+ { role: "user", parts: userParts }
226
+ ];
227
+ // -----------------------------------------------------------------------
228
+ // Tool runtime path — function calling with agentic loop
229
+ // -----------------------------------------------------------------------
230
+ const hasRuntimeTools = args.toolRuntime && args.toolCallContext;
231
+ const deferredNativeImageCalls = [];
232
+ if (hasRuntimeTools) {
233
+ const raw = await runToolLoop({
234
+ ai,
235
+ model: args.model,
236
+ system,
237
+ contents,
238
+ toolRuntime: args.toolRuntime,
239
+ toolCallContext: args.toolCallContext,
240
+ planSchema: requestEditPlanSchema,
241
+ onToken: args.onToken,
242
+ onToolExecution: args.onToolExecution,
243
+ onStatusUpdate: args.onStatusUpdate,
244
+ onImageProgress: args.onImageProgress,
245
+ deferredNativeImageCalls,
246
+ log: args.log,
247
+ signal: effectiveSignal,
248
+ });
249
+ const result = parseAndValidatePlan(raw, args, chatStrictPrimaryOpMode, schemaContext);
250
+ return {
251
+ ...result,
252
+ deferredNativeImageCalls: deferredNativeImageCalls.length > 0 ? deferredNativeImageCalls : undefined
253
+ };
254
+ }
255
+ // -----------------------------------------------------------------------
256
+ // JSON mode path — streaming or non-streaming (like OpenAI planner)
257
+ // -----------------------------------------------------------------------
258
+ let raw = "";
259
+ let usage = { ...ZERO_USAGE };
260
+ if (args.onToken) {
261
+ // Streaming path
262
+ let streamedOpsCount = 0;
263
+ let lastSummaryLen = 0;
264
+ let emittedChangeLogCount = 0;
265
+ const emittedFieldDraftByKey = new Map();
266
+ const stream = await raceWithAbort(ai.models.generateContentStream({
267
+ model: args.model,
268
+ contents,
269
+ config: {
270
+ systemInstruction: system,
271
+ responseMimeType: "application/json",
272
+ responseJsonSchema: requestGeminiEditPlanSchema,
273
+ temperature: 0,
274
+ safetySettings: PERMISSIVE_SAFETY_SETTINGS,
275
+ }
276
+ }), effectiveSignal);
277
+ for await (const chunk of stream) {
278
+ const text = extractGeminiText(chunk);
279
+ if (!text)
280
+ continue;
281
+ raw += text;
282
+ args.onToken(text);
283
+ // Progressive field draft extraction
284
+ if (args.onFieldDraft) {
285
+ const fieldDrafts = extractUpdatePropsFieldDraftsFromPlanBuffer(raw);
286
+ for (const draft of fieldDrafts) {
287
+ const key = `${draft.opIndex}:${draft.blockId}:${draft.editablePath}`;
288
+ const prev = emittedFieldDraftByKey.get(key);
289
+ if (prev === draft.value)
290
+ continue;
291
+ emittedFieldDraftByKey.set(key, draft.value);
292
+ args.onFieldDraft({ blockId: draft.blockId, editablePath: draft.editablePath, value: draft.value, ...(draft.pageSlug ? { pageSlug: draft.pageSlug } : {}) });
293
+ }
294
+ }
295
+ // Progressive summary + change log extraction
296
+ if (args.onSummaryChunk || args.onChangeLogEntry) {
297
+ const extracted = extractSummaryFromPlanBuffer(raw);
298
+ if (extracted.summary && extracted.summary.length > lastSummaryLen) {
299
+ args.onSummaryChunk?.(extracted.summary.slice(lastSummaryLen));
300
+ lastSummaryLen = extracted.summary.length;
301
+ }
302
+ for (let i = emittedChangeLogCount; i < extracted.changeLog.length; i++) {
303
+ args.onChangeLogEntry?.(extracted.changeLog[i]);
304
+ }
305
+ emittedChangeLogCount = extracted.changeLog.length;
306
+ }
307
+ // Progressive ops extraction
308
+ if (args.onPlannedOp) {
309
+ const next = extractOpsFromPlanBuffer(raw, streamedOpsCount);
310
+ streamedOpsCount = next.nextEmittedCount;
311
+ for (let idx = 0; idx < next.newOps.length; idx += 1) {
312
+ args.onPlannedOp(next.newOps[idx], streamedOpsCount - next.newOps.length + idx + 1);
313
+ }
314
+ }
315
+ // Extract usage from last chunk
316
+ const chunkUsage = extractGeminiUsage(chunk);
317
+ if (chunkUsage.totalTokens > 0)
318
+ usage = chunkUsage;
319
+ }
320
+ if (raw.trim().length === 0) {
321
+ throw toPlannerError("incomplete_output", "Model returned no planning output");
322
+ }
323
+ }
324
+ else {
325
+ // Non-streaming path
326
+ const response = await raceWithAbort(ai.models.generateContent({
327
+ model: args.model,
328
+ contents,
329
+ config: {
330
+ systemInstruction: system,
331
+ responseMimeType: "application/json",
332
+ responseJsonSchema: requestGeminiEditPlanSchema,
333
+ temperature: 0,
334
+ safetySettings: PERMISSIVE_SAFETY_SETTINGS,
335
+ }
336
+ }), effectiveSignal);
337
+ raw = extractGeminiText(response);
338
+ usage = extractGeminiUsage(response);
339
+ if (raw.trim().length === 0) {
340
+ throw toPlannerError("incomplete_output", "Model returned no planning output");
341
+ }
342
+ }
343
+ return parseAndValidatePlan(raw, args, chatStrictPrimaryOpMode, schemaContext, usage);
344
+ }
345
+ // ---------------------------------------------------------------------------
346
+ // Tool loop — agentic function calling for image gen, unsplash, etc.
347
+ // ---------------------------------------------------------------------------
348
+ const MAX_TOOL_TURNS = 6;
349
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
350
+ async function runToolLoop(args) {
351
+ // Build function declarations from tool registry + submit_edit_plan
352
+ const runtimeManifests = args.toolRuntime.registry.listManifests();
353
+ const functionDeclarations = [
354
+ {
355
+ name: "submit_edit_plan",
356
+ description: "Submit the structured EditPlan JSON.",
357
+ parameters: args.planSchema
358
+ },
359
+ ...runtimeManifests.map((m) => ({
360
+ name: m.name.replace(/\./g, "_"), // Gemini requires alphanumeric + underscore
361
+ description: m.description,
362
+ parameters: m.inputSchema
363
+ }))
364
+ ];
365
+ // Map aliased names back to runtime names
366
+ const aliasToRuntimeName = new Map();
367
+ for (const m of runtimeManifests) {
368
+ aliasToRuntimeName.set(m.name.replace(/\./g, "_"), m.name);
369
+ }
370
+ const messages = [...args.contents];
371
+ let submitPlanJson = "";
372
+ for (let turn = 0; turn < MAX_TOOL_TURNS; turn++) {
373
+ const callSignal = args.signal ?? AbortSignal.timeout(LLM_TIMEOUT_MS);
374
+ const response = await raceWithAbort(args.ai.models.generateContent({
375
+ model: args.model,
376
+ contents: messages,
377
+ config: {
378
+ systemInstruction: args.system,
379
+ temperature: 0,
380
+ safetySettings: PERMISSIVE_SAFETY_SETTINGS,
381
+ tools: [{ functionDeclarations }],
382
+ toolConfig: { functionCallingConfig: { mode: "ANY" } },
383
+ }
384
+ }), callSignal);
385
+ const text = extractGeminiText(response);
386
+ if (text && args.onToken)
387
+ args.onToken(text);
388
+ const functionCalls = extractGeminiFunctionCalls(response);
389
+ if (functionCalls.length === 0) {
390
+ // Model returned text instead of function call — treat as plan JSON
391
+ return text || submitPlanJson;
392
+ }
393
+ // Process function calls
394
+ const toolResults = [];
395
+ for (const fc of functionCalls) {
396
+ if (fc.name === "submit_edit_plan") {
397
+ submitPlanJson = JSON.stringify(fc.args);
398
+ toolResults.push({
399
+ functionResponse: { name: fc.name, response: { ok: true } }
400
+ });
401
+ continue;
402
+ }
403
+ const runtimeName = aliasToRuntimeName.get(fc.name) ?? fc.name;
404
+ const input = fc.args ?? {};
405
+ // Defer image tools
406
+ if (DEFERRABLE_IMAGE_TOOLS.has(runtimeName)) {
407
+ const placeholder = runtimeName === "image.generate"
408
+ ? GENERATING_IMAGE_PLACEHOLDER
409
+ : SEARCHING_IMAGE_PLACEHOLDER;
410
+ const placeholderData = runtimeName === "image.generate"
411
+ ? { imageUrl: placeholder, alt: String(input.prompt ?? "Generating image…"), width: 768, height: 512 }
412
+ : { items: [{ id: "placeholder", imageUrl: placeholder, thumbUrl: placeholder, alt: String(input.query ?? "Searching…"), author: "Placeholder", sourceUrl: "" }] };
413
+ args.deferredNativeImageCalls.push({
414
+ toolName: runtimeName,
415
+ input,
416
+ placeholderUrl: placeholder
417
+ });
418
+ toolResults.push({
419
+ functionResponse: { name: fc.name, response: placeholderData }
420
+ });
421
+ args.onToolExecution?.({
422
+ toolName: runtimeName,
423
+ ok: true,
424
+ latencyMs: 0,
425
+ attempts: 0,
426
+ traceId: args.toolCallContext.traceId,
427
+ sessionId: args.toolCallContext.sessionId,
428
+ siteId: args.toolCallContext.siteId,
429
+ plannerProvider: "gemini",
430
+ deferred: true
431
+ });
432
+ continue;
433
+ }
434
+ // Execute runtime tool
435
+ const startMs = Date.now();
436
+ const result = await executeToolCall({
437
+ runtime: args.toolRuntime,
438
+ toolName: runtimeName,
439
+ input,
440
+ context: {
441
+ siteId: args.toolCallContext.siteId,
442
+ sessionId: args.toolCallContext.sessionId,
443
+ userId: args.toolCallContext.userId,
444
+ traceId: args.toolCallContext.traceId,
445
+ plannerProvider: "gemini",
446
+ gdriveFolderId: args.toolCallContext.gdriveFolderId,
447
+ onStatusUpdate: args.onStatusUpdate,
448
+ onImageProgress: args.onImageProgress,
449
+ },
450
+ policy: args.toolRuntime.defaultPolicy
451
+ });
452
+ const latencyMs = Date.now() - startMs;
453
+ args.onToolExecution?.({
454
+ toolName: runtimeName,
455
+ ok: result.ok,
456
+ latencyMs,
457
+ attempts: result.attempts,
458
+ traceId: args.toolCallContext.traceId,
459
+ sessionId: args.toolCallContext.sessionId,
460
+ siteId: args.toolCallContext.siteId,
461
+ plannerProvider: "gemini",
462
+ ...(result.ok ? {} : { errorCode: result.error?.code })
463
+ });
464
+ toolResults.push({
465
+ functionResponse: {
466
+ name: fc.name,
467
+ response: result.ok
468
+ ? (typeof result.data === "object" && result.data !== null ? result.data : { result: result.data })
469
+ : { error: String(result.error?.message ?? "Tool execution failed") }
470
+ }
471
+ });
472
+ }
473
+ // If we got submit_edit_plan, we're done
474
+ if (submitPlanJson)
475
+ return submitPlanJson;
476
+ // Add model response + tool results to conversation
477
+ const modelParts = functionCalls.map((fc) => ({ functionCall: { name: fc.name, args: fc.args } }));
478
+ messages.push({ role: "model", parts: modelParts });
479
+ messages.push({ role: "user", parts: toolResults });
480
+ }
481
+ // Exhausted max turns — return whatever we have
482
+ if (submitPlanJson)
483
+ return submitPlanJson;
484
+ throw toPlannerError("incomplete_output", `Gemini tool loop exhausted ${MAX_TOOL_TURNS} turns without submitting a plan`);
485
+ }
486
+ // ---------------------------------------------------------------------------
487
+ // Shared parse + validate
488
+ // ---------------------------------------------------------------------------
489
+ function parseAndValidatePlan(raw, args, chatStrictPrimaryOpMode, schemaContext, usage = { ...ZERO_USAGE }) {
490
+ const jsonText = extractJsonObject(raw) ?? raw;
491
+ if (!jsonText.trim()) {
492
+ throw toPlannerError("malformed_output", "Model did not return JSON", true);
493
+ }
494
+ let parsedJson;
495
+ try {
496
+ parsedJson = JSON.parse(jsonText);
497
+ }
498
+ catch (err) {
499
+ try {
500
+ const meta = repairAndParseJsonWithMeta(jsonText);
501
+ parsedJson = meta.parsed;
502
+ if (meta.strategy !== "none") {
503
+ console.warn(`[gemini-planner] JSON repaired via ${meta.strategy}${meta.mutationCount ? ` (${meta.mutationCount} mutations)` : ""}${meta.discardedBytes ? ` (${meta.discardedBytes}B discarded)` : ""}`);
504
+ }
505
+ }
506
+ catch {
507
+ const posMatch = err instanceof SyntaxError ? /position (\d+)/i.exec(err.message) : null;
508
+ const pos = posMatch?.[1];
509
+ const snippet = pos ? jsonText.slice(Math.max(0, Number(pos) - 30), Number(pos) + 30) : jsonText.slice(0, 200);
510
+ throw toPlannerError("malformed_output", `Model returned malformed JSON: ${err.message}. Near: …${snippet}…`, true);
511
+ }
512
+ }
513
+ const rawCandidateResult = rawPlanCandidateSchema.safeParse(parsedJson);
514
+ if (!rawCandidateResult.success) {
515
+ const first = rawCandidateResult.error.issues[0];
516
+ const at = first?.path?.length ? ` at ${first.path.join(".")}` : "";
517
+ const detail = first?.message ?? "Raw planner output shape is invalid";
518
+ throw toPlannerError("malformed_output", `${detail}${at}`, true);
519
+ }
520
+ const parsed = normalizePlanCandidate(rawCandidateResult.data, {
521
+ defaultSlug: args.slug,
522
+ currentPage: args.currentPage,
523
+ userMessage: args.message
524
+ });
525
+ const planResult = editPlanSchema.safeParse(parsed);
526
+ if (!planResult.success) {
527
+ const first = planResult.error.issues[0];
528
+ const message = first?.message ?? "Invalid model output";
529
+ const path = first?.path?.length ? ` at ${first.path.join(".")}` : "";
530
+ const sample = JSON.stringify(parsed).slice(0, 700);
531
+ throw toPlannerError("schema_violation", `${message}${path}. Parsed sample: ${sample}`, true);
532
+ }
533
+ if (chatStrictPrimaryOpMode && planResult.data.intent === "edit_plan" && planResult.data.ops.length > 1) {
534
+ return {
535
+ plan: { ...planResult.data, ops: [planResult.data.ops[0]] },
536
+ usage,
537
+ schemaContext: schemaContext.meta
538
+ };
539
+ }
540
+ return { plan: planResult.data, usage, schemaContext: schemaContext.meta };
541
+ }
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Post-plan hallucination validator.
3
+ *
4
+ * Catches planner-generated `update_props` ops whose patches contain keys
5
+ * that aren't in the target block's schema — typically when the planner
6
+ * promises visual changes (colors, animations, gradients) that the block
7
+ * doesn't support. Strips the unsupported keys, appends a user-facing note
8
+ * to `summary_for_user`, and emits a telemetry signal for later analysis.
9
+ *
10
+ * Complements the STRICT SCHEMA DISCIPLINE prompt rule: defense in depth
11
+ * against LLMs that paper over schema limits in their summary text.
12
+ */
13
+ import { type EditPlan, type PageDoc } from "@avocadostudio-ai/shared";
14
+ export type HallucinatedProp = {
15
+ blockId: string;
16
+ blockType: string;
17
+ propName: string;
18
+ };
19
+ export type HallucinationValidationResult = {
20
+ plan: EditPlan;
21
+ hallucinatedProps: HallucinatedProp[];
22
+ };
23
+ /**
24
+ * Scan every `update_props` op in the plan, strip patch keys that are not
25
+ * in the block's schema, collect the removed (blockType, propName) pairs,
26
+ * and append a single merged note to `summary_for_user` when anything was
27
+ * stripped.
28
+ *
29
+ * Mutates `plan` in place (strips keys, appends the summary note, appends
30
+ * change_log entries) and returns the updated plan alongside the list of
31
+ * stripped props for logging.
32
+ */
33
+ export declare function validateAndStripHallucinatedProps(args: {
34
+ plan: EditPlan;
35
+ draft: Map<string, PageDoc>;
36
+ }): HallucinationValidationResult;
@@ -0,0 +1,110 @@
1
+ /**
2
+ * Post-plan hallucination validator.
3
+ *
4
+ * Catches planner-generated `update_props` ops whose patches contain keys
5
+ * that aren't in the target block's schema — typically when the planner
6
+ * promises visual changes (colors, animations, gradients) that the block
7
+ * doesn't support. Strips the unsupported keys, appends a user-facing note
8
+ * to `summary_for_user`, and emits a telemetry signal for later analysis.
9
+ *
10
+ * Complements the STRICT SCHEMA DISCIPLINE prompt rule: defense in depth
11
+ * against LLMs that paper over schema limits in their summary text.
12
+ */
13
+ import { blockSchemas, getBlockMeta } from "@avocadostudio-ai/shared";
14
+ /**
15
+ * Return the set of top-level prop keys allowed by a block type's registered
16
+ * schema. Returns null when the block type is unregistered (e.g. custom
17
+ * blocks from an external manifest) — in that case we cannot validate and
18
+ * skip the check.
19
+ */
20
+ function allowedPropKeysForBlockType(blockType) {
21
+ const schema = blockSchemas[blockType];
22
+ if (!schema)
23
+ return null;
24
+ const shape = schema.shape;
25
+ if (!shape || typeof shape !== "object")
26
+ return null;
27
+ return new Set(Object.keys(shape));
28
+ }
29
+ /**
30
+ * Look up the current block by id across all pages in the draft. Preferring
31
+ * the targeted page when supplied avoids cross-page collisions on id reuse.
32
+ */
33
+ function findBlockType(args) {
34
+ const { blockId, pageSlug, draft } = args;
35
+ if (pageSlug) {
36
+ const page = draft.get(pageSlug);
37
+ const hit = page?.blocks.find((b) => b.id === blockId);
38
+ if (hit)
39
+ return hit.type;
40
+ }
41
+ for (const page of draft.values()) {
42
+ const hit = page.blocks.find((b) => b.id === blockId);
43
+ if (hit)
44
+ return hit.type;
45
+ }
46
+ return undefined;
47
+ }
48
+ function humanBlockName(blockType) {
49
+ const meta = getBlockMeta(blockType);
50
+ return meta?.displayName ?? blockType;
51
+ }
52
+ /**
53
+ * Scan every `update_props` op in the plan, strip patch keys that are not
54
+ * in the block's schema, collect the removed (blockType, propName) pairs,
55
+ * and append a single merged note to `summary_for_user` when anything was
56
+ * stripped.
57
+ *
58
+ * Mutates `plan` in place (strips keys, appends the summary note, appends
59
+ * change_log entries) and returns the updated plan alongside the list of
60
+ * stripped props for logging.
61
+ */
62
+ export function validateAndStripHallucinatedProps(args) {
63
+ const { plan, draft } = args;
64
+ const hallucinatedProps = [];
65
+ for (const op of plan.ops) {
66
+ if (op.op !== "update_props")
67
+ continue;
68
+ const blockType = findBlockType({ blockId: op.blockId, pageSlug: op.pageSlug, draft });
69
+ if (!blockType)
70
+ continue;
71
+ const allowedKeys = allowedPropKeysForBlockType(blockType);
72
+ if (!allowedKeys)
73
+ continue;
74
+ const rawPatch = op.patch;
75
+ if (!rawPatch || typeof rawPatch !== "object")
76
+ continue;
77
+ const patchCandidate = rawPatch.props && typeof rawPatch.props === "object" && !Array.isArray(rawPatch.props)
78
+ ? rawPatch.props
79
+ : rawPatch;
80
+ for (const key of Object.keys(patchCandidate)) {
81
+ if (allowedKeys.has(key))
82
+ continue;
83
+ delete patchCandidate[key];
84
+ hallucinatedProps.push({ blockId: op.blockId, blockType, propName: key });
85
+ }
86
+ }
87
+ if (hallucinatedProps.length > 0) {
88
+ // Merge duplicates into a single readable note keyed by blockType. We
89
+ // intentionally avoid echoing the raw prop name back to the user —
90
+ // doing so is (a) jargon-y (users don't think in prop keys), and (b)
91
+ // makes the note trivially collide with eval banned-word checks that
92
+ // try to prove the planner didn't promise the unsupported behavior.
93
+ const byBlockType = new Map();
94
+ for (const entry of hallucinatedProps) {
95
+ const bucket = byBlockType.get(entry.blockType) ?? new Set();
96
+ bucket.add(entry.propName);
97
+ byBlockType.set(entry.blockType, bucket);
98
+ }
99
+ const noteParts = [];
100
+ for (const [blockType] of byBlockType) {
101
+ const name = humanBlockName(blockType);
102
+ noteParts.push(`Some requested styling isn't available on the ${name} block — applied the supported parts.`);
103
+ }
104
+ const note = noteParts.join(" ");
105
+ const summary = plan.summary_for_user?.trimEnd() ?? "";
106
+ plan.summary_for_user = summary.length > 0 ? `${summary}\n\n${note}` : note;
107
+ plan.change_log = [...plan.change_log, note];
108
+ }
109
+ return { plan, hallucinatedProps };
110
+ }