@paroicms/site-generator-plugin 0.9.0 → 0.10.0

package/gen-backend/dist/generator/fake-content-generator.ts/content-report.js

@@ -1,11 +1,15 @@
 export function createGeneratedContentReport() {
-    let contentCount = 0;
+    let totalEntryCount = 0;
+    const llmReports = [];
     return {
-        getContentCount() {
-            return contentCount;
+        getResults() {
+            return { entryCount: totalEntryCount, llmReports };
         },
-        addContentCount(count) {
-            contentCount += count;
+        add(entryCount, llmReport) {
+            totalEntryCount += entryCount;
+            if (llmReport) {
+                llmReports.push(llmReport);
+            }
         },
     };
 }

package/gen-backend/dist/generator/fake-content-generator.ts/create-database-with-fake-content.js

@@ -1,12 +1,13 @@
 import { getPartTypeByName, getRegularDocumentTypeByName, getRoutingDocumentTypeByName, } from "@paroicms/internal-anywhere-lib";
 import { createSimpleTranslator, } from "@paroicms/public-server-lib";
+import { updateGeneratedSiteStepSetAsCompleted, } from "../../db/db-write.queries.js";
 import { getRandomImagePath } from "../lib/images-lib.js";
 import { createTaskCollector } from "../lib/tasks.js";
 import { dedupMessages } from "./content-helpers.js";
 import { createGeneratedContentReport } from "./content-report.js";
 import { generateLocalizedFooterMention } from "./create-node-contents.js";
 import { generateFieldSetContent, generateMultipleFieldSetContents, } from "./generate-fake-content.js";
-export async function fillSiteWithFakeContent(ctx, { regSite, siteTitle }) {
+export async function fillSiteWithFakeContent(ctx, stepHandle, { regSite, localizedValues }) {
     const { service } = ctx;
     const { fqdn } = regSite;
     const report = createGeneratedContentReport();
@@ -15,7 +16,7 @@ export async function fillSiteWithFakeContent(ctx, { regSite, siteTitle }) {
         labels: siteSchema.l10n,
         logger: ctx.logger,
     });
-    await updateSiteFields(ctx, report, { fqdn, siteSchema, siteTitle });
+    await updateSiteFields(ctx, report, { fqdn, siteSchema, siteTitle: localizedValues.siteTitle });
     const tasks = createTaskCollector(ctx);
     fillRoutingDocumentAndAddChildren(ctx, tasks, report, {
         fqdn,
@@ -28,10 +29,17 @@ export async function fillSiteWithFakeContent(ctx, { regSite, siteTitle }) {
     const { promise } = tasks.runAll({ maxParallel: 10, rateLimitPerSecond: 3 });
     const { doneCount, errorMessages } = await promise;
     if (errorMessages.length > 0) {
-        ctx.logger.warn(`Failed to generate ${errorMessages.length} documents:\n  - ${errorMessages.join("\n  - ")}`);
+        ctx.logger.warn(`Failed to generate documents:\n  - ${errorMessages.join("\n  - ")}`);
     }
     ctx.logger.debug(`… Executed ${doneCount} generating tasks`);
-    return report;
+    const results = report.getResults();
+    await updateGeneratedSiteStepSetAsCompleted(ctx, stepHandle, {
+        status: "completed",
+        contentEntryCount: results.entryCount,
+        contentInputTokenCount: results.llmReports.reduce((acc, r) => acc + r.inputTokenCount, 0),
+        contentOutputTokenCount: results.llmReports.reduce((acc, r) => acc + (r.outputTokenCount ?? 0), 0),
+        contentErrors: errorMessages.length > 0 ? errorMessages.join("\n  - ") : null,
+    });
 }
 function fillRoutingDocumentAndAddChildren(ctx, tasks, report, siteOptions, nodeOptions) {
     const { routingIds, nodeType } = nodeOptions;
@@ -137,13 +145,12 @@ async function updateRoutingDocument(ctx, report, siteOptions, nodeOptions) {
         siteSchema,
         schemaI18n,
         withTitle: false,
-        debugName: nodeType.kebabName,
-    });
+        llmTaskName: nodeType.kebabName,
+    }, report);
     await ctx.service.connector.updateDocumentContent(fqdn, {
         nodeId: routingIds.nodeId,
         content: toRiDocumentContent(content, nodeType),
     });
-    report.addContentCount(1);
 }
 async function addRegularDocuments(ctx, report, siteOptions, nodeOptions) {
     ctx.logger.debug(`[TASK] Adding regular documents "${nodeOptions.nodeType.typeName}"…`);
@@ -160,8 +167,8 @@ async function addRegularDocuments(ctx, report, siteOptions, nodeOptions) {
         count: getDefaultNodeContentCount(nodeType),
         withTitle: true,
         tolerateErrors,
-        debugName: nodeType.kebabName,
-    });
+        llmTaskName: nodeType.kebabName,
+    }, report);
     const errorMessages = dedupMessages(tolerateErrors.errorMessages);
     if (errorMessages.length > 0) {
         ctx.logger.warn(`Error generating content for ${nodeType.typeName}:\n  - ${errorMessages.join("\n  - ")}`);
@@ -170,7 +177,6 @@ async function addRegularDocuments(ctx, report, siteOptions, nodeOptions) {
         parentNodeId,
         contents: list.map((content) => toRiDocumentContent(content, nodeType)),
     });
-    report.addContentCount(list.length);
 }
 async function addParts(ctx, report, siteOptions, nodeOptions) {
     ctx.logger.debug(`[TASK] Adding parts "${nodeOptions.nodeType.typeName}"…`);
@@ -187,8 +193,8 @@ async function addParts(ctx, report, siteOptions, nodeOptions) {
         count: getDefaultNodeContentCount(nodeType),
         withTitle: true,
         tolerateErrors,
-        debugName: nodeType.kebabName,
-    });
+        llmTaskName: nodeType.kebabName,
+    }, report);
     const errorMessages = dedupMessages(tolerateErrors.errorMessages);
     if (errorMessages.length > 0) {
         ctx.logger.warn(`Error generating content for ${nodeType.typeName}:\n  - ${errorMessages.join("\n  - ")}`);
@@ -197,7 +203,6 @@ async function addParts(ctx, report, siteOptions, nodeOptions) {
         parentNodeId,
         contents: list.map((content) => toRiPartContent(content, nodeType)),
     });
-    report.addContentCount(list.length);
 }
 function toRiDocumentContent(content, nodeType) {
     const { title, fields, featuredImage } = content;

package/gen-backend/dist/generator/fake-content-generator.ts/generate-fake-content.js

@@ -1,17 +1,17 @@
 import { camelToKebabCase, camelToTitleCase } from "../lib/utils.js";
 import { createNodeContents } from "./create-node-contents.js";
 import { invokeGenerateFakeContent, } from "./invoke-generate-fake-content.js";
-export async function generateFieldSetContent(ctx, options) {
+export async function generateFieldSetContent(ctx, options, report) {
     const list = await generateMultipleFieldSetContents(ctx, {
         ...options,
         count: 1,
-    });
+    }, report);
     if (list.length !== 1)
         throw new Error(`Expected one item, got ${list.length}`);
     return list[0];
 }
-export async function generateMultipleFieldSetContents(ctx, options) {
-    const { siteSchema, nodeType, documentType, schemaI18n, count, withTitle, tolerateErrors, debugName, } = options;
+export async function generateMultipleFieldSetContents(ctx, options, report) {
+    const { siteSchema, nodeType, documentType, schemaI18n, count, withTitle, tolerateErrors, llmTaskName, } = options;
     if (nodeType.kind === "site")
         throw new Error("Cannot generate content for site node type");
     // for a document, the LLM is best at generating the title, so we ask for it and remove it later
@@ -40,7 +40,7 @@ export async function generateMultipleFieldSetContents(ctx, options) {
         defaultLanguage,
     });
     const language = defaultLanguage ?? "en";
-    let generatedContents = outputTags.length > 0
+    let output = outputTags.length > 0
         ? await invokeGenerateFakeContent(ctx, {
             count,
             typeKind: nodeType.kind,
@@ -49,18 +49,22 @@ export async function generateMultipleFieldSetContents(ctx, options) {
             documentDescription,
             siteTheme,
             language,
-        }, outputTags, { tolerateErrors, debugName })
+        }, outputTags, { tolerateErrors, llmTaskName })
         : undefined;
-    if (skipTitle && generatedContents) {
-        generatedContents = generatedContents.map((content) => {
-            const { title, ...rest } = content;
-            return rest;
-        });
+    if (skipTitle && output) {
+        output = {
+            contents: output.contents.map((content) => {
+                const { title, ...rest } = content;
+                return rest;
+            }),
+            llmReport: output.llmReport,
+        };
     }
+    report.add(count, output?.llmReport);
     return createNodeContents({
         nodeType,
         count,
-        generatedContents,
+        generatedContents: output?.contents,
         outputTags,
         language,
     });

package/gen-backend/dist/generator/fake-content-generator.ts/invoke-generate-fake-content.js

@@ -1,15 +1,17 @@
-import { PromptTemplate } from "@langchain/core/prompts";
 import { languageLabelIn } from "@paroicms/public-anywhere-lib";
+import { batchInvokeMinistral } from "../lib/calling-llm-mistral.js";
 import { readPromptFile } from "../lib/create-prompt.js";
 import { debugBatchLlmOutputs } from "../lib/debug-utils.js";
 import { parseLlmResponseAsList } from "../lib/parse-llm-response.js";
-const singlePromptTpl = PromptTemplate.fromTemplate(await readPromptFile("generate-fake-content-single.md"));
-const multipleDocumentsPromptTpl = PromptTemplate.fromTemplate(await readPromptFile("generate-fake-content-multiple-documents.md"));
-const multiplePartsPromptTpl = PromptTemplate.fromTemplate(await readPromptFile("generate-fake-content-multiple-parts.md"));
+import { buildPromptTemplate } from "../lib/prompt-template.js";
+// Load and create prompt templates
+const singlePromptTemplate = buildPromptTemplate(await readPromptFile("generate-fake-content-single.md"));
+const multipleDocumentsPromptTemplate = buildPromptTemplate(await readPromptFile("generate-fake-content-multiple-documents.md"));
+const multiplePartsPromptTemplate = buildPromptTemplate(await readPromptFile("generate-fake-content-multiple-parts.md"));
 export async function invokeGenerateFakeContent(ctx, input, outputTags, options) {
     const { language, typeKind } = input;
     const single = input.count === 1;
-    const debugName = `fake-content-${options.debugName}${single ? "" : `-${input.count}`}`;
+    const llmTaskName = `fake-content-${options.llmTaskName}${single ? "" : `-${input.count}`}`;
     const tagAndDescriptions = outputTags
         .map(({ tagName, tagDescription }) => `<${tagName}>${tagDescription}</${tagName}>`)
         .join("\n\n");
@@ -36,22 +38,29 @@ export async function invokeGenerateFakeContent(ctx, input, outputTags, options)
         batchInputs.push(llmInput);
         startIndex = nextIndex;
     }
-    const debug = await debugBatchLlmOutputs(ctx, debugName, ctx.cheapModelName, batchInputs);
-    let contents = debug.storedContents;
-    if (!contents) {
-        const llmMessages = await (single
-            ? singlePromptTpl
+    const debug = await debugBatchLlmOutputs(ctx, llmTaskName, ctx.mistralModelName, undefined, batchInputs);
+    let llmOutput = debug.stored;
+    if (!llmOutput) {
+        // Select the appropriate prompt template
+        const promptTemplate = single
+            ? singlePromptTemplate
             : typeKind === "document"
-                ? multipleDocumentsPromptTpl
-                : multiplePartsPromptTpl)
-            .pipe(ctx.cheapModel)
-            .batch(batchInputs);
-        contents = await debug.getMessageContents(llmMessages);
+                ? multipleDocumentsPromptTemplate
+                : multiplePartsPromptTemplate;
+        // Process batch inputs
+        const messages = batchInputs.map(promptTemplate);
+        const results = await batchInvokeMinistral(ctx, messages, {
+            llmTaskName,
+            temperature: 0.1,
+            maxTokens: 50_000,
+            timeoutMs: 60_000,
+        });
+        llmOutput = await debug.getMessageContents(results);
     }
     const results = [];
-    for (const llmMessageContent of contents) {
+    for (const llmMessageContent of llmOutput.outputs) {
         const list = parseLlmResponseAsList(llmMessageContent, outputTags, options);
         results.push(...list.map((fields) => Object.fromEntries(Object.entries(fields).map(([fieldName, value]) => [fieldName, { [language]: value }]))));
     }
-    return results;
+    return { contents: results, llmReport: llmOutput.llmReport };
 }

package/gen-backend/dist/generator/lib/calling-llm-anthropic.js

@@ -0,0 +1,33 @@
+import { LlmError } from "./llm-utils.js";
+export async function invokeClaude(ctx, options) {
+    const { anthropic, anthropicModelName } = ctx;
+    const { prompt, llmTaskName, maxTokens, systemInstruction, temperature } = options;
+    const system = systemInstruction === "beFast"
+        ? "Please provide brief, direct answers without extensive deliberation. Focus on giving the most immediately useful information quickly."
+        : "Take max 20 seconds to think and provide a thorough response.";
+    const startTime = Date.now();
+    const response = await anthropic.messages.create({
+        model: anthropicModelName,
+        system,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: maxTokens,
+        temperature: temperature,
+    });
+    const llmReport = {
+        llmTaskName,
+        modelName: anthropicModelName,
+        inputTokenCount: response.usage.input_tokens,
+        durationMs: Date.now() - startTime,
+        outputTokenCount: response.usage.output_tokens,
+    };
+    const [first] = response.content;
+    if (first.type === "text") {
+        return {
+            messageContent: first.text,
+            report: llmReport,
+        };
+    }
+    ctx.logger.error(`Unexpected response type from Claude: ${JSON.stringify(response.content, null, 2)}`);
+    llmReport.errorMessage = `Unexpected response type: ${first.type}`;
+    throw new LlmError(`Unexpected response type: "${first.type}"`, llmReport);
+}

package/gen-backend/dist/generator/lib/calling-llm-mistral.js

@@ -0,0 +1,156 @@
+import { messageOf } from "@paroi/data-formatters-lib";
+let seq = 0;
+export async function batchInvokeMinistral(ctx, prompts, options) {
+    const startTime = Date.now();
+    const responses = await execBatchInvokeMinistral(ctx, prompts, options);
+    const llmMessages = responses
+        .map((msg) => msg.response.body.choices[0]?.message.content)
+        .filter(Boolean);
+    const llmReport = {
+        llmTaskName: options.llmTaskName,
+        modelName: ctx.mistralModelName,
+        inputTokenCount: responses
+            .map((msg) => msg.response.body.usage.prompt_tokens)
+            .reduce((a, b) => a + b, 0),
+        durationMs: Date.now() - startTime,
+        outputTokenCount: responses
+            .map((msg) => msg.response.body.usage.completion_tokens)
+            .reduce((a, b) => a + b, 0),
+    };
+    return {
+        llmMessages,
+        llmReport,
+    };
+}
+async function execBatchInvokeMinistral(ctx, prompts, options) {
+    const { mistral, mistralModelName, logger } = ctx;
+    const uploadedFileIds = [];
+    try {
+        const messages = prompts
+            .map((prompt, index) => ({
+            custom_id: `${index}`,
+            body: {
+                max_tokens: options.maxTokens,
+                temperature: options.temperature,
+                messages: [
+                    {
+                        role: "user",
+                        content: prompt,
+                    },
+                ],
+            },
+        }))
+            .map((request) => JSON.stringify(request))
+            .join("\n");
+        const batchData = await mistral.files.upload({
+            file: {
+                fileName: `batch-input-${seq++}.jsonl`,
+                content: Buffer.from(messages),
+            },
+            purpose: "batch",
+        });
+        uploadedFileIds.push(batchData.id);
+        const createdJob = await mistral.batch.jobs.create({
+            inputFiles: [batchData.id],
+            model: mistralModelName,
+            endpoint: "/v1/chat/completions",
+            metadata: { jobType: "batchInvoke" },
+            timeoutHours: 1,
+        });
+        const outputFileId = await waitJobCompletion(ctx, {
+            jobId: createdJob.id,
+            timeoutMs: options.timeoutMs,
+        });
+        uploadedFileIds.push(outputFileId);
+        const outputFileStream = await mistral.files.download({ fileId: outputFileId });
+        const result = await readAsString(outputFileStream);
+        try {
+            return result
+                .trim()
+                .split("\n")
+                .map((line) => JSON.parse(line));
+        }
+        catch (error) {
+            logger.error("[Mistral] Error parsing batch job result:", error, result);
+            throw new Error("Failed to parse batch job result");
+        }
+    }
+    finally {
+        for (const fileId of uploadedFileIds) {
+            try {
+                await mistral.files.delete({ fileId });
+            }
+            catch (error) {
+                logger.error("[Mistral] Error deleting uploaded file:", error, fileId);
+            }
+        }
+    }
+}
+async function waitJobCompletion(ctx, options) {
+    const { mistral, logger } = ctx;
+    const { jobId, timeoutMs } = options;
+    const startTime = Date.now();
+    let jobStatus;
+    let timeoutOccurred = false;
+    try {
+        while (true) {
+            jobStatus = await mistral.batch.jobs.get({ jobId });
+            const { status } = jobStatus;
+            if (status === "QUEUED" || status === "RUNNING" || status === "CANCELLATION_REQUESTED") {
+                const elapsedTime = Date.now() - startTime;
+                if (elapsedTime > timeoutMs) {
+                    timeoutOccurred = true;
+                    break;
+                }
+                await new Promise((resolve) => setTimeout(resolve, 2_000));
+                continue;
+            }
+            if (status === "FAILED" ||
+                status === "CANCELLED" ||
+                status === "TIMEOUT_EXCEEDED" ||
+                status === "SUCCESS") {
+                break;
+            }
+            throw new Error(`Unexpected batch job "${jobStatus.id}" status: "${status}"`);
+        }
+    }
+    catch (error) {
+        if (!jobStatus) {
+            throw new Error(`[Mistral] Failed to wait for batch job "${jobId}" completion: ${messageOf(error)}`);
+        }
+        logger.error(`[Mistral] Error while waiting for job "${jobId}" completion:`, error);
+    }
+    if (!jobStatus)
+        throw new Error("[Mistral] Should have a job status here");
+    if (timeoutOccurred) {
+        logger.debug(`[Mistral] Batch job "${jobId}" timed out after ${timeoutMs}ms. Attempting to cancel…`);
+        try {
+            jobStatus = await mistral.batch.jobs.cancel({ jobId });
+        }
+        catch (error) {
+            throw new Error(`[Mistral] Failed to cancel batch job "${jobId}" after timeout: ${messageOf(error)}`);
+        }
+    }
+    const { status, errors } = jobStatus;
+    if (status !== "SUCCESS") {
+        const errMessages = errors.map((e) => e.message).join(", ");
+        throw new Error(`[Mistral] Batch job ${jobStatus.id} failed with status "${status}": ${errMessages}`);
+    }
+    if (!jobStatus?.outputFile)
+        throw new Error("[Mistral] Missing output file");
+    return jobStatus.outputFile;
+}
+function readAsString(stream) {
+    return new Promise((resolve, reject) => {
+        const output = [];
+        stream.pipeTo(new WritableStream({
+            write(chunk) {
+                output.push(new TextDecoder("utf-8").decode(chunk));
+            },
+            close() {
+                resolve(output.join(""));
+            },
+            abort: reject,
+        }));
+    });
+}

package/gen-backend/dist/generator/lib/create-prompt.js

@@ -1,8 +1,8 @@
-import { PromptTemplate } from "@langchain/core/prompts";
 import { connectorPackageDir } from "@paroicms/connector";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { projectDir } from "../../context.js";
+import { buildPromptTemplate } from "./prompt-template.js";
 const contextContent = await readPromptFile("0-context.md");
 const siteSchemaTsDefs = await readFile(join(connectorPackageDir, "typeonly", "site-schema-json-types.d.ts"), "utf-8");
 const predefinedFields = JSON.parse(await readPromptFile("predefined-fields.json"));
@@ -27,7 +27,7 @@ ${contextContent}${schemaTypeDefTemplate}
 
 ${promptContent}
 `;
-    return PromptTemplate.fromTemplate(template);
+    return buildPromptTemplate(template);
 }
 export async function readPromptFile(fileName) {
     return await readFile(join(projectDir, "prompts", fileName), "utf-8");

package/gen-backend/dist/generator/lib/debug-utils.js

@@ -1,62 +1,61 @@
 import { messageOf } from "@paroi/data-formatters-lib";
+import { ensureDirectory } from "@paroicms/internal-server-lib";
 import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { estimateTokenCount } from "./llm-tokens.js";
 const debugSep = "\n\n========================\n\n";
-export async function debugLlmOutput(ctx, debugName, llmModelName, llmInput) {
-    const storedContents = await readDebugLlmOutputs(ctx, debugName);
-    const storedContent = storedContents?.[0];
-    if (storedContent) {
-        ctx.logger.info(`[${debugName}][${llmModelName}] Found debug output (skip calling LLM)`);
+export async function debugLlmOutput(ctx, llmTaskName, llmModelName, stepHandle, llmInput) {
+    const aggregatedInput = Object.values(llmInput).join("\n");
+    const inputTokenCount = aggregatedInput ? estimateTokenCount(aggregatedInput) : 0;
+    const stored = await readDebugLlmOutputs(ctx, { llmTaskName, inputTokenCount, llmModelName });
+    const singleStored = stored && stored.outputs.length === 1
+        ? {
+            output: stored.outputs[0],
+            llmReport: stored.llmReport,
+        }
+        : undefined;
+    if (singleStored) {
+        ctx.logger.info(`[${llmTaskName}][${llmModelName}] Found debug output (skip calling LLM)`);
     }
     else {
-        const aggregatedInput = Object.values(llmInput).join("\n");
-        const tokenCount = aggregatedInput ? await estimateTokenCount(aggregatedInput) : 0;
-        ctx.logger.debug(`[${debugName}][${llmModelName}] Calling LLM… User tokens: ~${tokenCount}`);
+        ctx.logger.debug(`[${llmTaskName}][${llmModelName}] Calling LLM… User tokens: ~${inputTokenCount}`);
     }
     const startTs = Date.now();
     return {
-        storedContent,
-        async getMessageContent(llmMessage) {
-            if (typeof llmMessage.content !== "string")
-                throw new Error("Expected a string");
-            const llmMessageContent = llmMessage.content;
-            const duration = Date.now() - startTs;
-            ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${llmMessageContent.length} - [${debugName}][${llmModelName}]`);
-            await writeDebugLlmInputOutputs(ctx, debugName, llmModelName, [
+        stored: singleStored,
+        async getMessageContent(llmMessage, llmReport) {
+            const llmMessageContent = llmMessage;
+            const totalTokens = llmReport.outputTokenCount ?? 0;
+            ctx.logger.debug(`… done. Duration: ${llmReport.durationMs} ms, Tokens: ~${totalTokens} - [${llmTaskName}][${llmModelName}]`);
+            await writeDebugLlmInputOutputs(ctx, stepHandle, [
                 {
                     llmInput,
                     llmMessageContent,
                 },
-            ]);
-            return llmMessageContent;
+            ], llmReport, startTs);
+            return { output: llmMessageContent, llmReport };
         },
     };
 }
-export async function debugBatchLlmOutputs(ctx, debugName, llmModelName, llmInputs) {
-    const storedContents = await readDebugLlmOutputs(ctx, debugName);
-    if (storedContents) {
-        ctx.logger.info(`[${debugName}][${llmModelName}] Found debug output (skip calling LLM)`);
+export async function debugBatchLlmOutputs(ctx, llmTaskName, llmModelName, stepHandle, llmInputs) {
+    const aggregatedInput = llmInputs
+        .map((llmInput) => Object.values(llmInput).join("\n"))
+        .join("\n\n");
+    const inputTokenCount = aggregatedInput ? estimateTokenCount(aggregatedInput) : 0;
+    const stored = await readDebugLlmOutputs(ctx, { llmTaskName, inputTokenCount, llmModelName });
+    if (stored) {
+        ctx.logger.info(`[${llmTaskName}][${llmModelName}] Found debug output (skip calling LLM)`);
     }
     else {
-        const aggregatedInput = llmInputs
-            .map((llmInput) => Object.values(llmInput).join("\n"))
-            .join("\n\n");
-        const tokenCount = aggregatedInput ? await estimateTokenCount(aggregatedInput) : 0;
-        ctx.logger.debug(`[${debugName}][${llmModelName}] Calling LLM… User tokens: ~${tokenCount}`);
+        ctx.logger.debug(`[${llmTaskName}][${llmModelName}] Calling LLM… User tokens: ~${inputTokenCount}`);
     }
     const startTs = Date.now();
     return {
-        storedContents,
-        async getMessageContents(llmMessages) {
-            const llmMessageContents = llmMessages.map((llmMessage) => {
-                if (typeof llmMessage.content !== "string")
-                    throw new Error("Expected a string");
-                return llmMessage.content;
-            });
+        stored,
+        async getMessageContents({ llmMessages, llmReport }) {
+            const llmMessageContents = llmMessages;
             const duration = Date.now() - startTs;
-            const totalTokens = llmMessageContents.reduce((sum, content) => sum + content.length, 0);
-            ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${totalTokens} - [${debugName}][${llmModelName}]`);
+            ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${llmReport.outputTokenCount} - [${llmTaskName}][${llmModelName}]`);
             if (llmMessageContents.length !== llmInputs.length) {
                 throw new Error(`Expected ${llmInputs.length} LLM outputs, but got ${llmMessageContents.length}`);
             }
@@ -66,16 +65,17 @@ export async function debugBatchLlmOutputs(ctx, debugName, llmModelName, llmInpu
                     llmMessageContent: llmMessageContents[i],
                 };
             });
-            await writeDebugLlmInputOutputs(ctx, debugName, llmModelName, list);
-            return llmMessageContents;
+            await writeDebugLlmInputOutputs(ctx, stepHandle, list, llmReport, startTs);
+            return { outputs: llmMessageContents, llmReport };
         },
     };
 }
-async function readDebugLlmOutputs(ctx, debugName) {
+async function readDebugLlmOutputs(ctx, options) {
     const { logger, debugDir } = ctx;
     if (!debugDir)
         return;
-    const debugFile = join(debugDir, `${debugName}.txt`);
+    const { llmTaskName, inputTokenCount, llmModelName } = options;
+    const debugFile = join(debugDir, `${llmTaskName}.txt`);
     try {
         const debugContent = await readFile(debugFile, "utf8");
         const list = debugContent.split(debugSep);
@@ -86,8 +86,15 @@ async function readDebugLlmOutputs(ctx, debugName) {
         for (let i = 1; i < list.length; i += 2) {
             outputs.push(list[i]);
         }
-        logger.debug(`… found debug output for ${debugName} (skip calling LLM)`);
-        return outputs;
+        const llmReport = {
+            llmTaskName,
+            modelName: llmModelName,
+            inputTokenCount,
+            durationMs: 0,
+            outputTokenCount: estimateTokenCount(outputs.join(" ")),
+        };
+        logger.debug(`… found debug output for ${llmTaskName} (skip calling LLM)`);
+        return { outputs, llmReport };
     }
     catch (error) {
         if (error.code !== "ENOENT") {
@@ -95,17 +102,36 @@ async function readDebugLlmOutputs(ctx, debugName) {
         }
     }
 }
-async function writeDebugLlmInputOutputs(ctx, debugName, llmModelName, list) {
-    const { debugDir } = ctx;
+async function writeDebugLlmInputOutputs(ctx, stepHandle, list, llmReport, startTs) {
+    const { debugDir, sessionId } = ctx;
     if (!debugDir)
         return;
-    const dt = new Date().toISOString();
-    const baseName = `${dt.substring(0, 19)}-${debugName}`;
-    const content = [`${llmModelName ?? "Unamed model"} - ${debugName} - ${dt}`];
+    const dt = new Date(startTs).toISOString();
+    const nameParts = [
+        dt.substring(0, 19).replace(/:/g, "-"),
+        stepHandle?.stepNumber,
+        llmReport.llmTaskName,
+        llmReport.errorMessage ? "ERROR" : undefined,
+    ].filter(Boolean);
+    const baseName = nameParts.join("-");
+    const header = [
+        `Model: ${llmReport.modelName}`,
+        `Task: ${llmReport.llmTaskName}`,
+        `Input tokens: ~${llmReport.inputTokenCount}`,
+        `Output tokens: ~${llmReport.outputTokenCount}`,
+        `Duration: ${llmReport.durationMs} ms`,
+        `Date: ${dt}`,
+    ];
+    if (llmReport.errorMessage) {
+        header.push(`Error: ${llmReport.errorMessage}`);
+    }
+    const content = [header.join("\n")];
     for (const { llmInput, llmMessageContent } of list) {
         content.push(debugSep, llmInputToDebugMessage(llmInput), debugSep, llmMessageContent);
     }
-    await writeFile(join(debugDir, `${baseName}.txt`), content.join(""));
+    const dir = join(debugDir, sessionId);
+    await ensureDirectory(dir);
+    await writeFile(join(dir, `${baseName}.txt`), content.join(""));
 }
 function llmInputToDebugMessage(input) {
     return Object.entries(input)

package/gen-backend/dist/generator/lib/llm-tokens.js

@@ -1,10 +1,8 @@
-import { TokenTextSplitter } from "langchain/text_splitter";
-export async function estimateTokenCount(text) {
-    const splitter = new TokenTextSplitter({
-        encodingName: "cl100k_base",
-        chunkSize: 1,
-        chunkOverlap: 0,
-    });
-    const chunks = await splitter.splitText(text);
-    return chunks.length;
+/**
+ * Estimates token count based on character count.
+ * Most LLMs use subword tokenization where 1 token is roughly 4 characters in English.
+ */
+export function estimateTokenCount(text) {
+    // Approximate token count using character count / 4 for English text
+    return Math.ceil(text.length / 4);
 }