@paroicms/site-generator-plugin 0.1.0

package/gen-backend/dist/generator/lib/create-prompt.js

@@ -0,0 +1,44 @@
+import { PromptTemplate } from "@langchain/core/prompts";
+import { connectorPackageDir } from "@paroicms/connector";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { projectDir } from "../../context.js";
+const contextContent = await readPromptFile("0-context.md");
+const siteSchemaTsDefs = await readFile(join(connectorPackageDir, "typeonly", "site-schema-json-types.d.ts"), "utf-8");
+const predefinedFields = JSON.parse(await readPromptFile("predefined-fields.json"));
+export async function createPromptTemplate(options) {
+    const { fileName, withSiteSchemaTsDefs } = options;
+    const promptContent = await readPromptFile(fileName);
+    const schemaTypeDefTemplate = withSiteSchemaTsDefs
+        ? `
+
+All the site structure is described in a site-schema in JSON format. Here is the TypeScript definition for the site schema:
+
+<site_schema_ts_defs>
+{siteSchemaTsDefs}
+</site_schema_ts_defs>`
+        : "";
+    const template = `
+# Context
+
+${contextContent}${schemaTypeDefTemplate}
+
+# Task to do
+
+${promptContent}
+`;
+    return PromptTemplate.fromTemplate(template);
+}
+export async function readPromptFile(fileName) {
+    return await readFile(join(projectDir, "prompts", fileName), "utf-8");
+}
+export function getPredefinedFields() {
+    if (!predefinedFields)
+        throw new Error("Predefined fields not loaded");
+    return predefinedFields;
+}
+export function getSiteSchemaTsDefs() {
+    if (!siteSchemaTsDefs)
+        throw new Error("Site-schema defs not loaded");
+    return siteSchemaTsDefs;
+}

package/gen-backend/dist/generator/lib/debug-utils.js

@@ -0,0 +1,118 @@
+import { messageOf } from "@paroi/data-formatters-lib";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { estimateTokenCount } from "./llm-tokens.js";
+const debugSep = "\n\n========================\n\n";
+export async function debugLlmOutput(ctx, debugName, llmModelName, llmInput) {
+    const storedContents = await readDebugLlmOutputs(ctx, debugName);
+    const storedContent = storedContents?.[0];
+    if (storedContent) {
+        ctx.logger.info(`[${debugName}][${llmModelName}] Found debug output (skip calling LLM)`);
+    }
+    else {
+        const aggregatedInput = Object.values(llmInput).join("\n");
+        const tokenCount = aggregatedInput ? await estimateTokenCount(aggregatedInput) : 0;
+        ctx.logger.debug(`[${debugName}][${llmModelName}] Calling LLM… User tokens: ~${tokenCount}`);
+    }
+    const startTs = Date.now();
+    return {
+        storedContent,
+        async getMessageContent(llmMessage) {
+            if (typeof llmMessage.content !== "string")
+                throw new Error("Expected a string");
+            const llmMessageContent = llmMessage.content;
+            const duration = Date.now() - startTs;
+            ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${llmMessageContent.length} - [${debugName}][${llmModelName}]`);
+            await writeDebugLlmInputOutputs(ctx, debugName, llmModelName, [
+                {
+                    llmInput,
+                    llmMessageContent,
+                },
+            ]);
+            return llmMessageContent;
+        },
+    };
+}
+export async function debugBatchLlmOutputs(ctx, debugName, llmModelName, llmInputs) {
+    const storedContents = await readDebugLlmOutputs(ctx, debugName);
+    if (storedContents) {
+        ctx.logger.info(`[${debugName}][${llmModelName}] Found debug output (skip calling LLM)`);
+    }
+    else {
+        const aggregatedInput = llmInputs
+            .map((llmInput) => Object.values(llmInput).join("\n"))
+            .join("\n\n");
+        const tokenCount = aggregatedInput ? await estimateTokenCount(aggregatedInput) : 0;
+        ctx.logger.debug(`[${debugName}][${llmModelName}] Calling LLM… User tokens: ~${tokenCount}`);
+    }
+    const startTs = Date.now();
+    return {
+        storedContents,
+        async getMessageContents(llmMessages) {
+            const llmMessageContents = llmMessages.map((llmMessage) => {
+                if (typeof llmMessage.content !== "string")
+                    throw new Error("Expected a string");
+                return llmMessage.content;
+            });
+            const duration = Date.now() - startTs;
+            const totalTokens = llmMessageContents.reduce((sum, content) => sum + content.length, 0);
+            ctx.logger.debug(`… done. Duration: ${duration} ms, Tokens: ~${totalTokens} - [${debugName}][${llmModelName}]`);
+            if (llmMessageContents.length !== llmInputs.length) {
+                throw new Error(`Expected ${llmInputs.length} LLM outputs, but got ${llmMessageContents.length}`);
+            }
+            const list = llmInputs.map((llmInput, i) => {
+                return {
+                    llmInput,
+                    llmMessageContent: llmMessageContents[i],
+                };
+            });
+            await writeDebugLlmInputOutputs(ctx, debugName, llmModelName, list);
+            return llmMessageContents;
+        },
+    };
+}
+async function readDebugLlmOutputs(ctx, debugName) {
+    const { logger, debugDir } = ctx;
+    if (!debugDir)
+        return;
+    const debugFile = join(debugDir, `${debugName}.txt`);
+    try {
+        const debugContent = await readFile(debugFile, "utf8");
+        const list = debugContent.split(debugSep);
+        if (list.length < 3)
+            return;
+        list.shift();
+        const outputs = [];
+        for (let i = 1; i < list.length; i += 2) {
+            outputs.push(list[i]);
+        }
+        logger.debug(`… found debug output for ${debugName} (skip calling LLM)`);
+        return outputs;
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            logger.error(`Error reading debug output from "${debugFile}": ${messageOf(error)}`);
+        }
+    }
+}
+async function writeDebugLlmInputOutputs(ctx, debugName, llmModelName, list) {
+    const { debugDir } = ctx;
+    if (!debugDir)
+        return;
+    const dt = new Date().toISOString();
+    const baseName = `${dt.substring(0, 19)}-${debugName}`;
+    const content = [`${llmModelName ?? "Unamed model"} - ${debugName} - ${dt}`];
+    for (const { llmInput, llmMessageContent } of list) {
+        content.push(debugSep, llmInputToDebugMessage(llmInput), debugSep, llmMessageContent);
+    }
+    await writeFile(join(debugDir, `${baseName}.txt`), content.join(""));
+}
+function llmInputToDebugMessage(input) {
+    return Object.entries(input)
+        .map(([key, value]) => {
+        return `<${key}>
+${value}
+</${key}>`;
+    })
+        .join("\n\n");
+}

package/gen-backend/dist/generator/lib/images-lib.js

@@ -0,0 +1,16 @@
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { projectDir } from "../../context.js";
+const availableExtensions = new Set([".jpeg", ".jpg", ".png", ".gif", ".svg", ".webp"]);
+const imageFileNames = [];
+export async function initializeImageNames() {
+    for (const file of await readdir(join(projectDir, "images"))) {
+        const ext = file.slice(file.lastIndexOf("."));
+        if (!availableExtensions.has(ext))
+            continue;
+        imageFileNames.push(file);
+    }
+}
+export function getRandomImagePath() {
+    return join(projectDir, "images", imageFileNames[Math.floor(Math.random() * imageFileNames.length)]);
+}

package/gen-backend/dist/generator/lib/llm-invoke-types.js

@@ -0,0 +1 @@
+export {};

package/gen-backend/dist/generator/lib/llm-tokens.js

@@ -0,0 +1,10 @@
+import { TokenTextSplitter } from "langchain/text_splitter";
+export async function estimateTokenCount(text) {
+    const splitter = new TokenTextSplitter({
+        encodingName: "cl100k_base",
+        chunkSize: 1,
+        chunkOverlap: 0,
+    });
+    const chunks = await splitter.splitText(text);
+    return chunks.length;
+}

package/gen-backend/dist/generator/lib/markdown-bulleted-list-parser.js

@@ -0,0 +1,147 @@
+export function parseMarkdownBulletedList(markdown) {
+    const lines = markdown.split(/\r?\n/);
+    const parsedLines = lines.map(parseLine).filter((line) => !!line);
+    return { home: convertParsedLinesToTree(parsedLines) };
+}
+export function convertParsedLinesToTree(lines) {
+    if (lines.length === 0) {
+        throw new Error("Input array cannot be empty");
+    }
+    const firstLine = lines[0];
+    // Ensure first line is a routing document with indent 0
+    if (firstLine.indent !== 0 || firstLine.kind !== "routingDocument") {
+        throw new Error("First line must be a routing document with zero indentation");
+    }
+    // Create root node
+    const root = {
+        kind: "routingDocument",
+        typeName: firstLine.typeName,
+    };
+    // Stack to keep track of parent nodes at each level
+    const stack = [root];
+    let currentDepth = 0;
+    // Process remaining lines
+    for (let i = 1; i < lines.length; ++i) {
+        const line = lines[i];
+        const node = createNode(line);
+        // If indent is greater than current level, add as child to last node in stack
+        if (line.indent > currentDepth) {
+            if (line.indent !== currentDepth + 1) {
+                throw new Error(`Invalid indentation level at line ${line.lineNumber}`);
+            }
+            const lastNode = stack[stack.length - 1];
+            appendChildToParentNode(node, lastNode);
+            stack.push(node);
+        }
+        // If indent is less than current level, pop stack until we reach correct level
+        else if (line.indent < currentDepth) {
+            while (line.indent <= currentDepth) {
+                stack.pop();
+                --currentDepth;
+            }
+            const lastNode = stack[stack.length - 1];
+            appendChildToParentNode(node, lastNode);
+            stack.push(node);
+        }
+        // If same indent, add as sibling
+        else {
+            stack.pop();
+            const lastNode = stack[stack.length - 1];
+            appendChildToParentNode(node, lastNode);
+            stack.push(node);
+        }
+        currentDepth = line.indent;
+    }
+    return root;
+}
+function appendChildToParentNode(child, parent) {
+    if (parent.kind === "routingDocument") {
+        parent.children ??= [];
+        parent.children.push(child);
+    }
+    else if (parent.kind === "regularDocument") {
+        if (child.kind === "routingDocument") {
+            throw new Error(`Regular document type "${parent.typeName}" cannot have ${child.kind} child "${child.typeName}"`);
+        }
+        parent.children ??= [];
+        parent.children.push(child);
+    }
+    else if (parent.kind === "part") {
+        if (child.kind !== "part") {
+            throw new Error(`Part type "${parent.typeName}" cannot have ${child.kind} child "${child.typeName}"`);
+        }
+        parent.children ??= [];
+        parent.children.push(child);
+    }
+}
+function createNode(line) {
+    switch (line.kind) {
+        case "routingDocument":
+            return {
+                kind: "routingDocument",
+                typeName: line.typeName,
+            };
+        case "regularDocument":
+            return {
+                kind: "regularDocument",
+                typeName: line.typeName,
+            };
+        case "part":
+            if (!line.listName) {
+                throw new Error("Part node must have a listName");
+            }
+            return {
+                kind: "part",
+                typeName: line.typeName,
+                listName: line.listName,
+            };
+        default:
+            throw new Error(`Unknown node kind: ${line.kind}`);
+    }
+}
+export function parseLine(input, index) {
+    if (!input.trim())
+        return;
+    const lineNumber = index + 1;
+    // Count indentation (2 spaces per level)
+    const indentSpaces = input.match(/^[ ]+/);
+    const bulletIndex = indentSpaces ? indentSpaces[0].length : 0;
+    const indent = bulletIndex / 2;
+    const bulletChar = input.charAt(bulletIndex);
+    if (bulletChar !== "*" && bulletChar !== "-") {
+        throw new Error(`Missing bullet at line ${lineNumber}`);
+    }
+    const cleanLine = input.substring(bulletIndex + 1).trim();
+    // list of `article` (regular documents)
+    const regularDocumentMatch = cleanLine.match(/^list of `([^`]+)` \(regular documents\)$/);
+    if (regularDocumentMatch) {
+        return {
+            kind: "regularDocument",
+            indent,
+            typeName: regularDocumentMatch[1],
+            lineNumber,
+        };
+    }
+    // list of `pageSection` (parts), list name: `partSections`
+    const partMatch = cleanLine.match(/^list of `([^`]+)` \(parts\),? list name: `([^`]+)`$/);
+    if (partMatch) {
+        return {
+            kind: "part",
+            indent,
+            typeName: partMatch[1],
+            listName: partMatch[2],
+            lineNumber,
+        };
+    }
+    // `pages` (routing document)
+    const routingDocumentMatch = cleanLine.match(/^`([^`]+)` \(routing document\)$/);
+    if (routingDocumentMatch) {
+        return {
+            kind: "routingDocument",
+            indent,
+            typeName: routingDocumentMatch[1],
+            lineNumber,
+        };
+    }
+    throw new Error(`Invalid line at line ${lineNumber}`);
+}

package/gen-backend/dist/generator/lib/parse-llm-response.js

@@ -0,0 +1,160 @@
+import { parse } from "yaml";
+export function parseLlmResponseAsProperties(llmResponse, outputTags) {
+    const rawTags = parseLlmRawTags(llmResponse, outputTags.map((tag) => tag.tagName));
+    const map = new Map(rawTags.map((tag) => [tag.tagName, tag]));
+    if (rawTags.length !== outputTags.length) {
+        const missingTags = outputTags.filter((tag) => !map.has(tag.tagName));
+        throw new Error(`Missing tags: ${missingTags.map((tag) => tag.tagName).join(", ")}`);
+    }
+    const resultObj = {};
+    for (const outputTag of outputTags) {
+        const raw = map.get(outputTag.tagName);
+        if (!raw) {
+            if (!outputTag.optional)
+                throw new Error(`Missing tag: ${outputTag.tagName}`);
+            continue;
+        }
+        resultObj[outputTag.key] = formatRawContent(raw.content, outputTag);
+    }
+    return resultObj;
+}
+export function parseLlmResponseAsList(llmResponse, outputTags, options = {}) {
+    const { tolerateErrors } = options;
+    const rawTags = parseLlmRawTags(llmResponse, outputTags.map((tag) => tag.tagName), options);
+    if (rawTags.length === 0)
+        return [];
+    const outputTagMap = new Map(outputTags.map((tag) => [tag.tagName, tag]));
+    const result = [];
+    let current = {};
+    for (const rawTag of rawTags) {
+        const outputTag = outputTagMap.get(rawTag.tagName);
+        if (!outputTag)
+            throw new Error(`Unexpected output tag "${rawTag.tagName}"`); // it's a real bug
+        if (rawTag.tagName in current) {
+            const rawTag = ensureProperties(current, outputTags, options);
+            if (rawTag) {
+                result.push(rawTag);
+            }
+            current = {};
+        }
+        if (rawTag.content === "") {
+            if (!outputTag.optional) {
+                const message = `Empty tag <${outputTag.tagName}>`;
+                if (!tolerateErrors)
+                    throw new Error(message);
+                tolerateErrors.errorMessages.push(message);
+                current = {};
+            }
+            continue;
+        }
+        current[outputTag.key] = formatRawContent(rawTag.content, outputTag);
+    }
+    if (Object.keys(current).length > 0) {
+        const rawTag = ensureProperties(current, outputTags, options);
+        if (rawTag) {
+            result.push(rawTag);
+        }
+    }
+    return result;
+}
+function ensureProperties(obj, outputTags, options) {
+    const { tolerateErrors } = options;
+    for (const tag of outputTags) {
+        if (!(tag.key in obj) && !tag.optional) {
+            const message = `Missing tag <${tag.tagName}>`;
+            if (!tolerateErrors)
+                throw new Error(message);
+            tolerateErrors.errorMessages.push(message);
+            return;
+        }
+    }
+    return obj;
+}
+function formatRawContent(rawContent, tag) {
+    const { format, optional } = tag;
+    if (!rawContent && optional)
+        return;
+    switch (format) {
+        case "yaml":
+            return parse(rawContent);
+        case "json":
+            return JSON.parse(rawContent);
+        case "markdown":
+        case "text":
+            return rawContent;
+        default:
+            throw new Error(`Unknown format "${format}"`);
+    }
+}
+export function parseLlmRawTags(llmResponse, tagNames, options = {}) {
+    const { tolerateErrors } = options;
+    const tagList = [];
+    // Create regex to match all opening and closing tags
+    const tagNamesPattern = tagNames.join("|");
+    const pattern = new RegExp(`<(${tagNamesPattern})>|</(${tagNamesPattern})>`, "g");
+    const matches = [];
+    let match;
+    // Find all tags (opening and closing) and store them with their positions
+    // biome-ignore lint/suspicious/noAssignInExpressions: regex exec requires assignment in loop
+    while ((match = pattern.exec(llmResponse)) !== null) {
+        const isOpening = match[1] !== undefined;
+        const tagName = isOpening ? match[1] : match[2];
+        matches.push({ isOpening, tagName, position: match.index });
+    }
+    // Process the tags to extract content
+    for (let i = 0; i < matches.length; ++i) {
+        const current = matches[i];
+        if (current.isOpening) {
+            // Find the next corresponding closing tag
+            let j = i + 1;
+            let foundClosing = false;
+            while (j < matches.length) {
+                const next = matches[j];
+                // If we encounter another opening tag of any type before finding our closing tag,
+                // it's an error if not tolerating errors
+                if (next.isOpening) {
+                    const message = `Missing closing tag for <${current.tagName}>`;
+                    if (!tolerateErrors)
+                        throw new Error(message);
+                    tolerateErrors.errorMessages.push(message);
+                    foundClosing = undefined;
+                    // If we are tolerating errors, we skip this opening tag entirely
+                    break;
+                }
+                if (!next.isOpening && next.tagName === current.tagName) {
+                    // Found a matching closing tag
+                    const contentStart = current.position + `<${current.tagName}>`.length;
+                    const contentEnd = next.position;
+                    const content = llmResponse.substring(contentStart, contentEnd).trim();
+                    tagList.push({
+                        tagName: current.tagName,
+                        content,
+                    });
+                    // Skip to after this closing tag
+                    i = j;
+                    foundClosing = true;
+                    break;
+                }
+                if (!next.isOpening && next.tagName !== current.tagName) {
+                    // Found a non-matching closing tag
+                    const message = `Mismatched tags: opening <${current.tagName}>, closing </${next.tagName}>`;
+                    if (!tolerateErrors)
+                        throw new Error(message);
+                    tolerateErrors.errorMessages.push(message);
+                    foundClosing = undefined;
+                    // If we are tolerating errors, we skip this current opening tag entirely
+                    break;
+                }
+                ++j;
+            }
+            // Handle case where no matching closing tag was found
+            if (foundClosing === false) {
+                const message = `Unclosed tag <${current.tagName}>`;
+                if (!tolerateErrors)
+                    throw new Error(message);
+                tolerateErrors.errorMessages.push(message);
+            }
+        }
+    }
+    return tagList;
+}

package/gen-backend/dist/generator/lib/tasks.js

@@ -0,0 +1,112 @@
+import { messageOf } from "@paroi/data-formatters-lib";
+export function createTaskCollector(ctx) {
+    const tasks = [];
+    return {
+        tasks,
+        add(task) {
+            tasks.push(task);
+        },
+        runAll(options) {
+            return runTasks(ctx, tasks, options);
+        },
+    };
+}
+export function runTasks(ctx, tasks, options) {
+    const { logger } = ctx;
+    const { maxParallel, rateLimitPerSecond } = options;
+    let resolve;
+    let reject;
+    const promise = new Promise((resolveCb, rejectCb) => {
+        resolve = resolveCb;
+        reject = rejectCb;
+    });
+    let timeoutId;
+    let runningCount = 0;
+    let index = 0;
+    let doneCount = 0;
+    const errorMessages = [];
+    // Track task start timestamps for rate limiting
+    const startTimestamps = [];
+    // Flag to indicate whether the task runner has been stopped
+    let mustStop = false;
+    // Resolve function for stop promise
+    let stopResolve;
+    const stopPromise = new Promise((resolveStop) => {
+        stopResolve = resolveStop;
+    });
+    function runNext() {
+        // Don't start new tasks if stopped
+        if (mustStop) {
+            if (runningCount === 0) {
+                // All running tasks have completed, we can resolve the stop promise
+                if (index < tasks.length) {
+                    reject(new Error("Stopped before all tasks completed"));
+                }
+                else {
+                    resolve({ doneCount, errorMessages });
+                }
+                stopResolve();
+            }
+            return;
+        }
+        if (index >= tasks.length) {
+            if (runningCount === 0) {
+                resolve({ doneCount, errorMessages });
+            }
+            return;
+        }
+        if (runningCount >= maxParallel)
+            return;
+        // Check rate limit if specified
+        if (rateLimitPerSecond !== undefined && rateLimitPerSecond > 0) {
+            const now = Date.now();
+            // Remove timestamps older than 1 second
+            const oneSecondAgo = now - 1000;
+            while (startTimestamps.length > 0 && startTimestamps[0] <= oneSecondAgo) {
+                startTimestamps.shift();
+            }
+            // Check if we've hit the rate limit
+            if (startTimestamps.length >= rateLimitPerSecond) {
+                // We've reached our rate limit, schedule retry after delay
+                const oldestTimestamp = startTimestamps[0];
+                const delayMs = Math.max(10, oldestTimestamp + 1000 - now);
+                clearTimeout(timeoutId);
+                timeoutId = setTimeout(runNext, delayMs);
+                return;
+            }
+            // Record this task start time
+            startTimestamps.push(now);
+        }
+        const task = tasks[index];
+        ++index;
+        ++runningCount;
+        task().then(() => {
+            ++doneCount;
+            --runningCount;
+            runNext();
+        }, (err) => {
+            logger.error(err);
+            errorMessages.push(messageOf(err));
+            --runningCount;
+            runNext();
+        });
+    }
+    for (let i = 0; i < maxParallel; ++i) {
+        runNext();
+    }
+    function stop() {
+        // Mark as stopped to prevent new tasks from starting
+        mustStop = true;
+        clearTimeout(timeoutId);
+        // If no tasks are running, resolve immediately
+        if (runningCount === 0) {
+            stopResolve();
+        }
+        // Otherwise, stopResolve will be called when runningCount reaches 0
+        return stopPromise;
+    }
+    return {
+        promise,
+        stop,
+    };
+}

package/gen-backend/dist/generator/lib/utils.js

@@ -0,0 +1,13 @@
+export function camelToKebabCase(s) {
+    return s
+        .replace(/[A-Z]/g, (m) => `-${m.toLowerCase()}`)
+        .replace(/^-+/, "")
+        .replace(/-+$/, "")
+        .replace(/--+/g, "-");
+}
+export function camelToTitleCase(camelCase) {
+    return camelCase
+        .replace(/([A-Z])/g, " $1")
+        .replace(/^./, (str) => str.toUpperCase())
+        .trimStart();
+}