@paroicms/site-generator-plugin 0.8.0 → 0.9.0

package/gen-backend/dist/commands/actions.js

@@ -0,0 +1,49 @@
+import { invokeMessageGuard } from "../generator/llm-queries/invoke-message-guard.js";
+import { invokeNewSiteAnalysis } from "../generator/llm-queries/invoke-new-site-analysis.js";
+import { invokeUpdateSiteSchema } from "../generator/llm-queries/invoke-update-site-schema.js";
+import { generateSite } from "../generator/site-generator/site-generator.js";
+import { newSession, verifySessionToken } from "./generator-session.js";
+export async function executeCommand(commandCtx, input) {
+    if (input.command === "newSession") {
+        return {
+            success: true,
+            result: await newSession(commandCtx, input),
+        };
+    }
+    const { sessionId } = verifySessionToken(commandCtx, input.sessionToken);
+    const ctx = {
+        ...commandCtx,
+        sessionId,
+    };
+    if (input.command === "createSiteSchema") {
+        const errorResponse = await invokeMessageGuard(ctx, input);
+        if (errorResponse)
+            return errorResponse;
+        return {
+            success: true,
+            result: await invokeNewSiteAnalysis(ctx, input),
+        };
+    }
+    if (input.command === "updateSiteSchema") {
+        const errorResponse = await invokeMessageGuard(ctx, input, {
+            skipOutOfScopeCheck: true,
+        });
+        if (errorResponse)
+            return errorResponse;
+        return {
+            success: true,
+            result: await invokeUpdateSiteSchema(ctx, input),
+        };
+    }
+    if (input.command === "generateSite") {
+        return {
+            success: true,
+            result: await generateSite(ctx, input),
+        };
+    }
+    ctx.logger.error(`Invalid command: ${input.command}`);
+    return {
+        success: false,
+        userMessage: "Invalid command",
+    };
+}

package/gen-backend/dist/db/db.queries.js

@@ -1,8 +1,8 @@
-import { strVal } from "@paroi/data-formatters-lib";
+import { strVal, strValOrUndef } from "@paroi/data-formatters-lib";
 export async function readSession(ctx, sessionId) {
     const { cn } = ctx;
     const row = await cn("PaGenSession")
-        .select("id", "status", "guardCount", "promptCount", "nodeTypeCount", "contentCount", "errorMessage")
+        .select("id", "createdAt", "status", "guardCount", "promptCount", "nodeTypeCount", "contentCount", "errorMessage")
         .where({ id: sessionId })
         .first();
     if (!row)
@@ -15,7 +15,7 @@ export async function readSession(ctx, sessionId) {
         promptCount: Number(row.promptCount),
         nodeTypeCount: Number(row.nodeTypeCount),
         contentCount: Number(row.contentCount),
-        errorMessage: strVal(row.errorMessage),
+        errorMessage: strValOrUndef(row.errorMessage),
     };
 }
 export async function insertSession(ctx, { sessionId }) {

package/gen-backend/dist/errors.js

@@ -0,0 +1,20 @@
+/**
+ * Custom API error class for handling API-specific errors
+ */
+export class ApiError extends Error {
+    statusCode;
+    constructor(message, statusCode = 400) {
+        super(message);
+        this.name = "ApiError";
+        this.statusCode = statusCode;
+    }
+}
+/**
+ * Create a new API error
+ * @param message Error message
+ * @param statusCode HTTP status code (default: 400)
+ * @returns ApiError instance
+ */
+export function createApiError(message, statusCode = 400) {
+    return new ApiError(message, statusCode);
+}

package/gen-backend/dist/generator/actions.js

@@ -0,0 +1,45 @@
+import { invokeMessageGuard } from "./llm-queries/invoke-message-guard.js";
+import { invokeNewSiteAnalysis } from "./llm-queries/invoke-new-site-analysis.js";
+import { invokeUpdateSiteSchema } from "./llm-queries/invoke-update-site-schema.js";
+import { newSession, verifySessionToken } from "./session/generator-session.js";
+import { generateSite } from "./site-generator/site-generator.js";
+export async function executeCommand(ctx, input) {
+    if (input.command === "newSession") {
+        return {
+            success: true,
+            result: await newSession(ctx, input),
+        };
+    }
+    verifySessionToken(input.sessionToken);
+    if (input.command === "createSiteSchema") {
+        const errorResponse = await invokeMessageGuard(ctx, input);
+        if (errorResponse)
+            return errorResponse;
+        return {
+            success: true,
+            result: await invokeNewSiteAnalysis(ctx, input),
+        };
+    }
+    if (input.command === "updateSiteSchema") {
+        const errorResponse = await invokeMessageGuard(ctx, input, {
+            skipOutOfScopeCheck: true,
+        });
+        if (errorResponse)
+            return errorResponse;
+        return {
+            success: true,
+            result: await invokeUpdateSiteSchema(ctx, input),
+        };
+    }
+    if (input.command === "generateSite") {
+        return {
+            success: true,
+            result: await generateSite(ctx, input),
+        };
+    }
+    ctx.logger.error(`Invalid command: ${input.command}`);
+    return {
+        success: false,
+        userMessage: "Invalid command",
+    };
+}

package/gen-backend/dist/generator/fake-content-generator.ts/augment-fields.js

@@ -0,0 +1,51 @@
+import { getRandomImagePath } from "../lib/images-lib.js";
+export function augmentWithComputedValues(list, nodeType, language) {
+    if (!nodeType.fields)
+        return;
+    for (const item of list) {
+        for (const f of nodeType.fields) {
+            const languageKey = f.storedOn === "node" ? "." : language;
+            if (f.storedAs === "mediaHandle") {
+                const value = getMediaGeneratedFieldContent(f, language);
+                if (value) {
+                    item[f.name] = value;
+                }
+            }
+            else {
+                const value = getDefaultValueForField(f.name);
+                if (value !== undefined) {
+                    item[f.name] = { [languageKey]: value };
+                }
+            }
+        }
+    }
+}
+function getMediaGeneratedFieldContent(f, language) {
+    const languageKey = f.storedOn === "node" ? "." : language;
+    if (f.dataType === "media") {
+        return { [languageKey]: { file: getRandomImagePath() } };
+    }
+    if (f.dataType === "gallery") {
+        return {
+            [languageKey]: {
+                files: [
+                    getRandomImagePath(),
+                    getRandomImagePath(),
+                    getRandomImagePath(),
+                    getRandomImagePath(),
+                    getRandomImagePath(),
+                    getRandomImagePath(),
+                ],
+            },
+        };
+    }
+}
+function getDefaultValueForField(fieldName) {
+    if (fieldName === "phone" || fieldName === "phone2")
+        return "0123456789";
+    if (fieldName === "phones")
+        return JSON.stringify(["0123456789", "9876543210"]);
+    if (fieldName === "updateDateTime")
+        return new Date().toISOString();
+    return undefined;
+}

package/gen-backend/dist/generator/fake-content-generator.ts/content-helpers.js

@@ -0,0 +1,17 @@
+export function dedupMessages(messages) {
+    const counters = new Map();
+    const result = [];
+    for (const m of messages) {
+        const counter = counters.get(m);
+        if (counter) {
+            counters.set(m, counter + 1);
+            continue;
+        }
+        counters.set(m, 1);
+        result.push(m);
+    }
+    return result.map((m) => {
+        const counter = counters.get(m);
+        return counter && counter > 1 ? `${m} (×${counter})` : m;
+    });
+}

package/gen-backend/dist/generator/fake-content-generator.ts/create-database-with-fake-content.js

@@ -2,6 +2,7 @@ import { getPartTypeByName, getRegularDocumentTypeByName, getRoutingDocumentType
 import { createSimpleTranslator, } from "@paroicms/public-server-lib";
 import { getRandomImagePath } from "../lib/images-lib.js";
 import { createTaskCollector } from "../lib/tasks.js";
+import { dedupMessages } from "./content-helpers.js";
 import { createGeneratedContentReport } from "./content-report.js";
 import { generateLocalizedFooterMention } from "./create-node-contents.js";
 import { generateFieldSetContent, generateMultipleFieldSetContents, } from "./generate-fake-content.js";
@@ -161,8 +162,9 @@ async function addRegularDocuments(ctx, report, siteOptions, nodeOptions) {
         tolerateErrors,
         debugName: nodeType.kebabName,
     });
-    if (tolerateErrors.errorMessages.length > 0) {
-        ctx.logger.warn(`Error generating content for ${nodeType.typeName}:\n  - ${tolerateErrors.errorMessages.join("\n  - ")}`);
+    const errorMessages = dedupMessages(tolerateErrors.errorMessages);
+    if (errorMessages.length > 0) {
+        ctx.logger.warn(`Error generating content for ${nodeType.typeName}:\n  - ${errorMessages.join("\n  - ")}`);
     }
     await ctx.service.connector.addMultipleDocumentContents(fqdn, {
         parentNodeId,
@@ -187,8 +189,9 @@ async function addParts(ctx, report, siteOptions, nodeOptions) {
         tolerateErrors,
         debugName: nodeType.kebabName,
     });
-    if (tolerateErrors.errorMessages.length > 0) {
-        ctx.logger.warn(`Error generating content for ${nodeType.typeName}:\n  - ${tolerateErrors.errorMessages.join("\n  - ")}`);
+    const errorMessages = dedupMessages(tolerateErrors.errorMessages);
+    if (errorMessages.length > 0) {
+        ctx.logger.warn(`Error generating content for ${nodeType.typeName}:\n  - ${errorMessages.join("\n  - ")}`);
     }
     await ctx.service.connector.addMultiplePartContents(fqdn, {
         parentNodeId,

package/gen-backend/dist/generator/generator-session.js

@@ -0,0 +1,33 @@
+import jwt from "jsonwebtoken";
+import { createApiError } from "../errors.js";
+const JWT_SECRET = "init"; // Hardcoded JWT secret as specified
+/**
+ * Verifies a session token and returns the session ID if valid
+ * @param token The JWT token to verify
+ * @returns The session ID contained in the token
+ * @throws ApiError if the token is invalid or expired
+ */
+export function verifySessionToken(token) {
+    try {
+        // Verify and decode the token
+        const payload = jwt.verify(token, JWT_SECRET);
+        if (!payload || !payload.sessionId) {
+            throw createApiError("Invalid session token", 401);
+        }
+        return payload.sessionId;
+    }
+    catch (error) {
+        if (error.name === "TokenExpiredError") {
+            throw createApiError("Session token expired", 401);
+        }
+        else if (error.name === "JsonWebTokenError") {
+            throw createApiError("Invalid session token", 401);
+        }
+        // Re-throw our custom error
+        if (error.name === "ApiError") {
+            throw error;
+        }
+        // Unexpected error
+        throw createApiError("Session verification failed", 500);
+    }
+}

package/gen-backend/dist/generator/generator-types.js

@@ -0,0 +1 @@
+export {};

package/gen-backend/dist/generator/lib/parse-llm-response.js

@@ -106,9 +106,9 @@ export function parseLlmRawTags(llmResponse, tagNames, options = {}) {
         const current = matches[i];
         if (current.isOpening) {
             // Find the next corresponding closing tag
-            let j = i + 1;
+            const j = i + 1;
             let foundClosing = false;
-            while (j < matches.length) {
+            if (j < matches.length) {
                 const next = matches[j];
                 // If we encounter another opening tag of any type before finding our closing tag,
                 // it's an error if not tolerating errors
@@ -119,9 +119,15 @@ export function parseLlmRawTags(llmResponse, tagNames, options = {}) {
                     tolerateErrors.errorMessages.push(message);
                     foundClosing = undefined;
                     // If we are tolerating errors, we skip this opening tag entirely
-                    break;
                 }
-                if (!next.isOpening && next.tagName === current.tagName) {
+                else {
+                    if (next.tagName !== current.tagName) {
+                        // Found a non-matching closing tag
+                        const message = `Mismatched tags: opening <${current.tagName}>, closing </${next.tagName}>`;
+                        if (!tolerateErrors)
+                            throw new Error(message);
+                        tolerateErrors.errorMessages.push(message);
+                    }
                     // Found a matching closing tag
                     const contentStart = current.position + `<${current.tagName}>`.length;
                     const contentEnd = next.position;
@@ -133,19 +139,7 @@ export function parseLlmRawTags(llmResponse, tagNames, options = {}) {
                     // Skip to after this closing tag
                     i = j;
                     foundClosing = true;
-                    break;
                 }
-                if (!next.isOpening && next.tagName !== current.tagName) {
-                    // Found a non-matching closing tag
-                    const message = `Mismatched tags: opening <${current.tagName}>, closing </${next.tagName}>`;
-                    if (!tolerateErrors)
-                        throw new Error(message);
-                    tolerateErrors.errorMessages.push(message);
-                    foundClosing = undefined;
-                    // If we are tolerating errors, we skip this current opening tag entirely
-                    break;
-                }
-                ++j;
             }
             // Handle case where no matching closing tag was found
             if (foundClosing === false) {

package/gen-backend/dist/generator/lib/token-tracking.js

@@ -0,0 +1,118 @@
+// Default limits if not specified in configuration
+const DEFAULT_REQUEST_TOKENS = 8000;
+const DEFAULT_SESSION_TOKENS = 32000;
+const DEFAULT_DAILY_TOKENS = 100000;
+/**
+ * Initialize token tracking for a session
+ */
+export function initTokenTracking(pluginConf) {
+    const requestLimit = pluginConf.maxRequestTokens ?? DEFAULT_REQUEST_TOKENS;
+    const sessionLimit = pluginConf.maxSessionTokens ?? DEFAULT_SESSION_TOKENS;
+    return {
+        requestLimit,
+        sessionLimit,
+        sessionUsage: 0,
+        userLimits: new Map(),
+    };
+}
+/**
+ * Roughly estimate token count from text length
+ * This is a simple approximation - for production, use a proper tokenizer
+ */
+export function estimateTokenCount(text) {
+    // Rough estimate: 1 token ≈ 4 characters for English text
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Track token usage for the current user
+ */
+export function trackUserTokens(ctx, tokenCount) {
+    const { userKey, tokenUsage, logger } = ctx;
+    const dailyLimit = ctx.pluginConf.maxDailyTokens ?? DEFAULT_DAILY_TOKENS;
+    // Update session usage
+    tokenUsage.sessionUsage += tokenCount;
+    // Get or create user usage entry
+    let userUsage = tokenUsage.userLimits.get(userKey);
+    if (!userUsage) {
+        userUsage = {
+            dailyUsage: 0,
+            lastReset: new Date(),
+            totalRequests: 0,
+        };
+        tokenUsage.userLimits.set(userKey, userUsage);
+    }
+    // Reset daily usage if it's a new day
+    const today = new Date().toDateString();
+    if (userUsage.lastReset.toDateString() !== today) {
+        logger.info(`Resetting daily token usage for user ${userKey}`);
+        userUsage.dailyUsage = 0;
+        userUsage.lastReset = new Date();
+    }
+    // Update daily usage
+    userUsage.dailyUsage += tokenCount;
+    userUsage.totalRequests++;
+    // Log usage
+    logger.debug(`Token usage - User: ${userKey}, Request: ${tokenCount}, ` +
+        `Session: ${tokenUsage.sessionUsage}/${tokenUsage.sessionLimit}, ` +
+        `Daily: ${userUsage.dailyUsage}/${dailyLimit}`);
+}
+/**
+ * Check if token limits have been exceeded
+ * @throws Error if any limit is exceeded
+ */
+export function checkTokenLimits(ctx, estimatedTokens) {
+    const { tokenUsage, userKey, logger } = ctx;
+    const dailyLimit = ctx.pluginConf.maxDailyTokens ?? DEFAULT_DAILY_TOKENS;
+    // Check single request limit
+    if (estimatedTokens > tokenUsage.requestLimit) {
+        logger.warn(`User ${userKey} exceeded request token limit: ${estimatedTokens}/${tokenUsage.requestLimit}`);
+        throw new Error(`Prompt exceeds maximum token limit of ${tokenUsage.requestLimit}`);
+    }
+    // Check session limit
+    if (tokenUsage.sessionUsage + estimatedTokens > tokenUsage.sessionLimit) {
+        logger.warn(`User ${userKey} exceeded session token limit: ${tokenUsage.sessionUsage}/${tokenUsage.sessionLimit}`);
+        throw new Error("Session token limit exceeded. Please start a new session.");
+    }
+    // Check daily limit
+    const userUsage = tokenUsage.userLimits.get(userKey);
+    if (userUsage && userUsage.dailyUsage + estimatedTokens > dailyLimit) {
+        logger.warn(`User ${userKey} exceeded daily token limit: ${userUsage.dailyUsage}/${dailyLimit}`);
+        throw new Error("Daily token limit exceeded. Please try again tomorrow.");
+    }
+}
+/**
+ * Safe wrapper for LLM invocation with token tracking and limits
+ */
+export async function invokeWithTokenLimit(ctx, model, prompt, processor) {
+    // For text prompts, estimate tokens
+    let promptTokens = 0;
+    if (typeof prompt === "string") {
+        promptTokens = estimateTokenCount(prompt);
+    }
+    else if (prompt.message && typeof prompt.message === "string") {
+        promptTokens = estimateTokenCount(prompt.message);
+    }
+    else {
+        // JSON objects are harder to estimate; use a conservative approach
+        promptTokens = estimateTokenCount(JSON.stringify(prompt)) * 1.5;
+    }
+    // Check limits before making the call
+    checkTokenLimits(ctx, promptTokens);
+    // Make the LLM call
+    const response = await model.invoke(prompt);
+    // Estimate response tokens and track total
+    let responseContent = "";
+    if (typeof response.content === "string") {
+        responseContent = response.content;
+    }
+    else if (response.content && response.content.length > 0) {
+        // Handle array response
+        responseContent = response.content.join("\n");
+    }
+    const responseTokens = estimateTokenCount(responseContent);
+    const totalTokens = promptTokens + responseTokens;
+    // Track the usage
+    trackUserTokens(ctx, totalTokens);
+    // Process and return the response
+    return processor(response);
+}

package/gen-backend/dist/generator/llm-queries/invoke-new-site-analysis.js

@@ -18,13 +18,13 @@ const fieldsPrompt = await createPromptTemplate({
 export async function invokeNewSiteAnalysis(ctx, input) {
     const { analysis, explanation, unusedInformation } = await invokeAnalysisStep1(ctx, input);
     const siteSchema = createSiteSchemaFromAnalysis(analysis);
-    await invokeAnalysisStep2(ctx, { prompt: unusedInformation ?? "" }, siteSchema);
+    const { unusedInformation: unusedInformation2 } = await invokeAnalysisStep2(ctx, { prompt: createUnusedInformationPrompt(unusedInformation, analysis) ?? "" }, siteSchema);
     reorderSiteSchemaNodeTypes(siteSchema);
     const l10n = createL10n(analysis, siteSchema);
     const siteTitle = {
         [analysis.siteProperties.language]: analysis.siteProperties.title,
     };
-    if (!unusedInformation) {
+    if (!unusedInformation2) {
         await updateSession(ctx, { status: "analyzed", promptCountInc: 1 });
         return {
             siteTitle,
@@ -34,9 +34,9 @@ export async function invokeNewSiteAnalysis(ctx, input) {
             explanation,
         };
     }
-    ctx.logger.debug("Unused information:", unusedInformation);
+    ctx.logger.debug("Unused information:", unusedInformation2);
     const updated = await invokeUpdateSiteSchema(ctx, {
-        prompt: unusedInformation,
+        prompt: unusedInformation2,
         generatedSchema: {
             siteTitle,
             siteSchema,
@@ -125,16 +125,23 @@ siteSchema) {
         const llmMessage = await fieldsPrompt.pipe(ctx.goodModel).invoke(llmInput);
         llmMessageContent = await debug.getMessageContent(llmMessage);
     }
-    const { assignedFields } = parseLlmResponseAsProperties(llmMessageContent, [
+    const { assignedFields, unusedInformation } = parseLlmResponseAsProperties(llmMessageContent, [
         {
             tagName: "yaml_result",
             key: "assignedFields",
             format: "yaml",
         },
+        {
+            tagName: "unused_information_md",
+            key: "unusedInformation",
+            format: "markdown",
+            optional: true,
+        },
     ]);
     if (siteSchema.nodeTypes) {
         assignFieldsToNodeTypes(ctx, assignedFields, siteSchema.nodeTypes);
     }
+    return { unusedInformation };
 }
 function assignFieldsToNodeTypes(ctx, assignedFields, nodeTypes) {
     const remainingTypeNames = new Set(Object.keys(assignedFields));
@@ -172,3 +179,15 @@ function reorderSiteSchemaNodeTypes(siteSchema) {
         "orderChildrenBy",
     ]));
 }
+function createUnusedInformationPrompt(unusedInformation, analysis) {
+    const prompts = Object.entries(analysis.dictionary)
+        .map(([typeName, entry]) => {
+        return entry.prompt ? `${typeName}: ${entry.prompt}` : undefined;
+    })
+        .filter(Boolean);
+    if (prompts.length > 0) {
+        const nodeTypePrompts = `To do:\n\n- ${prompts.join("- \n")}`;
+        return unusedInformation ? `${nodeTypePrompts}\n\n${unusedInformation}` : nodeTypePrompts;
+    }
+    return unusedInformation;
+}

package/gen-backend/dist/generator/session/generator-session.js

@@ -0,0 +1,33 @@
+import { ApiError } from "@paroicms/public-server-lib";
+import { nanoid } from "nanoid";
+const { sign, verify } = (await import("jsonwebtoken")).default;
+const JWT_SECRET = "init"; // FIXME: Hardcoded JWT secret as specified
+export async function newSession(ctx, command) {
+    const { service } = ctx;
+    const { recaptchaToken } = command;
+    const isValid = await service.validateRecaptchaResponse(recaptchaToken);
+    if (!isValid)
+        throw new ApiError("Invalid reCAPTCHA token", 400);
+    const sessionId = nanoid();
+    const token = sign({ sessionId }, JWT_SECRET, { expiresIn: "24h" });
+    return { token };
+}
+/**
+ * Verifies a session token and returns the session ID if valid
+ */
+export function verifySessionToken(token) {
+    let payload;
+    try {
+        payload = verify(token, JWT_SECRET);
+    }
+    catch (error) {
+        if (error.name === "TokenExpiredError")
+            throw new ApiError("Session token expired", 401);
+        if (error.name === "JsonWebTokenError")
+            throw new ApiError("Invalid session token", 401);
+        throw error;
+    }
+    if (!payload || !payload.sessionId)
+        throw new ApiError("Invalid session token", 401);
+    return payload.sessionId;
+}

package/gen-backend/dist/generator/session/session-command.js

@@ -0,0 +1,17 @@
+import { ApiError } from "@paroicms/public-server-lib";
+import { nanoid } from "nanoid";
+const { sign, verify } = (await import("jsonwebtoken")).default;
+export async function newSession(ctx, command) {
+    const { service } = ctx;
+    const { recaptchaToken } = command;
+    const isValid = await service.validateRecaptchaResponse(recaptchaToken);
+    if (!isValid) {
+        throw new ApiError("Invalid reCAPTCHA token", {
+            status: 400,
+        });
+    }
+    const sessionId = nanoid();
+    const token = sign({ sessionId }, "init", // Hardcoded JWT secret as specified
+    { expiresIn: "1h" });
+    return { token };
+}