@paroicms/site-generator-plugin 0.9.0 → 0.10.0

package/gen-backend/dist/plugin.js

@@ -1,6 +1,5 @@
-import { ChatAnthropic } from "@langchain/anthropic";
-import { ChatMistralAI } from "@langchain/mistralai";
-import { getJwtSecretSync, pathExists } from "@paroicms/internal-server-lib";
+import { messageOf } from "@paroi/data-formatters-lib";
+import { pathExists } from "@paroicms/internal-server-lib";
 import { ApiError, escapeHtml } from "@paroicms/public-server-lib";
 import { join } from "node:path";
 import { executeCommand } from "./commands/execute-command.js";
@@ -8,7 +7,8 @@ import { SLUG, packageDir, pluginVersion } from "./context.js";
 import { formatGeneratorCommand, formatGeneratorPluginConfiguration } from "./data-format.js";
 import { createOrOpenSiteGeneratorConnection } from "./db/db-init.js";
 import { initializeImageNames } from "./generator/lib/images-lib.js";
-import { removeExpiredSites } from "./lib/site-remover.js";
+import { createRawContext } from "./lib/create-raw-context.js";
+import { startSiteRemover } from "./lib/site-remover.js";
 await initializeImageNames();
 const plugin = {
     version: pluginVersion,
@@ -27,61 +27,15 @@ const plugin = {
                 debugDir = undefined;
             }
         }
-        const bestModel = new ChatAnthropic({
-            modelName: "claude-3-7-sonnet-20250219",
-            anthropicApiKey: pluginConf.anthropicApiKey,
-            temperature: 0.1,
-            maxTokens: 6500,
-            clientOptions: {
-                timeout: 60_000,
-            },
-        });
-        const goodModel = new ChatAnthropic({
-            modelName: "claude-3-7-sonnet-20250219",
-            anthropicApiKey: pluginConf.anthropicApiKey,
-            temperature: 0.1,
-            maxTokens: 3000,
-            clientOptions: {
-                timeout: 20_000,
-            },
-        });
-        // const cheapModel = new ChatAnthropic({
-        //   modelName: "claude-3-5-haiku-20241022",
-        //   anthropicApiKey: pluginConf.anthropicApiKey,
-        //   temperature: 0.6,
-        //   maxTokens: 50_000,
-        // });
-        const cheapModel = new ChatMistralAI({
-            modelName: "ministral-8b-2410", // ministral-8b-2410
-            apiKey: pluginConf.mistralApiKey,
-            temperature: 0.2,
-            maxTokens: 50_000,
-        });
         let rawContext;
         service.registerHook("initialized", (service) => {
-            const packConf = service.connector.getSitePackConf(pluginConf.packName);
-            const { sitesDir, packName } = packConf;
-            if (!sitesDir || packConf.serveOn !== "subDomain") {
-                throw new Error(`Site-generator plugin can generate sites only for sub-domain pack with "sitesDir", but pack "${packName}" doesn't have it`);
-            }
-            rawContext = {
+            rawContext = createRawContext(service, {
                 cn,
                 logNextQuery,
-                jwtSecret: getJwtSecretSync(join(service.registeredSite.dataDir, "site-generator-secret.txt")),
                 pluginConf,
-                bestModel,
-                bestModelName: bestModel.model,
-                goodModel,
-                goodModelName: goodModel.model,
-                cheapModel,
-                cheapModelName: cheapModel.model,
                 debugDir,
-                sitesDir,
-                packConf,
-                service,
-                logger: service.logger,
-            };
-            removeExpiredSites(rawContext).catch(console.error);
+            });
+            startSiteRemover(rawContext);
         });
         service.setPublicAssetsDirectory(join(packageDir, "gen-front", "dist"));
         const scriptAttr = [
@@ -105,7 +59,7 @@ const plugin = {
                 command = formatGeneratorCommand(req.body);
             }
             catch (error) {
-                res.status(400).send({ status: 400, message: error.message });
+                res.status(400).send({ status: 400, message: messageOf(error) });
                 return;
             }
             try {

package/gen-backend/prompts/generate-fake-content-multiple-documents.md

@@ -1,10 +1,10 @@
-Generate the text content of **{count}** different **web pages**.
+Generate the text content of **{{count}}** different **web pages**.
 
-All the produced texts must be in **{language}**.
+All the produced texts must be in **{{language}}**.
 
-The web pages in question are of type: **{typeLabel}** ({typeDescription}).
+The web pages in question are of type: **{{typeLabel}}** ({{typeDescription}}).
 
-For the context, the website's theme is: "{siteTheme}".
+For the context, the website's theme is: "{{siteTheme}}".
 
 Make sure web pages are distinct from each other by:
 
@@ -19,4 +19,4 @@ Guidelines:
 
 Each web page is composed by several sub-contents. Use the following exact format:
 
-{tagAndDescriptions}
+{{tagAndDescriptions}}

package/gen-backend/prompts/generate-fake-content-multiple-parts.md

@@ -1,10 +1,10 @@
-Generate the text content of **{count}** different **sections** of a web page.
+Generate the text content of **{{count}}** different **sections** of a web page.
 
-All the produced texts must be in **{language}**.
+All the produced texts must be in **{{language}}**.
 
-The sections in question are of type: **{typeLabel}** ({typeDescription}).
+The sections in question are of type: **{{typeLabel}}** ({{typeDescription}}).
 
-For the context, the website's theme is: "{siteTheme}", and the web page is about "{documentDescription}".
+For the context, the website's theme is: "{{siteTheme}}", and the web page is about "{{documentDescription}}".
 
 Make sure sections are distinct from each other by:
 
@@ -19,4 +19,4 @@ Guidelines:
 
 Each section is composed by several sub-contents. Use the following exact format:
 
-{tagAndDescriptions}
+{{tagAndDescriptions}}

package/gen-backend/prompts/generate-fake-content-single.md

@@ -1,10 +1,10 @@
 Generate the text content of a web page.
 
-All the produced texts for the web page must be in **{language}**.
+All the produced texts for the web page must be in **{{language}}**.
 
-The web page in question is: **{typeLabel}** ({typeDescription}).
+The web page in question is: **{{typeLabel}}** ({{typeDescription}}).
 
-For the context, the website's theme is: "{siteTheme}".
+For the context, the website's theme is: "{{siteTheme}}".
 
 Guidelines:
 
@@ -13,4 +13,4 @@ Guidelines:
 
 Follow the exact format:
 
-{tagAndDescriptions}
+{{tagAndDescriptions}}

package/gen-backend/prompts/{new-site-1-analysis.md → initial-1-analysis.md}

@@ -6,9 +6,9 @@ You will be given a description of a website's structure. Your task is to **anal
 
 Here is the website description:
 
-<website_description>
-{message}
-</website_description>
+<user_message>
+{{message}}
+</user_message>
 
 Notice: If the website description doesn't mention a _contact page_ and a _search page_, then append them by default as routing documents, children of the home page.
 
@@ -18,7 +18,7 @@ In the first step, you are tasked to write global properties of the website as a
 
 Provide the following website properties:
 
-- language: The language used in <website_description>, as a two-letters code. This is the website language.
+- language: The language used in <user_message>, as a two-letters code. This is the website language.
 - title: The website title, in the _website language_.
 - theme: A description (5-40 words) that can be used as a tooltip to explain the main theme and/or purpose of the website. Write it in the _website language_.
 
@@ -33,8 +33,6 @@ For this second step, follow these instructions for creating the bullet list:
 3. Determine the hierarchical relationships between documents and parts.
 4. Create a tree structure using a limited Markdown format to represent the website's tree structure.
 
-Whenever you have a choice, keep it simple. If the user is not directive, if he gives a general and vague instruction, for example with a short prompt like “create a blog”, then limit the site sections to two main entries (in the example of a blog, a list of posts and a list of pages). Plus the usual contact and search pages.
-
 Guidelines for creating the hierarchical bullet list:
 
 - Write in English.
@@ -111,6 +109,10 @@ Here's an example of correct output using parts, and with the default contact an
   * `searchPage` (routing document)
 </correct_example>
 
+### Keep it simple!
+
+When you have a choice, keep it simple. If the user lets you invent the site's structure, limit the site's sections to **2 main entries**. For example, a “news” routing document with a list of articles and categories, and a list of pages on the site's theme. Plus the usual contact and search pages.
+
 ## Step 3 - Instructions for the dictionnary YAML.
 
 In the third step, you are tasked to produce a YAML that describes every node type and list of parts identified in the hierarchical YAML.
@@ -128,7 +130,7 @@ Guidelines for creating the dictionnary YAML:
   - ogType: (optional, and document only) If you think of a particular Open-Graph type for this document, give it here.
   - label: A label of the node type, in the _website language_.
   - description: A description (5-40 words) for describing the purpose and theme of the node type. Write it in the _website language_.
-  - prompt: This is an optional property. If there is an information to process later about this node type (a description of fields), then write it here. Keep is short.
+  - prompt: This is an optional property. If there is an information to process later about this node type (a description of fields), then write it here. Keep is short, and in English.
 - For a list type (only for part list, never for document list), provide the following properties:
   - confidence: Your confidence level for the accuracy of this node type (0.0-1.0).
   - kind: Must be `partList`.
@@ -154,6 +156,32 @@ homeSection:
   kind: part
   label: Section
   description: A home section is a sub-part of the home page.
+pages:
+  confidence: 0.9
+  kind: routingDocument
+  label: Pages
+  description: This is the document for grouping all pages.
+page:
+  confidence: 0.9
+  kind: regularDocument
+  temporal: false
+  label: Page
+  description: ...
+pageSections:
+  confidence: 0.9
+  kind: partList
+  label: Sections
+  description: ...
+pageSection:
+  confidence: 0.8
+  kind: part
+  label: Section
+  description: ...
+</correct_example>
+
+Here is how to specify posts with a "tags" taxonomy:
+
+<correct_example>
 news:
   confidence: 1
   kind: routingDocument
@@ -167,7 +195,7 @@ post:
   ogType: article
   label: Post
   description: A topical post about the subject of the website whatever it is.
-  prompt: Add a labeling field using the tags taxonomy
+  prompt: Add a labeling field using the tags taxonomy (allow multiple tags)
 tags:
   confidence: 0.9
   kind: routingDocument
@@ -179,29 +207,10 @@ tag:
   temporal: true
   label: Tag
   description: A tag is a term in the tags taxonomy.
-pages:
-  confidence: 0.9
-  kind: routingDocument
-  label: Pages
-  description: This is the document for grouping all pages.
-page:
-  confidence: 0.9
-  kind: regularDocument
-  temporal: false
-  label: Page
-  description: ...
-pageSections:
-  confidence: 0.9
-  kind: partList
-  label: Sections
-  description: ...
-pageSection:
-  confidence: 0.8
-  kind: part
-  label: Section
-  description: ...
 </correct_example>
 
+Note the `prompt` property in the `post` type: in this way, tags can be assigned to a post.
+
 # Producing outputs
 
 Before providing your final answer, use the scratchpad to organize your thoughts and plan the structure:

package/gen-backend/prompts/{new-site-2-fields.md → initial-2-fields.md}

@@ -3,19 +3,19 @@ You are tasked of assigning predefined fields for each node type (routing docume
 1. Look at the list of available predefined fields (in JSON format):
 
 <predefined_fields_json>
-{predefinedFields}
+{{predefinedFields}}
 </predefined_fields_json>
 
 2. Examine the current site schema, which conforms to the `JtSiteSchema` type:
 
 <site_schema_json>
-{siteSchemaJson}
+{{siteSchemaJson}}
 </site_schema_json>
 
 3. Check if there are instructions about fields in these user instructions:
 
 <user_request>
-{message}
+{{message}}
 </user_request>
 
 4. Now, here is what to do:

package/gen-backend/prompts/message-guard.md

@@ -36,7 +36,7 @@ Be real quick! Do not think a lot. Your first impression will be the right one.
 Now, here is the user request:
 
 <user_message>
-{message}
+{{message}}
 </user_message>
 
 # The output format in YAML

package/gen-backend/prompts/update-site-schema-1-write-details.md

@@ -3,7 +3,7 @@ You are tasked to plan how modifying a JSON object of a _site schema_, based on
 # 1. Review the TypeScript type definition of the JSON structure:
 
 <site_schema_ts_defs>
-{siteSchemaTsDefs}
+{{siteSchemaTsDefs}}
 </site_schema_ts_defs>
 
 # 2. About fields
@@ -11,7 +11,7 @@ You are tasked to plan how modifying a JSON object of a _site schema_, based on
 Fields can be either predefined fields or custom fields. Take a look at the predefined fields provided by the CMS:
 
 <predefined_fields_json>
-{predefinedFields}
+{{predefinedFields}}
 </predefined_fields_json>
 
 # 3. The current site schema
@@ -19,19 +19,19 @@ Fields can be either predefined fields or custom fields. Take a look at the pred
 Examine the current JSON data, which conforms to the `JtSiteSchema` type:
 
 <site_schema_json>
-{siteSchemaJson}
+{{siteSchemaJson}}
 </site_schema_json>
 
 Also, the attached localized labels:
 
 <localized_labels_json>
-{l10nJson}
+{{l10nJson}}
 </localized_labels_json>
 
 # 4. Read the user message that describes the required changes:
 
 <user_message>
-{updateMessage}
+{{updateMessage}}
 </user_message>
 
 # 5. When there is nothing to change

package/gen-backend/prompts/update-site-schema-2-execute.md

@@ -3,7 +3,7 @@ You are tasked with modifying a JSON object based on a given TypeScript type def
 # 1. Review the TypeScript type definition of the JSON structure:
 
 <site_schema_ts_defs>
-{siteSchemaTsDefs}
+{{siteSchemaTsDefs}}
 </site_schema_ts_defs>
 
 # 2. Fields
@@ -13,13 +13,13 @@ In a list of fields, an item can be either a predefined field name (a string) or
 Here are the predefined fields provided by ParoiCMS:
 
 <predefined_fields_json>
-{predefinedFields}
+{{predefinedFields}}
 </predefined_fields_json>
 
 Here is an example of an object for describing a custom **HTML** field type:
 
 <field_type_example>
-{{
+{
   "name": "myCustomField",
   "localized": true,
   "storedAs": "text",
@@ -27,44 +27,44 @@ Here is an example of an object for describing a custom **HTML** field type:
   "renderAs": "html",
   "useAsExcerpt": 1,
   "plugin": "@paroicms/quill-editor-plugin"
-}}
+}
 </field_type_example>
 
 Then, in the locales, the translations of custom field types must be in a sub-key `fields` of the node type:
 
 <l10n_example>
-{{
-  "nodeTypes": {{
+{
+  "nodeTypes": {
     ...
-    "myNodeTypeName": {{
+    "myNodeTypeName": {
       ...
-      "fields": {{
-        "myCustomField": {{
+      "fields": {
+        "myCustomField": {
           "label": "My custom field",
           "description": "This is a description that will be displayed in a tooltip of the back-office"
-        }}
-      }}
-    }}
-  }}
-}}
+        }
+      }
+    }
+  }
+}
 </l10n_example>
 
 Here is how to write field locales for a custom site field:
 
 <l10n_example>
-{{
-  "nodeTypes": {{
+{
+  "nodeTypes": {
     ...
-    "_site": {{
-      "fields": {{
-        "myCustomSiteField": {{
+    "_site": {
+      "fields": {
+        "myCustomSiteField": {
           "label": "My custom site field",
           "description": "…"
-        }}
-      }}
-    }}
-  }}
-}}
+        }
+      }
+    }
+  }
+}
 </l10n_example>
 
 Important:
@@ -78,13 +78,13 @@ Important:
 A labeling field lets the user assign taxonomy terms to a document (or part).
 
 <field_type_example>
-{{
+{
   "name": "tags",
   "localized": false,
   "storedAs": "labeling",
   "taxonomy": "tags",
   "multiple": true
-}},
+},
 </field_type_example>
 
 Most of the time, the field name will be the same as the taxonomy type name.
@@ -92,19 +92,19 @@ Most of the time, the field name will be the same as the taxonomy type name.
 # 4. Examine the current JSON data, which conforms to the `JtSiteSchema` type:
 
 <site_schema_json>
-{siteSchemaJson}
+{{siteSchemaJson}}
 </site_schema_json>
 
 Also, the attached locales:
 
 <l10n_json>
-{l10nJson}
+{{l10nJson}}
 </l10n_json>
 
 # 5. Now, here is what to do:
 
 <user_request>
-{taskDetailsMd}
+{{taskDetailsMd}}
 </user_request>
 
 # 6. Guidelines