@paroicms/site-generator-plugin 0.7.0 → 0.8.1

package/gen-backend/dist/lib/site-remover.js

@@ -0,0 +1,39 @@
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+let runningId;
+export function startSiteRemover(ctx) {
+    clearInterval(runningId);
+    let sitesDir;
+    runningId = setInterval(async () => {
+        if (!sitesDir)
+            return;
+        try {
+            await removeExpiredSites(ctx);
+        }
+        catch (error) {
+            ctx.logger.error("[site-remover]", error);
+        }
+    }, 1000 * 60 * 60).unref(); // Check every hour
+    return {
+        stop() {
+            clearInterval(runningId);
+        },
+    };
+}
+export async function removeExpiredSites(ctx) {
+    const { sitesDir, packConf, service } = ctx;
+    const now = new Date();
+    const expirationTime = new Date(now.getTime() - 1000 * 60 * 60 * 24 * 2); // 2 days
+    // List entries in sitesDir, filter directories named as uuidv4, and remove them.
+    const uuidv4Regex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+    const entries = await readdir(sitesDir, { withFileTypes: true });
+    for (const entry of entries) {
+        if (!entry.isDirectory() || !uuidv4Regex.test(entry.name))
+            continue;
+        const dirPath = join(sitesDir, entry.name);
+        const st = await stat(dirPath);
+        if (st.ctime < expirationTime) {
+            await service.connector.removeSite(`${entry.name}.${packConf.parentDomain}`);
+        }
+    }
+}

package/gen-backend/dist/plugin.js

@@ -1,22 +1,24 @@
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatMistralAI } from "@langchain/mistralai";
-import { strVal } from "@paroi/data-formatters-lib";
-import { pathExists } from "@paroicms/internal-server-lib";
+import { getJwtSecretSync, pathExists } from "@paroicms/internal-server-lib";
 import { ApiError, escapeHtml } from "@paroicms/public-server-lib";
-import { readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { projectDir } from "./context.js";
+import { join } from "node:path";
+import { executeCommand } from "./commands/execute-command.js";
+import { SLUG, packageDir, pluginVersion } from "./context.js";
 import { formatGeneratorCommand, formatGeneratorPluginConfiguration } from "./data-format.js";
-import { executeCommand } from "./generator/actions.js";
+import { createOrOpenSiteGeneratorConnection } from "./db/db-init.js";
 import { initializeImageNames } from "./generator/lib/images-lib.js";
-const packageDir = dirname(projectDir);
-const version = strVal(JSON.parse(readFileSync(join(packageDir, "package.json"), "utf-8")).version);
-const SLUG = "site-generator";
+import { removeExpiredSites } from "./lib/site-remover.js";
 await initializeImageNames();
 const plugin = {
-    version,
+    version: pluginVersion,
     slug: SLUG,
     async siteInit(service) {
+        const { cn, logNextQuery } = await createOrOpenSiteGeneratorConnection({
+            sqliteFile: join(service.registeredSite.dataDir, "site-generator.sqlite"),
+            canCreate: true,
+            logger: service.logger,
+        });
         const pluginConf = formatGeneratorPluginConfiguration(service.configuration);
         let debugDir = pluginConf.debugDir;
         if (debugDir) {
@@ -29,7 +31,7 @@ const plugin = {
             modelName: "claude-3-7-sonnet-20250219",
             anthropicApiKey: pluginConf.anthropicApiKey,
             temperature: 0.1,
-            maxTokens: 4096,
+            maxTokens: 6500,
             clientOptions: {
                 timeout: 60_000,
             },
@@ -38,7 +40,7 @@ const plugin = {
             modelName: "claude-3-7-sonnet-20250219",
             anthropicApiKey: pluginConf.anthropicApiKey,
             temperature: 0.1,
-            maxTokens: 4096,
+            maxTokens: 3000,
             clientOptions: {
                 timeout: 20_000,
             },
@@ -55,6 +57,32 @@ const plugin = {
             temperature: 0.2,
             maxTokens: 50_000,
         });
+        let rawContext;
+        service.registerHook("initialized", (service) => {
+            const packConf = service.connector.getSitePackConf(pluginConf.packName);
+            const { sitesDir, packName } = packConf;
+            if (!sitesDir || packConf.serveOn !== "subDomain") {
+                throw new Error(`Site-generator plugin can generate sites only for sub-domain pack with "sitesDir", but pack "${packName}" doesn't have it`);
+            }
+            rawContext = {
+                cn,
+                logNextQuery,
+                jwtSecret: getJwtSecretSync(join(service.registeredSite.dataDir, "site-generator-secret.txt")),
+                pluginConf,
+                bestModel,
+                bestModelName: bestModel.model,
+                goodModel,
+                goodModelName: goodModel.model,
+                cheapModel,
+                cheapModelName: cheapModel.model,
+                debugDir,
+                sitesDir,
+                packConf,
+                service,
+                logger: service.logger,
+            };
+            removeExpiredSites(rawContext).catch(console.error);
+        });
         service.setPublicAssetsDirectory(join(packageDir, "gen-front", "dist"));
         const scriptAttr = [
             ["type", "module"],
@@ -64,24 +92,14 @@ const plugin = {
         ];
         service.addHeadTag(`<link rel="stylesheet" href="${escapeHtml(`${service.pluginAssetsUrl}/gen-front.css`)}">`, `<script ${scriptAttr.map(([key, val]) => `${key}="${escapeHtml(val)}"`).join(" ")}></script>`);
         service.setPublicApiHandler(async (service, httpContext, relativePath) => {
+            if (!rawContext)
+                throw new Error("should be initialized");
+            const ctx = rawContext;
             const { req, res } = httpContext;
             if (relativePath !== "") {
                 res.status(404).send({ status: 404 });
                 return;
             }
-            const ctx = {
-                pluginConf,
-                packConf: service.connector.getSitePackConf(pluginConf.packName),
-                service,
-                logger: service.logger,
-                bestModel,
-                bestModelName: bestModel.model,
-                goodModel,
-                goodModelName: goodModel.model,
-                cheapModel,
-                cheapModelName: cheapModel.model,
-                debugDir,
-            };
             let command;
             try {
                 command = formatGeneratorCommand(req.body);

package/gen-backend/prompts/0-context.md

@@ -1,4 +1,4 @@
-We use **ParoiCMS** technology. With this technology, a web page is called a **document**. A website is a tree of documents. The home page is a document, the site section with news posts is a document, each post is a document. Each document has its own path in the URL.
+We use **ParoiCMS** for creating a website. With this technology, a web page is called a **document**. A website is a tree of documents. The home page is a document, the site section with news posts is a document, each post is a document. Each document has its own path in the URL.
 
 There is a special kind of documents that we want to detect: **routing documents** are the site sections. They can't be duplicated. They are never items of a list. For example, the homepage document, the search-page document, the "about us" document, the parent page of blog posts are _routing documents_. Other documents are **regular documents**, and they are always items of a list.
 
@@ -6,4 +6,4 @@ A document always has the following base attributes: a localized _title_, a _pub
 
 A document can contain lists of **parts**. A _part_ is a sub-section of a document, or of another _part_. A part always has a _publish date_ and a _draft_ flag. It may contain a sequence of fields and/or a sequence of child parts. A part is always an item of a list.
 
-Important: In the current version, we don't support any taxonomy. No categories, no tags etc.
+Any routing document which is parent of regular documents can be used as a **taxonomy**. Then, the terms are the regular child documents. Then a taxonomy can be used in any document or part, by declaring a **labeling field**.

package/gen-backend/prompts/new-site-1-analysis.md

@@ -100,7 +100,9 @@ Here's an example of correct output using parts, and with the default contact an
 * `home` (routing document)
   * list of `homeSection` (parts), list name: `homeSections`
   * `news` (routing document)
-    * list of `article` (regular documents)
+    * list of `post` (regular documents)
+    * `tags` (routing document)
+      * list of `tag` (regular documents)
   * `pages` (routing document)
     * list of `page` (regular documents)
       * list of `pageSection` (parts), list name: `pageSections`
@@ -125,6 +127,7 @@ Guidelines for creating the dictionnary YAML:
   - ogType: (optional, and document only) If you think of a particular Open-Graph type for this document, give it here.
   - label: A label of the node type, in the _website language_.
   - description: A description (5-40 words) for describing the purpose and theme of the node type. Write it in the _website language_.
+  - prompt: This is an optional property. If there is an information to process later about this node type (a description of fields), then write it here. Keep is short.
 - For a list type (only for part list, never for document list), provide the following properties:
   - confidence: Your confidence level for the accuracy of this node type (0.0-1.0).
   - kind: Must be `partList`.
@@ -155,14 +158,26 @@ news:
   kind: routingDocument
   entryPage: true
   label: News
-  description: This is the blog section of the website. The news document contains all the topical articles.
-article:
+  description: This is the blog section of the website. The news document contains all the topical posts.
+post:
   confidence: 0.8
   kind: regularDocument
   temporal: true
   ogType: article
-  label: Article
-  description: A topical article about the subject of the website whatever it is.
+  label: Post
+  description: A topical post about the subject of the website whatever it is.
+  prompt: Add a labeling field using the tags taxonomy
+tags:
+  confidence: 0.9
+  kind: routingDocument
+  label: Tags
+  description: Tags taxonomy for post documents.
+tag:
+  confidence: 0.9
+  kind: regularDocument
+  temporal: true
+  label: Tag
+  description: A tag is a term in the tags taxonomy.
 pages:
   confidence: 0.9
   kind: routingDocument

package/gen-backend/prompts/new-site-2-fields.md

@@ -33,6 +33,7 @@ Guidelines for creating the dictionnary YAML:
   - By default, for most of node types, if you are not sure about what could be the best fields, then remember that a document is a webpage and just use a `[htmlContent]`.
   - Except if there are specific instructions in the website description, here is the default value for the `_site` node type: `["logo", "footerMention"]`.
 - Gallery of medias: there is a predefined field named `"gallery"`. It contains a list of medias. The theme can render it as a carousel, a slider, an image gallery, a slideshow, etc.
+- This task is about predefined fields only. Custom fields will be added in a further step.
 
 Here is an example of expected output:
 

package/gen-backend/prompts/update-site-schema-2-execute.md

@@ -73,7 +73,23 @@ Important:
 - Never add an unknown predefined field.
 - The type name of the "site" node type is omitted from the JSON but its value is always `_site`.
 
-# 3. Examine the current JSON data, which conforms to the `JtSiteSchema` type:
+# 3. Labeling fields (using a taxonomy)
+
+A labeling field lets the user assign taxonomy terms to a document (or part).
+
+<field_type_example>
+{{
+  "name": "tags",
+  "localized": false,
+  "storedAs": "labeling",
+  "taxonomy": "tags",
+  "multiple": true
+}},
+</field_type_example>
+
+Most of the time, the field name will be the same as the taxonomy type name.
+
+# 4. Examine the current JSON data, which conforms to the `JtSiteSchema` type:
 
 <site_schema_json>
 {siteSchemaJson}
@@ -85,19 +101,19 @@ Also, the attached locales:
 {l10nJson}
 </l10n_json>
 
-# 4. Now, here is what to do:
+# 5. Now, here is what to do:
 
 <user_request>
 {taskDetailsMd}
 </user_request>
 
-# 5. Guidelines
+# 6. Guidelines
 
 - Don't assume how the CMS works. If you are not sure how to do something, don't do it.
 - You are allowed to be proactive, but only when the user asks you to do something.
 - Remember to adhere strictly to the TypeScript typing when making changes. If the update message requests changes that would violate the typing, then prioritize maintaining the correct structure over making those specific changes.
 
-# 6. Output
+# 7. Output
 
 If there is a change in the site schema, then provide the updated site schema in JSON within <updated_site_schema_json> tags. Otherwise, let this tag empty.