@griddo/cx 11.9.7 → 11.9.8-rc.1

package/exporter/commands/upload-search-content.ts

@@ -1,39 +1,214 @@
-#!/usr/bin/env node
+import type { PostSearchInfoResponse } from "../shared/types/api";
+import type { AIEmbeddingsResponse, PostSearchInfoProps } from "../shared/types/global";
+import type { GatsbyPageData } from "../ssg-adapters/gatsby/shared/types";
 
-import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
 
-import { envs } from "../constants";
-import { getConfig } from "../utils/core-utils";
-import { getInstanceDomains } from "../utils/domains";
-import {
-	startAIEmbeddings,
-	uploadRenderedSearchContentToAPI,
-} from "../utils/searches";
+import { throwError, withErrorHandler } from "../core/errors";
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { post } from "../services/api";
+import { AuthService } from "../services/auth";
+import { getInstanceDomains } from "../services/domains";
+import { getRenderPathsHydratedWithDomainFromDB } from "../services/render";
+import { AI_EMBEDDINGS, SEARCH } from "../shared/endpoints";
+import { GRIDDO_AI_EMBEDDINGS, GRIDDO_SEARCH_FEATURE } from "../shared/envs";
+import { ReadFromStoreError, UploadSearchError } from "../shared/errors";
+
+/**
+ * Save in the BBDD the content of a page parsed without HTML tags.
+ *
+ * @param props Object with parts of the final page object to be saved in the BBDD.
+ */
+async function postSearchInfo(props: PostSearchInfoProps) {
+	const { title, description, image, pageId, languageId, siteId, url, content, template } = props;
+
+	const response = await post<PostSearchInfoResponse>({
+		endpoint: SEARCH,
+		body: {
+			title,
+			description,
+			image,
+			pageId,
+			languageId,
+			siteId,
+			url,
+			template,
+			content,
+		},
+		useApiCacheDir: false,
+		logToFile: false,
+	});
+
+	return response;
+}
+
+function prepareHTMLContentForSearch(content: string): string {
+	// 1. Remove script, style, and other unwanted block tags and their content.
+	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
+	const tagsToRemove = ["meta", "link", "style", "script", "noscript", "nav", "header", "footer"];
+	const removeTagsRegex = new RegExp(`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`, "gis");
+	let processedContent = content.replace(removeTagsRegex, "");
+
+	// 2. Strip all remaining HTML tags.
+	processedContent = processedContent.replace(/<[^>]+>/g, " ");
+
+	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
+	processedContent = processedContent.replace(/\s+/g, " ").trim();
+
+	return processedContent;
+}
+
+/**
+ * Function that search in the `/public` dir the content info of the pages and
+ * send it to the search table in the ddbb using the API.
+ * @todo Utilizar la carpeta page-data en lugar de la carpeta store puesto que
+ * esta ya no es persistente.
+ */
+async function uploadRenderedSearchContentToAPI(options: {
+	htmlContentDir: string;
+	jsonContentDir: string;
+}) {
+	const { htmlContentDir, jsonContentDir } = options;
+
+	if (!(await pathExists(jsonContentDir)) || !(await pathExists(htmlContentDir))) {
+		GriddoLog.info(
+			`Skipping uploading content to the search endpoint because it has not exported sites.`,
+		);
 
-async function main() {
-	if (!envs.GRIDDO_SEARCH_FEATURE) {
 		return;
 	}
 
-	const domains = await getInstanceDomains();
-	const config = getConfig();
-	for (const domain of domains) {
-		const { __exports_dist } = config.paths(domain);
+	// Get pages from gatsby page-data dir
+	const gatsbyPageDataPages = getPageDataPagesFromExports(jsonContentDir);
+
+	for await (const pageData of gatsbyPageDataPages) {
+		const { result } = pageData;
+		const { pageContext } = result;
+		const { page, openGraph, pageMetadata } = pageContext;
+
+		const { compose } = page.fullPath;
+
+		const htmlPath = path.resolve(`${htmlContentDir}/${compose}/index.html`);
+		const htmlContent = await fsp.readFile(htmlPath, "utf-8");
 
-		// If __exports_dist has not exported sites (directories), it does not
-		// upload search content.
-		if (fs.existsSync(__exports_dist)) {
-			await uploadRenderedSearchContentToAPI(__exports_dist, domain);
+		const pageObject: PostSearchInfoProps = {
+			siteId: page.site,
+			pageId: page.id,
+			// `pageMetadata.title` has already a fallback `metatitle ||
+			// pageTitle` so probably `title` never will be take the
+			// `openGraph?.title` value. Only when the `metatitle` and
+			// `pageTitle` are empty.
+			title: pageMetadata?.title || openGraph?.title,
+			languageId: page.language,
+			url: page.fullUrl,
+			template: page.template.templateType || page.templateId,
+			description: pageMetadata?.description || openGraph?.description,
+			image: openGraph.image,
+			// _content: prepareHTMLContentForSearch(htmlContent),
+			content: prepareHTMLContentForSearch(htmlContent),
+		};
+
+		try {
+			await postSearchInfo(pageObject);
+		} catch (error) {
+			throwError(UploadSearchError, error);
 		}
 	}
+}
+
+/**
+ * Walk recursively in a basePath and return an array of pages with json
+ * extension with the full absolute path and the path includes "page-data".
+ *
+ * @param basePath The path to walk recursively.
+ * @returns An array of pages with json extension with the full absolute path
+ * and the path includes "page-data".
+ */
+async function* walkRecursively(basePath: string): AsyncGenerator<string> {
+	const filesHandle = await fsp.opendir(basePath);
 
-	// Solo una vez en cada render de todos los dominios...
-	if (envs.GRIDDO_AI_EMBEDDINGS) {
-		await startAIEmbeddings();
+	for await (const fileDirent of filesHandle) {
+		if (fileDirent.isDirectory()) {
+			yield* walkRecursively(path.join(basePath, fileDirent.name));
+		} else if (
+			fileDirent.isFile() &&
+			path.extname(fileDirent.name) === ".json" &&
+			fileDirent.name.includes("page-data")
+		) {
+			yield path.join(basePath, fileDirent.name);
+		}
 	}
 }
 
-main().catch((err) => {
-	console.error("Error", err?.stdout?.toString() || err);
-	process.exit(1);
-});
+/**
+ * Walk recursively in a basePath and return an array of pages with json
+ * extension with the full absolute path and the path includes "page-data".
+ *
+ * @param basePath The path to walk recursively.
+ * @returns An array of pages with json extension with the full absolute path
+ * and the path includes "page-data".
+ */
+async function* getPageDataPagesFromExports<PageType extends GatsbyPageData>(
+	basePath: string,
+): AsyncGenerator<PageType> {
+	const jsonFiles = walkRecursively(basePath);
+
+	for await (const filePath of jsonFiles) {
+		try {
+			const fileContent = await fsp.readFile(filePath, "utf8");
+			const page = JSON.parse(fileContent) as PageType;
+
+			if (page.path) {
+				yield page;
+			}
+		} catch (error) {
+			throwError(ReadFromStoreError, error);
+		}
+	}
+}
+
+async function getContentDirectories(domain: string) {
+	const { __exports } = await getRenderPathsHydratedWithDomainFromDB({ domain });
+
+	return {
+		htmlContentDir: path.join(__exports, "dist"),
+		jsonContentDir: path.join(__exports, "dist", "page-data"),
+	};
+}
+
+async function uploadSearchContent() {
+	if (GRIDDO_SEARCH_FEATURE) {
+		const domains = await getInstanceDomains();
+		for (const domain of domains) {
+			const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
+
+			GriddoLog.info(
+				`Uploading search content for ${domain}: processing content for ${htmlContentDir.length} pages...`,
+			);
+			await uploadRenderedSearchContentToAPI({
+				htmlContentDir,
+				jsonContentDir,
+			});
+		}
+	}
+}
+
+async function aiEmbeddings() {
+	if (GRIDDO_SEARCH_FEATURE && GRIDDO_AI_EMBEDDINGS) {
+		GriddoLog.info(`Triggering AI embeddings...`);
+		await post<AIEmbeddingsResponse>({
+			endpoint: AI_EMBEDDINGS,
+			useApiCacheDir: false,
+		});
+	}
+}
+
+async function main() {
+	await AuthService.login();
+	await uploadSearchContent();
+	await aiEmbeddings();
+}
+
+withErrorHandler(main);

package/exporter/core/GriddoLog.ts

@@ -0,0 +1,45 @@
+import { GRIDDO_BUILD_LOGS, GRIDDO_VERBOSE_LOGS } from "../shared/envs";
+import { brush } from "../shared/npm-modules/brush";
+
+/**
+ * Clase estática para gestionar los logs de la aplicación.
+ * No se puede instanciar, se usa directamente: GriddoLogs.info("mensaje").
+ */
+class GriddoLog {
+	/** El constructor es privado para prevenir la instanciación de la clase. */
+	private constructor() {}
+
+	public static verbose(...str: unknown[]): void {
+		if (GRIDDO_VERBOSE_LOGS) {
+			console.log(brush.yellow("verbose"), brush.dim(str.join(" ")));
+		}
+	}
+
+	public static build(...str: unknown[]): void {
+		if (GRIDDO_BUILD_LOGS) {
+			GriddoLog.log(...str);
+		}
+	}
+
+	public static info(...str: unknown[]): void {
+		console.log(`${brush.blue("info")} ${str.join(" ")}`);
+	}
+
+	public static success(...str: unknown[]): void {
+		console.log(`${brush.green("success")} ${str.join(" ")}`);
+	}
+
+	public static error(...str: unknown[]): void {
+		console.error(`${brush.red("error")} ${str.join(" ")}`);
+	}
+
+	public static warn(...str: unknown[]): void {
+		console.warn(`${brush.yellow("warn")} ${str.join(" ")}`);
+	}
+
+	public static log(...args: Parameters<typeof console.log>): void {
+		console.log(...args);
+	}
+}
+
+export { GriddoLog };

package/exporter/core/check-env-health.ts

@@ -0,0 +1,200 @@
+import { GRIDDO_SKIP_BUILD_CHECKS } from "../shared/envs";
+import { CheckHealthError } from "../shared/errors";
+import { throwError } from "./errors";
+import { GriddoLog } from "./GriddoLog";
+
+/**
+ * Environment variables that must be present (but value doesn't matter)
+ */
+const REQUIRED_ENV_VARS: readonly string[] = [];
+
+/**
+ * Environment variables that must have specific values
+ */
+const REQUIRED_ENV_VALUES: readonly {
+	key: string;
+	expected: string;
+	description?: string;
+}[] = [
+	{
+		key: "GRIDDO_RENDER_BY_DOMAINS",
+		expected: "true",
+		description: "Must be set to 'true' for domain-based rendering",
+	},
+];
+
+/**
+ * Environment variables that are recommended but not required
+ * These will show warnings if missing but won't fail the health check
+ */
+const RECOMMENDED_ENV_VARS: readonly { key: string; description?: string }[] = [
+	{
+		key: "GRIDDO_BUILD_LOGS",
+		description: "Recommended for detailed render logs",
+	},
+	{
+		key: "GRIDDO_BUILD_LOGS_TO_FILE",
+		description: "Recommended for performance gain and simplify render logs",
+	},
+	{
+		key: "GRIDDO_API_CONCURRENCY_COUNT",
+		description: "Recommended for better performance and memory management",
+	},
+];
+
+/**
+ * Environment variables that must be present (but value doesn't matter)
+ */
+const DEPRECATED_ENV_VARS: readonly { key: string; description?: string }[] = [
+	{
+		key: "GRIDDO_RENDER_SITE",
+		description:
+			"This environment variable no longer has any effect because it is incompatible with incremental rendering.",
+	},
+	{
+		key: "GRIDDO_RENDER_PAGES",
+		description:
+			"This environment variable no longer has any effect because it is incompatible with incremental rendering.",
+	},
+	{
+		key: "REACT_APP_API_ENDPOINT",
+		description: "Use GRIDDO_API_URL",
+	},
+	{
+		key: "REACT_APP_PUBLIC_API_ENDPOINT",
+		description: "Use GRIDDO_PUBLIC_API_URL",
+	},
+	{
+		key: "GRIDDO_RENDER_ALL_SITES",
+		description: "This environment variable is deprecated, remove it",
+	},
+];
+
+/**
+ * Check if required environment variables exist with any value.
+ */
+function checkRequiredEnvVars(): { isValid: boolean; missing: string[] } {
+	const missing = REQUIRED_ENV_VARS.filter((envName) => !process.env[envName]);
+
+	if (missing.length > 0) {
+		GriddoLog.error(`Missing required environment variables: ${missing.join(", ")}`);
+		return { isValid: false, missing };
+	}
+
+	return { isValid: true, missing: [] };
+}
+
+/**
+ * Check if environment variables have the expected values.
+ */
+function checkEnvVarValues(): {
+	isValid: boolean;
+	mismatches: { key: string; expected: string; actual: string }[];
+} {
+	const mismatches: { key: string; expected: string; actual: string }[] = [];
+
+	for (const { key, expected } of REQUIRED_ENV_VALUES) {
+		const actual = process.env[key];
+
+		if (actual !== expected) {
+			mismatches.push({ key, expected, actual: actual || "(undefined)" });
+		}
+	}
+
+	if (mismatches.length > 0) {
+		return { isValid: false, mismatches };
+	}
+
+	return { isValid: true, mismatches: [] };
+}
+
+/**
+ * Check recommended environment variables and show warnings for missing ones.
+ */
+function checkRecommendedEnvVars() {
+	const missing = RECOMMENDED_ENV_VARS.filter(({ key }) => !process.env[key]);
+
+	if (missing.length > 0) {
+		GriddoLog.warn("Recommended environment variables not set:");
+
+		for (const { key, description } of missing) {
+			const desc = description ? ` # ${description}` : "";
+			GriddoLog.log(`     -> ${key}${desc}`);
+		}
+	}
+}
+
+/**
+ * Check if set enviroment variables have been deprecated
+ */
+function checkDeprecatedEnvVars() {
+	const deprecated = DEPRECATED_ENV_VARS.filter(({ key }) => process.env[key]);
+
+	if (deprecated.length > 0) {
+		const deprecatedMessages = deprecated.map(({ key, description }) => {
+			const desc = description ? ` # ${description}` : "";
+			return `  ${key}${desc}`;
+		});
+
+		GriddoLog.warn("Deprecated environment variables set:");
+		GriddoLog.warn(`${deprecatedMessages.join("\n")}\n`);
+	}
+
+	return { deprecated: deprecated.map((key) => key) };
+}
+
+/**
+ * Check if the environment is secure to launch a render.
+ * If something fails then log an error message and exit from the process.
+ * Otherwise just return true.
+ */
+export function checkEnvironmentHealth() {
+	// Bypass the check using this env var, handy for local renders.
+	if (GRIDDO_SKIP_BUILD_CHECKS) {
+		GriddoLog.info("Build health check skipped");
+		return true;
+	}
+
+	// Check required environment variables
+	const envVarsCheck = checkRequiredEnvVars();
+	const envValuesCheck = checkEnvVarValues();
+
+	// (warnings only)
+	checkRecommendedEnvVars();
+	checkDeprecatedEnvVars();
+
+	// Render is safe if all required checks pass
+	const isSafeToRender = envVarsCheck.isValid && envValuesCheck.isValid;
+
+	if (isSafeToRender) {
+		GriddoLog.info("Build health check passed");
+		return true;
+	}
+
+	// Generate consolidated error message
+	const allIssues: string[] = [];
+
+	if (!envVarsCheck.isValid) {
+		allIssues.push(`Missing: ${envVarsCheck.missing.join(", ")}`);
+	}
+
+	if (!envValuesCheck.isValid) {
+		const mismatchMessages = envValuesCheck.mismatches.map(
+			({ key, expected, actual }) => `${key}="${actual || "undefined"}" (expected "${expected}")`,
+		);
+		allIssues.push(`Incorrect values:\n  ${mismatchMessages.join("\n  ")}`);
+	}
+
+	// Show what needs to be set
+	const requiredVarsMessage = REQUIRED_ENV_VALUES.map(({ key, expected, description }) => {
+		const desc = description ? ` # ${description}` : "";
+		return `  ${key}="${expected}"${desc}`;
+	}).join("\n");
+
+	GriddoLog.error(`Environment health check failed:`);
+	GriddoLog.error(`  ${allIssues.join("\n  ")}`);
+	GriddoLog.warn("\nRequired environment variables:");
+	GriddoLog.error(requiredVarsMessage);
+
+	throwError(CheckHealthError);
+}

package/exporter/core/db-class.ts

@@ -0,0 +1,54 @@
+import type { RenderDB } from "../shared/types/render";
+
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { pkgDirSync } from "../shared/npm-modules/pkg-dir";
+import { GriddoLog } from "./GriddoLog";
+
+// Interfaz para abstraer la DB
+export interface Database {
+	read(): Promise<RenderDB>;
+	write(renderDB: RenderDB): Promise<void>;
+}
+
+export class JsonDatabase implements Database {
+	private readonly dbFilePath: string;
+
+	constructor(customDBPath?: string) {
+		this.dbFilePath =
+			customDBPath ??
+			path.join(
+				pkgDirSync({ cwd: path.resolve(__dirname, "../../..") }) ?? "",
+				".griddo/cache/db.json",
+			);
+	}
+
+	async read(customDBFilePath?: string): Promise<RenderDB> {
+		try {
+			const raw = await fsp.readFile(customDBFilePath || this.dbFilePath, "utf-8");
+			return JSON.parse(raw) as RenderDB;
+		} catch (error) {
+			if (error instanceof Error) {
+				GriddoLog.error(`Error reading DB file at ${this.dbFilePath}:`, error.message);
+			} else {
+				GriddoLog.error(`Unknown error reading DB file at ${this.dbFilePath}:`, error);
+			}
+			throw error;
+		}
+	}
+
+	async write(renderDB: RenderDB): Promise<void> {
+		try {
+			await fsp.mkdir(path.dirname(this.dbFilePath), { recursive: true });
+			await fsp.writeFile(this.dbFilePath, JSON.stringify(renderDB, null, "\t"));
+		} catch (error) {
+			if (error instanceof Error) {
+				GriddoLog.error(`Error writing to DB file at ${this.dbFilePath}:`, error.message);
+			} else {
+				GriddoLog.error(`Unknown error writing to DB file at ${this.dbFilePath}:`, error);
+			}
+			throw error;
+		}
+	}
+}

package/exporter/core/db.ts

@@ -0,0 +1,33 @@
+import type { RenderDB } from "../shared/types/render";
+
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { pkgDirSync } from "../shared/npm-modules/pkg-dir";
+import { GriddoLog } from "./GriddoLog";
+
+const root = pkgDirSync({ cwd: path.resolve(__dirname, "../../..") }) || "";
+const cache = path.join(root, ".griddo/cache");
+const dbFilePath = path.join(cache, "db.json");
+
+async function readDB(customDBPath = "") {
+	const file = customDBPath || dbFilePath;
+	try {
+		return JSON.parse(await fsp.readFile(file, "utf-8")) as RenderDB;
+	} catch (error) {
+		GriddoLog.error(`Failed to read DB file at ${file}:`, error);
+		throw error;
+	}
+}
+
+async function writeDB(renderDB: RenderDB, customDBPath = "") {
+	const file = customDBPath || dbFilePath;
+	try {
+		await fsp.writeFile(file, JSON.stringify(renderDB, null, "\t"));
+	} catch (error) {
+		GriddoLog.error(`Failed to write DB file at ${file}:`, error);
+		throw error;
+	}
+}
+
+export { readDB, writeDB };

package/exporter/core/dist-rollback.ts

@@ -0,0 +1,40 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { readDB } from "./db";
+import { pathExists } from "./fs";
+import { GriddoLog } from "./GriddoLog";
+
+async function distRollback(domain: string) {
+	const data = await readDB();
+	const { exportsDir, exportsDirBackup } = data.paths;
+
+	GriddoLog.info(`Cleaning exports dir for the domain ${domain}`);
+	GriddoLog.verbose(`Deleting ${path.join(exportsDir, domain)}...`);
+
+	// Probar rsync en lugar de borrar y copiar
+
+	// 1 - Borrar dist corrupto
+	await fsp.rm(path.join(exportsDir, domain), {
+		recursive: true,
+		force: true,
+	});
+
+	// 2 - Si hay backup, restaurar
+	if (await pathExists(path.join(exportsDirBackup, domain))) {
+		await fsp.cp(path.join(exportsDirBackup, domain), path.join(exportsDir, domain), {
+			recursive: true,
+		});
+
+		GriddoLog.info(`export-backup dir for the domain ${domain} found. Restoring before exit...`);
+		GriddoLog.verbose(
+			`Copying ${path.join(exportsDirBackup, domain)} -> ${path.join(exportsDir, domain)}...`,
+		);
+	} else {
+		GriddoLog.info(
+			"No export-backup found, skipping rollback. Next render will create a new exports dir from scratch...",
+		);
+	}
+}
+
+export { distRollback };

package/exporter/core/errors.ts

@@ -0,0 +1,82 @@
+import type { ErrorsType } from "../shared/errors";
+
+import path from "node:path";
+
+import { getRenderPathsHydratedWithDomainFromDB } from "../services/render";
+import { brush } from "../shared/npm-modules/brush";
+import { readDB, writeDB } from "./db";
+import { distRollback } from "./dist-rollback";
+import { GriddoLog } from "./GriddoLog";
+
+export type ErrorData = {
+	error: ErrorsType;
+	message: string;
+	expected?: string;
+	hint?: string;
+};
+
+export class RenderError extends Error {
+	constructor(originalError?: unknown) {
+		super(originalError instanceof Error ? originalError.message : String(originalError));
+
+		this.name = "InternalCXError";
+		this.stack = originalError instanceof Error ? originalError.stack : "";
+	}
+}
+
+/**
+ * Throws an error with the provided error message, expected value, and hint.
+ */
+function throwError(options: ErrorData, stack?: unknown): never {
+	const { error, message, expected, hint } = options;
+
+	const errorColor = GriddoLog.log(brush.red(`[ ${error} ]`));
+	const extraText = [expected, hint].filter(Boolean).join("\n");
+
+	GriddoLog.log(`
+${errorColor}
+${message}
+${extraText}
+
+${brush.red("stack")}
+${JSON.stringify(stack, null, 2)}`);
+
+	throw new RenderError(stack);
+}
+
+async function withErrorHandler<T>(fn: () => Promise<T>) {
+	try {
+		await fn();
+	} catch (error) {
+		if (error instanceof RenderError) {
+			GriddoLog.error("Internal Griddo RenderError");
+		} else if (error instanceof Error) {
+			GriddoLog.error(error.message);
+		} else {
+			GriddoLog.error(`An unexpected error occurred ${error}`);
+		}
+
+		try {
+			const { __root } = await getRenderPathsHydratedWithDomainFromDB();
+			const data = await readDB();
+			if (data.needsRollbackOnError) {
+				GriddoLog.info("Cleaning exports dir...");
+				GriddoLog.verbose(`Deleting ${path.join(__root, "exports")}...`);
+
+				await distRollback(data.currentRenderingDomain!);
+			} else {
+				GriddoLog.info("No rollback needed, skipping...");
+			}
+		} catch (_e) {
+			GriddoLog.info("Early render stage, no db.json created yet...");
+		}
+
+		const data = await readDB();
+		data.domains[data.currentRenderingDomain!].isRendering = false;
+		data.domains[data.currentRenderingDomain!].renderMode = "ERROR";
+		await writeDB(data);
+		throw error;
+	}
+}
+
+export { throwError, withErrorHandler };