@griddo/cx 11.9.17-rc.1 → 11.10.0

package/cli.mjs

@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+
+import { execSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+function getRoot(rootFlag) {
+	if (rootFlag) {
+		return path.resolve(rootFlag);
+	}
+	return path.resolve(__dirname, "../../");
+}
+
+function getDBPath(rootFlag) {
+	return path.join(getRoot(rootFlag), ".griddo/cache/db.json");
+}
+
+const COMMANDS_PATH = path.join(__dirname, "build/commands");
+
+export const AVAILABLE_COMMANDS = [
+	{
+		name: "render",
+		description: "Render the instance",
+		domainArgument: "optional",
+		rootArgument: true,
+		script: "render-full-instance",
+		isCommandPack: true,
+		commandPack: renderCommandPack,
+	},
+	{
+		name: "prepare-assets-directory",
+		description: "Prepare assets directory (internal local only)",
+		domainArgument: true,
+		script: "prepare-assets-directory",
+		rootArgument: true,
+		isCommandPack: false,
+	},
+	{
+		name: "prepare-domains-render",
+		description: "Prepare domains",
+		domainArgument: false,
+		script: "prepare-domains-render",
+		rootArgument: true,
+		isCommandPack: false,
+	},
+	{
+		name: "start-render",
+		description: "Start rendering for a domain",
+		domainArgument: true,
+		script: "start-render",
+		rootArgument: false,
+		isCommandPack: false,
+	},
+	{
+		name: "end-render",
+		description: "End rendering for a domain",
+		domainArgument: true,
+		script: "end-render",
+		isCommandPack: false,
+		rootArgument: true,
+	},
+	{
+		name: "upload-search-content",
+		description: "Upload search content for a domain",
+		domainArgument: false,
+		script: "upload-search-content",
+		isCommandPack: false,
+		rootArgument: true,
+	},
+	{
+		name: "reset-render",
+		description: "Reset render state",
+		domainArgument: false,
+		script: "reset-render",
+		isCommandPack: false,
+		rootArgument: false,
+	},
+];
+
+function showHelp() {
+	console.log("Griddo CX CLI - Available commands:\n");
+
+	AVAILABLE_COMMANDS.forEach((cmd) => {
+		const domainArg = cmd.domainArgument ? " --domain=<domain>" : "";
+		console.log(`  ${cmd.name}${domainArg} --root=<path> - ${cmd.description}`);
+	});
+
+	console.log("\nUsage:");
+	console.log("  griddo-render <command> --root=<path> [--domain=<domain>]");
+	console.log("  griddo-render --help");
+	console.log("\nExamples:");
+	console.log("  griddo-render render --domain=pro-pikachu --root=. ");
+	console.log("  griddo-render start-render --domain=mydomain.com --root=some-path");
+	console.log("  griddo-render prepare-domains-render --root=. ");
+	console.log("  griddo-render --help");
+}
+
+function findCommand(commandName) {
+	return AVAILABLE_COMMANDS.find((cmd) => cmd.name === commandName);
+}
+
+function validateCommand(commandName, config) {
+	const command = findCommand(commandName);
+
+	if (!command) {
+		console.error(`Error: Unknown command '${commandName}'`);
+		console.error("Use 'griddo-render --help' to see available commands");
+		process.exit(1);
+	}
+
+	if (!config.root && command.rootArgument) {
+		console.error(`Error: --root flag is required`);
+		console.error(`Usage: griddo-render ${commandName} --root=<path>`);
+		process.exit(1);
+	}
+
+	if (command.domainArgument === true && !config.domain) {
+		console.error(`Error: Command '${commandName}' requires a --domain argument`);
+		console.error(`Usage: griddo-render ${commandName} --domain=<domain> --root=<path>`);
+		process.exit(1);
+	}
+
+	if (command.domainArgument === false && config.domain) {
+		console.warn(
+			`Warning: Command '${commandName}' doesn't require a domain, ignoring '${config.domain}'`,
+		);
+	}
+
+	// Creamos un nuevo config validado para asegurar que el dominio se pasa solo cuando es necesario
+	const validatedConfig = {
+		...config,
+		domain: command.domainArgument ? config.domain : undefined,
+	};
+
+	return { command, config: validatedConfig };
+}
+
+function executeCommand(command, config) {
+	try {
+		const env = { ...process.env };
+		const scriptPath = path.join(COMMANDS_PATH, command.script);
+
+		if (command.isCommandPack) {
+			command.commandPack(config);
+			return;
+		}
+
+		const commandToRun = `node ${scriptPath}${config.domain ? ` ${config.domain}` : ""}`;
+
+		execSync(commandToRun, {
+			stdio: "inherit",
+			env,
+			cwd: process.cwd(),
+		});
+	} catch (error) {
+		console.error(`Error executing command ${command.name}`);
+		throw error;
+	}
+}
+
+function renderCommandPack(config) {
+	// Pequeño helper para no repetir la lógica de búsqueda y ejecución
+	const run = (commandName, domainOverride) => {
+		const commandToRun = findCommand(commandName);
+		// Pasamos una copia del config con el dominio que necesite este comando en particular
+		const runConfig = { ...config, domain: domainOverride ?? config.domain };
+		if (commandToRun) {
+			executeCommand(commandToRun, runConfig);
+		}
+	};
+
+	try {
+		run("prepare-domains-render", null);
+
+		let domainsToProcess = [];
+		if (config.domain) {
+			domainsToProcess = [config.domain];
+		} else {
+			const { sortedDomains } = JSON.parse(fs.readFileSync(getDBPath(config.root), "utf-8"));
+			domainsToProcess = sortedDomains;
+		}
+
+		for (const domain of domainsToProcess) {
+			run("start-render", domain);
+			run("prepare-assets-directory", domain);
+			run("end-render", domain);
+		}
+
+		// All domains
+		run("upload-search-content");
+	} catch (error) {
+		console.error("An error occurred during the render process:", error);
+		run("reset-render", null);
+	}
+}
+
+function main() {
+	const options = {
+		help: { type: "boolean", short: "h" },
+		domain: { type: "string" },
+		root: { type: "string" },
+	};
+
+	const { values, positionals } = parseArgs({
+		args: process.argv.slice(2),
+		options,
+		allowPositionals: true,
+	});
+
+	const commandName = positionals[0];
+
+	if (!commandName || values.help) {
+		showHelp();
+		return;
+	}
+
+	const initialConfig = {
+		domain: values.domain,
+		root: values.root,
+	};
+
+	const { command, config } = validateCommand(commandName, initialConfig);
+
+	executeCommand(command, config);
+}
+
+main();

package/exporter/build-esbuild.noop

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Build CX script file
+#
+# This file generates the necessary builds for CX.
+# - Setup, to generate necessary elements (tsconfig, etc..)
+# - Library, CX library that will be used by the SSG framework.
+# - Scripts, CX scripts that will be used by infra during the render process.
+#
+# @todo:
+# In the future, we will try to eliminate the bundling processes and use
+# TypeScript or ESModules directly.
+
+# start
+echo "📦 Building ..."
+
+# CLI options
+log="--log-level=error"
+production_node_opts="--bundle --platform=node --minify --sourcemap"
+debugging_node_opts="--bundle --platform=node --sourcemap"
+# react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
+
+# Manage --minify for debugging
+bundle_node_opts=$production_node_opts
+if [ "$1" = "--debug" ]; then
+    bundle_node_opts=$debugging_node_opts
+fi
+
+# library
+# esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
+# esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
+
+# scripts
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
+esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts} --outfile=./build/prepare-assets-directory.js
+
+# types
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/build.sh

@@ -1,21 +1,10 @@
-# Build CX script file
-#
-# This file generates the necessary builds for CX.
-# - Setup, to generate necessary elements (tsconfig, etc..)
-# - Library, CX library that will be used by the SSG framework.
-# - Scripts, CX scripts that will be used by infra during the render process.
-#
-# @todo:
-# In the future, we will try to eliminate the bundling processes and use
-# TypeScript or ESModules directly.
+#!/usr/bin/env bash
+# shellcheck disable=SC2086
 
-# start
-echo "📦 Building ..."
-
-# CLI options
+# CLI options for esbuild
 log="--log-level=error"
-production_node_opts="--bundle --platform=node --minify --sourcemap"
-debugging_node_opts="--bundle --platform=node --sourcemap"
+production_node_opts="--bundle --platform=node --minify --sourcemap --outfile=./build/commands/"
+debugging_node_opts="--bundle --platform=node --sourcemap --outfile=./build/commands/"
 react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
 
 # Manage --minify for debugging
@@ -25,15 +14,18 @@ if [ "$1" = "--debug" ]; then
 fi
 
 # library
-esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
+esbuild ${log} ./exporter/index.ts --bundle --platform=node  --outfile=./build/index.js
 esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
 
-# scripts
-esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
-esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
-esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
-esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
-esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
+# This .sh needs to be run from a npm script, so esbuild dependency is available
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts}end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts}upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts}reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts}start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts}prepare-domains-render.js
+esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts}prepare-assets-directory.js
 
 # types
-tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
+tsgo --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
+
+echo "Build completed successfully!"

package/exporter/commands/README.md

@@ -0,0 +1,142 @@
+# Comandos del Orquestador Griddo CX
+
+Este directorio contiene los comandos principales de Griddo Exporter (CX). Estos comandos forman parte del ciclo de vida completo del proceso de render de una instancia.
+
+La recomendación es llamarlos mediante el cli `node cli.mjs <comand-name> [options]`
+
+## Listado de comandos.
+
+### 1. `prepare-domains-render.ts`
+
+**Propósito**: Prepara los dominios para el proceso de renderizado.
+
+**Funcionalidades principales**:
+- Inicializa el entorno de renderizado creando la estructura de base de datos local (db.json)
+- Obtiene la lista de dominios disponibles desde la API
+- Calcula el número de páginas por dominio analizando sitios para publicar/despublicar
+- Determina el modo de renderizado para cada dominio (basado en si tiene contenido pendiente)
+- Ordena los dominios por número de páginas para optimizar el procesamiento
+- Crea archivos de configuración (`domains.json`) para uso de infraestructura (A deprecar: infra deberá leer db.json para obtener esa información)
+
+**Uso típico**: Se ejecuta antes de cualquier proceso de renderizado para preparar el estado inicial.
+
+**Características técnicas**:
+- No requiere argumentos (proceso global)
+- Crea la carpeta `.griddo` si no existe donde le indiquemos cuando le pasemos `--root` desde el cli.
+
+
+### 2. `start-render.ts`
+
+**Propósito**: Ejecuta el ciclo completo de renderizado para uno o múltiples dominios.
+
+**Funcionalidades principales**:
+- Autenticación con la API de Griddo
+- Verificación del estado de salud del entorno
+- Ejecución del proceso Gatsby para generar sitios estáticos
+
+**Uso**:
+```bash
+# Modo moderno (recomendado)
+start-render <dominio>
+```
+
+**Características técnicas**:
+- Puede procesar un dominio específico o todos los dominios disponibles
+- Utiliza el adaptador Gatsby para la generación de sitios
+- Requiere autenticación previa con la API
+
+### 3. `end-render.ts`
+
+**Propósito**: Finaliza el ciclo de renderizado notificando a la API sobre los cambios realizados.
+
+**Funcionalidades principales**:
+- Lee el reporte de construcción generado por Gatsby
+- Procesa cada sitio del dominio indicando qué se publicó/despublicó
+- Envía llamadas de finalización a la API para cada sitio
+- Maneja modos IDLE para sitios sin cambios
+
+**Uso**:
+```bash
+# Modo moderno
+end-render <dominio>
+```
+
+**Características técnicas**:
+- Lee `build-report.json` generado durante el proceso de construcción
+- Elimina hashes duplicados antes del envío
+- Puede saltar dominios marcados como IDLE
+
+### 4. `upload-search-content.ts`
+
+**Propósito**: Sube el contenido de las páginas generadas para el uso posterior en los buscadores de Griddo.
+
+**Funcionalidades principales**:
+- Extrae contenido de las páginas HTML generadas
+- Limpia el contenido HTML removiendo etiquetas, scripts y elementos no deseados
+- Procesa archivos `page-data.json` de Gatsby para obtener metadatos
+- Envía el contenido indexable a la API de búsqueda
+- Soporte opcional para embeddings de IA
+
+**Características técnicas**:
+- Solo se ejecuta si `GRIDDO_SEARCH_FEATURE` está habilitado
+- Procesa recursivamente directorios `page-data` y `public`
+- Normaliza el contenido eliminando múltiples espacios y saltos de línea
+- Envía datos estructurados: título, descripción, imagen, URL, contenido, template
+
+**Configuración**:
+- Activado por `GRIDDO_SEARCH_FEATURE=true`
+- Embeddings de IA con `GRIDDO_AI_EMBEDDINGS=true`
+
+### 5. `reset-render.ts`
+
+**Propósito**: Resetea el estado de renderizado en el sistema remoto.
+
+**Funcionalidades principales**:
+- Envía una llamada a la API para resetear el estado de renderizado
+- Útil para limpiar estados inconsistentes o errores de procesamiento
+
+**Uso típico**: Se utiliza cuando hay problemas con el estado del renderizado que requieren un reinicio limpio.
+
+**Características técnicas**:
+- No requiere argumentos
+- Comunicación directa con el endpoint `RESET_RENDER`
+- Operación atómica que afecta el estado global
+
+### 6. `prepare-assets-directory.ts` (USO SOLO EN DESARROLLO LOCAL)
+
+**Propósito**: Prepara el directorio de assets para un dominio específico.
+
+**Funcionalidades principales**:
+- Reorganiza la estructura de directorios de assets
+- Mueve assets desde ubicación compartida a ubicación específica del dominio
+
+**Uso**:
+```bash
+prepare-assets-directory <dominio>
+```
+
+**Características técnicas**:
+- Mueve el directorio `assets` compartido al directorio específico del dominio
+- Operación de bajo nivel para organización de archivos estáticos
+- Elimina y recrea la estructura según sea necesario
+
+## Flujo de Uso Típico
+
+1. **`prepare-domains-render`** - Preparar el entorno y obtener dominios
+2. **`start-render`** - Ejecutar la generación de sitios para un dominio
+3. **`end-render`** - Notificar finalización y cambios a la API
+4. **`upload-search-content`** - Subir contenido para búsqueda (si está habilitado)
+5. **`prepare-assets-directory`** - Organizar assets (si es necesario)
+
+## Variables de Entorno Relevantes
+
+- `GRIDDO_SEARCH_FEATURE`: Habilita subida de contenido para búsqueda
+- `GRIDDO_AI_EMBEDDINGS`: Habilita generación de embeddings de IA
+- `GRIDDO_API_URL`: URL base de la API de Griddo
+- `GRIDDO_BOT_USER` / `GRIDDO_BOT_PASSWORD`: Credenciales de autenticación
+
+## Notas de Desarrollo
+
+- Todos los comandos utilizan manejo de errores consistente a través de `withErrorHandler`
+- La mayoría requiere autenticación previa con `AuthService.login()`
+- Los comandos modernos esperan un argumento de dominio específico

package/exporter/commands/end-render.ts

@@ -1,105 +1,71 @@
-#!/usr/bin/env node
-
-import fs from "node:fs";
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import pkgDir from "pkg-dir";
-
+import { throwError, withErrorHandler } from "../core/errors";
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
 import { AuthService } from "../services/auth";
+import {
+	getRenderMetadataFromDB,
+	getRenderModeFromDB,
+	getRenderPathsHydratedWithDomainFromDB,
+} from "../services/render";
 import { endSiteRender } from "../services/sites";
-import { getConfig } from "../utils/core-utils";
-import { getInstanceDomains } from "../utils/domains";
-import { infoLog } from "../utils/loggin";
+import { ArtifactError } from "../shared/errors";
+import { RENDER_MODE, type Report } from "../shared/types/render";
+
+async function getRenderReport(options: {
+	domain: string;
+	exportsSiteDir: string;
+	buildReportFileName: string;
+}) {
+	const { buildReportFileName, domain, exportsSiteDir } = options;
+	const buildReportFile = path.resolve(exportsSiteDir, domain, "dist", buildReportFileName);
+
+	if (!(await pathExists(buildReportFile))) {
+		GriddoLog.info(`Build report file "${buildReportFile}" not found.`);
+		throwError(ArtifactError);
+	}
 
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
+}
 
-// Where we are going to find export dirs
-const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
+async function endRender() {
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { __sites } = await getRenderPathsHydratedWithDomainFromDB();
 
-// Where we are going to find archived exports
-const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
+	await AuthService.login();
 
-type Report = {
-	authControl:
-		| {
-				Authorization: string;
-				"Cache-Control": string;
-				lang?: string | undefined;
-		  }
-		| undefined;
-	sites: Array<{
-		siteId: number;
-		publishHashes: Array<string>;
-		siteHash: string | null;
-		unpublishHashes: Array<string>;
-		publishPagesIds: Array<number>;
-	}>;
-};
+	const [domain] = process.argv.slice(2);
 
-function getDomainRenderReport(domain: string) {
-	const reports: Array<Report> = [];
-	const { buildReportFileName } = getConfig();
+	const { renderMode, reason } = await getRenderModeFromDB(domain);
+	if (renderMode === RENDER_MODE.IDLE) {
+		GriddoLog.info(
+			`(From Current Render) [${domain}]: Skipping end-render as it is marked as IDLE with the reason ${reason}.`,
+		);
+		return;
+	}
 
-	const buildReportFile = path.resolve(
-		exportArchiveBasePath,
+	const report = await getRenderReport({
 		domain,
-		"dist",
+		exportsSiteDir: __sites,
 		buildReportFileName,
-	);
+	});
 
-	if (!fs.existsSync(buildReportFile)) {
-		console.info(`Build report file "${buildReportFile}" not found.`);
-	}
-
-	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
+	GriddoLog.info(`Sending ending call to ${domain} sites`);
 
-	reports.push(buildReport);
+	for (const site of report.sites) {
+		site.publishHashes = [...new Set(site.publishHashes)];
+		site.unpublishHashes = [...new Set(site.unpublishHashes)];
 
-	return reports;
+		const { siteId, ...body } = site;
+		GriddoLog.info(`Site (${siteId})`);
+		await endSiteRender(siteId, body);
+	}
 }
 
-(async () => {
-	const sendEndSiteRender = async (reports: Array<Report>) => {
-		for (const report of reports) {
-			const sites = report.sites;
-
-			for (const site of sites) {
-				const { siteId, ...body } = site;
-
-				console.info(`Site (${siteId})`);
-
-				await endSiteRender(siteId, body);
-			}
-		}
-	};
-
-	if (RENDER_BY_DOMAIN) {
-		await AuthService.login();
-		const domain = process.argv.splice(2)[0];
-		const reports = getDomainRenderReport(domain);
-
-		infoLog(`Sending ending call to ${domain} sites`);
-
-		for (const report of reports) {
-			for (const site of report.sites) {
-				site.publishHashes = [...new Set(site.publishHashes)];
-				site.unpublishHashes = [...new Set(site.unpublishHashes)];
-			}
-		}
-
-		await sendEndSiteRender(reports);
-	} else {
-		// Legacy render
-		const domains = await getInstanceDomains();
-		for (const domain of domains) {
-			const reports = getDomainRenderReport(domain);
-
-			infoLog(`Sending ending call to ${domain} sites`);
+async function main() {
+	await endRender();
+}
 
-			await sendEndSiteRender(reports);
-		}
-	}
-})().catch((err) => {
-	console.error("Error", err?.stdout?.toString() || err);
-	process.exit(1);
-});
+withErrorHandler(main);

package/exporter/commands/prepare-assets-directory.ts

@@ -0,0 +1,35 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { withErrorHandler } from "../core/errors";
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { getRenderModeFromDB, getRenderPathsHydratedWithDomainFromDB } from "../services/render";
+import { RENDER_MODE } from "../shared/types/render";
+
+async function prepareAssetsDir() {
+	const [domain] = process.argv.slice(2);
+
+	const { renderMode } = await getRenderModeFromDB(domain);
+
+	if (renderMode === RENDER_MODE.IDLE) {
+		return;
+	}
+
+	const { __exports } = await getRenderPathsHydratedWithDomainFromDB({ domain });
+	const assetsDir = path.join(__exports, "assets");
+	const domainAssetsDir = path.join(__exports, domain);
+
+	if (await pathExists(assetsDir)) {
+		await fsp.rm(domainAssetsDir, { force: true, recursive: true });
+		await fsp.rename(assetsDir, domainAssetsDir);
+	} else {
+		GriddoLog.warn("Assets directory not found");
+	}
+}
+
+async function main() {
+	await prepareAssetsDir();
+}
+
+withErrorHandler(main);