@griddo/cx 11.7.6 → 11.7.7-rc.0

package/exporter/adapters/gatsby/shared/sync-render.ts

@@ -0,0 +1,276 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { findFilesBySuffix } from "../../../utils/folders";
+import { infoLog, verboseLog } from "../../../utils/loggin";
+import { getAssetsDiffBetweenRenders, patchHtmlFile } from "./diff-assets";
+
+interface SyncRenderConfig {
+	src: string;
+	dst: string;
+	pagesToCreate: Array<number>;
+	pagesToDelete: Array<number>;
+	artifactsToCopyToExports: Array<string>;
+}
+interface CopyFilePath {
+	from: string;
+	to: string;
+}
+interface SyncState {
+	htmlToAdd: Array<CopyFilePath>;
+	jsonToAdd: Array<CopyFilePath>;
+	htmlToDelete: Array<string>;
+	jsonToDelete: Array<string>;
+}
+interface PageInfo {
+	id: number;
+	composePath: string; // p.ej. '/about-us' o '/'
+	htmlPath: string; // p.ej. '/path/to/public/about-us/index.html'
+	jsonPath: string; // p.ej. '/path/to/public/page-data/about-us/page-data.json'
+}
+interface GatsbyPageData {
+	result: {
+		pageContext: {
+			id: number;
+			fullPath: {
+				site: string;
+				domain: string;
+				domainUrl: string;
+				language: string;
+				page: string;
+				compose: string;
+			};
+		};
+	};
+}
+
+class SyncRender {
+	private bundleDir: string;
+	private currentRenderDir: string;
+	private pagesToCreate: Array<number>;
+	private pagesToDelete: Array<number>;
+	private assetArtifacts: Array<string>;
+
+	private state: SyncState = {
+		htmlToAdd: [],
+		jsonToAdd: [],
+		htmlToDelete: [],
+		jsonToDelete: [],
+	};
+
+	constructor(config: SyncRenderConfig) {
+		this.bundleDir = config.dst;
+		this.currentRenderDir = config.src;
+		this.pagesToCreate = config.pagesToCreate;
+		this.pagesToDelete = config.pagesToDelete;
+		this.assetArtifacts = config.artifactsToCopyToExports;
+	}
+
+	public async execute() {
+		await this.setPagesToDelete();
+		await this.setPagesToAdd();
+		await this.sync();
+		await this.assertAssetsAreValid();
+	}
+
+	private async assertAssetsAreValid() {
+		const isDifferent = await getAssetsDiffBetweenRenders(this.currentRenderDir, this.bundleDir);
+		if (isDifferent) {
+			infoLog(
+				"Warning: JavaScript and CSS bundles must have the same files and content. You can ignore this warning if the rendering process only has pages to unpublish.",
+			);
+		} else {
+			infoLog("Previous and current JavaScript and CSS packages successfully matched");
+		}
+	}
+
+	private async setPagesToDelete() {
+		// ¿Por qué se incluyen this.pagesToCreate en las candidatas para borrarse?
+		// El set de ids incluye tanto las páginas a borrar como las que se van
+		// a crear para manejar correctamente por ejemplo los cambios de slug.
+		// Si la página `id=3` tiene slug `slug=about` y despues viene con
+		// `slug=about-us` borramos la página `id=3` en el render previo y la
+		// volvemos a crear.
+		// Resumido lo que pasará es que las páginas que se van a crear se
+		// borran previamente por si ha cambiado el slug. Porque el slug hace
+		// que se guarden en otro directorio.
+		// ¿Qué pasaría si no la borrásemos? que tendríamos las dos páginas
+		// `/about/index.html` y `/about-us/index.html`
+		const candidateIdsToDelete = new Set<number>([...this.pagesToDelete, ...this.pagesToCreate]);
+		const existingPreviousPages = await this.scanPages(this.bundleDir);
+
+		for (const page of existingPreviousPages) {
+			if (candidateIdsToDelete.has(page.id)) {
+				this.state.htmlToDelete.push(page.htmlPath);
+				this.state.jsonToDelete.push(page.jsonPath);
+			}
+		}
+
+		verboseLog(`${this.state.htmlToDelete.length} pages HTML to delete`);
+		verboseLog(`${this.state.jsonToDelete.length} pages JSON to delete`);
+	}
+
+	private async setPagesToAdd() {
+		const candidateIdsToCreate = new Set<number>(this.pagesToCreate);
+		const newPages = await this.scanPages(this.currentRenderDir);
+
+		for (const page of newPages) {
+			if (candidateIdsToCreate.has(page.id)) {
+				const htmlTo = path.join(this.bundleDir, page.composePath, "index.html");
+				const normalizedCompose = page.composePath === "/" ? "index" : page.composePath;
+				const jsonTo = path.join(this.bundleDir, "page-data", normalizedCompose, "page-data.json");
+
+				this.state.htmlToAdd.push({ from: page.htmlPath, to: htmlTo });
+				this.state.jsonToAdd.push({ from: page.jsonPath, to: jsonTo });
+			}
+		}
+
+		verboseLog(`${this.state.htmlToAdd.length} pages HTML to create`);
+		verboseLog(`${this.state.jsonToAdd.length} pages JSON to create`);
+	}
+
+	private async sync() {
+		// Delete...
+		const allFilesToDelete = [...this.state.htmlToDelete, ...this.state.jsonToDelete];
+		for (const file of allFilesToDelete) {
+			try {
+				// Usar `force: true` para no fallar si el archivo ya no existe.
+				await fsp.rm(file, { force: true });
+				verboseLog(`Sync (remove) : ${file}`);
+			} catch (e) {
+				// El error solo se registraría si es un problema de permisos, etc.
+				console.error(`Failed to remove ${file}:`, e);
+			}
+		}
+
+		await this.restoreWebpackCompilationHash();
+
+		// Copy...
+		const allFilesToAdd = [...this.state.htmlToAdd, ...this.state.jsonToAdd];
+		for (const file of allFilesToAdd) {
+			try {
+				await fsp.mkdir(path.dirname(file.to), { recursive: true });
+				await fsp.copyFile(file.from, file.to);
+				verboseLog(`Sync (copy) : ${file.from} -> ${file.to}`);
+			} catch (e) {
+				console.error(`Failed to copy ${file.from} to ${file.to}:`, e);
+			}
+		}
+
+		// AVOID TO RESTORE webpackCompilationHash
+		// await this.copyAppDataJsonFile();
+		await this.copyAssetArtifacts();
+		await this.copySitemaps();
+	}
+
+	private async restoreWebpackCompilationHash() {
+		const appDataJson = path.join(this.bundleDir, "page-data", "app-data.json");
+		const webpackCompilationHash = JSON.parse(
+			await fsp.readFile(appDataJson, "utf8"),
+		).webpackCompilationHash;
+		console.log(
+			"Leyendo el webpackCompilationHash desde",
+			appDataJson,
+			"=",
+			webpackCompilationHash,
+		);
+
+		const changedHtmlFilePaths = this.state.htmlToAdd.map(({ from }) => from);
+
+		for (const htmlFilePath of changedHtmlFilePaths) {
+			await patchHtmlFile(htmlFilePath, webpackCompilationHash);
+		}
+	}
+
+	// private async copyAppDataJsonFile() {
+	// 	try {
+	// 		const src = path.join(this.currentRenderDir, "page-data", "app-data.json");
+	// 		const dst = path.join(this.bundleDir, "page-data", "app-data.json");
+	// 		await fsp.copyFile(src, dst);
+	// 		verboseLog("Copied app-data.json");
+	// 	} catch (e) {
+	// 		console.error("Failed to copy app-data.json:", e);
+	// 	}
+	// }
+
+	private async copyAssetArtifacts() {
+		for (const assetArtifact of this.assetArtifacts) {
+			const src = path.join(this.currentRenderDir, assetArtifact);
+			const dst = path.join(this.bundleDir, assetArtifact);
+			try {
+				await fsp.mkdir(path.dirname(dst), { recursive: true });
+				await fsp.copyFile(src, dst);
+				verboseLog(`Copied artifact: ${assetArtifact}`);
+			} catch (e) {
+				console.error(`Failed to copy artifact ${src} to ${dst}:`, e);
+			}
+		}
+	}
+
+	private async copySitemaps() {
+		const renderSrc = path.resolve(this.currentRenderDir);
+		const renderDst = path.resolve(this.bundleDir);
+
+		// Primero, elimina los sitemaps existentes en el destino
+		const xmlFilesInDst = findFilesBySuffix(renderDst, ".xml");
+		for await (const xmlFile of xmlFilesInDst) {
+			const basename = path.basename(xmlFile);
+			if (basename.startsWith("sitemap-") || basename === "sitemap.xml") {
+				await fsp.rm(xmlFile, { force: true });
+			}
+		}
+
+		// Segundo, copia los nuevos sitemaps desde el origen
+		const xmlFilesInSrc = findFilesBySuffix(renderSrc, ".xml");
+		for await (const src of xmlFilesInSrc) {
+			// Reconstruye la ruta de destino de forma más robusta
+			const relativePath = path.relative(renderSrc, src);
+			const dst = path.join(renderDst, relativePath);
+
+			try {
+				await fsp.mkdir(path.dirname(dst), { recursive: true });
+				await fsp.copyFile(src, dst);
+			} catch (e) {
+				console.error(`Failed to copy sitemap ${src} to ${dst}:`, e);
+			}
+		}
+
+		verboseLog(`Copied sitemap files.`);
+	}
+
+	/**
+	 * Escanea un directorio de build de Gatsby y extrae la información esencial de cada página.
+	 * Lee cada page-data.json una sola vez para optimizar el rendimiento.
+	 * @param dir Directorio a escanear (src o dst).
+	 * @returns Un array con la información de cada página encontrada.
+	 */
+	private async scanPages(dir: string): Promise<Array<PageInfo>> {
+		// Array de paths de page-data.json
+		const gatsbyPageDataFiles = findFilesBySuffix(path.join(dir, "page-data"), "page-data.json");
+		const pagesFound: Array<PageInfo> = [];
+
+		for await (const jsonFile of gatsbyPageDataFiles) {
+			try {
+				const content = JSON.parse(await fsp.readFile(jsonFile, "utf-8")) as GatsbyPageData;
+				const id = content.result.pageContext.id;
+				const composePath = content.result.pageContext.fullPath.compose;
+				// Esto es necesario porque Gatsby convierte la ruta "/" en un
+				// directorio llamado "index" en lo que archivos page-data.json
+				// se refiere.
+				const normalizedCompose = composePath === "/" ? "index" : composePath;
+
+				pagesFound.push({
+					id,
+					composePath,
+					htmlPath: path.join(dir, composePath, "index.html"),
+					jsonPath: path.join(dir, "page-data", normalizedCompose, "page-data.json"),
+				});
+			} catch (e) {
+				console.error(`Error reading or parsing page data from ${jsonFile}:`, e);
+			}
+		}
+		return pagesFound;
+	}
+}
+
+export { SyncRender };

package/exporter/adapters/gatsby/shared/types.ts

@@ -0,0 +1,35 @@
+interface SSG {
+	assetPrefix: string;
+	ssgArtifacts: {
+		disposables: string[];
+		cacheables: string[];
+	};
+}
+
+// Minimal representation of the page structure
+interface GatsbyPageData {
+	path: string;
+	result: {
+		pageContext: {
+			id: number;
+			title: string;
+			path: string;
+			openGraph: any;
+			pageMetadata: any;
+			page: {
+				site: number;
+				id: number;
+				fullPath: { compose: string };
+				language: number;
+				title: string;
+				fullUrl: string;
+				templateId: string;
+				template: {
+					templateType: string;
+				};
+			};
+		};
+	};
+}
+
+export type { GatsbyPageData, SSG };

package/exporter/build-esbuild.noop

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Build CX script file
+#
+# This file generates the necessary builds for CX.
+# - Setup, to generate necessary elements (tsconfig, etc..)
+# - Library, CX library that will be used by the SSG framework.
+# - Scripts, CX scripts that will be used by infra during the render process.
+#
+# @todo:
+# In the future, we will try to eliminate the bundling processes and use
+# TypeScript or ESModules directly.
+
+# start
+echo "📦 Building ..."
+
+# CLI options
+log="--log-level=error"
+production_node_opts="--bundle --platform=node --minify --sourcemap"
+debugging_node_opts="--bundle --platform=node --sourcemap"
+# react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
+
+# Manage --minify for debugging
+bundle_node_opts=$production_node_opts
+if [ "$1" = "--debug" ]; then
+    bundle_node_opts=$debugging_node_opts
+fi
+
+# library
+# esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
+# esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
+
+# scripts
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
+esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts} --outfile=./build/prepare-assets-directory.js
+
+# types
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/build.sh

@@ -1,39 +1,5 @@
-# Build CX script file
-#
-# This file generates the necessary builds for CX.
-# - Setup, to generate necessary elements (tsconfig, etc..)
-# - Library, CX library that will be used by the SSG framework.
-# - Scripts, CX scripts that will be used by infra during the render process.
-#
-# @todo:
-# In the future, we will try to eliminate the bundling processes and use
-# TypeScript or ESModules directly.
+#!/usr/bin/env bash
 
-# start
-echo "📦 Building ..."
-
-# CLI options
-log="--log-level=error"
-production_node_opts="--bundle --platform=node --minify --sourcemap"
-debugging_node_opts="--bundle --platform=node --sourcemap"
-react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
-
-# Manage --minify for debugging
-bundle_node_opts=$production_node_opts
-if [ "$1" = "--debug" ]; then
-    bundle_node_opts=$debugging_node_opts
-fi
-
-# library
-esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
-esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
-
-# scripts
-esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
-esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
-esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
-esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
-esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
-
-# types
-tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
+echo "📦 Compiling CX..."
+tsgo --project tsconfig.commands.json
+# tsgo --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/commands/end-render.ts

@@ -1,105 +1,109 @@
 #!/usr/bin/env node
 
-import fs from "node:fs";
-import path from "node:path";
+import type { Report } from "../types/render";
 
-import pkgDir from "pkg-dir";
+import fsp from "node:fs/promises";
+import path from "node:path";
 
+import { ArtifactError } from "../constants/errors";
 import { AuthService } from "../services/auth";
 import { endSiteRender } from "../services/sites";
-import { getConfig } from "../utils/core-utils";
 import { getInstanceDomains } from "../utils/domains";
+import { throwError } from "../utils/errors";
+import { pathExists } from "../utils/folders";
 import { infoLog } from "../utils/loggin";
+import { getRenderMetadataFromDB, getRenderMode, getRenderPathsFromDB } from "../utils/render";
 
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
-
-// Where we are going to find export dirs
-const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
-
-// Where we are going to find archived exports
-const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
-
-type Report = {
-	authControl:
-		| {
-				Authorization: string;
-				"Cache-Control": string;
-				lang?: string | undefined;
-		  }
-		| undefined;
-	sites: Array<{
-		siteId: number;
-		publishHashes: Array<string>;
-		siteHash: string | null;
-		unpublishHashes: Array<string>;
-		publishPagesIds: Array<number>;
-	}>;
-};
-
-function getDomainRenderReport(domain: string) {
-	const reports: Array<Report> = [];
-	const { buildReportFileName } = getConfig();
+const renderByDomains = !!process.env.GRIDDO_RENDER_BY_DOMAINS;
 
-	const buildReportFile = path.resolve(
-		exportArchiveBasePath,
-		domain,
-		"dist",
-		buildReportFileName,
-	);
+async function getRenderReport(options: {
+	domain: string;
+	exportsSiteDir: string;
+	buildReportFileName: string;
+}) {
+	const { buildReportFileName, domain, exportsSiteDir: exportArchiveBasePath } = options;
+	const buildReportFile = path.resolve(exportArchiveBasePath, domain, "dist", buildReportFileName);
 
-	if (!fs.existsSync(buildReportFile)) {
-		console.info(`Build report file "${buildReportFile}" not found.`);
+	if (!(await pathExists(buildReportFile))) {
+		infoLog(`Build report file "${buildReportFile}" not found.`);
+		throwError(ArtifactError);
 	}
 
-	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
-
-	reports.push(buildReport);
-
-	return reports;
+	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
 }
 
-(async () => {
-	const sendEndSiteRender = async (reports: Array<Report>) => {
-		for (const report of reports) {
-			const sites = report.sites;
+const sendEndSiteRender = async (report: Report) => {
+	const sites = report.sites;
 
-			for (const site of sites) {
-				const { siteId, ...body } = site;
+	for (const site of sites) {
+		const { siteId, ...body } = site;
+		infoLog(`Site (${siteId})`);
+		await endSiteRender(siteId, body);
+	}
+};
 
-				console.info(`Site (${siteId})`);
+async function endRender() {
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { __sites } = await getRenderPathsFromDB();
 
-				await endSiteRender(siteId, body);
-			}
+	await AuthService.login();
+	if (renderByDomains) {
+		const [domain] = process.argv.slice(2);
+		const { renderMode } = await getRenderMode(domain);
+
+		if (renderMode === "IDLE") {
+			infoLog(`Skipping build-end for domain ${domain} as it is marked as IDLE.`);
+			return;
 		}
-	};
 
-	if (RENDER_BY_DOMAIN) {
-		await AuthService.login();
-		const domain = process.argv.splice(2)[0];
-		const reports = getDomainRenderReport(domain);
+		const report = await getRenderReport({
+			domain,
+			exportsSiteDir: __sites,
+			buildReportFileName,
+		});
 
 		infoLog(`Sending ending call to ${domain} sites`);
 
-		for (const report of reports) {
-			for (const site of report.sites) {
-				site.publishHashes = [...new Set(site.publishHashes)];
-				site.unpublishHashes = [...new Set(site.unpublishHashes)];
-			}
+		for (const site of report.sites) {
+			site.publishHashes = [...new Set(site.publishHashes)];
+			site.unpublishHashes = [...new Set(site.unpublishHashes)];
 		}
 
-		await sendEndSiteRender(reports);
+		await sendEndSiteRender(report);
 	} else {
-		// Legacy render
+		console.log("- LEGACY RENDER MODE -");
 		const domains = await getInstanceDomains();
 		for (const domain of domains) {
-			const reports = getDomainRenderReport(domain);
+			const { renderMode } = await getRenderMode(domain);
+
+			if (renderMode === "IDLE") {
+				infoLog(`Skipping build-end for domain ${domain} as it is marked as IDLE.`);
+				return;
+			}
+
+			const report = await getRenderReport({
+				domain,
+				exportsSiteDir: __sites,
+				buildReportFileName,
+			});
 
 			infoLog(`Sending ending call to ${domain} sites`);
 
-			await sendEndSiteRender(reports);
+			for (const site of report.sites) {
+				site.publishHashes = [...new Set(site.publishHashes)];
+				site.unpublishHashes = [...new Set(site.unpublishHashes)];
+			}
+
+			await sendEndSiteRender(report);
 		}
 	}
-})().catch((err) => {
+}
+
+async function main() {
+	await endRender();
+}
+
+main().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/prepare-assets-directory.ts

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+import childProcess from "node:child_process";
+import path from "node:path";
+
+import { pathExists } from "../utils/folders";
+import { localLog } from "../utils/loggin";
+import { getRenderPathsFromDB } from "../utils/render";
+
+async function prepareAssetsDir() {
+	const [domain] = process.argv.slice(2);
+
+	const { __exports } = await getRenderPathsFromDB({ domain });
+
+	const assetsDir = path.join(__exports, "assets");
+	const domainAssetsDir = path.join(__exports, domain);
+
+	if (await pathExists(assetsDir)) {
+		childProcess.execSync(`rm -rf ${domainAssetsDir}`);
+		childProcess.execSync(`mv ${assetsDir} ${domainAssetsDir}`);
+	} else {
+		localLog("Assets directory not found");
+	}
+}
+
+async function main() {
+	await prepareAssetsDir();
+}
+
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});