@griddo/cx 11.7.5 → 11.7.6-rc.1

package/exporter/adapters/gatsby/shared/sync-render.ts

@@ -0,0 +1,294 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { verboseLog } from "../../../utils/loggin";
+
+type SyncRenderConfig = {
+	src: string;
+	dst: string;
+	pagesToCreate: Array<number>;
+	pagesToDelete: Array<number>;
+	artifactsToCopyToExports: Array<string>;
+};
+
+type CopyFilePath = {
+	from: string;
+	to: string;
+};
+
+type SyncState = {
+	htmlToAdd: Array<CopyFilePath>;
+	jsonToAdd: Array<CopyFilePath>;
+	htmlToDelete: Array<string>;
+	jsonToDelete: Array<string>;
+};
+
+class SyncRender {
+	private bundleDir: string;
+	private currentRenderDir: string;
+	private pagesToCreate: Array<number>;
+	private pagesToDelete: Array<number>;
+	private assetArtifacts: Array<string>;
+
+	private state: SyncState = {
+		htmlToAdd: [],
+		jsonToAdd: [],
+		htmlToDelete: [],
+		jsonToDelete: [],
+	};
+
+	constructor(config: SyncRenderConfig) {
+		this.bundleDir = config.dst;
+		this.currentRenderDir = config.src;
+		this.pagesToCreate = config.pagesToCreate;
+		this.pagesToDelete = config.pagesToDelete;
+		this.assetArtifacts = config.artifactsToCopyToExports;
+	}
+
+	// Se basa en los paths encontrados en page-data.json y no en recorrer físicamente el file-system
+	setPagesToDelete() {
+		// El set de ids están tanto las páginas que se van a crear como borrar.
+		// El por qué poner las que se van a crear es porque necesitamos quitar esas páginas con esos ids por si las que se van a crear tienen otro slug.
+		// Por ejemplo si se va crear la página id3 con slug /page-with-new-slug
+		// pero existía antes con el slug /page se borrará /page del sistema de
+		// ficheros y después se creará /page-with-new-slug ya que solo puede
+		// existir una página con el mismo id salvo que sea Multi-Page o
+		// Listado-Estático.
+		const idSet = new Set<number>([...this.pagesToDelete, ...this.pagesToCreate]);
+		const composePaths = this.getPageComposePaths(this.bundleDir);
+
+		for (const compose of composePaths) {
+			const normalizedCompose = compose === "/" ? "index" : compose;
+			const json = JSON.parse(
+				fs.readFileSync(
+					path.join(this.bundleDir, "page-data", normalizedCompose, "page-data.json"),
+					"utf8",
+				),
+			);
+			const id = json.result.pageContext.id;
+			const pagePath = json.result.pageContext.fullPath.compose as string;
+			// si el id de la página está en los ids para borrar (idSet) se añade
+			if (idSet.has(id)) {
+				try {
+					this.state.htmlToDelete.push(path.join(this.bundleDir, pagePath, "index.html"));
+					this.state.jsonToDelete.push(
+						path.join(
+							this.bundleDir,
+							"page-data",
+							pagePath === "/" ? "index" : pagePath,
+							"page-data.json",
+						),
+					);
+				} catch (e) {
+					console.log(e);
+				}
+			}
+		}
+
+		verboseLog(`${this.state.htmlToDelete.length} pages HTML to delete`);
+		verboseLog(`${this.state.jsonToDelete.length} pages JSON to delete`);
+	}
+
+	// Se basa en los paths encontrados en page-data.json y no en recorrer físicamente el file-system
+	// Cuando añadimos una página con id 3 (p.e.), hay que buscar todas las
+	// páginas con ese id en el bundle y borrar sus html/json. Esto va a pasar
+	// por ejemplo si cambiamos el slug de la página 3. Eso hará que se guarde
+	// en distintas carpetas pero no es una página que venga en "pagesToDelete"
+	setPagesToAdd() {
+		const idSet = new Set<number>(this.pagesToCreate);
+		const composePaths = this.getPageComposePaths(this.currentRenderDir);
+
+		for (const compose of composePaths) {
+			// 1 - leemos el page-data.json de la página para añadir
+			// Gatsby crea un directorio `index` para la página raíz `/`
+			const normalizedCompose = compose === "/" ? "index" : compose;
+			const json = JSON.parse(
+				fs.readFileSync(
+					path.join(this.currentRenderDir, "page-data", normalizedCompose, "page-data.json"),
+					"utf8",
+				),
+			);
+			// 2 - sacamos el id
+			const id = json.result.pageContext.id;
+
+			// 3 - obtenemos el src y dst del HTML
+			const htmlFrom = path.join(this.currentRenderDir, compose, "index.html");
+			const htmlTo = path.join(this.bundleDir, compose, "index.html");
+			// 4 - obtenemos el src/dst del JSON
+			const jsonFrom = path.join(
+				this.currentRenderDir,
+				"page-data",
+				normalizedCompose,
+				"page-data.json",
+			);
+			const jsonTo = path.join(this.bundleDir, "page-data", normalizedCompose, "page-data.json");
+
+			// Solo actuamos en la página si su id está en pagesToCreate
+			if (idSet.has(id)) {
+				try {
+					this.state.htmlToAdd.push({ from: htmlFrom, to: htmlTo });
+					this.state.jsonToAdd.push({ from: jsonFrom, to: jsonTo });
+				} catch (e) {
+					console.log(e);
+				}
+			}
+
+			verboseLog(`${this.state.htmlToAdd.length} pages HTML to create`);
+			verboseLog(`${this.state.jsonToAdd.length} pages JSON to create`);
+		}
+	}
+
+	/**
+	 * Realiza la sincronización de los archivos HTML y JSON entre el directorio
+	 * actual de renderizado y el directorio de exportación.
+	 */
+	sync() {
+		// remove html's
+		for (const html of this.state.htmlToDelete) {
+			try {
+				fs.rmSync(html);
+				verboseLog(`Sync (remove) : ${html}`);
+			} catch (e) {
+				console.log(`Does not exist ${html}`, e);
+			}
+		}
+
+		// remove json's
+		for (const json of this.state.jsonToDelete) {
+			try {
+				fs.rmSync(json);
+				verboseLog(`Sync (remove) : ${json}`);
+			} catch (e) {
+				console.log(`Does not exist ${json}`, e);
+			}
+		}
+
+		// copy html's
+		for (const html of this.state.htmlToAdd) {
+			fs.mkdirSync(path.dirname(html.to), { recursive: true });
+			fs.copyFileSync(html.from, html.to);
+			verboseLog(`Sync (copy) : ${html.from} -> ${html.to}`);
+		}
+
+		// copy json's
+		for (const json of this.state.jsonToAdd) {
+			fs.mkdirSync(path.dirname(json.to), { recursive: true });
+			fs.copyFileSync(json.from, json.to);
+			verboseLog(`Sync (copy) : ${json.from} -> ${json.to}`);
+		}
+
+		// update app-data.json (update the webpackCompilationHash)
+		this.copyAppDataJsonFile();
+
+		// copy artifacts
+		this.copyDirs();
+
+		// copy sitemaps
+		this.copySitemaps();
+	}
+
+	/**
+	 * Busca archivos por extensión de forma recursiva en un directorio.
+	 * @param dir Directorio base.
+	 * @param extension Extensión a buscar (ej: ".ts", ".js").
+	 * @returns Lista de rutas absolutas de archivos encontrados.
+	 */
+	private findFilesByExtension(dir: string, extension: string): Array<string> {
+		let results: Array<string> = [];
+
+		if (!fs.existsSync(dir)) {
+			return [];
+		}
+
+		const list = fs.readdirSync(dir, { withFileTypes: true });
+
+		for (const item of list) {
+			const fullPath = path.join(dir, item.name);
+
+			if (item.isDirectory()) {
+				results = results.concat(this.findFilesByExtension(fullPath, extension));
+			} else if (item.isFile() && fullPath.endsWith(extension)) {
+				results.push(fullPath);
+			}
+		}
+
+		return results;
+	}
+
+	private findAllPageDataJsons(dir: string) {
+		return this.findFilesByExtension(dir, ".json").filter((n) => n.endsWith("page-data.json"));
+	}
+
+	private getPageComposePaths(src: string) {
+		// [page-data/foo/bar/page-data.json, ...]
+		const allPageDataJson = this.findAllPageDataJsons(src);
+
+		// Array de fullPaths
+		// Entre ellos un fullPath puede ser: "/" (El root de la instancia)
+
+		// No sé si hacer esto en este momento...
+		// fullPath: "/foo/bar" => "foo/bar"
+		// fullPath: "/" => "index"
+		return allPageDataJson.map(
+			(p) => JSON.parse(fs.readFileSync(p, "utf-8")).result.pageContext.fullPath.compose,
+		);
+	}
+
+	private copyAppDataJsonFile() {
+		fs.copyFileSync(
+			path.join(this.currentRenderDir, "page-data", "app-data.json"),
+			path.join(this.bundleDir, "page-data", "app-data.json"),
+		);
+		verboseLog("Copied app-data.json");
+	}
+
+	private copyDirs() {
+		for (const assetArtifact of this.assetArtifacts) {
+			fs.copyFileSync(
+				path.join(this.currentRenderDir, assetArtifact),
+				path.join(this.bundleDir, assetArtifact),
+			);
+			verboseLog(
+				`Copied ${path.join(this.currentRenderDir, assetArtifact)} -> to -> ${path.join(
+					this.bundleDir,
+					assetArtifact,
+				)}`,
+			);
+		}
+	}
+
+	private copySitemaps() {
+		/*
+		1. buscar directorios que tengan como hijo un `sitemap.xml`
+		2. copiar todos los sitemap.xml y sitemap-* a /exports/sites/<domain>/<site>/ borrando previamente los .xmls que existan
+		*/
+		const renderSrc = path.resolve(this.currentRenderDir);
+		const renderDst = path.resolve(this.bundleDir);
+		const xmlFilesInSrc = this.findFilesByExtension(renderSrc, ".xml");
+		const xmlFilesInDst = this.findFilesByExtension(renderDst, ".xml");
+
+		// Eliminar los sitemaps de griddo que existan en el destino, se van a
+		// crear todos nuevos en la fase Meta.
+		for (const xmlFile of xmlFilesInDst) {
+			if (
+				path.basename(xmlFile).startsWith("sitemap-") ||
+				path.basename(xmlFile) === "sitemap.xml"
+			) {
+				fs.rmSync(xmlFile, { force: true });
+			}
+		}
+
+		for (const xmlFile of xmlFilesInSrc) {
+			const src = xmlFile;
+			// cambia la ruta, de src -> dst
+			const dst = xmlFile.replace(renderSrc, renderDst);
+
+			if (!fs.existsSync(path.dirname(dst))) {
+				fs.mkdirSync(path.dirname(dst), { recursive: true });
+			}
+			fs.copyFileSync(src, dst);
+		}
+	}
+}
+
+export { SyncRender };

package/exporter/adapters/gatsby/shared/types.ts

@@ -0,0 +1,35 @@
+interface SSG {
+	assetPrefix: string;
+	ssgArtifacts: {
+		disposables: string[];
+		cacheables: string[];
+	};
+}
+
+// Minimal representation of the page structure
+interface GatsbyPageData {
+	path: string;
+	result: {
+		pageContext: {
+			id: number;
+			title: string;
+			path: string;
+			openGraph: any;
+			pageMetadata: any;
+			page: {
+				site: number;
+				id: number;
+				fullPath: { compose: string };
+				language: number;
+				title: string;
+				fullUrl: string;
+				templateId: string;
+				template: {
+					templateType: string;
+				};
+			};
+		};
+	};
+}
+
+export type { GatsbyPageData, SSG };

package/exporter/build-esbuild.noop

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Build CX script file
+#
+# This file generates the necessary builds for CX.
+# - Setup, to generate necessary elements (tsconfig, etc..)
+# - Library, CX library that will be used by the SSG framework.
+# - Scripts, CX scripts that will be used by infra during the render process.
+#
+# @todo:
+# In the future, we will try to eliminate the bundling processes and use
+# TypeScript or ESModules directly.
+
+# start
+echo "📦 Building ..."
+
+# CLI options
+log="--log-level=error"
+production_node_opts="--bundle --platform=node --minify --sourcemap"
+debugging_node_opts="--bundle --platform=node --sourcemap"
+# react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
+
+# Manage --minify for debugging
+bundle_node_opts=$production_node_opts
+if [ "$1" = "--debug" ]; then
+    bundle_node_opts=$debugging_node_opts
+fi
+
+# library
+# esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
+# esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
+
+# scripts
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
+esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts} --outfile=./build/prepare-assets-directory.js
+
+# types
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/build.sh

@@ -1,39 +1,6 @@
-# Build CX script file
-#
-# This file generates the necessary builds for CX.
-# - Setup, to generate necessary elements (tsconfig, etc..)
-# - Library, CX library that will be used by the SSG framework.
-# - Scripts, CX scripts that will be used by infra during the render process.
-#
-# @todo:
-# In the future, we will try to eliminate the bundling processes and use
-# TypeScript or ESModules directly.
+#!/usr/bin/env bash
 
-# start
-echo "📦 Building ..."
-
-# CLI options
-log="--log-level=error"
-production_node_opts="--bundle --platform=node --minify --sourcemap"
-debugging_node_opts="--bundle --platform=node --sourcemap"
-react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
-
-# Manage --minify for debugging
-bundle_node_opts=$production_node_opts
-if [ "$1" = "--debug" ]; then
-    bundle_node_opts=$debugging_node_opts
-fi
-
-# library
-esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
-esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
-
-# scripts
-esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
-esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
-esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
-esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
-esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
-
-# types
-tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
+echo "📦 Compiling CX..."
+rm -rf ./build
+tsc --project tsconfig.commands.json
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/commands/end-render.ts

@@ -1,105 +1,80 @@
 #!/usr/bin/env node
 
-import fs from "node:fs";
-import path from "node:path";
+import type { Report } from "../types/render";
 
-import pkgDir from "pkg-dir";
+import fsp from "node:fs/promises";
+import path from "node:path";
 
+import { ArtifactError } from "../constants/errors";
 import { AuthService } from "../services/auth";
 import { endSiteRender } from "../services/sites";
-import { getConfig } from "../utils/core-utils";
-import { getInstanceDomains } from "../utils/domains";
+import { doLifeCycle } from "../utils/core-utils";
+import { throwError } from "../utils/errors";
+import { pathExists } from "../utils/folders";
 import { infoLog } from "../utils/loggin";
-
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
-
-// Where we are going to find export dirs
-const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
-
-// Where we are going to find archived exports
-const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
-
-type Report = {
-	authControl:
-		| {
-				Authorization: string;
-				"Cache-Control": string;
-				lang?: string | undefined;
-		  }
-		| undefined;
-	sites: Array<{
-		siteId: number;
-		publishHashes: Array<string>;
-		siteHash: string | null;
-		unpublishHashes: Array<string>;
-		publishPagesIds: Array<number>;
-	}>;
-};
-
-function getDomainRenderReport(domain: string) {
-	const reports: Array<Report> = [];
-	const { buildReportFileName } = getConfig();
-
-	const buildReportFile = path.resolve(
-		exportArchiveBasePath,
-		domain,
-		"dist",
-		buildReportFileName,
-	);
-
-	if (!fs.existsSync(buildReportFile)) {
-		console.info(`Build report file "${buildReportFile}" not found.`);
+import { getRenderMetadataFromDB, getRenderMode, getRenderPathsFromDB } from "../utils/render";
+
+async function getRenderReport(options: {
+	domain: string;
+	exportsSiteDir: string;
+	buildReportFileName: string;
+}) {
+	const { buildReportFileName, domain, exportsSiteDir: exportArchiveBasePath } = options;
+	const buildReportFile = path.resolve(exportArchiveBasePath, domain, "dist", buildReportFileName);
+
+	if (!(await pathExists(buildReportFile))) {
+		infoLog(`Build report file "${buildReportFile}" not found.`);
+		throwError(ArtifactError);
 	}
 
-	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
+	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
+}
 
-	reports.push(buildReport);
+const sendEndSiteRender = async (report: Report) => {
+	const sites = report.sites;
 
-	return reports;
-}
+	for (const site of sites) {
+		const { siteId, ...body } = site;
+		infoLog(`Site (${siteId})`);
+		await endSiteRender(siteId, body);
+	}
+};
 
-(async () => {
-	const sendEndSiteRender = async (reports: Array<Report>) => {
-		for (const report of reports) {
-			const sites = report.sites;
+async function endRenderAction() {
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { __sites } = await getRenderPathsFromDB();
 
-			for (const site of sites) {
-				const { siteId, ...body } = site;
+	const [domain] = process.argv.slice(2);
+	const { renderMode } = await getRenderMode(domain);
 
-				console.info(`Site (${siteId})`);
+	if (renderMode === "IDLE") {
+		infoLog(`Skipping build-end for domain ${domain} as it is marked as IDLE.`);
+		return;
+	}
 
-				await endSiteRender(siteId, body);
-			}
-		}
-	};
+	await AuthService.login();
 
-	if (RENDER_BY_DOMAIN) {
-		await AuthService.login();
-		const domain = process.argv.splice(2)[0];
-		const reports = getDomainRenderReport(domain);
+	const report = await getRenderReport({
+		domain,
+		exportsSiteDir: __sites,
+		buildReportFileName,
+	});
 
-		infoLog(`Sending ending call to ${domain} sites`);
+	infoLog(`Sending ending call to ${domain} sites`);
 
-		for (const report of reports) {
-			for (const site of report.sites) {
-				site.publishHashes = [...new Set(site.publishHashes)];
-				site.unpublishHashes = [...new Set(site.unpublishHashes)];
-			}
-		}
+	for (const site of report.sites) {
+		site.publishHashes = [...new Set(site.publishHashes)];
+		site.unpublishHashes = [...new Set(site.unpublishHashes)];
+	}
 
-		await sendEndSiteRender(reports);
-	} else {
-		// Legacy render
-		const domains = await getInstanceDomains();
-		for (const domain of domains) {
-			const reports = getDomainRenderReport(domain);
+	await sendEndSiteRender(report);
+}
 
-			infoLog(`Sending ending call to ${domain} sites`);
+async function main() {
+	await doLifeCycle("EndRender", endRenderAction);
+}
 
-			await sendEndSiteRender(reports);
-		}
-	}
-})().catch((err) => {
+main().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/prepare-assets-directory.ts

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+import childProcess from "node:child_process";
+import path from "node:path";
+
+import { doLifeCycle } from "../utils/core-utils";
+import { pathExists } from "../utils/folders";
+import { getRenderPathsFromDB } from "../utils/render";
+
+async function prepareAssetsAction() {
+	const [domain] = process.argv.slice(2);
+
+	const { __exports } = await getRenderPathsFromDB({ domain });
+
+	const assetsDir = path.join(__exports, "assets");
+	const domainAssetsDir = path.join(__exports, domain);
+
+	if (await pathExists(assetsDir)) {
+		childProcess.execSync(`rm -rf ${domainAssetsDir}`);
+		childProcess.execSync(`mv ${assetsDir} ${domainAssetsDir}`);
+	} else {
+		console.log("Assets directory not found");
+	}
+}
+
+async function main() {
+	await doLifeCycle("PrepareAssets(Internal)", prepareAssetsAction);
+}
+
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});