@griddo/cx 10.3.26 → 10.4.0

package/exporter/services/store.ts

@@ -6,14 +6,8 @@ import type {
 	GriddoSinglePage,
 	PageAdditionalInfo,
 } from "../types/pages";
-import type { Site } from "../types/sites";
-
-import fs from "node:fs";
-import fsp from "node:fs/promises";
-import path from "node:path";
 
 import dotenv from "dotenv";
-import fsx from "fs-extra";
 import pLimit from "p-limit";
 
 import { DistributorService } from "./distributors";
@@ -21,8 +15,7 @@ import { NavigationService } from "./navigation";
 import { SettingsService } from "./settings";
 import { SitesService } from "./sites";
 import { version as griddoVersion } from "../../package.json";
-import { createSha256, updatedSiteHash } from "../utils/cache";
-import { deleteSites } from "../utils/folders";
+import { updatedSiteHash } from "../utils/cache";
 import {
 	createGriddoListPages,
 	createGriddoMultiPages,
@@ -30,25 +23,27 @@ import {
 	getMultiPageElements,
 	getPaginatedPages,
 } from "../utils/pages";
-import { logInfo, removeProperties, siteList } from "../utils/shared";
+import { logBox, logInfo, siteList } from "../utils/shared";
 import { checkSites, getSiteData, unpublishSites } from "../utils/sites";
+import {
+	getPagesToCreateOrDelete,
+	removePagesFromStore,
+	savePagesInStore,
+	saveRenderInfoInStore,
+} from "../utils/store";
 
 dotenv.config();
 
 // Envs
-const API_URL = process.env.API_URL as string;
-const CXBRANCH = process.env.CXBRANCH;
-// prettier-ignore
-const REACT_APP_INSTANCE = process.env.GRIDDO_REACT_APP_INSTANCE || process.env.REACT_APP_INSTANCE;
-// prettier-ignore
-const API_CONCURRENCY_COUNT = parseInt(process.env.GRIDDO_API_CONCURRENCY_COUNT || "") || 10;
-const PUBLIC_API_URL = process.env.PUBLIC_API_URL as string;
-// prettier-ignore
-const RENDER_ID = process.env.GRIDDO_RENDERID || new Date().valueOf().toString();
+/* prettier-ignore */ const API_URL = process.env.API_URL as string;
+/* prettier-ignore */ const REACT_APP_INSTANCE = process.env.GRIDDO_REACT_APP_INSTANCE || process.env.REACT_APP_INSTANCE;
+/* prettier-ignore */ const API_CONCURRENCY_COUNT = parseInt(process.env.GRIDDO_API_CONCURRENCY_COUNT || "10");
+/* prettier-ignore */ const PUBLIC_API_URL = process.env.PUBLIC_API_URL as string;
+/* prettier-ignore */ const RENDER_ID = process.env.GRIDDO_RENDERID || new Date().valueOf().toString();
 
 // Consts
-const CREATED_PAGES: Array<number> = [];
-const BUILD_PROCESS_DATA: BuildProcessData = {};
+const createdPages: Array<number> = [];
+const buildProcessData: BuildProcessData = {};
 
 /**
  * Fetch, process and save object pages and sites data into the file system to
@@ -57,20 +52,10 @@ const BUILD_PROCESS_DATA: BuildProcessData = {};
 async function createStore(storeFolder: string, domain: string) {
 	console.info(`API calls with ${API_CONCURRENCY_COUNT} threads`);
 
-	// Remove old store
-	if (fs.existsSync(storeFolder)) {
-		fs.rmSync(storeFolder, { recursive: true });
-	}
-
-	// Create fresh folders for store and store/metadata
-	fs.mkdirSync(storeFolder);
-	fs.mkdirSync(path.resolve(storeFolder, "metadata"));
-
 	try {
 		// Get sites objects to publish and unpublish.
 		const { sitesToPublish, sitesToUnpublish } = await checkSites(domain);
 
-		// The settings will be extracted below in this process.
 		await SettingsService.getAll();
 
 		// If no activity in sites, exit
@@ -82,16 +67,19 @@ async function createStore(storeFolder: string, domain: string) {
 		console.info(`Sites to publish: ${siteList(sitesToPublish)}`);
 		console.info(`Sites to unpublish: ${siteList(sitesToUnpublish)}`);
 
-		// Unpublish (API) and delete sites
-		// TODO: Manage this to save end data to the __report-build__ file ???
+		// Send unpublished sites to API.
+		// @todo ¿mover esta llamada al final del proceso de render?
 		await unpublishSites(sitesToUnpublish);
 
-		// Delete (clean) sites to publish
-		// SPREAD: await deleteSites(sitesToPublish);
-		await deleteSites([...sitesToPublish]);
-
 		for (const site of sitesToPublish) {
-			const { id: siteId, slug: siteSlug, theme, favicon } = site;
+			const {
+				id: siteId,
+				slug: siteSlug,
+				theme,
+				favicon,
+				changedPages = [],
+				pages = [],
+			} = site;
 
 			const {
 				siteInfo,
@@ -116,15 +104,13 @@ async function createStore(storeFolder: string, domain: string) {
 				avoidDebugMetas,
 			} = SettingsService.settings;
 
-			BUILD_PROCESS_DATA[siteId] = {
+			buildProcessData[siteId] = {
 				siteHash,
 				unpublishHashes,
 				publishHashes: [],
 			};
 
 			const NavService = new NavigationService();
-
-			// Set `headers` and `footers` to the NavService to use later in this process.
 			NavService.navigations = { headers, footers };
 
 			site.languages = siteLangs;
@@ -159,7 +145,6 @@ async function createStore(storeFolder: string, domain: string) {
 					avoidHrefLangXDefault,
 					avoidDebugMetas,
 				},
-				BUILD_MODE: CXBRANCH,
 				siteScript,
 			};
 
@@ -170,9 +155,8 @@ async function createStore(storeFolder: string, domain: string) {
 			// -------------------------------------------------------------------------
 			// Async function that process every page for one site to use later in the
 			// `Promise.all` with pLimit.
-			const fetchPageAndSaveToStore = async (pageId: number) => {
-				// Here will be store every page returned by the API and processed by
-				// Griddo CX.
+			const fetchPageAndSaveInStore = async (pageId: number) => {
+				// Here will be store every page returned by the API and processed
 				let griddoPageObjects: Array<GriddoPageObject> = [];
 
 				// Get page data
@@ -182,7 +166,7 @@ async function createStore(storeFolder: string, domain: string) {
 				if (!page) return;
 
 				// Update Array of page ids
-				CREATED_PAGES.push(pageId);
+				createdPages.push(pageId);
 
 				// TODO: Use structuredClone() when node 18 is available.
 				// SHAME: This new pageAdditionalInfo needs to be a copy because
@@ -269,24 +253,60 @@ async function createStore(storeFolder: string, domain: string) {
 					];
 				}
 
-				// Update hashes
-				BUILD_PROCESS_DATA[siteId].publishHashes.push(page.hash);
+				// Upload only the valid pages hashes of pages that has been changed or created
+				if (page.hash !== null) {
+					buildProcessData[siteId].publishHashes.push(page.hash);
+				}
 
 				// Save build data to store
-				await savePagesInStore(storeFolder, griddoPageObjects); // gatsbyPageObjects is always an array regardless of whether it's just one page
-				setSitesToPublish(storeFolder, sitesToPublish);
-				setBuildProcessData(storeFolder, BUILD_PROCESS_DATA);
-				setCreatedPages(storeFolder, CREATED_PAGES);
+				savePagesInStore(storeFolder, griddoPageObjects);
 			};
 
-			// Create the pLimit array of promises
+			// Pages to be created or deleted in the store
+			const {
+				pagesInStore,
+				pagesMissingInStore,
+				pagesToDeleteFromStore,
+				pagesToWriteToStore,
+			} = await getPagesToCreateOrDelete(storeFolder, {
+				sitesToPublish,
+				pages,
+				validPagesIds,
+				changedPages,
+			});
+
+			// Delete pages from published, unpublished and deleted sites.
+			removePagesFromStore(storeFolder, pagesToDeleteFromStore);
+
+			// Debug time
+			const changedPageLogs = `Store:               ${pagesInStore.length} -> ${pagesInStore}
+				ValidPages:          ${validPagesIds.length} -> ${validPagesIds}
+				Changed:             ${changedPages.length} -> ${changedPages}
+				Deleted/Unpublished: ${pagesToDeleteFromStore.length} -> ${pagesToDeleteFromStore}
+				Missing in store:    ${pagesMissingInStore.length} -> ${pagesMissingInStore}
+				Finally to store:    ${pagesToWriteToStore.length} -> ${pagesToWriteToStore}`;
+
+			if (JSON.parse(process.env.GRIDDO_DEBUG_LOGS || "false")) {
+				logBox(changedPageLogs, `Site store: ${site.name}`, 1, 0);
+			}
+
+			// Create (async)
 			const limit = pLimit(API_CONCURRENCY_COUNT);
-			const pagesToRender = validPagesIds.map((id: number) =>
-				limit(() => fetchPageAndSaveToStore(id))
+			const pagesToStore = pagesToWriteToStore.map((id: number) =>
+				limit(() => fetchPageAndSaveInStore(id))
 			);
 
-			await Promise.all(pagesToRender);
+			await Promise.all(pagesToStore);
 		}
+
+		// Después de renderizar todos los sites y sus páginas (las que sean que
+		// hayan cambiado etc.) guardamos un archivo con la información del
+		// render para usos posteriores.
+		saveRenderInfoInStore(storeFolder, {
+			buildProcessData,
+			createdPages,
+			sitesToPublish,
+		});
 	} catch (e) {
 		const error = e as { message: string };
 		console.error(error.message);
@@ -294,58 +314,4 @@ async function createStore(storeFolder: string, domain: string) {
 	}
 }
 
-/**
- * Set the build process data in the Store.
- */
-function setBuildProcessData(
-	basePath: string,
-	buildProcessData: BuildProcessData
-) {
-	fsx.writeJSONSync(
-		path.resolve(basePath, "metadata", "buildProcessData.json"),
-		buildProcessData
-	);
-}
-
-/**
- * Set the created pages in the Store.
- */
-function setCreatedPages(basePath: string, pages: Array<number>) {
-	fsx.writeJSONSync(
-		path.resolve(basePath, "metadata", "createdPages.json"),
-		pages
-	);
-}
-
-/**
- * Set site to publish data in the Store.
- */
-function setSitesToPublish(basePath: string, sites: Array<Site>) {
-	fsx.writeJSONSync(
-		path.resolve(basePath, "metadata", "sitesToPublish.json"),
-		sites
-	);
-}
-
-/**
- * Save the pages into the file system.
- */
-async function savePagesInStore(
-	basePath: string,
-	pages: Array<GriddoPageObject>
-) {
-	pages.forEach((page) =>
-		removeProperties(page, ["editorID", "parentEditorID"])
-	);
-
-	const promises = pages.map(async (pageObject) => {
-		const filename = `${createSha256(pageObject.path)}.json`;
-		const fileContent = JSON.stringify(pageObject);
-		const filePath = path.resolve(basePath, filename);
-		await fsp.writeFile(filePath, fileContent);
-	});
-
-	await Promise.all(promises);
-}
-
 export { createStore };

package/exporter/types/global.ts

@@ -1,8 +1,10 @@
 import type { EndSiteRenderBody } from "./api";
 import type { Core, Fields } from "@griddo/core";
 
+import { Site } from "./sites";
+
 // TODO: JSDoc
-export interface Settings {
+interface Settings {
 	apiVersion?: string;
 	avoidCanonicalsOnSitemaps?: boolean;
 	avoidDebugMetas?: boolean;
@@ -27,10 +29,10 @@ export interface Settings {
 }
 
 // TODO: JSDoc
-export type Petition = Record<string, unknown>;
+type Petition = Record<string, unknown>;
 
 // TODO: JSDoc
-export interface PostSearchInfoProps {
+interface PostSearchInfoProps {
 	title?: string;
 	description: string | null | undefined;
 	image: string | null | undefined;
@@ -43,7 +45,7 @@ export interface PostSearchInfoProps {
 }
 
 // TODO: JSDoc
-export interface FetchDataProps {
+interface FetchDataProps {
 	page: Core.Page;
 	component: {
 		data?: Fields.Reference<unknown>;
@@ -53,17 +55,45 @@ export interface FetchDataProps {
 }
 
 // TODO: JSDoc
-export type Domains = Array<{
+type Domains = Array<{
 	id: number;
 	slug: string;
 	url: string;
 }>;
 
 /** Describes the type of build process data object. */
-export type BuildProcessData = Record<string, EndSiteRenderBody>;
+type BuildProcessData = Record<string, EndSiteRenderBody>;
 
 // TODO: JSDoc
-export type Robot = { path: string; content: string };
+type Robot = { path: string; content: string };
 
 // TODO: JSDoc
-export type Robots = Array<Robot>;
+type Robots = Array<Robot>;
+
+interface RenderInfo {
+	buildProcessData: BuildProcessData;
+	createdPages: Array<number>;
+	sitesToPublish: Array<Site>;
+}
+
+type LifeCycles = {
+	doRestore?: boolean;
+	doData?: boolean;
+	doSSG?: boolean;
+	doRelocation?: boolean;
+	doMeta?: boolean;
+	doArchive?: boolean;
+};
+
+export {
+	BuildProcessData,
+	Domains,
+	FetchDataProps,
+	LifeCycles,
+	Petition,
+	PostSearchInfoProps,
+	RenderInfo,
+	Robot,
+	Robots,
+	Settings,
+};

package/exporter/types/pages.ts

@@ -64,7 +64,6 @@ export type GriddoMultiPage = Core.Page & {
 // TODO: JSDoc
 export interface AdditionalInfo {
 	baseUrl: string;
-	BUILD_MODE?: string;
 	cloudinaryName?: string;
 	griddoVersion: string;
 	instance?: string;
@@ -118,7 +117,6 @@ export type GriddoPageObject = {
 	/** Page size in bytes */
 	size?: number;
 	context: {
-		BUILD_MODE?: string;
 		cloudinaryName?: string;
 		footer: Record<string, unknown> | null;
 		fullPath: Core.Page["fullPath"];

package/exporter/types/sites.ts

@@ -36,6 +36,7 @@ export interface Site
 	languageSites: Array<number>;
 	navigationModules: { header?: string; footer?: string } | null;
 	pages: Array<number>;
+	changedPages: Array<number>;
 	published: string;
 	rendering: boolean;
 	renderingHours: number;

package/exporter/utils/download-build-data.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 
 import { createAPICacheFolder } from "./cache";
 import { sanitizeAPICacheFolder } from "./shared";
+import { createStoreFolder } from "./store";
 import { createStore } from "../services/store";
 
 const storeFolder = path.resolve(__dirname, "../store/");
@@ -13,10 +14,11 @@ const apiCacheFolder = path.resolve(__dirname, "../apiCache");
  * `getBuildMetadata()` to get build and sites metadata as objects. Both from
  * exporter utils sites folder.
  */
-async function downloadBuildData(domain: string) {
+async function createBuildData(domain: string) {
 	createAPICacheFolder();
+	createStoreFolder(storeFolder);
 	await createStore(storeFolder, domain);
 	sanitizeAPICacheFolder(apiCacheFolder);
 }
 
-export { downloadBuildData };
+export { createBuildData };

package/exporter/utils/folders.ts

@@ -4,41 +4,47 @@ import path from "node:path";
 
 import fs from "fs-extra";
 
-import { logInfo } from "./shared";
+import { CXRootFolder, logInfo } from "./shared";
 
 /**
  * Delete sites folders that will be updated from de file system.
  *
  * @param updateSites An array of sites to be updated.
  */
-async function deleteSites(updatedSites: Array<Site>) {
+async function deleteSites(updatedSites: Array<Site>, DOMAIN: string) {
 	for await (const site of updatedSites) {
 		for (const domain of site.domains) {
-			const mappedDomain = Object.values(domain)[0];
-			const dir = path.resolve(__dirname, `../../dist${mappedDomain}`);
-			const pageDataDir = path.resolve(
+			const domainFromSite = Object.values(domain)[0];
+			const domainPath = JSON.parse(
+				process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH || "false"
+			)
+				? domainFromSite.split(DOMAIN).join("")
+				: domainFromSite;
+
+			const siteFolder = path.resolve(__dirname, `../public${domainPath}`);
+			const pageDataFolder = path.resolve(
 				__dirname,
-				`../../assets/page-data${mappedDomain}`
+				`../public/page-data${domainPath}`
 			);
 
-			logInfo(`Site dir ${dir}`);
-			logInfo(`Page data dir ${pageDataDir}`);
+			logInfo(`Site dir ${siteFolder}`);
+			logInfo(`Page data dir ${pageDataFolder}`);
 
 			// delete directory recursively
-			if (!fs.existsSync(dir)) return;
+			if (!fs.existsSync(siteFolder)) continue;
 
 			try {
-				await fs.rm(dir, { recursive: true });
-				logInfo(`${dir} is deleted!`);
+				await fs.rm(siteFolder, { recursive: true });
+				logInfo(`${siteFolder} was deleted!`);
 			} catch (err) {
 				console.log(err);
 			}
 
-			if (!fs.existsSync(pageDataDir)) return;
+			if (!fs.existsSync(pageDataFolder)) continue;
 
 			try {
-				await fs.rm(pageDataDir, { recursive: true });
-				console.info(`${pageDataDir} is deleted!`);
+				await fs.rm(pageDataFolder, { recursive: true });
+				console.info(`${pageDataFolder} was deleted!`);
 			} catch (err) {
 				console.log(err);
 			}
@@ -46,4 +52,83 @@ async function deleteSites(updatedSites: Array<Site>) {
 	}
 }
 
-export { deleteSites };
+/**
+ * Remove empty site dirst (with or without sitemaps xmls)
+ */
+const clearEmptyDirs = (baseFolder?: string) => {
+	const folder = baseFolder || path.resolve(CXRootFolder, "dist");
+
+	const isDir = fs.statSync(folder).isDirectory();
+	if (!isDir) {
+		return;
+	}
+
+	// archivos o direvtorios dentro de `folder`
+	let children = fs.readdirSync(folder);
+	// let children = childrenRaw.filter((file) => {
+	// 	return path.extname(file).toLowerCase() !== ".xml";
+	// });
+	const folderHasChildren = children.length > 0;
+
+	// Por cada uno de ellos miramos si tiene hijos, enviamos cada uno de ellos
+	// a `clearEmptyDirsWithXML`
+	if (folderHasChildren) {
+		const childrenCount = children.length;
+		const xmlCount = children.filter((file) => {
+			return path.extname(file).toLowerCase() === ".xml";
+		}).length;
+
+		// Todos los hijos son archivos .xml
+		// Podemos borrarlos y dejar el folder vacío.
+		if (childrenCount === xmlCount) {
+			children.forEach(function (xmlFile) {
+				const fullPath = path.join(folder, xmlFile);
+				fs.rmSync(fullPath);
+			});
+			children = fs.readdirSync(folder);
+		}
+
+		children.forEach(function (file) {
+			const fullPath = path.join(folder, file);
+			clearEmptyDirs(fullPath);
+		});
+
+		// re-evaluate files; after deleting subfolder we may have parent folder
+		// empty now...
+		children = fs.readdirSync(folder);
+	}
+
+	// Si no tiene hijos, lo borramos
+	if (children.length === 0) {
+		fs.rmdirSync(folder);
+		return;
+	}
+};
+
+const _clearEmptyDirs = (baseFolder?: string) => {
+	const folder = baseFolder || path.resolve(CXRootFolder, "dist");
+
+	const isDir = fs.statSync(folder).isDirectory();
+	if (!isDir) {
+		return;
+	}
+
+	let files = fs.readdirSync(folder);
+	if (files.length > 0) {
+		files.forEach(function (file) {
+			const fullPath = path.join(folder, file);
+			_clearEmptyDirs(fullPath);
+		});
+
+		// re-evaluate files; after deleting subfolder
+		// we may have parent folder empty now
+		files = fs.readdirSync(folder);
+	}
+
+	if (files.length === 0) {
+		fs.rmdirSync(folder);
+		return;
+	}
+};
+
+export { clearEmptyDirs, _clearEmptyDirs, deleteSites };

package/exporter/utils/health-checks.ts

@@ -30,7 +30,10 @@ function checkRenderHealthOrExit() {
 	}
 
 	// Error
-	logBox(`Error. The environment is not suitable for a build.`);
+	logBox(
+		`Error. The environment is not suitable for a build.`,
+		"Not valid environment"
+	);
 	process.exit(1);
 }
 

package/exporter/utils/instance.ts

@@ -46,7 +46,11 @@ function getComponentsLibAliases() {
 	return Object.keys(xsConfig?.compilerOptions?.paths).reduce(
 		(currentAlias, pathKey) => {
 			const [aliasKey] = pathKey.split("/");
-			const [pathAtJsConfig] = xsConfig?.compilerOptions?.paths[pathKey];
+			const [pathAtJsConfig] =
+				xsConfig &&
+				xsConfig.compilerOptions &&
+				xsConfig.compilerOptions.paths &&
+				xsConfig.compilerOptions.paths[pathKey];
 
 			const [relativePathToDir] = pathAtJsConfig.split("/*");
 

package/exporter/utils/messages.ts

@@ -0,0 +1,31 @@
+import chalk from "chalk";
+
+export const printWarningMessage = (message: string, location: string, expected: string) => {
+    const heading = "  > PRINT WARNING MESSAGE";
+    const longestLineLength = Math.max(message.length, location.length, heading.length, expected.length);
+    const line = '+'.padEnd(longestLineLength + 15, '-') + '+';
+
+    console.log(line);
+    console.log(chalk.keyword('orange').bold('|' + heading.padEnd(longestLineLength + 5)));
+    console.log('|');
+    console.log('|' + `  Location: "${location}"`.padEnd(longestLineLength + 5));
+    console.log('|' + `  Message: ${message}`.padEnd(longestLineLength + 5));
+    console.log('|' + `  Expected: ${expected}`.padEnd(longestLineLength + 5));
+    console.log('|');
+    console.log(line + '\n');
+};
+
+export const printErrorMessage = (message: string, location: string, expected: string) => {
+    const heading = "  > PRINT ERROR MESSAGE";
+    const longestLineLength = Math.max(message.length, location.length, heading.length, expected.length);
+    const line = '+'.padEnd(longestLineLength + 15, '-') + '+';
+
+    console.log(line);
+    console.log(chalk.keyword('red').bold('|' + heading.padEnd(longestLineLength + 5)));
+    console.log('|');
+    console.log('|' + `  Location: "${location}"`.padEnd(longestLineLength + 5));
+    console.log('|' + `  Message: ${message}`.padEnd(longestLineLength + 5));
+    console.log('|' + `  Expected: ${expected}`.padEnd(longestLineLength + 5));
+    console.log('|');
+    console.log(line + '\n');
+}

package/exporter/utils/pages.ts

@@ -110,8 +110,7 @@ function getPageMetaData(params: Core.Page): Core.Page["pageMetaData"] {
  */
 async function createGriddoPageObject(
 	page: GriddoSinglePage | GriddoListPage | GriddoMultiPage,
-	additionalInfo: PageAdditionalInfo,
-	excludeFromSearch = false
+	additionalInfo: PageAdditionalInfo
 ): Promise<GriddoPageObject> {
 	// Extract some props from page
 	const {
@@ -128,7 +127,6 @@ async function createGriddoPageObject(
 	// Extract some props from additionalInfo
 	const {
 		baseUrl,
-		BUILD_MODE,
 		cloudinaryName,
 		griddoVersion,
 		siteLangs,
@@ -187,7 +185,6 @@ async function createGriddoPageObject(
 			siteLangs,
 			cloudinaryName,
 			siteOptions,
-			BUILD_MODE,
 			griddoVersion,
 			renderDate,
 			siteScript,