@griddo/cx 11.7.11 → 11.7.12-rc.1

package/exporter/utils/render.ts

@@ -1,71 +1,203 @@
-import fs from "node:fs";
-import path from "node:path";
+import type { RenderModeTuple } from "../types/render";
 
-import { envs } from "../constants";
-import { throwError } from "../errors";
-import { RenderUUIDError } from "../errors/errors-data";
-import { apiRegister } from "../registers";
-import { insertAlert } from "./alerts";
-import { getConfig } from "./core-utils";
-import { debugLog } from "./loggin";
+import { execSync } from "node:child_process";
+import fsp from "node:fs/promises";
+import path from "node:path";
 
-const config = getConfig();
+import { RenderUUIDError } from "../constants/errors";
+import { AuthService } from "../services/auth";
+import { readDB, writeDB } from "../services/db";
+import { RENDER_MODE } from "../types/render";
+import { throwError } from "./errors";
+import { pathExists } from "./folders";
+import { infoLog, verboseLog } from "./loggin";
+import { getBuildMetadata } from "./store";
 
 /**
  * Creates a sentinel file with the current date and time.
  * This file is used to track later if node_modules/@griddo/cx was cleaned by a
- * npm install from a deploy.
+ * npm install coming from a deploy.
  */
-function createSentinelRenderFile() {
-	const { __cx } = config.paths();
-	const renderSentinelFile = path.join(__cx, ".render-sentinel");
-	fs.writeFileSync(renderSentinelFile, new Date().toISOString());
+async function markRenderAsStarted(options: { domain: string; basePath: string }) {
+	const { domain } = options;
+
+	const db = await readDB();
+	db.domains[domain].isRendering = true;
+	await writeDB(db);
+
+	// Creamos un archivo centinela, si al terminar el render este archivo no
+	// existe es que ha habido un deploy por medio y hay que invalidar el render
+	const { __ssg } = await getRenderPathsFromDB();
+
+	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
+	await fsp.writeFile(renderSentinelFile, new Date().toISOString());
 }
 
-function deleteSentinelRenderFile() {
-	const { __cx } = config.paths();
-	const renderSentinelFile = path.join(__cx, ".render-sentinel");
-	fs.unlinkSync(renderSentinelFile);
+async function markRenderAsCompleted(domain: string) {
+	const db = await readDB();
+	db.domains[domain].isRendering = false;
+	// db.domains[domain].renderMode = undefined;
+	// db.domains[domain].shouldBeRendered = false;
+	await writeDB(db);
+
+	// Borramos finalmente el archivo centinela
+	const { __ssg } = await getRenderPathsFromDB();
+	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
+	await fsp.unlink(renderSentinelFile);
 }
 
-function isValidRender() {
-	const { __cx } = config.paths();
-	const renderSentinelFile = path.join(__cx, ".render-sentinel");
-	if (!fs.existsSync(renderSentinelFile)) {
+async function assertRenderIsValid(domain: string) {
+	// Comprobamos que .render-sentinel exista, si no es que un deploy lo borro
+	// y hay que invalidar el render.
+	const { __ssg } = await getRenderPathsFromDB();
+	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
+	if (!(await pathExists(renderSentinelFile))) {
 		throwError(RenderUUIDError);
 	}
 }
 
-/**
- * Send the default registers through Griddo alerts.
- */
-async function sendGriddoDefaultAlerts() {
-	if (envs.GRIDDO_ALERT_FEATURE) {
-		const registerContent = apiRegister.get("API_RESPONSE_TOO_BIG");
+// @todo add more messages...
+async function resolveDomainRenderMode(options: { domain: string; shouldBeRendered: boolean }) {
+	const { domain, shouldBeRendered } = options;
+	const { __cache, __exports } = await getRenderPathsFromDB({ domain });
+
+	// missing exports
+	const missingExports = !(await pathExists(path.join(__exports)));
+	// render with errors
+	const db = await readDB();
+	const wasRenderWithError = db.domains[domain]?.isRendering;
+	// new deploy or machine
+	const isNewDeploy = await hasNewCommit(__cache);
+
+	if (missingExports) {
+		infoLog(`[${domain}]: Render from Scratch due to missing exports (first render)`);
+		return {
+			renderMode: RENDER_MODE.FROM_SCRATCH,
+			reason: "Missing exports directory",
+		};
+	}
 
-		if (registerContent.entries.length < 1) {
-			return;
+	if (wasRenderWithError) {
+		infoLog(`[${domain}]: Render from Scratch due to a previous render error`);
+		return {
+			renderMode: RENDER_MODE.FROM_SCRATCH,
+			reason: "error in previous render",
+		};
+	}
+
+	if (isNewDeploy) {
+		infoLog(`[${domain}]: Render from Scratch due to a changes in the repo (commit hash)`);
+		return { renderMode: RENDER_MODE.FROM_SCRATCH, reason: "new commit hash" };
+	}
+
+	if (!shouldBeRendered) {
+		infoLog(`[${domain}]: Skipping render as it is not marked for rendering (IDLE).`);
+		return { renderMode: RENDER_MODE.IDLE, reason: "" };
+	}
+
+	console.log(
+		`[${domain}]: {DEBUG} Render INCREMENTAL porque no se cumplen: missingExports (FROM_SCRATCH), wasRenderWithError (FROM_SCRATCH), isNewDeploy (FROM_SCRATCH) o !shouldBeRendered (IDLE)`,
+	);
+	return { renderMode: RENDER_MODE.INCREMENTAL, reason: "" };
+}
+
+async function hasNewCommit(basePath: string): Promise<boolean> {
+	const commitFile = path.join(basePath, "commit");
+	const currentCommit = execSync("git rev-parse HEAD").toString().trim();
+
+	if (await pathExists(commitFile)) {
+		const savedCommit = (await fsp.readFile(commitFile, "utf-8")).trim();
+		if (savedCommit === currentCommit) {
+			return false; // No hay nuevo commit
 		}
 
-		debugLog("\nRender register report\n");
-		debugLog(registerContent);
-		debugLog();
-
-		await insertAlert({
-			description: `API response size is too large (${envs.GRIDDO_API_MAX_RESPONSE_SIZE}KB).`,
-			area: "Griddo",
-			level: "W",
-			fullData: {
-				output: registerContent,
-				date: new Date().toISOString(),
-			},
-		});
+		return true;
+	}
+
+	return true;
+}
+
+async function updateCommitFile(options: { basePath: string }) {
+	const { basePath } = options;
+	const currentCommit = execSync("git rev-parse HEAD").toString().trim();
+	await fsp.writeFile(path.join(basePath, "commit"), currentCommit);
+}
+
+async function getRenderMode(domain: string): Promise<RenderModeTuple> {
+	const db = await readDB();
+
+	if (!db.domains[domain].renderMode) {
+		throw new Error(`Render mode not found for domain ${domain}`);
 	}
+
+	return {
+		renderMode: db.domains[domain].renderMode,
+		reason: db.domains[domain].renderModeReason,
+	};
+}
+
+async function getRenderPathsFromDB(options?: { domain?: string; dbFilePath?: string }) {
+	const { domain, dbFilePath } = options || {};
+
+	const db = await readDB(dbFilePath);
+	const paths = db.paths;
+
+	return {
+		__root: paths.root,
+		__cache: path.join(paths.cxCache, domain || ""),
+		__components: paths.components,
+		__cx: paths.cx,
+		__sites: paths.exportsDir,
+		__exports: path.join(paths.exportsDir, domain || ""),
+		__ssg: paths.ssg,
+		__exports_dist: path.join(paths.exportsDir, domain || "", "dist"),
+	};
+}
+
+async function getRenderMetadataFromDB() {
+	const db = await readDB();
+	return {
+		proDomainPrefix: db.proDomainPrefix,
+		griddoVersion: db.griddoVersion,
+		buildReportFileName: db.buildReportFileName,
+	};
+}
+
+/**
+ * Save a file with the end of build process to use as `build-end` signal.
+ */
+async function generateBuildReport(domain: string) {
+	const authControl = await AuthService.login();
+
+	const { __root } = await getRenderPathsFromDB();
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { buildProcessData } = await getBuildMetadata(domain);
+
+	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
+		...buildProcessData[siteID],
+		siteId: Number.parseInt(siteID),
+	}));
+
+	const report = {
+		authControl,
+		sites: buildSitesInfo,
+	};
+
+	const reportFilePath = path.join(__root, "dist", buildReportFileName);
+
+	await fsp.writeFile(reportFilePath, JSON.stringify(report));
+
+	verboseLog(`build report saved in ${reportFilePath}`);
 }
 
 export {
-	createSentinelRenderFile,
-	deleteSentinelRenderFile,
-	isValidRender,
-	sendGriddoDefaultAlerts,
+	assertRenderIsValid,
+	generateBuildReport,
+	getRenderMetadataFromDB,
+	getRenderMode,
+	getRenderPathsFromDB,
+	markRenderAsCompleted,
+	markRenderAsStarted,
+	resolveDomainRenderMode,
+	updateCommitFile,
 };

package/exporter/utils/sitemaps.ts

@@ -0,0 +1,129 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { AuthService } from "../services/auth";
+import { getSitemap } from "../services/sites";
+import { pathExists } from "./folders";
+import { verboseLog } from "./loggin";
+import { generateSitemapIndexXml, generateUrlsetXml } from "./npm-deps/xml-parser";
+import { getRenderPathsFromDB } from "./render";
+import { getBuildMetadata } from "./store";
+
+/**
+ * Generate sitemaps and save them into file system.
+ */
+async function generateSitemaps(domain: string) {
+	const { __root } = await getRenderPathsFromDB();
+	const { sitesToPublish } = await getBuildMetadata(domain);
+	const distDir = path.join(__root, "dist");
+	const templateSitemapPrefix = "sitemap-";
+
+	const isGriddoSitemapFile = (file: string) =>
+		file === "sitemap.xml" || (file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
+
+	for (const site of sitesToPublish) {
+		const { id, languages } = site;
+
+		for (const lang of languages) {
+			// Local copy of headers to avoid mutating the shared object
+			const localHeaders: Record<string, string> = AuthService.headers
+				? { ...AuthService.headers }
+				: {};
+			localHeaders.lang = lang.id.toString();
+
+			const response = await getSitemap({ siteId: id, headers: localHeaders });
+
+			if (!response) {
+				continue;
+			}
+
+			const { items: sitemapPagesGroup, url } = response;
+			const { home, domain } = url;
+
+			if (!home) {
+				continue;
+			}
+
+			const domainLanguage = site.domains.find(
+				(domain) => Object.keys(domain)[0] === lang.id.toString(),
+			);
+
+			if (!domainLanguage) {
+				continue;
+			}
+
+			const slug = Object.values(domainLanguage)[0];
+			const sitemaps: Array<string> = [];
+			const sitemapPageGroupKeys = Object.keys(sitemapPagesGroup);
+			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
+
+			/** delete every griddo sitemap file before create the new ones */
+			if (await pathExists(sitemapBasePath)) {
+				try {
+					const files = await fsp.readdir(sitemapBasePath);
+					for (const file of files) {
+						if (isGriddoSitemapFile(file)) {
+							const filePath = path.join(sitemapBasePath, file);
+							try {
+								await fsp.rm(filePath);
+							} catch (err) {
+								console.error(`Error deleting file ${filePath}:`, err);
+							}
+						}
+					}
+				} catch (err) {
+					console.error(`Error reading ${sitemapBasePath}:`, err);
+				}
+			}
+
+			for (const templateId of sitemapPageGroupKeys) {
+				const sitemapPages = sitemapPagesGroup[templateId];
+
+				if (!sitemapPages.length) {
+					continue;
+				}
+
+				const siteMap = generateUrlsetXml(sitemapPages);
+
+				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
+				const exactPath = path.join(sitemapBasePath, sitemapName);
+
+				await saveFile(exactPath, siteMap);
+
+				sitemaps.push(`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`);
+			}
+
+			if (!sitemaps.length) {
+				continue;
+			}
+
+			const siteMap = generateSitemapIndexXml(sitemaps);
+
+			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
+
+			await saveFile(exactPath, siteMap);
+			verboseLog(`sitemap generated in ${exactPath}`);
+		}
+	}
+}
+
+/**
+ * Saves the content to a file specified by its path. If the file exists, it will be overwritten.
+ *
+ * @param filePath The path of the file to save the content to.
+ * @param content The content to save to the file.
+ */
+async function saveFile(filePath: string, content: string) {
+	try {
+		const pathName = path.dirname(filePath);
+		if (!(await pathExists(pathName))) {
+			await fsp.mkdir(pathName, { recursive: true });
+		}
+		await fsp.writeFile(filePath, content);
+	} catch (error) {
+		console.log(error);
+		throw new Error(`Error saving a file`);
+	}
+}
+
+export { generateSitemaps };

package/exporter/utils/sites.ts

@@ -1,49 +1,35 @@
 import type { Site, SiteData } from "../types/sites";
 
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import fs from "fs-extra";
-import { parse } from "js2xmlparser";
-
-import { envs } from "../constants";
-import { getConfig } from "./core-utils";
-import { buildLog, verboseLog } from "./loggin";
-import { getBuildMetadata } from "./store";
-import { AuthService } from "../services/auth";
+import {
+	GRIDDO_RENDER_ALL_SITES,
+	GRIDDO_RENDER_PAGES,
+	GRIDDO_RENDER_SITE,
+} from "../constants/envs";
 import {
 	endSiteRender,
 	getAllSites,
 	getSiteInfo,
 	getSiteLanguages,
-	getSitemap,
 	getSiteSocials,
 	startSiteRender,
 } from "../services/sites";
-
-const config = getConfig();
+import { getRenderPathsFromDB } from "./render";
 
 /**
  * Check the instance sites and returns site prepared to be published and unpublished.
  */
 async function getSitesToRender(domain: string) {
-	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
-
-	// Login to API
-	await AuthService.login();
-
 	// Get all sites. An array of Site
 	const allSites = await getAllSites(domain);
 
 	// Filter the array of sites to get only the ones to build/render
-	const validSites = envs.GRIDDO_RENDER_ALL_SITES
-		? allSites.filter(
-				(site) =>
-					!envs.GRIDDO_RENDER_SITE || site.id === envs.GRIDDO_RENDER_SITE,
-			)
+	const validSites = GRIDDO_RENDER_ALL_SITES
+		? allSites.filter((site) => !GRIDDO_RENDER_SITE || site.id === GRIDDO_RENDER_SITE)
 		: allSites.filter((site) =>
-				envs.GRIDDO_RENDER_SITE
-					? site.id === envs.GRIDDO_RENDER_SITE
-					: !!site.shouldBeUpdated,
+				GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.shouldBeUpdated,
 			);
 
 	// If there are valid sites...
@@ -58,8 +44,7 @@ async function getSitesToRender(domain: string) {
 			site.domains = items
 				.filter(
 					(item) =>
-						item.domain &&
-						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
+						item.domain && (item.domain.slug === domain || item.domain.slug === `/${domain}`),
 				)
 				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
 		}
@@ -67,15 +52,11 @@ async function getSitesToRender(domain: string) {
 
 	// Save sites object to publish
 	const sitesToPublish = validSites.filter((site) =>
-		envs.GRIDDO_RENDER_SITE
-			? site.id === envs.GRIDDO_RENDER_SITE
-			: !!site.isPublished,
+		GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.isPublished,
 	);
 
 	// Save sites object to unpublish
-	const sitesToUnpublish = validSites.filter(
-		(site) => !site.isPublished && site.shouldBeUpdated,
-	);
+	const sitesToUnpublish = validSites.filter((site) => !site.isPublished && site.shouldBeUpdated);
 
 	return {
 		sitesToPublish,
@@ -90,7 +71,7 @@ async function getSitesToRender(domain: string) {
  * @see https://griddoio.notion.site/Sites-d7bb0b7cb8d24894a5337e1139fc3d09#2019d3255bda4d219c7e19cf28d0c4fe
  */
 async function unpublishSites(sites: Array<Site>) {
-	const { __cx } = config.paths();
+	const { __root } = await getRenderPathsFromDB();
 
 	for (const site of sites) {
 		// API
@@ -107,7 +88,8 @@ async function unpublishSites(sites: Array<Site>) {
 
 		// STORE
 		// Remove site directory from the Store
-		fs.rmSync(path.join(__cx, "store", site.id.toString()), {
+		// @todo: quitar. Como ya no hay persistenci de store, esto debería eliminarse de forma segura.
+		await fsp.rm(path.join(__root, "store", site.id.toString()), {
 			force: true,
 			recursive: true,
 		});
@@ -130,8 +112,8 @@ async function getSiteData(siteID: number) {
 
 	const { siteHash, unpublishHashes, publishIds } = buildData;
 	const { headers, footers } = siteInfo;
-	const validPagesIds = envs.GRIDDO_RENDER_PAGES.length
-		? envs.GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
+	const validPagesIds = GRIDDO_RENDER_PAGES.length
+		? GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
 		: publishIds;
 
 	const siteData: SiteData = {
@@ -149,164 +131,4 @@ async function getSiteData(siteID: number) {
 	return siteData;
 }
 
-/**
- * Save a file with the end of build process to use as `build-end` signal.
- */
-async function generateBuildReport() {
-	const { __cx } = config.paths();
-	const { buildReportFileName } = config;
-
-	const { buildProcessData } = await getBuildMetadata();
-
-	// Get the token
-	const authControl = await AuthService.login();
-
-	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
-		...buildProcessData[siteID],
-		siteId: Number.parseInt(siteID),
-	}));
-
-	const report = {
-		authControl,
-		sites: buildSitesInfo,
-	};
-
-	const reportFilePath = path.join(__cx, "dist", buildReportFileName);
-	fs.writeFileSync(reportFilePath, JSON.stringify(report));
-
-	verboseLog(`build report saved in ${reportFilePath}`);
-
-	buildLog(`Build report saved for ${buildSitesInfo.length} site(s)`);
-}
-
-/**
- * Generate sitemaps and save them into file system.
- */
-async function generateSitemaps() {
-	const { sitesToPublish } = await getBuildMetadata();
-	const { __cx } = config.paths();
-	const distDir = path.join(__cx, "dist");
-
-	const templateSitemapPrefix = "sitemap-";
-
-	const isGriddoSitemapFile = (file: string) =>
-		file === "sitemap.xml" ||
-		(file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
-
-	for (const site of sitesToPublish) {
-		const { id, languages } = site;
-
-		for (const lang of languages) {
-			if (AuthService.headers) {
-				AuthService.headers.lang = lang.id.toString();
-			}
-
-			const response = await getSitemap(id);
-
-			if (!response) continue;
-
-			const {
-				items: sitemapPagesGroup,
-				url: { home, domain },
-			} = response;
-
-			if (!home) continue;
-
-			const domainLanguage = site.domains.find(
-				(domain) => Object.keys(domain)[0] === lang.id.toString(),
-			);
-
-			if (!domainLanguage) continue;
-
-			const slug = Object.values(domainLanguage)[0];
-			const sitemaps: Array<string> = [];
-			const sitemapPageGroupKeys = Object.keys(sitemapPagesGroup);
-			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
-
-			/** delete every griddo sitemap file before create the new ones */
-			if (fs.existsSync(sitemapBasePath)) {
-				try {
-					const files = fs.readdirSync(sitemapBasePath);
-					for (const file of files) {
-						if (isGriddoSitemapFile(file)) {
-							const filePath = path.join(sitemapBasePath, file);
-							try {
-								fs.unlinkSync(filePath);
-							} catch (err) {
-								console.error(`Error removing ${filePath}:`, err);
-							}
-						}
-					}
-				} catch (err) {
-					console.error(`Error reading ${sitemapBasePath}:`, err);
-				}
-			}
-
-			for (const templateId of sitemapPageGroupKeys) {
-				const sitemapPages = sitemapPagesGroup[templateId];
-
-				if (!sitemapPages.length) continue;
-
-				const siteMap = parse("urlset", {
-					"@": {
-						xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
-						"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
-						"xsi:schemaLocation":
-							"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd",
-					},
-					url: sitemapPages,
-				});
-
-				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
-				const exactPath = path.join(sitemapBasePath, sitemapName);
-
-				saveFile(exactPath, siteMap);
-
-				sitemaps.push(
-					`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`,
-				);
-			}
-
-			if (!sitemaps.length) {
-				continue;
-			}
-
-			const siteMap = parse("sitemapindex", {
-				"@": { xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" },
-				sitemap: sitemaps.map((loc) => ({ loc })),
-			});
-
-			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
-
-			saveFile(exactPath, siteMap);
-			verboseLog(`sitemap generated in ${exactPath}`);
-		}
-	}
-}
-
-/**
- * Saves the content to a file specified by its path. If the file exists, it will be overwritten.
- *
- * @param filePath The path of the file to save the content to.
- * @param content The content to save to the file.
- */
-function saveFile(filePath: string, content: string) {
-	try {
-		const pathName = path.dirname(filePath);
-		if (!fs.existsSync(pathName)) {
-			fs.mkdirSync(pathName, { recursive: true });
-		}
-		fs.writeFileSync(filePath, content);
-	} catch (error) {
-		console.log(error);
-		throw new Error(`Error saving a file`);
-	}
-}
-
-export {
-	generateBuildReport,
-	generateSitemaps,
-	getSiteData,
-	getSitesToRender,
-	unpublishSites,
-};
+export { getSiteData, getSitesToRender, unpublishSites };