@griddo/cx 11.9.11 → 11.9.12-rc.0

package/exporter/services/render.ts

@@ -0,0 +1,229 @@
+import type { RenderModeTuple } from "../shared/types/render";
+
+import { execSync } from "node:child_process";
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { readDB, writeDB } from "../core/db";
+import { throwError } from "../core/errors";
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { RenderUUIDError } from "../shared/errors";
+import { brush } from "../shared/npm-modules/brush";
+import { RENDER_MODE } from "../shared/types/render";
+import { AuthService } from "./auth";
+import { getBuildMetadata } from "./manage-store";
+
+/**
+ * Creates a sentinel file with the current date and time.
+ * This file is used to track later if node_modules/@griddo/cx was cleaned by a
+ * npm install coming from a deploy.
+ */
+async function markRenderAsStarted(options: { domain: string; basePath: string }) {
+	const { domain } = options;
+
+	const db = await readDB();
+	db.domains[domain].isRendering = true;
+	await writeDB(db);
+
+	// Creamos un archivo centinela, si al terminar el render este archivo no
+	// existe es que ha habido un deploy por medio y hay que invalidar el render
+	const { __ssg } = await getRenderPathsHydratedWithDomainFromDB();
+
+	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
+	await fsp.writeFile(renderSentinelFile, new Date().toISOString());
+}
+
+async function markRenderAsCompleted(domain: string) {
+	const db = await readDB();
+	db.domains[domain].isRendering = false;
+	db.currentRenderingDomain = null;
+	db.domains[domain].renderMode = RENDER_MODE.COMPLETED;
+	// db.domains[domain].shouldBeRendered = false;
+	await writeDB(db);
+
+	// Borramos finalmente el archivo centinela
+	const { __ssg } = await getRenderPathsHydratedWithDomainFromDB();
+	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
+	await fsp.unlink(renderSentinelFile);
+}
+
+async function assertRenderIsValid(domain: string) {
+	// Comprobamos que .render-sentinel exista, si no es que un deploy lo borro
+	// y hay que invalidar el render.
+	const { __ssg } = await getRenderPathsHydratedWithDomainFromDB();
+	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
+	if (!(await pathExists(renderSentinelFile))) {
+		throwError(RenderUUIDError);
+	}
+}
+
+/**
+ * Determines the appropriate render mode for a given domain based on its current state,
+ * previous render errors, deployment status, and whether rendering is required.
+ *
+ * @param options - An object containing:
+ *   - `domain`: The domain name to resolve the render mode for.
+ *   - `shouldBeRendered`: A boolean indicating if the domain should be rendered.
+ * @returns An object with:
+ *   - `renderMode`: The resolved render mode (`FROM_SCRATCH`, `INCREMENTAL`, or `IDLE`).
+ *   - `reason`: A string describing the reason for the chosen render mode.
+ *
+ * @remarks
+ * The function checks for missing exports, previous render errors, new deployments,
+ * and whether rendering is necessary to decide the render mode.
+ *
+ * @todo
+ * Improve ifs and reason concatenations...
+ */
+async function resolveDomainRenderMode(options: { domain: string; shouldBeRendered: boolean }) {
+	const { domain, shouldBeRendered } = options;
+
+	const db = await readDB();
+
+	const { __cache, __exports } = await getRenderPathsHydratedWithDomainFromDB({ domain });
+	const exportsAreMissing = !(await pathExists(path.join(__exports)));
+	const previousRenderFailed = db.domains[domain]?.isRendering;
+	const newDeployDetected = await hasNewCommit(__cache);
+
+	if (exportsAreMissing) {
+		return {
+			renderMode: RENDER_MODE.FROM_SCRATCH,
+			reason: "missing exports directory",
+		};
+	}
+
+	if (previousRenderFailed) {
+		return {
+			renderMode: RENDER_MODE.FROM_SCRATCH,
+			reason: "error in previous render",
+		};
+	}
+
+	if (newDeployDetected) {
+		return {
+			renderMode: RENDER_MODE.FROM_SCRATCH,
+			reason: "new commit hash",
+		};
+	}
+
+	if (!shouldBeRendered) {
+		return {
+			renderMode: RENDER_MODE.IDLE,
+			reason: "no activity",
+		};
+	}
+
+	return {
+		renderMode: RENDER_MODE.INCREMENTAL,
+		reason: "has changes",
+	};
+}
+
+async function hasNewCommit(basePath: string): Promise<boolean> {
+	const commitFile = path.join(basePath, "commit");
+	const currentCommit = execSync("git rev-parse HEAD").toString().trim();
+
+	if (await pathExists(commitFile)) {
+		const savedCommit = (await fsp.readFile(commitFile, "utf-8")).trim();
+		if (savedCommit === currentCommit) {
+			return false; // No hay nuevo commit
+		}
+
+		return true;
+	}
+
+	return true;
+}
+
+async function updateCommitFile(options: { basePath: string }) {
+	const { basePath } = options;
+	const currentCommit = execSync("git rev-parse HEAD").toString().trim();
+	await fsp.writeFile(path.join(basePath, "commit"), currentCommit);
+}
+
+async function getRenderModeFromDB(domain: string): Promise<RenderModeTuple> {
+	const db = await readDB();
+
+	if (!db.domains[domain]) {
+		throw new Error(brush.red(`[!] Error: Domain ${domain} not found in DB`));
+	}
+
+	if (!db.domains[domain].renderMode) {
+		throw new Error(brush.red(`[!] Error: Render mode not found for domain ${domain}`));
+	}
+
+	return {
+		renderMode: db.domains[domain].renderMode,
+		reason: db.domains[domain].renderModeReason,
+	};
+}
+
+async function getRenderPathsHydratedWithDomainFromDB(options?: {
+	domain?: string;
+	dbFilePath?: string;
+}) {
+	const { domain, dbFilePath } = options || {};
+
+	const db = await readDB(dbFilePath);
+	const paths = db.paths;
+
+	return {
+		__root: paths.root,
+		__cache: path.join(paths.cxCache, domain || ""),
+		__components: paths.components,
+		__cx: paths.cx,
+		__sites: paths.exportsDir,
+		__exports: path.join(paths.exportsDir, domain || ""),
+		__exports_backup: path.join(paths.exportsDirBackup, domain || ""),
+		__ssg: paths.ssg,
+		__exports_dist: path.join(paths.exportsDir, domain || "", "dist"),
+	};
+}
+
+async function getRenderMetadataFromDB() {
+	const db = await readDB();
+	return {
+		griddoVersion: db.griddoVersion,
+		buildReportFileName: db.buildReportFileName,
+	};
+}
+
+/**
+ * Save a file with the end of build process to use as `end-render` signal.
+ */
+async function generateBuildReport(domain: string) {
+	const authControl = await AuthService.login();
+
+	const { __root } = await getRenderPathsHydratedWithDomainFromDB();
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { buildProcessData } = await getBuildMetadata(domain);
+
+	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
+		...buildProcessData[siteID],
+		siteId: Number.parseInt(siteID),
+	}));
+
+	const report = {
+		authControl,
+		sites: buildSitesInfo,
+	};
+
+	const reportFilePath = path.join(__root, "current-dist", buildReportFileName);
+
+	await fsp.writeFile(reportFilePath, JSON.stringify(report));
+
+	GriddoLog.verbose(`build report saved in ${reportFilePath}`);
+}
+
+export {
+	assertRenderIsValid,
+	generateBuildReport,
+	getRenderMetadataFromDB,
+	getRenderModeFromDB,
+	getRenderPathsHydratedWithDomainFromDB,
+	markRenderAsCompleted,
+	markRenderAsStarted,
+	resolveDomainRenderMode,
+	updateCommitFile,
+};

package/exporter/services/robots.ts

@@ -1,71 +1,43 @@
-import type { Robots } from "../types/global";
+import type { Robots } from "../shared/types/global";
 
-import fs from "node:fs";
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import { endpoints } from "../constants";
-import { get } from "../utils/api";
-import { getConfig } from "../utils/core-utils";
-import { verboseLog } from "../utils/loggin";
-
-const config = getConfig();
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { ROBOTS } from "../shared/endpoints";
+import { get } from "./api";
+import { getRenderPathsHydratedWithDomainFromDB } from "./render";
+
+async function fetchRobots() {
+	const apiRobots = await get<Robots>({ endpoint: ROBOTS });
+	return (
+		apiRobots
+			?.filter((r) => !!r.path)
+			.map(({ path, content }) => ({
+				path,
+				content: content || "User-agent: *\n\r\n\rAllow: /",
+			})) || []
+	);
+}
 
-/**
- * TODO: JSDoc
- */
-class RobotsService {
-	robots: Robots;
-	settings: Record<string, unknown>;
+async function generateRobots(domain: string) {
+	const { __root } = await getRenderPathsHydratedWithDomainFromDB({ domain });
+	const distDirectory = path.join(__root, "current-dist");
+	const robot = (await fetchRobots()).find(({ path }) => path === `/${domain}`);
 
-	constructor() {
-		this.robots = [];
-		this.settings = {};
+	if (!robot) {
+		GriddoLog.info(`Robots not found for ${domain}`);
+		return;
 	}
 
-	/**
-	 * TODO: JSDoc
-	 */
-	private async getRobots() {
-		try {
-			const apiRobots = await get<Robots>({ endpoint: endpoints.ROBOTS });
-			this.robots =
-				apiRobots
-					?.filter((r) => !!r.path)
-					.map(({ path, content }) => ({
-						path,
-						content: content || "User-agent: *\n\r\n\rAllow: /",
-					})) || [];
-		} catch (e) {
-			console.warn(`${this.constructor.name}: ${(e as Error).message}`);
-		}
-	}
-
-	/**
-	 * Write robots.txt files for the domain.
-	 */
-	async writeFiles(domain: string) {
-		const { __cx } = config.paths(domain);
-		const distDirectory = path.join(__cx, "dist");
-
-		await this.getRobots();
-
-		const robot = this.robots.find(({ path }) => path === `/${domain}`);
-
-		if (!robot) {
-			console.log(`Robots not found for ${domain}`);
-			return;
-		}
-
-		if (fs.existsSync(distDirectory)) {
-			const fileLocation = path.join(distDirectory, "robots.txt");
-			fs.writeFileSync(fileLocation, robot?.content);
-			verboseLog(`wrote robots.txt to ${fileLocation}`);
-		} else {
-			console.log(`${distDirectory} not found`);
-		}
+	if (await pathExists(distDirectory)) {
+		const fileLocation = path.join(distDirectory, "robots.txt");
+		await fsp.writeFile(fileLocation, robot?.content);
+		GriddoLog.verbose(`wrote robots.txt to ${fileLocation}`);
+	} else {
+		GriddoLog.info(`${distDirectory} not found`);
 	}
 }
 
-const robotsService = new RobotsService();
-
-export { robotsService as RobotsService };
+export { generateRobots };

package/exporter/services/sitemaps.ts

@@ -0,0 +1,129 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { generateSitemapIndexXml, generateUrlsetXml } from "../shared/npm-modules/xml-parser";
+import { AuthService } from "./auth";
+import { getBuildMetadata } from "./manage-store";
+import { getRenderPathsHydratedWithDomainFromDB } from "./render";
+import { getSitemap } from "./sites";
+
+/**
+ * Generate sitemaps and save them into file system.
+ */
+async function generateSitemaps(domain: string) {
+	const { __root } = await getRenderPathsHydratedWithDomainFromDB();
+	const { sitesToPublish } = await getBuildMetadata(domain);
+	const distDir = path.join(__root, "current-dist");
+	const templateSitemapPrefix = "sitemap-";
+
+	const isGriddoSitemapFile = (file: string) =>
+		file === "sitemap.xml" || (file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
+
+	for (const site of sitesToPublish) {
+		const { id, languages } = site;
+
+		for (const lang of languages) {
+			// Local copy of headers to avoid mutating the shared object
+			const localHeaders: Record<string, string> = AuthService.headers
+				? { ...AuthService.headers }
+				: {};
+			localHeaders.lang = lang.id.toString();
+
+			const response = await getSitemap({ siteId: id, headers: localHeaders });
+
+			if (!response) {
+				continue;
+			}
+
+			const { items: sitemapPagesGroup, url } = response;
+			const { home, domain } = url;
+
+			if (!home) {
+				continue;
+			}
+
+			const domainLanguage = site.domains.find(
+				(domain) => Object.keys(domain)[0] === lang.id.toString(),
+			);
+
+			if (!domainLanguage) {
+				continue;
+			}
+
+			const slug = Object.values(domainLanguage)[0];
+			const sitemaps: string[] = [];
+			const sitemapPageGroupKeys = Object.keys(sitemapPagesGroup);
+			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
+
+			/** delete every griddo sitemap file before create the new ones */
+			if (await pathExists(sitemapBasePath)) {
+				try {
+					const files = await fsp.readdir(sitemapBasePath);
+					for (const file of files) {
+						if (isGriddoSitemapFile(file)) {
+							const filePath = path.join(sitemapBasePath, file);
+							try {
+								await fsp.rm(filePath);
+							} catch (err) {
+								GriddoLog.error(`Error deleting file ${filePath}:`, err);
+							}
+						}
+					}
+				} catch (err) {
+					GriddoLog.error(`Error reading ${sitemapBasePath}:`, err);
+				}
+			}
+
+			for (const templateId of sitemapPageGroupKeys) {
+				const sitemapPages = sitemapPagesGroup[templateId];
+
+				if (!sitemapPages.length) {
+					continue;
+				}
+
+				const siteMap = generateUrlsetXml(sitemapPages);
+
+				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
+				const exactPath = path.join(sitemapBasePath, sitemapName);
+
+				await saveFile(exactPath, siteMap);
+				GriddoLog.verbose(`sitemap generated in ${exactPath}`);
+
+				sitemaps.push(`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`);
+			}
+
+			if (!sitemaps.length) {
+				continue;
+			}
+
+			const siteMap = generateSitemapIndexXml(sitemaps);
+
+			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
+
+			await saveFile(exactPath, siteMap);
+			GriddoLog.verbose(`sitemap generated in ${exactPath}`);
+		}
+	}
+}
+
+/**
+ * Saves the content to a file specified by its path. If the file exists, it will be overwritten.
+ *
+ * @param filePath The path of the file to save the content to.
+ * @param content The content to save to the file.
+ */
+async function saveFile(filePath: string, content: string) {
+	try {
+		const pathName = path.dirname(filePath);
+		if (!(await pathExists(pathName))) {
+			await fsp.mkdir(pathName, { recursive: true });
+		}
+		await fsp.writeFile(filePath, content);
+	} catch (_error) {
+		throw new Error(`Error saving a file`);
+	}
+}
+
+export { generateSitemaps };

package/exporter/services/sites.ts

@@ -1,3 +1,4 @@
+import type { Core } from "@griddo/core";
 import type {
 	AllSitesReponse,
 	EndPageInfoResponse,
@@ -9,19 +10,28 @@ import type {
 	SitemapAPIResponse,
 	SocialsResponse,
 	StartPageRenderResponse,
-} from "../types/api";
-import type { Site } from "../types/sites";
-import type { Core } from "@griddo/core";
-
-import { endpoints } from "../constants";
-import { get, post } from "../utils/api";
+} from "../shared/types/api";
+import type { Site } from "../shared/types/sites";
+
+import {
+	BUILD_END,
+	BUILD_START,
+	GET_ALL,
+	GET_PAGE,
+	GET_REFERENCE_FIELD_DATA,
+	GET_SITEMAP,
+	INFO,
+	LANGUAGES,
+	SOCIALS,
+} from "../shared/endpoints";
+import { get, post } from "./api";
 
 /**
  * Get a list of site objects.
  */
 async function getAllSites(domain: string) {
-	return await get<AllSitesReponse>({
-		endpoint: `${endpoints.GET_ALL}?domainSlug=${domain}`,
+	return get<AllSitesReponse>({
+		endpoint: `${GET_ALL}?domainSlug=${domain}`,
 	});
 }
 
@@ -29,7 +39,7 @@ async function getAllSites(domain: string) {
  * Get a list of site objects from a domain.
  */
 async function getAllSitesFromDomain(domain: string) {
-	const allSites = await get<AllSitesReponse>({ endpoint: endpoints.GET_ALL });
+	const allSites = await get<AllSitesReponse>({ endpoint: GET_ALL });
 
 	if (allSites.length) {
 		for (const site of allSites) {
@@ -40,8 +50,7 @@ async function getAllSitesFromDomain(domain: string) {
 			site.domains = items
 				.filter(
 					(item) =>
-						item.domain &&
-						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
+						item.domain && (item.domain.slug === domain || item.domain.slug === `/${domain}`),
 				)
 				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
 		}
@@ -54,8 +63,8 @@ async function getAllSitesFromDomain(domain: string) {
  * Fetch a page object from API.
  */
 async function getPage(id: number, cacheKey: string) {
-	return await get<PageResponse>({
-		endpoint: `${endpoints.GET_PAGE}/${id}`,
+	return get<PageResponse>({
+		endpoint: `${GET_PAGE}/${id}`,
 		cacheKey,
 	});
 }
@@ -64,27 +73,27 @@ async function getPage(id: number, cacheKey: string) {
  * Get site info
  */
 async function getSiteInfo(id: number, cacheKey = "") {
-	const [prefix, suffix] = endpoints.INFO;
+	const [prefix, suffix] = INFO;
 
-	return await get<Site>({
+	return get<Site>({
 		endpoint: `${prefix}${id}${suffix}`,
 		cacheKey,
 	});
 }
 
 async function getSiteLanguages(id: number, cacheKey = "") {
-	const [prefix, suffix] = endpoints.LANGUAGES;
+	const [prefix, suffix] = LANGUAGES;
 
-	return await get<LanguagesResponse>({
+	return get<LanguagesResponse>({
 		endpoint: `${prefix}${id}${suffix}`,
 		cacheKey,
 	});
 }
 
 async function startSiteRender(id: number) {
-	const [prefix, suffix] = endpoints.BUILD_START;
+	const [prefix, suffix] = BUILD_START;
 
-	return await get<StartPageRenderResponse>({
+	return get<StartPageRenderResponse>({
 		endpoint: `${prefix}${id}${suffix}`,
 	});
 }
@@ -93,12 +102,13 @@ async function startSiteRender(id: number) {
  * Send the end signal to API for a render site.
  */
 async function endSiteRender(id: number, body: EndSiteRenderBody) {
-	const [prefix, suffix] = endpoints.BUILD_END;
+	const [prefix, suffix] = BUILD_END;
 
-	return await post<EndPageInfoResponse>({
+	await post<EndPageInfoResponse>({
 		endpoint: `${prefix}${id}${suffix}`,
 		body,
 		useApiCacheDir: false,
+		logToFile: false,
 	});
 }
 
@@ -109,11 +119,11 @@ async function getReferenceFieldSiteData(
 	dataSiteId?: number,
 	dataLangID?: number,
 ) {
-	const [prefix, suffix] = endpoints.GET_REFERENCE_FIELD_DATA;
+	const [prefix, suffix] = GET_REFERENCE_FIELD_DATA;
 	const site = dataSiteId || page.site;
 	const lang = dataLangID || page.language;
 
-	return await post<ReferenceFieldResponse>({
+	return post<ReferenceFieldResponse>({
 		endpoint: `${prefix}${site}${suffix}`,
 		body,
 		headers: { lang },
@@ -121,18 +131,20 @@ async function getReferenceFieldSiteData(
 	});
 }
 
-async function getSitemap(id: number) {
-	const [prefix, suffix] = endpoints.GET_SITEMAP;
+async function getSitemap(options: { siteId: number; headers?: Record<string, string> }) {
+	const { siteId, headers } = options;
+	const [prefix, suffix] = GET_SITEMAP;
 
 	return get<SitemapAPIResponse>({
-		endpoint: `${prefix}${id}${suffix}`,
+		endpoint: `${prefix}${siteId}${suffix}`,
+		headers,
 	});
 }
 
 async function getSiteSocials(id: number, cacheKey = "") {
-	const [prefix, suffix] = endpoints.SOCIALS;
+	const [prefix, suffix] = SOCIALS;
 
-	return await get<SocialsResponse>({
+	return get<SocialsResponse>({
 		endpoint: `${prefix}${id}${suffix}`,
 		cacheKey,
 	});