@griddo/cx 11.7.11 → 11.7.12-rc.1

package/exporter/commands/end-render.ts

@@ -1,105 +1,107 @@
-#!/usr/bin/env node
+import type { Report } from "../types/render";
 
-import fs from "node:fs";
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import pkgDir from "pkg-dir";
-
+import { ArtifactError } from "../constants/errors";
 import { AuthService } from "../services/auth";
 import { endSiteRender } from "../services/sites";
-import { getConfig } from "../utils/core-utils";
 import { getInstanceDomains } from "../utils/domains";
+import { throwError } from "../utils/errors";
+import { pathExists } from "../utils/folders";
 import { infoLog } from "../utils/loggin";
+import { getRenderMetadataFromDB, getRenderMode, getRenderPathsFromDB } from "../utils/render";
 
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
-
-// Where we are going to find export dirs
-const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
-
-// Where we are going to find archived exports
-const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
-
-type Report = {
-	authControl:
-		| {
-				Authorization: string;
-				"Cache-Control": string;
-				lang?: string | undefined;
-		  }
-		| undefined;
-	sites: Array<{
-		siteId: number;
-		publishHashes: Array<string>;
-		siteHash: string | null;
-		unpublishHashes: Array<string>;
-		publishPagesIds: Array<number>;
-	}>;
-};
-
-function getDomainRenderReport(domain: string) {
-	const reports: Array<Report> = [];
-	const { buildReportFileName } = getConfig();
+const renderByDomains = !!process.env.GRIDDO_RENDER_BY_DOMAINS;
 
-	const buildReportFile = path.resolve(
-		exportArchiveBasePath,
-		domain,
-		"dist",
-		buildReportFileName,
-	);
+async function getRenderReport(options: {
+	domain: string;
+	exportsSiteDir: string;
+	buildReportFileName: string;
+}) {
+	const { buildReportFileName, domain, exportsSiteDir: exportArchiveBasePath } = options;
+	const buildReportFile = path.resolve(exportArchiveBasePath, domain, "dist", buildReportFileName);
 
-	if (!fs.existsSync(buildReportFile)) {
-		console.info(`Build report file "${buildReportFile}" not found.`);
+	if (!(await pathExists(buildReportFile))) {
+		infoLog(`Build report file "${buildReportFile}" not found.`);
+		throwError(ArtifactError);
 	}
 
-	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
-
-	reports.push(buildReport);
-
-	return reports;
+	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
 }
 
-(async () => {
-	const sendEndSiteRender = async (reports: Array<Report>) => {
-		for (const report of reports) {
-			const sites = report.sites;
+const sendEndSiteRender = async (report: Report) => {
+	const sites = report.sites;
 
-			for (const site of sites) {
-				const { siteId, ...body } = site;
+	for (const site of sites) {
+		const { siteId, ...body } = site;
+		infoLog(`Site (${siteId})`);
+		await endSiteRender(siteId, body);
+	}
+};
 
-				console.info(`Site (${siteId})`);
+async function endRender() {
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { __sites } = await getRenderPathsFromDB();
 
-				await endSiteRender(siteId, body);
-			}
+	await AuthService.login();
+	if (renderByDomains) {
+		const [domain] = process.argv.slice(2);
+		const { renderMode } = await getRenderMode(domain);
+
+		if (renderMode === "IDLE") {
+			infoLog(`Skipping build-end for domain ${domain} as it is marked as IDLE.`);
+			return;
 		}
-	};
 
-	if (RENDER_BY_DOMAIN) {
-		await AuthService.login();
-		const domain = process.argv.splice(2)[0];
-		const reports = getDomainRenderReport(domain);
+		const report = await getRenderReport({
+			domain,
+			exportsSiteDir: __sites,
+			buildReportFileName,
+		});
 
 		infoLog(`Sending ending call to ${domain} sites`);
 
-		for (const report of reports) {
-			for (const site of report.sites) {
-				site.publishHashes = [...new Set(site.publishHashes)];
-				site.unpublishHashes = [...new Set(site.unpublishHashes)];
-			}
+		for (const site of report.sites) {
+			site.publishHashes = [...new Set(site.publishHashes)];
+			site.unpublishHashes = [...new Set(site.unpublishHashes)];
 		}
 
-		await sendEndSiteRender(reports);
+		await sendEndSiteRender(report);
 	} else {
-		// Legacy render
+		console.log("- LEGACY RENDER MODE -");
 		const domains = await getInstanceDomains();
 		for (const domain of domains) {
-			const reports = getDomainRenderReport(domain);
+			const { renderMode } = await getRenderMode(domain);
+
+			if (renderMode === "IDLE") {
+				infoLog(`Skipping build-end for domain ${domain} as it is marked as IDLE.`);
+				return;
+			}
+
+			const report = await getRenderReport({
+				domain,
+				exportsSiteDir: __sites,
+				buildReportFileName,
+			});
 
 			infoLog(`Sending ending call to ${domain} sites`);
 
-			await sendEndSiteRender(reports);
+			for (const site of report.sites) {
+				site.publishHashes = [...new Set(site.publishHashes)];
+				site.unpublishHashes = [...new Set(site.unpublishHashes)];
+			}
+
+			await sendEndSiteRender(report);
 		}
 	}
-})().catch((err) => {
+}
+
+async function main() {
+	await endRender();
+}
+
+main().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/prepare-assets-directory.ts

@@ -0,0 +1,30 @@
+import childProcess from "node:child_process";
+import path from "node:path";
+
+import { pathExists } from "../utils/folders";
+import { getRenderPathsFromDB } from "../utils/render";
+
+async function prepareAssetsDir() {
+	const [domain] = process.argv.slice(2);
+
+	const { __exports } = await getRenderPathsFromDB({ domain });
+
+	const assetsDir = path.join(__exports, "assets");
+	const domainAssetsDir = path.join(__exports, domain);
+
+	if (await pathExists(assetsDir)) {
+		childProcess.execSync(`rm -rf ${domainAssetsDir}`);
+		childProcess.execSync(`mv ${assetsDir} ${domainAssetsDir}`);
+	} else {
+		console.log("Assets directory not found");
+	}
+}
+
+async function main() {
+	await prepareAssetsDir();
+}
+
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/commands/prepare-domains-render.ts

@@ -1,46 +1,158 @@
-import fs from "node:fs";
+import type { RenderDB } from "../types/render";
+
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import { getConfig } from "../utils/core-utils";
+import { version as griddoVersion } from "../../package.json";
+import { AuthService } from "../services/auth";
+import { readDB, writeDB } from "../services/db";
 import { getInstanceDomains } from "../utils/domains";
-import { infoLog, successLog } from "../utils/loggin";
+import { pathExists, removeDirs } from "../utils/folders";
+import { resolveComponentsPath } from "../utils/instance";
+import { showExporterVersion } from "../utils/loggin";
+import { pkgDir } from "../utils/npm-deps/pkg-dir";
+import { resolveDomainRenderMode } from "../utils/render";
 import { getSitesToRender } from "../utils/sites";
 
-(async () => {
-	infoLog("Checking domains size");
+async function getDomainsWithNumberOfPages(domains: Array<string>) {
+	const results = await Promise.all(
+		domains.map(async (domain) => {
+			// Get the sites to render for the domain
+			const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
 
-	const { __cx } = getConfig().paths();
+			// Only count pages (offlinePending && publishPending) if sitesToPublish
+			// is not empty. Otherwise we set the domain has having 0 pages.
+			const pages: Array<number> = [];
 
-	const domains = await getInstanceDomains();
-	const domainsInfo = [];
-	const domainsFilePath = path.join(__cx, "domains.json");
-
-	for (const domain of domains) {
-		const { sitesToPublish } = await getSitesToRender(domain);
-
-		// Only count pages (changedPages) if sitesToPublish is not empty.
-		// Otherwise we set the domain has having 0 pages.
-		const pages = [];
-		if (sitesToPublish.length > 0) {
-			for (const site of sitesToPublish) {
-				pages.push(site.changedPages.length);
+			if (sitesToPublish.length > 0) {
+				for (const site of sitesToPublish) {
+					// el site ya está publicado
+					if (site.isPublished && !site.shouldBeUpdated) {
+						const { pagesStatus } = site;
+						const { offlinePending, uploadPending } = pagesStatus;
+						const total = offlinePending.length + uploadPending.length;
+						pages.push(total);
+					}
+					// el site esta pendiente de publicar
+					if (site.isPublished && site.shouldBeUpdated) {
+						const { pagesStatus } = site;
+						const { offlinePending, uploadPending, active } = pagesStatus;
+						const total = offlinePending.length + uploadPending.length + active.length;
+						pages.push(total);
+					}
+				}
+			} else if (sitesToUnpublish.length > 0) {
+				// si tiene sites para despublicar marcamos el dominio como que tiene 1 página
+				pages.push(1);
+			} else {
+				// si eldominio no tiene nada ni para publicar ni despublicar lomarcamos como 0
+				pages.push(0);
 			}
-		} else {
-			pages.push(0);
-		}
 
-		const totalPages = pages.reduce((a, b) => a + b);
-		domainsInfo.push({ domain, totalPages });
-	}
+			const totalPages = pages.reduce((a, b) => a + b);
 
-	// sort domains, smaller first
+			return { domain, totalPages };
+		}),
+	);
+
+	return results;
+}
+
+function getDomainsSortedByNumberOfPages(
+	domainsInfo: Array<{
+		domain: string;
+		totalPages: number;
+	}>,
+) {
 	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
-	const domainSorted = domainsInfo.map(({ domain }) => domain);
 
-	fs.writeFileSync(domainsFilePath, JSON.stringify(domainSorted));
+	return domainsInfo.map(({ domain }) => domain);
+}
+
+async function initRender() {
+	const root = (await pkgDir({ cwd: path.resolve(__dirname, "../../..") })) || "";
+	const cx = root;
+	const ssg = path.resolve((await pkgDir({ cwd: __dirname })) || "");
+	const cxCache = path.resolve(root, ".cx-cache");
+	const components = resolveComponentsPath();
+	const exportsDir = path.join(root, "exports/sites");
+
+	const data: RenderDB = {
+		proDomainPrefix: "pro-",
+		griddoVersion,
+		buildReportFileName: "build-report.json",
+		sortedDomains: [],
+		domains: {},
+		paths: {
+			components,
+			cx,
+			cxCache,
+			exportsDir,
+			root,
+			ssg,
+		},
+	};
+
+	// create the main cache folder for cx if it doesn't exist.
+	if (!(await pathExists(cxCache))) {
+		await fsp.mkdir(cxCache, { recursive: true });
+	}
+
+	await writeDB(data);
+}
+
+async function prepareDomains() {
+	await AuthService.login();
+
+	const db = await readDB();
+
+	const __ssg = db.paths.ssg;
+	const domains = await getInstanceDomains();
+	const domainsWithNumberOfPages = await getDomainsWithNumberOfPages(domains);
+	const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPages);
+
+	// @deprecated use db.json (only for infra)
+	await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
+	await fsp.writeFile(path.join(__ssg, "domains.txt"), domainSorted.join(","));
+
+	db.sortedDomains = domainSorted;
+	db.domains = {};
+
+	for (const { domain, totalPages } of domainsWithNumberOfPages) {
+		const shouldBeRendered = totalPages > 0;
+		const { renderMode, reason } = await resolveDomainRenderMode({
+			domain,
+			shouldBeRendered,
+		});
+
+		db.domains[domain] = db.domains[domain] || {};
+		db.domains[domain].renderMode = renderMode;
+		db.domains[domain].shouldBeRendered = shouldBeRendered;
+		db.domains[domain].renderModeReason = reason;
+	}
+
+	await writeDB(db);
+}
+
+async function clean() {
+	const db = await readDB();
+	const { root: __root } = db.paths;
+	await removeDirs([path.join(__root, "apiCache")]);
+}
+
+/**
+ * Prepare domains to render.
+ * It creates a `domains.json` file with the sorted list of domains and writes a
+ * `RENDE-STATE.json` file to persist a state between all cx-infra commands.
+ */
+async function main() {
+	await showExporterVersion();
+	await initRender();
+	await prepareDomains();
+	await clean();
+}
 
-	successLog(`Checking domains size`);
-})().catch((err) => {
-	console.error(err);
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/reset-render.ts

@@ -1,11 +1,18 @@
-#!/usr/bin/env node
-
+import { RESET_RENDER } from "../constants/endpoints";
 import { AuthService } from "../services/auth";
-import { resetRender } from "../services/settings";
+import { post } from "../utils/api";
+
+async function resetRender() {
+	await post({
+		endpoint: RESET_RENDER,
+		useApiCacheDir: false,
+	});
+}
 
 async function main() {
 	await AuthService.login();
 	await resetRender();
+	console.log(`The render status has been reset.`);
 }
 
 main().catch((err) => {

package/exporter/commands/start-render.ts

@@ -1,68 +1,45 @@
-#!/usr/bin/env node
-import { renderDomainsWithGatsbyAdapter } from "../adapters/gatsby";
-import { RenderError } from "../errors";
+import { gatsbyRenderDomain } from "../adapters/gatsby";
+import { AuthService } from "../services/auth";
+import { brush } from "../utils/brush";
+import { checkEnvironmentHealth } from "../utils/check-environment-health";
 import { getInstanceDomains } from "../utils/domains";
-import { errorLabelLog, showExporterVersion } from "../utils/loggin";
-import { sendGriddoDefaultAlerts } from "../utils/render";
+import { RenderError } from "../utils/errors";
 
-const GRIDDO_RENDER_DOMAINS = process.env.GRIDDO_RENDER_DOMAINS || "";
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+const renderByDomains = !!process.env.GRIDDO_RENDER_BY_DOMAINS;
 
 async function startRender() {
 	try {
-		// DOMAINS
-		// This block is just for the local instance renders.
-		if (GRIDDO_RENDER_DOMAINS) {
-			console.log("USANDO EL .ENV");
-			showExporterVersion();
-			for (const domain of GRIDDO_RENDER_DOMAINS.split(",")) {
-				await renderDomainsWithGatsbyAdapter(domain);
-			}
-			await sendGriddoDefaultAlerts();
-			process.exit(0);
-		}
-
-		// This will render every domain opposed to the new logic of render each
-		// domain separately.
-		if (RENDER_BY_DOMAIN) {
-			const domain = process.argv.splice(2)[0];
-
-			if (!domain) {
-				console.log("Needs the domain name argument");
-				throw new RenderError();
-			}
-
-			showExporterVersion();
-			await renderDomainsWithGatsbyAdapter(domain);
-			await sendGriddoDefaultAlerts();
-
-			process.exit(0);
+		if (renderByDomains) {
+			const [domain] = process.argv.slice(2);
+			await gatsbyRenderDomain(domain);
 		} else {
-			showExporterVersion();
-
-			console.log("( Legacy Render Mode )\n");
-
+			console.log("- LEGACY RENDER MODE -");
 			const domains = await getInstanceDomains();
-
 			for (const domain of domains) {
-				await renderDomainsWithGatsbyAdapter(domain);
-				await sendGriddoDefaultAlerts();
+				await gatsbyRenderDomain(domain);
 			}
-			process.exit(0);
 		}
+
+		return;
 	} catch (error) {
 		if (error instanceof RenderError) {
-			errorLabelLog("GRIDDO_ERROR InternalCXError");
+			console.log(brush.red("GRIDDO_ERROR InternalCXError"));
 			process.exit(1);
 		}
 
-		errorLabelLog("GRIDDO_ERROR UnknownError");
+		console.log(brush.red("GRIDDO_ERROR UnknownError"));
 		console.error(error);
 		process.exit(1);
 	}
 }
 
-startRender().catch((err) => {
+async function main() {
+	checkEnvironmentHealth();
+	await AuthService.login();
+	await startRender();
+}
+
+main().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });