@griddo/cx 11.9.9-rc.0 → 11.9.9-rc.2

package/exporter/adapters/gatsby/utils.ts

@@ -0,0 +1,161 @@
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+
+import fsx from "fs-extra";
+
+import { envs } from "../../constants";
+import { throwError } from "../../errors";
+import { ErrorInSSGBuildProcess } from "../../errors/errors-data";
+import { getConfig } from "../../utils/core-utils";
+import { verboseLog } from "../../utils/loggin";
+
+const config = getConfig();
+
+/**
+ * Return the assetPrefix with the domain concatenated `assetPrefix/domain` only
+ * if the domain is "pro-" **and** there is a env `GRIDDO_ASSET_PREFIX` with any
+ * value different from `null`, `undefined` or `empty string`
+ *
+ * else...
+ * - If assetPrefix or domain is falsy, returns ""
+ * - If the domain is not "pro-", returns ""
+ */
+function getGatsbyAssetPrefixWithDomain(domain: string) {
+	const { proDomain } = getConfig();
+
+	if (!envs.GRIDDO_ASSET_PREFIX || !domain) {
+		return "";
+	}
+
+	if (!domain.startsWith(proDomain)) {
+		return "";
+	}
+
+	const assetPrefixWithDomain = `${envs.GRIDDO_ASSET_PREFIX}/${domain}`;
+
+	verboseLog(
+		`Reading process.env.GRIDDO_ASSET_PREFIX, ${envs.GRIDDO_ASSET_PREFIX}`,
+	);
+	verboseLog(
+		`Setting the asset prefix with the domain concatenated, ${assetPrefixWithDomain}`,
+	);
+
+	return assetPrefixWithDomain;
+}
+
+/**
+ * Spawn a new node process `yarn gatsby-build`
+ * @note This proccess (`yarn gatsby-build`) can not access to the custom Griddo
+ * `process.env` so it needs variables passed to it via the `env` prop.
+ */
+function runGatsbyBuildCommand(assetPrefixWithDomain: string) {
+	verboseLog(`read assetPrefixWithDomain, ${assetPrefixWithDomain}`);
+	verboseLog(
+		`using this NODE_OPTIONS in gatsby command: ${process.env.NODE_OPTIONS}`,
+	);
+
+	const { __ssg } = config.paths();
+
+	// Remove `--openssl-legacy-provider` from NODE_OPTIONS because this value
+	// break Gataby render.
+	const nodeOptionsWithoutLegacyOpenSSL = process.env.NODE_OPTIONS
+		? process.env.NODE_OPTIONS.replace(
+				/--openssl-legacy-provider\s*/g,
+				"",
+			).trim()
+		: "";
+
+	const command = spawnSync(
+		"yarn",
+		["gatsby-build", envs.GRIDDO_SSG_VERBOSE_LOGS ? "--verbose" : ""],
+		{
+			cwd: __ssg,
+			stdio: ["ignore", "inherit", "ignore"],
+			encoding: "utf8",
+			shell: true,
+			env: Object.assign(process.env, {
+				GRIDDO_EXPORTER: "true",
+				SPAWN_ASSET_PREFIX_WITH_DOMAIN: assetPrefixWithDomain,
+				NODE_OPTIONS: nodeOptionsWithoutLegacyOpenSSL,
+			}),
+		},
+	);
+
+	if (command.status !== 0) {
+		throwError(ErrorInSSGBuildProcess(command));
+	}
+}
+
+/**
+ * Update the Griddo's `/dist` dir with the contents from `public` dir only
+ * with files of type: js, json and css.
+ */
+async function createDistFromGatsbyPublic(
+	domain: string,
+	needsAssetPrefix: boolean,
+) {
+	const { __cx, __ssg } = config.paths(domain);
+
+	const cxDistDir = path.join(__cx, "dist");
+	const cxAssetsDir = path.join(__cx, "assets");
+	const cxDistDirWithDomain = path.join(__cx, "dist", domain);
+	const publicDir = path.join(__ssg, "public");
+
+	const validFilesFromPublic = fsx
+		.readdirSync(publicDir)
+		.filter(
+			(file) =>
+				path.extname(file) === ".js" ||
+				path.extname(file) === ".json" ||
+				path.extname(file) === ".css",
+		);
+
+	const pageDataSrc = `${publicDir}/page-data`;
+	const pageDataDest = `${cxAssetsDir}/page-data`;
+
+	const projectStaticSrc = path.join(__ssg, "static");
+	const projectStaticDest = cxAssetsDir;
+
+	const gatsbyStaticSrc = `${cxDistDir}/static`;
+	const gatsbyStaticDest = `${cxAssetsDir}/static`;
+
+	try {
+		fsx.mkdirSync(cxAssetsDir, { recursive: true });
+		fsx.copySync(publicDir, cxDistDir, { preserveTimestamps: true });
+
+		if (needsAssetPrefix) {
+			fsx.copySync(pageDataSrc, pageDataDest, { preserveTimestamps: true });
+			if (fsx.existsSync(projectStaticSrc)) {
+				fsx.copySync(projectStaticSrc, projectStaticDest, {
+					overwrite: false,
+					preserveTimestamps: true,
+				});
+			}
+			fsx.copySync(gatsbyStaticSrc, gatsbyStaticDest, {
+				overwrite: false,
+				preserveTimestamps: true,
+			});
+			if (fsx.existsSync(projectStaticSrc)) {
+				fsx.copySync(projectStaticSrc, cxDistDirWithDomain, {
+					overwrite: false,
+					preserveTimestamps: true,
+				});
+			}
+
+			// Copia el resto de archivos...
+			validFilesFromPublic.map((file) => {
+				const fileSrc = `${publicDir}/${file}`;
+				const fileDest = `${cxAssetsDir}/${file}`;
+				fsx.copySync(fileSrc, fileDest, { preserveTimestamps: true });
+			});
+		}
+	} catch (err) {
+		console.error(err);
+	}
+}
+
+export {
+	createDistFromGatsbyPublic,
+	getGatsbyAssetPrefixWithDomain,
+	runGatsbyBuildCommand,
+};

package/exporter/artifacts/README.md

@@ -0,0 +1,34 @@
+# CX Artifacs
+
+## How to add an artifact.
+
+An artifact is a **full path** to a file or directory if it has a fixed
+location or only a **name** when it is going to be moved (archived/restored)
+between directories in the rendering process and therefore doesn't have a
+single location.
+
+Only the exporter knows where the artifacts are archived and restored, so the
+artifacts don't need to know anything other than their own location or name.
+
+Si estás creando un adaptador, sus artefactos deberán ir en un archivo separado (igual que `artifacts/cx.ts`) y exportar una función por defecto que devuelva un `Artifacts`. Todo deberá estar exportado desde `artifacts/index.ts`.
+
+**Artifacts have four categories:**
+
+- **initials**: Are directories that need to be created for their
+  future use.
+
+- **archivables**: Are stored in `__exports` which is the one uploaded to the
+  server as the final render and restored in `__cx` and `__ssg` to be reused in the
+  render. Therefore, they don't have a single location and should be added only
+  with the artifact name (directory/file)
+
+- **cacheables**: Are stored in `__cache` and restored in `__cx` and `__ssg` to be
+  reused in each domain render. Therefore, they don't have a single location.
+
+- **disposables**: Are generated alwais in `__cx` and `__ssg` or fixed locations.
+  These artifacts are discarded (deleted) in each domain render. They are
+  temporary directories and/or files.
+
+## Example
+
+Check `cx.ts` inside `./artifacts` directory to see how to add an artifact.

package/exporter/artifacts/index.ts

@@ -0,0 +1,33 @@
+import type { Artifacts } from "../types/global";
+
+import path from "node:path";
+
+import { getConfig } from "../utils/core-utils";
+
+/**
+ * Returns the artifacts of CX.
+ */
+function getCxArtifacts(domain: string): Artifacts {
+	const config = getConfig();
+	const { __cx, __exports, __cache } = config.paths(domain);
+
+	return {
+		initials: [
+			__exports, // `/exports/<domain>`
+			__cache, // `__cx/.cx-cache/<domain>`
+			path.join(__cx, "store"),
+			path.join(__cx, "apiCache"),
+		],
+		disposables: [
+			path.join(__cx, "store"),
+			path.join(__cx, "apiCache"),
+			path.join(__cx, "render-metadata.json"),
+			path.join(__cx, "dist"),
+			path.join(__cx, "debug"),
+		],
+		cacheables: ["apiCache", "store"],
+		archivables: ["dist", "assets", "debug"],
+	};
+}
+
+export default getCxArtifacts;

package/exporter/build.sh

@@ -1,10 +1,21 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2086
+# Build CX script file
+#
+# This file generates the necessary builds for CX.
+# - Setup, to generate necessary elements (tsconfig, etc..)
+# - Library, CX library that will be used by the SSG framework.
+# - Scripts, CX scripts that will be used by infra during the render process.
+#
+# @todo:
+# In the future, we will try to eliminate the bundling processes and use
+# TypeScript or ESModules directly.
 
-# CLI options for esbuild
+# start
+echo "📦 Building ..."
+
+# CLI options
 log="--log-level=error"
-production_node_opts="--bundle --platform=node --minify --sourcemap --outfile=./build/commands/"
-debugging_node_opts="--bundle --platform=node --sourcemap --outfile=./build/commands/"
+production_node_opts="--bundle --platform=node --minify --sourcemap"
+debugging_node_opts="--bundle --platform=node --sourcemap"
 react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
 
 # Manage --minify for debugging
@@ -14,18 +25,15 @@ if [ "$1" = "--debug" ]; then
 fi
 
 # library
-esbuild ${log} ./exporter/index.ts --bundle --platform=node  --outfile=./build/index.js
+esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
 esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
 
-# This .sh needs to be run from a npm script, so esbuild dependency is available
-esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts}end-render.js
-esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts}upload-search-content.js
-esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts}reset-render.js
-esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts}start-render.js
-esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts}prepare-domains-render.js
-esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts}prepare-assets-directory.js
+# scripts
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
 
 # types
-tsgo --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
-
-echo "Build completed successfully!"
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/commands/end-render.ts

@@ -1,84 +1,105 @@
-import fsp from "node:fs/promises";
+#!/usr/bin/env node
+
+import fs from "node:fs";
 import path from "node:path";
 
-import { throwError, withErrorHandler } from "../core/errors";
-import { pathExists } from "../core/fs";
-import { GriddoLog } from "../core/GriddoLog";
+import pkgDir from "pkg-dir";
+
 import { AuthService } from "../services/auth";
-import { getInstanceDomains } from "../services/domains";
-import {
-	getRenderMetadataFromDB,
-	getRenderModeFromDB,
-	getRenderPathsHydratedWithDomainFromDB,
-} from "../services/render";
 import { endSiteRender } from "../services/sites";
-import { GRIDDO_RENDER_BY_DOMAINS } from "../shared/envs";
-import { ArtifactError } from "../shared/errors";
-import { RENDER_MODE, type Report } from "../shared/types/render";
-
-async function getRenderReport(options: {
-	domain: string;
-	exportsSiteDir: string;
-	buildReportFileName: string;
-}) {
-	const { buildReportFileName, domain, exportsSiteDir } = options;
-	const buildReportFile = path.resolve(exportsSiteDir, domain, "dist", buildReportFileName);
-
-	if (!(await pathExists(buildReportFile))) {
-		GriddoLog.info(`Build report file "${buildReportFile}" not found.`);
-		throwError(ArtifactError);
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import { infoLog } from "../utils/loggin";
+
+const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+
+// Where we are going to find export dirs
+const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
+
+// Where we are going to find archived exports
+const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
+
+type Report = {
+	authControl:
+		| {
+				Authorization: string;
+				"Cache-Control": string;
+				lang?: string | undefined;
+		  }
+		| undefined;
+	sites: Array<{
+		siteId: number;
+		publishHashes: Array<string>;
+		siteHash: string | null;
+		unpublishHashes: Array<string>;
+		publishPagesIds: Array<number>;
+	}>;
+};
+
+function getDomainRenderReport(domain: string) {
+	const reports: Array<Report> = [];
+	const { buildReportFileName } = getConfig();
+
+	const buildReportFile = path.resolve(
+		exportArchiveBasePath,
+		domain,
+		"dist",
+		buildReportFileName,
+	);
+
+	if (!fs.existsSync(buildReportFile)) {
+		console.info(`Build report file "${buildReportFile}" not found.`);
 	}
 
-	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
-}
+	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
 
-async function endRender() {
-	const { buildReportFileName } = await getRenderMetadataFromDB();
-	const { __sites } = await getRenderPathsHydratedWithDomainFromDB();
+	reports.push(buildReport);
 
-	await AuthService.login();
+	return reports;
+}
 
-	let domains = [];
+(async () => {
+	const sendEndSiteRender = async (reports: Array<Report>) => {
+		for (const report of reports) {
+			const sites = report.sites;
 
-	if (GRIDDO_RENDER_BY_DOMAINS) {
-		const [domainName] = process.argv.slice(2);
-		domains.push(domainName);
-	} else {
-		GriddoLog.warn("Legacy Render Mode");
-		domains = await getInstanceDomains();
-	}
+			for (const site of sites) {
+				const { siteId, ...body } = site;
 
-	for (const domain of domains) {
-		const { renderMode, reason } = await getRenderModeFromDB(domain);
+				console.info(`Site (${siteId})`);
 
-		if (renderMode === RENDER_MODE.IDLE) {
-			GriddoLog.info(
-				`(From Current Render) [${domain}]: Skipping end-render as it is marked as IDLE with the reason ${reason}.`,
-			);
-			return;
+				await endSiteRender(siteId, body);
+			}
 		}
+	};
 
-		const report = await getRenderReport({
-			domain,
-			exportsSiteDir: __sites,
-			buildReportFileName,
-		});
-
-		GriddoLog.info(`Sending ending call to ${domain} sites`);
+	if (RENDER_BY_DOMAIN) {
+		await AuthService.login();
+		const domain = process.argv.splice(2)[0];
+		const reports = getDomainRenderReport(domain);
 
-		for (const site of report.sites) {
-			site.publishHashes = [...new Set(site.publishHashes)];
-			site.unpublishHashes = [...new Set(site.unpublishHashes)];
+		infoLog(`Sending ending call to ${domain} sites`);
 
-			const { siteId, ...body } = site;
-			GriddoLog.info(`Site (${siteId})`);
-			await endSiteRender(siteId, body);
+		for (const report of reports) {
+			for (const site of report.sites) {
+				site.publishHashes = [...new Set(site.publishHashes)];
+				site.unpublishHashes = [...new Set(site.unpublishHashes)];
+			}
 		}
-	}
-}
 
-async function main() {
-	await endRender();
-}
+		await sendEndSiteRender(reports);
+	} else {
+		// Legacy render
+		const domains = await getInstanceDomains();
+		for (const domain of domains) {
+			const reports = getDomainRenderReport(domain);
 
-withErrorHandler(main);
+			infoLog(`Sending ending call to ${domain} sites`);
+
+			await sendEndSiteRender(reports);
+		}
+	}
+})().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/commands/move-assets.ts

@@ -0,0 +1,11 @@
+import { execSync } from "node:child_process";
+import path from "node:path";
+
+const domainName = "pro-griddo";
+const domainPath = path.resolve(
+	__dirname,
+	`../../../exports/sites/${domainName}`,
+);
+
+execSync(`mkdir -p ${domainPath}/assets/${domainName}`);
+execSync(`cp -R ${domainPath}/assets/* ${domainPath}/assets/${domainName}`);

package/exporter/commands/prepare-domains-render.ts

@@ -1,158 +1,46 @@
-import type { RenderDB } from "../shared/types/render";
-
-import fsp from "node:fs/promises";
+import fs from "node:fs";
 import path from "node:path";
 
-import { version as griddoVersion } from "../../package.json";
-import { checkEnvironmentHealth } from "../core/check-env-health";
-import { readDB, writeDB } from "../core/db";
-import { withErrorHandler } from "../core/errors";
-import { pathExists, rmDirs } from "../core/fs";
-import { GriddoLog } from "../core/GriddoLog";
-import { resolveComponentsPath } from "../core/instance";
-import { AuthService } from "../services/auth";
-import { getInstanceDomains } from "../services/domains";
-import { getSitesToRender } from "../services/manage-sites";
-import { resolveDomainRenderMode } from "../services/render";
-import { GRIDDO_SKIP_BUILD_CHECKS } from "../shared/envs";
-import { pkgDir } from "../shared/npm-modules/pkg-dir";
-
-async function getDomainsWithNumberOfPages(domains: string[]) {
-	const results = await Promise.all(
-		domains.map(async (domain) => {
-			// Get the sites to render for the domain
-			const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
-
-			// Only count pages (offlinePending && publishPending) if sitesToPublish
-			// is not empty. Otherwise we set the domain has having 0 pages.
-			const pages: number[] = [];
-
-			if (sitesToPublish.length > 0) {
-				for (const site of sitesToPublish) {
-					// el site ya está publicado
-					if (site.isPublished && !site.shouldBeUpdated) {
-						const { pagesStatus } = site;
-						const { offlinePending, uploadPending } = pagesStatus;
-						const total = offlinePending.length + uploadPending.length;
-						pages.push(total);
-					}
-					// el site esta pendiente de publicar
-					if (site.isPublished && site.shouldBeUpdated) {
-						const { pagesStatus } = site;
-						const { offlinePending, uploadPending, active } = pagesStatus;
-						const total = offlinePending.length + uploadPending.length + active.length;
-						pages.push(total);
-					}
-				}
-			} else if (sitesToUnpublish.length > 0) {
-				// si tiene sites para despublicar marcamos el dominio como que tiene 1 página
-				pages.push(1);
-			} else {
-				// si eldominio no tiene nada ni para publicar ni despublicar lomarcamos como 0
-				pages.push(0);
-			}
-
-			const totalPages = pages.reduce((a, b) => a + b);
-
-			return { domain, totalPages };
-		}),
-	);
-
-	return results;
-}
-
-function getDomainsSortedByNumberOfPages(domainsInfo: { domain: string; totalPages: number }[]) {
-	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
-
-	return domainsInfo.map(({ domain }) => domain);
-}
-
-async function initRender() {
-	const root = (await pkgDir({ cwd: path.resolve(__dirname, "../../..") })) || "";
-	const cx = root;
-	const ssg = path.resolve((await pkgDir({ cwd: __dirname })) || "");
-	const cxCache = path.resolve(root, ".griddo/cache");
-	const components = resolveComponentsPath();
-	const exportsDir = path.join(root, "exports/sites");
-	const exportsDirBackup = path.join(root, "exports-backup/sites");
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import { infoLog, successLog } from "../utils/loggin";
+import { getSitesToRender } from "../utils/sites";
 
-	const data: RenderDB = {
-		griddoVersion,
-		buildReportFileName: "build-report.json",
-		sortedDomains: [],
-		needsRollbackOnError: false,
-		domains: {},
-		currentRenderingDomain: null,
-		paths: {
-			components,
-			cx,
-			cxCache,
-			exportsDir,
-			root,
-			ssg,
-			exportsDirBackup,
-		},
-	};
+(async () => {
+	infoLog("Checking domains size");
 
-	// create the main cache folder for cx if it doesn't exist.
-	if (!(await pathExists(cxCache))) {
-		await fsp.mkdir(cxCache, { recursive: true });
-	}
-
-	await writeDB(data);
-}
-
-async function prepareDomains() {
-	await AuthService.login();
+	const { __cx } = getConfig().paths();
 
-	const db = await readDB();
-
-	const __ssg = db.paths.ssg;
 	const domains = await getInstanceDomains();
-	const domainsWithNumberOfPages = await getDomainsWithNumberOfPages(domains);
-	const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPages);
-
-	// @deprecated use db.json (only for infra)
-	await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
-
-	db.sortedDomains = domainSorted;
-	db.domains = {};
-
-	for (const { domain, totalPages } of domainsWithNumberOfPages) {
-		const shouldBeRendered = totalPages > 0;
-		const { renderMode, reason } = await resolveDomainRenderMode({
-			domain,
-			shouldBeRendered,
-		});
-
-		// Log RenderModes/Reason
-		GriddoLog.info(
-			`(From Initial Render) [${domain}]: Marked as ${renderMode} with the reason: ${reason}`,
-		);
+	const domainsInfo = [];
+	const domainsFilePath = path.join(__cx, "domains.json");
+
+	for (const domain of domains) {
+		const { sitesToPublish } = await getSitesToRender(domain);
+
+		// Only count pages (changedPages) if sitesToPublish is not empty.
+		// Otherwise we set the domain has having 0 pages.
+		const pages = [];
+		if (sitesToPublish.length > 0) {
+			for (const site of sitesToPublish) {
+				pages.push(site.changedPages.length);
+			}
+		} else {
+			pages.push(0);
+		}
 
-		db.domains[domain] = db.domains[domain] || {};
-		db.domains[domain].renderMode = renderMode;
-		db.domains[domain].shouldBeRendered = shouldBeRendered;
-		db.domains[domain].renderModeReason = reason;
+		const totalPages = pages.reduce((a, b) => a + b);
+		domainsInfo.push({ domain, totalPages });
 	}
 
-	await writeDB(db);
-}
-
-async function clean() {
-	const db = await readDB();
-	const { root } = db.paths;
-	await rmDirs([path.join(root, "apiCache")]);
-}
-
-async function main() {
-	GRIDDO_SKIP_BUILD_CHECKS
-		? GriddoLog.info("Build health check bypassed")
-		: checkEnvironmentHealth();
+	// sort domains, smaller first
+	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
+	const domainSorted = domainsInfo.map(({ domain }) => domain);
 
-	await initRender();
-	await prepareDomains();
-	await clean();
-}
+	fs.writeFileSync(domainsFilePath, JSON.stringify(domainSorted));
 
-withErrorHandler(main);
+	successLog(`Checking domains size`);
+})().catch((err) => {
+	console.error(err);
+	process.exit(1);
+});