@griddo/cx 11.7.6-rc.1 → 11.7.6

package/exporter/adapters/gatsby/index.ts

@@ -1,76 +1,162 @@
-import type { SSG } from "./shared/types";
+import type {
+	Artifacts,
+	LifeCyclesNames,
+	LifeCycleSteps,
+} from "../../types/global";
 
 import path from "node:path";
 
-import { GRIDDO_ASSET_PREFIX } from "../../constants/envs";
-import { getCxArtifacts } from "../../utils/artifacts";
-import { doLifeCycle } from "../../utils/core-utils";
+import {
+	createDistFromGatsbyPublic,
+	getGatsbyAssetPrefixWithDomain,
+	runGatsbyBuildCommand,
+} from "./utils";
+import getCxArtifacts from "../../artifacts";
+import { AuthService } from "../../services/auth";
+import {
+	createDomainMetadata,
+	doLifeCycle,
+	getConfig,
+	saveBuildEndLogs,
+} from "../../utils/core-utils";
+import { createDomainRenderData } from "../../utils/create-build-data";
+import {
+	copyArtifacts,
+	createArtifacts,
+	deleteDisposableSiteDirs,
+	moveArtifacts,
+	removeArtifacts,
+	removeVirtualPagesFromStore,
+	renameArtifact,
+} from "../../utils/folders";
 import { infoLog } from "../../utils/loggin";
-import { getRenderMetadataFromDB, getRenderMode, getRenderPathsFromDB } from "../../utils/render";
-import { cleanAction } from "./actions/clean";
-import { closeAction } from "./actions/close";
-import { dataAction } from "./actions/data";
-import { healthCheckAction } from "./actions/healthCheck";
-import { initAction } from "./actions/init";
-import { metaAction } from "./actions/meta";
-import { prepareAction } from "./actions/prepare";
-import { relocationAction } from "./actions/relocation";
-import { restoreAction } from "./actions/restore";
-import { ssgAction } from "./actions/ssg";
-import { syncAction } from "./actions/sync";
-import { RenderContext } from "./shared/context";
+import {
+	createSentinelRenderFile,
+	deleteSentinelRenderFile,
+	isValidRender,
+} from "../../utils/render";
 
 /**
- * Executes a complete Gatsby render process for a given domain.
- *
- * The render process consists of multiple lifecycle steps that handle cleaning,
- * preparing, building and syncing the site. It can run in two modes:
- * - FROM_SCRATCH: Performs a complete rebuild of the site
- * - INCREMENTAL: Only rebuilds changed pages while preserving the rest
+ * Render every instance domain with the Gatsby adapter.
  */
-export async function gatsbyRenderDomain(domain: string) {
-	const { renderMode, reason } = await getRenderMode(domain);
+export async function renderDomainsWithGatsbyAdapter(domain: string) {
+	await AuthService.login();
+	const config = getConfig();
 
-	if (renderMode === "IDLE") {
-		return;
-	}
+	infoLog(`Initializing render for the domain ${domain}\n`);
+
+	const {
+		//
+		__ssg,
+		__exports,
+		__cache,
+		__cx,
+		__components,
+	} = config.paths(domain);
 
-	const renderReason = reason ? `<${reason}>` : "";
-	infoLog(`Init render (${renderMode}) ${renderReason} for domain ${domain}\n`);
+	// Griddo artifacts
+	const cxArtifacts = getCxArtifacts(domain);
 
-	const renderMetadata = await getRenderMetadataFromDB();
-	const paths = await getRenderPathsFromDB({ domain });
-	const cxArtifacts = await getCxArtifacts(domain);
-	const domainNeedsAssetPrefix = GRIDDO_ASSET_PREFIX && domain.startsWith(renderMetadata.proDomain);
-	const assetPrefix = domainNeedsAssetPrefix ? `${GRIDDO_ASSET_PREFIX}/${domain}` : "";
-	const ssgArtifacts = {
+	// Gatsby artifacts
+	const gatsbyArtifacts: Pick<Artifacts, "disposables" | "cacheables"> = {
 		disposables: [
-			path.join(paths.__ssg, "public"),
-			path.join(paths.__ssg, "static"),
-			path.join(paths.__ssg, ".cache"),
-			path.join(paths.__ssg, "domains.json"), // Este disposable es de griddo pero está en la ruta __ssg
+			path.join(__ssg, "public"),
+			path.join(__ssg, "static"),
+			path.join(__ssg, ".cache"),
 		],
 		cacheables: [".cache"],
 	};
 
-	const context = new RenderContext<SSG>({
-		domain,
-		renderMode,
-		paths,
-		renderMetadata,
-		cxArtifacts,
-		ssg: { assetPrefix, ssgArtifacts },
-	});
-
-	await doLifeCycle("Init", async () => initAction(context));
-	await doLifeCycle("Clean", async () => cleanAction(context));
-	await doLifeCycle("Prepare", async () => prepareAction(context));
-	await doLifeCycle("Restore", async () => restoreAction(context));
-	await doLifeCycle("Data", async () => dataAction(context));
-	await doLifeCycle("SSG", async () => ssgAction(context));
-	await doLifeCycle("Relocation", async () => relocationAction(context));
-	await doLifeCycle("Meta", async () => metaAction(context));
-	await doLifeCycle("Sync", async () => syncAction(context));
-	await doLifeCycle("Close", async () => closeAction(context));
-	await doLifeCycle("HealthCheck", async () => healthCheckAction(context));
+	// These variables are involved in the build for create the dist directory
+	// from public and pass to the gatsby-build command via spawnSync.
+	const assetPrefix = getGatsbyAssetPrefixWithDomain(domain);
+	const needsAssetPrefix = !!assetPrefix && assetPrefix !== "";
+
+	// Compose artifacts
+	const disposableArtifacts = [
+		...cxArtifacts.disposables,
+		...gatsbyArtifacts.disposables,
+	];
+
+	// LifeCycles
+
+	const allLifeCycles: Array<{
+		name: LifeCyclesNames;
+		steps: LifeCycleSteps;
+	}> = [
+		{
+			name: "Init",
+			steps: [() => createSentinelRenderFile()],
+		},
+
+		{
+			name: "Clean",
+			steps: [() => removeArtifacts(disposableArtifacts)],
+		},
+
+		{
+			name: "Prepare",
+			steps: [() => createArtifacts(cxArtifacts.initials)],
+		},
+
+		{
+			name: "Restore",
+			steps: [
+				() => copyArtifacts(__components, __ssg, ["static"]),
+				() => copyArtifacts(__exports, __cx, cxArtifacts.archivables),
+				() =>
+					renameArtifact(path.join(__cx, "dist"), path.join(__ssg, "public")),
+				() => moveArtifacts(__cache, __cx, cxArtifacts.cacheables),
+				() => moveArtifacts(__cache, __ssg, gatsbyArtifacts.cacheables),
+			],
+		},
+
+		{
+			name: "Data",
+			steps: [() => createDomainRenderData(domain)],
+		},
+
+		{
+			name: "SSG",
+			steps: [() => runGatsbyBuildCommand(assetPrefix)],
+		},
+
+		{
+			name: "Relocation",
+			steps: [() => createDistFromGatsbyPublic(domain, needsAssetPrefix)],
+		},
+
+		{
+			name: "Meta",
+			steps: [() => createDomainMetadata(domain), () => saveBuildEndLogs()],
+		},
+
+		{
+			name: "Archive",
+			steps: [
+				() => removeVirtualPagesFromStore(),
+				() => deleteDisposableSiteDirs(),
+				() =>
+					moveArtifacts(__cx, __exports, cxArtifacts.archivables, {
+						withBackup: true,
+					}),
+				() => moveArtifacts(__cx, __cache, cxArtifacts.cacheables),
+				() => moveArtifacts(__ssg, __cache, gatsbyArtifacts.cacheables),
+			],
+		},
+
+		{
+			name: "Close",
+			steps: [() => removeArtifacts(disposableArtifacts)],
+		},
+
+		{
+			name: "HealthCheck",
+			steps: [() => isValidRender(), () => deleteSentinelRenderFile()],
+		},
+	];
+
+	for (const { name, steps } of allLifeCycles) {
+		await doLifeCycle(name, steps);
+	}
 }

package/exporter/adapters/gatsby/utils.ts

@@ -0,0 +1,161 @@
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+
+import fsx from "fs-extra";
+
+import { envs } from "../../constants";
+import { throwError } from "../../errors";
+import { ErrorInSSGBuildProcess } from "../../errors/errors-data";
+import { getConfig } from "../../utils/core-utils";
+import { verboseLog } from "../../utils/loggin";
+
+const config = getConfig();
+
+/**
+ * Return the assetPrefix with the domain concatenated `assetPrefix/domain` only
+ * if the domain is "pro-" **and** there is a env `GRIDDO_ASSET_PREFIX` with any
+ * value different from `null`, `undefined` or `empty string`
+ *
+ * else...
+ * - If assetPrefix or domain is falsy, returns ""
+ * - If the domain is not "pro-", returns ""
+ */
+function getGatsbyAssetPrefixWithDomain(domain: string) {
+	const { proDomain } = getConfig();
+
+	if (!envs.GRIDDO_ASSET_PREFIX || !domain) {
+		return "";
+	}
+
+	if (!domain.startsWith(proDomain)) {
+		return "";
+	}
+
+	const assetPrefixWithDomain = `${envs.GRIDDO_ASSET_PREFIX}/${domain}`;
+
+	verboseLog(
+		`Reading process.env.GRIDDO_ASSET_PREFIX, ${envs.GRIDDO_ASSET_PREFIX}`,
+	);
+	verboseLog(
+		`Setting the asset prefix with the domain concatenated, ${assetPrefixWithDomain}`,
+	);
+
+	return assetPrefixWithDomain;
+}
+
+/**
+ * Spawn a new node process `yarn gatsby-build`
+ * @note This proccess (`yarn gatsby-build`) can not access to the custom Griddo
+ * `process.env` so it needs variables passed to it via the `env` prop.
+ */
+function runGatsbyBuildCommand(assetPrefixWithDomain: string) {
+	verboseLog(`read assetPrefixWithDomain, ${assetPrefixWithDomain}`);
+	verboseLog(
+		`using this NODE_OPTIONS in gatsby command: ${process.env.NODE_OPTIONS}`,
+	);
+
+	const { __ssg } = config.paths();
+
+	// Remove `--openssl-legacy-provider` from NODE_OPTIONS because this value
+	// break Gataby render.
+	const nodeOptionsWithoutLegacyOpenSSL = process.env.NODE_OPTIONS
+		? process.env.NODE_OPTIONS.replace(
+				/--openssl-legacy-provider\s*/g,
+				"",
+			).trim()
+		: "";
+
+	const command = spawnSync(
+		"yarn",
+		["gatsby-build", envs.GRIDDO_SSG_VERBOSE_LOGS ? "--verbose" : ""],
+		{
+			cwd: __ssg,
+			stdio: ["ignore", "inherit", "ignore"],
+			encoding: "utf8",
+			shell: true,
+			env: Object.assign(process.env, {
+				GRIDDO_EXPORTER: "true",
+				SPAWN_ASSET_PREFIX_WITH_DOMAIN: assetPrefixWithDomain,
+				NODE_OPTIONS: nodeOptionsWithoutLegacyOpenSSL,
+			}),
+		},
+	);
+
+	if (command.status !== 0) {
+		throwError(ErrorInSSGBuildProcess(command));
+	}
+}
+
+/**
+ * Update the Griddo's `/dist` dir with the contents from `public` dir only
+ * with files of type: js, json and css.
+ */
+async function createDistFromGatsbyPublic(
+	domain: string,
+	needsAssetPrefix: boolean,
+) {
+	const { __cx, __ssg } = config.paths(domain);
+
+	const cxDistDir = path.join(__cx, "dist");
+	const cxAssetsDir = path.join(__cx, "assets");
+	const cxDistDirWithDomain = path.join(__cx, "dist", domain);
+	const publicDir = path.join(__ssg, "public");
+
+	const validFilesFromPublic = fsx
+		.readdirSync(publicDir)
+		.filter(
+			(file) =>
+				path.extname(file) === ".js" ||
+				path.extname(file) === ".json" ||
+				path.extname(file) === ".css",
+		);
+
+	const pageDataSrc = `${publicDir}/page-data`;
+	const pageDataDest = `${cxAssetsDir}/page-data`;
+
+	const projectStaticSrc = path.join(__ssg, "static");
+	const projectStaticDest = cxAssetsDir;
+
+	const gatsbyStaticSrc = `${cxDistDir}/static`;
+	const gatsbyStaticDest = `${cxAssetsDir}/static`;
+
+	try {
+		fsx.mkdirSync(cxAssetsDir, { recursive: true });
+		fsx.copySync(publicDir, cxDistDir, { preserveTimestamps: true });
+
+		if (needsAssetPrefix) {
+			fsx.copySync(pageDataSrc, pageDataDest, { preserveTimestamps: true });
+			if (fsx.existsSync(projectStaticSrc)) {
+				fsx.copySync(projectStaticSrc, projectStaticDest, {
+					overwrite: false,
+					preserveTimestamps: true,
+				});
+			}
+			fsx.copySync(gatsbyStaticSrc, gatsbyStaticDest, {
+				overwrite: false,
+				preserveTimestamps: true,
+			});
+			if (fsx.existsSync(projectStaticSrc)) {
+				fsx.copySync(projectStaticSrc, cxDistDirWithDomain, {
+					overwrite: false,
+					preserveTimestamps: true,
+				});
+			}
+
+			// Copia el resto de archivos...
+			validFilesFromPublic.map((file) => {
+				const fileSrc = `${publicDir}/${file}`;
+				const fileDest = `${cxAssetsDir}/${file}`;
+				fsx.copySync(fileSrc, fileDest, { preserveTimestamps: true });
+			});
+		}
+	} catch (err) {
+		console.error(err);
+	}
+}
+
+export {
+	createDistFromGatsbyPublic,
+	getGatsbyAssetPrefixWithDomain,
+	runGatsbyBuildCommand,
+};

package/exporter/artifacts/README.md

@@ -0,0 +1,34 @@
+# CX Artifacs
+
+## How to add an artifact.
+
+An artifact is a **full path** to a file or directory if it has a fixed
+location or only a **name** when it is going to be moved (archived/restored)
+between directories in the rendering process and therefore doesn't have a
+single location.
+
+Only the exporter knows where the artifacts are archived and restored, so the
+artifacts don't need to know anything other than their own location or name.
+
+Si estás creando un adaptador, sus artefactos deberán ir en un archivo separado (igual que `artifacts/cx.ts`) y exportar una función por defecto que devuelva un `Artifacts`. Todo deberá estar exportado desde `artifacts/index.ts`.
+
+**Artifacts have four categories:**
+
+- **initials**: Are directories that need to be created for their
+  future use.
+
+- **archivables**: Are stored in `__exports` which is the one uploaded to the
+  server as the final render and restored in `__cx` and `__ssg` to be reused in the
+  render. Therefore, they don't have a single location and should be added only
+  with the artifact name (directory/file)
+
+- **cacheables**: Are stored in `__cache` and restored in `__cx` and `__ssg` to be
+  reused in each domain render. Therefore, they don't have a single location.
+
+- **disposables**: Are generated alwais in `__cx` and `__ssg` or fixed locations.
+  These artifacts are discarded (deleted) in each domain render. They are
+  temporary directories and/or files.
+
+## Example
+
+Check `cx.ts` inside `./artifacts` directory to see how to add an artifact.

package/exporter/{utils/artifacts.ts → artifacts/index.ts}

@@ -2,13 +2,14 @@ import type { Artifacts } from "../types/global";
 
 import path from "node:path";
 
-import { getRenderPathsFromDB } from "./render";
+import { getConfig } from "../utils/core-utils";
 
 /**
  * Returns the artifacts of CX.
  */
-async function getCxArtifacts(domain: string): Promise<Artifacts> {
-	const { __cx, __exports, __cache } = await getRenderPathsFromDB({ domain });
+function getCxArtifacts(domain: string): Artifacts {
+	const config = getConfig();
+	const { __cx, __exports, __cache } = config.paths(domain);
 
 	return {
 		initials: [
@@ -20,15 +21,13 @@ async function getCxArtifacts(domain: string): Promise<Artifacts> {
 		disposables: [
 			path.join(__cx, "store"),
 			path.join(__cx, "apiCache"),
+			path.join(__cx, "render-metadata.json"),
 			path.join(__cx, "dist"),
-			path.join(__cx, "dist-restored"),
 			path.join(__cx, "debug"),
-			path.join(__cx, "assets"),
 		],
-		cacheables: [],
-		archivables: ["dist", "debug", "assets"],
-		restaurable: ["dist", "debug"],
+		cacheables: ["apiCache", "store"],
+		archivables: ["dist", "assets", "debug"],
 	};
 }
 
-export { getCxArtifacts };
+export default getCxArtifacts;

package/exporter/build.sh

@@ -1,6 +1,39 @@
-#!/usr/bin/env bash
+# Build CX script file
+#
+# This file generates the necessary builds for CX.
+# - Setup, to generate necessary elements (tsconfig, etc..)
+# - Library, CX library that will be used by the SSG framework.
+# - Scripts, CX scripts that will be used by infra during the render process.
+#
+# @todo:
+# In the future, we will try to eliminate the bundling processes and use
+# TypeScript or ESModules directly.
 
-echo "📦 Compiling CX..."
-rm -rf ./build
-tsc --project tsconfig.commands.json
-tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
+# start
+echo "📦 Building ..."
+
+# CLI options
+log="--log-level=error"
+production_node_opts="--bundle --platform=node --minify --sourcemap"
+debugging_node_opts="--bundle --platform=node --sourcemap"
+react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
+
+# Manage --minify for debugging
+bundle_node_opts=$production_node_opts
+if [ "$1" = "--debug" ]; then
+    bundle_node_opts=$debugging_node_opts
+fi
+
+# library
+esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
+esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
+
+# scripts
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
+
+# types
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/commands/end-render.ts

@@ -1,80 +1,105 @@
 #!/usr/bin/env node
 
-import type { Report } from "../types/render";
-
-import fsp from "node:fs/promises";
+import fs from "node:fs";
 import path from "node:path";
 
-import { ArtifactError } from "../constants/errors";
+import pkgDir from "pkg-dir";
+
 import { AuthService } from "../services/auth";
 import { endSiteRender } from "../services/sites";
-import { doLifeCycle } from "../utils/core-utils";
-import { throwError } from "../utils/errors";
-import { pathExists } from "../utils/folders";
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
 import { infoLog } from "../utils/loggin";
-import { getRenderMetadataFromDB, getRenderMode, getRenderPathsFromDB } from "../utils/render";
-
-async function getRenderReport(options: {
-	domain: string;
-	exportsSiteDir: string;
-	buildReportFileName: string;
-}) {
-	const { buildReportFileName, domain, exportsSiteDir: exportArchiveBasePath } = options;
-	const buildReportFile = path.resolve(exportArchiveBasePath, domain, "dist", buildReportFileName);
-
-	if (!(await pathExists(buildReportFile))) {
-		infoLog(`Build report file "${buildReportFile}" not found.`);
-		throwError(ArtifactError);
-	}
 
-	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
-}
+const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+
+// Where we are going to find export dirs
+const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
+
+// Where we are going to find archived exports
+const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
+
+type Report = {
+	authControl:
+		| {
+				Authorization: string;
+				"Cache-Control": string;
+				lang?: string | undefined;
+		  }
+		| undefined;
+	sites: Array<{
+		siteId: number;
+		publishHashes: Array<string>;
+		siteHash: string | null;
+		unpublishHashes: Array<string>;
+		publishPagesIds: Array<number>;
+	}>;
+};
 
-const sendEndSiteRender = async (report: Report) => {
-	const sites = report.sites;
+function getDomainRenderReport(domain: string) {
+	const reports: Array<Report> = [];
+	const { buildReportFileName } = getConfig();
 
-	for (const site of sites) {
-		const { siteId, ...body } = site;
-		infoLog(`Site (${siteId})`);
-		await endSiteRender(siteId, body);
+	const buildReportFile = path.resolve(
+		exportArchiveBasePath,
+		domain,
+		"dist",
+		buildReportFileName,
+	);
+
+	if (!fs.existsSync(buildReportFile)) {
+		console.info(`Build report file "${buildReportFile}" not found.`);
 	}
-};
 
-async function endRenderAction() {
-	const { buildReportFileName } = await getRenderMetadataFromDB();
-	const { __sites } = await getRenderPathsFromDB();
+	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
 
-	const [domain] = process.argv.slice(2);
-	const { renderMode } = await getRenderMode(domain);
+	reports.push(buildReport);
 
-	if (renderMode === "IDLE") {
-		infoLog(`Skipping build-end for domain ${domain} as it is marked as IDLE.`);
-		return;
-	}
+	return reports;
+}
 
-	await AuthService.login();
+(async () => {
+	const sendEndSiteRender = async (reports: Array<Report>) => {
+		for (const report of reports) {
+			const sites = report.sites;
 
-	const report = await getRenderReport({
-		domain,
-		exportsSiteDir: __sites,
-		buildReportFileName,
-	});
+			for (const site of sites) {
+				const { siteId, ...body } = site;
 
-	infoLog(`Sending ending call to ${domain} sites`);
+				console.info(`Site (${siteId})`);
 
-	for (const site of report.sites) {
-		site.publishHashes = [...new Set(site.publishHashes)];
-		site.unpublishHashes = [...new Set(site.unpublishHashes)];
-	}
+				await endSiteRender(siteId, body);
+			}
+		}
+	};
 
-	await sendEndSiteRender(report);
-}
+	if (RENDER_BY_DOMAIN) {
+		await AuthService.login();
+		const domain = process.argv.splice(2)[0];
+		const reports = getDomainRenderReport(domain);
 
-async function main() {
-	await doLifeCycle("EndRender", endRenderAction);
-}
+		infoLog(`Sending ending call to ${domain} sites`);
+
+		for (const report of reports) {
+			for (const site of report.sites) {
+				site.publishHashes = [...new Set(site.publishHashes)];
+				site.unpublishHashes = [...new Set(site.unpublishHashes)];
+			}
+		}
+
+		await sendEndSiteRender(reports);
+	} else {
+		// Legacy render
+		const domains = await getInstanceDomains();
+		for (const domain of domains) {
+			const reports = getDomainRenderReport(domain);
 
-main().catch((err) => {
+			infoLog(`Sending ending call to ${domain} sites`);
+
+			await sendEndSiteRender(reports);
+		}
+	}
+})().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/move-assets.ts

@@ -0,0 +1,11 @@
+import { execSync } from "node:child_process";
+import path from "node:path";
+
+const domainName = "pro-griddo";
+const domainPath = path.resolve(
+	__dirname,
+	`../../../exports/sites/${domainName}`,
+);
+
+execSync(`mkdir -p ${domainPath}/assets/${domainName}`);
+execSync(`cp -R ${domainPath}/assets/* ${domainPath}/assets/${domainName}`);