@griddo/cx 11.9.8-rc.1 → 11.9.8-rc.2

package/exporter/adapters/gatsby/index.ts

@@ -0,0 +1,162 @@
+import type {
+	Artifacts,
+	LifeCyclesNames,
+	LifeCycleSteps,
+} from "../../types/global";
+
+import path from "node:path";
+
+import {
+	createDistFromGatsbyPublic,
+	getGatsbyAssetPrefixWithDomain,
+	runGatsbyBuildCommand,
+} from "./utils";
+import getCxArtifacts from "../../artifacts";
+import { AuthService } from "../../services/auth";
+import {
+	createDomainMetadata,
+	doLifeCycle,
+	getConfig,
+	saveBuildEndLogs,
+} from "../../utils/core-utils";
+import { createDomainRenderData } from "../../utils/create-build-data";
+import {
+	copyArtifacts,
+	createArtifacts,
+	deleteDisposableSiteDirs,
+	moveArtifacts,
+	removeArtifacts,
+	removeVirtualPagesFromStore,
+	renameArtifact,
+} from "../../utils/folders";
+import { infoLog } from "../../utils/loggin";
+import {
+	createSentinelRenderFile,
+	deleteSentinelRenderFile,
+	isValidRender,
+} from "../../utils/render";
+
+/**
+ * Render every instance domain with the Gatsby adapter.
+ */
+export async function renderDomainsWithGatsbyAdapter(domain: string) {
+	await AuthService.login();
+	const config = getConfig();
+
+	infoLog(`Initializing render for the domain ${domain}\n`);
+
+	const {
+		//
+		__ssg,
+		__exports,
+		__cache,
+		__cx,
+		__components,
+	} = config.paths(domain);
+
+	// Griddo artifacts
+	const cxArtifacts = getCxArtifacts(domain);
+
+	// Gatsby artifacts
+	const gatsbyArtifacts: Pick<Artifacts, "disposables" | "cacheables"> = {
+		disposables: [
+			path.join(__ssg, "public"),
+			path.join(__ssg, "static"),
+			path.join(__ssg, ".cache"),
+		],
+		cacheables: [".cache"],
+	};
+
+	// These variables are involved in the build for create the dist directory
+	// from public and pass to the gatsby-build command via spawnSync.
+	const assetPrefix = getGatsbyAssetPrefixWithDomain(domain);
+	const needsAssetPrefix = !!assetPrefix && assetPrefix !== "";
+
+	// Compose artifacts
+	const disposableArtifacts = [
+		...cxArtifacts.disposables,
+		...gatsbyArtifacts.disposables,
+	];
+
+	// LifeCycles
+
+	const allLifeCycles: Array<{
+		name: LifeCyclesNames;
+		steps: LifeCycleSteps;
+	}> = [
+		{
+			name: "Init",
+			steps: [() => createSentinelRenderFile()],
+		},
+
+		{
+			name: "Clean",
+			steps: [() => removeArtifacts(disposableArtifacts)],
+		},
+
+		{
+			name: "Prepare",
+			steps: [() => createArtifacts(cxArtifacts.initials)],
+		},
+
+		{
+			name: "Restore",
+			steps: [
+				() => copyArtifacts(__components, __ssg, ["static"]),
+				() => copyArtifacts(__exports, __cx, cxArtifacts.archivables),
+				() =>
+					renameArtifact(path.join(__cx, "dist"), path.join(__ssg, "public")),
+				() => moveArtifacts(__cache, __cx, cxArtifacts.cacheables),
+				() => moveArtifacts(__cache, __ssg, gatsbyArtifacts.cacheables),
+			],
+		},
+
+		{
+			name: "Data",
+			steps: [() => createDomainRenderData(domain)],
+		},
+
+		{
+			name: "SSG",
+			steps: [() => runGatsbyBuildCommand(assetPrefix)],
+		},
+
+		{
+			name: "Relocation",
+			steps: [() => createDistFromGatsbyPublic(domain, needsAssetPrefix)],
+		},
+
+		{
+			name: "Meta",
+			steps: [() => createDomainMetadata(domain), () => saveBuildEndLogs()],
+		},
+
+		{
+			name: "Archive",
+			steps: [
+				() => removeVirtualPagesFromStore(),
+				() => deleteDisposableSiteDirs(),
+				() =>
+					moveArtifacts(__cx, __exports, cxArtifacts.archivables, {
+						withBackup: true,
+					}),
+				() => moveArtifacts(__cx, __cache, cxArtifacts.cacheables),
+				() => moveArtifacts(__ssg, __cache, gatsbyArtifacts.cacheables),
+			],
+		},
+
+		{
+			name: "Close",
+			steps: [() => removeArtifacts(disposableArtifacts)],
+		},
+
+		{
+			name: "HealthCheck",
+			steps: [() => isValidRender(), () => deleteSentinelRenderFile()],
+		},
+	];
+
+	for (const { name, steps } of allLifeCycles) {
+		await doLifeCycle(name, steps);
+	}
+}

package/exporter/adapters/gatsby/utils.ts

@@ -0,0 +1,161 @@
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+
+import fsx from "fs-extra";
+
+import { envs } from "../../constants";
+import { throwError } from "../../errors";
+import { ErrorInSSGBuildProcess } from "../../errors/errors-data";
+import { getConfig } from "../../utils/core-utils";
+import { verboseLog } from "../../utils/loggin";
+
+const config = getConfig();
+
+/**
+ * Return the assetPrefix with the domain concatenated `assetPrefix/domain` only
+ * if the domain is "pro-" **and** there is a env `GRIDDO_ASSET_PREFIX` with any
+ * value different from `null`, `undefined` or `empty string`
+ *
+ * else...
+ * - If assetPrefix or domain is falsy, returns ""
+ * - If the domain is not "pro-", returns ""
+ */
+function getGatsbyAssetPrefixWithDomain(domain: string) {
+	const { proDomain } = getConfig();
+
+	if (!envs.GRIDDO_ASSET_PREFIX || !domain) {
+		return "";
+	}
+
+	if (!domain.startsWith(proDomain)) {
+		return "";
+	}
+
+	const assetPrefixWithDomain = `${envs.GRIDDO_ASSET_PREFIX}/${domain}`;
+
+	verboseLog(
+		`Reading process.env.GRIDDO_ASSET_PREFIX, ${envs.GRIDDO_ASSET_PREFIX}`,
+	);
+	verboseLog(
+		`Setting the asset prefix with the domain concatenated, ${assetPrefixWithDomain}`,
+	);
+
+	return assetPrefixWithDomain;
+}
+
+/**
+ * Spawn a new node process `yarn gatsby-build`
+ * @note This proccess (`yarn gatsby-build`) can not access to the custom Griddo
+ * `process.env` so it needs variables passed to it via the `env` prop.
+ */
+function runGatsbyBuildCommand(assetPrefixWithDomain: string) {
+	verboseLog(`read assetPrefixWithDomain, ${assetPrefixWithDomain}`);
+	verboseLog(
+		`using this NODE_OPTIONS in gatsby command: ${process.env.NODE_OPTIONS}`,
+	);
+
+	const { __ssg } = config.paths();
+
+	// Remove `--openssl-legacy-provider` from NODE_OPTIONS because this value
+	// break Gataby render.
+	const nodeOptionsWithoutLegacyOpenSSL = process.env.NODE_OPTIONS
+		? process.env.NODE_OPTIONS.replace(
+				/--openssl-legacy-provider\s*/g,
+				"",
+			).trim()
+		: "";
+
+	const command = spawnSync(
+		"yarn",
+		["gatsby-build", envs.GRIDDO_SSG_VERBOSE_LOGS ? "--verbose" : ""],
+		{
+			cwd: __ssg,
+			stdio: ["ignore", "inherit", "ignore"],
+			encoding: "utf8",
+			shell: true,
+			env: Object.assign(process.env, {
+				GRIDDO_EXPORTER: "true",
+				SPAWN_ASSET_PREFIX_WITH_DOMAIN: assetPrefixWithDomain,
+				NODE_OPTIONS: nodeOptionsWithoutLegacyOpenSSL,
+			}),
+		},
+	);
+
+	if (command.status !== 0) {
+		throwError(ErrorInSSGBuildProcess(command));
+	}
+}
+
+/**
+ * Update the Griddo's `/dist` dir with the contents from `public` dir only
+ * with files of type: js, json and css.
+ */
+async function createDistFromGatsbyPublic(
+	domain: string,
+	needsAssetPrefix: boolean,
+) {
+	const { __cx, __ssg } = config.paths(domain);
+
+	const cxDistDir = path.join(__cx, "dist");
+	const cxAssetsDir = path.join(__cx, "assets");
+	const cxDistDirWithDomain = path.join(__cx, "dist", domain);
+	const publicDir = path.join(__ssg, "public");
+
+	const validFilesFromPublic = fsx
+		.readdirSync(publicDir)
+		.filter(
+			(file) =>
+				path.extname(file) === ".js" ||
+				path.extname(file) === ".json" ||
+				path.extname(file) === ".css",
+		);
+
+	const pageDataSrc = `${publicDir}/page-data`;
+	const pageDataDest = `${cxAssetsDir}/page-data`;
+
+	const projectStaticSrc = path.join(__ssg, "static");
+	const projectStaticDest = cxAssetsDir;
+
+	const gatsbyStaticSrc = `${cxDistDir}/static`;
+	const gatsbyStaticDest = `${cxAssetsDir}/static`;
+
+	try {
+		fsx.mkdirSync(cxAssetsDir, { recursive: true });
+		fsx.copySync(publicDir, cxDistDir, { preserveTimestamps: true });
+
+		if (needsAssetPrefix) {
+			fsx.copySync(pageDataSrc, pageDataDest, { preserveTimestamps: true });
+			if (fsx.existsSync(projectStaticSrc)) {
+				fsx.copySync(projectStaticSrc, projectStaticDest, {
+					overwrite: false,
+					preserveTimestamps: true,
+				});
+			}
+			fsx.copySync(gatsbyStaticSrc, gatsbyStaticDest, {
+				overwrite: false,
+				preserveTimestamps: true,
+			});
+			if (fsx.existsSync(projectStaticSrc)) {
+				fsx.copySync(projectStaticSrc, cxDistDirWithDomain, {
+					overwrite: false,
+					preserveTimestamps: true,
+				});
+			}
+
+			// Copia el resto de archivos...
+			validFilesFromPublic.map((file) => {
+				const fileSrc = `${publicDir}/${file}`;
+				const fileDest = `${cxAssetsDir}/${file}`;
+				fsx.copySync(fileSrc, fileDest, { preserveTimestamps: true });
+			});
+		}
+	} catch (err) {
+		console.error(err);
+	}
+}
+
+export {
+	createDistFromGatsbyPublic,
+	getGatsbyAssetPrefixWithDomain,
+	runGatsbyBuildCommand,
+};

package/exporter/artifacts/README.md

@@ -0,0 +1,34 @@
+# CX Artifacs
+
+## How to add an artifact.
+
+An artifact is a **full path** to a file or directory if it has a fixed
+location or only a **name** when it is going to be moved (archived/restored)
+between directories in the rendering process and therefore doesn't have a
+single location.
+
+Only the exporter knows where the artifacts are archived and restored, so the
+artifacts don't need to know anything other than their own location or name.
+
+Si estás creando un adaptador, sus artefactos deberán ir en un archivo separado (igual que `artifacts/cx.ts`) y exportar una función por defecto que devuelva un `Artifacts`. Todo deberá estar exportado desde `artifacts/index.ts`.
+
+**Artifacts have four categories:**
+
+- **initials**: Are directories that need to be created for their
+  future use.
+
+- **archivables**: Are stored in `__exports` which is the one uploaded to the
+  server as the final render and restored in `__cx` and `__ssg` to be reused in the
+  render. Therefore, they don't have a single location and should be added only
+  with the artifact name (directory/file)
+
+- **cacheables**: Are stored in `__cache` and restored in `__cx` and `__ssg` to be
+  reused in each domain render. Therefore, they don't have a single location.
+
+- **disposables**: Are generated alwais in `__cx` and `__ssg` or fixed locations.
+  These artifacts are discarded (deleted) in each domain render. They are
+  temporary directories and/or files.
+
+## Example
+
+Check `cx.ts` inside `./artifacts` directory to see how to add an artifact.

package/exporter/artifacts/index.ts

@@ -0,0 +1,33 @@
+import type { Artifacts } from "../types/global";
+
+import path from "node:path";
+
+import { getConfig } from "../utils/core-utils";
+
+/**
+ * Returns the artifacts of CX.
+ */
+function getCxArtifacts(domain: string): Artifacts {
+	const config = getConfig();
+	const { __cx, __exports, __cache } = config.paths(domain);
+
+	return {
+		initials: [
+			__exports, // `/exports/<domain>`
+			__cache, // `__cx/.cx-cache/<domain>`
+			path.join(__cx, "store"),
+			path.join(__cx, "apiCache"),
+		],
+		disposables: [
+			path.join(__cx, "store"),
+			path.join(__cx, "apiCache"),
+			path.join(__cx, "render-metadata.json"),
+			path.join(__cx, "dist"),
+			path.join(__cx, "debug"),
+		],
+		cacheables: ["apiCache", "store"],
+		archivables: ["dist", "assets", "debug"],
+	};
+}
+
+export default getCxArtifacts;

package/exporter/build.sh

@@ -1,10 +1,22 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2086
+# Build CX script file
+#
+# This file generates the necessary builds for CX.
+# - Setup, to generate necessary elements (tsconfig, etc..)
+# - Library, CX library that will be used by the SSG framework.
+# - Scripts, CX scripts that will be used by infra during the render process.
+#
+# @todo:
+# In the future, we will try to eliminate the bundling processes and use
+# TypeScript or ESModules directly.
 
-# CLI options for esbuild
+# start
+echo "📦 Building ..."
+
+# CLI options
 log="--log-level=error"
-production_node_opts="--bundle --platform=node --minify --sourcemap --outfile=./build/commands/"
-debugging_node_opts="--bundle --platform=node --sourcemap --outfile=./build/commands/"
+production_node_opts="--bundle --platform=node --minify --sourcemap"
+debugging_node_opts="--bundle --platform=node --sourcemap"
+react_opts="--platform=node --external:@griddo-instance --external:react --external:react-dom --bundle --minify"
 
 # Manage --minify for debugging
 bundle_node_opts=$production_node_opts
@@ -12,17 +24,16 @@ if [ "$1" = "--debug" ]; then
     bundle_node_opts=$debugging_node_opts
 fi
 
-# This .sh needs to be run from a npm script, so esbuild dependency is available
-esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts}end-render.js
-esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts}upload-search-content.js
-esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts}reset-render.js
-esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts}start-render.js
-esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts}prepare-domains-render.js
-esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts}prepare-assets-directory.js
-esbuild ${log} ./exporter/commands/start-embeddings.ts ${bundle_node_opts}start-embeddings.js
-esbuild ${log} ./exporter/index.ts --bundle --platform=node  --outfile=./build/index.js
+# library
+esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.js
+esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
 
-# types
-tsgo --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
+# scripts
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
 
-echo "Build completed successfully!"
+# types
+tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/commands/end-render.ts

@@ -1,84 +1,105 @@
-import fsp from "node:fs/promises";
+#!/usr/bin/env node
+
+import fs from "node:fs";
 import path from "node:path";
 
-import { throwError, withErrorHandler } from "../core/errors";
-import { pathExists } from "../core/fs";
-import { GriddoLog } from "../core/GriddoLog";
+import pkgDir from "pkg-dir";
+
 import { AuthService } from "../services/auth";
-import { getInstanceDomains } from "../services/domains";
-import {
-	getRenderMetadataFromDB,
-	getRenderModeFromDB,
-	getRenderPathsHydratedWithDomainFromDB,
-} from "../services/render";
 import { endSiteRender } from "../services/sites";
-import { GRIDDO_RENDER_BY_DOMAINS } from "../shared/envs";
-import { ArtifactError } from "../shared/errors";
-import { RENDER_MODE, type Report } from "../shared/types/render";
-
-async function getRenderReport(options: {
-	domain: string;
-	exportsSiteDir: string;
-	buildReportFileName: string;
-}) {
-	const { buildReportFileName, domain, exportsSiteDir } = options;
-	const buildReportFile = path.resolve(exportsSiteDir, domain, "dist", buildReportFileName);
-
-	if (!(await pathExists(buildReportFile))) {
-		GriddoLog.info(`Build report file "${buildReportFile}" not found.`);
-		throwError(ArtifactError);
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import { infoLog } from "../utils/loggin";
+
+const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+
+// Where we are going to find export dirs
+const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
+
+// Where we are going to find archived exports
+const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
+
+type Report = {
+	authControl:
+		| {
+				Authorization: string;
+				"Cache-Control": string;
+				lang?: string | undefined;
+		  }
+		| undefined;
+	sites: Array<{
+		siteId: number;
+		publishHashes: Array<string>;
+		siteHash: string | null;
+		unpublishHashes: Array<string>;
+		publishPagesIds: Array<number>;
+	}>;
+};
+
+function getDomainRenderReport(domain: string) {
+	const reports: Array<Report> = [];
+	const { buildReportFileName } = getConfig();
+
+	const buildReportFile = path.resolve(
+		exportArchiveBasePath,
+		domain,
+		"dist",
+		buildReportFileName,
+	);
+
+	if (!fs.existsSync(buildReportFile)) {
+		console.info(`Build report file "${buildReportFile}" not found.`);
 	}
 
-	return JSON.parse(await fsp.readFile(buildReportFile, "utf-8")) as Report;
-}
+	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
 
-async function endRender() {
-	const { buildReportFileName } = await getRenderMetadataFromDB();
-	const { __sites } = await getRenderPathsHydratedWithDomainFromDB();
+	reports.push(buildReport);
 
-	await AuthService.login();
+	return reports;
+}
 
-	let domains = [];
+(async () => {
+	const sendEndSiteRender = async (reports: Array<Report>) => {
+		for (const report of reports) {
+			const sites = report.sites;
 
-	if (GRIDDO_RENDER_BY_DOMAINS) {
-		const [domainName] = process.argv.slice(2);
-		domains.push(domainName);
-	} else {
-		GriddoLog.warn("Legacy Render Mode");
-		domains = await getInstanceDomains();
-	}
+			for (const site of sites) {
+				const { siteId, ...body } = site;
 
-	for (const domain of domains) {
-		const { renderMode, reason } = await getRenderModeFromDB(domain);
+				console.info(`Site (${siteId})`);
 
-		if (renderMode === RENDER_MODE.IDLE) {
-			GriddoLog.info(
-				`(From Current Render) [${domain}]: Skipping end-render as it is marked as IDLE with the reason ${reason}.`,
-			);
-			return;
+				await endSiteRender(siteId, body);
+			}
 		}
+	};
 
-		const report = await getRenderReport({
-			domain,
-			exportsSiteDir: __sites,
-			buildReportFileName,
-		});
-
-		GriddoLog.info(`Sending ending call to ${domain} sites`);
+	if (RENDER_BY_DOMAIN) {
+		await AuthService.login();
+		const domain = process.argv.splice(2)[0];
+		const reports = getDomainRenderReport(domain);
 
-		for (const site of report.sites) {
-			site.publishHashes = [...new Set(site.publishHashes)];
-			site.unpublishHashes = [...new Set(site.unpublishHashes)];
+		infoLog(`Sending ending call to ${domain} sites`);
 
-			const { siteId, ...body } = site;
-			GriddoLog.info(`Site (${siteId})`);
-			await endSiteRender(siteId, body);
+		for (const report of reports) {
+			for (const site of report.sites) {
+				site.publishHashes = [...new Set(site.publishHashes)];
+				site.unpublishHashes = [...new Set(site.unpublishHashes)];
+			}
 		}
-	}
-}
 
-async function main() {
-	await endRender();
-}
+		await sendEndSiteRender(reports);
+	} else {
+		// Legacy render
+		const domains = await getInstanceDomains();
+		for (const domain of domains) {
+			const reports = getDomainRenderReport(domain);
 
-withErrorHandler(main);
+			infoLog(`Sending ending call to ${domain} sites`);
+
+			await sendEndSiteRender(reports);
+		}
+	}
+})().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/commands/move-assets.ts

@@ -0,0 +1,11 @@
+import { execSync } from "node:child_process";
+import path from "node:path";
+
+const domainName = "pro-griddo";
+const domainPath = path.resolve(
+	__dirname,
+	`../../../exports/sites/${domainName}`,
+);
+
+execSync(`mkdir -p ${domainPath}/assets/${domainName}`);
+execSync(`cp -R ${domainPath}/assets/* ${domainPath}/assets/${domainName}`);