@griddo/cx 11.2.4 → 11.2.5-rc.1

package/build/utils/core-utils.d.ts

@@ -69,7 +69,7 @@ declare function sanitizeAPICacheDir(): void;
  * @param functions - Functions to be executed to measure their execution time.
  * @returns A promise that resolves with the total execution time in seconds.
  */
-declare function measureExecutionTime(functions: Array<(...args: Array<unknown>) => unknown | Promise<unknown>>, delayInMS?: number): Promise<number>;
+declare function measureExecutionTime(functions: Array<(...args: Array<unknown>) => unknown | Promise<unknown>>): Promise<number>;
 /**
  * Pause the process until a key is pressed.
  *
@@ -90,11 +90,7 @@ declare function pause(title: string): Promise<void> | undefined;
  * @param options.bypass - Skip the step functions.
  * @returns - A promise that resolves when the life cycle process is completed.
  */
-declare function doLifeCycle(name: LifeCyclesNames, options: {
-    steps: LifeCycleSteps;
-    delay?: number;
-    bypass?: boolean;
-}): Promise<void>;
+declare function doLifeCycle(name: LifeCyclesNames, steps: LifeCycleSteps): Promise<void>;
 /**
  * Creates additional files after the render: sitemaps, robots and a report of
  * the finished render.
@@ -102,7 +98,7 @@ declare function doLifeCycle(name: LifeCyclesNames, options: {
  * @async
  * @param domain
  */
-declare function createRenderMetadata(domain: string): Promise<void>;
+declare function createDomainMetadata(domain: string): Promise<void>;
 declare function removeAllSiteDirsFromStore(): void;
 /**
  * Save render information to a file to use as debug log.
@@ -110,4 +106,4 @@ declare function removeAllSiteDirsFromStore(): void;
  * This information will **not** be sent to any API.
  */
 declare function saveBuildEndLogs(): void;
-export { createRenderMetadata, delay, doLifeCycle, getConfig, getSafeSiteId, instanceRootDir, isTruthy, measureExecutionTime, msToSec, pause, removeAllSiteDirsFromStore, removeProperties, sanitizeAPICacheDir, saveBuildEndLogs, walk, walkStore, };
+export { createDomainMetadata, delay, doLifeCycle, getConfig, getSafeSiteId, instanceRootDir, isTruthy, measureExecutionTime, msToSec, pause, removeAllSiteDirsFromStore, removeProperties, sanitizeAPICacheDir, saveBuildEndLogs, walk, walkStore, };

package/build/utils/create-build-data.d.ts

@@ -4,5 +4,5 @@
  * `getBuildMetadata()` to get build and sites metadata as objects. Both from
  * exporter utils sites dir.
  */
-declare function createBuildData(domain: string): Promise<void>;
-export { createBuildData };
+declare function createDomainRenderData(domain: string): Promise<void>;
+export { createDomainRenderData };

package/build/utils/domains.d.ts

@@ -1,5 +1,13 @@
+import type { Domains } from "../types/global";
 /**
  * Return an array of domains name (string) of the current instance.
  */
 declare function getInstanceDomains(): Promise<string[]>;
-export { getInstanceDomains };
+/**
+ * Return an unique array of domains, filtered without "/".
+ *
+ * @param domains An array of domains
+ * @see Domains
+ */
+declare function getDomainSlugs(domains: Domains): string[];
+export { getInstanceDomains, getDomainSlugs };

package/build/utils/publish.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Spawn a new node process to upload the domain content.
+ * @note This proccess can not access to the custom Griddo `process.env` so it
+ * needs variables passed to it via the `env` prop
+ */
+declare function publishDomain(domain: string): string;
+export { publishDomain };

package/build/utils/render.d.ts

@@ -12,4 +12,4 @@ declare function finishRender(): undefined;
  * Send the default registers through Griddo alerts.
  */
 declare function sendGriddoDefaultAlerts(): void;
-export { createSentinelRenderFile, deleteSentinelRenderFile, finishRender, initRender, isValidRenderProcessOrThrow, sendGriddoDefaultAlerts, };
+export { createSentinelRenderFile, deleteSentinelRenderFile, finishRender, initRender, isValidRenderProcessOrThrow, sendGriddoDefaultAlerts };

package/build/utils/sites.d.ts

@@ -2,7 +2,7 @@ import type { Site, SiteData } from "../types/sites";
 /**
  * Check the instance sites and returns site prepared to be published and unpublished.
  */
-declare function checkSites(domain: string): Promise<{
+declare function getSitesToRender(domain: string): Promise<{
     sitesToPublish: Site[];
     sitesToUnpublish: Site[];
 }>;
@@ -28,4 +28,4 @@ declare function generateBuildReport(): Promise<void>;
  * Generate sitemaps and save them into file system.
  */
 declare function generateSitemaps(): Promise<void>;
-export { checkSites, generateBuildReport, generateSitemaps, getSiteData, unpublishSites, };
+export { generateBuildReport, generateSitemaps, getSiteData, getSitesToRender, unpublishSites, };

package/build/utils/store.d.ts

@@ -59,7 +59,7 @@ declare function savePagesInStore(siteFolderName: string, pages: Array<GriddoPag
  * Remove files from dir.
  * @param filenames - An array of ids representing file page names.
  */
-declare function removePagesFromStore(siteDirName: string, filenames: Array<number>): void;
+declare function removeSitePagesFromStore(siteDirName: string, filenames: Array<number>): void;
 /**
  * Returns pages that need to be created or deleted for a site in the store based on the provided arguments.
  * @param sitePages - Properties for page retrieval.
@@ -69,7 +69,7 @@ declare function removePagesFromStore(siteDirName: string, filenames: Array<numb
  * @param props.changedPages - Array of pages that have been modified in the site.
  * @returns - An object containing the pages to be created and deleted in the site.
  */
-declare function getPagesToCreateOrDelete(sitePages: {
+declare function getSitePagesToCreateOrDelete(sitePages: {
     sitesToPublish: Array<Site>;
     validPagesIds: Array<number>;
     changedPages: Array<number>;
@@ -80,10 +80,35 @@ declare function getPagesToCreateOrDelete(sitePages: {
     pagesToDeleteFromStore: number[];
     pagesToWriteToStore: number[];
 }>;
+/**
+ * Returns pages that need to be created or deleted for a site in the store based on the provided arguments.
+ * @param sitePages - Properties for page retrieval.
+ * @param props.storeDir - Absolute path of the Griddo store.
+ * @param props.pages - Exhaustive array of all pages in the site.
+ * @param props.validPagesIds - Array of valid pages in the site.
+ * @param props.changedPages - Array of pages that have been modified in the site.
+ * @returns - An object containing the pages to be created and deleted in the site.
+ */
+declare function getSitePagesToCreateOrDeleteCached(sitePages: {
+    sitesToPublish: Array<Site>;
+    validPagesIds: Array<number>;
+    changedPages: Array<number>;
+    siteDirName: string;
+    domain: string;
+}): Promise<{
+    pagesInStore: number[];
+    pagesMissingInStore: number[];
+    pagesToDeleteFromStore: number[];
+    pagesToWriteToStore: number[];
+}>;
+/**
+ * Removes sites from store that are not in `sitesToPublish` array.
+ */
+declare function deleteZombieSitesFoldersFromStore(sitesToPublish: Array<Site>): void;
 /**
  * Removes sites that exist in the store that should no longer be there because
  * the domain has been changed. This is necessary because those sites will not
  * be marked as `sitesToUnpublish` so they must be removed manually.
  */
 declare function removeOrphanSites(sitesToPublish: Array<Site>, domain: string): void;
-export { createStoreDir, getBuildMetadata, getBuildPagesFromCachedStore, getBuildPagesFromStore, getBuildPagesPath, getPageInStoreDir, getPagesToCreateOrDelete, removeOrphanSites, removePagesFromStore, savePagesInStore, saveRenderInfoInStore, };
+export { createStoreDir, deleteZombieSitesFoldersFromStore, getBuildMetadata, getBuildPagesFromCachedStore, getBuildPagesFromStore, getBuildPagesPath, getPageInStoreDir, getSitePagesToCreateOrDelete, getSitePagesToCreateOrDeleteCached, removeOrphanSites, removeSitePagesFromStore, savePagesInStore, saveRenderInfoInStore, };

package/exporter/adapters/gatsby/index.ts

@@ -12,14 +12,14 @@ import {
 	runGatsbyBuildCommand,
 } from "./utils";
 import getCxArtifacts from "../../artifacts";
+import { AuthService } from "../../services/auth";
 import {
-	createRenderMetadata,
+	createDomainMetadata,
 	doLifeCycle,
 	getConfig,
 	saveBuildEndLogs,
 } from "../../utils/core-utils";
-import { createBuildData } from "../../utils/create-build-data";
-import { getInstanceDomains } from "../../utils/domains";
+import { createDomainRenderData } from "../../utils/create-build-data";
 import {
 	clearEmptyDirs,
 	copyArtifacts,
@@ -36,132 +36,130 @@ import {
 	isValidRenderProcessOrThrow,
 } from "../../utils/render";
 
-const config = getConfig();
-
 /**
  * Render every instance domain with the Gatsby adapter.
  */
-export async function renderDomainsWithGatsbyAdapter() {
-	const domains = await getInstanceDomains();
-
-	for (const domain of domains) {
-		infoLog(`Initializing render for the domain ${domain}`);
-
-		createSentinelRenderFile();
-
-		const {
-			//
-			__ssg,
-			__exports,
-			__cache,
-			__cx,
-			__components,
-		} = config.paths(domain);
-
-		// Griddo artifacts
-		const cxArtifacts = getCxArtifacts(domain);
-
-		// Gatsby artifacts
-		const gatsbyArtifacts: Artifacts = {
-			disposables: [
-				path.join(__ssg, "public"),
-				path.join(__ssg, "static"),
-				path.join(__ssg, ".cache"),
+export async function renderDomainsWithGatsbyAdapter(domain: string) {
+	await AuthService.login();
+	const config = getConfig();
+
+	infoLog(`Initializing render for the domain ${domain}\n`);
+
+	const {
+		//
+		__ssg,
+		__exports,
+		__cache,
+		__cx,
+		__components,
+	} = config.paths(domain);
+
+	// Griddo artifacts
+	const cxArtifacts = getCxArtifacts(domain);
+
+	// Gatsby artifacts
+	const gatsbyArtifacts: Pick<Artifacts, "disposables" | "cacheables"> = {
+		disposables: [
+			path.join(__ssg, "public"),
+			path.join(__ssg, "static"),
+			path.join(__ssg, ".cache"),
+		],
+		cacheables: [".cache"],
+	};
+
+	// These variables are involved in the build for create the dist directory
+	// from public and pass to the gatsby-build command via spawnSync.
+	const assetPrefix = getGatsbyAssetPrefixWithDomain(domain);
+	const needsAssetPrefix = !!assetPrefix && assetPrefix !== "";
+
+	// Compose artifacts
+	const disposableArtifacts = [
+		...cxArtifacts.disposables,
+		...gatsbyArtifacts.disposables,
+	];
+
+	// LifeCycles
+
+	const allLifeCycles: Array<{
+		name: LifeCyclesNames;
+		steps: LifeCycleSteps;
+	}> = [
+		{
+			name: "Init",
+			steps: [() => createSentinelRenderFile()],
+		},
+
+		{
+			name: "Clean",
+			steps: [() => removeArtifacts(disposableArtifacts)],
+		},
+
+		{
+			name: "Prepare",
+			steps: [() => createArtifacts(cxArtifacts.initials)],
+		},
+
+		{
+			name: "Restore",
+			steps: [
+				() => copyArtifacts(__components, __ssg, ["static"]),
+				() => copyArtifacts(__exports, __cx, cxArtifacts.archivables),
+				() =>
+					renameArtifact(path.join(__cx, "dist"), path.join(__ssg, "public")),
+				() => moveArtifacts(__cache, __cx, cxArtifacts.cacheables),
+				() => moveArtifacts(__cache, __ssg, gatsbyArtifacts.cacheables),
+			],
+		},
+
+		{
+			name: "Data",
+			steps: [() => createDomainRenderData(domain)],
+		},
+
+		{
+			name: "SSG",
+			steps: [() => runGatsbyBuildCommand(assetPrefix)],
+		},
+
+		{
+			name: "Relocation",
+			steps: [() => createDistFromGatsbyPublic(domain, needsAssetPrefix)],
+		},
+
+		{
+			name: "Meta",
+			steps: [() => createDomainMetadata(domain), () => saveBuildEndLogs()],
+		},
+
+		{
+			name: "Archive",
+			steps: [
+				() => removeVirtualPagesFromStore(),
+				() => clearEmptyDirs(path.join(__cx, "dist")),
+				() =>
+					moveArtifacts(__cx, __exports, cxArtifacts.archivables, {
+						withBackup: true,
+					}),
+				() => moveArtifacts(__cx, __cache, cxArtifacts.cacheables),
+				() => moveArtifacts(__ssg, __cache, gatsbyArtifacts.cacheables),
 			],
-			cacheables: [".cache"],
-			initials: [],
-			archivables: [],
-		};
-		// Compose artifacts
-		const disposableArtifacts = [
-			...cxArtifacts.disposables,
-			...gatsbyArtifacts.disposables,
-		];
-
-		// These variables are involved in the build for create the dist directory
-		// from public and pass to the gatsby-build command via spawnSync.
-		const assetPrefix = getGatsbyAssetPrefixWithDomain(domain);
-		const needsAssetPrefix = !!assetPrefix && assetPrefix !== "";
-
-		// LifeCycles
-
-		const allLifeCycles: Array<{
-			name: LifeCyclesNames;
-			steps: LifeCycleSteps;
-		}> = [
-			{
-				name: "Clean",
-				steps: [() => removeArtifacts(disposableArtifacts)],
-			},
-
-			{
-				name: "Prepare",
-				steps: [() => createArtifacts(cxArtifacts.initials)],
-			},
-
-			{
-				name: "Restore",
-				steps: [
-					() => copyArtifacts(__components, __ssg, ["static"]),
-					() => copyArtifacts(__exports, __cx, cxArtifacts.archivables),
-					() =>
-						renameArtifact(path.join(__cx, "dist"), path.join(__ssg, "public")),
-					() => moveArtifacts(__cache, __cx, cxArtifacts.cacheables),
-					() => moveArtifacts(__cache, __ssg, gatsbyArtifacts.cacheables),
-				],
-			},
-
-			{
-				name: "Data",
-				steps: [() => createBuildData(domain)],
-			},
-
-			{
-				name: "SSG",
-				steps: [() => runGatsbyBuildCommand(assetPrefix)],
-			},
-
-			{
-				name: "Relocation",
-				steps: [() => createDistFromGatsbyPublic(domain, needsAssetPrefix)],
-			},
-
-			{
-				name: "Meta",
-				steps: [() => createRenderMetadata(domain), () => saveBuildEndLogs()],
-			},
-
-			{
-				name: "Archive",
-				steps: [
-					() => removeVirtualPagesFromStore(),
-					() => clearEmptyDirs(path.join(__cx, "dist")),
-					() =>
-						moveArtifacts(__cx, __exports, cxArtifacts.archivables, {
-							withBackup: true,
-						}),
-					() => moveArtifacts(__cx, __cache, cxArtifacts.cacheables),
-					() => moveArtifacts(__ssg, __cache, gatsbyArtifacts.cacheables),
-				],
-			},
-
-			{
-				name: "Close",
-				steps: [() => removeArtifacts(disposableArtifacts)],
-			},
-
-			{
-				name: "HealthCheck",
-				steps: [() => isValidRenderProcessOrThrow()],
-			},
-		];
-
-		for (const { name, steps } of allLifeCycles) {
-			await doLifeCycle(name, {
-				steps: steps,
-			});
-		}
-
-		deleteSentinelRenderFile();
+		},
+
+		{
+			name: "Clean",
+			steps: [() => removeArtifacts(disposableArtifacts)],
+		},
+
+		{
+			name: "HealthCheck",
+			steps: [
+				() => isValidRenderProcessOrThrow(),
+				() => deleteSentinelRenderFile(),
+			],
+		},
+	];
+
+	for (const { name, steps } of allLifeCycles) {
+		await doLifeCycle(name, steps);
 	}
 }

package/exporter/build.sh

@@ -29,11 +29,11 @@ esbuild ${log} ./exporter/index.ts ${bundle_node_opts} --outfile=./build/index.j
 esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/index.js
 
 # scripts
-esbuild ${log} ./exporter/scripts/run-start-render.ts ${bundle_node_opts} --outfile=./build/run-start-render.js
-esbuild ${log} ./exporter/scripts/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
-esbuild ${log} ./exporter/scripts/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
-esbuild ${log} ./exporter/scripts/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
-esbuild ${log} ./exporter/scripts/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts} --outfile=./build/end-render.js
+esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts} --outfile=./build/upload-search-content.js
+esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts} --outfile=./build/reset-render.js
+esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts} --outfile=./build/start-render.js
+esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts} --outfile=./build/prepare-domains-render.js
 
 # types
 tsc --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json

package/exporter/commands/end-render.ts

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+
+import fs from "node:fs";
+import path from "node:path";
+
+import pkgDir from "pkg-dir";
+
+import { AuthService } from "../services/auth";
+import { endSiteRender } from "../services/sites";
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import { infoLog } from "../utils/loggin";
+
+const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+
+// Where we are going to find export dirs
+const execBasePath = pkgDir.sync(path.resolve(__dirname, "../.."))!;
+
+// Where we are going to find archived exports
+const exportArchiveBasePath = path.resolve(execBasePath, "exports/sites");
+
+type Report = {
+	authControl:
+		| {
+				Authorization: string;
+				"Cache-Control": string;
+				lang?: string | undefined;
+		  }
+		| undefined;
+	sites: Array<{
+		siteId: number;
+		publishHashes: Array<string>;
+		siteHash: string | null;
+		unpublishHashes: Array<string>;
+		publishPagesIds: Array<number>;
+	}>;
+};
+
+// function getExportedDomains() {
+// 	const domains = fs
+// 		.readdirSync(exportArchiveBasePath)
+// 		// Avoid empty files, usually .DS_Store from local renders.
+// 		.filter((file) => !file.startsWith("."));
+
+// 	return domains;
+// }
+
+function getDomainRenderReport(domain: string) {
+	const reports: Array<Report> = [];
+	const { buildReportFileName } = getConfig();
+
+	const buildReportFile = path.resolve(
+		exportArchiveBasePath,
+		domain,
+		"dist",
+		buildReportFileName,
+	);
+
+	if (!fs.existsSync(buildReportFile)) {
+		console.info(`Build report file "${buildReportFile}" not found.`);
+	}
+
+	const buildReport = JSON.parse(fs.readFileSync(buildReportFile, "utf-8"));
+
+	reports.push(buildReport);
+
+	return reports;
+}
+
+(async () => {
+	const sendEndSiteRender = async (reports: Array<Report>) => {
+		for (const report of reports) {
+			const sites = report.sites;
+
+			for (const site of sites) {
+				const { siteId, ...body } = site;
+
+				console.info(`Site (${siteId})`);
+
+				await endSiteRender(siteId, body);
+			}
+		}
+	};
+
+	if (RENDER_BY_DOMAIN) {
+		await AuthService.login();
+		const domain = process.argv.splice(2)[0];
+		const reports = getDomainRenderReport(domain);
+
+		infoLog(`Sending ending call to ${domain} sites`);
+
+		sendEndSiteRender(reports);
+	} else {
+		// Legacy render
+		const domains = await getInstanceDomains();
+		for (const domain of domains) {
+			const reports = getDomainRenderReport(domain);
+
+			infoLog(`Sending ending call to ${domain} sites`);
+
+			sendEndSiteRender(reports);
+		}
+	}
+})().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/commands/prepare-domains-render.ts

@@ -0,0 +1,89 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import { infoLog, successLog } from "../utils/loggin";
+import { getSitesToRender } from "../utils/sites";
+
+const { GRIDDO_EXPERIMENTAL_DOMAIN_RENDERING_ORDER } = process.env;
+
+(async () => {
+	infoLog("Checking domains size");
+
+	const { __cx } = getConfig().paths();
+
+	// Get domains and calculate order
+	const domains = await getInstanceDomains();
+	const domainsInfo = [];
+	const domainsFilePath = path.join(__cx, "domains.json");
+
+	// Domains order from .env file
+	if (GRIDDO_EXPERIMENTAL_DOMAIN_RENDERING_ORDER) {
+		const domainsOrderArray: Array<string> = JSON.parse(
+			GRIDDO_EXPERIMENTAL_DOMAIN_RENDERING_ORDER,
+		);
+		const validDomains = domainsOrderArray.every((domain) =>
+			domains.includes(domain),
+		);
+		if (validDomains) {
+			fs.writeFileSync(domainsFilePath, JSON.stringify(domainsOrderArray));
+			successLog(`Checking domains size`);
+			return;
+		} else {
+			console.log("Domains in .env are not valid, using the API");
+		}
+	}
+
+	for (const domain of domains) {
+		const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
+
+		if (process.env.GRIDDO_EXPERIMENTAL_SKIP_EMPTY_DOMAINS) {
+			if (sitesToPublish.length === 0 && sitesToUnpublish.length === 0) {
+				infoLog(`Skipping rendering domain ${domain}`);
+
+				// Delete assets/* y dist/*
+				// vanishDomainAssets(domain);
+				continue;
+			}
+		}
+
+		// Only count pages (changedPages) if sitesToPublish is not empty.
+		// Otherwise we set the domain has having 0 pages.
+		const pages = [];
+		if (sitesToPublish.length > 0) {
+			for (const site of sitesToPublish) {
+				pages.push(site.changedPages.length);
+			}
+		} else {
+			pages.push(0);
+		}
+
+		// eslint-disable-next-line no-array-reduce/no-reduce
+		const totalPages = pages.reduce((a, b) => a + b);
+		domainsInfo.push({ domain, totalPages });
+	}
+
+	console.log(domainsInfo);
+	// sort domains, smaller first
+	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
+	const domainSorted = domainsInfo.map(({ domain }) => domain);
+
+	fs.writeFileSync(domainsFilePath, JSON.stringify(domainSorted));
+
+	successLog(`Checking domains size`);
+})().catch((err) => {
+	console.error(err);
+	process.exit(1);
+});
+
+// function vanishDomainAssets(domain: string) {
+// 	const { __exports } = getConfig().paths(domain);
+
+// 	if (fs.existsSync(__exports)) {
+// 		fs.rmSync(__exports, { recursive: true, force: true });
+// 	}
+
+// 	fs.mkdirSync(path.join(__exports, "dist"), { recursive: true });
+// 	fs.mkdirSync(path.join(__exports, "assets"), { recursive: true });
+// }