@griddo/cx 11.7.6-rc.1 → 11.7.6

package/exporter/utils/core-utils.ts

@@ -1,16 +1,50 @@
 import type { APIResponses } from "../types/api";
-import type { LifeCycleAction, LifeCyclesNames } from "../types/global";
-
-import fsp from "node:fs/promises";
+import type {
+	CXConfig,
+	LifeCyclesNames,
+	LifeCycleSteps,
+} from "../types/global";
+import type { APIPageObject } from "../types/pages";
+
+import fs from "node:fs";
 import path from "node:path";
 
-import { LifecycleExecutionError } from "../constants/errors";
-import { readDB } from "../services/db";
-import { brush } from "./brush";
-import { throwError } from "./errors";
-import { pathExists, prependFileSync } from "./folders";
-import { infoLog, successLog } from "./loggin";
-import { getRenderPathsFromDB } from "./render";
+import fsx from "fs-extra";
+import kleur from "kleur";
+import { bgRed, black } from "kleur/colors";
+import pkgDir from "pkg-dir";
+
+import { envs } from "../constants";
+import { throwError } from "../errors";
+import { prependFileSync } from "./folders";
+import { boxLog, infoLog, successLog } from "./loggin";
+import { generateBuildReport, generateSitemaps } from "./sites";
+import { LifecycleExecutionError } from "../errors/errors-data";
+import { RobotsService } from "../services/robots";
+
+const config = getConfig();
+
+const instanceRootDir = pkgDir.sync()!; // instance root dir
+
+/**
+ * Returns the configuration file content.
+ *
+ * @example
+ * const config = getConfig()
+ * const { __cx } = config.paths()
+ * const { griddoVersion, proDomain } = config
+ */
+function getConfig(): CXConfig {
+	try {
+		const configModule = require("../../cx.config.js");
+		const config = configModule.default;
+
+		return config;
+	} catch (error) {
+		console.log(error);
+		throw new Error("Error while reading configuration file");
+	}
+}
 
 /**
  * Returns true for "true", "on", true and positive numbers.
@@ -56,25 +90,53 @@ function isTruthy(value: any): boolean {
 	return false;
 }
 
+/**
+ * Walk a directory and returns the file pathts.
+ *
+ * @param dir A directory path.
+ */
+function walk(dir: string) {
+	const results: Array<string> = [];
+	const list = fs.readdirSync(dir);
+	for (const file of list) {
+		results.push(path.join(dir, file));
+	}
+
+	return results;
+}
+
 /**
  * Walk a directory and returns the JSON file absolute paths with one level of depth.
+ * Bypass the `metadata` folder.
  * /abs/.../sotre/<siteId>/jsonfile.json
  * /abs/.../sotre/<siteId>/jsonfile.json
  * /abs/.../sotre/<siteId>/jsonfile.json
  * /abs/.../sotre/<siteId>/jsonfile.json
  */
-async function* walkStore(storeDir: string): AsyncGenerator<string> {
-	const rootDirHandle = await fsp.opendir(storeDir);
-	for await (const siteDirent of rootDirHandle) {
-		if (siteDirent.isDirectory()) {
-			const siteDirPath = path.join(storeDir, siteDirent.name);
-			const siteDirHandle = await fsp.opendir(siteDirPath);
-			for await (const jsonFileDirent of siteDirHandle) {
-				const jsonFilePath = path.join(siteDirPath, jsonFileDirent.name);
-				yield jsonFilePath;
-			}
+function walkStore(dir: string): Array<string> {
+	const results: Array<string> = [];
+
+	// Listamos todas las subcarpetas y evitamos entrar en 'metadata'
+	const subdirs = fs.readdirSync(dir).filter((file) => {
+		const fullPath = path.join(dir, file);
+		return fs.statSync(fullPath).isDirectory() && file !== "metadata";
+	});
+
+	// Iteramos sobre las subcarpetas
+	for (const subdir of subdirs) {
+		const subdirPath = path.join(dir, subdir);
+
+		// Listamos los archivos en cada subcarpeta y filtramos los .json
+		const files = fs
+			.readdirSync(subdirPath)
+			.filter((file) => file.endsWith(".json"));
+
+		for (const file of files) {
+			results.push(path.join(subdirPath, file));
 		}
 	}
+
+	return results;
 }
 
 /**
@@ -125,6 +187,70 @@ function removeProperties(obj: Record<string, unknown>, props: Array<string>) {
 	}
 }
 
+/**
+ * Remove unused files (old) inside the `apiCache` dir
+ *
+ * @todo remove other file types: sites, socials, etc..
+ */
+function sanitizeAPICacheDir() {
+	const { __cx } = config.paths();
+	const dirPath = path.join(__cx, "apiCache");
+	const allCachedFiles = fs.readdirSync(dirPath);
+
+	// Object to store the the more rencent file names
+	// Record<string, string> = { "234856872634", "3268746238747238.json"};
+	const filesByIdMap: Record<string, string> = {};
+	//                      ^id     ^path
+
+	// Page files.
+	// We only need files that describes a page object.
+	const pageFilePaths = allCachedFiles.filter((fileName) => {
+		const filePath = `${dirPath}/${fileName}`;
+		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
+		const { id, entity, fullUrl } = fileObject;
+
+		// Is a page file if has id, entity and fullUrl
+		return !!(id && entity && fullUrl);
+	});
+
+	// Fill the filesById object
+	for (const fileName of pageFilePaths) {
+		const filePath = `${dirPath}/${fileName}`;
+		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
+		const fileCreationDate = fs.statSync(filePath).mtimeMs;
+
+		const { id } = fileObject;
+
+		// Is a valid page if doesn't exists in the store object or is newer
+		// that the stored one.
+		const validPageFile =
+			!filesByIdMap[id] ||
+			fileCreationDate > fs.statSync(`${dirPath}/${filesByIdMap[id]}`).mtimeMs;
+
+		if (validPageFile) {
+			filesByIdMap[id] = fileName;
+		}
+	}
+
+	let counter = 0;
+
+	// Delete files using the store object filesById as reference map.
+	for (const fileName of pageFilePaths) {
+		const filePath = `${dirPath}/${fileName}`;
+		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
+
+		const { id } = fileObject;
+
+		// If the filename is not present in the map, remove it!
+		if (fileName !== filesByIdMap[id]) {
+			fs.unlinkSync(filePath);
+			counter++;
+		}
+	}
+
+	console.log(`Sanitize apiCache dir for ${counter} files`);
+}
+
 /**
  * Measures the execution time of a series of sync or async functions.
  *
@@ -132,55 +258,84 @@ function removeProperties(obj: Record<string, unknown>, props: Array<string>) {
  * @param functions - Functions to be executed to measure their execution time.
  * @returns A promise that resolves with the total execution time in seconds.
  */
-async function executeAndReturnExecutionTime<T>(func: () => T | Promise<T>) {
-	const start = process.hrtime.bigint();
-	await Promise.resolve(func());
-	const end = process.hrtime.bigint();
-	return Number((Number(end - start) / 1e9).toFixed(3));
+async function measureExecutionTime(
+	functions: Array<(...args: Array<unknown>) => unknown | Promise<unknown>>,
+) {
+	const start = process.hrtime();
+
+	for (const func of functions) {
+		await func();
+	}
+
+	const [seconds, miliseconds] = process.hrtime(start);
+	const timeInSeconds = seconds + miliseconds / 1e9;
+
+	return +timeInSeconds.toFixed(3);
+}
+
+/**
+ * Pause the process until a key is pressed.
+ *
+ * @param title
+ * @returns
+ */
+function pause(title: string) {
+	const isPauseEnabled = !!envs.GRIDDO_RENDER_BREAKPOINTS_FEATURE;
+
+	if (!isPauseEnabled) {
+		return;
+	}
+
+	return new Promise<void>((resolve) => {
+		console.log("\n");
+		boxLog(`⌛️ ${title}`, "", 1, 0);
+		process.stdin.once("data", () => {
+			resolve();
+		});
+	});
 }
 
 /**
- * Executes a life cycle process, which involves executing a callback
- * function, printing to the console, and handling errors with optional retries.
+ * Executes a life cycle process, which involves executing an array of
+ * functions, printing to the console, and handling errors with optional
+ * retries.
  *
  * @async
  * @param name - The name of the life cycle.
- * @param action - A callback function to execute.
- * @param options - Optional configuration for the life cycle process.
- * @param options.maxRetrys - The maximum number of retries for the life cycle.
- * @param options.enabled - Whether the life cycle process should be executed.
+ * @param options - The arguments object.
+ * @param options.steps - An array of functions to execute.
  * @returns - A promise that resolves when the life cycle process is completed.
  */
-async function doLifeCycle(
-	name: string,
-	action: LifeCycleAction,
-	options?: { maxRetrys?: number; spinner?: boolean },
-) {
-	const { maxRetrys = 3 } = options || {};
+async function doLifeCycle(name: LifeCyclesNames, steps: LifeCycleSteps) {
+	const attemptsByLifeCycleName: Record<LifeCyclesNames, number> = {
+		Init: envs.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS,
+		Archive: envs.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS,
+		Data: envs.GRIDDO_DATA_LIFECYCLE_MAX_ATTEMPTS,
+		Meta: envs.GRIDDO_META_LIFECYCLE_MAX_ATTEMPTS,
+		Relocation: envs.GRIDDO_RELOCATION_LIFECYCLE_MAX_ATTEMPTS,
+		Clean: envs.GRIDDO_CLEAN_LIFECYCLE_MAX_ATTEMPTS,
+		Restore: envs.GRIDDO_RESTORE_LIFECYCLE_MAX_ATTEMPTS,
+		Prepare: envs.GRIDDO_PREPARE_LIFECYCLE_MAX_ATTEMPTS,
+		Close: envs.GRIDDO_CLOSE_LIFECYCLE_MAX_ATTEMPTS,
+		SSG: envs.GRIDDO_SSG_LIFECYCLE_MAX_ATTEMPTS,
+		HealthCheck: 1,
+		__DEBUG__: 1,
+	};
 
 	let trysCount = 0;
+	const maxTrysAccepted = attemptsByLifeCycleName[name] || 1;
 
-	while (trysCount < maxRetrys) {
+	while (trysCount < maxTrysAccepted) {
 		try {
 			infoLog(`start ${name} life-cycle`);
-
-			// let counter = 0;
-			// process.stdout.write("\x1B[?25l");
-			// const spinnerChars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
-			// const interval = setInterval(() => {
-			// 	const spinner = spinnerChars[counter % spinnerChars.length];
-			// 	process.stdout.write(`\r${spinner} ${name} life-cycle...`);
-			// 	counter++;
-			// }, 60);
-
-			const exeTime = await executeAndReturnExecutionTime(action);
-			// clearInterval(interval);
-			// process.stdout.write("\x1B[?25h");
+			const exeTime = await measureExecutionTime(steps);
 			successLog(`${name} - ${exeTime}s\n`);
+			// if no errors, print and go out
+			await pause(`${name} LifeCycle`);
 			break;
 		} catch (error) {
-			const errorString = brush.red(` Error in ${name} LifeCycle `);
-			const attemptsString = brush.yellow(`Attempt (${trysCount + 1})`);
+			const errorString = bgRed(black(` Error in ${name} LifeCycle `));
+			const attemptsString = kleur.yellow(`Attempt (${trysCount + 1})`);
 			console.log(`\n${errorString} ${attemptsString}\n`);
 			console.log(error);
 			console.log();
@@ -189,23 +344,38 @@ async function doLifeCycle(
 		}
 	}
 
-	if (trysCount === maxRetrys) {
-		throwError(LifecycleExecutionError(maxRetrys, name));
+	if (trysCount === maxTrysAccepted) {
+		throwError(LifecycleExecutionError(maxTrysAccepted, name));
 	}
 }
 
-async function removeAllSiteDirsFromStore() {
-	const { __cx } = await getRenderPathsFromDB();
+/**
+ * Creates additional files after the render: sitemaps, robots and a report of
+ * the finished render.
+ *
+ * @async
+ * @param domain
+ */
+async function createDomainMetadata(domain: string) {
+	await generateBuildReport();
+	await generateSitemaps();
+	await RobotsService.writeFiles(domain);
+}
+
+function removeAllSiteDirsFromStore() {
+	const { __cx } = config.paths();
 	const storeDir = path.join(__cx, "store");
-	const allDirs = await fsp.readdir(storeDir);
+	const allDirs = fs.readdirSync(storeDir);
 
 	const allSiteDirs = allDirs.filter((dirname) => dirname !== "metadata");
 
-	const allSiteDirsFullPath = allSiteDirs.map((name) => path.join(storeDir, name));
+	const allSiteDirsFullPath = allSiteDirs.map((name) =>
+		path.join(storeDir, name),
+	);
 
 	// Remove all site directories except "metadata" directory.
 	for (const site of allSiteDirsFullPath) {
-		await fsp.rm(site, { recursive: true, force: true });
+		fs.rmSync(site, { recursive: true, force: true });
 	}
 }
 
@@ -231,21 +401,21 @@ function getFormattedDateTime() {
  *
  * This information will **not** be sent to any API.
  */
-async function saveBuildEndLogs(options: { basePath: string; domain: string }) {
-	const { basePath, domain } = options;
-
-	const db = await readDB();
-	const { sitesToPublish, createdPages, buildProcessData } = db.domains[domain].renderInfo || {};
+function saveBuildEndLogs() {
+	const { __cx } = config.paths();
+	const { sitesToPublish, createdPages, buildProcessData } = fsx.readJSONSync(
+		path.join(__cx, "render-metadata.json"),
+	);
 
 	const dateString = getFormattedDateTime();
-	const buildEndLogsDir = path.join(basePath, "debug", "build-end");
+	const buildEndLogsDir = path.join(__cx, "debug", "build-end");
 
-	if (!(await pathExists(buildEndLogsDir))) {
-		await fsp.mkdir(buildEndLogsDir, { recursive: true });
+	if (!fs.existsSync(buildEndLogsDir)) {
+		fs.mkdirSync(buildEndLogsDir, { recursive: true });
 	}
 
 	// Log
-	await fsp.writeFile(
+	fs.writeFileSync(
 		path.join(buildEndLogsDir, `${dateString}.json`),
 		JSON.stringify(
 			{
@@ -262,36 +432,27 @@ async function saveBuildEndLogs(options: { basePath: string; domain: string }) {
 	);
 
 	// Update index
-	await prependFileSync(
+	prependFileSync(
 		path.join(buildEndLogsDir, "index.html"),
 		`<pre><a href="${dateString}.json">${dateString}</a></pre>\n`,
 	);
 }
 
-const lifeCycleNames: Record<LifeCyclesNames, LifeCyclesNames> = {
-	Sync: "Sync",
-	Clean: "Clean",
-	Close: "Close",
-	Data: "Data",
-	HealthCheck: "HealthCheck",
-	Init: "Init",
-	Meta: "Meta",
-	Prepare: "Prepare",
-	Relocation: "Relocation",
-	Restore: "Restore",
-	SSG: "SSG",
-} as const;
-
 export {
+	createDomainMetadata,
 	delay,
 	doLifeCycle,
-	executeAndReturnExecutionTime,
+	getConfig,
 	getSafeSiteId,
+	instanceRootDir,
 	isTruthy,
-	lifeCycleNames,
+	measureExecutionTime,
 	msToSec,
+	pause,
 	removeAllSiteDirsFromStore,
 	removeProperties,
+	sanitizeAPICacheDir,
 	saveBuildEndLogs,
+	walk,
 	walkStore,
 };

package/exporter/utils/create-build-data.ts

@@ -0,0 +1,17 @@
+import { createAPICacheDir } from "./cache";
+import { sanitizeAPICacheDir } from "./core-utils";
+import { createStore } from "../services/store";
+
+/**
+ * Download all data: sites, pages etc.. from the instance private Griddo API.
+ * Then you can use the generator funcion `getBuildPagesFromStore()` to get the pages and
+ * `getBuildMetadata()` to get build and sites metadata as objects. Both from
+ * exporter utils sites dir.
+ */
+async function createDomainRenderData(domain: string) {
+	createAPICacheDir();
+	await createStore(domain);
+	sanitizeAPICacheDir();
+}
+
+export { createDomainRenderData };

package/exporter/utils/domains.ts

@@ -1,20 +1,17 @@
 import type { Domains } from "../types/global";
 
-import { DOMAINS } from "../constants/endpoints";
-import { NoDomainsFoundError } from "../constants/errors";
-import { get } from "./api";
-import { throwError } from "./errors";
 import { verboseLog } from "./loggin";
+import { throwError } from "../errors";
+import { NoDomainsFoundError } from "../errors/errors-data";
+import { AuthService } from "../services/auth";
+import { getAllDomains } from "../services/domains";
 
 /**
  * Return an array of domains name (string) of the current instance.
  */
 async function getInstanceDomains() {
-	// await AuthService.login();
-	const domains = await get<Domains>({
-		endpoint: DOMAINS,
-		useApiCacheDir: false,
-	});
+	await AuthService.login();
+	const domains = await getAllDomains();
 
 	if (!domains.length) {
 		throwError(NoDomainsFoundError);
@@ -39,4 +36,4 @@ function getDomainSlugs(domains: Domains) {
 	return [...new Set(filteredDomains)];
 }
 
-export { getDomainSlugs, getInstanceDomains };
+export { getInstanceDomains, getDomainSlugs };