@griddo/cx 11.7.5 → 11.7.6-rc.0

package/exporter/utils/core-utils.ts

@@ -1,50 +1,16 @@
 import type { APIResponses } from "../types/api";
-import type {
-	CXConfig,
-	LifeCyclesNames,
-	LifeCycleSteps,
-} from "../types/global";
-import type { APIPageObject } from "../types/pages";
-
-import fs from "node:fs";
-import path from "node:path";
-
-import fsx from "fs-extra";
-import kleur from "kleur";
-import { bgRed, black } from "kleur/colors";
-import pkgDir from "pkg-dir";
+import type { LifeCycleAction, LifeCyclesNames } from "../types/global";
 
-import { envs } from "../constants";
-import { throwError } from "../errors";
-import { prependFileSync } from "./folders";
-import { boxLog, infoLog, successLog } from "./loggin";
-import { generateBuildReport, generateSitemaps } from "./sites";
-import { LifecycleExecutionError } from "../errors/errors-data";
-import { RobotsService } from "../services/robots";
-
-const config = getConfig();
+import fsp from "node:fs/promises";
+import path from "node:path";
 
-const instanceRootDir = pkgDir.sync()!; // instance root dir
-
-/**
- * Returns the configuration file content.
- *
- * @example
- * const config = getConfig()
- * const { __cx } = config.paths()
- * const { griddoVersion, proDomain } = config
- */
-function getConfig(): CXConfig {
-	try {
-		const configModule = require("../../cx.config.js");
-		const config = configModule.default;
-
-		return config;
-	} catch (error) {
-		console.log(error);
-		throw new Error("Error while reading configuration file");
-	}
-}
+import { LifecycleExecutionError } from "../constants/errors";
+import { readDB } from "../services/db";
+import { brush } from "./brush";
+import { throwError } from "./errors";
+import { pathExists, prependFileSync } from "./folders";
+import { infoLog, successLog } from "./loggin";
+import { getRenderPathsFromDB } from "./render";
 
 /**
  * Returns true for "true", "on", true and positive numbers.
@@ -90,53 +56,25 @@ function isTruthy(value: any): boolean {
 	return false;
 }
 
-/**
- * Walk a directory and returns the file pathts.
- *
- * @param dir A directory path.
- */
-function walk(dir: string) {
-	const results: Array<string> = [];
-	const list = fs.readdirSync(dir);
-	for (const file of list) {
-		results.push(path.join(dir, file));
-	}
-
-	return results;
-}
-
 /**
  * Walk a directory and returns the JSON file absolute paths with one level of depth.
- * Bypass the `metadata` folder.
  * /abs/.../sotre/<siteId>/jsonfile.json
  * /abs/.../sotre/<siteId>/jsonfile.json
  * /abs/.../sotre/<siteId>/jsonfile.json
  * /abs/.../sotre/<siteId>/jsonfile.json
  */
-function walkStore(dir: string): Array<string> {
-	const results: Array<string> = [];
-
-	// Listamos todas las subcarpetas y evitamos entrar en 'metadata'
-	const subdirs = fs.readdirSync(dir).filter((file) => {
-		const fullPath = path.join(dir, file);
-		return fs.statSync(fullPath).isDirectory() && file !== "metadata";
-	});
-
-	// Iteramos sobre las subcarpetas
-	for (const subdir of subdirs) {
-		const subdirPath = path.join(dir, subdir);
-
-		// Listamos los archivos en cada subcarpeta y filtramos los .json
-		const files = fs
-			.readdirSync(subdirPath)
-			.filter((file) => file.endsWith(".json"));
-
-		for (const file of files) {
-			results.push(path.join(subdirPath, file));
+async function* walkStore(storeDir: string): AsyncGenerator<string> {
+	const rootDirHandle = await fsp.opendir(storeDir);
+	for await (const siteDirent of rootDirHandle) {
+		if (siteDirent.isDirectory()) {
+			const siteDirPath = path.join(storeDir, siteDirent.name);
+			const siteDirHandle = await fsp.opendir(siteDirPath);
+			for await (const jsonFileDirent of siteDirHandle) {
+				const jsonFilePath = path.join(siteDirPath, jsonFileDirent.name);
+				yield jsonFilePath;
+			}
 		}
 	}
-
-	return results;
 }
 
 /**
@@ -187,70 +125,6 @@ function removeProperties(obj: Record<string, unknown>, props: Array<string>) {
 	}
 }
 
-/**
- * Remove unused files (old) inside the `apiCache` dir
- *
- * @todo remove other file types: sites, socials, etc..
- */
-function sanitizeAPICacheDir() {
-	const { __cx } = config.paths();
-	const dirPath = path.join(__cx, "apiCache");
-	const allCachedFiles = fs.readdirSync(dirPath);
-
-	// Object to store the the more rencent file names
-	// Record<string, string> = { "234856872634", "3268746238747238.json"};
-	const filesByIdMap: Record<string, string> = {};
-	//                      ^id     ^path
-
-	// Page files.
-	// We only need files that describes a page object.
-	const pageFilePaths = allCachedFiles.filter((fileName) => {
-		const filePath = `${dirPath}/${fileName}`;
-		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
-		const { id, entity, fullUrl } = fileObject;
-
-		// Is a page file if has id, entity and fullUrl
-		return !!(id && entity && fullUrl);
-	});
-
-	// Fill the filesById object
-	for (const fileName of pageFilePaths) {
-		const filePath = `${dirPath}/${fileName}`;
-		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
-		const fileCreationDate = fs.statSync(filePath).mtimeMs;
-
-		const { id } = fileObject;
-
-		// Is a valid page if doesn't exists in the store object or is newer
-		// that the stored one.
-		const validPageFile =
-			!filesByIdMap[id] ||
-			fileCreationDate > fs.statSync(`${dirPath}/${filesByIdMap[id]}`).mtimeMs;
-
-		if (validPageFile) {
-			filesByIdMap[id] = fileName;
-		}
-	}
-
-	let counter = 0;
-
-	// Delete files using the store object filesById as reference map.
-	for (const fileName of pageFilePaths) {
-		const filePath = `${dirPath}/${fileName}`;
-		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
-
-		const { id } = fileObject;
-
-		// If the filename is not present in the map, remove it!
-		if (fileName !== filesByIdMap[id]) {
-			fs.unlinkSync(filePath);
-			counter++;
-		}
-	}
-
-	console.log(`Sanitize apiCache dir for ${counter} files`);
-}
-
 /**
  * Measures the execution time of a series of sync or async functions.
  *
@@ -258,84 +132,55 @@ function sanitizeAPICacheDir() {
  * @param functions - Functions to be executed to measure their execution time.
  * @returns A promise that resolves with the total execution time in seconds.
  */
-async function measureExecutionTime(
-	functions: Array<(...args: Array<unknown>) => unknown | Promise<unknown>>,
-) {
-	const start = process.hrtime();
-
-	for (const func of functions) {
-		await func();
-	}
-
-	const [seconds, miliseconds] = process.hrtime(start);
-	const timeInSeconds = seconds + miliseconds / 1e9;
-
-	return +timeInSeconds.toFixed(3);
-}
-
-/**
- * Pause the process until a key is pressed.
- *
- * @param title
- * @returns
- */
-function pause(title: string) {
-	const isPauseEnabled = !!envs.GRIDDO_RENDER_BREAKPOINTS_FEATURE;
-
-	if (!isPauseEnabled) {
-		return;
-	}
-
-	return new Promise<void>((resolve) => {
-		console.log("\n");
-		boxLog(`⌛️ ${title}`, "", 1, 0);
-		process.stdin.once("data", () => {
-			resolve();
-		});
-	});
+async function executeAndReturnExecutionTime<T>(func: () => T | Promise<T>) {
+	const start = process.hrtime.bigint();
+	await Promise.resolve(func());
+	const end = process.hrtime.bigint();
+	return Number((Number(end - start) / 1e9).toFixed(3));
 }
 
 /**
- * Executes a life cycle process, which involves executing an array of
- * functions, printing to the console, and handling errors with optional
- * retries.
+ * Executes a life cycle process, which involves executing a callback
+ * function, printing to the console, and handling errors with optional retries.
  *
  * @async
  * @param name - The name of the life cycle.
- * @param options - The arguments object.
- * @param options.steps - An array of functions to execute.
+ * @param action - A callback function to execute.
+ * @param options - Optional configuration for the life cycle process.
+ * @param options.maxRetrys - The maximum number of retries for the life cycle.
+ * @param options.enabled - Whether the life cycle process should be executed.
  * @returns - A promise that resolves when the life cycle process is completed.
  */
-async function doLifeCycle(name: LifeCyclesNames, steps: LifeCycleSteps) {
-	const attemptsByLifeCycleName: Record<LifeCyclesNames, number> = {
-		Init: envs.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS,
-		Archive: envs.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS,
-		Data: envs.GRIDDO_DATA_LIFECYCLE_MAX_ATTEMPTS,
-		Meta: envs.GRIDDO_META_LIFECYCLE_MAX_ATTEMPTS,
-		Relocation: envs.GRIDDO_RELOCATION_LIFECYCLE_MAX_ATTEMPTS,
-		Clean: envs.GRIDDO_CLEAN_LIFECYCLE_MAX_ATTEMPTS,
-		Restore: envs.GRIDDO_RESTORE_LIFECYCLE_MAX_ATTEMPTS,
-		Prepare: envs.GRIDDO_PREPARE_LIFECYCLE_MAX_ATTEMPTS,
-		Close: envs.GRIDDO_CLOSE_LIFECYCLE_MAX_ATTEMPTS,
-		SSG: envs.GRIDDO_SSG_LIFECYCLE_MAX_ATTEMPTS,
-		HealthCheck: 1,
-		__DEBUG__: 1,
-	};
+async function doLifeCycle(
+	name: string,
+	action: LifeCycleAction,
+	options?: { maxRetrys?: number; spinner?: boolean },
+) {
+	const { maxRetrys = 3 } = options || {};
 
 	let trysCount = 0;
-	const maxTrysAccepted = attemptsByLifeCycleName[name] || 1;
 
-	while (trysCount < maxTrysAccepted) {
+	while (trysCount < maxRetrys) {
 		try {
 			infoLog(`start ${name} life-cycle`);
-			const exeTime = await measureExecutionTime(steps);
+
+			// let counter = 0;
+			// process.stdout.write("\x1B[?25l");
+			// const spinnerChars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+			// const interval = setInterval(() => {
+			// 	const spinner = spinnerChars[counter % spinnerChars.length];
+			// 	process.stdout.write(`\r${spinner} ${name} life-cycle...`);
+			// 	counter++;
+			// }, 60);
+
+			const exeTime = await executeAndReturnExecutionTime(action);
+			// clearInterval(interval);
+			// process.stdout.write("\x1B[?25h");
 			successLog(`${name} - ${exeTime}s\n`);
-			// if no errors, print and go out
-			await pause(`${name} LifeCycle`);
 			break;
 		} catch (error) {
-			const errorString = bgRed(black(` Error in ${name} LifeCycle `));
-			const attemptsString = kleur.yellow(`Attempt (${trysCount + 1})`);
+			const errorString = brush.red(` Error in ${name} LifeCycle `);
+			const attemptsString = brush.yellow(`Attempt (${trysCount + 1})`);
 			console.log(`\n${errorString} ${attemptsString}\n`);
 			console.log(error);
 			console.log();
@@ -344,38 +189,23 @@ async function doLifeCycle(name: LifeCyclesNames, steps: LifeCycleSteps) {
 		}
 	}
 
-	if (trysCount === maxTrysAccepted) {
-		throwError(LifecycleExecutionError(maxTrysAccepted, name));
+	if (trysCount === maxRetrys) {
+		throwError(LifecycleExecutionError(maxRetrys, name));
 	}
 }
 
-/**
- * Creates additional files after the render: sitemaps, robots and a report of
- * the finished render.
- *
- * @async
- * @param domain
- */
-async function createDomainMetadata(domain: string) {
-	await generateBuildReport();
-	await generateSitemaps();
-	await RobotsService.writeFiles(domain);
-}
-
-function removeAllSiteDirsFromStore() {
-	const { __cx } = config.paths();
+async function removeAllSiteDirsFromStore() {
+	const { __cx } = await getRenderPathsFromDB();
 	const storeDir = path.join(__cx, "store");
-	const allDirs = fs.readdirSync(storeDir);
+	const allDirs = await fsp.readdir(storeDir);
 
 	const allSiteDirs = allDirs.filter((dirname) => dirname !== "metadata");
 
-	const allSiteDirsFullPath = allSiteDirs.map((name) =>
-		path.join(storeDir, name),
-	);
+	const allSiteDirsFullPath = allSiteDirs.map((name) => path.join(storeDir, name));
 
 	// Remove all site directories except "metadata" directory.
 	for (const site of allSiteDirsFullPath) {
-		fs.rmSync(site, { recursive: true, force: true });
+		await fsp.rm(site, { recursive: true, force: true });
 	}
 }
 
@@ -401,21 +231,21 @@ function getFormattedDateTime() {
  *
  * This information will **not** be sent to any API.
  */
-function saveBuildEndLogs() {
-	const { __cx } = config.paths();
-	const { sitesToPublish, createdPages, buildProcessData } = fsx.readJSONSync(
-		path.join(__cx, "render-metadata.json"),
-	);
+async function saveBuildEndLogs(options: { basePath: string; domain: string }) {
+	const { basePath, domain } = options;
+
+	const db = await readDB();
+	const { sitesToPublish, createdPages, buildProcessData } = db.domains[domain].renderInfo || {};
 
 	const dateString = getFormattedDateTime();
-	const buildEndLogsDir = path.join(__cx, "debug", "build-end");
+	const buildEndLogsDir = path.join(basePath, "debug", "build-end");
 
-	if (!fs.existsSync(buildEndLogsDir)) {
-		fs.mkdirSync(buildEndLogsDir, { recursive: true });
+	if (!(await pathExists(buildEndLogsDir))) {
+		await fsp.mkdir(buildEndLogsDir, { recursive: true });
 	}
 
 	// Log
-	fs.writeFileSync(
+	await fsp.writeFile(
 		path.join(buildEndLogsDir, `${dateString}.json`),
 		JSON.stringify(
 			{
@@ -432,27 +262,36 @@ function saveBuildEndLogs() {
 	);
 
 	// Update index
-	prependFileSync(
+	await prependFileSync(
 		path.join(buildEndLogsDir, "index.html"),
 		`<pre><a href="${dateString}.json">${dateString}</a></pre>\n`,
 	);
 }
 
+const lifeCycleNames: Record<LifeCyclesNames, LifeCyclesNames> = {
+	Sync: "Sync",
+	Clean: "Clean",
+	Close: "Close",
+	Data: "Data",
+	HealthCheck: "HealthCheck",
+	Init: "Init",
+	Meta: "Meta",
+	Prepare: "Prepare",
+	Relocation: "Relocation",
+	Restore: "Restore",
+	SSG: "SSG",
+} as const;
+
 export {
-	createDomainMetadata,
 	delay,
 	doLifeCycle,
-	getConfig,
+	executeAndReturnExecutionTime,
 	getSafeSiteId,
-	instanceRootDir,
 	isTruthy,
-	measureExecutionTime,
+	lifeCycleNames,
 	msToSec,
-	pause,
 	removeAllSiteDirsFromStore,
 	removeProperties,
-	sanitizeAPICacheDir,
 	saveBuildEndLogs,
-	walk,
 	walkStore,
 };

package/exporter/utils/domains.ts

@@ -1,17 +1,20 @@
 import type { Domains } from "../types/global";
 
+import { DOMAINS } from "../constants/endpoints";
+import { NoDomainsFoundError } from "../constants/errors";
+import { get } from "./api";
+import { throwError } from "./errors";
 import { verboseLog } from "./loggin";
-import { throwError } from "../errors";
-import { NoDomainsFoundError } from "../errors/errors-data";
-import { AuthService } from "../services/auth";
-import { getAllDomains } from "../services/domains";
 
 /**
  * Return an array of domains name (string) of the current instance.
  */
 async function getInstanceDomains() {
-	await AuthService.login();
-	const domains = await getAllDomains();
+	// await AuthService.login();
+	const domains = await get<Domains>({
+		endpoint: DOMAINS,
+		useApiCacheDir: false,
+	});
 
 	if (!domains.length) {
 		throwError(NoDomainsFoundError);
@@ -36,4 +39,4 @@ function getDomainSlugs(domains: Domains) {
 	return [...new Set(filteredDomains)];
 }
 
-export { getInstanceDomains, getDomainSlugs };
+export { getDomainSlugs, getInstanceDomains };

package/exporter/{errors/index.ts → utils/errors.ts}

@@ -1,6 +1,6 @@
-import type { ErrorsType } from "./errors-data";
+import type { ErrorsType } from "../constants/errors";
 
-import kleur from "kleur";
+import { brush } from "./brush";
 
 export type ErrorData = {
 	error: ErrorsType;
@@ -10,20 +10,21 @@ export type ErrorData = {
 };
 
 export class RenderError extends Error {
-	constructor() {
-		super();
+	constructor(originalError?: unknown) {
+		super(originalError instanceof Error ? originalError.message : String(originalError));
+
 		this.name = "InternalCXError";
-		this.stack = "";
+		this.stack = originalError instanceof Error ? originalError.stack : "";
 	}
 }
 
 /**
  * Throws an error with the provided error message, expected value, and hint.
  */
-function throwError(options: ErrorData, stack?: unknown) {
+function throwError(options: ErrorData, stack?: unknown): never {
 	const { error, message, expected, hint } = options;
 
-	const errorColor = kleur.red("[ " + error + " ]");
+	const errorColor = console.log(brush.red(`[ ${error} ]`));
 	const extraText = [expected, hint].filter(Boolean).join("\n");
 
 	console.log(`
@@ -31,10 +32,10 @@ ${errorColor}
 ${message}
 ${extraText}
 
-${kleur.red("stack")}
+${brush.red("stack")}
 ${JSON.stringify(stack, null, 2)}`);
 
-	throw new RenderError();
+	throw new RenderError(stack);
 }
 
 export { throwError };