@griddo/cx 11.9.12-rc.0 → 11.9.13

package/exporter/utils/cache.ts

@@ -0,0 +1,142 @@
+import type { Petition } from "../types/global";
+import type { HashSites, SiteHash } from "../types/sites";
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import fsx from "fs-extra";
+
+import { getConfig } from "./core-utils";
+
+const config = getConfig();
+
+/**
+ * Creates an `apiCache` dir to store pages fetched from them API.
+ */
+function createAPICacheDir() {
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
+
+	if (!fs.existsSync(apiCacheDir)) {
+		fs.mkdirSync(apiCacheDir);
+	}
+
+	console.info("Cache initialized");
+}
+
+/**
+ * Generate a filename with a hash using a Petition object
+ *
+ * @todo Merge with createSha256
+ * @param petition An object
+ */
+function generateFilenameWithHash(petition: Petition) {
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
+
+	const hashSum = crypto.createHash("sha256");
+	hashSum.update(JSON.stringify(petition));
+
+	return `${apiCacheDir}/${hashSum.digest("hex")}`;
+}
+
+/**
+ * Generate a filename with a hash using a string.
+ *
+ * @todo Merge with generateFilenameWithHash
+ * @param data A string to create a sha256 based on.
+ */
+function createSha256(data: string) {
+	const uniqueString = crypto.randomBytes(16).toString("hex");
+	const hash = crypto.createHash("sha256");
+	hash.update(`${data}${uniqueString}`);
+
+	return hash.digest("hex");
+}
+
+/**
+ * Save a file using a hash name.
+ *
+ * @param petition An object.
+ * @param content Content to be saved.
+ */
+function saveCache<T>(petition: Petition, content: T) {
+	const stringContent =
+		typeof content === "string" ? content : JSON.stringify(content);
+	const filename = generateFilenameWithHash(petition);
+	const filepath = path.dirname(filename);
+	if (!fs.existsSync(filepath)) {
+		fs.mkdirSync(filepath, { recursive: true });
+	}
+	fs.writeFileSync(filename, stringContent, "utf8");
+}
+
+/**
+ * Search in the `apiCache` dir for a file using the petition as hash generator.
+ * Return the file content if found or null if not.
+ *
+ * @param petition An object
+ */
+function searchCacheData<T>(petition: Petition) {
+	try {
+		const file = generateFilenameWithHash(petition);
+		const jsonData = fsx.readJSONSync(file, {
+			encoding: "utf-8",
+		});
+		return jsonData as T;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Get site hashes from the file as an object
+ */
+function getHashSites(): HashSites {
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
+	const siteHasFilename = `${apiCacheDir}/siteHash.json`;
+
+	try {
+		const jsonData = fsx.readJSONSync(siteHasFilename, {
+			encoding: "utf-8",
+		});
+		return jsonData || {};
+	} catch {
+		return {};
+	}
+}
+
+/**
+ * Update (write) the site hash file.
+ *
+ * @param siteId The id of the site.
+ * @param siteHash The has of the site.
+ */
+function updatedSiteHash(siteId: number, siteHash: SiteHash) {
+	const allHash = getHashSites();
+	const lastHash = allHash[siteId];
+	const currentHash = siteHash || lastHash || new Date().valueOf();
+
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
+	const siteHasFilename = `${apiCacheDir}/siteHash.json`;
+
+	if (currentHash !== lastHash) {
+		allHash[siteId] = currentHash;
+		fs.writeFileSync(siteHasFilename, JSON.stringify(allHash), {
+			encoding: "utf-8",
+		});
+	}
+
+	return currentHash.toString();
+}
+
+export {
+	createAPICacheDir,
+	createSha256,
+	saveCache,
+	searchCacheData,
+	updatedSiteHash,
+};

package/exporter/utils/core-utils.ts

@@ -0,0 +1,458 @@
+import type { APIResponses } from "../types/api";
+import type {
+	CXConfig,
+	LifeCyclesNames,
+	LifeCycleSteps,
+} from "../types/global";
+import type { APIPageObject } from "../types/pages";
+
+import fs from "node:fs";
+import path from "node:path";
+
+import fsx from "fs-extra";
+import kleur from "kleur";
+import { bgRed, black } from "kleur/colors";
+import pkgDir from "pkg-dir";
+
+import { envs } from "../constants";
+import { throwError } from "../errors";
+import { prependFileSync } from "./folders";
+import { boxLog, infoLog, successLog } from "./loggin";
+import { generateBuildReport, generateSitemaps } from "./sites";
+import { LifecycleExecutionError } from "../errors/errors-data";
+import { RobotsService } from "../services/robots";
+
+const config = getConfig();
+
+const instanceRootDir = pkgDir.sync()!; // instance root dir
+
+/**
+ * Returns the configuration file content.
+ *
+ * @example
+ * const config = getConfig()
+ * const { __cx } = config.paths()
+ * const { griddoVersion, proDomain } = config
+ */
+function getConfig(): CXConfig {
+	try {
+		const configModule = require("../../cx.config.js");
+		const config = configModule.default;
+
+		return config;
+	} catch (error) {
+		console.log(error);
+		throw new Error("Error while reading configuration file");
+	}
+}
+
+/**
+ * Returns true for "true", "on", true and positive numbers.
+ * Returns false for "false", "off", false, 0, negative integers and anything else.
+ */
+function isTruthy(value: any): boolean {
+	// Return if Boolean
+	if (typeof value === "boolean") {
+		return value;
+	}
+
+	// Return false if null or undefined
+	if (value === undefined || value === null) {
+		return false;
+	}
+
+	// If the String is true or false
+	if (value.toLowerCase() === "true") {
+		return true;
+	}
+	if (value.toLowerCase() === "false") {
+		return false;
+	}
+
+	// "on" | "off", We love them in Griddo ^^.
+	if (value.toLowerCase() === "on") {
+		return true;
+	}
+	if (value.toLowerCase() === "off") {
+		return false;
+	}
+
+	// Now check if it's a number
+	const number = Number.parseInt(value, 10);
+	if (Number.isNaN(number)) {
+		return false;
+	}
+	if (number > 0) {
+		return true;
+	}
+
+	// Default to false
+	return false;
+}
+
+/**
+ * Walk a directory and returns the file pathts.
+ *
+ * @param dir A directory path.
+ */
+function walk(dir: string) {
+	const results: Array<string> = [];
+	const list = fs.readdirSync(dir);
+	for (const file of list) {
+		results.push(path.join(dir, file));
+	}
+
+	return results;
+}
+
+/**
+ * Walk a directory and returns the JSON file absolute paths with one level of depth.
+ * Bypass the `metadata` folder.
+ * /abs/.../sotre/<siteId>/jsonfile.json
+ * /abs/.../sotre/<siteId>/jsonfile.json
+ * /abs/.../sotre/<siteId>/jsonfile.json
+ * /abs/.../sotre/<siteId>/jsonfile.json
+ */
+function walkStore(dir: string): Array<string> {
+	const results: Array<string> = [];
+
+	// Listamos todas las subcarpetas y evitamos entrar en 'metadata'
+	const subdirs = fs.readdirSync(dir).filter((file) => {
+		const fullPath = path.join(dir, file);
+		return fs.statSync(fullPath).isDirectory() && file !== "metadata";
+	});
+
+	// Iteramos sobre las subcarpetas
+	for (const subdir of subdirs) {
+		const subdirPath = path.join(dir, subdir);
+
+		// Listamos los archivos en cada subcarpeta y filtramos los .json
+		const files = fs
+			.readdirSync(subdirPath)
+			.filter((file) => file.endsWith(".json"));
+
+		for (const file of files) {
+			results.push(path.join(subdirPath, file));
+		}
+	}
+
+	return results;
+}
+
+/**
+ * Custom delay using the "promise hack",
+ *
+ * @param ms Amount of miliseconds to be delayed
+ */
+function delay(ms: number) {
+	return new Promise((res) => setTimeout(res, ms));
+}
+
+/**
+ * Converts milliseconds to seconds with a fixed number of decimals.
+ *
+ * @param ms The number in milliseconds.
+ * @param fixed The amount of fixed decimals.
+ * @returns The converted number in seconds with the fixed number of decimals.
+ */
+function msToSec(ms: number, decimals = 3) {
+	return (ms / 1000).toFixed(decimals);
+}
+
+/**
+ * Return a siteID from a response object if exist
+ * @param response A response object
+ */
+function getSafeSiteId(response: APIResponses) {
+	if (typeof response === "string") {
+		return undefined;
+	}
+
+	return "site" in response && response.site ? response?.site : undefined;
+}
+
+/**
+ * Remove props from an object
+ *
+ * @param obj The object
+ * @param props An array of props to be removed
+ */
+function removeProperties(obj: Record<string, unknown>, props: Array<string>) {
+	for (const key in obj) {
+		if (props.includes(key)) {
+			delete obj[key];
+		} else if (typeof obj[key] === "object") {
+			removeProperties(obj[key] as Record<string, unknown>, props);
+		}
+	}
+}
+
+/**
+ * Remove unused files (old) inside the `apiCache` dir
+ *
+ * @todo remove other file types: sites, socials, etc..
+ */
+function sanitizeAPICacheDir() {
+	const { __cx } = config.paths();
+	const dirPath = path.join(__cx, "apiCache");
+	const allCachedFiles = fs.readdirSync(dirPath);
+
+	// Object to store the the more rencent file names
+	// Record<string, string> = { "234856872634", "3268746238747238.json"};
+	const filesByIdMap: Record<string, string> = {};
+	//                      ^id     ^path
+
+	// Page files.
+	// We only need files that describes a page object.
+	const pageFilePaths = allCachedFiles.filter((fileName) => {
+		const filePath = `${dirPath}/${fileName}`;
+		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
+		const { id, entity, fullUrl } = fileObject;
+
+		// Is a page file if has id, entity and fullUrl
+		return !!(id && entity && fullUrl);
+	});
+
+	// Fill the filesById object
+	for (const fileName of pageFilePaths) {
+		const filePath = `${dirPath}/${fileName}`;
+		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
+		const fileCreationDate = fs.statSync(filePath).mtimeMs;
+
+		const { id } = fileObject;
+
+		// Is a valid page if doesn't exists in the store object or is newer
+		// that the stored one.
+		const validPageFile =
+			!filesByIdMap[id] ||
+			fileCreationDate > fs.statSync(`${dirPath}/${filesByIdMap[id]}`).mtimeMs;
+
+		if (validPageFile) {
+			filesByIdMap[id] = fileName;
+		}
+	}
+
+	let counter = 0;
+
+	// Delete files using the store object filesById as reference map.
+	for (const fileName of pageFilePaths) {
+		const filePath = `${dirPath}/${fileName}`;
+		const fileObject = fsx.readJSONSync(filePath, "utf-8") as APIPageObject;
+
+		const { id } = fileObject;
+
+		// If the filename is not present in the map, remove it!
+		if (fileName !== filesByIdMap[id]) {
+			fs.unlinkSync(filePath);
+			counter++;
+		}
+	}
+
+	console.log(`Sanitize apiCache dir for ${counter} files`);
+}
+
+/**
+ * Measures the execution time of a series of sync or async functions.
+ *
+ * @async
+ * @param functions - Functions to be executed to measure their execution time.
+ * @returns A promise that resolves with the total execution time in seconds.
+ */
+async function measureExecutionTime(
+	functions: Array<(...args: Array<unknown>) => unknown | Promise<unknown>>,
+) {
+	const start = process.hrtime();
+
+	for (const func of functions) {
+		await func();
+	}
+
+	const [seconds, miliseconds] = process.hrtime(start);
+	const timeInSeconds = seconds + miliseconds / 1e9;
+
+	return +timeInSeconds.toFixed(3);
+}
+
+/**
+ * Pause the process until a key is pressed.
+ *
+ * @param title
+ * @returns
+ */
+function pause(title: string) {
+	const isPauseEnabled = !!envs.GRIDDO_RENDER_BREAKPOINTS_FEATURE;
+
+	if (!isPauseEnabled) {
+		return;
+	}
+
+	return new Promise<void>((resolve) => {
+		console.log("\n");
+		boxLog(`⌛️ ${title}`, "", 1, 0);
+		process.stdin.once("data", () => {
+			resolve();
+		});
+	});
+}
+
+/**
+ * Executes a life cycle process, which involves executing an array of
+ * functions, printing to the console, and handling errors with optional
+ * retries.
+ *
+ * @async
+ * @param name - The name of the life cycle.
+ * @param options - The arguments object.
+ * @param options.steps - An array of functions to execute.
+ * @returns - A promise that resolves when the life cycle process is completed.
+ */
+async function doLifeCycle(name: LifeCyclesNames, steps: LifeCycleSteps) {
+	const attemptsByLifeCycleName: Record<LifeCyclesNames, number> = {
+		Init: envs.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS,
+		Archive: envs.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS,
+		Data: envs.GRIDDO_DATA_LIFECYCLE_MAX_ATTEMPTS,
+		Meta: envs.GRIDDO_META_LIFECYCLE_MAX_ATTEMPTS,
+		Relocation: envs.GRIDDO_RELOCATION_LIFECYCLE_MAX_ATTEMPTS,
+		Clean: envs.GRIDDO_CLEAN_LIFECYCLE_MAX_ATTEMPTS,
+		Restore: envs.GRIDDO_RESTORE_LIFECYCLE_MAX_ATTEMPTS,
+		Prepare: envs.GRIDDO_PREPARE_LIFECYCLE_MAX_ATTEMPTS,
+		Close: envs.GRIDDO_CLOSE_LIFECYCLE_MAX_ATTEMPTS,
+		SSG: envs.GRIDDO_SSG_LIFECYCLE_MAX_ATTEMPTS,
+		HealthCheck: 1,
+		__DEBUG__: 1,
+	};
+
+	let trysCount = 0;
+	const maxTrysAccepted = attemptsByLifeCycleName[name] || 1;
+
+	while (trysCount < maxTrysAccepted) {
+		try {
+			infoLog(`start ${name} life-cycle`);
+			const exeTime = await measureExecutionTime(steps);
+			successLog(`${name} - ${exeTime}s\n`);
+			// if no errors, print and go out
+			await pause(`${name} LifeCycle`);
+			break;
+		} catch (error) {
+			const errorString = bgRed(black(` Error in ${name} LifeCycle `));
+			const attemptsString = kleur.yellow(`Attempt (${trysCount + 1})`);
+			console.log(`\n${errorString} ${attemptsString}\n`);
+			console.log(error);
+			console.log();
+
+			trysCount++;
+		}
+	}
+
+	if (trysCount === maxTrysAccepted) {
+		throwError(LifecycleExecutionError(maxTrysAccepted, name));
+	}
+}
+
+/**
+ * Creates additional files after the render: sitemaps, robots and a report of
+ * the finished render.
+ *
+ * @async
+ * @param domain
+ */
+async function createDomainMetadata(domain: string) {
+	await generateBuildReport();
+	await generateSitemaps();
+	await RobotsService.writeFiles(domain);
+}
+
+function removeAllSiteDirsFromStore() {
+	const { __cx } = config.paths();
+	const storeDir = path.join(__cx, "store");
+	const allDirs = fs.readdirSync(storeDir);
+
+	const allSiteDirs = allDirs.filter((dirname) => dirname !== "metadata");
+
+	const allSiteDirsFullPath = allSiteDirs.map((name) =>
+		path.join(storeDir, name),
+	);
+
+	// Remove all site directories except "metadata" directory.
+	for (const site of allSiteDirsFullPath) {
+		fs.rmSync(site, { recursive: true, force: true });
+	}
+}
+
+function getFormattedDateTime() {
+	const now = new Date();
+	const date = [
+		String(now.getDate()).padStart(2, "0"),
+		String(now.getMonth() + 1).padStart(2, "0"),
+		now.getFullYear(),
+	].join("-");
+
+	const time = [
+		String(now.getHours()).padStart(2, "0"),
+		String(now.getMinutes()).padStart(2, "0"),
+		String(now.getSeconds()).padStart(2, "0"),
+	].join(":");
+
+	return `${date}_${time}`;
+}
+
+/**
+ * Save render information to a file to use as debug log.
+ *
+ * This information will **not** be sent to any API.
+ */
+function saveBuildEndLogs() {
+	const { __cx } = config.paths();
+	const { sitesToPublish, createdPages, buildProcessData } = fsx.readJSONSync(
+		path.join(__cx, "render-metadata.json"),
+	);
+
+	const dateString = getFormattedDateTime();
+	const buildEndLogsDir = path.join(__cx, "debug", "build-end");
+
+	if (!fs.existsSync(buildEndLogsDir)) {
+		fs.mkdirSync(buildEndLogsDir, { recursive: true });
+	}
+
+	// Log
+	fs.writeFileSync(
+		path.join(buildEndLogsDir, `${dateString}.json`),
+		JSON.stringify(
+			{
+				date: new Date().toLocaleString(),
+				log: {
+					sitesToPublish,
+					createdPages,
+					buildProcessData,
+				},
+			},
+			null,
+			2,
+		),
+	);
+
+	// Update index
+	prependFileSync(
+		path.join(buildEndLogsDir, "index.html"),
+		`<pre><a href="${dateString}.json">${dateString}</a></pre>\n`,
+	);
+}
+
+export {
+	createDomainMetadata,
+	delay,
+	doLifeCycle,
+	getConfig,
+	getSafeSiteId,
+	instanceRootDir,
+	isTruthy,
+	measureExecutionTime,
+	msToSec,
+	pause,
+	removeAllSiteDirsFromStore,
+	removeProperties,
+	sanitizeAPICacheDir,
+	saveBuildEndLogs,
+	walk,
+	walkStore,
+};

package/exporter/utils/create-build-data.ts

@@ -0,0 +1,17 @@
+import { createAPICacheDir } from "./cache";
+import { sanitizeAPICacheDir } from "./core-utils";
+import { createStore } from "../services/store";
+
+/**
+ * Download all data: sites, pages etc.. from the instance private Griddo API.
+ * Then you can use the generator funcion `getBuildPagesFromStore()` to get the pages and
+ * `getBuildMetadata()` to get build and sites metadata as objects. Both from
+ * exporter utils sites dir.
+ */
+async function createDomainRenderData(domain: string) {
+	createAPICacheDir();
+	await createStore(domain);
+	sanitizeAPICacheDir();
+}
+
+export { createDomainRenderData };

package/exporter/utils/domains.ts

@@ -0,0 +1,39 @@
+import type { Domains } from "../types/global";
+
+import { verboseLog } from "./loggin";
+import { throwError } from "../errors";
+import { NoDomainsFoundError } from "../errors/errors-data";
+import { AuthService } from "../services/auth";
+import { getAllDomains } from "../services/domains";
+
+/**
+ * Return an array of domains name (string) of the current instance.
+ */
+async function getInstanceDomains() {
+	await AuthService.login();
+	const domains = await getAllDomains();
+
+	if (!domains.length) {
+		throwError(NoDomainsFoundError);
+	}
+
+	verboseLog(`getting domains names (${domains.length})`);
+
+	return getDomainSlugs(domains);
+}
+
+/**
+ * Return an unique array of domains, filtered without "/".
+ *
+ * @param domains An array of domains
+ * @see Domains
+ */
+function getDomainSlugs(domains: Domains) {
+	const filteredDomains = domains
+		.filter(({ slug }) => !!slug)
+		.map(({ slug }) => slug.replace("/", ""));
+
+	return [...new Set(filteredDomains)];
+}
+
+export { getInstanceDomains, getDomainSlugs };