@griddo/cx 11.7.5 → 11.7.6-rc.0

package/exporter/utils/sites.ts

@@ -1,14 +1,16 @@
 import type { Site, SiteData } from "../types/sites";
 
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import fs from "fs-extra";
 import { parse } from "js2xmlparser";
 
-import { envs } from "../constants";
-import { getConfig } from "./core-utils";
-import { buildLog, verboseLog } from "./loggin";
-import { getBuildMetadata } from "./store";
+import {
+	GRIDDO_API_URL,
+	GRIDDO_RENDER_ALL_SITES,
+	GRIDDO_RENDER_PAGES,
+	GRIDDO_RENDER_SITE,
+} from "../constants/envs";
 import { AuthService } from "../services/auth";
 import {
 	endSiteRender,
@@ -19,31 +21,28 @@ import {
 	getSiteSocials,
 	startSiteRender,
 } from "../services/sites";
-
-const config = getConfig();
+import { pathExists } from "./folders";
+import { verboseLog } from "./loggin";
+import { getRenderMetadataFromDB, getRenderPathsFromDB } from "./render";
+import { getBuildMetadata } from "./store";
 
 /**
  * Check the instance sites and returns site prepared to be published and unpublished.
  */
 async function getSitesToRender(domain: string) {
-	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
+	console.info(`API URL ${GRIDDO_API_URL as string}`);
 
 	// Login to API
-	await AuthService.login();
+	// await AuthService.login();
 
 	// Get all sites. An array of Site
 	const allSites = await getAllSites(domain);
 
 	// Filter the array of sites to get only the ones to build/render
-	const validSites = envs.GRIDDO_RENDER_ALL_SITES
-		? allSites.filter(
-				(site) =>
-					!envs.GRIDDO_RENDER_SITE || site.id === envs.GRIDDO_RENDER_SITE,
-			)
+	const validSites = GRIDDO_RENDER_ALL_SITES
+		? allSites.filter((site) => !GRIDDO_RENDER_SITE || site.id === GRIDDO_RENDER_SITE)
 		: allSites.filter((site) =>
-				envs.GRIDDO_RENDER_SITE
-					? site.id === envs.GRIDDO_RENDER_SITE
-					: !!site.shouldBeUpdated,
+				GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.shouldBeUpdated,
 			);
 
 	// If there are valid sites...
@@ -58,8 +57,7 @@ async function getSitesToRender(domain: string) {
 			site.domains = items
 				.filter(
 					(item) =>
-						item.domain &&
-						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
+						item.domain && (item.domain.slug === domain || item.domain.slug === `/${domain}`),
 				)
 				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
 		}
@@ -67,15 +65,11 @@ async function getSitesToRender(domain: string) {
 
 	// Save sites object to publish
 	const sitesToPublish = validSites.filter((site) =>
-		envs.GRIDDO_RENDER_SITE
-			? site.id === envs.GRIDDO_RENDER_SITE
-			: !!site.isPublished,
+		GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.isPublished,
 	);
 
 	// Save sites object to unpublish
-	const sitesToUnpublish = validSites.filter(
-		(site) => !site.isPublished && site.shouldBeUpdated,
-	);
+	const sitesToUnpublish = validSites.filter((site) => !site.isPublished && site.shouldBeUpdated);
 
 	return {
 		sitesToPublish,
@@ -90,7 +84,7 @@ async function getSitesToRender(domain: string) {
  * @see https://griddoio.notion.site/Sites-d7bb0b7cb8d24894a5337e1139fc3d09#2019d3255bda4d219c7e19cf28d0c4fe
  */
 async function unpublishSites(sites: Array<Site>) {
-	const { __cx } = config.paths();
+	const { __cx } = await getRenderPathsFromDB();
 
 	for (const site of sites) {
 		// API
@@ -107,7 +101,8 @@ async function unpublishSites(sites: Array<Site>) {
 
 		// STORE
 		// Remove site directory from the Store
-		fs.rmSync(path.join(__cx, "store", site.id.toString()), {
+		// @todo: quitar. Como ya no hay persistenci de store, esto debería eliminarse de forma segura.
+		await fsp.rm(path.join(__cx, "store", site.id.toString()), {
 			force: true,
 			recursive: true,
 		});
@@ -130,8 +125,8 @@ async function getSiteData(siteID: number) {
 
 	const { siteHash, unpublishHashes, publishIds } = buildData;
 	const { headers, footers } = siteInfo;
-	const validPagesIds = envs.GRIDDO_RENDER_PAGES.length
-		? envs.GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
+	const validPagesIds = GRIDDO_RENDER_PAGES.length
+		? GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
 		: publishIds;
 
 	const siteData: SiteData = {
@@ -152,15 +147,13 @@ async function getSiteData(siteID: number) {
 /**
  * Save a file with the end of build process to use as `build-end` signal.
  */
-async function generateBuildReport() {
-	const { __cx } = config.paths();
-	const { buildReportFileName } = config;
-
-	const { buildProcessData } = await getBuildMetadata();
-
-	// Get the token
+async function generateBuildReport(domain: string) {
 	const authControl = await AuthService.login();
 
+	const { __cx } = await getRenderPathsFromDB();
+	const { buildReportFileName } = await getRenderMetadataFromDB();
+	const { buildProcessData } = await getBuildMetadata(domain);
+
 	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
 		...buildProcessData[siteID],
 		siteId: Number.parseInt(siteID),
@@ -172,51 +165,54 @@ async function generateBuildReport() {
 	};
 
 	const reportFilePath = path.join(__cx, "dist", buildReportFileName);
-	fs.writeFileSync(reportFilePath, JSON.stringify(report));
 
-	verboseLog(`build report saved in ${reportFilePath}`);
+	await fsp.writeFile(reportFilePath, JSON.stringify(report));
 
-	buildLog(`Build report saved for ${buildSitesInfo.length} site(s)`);
+	verboseLog(`build report saved in ${reportFilePath}`);
 }
 
 /**
  * Generate sitemaps and save them into file system.
  */
-async function generateSitemaps() {
-	const { sitesToPublish } = await getBuildMetadata();
-	const { __cx } = config.paths();
+async function generateSitemaps(domain: string) {
+	const { __cx } = await getRenderPathsFromDB();
+	const { sitesToPublish } = await getBuildMetadata(domain);
 	const distDir = path.join(__cx, "dist");
-
 	const templateSitemapPrefix = "sitemap-";
 
 	const isGriddoSitemapFile = (file: string) =>
-		file === "sitemap.xml" ||
-		(file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
+		file === "sitemap.xml" || (file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
 
 	for (const site of sitesToPublish) {
 		const { id, languages } = site;
 
 		for (const lang of languages) {
-			if (AuthService.headers) {
-				AuthService.headers.lang = lang.id.toString();
-			}
+			// Local copy of headers to avoid mutating the shared object
+			const localHeaders: Record<string, string> = AuthService.headers
+				? { ...AuthService.headers }
+				: {};
+			localHeaders.lang = lang.id.toString();
 
-			const response = await getSitemap(id);
+			const response = await getSitemap({ siteId: id, headers: localHeaders });
 
-			if (!response) continue;
+			if (!response) {
+				continue;
+			}
 
-			const {
-				items: sitemapPagesGroup,
-				url: { home, domain },
-			} = response;
+			const { items: sitemapPagesGroup, url } = response;
+			const { home, domain } = url;
 
-			if (!home) continue;
+			if (!home) {
+				continue;
+			}
 
 			const domainLanguage = site.domains.find(
 				(domain) => Object.keys(domain)[0] === lang.id.toString(),
 			);
 
-			if (!domainLanguage) continue;
+			if (!domainLanguage) {
+				continue;
+			}
 
 			const slug = Object.values(domainLanguage)[0];
 			const sitemaps: Array<string> = [];
@@ -224,16 +220,16 @@ async function generateSitemaps() {
 			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
 
 			/** delete every griddo sitemap file before create the new ones */
-			if (fs.existsSync(sitemapBasePath)) {
+			if (await pathExists(sitemapBasePath)) {
 				try {
-					const files = fs.readdirSync(sitemapBasePath);
+					const files = await fsp.readdir(sitemapBasePath);
 					for (const file of files) {
 						if (isGriddoSitemapFile(file)) {
 							const filePath = path.join(sitemapBasePath, file);
 							try {
-								fs.unlinkSync(filePath);
+								await fsp.rm(filePath);
 							} catch (err) {
-								console.error(`Error removing ${filePath}:`, err);
+								console.error(`Error deleting file ${filePath}:`, err);
 							}
 						}
 					}
@@ -245,7 +241,9 @@ async function generateSitemaps() {
 			for (const templateId of sitemapPageGroupKeys) {
 				const sitemapPages = sitemapPagesGroup[templateId];
 
-				if (!sitemapPages.length) continue;
+				if (!sitemapPages.length) {
+					continue;
+				}
 
 				const siteMap = parse("urlset", {
 					"@": {
@@ -260,11 +258,9 @@ async function generateSitemaps() {
 				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
 				const exactPath = path.join(sitemapBasePath, sitemapName);
 
-				saveFile(exactPath, siteMap);
+				await saveFile(exactPath, siteMap);
 
-				sitemaps.push(
-					`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`,
-				);
+				sitemaps.push(`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`);
 			}
 
 			if (!sitemaps.length) {
@@ -278,7 +274,7 @@ async function generateSitemaps() {
 
 			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
 
-			saveFile(exactPath, siteMap);
+			await saveFile(exactPath, siteMap);
 			verboseLog(`sitemap generated in ${exactPath}`);
 		}
 	}
@@ -290,23 +286,17 @@ async function generateSitemaps() {
  * @param filePath The path of the file to save the content to.
  * @param content The content to save to the file.
  */
-function saveFile(filePath: string, content: string) {
+async function saveFile(filePath: string, content: string) {
 	try {
 		const pathName = path.dirname(filePath);
-		if (!fs.existsSync(pathName)) {
-			fs.mkdirSync(pathName, { recursive: true });
+		if (!(await pathExists(pathName))) {
+			await fsp.mkdir(pathName, { recursive: true });
 		}
-		fs.writeFileSync(filePath, content);
+		await fsp.writeFile(filePath, content);
 	} catch (error) {
 		console.log(error);
 		throw new Error(`Error saving a file`);
 	}
 }
 
-export {
-	generateBuildReport,
-	generateSitemaps,
-	getSiteData,
-	getSitesToRender,
-	unpublishSites,
-};
+export { generateBuildReport, generateSitemaps, getSiteData, getSitesToRender, unpublishSites };

package/exporter/utils/store.ts

@@ -3,99 +3,27 @@ import type { RenderInfo } from "../types/global";
 import type { GriddoPageObject } from "../types/pages";
 import type { Site } from "../types/sites";
 
-import fs from "node:fs";
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import fsx from "fs-extra";
-
-import { throwError } from "../errors";
-import { getConfig, removeProperties, walk, walkStore } from "./core-utils";
+import { WriteToStoreError } from "../constants/errors";
+import { readDB, writeDB } from "../services/db";
+import { removeProperties } from "./core-utils";
+import { throwError } from "./errors";
+import { pathExists } from "./folders";
 import { verboseLog } from "./loggin";
-import { ReadFromStoreError, WriteToStoreError } from "../errors/errors-data";
-
-const config = getConfig();
-
-/**
- * Read all path pages stored in the `store` Griddo directory and returns the
- * absolute file path.
- *
- * @param domain - The domain to get the pages from.
- */
-function getBuildPagesFromCachedStore<PageType extends GriddoPageObject>(
-	domain: string,
-) {
-	const { __cache } = config.paths(domain);
-	return getBuildPagesFromStore<PageType>({
-		basePath: path.join(__cache, "store"),
-		withSizeProp: false,
-	});
-}
+import { getRenderPathsFromDB } from "./render";
 
 /**
- * Read all pages stored in the `store` Griddo directory and returns one by one
- * with a generator.
- *
- * @param basePath - Base directory to get pages from.
- * @param options.withSizeProp - Add size prop to the page object.
- * @todo throw error if the basePath is not an store folder
+ * Get the build metadata from the Store.
  */
-function* getBuildPagesFromStore<PageType extends GriddoPageObject>(args?: {
-	basePath?: string;
-	withSizeProp?: boolean;
-}) {
-	const { basePath, withSizeProp } = args || {};
-	const { __cx } = config.paths();
-	const pagesDirPath = basePath || path.join(__cx, "store");
-
-	const jsonFilePaths = walkStore(pagesDirPath).filter(
-		(file) => path.extname(file) === ".json",
-	);
-
-	for (const filePath of jsonFilePaths) {
-		try {
-			const page = fsx.readJsonSync(filePath, {
-				encoding: "utf-8",
-			}) as PageType;
-
-			if (withSizeProp) {
-				const fileStats = fs.statSync(filePath);
-				page.size = fileStats.size / 1024;
-			}
+async function getBuildMetadata(domain: string): Promise<BuildMetaData> {
+	const db = await readDB();
+	const { sitesToPublish, createdPages, buildProcessData } = db.domains[domain].renderInfo || {};
 
-			// SECURITY: Only returns valid page objects
-			if (page.path) {
-				yield page;
-			}
-		} catch (error) {
-			throwError(ReadFromStoreError, error);
-		}
+	if (!sitesToPublish || !createdPages || !buildProcessData) {
+		throw new Error("Build metadata not found");
 	}
-}
-
-/**
- * Read all pages stored in `store` Griddo directory and return the absolute
- * path.
- */
-function getBuildPagesPath() {
-	const config = getConfig();
-	const { __cx } = config.paths();
-
-	const PAGES_DIR = path.join(__cx, "store");
-	const PAGE_FILES = walk(PAGES_DIR).filter(
-		(file) => path.extname(file) === ".json",
-	);
-
-	return PAGE_FILES;
-}
-
-/**
- * Get the build metadata from the Store.
- */
-async function getBuildMetadata(): Promise<BuildMetaData> {
-	const { __cx } = config.paths();
-	const { sitesToPublish, createdPages, buildProcessData } = fsx.readJSONSync(
-		path.join(__cx, "render-metadata.json"),
-	);
 
 	return {
 		buildProcessData,
@@ -108,13 +36,11 @@ async function getBuildMetadata(): Promise<BuildMetaData> {
  * Write render info into a file.
  * @param renderInfo - Data that will be saved related to the render process.
  */
-function saveRenderInfoInStore(renderInfo: RenderInfo) {
-	const { __cx } = config.paths();
-
-	fs.writeFileSync(
-		path.join(__cx, "render-metadata.json"),
-		JSON.stringify(renderInfo),
-	);
+async function saveRenderInfoInStore(renderInfo: RenderInfo, domain: string) {
+	const db = await readDB();
+	db.domains[domain] = db.domains[domain] || {};
+	db.domains[domain].renderInfo = renderInfo;
+	await writeDB(db);
 }
 
 /**
@@ -133,59 +59,49 @@ function saveRenderInfoInStore(renderInfo: RenderInfo) {
 // }
 
 /**
- * Return an array of ids only from `.json` files (no dirs) in the `basePath` dir based in the file name.
+ * Return an array of paths only from `.json` files (no dirs) in the `basePath` dir based in the file name.
  * @param basePath - Absolute path of the dir from which files will be read.
  * @returns A Array<number> of pages ids in `basePath` dir.
  */
-function getPageInStoreDir(basePath: string): Array<number> {
-	const filesInStore = fs.readdirSync(basePath);
+async function getPageInStoreDir(basePath: string) {
+	const filesInStore = await fsp.readdir(basePath);
+
+	return filesInStore
+		.filter(async (file) => {
+			const fullPathFile = `${basePath}/${file}`;
+			const stat = await fsp.stat(fullPathFile);
+			// Si es un directorio, no lo incluimos.
+			if (stat?.isDirectory()) {
+				return false;
+			}
 
-	return (
-		filesInStore
-			.filter((file) => {
-				const fullPathFile = `${basePath}/${file}`;
-				const stat = fs.statSync(fullPathFile);
-				// Si es un directorio, no lo incluimos.
-				if (stat?.isDirectory()) {
-					return false;
-				}
-				// Si es un archivo pero no tiene la extensión `.json`, no lo incluimos
-				if (path.extname(file) !== ".json") {
-					return false;
-				}
-				// Si es un archivo con el nombre NO pasable a número (NaN), no lo incluimos.
-				const baseFilename = path.basename(file, path.extname(file));
-				if (Number.isNaN(Number.parseInt(baseFilename))) {
-					return false;
-				}
+			// Si es un archivo pero no tiene la extensión `.json`, no lo incluimos
+			if (path.extname(file) !== ".json") {
+				return false;
+			}
 
-				// no es dir, es json y con nombre pasable a número, ok :)
-				return true;
-			})
-			// Tenemos solo jsons donde el nombre es un "número": "21345.json"
-			.map((page) => {
-				const filename = path.basename(page, path.extname(page));
-				return Number.parseInt(filename);
-			})
-	);
+			// no es dir, es json.
+			return true;
+		})
+		.map((page) => {
+			return path.join(basePath, page);
+		});
 }
 
 /**
  * Save the pages into the file system.
  * @param pages - An array of Griddo page objects to be saved.
  */
-function saveSitePagesInStore(
-	siteDirName: string,
-	pages: Array<GriddoPageObject>,
-) {
-	const { __cx } = config.paths();
+async function saveSitePagesInStore(siteDirName: string, pages: Array<GriddoPageObject>) {
+	const { __cx } = await getRenderPathsFromDB();
 
 	try {
 		for (const page of pages) {
 			removeProperties(page, ["editorID", "parentEditorID"]);
 			const filename = `${page.context.page.id}.json`;
 			const filePath = path.join(__cx, "store", siteDirName, filename);
-			fsx.writeJSONSync(filePath, page);
+			// En los listados estáticos y multipage los ids de las páginas son iguales
+			await writeUniqueFileSync(filePath, JSON.stringify(page));
 		}
 	} catch (error) {
 		throwError(WriteToStoreError, error);
@@ -203,23 +119,15 @@ function saveSitePagesInStore(
  * @param siteDirName - The name of the site directory containing the pages.
  * @param pageIdsArray - A two-dimensional array where each inner array contains numbers representing page files to be removed.
  */
-function removeSitePagesFromStore(
-	siteDirName: string,
-	pageIdsArray: Array<Array<number>>,
-) {
-	const { __cx } = config.paths();
+async function removeSitePagesFromStore(siteDirName: string, pageIdsArray: Array<Array<number>>) {
+	const { __cx } = await getRenderPathsFromDB();
 
-	const processPageIdsArray = (pageIds: Array<number>) => {
+	const processPageIdsArray = async (pageIds: Array<number>) => {
 		for (const filename of pageIds) {
-			const filePath = path.join(
-				__cx,
-				"store",
-				siteDirName,
-				`${filename}.json`,
-			);
+			const filePath = path.join(__cx, "store", siteDirName, `${filename}.json`);
 			try {
-				if (fs.existsSync(filePath)) {
-					fs.unlinkSync(filePath);
+				if (await pathExists(filePath)) {
+					await fsp.unlink(filePath);
 					verboseLog(`Removed file: ${filePath}`);
 				}
 			} catch (error) {
@@ -230,7 +138,7 @@ function removeSitePagesFromStore(
 	};
 
 	for (const pageIds of pageIdsArray) {
-		processPageIdsArray(pageIds);
+		await processPageIdsArray(pageIds);
 	}
 }
 
@@ -239,48 +147,33 @@ function removeSitePagesFromStore(
  * the domain has been changed. This is necessary because those sites will not
  * be marked as `sitesToUnpublish` so they must be removed manually.
  */
-function removeOrphanSites(sitesToPublish: Array<Site>, domain: string) {
-	const { __cx } = getConfig().paths(domain);
-	const currentSitesInStore = fs.readdirSync(path.join(__cx, "store"));
+async function removeOrphanSites(sitesToPublish: Array<Site>, domain: string) {
+	const { __cx } = await getRenderPathsFromDB({ domain });
+	const storePath = path.join(__cx, "store");
+
+	if (!(await pathExists(storePath))) {
+		return;
+	}
+
+	const currentSitesInStore = await fsp.readdir(storePath);
 	const sitesFromAPI = sitesToPublish.map(({ id }) => `${id}`);
-	const sitesToDelete = currentSitesInStore.filter(
-		(site) => !sitesFromAPI.includes(site),
-	);
+	const sitesToDelete = currentSitesInStore.filter((site) => !sitesFromAPI.includes(site));
 
 	for (const site of sitesToDelete) {
-		fs.rmSync(path.join(__cx, "store", site), { recursive: true, force: true });
+		await fsp.rm(path.join(storePath, site), { recursive: true, force: true });
 		verboseLog(`Removed orphan site id: ${site}`);
 	}
 }
 
-/**
- * Returns the list of `activePages` pages that do not currently exist in a
- * site's store (probably due to a failed render).
- */
-function getMissingPublishedPagesInStore(
-	siteName: string,
-	activePages: Array<number>,
-) {
-	const { __cx } = config.paths();
-	const pagesInStore = getPageInStoreDir(path.join(__cx, "store", siteName));
-	const missingActivePages: Array<number> = [];
-	for (const activePage of activePages) {
-		if (!pagesInStore.includes(activePage)) {
-			missingActivePages.push(activePage);
-		}
-	}
-	return missingActivePages;
-}
-
 /**
  * Get pages that are not active anymore but are in the store.
  */
-function getZombiePagesInStore(siteDir: string, activePages: Array<number>) {
-	const { __cx } = getConfig().paths();
+async function getZombiePagesInStore(siteDir: string, activePages: Array<number>) {
+	const { __cx } = await getRenderPathsFromDB();
 	const storeDir = path.join(__cx, "store", siteDir);
-	const sitePages = fs.readdirSync(storeDir);
+	const sitePages = await fsp.readdir(storeDir);
 
-	let zombiePages: Array<number> = [];
+	const zombiePages: Array<number> = [];
 	for (const pageFileName of sitePages) {
 		const pageFileNameWithoutExtension = pageFileName.split(".")[0];
 		const pageInStore = Number.parseInt(pageFileNameWithoutExtension);
@@ -299,16 +192,30 @@ function getZombiePagesInStore(siteDir: string, activePages: Array<number>) {
 	return zombiePages;
 }
 
+async function writeUniqueFileSync(filePath: string, content: string) {
+	const dir = path.dirname(filePath);
+	const ext = path.extname(filePath);
+	const base = path.basename(filePath, ext);
+
+	let uniquePath = filePath;
+	let counter = 1;
+
+	while (await pathExists(uniquePath)) {
+		const newName = `${base}-${counter}${ext}`;
+		uniquePath = path.join(dir, newName);
+		counter++;
+	}
+
+	await fsp.writeFile(uniquePath, content);
+}
+
 export {
 	getBuildMetadata,
-	getBuildPagesFromCachedStore,
-	getBuildPagesFromStore,
-	getBuildPagesPath,
-	getMissingPublishedPagesInStore,
 	getPageInStoreDir,
 	getZombiePagesInStore,
 	removeOrphanSites,
 	removeSitePagesFromStore,
-	saveSitePagesInStore,
 	saveRenderInfoInStore,
+	saveSitePagesInStore,
+	writeUniqueFileSync,
 };