@griddo/cx 11.9.9-rc.0 → 11.9.9-rc.2

package/exporter/utils/sites.ts

@@ -0,0 +1,312 @@
+import type { Site, SiteData } from "../types/sites";
+
+import path from "node:path";
+
+import fs from "fs-extra";
+import { parse } from "js2xmlparser";
+
+import { envs } from "../constants";
+import { getConfig } from "./core-utils";
+import { buildLog, verboseLog } from "./loggin";
+import { getBuildMetadata } from "./store";
+import { AuthService } from "../services/auth";
+import {
+	endSiteRender,
+	getAllSites,
+	getSiteInfo,
+	getSiteLanguages,
+	getSitemap,
+	getSiteSocials,
+	startSiteRender,
+} from "../services/sites";
+
+const config = getConfig();
+
+/**
+ * Check the instance sites and returns site prepared to be published and unpublished.
+ */
+async function getSitesToRender(domain: string) {
+	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
+
+	// Login to API
+	await AuthService.login();
+
+	// Get all sites. An array of Site
+	const allSites = await getAllSites(domain);
+
+	// Filter the array of sites to get only the ones to build/render
+	const validSites = envs.GRIDDO_RENDER_ALL_SITES
+		? allSites.filter(
+				(site) =>
+					!envs.GRIDDO_RENDER_SITE || site.id === envs.GRIDDO_RENDER_SITE,
+			)
+		: allSites.filter((site) =>
+				envs.GRIDDO_RENDER_SITE
+					? site.id === envs.GRIDDO_RENDER_SITE
+					: !!site.shouldBeUpdated,
+			);
+
+	// If there are valid sites...
+	// En este paso se añade al objeto `Site` la información de los dominios
+	// utilizando los idiomas.
+	if (validSites.length) {
+		for (const site of validSites) {
+			const { items } = await getSiteLanguages(site.id);
+
+			// Añadimos la prop domains con el dominio "cocinado" con los
+			// idiomas y teniendo en cuenta solo el dominio actual.
+			site.domains = items
+				.filter(
+					(item) =>
+						item.domain &&
+						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
+				)
+				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
+		}
+	}
+
+	// Save sites object to publish
+	const sitesToPublish = validSites.filter((site) =>
+		envs.GRIDDO_RENDER_SITE
+			? site.id === envs.GRIDDO_RENDER_SITE
+			: !!site.isPublished,
+	);
+
+	// Save sites object to unpublish
+	const sitesToUnpublish = validSites.filter(
+		(site) => !site.isPublished && site.shouldBeUpdated,
+	);
+
+	return {
+		sitesToPublish,
+		sitesToUnpublish,
+	};
+}
+
+/**
+ * Unpublish an array of sites sending the information to the API.
+ *
+ * @param sites An array of sites
+ * @see https://griddoio.notion.site/Sites-d7bb0b7cb8d24894a5337e1139fc3d09#2019d3255bda4d219c7e19cf28d0c4fe
+ */
+async function unpublishSites(sites: Array<Site>) {
+	const { __cx } = config.paths();
+
+	for (const site of sites) {
+		// API
+		const buildInfo = await startSiteRender(site.id);
+		const { siteHash } = buildInfo;
+		const body = {
+			siteHash,
+			publishHashes: [],
+			unpublishHashes: [],
+			publishPagesIds: [],
+		};
+
+		await endSiteRender(site.id, body);
+
+		// STORE
+		// Remove site directory from the Store
+		fs.rmSync(path.join(__cx, "store", site.id.toString()), {
+			force: true,
+			recursive: true,
+		});
+	}
+}
+
+/**
+ * Return a single site generic data.
+ *
+ * @param siteID The site id.
+ * @see SiteData
+ */
+async function getSiteData(siteID: number) {
+	const buildData = await startSiteRender(siteID);
+	const siteInfo = await getSiteInfo(siteID);
+	const siteLangs = await getSiteLanguages(siteID);
+	const socials = await getSiteSocials(siteID);
+	const siteLangsInfo = siteLangs.items;
+	const defaultLang = siteLangsInfo.find((lang) => lang.isDefault);
+
+	const { siteHash, unpublishHashes, publishIds } = buildData;
+	const { headers, footers } = siteInfo;
+	const validPagesIds = envs.GRIDDO_RENDER_PAGES.length
+		? envs.GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
+		: publishIds;
+
+	const siteData: SiteData = {
+		siteInfo,
+		validPagesIds,
+		siteHash,
+		unpublishHashes,
+		siteLangs: siteLangsInfo,
+		defaultLang,
+		headers,
+		footers,
+		socials,
+	};
+
+	return siteData;
+}
+
+/**
+ * Save a file with the end of build process to use as `build-end` signal.
+ */
+async function generateBuildReport() {
+	const { __cx } = config.paths();
+	const { buildReportFileName } = config;
+
+	const { buildProcessData } = await getBuildMetadata();
+
+	// Get the token
+	const authControl = await AuthService.login();
+
+	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
+		...buildProcessData[siteID],
+		siteId: Number.parseInt(siteID),
+	}));
+
+	const report = {
+		authControl,
+		sites: buildSitesInfo,
+	};
+
+	const reportFilePath = path.join(__cx, "dist", buildReportFileName);
+	fs.writeFileSync(reportFilePath, JSON.stringify(report));
+
+	verboseLog(`build report saved in ${reportFilePath}`);
+
+	buildLog(`Build report saved for ${buildSitesInfo.length} site(s)`);
+}
+
+/**
+ * Generate sitemaps and save them into file system.
+ */
+async function generateSitemaps() {
+	const { sitesToPublish } = await getBuildMetadata();
+	const { __cx } = config.paths();
+	const distDir = path.join(__cx, "dist");
+
+	const templateSitemapPrefix = "sitemap-";
+
+	const isGriddoSitemapFile = (file: string) =>
+		file === "sitemap.xml" ||
+		(file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
+
+	for (const site of sitesToPublish) {
+		const { id, languages } = site;
+
+		for (const lang of languages) {
+			if (AuthService.headers) {
+				AuthService.headers.lang = lang.id.toString();
+			}
+
+			const response = await getSitemap(id);
+
+			if (!response) continue;
+
+			const {
+				items: sitemapPagesGroup,
+				url: { home, domain },
+			} = response;
+
+			if (!home) continue;
+
+			const domainLanguage = site.domains.find(
+				(domain) => Object.keys(domain)[0] === lang.id.toString(),
+			);
+
+			if (!domainLanguage) continue;
+
+			const slug = Object.values(domainLanguage)[0];
+			const sitemaps: Array<string> = [];
+			const sitemapPageGroupKeys = Object.keys(sitemapPagesGroup);
+			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
+
+			/** delete every griddo sitemap file before create the new ones */
+			if (fs.existsSync(sitemapBasePath)) {
+				try {
+					const files = fs.readdirSync(sitemapBasePath);
+					for (const file of files) {
+						if (isGriddoSitemapFile(file)) {
+							const filePath = path.join(sitemapBasePath, file);
+							try {
+								fs.unlinkSync(filePath);
+							} catch (err) {
+								console.error(`Error removing ${filePath}:`, err);
+							}
+						}
+					}
+				} catch (err) {
+					console.error(`Error reading ${sitemapBasePath}:`, err);
+				}
+			}
+
+			for (const templateId of sitemapPageGroupKeys) {
+				const sitemapPages = sitemapPagesGroup[templateId];
+
+				if (!sitemapPages.length) continue;
+
+				const siteMap = parse("urlset", {
+					"@": {
+						xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
+						"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+						"xsi:schemaLocation":
+							"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd",
+					},
+					url: sitemapPages,
+				});
+
+				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
+				const exactPath = path.join(sitemapBasePath, sitemapName);
+
+				saveFile(exactPath, siteMap);
+
+				sitemaps.push(
+					`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`,
+				);
+			}
+
+			if (!sitemaps.length) {
+				continue;
+			}
+
+			const siteMap = parse("sitemapindex", {
+				"@": { xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" },
+				sitemap: sitemaps.map((loc) => ({ loc })),
+			});
+
+			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
+
+			saveFile(exactPath, siteMap);
+			verboseLog(`sitemap generated in ${exactPath}`);
+		}
+	}
+}
+
+/**
+ * Saves the content to a file specified by its path. If the file exists, it will be overwritten.
+ *
+ * @param filePath The path of the file to save the content to.
+ * @param content The content to save to the file.
+ */
+function saveFile(filePath: string, content: string) {
+	try {
+		const pathName = path.dirname(filePath);
+		if (!fs.existsSync(pathName)) {
+			fs.mkdirSync(pathName, { recursive: true });
+		}
+		fs.writeFileSync(filePath, content);
+	} catch (error) {
+		console.log(error);
+		throw new Error(`Error saving a file`);
+	}
+}
+
+export {
+	generateBuildReport,
+	generateSitemaps,
+	getSiteData,
+	getSitesToRender,
+	unpublishSites,
+};

package/exporter/utils/store.ts

@@ -0,0 +1,314 @@
+import type { BuildMetaData } from "../types/api";
+import type { RenderInfo } from "../types/global";
+import type { GriddoPageObject } from "../types/pages";
+import type { Site } from "../types/sites";
+
+import fs from "node:fs";
+import path from "node:path";
+
+import fsx from "fs-extra";
+
+import { throwError } from "../errors";
+import { getConfig, removeProperties, walk, walkStore } from "./core-utils";
+import { verboseLog } from "./loggin";
+import { ReadFromStoreError, WriteToStoreError } from "../errors/errors-data";
+
+const config = getConfig();
+
+/**
+ * Read all path pages stored in the `store` Griddo directory and returns the
+ * absolute file path.
+ *
+ * @param domain - The domain to get the pages from.
+ */
+function getBuildPagesFromCachedStore<PageType extends GriddoPageObject>(
+	domain: string,
+) {
+	const { __cache } = config.paths(domain);
+	return getBuildPagesFromStore<PageType>({
+		basePath: path.join(__cache, "store"),
+		withSizeProp: false,
+	});
+}
+
+/**
+ * Read all pages stored in the `store` Griddo directory and returns one by one
+ * with a generator.
+ *
+ * @param basePath - Base directory to get pages from.
+ * @param options.withSizeProp - Add size prop to the page object.
+ * @todo throw error if the basePath is not an store folder
+ */
+function* getBuildPagesFromStore<PageType extends GriddoPageObject>(args?: {
+	basePath?: string;
+	withSizeProp?: boolean;
+}) {
+	const { basePath, withSizeProp } = args || {};
+	const { __cx } = config.paths();
+	const pagesDirPath = basePath || path.join(__cx, "store");
+
+	const jsonFilePaths = walkStore(pagesDirPath).filter(
+		(file) => path.extname(file) === ".json",
+	);
+
+	for (const filePath of jsonFilePaths) {
+		try {
+			const page = fsx.readJsonSync(filePath, {
+				encoding: "utf-8",
+			}) as PageType;
+
+			if (withSizeProp) {
+				const fileStats = fs.statSync(filePath);
+				page.size = fileStats.size / 1024;
+			}
+
+			// SECURITY: Only returns valid page objects
+			if (page.path) {
+				yield page;
+			}
+		} catch (error) {
+			throwError(ReadFromStoreError, error);
+		}
+	}
+}
+
+/**
+ * Read all pages stored in `store` Griddo directory and return the absolute
+ * path.
+ */
+function getBuildPagesPath() {
+	const config = getConfig();
+	const { __cx } = config.paths();
+
+	const PAGES_DIR = path.join(__cx, "store");
+	const PAGE_FILES = walk(PAGES_DIR).filter(
+		(file) => path.extname(file) === ".json",
+	);
+
+	return PAGE_FILES;
+}
+
+/**
+ * Get the build metadata from the Store.
+ */
+async function getBuildMetadata(): Promise<BuildMetaData> {
+	const { __cx } = config.paths();
+	const { sitesToPublish, createdPages, buildProcessData } = fsx.readJSONSync(
+		path.join(__cx, "render-metadata.json"),
+	);
+
+	return {
+		buildProcessData,
+		createdPages,
+		sitesToPublish,
+	};
+}
+
+/**
+ * Write render info into a file.
+ * @param renderInfo - Data that will be saved related to the render process.
+ */
+function saveRenderInfoInStore(renderInfo: RenderInfo) {
+	const { __cx } = config.paths();
+
+	fs.writeFileSync(
+		path.join(__cx, "render-metadata.json"),
+		JSON.stringify(renderInfo),
+	);
+}
+
+/**
+ * Return an array of ids only from `.json` files (no dirs) in the `basePath` dir based in the file name and filtered by the pages ids in a concrete site.
+ * @param basePath - Absolute path of the dir from which files will be read.
+ * @returns A Array<number> of pages ids in `basePath` dir filtered by pages ids in a concrete site.
+ */
+// function getPagesInStoreDirBySitePages(
+// 	basePath: string,
+// 	pages: Array<number>,
+// ): Array<number> {
+// 	const allPagesInStore = getPageInStoreDir(basePath);
+// 	const pagesBySite = allPagesInStore.filter((page) => pages.includes(page));
+
+// 	return pagesBySite;
+// }
+
+/**
+ * Return an array of ids only from `.json` files (no dirs) in the `basePath` dir based in the file name.
+ * @param basePath - Absolute path of the dir from which files will be read.
+ * @returns A Array<number> of pages ids in `basePath` dir.
+ */
+function getPageInStoreDir(basePath: string): Array<number> {
+	const filesInStore = fs.readdirSync(basePath);
+
+	return (
+		filesInStore
+			.filter((file) => {
+				const fullPathFile = `${basePath}/${file}`;
+				const stat = fs.statSync(fullPathFile);
+				// Si es un directorio, no lo incluimos.
+				if (stat?.isDirectory()) {
+					return false;
+				}
+				// Si es un archivo pero no tiene la extensión `.json`, no lo incluimos
+				if (path.extname(file) !== ".json") {
+					return false;
+				}
+				// Si es un archivo con el nombre NO pasable a número (NaN), no lo incluimos.
+				const baseFilename = path.basename(file, path.extname(file));
+				if (Number.isNaN(Number.parseInt(baseFilename))) {
+					return false;
+				}
+
+				// no es dir, es json y con nombre pasable a número, ok :)
+				return true;
+			})
+			// Tenemos solo jsons donde el nombre es un "número": "21345.json"
+			.map((page) => {
+				const filename = path.basename(page, path.extname(page));
+				return Number.parseInt(filename);
+			})
+	);
+}
+
+/**
+ * Save the pages into the file system.
+ * @param pages - An array of Griddo page objects to be saved.
+ */
+function saveSitePagesInStore(
+	siteDirName: string,
+	pages: Array<GriddoPageObject>,
+) {
+	const { __cx } = config.paths();
+
+	try {
+		for (const page of pages) {
+			removeProperties(page, ["editorID", "parentEditorID"]);
+			const filename = `${page.context.page.id}.json`;
+			const filePath = path.join(__cx, "store", siteDirName, filename);
+			fsx.writeJSONSync(filePath, page);
+		}
+	} catch (error) {
+		throwError(WriteToStoreError, error);
+	}
+}
+
+/**
+ * Removes JSON files from the specified site directory based on the provided arrays of page numbers.
+ * For each number in the arrays, a corresponding file with the format `<number>.json` is removed if it exists.
+ *
+ * WARNING: This function may end up handling large arrays, so don't be tempted
+ * to refactor it to use something cool like the spread operator and make a
+ * single array, etc..
+ *
+ * @param siteDirName - The name of the site directory containing the pages.
+ * @param pageIdsArray - A two-dimensional array where each inner array contains numbers representing page files to be removed.
+ */
+function removeSitePagesFromStore(
+	siteDirName: string,
+	pageIdsArray: Array<Array<number>>,
+) {
+	const { __cx } = config.paths();
+
+	const processPageIdsArray = (pageIds: Array<number>) => {
+		for (const filename of pageIds) {
+			const filePath = path.join(
+				__cx,
+				"store",
+				siteDirName,
+				`${filename}.json`,
+			);
+			try {
+				if (fs.existsSync(filePath)) {
+					fs.unlinkSync(filePath);
+					verboseLog(`Removed file: ${filePath}`);
+				}
+			} catch (error) {
+				console.log(error);
+				throw new Error(`Error removing file ${filename}`);
+			}
+		}
+	};
+
+	for (const pageIds of pageIdsArray) {
+		processPageIdsArray(pageIds);
+	}
+}
+
+/**
+ * Removes sites that exist in the store that should no longer be there because
+ * the domain has been changed. This is necessary because those sites will not
+ * be marked as `sitesToUnpublish` so they must be removed manually.
+ */
+function removeOrphanSites(sitesToPublish: Array<Site>, domain: string) {
+	const { __cx } = getConfig().paths(domain);
+	const currentSitesInStore = fs.readdirSync(path.join(__cx, "store"));
+	const sitesFromAPI = sitesToPublish.map(({ id }) => `${id}`);
+	const sitesToDelete = currentSitesInStore.filter(
+		(site) => !sitesFromAPI.includes(site),
+	);
+
+	for (const site of sitesToDelete) {
+		fs.rmSync(path.join(__cx, "store", site), { recursive: true, force: true });
+		verboseLog(`Removed orphan site id: ${site}`);
+	}
+}
+
+/**
+ * Returns the list of `activePages` pages that do not currently exist in a
+ * site's store (probably due to a failed render).
+ */
+function getMissingPublishedPagesInStore(
+	siteName: string,
+	activePages: Array<number>,
+) {
+	const { __cx } = config.paths();
+	const pagesInStore = getPageInStoreDir(path.join(__cx, "store", siteName));
+	const missingActivePages: Array<number> = [];
+	for (const activePage of activePages) {
+		if (!pagesInStore.includes(activePage)) {
+			missingActivePages.push(activePage);
+		}
+	}
+	return missingActivePages;
+}
+
+/**
+ * Get pages that are not active anymore but are in the store.
+ */
+function getZombiePagesInStore(siteDir: string, activePages: Array<number>) {
+	const { __cx } = getConfig().paths();
+	const storeDir = path.join(__cx, "store", siteDir);
+	const sitePages = fs.readdirSync(storeDir);
+
+	let zombiePages: Array<number> = [];
+	for (const pageFileName of sitePages) {
+		const pageFileNameWithoutExtension = pageFileName.split(".")[0];
+		const pageInStore = Number.parseInt(pageFileNameWithoutExtension);
+
+		// What the hell is `pageIdInStore > 0` ??
+		// Las páginas de listados estáticos y multipage tiene ids "basura", que
+		// en este caso son ids negativos y siempre so borran al final del render.
+		// En este caso hay que ignorarlas porque NUNCA van a estar en el
+		// `activePages`, estaríamos borrándolas siempre antes de la fase de
+		// SSG.
+		if (!activePages.includes(pageInStore) && pageInStore > 0) {
+			zombiePages.push(pageInStore);
+		}
+	}
+
+	return zombiePages;
+}
+
+export {
+	getBuildMetadata,
+	getBuildPagesFromCachedStore,
+	getBuildPagesFromStore,
+	getBuildPagesPath,
+	getMissingPublishedPagesInStore,
+	getPageInStoreDir,
+	getZombiePagesInStore,
+	removeOrphanSites,
+	removeSitePagesFromStore,
+	saveSitePagesInStore,
+	saveRenderInfoInStore,
+};