@griddo/cx 10.3.13 → 10.3.15

package/{src → exporter}/services/robots.ts

@@ -1,7 +1,7 @@
 import type { Robots } from "../types/global";
 
-import fs from "fs";
-import path from "path";
+import fs from "node:fs";
+import path from "node:path";
 
 dotenv.config();
 
@@ -28,7 +28,7 @@ class RobotsService {
 	/**
 	 * TODO: JSDoc
 	 */
-	async loadRobots() {
+	private async getRobots() {
 		try {
 			const apiRobots = await get<Robots>({
 				endpoint: this.endpoint,
@@ -46,10 +46,11 @@ class RobotsService {
 	}
 
 	/**
-	 * TODO: JSDoc
+	 * Write robots.txt files for the current rendering domain.
 	 */
 	async writeFiles(basePath: string) {
-		// Takes Robot for the current rendering domain
+		await this.getRobots();
+
 		const robot = this.robots.find(
 			({ path }) => path === `/${process.env.DOMAIN}`
 		);
@@ -60,7 +61,9 @@ class RobotsService {
 		}
 
 		if (fs.existsSync(basePath)) {
-			const fileLocation = process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH
+			const fileLocation = JSON.parse(
+				process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH || "false"
+			)
 				? path.join(basePath, "robots.txt")
 				: path.join(basePath, robot.path, "robots.txt");
 			fs.writeFileSync(fileLocation, robot?.content);

package/exporter/services/store.ts

@@ -0,0 +1,351 @@
+import type { BuildProcessData } from "../types/global";
+import type {
+	GriddoListPage,
+	GriddoMultiPage,
+	GriddoPageObject,
+	GriddoSinglePage,
+	PageAdditionalInfo,
+} from "../types/pages";
+import type { Site } from "../types/sites";
+
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import dotenv from "dotenv";
+import fsx from "fs-extra";
+import pLimit from "p-limit";
+
+import { DistributorService } from "./distributors";
+import { NavigationService } from "./navigation";
+import { SettingsService } from "./settings";
+import { SitesService } from "./sites";
+import { version as griddoVersion } from "../../package.json";
+import { createSha256, updatedSiteHash } from "../utils/cache";
+import { deleteSites } from "../utils/folders";
+import {
+	createGriddoListPages,
+	createGriddoMultiPages,
+	createGriddoSinglePage,
+	getMultiPageElements,
+	getPaginatedPages,
+} from "../utils/pages";
+import { logInfo, removeProperties, siteList } from "../utils/shared";
+import { checkSites, getSiteData, unpublishSites } from "../utils/sites";
+
+dotenv.config();
+
+// Envs
+const API_URL = process.env.API_URL as string;
+const CXBRANCH = process.env.CXBRANCH;
+// prettier-ignore
+const REACT_APP_INSTANCE = process.env.GRIDDO_REACT_APP_INSTANCE || process.env.REACT_APP_INSTANCE;
+// prettier-ignore
+const API_CONCURRENCY_COUNT = parseInt(process.env.GRIDDO_API_CONCURRENCY_COUNT || "") || 10;
+const PUBLIC_API_URL = process.env.PUBLIC_API_URL as string;
+// prettier-ignore
+const RENDER_ID = process.env.GRIDDO_RENDERID || new Date().valueOf().toString();
+
+// Consts
+const CREATED_PAGES: Array<number> = [];
+const BUILD_PROCESS_DATA: BuildProcessData = {};
+
+/**
+ * Fetch, process and save object pages and sites data into the file system to
+ * be consumed by other services (Griddo itself, Adapters, etc.)
+ */
+async function createStore(storeFolder: string) {
+	console.info(`API calls with ${API_CONCURRENCY_COUNT} threads`);
+
+	// Remove old store
+	if (fs.existsSync(storeFolder)) {
+		fs.rmSync(storeFolder, { recursive: true });
+	}
+
+	// Create fresh folders for store and store/metadata
+	fs.mkdirSync(storeFolder);
+	fs.mkdirSync(path.resolve(storeFolder, "metadata"));
+
+	try {
+		// Get sites objects to publish and unpublish.
+		const { sitesToPublish, sitesToUnpublish } = await checkSites();
+
+		// The settings will be extracted below in this process.
+		await SettingsService.getAll();
+
+		// If no activity in sites, exit
+		if (!(sitesToPublish.length || sitesToUnpublish.length)) {
+			console.warn("There are no sites to update");
+			process.exit(0);
+		}
+
+		console.info(`Sites to publish: ${siteList(sitesToPublish)}`);
+		console.info(`Sites to unpublish: ${siteList(sitesToUnpublish)}`);
+
+		// Unpublish (API) and delete sites
+		// TODO: Manage this to save end data to the __report-build__ file ???
+		await unpublishSites(sitesToUnpublish);
+
+		// Delete (clean) sites to publish
+		// SPREAD: await deleteSites(sitesToPublish);
+		await deleteSites([...sitesToPublish]);
+
+		for (const site of sitesToPublish) {
+			const { id: siteId, slug: siteSlug, theme, favicon } = site;
+
+			const {
+				siteInfo,
+				validPagesIds,
+				siteHash,
+				unpublishHashes,
+				siteLangs,
+				defaultLang,
+				headers,
+				footers,
+				socials,
+			} = await getSiteData(siteId);
+
+			const {
+				cloudinaryName,
+				useMetaTitle,
+				useMetaKeywords,
+				showBasicMetaRobots,
+				avoidHrefLangsOnCanonicals,
+				avoidSelfReferenceCanonicals,
+				avoidHrefLangXDefault,
+				avoidDebugMetas,
+			} = SettingsService.settings;
+
+			BUILD_PROCESS_DATA[siteId] = {
+				siteHash,
+				unpublishHashes,
+				publishHashes: [],
+			};
+
+			const NavService = new NavigationService();
+
+			// Set `headers` and `footers` to the NavService to use later in this process.
+			NavService.navigations = { headers, footers };
+
+			site.languages = siteLangs;
+
+			const shouldUpdateSite = updatedSiteHash(siteId, siteHash);
+
+			const siteScript = siteInfo.siteScript;
+
+			const siteMetadata = {
+				siteUrl: siteInfo.slug,
+				title: siteInfo.name,
+				favicon,
+			};
+
+			const additionalInfo = {
+				baseUrl: API_URL,
+				publicBaseUrl: PUBLIC_API_URL,
+				instance: REACT_APP_INSTANCE,
+				siteSlug,
+				theme,
+				siteMetadata,
+				socials,
+				siteLangs,
+				cloudinaryName,
+				griddoVersion,
+				siteOptions: {
+					useMetaTitle,
+					useMetaKeywords,
+					showBasicMetaRobots,
+					avoidHrefLangsOnCanonicals,
+					avoidSelfReferenceCanonicals,
+					avoidHrefLangXDefault,
+					avoidDebugMetas,
+				},
+				BUILD_MODE: CXBRANCH,
+				siteScript,
+			};
+
+			logInfo(`${site.name} site`);
+
+			// -------------------------------------------------------------------------
+			// Pages loop promise creation
+			// -------------------------------------------------------------------------
+			// Async function that process every page for one site to use later in the
+			// `Promise.all` with pLimit.
+			const fetchPageAndSaveToStore = async (pageId: number) => {
+				// Here will be store every page returned by the API and processed by
+				// Griddo CX.
+				let griddoPageObjects: Array<GriddoPageObject> = [];
+
+				// Get page data
+				const page = await SitesService.getPage(pageId, shouldUpdateSite);
+
+				// Probably a 404
+				if (!page) return;
+
+				// Update Array of page ids
+				CREATED_PAGES.push(pageId);
+
+				// TODO: Use structuredClone() when node 18 is available.
+				// SHAME: This new pageAdditionalInfo needs to be a copy because
+				// additionalInfo referenced from outside this function and
+				// its used by other pages.
+				const pageAdditionalInfo = JSON.parse(
+					JSON.stringify(additionalInfo)
+				) as PageAdditionalInfo;
+
+				// Updated with navigations (header & footer)
+				pageAdditionalInfo.navigations = NavService.getPageNavigations(page);
+
+				// Content Type Data Query
+				// Where the pair `hasDistributorData:true` and `data` prop exists, this
+				// function will attach a `queriedItems` prop with the result of the fetch
+				// for the data.
+				const template = await DistributorService.getDistributorData({
+					page,
+					cacheKey: RENDER_ID,
+				});
+
+				// MultiPage Query
+				// Where the pair `hasGriddoMultiPage:true` prop exists, this function
+				// will process the schema and return a multiPageElemtens array to use
+				// in `createGriddoMultiPages()`
+				const multiPageElements = await getMultiPageElements(template);
+
+				// -----------------------------------------------------------------------
+				// Build Griddo page objects.
+				// -----------------------------------------------------------------------
+				// A page from the API could be one of those:
+				// - List with pagination (static list template): needs to be splitted in n pages
+				// - Multi-page module: needs to be splitted in n pages
+				// - Single: just one page
+
+				// Griddo page types
+				const isStaticListPage = page?.mode === "list";
+				const isMultiPage = !isStaticListPage && multiPageElements;
+				const isSinglePage = !isMultiPage && !isStaticListPage;
+
+				// >> List (static list template) <<
+				if (isStaticListPage) {
+					const griddoListPage = {
+						page: page,
+						pages: getPaginatedPages(template),
+						isRoot: false,
+						defaultLang: defaultLang,
+						template: template,
+						totalQueriedItems: template.queriedItems,
+					} as GriddoListPage;
+
+					// pageObjects = await createGriddoListPages({ adapter: "Gatsby" })
+					griddoPageObjects = await createGriddoListPages(
+						griddoListPage,
+						pageAdditionalInfo
+					);
+				}
+
+				// >> Multi-page <<
+				if (isMultiPage) {
+					const griddoMultipage = page as GriddoMultiPage;
+
+					// The `template` key with the (maybe) distributor data items queried
+					griddoMultipage.template = template;
+					griddoMultipage.multiPageElements = multiPageElements;
+					griddoMultipage.defaultLang = defaultLang;
+
+					griddoPageObjects = await createGriddoMultiPages(
+						griddoMultipage,
+						pageAdditionalInfo
+					);
+				}
+
+				// >> Single template <<
+				if (isSinglePage) {
+					const griddoSinglePage = page as GriddoSinglePage;
+
+					// The `template` key with the (maybe) distributor data items queried
+					griddoSinglePage.template = template;
+					griddoSinglePage.defaultLang = defaultLang;
+
+					griddoPageObjects = [
+						await createGriddoSinglePage(griddoSinglePage, pageAdditionalInfo),
+					];
+				}
+
+				// Update hashes
+				BUILD_PROCESS_DATA[siteId].publishHashes.push(page.hash);
+
+				// Save build data to store
+				await savePagesInStore(storeFolder, griddoPageObjects); // gatsbyPageObjects is always an array regardless of whether it's just one page
+				setSitesToPublish(storeFolder, sitesToPublish);
+				setBuildProcessData(storeFolder, BUILD_PROCESS_DATA);
+				setCreatedPages(storeFolder, CREATED_PAGES);
+			};
+
+			// Create the pLimit array of promises
+			const limit = pLimit(API_CONCURRENCY_COUNT);
+			const pagesToRender = validPagesIds.map((id: number) =>
+				limit(() => fetchPageAndSaveToStore(id))
+			);
+
+			await Promise.all(pagesToRender);
+		}
+	} catch (e) {
+		const error = e as { message: string };
+		console.error(error.message);
+		process.exit(1);
+	}
+}
+
+/**
+ * Set the build process data in the Store.
+ */
+function setBuildProcessData(
+	basePath: string,
+	buildProcessData: BuildProcessData
+) {
+	fsx.writeJSONSync(
+		path.resolve(basePath, "metadata", "buildProcessData.json"),
+		buildProcessData
+	);
+}
+
+/**
+ * Set the created pages in the Store.
+ */
+function setCreatedPages(basePath: string, pages: Array<number>) {
+	fsx.writeJSONSync(
+		path.resolve(basePath, "metadata", "createdPages.json"),
+		pages
+	);
+}
+
+/**
+ * Set site to publish data in the Store.
+ */
+function setSitesToPublish(basePath: string, sites: Array<Site>) {
+	fsx.writeJSONSync(
+		path.resolve(basePath, "metadata", "sitesToPublish.json"),
+		sites
+	);
+}
+
+/**
+ * Save the pages into the file system.
+ */
+async function savePagesInStore(
+	basePath: string,
+	pages: Array<GriddoPageObject>
+) {
+	pages.forEach((page) =>
+		removeProperties(page, ["editorID", "parentEditorID"])
+	);
+
+	const promises = pages.map(async (pageObject) => {
+		const filename = `${createSha256(pageObject.path)}.json`;
+		const fileContent = JSON.stringify(pageObject);
+		const filePath = path.resolve(basePath, filename);
+		await fsp.writeFile(filePath, fileContent);
+	});
+
+	await Promise.all(promises);
+}
+
+export { createStore };

package/{src → exporter}/types/api.ts

@@ -179,3 +179,9 @@ export interface ShowApiErrorOptions {
 		headers?: unknown;
 	};
 }
+
+export interface BuildMetaData {
+	buildProcessData: Record<string, EndSiteRenderBody>;
+	createdPages: Array<number>;
+	sitesToPublish: Array<Site>;
+}

package/{src → exporter}/types/pages.ts

@@ -96,14 +96,28 @@ export interface PageAdditionalInfo extends AdditionalInfo {
 }
 
 // TODO: JSDoc
-export type GatsbyPageObject = {
+export type MultiPageElement = {
+	component: string;
+	title: string | Required<Fields.Heading>;
+	elements: Array<Record<string, unknown>>;
+	componentModules: Array<Record<string, unknown>>;
+	sectionSlug: string;
+	metaTitle: string;
+	metaDescription: string;
+};
+
+// TODO: JSDoc
+export type MultiPageElements = Array<MultiPageElement>;
+
+/**
+ * Gatsby page object format.
+ */
+export type GriddoPageObject = {
 	matchPath?: string;
 	path: string;
-	component: string;
 	/** Page size in bytes */
 	size?: number;
 	context: {
-		// Page
 		BUILD_MODE?: string;
 		cloudinaryName?: string;
 		footer: Record<string, unknown> | null;
@@ -141,20 +155,7 @@ export type GatsbyPageObject = {
 		siteOptions: AdditionalInfo["siteOptions"];
 		siteScript: string;
 		socials: SocialsResponse;
+		// The three Griddo page types
 		page: GriddoSinglePage | GriddoListPage | GriddoMultiPage;
 	};
 };
-
-// TODO: JSDoc
-export type MultiPageElement = {
-	component: string;
-	title: string | Required<Fields.Heading>;
-	elements: Array<Record<string, unknown>>;
-	componentModules: Array<Record<string, unknown>>;
-	sectionSlug: string;
-	metaTitle: string;
-	metaDescription: string;
-};
-
-// TODO: JSDoc
-export type MultiPageElements = Array<MultiPageElement>;

package/{src → exporter}/types/sites.ts

@@ -4,7 +4,7 @@ import type { Core } from "@griddo/core";
 
 /**
  * Describes a Griddo site object from API.
- * This takes some type props from Core.Site which is a Site for the Gatsby template.tsx.
+ * This takes some type props from Core.Site.
  */
 export interface Site
 	extends Required<

package/{src → exporter}/utils/api.ts

@@ -89,13 +89,16 @@ async function requestAPI<T extends APIResponses>(
 		}
 
 		if (attempt > parseInt(RETRY_ATTEMPTS)) {
-			console.log(
-				`Max attempts ${RETRY_ATTEMPTS} reached\n- ${method.toUpperCase()} ${endpoint}\n- BODY: ${JSON.stringify(
-					body
-				)}\n- HEADERS: ${JSON.stringify(headers)}\n- RESPONSE: ${
-					(error.response?.status, JSON.stringify(error.response?.data))
-				}`
-			);
+			console.log(`
+Max attempts ${RETRY_ATTEMPTS} reached
+--------------------------------------
+- ${method.toUpperCase()} ${endpoint}
+- BODY: ${JSON.stringify(body)}
+- HEADERS: ${JSON.stringify(headers)}
+- RESPONSE: ${(error.response?.status, JSON.stringify(error.response?.data))}
+--------------------------------------
+`);
+
 			process.exit(1);
 		}
 

package/{src → exporter}/utils/cache.ts

@@ -1,25 +1,25 @@
 import type { Petition } from "../types/global";
 import type { HashSites, SiteHash } from "../types/sites";
 
-import crypto from "crypto";
-import fs from "fs";
-import path from "path";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
 
 import fsx from "fs-extra";
 
 // Consts
-const API_CACHE_DIR_PATH = path.resolve(__dirname, "./../../apiCache");
+const API_CACHE_DIR_PATH = path.resolve(__dirname, "./../apiCache");
 const SITE_HASH_FILENAME = `${API_CACHE_DIR_PATH}/siteHash.json`;
 
 /**
- *  Creates an `apiCache` folder to store pages from API
+ * Creates an `apiCache` folder to store pages fetched from them API.
  */
-function initCache() {
+function createAPICacheFolder() {
 	if (!fs.existsSync(API_CACHE_DIR_PATH)) {
 		fs.mkdirSync(API_CACHE_DIR_PATH);
 	}
 
-	console.log("\n🪄 Cache initialized");
+	console.info("Cache initialized");
 }
 
 /**
@@ -119,4 +119,10 @@ function updatedSiteHash(siteId: number, siteHash: SiteHash) {
 	return currentHash.toString();
 }
 
-export { createSha256, updatedSiteHash, searchCacheData, saveCache, initCache };
+export {
+	createSha256,
+	updatedSiteHash,
+	searchCacheData,
+	saveCache,
+	createAPICacheFolder,
+};

package/{src → exporter}/utils/domains.ts

@@ -4,9 +4,9 @@ import { AuthService } from "../services/auth";
 import { DomainsService } from "../services/domains";
 
 /**
- * Return an array of slug domains.
+ * Return an array of domains name (string) of the current instance.
  */
-async function findDomains() {
+async function getInstanceDomains() {
 	await AuthService.login();
 	const domains = await DomainsService.getAll();
 
@@ -27,4 +27,4 @@ function getDomainSlugs(domains: Domains) {
 	return [...new Set(filteredDomains)];
 }
 
-export { findDomains };
+export { getInstanceDomains };

package/exporter/utils/download-build-data.ts

@@ -0,0 +1,22 @@
+import path from "node:path";
+
+import { createAPICacheFolder } from "./cache";
+import { sanitizeAPICacheFolder } from "./shared";
+import { createStore } from "../services/store";
+
+const storeFolder = path.resolve(__dirname, "../store/");
+const apiCacheFolder = path.resolve(__dirname, "../apiCache");
+
+/**
+ * Download all data: sites, pages etc.. from the instance private Griddo API.
+ * Then you can use the generator funcion `getBuildPages()` to get the pages and
+ * `getBuildMetadata()` to get build and sites metadata as objects. Both from
+ * exporter utils sites folder.
+ */
+async function downloadBuildData() {
+	createAPICacheFolder();
+	await createStore(storeFolder);
+	sanitizeAPICacheFolder(apiCacheFolder);
+}
+
+export { downloadBuildData };

package/exporter/utils/folders.ts

@@ -0,0 +1,49 @@
+import type { Site } from "../types/sites";
+
+import path from "node:path";
+
+import fs from "fs-extra";
+
+import { logInfo } from "./shared";
+
+/**
+ * Delete sites folders that will be updated from de file system.
+ *
+ * @param updateSites An array of sites to be updated.
+ */
+async function deleteSites(updatedSites: Array<Site>) {
+	for await (const site of updatedSites) {
+		for (const domain of site.domains) {
+			const mappedDomain = Object.values(domain)[0];
+			const dir = path.resolve(__dirname, `../../dist${mappedDomain}`);
+			const pageDataDir = path.resolve(
+				__dirname,
+				`../../assets/page-data${mappedDomain}`
+			);
+
+			logInfo(`Site dir ${dir}`);
+			logInfo(`Page data dir ${pageDataDir}`);
+
+			// delete directory recursively
+			if (!fs.existsSync(dir)) return;
+
+			try {
+				await fs.rm(dir, { recursive: true });
+				logInfo(`${dir} is deleted!`);
+			} catch (err) {
+				console.log(err);
+			}
+
+			if (!fs.existsSync(pageDataDir)) return;
+
+			try {
+				await fs.rm(pageDataDir, { recursive: true });
+				console.info(`${pageDataDir} is deleted!`);
+			} catch (err) {
+				console.log(err);
+			}
+		}
+	}
+}
+
+export { deleteSites };

package/{src → exporter}/utils/health-checks.ts

@@ -5,12 +5,13 @@ const GRIDDO_ENVS_VARS: ReadonlyArray<string> = [];
 
 /**
  * Check if the environment is secure to launch a render.
- * If something fails throws an error and exit
+ * If something fails then log an error message and exit from the process.
+ * Otherwise just return true.
  */
-function buildHealthCheck() {
+function checkRenderHealthOrExit() {
 	// Bypass the check using this env var, handy for local renders.
 	if (GRIDDO_SKIP_BUILD_CHECKS) {
-		console.info(`🫥 Build health check skipped`);
+		console.info(`Build health check skipped`);
 
 		return true;
 	}
@@ -23,7 +24,7 @@ function buildHealthCheck() {
 
 	// The environment is suitable for a build-render
 	if (renderingIsSave) {
-		console.info("✅ Build health check passed");
+		console.info("Build health check passed");
 
 		return true;
 	}
@@ -47,8 +48,8 @@ function checkEnvsVars(envs: ReadonlyArray<string>) {
 
 	if (missingEnvs.length) {
 		const missingEnvsMsg = missingEnvs.join("\n");
-		console.log(
-			`ENV CHECK FAILED. These envs vars are missing: \n\n${missingEnvsMsg}`
+		console.error(
+			`Env check failed. These envs vars are missing: \n\n${missingEnvsMsg}`
 		);
 
 		return false;
@@ -57,4 +58,4 @@ function checkEnvsVars(envs: ReadonlyArray<string>) {
 	return true;
 }
 
-export { buildHealthCheck };
+export { checkRenderHealthOrExit };

package/{src → exporter}/utils/instance.ts

@@ -1,4 +1,4 @@
-import path from "path";
+import path from "node:path";
 
 import findUp from "find-up";
 import pkgDir from "pkg-dir";