@griddo/cx 1.75.178 → 1.75.180

package/src/services/store.ts

@@ -0,0 +1,513 @@
+//                                                                            //
+//                            ┌───────────────────┐                           //
+//                            │   Page from API   │ ∞ loop                    //
+//                            └─────────┬─────────┘                           //
+//                                    ┌─┼─┐                                   //
+//                                 navigations                                //
+//                                 distributors                               //
+//                                     etc..                                  //
+//                                    └─┼─┘                                   //
+//                       ┌──────────────┼─────────────┐                       //
+//                       ▼              ▼             ▼                       //
+//                ┌─────────────┐ ┌───────────┐ ┌───────────┐                 //
+//                │ Single page │ │ Multipage │ │ Listpages │                 //
+//                └─────────────┘ └───────────┘ └───────────┘                 //
+//                       ▼              ▼             ▼                       //
+//                       └──────┐       |       ┌─────┘                       //
+//                              ▼       ▼       ▼                             //
+//                        ┌───────  S T O R E  ─────┐                         //
+//                        │  Array of page objects  │                         //
+//                        ├─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┤                         //
+//                        │      Build metadata     │                         //
+//                        └─────────────────────────┘                         //
+
+// Types
+import type { BuildProcessData, StoreMode } from "../types/global";
+import type { GatsbyPageObject, PageAdditionalInfo } from "../types/pages";
+import type { Site } from "../types/sites";
+
+// External libraries
+import chalk from "chalk";
+import dotenv from "dotenv";
+import fs from "fs";
+import fsp from "fs/promises";
+import pLimit from "p-limit";
+import path from "path";
+import v8 from "v8";
+
+// Services
+import { DistributorService } from "./distributors";
+import { NavigationService } from "./navigation";
+import { RobotsService } from "./robots";
+import { SettingsService } from "./settings";
+import { SitesService } from "./sites";
+
+// Utils
+import { version as griddoVersion } from "../../package.json";
+import { createSha256, updatedSiteHash } from "../utils/cache";
+import { deleteSites } from "../utils/folders";
+import {
+	createGriddoListPages,
+	createGriddoMultiPages,
+	createGriddoSinglePage,
+	getMultiPageElements,
+	getPaginatedPages,
+} from "../utils/pages";
+import { logInfo, walk } from "../utils/shared";
+import { checkSites, getSiteData, unpublishSites } from "../utils/sites";
+
+dotenv.config();
+
+// Envs
+const API_URL = process.env.API_URL as string;
+const CXBRANCH = process.env.CXBRANCH;
+const REACT_APP_INSTANCE = process.env.GRIDDO_REACT_APP_INSTANCE;
+const GRIDDO_API_CONCURRENCY_COUNT = process.env.GRIDDO_API_CONCURRENCY_COUNT;
+const PUBLIC_API_URL = process.env.PUBLIC_API_URL as string;
+const RENDER_ID = process.env.RENDERID || new Date().valueOf().toString();
+
+// Constants shared between Gatsby phases.
+const CREATED_PAGES: Array<number> = [];
+const BUILD_PROCESS_DATA: BuildProcessData = {};
+
+/**
+ * Store service to fetch, process and save temporarily final pages and site
+ * data to be consumed by external services (Griddo itself, Gatsby, etc.)
+ *
+ * Modes:
+ * - File mode will save the final page objects in individual files in the file system.
+ * - Memory mode will save the final page objects in an array of objects in memory.
+ *
+ * @example
+ * import { StoreService } from "./store"
+ *
+ * // Init the store service.
+ * StoreService.init("foo/pages", { mode: "file" })
+ * // Save the data into files.
+ * await StoreService.saveBuildData()
+ * // Consume the pages as an interator.
+ * const pages = StoreService.getPages()
+ */
+class StoreService {
+	static basePath: string;
+	static mode: "file" | "memory";
+	static pages: Array<GatsbyPageObject> = [];
+	static sitesToPublish: Array<Site>;
+	static buildProcessData: BuildProcessData;
+	static createdPages: Array<number>;
+
+	// Public methods
+
+	/**
+	 * Initialize the Store with a path to store the files in file mode and an option object to set the mode.
+	 *
+	 * @param basePath The path to store the page files.
+	 * @param options An object configuration
+	 * @param options.mode Store mode
+	 * @todo Avoid initialize the Store with a `basePath` in memory mode.
+	 */
+	public static init(basePath: string, options: { mode: StoreMode }) {
+		const { mode } = options;
+		const griddoCxThreads = parseInt(GRIDDO_API_CONCURRENCY_COUNT || "10");
+		const modeString = mode === "file" ? "File mode" : "Memory mode";
+
+		console.log(`🧶 Processing API page calls with ${griddoCxThreads} threads`);
+		console.log(`📦 Griddo Store initialized - ${chalk.italic(modeString)}`);
+
+		if (mode === "file") {
+			this.basePath = basePath;
+			this.mode = mode;
+
+			if (fs.existsSync(basePath)) {
+				fs.rmSync(basePath, { recursive: true });
+			}
+
+			if (!fs.existsSync(basePath)) {
+				fs.mkdirSync(basePath);
+			}
+
+			if (!fs.existsSync(path.resolve(basePath, "pages"))) {
+				fs.mkdirSync(path.resolve(basePath, "pages"));
+			}
+		}
+	}
+
+	/**
+	 * Save the build data to the Store.
+	 */
+	public static async saveBuildData() {
+		try {
+			// Get sites objects to publish and unpublish.
+			const { sitesToPublish, sitesToUnpublish } = await checkSites();
+
+			// The settings will be extracted below in this process.
+			await SettingsService.getAll();
+
+			// If no activity in sites, exit
+			if (!(sitesToPublish.length || sitesToUnpublish.length)) {
+				console.warn("There are no sites to update");
+
+				process.exit(0);
+			}
+
+			// Publish and unpublis sites
+			console.log(
+				chalk.green(
+					"\n⬆️ ",
+					`(${sitesToPublish.length}) Sites to publish:`,
+					sitesToPublish.map(({ name }) => name).join(", "),
+					"\n"
+				)
+			);
+
+			console.log(
+				chalk.red(
+					"⬇️ ",
+					`(${sitesToUnpublish.length}) Sites to unpublish:`,
+					sitesToUnpublish.map(({ name }) => name).join(", "),
+					"\n"
+				)
+			);
+
+			// Unpublish (API) and delete sites
+			await unpublishSites(sitesToUnpublish);
+
+			// Delete (clean) sites to publish
+			await deleteSites([...sitesToPublish]);
+
+			// Set robots information to use later in the `onPostBuild` phase.
+			await RobotsService.loadRobots();
+
+			// Array of sites (Promises)
+			const promisesOfSitesToPublish = sitesToPublish.map(async (site) => {
+				// Gets
+				const { id: siteId, slug: siteSlug, theme, favicon } = site;
+
+				const {
+					siteInfo,
+					validPagesIds,
+					siteHash,
+					unpublishHashes,
+					siteLangs,
+					defaultLang,
+					headers,
+					footers,
+					socials,
+					// TODO: Check the false here...
+				} = await getSiteData(siteId, false);
+
+				const {
+					cloudinaryName,
+					useMetaTitle,
+					showBasicMetaRobots,
+					avoidHrefLangsOnCanonicals,
+					avoidSelfReferenceCanonicals,
+					avoidHrefLangXDefault,
+					avoidDebugMetas,
+				} = SettingsService.settings;
+
+				// Sets
+
+				BUILD_PROCESS_DATA[siteId] = {
+					siteHash,
+					unpublishHashes,
+					publishHashes: [],
+				};
+
+				const NavService = new NavigationService();
+
+				// Set `headers` and `footers` to the NavService to use later in this process.
+				NavService.navigations = { headers, footers };
+
+				site.languages = siteLangs;
+
+				const shouldUpdateSite = updatedSiteHash(siteId, siteHash);
+
+				const siteScript = siteInfo.siteScript;
+
+				const siteMetadata = {
+					siteUrl: siteInfo.slug,
+					title: siteInfo.name,
+					favicon,
+				};
+
+				const additionalInfo = {
+					baseUrl: API_URL,
+					publicBaseUrl: PUBLIC_API_URL,
+					instance: REACT_APP_INSTANCE,
+					siteSlug,
+					theme,
+					siteMetadata,
+					socials,
+					siteLangs,
+					cloudinaryName,
+					griddoVersion,
+					siteOptions: {
+						useMetaTitle,
+						showBasicMetaRobots,
+						avoidHrefLangsOnCanonicals,
+						avoidSelfReferenceCanonicals,
+						avoidHrefLangXDefault,
+						avoidDebugMetas,
+					},
+					BUILD_MODE: CXBRANCH,
+					siteScript,
+				};
+
+				logInfo(`${validPagesIds.length} valid pages`);
+
+				// -------------------------------------------------------------------------
+				// Pages loop promise creation
+				// -------------------------------------------------------------------------
+				// Async function that process every page for one site to use later in the
+				// `Promise.all` with pLimit.
+				const fetchPageAndSaveToStore = async (pageId: number) => {
+					// Here will be store every page returned by the API and processed by
+					// Griddo CX.
+					let gatsbyPageObjects: Array<GatsbyPageObject> = [];
+
+					// Get page data
+					const page = await SitesService.getPage(pageId, shouldUpdateSite);
+
+					if (!page) return;
+
+					// Update Array of page ids
+					CREATED_PAGES.push(pageId);
+
+					// TODO: Use structuredClone() when node 18 is available.
+					const pageAdditionalInfo = JSON.parse(
+						JSON.stringify(additionalInfo)
+					) as PageAdditionalInfo;
+
+					// Updated with navigations (header & footer)
+					// NavService.getPageNavigations returns { header, footer }
+					pageAdditionalInfo.navigations = NavService.getPageNavigations(page);
+
+					// Content Type Data Query
+					// Where the pair `hasDistributorData:true` and `data` prop exists, this
+					// function will attach a `queriedItems` prop with the result of the fetch
+					// for the data.
+					const template = await DistributorService.getDistributorData({
+						page,
+						cached: !!RENDER_ID,
+					});
+
+					// MultiPage Query
+					// Where the pair `hasGriddoMultiPage:true` prop exists, this function
+					// will process the schema and return a multiPageElemtens array to use
+					// in `createGriddoMultipages()`
+					const multiPageElements = await getMultiPageElements(template);
+
+					// -----------------------------------------------------------------------
+					// Build page objects
+					// -----------------------------------------------------------------------
+					// Three types of pages
+					// - List with pagination (n pages)
+					// - Multi-page module (n pages)
+					// - Single (1 page)
+
+					// List
+					if (page?.mode === "list") {
+						// of data items and the `itemsPerPage` prop.
+						gatsbyPageObjects = await createGriddoListPages(
+							{
+								...page, // Revisar la inclusión de este ...page.
+								// The first page, the one without the `/2, /3, /4, etc..` slug
+								page,
+								// The other pages
+								// In this case `getPaginatedPages` doesn't require the whole page object, just the `template` part.
+								// This is because only the `template` prop will be different for each page.
+								pages: getPaginatedPages(template),
+								// The `template` key with the (maybe) distributor data items queried
+								template,
+								// Pass all data items
+								totalQueriedItems: template.queriedItems,
+								// Default site language
+								defaultLang,
+							},
+							pageAdditionalInfo
+						);
+					}
+
+					// >> Multi-page
+					if (multiPageElements) {
+						gatsbyPageObjects = await createGriddoMultiPages(
+							{
+								// The whole page object
+								...page,
+								// The `template` key with the (maybe) distributor data items queried
+								template,
+								// Array of object of every element of the MultiPage
+								multiPageElements,
+								// Default site language
+								defaultLang,
+							},
+							pageAdditionalInfo
+						);
+					}
+
+					// >> Single template
+					if (!multiPageElements && page.mode !== "list") {
+						gatsbyPageObjects = [
+							await createGriddoSinglePage(
+								{
+									// The whole page object
+									...page,
+									// The `template` key with the (maybe) distributor data items queried
+									template,
+									// Default site language
+									defaultLang,
+								},
+								pageAdditionalInfo
+							),
+						];
+					}
+
+					// Update hashes
+					BUILD_PROCESS_DATA[siteId].publishHashes.push(page.hash);
+
+					// Save build data to store
+					await this.savePages(gatsbyPageObjects); // gatsbyPageObjects is always an array regardless of whether it's just one page
+					this.saveSitesToPublish(sitesToPublish);
+					this.saveBuildProcessData(BUILD_PROCESS_DATA);
+					this.saveCreatedPages(CREATED_PAGES);
+				};
+
+				const griddoCxThreads = parseInt(GRIDDO_API_CONCURRENCY_COUNT || "10");
+
+				// Create the pLimit array of promises
+				const limit = pLimit(griddoCxThreads);
+				const pagesToRender = validPagesIds.map((id: number) =>
+					limit(() => fetchPageAndSaveToStore(id))
+				);
+
+				await Promise.all(pagesToRender);
+			});
+
+			await Promise.all(promisesOfSitesToPublish);
+		} catch (e) {
+			const error = e as { message: string };
+
+			console.error(error.message);
+
+			process.exit(1);
+		}
+	}
+
+	/**
+	 * Get the pages saved previously from the Store.
+	 */
+	public static getPages() {
+		if (this.mode === "file") {
+			return this.getPagesFromFiles();
+		} else {
+			return this.getPagesFromMemory();
+		}
+	}
+
+	/**
+	 * Save the pages into the Store.
+	 *
+	 * @param pages An array of pages to save in the store.
+	 */
+	public static async savePages(pages: Array<GatsbyPageObject>) {
+		if (this.mode === "file") {
+			await this.savePagesToFiles(pages);
+		} else {
+			this.savePagesToMemory(pages);
+		}
+	}
+
+	/**
+	 * Save site to publish data in the Store.
+	 *
+	 * @param sites An array of sites to be published.
+	 */
+	public static saveSitesToPublish(sites: Array<Site>) {
+		this.sitesToPublish = sites;
+	}
+
+	/**
+	 * Save the build process data in the Store.
+	 *
+	 * @param buildProcessData An object that represents a build process data.
+	 */
+	public static saveBuildProcessData(buildProcessData: BuildProcessData) {
+		this.buildProcessData = buildProcessData;
+	}
+
+	/**
+	 * Save the created pages in the Store.
+	 *
+	 * @param pages An array of pages ids.
+	 */
+	public static saveCreatedPages(pages: Array<number>) {
+		this.createdPages = pages;
+	}
+
+	/**
+	 * Get the build metadata from the Store.
+	 */
+	public static getMetadata() {
+		return {
+			buildProcessData: this.buildProcessData,
+			createdPages: this.createdPages,
+			sitesToPublish: this.sitesToPublish,
+		};
+	}
+
+	/**
+	 * Return an iterator with pages read from the file system.
+	 */
+	private static *getPagesFromFiles() {
+		const pagesDirPath = path.resolve(this.basePath, "pages");
+		const jsonFilePaths = walk(pagesDirPath).filter(
+			(file) => path.extname(file) === ".json"
+		);
+
+		for (const filePath of jsonFilePaths) {
+			const page = require(filePath) as GatsbyPageObject;
+			page.size = fs.statSync(filePath).size / 1024;
+			yield { ...page };
+		}
+	}
+
+	/**
+	 * Get the pages from the Store from the memory.
+	 */
+	private static getPagesFromMemory() {
+		return this.pages;
+	}
+
+	/**
+	 * Save pages to the file system.
+	 *
+	 * @param pages An array of final pages to save into the file system.
+	 */
+	private static async savePagesToFiles(pages: Array<GatsbyPageObject>) {
+		const promises = pages.map(async (pageObject) => {
+			const filename = `${createSha256(pageObject.path)}.json`;
+			const fileContent = JSON.stringify(pageObject);
+			const filePath = path.resolve(this.basePath, "pages", filename);
+			await fsp.writeFile(filePath, fileContent);
+		});
+
+		await Promise.all(promises);
+	}
+
+	/**
+	 * Save pages to the memory.
+	 *
+	 * @param pages An array of final pages to save into the memory
+	 */
+	private static savePagesToMemory(pages: Array<GatsbyPageObject>) {
+		const pagesWithSize = pages.map((page) => {
+			const size = v8.serialize(page).byteLength / 1024;
+
+			return { ...page, size };
+		});
+		this.pages.push(...pagesWithSize);
+	}
+}
+
+export { StoreService };

package/src/types/api.ts

@@ -0,0 +1,155 @@
+// Types
+import type { Core, Fields } from "@griddo/core";
+import { Domains, Robots, Settings } from "./global";
+import type { APIPageObject } from "./pages";
+import type { Site } from "./sites";
+
+/** Describes the props of the body object for the API */
+interface Body {
+	/** The title of the page. */
+	title?: string;
+	/** The description of the page. */
+	description?: string;
+	/** The URL of the page's image. */
+	image?: string;
+	/** The ID of the page. */
+	pageId?: number;
+	/** The ID of the page's language. */
+	languageId?: number;
+	/** The ID of the page's site. */
+	siteId?: number;
+	/** The URL of the page. */
+	url?: string;
+	/** The template of the page. */
+	template?: string | null | number;
+	/** The content of the page. */
+	content?: string;
+	/** Additional properties. */
+	[key: string]: unknown;
+}
+
+/** Describes an item in a sitemap. */
+interface SitemapItem {
+	/** The URL of the page. */
+	loc: string;
+	/** The date the page was last modified. */
+	lastmod: string;
+	/** The priority of the page. */
+	priority: number;
+}
+
+/** Describes a URL. */
+interface URL {
+	/** The path of the URL. */
+	path: string;
+	/** The domain of the URL. */
+	domain: string;
+	/** The home of the URL. */
+	home: string;
+}
+
+/** Describes the items returned by a sitemap response. */
+export type SitemapItems = Record<string, Array<SitemapItem>>;
+
+/** Describes the response object returned by a Griddo distributor. */
+export type DistributorResponse = Fields.QueriedData<unknown>;
+
+/** Describes the response object returned by socials from Griddo. */
+export type SocialsResponse = Record<Core.SocialName, string>;
+
+/** Describes the response object returned by a sitemap. */
+export interface SitemapAPIResponse {
+	/** Number of items */
+	totalItems: number;
+	/** Page id */
+	page: number;
+	/** Item response by API */
+	items: SitemapItems;
+	/** Url  */
+	url: URL;
+}
+
+/** Descripbes Language responses from Griddo API. */
+export interface LanguagesResponse {
+	/** Total of languages */
+	totalItems: number;
+	/** Array of languages objects */
+	items: Array<Core.SiteLanguage>;
+}
+
+/** Describes common props for api responses. */
+interface APIRequest {
+	/** The URL of the API endpoint. */
+	endpoint: string;
+	/** The parameters to be sent in the request body. */
+	body?: Body;
+	/** Indicates whether to use the cache to get the response from the API. */
+	cached?: unknown;
+	/** Number of connection attempts (in case it fails on the first attempt). */
+	attempt?: number;
+}
+
+/** Type with the POST request properties. */
+export interface PostAPI extends APIRequest {
+	headers?: { lang?: number } & Record<string, unknown>;
+}
+
+/** Type with the GET request properties. */
+export type GetAPI = APIRequest;
+
+/** Type with the PUT request properties. */
+export type PutAPI = APIRequest;
+
+/** Response type for the start render API call */
+export interface StartPageRenderResponse {
+	/** Site hashes to publish */
+	publishIds: Array<number>;
+	/** Current site hash */
+	siteHash: string | null;
+	/** Site hashes to unpublish */
+	unpublishHashes: Array<string>;
+}
+
+/** Response type for the end render API call */
+export interface EndPageInfoResponse {
+	/** Response code */
+	code: number;
+	/** Response message */
+	message: string;
+}
+
+/** Describes a response type for GET all sites */
+export type AllSitesReponse = Array<Site>;
+
+/** Describes a response type for GET all pages */
+export type AllPagesResponse = Array<APIPageObject>;
+
+/** Describes a response type for GET one page */
+export type PageResponse = APIPageObject;
+
+/** Error type for try/catch */
+export interface Error {
+	response: {
+		status: number;
+		statusText: string;
+		data: Record<string, unknown>;
+	};
+	message: string;
+	stack: string;
+}
+
+/** All types for the API response used in the api functions get, put, post */
+export type APIResponses =
+	| Domains
+	| Robots
+	| Settings
+	| AllSitesReponse
+	| Site
+	| LanguagesResponse
+	| StartPageRenderResponse
+	| SitemapAPIResponse
+	| SocialsResponse
+	| PageResponse
+	| EndPageInfoResponse
+	| DistributorResponse
+	| AllPagesResponse;