@griddo/cx 11.7.6 → 11.7.7-rc.0

package/exporter/services/store.ts

@@ -1,4 +1,4 @@
-import type { BuildProcessData } from "../types/global";
+import type { BuildProcessData, Settings } from "../types/global";
 import type {
 	GriddoListPage,
 	GriddoMultiPage,
@@ -6,20 +6,24 @@ import type {
 	GriddoSinglePage,
 	PageAdditionalInfo,
 } from "../types/pages";
+import type { RenderMode } from "../types/render";
 
-import fs from "node:fs";
+import fsp from "node:fs/promises";
 import path from "node:path";
 
 import pLimit from "p-limit";
 
-import { envs } from "../constants";
-import { NavigationService } from "./navigation";
-import { getReferenceFieldData } from "./reference-fields";
-import { getAllSettings } from "./settings";
-import { getPage } from "./sites";
+import { SETTINGS } from "../constants/endpoints";
+import {
+	GRIDDO_API_CONCURRENCY_COUNT,
+	GRIDDO_API_URL,
+	GRIDDO_PUBLIC_API_URL,
+	GRIDDO_REACT_APP_INSTANCE,
+} from "../constants/envs";
+import { get } from "../utils/api";
 import { updatedSiteHash } from "../utils/cache";
-import { getConfig, removeAllSiteDirsFromStore } from "../utils/core-utils";
-import { buildLog, listSitesLog } from "../utils/loggin";
+import { removeAllSiteDirsFromStore } from "../utils/core-utils";
+import { infoLog, listSitesLog } from "../utils/loggin";
 import {
 	createGriddoListPages,
 	createGriddoMultiPages,
@@ -29,26 +33,38 @@ import {
 } from "../utils/pages";
 import { getSiteData, getSitesToRender, unpublishSites } from "../utils/sites";
 import {
-	getMissingPublishedPagesInStore,
 	getZombiePagesInStore,
 	removeOrphanSites,
 	removeSitePagesFromStore,
 	saveRenderInfoInStore,
 	saveSitePagesInStore,
 } from "../utils/store";
+import { NavigationService } from "./navigation";
+import { getReferenceFieldData } from "./reference-fields";
+import { getPage } from "./sites";
 
-const renderId = new Date().valueOf().toString();
-const { griddoVersion, paths } = getConfig();
-const { __cx } = paths();
-const storeDir = path.join(__cx, "store");
+const renderId = Date.now().toString();
+
+function guessSiteNeedsToBeRenderedFromScratch() {
+	return false;
+}
 
 /**
  * Fetch, process and save object pages and sites data into the file system to
  * be consumed by other services (Griddo itself, Adapters, etc.)
  */
-async function createStore(domain: string) {
-	console.info(`API calls with ${envs.GRIDDO_API_CONCURRENCY_COUNT} threads`);
-	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
+async function createStore(options: {
+	domain: string;
+	renderMode: RenderMode;
+	griddoVersion: string;
+	basePath: string;
+}) {
+	console.info(`API calls with ${GRIDDO_API_CONCURRENCY_COUNT} threads`);
+	console.info(`API URL ${GRIDDO_API_URL as string}`);
+
+	const { renderMode, domain, griddoVersion, basePath } = options;
+	const renderFromScratch = renderMode === "FROM_SCRATCH";
+	const storeDir = path.join(basePath, "store");
 
 	try {
 		// Vars to save later in the report file
@@ -58,8 +74,7 @@ async function createStore(domain: string) {
 		// Get sites objects to publish and unpublish from this domain
 		const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
 
-		const domainHasSites =
-			sitesToPublish.length > 0 || sitesToUnpublish.length > 0;
+		const domainHasSites = sitesToPublish.length > 0 || sitesToUnpublish.length > 0;
 
 		if (!domainHasSites) {
 			console.warn(`There are no sites to update in the domain ${domain}`);
@@ -69,20 +84,39 @@ async function createStore(domain: string) {
 		listSitesLog("Sites to unpublish:", sitesToUnpublish);
 
 		// Send unpublished sites to API.
-		// @todo ¿mover esta llamada al final del proceso de render?
-		// Aquí también se están eliminado físicamente los archivos json del
-		// store de los sites para despublicar.
 		await unpublishSites(sitesToUnpublish);
 
 		// Eliminamos posibles sites que estén en el store pero que no deberían
 		// porque han sido cambiados de dominio. Si han sido despublicados ya se
 		// despublican bien porque entran en `sitesToUnpublish`
-		removeOrphanSites(sitesToPublish, domain);
+		await removeOrphanSites(sitesToPublish, domain);
+
+		const allPagesToRemoveFromBuild: Array<number> = [];
+		// let numberOfFinalActivePages: Array<number> = [];
+
+		for (const site of sitesToUnpublish) {
+			const { pagesStatus } = site;
+
+			// Añadimos a allPagesToRemoveFromBuild todas las páginas de un site
+			// para despublicar.
+			allPagesToRemoveFromBuild.push(
+				...pagesStatus.active,
+				...pagesStatus.offlinePending,
+				...pagesStatus.uploadPending,
+				...pagesStatus.deleted,
+				...pagesStatus.offline,
+			);
+		}
 
-		// Solo los sites to publish...
 		for (const site of sitesToPublish) {
 			const { id: siteId, slug: siteSlug, theme, favicon, pagesStatus } = site;
 			const siteDirName = siteId.toString();
+			// numberOfFinalActivePages.push(
+			// 	...pagesStatus.active,
+			// 	...pagesStatus.uploadPending,
+			// );
+
+			allPagesToRemoveFromBuild.push(...pagesStatus.offlinePending, ...pagesStatus.deleted);
 
 			const {
 				siteInfo,
@@ -104,7 +138,7 @@ async function createStore(domain: string) {
 				avoidSelfReferenceCanonicals,
 				avoidHrefLangXDefault,
 				avoidDebugMetas,
-			} = await getAllSettings();
+			} = await get<Settings>({ endpoint: SETTINGS });
 
 			buildProcessData[siteId] = {
 				siteHash,
@@ -118,7 +152,7 @@ async function createStore(domain: string) {
 
 			site.languages = siteLangs;
 
-			const shouldUpdateSite = updatedSiteHash(siteId, siteHash);
+			const shouldUpdateSite = await updatedSiteHash(siteId, siteHash);
 
 			const siteScript = siteInfo.siteScript;
 
@@ -129,9 +163,9 @@ async function createStore(domain: string) {
 			};
 
 			const additionalInfo = {
-				baseUrl: envs.GRIDDO_API_URL,
-				publicBaseUrl: envs.GRIDDO_PUBLIC_API_URL,
-				instance: envs.GRIDDO_REACT_APP_INSTANCE,
+				baseUrl: GRIDDO_API_URL,
+				publicBaseUrl: GRIDDO_PUBLIC_API_URL,
+				instance: GRIDDO_REACT_APP_INSTANCE,
 				siteSlug,
 				theme,
 				siteMetadata,
@@ -151,10 +185,10 @@ async function createStore(domain: string) {
 				siteScript,
 			};
 
-			buildLog(`${site.name} site`);
+			infoLog(`Getting pages from ${site.name} site`);
 
 			/// Creates the store directory for each site using the id
-			fs.mkdirSync(path.join(storeDir, siteDirName), {
+			await fsp.mkdir(path.join(storeDir, siteDirName), {
 				recursive: true,
 			});
 
@@ -163,10 +197,7 @@ async function createStore(domain: string) {
 			// -------------------------------------------------------------------------
 			// Async function that process every page for one site to use later in the
 			// `Promise.all` with pLimit.
-			const fetchSitePageAndSaveInStore = async (
-				siteIdName: string,
-				pageId: number,
-			) => {
+			const fetchSitePageAndSaveInStore = async (siteIdName: string, pageId: number) => {
 				// Here will be store every page returned by the API and processed
 				let griddoPageObjects: Array<GriddoPageObject> = [];
 
@@ -185,9 +216,7 @@ async function createStore(domain: string) {
 				// SHAME: This new pageAdditionalInfo needs to be a copy because
 				// additionalInfo referenced from outside this function and
 				// its used by other pages.
-				const pageAdditionalInfo = JSON.parse(
-					JSON.stringify(additionalInfo),
-				) as PageAdditionalInfo;
+				const pageAdditionalInfo = JSON.parse(JSON.stringify(additionalInfo)) as PageAdditionalInfo;
 
 				// Updated with navigations (header & footer)
 				pageAdditionalInfo.navigations = NavService.getPageNavigations(page);
@@ -217,8 +246,7 @@ async function createStore(domain: string) {
 				// - Single: just one page
 
 				// Griddo page types
-				const isStaticListPage =
-					page?.mode === "list" || page?.mode === "paginated-data";
+				const isStaticListPage = page?.mode === "list" || page?.mode === "paginated-data";
 				const isMultiPage = !isStaticListPage && multiPageElements;
 				const isSinglePage = !isMultiPage && !isStaticListPage;
 
@@ -234,10 +262,12 @@ async function createStore(domain: string) {
 					} as GriddoListPage;
 
 					// pageObjects = await createGriddoListPages({ adapter: "Gatsby" })
-					griddoPageObjects = await createGriddoListPages(
-						griddoListPage,
-						pageAdditionalInfo,
-					);
+					griddoPageObjects = await createGriddoListPages(griddoListPage, pageAdditionalInfo);
+
+					// ¿Por qué la añadimos a las páginas para borrar?
+					// Porque este tipo de páginas no está manejada por API y se crean varias páginas con el mismo ID.
+					// Esto hace que no se pueda rastrear bien el render.
+					allPagesToRemoveFromBuild.push(page.id);
 				}
 
 				// >> Multi-page <<
@@ -249,10 +279,12 @@ async function createStore(domain: string) {
 					griddoMultipage.multiPageElements = multiPageElements;
 					griddoMultipage.defaultLang = defaultLang;
 
-					griddoPageObjects = await createGriddoMultiPages(
-						griddoMultipage,
-						pageAdditionalInfo,
-					);
+					griddoPageObjects = await createGriddoMultiPages(griddoMultipage, pageAdditionalInfo);
+
+					// ¿Por qué la añadimos a las páginas para borrar?
+					// Porque este tipo de páginas no está manejada por API y se crean varias páginas con el mismo ID.
+					// Esto hace que no se pueda rastrear bien el render.
+					allPagesToRemoveFromBuild.push(page.id);
 				}
 
 				// >> Single template <<
@@ -263,9 +295,7 @@ async function createStore(domain: string) {
 					griddoSinglePage.template = template;
 					griddoSinglePage.defaultLang = defaultLang;
 
-					griddoPageObjects = [
-						await createGriddoSinglePage(griddoSinglePage, pageAdditionalInfo),
-					];
+					griddoPageObjects = [await createGriddoSinglePage(griddoSinglePage, pageAdditionalInfo)];
 				}
 
 				// Upload only the valid pages hashes or ids of pages that has
@@ -276,25 +306,26 @@ async function createStore(domain: string) {
 				}
 
 				// Save build data to store
-				saveSitePagesInStore(siteIdName, griddoPageObjects);
+				await saveSitePagesInStore(siteIdName, griddoPageObjects);
 			};
 
-			// Pages that needs to be fetched from the API and written to the
-			// store.
-			const pagesToFetchFromAPI = [
-				// These pages are news or has been updated.
-				...pagesStatus.uploadPending,
-				// You should take into account the pages that, even though they are
-				// already published and therefore should be avoided from requesting
-				// them to the API, are not in the store, probably due to a failed
-				// render. In that case, you have to request them again from the API
-				// to include them in the store.
-				...getMissingPublishedPagesInStore(siteDirName, pagesStatus.active),
-			];
+			// Pages that needs to be fetched from the API and written to the store.
+			const pagesToFetchFromAPI = pagesStatus.uploadPending;
+
+			// Esto es necesario porque en la BBDD las páginas pueden estar
+			// marcadas como activas (online) pero si es una máquina nueva hace
+			// falta hacerlas de nuevo...
+			// const sitePendingOfPublishing = site.isPublished && site.shouldBeUpdated;
+			const sitePendingOfPublishing = guessSiteNeedsToBeRenderedFromScratch();
+			// @todo: necesito tener un sistema par publicar todo de un site (cuando un site esté pending to publish)
+			// algo como site.status = pending | published | unpublished...
+			if (renderFromScratch || sitePendingOfPublishing) {
+				pagesToFetchFromAPI.push(...pagesStatus.active);
+			}
 
 			// Create pages to the store. First requesting from API,
 			// transform, etc., and then writing them to disk.
-			const limit = pLimit(envs.GRIDDO_API_CONCURRENCY_COUNT);
+			const limit = pLimit(GRIDDO_API_CONCURRENCY_COUNT);
 			const pagesToStore = pagesToFetchFromAPI.map((id: number) =>
 				limit(() => fetchSitePageAndSaveInStore(siteDirName, id)),
 			);
@@ -302,13 +333,13 @@ async function createStore(domain: string) {
 			await Promise.all(pagesToStore);
 
 			// @ts-expect-error Mutate pagesStatus for reading pourposes.
-			pagesStatus.zombie = getZombiePagesInStore(siteDirName, [
+			pagesStatus.zombie = await getZombiePagesInStore(siteDirName, [
 				...pagesStatus.active,
 				...pagesStatus.uploadPending,
 			]);
 
 			// Delete pages from the store removing them from disk.
-			removeSitePagesFromStore(siteDirName, [
+			await removeSitePagesFromStore(siteDirName, [
 				pagesStatus.deleted,
 				pagesStatus.offline,
 				pagesStatus.offlinePending,
@@ -336,20 +367,31 @@ async function createStore(domain: string) {
 
 		if (domainHasSites) {
 			// ( 1 )
-			saveRenderInfoInStore({
-				buildProcessData,
-				createdPages,
-				sitesToPublish,
-			});
+			await saveRenderInfoInStore(
+				{
+					buildProcessData,
+					createdPages,
+					sitesToPublish,
+				},
+				domain,
+			);
 		} else {
 			// ( 2 )
-			removeAllSiteDirsFromStore();
-			saveRenderInfoInStore({
-				buildProcessData,
-				createdPages: [],
-				sitesToPublish: [],
-			});
+			await removeAllSiteDirsFromStore();
+			await saveRenderInfoInStore(
+				{
+					buildProcessData,
+					createdPages: [],
+					sitesToPublish: [],
+				},
+				domain,
+			);
 		}
+
+		return {
+			pagesToCreate: createdPages,
+			pagesToDelete: allPagesToRemoveFromBuild,
+		};
 	} catch (e) {
 		const error = e as { message: string };
 		console.error(error.message);

package/exporter/types/api.ts

@@ -1,7 +1,7 @@
+import type { Core, Fields } from "@griddo/core";
 import type { Domains, Robots, Settings } from "./global";
 import type { APIPageObject } from "./pages";
 import type { Site } from "./sites";
-import type { Core, Fields } from "@griddo/core";
 
 /** EndSiteRender Body */
 export type EndSiteRenderBody = {
@@ -29,29 +29,28 @@ export interface ReferenceFieldBody {
 }
 
 /** Describes the props of the body object for the get page */
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-interface PageBody {
-	/** The title of the page. */
-	title?: string;
-	/** The description of the page. */
-	description?: string;
-	/** The URL of the page's image. */
-	image?: string;
-	/** The ID of the page. */
-	pageId?: number;
-	/** The ID of the page's language. */
-	languageId?: number;
-	/** The ID of the page's site. */
-	siteId?: number;
-	/** The URL of the page. */
-	url?: string;
-	/** The template of the page. */
-	template?: string | null | number;
-	/** The content of the page. */
-	content?: string;
-	/** Additional properties. */
-	[key: string]: unknown;
-}
+// interface PageBody {
+// 	/** The title of the page. */
+// 	title?: string;
+// 	/** The description of the page. */
+// 	description?: string;
+// 	/** The URL of the page's image. */
+// 	image?: string;
+// 	/** The ID of the page. */
+// 	pageId?: number;
+// 	/** The ID of the page's language. */
+// 	languageId?: number;
+// 	/** The ID of the page's site. */
+// 	siteId?: number;
+// 	/** The URL of the page. */
+// 	url?: string;
+// 	/** The template of the page. */
+// 	template?: string | null | number;
+// 	/** The content of the page. */
+// 	content?: string;
+// 	/** Additional properties. */
+// 	[key: string]: unknown;
+// }
 
 /** Describes an item in a sitemap. */
 interface SitemapItem {
@@ -120,10 +119,8 @@ export interface APIRequest {
 	attempt?: number;
 	/**
 	 * Headers for the post api fetch
-	 * @shame any
-	 * @todo type this correctly from axios types
 	 */
-	headers?: any; // Record<string, unknown>;
+	headers?: Record<string, unknown>;
 	/* Save results in apiCache folder */
 	useApiCacheDir?: boolean;
 }

package/exporter/types/global.ts

@@ -1,6 +1,6 @@
+import type { Core, Fields } from "@griddo/core";
 import type { EndSiteRenderBody } from "./api";
 import type { Site } from "./sites";
-import type { Core, Fields } from "@griddo/core";
 
 interface Settings {
 	apiVersion?: string;
@@ -70,9 +70,7 @@ interface RenderInfo {
 	sitesToPublish: Array<Site>;
 }
 
-type LifeCycleSteps = Array<
-	(...args: Array<unknown>) => unknown | Promise<unknown>
->;
+type LifeCycleAction = (...args: Array<unknown>) => void | Promise<void>;
 
 type LifeCyclesNames =
 	| "Init"
@@ -82,12 +80,11 @@ type LifeCyclesNames =
 	| "SSG"
 	| "Relocation"
 	| "Meta"
-	| "Archive"
+	| "Sync"
 	| "Clean"
 	| "Close"
 	| "HealthCheck"
-	| "Close"
-	| "__DEBUG__";
+	| "Close";
 
 type PlaceholderPath =
 	| "__exports"
@@ -97,13 +94,12 @@ type PlaceholderPath =
 	| "__components"
 	| "__root"
 	| "__exports_dist"
-	| "__cx_dist";
+	| "__sites";
 
 interface CXConfig {
 	proDomain: string;
 	griddoVersion: string;
 	buildReportFileName: string;
-	paths: (domain?: string) => Record<PlaceholderPath, string>;
 }
 
 interface AIEmbeddingsResponse {
@@ -116,6 +112,7 @@ interface Artifacts {
 	cacheables: Array<string>;
 	initials: Array<string>;
 	archivables: Array<string>;
+	restaurable: Array<string>;
 }
 
 export type {
@@ -134,5 +131,5 @@ export type {
 	Robot,
 	Robots,
 	Settings,
-	LifeCycleSteps,
+	LifeCycleAction,
 };

package/exporter/types/navigation.ts

@@ -21,5 +21,5 @@ export type Footer = {
 	setAsDefault: boolean;
 	theme: null | string;
 	title: string;
-	type: "header";
+	type: "footer";
 };

package/exporter/types/pages.ts

@@ -1,7 +1,7 @@
+import type { Core, Fields } from "@griddo/core";
 import type { SocialsResponse } from "./api";
 import type { Settings } from "./global";
 import type { Site } from "./sites";
-import type { Core, Fields } from "@griddo/core";
 
 // TODO: In @griddo/core the type Core.Page has header/footer as React-Types,
 // but API return `number | null`.
@@ -156,8 +156,7 @@ export interface Dimensions {
 	dimensionsSelect?: Array<string>;
 }
 
-export interface GatsbyPageTemplateProps
-	extends Omit<GriddoPageObject, "context"> {
+export interface GatsbyPageTemplateProps extends Omit<GriddoPageObject, "context"> {
 	pageContext: GriddoPageObject["context"] & {
 		page: Core.Page;
 	};

package/exporter/types/render.ts

@@ -0,0 +1,59 @@
+import type { RenderInfo } from "./global";
+
+const RENDER_MODE = {
+	FROM_SCRATCH: "FROM_SCRATCH",
+	INCREMENTAL: "INCREMENTAL",
+	IDLE: "IDLE",
+} as const;
+
+type RenderMode = (typeof RENDER_MODE)[keyof typeof RENDER_MODE];
+
+export interface RenderModeTuple {
+	renderMode: RenderMode;
+	reason?: string;
+}
+
+type DomainLike = string;
+
+type RenderDB = {
+	proDomain: string;
+	griddoVersion: string;
+	buildReportFileName: string;
+	sortedDomains: Array<DomainLike>;
+	paths: {
+		root: string;
+		cx: string;
+		ssg: string;
+		cxCache: string;
+		components: string;
+		exportsDir: string;
+	};
+	domains: {
+		[key: DomainLike]: {
+			renderMode?: RenderMode;
+			shouldBeRendered?: boolean;
+			isRendering?: boolean;
+			renderInfo?: RenderInfo;
+			renderModeReason?: string;
+		};
+	};
+};
+
+interface Report {
+	authControl:
+		| {
+				Authorization: string;
+				"Cache-Control": string;
+				lang?: string | undefined;
+		  }
+		| undefined;
+	sites: Array<{
+		siteId: number;
+		publishHashes: Array<string>;
+		siteHash: string | null;
+		unpublishHashes: Array<string>;
+		publishPagesIds: Array<number>;
+	}>;
+}
+
+export { RENDER_MODE, type RenderDB, type RenderMode, type Report };

package/exporter/types/sites.ts

@@ -1,6 +1,6 @@
+import type { Core } from "@griddo/core";
 import type { SocialsResponse } from "./api";
 import type { Footer, Header } from "./navigation";
-import type { Core } from "@griddo/core";
 
 /**
  * Describes a Griddo site object from API.
@@ -43,7 +43,6 @@ export interface Site
 	 */
 	pages: Array<number>;
 	onlinePages: Array<number>;
-	changedPages: Array<number>;
 	published: string;
 	rendering: boolean;
 	renderingHours: number;