@griddo/cx 1.75.222 → 1.75.224

package/gatsby-node.ts

@@ -1,4 +1,3 @@
-import type { StoreMode } from "./src/types/global";
 import type { GatsbyNode } from "gatsby";
 
 import path from "path";
@@ -9,20 +8,24 @@ import { RobotsService } from "./src/services/robots";
 import { StoreService } from "./src/services/store";
 import { initCache } from "./src/utils/cache";
 import { prepareStaticFolder, updateDist } from "./src/utils/folders";
-import { logInfo, logPageSize, msToSec, splash } from "./src/utils/shared";
+import {
+	logInfo,
+	logPageSize,
+	msToSec,
+	// sanitizeApiCache,
+	splash,
+} from "./src/utils/shared";
 import { generateBuildReport, generateSitemaps } from "./src/utils/sites";
 
 dotenv.config();
 
-// Envs
-const GRIDDO_STORE_MODE = (process.env.GRIDDO_STORE_MODE ||
-	"file") as StoreMode;
-
 // Consts
 const PUBLIC_FOLDER = path.resolve(__dirname, "./public");
 const STORE_DIR_PATH = path.resolve(__dirname, "./store/");
+// const API_CACHE_FOLDER = path.resolve(__dirname, "./apiCache");
 const REPORT_FILE = path.resolve(PUBLIC_FOLDER, "__build-report__.json");
-const envVars: Record<string, any> = {};
+const griddoEnvs: Record<string, any> = {};
+const otherEnvs: Record<string, any> = {};
 
 // -----------------------------------------------------------------------------
 // onPreInit
@@ -34,21 +37,26 @@ export const onPreInit = async () => {
 
 	Object.keys(process.env)
 		.filter((key) => key.startsWith("GRIDDO"))
-		.forEach((key) => (envVars[key] = process.env[key]));
+		.forEach((key) => (griddoEnvs[key] = process.env[key]));
+
+	Object.keys(process.env)
+		.filter((key) => !key.startsWith("GRIDDO"))
+		.forEach((key) => (otherEnvs[key] = process.env[key]));
 
 	console.log("%%%");
 	console.log("- GRIDDO ENVS VARS -");
-	console.log(envVars);
-	console.log("- ALL ENVS VARS -");
-	console.log(process.env);
+	console.log(griddoEnvs);
+	console.log("- OTHER ENVS -");
+	console.log(otherEnvs);
 	console.log("%%%");
 
-	StoreService.init(STORE_DIR_PATH, {
-		sanitize: true,
-		mode: GRIDDO_STORE_MODE,
-	});
+	StoreService.init(STORE_DIR_PATH, { sanitize: true });
 	await StoreService.createBuildSource();
 
+	// Remove old cahe page files
+	// TODO: Use disabled
+	// sanitizeApiCache(API_CACHE_FOLDER);
+
 	console.log("\n✨ onPreInit\n");
 };
 

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@griddo/cx",
 	"description": "Griddo SSG based on Gatsby",
-	"version": "1.75.222",
+	"version": "1.75.224",
 	"authors": [
 		"Álvaro Sánchez' <alvaro.sanches@secuoyas.com>",
 		"Diego M. Béjar <diego.bejar@secuoyas.com>",
@@ -64,7 +64,7 @@
 		"react-helmet": "^6.0.0"
 	},
 	"devDependencies": {
-		"@griddo/eslint-config-back": "^1.75.222",
+		"@griddo/eslint-config-back": "^1.75.224",
 		"@types/babel__core": "^7.20.0",
 		"@types/babel__preset-env": "^7.9.2",
 		"@types/csvtojson": "^2.0.0",
@@ -112,5 +112,5 @@
 	"publishConfig": {
 		"access": "public"
 	},
-	"gitHead": "f417b394d6830b80fc15f2fd691b620a34d74b2e"
+	"gitHead": "1c4e2c06dd6fbee54a37eba2c8f2ebd27b38781f"
 }

package/src/services/distributors.ts

@@ -78,14 +78,14 @@ class DistributorService {
 	 * @param props
 	 * @param props.page The Page object
 	 * @param props.component.data The ReferenceField, always in the data prop.
-	 * @param props.cached Boolean that indicates cache use.
+	 * @param props.cacheKey A cache key to manage cached files or using fetch
 	 * @returns The ContentType data.
 	 */
 	static async fetchContentTypeData(props: FetchDataProps) {
 		const {
 			page,
 			component: { data },
-			cached,
+			cacheKey,
 		} = props;
 
 		// Distrubutor with `hasDistributorDat: true` but without `data` prop
@@ -98,7 +98,11 @@ class DistributorService {
 		}
 
 		const body = this.getBody(data, page);
-		const response = await SitesService.getDistributorData(page, body, cached);
+		const response = await SitesService.getDistributorData(
+			page,
+			body,
+			cacheKey
+		);
 
 		return response;
 	}
@@ -108,14 +112,15 @@ class DistributorService {
 	 *
 	 * @param props
 	 * @param props.page The APIPage object
-	 * @param props.cached Boolean that indicates cache use.
+	 * @param props.cacheKey A cache key to manage cached files or using fetch
 	 */
 	static async getDistributorData({
 		page,
-		cached = false,
+		cacheKey = "",
 	}: {
 		page: APIPageObject;
-		cached: boolean;
+		/** Reference id to manage cache between renders. */
+		cacheKey: string;
 	}) {
 		try {
 			const { template } = page;
@@ -155,7 +160,7 @@ class DistributorService {
 					if (component.hasDistributorData) {
 						component.queriedItems = await this.fetchContentTypeData({
 							page,
-							cached,
+							cacheKey,
 							component,
 						});
 					}

package/src/services/sites.ts

@@ -52,13 +52,13 @@ class SitesService {
 	/**
 	 * Fetch a page object from API.
 	 */
-	static async getPage(id: number, cached: unknown) {
+	static async getPage(id: number, cacheKey: string) {
 		const {
 			GET_PAGE: [prefix, suffix],
 		} = ENDPOINTS;
 		const response = await get<PageResponse>({
 			endpoint: `${prefix}${id}${suffix}`,
-			cached,
+			cacheKey,
 		});
 
 		return response;
@@ -67,13 +67,13 @@ class SitesService {
 	/**
 	 * Get site info
 	 */
-	static async getInfo(id: number, cached: boolean) {
+	static async getInfo(id: number, cacheKey = "") {
 		const {
 			INFO: [prefix, suffix],
 		} = ENDPOINTS;
 		const response = await get<Site>({
 			endpoint: `${prefix}${id}${suffix}`,
-			cached,
+			cacheKey,
 		});
 
 		return response;
@@ -82,13 +82,13 @@ class SitesService {
 	/**
 	 * TODO: JSDoc
 	 */
-	static async getLanguages(id: number, cached = false) {
+	static async getLanguages(id: number, cacheKey = "") {
 		const {
 			LANGUAGES: [prefix, suffix],
 		} = ENDPOINTS;
 		const response = await get<LanguagesResponse>({
 			endpoint: `${prefix}${id}${suffix}`,
-			cached,
+			cacheKey,
 		});
 
 		return response;
@@ -129,7 +129,7 @@ class SitesService {
 	static async getDistributorData(
 		page: Core.Page,
 		body: DistributorBody,
-		cached: boolean
+		cacheKey: string
 	) {
 		const {
 			GET_DISTRIBUTOR_DATA: [prefix, suffix],
@@ -139,7 +139,7 @@ class SitesService {
 			endpoint: `${prefix}${id}${suffix}`,
 			body,
 			headers: { lang },
-			cached,
+			cacheKey,
 		});
 
 		return response;
@@ -162,13 +162,13 @@ class SitesService {
 	/**
 	 * TODO: JSDoc
 	 */
-	static async getSocials(id: number, cached: boolean) {
+	static async getSocials(id: number, cacheKey = "") {
 		const {
 			SOCIALS: [prefix, suffix],
 		} = ENDPOINTS;
 		const response = await get<SocialsResponse>({
 			endpoint: `${prefix}${id}${suffix}`,
-			cached,
+			cacheKey,
 		});
 
 		return response;
@@ -177,13 +177,13 @@ class SitesService {
 	/**
 	 * TODO: JSDoc
 	 */
-	static async getPages(id: number, cached = false) {
+	static async getPages(id: number, cacheKey = "") {
 		const {
 			GET_PAGES: [prefix, suffix],
 		} = ENDPOINTS;
 		const response = await get<AllPagesResponse>({
 			endpoint: `${prefix}${id}${suffix}`,
-			cached,
+			cacheKey,
 		});
 
 		return response;

package/src/services/store.ts

@@ -21,17 +21,17 @@
 //                        │      Build metadata     │                         //
 //                        └─────────────────────────┘                         //
 
-import type { BuildProcessData, StoreMode } from "../types/global";
+import type { BuildProcessData } from "../types/global";
 import type { GatsbyPageObject, PageAdditionalInfo } from "../types/pages";
 import type { Site } from "../types/sites";
 
 import fs from "fs";
 import fsp from "fs/promises";
 import path from "path";
-import v8 from "v8";
 
 import chalk from "chalk";
 import dotenv from "dotenv";
+import fsx from "fs-extra";
 import pLimit from "p-limit";
 
 import { DistributorService } from "./distributors";
@@ -73,15 +73,11 @@ const UNWANTED_PAGE_PROPS = ["editorID", "parentEditorID"];
  * Store service to fetch, process and save object pages and sites data to be
  * consumed by other services (Griddo itself, Gatsby, etc.)
  *
- * Modes:
- * - { File mode } save the objects as json files in the file system (json DB)
- * - { Memory mode } save the objects in an array.
- *
  * @example
  * import { StoreService } from "./store"
  *
  * // Init the store service.
- * StoreService.init("foo/pages", { mode: "file", ... })
+ * StoreService.init("foo/pages", { sanitize: true })
  * // Save the data into files.
  * await StoreService.createBuildSource()
  * // Consume the pages as an interator.
@@ -89,7 +85,6 @@ const UNWANTED_PAGE_PROPS = ["editorID", "parentEditorID"];
  */
 class StoreService {
 	static basePath: string;
-	static mode: "file" | "memory";
 	static sanitize? = false;
 	static pages: Array<GatsbyPageObject> = [];
 	static sitesToPublish: Array<Site>;
@@ -102,39 +97,28 @@ class StoreService {
 	 * @param basePath The path to store the page files.
 	 * @param options An object configuration
 	 * @param options.mode Store mode
-	 * @todo Avoid initialize the Store with a `basePath` in memory mode.
 	 */
-	public static init(
-		basePath: string,
-		options: { mode: StoreMode; sanitize?: boolean }
-	) {
+	public static init(basePath: string, options: { sanitize?: boolean }) {
 		this.sanitize = options?.sanitize;
-		const { mode } = options;
 		const griddoCxThreads = parseInt(GRIDDO_API_CONCURRENCY_COUNT || "10");
-		const modeString = mode === "file" ? "File mode" : "Memory mode";
 
 		console.log(`🧶 Processing API page calls with ${griddoCxThreads} threads`);
-		console.log(`📦 Griddo Store initialized - ${chalk.italic(modeString)}`);
+		console.log(`📦 Griddo Store initialized`);
 
 		if (this.sanitize) {
 			console.log(`🏥 Sanitize pages enabled`);
 		}
 
-		if (mode === "file") {
-			this.basePath = basePath;
-			this.mode = mode;
-
-			if (fs.existsSync(basePath)) {
-				fs.rmSync(basePath, { recursive: true });
-			}
+		this.basePath = basePath;
 
-			if (!fs.existsSync(basePath)) {
-				fs.mkdirSync(basePath);
-			}
+		// Remove old store
+		if (fs.existsSync(basePath)) {
+			fs.rmSync(basePath, { recursive: true });
+		}
 
-			if (!fs.existsSync(path.resolve(basePath))) {
-				fs.mkdirSync(path.resolve(basePath));
-			}
+		// Create new store
+		if (!fs.existsSync(basePath)) {
+			fs.mkdirSync(basePath);
 		}
 	}
 
@@ -186,7 +170,7 @@ class StoreService {
 					headers,
 					footers,
 					socials,
-				} = await getSiteData(siteId, false);
+				} = await getSiteData(siteId);
 
 				const {
 					cloudinaryName,
@@ -280,7 +264,7 @@ class StoreService {
 					// for the data.
 					const template = await DistributorService.getDistributorData({
 						page,
-						cached: !!RENDER_ID,
+						cacheKey: RENDER_ID,
 					});
 
 					// MultiPage Query
@@ -358,9 +342,9 @@ class StoreService {
 
 					// Save build data to store
 					await this.savePages(gatsbyPageObjects); // gatsbyPageObjects is always an array regardless of whether it's just one page
-					this.saveSitesToPublish(sitesToPublish);
-					this.saveBuildProcessData(BUILD_PROCESS_DATA);
-					this.saveCreatedPages(CREATED_PAGES);
+					this.setSitesToPublish(sitesToPublish);
+					this.setBuildProcessData(BUILD_PROCESS_DATA);
+					this.setCreatedPages(CREATED_PAGES);
 				};
 
 				const griddoCxThreads = parseInt(GRIDDO_API_CONCURRENCY_COUNT || "10");
@@ -383,57 +367,49 @@ class StoreService {
 	}
 
 	/**
-	 * Get all pages data from the Store.
-	 * If the Store is in file mode `getPages` will return a Generator of pages.
-	 * If the Store is in memory mode `getPages` will return an array of pages.
-	 */
-	public static getPages() {
-		return this.mode === "file"
-			? this.getPagesFromFiles()
-			: this.getPagesFromMemory();
-	}
-
-	/**
-	 * Save the pages into the Store.
+	 * Save the pages into the file system.
 	 *
 	 * @param pages An array of pages to save in the store.
 	 */
 	public static async savePages(pages: Array<GatsbyPageObject>) {
-		// WARNS: This mutate the page objects and could hurt the build render
-		// performance.
 		if (this.sanitize) {
 			pages.forEach((page) => removeProperties(page, UNWANTED_PAGE_PROPS));
 		}
 
-		return this.mode === "file"
-			? await this.savePagesToFiles(pages)
-			: this.savePagesToMemory(pages);
+		const promises = pages.map(async (pageObject) => {
+			const filename = `${createSha256(pageObject.path)}.json`;
+			const fileContent = JSON.stringify(pageObject);
+			const filePath = path.resolve(this.basePath, filename);
+			await fsp.writeFile(filePath, fileContent);
+		});
+
+		await Promise.all(promises);
 	}
 
 	/**
-	 * Save site to publish data in the Store.
+	 * Set site to publish data in the Store.
 	 *
 	 * @param sites An array of sites to be published.
 	 */
-	public static saveSitesToPublish(sites: Array<Site>) {
+	public static setSitesToPublish(sites: Array<Site>) {
 		this.sitesToPublish = sites;
 	}
 
 	/**
-	 * Save the build process data in the Store.
+	 * Set the build process data in the Store.
 	 *
 	 * @param buildProcessData An object that represents a build process data.
 	 */
-	public static saveBuildProcessData(buildProcessData: BuildProcessData) {
+	public static setBuildProcessData(buildProcessData: BuildProcessData) {
 		this.buildProcessData = buildProcessData;
 	}
 
 	/**
-	 * Save the created pages in the Store.
+	 * Set the created pages in the Store.
 	 *
 	 * @param pages An array of pages ids.
 	 */
-	public static saveCreatedPages(pages: Array<number>) {
+	public static setCreatedPages(pages: Array<number>) {
 		this.createdPages = pages;
 	}
 
@@ -451,7 +427,7 @@ class StoreService {
 	/**
 	 * Return an iterator with pages read from the file system.
 	 */
-	private static *getPagesFromFiles() {
+	public static *getPages() {
 		const pagesDirPath = path.resolve(this.basePath);
 		const jsonFilePaths = walk(pagesDirPath).filter(
 			(file) => path.extname(file) === ".json"
@@ -459,52 +435,17 @@ class StoreService {
 
 		for (const filePath of jsonFilePaths) {
 			try {
-				const jsonContent = fs.readFileSync(filePath, "utf-8");
-				const page = JSON.parse(jsonContent) as GatsbyPageObject;
-				page.size = fs.statSync(filePath).size / 1024;
+				const fileStats = fs.statSync(filePath);
+				const page = fsx.readJSONSync(filePath, {
+					encoding: "utf-8",
+				}) as GatsbyPageObject;
+				page.size = fileStats.size / 1024;
 				yield page;
-			} catch {
-				console.error(`Error: The file ${filePath} doesn't exist`);
+			} catch (error) {
+				console.error(`Error: The file ${filePath} doesn't exist`, error);
 			}
 		}
 	}
-
-	/**
-	 * Get the pages from the Store from the memory.
-	 */
-	private static getPagesFromMemory() {
-		return this.pages;
-	}
-
-	/**
-	 * Save pages to the file system.
-	 *
-	 * @param pages An array of final pages to save into the file system.
-	 */
-	private static async savePagesToFiles(pages: Array<GatsbyPageObject>) {
-		const promises = pages.map(async (pageObject) => {
-			const filename = `${createSha256(pageObject.path)}.json`;
-			const fileContent = JSON.stringify(pageObject);
-			const filePath = path.resolve(this.basePath, filename);
-			await fsp.writeFile(filePath, fileContent);
-		});
-
-		await Promise.all(promises);
-	}
-
-	/**
-	 * Save pages to the memory.
-	 *
-	 * @param pages An array of final pages to save into the memory
-	 */
-	private static savePagesToMemory(pages: Array<GatsbyPageObject>) {
-		const pagesWithSize = pages.map((page) => {
-			const size = v8.serialize(page).byteLength / 1024;
-
-			return { ...page, size };
-		});
-		this.pages.push(...pagesWithSize);
-	}
 }
 
 export { StoreService };

package/src/types/api.ts

@@ -112,8 +112,8 @@ export interface APIRequest {
 	endpoint: string;
 	/** The parameters to be sent in the request body. */
 	body?: unknown;
-	/** Indicates whether to use the cache to get the response from the API. */
-	cached?: unknown;
+	/** Reference id to manage cache between renders. */
+	cacheKey?: string;
 	/** Number of connection attempts (in case it fails on the first attempt). */
 	attempt?: number;
 	/** Headers for the post api fetch */

package/src/types/global.ts

@@ -1,9 +1,6 @@
 import type { EndSiteRenderBody } from "./api";
 import type { Core, Fields } from "@griddo/core";
 
-// TODO: JSDoc
-export type StoreMode = "memory" | "file";
-
 // TODO: JSDoc
 export interface Settings {
 	apiVersion?: string;
@@ -50,7 +47,8 @@ export interface FetchDataProps {
 	component: {
 		data: Fields.Reference<unknown>;
 	};
-	cached: boolean;
+	/** Reference id to manage cache between renders. */
+	cacheKey: string;
 }
 
 // TODO: JSDoc

package/src/types/pages.ts

@@ -5,7 +5,7 @@ import type { Core, Fields } from "@griddo/core";
 
 // TODO: In @griddo/core the type Core.Page has header/footer as React-Types,
 // but API return `number | null`.
-export type APIPageObject = Core.Page & {
+export type APIPageObject = Core.Page & { id: number } & {
 	header: number | null;
 	footer: number | null;
 };