@griddo/cx 11.7.12-rc.3 → 11.7.12

package/exporter/utils/render.ts

@@ -1,203 +1,71 @@
-import type { RenderModeTuple } from "../types/render";
-
-import { execSync } from "node:child_process";
-import fsp from "node:fs/promises";
+import fs from "node:fs";
 import path from "node:path";
 
-import { RenderUUIDError } from "../constants/errors";
-import { AuthService } from "../services/auth";
-import { readDB, writeDB } from "../services/db";
-import { RENDER_MODE } from "../types/render";
-import { throwError } from "./errors";
-import { pathExists } from "./folders";
-import { infoLog, verboseLog } from "./loggin";
-import { getBuildMetadata } from "./store";
+import { envs } from "../constants";
+import { throwError } from "../errors";
+import { RenderUUIDError } from "../errors/errors-data";
+import { apiRegister } from "../registers";
+import { insertAlert } from "./alerts";
+import { getConfig } from "./core-utils";
+import { debugLog } from "./loggin";
+
+const config = getConfig();
 
 /**
  * Creates a sentinel file with the current date and time.
  * This file is used to track later if node_modules/@griddo/cx was cleaned by a
- * npm install coming from a deploy.
+ * npm install from a deploy.
  */
-async function markRenderAsStarted(options: { domain: string; basePath: string }) {
-	const { domain } = options;
-
-	const db = await readDB();
-	db.domains[domain].isRendering = true;
-	await writeDB(db);
-
-	// Creamos un archivo centinela, si al terminar el render este archivo no
-	// existe es que ha habido un deploy por medio y hay que invalidar el render
-	const { __ssg } = await getRenderPathsFromDB();
-
-	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
-	await fsp.writeFile(renderSentinelFile, new Date().toISOString());
+function createSentinelRenderFile() {
+	const { __cx } = config.paths();
+	const renderSentinelFile = path.join(__cx, ".render-sentinel");
+	fs.writeFileSync(renderSentinelFile, new Date().toISOString());
 }
 
-async function markRenderAsCompleted(domain: string) {
-	const db = await readDB();
-	db.domains[domain].isRendering = false;
-	// db.domains[domain].renderMode = undefined;
-	// db.domains[domain].shouldBeRendered = false;
-	await writeDB(db);
-
-	// Borramos finalmente el archivo centinela
-	const { __ssg } = await getRenderPathsFromDB();
-	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
-	await fsp.unlink(renderSentinelFile);
+function deleteSentinelRenderFile() {
+	const { __cx } = config.paths();
+	const renderSentinelFile = path.join(__cx, ".render-sentinel");
+	fs.unlinkSync(renderSentinelFile);
 }
 
-async function assertRenderIsValid(domain: string) {
-	// Comprobamos que .render-sentinel exista, si no es que un deploy lo borro
-	// y hay que invalidar el render.
-	const { __ssg } = await getRenderPathsFromDB();
-	const renderSentinelFile = path.join(__ssg, `.render-sentinel-${domain}`);
-	if (!(await pathExists(renderSentinelFile))) {
+function isValidRender() {
+	const { __cx } = config.paths();
+	const renderSentinelFile = path.join(__cx, ".render-sentinel");
+	if (!fs.existsSync(renderSentinelFile)) {
 		throwError(RenderUUIDError);
 	}
 }
 
-// @todo add more messages...
-async function resolveDomainRenderMode(options: { domain: string; shouldBeRendered: boolean }) {
-	const { domain, shouldBeRendered } = options;
-	const { __cache, __exports } = await getRenderPathsFromDB({ domain });
-
-	// missing exports
-	const missingExports = !(await pathExists(path.join(__exports)));
-	// render with errors
-	const db = await readDB();
-	const wasRenderWithError = db.domains[domain]?.isRendering;
-	// new deploy or machine
-	const isNewDeploy = await hasNewCommit(__cache);
-
-	if (missingExports) {
-		infoLog(`[${domain}]: Render from Scratch due to missing exports (first render)`);
-		return {
-			renderMode: RENDER_MODE.FROM_SCRATCH,
-			reason: "Missing exports directory",
-		};
-	}
-
-	if (wasRenderWithError) {
-		infoLog(`[${domain}]: Render from Scratch due to a previous render error`);
-		return {
-			renderMode: RENDER_MODE.FROM_SCRATCH,
-			reason: "error in previous render",
-		};
-	}
-
-	if (isNewDeploy) {
-		infoLog(`[${domain}]: Render from Scratch due to a changes in the repo (commit hash)`);
-		return { renderMode: RENDER_MODE.FROM_SCRATCH, reason: "new commit hash" };
-	}
-
-	if (!shouldBeRendered) {
-		infoLog(`[${domain}]: Skipping render as it is not marked for rendering (IDLE).`);
-		return { renderMode: RENDER_MODE.IDLE, reason: "" };
-	}
-
-	console.log(
-		`[${domain}]: {DEBUG} Render INCREMENTAL porque no se cumplen: missingExports (FROM_SCRATCH), wasRenderWithError (FROM_SCRATCH), isNewDeploy (FROM_SCRATCH) o !shouldBeRendered (IDLE)`,
-	);
-	return { renderMode: RENDER_MODE.INCREMENTAL, reason: "" };
-}
-
-async function hasNewCommit(basePath: string): Promise<boolean> {
-	const commitFile = path.join(basePath, "commit");
-	const currentCommit = execSync("git rev-parse HEAD").toString().trim();
-
-	if (await pathExists(commitFile)) {
-		const savedCommit = (await fsp.readFile(commitFile, "utf-8")).trim();
-		if (savedCommit === currentCommit) {
-			return false; // No hay nuevo commit
-		}
-
-		return true;
-	}
-
-	return true;
-}
-
-async function updateCommitFile(options: { basePath: string }) {
-	const { basePath } = options;
-	const currentCommit = execSync("git rev-parse HEAD").toString().trim();
-	await fsp.writeFile(path.join(basePath, "commit"), currentCommit);
-}
-
-async function getRenderMode(domain: string): Promise<RenderModeTuple> {
-	const db = await readDB();
-
-	if (!db.domains[domain].renderMode) {
-		throw new Error(`Render mode not found for domain ${domain}`);
-	}
-
-	return {
-		renderMode: db.domains[domain].renderMode,
-		reason: db.domains[domain].renderModeReason,
-	};
-}
-
-async function getRenderPathsFromDB(options?: { domain?: string; dbFilePath?: string }) {
-	const { domain, dbFilePath } = options || {};
-
-	const db = await readDB(dbFilePath);
-	const paths = db.paths;
-
-	return {
-		__root: paths.root,
-		__cache: path.join(paths.cxCache, domain || ""),
-		__components: paths.components,
-		__cx: paths.cx,
-		__sites: paths.exportsDir,
-		__exports: path.join(paths.exportsDir, domain || ""),
-		__ssg: paths.ssg,
-		__exports_dist: path.join(paths.exportsDir, domain || "", "dist"),
-	};
-}
-
-async function getRenderMetadataFromDB() {
-	const db = await readDB();
-	return {
-		proDomainPrefix: db.proDomainPrefix,
-		griddoVersion: db.griddoVersion,
-		buildReportFileName: db.buildReportFileName,
-	};
-}
-
 /**
- * Save a file with the end of build process to use as `build-end` signal.
+ * Send the default registers through Griddo alerts.
  */
-async function generateBuildReport(domain: string) {
-	const authControl = await AuthService.login();
-
-	const { __root } = await getRenderPathsFromDB();
-	const { buildReportFileName } = await getRenderMetadataFromDB();
-	const { buildProcessData } = await getBuildMetadata(domain);
+async function sendGriddoDefaultAlerts() {
+	if (envs.GRIDDO_ALERT_FEATURE) {
+		const registerContent = apiRegister.get("API_RESPONSE_TOO_BIG");
 
-	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
-		...buildProcessData[siteID],
-		siteId: Number.parseInt(siteID),
-	}));
-
-	const report = {
-		authControl,
-		sites: buildSitesInfo,
-	};
-
-	const reportFilePath = path.join(__root, "dist", buildReportFileName);
-
-	await fsp.writeFile(reportFilePath, JSON.stringify(report));
+		if (registerContent.entries.length < 1) {
+			return;
+		}
 
-	verboseLog(`build report saved in ${reportFilePath}`);
+		debugLog("\nRender register report\n");
+		debugLog(registerContent);
+		debugLog();
+
+		await insertAlert({
+			description: `API response size is too large (${envs.GRIDDO_API_MAX_RESPONSE_SIZE}KB).`,
+			area: "Griddo",
+			level: "W",
+			fullData: {
+				output: registerContent,
+				date: new Date().toISOString(),
+			},
+		});
+	}
 }
 
 export {
-	assertRenderIsValid,
-	generateBuildReport,
-	getRenderMetadataFromDB,
-	getRenderMode,
-	getRenderPathsFromDB,
-	markRenderAsCompleted,
-	markRenderAsStarted,
-	resolveDomainRenderMode,
-	updateCommitFile,
+	createSentinelRenderFile,
+	deleteSentinelRenderFile,
+	isValidRender,
+	sendGriddoDefaultAlerts,
 };

package/exporter/utils/searches.ts

@@ -0,0 +1,156 @@
+import type { PostSearchInfoResponse } from "../types/api";
+import type {
+	AIEmbeddingsResponse,
+	PostSearchInfoProps,
+} from "../types/global";
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { post } from "./api";
+import { getConfig } from "./core-utils";
+import { getBuildPagesFromCachedStore } from "./store";
+import { endpoints } from "../constants";
+
+/**
+ * Save in the BBDD the content of a page parsed without HTML tags.
+ *
+ * @param props Object with parts of the final page object to be saved in the BBDD.
+ */
+async function postSearchInfo(props: PostSearchInfoProps) {
+	const {
+		title,
+		description,
+		image,
+		pageId,
+		languageId,
+		siteId,
+		url,
+		content,
+		template,
+	} = props;
+
+	const response = await post<PostSearchInfoResponse>({
+		endpoint: endpoints.SEARCH,
+		body: {
+			title,
+			description,
+			image,
+			pageId,
+			languageId,
+			siteId,
+			url,
+			template,
+			content,
+		},
+		useApiCacheDir: false,
+	});
+
+	return response;
+}
+
+/**
+ * Remove the every cntent inside some HTML tags including them.
+ *
+ * @param content A string with the content.
+ */
+function prepareHTMLContentForSearch(content: string): string {
+	// 1. Remove script, style, and other unwanted block tags and their content.
+	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
+	const tagsToRemove = [
+		"meta",
+		"link",
+		"style",
+		"script",
+		"noscript",
+		"nav",
+		"header",
+		"footer",
+	];
+	const removeTagsRegex = new RegExp(
+		`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`,
+		"gis",
+	);
+	let processedContent = content.replace(removeTagsRegex, "");
+
+	// 2. Strip all remaining HTML tags.
+	processedContent = processedContent.replace(/<[^>]+>/g, " ");
+
+	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
+	processedContent = processedContent.replace(/\s+/g, " ").trim();
+
+	return processedContent;
+}
+
+/**
+ * Function that search in the `/public` dir the content info of the pages and
+ * send it to the search table in the ddbb using the API.
+ */
+async function uploadRenderedSearchContentToAPI(
+	distDomainPath: string,
+	domain: string,
+) {
+	const config = getConfig();
+	const { __cache } = config.paths(domain);
+
+	// Extraemos el path de la carpeta store dentro de un dominio
+	const storeFolder = path.join(__cache, "store");
+
+	// Este caso sería que el dominio existe en la instancia pero no hay sites
+	// renderizardos bajo el mismo. Por lo que `store` no existe.
+	if (!fs.existsSync(storeFolder)) {
+		console.log(
+			`Skipping uploading content to the search endpoint for the domain ${domain}, it has not exported sites.`,
+		);
+		return;
+	}
+
+	// Obtenemos las páginas del store cacheado de un dominio
+	const storePages = getBuildPagesFromCachedStore(domain);
+
+	for await (const store of storePages) {
+		const {
+			context: { page, openGraph, pageMetadata },
+		} = store;
+
+		const { compose } = page.fullPath;
+
+		const contextPath = path.resolve(`${distDomainPath}/${compose}/index.html`);
+
+		const content = fs.readFileSync(contextPath).toString();
+
+		const pageObject: PostSearchInfoProps = {
+			siteId: page.site,
+			pageId: page.id,
+			// `pageMetadata.title` has already a fallback `metatitle ||
+			// pageTitle` so probably `title` never will be take the
+			// `openGraph?.title` value. Only when the `metatitle` and
+			// `pageTitle` are empty.
+			title: pageMetadata?.title || openGraph?.title,
+			languageId: page.language,
+			url: page.fullUrl,
+			template: page.template.templateType || page.templateId,
+			description: pageMetadata?.description || openGraph?.description,
+			image: openGraph.image,
+			content: prepareHTMLContentForSearch(content),
+		};
+
+		await postSearchInfo(pageObject);
+	}
+}
+
+async function startAIEmbeddings() {
+	try {
+		await post<AIEmbeddingsResponse>({
+			endpoint: endpoints.AI_EMBEDDINGS,
+			useApiCacheDir: false,
+		});
+	} catch (error) {
+		console.warn(
+			"There was an error with the ai embeddings",
+			(error as Error).message,
+		);
+	}
+}
+
+export { postSearchInfo, startAIEmbeddings, uploadRenderedSearchContentToAPI };

package/exporter/utils/sites.ts

@@ -1,35 +1,49 @@
 import type { Site, SiteData } from "../types/sites";
 
-import fsp from "node:fs/promises";
 import path from "node:path";
 
-import {
-	GRIDDO_RENDER_ALL_SITES,
-	GRIDDO_RENDER_PAGES,
-	GRIDDO_RENDER_SITE,
-} from "../constants/envs";
+import fs from "fs-extra";
+import { parse } from "js2xmlparser";
+
+import { envs } from "../constants";
+import { getConfig } from "./core-utils";
+import { buildLog, verboseLog } from "./loggin";
+import { getBuildMetadata } from "./store";
+import { AuthService } from "../services/auth";
 import {
 	endSiteRender,
 	getAllSites,
 	getSiteInfo,
 	getSiteLanguages,
+	getSitemap,
 	getSiteSocials,
 	startSiteRender,
 } from "../services/sites";
-import { getRenderPathsFromDB } from "./render";
+
+const config = getConfig();
 
 /**
  * Check the instance sites and returns site prepared to be published and unpublished.
  */
 async function getSitesToRender(domain: string) {
+	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
+
+	// Login to API
+	await AuthService.login();
+
 	// Get all sites. An array of Site
 	const allSites = await getAllSites(domain);
 
 	// Filter the array of sites to get only the ones to build/render
-	const validSites = GRIDDO_RENDER_ALL_SITES
-		? allSites.filter((site) => !GRIDDO_RENDER_SITE || site.id === GRIDDO_RENDER_SITE)
+	const validSites = envs.GRIDDO_RENDER_ALL_SITES
+		? allSites.filter(
+				(site) =>
+					!envs.GRIDDO_RENDER_SITE || site.id === envs.GRIDDO_RENDER_SITE,
+			)
 		: allSites.filter((site) =>
-				GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.shouldBeUpdated,
+				envs.GRIDDO_RENDER_SITE
+					? site.id === envs.GRIDDO_RENDER_SITE
+					: !!site.shouldBeUpdated,
 			);
 
 	// If there are valid sites...
@@ -44,7 +58,8 @@ async function getSitesToRender(domain: string) {
 			site.domains = items
 				.filter(
 					(item) =>
-						item.domain && (item.domain.slug === domain || item.domain.slug === `/${domain}`),
+						item.domain &&
+						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
 				)
 				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
 		}
@@ -52,11 +67,15 @@ async function getSitesToRender(domain: string) {
 
 	// Save sites object to publish
 	const sitesToPublish = validSites.filter((site) =>
-		GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.isPublished,
+		envs.GRIDDO_RENDER_SITE
+			? site.id === envs.GRIDDO_RENDER_SITE
+			: !!site.isPublished,
 	);
 
 	// Save sites object to unpublish
-	const sitesToUnpublish = validSites.filter((site) => !site.isPublished && site.shouldBeUpdated);
+	const sitesToUnpublish = validSites.filter(
+		(site) => !site.isPublished && site.shouldBeUpdated,
+	);
 
 	return {
 		sitesToPublish,
@@ -71,7 +90,7 @@ async function getSitesToRender(domain: string) {
  * @see https://griddoio.notion.site/Sites-d7bb0b7cb8d24894a5337e1139fc3d09#2019d3255bda4d219c7e19cf28d0c4fe
  */
 async function unpublishSites(sites: Array<Site>) {
-	const { __root } = await getRenderPathsFromDB();
+	const { __cx } = config.paths();
 
 	for (const site of sites) {
 		// API
@@ -88,8 +107,7 @@ async function unpublishSites(sites: Array<Site>) {
 
 		// STORE
 		// Remove site directory from the Store
-		// @todo: quitar. Como ya no hay persistenci de store, esto debería eliminarse de forma segura.
-		await fsp.rm(path.join(__root, "store", site.id.toString()), {
+		fs.rmSync(path.join(__cx, "store", site.id.toString()), {
 			force: true,
 			recursive: true,
 		});
@@ -112,8 +130,8 @@ async function getSiteData(siteID: number) {
 
 	const { siteHash, unpublishHashes, publishIds } = buildData;
 	const { headers, footers } = siteInfo;
-	const validPagesIds = GRIDDO_RENDER_PAGES.length
-		? GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
+	const validPagesIds = envs.GRIDDO_RENDER_PAGES.length
+		? envs.GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
 		: publishIds;
 
 	const siteData: SiteData = {
@@ -131,4 +149,164 @@ async function getSiteData(siteID: number) {
 	return siteData;
 }
 
-export { getSiteData, getSitesToRender, unpublishSites };
+/**
+ * Save a file with the end of build process to use as `build-end` signal.
+ */
+async function generateBuildReport() {
+	const { __cx } = config.paths();
+	const { buildReportFileName } = config;
+
+	const { buildProcessData } = await getBuildMetadata();
+
+	// Get the token
+	const authControl = await AuthService.login();
+
+	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
+		...buildProcessData[siteID],
+		siteId: Number.parseInt(siteID),
+	}));
+
+	const report = {
+		authControl,
+		sites: buildSitesInfo,
+	};
+
+	const reportFilePath = path.join(__cx, "dist", buildReportFileName);
+	fs.writeFileSync(reportFilePath, JSON.stringify(report));
+
+	verboseLog(`build report saved in ${reportFilePath}`);
+
+	buildLog(`Build report saved for ${buildSitesInfo.length} site(s)`);
+}
+
+/**
+ * Generate sitemaps and save them into file system.
+ */
+async function generateSitemaps() {
+	const { sitesToPublish } = await getBuildMetadata();
+	const { __cx } = config.paths();
+	const distDir = path.join(__cx, "dist");
+
+	const templateSitemapPrefix = "sitemap-";
+
+	const isGriddoSitemapFile = (file: string) =>
+		file === "sitemap.xml" ||
+		(file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
+
+	for (const site of sitesToPublish) {
+		const { id, languages } = site;
+
+		for (const lang of languages) {
+			if (AuthService.headers) {
+				AuthService.headers.lang = lang.id.toString();
+			}
+
+			const response = await getSitemap(id);
+
+			if (!response) continue;
+
+			const {
+				items: sitemapPagesGroup,
+				url: { home, domain },
+			} = response;
+
+			if (!home) continue;
+
+			const domainLanguage = site.domains.find(
+				(domain) => Object.keys(domain)[0] === lang.id.toString(),
+			);
+
+			if (!domainLanguage) continue;
+
+			const slug = Object.values(domainLanguage)[0];
+			const sitemaps: Array<string> = [];
+			const sitemapPageGroupKeys = Object.keys(sitemapPagesGroup);
+			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
+
+			/** delete every griddo sitemap file before create the new ones */
+			if (fs.existsSync(sitemapBasePath)) {
+				try {
+					const files = fs.readdirSync(sitemapBasePath);
+					for (const file of files) {
+						if (isGriddoSitemapFile(file)) {
+							const filePath = path.join(sitemapBasePath, file);
+							try {
+								fs.unlinkSync(filePath);
+							} catch (err) {
+								console.error(`Error removing ${filePath}:`, err);
+							}
+						}
+					}
+				} catch (err) {
+					console.error(`Error reading ${sitemapBasePath}:`, err);
+				}
+			}
+
+			for (const templateId of sitemapPageGroupKeys) {
+				const sitemapPages = sitemapPagesGroup[templateId];
+
+				if (!sitemapPages.length) continue;
+
+				const siteMap = parse("urlset", {
+					"@": {
+						xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
+						"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+						"xsi:schemaLocation":
+							"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd",
+					},
+					url: sitemapPages,
+				});
+
+				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
+				const exactPath = path.join(sitemapBasePath, sitemapName);
+
+				saveFile(exactPath, siteMap);
+
+				sitemaps.push(
+					`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`,
+				);
+			}
+
+			if (!sitemaps.length) {
+				continue;
+			}
+
+			const siteMap = parse("sitemapindex", {
+				"@": { xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" },
+				sitemap: sitemaps.map((loc) => ({ loc })),
+			});
+
+			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
+
+			saveFile(exactPath, siteMap);
+			verboseLog(`sitemap generated in ${exactPath}`);
+		}
+	}
+}
+
+/**
+ * Saves the content to a file specified by its path. If the file exists, it will be overwritten.
+ *
+ * @param filePath The path of the file to save the content to.
+ * @param content The content to save to the file.
+ */
+function saveFile(filePath: string, content: string) {
+	try {
+		const pathName = path.dirname(filePath);
+		if (!fs.existsSync(pathName)) {
+			fs.mkdirSync(pathName, { recursive: true });
+		}
+		fs.writeFileSync(filePath, content);
+	} catch (error) {
+		console.log(error);
+		throw new Error(`Error saving a file`);
+	}
+}
+
+export {
+	generateBuildReport,
+	generateSitemaps,
+	getSiteData,
+	getSitesToRender,
+	unpublishSites,
+};