@griddo/cx 11.7.6-rc.2 → 11.7.6

package/exporter/utils/searches.ts

@@ -0,0 +1,156 @@
+import type { PostSearchInfoResponse } from "../types/api";
+import type {
+	AIEmbeddingsResponse,
+	PostSearchInfoProps,
+} from "../types/global";
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { post } from "./api";
+import { getConfig } from "./core-utils";
+import { getBuildPagesFromCachedStore } from "./store";
+import { endpoints } from "../constants";
+
+/**
+ * Save in the BBDD the content of a page parsed without HTML tags.
+ *
+ * @param props Object with parts of the final page object to be saved in the BBDD.
+ */
+async function postSearchInfo(props: PostSearchInfoProps) {
+	const {
+		title,
+		description,
+		image,
+		pageId,
+		languageId,
+		siteId,
+		url,
+		content,
+		template,
+	} = props;
+
+	const response = await post<PostSearchInfoResponse>({
+		endpoint: endpoints.SEARCH,
+		body: {
+			title,
+			description,
+			image,
+			pageId,
+			languageId,
+			siteId,
+			url,
+			template,
+			content,
+		},
+		useApiCacheDir: false,
+	});
+
+	return response;
+}
+
+/**
+ * Remove the every cntent inside some HTML tags including them.
+ *
+ * @param content A string with the content.
+ */
+function prepareHTMLContentForSearch(content: string): string {
+	// 1. Remove script, style, and other unwanted block tags and their content.
+	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
+	const tagsToRemove = [
+		"meta",
+		"link",
+		"style",
+		"script",
+		"noscript",
+		"nav",
+		"header",
+		"footer",
+	];
+	const removeTagsRegex = new RegExp(
+		`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`,
+		"gis",
+	);
+	let processedContent = content.replace(removeTagsRegex, "");
+
+	// 2. Strip all remaining HTML tags.
+	processedContent = processedContent.replace(/<[^>]+>/g, " ");
+
+	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
+	processedContent = processedContent.replace(/\s+/g, " ").trim();
+
+	return processedContent;
+}
+
+/**
+ * Function that search in the `/public` dir the content info of the pages and
+ * send it to the search table in the ddbb using the API.
+ */
+async function uploadRenderedSearchContentToAPI(
+	distDomainPath: string,
+	domain: string,
+) {
+	const config = getConfig();
+	const { __cache } = config.paths(domain);
+
+	// Extraemos el path de la carpeta store dentro de un dominio
+	const storeFolder = path.join(__cache, "store");
+
+	// Este caso sería que el dominio existe en la instancia pero no hay sites
+	// renderizardos bajo el mismo. Por lo que `store` no existe.
+	if (!fs.existsSync(storeFolder)) {
+		console.log(
+			`Skipping uploading content to the search endpoint for the domain ${domain}, it has not exported sites.`,
+		);
+		return;
+	}
+
+	// Obtenemos las páginas del store cacheado de un dominio
+	const storePages = getBuildPagesFromCachedStore(domain);
+
+	for await (const store of storePages) {
+		const {
+			context: { page, openGraph, pageMetadata },
+		} = store;
+
+		const { compose } = page.fullPath;
+
+		const contextPath = path.resolve(`${distDomainPath}/${compose}/index.html`);
+
+		const content = fs.readFileSync(contextPath).toString();
+
+		const pageObject: PostSearchInfoProps = {
+			siteId: page.site,
+			pageId: page.id,
+			// `pageMetadata.title` has already a fallback `metatitle ||
+			// pageTitle` so probably `title` never will be take the
+			// `openGraph?.title` value. Only when the `metatitle` and
+			// `pageTitle` are empty.
+			title: pageMetadata?.title || openGraph?.title,
+			languageId: page.language,
+			url: page.fullUrl,
+			template: page.template.templateType || page.templateId,
+			description: pageMetadata?.description || openGraph?.description,
+			image: openGraph.image,
+			content: prepareHTMLContentForSearch(content),
+		};
+
+		await postSearchInfo(pageObject);
+	}
+}
+
+async function startAIEmbeddings() {
+	try {
+		await post<AIEmbeddingsResponse>({
+			endpoint: endpoints.AI_EMBEDDINGS,
+			useApiCacheDir: false,
+		});
+	} catch (error) {
+		console.warn(
+			"There was an error with the ai embeddings",
+			(error as Error).message,
+		);
+	}
+}
+
+export { postSearchInfo, startAIEmbeddings, uploadRenderedSearchContentToAPI };

package/exporter/utils/sites.ts

@@ -1,16 +1,14 @@
 import type { Site, SiteData } from "../types/sites";
 
-import fsp from "node:fs/promises";
 import path from "node:path";
 
-import { create } from "xmlbuilder2";
+import fs from "fs-extra";
+import { parse } from "js2xmlparser";
 
-import {
-	GRIDDO_API_URL,
-	GRIDDO_RENDER_ALL_SITES,
-	GRIDDO_RENDER_PAGES,
-	GRIDDO_RENDER_SITE,
-} from "../constants/envs";
+import { envs } from "../constants";
+import { getConfig } from "./core-utils";
+import { buildLog, verboseLog } from "./loggin";
+import { getBuildMetadata } from "./store";
 import { AuthService } from "../services/auth";
 import {
 	endSiteRender,
@@ -21,28 +19,31 @@ import {
 	getSiteSocials,
 	startSiteRender,
 } from "../services/sites";
-import { pathExists } from "./folders";
-import { verboseLog } from "./loggin";
-import { getRenderMetadataFromDB, getRenderPathsFromDB } from "./render";
-import { getBuildMetadata } from "./store";
+
+const config = getConfig();
 
 /**
  * Check the instance sites and returns site prepared to be published and unpublished.
  */
 async function getSitesToRender(domain: string) {
-	console.info(`API URL ${GRIDDO_API_URL as string}`);
+	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
 
 	// Login to API
-	// await AuthService.login();
+	await AuthService.login();
 
 	// Get all sites. An array of Site
 	const allSites = await getAllSites(domain);
 
 	// Filter the array of sites to get only the ones to build/render
-	const validSites = GRIDDO_RENDER_ALL_SITES
-		? allSites.filter((site) => !GRIDDO_RENDER_SITE || site.id === GRIDDO_RENDER_SITE)
+	const validSites = envs.GRIDDO_RENDER_ALL_SITES
+		? allSites.filter(
+				(site) =>
+					!envs.GRIDDO_RENDER_SITE || site.id === envs.GRIDDO_RENDER_SITE,
+			)
 		: allSites.filter((site) =>
-				GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.shouldBeUpdated,
+				envs.GRIDDO_RENDER_SITE
+					? site.id === envs.GRIDDO_RENDER_SITE
+					: !!site.shouldBeUpdated,
 			);
 
 	// If there are valid sites...
@@ -57,7 +58,8 @@ async function getSitesToRender(domain: string) {
 			site.domains = items
 				.filter(
 					(item) =>
-						item.domain && (item.domain.slug === domain || item.domain.slug === `/${domain}`),
+						item.domain &&
+						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
 				)
 				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
 		}
@@ -65,11 +67,15 @@ async function getSitesToRender(domain: string) {
 
 	// Save sites object to publish
 	const sitesToPublish = validSites.filter((site) =>
-		GRIDDO_RENDER_SITE ? site.id === GRIDDO_RENDER_SITE : !!site.isPublished,
+		envs.GRIDDO_RENDER_SITE
+			? site.id === envs.GRIDDO_RENDER_SITE
+			: !!site.isPublished,
 	);
 
 	// Save sites object to unpublish
-	const sitesToUnpublish = validSites.filter((site) => !site.isPublished && site.shouldBeUpdated);
+	const sitesToUnpublish = validSites.filter(
+		(site) => !site.isPublished && site.shouldBeUpdated,
+	);
 
 	return {
 		sitesToPublish,
@@ -84,7 +90,7 @@ async function getSitesToRender(domain: string) {
  * @see https://griddoio.notion.site/Sites-d7bb0b7cb8d24894a5337e1139fc3d09#2019d3255bda4d219c7e19cf28d0c4fe
  */
 async function unpublishSites(sites: Array<Site>) {
-	const { __root } = await getRenderPathsFromDB();
+	const { __cx } = config.paths();
 
 	for (const site of sites) {
 		// API
@@ -101,8 +107,7 @@ async function unpublishSites(sites: Array<Site>) {
 
 		// STORE
 		// Remove site directory from the Store
-		// @todo: quitar. Como ya no hay persistenci de store, esto debería eliminarse de forma segura.
-		await fsp.rm(path.join(__root, "store", site.id.toString()), {
+		fs.rmSync(path.join(__cx, "store", site.id.toString()), {
 			force: true,
 			recursive: true,
 		});
@@ -125,8 +130,8 @@ async function getSiteData(siteID: number) {
 
 	const { siteHash, unpublishHashes, publishIds } = buildData;
 	const { headers, footers } = siteInfo;
-	const validPagesIds = GRIDDO_RENDER_PAGES.length
-		? GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
+	const validPagesIds = envs.GRIDDO_RENDER_PAGES.length
+		? envs.GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
 		: publishIds;
 
 	const siteData: SiteData = {
@@ -147,12 +152,14 @@ async function getSiteData(siteID: number) {
 /**
  * Save a file with the end of build process to use as `build-end` signal.
  */
-async function generateBuildReport(domain: string) {
-	const authControl = await AuthService.login();
+async function generateBuildReport() {
+	const { __cx } = config.paths();
+	const { buildReportFileName } = config;
+
+	const { buildProcessData } = await getBuildMetadata();
 
-	const { __root } = await getRenderPathsFromDB();
-	const { buildReportFileName } = await getRenderMetadataFromDB();
-	const { buildProcessData } = await getBuildMetadata(domain);
+	// Get the token
+	const authControl = await AuthService.login();
 
 	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
 		...buildProcessData[siteID],
@@ -164,55 +171,52 @@ async function generateBuildReport(domain: string) {
 		sites: buildSitesInfo,
 	};
 
-	const reportFilePath = path.join(__root, "dist", buildReportFileName);
-
-	await fsp.writeFile(reportFilePath, JSON.stringify(report));
+	const reportFilePath = path.join(__cx, "dist", buildReportFileName);
+	fs.writeFileSync(reportFilePath, JSON.stringify(report));
 
 	verboseLog(`build report saved in ${reportFilePath}`);
+
+	buildLog(`Build report saved for ${buildSitesInfo.length} site(s)`);
 }
 
 /**
  * Generate sitemaps and save them into file system.
  */
-async function generateSitemaps(domain: string) {
-	const { __root } = await getRenderPathsFromDB();
-	const { sitesToPublish } = await getBuildMetadata(domain);
-	const distDir = path.join(__root, "dist");
+async function generateSitemaps() {
+	const { sitesToPublish } = await getBuildMetadata();
+	const { __cx } = config.paths();
+	const distDir = path.join(__cx, "dist");
+
 	const templateSitemapPrefix = "sitemap-";
 
 	const isGriddoSitemapFile = (file: string) =>
-		file === "sitemap.xml" || (file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
+		file === "sitemap.xml" ||
+		(file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
 
 	for (const site of sitesToPublish) {
 		const { id, languages } = site;
 
 		for (const lang of languages) {
-			// Local copy of headers to avoid mutating the shared object
-			const localHeaders: Record<string, string> = AuthService.headers
-				? { ...AuthService.headers }
-				: {};
-			localHeaders.lang = lang.id.toString();
+			if (AuthService.headers) {
+				AuthService.headers.lang = lang.id.toString();
+			}
 
-			const response = await getSitemap({ siteId: id, headers: localHeaders });
+			const response = await getSitemap(id);
 
-			if (!response) {
-				continue;
-			}
+			if (!response) continue;
 
-			const { items: sitemapPagesGroup, url } = response;
-			const { home, domain } = url;
+			const {
+				items: sitemapPagesGroup,
+				url: { home, domain },
+			} = response;
 
-			if (!home) {
-				continue;
-			}
+			if (!home) continue;
 
 			const domainLanguage = site.domains.find(
 				(domain) => Object.keys(domain)[0] === lang.id.toString(),
 			);
 
-			if (!domainLanguage) {
-				continue;
-			}
+			if (!domainLanguage) continue;
 
 			const slug = Object.values(domainLanguage)[0];
 			const sitemaps: Array<string> = [];
@@ -220,16 +224,16 @@ async function generateSitemaps(domain: string) {
 			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
 
 			/** delete every griddo sitemap file before create the new ones */
-			if (await pathExists(sitemapBasePath)) {
+			if (fs.existsSync(sitemapBasePath)) {
 				try {
-					const files = await fsp.readdir(sitemapBasePath);
+					const files = fs.readdirSync(sitemapBasePath);
 					for (const file of files) {
 						if (isGriddoSitemapFile(file)) {
 							const filePath = path.join(sitemapBasePath, file);
 							try {
-								await fsp.rm(filePath);
+								fs.unlinkSync(filePath);
 							} catch (err) {
-								console.error(`Error deleting file ${filePath}:`, err);
+								console.error(`Error removing ${filePath}:`, err);
 							}
 						}
 					}
@@ -241,41 +245,40 @@ async function generateSitemaps(domain: string) {
 			for (const templateId of sitemapPageGroupKeys) {
 				const sitemapPages = sitemapPagesGroup[templateId];
 
-				if (!sitemapPages.length) {
-					continue;
-				}
+				if (!sitemapPages.length) continue;
 
-				const siteMap = create({ version: "1.0", encoding: "UTF-8" })
-					.ele("urlset", {
+				const siteMap = parse("urlset", {
+					"@": {
 						xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
 						"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
 						"xsi:schemaLocation":
-							"http://www.sitemaps.org/schemas/sitemap/0.9 " +
-							"http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd",
-					})
-					.ele(sitemapPages.map((page) => ({ url: page })))
-					.end({ prettyPrint: true });
+							"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd",
+					},
+					url: sitemapPages,
+				});
 
 				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
 				const exactPath = path.join(sitemapBasePath, sitemapName);
 
-				await saveFile(exactPath, siteMap);
+				saveFile(exactPath, siteMap);
 
-				sitemaps.push(`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`);
+				sitemaps.push(
+					`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`,
+				);
 			}
 
 			if (!sitemaps.length) {
 				continue;
 			}
 
-			const siteMap = create({ version: "1.0", encoding: "UTF-8" })
-				.ele("sitemapindex", { xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" })
-				.ele(sitemaps.map((loc) => ({ sitemap: { loc } })))
-				.end({ prettyPrint: true });
+			const siteMap = parse("sitemapindex", {
+				"@": { xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" },
+				sitemap: sitemaps.map((loc) => ({ loc })),
+			});
 
 			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
 
-			await saveFile(exactPath, siteMap);
+			saveFile(exactPath, siteMap);
 			verboseLog(`sitemap generated in ${exactPath}`);
 		}
 	}
@@ -287,17 +290,23 @@ async function generateSitemaps(domain: string) {
  * @param filePath The path of the file to save the content to.
  * @param content The content to save to the file.
  */
-async function saveFile(filePath: string, content: string) {
+function saveFile(filePath: string, content: string) {
 	try {
 		const pathName = path.dirname(filePath);
-		if (!(await pathExists(pathName))) {
-			await fsp.mkdir(pathName, { recursive: true });
+		if (!fs.existsSync(pathName)) {
+			fs.mkdirSync(pathName, { recursive: true });
 		}
-		await fsp.writeFile(filePath, content);
+		fs.writeFileSync(filePath, content);
 	} catch (error) {
 		console.log(error);
 		throw new Error(`Error saving a file`);
 	}
 }
 
-export { generateBuildReport, generateSitemaps, getSiteData, getSitesToRender, unpublishSites };
+export {
+	generateBuildReport,
+	generateSitemaps,
+	getSiteData,
+	getSitesToRender,
+	unpublishSites,
+};