@griddo/cx 11.7.6 → 11.7.7-rc.0

package/exporter/commands/prepare-domains-render.ts

@@ -1,46 +1,159 @@
-import fs from "node:fs";
+#!/usr/bin/env node
+
+import type { RenderDB } from "../types/render";
+
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import { getConfig } from "../utils/core-utils";
+import { version as griddoVersion } from "../../package.json";
+import { AuthService } from "../services/auth";
+import { readDB, writeDB } from "../services/db";
 import { getInstanceDomains } from "../utils/domains";
-import { infoLog, successLog } from "../utils/loggin";
+import { pathExists, pkgDir, removeDirs } from "../utils/folders";
+import { resolveComponentsPath } from "../utils/instance";
+import { showExporterVersion } from "../utils/loggin";
+import { resolveDomainRenderMode } from "../utils/render";
 import { getSitesToRender } from "../utils/sites";
 
-(async () => {
-	infoLog("Checking domains size");
+async function getDomainsWithNumberOfPages(domains: Array<string>) {
+	const results = await Promise.all(
+		domains.map(async (domain) => {
+			// Get the sites to render for the domain
+			const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
 
-	const { __cx } = getConfig().paths();
+			// Only count pages (offlinePending && publishPending) if sitesToPublish
+			// is not empty. Otherwise we set the domain has having 0 pages.
+			const pages: Array<number> = [];
 
-	const domains = await getInstanceDomains();
-	const domainsInfo = [];
-	const domainsFilePath = path.join(__cx, "domains.json");
-
-	for (const domain of domains) {
-		const { sitesToPublish } = await getSitesToRender(domain);
-
-		// Only count pages (changedPages) if sitesToPublish is not empty.
-		// Otherwise we set the domain has having 0 pages.
-		const pages = [];
-		if (sitesToPublish.length > 0) {
-			for (const site of sitesToPublish) {
-				pages.push(site.changedPages.length);
+			if (sitesToPublish.length > 0) {
+				for (const site of sitesToPublish) {
+					// el site ya está publicado
+					if (site.isPublished && !site.shouldBeUpdated) {
+						const { pagesStatus } = site;
+						const { offlinePending, uploadPending } = pagesStatus;
+						const total = offlinePending.length + uploadPending.length;
+						pages.push(total);
+					}
+					// el site esta pendiente de publicar
+					if (site.isPublished && site.shouldBeUpdated) {
+						const { pagesStatus } = site;
+						const { offlinePending, uploadPending, active } = pagesStatus;
+						const total = offlinePending.length + uploadPending.length + active.length;
+						pages.push(total);
+					}
+				}
+			} else if (sitesToUnpublish.length > 0) {
+				// si tiene sites para despublicar marcamos el dominio como que tiene 1 página
+				pages.push(1);
+			} else {
+				// si eldominio no tiene nada ni para publicar ni despublicar lomarcamos como 0
+				pages.push(0);
 			}
-		} else {
-			pages.push(0);
-		}
 
-		const totalPages = pages.reduce((a, b) => a + b);
-		domainsInfo.push({ domain, totalPages });
-	}
+			const totalPages = pages.reduce((a, b) => a + b);
+
+			return { domain, totalPages };
+		}),
+	);
+
+	return results;
+}
 
-	// sort domains, smaller first
+function getDomainsSortedByNumberOfPages(
+	domainsInfo: Array<{
+		domain: string;
+		totalPages: number;
+	}>,
+) {
 	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
-	const domainSorted = domainsInfo.map(({ domain }) => domain);
 
-	fs.writeFileSync(domainsFilePath, JSON.stringify(domainSorted));
+	return domainsInfo.map(({ domain }) => domain);
+}
+
+async function initRender() {
+	const root = (await pkgDir({ cwd: path.resolve(__dirname, "../../..") })) || "";
+	const cx = root;
+	const ssg = path.resolve((await pkgDir({ cwd: __dirname })) || "");
+	const cxCache = path.resolve(root, ".cx-cache");
+	const components = resolveComponentsPath();
+	const exportsDir = path.join(root, "exports/sites");
+
+	const data: RenderDB = {
+		proDomain: "pro-",
+		griddoVersion,
+		buildReportFileName: "build-report.json",
+		sortedDomains: [],
+		domains: {},
+		paths: {
+			components,
+			cx,
+			cxCache,
+			exportsDir,
+			root,
+			ssg,
+		},
+	};
+
+	// create the main cache folder for cx if it doesn't exist.
+	if (!(await pathExists(cxCache))) {
+		await fsp.mkdir(cxCache, { recursive: true });
+	}
+
+	await writeDB(data);
+}
+
+async function prepareDomains() {
+	await AuthService.login();
+
+	const db = await readDB();
+
+	const __ssg = db.paths.ssg;
+	const domains = await getInstanceDomains();
+	const domainsWithNumberOfPages = await getDomainsWithNumberOfPages(domains);
+	const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPages);
+
+	// @deprecated use db.json (only for infra)
+	await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
+	await fsp.writeFile(path.join(__ssg, "domains.txt"), domainSorted.join(","));
+
+	db.sortedDomains = domainSorted;
+	db.domains = {};
+
+	for (const { domain, totalPages } of domainsWithNumberOfPages) {
+		const shouldBeRendered = totalPages > 0;
+		const { renderMode, reason } = await resolveDomainRenderMode({
+			domain,
+			shouldBeRendered,
+		});
+
+		db.domains[domain] = db.domains[domain] || {};
+		db.domains[domain].renderMode = renderMode;
+		db.domains[domain].shouldBeRendered = shouldBeRendered;
+		db.domains[domain].renderModeReason = reason;
+	}
+
+	await writeDB(db);
+}
+
+async function clean() {
+	const db = await readDB();
+	const { root: __root } = db.paths;
+	await removeDirs([path.join(__root, "apiCache")]);
+}
+
+/**
+ * Prepare domains to render.
+ * It creates a `domains.json` file with the sorted list of domains and writes a
+ * `RENDE-STATE.json` file to persist a state between all cx-infra commands.
+ */
+async function main() {
+	await showExporterVersion();
+	await initRender();
+	await prepareDomains();
+	await clean();
+}
 
-	successLog(`Checking domains size`);
-})().catch((err) => {
-	console.error(err);
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/reset-render.ts

@@ -1,11 +1,20 @@
 #!/usr/bin/env node
 
+import { RESET_RENDER } from "../constants/endpoints";
 import { AuthService } from "../services/auth";
-import { resetRender } from "../services/settings";
+import { post } from "../utils/api";
+
+async function resetRender() {
+	await post({
+		endpoint: RESET_RENDER,
+		useApiCacheDir: false,
+	});
+}
 
 async function main() {
 	await AuthService.login();
 	await resetRender();
+	console.log(`The render status has been reset.`);
 }
 
 main().catch((err) => {

package/exporter/commands/start-render.ts

@@ -1,68 +1,47 @@
 #!/usr/bin/env node
-import { renderDomainsWithGatsbyAdapter } from "../adapters/gatsby";
-import { RenderError } from "../errors";
+
+import { gatsbyRenderDomain } from "../adapters/gatsby";
+import { AuthService } from "../services/auth";
+import { brush } from "../utils/brush";
+import { checkHealth } from "../utils/check-health";
 import { getInstanceDomains } from "../utils/domains";
-import { errorLabelLog, showExporterVersion } from "../utils/loggin";
-import { sendGriddoDefaultAlerts } from "../utils/render";
+import { RenderError } from "../utils/errors";
 
-const GRIDDO_RENDER_DOMAINS = process.env.GRIDDO_RENDER_DOMAINS || "";
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+const renderByDomains = !!process.env.GRIDDO_RENDER_BY_DOMAINS;
 
 async function startRender() {
 	try {
-		// DOMAINS
-		// This block is just for the local instance renders.
-		if (GRIDDO_RENDER_DOMAINS) {
-			console.log("USANDO EL .ENV");
-			showExporterVersion();
-			for (const domain of GRIDDO_RENDER_DOMAINS.split(",")) {
-				await renderDomainsWithGatsbyAdapter(domain);
-			}
-			await sendGriddoDefaultAlerts();
-			process.exit(0);
-		}
-
-		// This will render every domain opposed to the new logic of render each
-		// domain separately.
-		if (RENDER_BY_DOMAIN) {
-			const domain = process.argv.splice(2)[0];
-
-			if (!domain) {
-				console.log("Needs the domain name argument");
-				throw new RenderError();
-			}
-
-			showExporterVersion();
-			await renderDomainsWithGatsbyAdapter(domain);
-			await sendGriddoDefaultAlerts();
-
-			process.exit(0);
+		if (renderByDomains) {
+			const [domain] = process.argv.slice(2);
+			await gatsbyRenderDomain(domain);
 		} else {
-			showExporterVersion();
-
-			console.log("( Legacy Render Mode )\n");
-
+			console.log("- LEGACY RENDER MODE -");
 			const domains = await getInstanceDomains();
-
 			for (const domain of domains) {
-				await renderDomainsWithGatsbyAdapter(domain);
-				await sendGriddoDefaultAlerts();
+				await gatsbyRenderDomain(domain);
 			}
-			process.exit(0);
 		}
+
+		return;
 	} catch (error) {
 		if (error instanceof RenderError) {
-			errorLabelLog("GRIDDO_ERROR InternalCXError");
+			console.log(brush.red("GRIDDO_ERROR InternalCXError"));
 			process.exit(1);
 		}
 
-		errorLabelLog("GRIDDO_ERROR UnknownError");
+		console.log(brush.red("GRIDDO_ERROR UnknownError"));
 		console.error(error);
 		process.exit(1);
 	}
 }
 
-startRender().catch((err) => {
+async function main() {
+	checkHealth();
+	await AuthService.login();
+	await startRender();
+}
+
+main().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);
 });

package/exporter/commands/upload-search-content.ts

@@ -1,38 +1,212 @@
 #!/usr/bin/env node
 
-import fs from "node:fs";
+import type { GatsbyPageData } from "../adapters/gatsby/shared/types";
+import type { PostSearchInfoResponse } from "../types/api";
+import type { AIEmbeddingsResponse, PostSearchInfoProps } from "../types/global";
 
-import { envs } from "../constants";
-import { getConfig } from "../utils/core-utils";
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { AI_EMBEDDINGS, SEARCH } from "../constants/endpoints";
+import { GRIDDO_AI_EMBEDDINGS, GRIDDO_SEARCH_FEATURE } from "../constants/envs";
+import { ReadFromStoreError, UploadSearchError } from "../constants/errors";
+import { AuthService } from "../services/auth";
+import { post } from "../utils/api";
 import { getInstanceDomains } from "../utils/domains";
-import {
-	startAIEmbeddings,
-	uploadRenderedSearchContentToAPI,
-} from "../utils/searches";
+import { throwError } from "../utils/errors";
+import { pathExists } from "../utils/folders";
+import { getRenderPathsFromDB } from "../utils/render";
+
+/**
+ * Save in the BBDD the content of a page parsed without HTML tags.
+ *
+ * @param props Object with parts of the final page object to be saved in the BBDD.
+ */
+async function postSearchInfo(props: PostSearchInfoProps) {
+	const { title, description, image, pageId, languageId, siteId, url, content, template } = props;
+
+	const response = await post<PostSearchInfoResponse>({
+		endpoint: SEARCH,
+		body: {
+			title,
+			description,
+			image,
+			pageId,
+			languageId,
+			siteId,
+			url,
+			template,
+			content,
+		},
+		useApiCacheDir: false,
+	});
+
+	return response;
+}
+
+function prepareHTMLContentForSearch(content: string): string {
+	// 1. Remove script, style, and other unwanted block tags and their content.
+	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
+	const tagsToRemove = ["meta", "link", "style", "script", "noscript", "nav", "header", "footer"];
+	const removeTagsRegex = new RegExp(`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`, "gis");
+	let processedContent = content.replace(removeTagsRegex, "");
+
+	// 2. Strip all remaining HTML tags.
+	processedContent = processedContent.replace(/<[^>]+>/g, " ");
+
+	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
+	processedContent = processedContent.replace(/\s+/g, " ").trim();
+
+	return processedContent;
+}
+
+/**
+ * Function that search in the `/public` dir the content info of the pages and
+ * send it to the search table in the ddbb using the API.
+ * @todo Utilizar la carpeta page-data en lugar de la carpeta store puesto que
+ * esta ya no es persistente.
+ */
+async function uploadRenderedSearchContentToAPI(options: {
+	htmlContentDir: string;
+	jsonContentDir: string;
+}) {
+	const { htmlContentDir, jsonContentDir } = options;
+
+	if (!(await pathExists(jsonContentDir)) || !(await pathExists(htmlContentDir))) {
+		console.log(
+			`Skipping uploading content to the search endpoint because it has not exported sites.`,
+		);
 
-async function main() {
-	if (!envs.GRIDDO_SEARCH_FEATURE) {
 		return;
 	}
 
-	const domains = await getInstanceDomains();
-	const config = getConfig();
-	for (const domain of domains) {
-		const { __exports_dist } = config.paths(domain);
+	// Get pages from gatsby page-data dir
+	const gatsbyPageDataPages = getPageDataPagesFromExports(jsonContentDir);
+
+	for await (const pageData of gatsbyPageDataPages) {
+		const { result } = pageData;
+		const { pageContext } = result;
+		const { page, openGraph, pageMetadata } = pageContext;
+
+		const { compose } = page.fullPath;
 
-		// If __exports_dist has not exported sites (directories), it does not
-		// upload search content.
-		if (fs.existsSync(__exports_dist)) {
-			await uploadRenderedSearchContentToAPI(__exports_dist, domain);
+		const htmlPath = path.resolve(`${htmlContentDir}/${compose}/index.html`);
+		const htmlContent = await fsp.readFile(htmlPath, "utf-8");
+
+		const pageObject: PostSearchInfoProps = {
+			siteId: page.site,
+			pageId: page.id,
+			// `pageMetadata.title` has already a fallback `metatitle ||
+			// pageTitle` so probably `title` never will be take the
+			// `openGraph?.title` value. Only when the `metatitle` and
+			// `pageTitle` are empty.
+			title: pageMetadata?.title || openGraph?.title,
+			languageId: page.language,
+			url: page.fullUrl,
+			template: page.template.templateType || page.templateId,
+			description: pageMetadata?.description || openGraph?.description,
+			image: openGraph.image,
+			// _content: prepareHTMLContentForSearch(htmlContent),
+			content: prepareHTMLContentForSearch(htmlContent),
+		};
+
+		try {
+			await postSearchInfo(pageObject);
+		} catch (error) {
+			throwError(UploadSearchError, error);
 		}
 	}
+}
 
-	// Solo una vez en cada render de todos los dominios...
-	if (envs.GRIDDO_AI_EMBEDDINGS) {
-		await startAIEmbeddings();
+/**
+ * Walk recursively in a basePath and return an array of pages with json
+ * extension with the full absolute path and the path includes "page-data".
+ *
+ * @param basePath The path to walk recursively.
+ * @returns An array of pages with json extension with the full absolute path
+ * and the path includes "page-data".
+ */
+async function* walkRecursively(basePath: string): AsyncGenerator<string> {
+	const filesHandle = await fsp.opendir(basePath);
+
+	for await (const fileDirent of filesHandle) {
+		if (fileDirent.isDirectory()) {
+			yield* walkRecursively(path.join(basePath, fileDirent.name));
+		} else if (
+			fileDirent.isFile() &&
+			path.extname(fileDirent.name) === ".json" &&
+			fileDirent.name.includes("page-data")
+		) {
+			yield path.join(basePath, fileDirent.name);
+		}
 	}
 }
 
+/**
+ * Walk recursively in a basePath and return an array of pages with json
+ * extension with the full absolute path and the path includes "page-data".
+ *
+ * @param basePath The path to walk recursively.
+ * @returns An array of pages with json extension with the full absolute path
+ * and the path includes "page-data".
+ */
+async function* getPageDataPagesFromExports<PageType extends GatsbyPageData>(
+	basePath: string,
+): AsyncGenerator<PageType> {
+	const jsonFiles = walkRecursively(basePath);
+
+	for await (const filePath of jsonFiles) {
+		try {
+			const fileContent = await fsp.readFile(filePath, "utf8");
+			const page = JSON.parse(fileContent) as PageType;
+
+			if (page.path) {
+				yield page;
+			}
+		} catch (error) {
+			throwError(ReadFromStoreError, error);
+		}
+	}
+}
+
+async function getContentDirectories(domain: string) {
+	const { __exports } = await getRenderPathsFromDB({ domain });
+
+	return {
+		htmlContentDir: path.join(__exports, "dist"),
+		jsonContentDir: path.join(__exports, "dist", "page-data"),
+	};
+}
+
+async function uploadSearchContent() {
+	if (GRIDDO_SEARCH_FEATURE) {
+		await AuthService.login();
+		const domains = await getInstanceDomains();
+		for (const domain of domains) {
+			const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
+			await uploadRenderedSearchContentToAPI({
+				htmlContentDir,
+				jsonContentDir,
+			});
+		}
+	}
+}
+
+async function aiEmbeddings() {
+	if (GRIDDO_SEARCH_FEATURE && GRIDDO_AI_EMBEDDINGS) {
+		await AuthService.login();
+		await post<AIEmbeddingsResponse>({
+			endpoint: AI_EMBEDDINGS,
+			useApiCacheDir: false,
+		});
+	}
+}
+
+async function main() {
+	await uploadSearchContent();
+	await aiEmbeddings();
+}
+
 main().catch((err) => {
 	console.error("Error", err?.stdout?.toString() || err);
 	process.exit(1);