@griddo/cx 11.9.8-rc.1 → 11.9.8-rc.2

package/exporter/commands/prepare-domains-render.ts

@@ -1,154 +1,46 @@
-import type { RenderDB } from "../shared/types/render";
-
-import fsp from "node:fs/promises";
+import fs from "node:fs";
 import path from "node:path";
 
-import { version as griddoVersion } from "../../package.json";
-import { checkEnvironmentHealth } from "../core/check-env-health";
-import { readDB, writeDB } from "../core/db";
-import { withErrorHandler } from "../core/errors";
-import { pathExists, rmDirs } from "../core/fs";
-import { GriddoLog } from "../core/GriddoLog";
-import { resolveComponentsPath } from "../core/instance";
-import { AuthService } from "../services/auth";
-import { getInstanceDomains } from "../services/domains";
-import { getSitesToRender } from "../services/manage-sites";
-import { resolveDomainRenderMode } from "../services/render";
-import { pkgDir } from "../shared/npm-modules/pkg-dir";
-
-async function getDomainsWithNumberOfPages(domains: string[]) {
-	const results = await Promise.all(
-		domains.map(async (domain) => {
-			// Get the sites to render for the domain
-			const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
-
-			// Only count pages (offlinePending && publishPending) if sitesToPublish
-			// is not empty. Otherwise we set the domain has having 0 pages.
-			const pages: number[] = [];
-
-			if (sitesToPublish.length > 0) {
-				for (const site of sitesToPublish) {
-					// el site ya está publicado
-					if (site.isPublished && !site.shouldBeUpdated) {
-						const { pagesStatus } = site;
-						const { offlinePending, uploadPending } = pagesStatus;
-						const total = offlinePending.length + uploadPending.length;
-						pages.push(total);
-					}
-					// el site esta pendiente de publicar
-					if (site.isPublished && site.shouldBeUpdated) {
-						const { pagesStatus } = site;
-						const { offlinePending, uploadPending, active } = pagesStatus;
-						const total = offlinePending.length + uploadPending.length + active.length;
-						pages.push(total);
-					}
-				}
-			} else if (sitesToUnpublish.length > 0) {
-				// si tiene sites para despublicar marcamos el dominio como que tiene 1 página
-				pages.push(1);
-			} else {
-				// si eldominio no tiene nada ni para publicar ni despublicar lomarcamos como 0
-				pages.push(0);
-			}
-
-			const totalPages = pages.reduce((a, b) => a + b);
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import { infoLog, successLog } from "../utils/loggin";
+import { getSitesToRender } from "../utils/sites";
 
-			return { domain, totalPages };
-		}),
-	);
+(async () => {
+	infoLog("Checking domains size");
 
-	return results;
-}
+	const { __cx } = getConfig().paths();
 
-function getDomainsSortedByNumberOfPages(domainsInfo: { domain: string; totalPages: number }[]) {
-	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
-
-	return domainsInfo.map(({ domain }) => domain);
-}
-
-async function initRender() {
-	const root = (await pkgDir({ cwd: path.resolve(__dirname, "../../..") })) || "";
-	const cx = root;
-	const ssg = path.resolve((await pkgDir({ cwd: __dirname })) || "");
-	const cxCache = path.resolve(root, ".griddo/cache");
-	const components = resolveComponentsPath();
-	const exportsDir = path.join(root, "exports/sites");
-	const exportsDirBackup = path.join(root, "exports-backup/sites");
-
-	const data: RenderDB = {
-		griddoVersion,
-		buildReportFileName: "build-report.json",
-		sortedDomains: [],
-		needsRollbackOnError: false,
-		domains: {},
-		currentRenderingDomain: null,
-		paths: {
-			components,
-			cx,
-			cxCache,
-			exportsDir,
-			root,
-			ssg,
-			exportsDirBackup,
-		},
-	};
-
-	// create the main cache folder for cx if it doesn't exist.
-	if (!(await pathExists(cxCache))) {
-		await fsp.mkdir(cxCache, { recursive: true });
-	}
-
-	await writeDB(data);
-}
-
-async function prepareDomains() {
-	await AuthService.login();
-
-	const db = await readDB();
-
-	const __ssg = db.paths.ssg;
 	const domains = await getInstanceDomains();
-	const domainsWithNumberOfPages = await getDomainsWithNumberOfPages(domains);
-	const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPages);
-
-	// @deprecated use db.json (only for infra)
-	await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
-
-	db.sortedDomains = domainSorted;
-	db.domains = {};
-
-	for (const { domain, totalPages } of domainsWithNumberOfPages) {
-		const shouldBeRendered = totalPages > 0;
-		const { renderMode, reason } = await resolveDomainRenderMode({
-			domain,
-			shouldBeRendered,
-		});
-
-		// Log RenderModes/Reason
-		GriddoLog.info(
-			`(From Initial Render) [${domain}]: Marked as ${renderMode} with the reason: ${reason}`,
-		);
+	const domainsInfo = [];
+	const domainsFilePath = path.join(__cx, "domains.json");
+
+	for (const domain of domains) {
+		const { sitesToPublish } = await getSitesToRender(domain);
+
+		// Only count pages (changedPages) if sitesToPublish is not empty.
+		// Otherwise we set the domain has having 0 pages.
+		const pages = [];
+		if (sitesToPublish.length > 0) {
+			for (const site of sitesToPublish) {
+				pages.push(site.changedPages.length);
+			}
+		} else {
+			pages.push(0);
+		}
 
-		db.domains[domain] = db.domains[domain] || {};
-		db.domains[domain].renderMode = renderMode;
-		db.domains[domain].shouldBeRendered = shouldBeRendered;
-		db.domains[domain].renderModeReason = reason;
+		const totalPages = pages.reduce((a, b) => a + b);
+		domainsInfo.push({ domain, totalPages });
 	}
 
-	await writeDB(db);
-}
-
-async function clean() {
-	const db = await readDB();
-	const { root } = db.paths;
-	await rmDirs([path.join(root, "apiCache")]);
-}
+	// sort domains, smaller first
+	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
+	const domainSorted = domainsInfo.map(({ domain }) => domain);
 
-async function main() {
-	checkEnvironmentHealth();
-	await initRender();
-	await prepareDomains();
-	await clean();
-}
+	fs.writeFileSync(domainsFilePath, JSON.stringify(domainSorted));
 
-withErrorHandler(main);
+	successLog(`Checking domains size`);
+})().catch((err) => {
+	console.error(err);
+	process.exit(1);
+});

package/exporter/commands/reset-render.ts

@@ -1,19 +1,14 @@
-import { GriddoLog } from "../core/GriddoLog";
-import { post } from "../services/api";
-import { AuthService } from "../services/auth";
-import { RESET_RENDER } from "../shared/endpoints";
+#!/usr/bin/env node
 
-async function resetRender() {
-	await post({
-		endpoint: RESET_RENDER,
-		useApiCacheDir: false,
-	});
-}
+import { AuthService } from "../services/auth";
+import { resetRender } from "../services/settings";
 
 async function main() {
 	await AuthService.login();
 	await resetRender();
-	GriddoLog.info("The render status has been reset.");
 }
 
-main().catch(console.error);
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/commands/start-render.ts

@@ -1,30 +1,68 @@
-import { withErrorHandler } from "../core/errors";
-import { GriddoLog } from "../core/GriddoLog";
-import { showExporterVersion } from "../core/print-logos";
-import { AuthService } from "../services/auth";
-import { getInstanceDomains } from "../services/domains";
-import { GRIDDO_RENDER_BY_DOMAINS } from "../shared/envs";
-import { gatsbyRenderDomain } from "../ssg-adapters/gatsby";
-
-async function legacyRender() {
-	GriddoLog.warn("Legacy Render Mode");
-	const domains = await getInstanceDomains();
-	for (const domain of domains) {
-		await gatsbyRenderDomain(domain);
-	}
-}
+#!/usr/bin/env node
+import { renderDomainsWithGatsbyAdapter } from "../adapters/gatsby";
+import { RenderError } from "../errors";
+import { getInstanceDomains } from "../utils/domains";
+import { errorLabelLog, showExporterVersion } from "../utils/loggin";
+import { sendGriddoDefaultAlerts } from "../utils/render";
 
-async function render() {
-	const [domain] = process.argv.slice(2);
-	await gatsbyRenderDomain(domain);
-}
+const GRIDDO_RENDER_DOMAINS = process.env.GRIDDO_RENDER_DOMAINS || "";
+const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
+
+async function startRender() {
+	try {
+		// DOMAINS
+		// This block is just for the local instance renders.
+		if (GRIDDO_RENDER_DOMAINS) {
+			console.log("USANDO EL .ENV");
+			showExporterVersion();
+			for (const domain of GRIDDO_RENDER_DOMAINS.split(",")) {
+				await renderDomainsWithGatsbyAdapter(domain);
+			}
+			await sendGriddoDefaultAlerts();
+			process.exit(0);
+		}
+
+		// This will render every domain opposed to the new logic of render each
+		// domain separately.
+		if (RENDER_BY_DOMAIN) {
+			const domain = process.argv.splice(2)[0];
+
+			if (!domain) {
+				console.log("Needs the domain name argument");
+				throw new RenderError();
+			}
 
-async function main() {
-	await showExporterVersion();
-	await AuthService.login();
-	GRIDDO_RENDER_BY_DOMAINS // This is a legacy flag, we should remove it in the future
-		? await render()
-		: await legacyRender();
+			showExporterVersion();
+			await renderDomainsWithGatsbyAdapter(domain);
+			await sendGriddoDefaultAlerts();
+
+			process.exit(0);
+		} else {
+			showExporterVersion();
+
+			console.log("( Legacy Render Mode )\n");
+
+			const domains = await getInstanceDomains();
+
+			for (const domain of domains) {
+				await renderDomainsWithGatsbyAdapter(domain);
+				await sendGriddoDefaultAlerts();
+			}
+			process.exit(0);
+		}
+	} catch (error) {
+		if (error instanceof RenderError) {
+			errorLabelLog("GRIDDO_ERROR InternalCXError");
+			process.exit(1);
+		}
+
+		errorLabelLog("GRIDDO_ERROR UnknownError");
+		console.error(error);
+		process.exit(1);
+	}
 }
 
-withErrorHandler(main);
+startRender().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/commands/upload-search-content.ts

@@ -1,214 +1,39 @@
-import type { PostSearchInfoResponse } from "../shared/types/api";
-import type { AIEmbeddingsResponse, PostSearchInfoProps } from "../shared/types/global";
-import type { GatsbyPageData } from "../ssg-adapters/gatsby/shared/types";
+#!/usr/bin/env node
 
-import fsp from "node:fs/promises";
-import path from "node:path";
+import fs from "node:fs";
 
-import { throwError, withErrorHandler } from "../core/errors";
-import { pathExists } from "../core/fs";
-import { GriddoLog } from "../core/GriddoLog";
-import { post } from "../services/api";
-import { AuthService } from "../services/auth";
-import { getInstanceDomains } from "../services/domains";
-import { getRenderPathsHydratedWithDomainFromDB } from "../services/render";
-import { AI_EMBEDDINGS, SEARCH } from "../shared/endpoints";
-import { GRIDDO_AI_EMBEDDINGS, GRIDDO_SEARCH_FEATURE } from "../shared/envs";
-import { ReadFromStoreError, UploadSearchError } from "../shared/errors";
-
-/**
- * Save in the BBDD the content of a page parsed without HTML tags.
- *
- * @param props Object with parts of the final page object to be saved in the BBDD.
- */
-async function postSearchInfo(props: PostSearchInfoProps) {
-	const { title, description, image, pageId, languageId, siteId, url, content, template } = props;
-
-	const response = await post<PostSearchInfoResponse>({
-		endpoint: SEARCH,
-		body: {
-			title,
-			description,
-			image,
-			pageId,
-			languageId,
-			siteId,
-			url,
-			template,
-			content,
-		},
-		useApiCacheDir: false,
-		logToFile: false,
-	});
-
-	return response;
-}
-
-function prepareHTMLContentForSearch(content: string): string {
-	// 1. Remove script, style, and other unwanted block tags and their content.
-	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
-	const tagsToRemove = ["meta", "link", "style", "script", "noscript", "nav", "header", "footer"];
-	const removeTagsRegex = new RegExp(`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`, "gis");
-	let processedContent = content.replace(removeTagsRegex, "");
-
-	// 2. Strip all remaining HTML tags.
-	processedContent = processedContent.replace(/<[^>]+>/g, " ");
-
-	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
-	processedContent = processedContent.replace(/\s+/g, " ").trim();
-
-	return processedContent;
-}
-
-/**
- * Function that search in the `/public` dir the content info of the pages and
- * send it to the search table in the ddbb using the API.
- * @todo Utilizar la carpeta page-data en lugar de la carpeta store puesto que
- * esta ya no es persistente.
- */
-async function uploadRenderedSearchContentToAPI(options: {
-	htmlContentDir: string;
-	jsonContentDir: string;
-}) {
-	const { htmlContentDir, jsonContentDir } = options;
-
-	if (!(await pathExists(jsonContentDir)) || !(await pathExists(htmlContentDir))) {
-		GriddoLog.info(
-			`Skipping uploading content to the search endpoint because it has not exported sites.`,
-		);
+import { envs } from "../constants";
+import { getConfig } from "../utils/core-utils";
+import { getInstanceDomains } from "../utils/domains";
+import {
+	startAIEmbeddings,
+	uploadRenderedSearchContentToAPI,
+} from "../utils/searches";
 
+async function main() {
+	if (!envs.GRIDDO_SEARCH_FEATURE) {
 		return;
 	}
 
-	// Get pages from gatsby page-data dir
-	const gatsbyPageDataPages = getPageDataPagesFromExports(jsonContentDir);
-
-	for await (const pageData of gatsbyPageDataPages) {
-		const { result } = pageData;
-		const { pageContext } = result;
-		const { page, openGraph, pageMetadata } = pageContext;
-
-		const { compose } = page.fullPath;
-
-		const htmlPath = path.resolve(`${htmlContentDir}/${compose}/index.html`);
-		const htmlContent = await fsp.readFile(htmlPath, "utf-8");
+	const domains = await getInstanceDomains();
+	const config = getConfig();
+	for (const domain of domains) {
+		const { __exports_dist } = config.paths(domain);
 
-		const pageObject: PostSearchInfoProps = {
-			siteId: page.site,
-			pageId: page.id,
-			// `pageMetadata.title` has already a fallback `metatitle ||
-			// pageTitle` so probably `title` never will be take the
-			// `openGraph?.title` value. Only when the `metatitle` and
-			// `pageTitle` are empty.
-			title: pageMetadata?.title || openGraph?.title,
-			languageId: page.language,
-			url: page.fullUrl,
-			template: page.template.templateType || page.templateId,
-			description: pageMetadata?.description || openGraph?.description,
-			image: openGraph.image,
-			// _content: prepareHTMLContentForSearch(htmlContent),
-			content: prepareHTMLContentForSearch(htmlContent),
-		};
-
-		try {
-			await postSearchInfo(pageObject);
-		} catch (error) {
-			throwError(UploadSearchError, error);
+		// If __exports_dist has not exported sites (directories), it does not
+		// upload search content.
+		if (fs.existsSync(__exports_dist)) {
+			await uploadRenderedSearchContentToAPI(__exports_dist, domain);
 		}
 	}
-}
-
-/**
- * Walk recursively in a basePath and return an array of pages with json
- * extension with the full absolute path and the path includes "page-data".
- *
- * @param basePath The path to walk recursively.
- * @returns An array of pages with json extension with the full absolute path
- * and the path includes "page-data".
- */
-async function* walkRecursively(basePath: string): AsyncGenerator<string> {
-	const filesHandle = await fsp.opendir(basePath);
 
-	for await (const fileDirent of filesHandle) {
-		if (fileDirent.isDirectory()) {
-			yield* walkRecursively(path.join(basePath, fileDirent.name));
-		} else if (
-			fileDirent.isFile() &&
-			path.extname(fileDirent.name) === ".json" &&
-			fileDirent.name.includes("page-data")
-		) {
-			yield path.join(basePath, fileDirent.name);
-		}
+	// Solo una vez en cada render de todos los dominios...
+	if (envs.GRIDDO_AI_EMBEDDINGS) {
+		await startAIEmbeddings();
 	}
 }
 
-/**
- * Walk recursively in a basePath and return an array of pages with json
- * extension with the full absolute path and the path includes "page-data".
- *
- * @param basePath The path to walk recursively.
- * @returns An array of pages with json extension with the full absolute path
- * and the path includes "page-data".
- */
-async function* getPageDataPagesFromExports<PageType extends GatsbyPageData>(
-	basePath: string,
-): AsyncGenerator<PageType> {
-	const jsonFiles = walkRecursively(basePath);
-
-	for await (const filePath of jsonFiles) {
-		try {
-			const fileContent = await fsp.readFile(filePath, "utf8");
-			const page = JSON.parse(fileContent) as PageType;
-
-			if (page.path) {
-				yield page;
-			}
-		} catch (error) {
-			throwError(ReadFromStoreError, error);
-		}
-	}
-}
-
-async function getContentDirectories(domain: string) {
-	const { __exports } = await getRenderPathsHydratedWithDomainFromDB({ domain });
-
-	return {
-		htmlContentDir: path.join(__exports, "dist"),
-		jsonContentDir: path.join(__exports, "dist", "page-data"),
-	};
-}
-
-async function uploadSearchContent() {
-	if (GRIDDO_SEARCH_FEATURE) {
-		const domains = await getInstanceDomains();
-		for (const domain of domains) {
-			const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
-
-			GriddoLog.info(
-				`Uploading search content for ${domain}: processing content for ${htmlContentDir.length} pages...`,
-			);
-			await uploadRenderedSearchContentToAPI({
-				htmlContentDir,
-				jsonContentDir,
-			});
-		}
-	}
-}
-
-async function aiEmbeddings() {
-	if (GRIDDO_SEARCH_FEATURE && GRIDDO_AI_EMBEDDINGS) {
-		GriddoLog.info(`Triggering AI embeddings...`);
-		await post<AIEmbeddingsResponse>({
-			endpoint: AI_EMBEDDINGS,
-			useApiCacheDir: false,
-		});
-	}
-}
-
-async function main() {
-	await AuthService.login();
-	await uploadSearchContent();
-	await aiEmbeddings();
-}
-
-withErrorHandler(main);
+main().catch((err) => {
+	console.error("Error", err?.stdout?.toString() || err);
+	process.exit(1);
+});

package/exporter/{shared → constants}/endpoints.ts

@@ -1,5 +1,6 @@
-const GRIDDO_API_URL = process.env.GRIDDO_API_URL;
-const GRIDDO_PUBLIC_API_URL = process.env.GRIDDO_PUBLIC_API_URL;
+import { GRIDDO_API_URL, GRIDDO_PUBLIC_API_URL } from "./envs";
+
+const WITH_URI = `${GRIDDO_API_URL}/site/`;
 
 const AI_EMBEDDINGS = `${GRIDDO_API_URL}/ai/embeddings`;
 const ALERT = `${GRIDDO_PUBLIC_API_URL}/alert`;
@@ -12,16 +13,14 @@ const ROBOTS = `${GRIDDO_API_URL}/domains/robots`;
 const SEARCH = `${GRIDDO_API_URL}/search`;
 const SETTINGS = `${GRIDDO_API_URL}/settings`;
 
-// Site
-const SITE_URI = `${GRIDDO_API_URL}/site/`;
-const BUILD_END = [SITE_URI, "/build/end"];
-const BUILD_START = [SITE_URI, "/build/start"];
-const GET_PAGES = [SITE_URI, "/pages?pagination=false"];
-const GET_REFERENCE_FIELD_DATA = [SITE_URI, "/distributor"];
-const GET_SITEMAP = [SITE_URI, "/sitemap"];
-const INFO = [SITE_URI, "/all"];
-const LANGUAGES = [SITE_URI, "/languages"];
-const SOCIALS = [SITE_URI, "/socials"];
+const BUILD_END = [WITH_URI, "/build/end"];
+const BUILD_START = [WITH_URI, "/build/start"];
+const GET_PAGES = [WITH_URI, "/pages?pagination=false"];
+const GET_REFERENCE_FIELD_DATA = [WITH_URI, "/distributor"];
+const GET_SITEMAP = [WITH_URI, "/sitemap"];
+const INFO = [WITH_URI, "/all"];
+const LANGUAGES = [WITH_URI, "/languages"];
+const SOCIALS = [WITH_URI, "/socials"];
 
 export {
 	AI_EMBEDDINGS,