@griddo/cx 11.9.17 → 11.10.1

package/exporter/commands/prepare-domains-render.ts

@@ -1,46 +1,163 @@
-import fs from "node:fs";
+import type { RenderDB } from "../shared/types/render";
+
+import { execSync } from "node:child_process";
+import fsp from "node:fs/promises";
 import path from "node:path";
 
-import { getConfig } from "../utils/core-utils";
-import { getInstanceDomains } from "../utils/domains";
-import { infoLog, successLog } from "../utils/loggin";
-import { getSitesToRender } from "../utils/sites";
+import { version as griddoVersion } from "../../package.json";
+import { checkEnvironmentHealth } from "../core/check-env-health";
+import { readDB, writeDB } from "../core/db";
+import { pathExists, rmDirs } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { resolveComponentsPath } from "../core/instance";
+import { AuthService } from "../services/auth";
+import { getInstanceDomains } from "../services/domains";
+import { getSitesToRender } from "../services/manage-sites";
+import { resolveDomainRenderMode } from "../services/render";
+import { GRIDDO_SKIP_BUILD_CHECKS } from "../shared/envs";
+import { pkgDir } from "../shared/npm-modules/pkg-dir";
+
+async function getDomainsWithNumberOfPagesWithActivity(domains: string[]) {
+	const results = await Promise.all(
+		domains.map(async (domain) => {
+			const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
+			const allSites = [...sitesToPublish, ...sitesToUnpublish];
 
-(async () => {
-	infoLog("Checking domains size");
+			if (allSites.length < 1) {
+				GriddoLog.warn(`No sites to publish or unpublish for domain: ${domain}`);
+				return { domain, totalPendingPages: 0 };
+			}
 
-	const { __cx } = getConfig().paths();
+			const pages: number[] = [];
+			for (const site of allSites) {
+				const { offlinePending, uploadPending, active } = site.pagesStatus;
 
-	const domains = await getInstanceDomains();
-	const domainsInfo = [];
-	const domainsFilePath = path.join(__cx, "domains.json");
-
-	for (const domain of domains) {
-		const { sitesToPublish } = await getSitesToRender(domain);
-
-		// Only count pages (changedPages) if sitesToPublish is not empty.
-		// Otherwise we set the domain has having 0 pages.
-		const pages = [];
-		if (sitesToPublish.length > 0) {
-			for (const site of sitesToPublish) {
-				pages.push(site.changedPages.length);
+				// HACK/SHAME
+				// Si un site no tiene páginas (está totalmente vacío) pero está
+				// pendiente de ser publicado/despublicado
+				// `shouldBeUpdated=true`, marcamos como que tiene 1 página.
+				// Esto es solo para indicarle a CX que tenga en cuenta el site
+				// para el incremental render.
+				// HACK/SHAME
+				const siteHasNoPagesToRender =
+					offlinePending.length === 0 && uploadPending.length === 0 && active.length === 0;
+				const ghostPage = site.shouldBeUpdated && siteHasNoPagesToRender ? 1 : 0;
+				const pagesWithPendingActivity = offlinePending.length + uploadPending.length + ghostPage;
+				pages.push(pagesWithPendingActivity);
 			}
-		} else {
-			pages.push(0);
-		}
 
-		const totalPages = pages.reduce((a, b) => a + b);
-		domainsInfo.push({ domain, totalPages });
+			const totalPendingPages = pages.reduce((a, b) => a + b, 0);
+
+			return { domain, totalPendingPages };
+		}),
+	);
+
+	return results;
+}
+
+function getDomainsSortedByNumberOfPages(
+	domainsInfo: { domain: string; totalPendingPages: number }[],
+) {
+	domainsInfo.sort((a, b) => a.totalPendingPages - b.totalPendingPages);
+
+	return domainsInfo.map(({ domain }) => domain);
+}
+
+async function initRender() {
+	const root = (await pkgDir({ cwd: path.resolve(__dirname, "../../..") })) || "";
+	const cx = root;
+	const ssg = path.resolve((await pkgDir({ cwd: __dirname })) || "");
+	const cxCache = path.resolve(root, ".griddo/cache");
+	const components = resolveComponentsPath();
+	const exportsDir = path.join(root, "exports/sites");
+	const exportsDirBackup = path.join(root, "exports-backup/sites");
+
+	const data: RenderDB = {
+		griddoVersion,
+		buildReportFileName: "build-report.json",
+		sortedDomains: [],
+		needsRollbackOnError: false,
+		domains: {},
+		currentRenderingDomain: null,
+		commitHash: execSync("git rev-parse HEAD").toString().trim(),
+		commitMessage: execSync("git log -1 --pretty=%B").toString().trim(),
+		paths: {
+			components,
+			cx,
+			cxCache,
+			exportsDir,
+			root,
+			ssg,
+			exportsDirBackup,
+		},
+	};
+
+	// create the main cache folder for cx if it doesn't exist.
+	if (!(await pathExists(cxCache))) {
+		await fsp.mkdir(cxCache, { recursive: true });
 	}
 
-	// sort domains, smaller first
-	domainsInfo.sort((a, b) => a.totalPages - b.totalPages);
-	const domainSorted = domainsInfo.map(({ domain }) => domain);
+	await writeDB(data);
+}
+
+async function prepareDomains() {
+	await AuthService.login();
 
-	fs.writeFileSync(domainsFilePath, JSON.stringify(domainSorted));
+	const db = await readDB();
 
-	successLog(`Checking domains size`);
-})().catch((err) => {
-	console.error(err);
+	const __ssg = db.paths.ssg;
+	const domains = await getInstanceDomains();
+	const domainsWithNumberOfPendingPages = await getDomainsWithNumberOfPagesWithActivity(domains);
+	const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPendingPages);
+
+	// @deprecated use db.json (only for infra)
+	await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
+
+	db.sortedDomains = domainSorted;
+	db.domains = {};
+
+	for (const { domain, totalPendingPages } of domainsWithNumberOfPendingPages) {
+		const shouldBeRendered = totalPendingPages > 0;
+		const { renderMode, reason } = await resolveDomainRenderMode({
+			domain,
+			shouldBeRendered,
+		});
+
+		// Log RenderModes/Reason
+		GriddoLog.info(
+			`(From Initial Render) [${domain}]: Marked as ${renderMode} with the reason: ${reason}`,
+		);
+
+		db.domains[domain] = db.domains[domain] || {};
+		db.domains[domain].renderMode = renderMode;
+		db.domains[domain].shouldBeRendered = shouldBeRendered;
+		db.domains[domain].renderModeReason = reason;
+	}
+
+	await writeDB(db);
+}
+
+async function clean() {
+	const db = await readDB();
+	const { root } = db.paths;
+	await rmDirs([path.join(root, "apiCache")]);
+}
+
+async function main() {
+	GRIDDO_SKIP_BUILD_CHECKS
+		? GriddoLog.info("Build health check bypassed")
+		: checkEnvironmentHealth();
+
+	await initRender();
+	await prepareDomains();
+	await clean();
+}
+
+main().catch((error) => {
+	if (error instanceof Error) {
+		GriddoLog.error(error.message);
+	} else {
+		GriddoLog.error(`An unexpected error occurred ${error}`);
+	}
 	process.exit(1);
 });

package/exporter/commands/reset-render.ts

@@ -1,14 +1,19 @@
-#!/usr/bin/env node
-
+import { GriddoLog } from "../core/GriddoLog";
+import { post } from "../services/api";
 import { AuthService } from "../services/auth";
-import { resetRender } from "../services/settings";
+import { RESET_RENDER } from "../shared/endpoints";
 
-async function main() {
+async function resetRender() {
 	await AuthService.login();
+	await post({
+		endpoint: RESET_RENDER,
+		useApiCacheDir: false,
+	});
+	GriddoLog.info("The render status of all domains has been restored.");
+}
+
+async function main() {
 	await resetRender();
 }
 
-main().catch((err) => {
-	console.error("Error", err?.stdout?.toString() || err);
-	process.exit(1);
-});
+main().catch(console.error);

package/exporter/commands/single-domain-upload-search-content.ts

@@ -0,0 +1,206 @@
+import type { PostSearchInfoResponse } from "../shared/types/api";
+import type { PostSearchInfoProps } from "../shared/types/global";
+import type { GatsbyPageData } from "../ssg-adapters/gatsby/shared/types";
+
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import { throwError, withErrorHandler } from "../core/errors";
+import { pathExists } from "../core/fs";
+import { GriddoLog } from "../core/GriddoLog";
+import { post } from "../services/api";
+import { AuthService } from "../services/auth";
+import { getRenderModeFromDB, getRenderPathsHydratedWithDomainFromDB } from "../services/render";
+import { SEARCH } from "../shared/endpoints";
+import { GRIDDO_SEARCH_FEATURE } from "../shared/envs";
+import { ReadFromStoreError, UploadSearchError } from "../shared/errors";
+import { RENDER_MODE } from "../shared/types/render";
+
+/**
+ * Save in the BBDD the content of a page parsed without HTML tags.
+ *
+ * @param props Object with parts of the final page object to be saved in the BBDD.
+ */
+async function postSearchInfo(props: PostSearchInfoProps) {
+	const { title, description, image, pageId, languageId, siteId, url, content, template } = props;
+
+	const response = await post<PostSearchInfoResponse>({
+		endpoint: SEARCH,
+		body: {
+			title,
+			description,
+			image,
+			pageId,
+			languageId,
+			siteId,
+			url,
+			template,
+			content,
+		},
+		useApiCacheDir: false,
+		logToFile: false,
+	});
+
+	return response;
+}
+
+function prepareHTMLContentForSearch(content: string): string {
+	// 1. Remove script, style, and other unwanted block tags and their content.
+	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
+	const tagsToRemove = ["meta", "link", "style", "script", "noscript", "nav", "header", "footer"];
+	const removeTagsRegex = new RegExp(`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`, "gis");
+	let processedContent = content.replace(removeTagsRegex, "");
+
+	// 2. Strip all remaining HTML tags.
+	processedContent = processedContent.replace(/<[^>]+>/g, " ");
+
+	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
+	processedContent = processedContent.replace(/\s+/g, " ").trim();
+
+	return processedContent;
+}
+
+async function getContentDirectories(domain: string) {
+	const { __exports } = await getRenderPathsHydratedWithDomainFromDB({ domain });
+
+	return {
+		htmlContentDir: path.join(__exports, "dist"),
+		jsonContentDir: path.join(__exports, "dist", "page-data"),
+	};
+}
+
+/**
+ * Walk recursively in a basePath and return an array of pages with json
+ * extension with the full absolute path and the path includes "page-data".
+ *
+ * @param basePath The path to walk recursively.
+ * @returns An array of pages with json extension with the full absolute path
+ * and the path includes "page-data".
+ */
+async function* walkRecursively(basePath: string): AsyncGenerator<string> {
+	const filesHandle = await fsp.opendir(basePath);
+
+	for await (const fileDirent of filesHandle) {
+		if (fileDirent.isDirectory()) {
+			yield* walkRecursively(path.join(basePath, fileDirent.name));
+		} else if (
+			fileDirent.isFile() &&
+			path.extname(fileDirent.name) === ".json" &&
+			fileDirent.name.includes("page-data")
+		) {
+			yield path.join(basePath, fileDirent.name);
+		}
+	}
+}
+
+/**
+ * Walk recursively in a basePath and return an array of pages with json
+ * extension with the full absolute path and the path includes "page-data".
+ *
+ * @param basePath The path to walk recursively.
+ * @returns An array of pages with json extension with the full absolute path
+ * and the path includes "page-data".
+ */
+async function* getPageDataPagesFromExports<PageType extends GatsbyPageData>(
+	basePath: string,
+): AsyncGenerator<PageType> {
+	const jsonFiles = walkRecursively(basePath);
+
+	for await (const filePath of jsonFiles) {
+		try {
+			const fileContent = await fsp.readFile(filePath, "utf8");
+			const page = JSON.parse(fileContent) as PageType;
+
+			if (page.path) {
+				yield page;
+			}
+		} catch (error) {
+			throwError(ReadFromStoreError, error);
+		}
+	}
+}
+
+/**
+ * Function that search in the `/public` dir the content info of the pages and
+ * send it to the search table in the ddbb using the API.
+ * @todo Utilizar la carpeta page-data en lugar de la carpeta store puesto que
+ * esta ya no es persistente.
+ */
+async function uploadRenderedSearchContentToAPI(options: {
+	htmlContentDir: string;
+	jsonContentDir: string;
+}) {
+	const { htmlContentDir, jsonContentDir } = options;
+
+	if (!(await pathExists(jsonContentDir)) || !(await pathExists(htmlContentDir))) {
+		GriddoLog.info(
+			`(From Current Render) Skipping uploading content to the search endpoint because it has not exported sites.`,
+		);
+
+		return;
+	}
+
+	// Get pages from gatsby page-data dir
+	const gatsbyPageDataPages = getPageDataPagesFromExports(jsonContentDir);
+
+	for await (const pageData of gatsbyPageDataPages) {
+		const { result } = pageData;
+		const { pageContext } = result;
+		const { page, openGraph, pageMetadata } = pageContext;
+
+		const { compose } = page.fullPath;
+
+		const htmlPath = path.resolve(`${htmlContentDir}/${compose}/index.html`);
+		const htmlContent = await fsp.readFile(htmlPath, "utf-8");
+
+		const pageObject: PostSearchInfoProps = {
+			siteId: page.site,
+			pageId: page.id,
+			// `pageMetadata.title` has already a fallback `metatitle ||
+			// pageTitle` so probably `title` never will be take the
+			// `openGraph?.title` value. Only when the `metatitle` and
+			// `pageTitle` are empty.
+			title: pageMetadata?.title || openGraph?.title,
+			languageId: page.language,
+			url: page.fullUrl,
+			template: page.template.templateType || page.templateId,
+			description: pageMetadata?.description || openGraph?.description,
+			image: openGraph.image,
+			// _content: prepareHTMLContentForSearch(htmlContent),
+			content: prepareHTMLContentForSearch(htmlContent),
+		};
+
+		try {
+			await postSearchInfo(pageObject);
+		} catch (error) {
+			throwError(UploadSearchError, error);
+		}
+	}
+}
+
+async function uploadSearchContent() {
+	const [domain] = process.argv.slice(2);
+	const { renderMode, reason } = await getRenderModeFromDB(domain);
+
+	if (renderMode === RENDER_MODE.IDLE) {
+		GriddoLog.info(
+			`(From Current Render) [${domain}]: Skipping upload-search-content as it is marked as IDLE with the reason ${reason}.`,
+		);
+		return;
+	}
+
+	const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
+	await uploadRenderedSearchContentToAPI({
+		htmlContentDir,
+		jsonContentDir,
+	});
+}
+
+async function main() {
+	if (GRIDDO_SEARCH_FEATURE) {
+		await AuthService.login();
+		await uploadSearchContent();
+	}
+}
+
+withErrorHandler(main);

package/exporter/commands/start-render.ts

@@ -1,68 +1,17 @@
-#!/usr/bin/env node
-import { renderDomainsWithGatsbyAdapter } from "../adapters/gatsby";
-import { RenderError } from "../errors";
-import { getInstanceDomains } from "../utils/domains";
-import { errorLabelLog, showExporterVersion } from "../utils/loggin";
-import { sendGriddoDefaultAlerts } from "../utils/render";
-
-const GRIDDO_RENDER_DOMAINS = process.env.GRIDDO_RENDER_DOMAINS || "";
-const RENDER_BY_DOMAIN = process.env.GRIDDO_RENDER_BY_DOMAINS || "";
-
-async function startRender() {
-	try {
-		// DOMAINS
-		// This block is just for the local instance renders.
-		if (GRIDDO_RENDER_DOMAINS) {
-			console.log("USANDO EL .ENV");
-			showExporterVersion();
-			for (const domain of GRIDDO_RENDER_DOMAINS.split(",")) {
-				await renderDomainsWithGatsbyAdapter(domain);
-			}
-			await sendGriddoDefaultAlerts();
-			process.exit(0);
-		}
-
-		// This will render every domain opposed to the new logic of render each
-		// domain separately.
-		if (RENDER_BY_DOMAIN) {
-			const domain = process.argv.splice(2)[0];
-
-			if (!domain) {
-				console.log("Needs the domain name argument");
-				throw new RenderError();
-			}
-
-			showExporterVersion();
-			await renderDomainsWithGatsbyAdapter(domain);
-			await sendGriddoDefaultAlerts();
-
-			process.exit(0);
-		} else {
-			showExporterVersion();
-
-			console.log("( Legacy Render Mode )\n");
-
-			const domains = await getInstanceDomains();
-
-			for (const domain of domains) {
-				await renderDomainsWithGatsbyAdapter(domain);
-				await sendGriddoDefaultAlerts();
-			}
-			process.exit(0);
-		}
-	} catch (error) {
-		if (error instanceof RenderError) {
-			errorLabelLog("GRIDDO_ERROR InternalCXError");
-			process.exit(1);
-		}
+import { withErrorHandler } from "../core/errors";
+import { showExporterVersion } from "../core/print-logos";
+import { AuthService } from "../services/auth";
+import { gatsbyRenderDomain } from "../ssg-adapters/gatsby";
+
+async function render() {
+	await AuthService.login();
+	const [domain] = process.argv.slice(2);
+	await gatsbyRenderDomain(domain);
+}
 
-		errorLabelLog("GRIDDO_ERROR UnknownError");
-		console.error(error);
-		process.exit(1);
-	}
+async function main() {
+	await showExporterVersion();
+	await render();
 }
 
-startRender().catch((err) => {
-	console.error("Error", err?.stdout?.toString() || err);
-	process.exit(1);
-});
+withErrorHandler(main);