@griddo/cx 11.9.17 → 11.10.0

package/exporter/utils/searches.ts

@@ -1,156 +0,0 @@
-import type { PostSearchInfoResponse } from "../types/api";
-import type {
-	AIEmbeddingsResponse,
-	PostSearchInfoProps,
-} from "../types/global";
-
-import fs from "node:fs";
-import path from "node:path";
-
-import { post } from "./api";
-import { getConfig } from "./core-utils";
-import { getBuildPagesFromCachedStore } from "./store";
-import { endpoints } from "../constants";
-
-/**
- * Save in the BBDD the content of a page parsed without HTML tags.
- *
- * @param props Object with parts of the final page object to be saved in the BBDD.
- */
-async function postSearchInfo(props: PostSearchInfoProps) {
-	const {
-		title,
-		description,
-		image,
-		pageId,
-		languageId,
-		siteId,
-		url,
-		content,
-		template,
-	} = props;
-
-	const response = await post<PostSearchInfoResponse>({
-		endpoint: endpoints.SEARCH,
-		body: {
-			title,
-			description,
-			image,
-			pageId,
-			languageId,
-			siteId,
-			url,
-			template,
-			content,
-		},
-		useApiCacheDir: false,
-	});
-
-	return response;
-}
-
-/**
- * Remove the every cntent inside some HTML tags including them.
- *
- * @param content A string with the content.
- */
-function prepareHTMLContentForSearch(content: string): string {
-	// 1. Remove script, style, and other unwanted block tags and their content.
-	// The regex looks for <tag...>...</tag> where tag is one of the specified ones.
-	const tagsToRemove = [
-		"meta",
-		"link",
-		"style",
-		"script",
-		"noscript",
-		"nav",
-		"header",
-		"footer",
-	];
-	const removeTagsRegex = new RegExp(
-		`<(${tagsToRemove.join("|")})\\b[^>]*>.*?<\\/\\1>`,
-		"gis",
-	);
-	let processedContent = content.replace(removeTagsRegex, "");
-
-	// 2. Strip all remaining HTML tags.
-	processedContent = processedContent.replace(/<[^>]+>/g, " ");
-
-	// 3. Normalize whitespace: replace multiple spaces/newlines with a single space and trim.
-	processedContent = processedContent.replace(/\s+/g, " ").trim();
-
-	return processedContent;
-}
-
-/**
- * Function that search in the `/public` dir the content info of the pages and
- * send it to the search table in the ddbb using the API.
- */
-async function uploadRenderedSearchContentToAPI(
-	distDomainPath: string,
-	domain: string,
-) {
-	const config = getConfig();
-	const { __cache } = config.paths(domain);
-
-	// Extraemos el path de la carpeta store dentro de un dominio
-	const storeFolder = path.join(__cache, "store");
-
-	// Este caso sería que el dominio existe en la instancia pero no hay sites
-	// renderizardos bajo el mismo. Por lo que `store` no existe.
-	if (!fs.existsSync(storeFolder)) {
-		console.log(
-			`Skipping uploading content to the search endpoint for the domain ${domain}, it has not exported sites.`,
-		);
-		return;
-	}
-
-	// Obtenemos las páginas del store cacheado de un dominio
-	const storePages = getBuildPagesFromCachedStore(domain);
-
-	for await (const store of storePages) {
-		const {
-			context: { page, openGraph, pageMetadata },
-		} = store;
-
-		const { compose } = page.fullPath;
-
-		const contextPath = path.resolve(`${distDomainPath}/${compose}/index.html`);
-
-		const content = fs.readFileSync(contextPath).toString();
-
-		const pageObject: PostSearchInfoProps = {
-			siteId: page.site,
-			pageId: page.id,
-			// `pageMetadata.title` has already a fallback `metatitle ||
-			// pageTitle` so probably `title` never will be take the
-			// `openGraph?.title` value. Only when the `metatitle` and
-			// `pageTitle` are empty.
-			title: pageMetadata?.title || openGraph?.title,
-			languageId: page.language,
-			url: page.fullUrl,
-			template: page.template.templateType || page.templateId,
-			description: pageMetadata?.description || openGraph?.description,
-			image: openGraph.image,
-			content: prepareHTMLContentForSearch(content),
-		};
-
-		await postSearchInfo(pageObject);
-	}
-}
-
-async function startAIEmbeddings() {
-	try {
-		await post<AIEmbeddingsResponse>({
-			endpoint: endpoints.AI_EMBEDDINGS,
-			useApiCacheDir: false,
-		});
-	} catch (error) {
-		console.warn(
-			"There was an error with the ai embeddings",
-			(error as Error).message,
-		);
-	}
-}
-
-export { postSearchInfo, startAIEmbeddings, uploadRenderedSearchContentToAPI };

package/exporter/utils/sites.ts

@@ -1,312 +0,0 @@
-import type { Site, SiteData } from "../types/sites";
-
-import path from "node:path";
-
-import fs from "fs-extra";
-import { parse } from "js2xmlparser";
-
-import { envs } from "../constants";
-import { getConfig } from "./core-utils";
-import { buildLog, verboseLog } from "./loggin";
-import { getBuildMetadata } from "./store";
-import { AuthService } from "../services/auth";
-import {
-	endSiteRender,
-	getAllSites,
-	getSiteInfo,
-	getSiteLanguages,
-	getSitemap,
-	getSiteSocials,
-	startSiteRender,
-} from "../services/sites";
-
-const config = getConfig();
-
-/**
- * Check the instance sites and returns site prepared to be published and unpublished.
- */
-async function getSitesToRender(domain: string) {
-	console.info(`API URL ${envs.GRIDDO_API_URL as string}`);
-
-	// Login to API
-	await AuthService.login();
-
-	// Get all sites. An array of Site
-	const allSites = await getAllSites(domain);
-
-	// Filter the array of sites to get only the ones to build/render
-	const validSites = envs.GRIDDO_RENDER_ALL_SITES
-		? allSites.filter(
-				(site) =>
-					!envs.GRIDDO_RENDER_SITE || site.id === envs.GRIDDO_RENDER_SITE,
-			)
-		: allSites.filter((site) =>
-				envs.GRIDDO_RENDER_SITE
-					? site.id === envs.GRIDDO_RENDER_SITE
-					: !!site.shouldBeUpdated,
-			);
-
-	// If there are valid sites...
-	// En este paso se añade al objeto `Site` la información de los dominios
-	// utilizando los idiomas.
-	if (validSites.length) {
-		for (const site of validSites) {
-			const { items } = await getSiteLanguages(site.id);
-
-			// Añadimos la prop domains con el dominio "cocinado" con los
-			// idiomas y teniendo en cuenta solo el dominio actual.
-			site.domains = items
-				.filter(
-					(item) =>
-						item.domain &&
-						(item.domain.slug === domain || item.domain.slug === `/${domain}`),
-				)
-				.map((item) => ({ [item.id]: `${item.domain.slug}${item.path}` }));
-		}
-	}
-
-	// Save sites object to publish
-	const sitesToPublish = validSites.filter((site) =>
-		envs.GRIDDO_RENDER_SITE
-			? site.id === envs.GRIDDO_RENDER_SITE
-			: !!site.isPublished,
-	);
-
-	// Save sites object to unpublish
-	const sitesToUnpublish = validSites.filter(
-		(site) => !site.isPublished && site.shouldBeUpdated,
-	);
-
-	return {
-		sitesToPublish,
-		sitesToUnpublish,
-	};
-}
-
-/**
- * Unpublish an array of sites sending the information to the API.
- *
- * @param sites An array of sites
- * @see https://griddoio.notion.site/Sites-d7bb0b7cb8d24894a5337e1139fc3d09#2019d3255bda4d219c7e19cf28d0c4fe
- */
-async function unpublishSites(sites: Array<Site>) {
-	const { __cx } = config.paths();
-
-	for (const site of sites) {
-		// API
-		const buildInfo = await startSiteRender(site.id);
-		const { siteHash } = buildInfo;
-		const body = {
-			siteHash,
-			publishHashes: [],
-			unpublishHashes: [],
-			publishPagesIds: [],
-		};
-
-		await endSiteRender(site.id, body);
-
-		// STORE
-		// Remove site directory from the Store
-		fs.rmSync(path.join(__cx, "store", site.id.toString()), {
-			force: true,
-			recursive: true,
-		});
-	}
-}
-
-/**
- * Return a single site generic data.
- *
- * @param siteID The site id.
- * @see SiteData
- */
-async function getSiteData(siteID: number) {
-	const buildData = await startSiteRender(siteID);
-	const siteInfo = await getSiteInfo(siteID);
-	const siteLangs = await getSiteLanguages(siteID);
-	const socials = await getSiteSocials(siteID);
-	const siteLangsInfo = siteLangs.items;
-	const defaultLang = siteLangsInfo.find((lang) => lang.isDefault);
-
-	const { siteHash, unpublishHashes, publishIds } = buildData;
-	const { headers, footers } = siteInfo;
-	const validPagesIds = envs.GRIDDO_RENDER_PAGES.length
-		? envs.GRIDDO_RENDER_PAGES.filter((item) => publishIds.includes(item))
-		: publishIds;
-
-	const siteData: SiteData = {
-		siteInfo,
-		validPagesIds,
-		siteHash,
-		unpublishHashes,
-		siteLangs: siteLangsInfo,
-		defaultLang,
-		headers,
-		footers,
-		socials,
-	};
-
-	return siteData;
-}
-
-/**
- * Save a file with the end of build process to use as `build-end` signal.
- */
-async function generateBuildReport() {
-	const { __cx } = config.paths();
-	const { buildReportFileName } = config;
-
-	const { buildProcessData } = await getBuildMetadata();
-
-	// Get the token
-	const authControl = await AuthService.login();
-
-	const buildSitesInfo = Object.keys(buildProcessData).map((siteID) => ({
-		...buildProcessData[siteID],
-		siteId: Number.parseInt(siteID),
-	}));
-
-	const report = {
-		authControl,
-		sites: buildSitesInfo,
-	};
-
-	const reportFilePath = path.join(__cx, "dist", buildReportFileName);
-	fs.writeFileSync(reportFilePath, JSON.stringify(report));
-
-	verboseLog(`build report saved in ${reportFilePath}`);
-
-	buildLog(`Build report saved for ${buildSitesInfo.length} site(s)`);
-}
-
-/**
- * Generate sitemaps and save them into file system.
- */
-async function generateSitemaps() {
-	const { sitesToPublish } = await getBuildMetadata();
-	const { __cx } = config.paths();
-	const distDir = path.join(__cx, "dist");
-
-	const templateSitemapPrefix = "sitemap-";
-
-	const isGriddoSitemapFile = (file: string) =>
-		file === "sitemap.xml" ||
-		(file.startsWith(templateSitemapPrefix) && file.endsWith(".xml"));
-
-	for (const site of sitesToPublish) {
-		const { id, languages } = site;
-
-		for (const lang of languages) {
-			if (AuthService.headers) {
-				AuthService.headers.lang = lang.id.toString();
-			}
-
-			const response = await getSitemap(id);
-
-			if (!response) continue;
-
-			const {
-				items: sitemapPagesGroup,
-				url: { home, domain },
-			} = response;
-
-			if (!home) continue;
-
-			const domainLanguage = site.domains.find(
-				(domain) => Object.keys(domain)[0] === lang.id.toString(),
-			);
-
-			if (!domainLanguage) continue;
-
-			const slug = Object.values(domainLanguage)[0];
-			const sitemaps: Array<string> = [];
-			const sitemapPageGroupKeys = Object.keys(sitemapPagesGroup);
-			const sitemapBasePath = path.join(distDir, slug.replace(domain, ""));
-
-			/** delete every griddo sitemap file before create the new ones */
-			if (fs.existsSync(sitemapBasePath)) {
-				try {
-					const files = fs.readdirSync(sitemapBasePath);
-					for (const file of files) {
-						if (isGriddoSitemapFile(file)) {
-							const filePath = path.join(sitemapBasePath, file);
-							try {
-								fs.unlinkSync(filePath);
-							} catch (err) {
-								console.error(`Error removing ${filePath}:`, err);
-							}
-						}
-					}
-				} catch (err) {
-					console.error(`Error reading ${sitemapBasePath}:`, err);
-				}
-			}
-
-			for (const templateId of sitemapPageGroupKeys) {
-				const sitemapPages = sitemapPagesGroup[templateId];
-
-				if (!sitemapPages.length) continue;
-
-				const siteMap = parse("urlset", {
-					"@": {
-						xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
-						"xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
-						"xsi:schemaLocation":
-							"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd",
-					},
-					url: sitemapPages,
-				});
-
-				const sitemapName = `/${templateSitemapPrefix}${templateId.toLowerCase()}.xml`;
-				const exactPath = path.join(sitemapBasePath, sitemapName);
-
-				saveFile(exactPath, siteMap);
-
-				sitemaps.push(
-					`${home.endsWith("/") ? home.slice(0, -1) : home}${sitemapName}`,
-				);
-			}
-
-			if (!sitemaps.length) {
-				continue;
-			}
-
-			const siteMap = parse("sitemapindex", {
-				"@": { xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9" },
-				sitemap: sitemaps.map((loc) => ({ loc })),
-			});
-
-			const exactPath = path.join(sitemapBasePath, "sitemap.xml");
-
-			saveFile(exactPath, siteMap);
-			verboseLog(`sitemap generated in ${exactPath}`);
-		}
-	}
-}
-
-/**
- * Saves the content to a file specified by its path. If the file exists, it will be overwritten.
- *
- * @param filePath The path of the file to save the content to.
- * @param content The content to save to the file.
- */
-function saveFile(filePath: string, content: string) {
-	try {
-		const pathName = path.dirname(filePath);
-		if (!fs.existsSync(pathName)) {
-			fs.mkdirSync(pathName, { recursive: true });
-		}
-		fs.writeFileSync(filePath, content);
-	} catch (error) {
-		console.log(error);
-		throw new Error(`Error saving a file`);
-	}
-}
-
-export {
-	generateBuildReport,
-	generateSitemaps,
-	getSiteData,
-	getSitesToRender,
-	unpublishSites,
-};