@griddo/cx 11.13.3 → 11.14.0

package/exporter/services/indexnow-payload.ts

@@ -0,0 +1,195 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import pLimit from "p-limit";
+
+import { GriddoLog } from "../core/GriddoLog";
+import { GRIDDO_API_CONCURRENCY_COUNT, GRIDDO_INDEXNOW_ENABLED } from "../shared/envs";
+import type { GriddoPageObject } from "../shared/types/pages";
+import type { Site } from "../shared/types/sites";
+import { getRenderPathsHydratedWithDomainFromDB } from "./render";
+import { getPage } from "./sites";
+
+export interface IndexNowUpsertEntry {
+	id: number;
+	url: string;
+	siteId: number;
+}
+
+export interface IndexNowDeleteEntry {
+	id: number;
+	url: string;
+	siteId: number;
+	reason: "offlinePending" | "siteUnpublished";
+}
+
+export interface IndexNowPayloadFile {
+	domain: string;
+	renderedAt: string;
+	upsert: IndexNowUpsertEntry[];
+	delete: IndexNowDeleteEntry[];
+	skipped: {
+		noindex: number;
+		noUrlResolved: number;
+		paginatedSecondary: number;
+	};
+}
+
+interface FlushOptions {
+	domain: string;
+	sitesToPublish: Site[];
+	sitesToUnpublish: Site[];
+}
+
+interface CollectOptions {
+	siteId: number;
+	pageId: number;
+	pageObjects: GriddoPageObject[];
+	uploadPending: number[];
+}
+
+/**
+ * Collects URLs that should be notified to IndexNow during `createStore`.
+ * Intended usage:
+ * - `collect()` during the per-page loop (reuses the already-fetched page
+ *   objects, so upserts cost 0 extra API calls).
+ * - `flush()` once per render to resolve delete URLs via `getPage` and write
+ *   the payload file to `.griddo/cache/<domain>/indexnow-payload.json`.
+ */
+class IndexNowCollector {
+	public readonly enabled: boolean;
+	private upserts: IndexNowUpsertEntry[] = [];
+	private skipped = { noindex: 0, noUrlResolved: 0, paginatedSecondary: 0 };
+
+	constructor(enabled: boolean = GRIDDO_INDEXNOW_ENABLED) {
+		this.enabled = enabled;
+	}
+
+	collect(options: CollectOptions): void {
+		if (!this.enabled) return;
+		const { siteId, pageId, pageObjects, uploadPending } = options;
+		if (!uploadPending.includes(pageId)) return;
+
+		for (const obj of pageObjects) {
+			const entry = this.extractCanonical(obj, siteId, pageId);
+			if (entry) this.upserts.push(entry);
+		}
+	}
+
+	async flush(options: FlushOptions): Promise<void> {
+		if (!this.enabled) return;
+		const { domain, sitesToPublish, sitesToUnpublish } = options;
+
+		const { __cache } = await getRenderPathsHydratedWithDomainFromDB({ domain });
+		const payloadPath = path.join(__cache, "indexnow-payload.json");
+
+		// Always drop the previous payload first so a failure leaves no stale file.
+		await safeUnlink(payloadPath);
+
+		const deletes: IndexNowDeleteEntry[] = [];
+		const limit = pLimit(GRIDDO_API_CONCURRENCY_COUNT);
+		const jobs: Promise<void>[] = [];
+
+		for (const site of sitesToPublish) {
+			for (const id of site.pagesStatus.offlinePending) {
+				jobs.push(
+					limit(async () => {
+						const url = await this.resolveUrlForId(id);
+						if (url) {
+							deletes.push({ id, url, siteId: site.id, reason: "offlinePending" });
+						} else {
+							this.skipped.noUrlResolved++;
+						}
+					}),
+				);
+			}
+		}
+
+		for (const site of sitesToUnpublish) {
+			const ids = new Set<number>([
+				...site.pagesStatus.active,
+				...site.pagesStatus.offlinePending,
+			]);
+			for (const id of ids) {
+				jobs.push(
+					limit(async () => {
+						const url = await this.resolveUrlForId(id);
+						if (url) {
+							deletes.push({ id, url, siteId: site.id, reason: "siteUnpublished" });
+						} else {
+							this.skipped.noUrlResolved++;
+						}
+					}),
+				);
+			}
+		}
+
+		await Promise.all(jobs);
+
+		const payload: IndexNowPayloadFile = {
+			domain,
+			renderedAt: new Date().toISOString(),
+			upsert: this.upserts,
+			delete: deletes,
+			skipped: this.skipped,
+		};
+
+		await fsp.mkdir(__cache, { recursive: true });
+		await fsp.writeFile(payloadPath, JSON.stringify(payload, null, 2), "utf-8");
+		GriddoLog.verbose(
+			`indexnow payload written (upsert:${payload.upsert.length} delete:${payload.delete.length} skipped:${JSON.stringify(payload.skipped)})`,
+		);
+	}
+
+	private extractCanonical(
+		obj: GriddoPageObject,
+		siteId: number,
+		pageId: number,
+	): IndexNowUpsertEntry | null {
+		const page = obj.context.page as { fullUrl?: string; template?: { pageNumber?: number } };
+		const meta = obj.context.pageMetadata;
+
+		// Skip paginated secondary pages (/2/, /3/...). Page 1 is the canonical entry.
+		const pageNumber = page?.template?.pageNumber;
+		if (typeof pageNumber === "number" && pageNumber > 1) {
+			this.skipped.paginatedSecondary++;
+			return null;
+		}
+
+		const url = page?.fullUrl;
+		if (!url) {
+			this.skipped.noUrlResolved++;
+			return null;
+		}
+
+		if (meta?.index === "noindex") {
+			this.skipped.noindex++;
+			return null;
+		}
+
+		return { id: pageId, url, siteId };
+	}
+
+	private async resolveUrlForId(id: number): Promise<string | null> {
+		try {
+			const page = await getPage(id, "indexnow-payload");
+			const url = (page as { fullUrl?: string } | null)?.fullUrl;
+			return url ?? null;
+		} catch (err) {
+			GriddoLog.warn(
+				`indexnow: getPage(${id}) failed: ${(err as Error)?.message ?? String(err)}`,
+			);
+			return null;
+		}
+	}
+}
+
+async function safeUnlink(p: string): Promise<void> {
+	try {
+		await fsp.unlink(p);
+	} catch {
+		// Missing file is fine; nothing to clear.
+	}
+}
+
+export { IndexNowCollector };

package/exporter/services/llms.ts

@@ -71,7 +71,20 @@ async function generateLlmsTxt(domain: string): Promise<void> {
 		const pageLinks = llmsResponse
 			.map(({ title, url, socialDescription }) => {
 				const description = socialDescription ? `: ${socialDescription}` : "";
-				return `- [${title}](${removeTrailingSlash(url)}${pageIndexName})${description}`;
+				const strippedUrl = removeTrailingSlash(url);
+				// Home: pathname "/" → /index.md evita que quede dominio.com.md
+				const isHome = (() => {
+					try {
+						return new URL(url).pathname === "/";
+					} catch {
+						return url === "/" || url === "";
+					}
+				})();
+				const href =
+					GRIDDO_RENDER_ENABLED_LLM_MD && isHome
+						? `${strippedUrl}/index.md`
+						: `${strippedUrl}${pageIndexName}`;
+				return `- [${title}](${href})${description}`;
 			})
 			.join("\n");
 

package/exporter/services/render.ts

@@ -181,6 +181,11 @@ async function getRenderPathsHydratedWithDomainFromDB(options?: {
 	};
 }
 
+async function getDomainDryRunFromDB(domain: string): Promise<boolean> {
+	const db = await readDB();
+	return !!db.domains[domain]?.dryRun;
+}
+
 async function getRenderMetadataFromDB() {
 	const db = await readDB();
 	return {
@@ -228,6 +233,7 @@ async function generateBuildReport(domain: string) {
 export {
 	assertRenderIsValid,
 	generateBuildReport,
+	getDomainDryRunFromDB,
 	getRenderMetadataFromDB,
 	getRenderModeFromDB,
 	getRenderPathsHydratedWithDomainFromDB,

package/exporter/services/store.ts

@@ -35,6 +35,7 @@ import {
 	getMultiPageElements,
 	getPaginatedPages,
 } from "./pages";
+import { IndexNowCollector } from "./indexnow-payload";
 import { getReferenceFieldData } from "./reference-fields";
 import { getRenderPathsHydratedWithDomainFromDB } from "./render";
 import { getPage } from "./sites";
@@ -80,6 +81,9 @@ async function createStore(options: {
 	const createdPages: number[] = [];
 	const buildProcessData: BuildProcessData = {};
 
+	// IndexNow delta collector. No-op when the feature flag is off.
+	const indexNowCollector = new IndexNowCollector();
+
 	// Get sites objects to publish and unpublish from this domain
 	const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
 
@@ -115,6 +119,11 @@ async function createStore(options: {
 		const { id: siteId, slug: siteSlug, theme, favicon, pagesStatus } = site;
 		const siteDirName = siteId.toString();
 
+		// Snapshot of real pending-to-upload IDs before FROM_SCRATCH merges
+		// `active` into `pagesStatus.uploadPending` below. Used by IndexNow so
+		// unchanged `active` pages are not reported as upserts on FROM_SCRATCH.
+		const uploadPendingForIndexNow = [...pagesStatus.uploadPending];
+
 		allPagesToRemoveFromBuild.push(...pagesStatus.offlinePending, ...pagesStatus.deleted);
 
 		const {
@@ -312,6 +321,13 @@ async function createStore(options: {
 			// Save build data to store
 			await saveSitePagesInStore(siteIdName, griddoPageObjects);
 
+			indexNowCollector.collect({
+				siteId,
+				pageId: page.id,
+				pageObjects: griddoPageObjects,
+				uploadPending: uploadPendingForIndexNow,
+			});
+
 			// Update progress counter and log
 			progressCounter.current += 1;
 			progress.update(progressCounter.current);
@@ -355,6 +371,14 @@ async function createStore(options: {
 	// infor del render normalmente.
 	await saveRenderInfoInStore({ buildProcessData, createdPages, sitesToPublish }, domain);
 
+	try {
+		await indexNowCollector.flush({ domain, sitesToPublish, sitesToUnpublish });
+	} catch (err) {
+		GriddoLog.warn(
+			`indexnow: failed to write delta for ${domain}: ${(err as Error)?.message ?? String(err)}`,
+		);
+	}
+
 	return {
 		pagesToCreate: createdPages,
 		pagesToDelete: allPagesToRemoveFromBuild,

package/exporter/shared/envs.ts

@@ -35,6 +35,7 @@ function safeParseInt(value: string, fallback: number): number {
 
 // Rendering
 const GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS);
+const GRIDDO_INDEXNOW_ENABLED = envIsTruthy(env.GRIDDO_INDEXNOW_ENABLED);
 const GRIDDO_API_CONCURRENCY_COUNT = safeParseInt(env.GRIDDO_API_CONCURRENCY_COUNT || env.GRIDDO_RENDER_CONCURRENCY_COUNT || "10", 10);
 const GRIDDO_ASSET_PREFIX = env.GRIDDO_ASSET_PREFIX || env.ASSET_PREFIX || env.GRIDDO_RENDER_ASSET_PREFIX;
 const GRIDDO_BUILD_LOGS = envIsTruthy(env.GRIDDO_BUILD_LOGS || env.GRIDDO_RENDER_BUILD_LOGS);
@@ -64,6 +65,7 @@ export {
 	GRIDDO_BOT_USER,
 	GRIDDO_BUILD_LOGS,
 	GRIDDO_BUILD_LOGS_BUFFER_SIZE,
+	GRIDDO_INDEXNOW_ENABLED,
 	GRIDDO_PUBLIC_API_URL,
 	GRIDDO_REACT_APP_INSTANCE,
 	GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS,

package/exporter/ssg-adapters/gatsby/actions/meta.ts

@@ -1,5 +1,6 @@
 import type { RenderContext } from "@shared/context";
 
+import { generateIndexNowKeyFile } from "@services/indexnow-key-file";
 import { generateRobots } from "@services/robots";
 import { generateSitemaps } from "@services/sitemaps";
 
@@ -8,4 +9,5 @@ export async function metaAction(context: RenderContext) {
 
 	await generateSitemaps(domain);
 	await generateRobots(domain);
+	await generateIndexNowKeyFile(domain);
 }

package/exporter/ssg-adapters/gatsby/actions/sync.ts

@@ -5,6 +5,7 @@ import fsp from "node:fs/promises";
 import path from "node:path";
 
 import { deleteDisposableSiteDirs, deleteEmptyDirectories, mvDirs } from "../../../core/fs";
+import { getIndexNowKeyFilename } from "../../../services/indexnow-key-file";
 import { RENDER_MODE } from "../../../shared/types/render";
 import { extractAssetsFromDist } from "../shared/extract-assets";
 import { SyncRender } from "../shared/sync-render";
@@ -42,12 +43,16 @@ export async function syncAction(context: RenderContext<SSG>) {
 	if (renderMode === RENDER_MODE.INCREMENTAL) {
 		await deleteDisposableSiteDirs(previousDist);
 
+		const artifactsToCopyToExports = ["build-report.json", "llms.txt", "robots.txt"];
+		const indexNowKeyFile = getIndexNowKeyFilename(domain);
+		if (indexNowKeyFile) artifactsToCopyToExports.push(indexNowKeyFile);
+
 		const syncRender = new SyncRender({
 			src: currentDist,
 			dst: previousDist,
 			pagesToCreate,
 			pagesToDelete,
-			artifactsToCopyToExports: ["build-report.json", "llms.txt", "robots.txt"],
+			artifactsToCopyToExports,
 		});
 
 		await syncRender.execute();

package/exporter/ssg-adapters/gatsby/shared/sync-render.ts

@@ -144,11 +144,7 @@ class SyncRender {
 				// ../page-data/about-us/page-data.json // página con slug
 				// ../page-data/programs/page-data.json // página con slug
 				// ../page-data/index/page-data.json // <---- ¡página root index!
-				//
-				// Ojo: `composePath` viene normalizado por `removeTrailingSlash`
-				// (ver `scanPages`), así que la home llega como "" y no como "/".
-				const normalizedCompose =
-					page.composePath === "" || page.composePath === "/" ? "index" : page.composePath;
+				const normalizedCompose = page.composePath === "/" ? "index" : page.composePath;
 				const jsonTo = path.join(this.bundleDir, "page-data", normalizedCompose, "page-data.json");
 
 				this.state.htmlToAdd.push({ from: page.htmlPath, to: htmlTo });

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@griddo/cx",
 	"description": "Griddo SSG based on Gatsby",
-	"version": "11.13.3",
+	"version": "11.14.0",
 	"authors": [
 		"Hisco <francis.vega@griddo.io>"
 	],
@@ -40,6 +40,7 @@
 		"start-render": "node ./build/commands/start-render",
 		"end-render": "node ./build/commands/end-render",
 		"upload-search-content": "node ./build/commands/upload-search-content",
+		"notify-indexnow": "node ./build/commands/notify-indexnow",
 		"reset-render": "node ./build/commands/reset-render",
 		"// ONLY LOCAL SCRIPTS": "",
 		"prepare-assets-directory": "node ./build/commands/prepare-assets-directory",
@@ -61,7 +62,7 @@
 	},
 	"devDependencies": {
 		"@biomejs/biome": "2.3.4",
-		"@griddo/core": "11.13.3",
+		"@griddo/core": "11.14.0",
 		"@types/node": "20.19.4",
 		"@typescript/native-preview": "7.0.0-dev.20260401.1",
 		"cheerio": "1.1.2",
@@ -95,5 +96,5 @@
 	"publishConfig": {
 		"access": "public"
 	},
-	"gitHead": "6dd412b5408e057baa6c1c89f4815c85c7e16a45"
+	"gitHead": "d2c96b30e53ce16a9d06df7fd512308ae6fff64c"
 }