@griddo/cx 11.13.3 → 11.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/build/commands/end-render.js +31 -31
  2. package/build/commands/end-render.js.map +4 -4
  3. package/build/commands/notify-indexnow.d.ts +1 -0
  4. package/build/commands/notify-indexnow.js +161 -0
  5. package/build/commands/notify-indexnow.js.map +7 -0
  6. package/build/commands/prepare-assets-directory.js +12 -12
  7. package/build/commands/prepare-assets-directory.js.map +3 -3
  8. package/build/commands/prepare-domains-render.js +12 -12
  9. package/build/commands/prepare-domains-render.js.map +3 -3
  10. package/build/commands/reset-render.js +23 -23
  11. package/build/commands/reset-render.js.map +3 -3
  12. package/build/commands/start-render.js +51 -51
  13. package/build/commands/start-render.js.map +4 -4
  14. package/build/commands/upload-search-content.js +12 -12
  15. package/build/commands/upload-search-content.js.map +3 -3
  16. package/build/index.js +5 -3
  17. package/build/services/indexnow-key-file.d.ts +22 -0
  18. package/build/services/indexnow-notify.d.ts +29 -0
  19. package/build/services/indexnow-payload.d.ts +54 -0
  20. package/build/services/render.d.ts +2 -1
  21. package/build/shared/envs.d.ts +2 -1
  22. package/cli.mjs +9 -0
  23. package/exporter/build.sh +1 -0
  24. package/exporter/commands/end-render.ts +11 -0
  25. package/exporter/commands/notify-indexnow.ts +19 -0
  26. package/exporter/services/generate-md.ts +7 -10
  27. package/exporter/services/indexnow-key-file.ts +58 -0
  28. package/exporter/services/indexnow-notify.ts +196 -0
  29. package/exporter/services/indexnow-payload.ts +195 -0
  30. package/exporter/services/llms.ts +14 -1
  31. package/exporter/services/render.ts +6 -0
  32. package/exporter/services/store.ts +24 -0
  33. package/exporter/shared/envs.ts +2 -0
  34. package/exporter/ssg-adapters/gatsby/actions/meta.ts +2 -0
  35. package/exporter/ssg-adapters/gatsby/actions/sync.ts +6 -1
  36. package/exporter/ssg-adapters/gatsby/shared/sync-render.ts +1 -5
  37. package/package.json +4 -3
package/build/index.js CHANGED
@@ -24918,6 +24918,7 @@ function safeParseInt(value, fallback) {
24918
24918
  return Number.isNaN(parsed) ? fallback : parsed;
24919
24919
  }
24920
24920
  var GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS);
24921
+ var GRIDDO_INDEXNOW_ENABLED = envIsTruthy(env.GRIDDO_INDEXNOW_ENABLED);
24921
24922
  var GRIDDO_API_CONCURRENCY_COUNT = safeParseInt(env.GRIDDO_API_CONCURRENCY_COUNT || env.GRIDDO_RENDER_CONCURRENCY_COUNT || "10", 10);
24922
24923
  var GRIDDO_ASSET_PREFIX = env.GRIDDO_ASSET_PREFIX || env.ASSET_PREFIX || env.GRIDDO_RENDER_ASSET_PREFIX;
24923
24924
  var GRIDDO_BUILD_LOGS = envIsTruthy(env.GRIDDO_BUILD_LOGS || env.GRIDDO_RENDER_BUILD_LOGS);
@@ -25327,7 +25328,7 @@ var SITE_URI = `${GRIDDO_API_URL2}/site/`;
25327
25328
  var package_default = {
25328
25329
  name: "@griddo/cx",
25329
25330
  description: "Griddo SSG based on Gatsby",
25330
- version: "11.13.3",
25331
+ version: "11.14.0",
25331
25332
  authors: [
25332
25333
  "Hisco <francis.vega@griddo.io>"
25333
25334
  ],
@@ -25366,6 +25367,7 @@ var package_default = {
25366
25367
  "start-render": "node ./build/commands/start-render",
25367
25368
  "end-render": "node ./build/commands/end-render",
25368
25369
  "upload-search-content": "node ./build/commands/upload-search-content",
25370
+ "notify-indexnow": "node ./build/commands/notify-indexnow",
25369
25371
  "reset-render": "node ./build/commands/reset-render",
25370
25372
  "// ONLY LOCAL SCRIPTS": "",
25371
25373
  "prepare-assets-directory": "node ./build/commands/prepare-assets-directory",
@@ -25387,7 +25389,7 @@ var package_default = {
25387
25389
  },
25388
25390
  devDependencies: {
25389
25391
  "@biomejs/biome": "2.3.4",
25390
- "@griddo/core": "11.13.3",
25392
+ "@griddo/core": "11.14.0",
25391
25393
  "@types/node": "20.19.4",
25392
25394
  "@typescript/native-preview": "7.0.0-dev.20260401.1",
25393
25395
  cheerio: "1.1.2",
@@ -25421,7 +25423,7 @@ var package_default = {
25421
25423
  publishConfig: {
25422
25424
  access: "public"
25423
25425
  },
25424
- gitHead: "6dd412b5408e057baa6c1c89f4815c85c7e16a45"
25426
+ gitHead: "d2c96b30e53ce16a9d06df7fd512308ae6fff64c"
25425
25427
  };
25426
25428
 
25427
25429
  // exporter/shared/headers.ts
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Derive the IndexNow key for a given domain. It is a deterministic function
3
+ * of the domain slug so every machine and every rerun produces the same key.
4
+ * IndexNow only requires the key to be 8-128 alphanumeric chars and to match
5
+ * the file served at the host root; it does not need to be secret (ownership
6
+ * is proven by the ability to serve the file, not by the key being private).
7
+ */
8
+ declare function getIndexNowKeyForDomain(domain: string): string;
9
+ /**
10
+ * Filename served at the public root of every domain so IndexNow can verify
11
+ * ownership of the host. Returns `null` when the feature is disabled, which
12
+ * also means the file will not be written.
13
+ */
14
+ declare function getIndexNowKeyFilename(domain: string): string | null;
15
+ /**
16
+ * Write the IndexNow host-key file to the domain `current-dist` so it ends up
17
+ * served at `https://<host>/<key>.txt`. On FROM_SCRATCH the file is carried
18
+ * over via the `current-dist -> dist` rename; on INCREMENTAL it is pulled in
19
+ * through the `artifactsToCopyToExports` list (see sync action).
20
+ */
21
+ declare function generateIndexNowKeyFile(domain: string): Promise<void>;
22
+ export { generateIndexNowKeyFile, getIndexNowKeyFilename, getIndexNowKeyForDomain };
@@ -0,0 +1,29 @@
1
+ export interface NotifyDomainResult {
2
+ domain: string;
3
+ sent: number;
4
+ upsertCount: number;
5
+ deleteCount: number;
6
+ requests: number;
7
+ succeeded: boolean;
8
+ }
9
+ /**
10
+ * Read the payload emitted by `IndexNowCollector`, group URLs by host and POST
11
+ * each batch to the IndexNow endpoint. Sends upserts and deletes together in a
12
+ * single request per host (dedup by URL). Deletes the payload file only when
13
+ * every request succeeded, so a partial failure leaves the file in place for a
14
+ * later retry.
15
+ */
16
+ declare function notifyDomain(options: {
17
+ domain: string;
18
+ key: string;
19
+ }): Promise<NotifyDomainResult | null>;
20
+ /**
21
+ * Orchestrate the IndexNow notification for a single domain. Applies the
22
+ * feature flag and dry-run skips, resolves the host key and delegates to
23
+ * `notifyDomain`. Logs a summary line when a request actually ran.
24
+ */
25
+ declare function notifyIndexNowForDomain(options: {
26
+ slug: string;
27
+ dryRun: boolean;
28
+ }): Promise<void>;
29
+ export { notifyDomain, notifyIndexNowForDomain };
@@ -0,0 +1,54 @@
1
+ import type { GriddoPageObject } from "../shared/types/pages";
2
+ import type { Site } from "../shared/types/sites";
3
+ export interface IndexNowUpsertEntry {
4
+ id: number;
5
+ url: string;
6
+ siteId: number;
7
+ }
8
+ export interface IndexNowDeleteEntry {
9
+ id: number;
10
+ url: string;
11
+ siteId: number;
12
+ reason: "offlinePending" | "siteUnpublished";
13
+ }
14
+ export interface IndexNowPayloadFile {
15
+ domain: string;
16
+ renderedAt: string;
17
+ upsert: IndexNowUpsertEntry[];
18
+ delete: IndexNowDeleteEntry[];
19
+ skipped: {
20
+ noindex: number;
21
+ noUrlResolved: number;
22
+ paginatedSecondary: number;
23
+ };
24
+ }
25
+ interface FlushOptions {
26
+ domain: string;
27
+ sitesToPublish: Site[];
28
+ sitesToUnpublish: Site[];
29
+ }
30
+ interface CollectOptions {
31
+ siteId: number;
32
+ pageId: number;
33
+ pageObjects: GriddoPageObject[];
34
+ uploadPending: number[];
35
+ }
36
+ /**
37
+ * Collects URLs that should be notified to IndexNow during `createStore`.
38
+ * Intended usage:
39
+ * - `collect()` during the per-page loop (reuses the already-fetched page
40
+ * objects, so upserts cost 0 extra API calls).
41
+ * - `flush()` once per render to resolve delete URLs via `getPage` and write
42
+ * the payload file to `.griddo/cache/<domain>/indexnow-payload.json`.
43
+ */
44
+ declare class IndexNowCollector {
45
+ readonly enabled: boolean;
46
+ private upserts;
47
+ private skipped;
48
+ constructor(enabled?: boolean);
49
+ collect(options: CollectOptions): void;
50
+ flush(options: FlushOptions): Promise<void>;
51
+ private extractCanonical;
52
+ private resolveUrlForId;
53
+ }
54
+ export { IndexNowCollector };
@@ -59,6 +59,7 @@ declare function getRenderPathsHydratedWithDomainFromDB(options?: {
59
59
  __ssg: string;
60
60
  __exports_dist: string;
61
61
  }>;
62
+ declare function getDomainDryRunFromDB(domain: string): Promise<boolean>;
62
63
  declare function getRenderMetadataFromDB(): Promise<{
63
64
  griddoVersion: string;
64
65
  buildReportFileName: string;
@@ -67,4 +68,4 @@ declare function getRenderMetadataFromDB(): Promise<{
67
68
  * Save a file with the end of build process to use as `end-render` signal.
68
69
  */
69
70
  declare function generateBuildReport(domain: string): Promise<void>;
70
- export { assertRenderIsValid, generateBuildReport, getRenderMetadataFromDB, getRenderModeFromDB, getRenderPathsHydratedWithDomainFromDB, markRenderAsCompleted, markRenderAsStarted, resolveDomainRenderMode, updateCommitFile, };
71
+ export { assertRenderIsValid, generateBuildReport, getDomainDryRunFromDB, getRenderMetadataFromDB, getRenderModeFromDB, getRenderPathsHydratedWithDomainFromDB, markRenderAsCompleted, markRenderAsStarted, resolveDomainRenderMode, updateCommitFile, };
@@ -3,6 +3,7 @@ declare const GRIDDO_PUBLIC_API_URL: string | undefined;
3
3
  declare const GRIDDO_BOT_USER: string | undefined;
4
4
  declare const GRIDDO_BOT_PASSWORD: string | undefined;
5
5
  declare const GRIDDO_AI_EMBEDDINGS: boolean;
6
+ declare const GRIDDO_INDEXNOW_ENABLED: boolean;
6
7
  declare const GRIDDO_API_CONCURRENCY_COUNT: number;
7
8
  declare const GRIDDO_ASSET_PREFIX: string | undefined;
8
9
  declare const GRIDDO_BUILD_LOGS: boolean;
@@ -22,4 +23,4 @@ declare const GRIDDO_RENDER_API_TIMEOUT_MS: number;
22
23
  declare const GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD: number;
23
24
  declare const GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS: number;
24
25
  declare const GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS: number;
25
- export { GRIDDO_AI_EMBEDDINGS, GRIDDO_API_CONCURRENCY_COUNT, GRIDDO_API_URL, GRIDDO_ASSET_PREFIX, GRIDDO_BOT_PASSWORD, GRIDDO_BOT_USER, GRIDDO_BUILD_LOGS, GRIDDO_BUILD_LOGS_BUFFER_SIZE, GRIDDO_PUBLIC_API_URL, GRIDDO_REACT_APP_INSTANCE, GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS, GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS, GRIDDO_RENDER_API_TIMEOUT_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD, GRIDDO_RENDER_DISABLE_LLMS_TXT, GRIDDO_RENDER_ENABLED_LLM_MD, GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS, GRIDDO_SEARCH_FEATURE, GRIDDO_SKIP_BUILD_CHECKS, GRIDDO_SSG_BUNDLE_ANALYZER, GRIDDO_SSG_VERBOSE_LOGS, GRIDDO_USE_DIST_BACKUP, GRIDDO_VERBOSE_LOGS, };
26
+ export { GRIDDO_AI_EMBEDDINGS, GRIDDO_API_CONCURRENCY_COUNT, GRIDDO_API_URL, GRIDDO_ASSET_PREFIX, GRIDDO_BOT_PASSWORD, GRIDDO_BOT_USER, GRIDDO_BUILD_LOGS, GRIDDO_BUILD_LOGS_BUFFER_SIZE, GRIDDO_INDEXNOW_ENABLED, GRIDDO_PUBLIC_API_URL, GRIDDO_REACT_APP_INSTANCE, GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS, GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS, GRIDDO_RENDER_API_TIMEOUT_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD, GRIDDO_RENDER_DISABLE_LLMS_TXT, GRIDDO_RENDER_ENABLED_LLM_MD, GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS, GRIDDO_SEARCH_FEATURE, GRIDDO_SKIP_BUILD_CHECKS, GRIDDO_SSG_BUNDLE_ANALYZER, GRIDDO_SSG_VERBOSE_LOGS, GRIDDO_USE_DIST_BACKUP, GRIDDO_VERBOSE_LOGS, };
package/cli.mjs CHANGED
@@ -85,6 +85,15 @@ export const AVAILABLE_COMMANDS = [
85
85
  rootArgument: true,
86
86
  noStaticFiles: false,
87
87
  },
88
+ {
89
+ name: "notify-indexnow",
90
+ description: "Notify IndexNow with the render delta of every domain",
91
+ domainArgument: false,
92
+ script: "notify-indexnow",
93
+ isCommandPack: false,
94
+ rootArgument: true,
95
+ noStaticFiles: false,
96
+ },
88
97
  {
89
98
  name: "reset-render",
90
99
  description: "Reset render state",
package/exporter/build.sh CHANGED
@@ -20,6 +20,7 @@ esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/
20
20
  # This .sh needs to be run from a npm script, so esbuild dependency is available
21
21
  esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts}end-render.js
22
22
  esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts}upload-search-content.js
23
+ esbuild ${log} ./exporter/commands/notify-indexnow.ts ${bundle_node_opts}notify-indexnow.js
23
24
  esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts}reset-render.js
24
25
  esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts}start-render.js
25
26
  esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts}prepare-domains-render.js
@@ -5,7 +5,9 @@ import { throwError, withErrorHandler } from "../core/errors";
5
5
  import { pathExists } from "../core/fs";
6
6
  import { GriddoLog } from "../core/GriddoLog";
7
7
  import { AuthService } from "../services/auth";
8
+ import { notifyIndexNowForDomain } from "../services/indexnow-notify";
8
9
  import {
10
+ getDomainDryRunFromDB,
9
11
  getRenderMetadataFromDB,
10
12
  getRenderModeFromDB,
11
13
  getRenderPathsHydratedWithDomainFromDB,
@@ -62,6 +64,15 @@ async function endRender() {
62
64
  GriddoLog.info(`Site (${siteId})`);
63
65
  await endSiteRender(siteId, body);
64
66
  }
67
+
68
+ try {
69
+ const dryRun = await getDomainDryRunFromDB(domain);
70
+ await notifyIndexNowForDomain({ slug: domain, dryRun });
71
+ } catch (err) {
72
+ GriddoLog.warn(
73
+ `indexnow: notify failed for ${domain}: ${(err as Error)?.message ?? String(err)}`,
74
+ );
75
+ }
65
76
  }
66
77
 
67
78
  async function main() {
@@ -0,0 +1,19 @@
1
+ import { withErrorHandler } from "../core/errors";
2
+ import { AuthService } from "../services/auth";
3
+ import { getInstanceDomains } from "../services/domains";
4
+ import { notifyIndexNowForDomain } from "../services/indexnow-notify";
5
+
6
+ async function notifyIndexNow() {
7
+ const domains = await getInstanceDomains();
8
+
9
+ for (const { slug, dryRun } of domains) {
10
+ await notifyIndexNowForDomain({ slug, dryRun });
11
+ }
12
+ }
13
+
14
+ async function main() {
15
+ await AuthService.login();
16
+ await notifyIndexNow();
17
+ }
18
+
19
+ withErrorHandler(main);
@@ -28,20 +28,17 @@ export async function generateMdsFromHtmls(domain: string) {
28
28
 
29
29
  try {
30
30
  for await (const page of pages) {
31
- // Es home de dominio
32
- // Evitamos hacer markdown del home principal del dominio porque ahí irá el llms.txt
33
- if (page.path === path.join(distDirectory, "index.html")) continue;
34
-
35
31
  // No es index.html
36
32
  // evita parsear algunos htmls que se cuelan de gatsby...
37
33
  if (!page.path.endsWith("index.html")) continue;
38
34
 
39
- const pathWithoutIndexHtml = page.path.split(/\/index\.html$/);
40
- let file = "";
41
- if (pathWithoutIndexHtml.length === 2) {
42
- file = pathWithoutIndexHtml[0];
43
- }
44
- await fsp.writeFile(`${file}.md`, page.content);
35
+ // Home del dominio: escribimos dist/index.md porque dominio.com.md en browser queda raro
36
+ const isHome = page.path === path.join(distDirectory, "index.html");
37
+ const outPath = isHome
38
+ ? path.join(distDirectory, "index.md")
39
+ : `${page.path.replace(/\/index\.html$/, "")}.md`;
40
+
41
+ await fsp.writeFile(outPath, page.content);
45
42
  GriddoLog.info(`Generating MD for a ${page.path}`);
46
43
  }
47
44
  } catch (error) {
@@ -0,0 +1,58 @@
1
+ import { createHash } from "node:crypto";
2
+ import fsp from "node:fs/promises";
3
+ import path from "node:path";
4
+
5
+ import { GriddoLog } from "../core/GriddoLog";
6
+ import { GRIDDO_INDEXNOW_ENABLED } from "../shared/envs";
7
+ import { getRenderPathsHydratedWithDomainFromDB } from "./render";
8
+
9
+ const KEY_LENGTH = 32;
10
+
11
+ /**
12
+ * Derive the IndexNow key for a given domain. It is a deterministic function
13
+ * of the domain slug so every machine and every rerun produces the same key.
14
+ * IndexNow only requires the key to be 8-128 alphanumeric chars and to match
15
+ * the file served at the host root; it does not need to be secret (ownership
16
+ * is proven by the ability to serve the file, not by the key being private).
17
+ */
18
+ function getIndexNowKeyForDomain(domain: string): string {
19
+ return createHash("sha256").update(domain, "utf-8").digest("hex").slice(0, KEY_LENGTH);
20
+ }
21
+
22
+ /**
23
+ * Filename served at the public root of every domain so IndexNow can verify
24
+ * ownership of the host. Returns `null` when the feature is disabled, which
25
+ * also means the file will not be written.
26
+ */
27
+ function getIndexNowKeyFilename(domain: string): string | null {
28
+ if (!GRIDDO_INDEXNOW_ENABLED) return null;
29
+ return `${getIndexNowKeyForDomain(domain)}.txt`;
30
+ }
31
+
32
+ /**
33
+ * Write the IndexNow host-key file to the domain `current-dist` so it ends up
34
+ * served at `https://<host>/<key>.txt`. On FROM_SCRATCH the file is carried
35
+ * over via the `current-dist -> dist` rename; on INCREMENTAL it is pulled in
36
+ * through the `artifactsToCopyToExports` list (see sync action).
37
+ */
38
+ async function generateIndexNowKeyFile(domain: string): Promise<void> {
39
+ const filename = getIndexNowKeyFilename(domain);
40
+ if (!filename) return;
41
+
42
+ const key = getIndexNowKeyForDomain(domain);
43
+ const { __root } = await getRenderPathsHydratedWithDomainFromDB({ domain });
44
+ const distDirectory = path.join(__root, "current-dist");
45
+ const filePath = path.join(distDirectory, filename);
46
+
47
+ try {
48
+ await fsp.mkdir(distDirectory, { recursive: true });
49
+ await fsp.writeFile(filePath, key, "utf-8");
50
+ GriddoLog.verbose(`indexnow: wrote host-key file ${filePath}`);
51
+ } catch (err) {
52
+ GriddoLog.warn(
53
+ `indexnow: failed to write host-key file ${filePath}: ${(err as Error)?.message ?? String(err)}`,
54
+ );
55
+ }
56
+ }
57
+
58
+ export { generateIndexNowKeyFile, getIndexNowKeyFilename, getIndexNowKeyForDomain };
@@ -0,0 +1,196 @@
1
+ import type { IndexNowPayloadFile } from "./indexnow-payload";
2
+
3
+ import fsp from "node:fs/promises";
4
+ import path from "node:path";
5
+
6
+ import { GriddoLog } from "../core/GriddoLog";
7
+ import { GRIDDO_INDEXNOW_ENABLED } from "../shared/envs";
8
+ import { getIndexNowKeyForDomain } from "./indexnow-key-file";
9
+ import { getRenderPathsHydratedWithDomainFromDB } from "./render";
10
+
11
+ const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
12
+ const INDEXNOW_MAX_URLS_PER_REQUEST = 10_000;
13
+
14
+ export interface NotifyDomainResult {
15
+ domain: string;
16
+ sent: number;
17
+ upsertCount: number;
18
+ deleteCount: number;
19
+ requests: number;
20
+ succeeded: boolean;
21
+ }
22
+
23
+ /**
24
+ * Read the payload emitted by `IndexNowCollector`, group URLs by host and POST
25
+ * each batch to the IndexNow endpoint. Sends upserts and deletes together in a
26
+ * single request per host (dedup by URL). Deletes the payload file only when
27
+ * every request succeeded, so a partial failure leaves the file in place for a
28
+ * later retry.
29
+ */
30
+ async function notifyDomain(options: {
31
+ domain: string;
32
+ key: string;
33
+ }): Promise<NotifyDomainResult | null> {
34
+ const { domain, key } = options;
35
+ const { __cache } = await getRenderPathsHydratedWithDomainFromDB({ domain });
36
+ const payloadPath = path.join(__cache, "indexnow-payload.json");
37
+
38
+ const payload = await readPayload(payloadPath);
39
+ if (!payload) {
40
+ GriddoLog.verbose(`indexnow: no payload for ${domain}, skipping`);
41
+ return null;
42
+ }
43
+
44
+ const upsertCount = payload.upsert.length;
45
+ const deleteCount = payload.delete.length;
46
+
47
+ const urls = collectUrls(payload);
48
+ if (urls.length === 0) {
49
+ GriddoLog.verbose(`indexnow: empty payload for ${domain}, removing file`);
50
+ await safeUnlink(payloadPath);
51
+ return { domain, sent: 0, upsertCount, deleteCount, requests: 0, succeeded: true };
52
+ }
53
+
54
+ const byHost = groupByHost(urls);
55
+ let requests = 0;
56
+ let succeeded = true;
57
+
58
+ for (const [host, hostUrls] of byHost) {
59
+ for (const batch of batchUrls(hostUrls, INDEXNOW_MAX_URLS_PER_REQUEST)) {
60
+ requests++;
61
+ const ok = await postBatch({ host, key, urlList: batch });
62
+ if (!ok) succeeded = false;
63
+ }
64
+ }
65
+
66
+ if (succeeded) {
67
+ await safeUnlink(payloadPath);
68
+ }
69
+
70
+ return { domain, sent: urls.length, upsertCount, deleteCount, requests, succeeded };
71
+ }
72
+
73
+ function collectUrls(payload: IndexNowPayloadFile): string[] {
74
+ const set = new Set<string>();
75
+ for (const entry of payload.upsert) set.add(entry.url);
76
+ for (const entry of payload.delete) set.add(entry.url);
77
+ return Array.from(set);
78
+ }
79
+
80
+ function groupByHost(urls: string[]): Map<string, string[]> {
81
+ const byHost = new Map<string, string[]>();
82
+ for (const url of urls) {
83
+ let host: string;
84
+ try {
85
+ host = new URL(url).host;
86
+ } catch {
87
+ GriddoLog.warn(`indexnow: invalid URL skipped: ${url}`);
88
+ continue;
89
+ }
90
+ const list = byHost.get(host);
91
+ if (list) list.push(url);
92
+ else byHost.set(host, [url]);
93
+ }
94
+ return byHost;
95
+ }
96
+
97
+ function* batchUrls(urls: string[], size: number): Generator<string[]> {
98
+ for (let i = 0; i < urls.length; i += size) {
99
+ yield urls.slice(i, i + size);
100
+ }
101
+ }
102
+
103
+ async function postBatch(options: {
104
+ host: string;
105
+ key: string;
106
+ urlList: string[];
107
+ }): Promise<boolean> {
108
+ const { host, key, urlList } = options;
109
+ const payload = {
110
+ host,
111
+ key,
112
+ keyLocation: `https://${host}/${key}.txt`,
113
+ urlList,
114
+ };
115
+
116
+ try {
117
+ const response = await fetch(INDEXNOW_ENDPOINT, {
118
+ method: "POST",
119
+ headers: { "Content-Type": "application/json; charset=utf-8" },
120
+ body: JSON.stringify(payload),
121
+ });
122
+
123
+ if (!response.ok) {
124
+ const bodyText = await safeReadBody(response);
125
+ GriddoLog.warn(
126
+ `indexnow: ${host} HTTP ${response.status} (${urlList.length} urls)${bodyText ? `: ${bodyText}` : ""}`,
127
+ );
128
+ return false;
129
+ }
130
+
131
+ GriddoLog.verbose(`indexnow: ${host} pinged (${urlList.length} urls, HTTP ${response.status})`);
132
+ return true;
133
+ } catch (err) {
134
+ GriddoLog.warn(`indexnow: ${host} request failed: ${(err as Error)?.message ?? String(err)}`);
135
+ return false;
136
+ }
137
+ }
138
+
139
+ async function safeReadBody(response: Response): Promise<string> {
140
+ try {
141
+ const text = await response.text();
142
+ return text.slice(0, 200);
143
+ } catch {
144
+ return "";
145
+ }
146
+ }
147
+
148
+ async function readPayload(p: string): Promise<IndexNowPayloadFile | null> {
149
+ try {
150
+ const raw = await fsp.readFile(p, "utf-8");
151
+ return JSON.parse(raw) as IndexNowPayloadFile;
152
+ } catch {
153
+ return null;
154
+ }
155
+ }
156
+
157
+ async function safeUnlink(p: string): Promise<void> {
158
+ try {
159
+ await fsp.unlink(p);
160
+ } catch {
161
+ // Missing file is fine.
162
+ }
163
+ }
164
+
165
+ /**
166
+ * Orchestrate the IndexNow notification for a single domain. Applies the
167
+ * feature flag and dry-run skips, resolves the host key and delegates to
168
+ * `notifyDomain`. Logs a summary line when a request actually ran.
169
+ */
170
+ async function notifyIndexNowForDomain(options: {
171
+ slug: string;
172
+ dryRun: boolean;
173
+ }): Promise<void> {
174
+ const { slug, dryRun } = options;
175
+
176
+ if (!GRIDDO_INDEXNOW_ENABLED) {
177
+ GriddoLog.info("indexnow: feature disabled (GRIDDO_INDEXNOW_ENABLED), skipping");
178
+ return;
179
+ }
180
+
181
+ if (dryRun) {
182
+ GriddoLog.verbose(`indexnow: domain ${slug} ignored by dry-render`);
183
+ return;
184
+ }
185
+
186
+ const key = getIndexNowKeyForDomain(slug);
187
+ const result = await notifyDomain({ domain: slug, key });
188
+ if (!result) return;
189
+
190
+ const status = result.succeeded ? "ok" : "partial failure";
191
+ GriddoLog.info(
192
+ `indexnow: ${result.domain} -> ${result.sent} urls (${result.upsertCount} upsert, ${result.deleteCount} delete) in ${result.requests} request(s) [${status}]`,
193
+ );
194
+ }
195
+
196
+ export { notifyDomain, notifyIndexNowForDomain };