@griddo/cx 11.13.3 → 11.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/end-render.js +31 -31
- package/build/commands/end-render.js.map +4 -4
- package/build/commands/notify-indexnow.d.ts +1 -0
- package/build/commands/notify-indexnow.js +161 -0
- package/build/commands/notify-indexnow.js.map +7 -0
- package/build/commands/prepare-assets-directory.js +12 -12
- package/build/commands/prepare-assets-directory.js.map +3 -3
- package/build/commands/prepare-domains-render.js +12 -12
- package/build/commands/prepare-domains-render.js.map +3 -3
- package/build/commands/reset-render.js +23 -23
- package/build/commands/reset-render.js.map +3 -3
- package/build/commands/start-render.js +51 -51
- package/build/commands/start-render.js.map +4 -4
- package/build/commands/upload-search-content.js +12 -12
- package/build/commands/upload-search-content.js.map +3 -3
- package/build/index.js +5 -3
- package/build/services/indexnow-key-file.d.ts +22 -0
- package/build/services/indexnow-notify.d.ts +29 -0
- package/build/services/indexnow-payload.d.ts +54 -0
- package/build/services/render.d.ts +2 -1
- package/build/shared/envs.d.ts +2 -1
- package/cli.mjs +9 -0
- package/exporter/build.sh +1 -0
- package/exporter/commands/end-render.ts +11 -0
- package/exporter/commands/notify-indexnow.ts +19 -0
- package/exporter/services/generate-md.ts +7 -10
- package/exporter/services/indexnow-key-file.ts +58 -0
- package/exporter/services/indexnow-notify.ts +196 -0
- package/exporter/services/indexnow-payload.ts +195 -0
- package/exporter/services/llms.ts +14 -1
- package/exporter/services/render.ts +6 -0
- package/exporter/services/store.ts +24 -0
- package/exporter/shared/envs.ts +2 -0
- package/exporter/ssg-adapters/gatsby/actions/meta.ts +2 -0
- package/exporter/ssg-adapters/gatsby/actions/sync.ts +6 -1
- package/exporter/ssg-adapters/gatsby/shared/sync-render.ts +1 -5
- package/package.json +4 -3
package/build/index.js
CHANGED
|
@@ -24918,6 +24918,7 @@ function safeParseInt(value, fallback) {
|
|
|
24918
24918
|
return Number.isNaN(parsed) ? fallback : parsed;
|
|
24919
24919
|
}
|
|
24920
24920
|
var GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS);
|
|
24921
|
+
var GRIDDO_INDEXNOW_ENABLED = envIsTruthy(env.GRIDDO_INDEXNOW_ENABLED);
|
|
24921
24922
|
var GRIDDO_API_CONCURRENCY_COUNT = safeParseInt(env.GRIDDO_API_CONCURRENCY_COUNT || env.GRIDDO_RENDER_CONCURRENCY_COUNT || "10", 10);
|
|
24922
24923
|
var GRIDDO_ASSET_PREFIX = env.GRIDDO_ASSET_PREFIX || env.ASSET_PREFIX || env.GRIDDO_RENDER_ASSET_PREFIX;
|
|
24923
24924
|
var GRIDDO_BUILD_LOGS = envIsTruthy(env.GRIDDO_BUILD_LOGS || env.GRIDDO_RENDER_BUILD_LOGS);
|
|
@@ -25327,7 +25328,7 @@ var SITE_URI = `${GRIDDO_API_URL2}/site/`;
|
|
|
25327
25328
|
var package_default = {
|
|
25328
25329
|
name: "@griddo/cx",
|
|
25329
25330
|
description: "Griddo SSG based on Gatsby",
|
|
25330
|
-
version: "11.
|
|
25331
|
+
version: "11.14.0",
|
|
25331
25332
|
authors: [
|
|
25332
25333
|
"Hisco <francis.vega@griddo.io>"
|
|
25333
25334
|
],
|
|
@@ -25366,6 +25367,7 @@ var package_default = {
|
|
|
25366
25367
|
"start-render": "node ./build/commands/start-render",
|
|
25367
25368
|
"end-render": "node ./build/commands/end-render",
|
|
25368
25369
|
"upload-search-content": "node ./build/commands/upload-search-content",
|
|
25370
|
+
"notify-indexnow": "node ./build/commands/notify-indexnow",
|
|
25369
25371
|
"reset-render": "node ./build/commands/reset-render",
|
|
25370
25372
|
"// ONLY LOCAL SCRIPTS": "",
|
|
25371
25373
|
"prepare-assets-directory": "node ./build/commands/prepare-assets-directory",
|
|
@@ -25387,7 +25389,7 @@ var package_default = {
|
|
|
25387
25389
|
},
|
|
25388
25390
|
devDependencies: {
|
|
25389
25391
|
"@biomejs/biome": "2.3.4",
|
|
25390
|
-
"@griddo/core": "11.
|
|
25392
|
+
"@griddo/core": "11.14.0",
|
|
25391
25393
|
"@types/node": "20.19.4",
|
|
25392
25394
|
"@typescript/native-preview": "7.0.0-dev.20260401.1",
|
|
25393
25395
|
cheerio: "1.1.2",
|
|
@@ -25421,7 +25423,7 @@ var package_default = {
|
|
|
25421
25423
|
publishConfig: {
|
|
25422
25424
|
access: "public"
|
|
25423
25425
|
},
|
|
25424
|
-
gitHead: "
|
|
25426
|
+
gitHead: "d2c96b30e53ce16a9d06df7fd512308ae6fff64c"
|
|
25425
25427
|
};
|
|
25426
25428
|
|
|
25427
25429
|
// exporter/shared/headers.ts
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Derive the IndexNow key for a given domain. It is a deterministic function
|
|
3
|
+
* of the domain slug so every machine and every rerun produces the same key.
|
|
4
|
+
* IndexNow only requires the key to be 8-128 alphanumeric chars and to match
|
|
5
|
+
* the file served at the host root; it does not need to be secret (ownership
|
|
6
|
+
* is proven by the ability to serve the file, not by the key being private).
|
|
7
|
+
*/
|
|
8
|
+
declare function getIndexNowKeyForDomain(domain: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* Filename served at the public root of every domain so IndexNow can verify
|
|
11
|
+
* ownership of the host. Returns `null` when the feature is disabled, which
|
|
12
|
+
* also means the file will not be written.
|
|
13
|
+
*/
|
|
14
|
+
declare function getIndexNowKeyFilename(domain: string): string | null;
|
|
15
|
+
/**
|
|
16
|
+
* Write the IndexNow host-key file to the domain `current-dist` so it ends up
|
|
17
|
+
* served at `https://<host>/<key>.txt`. On FROM_SCRATCH the file is carried
|
|
18
|
+
* over via the `current-dist -> dist` rename; on INCREMENTAL it is pulled in
|
|
19
|
+
* through the `artifactsToCopyToExports` list (see sync action).
|
|
20
|
+
*/
|
|
21
|
+
declare function generateIndexNowKeyFile(domain: string): Promise<void>;
|
|
22
|
+
export { generateIndexNowKeyFile, getIndexNowKeyFilename, getIndexNowKeyForDomain };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface NotifyDomainResult {
|
|
2
|
+
domain: string;
|
|
3
|
+
sent: number;
|
|
4
|
+
upsertCount: number;
|
|
5
|
+
deleteCount: number;
|
|
6
|
+
requests: number;
|
|
7
|
+
succeeded: boolean;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Read the payload emitted by `IndexNowCollector`, group URLs by host and POST
|
|
11
|
+
* each batch to the IndexNow endpoint. Sends upserts and deletes together in a
|
|
12
|
+
* single request per host (dedup by URL). Deletes the payload file only when
|
|
13
|
+
* every request succeeded, so a partial failure leaves the file in place for a
|
|
14
|
+
* later retry.
|
|
15
|
+
*/
|
|
16
|
+
declare function notifyDomain(options: {
|
|
17
|
+
domain: string;
|
|
18
|
+
key: string;
|
|
19
|
+
}): Promise<NotifyDomainResult | null>;
|
|
20
|
+
/**
|
|
21
|
+
* Orchestrate the IndexNow notification for a single domain. Applies the
|
|
22
|
+
* feature flag and dry-run skips, resolves the host key and delegates to
|
|
23
|
+
* `notifyDomain`. Logs a summary line when a request actually ran.
|
|
24
|
+
*/
|
|
25
|
+
declare function notifyIndexNowForDomain(options: {
|
|
26
|
+
slug: string;
|
|
27
|
+
dryRun: boolean;
|
|
28
|
+
}): Promise<void>;
|
|
29
|
+
export { notifyDomain, notifyIndexNowForDomain };
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { GriddoPageObject } from "../shared/types/pages";
|
|
2
|
+
import type { Site } from "../shared/types/sites";
|
|
3
|
+
export interface IndexNowUpsertEntry {
|
|
4
|
+
id: number;
|
|
5
|
+
url: string;
|
|
6
|
+
siteId: number;
|
|
7
|
+
}
|
|
8
|
+
export interface IndexNowDeleteEntry {
|
|
9
|
+
id: number;
|
|
10
|
+
url: string;
|
|
11
|
+
siteId: number;
|
|
12
|
+
reason: "offlinePending" | "siteUnpublished";
|
|
13
|
+
}
|
|
14
|
+
export interface IndexNowPayloadFile {
|
|
15
|
+
domain: string;
|
|
16
|
+
renderedAt: string;
|
|
17
|
+
upsert: IndexNowUpsertEntry[];
|
|
18
|
+
delete: IndexNowDeleteEntry[];
|
|
19
|
+
skipped: {
|
|
20
|
+
noindex: number;
|
|
21
|
+
noUrlResolved: number;
|
|
22
|
+
paginatedSecondary: number;
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
interface FlushOptions {
|
|
26
|
+
domain: string;
|
|
27
|
+
sitesToPublish: Site[];
|
|
28
|
+
sitesToUnpublish: Site[];
|
|
29
|
+
}
|
|
30
|
+
interface CollectOptions {
|
|
31
|
+
siteId: number;
|
|
32
|
+
pageId: number;
|
|
33
|
+
pageObjects: GriddoPageObject[];
|
|
34
|
+
uploadPending: number[];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Collects URLs that should be notified to IndexNow during `createStore`.
|
|
38
|
+
* Intended usage:
|
|
39
|
+
* - `collect()` during the per-page loop (reuses the already-fetched page
|
|
40
|
+
* objects, so upserts cost 0 extra API calls).
|
|
41
|
+
* - `flush()` once per render to resolve delete URLs via `getPage` and write
|
|
42
|
+
* the payload file to `.griddo/cache/<domain>/indexnow-payload.json`.
|
|
43
|
+
*/
|
|
44
|
+
declare class IndexNowCollector {
|
|
45
|
+
readonly enabled: boolean;
|
|
46
|
+
private upserts;
|
|
47
|
+
private skipped;
|
|
48
|
+
constructor(enabled?: boolean);
|
|
49
|
+
collect(options: CollectOptions): void;
|
|
50
|
+
flush(options: FlushOptions): Promise<void>;
|
|
51
|
+
private extractCanonical;
|
|
52
|
+
private resolveUrlForId;
|
|
53
|
+
}
|
|
54
|
+
export { IndexNowCollector };
|
|
@@ -59,6 +59,7 @@ declare function getRenderPathsHydratedWithDomainFromDB(options?: {
|
|
|
59
59
|
__ssg: string;
|
|
60
60
|
__exports_dist: string;
|
|
61
61
|
}>;
|
|
62
|
+
declare function getDomainDryRunFromDB(domain: string): Promise<boolean>;
|
|
62
63
|
declare function getRenderMetadataFromDB(): Promise<{
|
|
63
64
|
griddoVersion: string;
|
|
64
65
|
buildReportFileName: string;
|
|
@@ -67,4 +68,4 @@ declare function getRenderMetadataFromDB(): Promise<{
|
|
|
67
68
|
* Save a file with the end of build process to use as `end-render` signal.
|
|
68
69
|
*/
|
|
69
70
|
declare function generateBuildReport(domain: string): Promise<void>;
|
|
70
|
-
export { assertRenderIsValid, generateBuildReport, getRenderMetadataFromDB, getRenderModeFromDB, getRenderPathsHydratedWithDomainFromDB, markRenderAsCompleted, markRenderAsStarted, resolveDomainRenderMode, updateCommitFile, };
|
|
71
|
+
export { assertRenderIsValid, generateBuildReport, getDomainDryRunFromDB, getRenderMetadataFromDB, getRenderModeFromDB, getRenderPathsHydratedWithDomainFromDB, markRenderAsCompleted, markRenderAsStarted, resolveDomainRenderMode, updateCommitFile, };
|
package/build/shared/envs.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ declare const GRIDDO_PUBLIC_API_URL: string | undefined;
|
|
|
3
3
|
declare const GRIDDO_BOT_USER: string | undefined;
|
|
4
4
|
declare const GRIDDO_BOT_PASSWORD: string | undefined;
|
|
5
5
|
declare const GRIDDO_AI_EMBEDDINGS: boolean;
|
|
6
|
+
declare const GRIDDO_INDEXNOW_ENABLED: boolean;
|
|
6
7
|
declare const GRIDDO_API_CONCURRENCY_COUNT: number;
|
|
7
8
|
declare const GRIDDO_ASSET_PREFIX: string | undefined;
|
|
8
9
|
declare const GRIDDO_BUILD_LOGS: boolean;
|
|
@@ -22,4 +23,4 @@ declare const GRIDDO_RENDER_API_TIMEOUT_MS: number;
|
|
|
22
23
|
declare const GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD: number;
|
|
23
24
|
declare const GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS: number;
|
|
24
25
|
declare const GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS: number;
|
|
25
|
-
export { GRIDDO_AI_EMBEDDINGS, GRIDDO_API_CONCURRENCY_COUNT, GRIDDO_API_URL, GRIDDO_ASSET_PREFIX, GRIDDO_BOT_PASSWORD, GRIDDO_BOT_USER, GRIDDO_BUILD_LOGS, GRIDDO_BUILD_LOGS_BUFFER_SIZE, GRIDDO_PUBLIC_API_URL, GRIDDO_REACT_APP_INSTANCE, GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS, GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS, GRIDDO_RENDER_API_TIMEOUT_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD, GRIDDO_RENDER_DISABLE_LLMS_TXT, GRIDDO_RENDER_ENABLED_LLM_MD, GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS, GRIDDO_SEARCH_FEATURE, GRIDDO_SKIP_BUILD_CHECKS, GRIDDO_SSG_BUNDLE_ANALYZER, GRIDDO_SSG_VERBOSE_LOGS, GRIDDO_USE_DIST_BACKUP, GRIDDO_VERBOSE_LOGS, };
|
|
26
|
+
export { GRIDDO_AI_EMBEDDINGS, GRIDDO_API_CONCURRENCY_COUNT, GRIDDO_API_URL, GRIDDO_ASSET_PREFIX, GRIDDO_BOT_PASSWORD, GRIDDO_BOT_USER, GRIDDO_BUILD_LOGS, GRIDDO_BUILD_LOGS_BUFFER_SIZE, GRIDDO_INDEXNOW_ENABLED, GRIDDO_PUBLIC_API_URL, GRIDDO_REACT_APP_INSTANCE, GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS, GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS, GRIDDO_RENDER_API_TIMEOUT_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS, GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD, GRIDDO_RENDER_DISABLE_LLMS_TXT, GRIDDO_RENDER_ENABLED_LLM_MD, GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS, GRIDDO_SEARCH_FEATURE, GRIDDO_SKIP_BUILD_CHECKS, GRIDDO_SSG_BUNDLE_ANALYZER, GRIDDO_SSG_VERBOSE_LOGS, GRIDDO_USE_DIST_BACKUP, GRIDDO_VERBOSE_LOGS, };
|
package/cli.mjs
CHANGED
|
@@ -85,6 +85,15 @@ export const AVAILABLE_COMMANDS = [
|
|
|
85
85
|
rootArgument: true,
|
|
86
86
|
noStaticFiles: false,
|
|
87
87
|
},
|
|
88
|
+
{
|
|
89
|
+
name: "notify-indexnow",
|
|
90
|
+
description: "Notify IndexNow with the render delta of every domain",
|
|
91
|
+
domainArgument: false,
|
|
92
|
+
script: "notify-indexnow",
|
|
93
|
+
isCommandPack: false,
|
|
94
|
+
rootArgument: true,
|
|
95
|
+
noStaticFiles: false,
|
|
96
|
+
},
|
|
88
97
|
{
|
|
89
98
|
name: "reset-render",
|
|
90
99
|
description: "Reset render state",
|
package/exporter/build.sh
CHANGED
|
@@ -20,6 +20,7 @@ esbuild ${log} ./exporter/react/index.tsx ${react_opts} --outfile=./build/react/
|
|
|
20
20
|
# This .sh needs to be run from a npm script, so esbuild dependency is available
|
|
21
21
|
esbuild ${log} ./exporter/commands/end-render.ts ${bundle_node_opts}end-render.js
|
|
22
22
|
esbuild ${log} ./exporter/commands/upload-search-content.ts ${bundle_node_opts}upload-search-content.js
|
|
23
|
+
esbuild ${log} ./exporter/commands/notify-indexnow.ts ${bundle_node_opts}notify-indexnow.js
|
|
23
24
|
esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts}reset-render.js
|
|
24
25
|
esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts}start-render.js
|
|
25
26
|
esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts}prepare-domains-render.js
|
|
@@ -5,7 +5,9 @@ import { throwError, withErrorHandler } from "../core/errors";
|
|
|
5
5
|
import { pathExists } from "../core/fs";
|
|
6
6
|
import { GriddoLog } from "../core/GriddoLog";
|
|
7
7
|
import { AuthService } from "../services/auth";
|
|
8
|
+
import { notifyIndexNowForDomain } from "../services/indexnow-notify";
|
|
8
9
|
import {
|
|
10
|
+
getDomainDryRunFromDB,
|
|
9
11
|
getRenderMetadataFromDB,
|
|
10
12
|
getRenderModeFromDB,
|
|
11
13
|
getRenderPathsHydratedWithDomainFromDB,
|
|
@@ -62,6 +64,15 @@ async function endRender() {
|
|
|
62
64
|
GriddoLog.info(`Site (${siteId})`);
|
|
63
65
|
await endSiteRender(siteId, body);
|
|
64
66
|
}
|
|
67
|
+
|
|
68
|
+
try {
|
|
69
|
+
const dryRun = await getDomainDryRunFromDB(domain);
|
|
70
|
+
await notifyIndexNowForDomain({ slug: domain, dryRun });
|
|
71
|
+
} catch (err) {
|
|
72
|
+
GriddoLog.warn(
|
|
73
|
+
`indexnow: notify failed for ${domain}: ${(err as Error)?.message ?? String(err)}`,
|
|
74
|
+
);
|
|
75
|
+
}
|
|
65
76
|
}
|
|
66
77
|
|
|
67
78
|
async function main() {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { withErrorHandler } from "../core/errors";
|
|
2
|
+
import { AuthService } from "../services/auth";
|
|
3
|
+
import { getInstanceDomains } from "../services/domains";
|
|
4
|
+
import { notifyIndexNowForDomain } from "../services/indexnow-notify";
|
|
5
|
+
|
|
6
|
+
async function notifyIndexNow() {
|
|
7
|
+
const domains = await getInstanceDomains();
|
|
8
|
+
|
|
9
|
+
for (const { slug, dryRun } of domains) {
|
|
10
|
+
await notifyIndexNowForDomain({ slug, dryRun });
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async function main() {
|
|
15
|
+
await AuthService.login();
|
|
16
|
+
await notifyIndexNow();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
withErrorHandler(main);
|
|
@@ -28,20 +28,17 @@ export async function generateMdsFromHtmls(domain: string) {
|
|
|
28
28
|
|
|
29
29
|
try {
|
|
30
30
|
for await (const page of pages) {
|
|
31
|
-
// Es home de dominio
|
|
32
|
-
// Evitamos hacer markdown del home principal del dominio porque ahí irá el llms.txt
|
|
33
|
-
if (page.path === path.join(distDirectory, "index.html")) continue;
|
|
34
|
-
|
|
35
31
|
// No es index.html
|
|
36
32
|
// evita parsear algunos htmls que se cuelan de gatsby...
|
|
37
33
|
if (!page.path.endsWith("index.html")) continue;
|
|
38
34
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
35
|
+
// Home del dominio: escribimos dist/index.md porque dominio.com.md en browser queda raro
|
|
36
|
+
const isHome = page.path === path.join(distDirectory, "index.html");
|
|
37
|
+
const outPath = isHome
|
|
38
|
+
? path.join(distDirectory, "index.md")
|
|
39
|
+
: `${page.path.replace(/\/index\.html$/, "")}.md`;
|
|
40
|
+
|
|
41
|
+
await fsp.writeFile(outPath, page.content);
|
|
45
42
|
GriddoLog.info(`Generating MD for a ${page.path}`);
|
|
46
43
|
}
|
|
47
44
|
} catch (error) {
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import fsp from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
|
|
5
|
+
import { GriddoLog } from "../core/GriddoLog";
|
|
6
|
+
import { GRIDDO_INDEXNOW_ENABLED } from "../shared/envs";
|
|
7
|
+
import { getRenderPathsHydratedWithDomainFromDB } from "./render";
|
|
8
|
+
|
|
9
|
+
const KEY_LENGTH = 32;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Derive the IndexNow key for a given domain. It is a deterministic function
|
|
13
|
+
* of the domain slug so every machine and every rerun produces the same key.
|
|
14
|
+
* IndexNow only requires the key to be 8-128 alphanumeric chars and to match
|
|
15
|
+
* the file served at the host root; it does not need to be secret (ownership
|
|
16
|
+
* is proven by the ability to serve the file, not by the key being private).
|
|
17
|
+
*/
|
|
18
|
+
function getIndexNowKeyForDomain(domain: string): string {
|
|
19
|
+
return createHash("sha256").update(domain, "utf-8").digest("hex").slice(0, KEY_LENGTH);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Filename served at the public root of every domain so IndexNow can verify
|
|
24
|
+
* ownership of the host. Returns `null` when the feature is disabled, which
|
|
25
|
+
* also means the file will not be written.
|
|
26
|
+
*/
|
|
27
|
+
function getIndexNowKeyFilename(domain: string): string | null {
|
|
28
|
+
if (!GRIDDO_INDEXNOW_ENABLED) return null;
|
|
29
|
+
return `${getIndexNowKeyForDomain(domain)}.txt`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Write the IndexNow host-key file to the domain `current-dist` so it ends up
|
|
34
|
+
* served at `https://<host>/<key>.txt`. On FROM_SCRATCH the file is carried
|
|
35
|
+
* over via the `current-dist -> dist` rename; on INCREMENTAL it is pulled in
|
|
36
|
+
* through the `artifactsToCopyToExports` list (see sync action).
|
|
37
|
+
*/
|
|
38
|
+
async function generateIndexNowKeyFile(domain: string): Promise<void> {
|
|
39
|
+
const filename = getIndexNowKeyFilename(domain);
|
|
40
|
+
if (!filename) return;
|
|
41
|
+
|
|
42
|
+
const key = getIndexNowKeyForDomain(domain);
|
|
43
|
+
const { __root } = await getRenderPathsHydratedWithDomainFromDB({ domain });
|
|
44
|
+
const distDirectory = path.join(__root, "current-dist");
|
|
45
|
+
const filePath = path.join(distDirectory, filename);
|
|
46
|
+
|
|
47
|
+
try {
|
|
48
|
+
await fsp.mkdir(distDirectory, { recursive: true });
|
|
49
|
+
await fsp.writeFile(filePath, key, "utf-8");
|
|
50
|
+
GriddoLog.verbose(`indexnow: wrote host-key file ${filePath}`);
|
|
51
|
+
} catch (err) {
|
|
52
|
+
GriddoLog.warn(
|
|
53
|
+
`indexnow: failed to write host-key file ${filePath}: ${(err as Error)?.message ?? String(err)}`,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export { generateIndexNowKeyFile, getIndexNowKeyFilename, getIndexNowKeyForDomain };
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import type { IndexNowPayloadFile } from "./indexnow-payload";
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { GriddoLog } from "../core/GriddoLog";
|
|
7
|
+
import { GRIDDO_INDEXNOW_ENABLED } from "../shared/envs";
|
|
8
|
+
import { getIndexNowKeyForDomain } from "./indexnow-key-file";
|
|
9
|
+
import { getRenderPathsHydratedWithDomainFromDB } from "./render";
|
|
10
|
+
|
|
11
|
+
const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
|
|
12
|
+
const INDEXNOW_MAX_URLS_PER_REQUEST = 10_000;
|
|
13
|
+
|
|
14
|
+
export interface NotifyDomainResult {
|
|
15
|
+
domain: string;
|
|
16
|
+
sent: number;
|
|
17
|
+
upsertCount: number;
|
|
18
|
+
deleteCount: number;
|
|
19
|
+
requests: number;
|
|
20
|
+
succeeded: boolean;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Read the payload emitted by `IndexNowCollector`, group URLs by host and POST
|
|
25
|
+
* each batch to the IndexNow endpoint. Sends upserts and deletes together in a
|
|
26
|
+
* single request per host (dedup by URL). Deletes the payload file only when
|
|
27
|
+
* every request succeeded, so a partial failure leaves the file in place for a
|
|
28
|
+
* later retry.
|
|
29
|
+
*/
|
|
30
|
+
async function notifyDomain(options: {
|
|
31
|
+
domain: string;
|
|
32
|
+
key: string;
|
|
33
|
+
}): Promise<NotifyDomainResult | null> {
|
|
34
|
+
const { domain, key } = options;
|
|
35
|
+
const { __cache } = await getRenderPathsHydratedWithDomainFromDB({ domain });
|
|
36
|
+
const payloadPath = path.join(__cache, "indexnow-payload.json");
|
|
37
|
+
|
|
38
|
+
const payload = await readPayload(payloadPath);
|
|
39
|
+
if (!payload) {
|
|
40
|
+
GriddoLog.verbose(`indexnow: no payload for ${domain}, skipping`);
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const upsertCount = payload.upsert.length;
|
|
45
|
+
const deleteCount = payload.delete.length;
|
|
46
|
+
|
|
47
|
+
const urls = collectUrls(payload);
|
|
48
|
+
if (urls.length === 0) {
|
|
49
|
+
GriddoLog.verbose(`indexnow: empty payload for ${domain}, removing file`);
|
|
50
|
+
await safeUnlink(payloadPath);
|
|
51
|
+
return { domain, sent: 0, upsertCount, deleteCount, requests: 0, succeeded: true };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const byHost = groupByHost(urls);
|
|
55
|
+
let requests = 0;
|
|
56
|
+
let succeeded = true;
|
|
57
|
+
|
|
58
|
+
for (const [host, hostUrls] of byHost) {
|
|
59
|
+
for (const batch of batchUrls(hostUrls, INDEXNOW_MAX_URLS_PER_REQUEST)) {
|
|
60
|
+
requests++;
|
|
61
|
+
const ok = await postBatch({ host, key, urlList: batch });
|
|
62
|
+
if (!ok) succeeded = false;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (succeeded) {
|
|
67
|
+
await safeUnlink(payloadPath);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return { domain, sent: urls.length, upsertCount, deleteCount, requests, succeeded };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function collectUrls(payload: IndexNowPayloadFile): string[] {
|
|
74
|
+
const set = new Set<string>();
|
|
75
|
+
for (const entry of payload.upsert) set.add(entry.url);
|
|
76
|
+
for (const entry of payload.delete) set.add(entry.url);
|
|
77
|
+
return Array.from(set);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function groupByHost(urls: string[]): Map<string, string[]> {
|
|
81
|
+
const byHost = new Map<string, string[]>();
|
|
82
|
+
for (const url of urls) {
|
|
83
|
+
let host: string;
|
|
84
|
+
try {
|
|
85
|
+
host = new URL(url).host;
|
|
86
|
+
} catch {
|
|
87
|
+
GriddoLog.warn(`indexnow: invalid URL skipped: ${url}`);
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const list = byHost.get(host);
|
|
91
|
+
if (list) list.push(url);
|
|
92
|
+
else byHost.set(host, [url]);
|
|
93
|
+
}
|
|
94
|
+
return byHost;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function* batchUrls(urls: string[], size: number): Generator<string[]> {
|
|
98
|
+
for (let i = 0; i < urls.length; i += size) {
|
|
99
|
+
yield urls.slice(i, i + size);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function postBatch(options: {
|
|
104
|
+
host: string;
|
|
105
|
+
key: string;
|
|
106
|
+
urlList: string[];
|
|
107
|
+
}): Promise<boolean> {
|
|
108
|
+
const { host, key, urlList } = options;
|
|
109
|
+
const payload = {
|
|
110
|
+
host,
|
|
111
|
+
key,
|
|
112
|
+
keyLocation: `https://${host}/${key}.txt`,
|
|
113
|
+
urlList,
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
const response = await fetch(INDEXNOW_ENDPOINT, {
|
|
118
|
+
method: "POST",
|
|
119
|
+
headers: { "Content-Type": "application/json; charset=utf-8" },
|
|
120
|
+
body: JSON.stringify(payload),
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
if (!response.ok) {
|
|
124
|
+
const bodyText = await safeReadBody(response);
|
|
125
|
+
GriddoLog.warn(
|
|
126
|
+
`indexnow: ${host} HTTP ${response.status} (${urlList.length} urls)${bodyText ? `: ${bodyText}` : ""}`,
|
|
127
|
+
);
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
GriddoLog.verbose(`indexnow: ${host} pinged (${urlList.length} urls, HTTP ${response.status})`);
|
|
132
|
+
return true;
|
|
133
|
+
} catch (err) {
|
|
134
|
+
GriddoLog.warn(`indexnow: ${host} request failed: ${(err as Error)?.message ?? String(err)}`);
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function safeReadBody(response: Response): Promise<string> {
|
|
140
|
+
try {
|
|
141
|
+
const text = await response.text();
|
|
142
|
+
return text.slice(0, 200);
|
|
143
|
+
} catch {
|
|
144
|
+
return "";
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function readPayload(p: string): Promise<IndexNowPayloadFile | null> {
|
|
149
|
+
try {
|
|
150
|
+
const raw = await fsp.readFile(p, "utf-8");
|
|
151
|
+
return JSON.parse(raw) as IndexNowPayloadFile;
|
|
152
|
+
} catch {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async function safeUnlink(p: string): Promise<void> {
|
|
158
|
+
try {
|
|
159
|
+
await fsp.unlink(p);
|
|
160
|
+
} catch {
|
|
161
|
+
// Missing file is fine.
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Orchestrate the IndexNow notification for a single domain. Applies the
|
|
167
|
+
* feature flag and dry-run skips, resolves the host key and delegates to
|
|
168
|
+
* `notifyDomain`. Logs a summary line when a request actually ran.
|
|
169
|
+
*/
|
|
170
|
+
async function notifyIndexNowForDomain(options: {
|
|
171
|
+
slug: string;
|
|
172
|
+
dryRun: boolean;
|
|
173
|
+
}): Promise<void> {
|
|
174
|
+
const { slug, dryRun } = options;
|
|
175
|
+
|
|
176
|
+
if (!GRIDDO_INDEXNOW_ENABLED) {
|
|
177
|
+
GriddoLog.info("indexnow: feature disabled (GRIDDO_INDEXNOW_ENABLED), skipping");
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (dryRun) {
|
|
182
|
+
GriddoLog.verbose(`indexnow: domain ${slug} ignored by dry-render`);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const key = getIndexNowKeyForDomain(slug);
|
|
187
|
+
const result = await notifyDomain({ domain: slug, key });
|
|
188
|
+
if (!result) return;
|
|
189
|
+
|
|
190
|
+
const status = result.succeeded ? "ok" : "partial failure";
|
|
191
|
+
GriddoLog.info(
|
|
192
|
+
`indexnow: ${result.domain} -> ${result.sent} urls (${result.upsertCount} upsert, ${result.deleteCount} delete) in ${result.requests} request(s) [${status}]`,
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export { notifyDomain, notifyIndexNowForDomain };
|