@griddo/cx 11.13.0 → 11.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/build/commands/end-render.js +150 -20
- package/build/commands/end-render.js.map +4 -4
- package/build/commands/prepare-assets-directory.js +146 -8
- package/build/commands/prepare-assets-directory.js.map +4 -4
- package/build/commands/prepare-domains-render.js +158 -28
- package/build/commands/prepare-domains-render.js.map +4 -4
- package/build/commands/reset-render.js +150 -20
- package/build/commands/reset-render.js.map +4 -4
- package/build/commands/start-render.js +179 -49
- package/build/commands/start-render.js.map +4 -4
- package/build/commands/upload-search-content.js +151 -21
- package/build/commands/upload-search-content.js.map +4 -4
- package/build/core/check-env-health.d.ts +47 -1
- package/build/core/http/create-adapter.d.ts +13 -0
- package/build/core/http/index.d.ts +6 -0
- package/build/core/http/types.d.ts +20 -0
- package/build/core/http/undici-adapter.d.ts +7 -0
- package/build/core/http/with-circuit-breaker.d.ts +24 -0
- package/build/core/http/with-retry.d.ts +32 -0
- package/build/index.js +25026 -42
- package/build/react/GriddoIntegrations/utils.d.ts +1 -1
- package/build/services/manage-store.d.ts +8 -1
- package/build/services/pages.d.ts +73 -2
- package/build/services/reference-fields.d.ts +1 -1
- package/build/shared/envs.d.ts +5 -2
- package/build/shared/types/api.d.ts +0 -2
- package/cli.mjs +28 -10
- package/exporter/commands/README.md +1 -1
- package/exporter/commands/end-render.ts +1 -1
- package/exporter/commands/prepare-domains-render.ts +1 -1
- package/exporter/commands/{single-domain-upload-search-content.ts → single-domain-upload-search-content.noop} +2 -4
- package/exporter/commands/upload-search-content.ts +1 -4
- package/exporter/core/check-env-health.ts +1 -1
- package/exporter/core/errors.ts +13 -13
- package/exporter/core/fs.ts +35 -31
- package/exporter/core/http/create-adapter.ts +58 -0
- package/exporter/core/http/index.ts +7 -0
- package/exporter/core/http/types.ts +22 -0
- package/exporter/core/http/undici-adapter.ts +53 -0
- package/exporter/core/http/with-circuit-breaker.ts +86 -0
- package/exporter/core/http/with-retry.ts +87 -0
- package/exporter/services/api.ts +22 -66
- package/exporter/services/auth.ts +11 -2
- package/exporter/services/domains.ts +6 -1
- package/exporter/services/llms.ts +1 -1
- package/exporter/services/manage-store.ts +16 -18
- package/exporter/services/pages.ts +7 -0
- package/exporter/services/reference-fields.ts +3 -5
- package/exporter/services/render.ts +3 -7
- package/exporter/services/store.ts +10 -4
- package/exporter/shared/envs.ts +20 -6
- package/exporter/shared/types/api.ts +0 -2
- package/exporter/ssg-adapters/gatsby/index.ts +4 -2
- package/exporter/ssg-adapters/gatsby/shared/sync-render.ts +5 -1
- package/package.json +15 -16
- package/tsconfig.commands.json +9 -22
- package/tsconfig.exporter.json +3 -4
- package/tsconfig.json +2 -3
- package/build/commands/single-domain-upload-search-content.d.ts +0 -1
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { HttpAdapter, HttpRequest, HttpResponse } from "./types";
|
|
2
|
+
|
|
3
|
+
type CircuitState = "closed" | "open" | "half-open";
|
|
4
|
+
|
|
5
|
+
export interface CircuitBreakerOptions {
|
|
6
|
+
/** Number of consecutive failures before opening the circuit. */
|
|
7
|
+
failureThreshold: number;
|
|
8
|
+
/** Time in ms before switching from open to half-open. */
|
|
9
|
+
cooldownMs: number;
|
|
10
|
+
/** Called when the circuit opens. */
|
|
11
|
+
onOpen?: () => void;
|
|
12
|
+
/** Called when the circuit closes (recovery). */
|
|
13
|
+
onClose?: () => void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export class CircuitOpenError extends Error {
|
|
17
|
+
constructor(cooldownMs: number) {
|
|
18
|
+
super(`Circuit breaker is open. Cooldown: ${cooldownMs}ms`);
|
|
19
|
+
this.name = "CircuitOpenError";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Decorator that adds circuit breaker logic to an HttpAdapter.
|
|
25
|
+
*
|
|
26
|
+
* States:
|
|
27
|
+
* - CLOSED: requests pass through normally, failures are counted
|
|
28
|
+
* - OPEN: all requests fail immediately with CircuitOpenError
|
|
29
|
+
* - HALF-OPEN: one probe request is allowed; success → CLOSED, failure → OPEN
|
|
30
|
+
*/
|
|
31
|
+
function withCircuitBreaker(adapter: HttpAdapter, options: CircuitBreakerOptions): HttpAdapter {
|
|
32
|
+
const { failureThreshold, cooldownMs, onOpen, onClose } = options;
|
|
33
|
+
|
|
34
|
+
let state: CircuitState = "closed";
|
|
35
|
+
let failures = 0;
|
|
36
|
+
let openedAt = 0;
|
|
37
|
+
|
|
38
|
+
function recordSuccess() {
|
|
39
|
+
if (state === "half-open") {
|
|
40
|
+
state = "closed";
|
|
41
|
+
failures = 0;
|
|
42
|
+
onClose?.();
|
|
43
|
+
} else {
|
|
44
|
+
failures = 0;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function recordFailure() {
|
|
49
|
+
failures++;
|
|
50
|
+
if (failures >= failureThreshold) {
|
|
51
|
+
state = "open";
|
|
52
|
+
openedAt = Date.now();
|
|
53
|
+
onOpen?.();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function shouldAllowRequest(): boolean {
|
|
58
|
+
if (state === "closed") return true;
|
|
59
|
+
|
|
60
|
+
if (state === "open" && Date.now() - openedAt >= cooldownMs) {
|
|
61
|
+
state = "half-open";
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return state === "half-open";
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return {
|
|
69
|
+
async request(req: HttpRequest): Promise<HttpResponse> {
|
|
70
|
+
if (!shouldAllowRequest()) {
|
|
71
|
+
throw new CircuitOpenError(cooldownMs);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
try {
|
|
75
|
+
const response = await adapter.request(req);
|
|
76
|
+
recordSuccess();
|
|
77
|
+
return response;
|
|
78
|
+
} catch (e) {
|
|
79
|
+
recordFailure();
|
|
80
|
+
throw e;
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export { withCircuitBreaker };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { HttpAdapter, HttpRequest, HttpResponse } from "./types";
|
|
2
|
+
|
|
3
|
+
export interface RetryOptions {
|
|
4
|
+
/** Max number of attempts (including the first one). */
|
|
5
|
+
attempts: number;
|
|
6
|
+
/** Base delay between retries in milliseconds. */
|
|
7
|
+
delayMs: number;
|
|
8
|
+
/** Backoff strategy: "fixed" (same delay) or "exponential" (1x, 2x, 4x...). Default: "fixed". */
|
|
9
|
+
backoff?: "fixed" | "exponential";
|
|
10
|
+
/** Add random jitter (±50%) to the delay to avoid thundering herd. Default: false. */
|
|
11
|
+
jitter?: boolean;
|
|
12
|
+
/** Decide whether a successful response should be retried (e.g. 5xx). */
|
|
13
|
+
retryOn?: (response: HttpResponse) => boolean;
|
|
14
|
+
/** Called before each retry. */
|
|
15
|
+
onRetry?: (context: {
|
|
16
|
+
request: HttpRequest;
|
|
17
|
+
attempt: number;
|
|
18
|
+
delayMs: number;
|
|
19
|
+
error?: Error;
|
|
20
|
+
response?: HttpResponse;
|
|
21
|
+
}) => void;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function wait(ms: number): Promise<void> {
|
|
25
|
+
return new Promise((res) => setTimeout(res, ms));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function computeDelay(
|
|
29
|
+
baseMs: number,
|
|
30
|
+
attempt: number,
|
|
31
|
+
backoff: "fixed" | "exponential",
|
|
32
|
+
jitter: boolean,
|
|
33
|
+
): number {
|
|
34
|
+
let ms = backoff === "exponential" ? baseMs * 2 ** (attempt - 1) : baseMs;
|
|
35
|
+
|
|
36
|
+
if (jitter) {
|
|
37
|
+
ms = Math.round(ms * (0.5 + Math.random()));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return ms;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Decorator that adds retry logic to an HttpAdapter.
|
|
45
|
+
*
|
|
46
|
+
* Retries on:
|
|
47
|
+
* - Transport errors (network failures, DNS, connection refused, timeouts)
|
|
48
|
+
* - Responses matching `retryOn` predicate (e.g. status >= 500)
|
|
49
|
+
*
|
|
50
|
+
* Supports fixed and exponential backoff with optional jitter.
|
|
51
|
+
*/
|
|
52
|
+
function withRetry(adapter: HttpAdapter, options: RetryOptions): HttpAdapter {
|
|
53
|
+
const { attempts, delayMs, backoff = "fixed", jitter = false, retryOn, onRetry } = options;
|
|
54
|
+
|
|
55
|
+
return {
|
|
56
|
+
async request(req: HttpRequest): Promise<HttpResponse> {
|
|
57
|
+
let lastError: Error | undefined;
|
|
58
|
+
|
|
59
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
60
|
+
try {
|
|
61
|
+
const response = await adapter.request(req);
|
|
62
|
+
|
|
63
|
+
if (retryOn?.(response) && attempt < attempts) {
|
|
64
|
+
const ms = computeDelay(delayMs, attempt, backoff, jitter);
|
|
65
|
+
onRetry?.({ request: req, attempt, delayMs: ms, response });
|
|
66
|
+
await wait(ms);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return response;
|
|
71
|
+
} catch (e) {
|
|
72
|
+
lastError = e as Error;
|
|
73
|
+
|
|
74
|
+
if (attempt < attempts) {
|
|
75
|
+
const ms = computeDelay(delayMs, attempt, backoff, jitter);
|
|
76
|
+
onRetry?.({ request: req, attempt, delayMs: ms, error: lastError });
|
|
77
|
+
await wait(ms);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
throw lastError;
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export { withRetry };
|
package/exporter/services/api.ts
CHANGED
|
@@ -12,20 +12,18 @@ import crypto from "node:crypto";
|
|
|
12
12
|
import fsp from "node:fs/promises";
|
|
13
13
|
import path from "node:path";
|
|
14
14
|
|
|
15
|
-
import {
|
|
16
|
-
GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS,
|
|
17
|
-
GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS,
|
|
18
|
-
} from "@shared/envs";
|
|
19
|
-
|
|
20
15
|
import { RenderError } from "../core/errors";
|
|
21
16
|
import { pathExists } from "../core/fs";
|
|
22
17
|
import { GriddoLog } from "../core/GriddoLog";
|
|
18
|
+
import { getHttpAdapter } from "../core/http";
|
|
23
19
|
import { addLogToBuffer } from "../core/logger";
|
|
24
20
|
import { brush } from "../shared/brush";
|
|
25
21
|
import { DEFAULT_HEADERS } from "../shared/headers";
|
|
26
22
|
import { AuthService } from "./auth";
|
|
27
23
|
import { getRenderPathsHydratedWithDomainFromDB } from "./render";
|
|
28
24
|
|
|
25
|
+
const adapter = getHttpAdapter();
|
|
26
|
+
|
|
29
27
|
/**
|
|
30
28
|
* Make a GET/PUT/POST request to the Griddo API.
|
|
31
29
|
*
|
|
@@ -44,15 +42,7 @@ async function requestAPI<T extends APIResponses>(
|
|
|
44
42
|
method: string,
|
|
45
43
|
appendToLog = "",
|
|
46
44
|
): Promise<T> {
|
|
47
|
-
const {
|
|
48
|
-
endpoint,
|
|
49
|
-
body,
|
|
50
|
-
cacheKey = "",
|
|
51
|
-
attempt = 1,
|
|
52
|
-
headers,
|
|
53
|
-
useApiCacheDir = true,
|
|
54
|
-
logToFile = true,
|
|
55
|
-
} = props;
|
|
45
|
+
const { endpoint, body, cacheKey = "", headers, useApiCacheDir = true, logToFile = true } = props;
|
|
56
46
|
const cacheOptions = { endpoint, body, headers, cacheKey };
|
|
57
47
|
|
|
58
48
|
// Cache
|
|
@@ -75,23 +65,25 @@ async function requestAPI<T extends APIResponses>(
|
|
|
75
65
|
try {
|
|
76
66
|
const start = new Date();
|
|
77
67
|
|
|
78
|
-
// Prepare
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
fetchOptions.body = JSON.stringify(body);
|
|
90
|
-
if (!fetchOptions.headers) fetchOptions.headers = {};
|
|
91
|
-
(fetchOptions.headers as Record<string, string>)["Content-Type"] = "application/json";
|
|
68
|
+
// Prepare request
|
|
69
|
+
const requestHeaders = Object.assign(
|
|
70
|
+
{},
|
|
71
|
+
DEFAULT_HEADERS,
|
|
72
|
+
headers,
|
|
73
|
+
AuthService.headers,
|
|
74
|
+
) as Record<string, string>;
|
|
75
|
+
|
|
76
|
+
const hasBody = method.toLowerCase() !== "get" && body;
|
|
77
|
+
if (hasBody) {
|
|
78
|
+
requestHeaders["Content-Type"] = "application/json";
|
|
92
79
|
}
|
|
93
80
|
|
|
94
|
-
const response = await
|
|
81
|
+
const response = await adapter.request({
|
|
82
|
+
url: endpoint,
|
|
83
|
+
method: method.toUpperCase(),
|
|
84
|
+
headers: requestHeaders,
|
|
85
|
+
body: hasBody ? JSON.stringify(body) : undefined,
|
|
86
|
+
});
|
|
95
87
|
|
|
96
88
|
// Handle non-2xx responses
|
|
97
89
|
if (!response.ok) {
|
|
@@ -120,38 +112,11 @@ async function requestAPI<T extends APIResponses>(
|
|
|
120
112
|
} catch (e) {
|
|
121
113
|
const error = e as Error;
|
|
122
114
|
|
|
123
|
-
if (attempt > GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS) {
|
|
124
|
-
GriddoLog.log(`
|
|
125
|
-
Max attempts ${GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS} reached
|
|
126
|
-
--------------------------------------
|
|
127
|
-
- ${method.toUpperCase()} ${endpoint}
|
|
128
|
-
- BODY: ${JSON.stringify(body)}
|
|
129
|
-
- HEADERS: ${JSON.stringify(headers)}
|
|
130
|
-
- ERROR: ${error.message}
|
|
131
|
-
--------------------------------------
|
|
132
|
-
`);
|
|
133
|
-
throw new RenderError(error);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
115
|
showApiError(error, {
|
|
137
116
|
callInfo: { endpoint, body },
|
|
138
117
|
});
|
|
139
118
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
await delay(GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS * 1000);
|
|
143
|
-
|
|
144
|
-
return requestAPI<T>(
|
|
145
|
-
{
|
|
146
|
-
endpoint,
|
|
147
|
-
body,
|
|
148
|
-
headers,
|
|
149
|
-
cacheKey,
|
|
150
|
-
attempt: attempt + 1,
|
|
151
|
-
},
|
|
152
|
-
method,
|
|
153
|
-
appendToLog,
|
|
154
|
-
);
|
|
119
|
+
throw new RenderError(error);
|
|
155
120
|
}
|
|
156
121
|
}
|
|
157
122
|
|
|
@@ -238,15 +203,6 @@ function getSafeSiteId(response: APIResponses) {
|
|
|
238
203
|
return "site" in response && response.site ? response.site : undefined;
|
|
239
204
|
}
|
|
240
205
|
|
|
241
|
-
/**
|
|
242
|
-
* Custom delay using the "promise hack",
|
|
243
|
-
*
|
|
244
|
-
* @param ms Amount of miliseconds to be delayed
|
|
245
|
-
*/
|
|
246
|
-
function delay(ms: number): Promise<void> {
|
|
247
|
-
return new Promise((res) => setTimeout(res, ms));
|
|
248
|
-
}
|
|
249
|
-
|
|
250
206
|
/**
|
|
251
207
|
* Converts milliseconds to seconds with a fixed number of decimals.
|
|
252
208
|
*
|
|
@@ -1,17 +1,21 @@
|
|
|
1
1
|
import type { AuthHeaders } from "../shared/types/api";
|
|
2
2
|
|
|
3
3
|
import { throwError } from "../core/errors";
|
|
4
|
+
import { getHttpAdapter } from "../core/http";
|
|
4
5
|
import { LOGIN } from "../shared/endpoints";
|
|
5
6
|
import { GRIDDO_BOT_PASSWORD, GRIDDO_BOT_USER } from "../shared/envs";
|
|
6
7
|
import { LoginError } from "../shared/errors";
|
|
7
8
|
import { DEFAULT_HEADERS } from "../shared/headers";
|
|
8
9
|
|
|
10
|
+
const adapter = getHttpAdapter();
|
|
11
|
+
|
|
9
12
|
class AuthService {
|
|
10
13
|
headers: AuthHeaders | undefined;
|
|
11
14
|
|
|
12
15
|
async login() {
|
|
13
16
|
try {
|
|
14
|
-
const response = await
|
|
17
|
+
const response = await adapter.request({
|
|
18
|
+
url: LOGIN,
|
|
15
19
|
method: "POST",
|
|
16
20
|
headers: Object.assign({}, DEFAULT_HEADERS, {
|
|
17
21
|
"Content-Type": "application/json",
|
|
@@ -27,7 +31,12 @@ class AuthService {
|
|
|
27
31
|
throw new Error("Error while login in the API");
|
|
28
32
|
}
|
|
29
33
|
|
|
30
|
-
const { token } = await response.json();
|
|
34
|
+
const { token } = await response.json<{ token: string }>();
|
|
35
|
+
|
|
36
|
+
if (!token || typeof token !== "string") {
|
|
37
|
+
throw new Error("Login response missing valid token");
|
|
38
|
+
}
|
|
39
|
+
|
|
31
40
|
this.headers = {
|
|
32
41
|
Authorization: `bearer ${token}`,
|
|
33
42
|
"Cache-Control": "no-store",
|
|
@@ -27,7 +27,12 @@ async function getInstanceDomains() {
|
|
|
27
27
|
return { ...domain, slug: domain.slug.replace("/", "") };
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
const seen = new Set<string>();
|
|
31
|
+
return filteredDomains.filter((d) => {
|
|
32
|
+
if (seen.has(d.slug)) return false;
|
|
33
|
+
seen.add(d.slug);
|
|
34
|
+
return true;
|
|
35
|
+
});
|
|
31
36
|
}
|
|
32
37
|
|
|
33
38
|
export { getInstanceDomains };
|
|
@@ -36,7 +36,7 @@ async function getClientLLMsTxtTemplate(filePath: string) {
|
|
|
36
36
|
|
|
37
37
|
async function generateLlmsTxt(domain: string): Promise<void> {
|
|
38
38
|
if (GRIDDO_RENDER_DISABLE_LLMS_TXT) {
|
|
39
|
-
GriddoLog.verbose(
|
|
39
|
+
GriddoLog.verbose(`${domain} skipped llms.txt generation: disabled by environment variable.`);
|
|
40
40
|
return;
|
|
41
41
|
}
|
|
42
42
|
|
|
@@ -50,26 +50,17 @@ async function saveRenderInfoInStore(renderInfo: RenderInfo, domain: string) {
|
|
|
50
50
|
async function getPageInStoreDir(basePath: string) {
|
|
51
51
|
const filesInStore = await fsp.readdir(basePath);
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
.
|
|
53
|
+
const checks = await Promise.all(
|
|
54
|
+
filesInStore.map(async (file) => {
|
|
55
55
|
const fullPathFile = `${basePath}/${file}`;
|
|
56
56
|
const stat = await fsp.stat(fullPathFile);
|
|
57
|
-
|
|
58
|
-
if (
|
|
59
|
-
return false;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// Si es un archivo pero no tiene la extensión `.json`, no lo incluimos
|
|
63
|
-
if (path.extname(file) !== ".json") {
|
|
64
|
-
return false;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
// no es dir, es json.
|
|
57
|
+
if (stat?.isDirectory()) return false;
|
|
58
|
+
if (path.extname(file) !== ".json") return false;
|
|
68
59
|
return true;
|
|
69
|
-
})
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
60
|
+
}),
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
return filesInStore.filter((_, i) => checks[i]).map((page) => path.join(basePath, page));
|
|
73
64
|
}
|
|
74
65
|
|
|
75
66
|
/**
|
|
@@ -140,15 +131,21 @@ async function writeUniqueFileSync(filePath: string, content: string) {
|
|
|
140
131
|
* @param props An array of props to be removed
|
|
141
132
|
*/
|
|
142
133
|
function removeProperties(obj: Record<string, unknown>, propsToRemove: Set<string>) {
|
|
134
|
+
const seen = new WeakSet<object>();
|
|
135
|
+
|
|
143
136
|
function remove(currentObj: Record<string, unknown>) {
|
|
144
137
|
if (!currentObj || typeof currentObj !== "object" || Array.isArray(currentObj)) {
|
|
145
138
|
return;
|
|
146
139
|
}
|
|
147
140
|
|
|
141
|
+
if (seen.has(currentObj)) {
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
seen.add(currentObj);
|
|
145
|
+
|
|
148
146
|
for (const key in currentObj) {
|
|
149
147
|
if (Object.hasOwn(currentObj, key)) {
|
|
150
148
|
if (propsToRemove.has(key)) {
|
|
151
|
-
// Búsqueda O(1) en lugar de O(n)
|
|
152
149
|
delete currentObj[key];
|
|
153
150
|
} else {
|
|
154
151
|
const value = currentObj[key];
|
|
@@ -167,6 +164,7 @@ export {
|
|
|
167
164
|
getBuildMetadata,
|
|
168
165
|
getPageInStoreDir,
|
|
169
166
|
removeOrphanSites,
|
|
167
|
+
removeProperties,
|
|
170
168
|
saveRenderInfoInStore,
|
|
171
169
|
saveSitePagesInStore,
|
|
172
170
|
writeUniqueFileSync,
|
|
@@ -482,9 +482,16 @@ function addPageNumberToTitle(title: string, pageNumber: number) {
|
|
|
482
482
|
}
|
|
483
483
|
|
|
484
484
|
export {
|
|
485
|
+
addPageNumberToTitle,
|
|
486
|
+
addPageNumberToUrl,
|
|
485
487
|
createGriddoListPages,
|
|
486
488
|
createGriddoMultiPages,
|
|
487
489
|
createGriddoSinglePage,
|
|
488
490
|
getMultiPageElements,
|
|
491
|
+
getOpenGraph,
|
|
492
|
+
getPage,
|
|
493
|
+
getPageCluster,
|
|
494
|
+
getPageMetadata,
|
|
489
495
|
getPaginatedPages,
|
|
496
|
+
removeDuplicateTrailing,
|
|
490
497
|
};
|
|
@@ -88,9 +88,7 @@ async function fetchContentTypeData(props: FetchDataProps) {
|
|
|
88
88
|
|
|
89
89
|
// Avoid fetch ReferenceField with empty `data.sources`
|
|
90
90
|
if (Array.isArray(data.sources) && data.sources.length < 1) {
|
|
91
|
-
GriddoLog.
|
|
92
|
-
`Warning: Page with id: ${page.id} has a ReferenceField with empty \`data.sources\``,
|
|
93
|
-
);
|
|
91
|
+
GriddoLog.warn(`Page with id: ${page.id} has a ReferenceField with empty \`data.sources\``);
|
|
94
92
|
|
|
95
93
|
return [];
|
|
96
94
|
}
|
|
@@ -99,8 +97,8 @@ async function fetchContentTypeData(props: FetchDataProps) {
|
|
|
99
97
|
|
|
100
98
|
// Inform that the ReferenceField has not `data.sources`
|
|
101
99
|
if (!data.sources && data.mode === "auto") {
|
|
102
|
-
GriddoLog.
|
|
103
|
-
`
|
|
100
|
+
GriddoLog.warn(
|
|
101
|
+
`Page with id: ${page.id} has a ReferenceField with \`undefined\` \`data.sources\``,
|
|
104
102
|
);
|
|
105
103
|
}
|
|
106
104
|
|
|
@@ -124,16 +124,12 @@ async function hasNewCommit(basePath: string): Promise<boolean> {
|
|
|
124
124
|
const commitFile = path.join(basePath, "commit");
|
|
125
125
|
const currentCommit = execSync("git rev-parse HEAD").toString().trim();
|
|
126
126
|
|
|
127
|
-
|
|
127
|
+
try {
|
|
128
128
|
const savedCommit = (await fsp.readFile(commitFile, "utf-8")).trim();
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
129
|
+
return savedCommit !== currentCommit;
|
|
130
|
+
} catch {
|
|
133
131
|
return true;
|
|
134
132
|
}
|
|
135
|
-
|
|
136
|
-
return true;
|
|
137
133
|
}
|
|
138
134
|
|
|
139
135
|
async function updateCommitFile(options: { basePath: string }) {
|
|
@@ -334,11 +334,17 @@ async function createStore(options: {
|
|
|
334
334
|
const totalPages = pagesToFetchFromAPI.length;
|
|
335
335
|
const progressCounter = { current: 0 };
|
|
336
336
|
const progress = new siteFetchProgressBar(site.name, totalPages);
|
|
337
|
-
const pagesToStore = pagesToFetchFromAPI.map((id: number) =>
|
|
338
|
-
limit(() => fetchSitePageAndSaveInStore(siteDirName, id, progressCounter, progress)),
|
|
339
|
-
);
|
|
340
337
|
|
|
341
|
-
|
|
338
|
+
// Drain in batches to avoid holding all promise wrappers in memory
|
|
339
|
+
const BATCH_SIZE = 100;
|
|
340
|
+
for (let i = 0; i < totalPages; i += BATCH_SIZE) {
|
|
341
|
+
const batch = pagesToFetchFromAPI.slice(i, i + BATCH_SIZE);
|
|
342
|
+
await Promise.all(
|
|
343
|
+
batch.map((id: number) =>
|
|
344
|
+
limit(() => fetchSitePageAndSaveInStore(siteDirName, id, progressCounter, progress)),
|
|
345
|
+
),
|
|
346
|
+
);
|
|
347
|
+
}
|
|
342
348
|
}
|
|
343
349
|
}
|
|
344
350
|
|
package/exporter/shared/envs.ts
CHANGED
|
@@ -25,12 +25,20 @@ const GRIDDO_PUBLIC_API_URL = env.GRIDDO_PUBLIC_API_URL || env.PUBLIC_API_URL;
|
|
|
25
25
|
const GRIDDO_BOT_USER = env.botEmail || env.GRIDDO_BOT_USER;
|
|
26
26
|
const GRIDDO_BOT_PASSWORD = env.botPassword || env.GRIDDO_BOT_PASSWORD;
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Parses an integer from an env var string with a safe fallback.
|
|
30
|
+
*/
|
|
31
|
+
function safeParseInt(value: string, fallback: number): number {
|
|
32
|
+
const parsed = Number.parseInt(value);
|
|
33
|
+
return Number.isNaN(parsed) ? fallback : parsed;
|
|
34
|
+
}
|
|
35
|
+
|
|
28
36
|
// Rendering
|
|
29
|
-
const GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS
|
|
30
|
-
const GRIDDO_API_CONCURRENCY_COUNT =
|
|
37
|
+
const GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS);
|
|
38
|
+
const GRIDDO_API_CONCURRENCY_COUNT = safeParseInt(env.GRIDDO_API_CONCURRENCY_COUNT || env.GRIDDO_RENDER_CONCURRENCY_COUNT || "10", 10);
|
|
31
39
|
const GRIDDO_ASSET_PREFIX = env.GRIDDO_ASSET_PREFIX || env.ASSET_PREFIX || env.GRIDDO_RENDER_ASSET_PREFIX;
|
|
32
40
|
const GRIDDO_BUILD_LOGS = envIsTruthy(env.GRIDDO_BUILD_LOGS || env.GRIDDO_RENDER_BUILD_LOGS);
|
|
33
|
-
const GRIDDO_BUILD_LOGS_BUFFER_SIZE =
|
|
41
|
+
const GRIDDO_BUILD_LOGS_BUFFER_SIZE = safeParseInt(env.GRIDDO_RENDER_BUILD_LOGS_BUFFER_SIZE || "500", 500);
|
|
34
42
|
const GRIDDO_REACT_APP_INSTANCE = env.GRIDDO_REACT_APP_INSTANCE || env.REACT_APP_INSTANCE || env.GRIDDO_APP_INSTANCE || env.GRIDDO_EDITOR_APP_INSTANCIE;
|
|
35
43
|
const GRIDDO_RENDER_DISABLE_LLMS_TXT = envIsTruthy(env.GRIDDO_RENDER_DISABLE_LLMS_TXT);
|
|
36
44
|
const GRIDDO_SEARCH_FEATURE = envIsTruthy(env.GRIDDO_SEARCH_FEATURE || env.GRIDDO_RENDER_SEARCH_FEATURE);
|
|
@@ -39,10 +47,13 @@ const GRIDDO_SSG_BUNDLE_ANALYZER = envIsTruthy(env.GRIDDO_RENDER_SSG_BUNDLE_ANAL
|
|
|
39
47
|
const GRIDDO_SSG_VERBOSE_LOGS = envIsTruthy(env.GRIDDO_RENDER_SSG_VERBOSE_LOGS);
|
|
40
48
|
const GRIDDO_USE_DIST_BACKUP = envIsTruthy(env.GRIDDO_RENDER_USE_DIST_BACKUP);
|
|
41
49
|
const GRIDDO_VERBOSE_LOGS = envIsTruthy(env.GRIDDO_VERBOSE_LOGS || env.GRIDDO_RENDER_VERBOSE_LOGS);
|
|
42
|
-
const GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS = Number.parseInt(env.GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS || "4");
|
|
43
|
-
const GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS = Number.parseInt(env.GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS || "4");
|
|
44
|
-
const GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS = Number.parseInt(env.GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS || "1");
|
|
45
50
|
const GRIDDO_RENDER_ENABLED_LLM_MD = envIsTruthy(env.GRIDDO_RENDER_ENABLED_LLM_MD);
|
|
51
|
+
const GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS = safeParseInt(env.GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS || "4", 4);
|
|
52
|
+
const GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS = safeParseInt(env.GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS || "4", 4);
|
|
53
|
+
const GRIDDO_RENDER_API_TIMEOUT_MS = safeParseInt(env.GRIDDO_RENDER_API_TIMEOUT_MS || "300000", 300000);
|
|
54
|
+
const GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD = safeParseInt(env.GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD || "10", 10);
|
|
55
|
+
const GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS = safeParseInt(env.GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS || "30000", 30000);
|
|
56
|
+
const GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS = safeParseInt(env.GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS || "1", 1);
|
|
46
57
|
|
|
47
58
|
export {
|
|
48
59
|
GRIDDO_AI_EMBEDDINGS,
|
|
@@ -57,6 +68,9 @@ export {
|
|
|
57
68
|
GRIDDO_REACT_APP_INSTANCE,
|
|
58
69
|
GRIDDO_RENDER_API_FETCH_RETRY_ATTEMPTS,
|
|
59
70
|
GRIDDO_RENDER_API_FETCH_RETRY_WAIT_SECONDS,
|
|
71
|
+
GRIDDO_RENDER_API_TIMEOUT_MS,
|
|
72
|
+
GRIDDO_RENDER_CIRCUIT_BREAKER_COOLDOWN_MS,
|
|
73
|
+
GRIDDO_RENDER_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
|
|
60
74
|
GRIDDO_RENDER_DISABLE_LLMS_TXT,
|
|
61
75
|
GRIDDO_RENDER_ENABLED_LLM_MD,
|
|
62
76
|
GRIDDO_RENDER_LIFECYCLE_RETRY_ATTEMPTS,
|
|
@@ -115,8 +115,6 @@ export interface APIRequest {
|
|
|
115
115
|
body?: any;
|
|
116
116
|
/** Reference id to manage cache between renders. */
|
|
117
117
|
cacheKey?: string;
|
|
118
|
-
/** Number of connection attempts (in case it fails on the first attempt). */
|
|
119
|
-
attempt?: number;
|
|
120
118
|
/**
|
|
121
119
|
* Headers for the post api fetch
|
|
122
120
|
*/
|
|
@@ -57,7 +57,7 @@ export async function gatsbyRenderDomain(domain: string) {
|
|
|
57
57
|
|
|
58
58
|
if (renderMode === RENDER_MODE.IDLE && derivedRenderMode === RENDER_MODE.IDLE) {
|
|
59
59
|
GriddoLog.info(
|
|
60
|
-
`(
|
|
60
|
+
`(Pre-check) ${domain} skipped start-render it is marked as <${RENDER_MODE.IDLE}> with the reason <${reason}>`,
|
|
61
61
|
);
|
|
62
62
|
return;
|
|
63
63
|
}
|
|
@@ -68,7 +68,9 @@ export async function gatsbyRenderDomain(domain: string) {
|
|
|
68
68
|
|
|
69
69
|
// Render mode reason to log information to the terminal
|
|
70
70
|
const renderModeReason = derivedRenderModeReason ? ` <${derivedRenderModeReason}>` : "";
|
|
71
|
-
GriddoLog.info(
|
|
71
|
+
GriddoLog.info(
|
|
72
|
+
`Init render ${domain} <${derivedRenderMode}> with the reason${renderModeReason}\n`,
|
|
73
|
+
);
|
|
72
74
|
|
|
73
75
|
// Render context
|
|
74
76
|
const context = new RenderContext<SSG>({
|
|
@@ -144,7 +144,11 @@ class SyncRender {
|
|
|
144
144
|
// ../page-data/about-us/page-data.json // página con slug
|
|
145
145
|
// ../page-data/programs/page-data.json // página con slug
|
|
146
146
|
// ../page-data/index/page-data.json // <---- ¡página root index!
|
|
147
|
-
|
|
147
|
+
//
|
|
148
|
+
// Ojo: `composePath` viene normalizado por `removeTrailingSlash`
|
|
149
|
+
// (ver `scanPages`), así que la home llega como "" y no como "/".
|
|
150
|
+
const normalizedCompose =
|
|
151
|
+
page.composePath === "" || page.composePath === "/" ? "index" : page.composePath;
|
|
148
152
|
const jsonTo = path.join(this.bundleDir, "page-data", normalizedCompose, "page-data.json");
|
|
149
153
|
|
|
150
154
|
this.state.htmlToAdd.push({ from: page.htmlPath, to: htmlTo });
|