@griddo/cx 11.7.6-rc.1 → 11.7.6-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/build/adapters/gatsby/actions/clean.js.map +1 -1
- package/build/adapters/gatsby/actions/close.js.map +1 -1
- package/build/adapters/gatsby/actions/data.js +3 -3
- package/build/adapters/gatsby/actions/data.js.map +1 -1
- package/build/adapters/gatsby/actions/healthCheck.js.map +1 -1
- package/build/adapters/gatsby/actions/init.js +2 -2
- package/build/adapters/gatsby/actions/init.js.map +1 -1
- package/build/adapters/gatsby/actions/meta.js +2 -2
- package/build/adapters/gatsby/actions/meta.js.map +1 -1
- package/build/adapters/gatsby/actions/prepare.js.map +1 -1
- package/build/adapters/gatsby/actions/relocation.js +2 -2
- package/build/adapters/gatsby/actions/relocation.js.map +1 -1
- package/build/adapters/gatsby/actions/restore.js +4 -4
- package/build/adapters/gatsby/actions/restore.js.map +1 -1
- package/build/adapters/gatsby/actions/ssg.js.map +1 -1
- package/build/adapters/gatsby/actions/sync.js +10 -9
- package/build/adapters/gatsby/actions/sync.js.map +1 -1
- package/build/adapters/gatsby/index.js +10 -6
- package/build/adapters/gatsby/index.js.map +1 -1
- package/build/adapters/gatsby/shared/context.js +8 -2
- package/build/adapters/gatsby/shared/context.js.map +1 -1
- package/build/adapters/gatsby/shared/diff-assets.js +101 -0
- package/build/adapters/gatsby/shared/diff-assets.js.map +1 -0
- package/build/adapters/gatsby/shared/extract-assets.js +11 -11
- package/build/adapters/gatsby/shared/extract-assets.js.map +1 -1
- package/build/adapters/gatsby/shared/gatsby-build.js.map +1 -1
- package/build/adapters/gatsby/shared/sync-render.js +162 -158
- package/build/adapters/gatsby/shared/sync-render.js.map +1 -1
- package/build/commands/end-render.js +44 -19
- package/build/commands/end-render.js.map +1 -1
- package/build/commands/prepare-assets-directory.js +4 -4
- package/build/commands/prepare-assets-directory.js.map +1 -1
- package/build/commands/prepare-domains-render.js +14 -19
- package/build/commands/prepare-domains-render.js.map +1 -1
- package/build/commands/reset-render.js +4 -4
- package/build/commands/reset-render.js.map +1 -1
- package/build/commands/start-render.js +20 -10
- package/build/commands/start-render.js.map +1 -1
- package/build/commands/upload-search-content.js +8 -17
- package/build/commands/upload-search-content.js.map +1 -1
- package/build/constants/endpoints.js.map +1 -1
- package/build/constants/envs.js +2 -7
- package/build/constants/envs.js.map +1 -1
- package/build/constants/errors.js +7 -9
- package/build/constants/errors.js.map +1 -1
- package/build/services/auth.js +1 -0
- package/build/services/auth.js.map +1 -1
- package/build/services/db-class.js +49 -0
- package/build/services/db-class.js.map +1 -0
- package/build/services/db.js.map +1 -1
- package/build/services/navigation.js +3 -0
- package/build/services/navigation.js.map +1 -1
- package/build/services/reference-fields.js.map +1 -1
- package/build/services/robots.js +4 -2
- package/build/services/robots.js.map +1 -1
- package/build/services/sites.js.map +1 -1
- package/build/services/store.js +2 -2
- package/build/services/store.js.map +1 -1
- package/build/types/render.js.map +1 -1
- package/build/utils/api.js.map +1 -1
- package/build/utils/artifacts.js +11 -11
- package/build/utils/artifacts.js.map +1 -1
- package/build/utils/brush.js +2 -2
- package/build/utils/brush.js.map +1 -1
- package/build/utils/cache.js +6 -6
- package/build/utils/cache.js.map +1 -1
- package/build/utils/check-health.js.map +1 -1
- package/build/utils/core-utils.js +35 -3
- package/build/utils/core-utils.js.map +1 -1
- package/build/utils/domains.js.map +1 -1
- package/build/utils/errors.js.map +1 -1
- package/build/utils/folders.js +30 -7
- package/build/utils/folders.js.map +1 -1
- package/build/utils/images.js.map +1 -1
- package/build/utils/instance.js.map +1 -1
- package/build/utils/loggin.js +9 -3
- package/build/utils/loggin.js.map +1 -1
- package/build/utils/pages.js.map +1 -1
- package/build/utils/render.js +1 -1
- package/build/utils/render.js.map +1 -1
- package/build/utils/sites.js +20 -19
- package/build/utils/sites.js.map +1 -1
- package/build/utils/store.js +8 -8
- package/build/utils/store.js.map +1 -1
- package/exporter/adapters/gatsby/actions/data.ts +3 -3
- package/exporter/adapters/gatsby/actions/init.ts +2 -2
- package/exporter/adapters/gatsby/actions/meta.ts +2 -2
- package/exporter/adapters/gatsby/actions/relocation.ts +2 -2
- package/exporter/adapters/gatsby/actions/restore.ts +4 -4
- package/exporter/adapters/gatsby/actions/sync.ts +10 -9
- package/exporter/adapters/gatsby/index.ts +13 -8
- package/exporter/adapters/gatsby/shared/diff-assets.ts +113 -0
- package/exporter/adapters/gatsby/shared/extract-assets.ts +11 -11
- package/exporter/adapters/gatsby/shared/sync-render.ts +195 -212
- package/exporter/build.sh +2 -3
- package/exporter/commands/end-render.ts +54 -25
- package/exporter/commands/prepare-assets-directory.ts +4 -4
- package/exporter/commands/prepare-domains-render.ts +15 -20
- package/exporter/commands/reset-render.ts +4 -4
- package/exporter/commands/start-render.ts +19 -11
- package/exporter/commands/upload-search-content.ts +9 -20
- package/exporter/constants/envs.ts +0 -6
- package/exporter/constants/errors.ts +12 -4
- package/exporter/services/db-class.ts +54 -0
- package/exporter/services/robots.ts +2 -2
- package/exporter/services/store.ts +2 -2
- package/exporter/types/global.ts +0 -1
- package/exporter/utils/artifacts.ts +11 -11
- package/exporter/utils/cache.ts +6 -6
- package/exporter/utils/core-utils.ts +46 -4
- package/exporter/utils/folders.ts +30 -7
- package/exporter/utils/loggin.ts +9 -3
- package/exporter/utils/render.ts +3 -1
- package/exporter/utils/sites.ts +18 -17
- package/exporter/utils/store.ts +8 -8
- package/package.json +10 -11
- package/src/gatsby-node-utils.ts +2 -2
- package/tsconfig.json +3 -2
- package/build/exporter/adapters/gatsby/actions/clean.d.ts +0 -3
- package/build/exporter/adapters/gatsby/actions/close.d.ts +0 -3
- package/build/exporter/adapters/gatsby/actions/data.d.ts +0 -2
- package/build/exporter/adapters/gatsby/actions/healthCheck.d.ts +0 -2
- package/build/exporter/adapters/gatsby/actions/init.d.ts +0 -2
- package/build/exporter/adapters/gatsby/actions/meta.d.ts +0 -2
- package/build/exporter/adapters/gatsby/actions/prepare.d.ts +0 -2
- package/build/exporter/adapters/gatsby/actions/relocation.d.ts +0 -2
- package/build/exporter/adapters/gatsby/actions/restore.d.ts +0 -3
- package/build/exporter/adapters/gatsby/actions/ssg.d.ts +0 -3
- package/build/exporter/adapters/gatsby/actions/sync.d.ts +0 -3
- package/build/exporter/adapters/gatsby/index.d.ts +0 -9
- package/build/exporter/adapters/gatsby/shared/context.d.ts +0 -38
- package/build/exporter/adapters/gatsby/shared/extract-assets.d.ts +0 -7
- package/build/exporter/adapters/gatsby/shared/gatsby-build.d.ts +0 -7
- package/build/exporter/adapters/gatsby/shared/sync-render.d.ts +0 -36
- package/build/exporter/adapters/gatsby/shared/types.d.ts +0 -34
- package/build/exporter/commands/end-render.d.ts +0 -2
- package/build/exporter/commands/prepare-assets-directory.d.ts +0 -2
- package/build/exporter/commands/prepare-domains-render.d.ts +0 -2
- package/build/exporter/commands/reset-render.d.ts +0 -2
- package/build/exporter/commands/start-render.d.ts +0 -2
- package/build/exporter/commands/upload-search-content.d.ts +0 -2
- package/build/exporter/constants/endpoints.d.ts +0 -19
- package/build/exporter/constants/envs.d.ts +0 -31
- package/build/exporter/constants/errors.d.ts +0 -24
- package/build/exporter/services/auth.d.ts +0 -10
- package/build/exporter/services/db.d.ts +0 -4
- package/build/exporter/services/navigation.d.ts +0 -50
- package/build/exporter/services/reference-fields.d.ts +0 -20
- package/build/exporter/services/robots.d.ts +0 -13
- package/build/exporter/services/sites.d.ts +0 -32
- package/build/exporter/services/store.d.ts +0 -15
- package/build/exporter/types/api.d.ts +0 -141
- package/build/exporter/types/global.d.ts +0 -84
- package/build/exporter/types/navigation.d.ts +0 -28
- package/build/exporter/types/pages.d.ts +0 -144
- package/build/exporter/types/render.d.ts +0 -50
- package/build/exporter/types/sites.d.ts +0 -56
- package/build/exporter/types/templates.d.ts +0 -8
- package/build/exporter/utils/api.d.ts +0 -23
- package/build/exporter/utils/artifacts.d.ts +0 -6
- package/build/exporter/utils/brush.d.ts +0 -18
- package/build/exporter/utils/cache.d.ts +0 -30
- package/build/exporter/utils/check-health.d.ts +0 -7
- package/build/exporter/utils/core-utils.d.ts +0 -77
- package/build/exporter/utils/domains.d.ts +0 -13
- package/build/exporter/utils/errors.d.ts +0 -15
- package/build/exporter/utils/folders.d.ts +0 -66
- package/build/exporter/utils/images.d.ts +0 -16
- package/build/exporter/utils/instance.d.ts +0 -21
- package/build/exporter/utils/loggin.d.ts +0 -37
- package/build/exporter/utils/pages.d.ts +0 -34
- package/build/exporter/utils/render.d.ts +0 -49
- package/build/exporter/utils/sites.d.ts +0 -31
- package/build/exporter/utils/store.d.ts +0 -53
- package/start-render.js +0 -5
|
@@ -8,7 +8,6 @@ import path from "node:path";
|
|
|
8
8
|
import { version as griddoVersion } from "../../package.json";
|
|
9
9
|
import { AuthService } from "../services/auth";
|
|
10
10
|
import { readDB, writeDB } from "../services/db";
|
|
11
|
-
import { doLifeCycle } from "../utils/core-utils";
|
|
12
11
|
import { getInstanceDomains } from "../utils/domains";
|
|
13
12
|
import { pathExists, pkgDir, removeDirs } from "../utils/folders";
|
|
14
13
|
import { resolveComponentsPath } from "../utils/instance";
|
|
@@ -71,11 +70,11 @@ function getDomainsSortedByNumberOfPages(
|
|
|
71
70
|
return domainsInfo.map(({ domain }) => domain);
|
|
72
71
|
}
|
|
73
72
|
|
|
74
|
-
async function
|
|
73
|
+
async function initRender() {
|
|
75
74
|
const root = (await pkgDir({ cwd: path.resolve(__dirname, "../../..") })) || "";
|
|
76
75
|
const cx = root;
|
|
77
76
|
const ssg = path.resolve((await pkgDir({ cwd: __dirname })) || "");
|
|
78
|
-
const cxCache = path.resolve(
|
|
77
|
+
const cxCache = path.resolve(root, ".cx-cache");
|
|
79
78
|
const components = resolveComponentsPath();
|
|
80
79
|
const exportsDir = path.join(root, "exports/sites");
|
|
81
80
|
|
|
@@ -90,7 +89,7 @@ async function initRenderAction() {
|
|
|
90
89
|
cx,
|
|
91
90
|
cxCache,
|
|
92
91
|
exportsDir,
|
|
93
|
-
root
|
|
92
|
+
root,
|
|
94
93
|
ssg,
|
|
95
94
|
},
|
|
96
95
|
};
|
|
@@ -103,24 +102,22 @@ async function initRenderAction() {
|
|
|
103
102
|
await writeDB(data);
|
|
104
103
|
}
|
|
105
104
|
|
|
106
|
-
async function
|
|
105
|
+
async function prepareDomains() {
|
|
107
106
|
await AuthService.login();
|
|
108
107
|
|
|
109
|
-
|
|
110
|
-
const __ssg = (await readDB()).paths.ssg;
|
|
108
|
+
const db = await readDB();
|
|
111
109
|
|
|
110
|
+
const __ssg = db.paths.ssg;
|
|
112
111
|
const domains = await getInstanceDomains();
|
|
113
112
|
const domainsWithNumberOfPages = await getDomainsWithNumberOfPages(domains);
|
|
114
113
|
const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPages);
|
|
115
114
|
|
|
116
115
|
// @deprecated use db.json (only for infra)
|
|
117
116
|
await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
|
|
117
|
+
await fsp.writeFile(path.join(__ssg, "domains.txt"), domainSorted.join(","));
|
|
118
118
|
|
|
119
|
-
// new behavior
|
|
120
|
-
const db = await readDB();
|
|
121
119
|
db.sortedDomains = domainSorted;
|
|
122
120
|
db.domains = {};
|
|
123
|
-
await writeDB(db);
|
|
124
121
|
|
|
125
122
|
for (const { domain, totalPages } of domainsWithNumberOfPages) {
|
|
126
123
|
const shouldBeRendered = totalPages > 0;
|
|
@@ -129,20 +126,19 @@ async function prepareDomainsAction() {
|
|
|
129
126
|
shouldBeRendered,
|
|
130
127
|
});
|
|
131
128
|
|
|
132
|
-
// Update DB
|
|
133
|
-
const db = await readDB();
|
|
134
129
|
db.domains[domain] = db.domains[domain] || {};
|
|
135
130
|
db.domains[domain].renderMode = renderMode;
|
|
136
131
|
db.domains[domain].shouldBeRendered = shouldBeRendered;
|
|
137
132
|
db.domains[domain].renderModeReason = reason;
|
|
138
|
-
await writeDB(db);
|
|
139
133
|
}
|
|
134
|
+
|
|
135
|
+
await writeDB(db);
|
|
140
136
|
}
|
|
141
137
|
|
|
142
|
-
async function
|
|
138
|
+
async function clean() {
|
|
143
139
|
const db = await readDB();
|
|
144
|
-
const {
|
|
145
|
-
await removeDirs([path.join(
|
|
140
|
+
const { root: __root } = db.paths;
|
|
141
|
+
await removeDirs([path.join(__root, "apiCache")]);
|
|
146
142
|
}
|
|
147
143
|
|
|
148
144
|
/**
|
|
@@ -152,10 +148,9 @@ async function cleanAction() {
|
|
|
152
148
|
*/
|
|
153
149
|
async function main() {
|
|
154
150
|
await showExporterVersion();
|
|
155
|
-
|
|
156
|
-
await
|
|
157
|
-
await
|
|
158
|
-
await doLifeCycle("Clean", cleanAction);
|
|
151
|
+
await initRender();
|
|
152
|
+
await prepareDomains();
|
|
153
|
+
await clean();
|
|
159
154
|
}
|
|
160
155
|
|
|
161
156
|
main().catch((err) => {
|
|
@@ -3,10 +3,8 @@
|
|
|
3
3
|
import { RESET_RENDER } from "../constants/endpoints";
|
|
4
4
|
import { AuthService } from "../services/auth";
|
|
5
5
|
import { post } from "../utils/api";
|
|
6
|
-
import { doLifeCycle } from "../utils/core-utils";
|
|
7
6
|
|
|
8
|
-
async function
|
|
9
|
-
await AuthService.login();
|
|
7
|
+
async function resetRender() {
|
|
10
8
|
await post({
|
|
11
9
|
endpoint: RESET_RENDER,
|
|
12
10
|
useApiCacheDir: false,
|
|
@@ -14,7 +12,9 @@ async function resetRenderAction() {
|
|
|
14
12
|
}
|
|
15
13
|
|
|
16
14
|
async function main() {
|
|
17
|
-
await
|
|
15
|
+
await AuthService.login();
|
|
16
|
+
await resetRender();
|
|
17
|
+
console.log(`The render status has been reset.`);
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
main().catch((err) => {
|
|
@@ -2,35 +2,43 @@
|
|
|
2
2
|
|
|
3
3
|
import { gatsbyRenderDomain } from "../adapters/gatsby";
|
|
4
4
|
import { AuthService } from "../services/auth";
|
|
5
|
+
import { brush } from "../utils/brush";
|
|
5
6
|
import { checkHealth } from "../utils/check-health";
|
|
6
|
-
import {
|
|
7
|
+
import { getInstanceDomains } from "../utils/domains";
|
|
7
8
|
import { RenderError } from "../utils/errors";
|
|
8
|
-
import { errorLabelLog } from "../utils/loggin";
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
checkHealth();
|
|
12
|
-
|
|
13
|
-
await AuthService.login();
|
|
10
|
+
const renderByDomains = !!process.env.GRIDDO_RENDER_BY_DOMAINS;
|
|
14
11
|
|
|
12
|
+
async function startRender() {
|
|
15
13
|
try {
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
if (renderByDomains) {
|
|
15
|
+
const [domain] = process.argv.slice(2);
|
|
16
|
+
await gatsbyRenderDomain(domain);
|
|
17
|
+
} else {
|
|
18
|
+
console.log("- LEGACY RENDER MODE -");
|
|
19
|
+
const domains = await getInstanceDomains();
|
|
20
|
+
for (const domain of domains) {
|
|
21
|
+
await gatsbyRenderDomain(domain);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
18
24
|
|
|
19
25
|
return;
|
|
20
26
|
} catch (error) {
|
|
21
27
|
if (error instanceof RenderError) {
|
|
22
|
-
|
|
28
|
+
console.log(brush.red("GRIDDO_ERROR InternalCXError"));
|
|
23
29
|
process.exit(1);
|
|
24
30
|
}
|
|
25
31
|
|
|
26
|
-
|
|
32
|
+
console.log(brush.red("GRIDDO_ERROR UnknownError"));
|
|
27
33
|
console.error(error);
|
|
28
34
|
process.exit(1);
|
|
29
35
|
}
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
async function main() {
|
|
33
|
-
|
|
39
|
+
checkHealth();
|
|
40
|
+
await AuthService.login();
|
|
41
|
+
await startRender();
|
|
34
42
|
}
|
|
35
43
|
|
|
36
44
|
main().catch((err) => {
|
|
@@ -11,12 +11,11 @@ import { AI_EMBEDDINGS, SEARCH } from "../constants/endpoints";
|
|
|
11
11
|
import { GRIDDO_AI_EMBEDDINGS, GRIDDO_SEARCH_FEATURE } from "../constants/envs";
|
|
12
12
|
import { ReadFromStoreError, UploadSearchError } from "../constants/errors";
|
|
13
13
|
import { AuthService } from "../services/auth";
|
|
14
|
-
import { readDB } from "../services/db";
|
|
15
14
|
import { post } from "../utils/api";
|
|
16
|
-
import {
|
|
15
|
+
import { getInstanceDomains } from "../utils/domains";
|
|
17
16
|
import { throwError } from "../utils/errors";
|
|
18
17
|
import { pathExists } from "../utils/folders";
|
|
19
|
-
import {
|
|
18
|
+
import { getRenderPathsFromDB } from "../utils/render";
|
|
20
19
|
|
|
21
20
|
/**
|
|
22
21
|
* Save in the BBDD the content of a page parsed without HTML tags.
|
|
@@ -180,11 +179,10 @@ async function getContentDirectories(domain: string) {
|
|
|
180
179
|
}
|
|
181
180
|
|
|
182
181
|
async function uploadSearchContent() {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
const
|
|
187
|
-
if (renderMode !== "IDLE" && GRIDDO_SEARCH_FEATURE) {
|
|
182
|
+
if (GRIDDO_SEARCH_FEATURE) {
|
|
183
|
+
await AuthService.login();
|
|
184
|
+
const domains = await getInstanceDomains();
|
|
185
|
+
for (const domain of domains) {
|
|
188
186
|
const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
|
|
189
187
|
await uploadRenderedSearchContentToAPI({
|
|
190
188
|
htmlContentDir,
|
|
@@ -194,18 +192,9 @@ async function uploadSearchContent() {
|
|
|
194
192
|
}
|
|
195
193
|
}
|
|
196
194
|
|
|
197
|
-
|
|
198
|
-
// const renderMode = await getRenderMode(domain);
|
|
199
|
-
// if (renderMode !== "IDLE" && GRIDDO_SEARCH_FEATURE) {
|
|
200
|
-
// const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
|
|
201
|
-
// await uploadRenderedSearchContentToAPI({ htmlContentDir, jsonContentDir });
|
|
202
|
-
// }
|
|
203
|
-
// }
|
|
204
|
-
|
|
205
|
-
async function aiEmbeddingsAction() {
|
|
195
|
+
async function aiEmbeddings() {
|
|
206
196
|
if (GRIDDO_SEARCH_FEATURE && GRIDDO_AI_EMBEDDINGS) {
|
|
207
197
|
await AuthService.login();
|
|
208
|
-
|
|
209
198
|
await post<AIEmbeddingsResponse>({
|
|
210
199
|
endpoint: AI_EMBEDDINGS,
|
|
211
200
|
useApiCacheDir: false,
|
|
@@ -214,8 +203,8 @@ async function aiEmbeddingsAction() {
|
|
|
214
203
|
}
|
|
215
204
|
|
|
216
205
|
async function main() {
|
|
217
|
-
await
|
|
218
|
-
await
|
|
206
|
+
await uploadSearchContent();
|
|
207
|
+
await aiEmbeddings();
|
|
219
208
|
}
|
|
220
209
|
|
|
221
210
|
main().catch((err) => {
|
|
@@ -64,10 +64,6 @@ const GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS = Number.parseInt(
|
|
|
64
64
|
process.env.GRIDDO_ARCHIVE_LIFECYCLE_MAX_ATTEMPTS || "1",
|
|
65
65
|
);
|
|
66
66
|
|
|
67
|
-
// Testing
|
|
68
|
-
const GRIDDO_FIXTURES_DOMAIN_NAMES = process.env.GRIDDO_CX_FIXTURES_DOMAIN_NAMES;
|
|
69
|
-
const GRIDDO_FIXTURES_SITE_NAMES = process.env.GRIDDO_CX_FIXTURES_SITE_NAMES;
|
|
70
|
-
|
|
71
67
|
export {
|
|
72
68
|
GRIDDO_AI_EMBEDDINGS,
|
|
73
69
|
GRIDDO_API_CONCURRENCY_COUNT,
|
|
@@ -81,8 +77,6 @@ export {
|
|
|
81
77
|
GRIDDO_CLOSE_LIFECYCLE_MAX_ATTEMPTS,
|
|
82
78
|
GRIDDO_DATA_LIFECYCLE_MAX_ATTEMPTS,
|
|
83
79
|
GRIDDO_DEBUG_LOGS,
|
|
84
|
-
GRIDDO_FIXTURES_DOMAIN_NAMES,
|
|
85
|
-
GRIDDO_FIXTURES_SITE_NAMES,
|
|
86
80
|
GRIDDO_INIT_LIFECYCLE_MAX_ATTEMPTS,
|
|
87
81
|
GRIDDO_META_LIFECYCLE_MAX_ATTEMPTS,
|
|
88
82
|
GRIDDO_PREPARE_LIFECYCLE_MAX_ATTEMPTS,
|
|
@@ -11,8 +11,9 @@ import type { SpawnSyncReturns } from "node:child_process";
|
|
|
11
11
|
import type { ErrorData } from "../utils/errors";
|
|
12
12
|
|
|
13
13
|
type ErrorsType =
|
|
14
|
-
| "UploadSearchError"
|
|
15
14
|
| "ArtifactError"
|
|
15
|
+
| "BundlesInconsistencyError"
|
|
16
|
+
| "CheckHealthError"
|
|
16
17
|
| "ErrorInSSGBuildProcess"
|
|
17
18
|
| "LifecycleExecutionError"
|
|
18
19
|
| "LoginError"
|
|
@@ -21,8 +22,8 @@ type ErrorsType =
|
|
|
21
22
|
| "ReadFromStoreError"
|
|
22
23
|
| "ReferenceFieldSourcesNotFoundError"
|
|
23
24
|
| "RenderUUIDError"
|
|
24
|
-
| "
|
|
25
|
-
| "
|
|
25
|
+
| "UploadSearchError"
|
|
26
|
+
| "WriteToStoreError";
|
|
26
27
|
|
|
27
28
|
const ArtifactError: ErrorData = {
|
|
28
29
|
error: "ArtifactError",
|
|
@@ -105,8 +106,16 @@ const CheckHealthError: ErrorData = {
|
|
|
105
106
|
hint: "Are the environment variables correctly set?",
|
|
106
107
|
};
|
|
107
108
|
|
|
109
|
+
const BundlesInconsistencyError: ErrorData = {
|
|
110
|
+
error: "BundlesInconsistencyError",
|
|
111
|
+
message: "There was a problem with the Javascript and CSS bundles diffs.",
|
|
112
|
+
expected: "The bundles must have the same files and content.",
|
|
113
|
+
};
|
|
114
|
+
|
|
108
115
|
export {
|
|
109
116
|
ArtifactError,
|
|
117
|
+
BundlesInconsistencyError,
|
|
118
|
+
CheckHealthError,
|
|
110
119
|
ErrorInSSGBuildProcess,
|
|
111
120
|
LifecycleExecutionError,
|
|
112
121
|
LoginError,
|
|
@@ -117,6 +126,5 @@ export {
|
|
|
117
126
|
RenderUUIDError,
|
|
118
127
|
UploadSearchError,
|
|
119
128
|
WriteToStoreError,
|
|
120
|
-
CheckHealthError,
|
|
121
129
|
type ErrorsType,
|
|
122
130
|
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { RenderDB } from "../types/render";
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { pkgDirSync } from "../utils/folders";
|
|
7
|
+
|
|
8
|
+
// Interfaz para abstraer la DB
|
|
9
|
+
export interface Database {
|
|
10
|
+
read(): Promise<RenderDB>;
|
|
11
|
+
write(renderDB: RenderDB): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export class JsonDatabase implements Database {
|
|
15
|
+
private readonly dbFilePath: string;
|
|
16
|
+
|
|
17
|
+
constructor(customDBPath?: string) {
|
|
18
|
+
this.dbFilePath =
|
|
19
|
+
customDBPath ??
|
|
20
|
+
path.join(
|
|
21
|
+
pkgDirSync({ cwd: path.resolve(__dirname, "../../..") }) ?? "",
|
|
22
|
+
".cx-cache",
|
|
23
|
+
"db.json",
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async read(customDBFilePath?: string): Promise<RenderDB> {
|
|
28
|
+
try {
|
|
29
|
+
const raw = await fsp.readFile(customDBFilePath || this.dbFilePath, "utf-8");
|
|
30
|
+
return JSON.parse(raw) as RenderDB;
|
|
31
|
+
} catch (error) {
|
|
32
|
+
if (error instanceof Error) {
|
|
33
|
+
console.error(`Error reading DB file at ${this.dbFilePath}:`, error.message);
|
|
34
|
+
} else {
|
|
35
|
+
console.error(`Unknown error reading DB file at ${this.dbFilePath}:`, error);
|
|
36
|
+
}
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async write(renderDB: RenderDB): Promise<void> {
|
|
42
|
+
try {
|
|
43
|
+
await fsp.mkdir(path.dirname(this.dbFilePath), { recursive: true });
|
|
44
|
+
await fsp.writeFile(this.dbFilePath, JSON.stringify(renderDB, null, 2));
|
|
45
|
+
} catch (error) {
|
|
46
|
+
if (error instanceof Error) {
|
|
47
|
+
console.error(`Error writing to DB file at ${this.dbFilePath}:`, error.message);
|
|
48
|
+
} else {
|
|
49
|
+
console.error(`Unknown error writing to DB file at ${this.dbFilePath}:`, error);
|
|
50
|
+
}
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -37,8 +37,8 @@ class RobotsService {
|
|
|
37
37
|
* Write robots.txt files for the domain.
|
|
38
38
|
*/
|
|
39
39
|
async writeFiles(domain: string) {
|
|
40
|
-
const {
|
|
41
|
-
const distDirectory = path.join(
|
|
40
|
+
const { __root } = await getRenderPathsFromDB({ domain });
|
|
41
|
+
const distDirectory = path.join(__root, "dist");
|
|
42
42
|
|
|
43
43
|
await this.getRobots();
|
|
44
44
|
|
|
@@ -8,7 +8,7 @@ import type {
|
|
|
8
8
|
} from "../types/pages";
|
|
9
9
|
import type { RenderMode } from "../types/render";
|
|
10
10
|
|
|
11
|
-
import
|
|
11
|
+
import fsp from "node:fs/promises";
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
|
|
14
14
|
import pLimit from "p-limit";
|
|
@@ -188,7 +188,7 @@ async function createStore(options: {
|
|
|
188
188
|
infoLog(`Getting pages from ${site.name} site`);
|
|
189
189
|
|
|
190
190
|
/// Creates the store directory for each site using the id
|
|
191
|
-
|
|
191
|
+
await fsp.mkdir(path.join(storeDir, siteDirName), {
|
|
192
192
|
recursive: true,
|
|
193
193
|
});
|
|
194
194
|
|
package/exporter/types/global.ts
CHANGED
|
@@ -8,22 +8,22 @@ import { getRenderPathsFromDB } from "./render";
|
|
|
8
8
|
* Returns the artifacts of CX.
|
|
9
9
|
*/
|
|
10
10
|
async function getCxArtifacts(domain: string): Promise<Artifacts> {
|
|
11
|
-
const {
|
|
11
|
+
const { __exports, __cache, __root } = await getRenderPathsFromDB({ domain });
|
|
12
12
|
|
|
13
13
|
return {
|
|
14
14
|
initials: [
|
|
15
|
-
__exports, //
|
|
16
|
-
__cache, //
|
|
17
|
-
path.join(
|
|
18
|
-
path.join(
|
|
15
|
+
__exports, // `<root>/exports/<domain>`
|
|
16
|
+
__cache, // `<root>/.cx-cache/<domain>`
|
|
17
|
+
path.join(__root, "store"),
|
|
18
|
+
path.join(__root, "apiCache"),
|
|
19
19
|
],
|
|
20
20
|
disposables: [
|
|
21
|
-
path.join(
|
|
22
|
-
path.join(
|
|
23
|
-
path.join(
|
|
24
|
-
path.join(
|
|
25
|
-
path.join(
|
|
26
|
-
path.join(
|
|
21
|
+
path.join(__root, "store"),
|
|
22
|
+
path.join(__root, "apiCache"),
|
|
23
|
+
path.join(__root, "dist"),
|
|
24
|
+
path.join(__root, "dist-restored"),
|
|
25
|
+
path.join(__root, "debug"),
|
|
26
|
+
path.join(__root, "assets"),
|
|
27
27
|
],
|
|
28
28
|
cacheables: [],
|
|
29
29
|
archivables: ["dist", "debug", "assets"],
|
package/exporter/utils/cache.ts
CHANGED
|
@@ -27,8 +27,8 @@ async function createAPICacheDir(options: { basePath: string }) {
|
|
|
27
27
|
* @param petition An object
|
|
28
28
|
*/
|
|
29
29
|
async function generateFilenameWithHash(petition: Petition) {
|
|
30
|
-
const {
|
|
31
|
-
const apiCacheDir = path.join(
|
|
30
|
+
const { __root } = await getRenderPathsFromDB();
|
|
31
|
+
const apiCacheDir = path.join(__root, "apiCache");
|
|
32
32
|
|
|
33
33
|
const hashSum = crypto.createHash("sha256");
|
|
34
34
|
hashSum.update(JSON.stringify(petition));
|
|
@@ -76,8 +76,8 @@ async function searchCacheData<T>(petition: Petition) {
|
|
|
76
76
|
* Get site hashes from the file as an object
|
|
77
77
|
*/
|
|
78
78
|
async function getHashSites(): Promise<HashSites> {
|
|
79
|
-
const {
|
|
80
|
-
const apiCacheDir = path.join(
|
|
79
|
+
const { __root } = await getRenderPathsFromDB();
|
|
80
|
+
const apiCacheDir = path.join(__root, "apiCache");
|
|
81
81
|
const siteHasFilename = `${apiCacheDir}/siteHash.json`;
|
|
82
82
|
|
|
83
83
|
try {
|
|
@@ -100,8 +100,8 @@ async function updatedSiteHash(siteId: number, siteHash: SiteHash) {
|
|
|
100
100
|
const lastHash = allHash[siteId];
|
|
101
101
|
const currentHash = siteHash || lastHash || Date.now();
|
|
102
102
|
|
|
103
|
-
const {
|
|
104
|
-
const apiCacheDir = path.join(
|
|
103
|
+
const { __root } = await getRenderPathsFromDB();
|
|
104
|
+
const apiCacheDir = path.join(__root, "apiCache");
|
|
105
105
|
const siteHasFilename = `${apiCacheDir}/siteHash.json`;
|
|
106
106
|
|
|
107
107
|
if (currentHash !== lastHash) {
|
|
@@ -154,9 +154,14 @@ async function executeAndReturnExecutionTime<T>(func: () => T | Promise<T>) {
|
|
|
154
154
|
async function doLifeCycle(
|
|
155
155
|
name: string,
|
|
156
156
|
action: LifeCycleAction,
|
|
157
|
-
options?: { maxRetrys?: number; spinner?: boolean },
|
|
157
|
+
options?: { maxRetrys?: number; spinner?: boolean; skip?: undefined | false | string },
|
|
158
158
|
) {
|
|
159
|
-
const { maxRetrys = 3 } = options || {};
|
|
159
|
+
const { maxRetrys = 3, skip } = options || {};
|
|
160
|
+
|
|
161
|
+
if (skip) {
|
|
162
|
+
console.log(`Skiping life-cycle. Reason: ${skip}`);
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
160
165
|
|
|
161
166
|
let trysCount = 0;
|
|
162
167
|
|
|
@@ -195,8 +200,8 @@ async function doLifeCycle(
|
|
|
195
200
|
}
|
|
196
201
|
|
|
197
202
|
async function removeAllSiteDirsFromStore() {
|
|
198
|
-
const {
|
|
199
|
-
const storeDir = path.join(
|
|
203
|
+
const { __root } = await getRenderPathsFromDB();
|
|
204
|
+
const storeDir = path.join(__root, "store");
|
|
200
205
|
const allDirs = await fsp.readdir(storeDir);
|
|
201
206
|
|
|
202
207
|
const allSiteDirs = allDirs.filter((dirname) => dirname !== "metadata");
|
|
@@ -282,6 +287,42 @@ const lifeCycleNames: Record<LifeCyclesNames, LifeCyclesNames> = {
|
|
|
282
287
|
SSG: "SSG",
|
|
283
288
|
} as const;
|
|
284
289
|
|
|
290
|
+
/**
|
|
291
|
+
* Busca de forma recursiva si ALGUNA de las claves de un array existe como propiedad propia
|
|
292
|
+
* en un objeto anidado. La búsqueda se detiene en cuanto se encuentra la primera coincidencia.
|
|
293
|
+
*
|
|
294
|
+
* @param obj El objeto a inspeccionar.
|
|
295
|
+
* @param keysToFind Un array de claves que se buscan.
|
|
296
|
+
* @param visited Un WeakSet para evitar ciclos de referencia.
|
|
297
|
+
* @returns Retorna `true` si al menos una de las claves se encuentra, de lo contrario `false`.
|
|
298
|
+
*/
|
|
299
|
+
function hasAnyKeyDeep<T extends object>(
|
|
300
|
+
obj: T,
|
|
301
|
+
keysToFind: string[],
|
|
302
|
+
visited = new WeakSet<object>(),
|
|
303
|
+
): boolean {
|
|
304
|
+
// 1. Casos base: no procesar no-objetos, si ya lo visitamos, o si el array de claves está vacío.
|
|
305
|
+
if (typeof obj !== "object" || obj === null || visited.has(obj)) {
|
|
306
|
+
return false;
|
|
307
|
+
}
|
|
308
|
+
if (keysToFind.length === 0) {
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
visited.add(obj);
|
|
312
|
+
|
|
313
|
+
// 2. Comprobar si ALGUNA de las claves buscadas es una propiedad propia del objeto actual.
|
|
314
|
+
// Usamos .some() porque se detiene en la primera coincidencia (eficiencia).
|
|
315
|
+
const foundKey = keysToFind.some((key) => Object.hasOwn(obj, key));
|
|
316
|
+
if (foundKey) {
|
|
317
|
+
return true;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// 3. Si no se encontró ninguna clave en este nivel, recorrer los valores y buscar recursivamente.
|
|
321
|
+
return Object.values(obj).some(
|
|
322
|
+
(value) => typeof value === "object" && hasAnyKeyDeep(value, keysToFind, visited),
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
|
|
285
326
|
export {
|
|
286
327
|
delay,
|
|
287
328
|
doLifeCycle,
|
|
@@ -294,4 +335,5 @@ export {
|
|
|
294
335
|
removeProperties,
|
|
295
336
|
saveBuildEndLogs,
|
|
296
337
|
walkStore,
|
|
338
|
+
hasAnyKeyDeep,
|
|
297
339
|
};
|
|
@@ -118,7 +118,7 @@ async function copyDirs(
|
|
|
118
118
|
} catch (error) {
|
|
119
119
|
if (withBackup) {
|
|
120
120
|
await restoreBackup(dstCompose);
|
|
121
|
-
|
|
121
|
+
verboseLog("Backup has been restored.");
|
|
122
122
|
}
|
|
123
123
|
|
|
124
124
|
throwError(ArtifactError, error);
|
|
@@ -213,7 +213,7 @@ async function deleteBackup(src: string, suffix = "-BACKUP") {
|
|
|
213
213
|
|
|
214
214
|
try {
|
|
215
215
|
await fsp.rm(dst, { recursive: true, force: true });
|
|
216
|
-
|
|
216
|
+
verboseLog(`Backup ${dst} has been deleted`);
|
|
217
217
|
} catch (error) {
|
|
218
218
|
console.log(error);
|
|
219
219
|
throw new Error(`Error while delete ${dst} backup`);
|
|
@@ -234,7 +234,7 @@ async function createBackup(src: string, suffix = "-BACKUP") {
|
|
|
234
234
|
|
|
235
235
|
try {
|
|
236
236
|
await fsp.rename(src, dst);
|
|
237
|
-
|
|
237
|
+
verboseLog(`Backup of ${src} has been created in ${dst}`);
|
|
238
238
|
} catch (error) {
|
|
239
239
|
console.log(`Error while coping ${src} to ${dst} backup`);
|
|
240
240
|
throwError(ArtifactError, error);
|
|
@@ -310,7 +310,7 @@ async function deleteEmptyDirectories(dirPath: string) {
|
|
|
310
310
|
} catch (err: any) {
|
|
311
311
|
// Si el directorio no existe o no se puede leer (ej. permisos), lo saltamos.
|
|
312
312
|
if (err.code === "ENOENT") {
|
|
313
|
-
console.warn(`
|
|
313
|
+
console.warn(`The directory "${dirPath}" does not exist, skipping it.`);
|
|
314
314
|
return;
|
|
315
315
|
}
|
|
316
316
|
console.error(`Error al leer el directorio "${dirPath}":`, err);
|
|
@@ -329,7 +329,7 @@ async function deleteEmptyDirectories(dirPath: string) {
|
|
|
329
329
|
if (remainingFiles.length === 0) {
|
|
330
330
|
try {
|
|
331
331
|
await fsp.rmdir(dirPath);
|
|
332
|
-
|
|
332
|
+
verboseLog(`Remove empty directory: ${dirPath}`);
|
|
333
333
|
} catch (err: any) {
|
|
334
334
|
// Puede que haya habido un problema de concurrencia o permisos
|
|
335
335
|
if (err.code === "ENOENT") {
|
|
@@ -344,9 +344,9 @@ async function deleteEmptyDirectories(dirPath: string) {
|
|
|
344
344
|
} catch (err: any) {
|
|
345
345
|
if (err.code === "ENOENT") {
|
|
346
346
|
// El directorio ya no existe, no es un error para nosotros en este contexto
|
|
347
|
-
console.warn(`
|
|
347
|
+
console.warn(`The directory "${dirPath}" does not exist or has already been processed.`);
|
|
348
348
|
} else {
|
|
349
|
-
console.error(`
|
|
349
|
+
console.error(`General error general while processing "${dirPath}":`, err);
|
|
350
350
|
throw err;
|
|
351
351
|
}
|
|
352
352
|
}
|
|
@@ -373,11 +373,34 @@ function pkgDirSync(options?: { readonly cwd?: string }) {
|
|
|
373
373
|
return filePath && path.dirname(filePath);
|
|
374
374
|
}
|
|
375
375
|
|
|
376
|
+
/**
|
|
377
|
+
* Busca recursivamente archivos que terminen con un sufijo específico dentro de un directorio.
|
|
378
|
+
* Esta función es un generador asíncrono, lo que la hace muy eficiente en uso de memoria.
|
|
379
|
+
*
|
|
380
|
+
* @param dir El directorio base para comenzar la búsqueda.
|
|
381
|
+
* @param suffix El sufijo con el que deben terminar los nombres de archivo (ej: 'page-data.json').
|
|
382
|
+
* @returns Un generador asíncrono que produce la ruta completa de cada archivo encontrado.
|
|
383
|
+
* @throws Si el directorio inicial `dir` no existe o no se puede leer.
|
|
384
|
+
*/
|
|
385
|
+
async function* findFilesBySuffix(dir: string, suffix: string): AsyncGenerator<string> {
|
|
386
|
+
const dirHandle = await fsp.opendir(dir);
|
|
387
|
+
for await (const item of dirHandle) {
|
|
388
|
+
const fullPath = path.join(dir, item.name);
|
|
389
|
+
if (item.isDirectory()) {
|
|
390
|
+
// yield* para encadenar otro generator.
|
|
391
|
+
yield* findFilesBySuffix(fullPath, suffix);
|
|
392
|
+
} else if (item.isFile() && item.name.endsWith(suffix)) {
|
|
393
|
+
yield fullPath;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
376
398
|
export {
|
|
377
399
|
copyDirs,
|
|
378
400
|
createDirs,
|
|
379
401
|
deleteDisposableSiteDirs,
|
|
380
402
|
deleteEmptyDirectories,
|
|
403
|
+
findFilesBySuffix,
|
|
381
404
|
moveDirs,
|
|
382
405
|
pathExists,
|
|
383
406
|
pkgDir,
|
package/exporter/utils/loggin.ts
CHANGED
|
@@ -103,16 +103,22 @@ function listSitesLog(title: string, sites: Array<Site>) {
|
|
|
103
103
|
${title} ${sitesOutput}`);
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
-
|
|
107
|
-
|
|
106
|
+
/**
|
|
107
|
+
* Console log with a blue color in the info prefix.
|
|
108
|
+
* @param str The string to be logged.
|
|
109
|
+
*/
|
|
110
|
+
function localLog(str: string) {
|
|
111
|
+
if (process.env.GRIDDO_IS_LOCAL_ENV) {
|
|
112
|
+
console.log(str);
|
|
113
|
+
}
|
|
108
114
|
}
|
|
109
115
|
|
|
110
116
|
export {
|
|
111
117
|
boxLog,
|
|
112
118
|
buildLog,
|
|
113
|
-
errorLabelLog,
|
|
114
119
|
infoLog,
|
|
115
120
|
listSitesLog,
|
|
121
|
+
localLog,
|
|
116
122
|
showExporterVersion,
|
|
117
123
|
successLog,
|
|
118
124
|
verboseLog,
|