@griddo/cx 11.10.16 → 11.10.17-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/end-render.js +4 -4
- package/build/commands/end-render.js.map +3 -3
- package/build/commands/prepare-assets-directory.js +2 -2
- package/build/commands/prepare-assets-directory.js.map +3 -3
- package/build/commands/prepare-domains-render.js +5 -5
- package/build/commands/prepare-domains-render.js.map +3 -3
- package/build/commands/reset-render.js +9 -9
- package/build/commands/reset-render.js.map +3 -3
- package/build/commands/start-render.js +24 -20
- package/build/commands/start-render.js.map +4 -4
- package/build/commands/upload-search-content.js +7 -7
- package/build/commands/upload-search-content.js.map +3 -3
- package/build/index.js +4 -2
- package/build/services/domains.d.ts +5 -1
- package/build/services/llms.d.ts +2 -0
- package/build/services/sites.d.ts +1 -1
- package/build/shared/endpoints.d.ts +2 -1
- package/build/shared/envs.d.ts +2 -1
- package/build/shared/types/global.d.ts +7 -1
- package/build/shared/types/render.d.ts +1 -0
- package/build/ssg-adapters/gatsby/actions/meta.d.ts +1 -1
- package/exporter/build.sh +1 -0
- package/exporter/commands/generate-md.noop +109 -0
- package/exporter/commands/prepare-domains-render.ts +19 -13
- package/exporter/commands/upload-search-content.ts +1 -1
- package/exporter/services/domains.ts +3 -1
- package/exporter/services/llms.ts +51 -0
- package/exporter/services/reference-fields.ts +4 -8
- package/exporter/shared/endpoints.ts +5 -0
- package/exporter/shared/envs.ts +2 -0
- package/exporter/shared/types/global.ts +10 -2
- package/exporter/shared/types/render.ts +1 -0
- package/exporter/ssg-adapters/gatsby/actions/meta.ts +6 -4
- package/exporter/ssg-adapters/gatsby/actions/sync.ts +1 -1
- package/package.json +2 -2
- package/tsconfig.commands.json +2 -1
- package/tsconfig.exporter.json +2 -1
- package/tsconfig.json +2 -1
package/build/index.js
CHANGED
|
@@ -135,6 +135,7 @@ var GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS);
|
|
|
135
135
|
var GRIDDO_VERBOSE_LOGS = envIsTruthy(env.GRIDDO_VERBOSE_LOGS);
|
|
136
136
|
var GRIDDO_USE_DIST_BACKUP = envIsTruthy(env.GRIDDO_USE_DIST_BACKUP);
|
|
137
137
|
var GRIDDO_SSG_BUNDLE_ANALYZER = envIsTruthy(env.GRIDDO_SSG_BUNDLE_ANALYZER);
|
|
138
|
+
var GRIDDO_RENDER_DISABLE_LLMS_TXT = envIsTruthy(env.GRIDDO_RENDER_DISABLE_LLMS_TXT);
|
|
138
139
|
|
|
139
140
|
// exporter/core/GriddoLog.ts
|
|
140
141
|
var GriddoLog = class _GriddoLog {
|
|
@@ -343,13 +344,14 @@ var RESET_RENDER = `${GRIDDO_API_URL2}/debug/reset-render`;
|
|
|
343
344
|
var ROBOTS = `${GRIDDO_API_URL2}/domains/robots`;
|
|
344
345
|
var SEARCH = `${GRIDDO_API_URL2}/search`;
|
|
345
346
|
var SETTINGS = `${GRIDDO_API_URL2}/settings`;
|
|
347
|
+
var DOMAIN_URI = `${GRIDDO_API_URL2}/domains/`;
|
|
346
348
|
var SITE_URI = `${GRIDDO_API_URL2}/site/`;
|
|
347
349
|
|
|
348
350
|
// package.json
|
|
349
351
|
var package_default = {
|
|
350
352
|
name: "@griddo/cx",
|
|
351
353
|
description: "Griddo SSG based on Gatsby",
|
|
352
|
-
version: "11.10.
|
|
354
|
+
version: "11.10.17-rc.1",
|
|
353
355
|
authors: [
|
|
354
356
|
"Hisco <francis.vega@griddo.io>"
|
|
355
357
|
],
|
|
@@ -442,7 +444,7 @@ var package_default = {
|
|
|
442
444
|
publishConfig: {
|
|
443
445
|
access: "public"
|
|
444
446
|
},
|
|
445
|
-
gitHead: "
|
|
447
|
+
gitHead: "00364b55b0b2954da9c57dfc53bcb84530219e06"
|
|
446
448
|
};
|
|
447
449
|
|
|
448
450
|
// exporter/shared/headers.ts
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Return an array of domains name (string) of the current instance.
|
|
3
3
|
*/
|
|
4
|
-
declare function getInstanceDomains(): Promise<
|
|
4
|
+
declare function getInstanceDomains(): Promise<{
|
|
5
|
+
id: number;
|
|
6
|
+
url: string;
|
|
7
|
+
slug: string;
|
|
8
|
+
}[]>;
|
|
5
9
|
export { getInstanceDomains };
|
|
@@ -12,7 +12,7 @@ declare function getAllSitesFromDomain(domain: string): Promise<Site[]>;
|
|
|
12
12
|
/**
|
|
13
13
|
* Fetch a page object from API.
|
|
14
14
|
*/
|
|
15
|
-
declare function getPage(id: number, cacheKey: string): Promise<import("
|
|
15
|
+
declare function getPage(id: number, cacheKey: string): Promise<import("@shared/types/pages").APIPageObject>;
|
|
16
16
|
/**
|
|
17
17
|
* Get site info
|
|
18
18
|
*/
|
|
@@ -8,6 +8,7 @@ declare const RESET_RENDER: string;
|
|
|
8
8
|
declare const ROBOTS: string;
|
|
9
9
|
declare const SEARCH: string;
|
|
10
10
|
declare const SETTINGS: string;
|
|
11
|
+
declare const LLMS: string[];
|
|
11
12
|
declare const BUILD_END: string[];
|
|
12
13
|
declare const BUILD_START: string[];
|
|
13
14
|
declare const GET_REFERENCE_FIELD_DATA: string[];
|
|
@@ -15,4 +16,4 @@ declare const GET_SITEMAP: string[];
|
|
|
15
16
|
declare const INFO: string[];
|
|
16
17
|
declare const LANGUAGES: string[];
|
|
17
18
|
declare const SOCIALS: string[];
|
|
18
|
-
export { AI_EMBEDDINGS, ALERT, BUILD_END, BUILD_START, DOMAINS, GET_ALL, GET_PAGE, GET_REFERENCE_FIELD_DATA, GET_SITEMAP, INFO, LANGUAGES, LOGIN, RESET_RENDER, ROBOTS, SEARCH, SETTINGS, SOCIALS, };
|
|
19
|
+
export { AI_EMBEDDINGS, ALERT, BUILD_END, BUILD_START, DOMAINS, GET_ALL, GET_PAGE, GET_REFERENCE_FIELD_DATA, GET_SITEMAP, INFO, LANGUAGES, LLMS, LOGIN, RESET_RENDER, ROBOTS, SEARCH, SETTINGS, SOCIALS, };
|
package/build/shared/envs.d.ts
CHANGED
|
@@ -14,4 +14,5 @@ declare const GRIDDO_AI_EMBEDDINGS: boolean;
|
|
|
14
14
|
declare const GRIDDO_VERBOSE_LOGS: boolean;
|
|
15
15
|
declare const GRIDDO_USE_DIST_BACKUP: boolean;
|
|
16
16
|
declare const GRIDDO_SSG_BUNDLE_ANALYZER: boolean;
|
|
17
|
-
|
|
17
|
+
declare const GRIDDO_RENDER_DISABLE_LLMS_TXT: boolean;
|
|
18
|
+
export { GRIDDO_AI_EMBEDDINGS, GRIDDO_API_CONCURRENCY_COUNT, GRIDDO_API_URL, GRIDDO_ASSET_PREFIX, GRIDDO_BOT_PASSWORD, GRIDDO_BOT_USER, GRIDDO_BUILD_LOGS, GRIDDO_BUILD_LOGS_BUFFER_SIZE, GRIDDO_PUBLIC_API_URL, GRIDDO_REACT_APP_INSTANCE, GRIDDO_RENDER_DISABLE_LLMS_TXT, GRIDDO_SEARCH_FEATURE, GRIDDO_SKIP_BUILD_CHECKS, GRIDDO_SSG_BUNDLE_ANALYZER, GRIDDO_SSG_VERBOSE_LOGS, GRIDDO_USE_DIST_BACKUP, GRIDDO_VERBOSE_LOGS, };
|
|
@@ -57,6 +57,12 @@ type Robot = {
|
|
|
57
57
|
content: string;
|
|
58
58
|
};
|
|
59
59
|
type Robots = Robot[];
|
|
60
|
+
type LLMs = {
|
|
61
|
+
id: number;
|
|
62
|
+
title: string;
|
|
63
|
+
url: string;
|
|
64
|
+
socialDescription: string;
|
|
65
|
+
}[];
|
|
60
66
|
interface RenderInfo {
|
|
61
67
|
buildProcessData: BuildProcessData;
|
|
62
68
|
createdPages: number[];
|
|
@@ -80,4 +86,4 @@ interface RenderArtifacts {
|
|
|
80
86
|
archivables: string[];
|
|
81
87
|
restaurable: string[];
|
|
82
88
|
}
|
|
83
|
-
export type { AIEmbeddingsResponse,
|
|
89
|
+
export type { AIEmbeddingsResponse, BuildProcessData, CXConfig, Domain, Domains, FetchDataProps, LifeCycleAction, LifeCyclesNames, LLMs, Petition, PlaceholderPath, PostSearchInfoProps, RenderArtifacts, RenderInfo, Robot, Robots, Settings, };
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import type { RenderContext } from "
|
|
1
|
+
import type { RenderContext } from "@shared/context";
|
|
2
2
|
export declare function metaAction(context: RenderContext): Promise<void>;
|
package/exporter/build.sh
CHANGED
|
@@ -24,6 +24,7 @@ esbuild ${log} ./exporter/commands/reset-render.ts ${bundle_node_opts}reset-rend
|
|
|
24
24
|
esbuild ${log} ./exporter/commands/start-render.ts ${bundle_node_opts}start-render.js
|
|
25
25
|
esbuild ${log} ./exporter/commands/prepare-domains-render.ts ${bundle_node_opts}prepare-domains-render.js
|
|
26
26
|
esbuild ${log} ./exporter/commands/prepare-assets-directory.ts ${bundle_node_opts}prepare-assets-directory.js
|
|
27
|
+
# esbuild ${log} ./exporter/commands/generate-md.ts ${bundle_node_opts}generate-md.js
|
|
27
28
|
|
|
28
29
|
# types
|
|
29
30
|
tsgo --emitDeclarationOnly --declaration --outDir build --project tsconfig.exporter.json
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import fsp from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
import TurndownService from "turndown";
|
|
5
|
+
|
|
6
|
+
// import { gfm } from "turndown-plugin-gfm";
|
|
7
|
+
|
|
8
|
+
import { throwError, withErrorHandler } from "../core/errors";
|
|
9
|
+
import { GriddoLog } from "../core/GriddoLog";
|
|
10
|
+
import { getRenderPathsHydratedWithDomainFromDB } from "../services/render";
|
|
11
|
+
import { ReadFromStoreError } from "../shared/errors";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Converts HTML to Markdown.
|
|
15
|
+
* @param html - The HTML content to convert.
|
|
16
|
+
* @returns The Markdown content.
|
|
17
|
+
*/
|
|
18
|
+
function htmlToMarkdown(html: string): string {
|
|
19
|
+
const turndown = new TurndownService({
|
|
20
|
+
headingStyle: "atx",
|
|
21
|
+
codeBlockStyle: "fenced",
|
|
22
|
+
bulletListMarker: "-",
|
|
23
|
+
emDelimiter: "*",
|
|
24
|
+
strongDelimiter: "**",
|
|
25
|
+
linkStyle: "inlined", // evita ruido
|
|
26
|
+
hr: "---",
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
// añade soporte GitHub-like
|
|
30
|
+
// turndown.use(gfm);
|
|
31
|
+
|
|
32
|
+
// elimina tags que hacen ruido para LLMs
|
|
33
|
+
turndown.remove([
|
|
34
|
+
"style",
|
|
35
|
+
"script",
|
|
36
|
+
"noscript",
|
|
37
|
+
"iframe",
|
|
38
|
+
"form",
|
|
39
|
+
"input",
|
|
40
|
+
"button",
|
|
41
|
+
"nav",
|
|
42
|
+
"footer",
|
|
43
|
+
"object",
|
|
44
|
+
]);
|
|
45
|
+
|
|
46
|
+
let md = turndown.turndown(html);
|
|
47
|
+
|
|
48
|
+
md = md
|
|
49
|
+
.replace(/ /g, " ")
|
|
50
|
+
.replace(/&/g, "&")
|
|
51
|
+
.replace(/\n{3,}/g, "\n\n") // compacta saltos
|
|
52
|
+
.trim();
|
|
53
|
+
|
|
54
|
+
return md;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Walk recursively in a basePath and return an array of pages with json
|
|
59
|
+
* extension with the full absolute path and the path includes "page-data".
|
|
60
|
+
*
|
|
61
|
+
* @param basePath The path to walk recursively.
|
|
62
|
+
* @returns An array of pages with json extension with the full absolute path
|
|
63
|
+
* and the path includes "page-data".
|
|
64
|
+
*/
|
|
65
|
+
async function* walkRecursively(basePath: string): AsyncGenerator<string> {
|
|
66
|
+
const filesHandle = await fsp.opendir(basePath);
|
|
67
|
+
|
|
68
|
+
for await (const fileDirent of filesHandle) {
|
|
69
|
+
if (fileDirent.isDirectory()) {
|
|
70
|
+
yield* walkRecursively(path.join(basePath, fileDirent.name));
|
|
71
|
+
} else if (fileDirent.isFile() && path.extname(fileDirent.name) === ".html") {
|
|
72
|
+
yield path.join(basePath, fileDirent.name);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function* getPageDataPagesFromExports(
|
|
78
|
+
basePath: string,
|
|
79
|
+
): AsyncGenerator<{ path: string; content: string }> {
|
|
80
|
+
const jsonFiles = walkRecursively(basePath);
|
|
81
|
+
|
|
82
|
+
for await (const filePath of jsonFiles) {
|
|
83
|
+
try {
|
|
84
|
+
const fileContent = await fsp.readFile(filePath, "utf8");
|
|
85
|
+
const content = htmlToMarkdown(fileContent);
|
|
86
|
+
yield { path: filePath, content };
|
|
87
|
+
} catch (error) {
|
|
88
|
+
throwError(ReadFromStoreError, error);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function generateMD() {
|
|
94
|
+
const [domain] = process.argv.slice(2);
|
|
95
|
+
const { __exports } = await getRenderPathsHydratedWithDomainFromDB({ domain });
|
|
96
|
+
|
|
97
|
+
const pages = getPageDataPagesFromExports(path.join(__exports, "dist"));
|
|
98
|
+
|
|
99
|
+
for await (const page of pages) {
|
|
100
|
+
await fsp.writeFile(path.join(page.path.replace(".html", ".md")), page.content);
|
|
101
|
+
GriddoLog.info(`Generating MD for a ${page.path}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function main() {
|
|
106
|
+
await generateMD();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
withErrorHandler(main);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Domain } from "@shared/types/global";
|
|
1
2
|
import type { RenderDB } from "../shared/types/render";
|
|
2
3
|
|
|
3
4
|
import { execSync } from "node:child_process";
|
|
@@ -17,10 +18,10 @@ import { resolveDomainRenderMode } from "../services/render";
|
|
|
17
18
|
import { GRIDDO_SKIP_BUILD_CHECKS } from "../shared/envs";
|
|
18
19
|
import { pkgDir } from "../shared/npm-modules/pkg-dir";
|
|
19
20
|
|
|
20
|
-
async function getDomainsWithNumberOfPagesWithActivity(domains:
|
|
21
|
+
async function getDomainsWithNumberOfPagesWithActivity(domains: Domain[]) {
|
|
21
22
|
const results = await Promise.all(
|
|
22
23
|
domains.map(async (domain) => {
|
|
23
|
-
const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain);
|
|
24
|
+
const { sitesToPublish, sitesToUnpublish } = await getSitesToRender(domain.slug);
|
|
24
25
|
const allSites = [...sitesToPublish, ...sitesToUnpublish];
|
|
25
26
|
|
|
26
27
|
if (allSites.length < 1) {
|
|
@@ -56,7 +57,7 @@ async function getDomainsWithNumberOfPagesWithActivity(domains: string[]) {
|
|
|
56
57
|
}
|
|
57
58
|
|
|
58
59
|
function getDomainsSortedByNumberOfPages(
|
|
59
|
-
domainsInfo: { domain:
|
|
60
|
+
domainsInfo: { domain: Domain; totalPendingPages: number }[],
|
|
60
61
|
) {
|
|
61
62
|
domainsInfo.sort((a, b) => a.totalPendingPages - b.totalPendingPages);
|
|
62
63
|
|
|
@@ -110,28 +111,33 @@ async function prepareDomains() {
|
|
|
110
111
|
const domainsWithNumberOfPendingPages = await getDomainsWithNumberOfPagesWithActivity(domains);
|
|
111
112
|
const domainSorted = getDomainsSortedByNumberOfPages(domainsWithNumberOfPendingPages);
|
|
112
113
|
|
|
113
|
-
|
|
114
|
-
await fsp.writeFile(path.join(__ssg, "domains.json"), JSON.stringify(domainSorted));
|
|
115
|
-
|
|
116
|
-
db.sortedDomains = domainSorted;
|
|
114
|
+
db.sortedDomains = domainSorted.map(({ slug }) => slug);
|
|
117
115
|
db.domains = {};
|
|
118
116
|
|
|
117
|
+
// @deprecated use db.json (only for infra)
|
|
118
|
+
await fsp.writeFile(
|
|
119
|
+
path.join(__ssg, "domains.json"),
|
|
120
|
+
JSON.stringify(domainSorted.map(({ slug }) => slug)),
|
|
121
|
+
);
|
|
122
|
+
|
|
119
123
|
for (const { domain, totalPendingPages } of domainsWithNumberOfPendingPages) {
|
|
124
|
+
const domainSlug = domain.slug;
|
|
120
125
|
const shouldBeRendered = totalPendingPages > 0;
|
|
121
126
|
const { renderMode, reason } = await resolveDomainRenderMode({
|
|
122
|
-
domain,
|
|
127
|
+
domain: domainSlug,
|
|
123
128
|
shouldBeRendered,
|
|
124
129
|
});
|
|
125
130
|
|
|
126
131
|
// Log RenderModes/Reason
|
|
127
132
|
GriddoLog.info(
|
|
128
|
-
`(From Initial Render) [${
|
|
133
|
+
`(From Initial Render) [${domainSlug}]: Marked as ${renderMode} with the reason: ${reason}`,
|
|
129
134
|
);
|
|
130
135
|
|
|
131
|
-
db.domains[
|
|
132
|
-
db.domains[
|
|
133
|
-
db.domains[
|
|
134
|
-
db.domains[
|
|
136
|
+
db.domains[domainSlug] = db.domains[domainSlug] || {};
|
|
137
|
+
db.domains[domainSlug].id = domain.id;
|
|
138
|
+
db.domains[domainSlug].renderMode = renderMode;
|
|
139
|
+
db.domains[domainSlug].shouldBeRendered = shouldBeRendered;
|
|
140
|
+
db.domains[domainSlug].renderModeReason = reason;
|
|
135
141
|
}
|
|
136
142
|
|
|
137
143
|
await writeDB(db);
|
|
@@ -185,7 +185,7 @@ async function getContentDirectories(domain: string) {
|
|
|
185
185
|
|
|
186
186
|
async function uploadSearchContent() {
|
|
187
187
|
if (GRIDDO_SEARCH_FEATURE) {
|
|
188
|
-
const domains = await getInstanceDomains();
|
|
188
|
+
const domains = (await getInstanceDomains()).map(({ slug }) => slug);
|
|
189
189
|
for (const domain of domains) {
|
|
190
190
|
const { htmlContentDir, jsonContentDir } = await getContentDirectories(domain);
|
|
191
191
|
|
|
@@ -23,7 +23,9 @@ async function getInstanceDomains() {
|
|
|
23
23
|
|
|
24
24
|
const filteredDomains = domains
|
|
25
25
|
.filter(({ slug }) => !!slug)
|
|
26
|
-
.map((
|
|
26
|
+
.map((domain) => {
|
|
27
|
+
return { ...domain, slug: domain.slug.replace("/", "") };
|
|
28
|
+
});
|
|
27
29
|
|
|
28
30
|
return [...new Set(filteredDomains)];
|
|
29
31
|
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { LLMs } from "../shared/types/global";
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { readDB } from "@core/db";
|
|
7
|
+
import { getRenderPathsHydratedWithDomainFromDB } from "@services/render";
|
|
8
|
+
import { GRIDDO_RENDER_DISABLE_LLMS_TXT } from "@shared/envs";
|
|
9
|
+
|
|
10
|
+
import { GriddoLog } from "../core/GriddoLog";
|
|
11
|
+
import { LLMS } from "../shared/endpoints";
|
|
12
|
+
import { get } from "./api";
|
|
13
|
+
|
|
14
|
+
async function generateLLMs(domain: string) {
|
|
15
|
+
if (GRIDDO_RENDER_DISABLE_LLMS_TXT) {
|
|
16
|
+
GriddoLog.verbose(`LLMs generation disabled by environment variable`);
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const db = await readDB();
|
|
21
|
+
const domainId = db.domains[domain].id;
|
|
22
|
+
|
|
23
|
+
const [prefix, suffix] = LLMS;
|
|
24
|
+
const llmsResponse = await get<LLMs>({ endpoint: `${prefix}${domainId}${suffix}` });
|
|
25
|
+
|
|
26
|
+
const contentDistilled = llmsResponse
|
|
27
|
+
.map(({ title, url, socialDescription }) => {
|
|
28
|
+
const description = socialDescription ? `: ${socialDescription}` : "";
|
|
29
|
+
return `- [${title}](${url})${description}`;
|
|
30
|
+
})
|
|
31
|
+
.join("\n");
|
|
32
|
+
|
|
33
|
+
const { __root } = await getRenderPathsHydratedWithDomainFromDB({ domain });
|
|
34
|
+
const distDirectory = path.join(__root, "current-dist");
|
|
35
|
+
|
|
36
|
+
try {
|
|
37
|
+
const head = `llms.txt for the domain ${domain}`;
|
|
38
|
+
const generated = `Generated: ${new Date().toISOString().replace("T", " ").slice(0, 19)}`;
|
|
39
|
+
const frontmatter = `${head}\n${generated}`;
|
|
40
|
+
|
|
41
|
+
await fsp.writeFile(
|
|
42
|
+
path.join(distDirectory, "llms.txt"),
|
|
43
|
+
`${frontmatter}\n\n${contentDistilled}`,
|
|
44
|
+
);
|
|
45
|
+
GriddoLog.verbose(`wrote llms.txt to ${path.join(distDirectory, "llms.txt")}`);
|
|
46
|
+
} catch (error) {
|
|
47
|
+
GriddoLog.error(`Error writing llms.txt to ${path.join(distDirectory, "llms.txt")}:`, error);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export { generateLLMs };
|
|
@@ -23,8 +23,7 @@ function getBody(data: Fields.Reference<unknown>, page: Core.Page) {
|
|
|
23
23
|
allLanguages = false,
|
|
24
24
|
preferenceLanguage = false,
|
|
25
25
|
referenceId,
|
|
26
|
-
|
|
27
|
-
// fields,
|
|
26
|
+
fields,
|
|
28
27
|
} = data;
|
|
29
28
|
|
|
30
29
|
if (mode === "auto") {
|
|
@@ -36,8 +35,7 @@ function getBody(data: Fields.Reference<unknown>, page: Core.Page) {
|
|
|
36
35
|
fullRelations,
|
|
37
36
|
allLanguages,
|
|
38
37
|
preferenceLanguage,
|
|
39
|
-
|
|
40
|
-
// fields,
|
|
38
|
+
fields,
|
|
41
39
|
};
|
|
42
40
|
}
|
|
43
41
|
|
|
@@ -46,8 +44,7 @@ function getBody(data: Fields.Reference<unknown>, page: Core.Page) {
|
|
|
46
44
|
mode,
|
|
47
45
|
fixed,
|
|
48
46
|
fullRelations,
|
|
49
|
-
|
|
50
|
-
// fields
|
|
47
|
+
fields,
|
|
51
48
|
};
|
|
52
49
|
}
|
|
53
50
|
|
|
@@ -58,8 +55,7 @@ function getBody(data: Fields.Reference<unknown>, page: Core.Page) {
|
|
|
58
55
|
quantity,
|
|
59
56
|
fullRelations,
|
|
60
57
|
referenceId: referenceId || page?.structuredDataContent?.id,
|
|
61
|
-
|
|
62
|
-
// fields,
|
|
58
|
+
fields,
|
|
63
59
|
};
|
|
64
60
|
}
|
|
65
61
|
|
|
@@ -12,6 +12,10 @@ const ROBOTS = `${GRIDDO_API_URL}/domains/robots`;
|
|
|
12
12
|
const SEARCH = `${GRIDDO_API_URL}/search`;
|
|
13
13
|
const SETTINGS = `${GRIDDO_API_URL}/settings`;
|
|
14
14
|
|
|
15
|
+
// Domain
|
|
16
|
+
const DOMAIN_URI = `${GRIDDO_API_URL}/domains/`;
|
|
17
|
+
const LLMS = [DOMAIN_URI, "/llms"];
|
|
18
|
+
|
|
15
19
|
// Site
|
|
16
20
|
const SITE_URI = `${GRIDDO_API_URL}/site/`;
|
|
17
21
|
const BUILD_END = [SITE_URI, "/build/end"];
|
|
@@ -34,6 +38,7 @@ export {
|
|
|
34
38
|
GET_SITEMAP,
|
|
35
39
|
INFO,
|
|
36
40
|
LANGUAGES,
|
|
41
|
+
LLMS,
|
|
37
42
|
LOGIN,
|
|
38
43
|
RESET_RENDER,
|
|
39
44
|
ROBOTS,
|
package/exporter/shared/envs.ts
CHANGED
|
@@ -37,6 +37,7 @@ const GRIDDO_AI_EMBEDDINGS = envIsTruthy(env.GRIDDO_AI_EMBEDDINGS);
|
|
|
37
37
|
const GRIDDO_VERBOSE_LOGS = envIsTruthy(env.GRIDDO_VERBOSE_LOGS);
|
|
38
38
|
const GRIDDO_USE_DIST_BACKUP = envIsTruthy(env.GRIDDO_USE_DIST_BACKUP);
|
|
39
39
|
const GRIDDO_SSG_BUNDLE_ANALYZER = envIsTruthy(env.GRIDDO_SSG_BUNDLE_ANALYZER);
|
|
40
|
+
const GRIDDO_RENDER_DISABLE_LLMS_TXT = envIsTruthy(env.GRIDDO_RENDER_DISABLE_LLMS_TXT);
|
|
40
41
|
|
|
41
42
|
export {
|
|
42
43
|
GRIDDO_AI_EMBEDDINGS,
|
|
@@ -49,6 +50,7 @@ export {
|
|
|
49
50
|
GRIDDO_BUILD_LOGS_BUFFER_SIZE,
|
|
50
51
|
GRIDDO_PUBLIC_API_URL,
|
|
51
52
|
GRIDDO_REACT_APP_INSTANCE,
|
|
53
|
+
GRIDDO_RENDER_DISABLE_LLMS_TXT,
|
|
52
54
|
GRIDDO_SEARCH_FEATURE,
|
|
53
55
|
GRIDDO_SKIP_BUILD_CHECKS,
|
|
54
56
|
GRIDDO_SSG_BUNDLE_ANALYZER,
|
|
@@ -64,6 +64,13 @@ type Robot = { path: string; content: string };
|
|
|
64
64
|
|
|
65
65
|
type Robots = Robot[];
|
|
66
66
|
|
|
67
|
+
type LLMs = {
|
|
68
|
+
id: number;
|
|
69
|
+
title: string;
|
|
70
|
+
url: string;
|
|
71
|
+
socialDescription: string;
|
|
72
|
+
}[];
|
|
73
|
+
|
|
67
74
|
interface RenderInfo {
|
|
68
75
|
buildProcessData: BuildProcessData;
|
|
69
76
|
createdPages: number[];
|
|
@@ -116,19 +123,20 @@ interface RenderArtifacts {
|
|
|
116
123
|
|
|
117
124
|
export type {
|
|
118
125
|
AIEmbeddingsResponse,
|
|
119
|
-
RenderArtifacts,
|
|
120
126
|
BuildProcessData,
|
|
121
127
|
CXConfig,
|
|
122
128
|
Domain,
|
|
123
129
|
Domains,
|
|
124
130
|
FetchDataProps,
|
|
131
|
+
LifeCycleAction,
|
|
125
132
|
LifeCyclesNames,
|
|
133
|
+
LLMs,
|
|
126
134
|
Petition,
|
|
127
135
|
PlaceholderPath,
|
|
128
136
|
PostSearchInfoProps,
|
|
137
|
+
RenderArtifacts,
|
|
129
138
|
RenderInfo,
|
|
130
139
|
Robot,
|
|
131
140
|
Robots,
|
|
132
141
|
Settings,
|
|
133
|
-
LifeCycleAction,
|
|
134
142
|
};
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { RenderContext } from "
|
|
1
|
+
import type { RenderContext } from "@shared/context";
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { generateLLMs } from "@services/llms";
|
|
4
|
+
import { generateBuildReport } from "@services/render";
|
|
5
|
+
import { generateRobots } from "@services/robots";
|
|
6
|
+
import { generateSitemaps } from "@services/sitemaps";
|
|
6
7
|
|
|
7
8
|
export async function metaAction(context: RenderContext) {
|
|
8
9
|
const { domain } = context;
|
|
@@ -10,4 +11,5 @@ export async function metaAction(context: RenderContext) {
|
|
|
10
11
|
await generateBuildReport(domain);
|
|
11
12
|
await generateSitemaps(domain);
|
|
12
13
|
await generateRobots(domain);
|
|
14
|
+
await generateLLMs(domain);
|
|
13
15
|
}
|
|
@@ -45,7 +45,7 @@ export async function syncAction(context: RenderContext<SSG>) {
|
|
|
45
45
|
dst: previousDist,
|
|
46
46
|
pagesToCreate,
|
|
47
47
|
pagesToDelete,
|
|
48
|
-
artifactsToCopyToExports: ["build-report.json"],
|
|
48
|
+
artifactsToCopyToExports: ["build-report.json", "llms.txt", "robots.txt"],
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
await syncRender.execute();
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@griddo/cx",
|
|
3
3
|
"description": "Griddo SSG based on Gatsby",
|
|
4
|
-
"version": "11.10.
|
|
4
|
+
"version": "11.10.17-rc.1",
|
|
5
5
|
"authors": [
|
|
6
6
|
"Hisco <francis.vega@griddo.io>"
|
|
7
7
|
],
|
|
@@ -94,5 +94,5 @@
|
|
|
94
94
|
"publishConfig": {
|
|
95
95
|
"access": "public"
|
|
96
96
|
},
|
|
97
|
-
"gitHead": "
|
|
97
|
+
"gitHead": "00364b55b0b2954da9c57dfc53bcb84530219e06"
|
|
98
98
|
}
|
package/tsconfig.commands.json
CHANGED
package/tsconfig.exporter.json
CHANGED