@bagelink/blox 1.15.273 → 1.15.275
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{prerender-1fZ-6d-3.js → prerender-BCRyYSNg.js} +18 -1
- package/dist/{prerender-52IAxpqw.cjs → prerender-DQ7D6auC.cjs} +17 -0
- package/dist/shard-runner-CZnX4qnn.cjs +123 -0
- package/dist/shard-runner-D4gCfCi3.js +101 -0
- package/dist/ssg/cli.cjs +45 -11
- package/dist/ssg/cli.mjs +42 -8
- package/dist/ssg/index.cjs +20 -20
- package/dist/ssg/index.mjs +2 -2
- package/dist/ssg/shard-runner.d.ts +45 -0
- package/dist/ssg/shard-runner.d.ts.map +1 -0
- package/package.json +1 -1
|
@@ -111,6 +111,12 @@ function extractCollectionRefs(pages) {
|
|
|
111
111
|
}
|
|
112
112
|
return refs;
|
|
113
113
|
}
|
|
114
|
+
const cmsRoutes = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
|
|
115
|
+
__proto__: null,
|
|
116
|
+
expandLocalePaths,
|
|
117
|
+
fetchCmsPrerenderPaths,
|
|
118
|
+
fetchCmsSiteData
|
|
119
|
+
}, Symbol.toStringTag, { value: "Module" }));
|
|
114
120
|
async function polyfillBloxSsgGlobals(options) {
|
|
115
121
|
const { env } = await import("node:process");
|
|
116
122
|
const pageUrl = (options == null ? void 0 : options.pageUrl) ?? env.BAGELINK_API_URL ?? "https://example.com/";
|
|
@@ -547,6 +553,15 @@ async function mergeLlmParts(options) {
|
|
|
547
553
|
}
|
|
548
554
|
return { pages: pages.length };
|
|
549
555
|
}
|
|
556
|
+
const llmArtifacts = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
|
|
557
|
+
__proto__: null,
|
|
558
|
+
buildJsonLd,
|
|
559
|
+
extractPageText,
|
|
560
|
+
generateLlmsFullTxt,
|
|
561
|
+
generateLlmsTxt,
|
|
562
|
+
isPathExcluded,
|
|
563
|
+
mergeLlmParts
|
|
564
|
+
}, Symbol.toStringTag, { value: "Module" }));
|
|
550
565
|
async function prerender({
|
|
551
566
|
root = process.cwd(),
|
|
552
567
|
clientOutDir = "dist/client",
|
|
@@ -887,5 +902,7 @@ export {
|
|
|
887
902
|
generateSitemapXml as m,
|
|
888
903
|
getSiteIconFlags as n,
|
|
889
904
|
mergeLlmParts as o,
|
|
890
|
-
polyfillBloxSsgGlobals as p
|
|
905
|
+
polyfillBloxSsgGlobals as p,
|
|
906
|
+
cmsRoutes as q,
|
|
907
|
+
llmArtifacts as r
|
|
891
908
|
};
|
|
@@ -134,6 +134,12 @@ function extractCollectionRefs(pages) {
|
|
|
134
134
|
}
|
|
135
135
|
return refs;
|
|
136
136
|
}
|
|
137
|
+
const cmsRoutes = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
|
|
138
|
+
__proto__: null,
|
|
139
|
+
expandLocalePaths,
|
|
140
|
+
fetchCmsPrerenderPaths,
|
|
141
|
+
fetchCmsSiteData
|
|
142
|
+
}, Symbol.toStringTag, { value: "Module" }));
|
|
137
143
|
async function polyfillBloxSsgGlobals(options) {
|
|
138
144
|
const { env } = await import("node:process");
|
|
139
145
|
const pageUrl = (options == null ? void 0 : options.pageUrl) ?? env.BAGELINK_API_URL ?? "https://example.com/";
|
|
@@ -570,6 +576,15 @@ async function mergeLlmParts(options) {
|
|
|
570
576
|
}
|
|
571
577
|
return { pages: pages.length };
|
|
572
578
|
}
|
|
579
|
+
const llmArtifacts = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
|
|
580
|
+
__proto__: null,
|
|
581
|
+
buildJsonLd,
|
|
582
|
+
extractPageText,
|
|
583
|
+
generateLlmsFullTxt,
|
|
584
|
+
generateLlmsTxt,
|
|
585
|
+
isPathExcluded,
|
|
586
|
+
mergeLlmParts
|
|
587
|
+
}, Symbol.toStringTag, { value: "Module" }));
|
|
573
588
|
async function prerender({
|
|
574
589
|
root = process.cwd(),
|
|
575
590
|
clientOutDir = "dist/client",
|
|
@@ -897,6 +912,7 @@ function discoverInternalLinks(html) {
|
|
|
897
912
|
exports.buildJsonLd = buildJsonLd;
|
|
898
913
|
exports.buildPageHead = buildPageHead;
|
|
899
914
|
exports.buildSiteHead = buildSiteHead;
|
|
915
|
+
exports.cmsRoutes = cmsRoutes;
|
|
900
916
|
exports.extractPageText = extractPageText;
|
|
901
917
|
exports.fetchCmsPrerenderPaths = fetchCmsPrerenderPaths;
|
|
902
918
|
exports.fetchCmsSiteData = fetchCmsSiteData;
|
|
@@ -906,6 +922,7 @@ exports.generateNetlifyRedirects = generateNetlifyRedirects;
|
|
|
906
922
|
exports.generateRobotsTxt = generateRobotsTxt;
|
|
907
923
|
exports.generateSitemapXml = generateSitemapXml;
|
|
908
924
|
exports.getSiteIconFlags = getSiteIconFlags;
|
|
925
|
+
exports.llmArtifacts = llmArtifacts;
|
|
909
926
|
exports.mergeLlmParts = mergeLlmParts;
|
|
910
927
|
exports.polyfillBloxSsgGlobals = polyfillBloxSsgGlobals;
|
|
911
928
|
exports.prerender = prerender;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __copyProps = (to, from, except, desc) => {
|
|
9
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
10
|
+
for (let key of __getOwnPropNames(from))
|
|
11
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
12
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
13
|
+
}
|
|
14
|
+
return to;
|
|
15
|
+
};
|
|
16
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
17
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
18
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
19
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
20
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
21
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
22
|
+
mod
|
|
23
|
+
));
|
|
24
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
25
|
+
const node_child_process = require("node:child_process");
|
|
26
|
+
const os = require("node:os");
|
|
27
|
+
const path = require("node:path");
|
|
28
|
+
const process = require("node:process");
|
|
29
|
+
const AUTO_SHARD_CAP = Number(process.env.BLOX_SSG_SHARD_CAP) || 6;
|
|
30
|
+
const PREFETCH_BUDGET = Number(process.env.BLOX_SSG_PREFETCH_BUDGET) || 24;
|
|
31
|
+
function resolveShardCount(shards) {
|
|
32
|
+
if (shards === "auto") {
|
|
33
|
+
const cores = os.cpus().length || 4;
|
|
34
|
+
return Math.max(1, Math.min(cores, AUTO_SHARD_CAP));
|
|
35
|
+
}
|
|
36
|
+
return Math.max(1, Math.floor(shards));
|
|
37
|
+
}
|
|
38
|
+
async function enumerateRoutes(cmd) {
|
|
39
|
+
return new Promise((resolve, reject) => {
|
|
40
|
+
const child = node_child_process.spawn(cmd, { shell: true, stdio: ["ignore", "pipe", "inherit"] });
|
|
41
|
+
let out = "";
|
|
42
|
+
child.stdout.on("data", (d) => {
|
|
43
|
+
out += d.toString();
|
|
44
|
+
});
|
|
45
|
+
child.on("error", reject);
|
|
46
|
+
child.on(
|
|
47
|
+
"close",
|
|
48
|
+
(code) => code === 0 ? resolve(out.split(/\s+/).filter(Boolean)) : reject(new Error(`routes-cmd exited ${code}`))
|
|
49
|
+
);
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
function runShard(index, total, paths, opts, llmPartsDir) {
|
|
53
|
+
return new Promise((resolve) => {
|
|
54
|
+
const args = [
|
|
55
|
+
`--mode=${opts.mode}`,
|
|
56
|
+
`--shard=${index}/${total}`,
|
|
57
|
+
"--no-crawl",
|
|
58
|
+
`--llm-parts=${path.join(llmPartsDir, `part.${index}.json`)}`,
|
|
59
|
+
...opts.passthroughArgs,
|
|
60
|
+
...paths
|
|
61
|
+
];
|
|
62
|
+
const child = node_child_process.spawn("blox-ssg", args, {
|
|
63
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
64
|
+
env: {
|
|
65
|
+
...process.env,
|
|
66
|
+
// Divide the global prefetch budget across shards so N processes
|
|
67
|
+
// don't collectively overwhelm the API.
|
|
68
|
+
BLOX_SSG_PREFETCH: String(Math.max(2, Math.floor(PREFETCH_BUDGET / total)))
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
child.on("close", (code) => resolve(code ?? 0));
|
|
72
|
+
child.on("error", (err) => {
|
|
73
|
+
console.error(`[blox-ssg] shard ${index} failed to spawn: ${err.message}`);
|
|
74
|
+
resolve(1);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
async function runSharded(opts) {
|
|
79
|
+
const total = resolveShardCount(opts.shards);
|
|
80
|
+
const paths = [...opts.extraPaths];
|
|
81
|
+
if (opts.routesCmd) {
|
|
82
|
+
const enumerated = await enumerateRoutes(opts.routesCmd);
|
|
83
|
+
for (const p of enumerated) if (!paths.includes(p)) paths.push(p);
|
|
84
|
+
console.log(`[blox-ssg] routes-cmd enumerated ${enumerated.length} route(s)`);
|
|
85
|
+
}
|
|
86
|
+
console.log(`[blox-ssg] sharded run · ${total} shard(s) · ${paths.length} extra path(s)`);
|
|
87
|
+
const llmPartsDir = path.resolve("dist", ".llm-parts");
|
|
88
|
+
await (await import("node:fs/promises")).mkdir(llmPartsDir, { recursive: true }).catch(() => {
|
|
89
|
+
});
|
|
90
|
+
const start = Date.now();
|
|
91
|
+
const codes = total === 1 ? [await runShard(0, 1, paths, opts, llmPartsDir)] : await Promise.all(Array.from({ length: total }, (_, i) => runShard(i, total, paths, opts, llmPartsDir)));
|
|
92
|
+
const failed = codes.findIndex((c) => c !== 0);
|
|
93
|
+
if (failed !== -1) {
|
|
94
|
+
console.error(`[blox-ssg] shard ${failed} exited with code ${codes[failed]}`);
|
|
95
|
+
return codes[failed];
|
|
96
|
+
}
|
|
97
|
+
if (process.env.BLOX_SSG_LLM !== "0") {
|
|
98
|
+
try {
|
|
99
|
+
const [{ mergeLlmParts }, { fetchCmsSiteData }] = await Promise.all([
|
|
100
|
+
Promise.resolve().then(() => require("./prerender-DQ7D6auC.cjs")).then((n) => n.llmArtifacts),
|
|
101
|
+
Promise.resolve().then(() => require("./prerender-DQ7D6auC.cjs")).then((n) => n.cmsRoutes)
|
|
102
|
+
]);
|
|
103
|
+
let website = null;
|
|
104
|
+
try {
|
|
105
|
+
website = (await fetchCmsSiteData(opts.apiBase, opts.websiteName)).website;
|
|
106
|
+
} catch {
|
|
107
|
+
}
|
|
108
|
+
const partFiles = Array.from({ length: total }, (_, i) => path.join(llmPartsDir, `part.${i}.json`));
|
|
109
|
+
const { pages } = await mergeLlmParts({
|
|
110
|
+
partFiles,
|
|
111
|
+
outDir: path.resolve("dist"),
|
|
112
|
+
website,
|
|
113
|
+
config: { fullText: process.env.BLOX_SSG_LLM_FULL !== "0" }
|
|
114
|
+
});
|
|
115
|
+
console.log(`[blox-ssg] merged llms.txt (${pages} pages)`);
|
|
116
|
+
} catch (e) {
|
|
117
|
+
console.warn(`[blox-ssg] llms merge skipped: ${e.message}`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
console.log(`[blox-ssg] sharded run done in ${((Date.now() - start) / 1e3).toFixed(1)}s`);
|
|
121
|
+
return 0;
|
|
122
|
+
}
|
|
123
|
+
exports.runSharded = runSharded;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
const AUTO_SHARD_CAP = Number(process.env.BLOX_SSG_SHARD_CAP) || 6;
|
|
6
|
+
const PREFETCH_BUDGET = Number(process.env.BLOX_SSG_PREFETCH_BUDGET) || 24;
|
|
7
|
+
function resolveShardCount(shards) {
|
|
8
|
+
if (shards === "auto") {
|
|
9
|
+
const cores = os.cpus().length || 4;
|
|
10
|
+
return Math.max(1, Math.min(cores, AUTO_SHARD_CAP));
|
|
11
|
+
}
|
|
12
|
+
return Math.max(1, Math.floor(shards));
|
|
13
|
+
}
|
|
14
|
+
async function enumerateRoutes(cmd) {
|
|
15
|
+
return new Promise((resolve, reject) => {
|
|
16
|
+
const child = spawn(cmd, { shell: true, stdio: ["ignore", "pipe", "inherit"] });
|
|
17
|
+
let out = "";
|
|
18
|
+
child.stdout.on("data", (d) => {
|
|
19
|
+
out += d.toString();
|
|
20
|
+
});
|
|
21
|
+
child.on("error", reject);
|
|
22
|
+
child.on(
|
|
23
|
+
"close",
|
|
24
|
+
(code) => code === 0 ? resolve(out.split(/\s+/).filter(Boolean)) : reject(new Error(`routes-cmd exited ${code}`))
|
|
25
|
+
);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
function runShard(index, total, paths, opts, llmPartsDir) {
|
|
29
|
+
return new Promise((resolve) => {
|
|
30
|
+
const args = [
|
|
31
|
+
`--mode=${opts.mode}`,
|
|
32
|
+
`--shard=${index}/${total}`,
|
|
33
|
+
"--no-crawl",
|
|
34
|
+
`--llm-parts=${path.join(llmPartsDir, `part.${index}.json`)}`,
|
|
35
|
+
...opts.passthroughArgs,
|
|
36
|
+
...paths
|
|
37
|
+
];
|
|
38
|
+
const child = spawn("blox-ssg", args, {
|
|
39
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
40
|
+
env: {
|
|
41
|
+
...process.env,
|
|
42
|
+
// Divide the global prefetch budget across shards so N processes
|
|
43
|
+
// don't collectively overwhelm the API.
|
|
44
|
+
BLOX_SSG_PREFETCH: String(Math.max(2, Math.floor(PREFETCH_BUDGET / total)))
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
child.on("close", (code) => resolve(code ?? 0));
|
|
48
|
+
child.on("error", (err) => {
|
|
49
|
+
console.error(`[blox-ssg] shard ${index} failed to spawn: ${err.message}`);
|
|
50
|
+
resolve(1);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
async function runSharded(opts) {
|
|
55
|
+
const total = resolveShardCount(opts.shards);
|
|
56
|
+
const paths = [...opts.extraPaths];
|
|
57
|
+
if (opts.routesCmd) {
|
|
58
|
+
const enumerated = await enumerateRoutes(opts.routesCmd);
|
|
59
|
+
for (const p of enumerated) if (!paths.includes(p)) paths.push(p);
|
|
60
|
+
console.log(`[blox-ssg] routes-cmd enumerated ${enumerated.length} route(s)`);
|
|
61
|
+
}
|
|
62
|
+
console.log(`[blox-ssg] sharded run · ${total} shard(s) · ${paths.length} extra path(s)`);
|
|
63
|
+
const llmPartsDir = path.resolve("dist", ".llm-parts");
|
|
64
|
+
await (await import("node:fs/promises")).mkdir(llmPartsDir, { recursive: true }).catch(() => {
|
|
65
|
+
});
|
|
66
|
+
const start = Date.now();
|
|
67
|
+
const codes = total === 1 ? [await runShard(0, 1, paths, opts, llmPartsDir)] : await Promise.all(Array.from({ length: total }, (_, i) => runShard(i, total, paths, opts, llmPartsDir)));
|
|
68
|
+
const failed = codes.findIndex((c) => c !== 0);
|
|
69
|
+
if (failed !== -1) {
|
|
70
|
+
console.error(`[blox-ssg] shard ${failed} exited with code ${codes[failed]}`);
|
|
71
|
+
return codes[failed];
|
|
72
|
+
}
|
|
73
|
+
if (process.env.BLOX_SSG_LLM !== "0") {
|
|
74
|
+
try {
|
|
75
|
+
const [{ mergeLlmParts }, { fetchCmsSiteData }] = await Promise.all([
|
|
76
|
+
import("./prerender-BCRyYSNg.js").then((n) => n.r),
|
|
77
|
+
import("./prerender-BCRyYSNg.js").then((n) => n.q)
|
|
78
|
+
]);
|
|
79
|
+
let website = null;
|
|
80
|
+
try {
|
|
81
|
+
website = (await fetchCmsSiteData(opts.apiBase, opts.websiteName)).website;
|
|
82
|
+
} catch {
|
|
83
|
+
}
|
|
84
|
+
const partFiles = Array.from({ length: total }, (_, i) => path.join(llmPartsDir, `part.${i}.json`));
|
|
85
|
+
const { pages } = await mergeLlmParts({
|
|
86
|
+
partFiles,
|
|
87
|
+
outDir: path.resolve("dist"),
|
|
88
|
+
website,
|
|
89
|
+
config: { fullText: process.env.BLOX_SSG_LLM_FULL !== "0" }
|
|
90
|
+
});
|
|
91
|
+
console.log(`[blox-ssg] merged llms.txt (${pages} pages)`);
|
|
92
|
+
} catch (e) {
|
|
93
|
+
console.warn(`[blox-ssg] llms merge skipped: ${e.message}`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
console.log(`[blox-ssg] sharded run done in ${((Date.now() - start) / 1e3).toFixed(1)}s`);
|
|
97
|
+
return 0;
|
|
98
|
+
}
|
|
99
|
+
export {
|
|
100
|
+
runSharded
|
|
101
|
+
};
|
package/dist/ssg/cli.cjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
"use strict";
|
|
3
3
|
const process = require("node:process");
|
|
4
|
-
const
|
|
4
|
+
const cmsRoutes = require("../prerender-DQ7D6auC.cjs");
|
|
5
5
|
function installFetchTimeout() {
|
|
6
6
|
const ms = Number(process.env.BLOX_SSG_FETCH_TIMEOUT_MS) || 2e4;
|
|
7
7
|
const original = globalThis.fetch;
|
|
@@ -48,12 +48,21 @@ async function main() {
|
|
|
48
48
|
let shardIndex = 0;
|
|
49
49
|
let shardTotal = 1;
|
|
50
50
|
let llmPartsFile = "";
|
|
51
|
+
let shardsOpt = null;
|
|
52
|
+
let routesCmd = "";
|
|
53
|
+
const passthroughArgs = [];
|
|
51
54
|
const excludePaths = ["/_blox_preview"];
|
|
52
55
|
for (const a of argv) {
|
|
53
56
|
if (a === "--crawl") {
|
|
54
57
|
crawl = true;
|
|
58
|
+
passthroughArgs.push(a);
|
|
55
59
|
} else if (a === "--no-crawl") {
|
|
56
60
|
crawl = false;
|
|
61
|
+
} else if (a.startsWith("--shards=")) {
|
|
62
|
+
const v = a.slice("--shards=".length);
|
|
63
|
+
shardsOpt = v === "auto" ? "auto" : Number.isInteger(Number(v)) ? Number(v) : null;
|
|
64
|
+
} else if (a.startsWith("--routes-cmd=")) {
|
|
65
|
+
routesCmd = a.slice("--routes-cmd=".length);
|
|
57
66
|
} else if (a.startsWith("--shard=")) {
|
|
58
67
|
const [i, n] = a.slice("--shard=".length).split("/").map(Number);
|
|
59
68
|
if (Number.isInteger(i) && Number.isInteger(n) && n > 0 && i >= 0 && i < n) {
|
|
@@ -68,29 +77,54 @@ Usage:
|
|
|
68
77
|
blox-ssg [options] [extra-paths...]
|
|
69
78
|
|
|
70
79
|
Options:
|
|
71
|
-
--mode=<env>
|
|
72
|
-
--crawl
|
|
73
|
-
--no-crawl
|
|
74
|
-
--exclude=<path>
|
|
75
|
-
--
|
|
80
|
+
--mode=<env> Environment from bgl.config.ts (default: production)
|
|
81
|
+
--crawl Discover internal links from rendered HTML
|
|
82
|
+
--no-crawl Disable crawling (default)
|
|
83
|
+
--exclude=<path> Exclude path prefix from rendering
|
|
84
|
+
--shards=<n|auto> Render in N parallel child processes (auto = min(cores, cap)).
|
|
85
|
+
Avoids the single-process happy-dom leak on large sites and
|
|
86
|
+
parallelizes CPU-bound SSR. Merges each shard's llms parts.
|
|
87
|
+
--shard=<i/n> (internal) Render only this shard's slice — set by --shards.
|
|
88
|
+
--routes-cmd=<cmd> Shell command whose stdout (whitespace-separated) is the
|
|
89
|
+
route list. For router-driven sites with no CMS pages.
|
|
90
|
+
--help, -h Show this help
|
|
76
91
|
|
|
77
92
|
Environment:
|
|
78
|
-
BAGELINK_API_URL
|
|
79
|
-
WEBSITE_NAME
|
|
93
|
+
BAGELINK_API_URL Override API base URL
|
|
94
|
+
WEBSITE_NAME Override CMS website name
|
|
95
|
+
BLOX_SSG_SHARD_CAP Cap for --shards=auto (default 6)
|
|
96
|
+
BLOX_SSG_PREFETCH_BUDGET Global /resolve-path prefetch budget across shards (24)
|
|
80
97
|
`);
|
|
81
98
|
process.exit(0);
|
|
82
99
|
} else if (a.startsWith("--mode=")) {
|
|
83
100
|
mode = a.slice("--mode=".length);
|
|
84
101
|
} else if (a.startsWith("--exclude=")) {
|
|
85
102
|
excludePaths.push(a.slice("--exclude=".length));
|
|
103
|
+
passthroughArgs.push(a);
|
|
86
104
|
} else if (a.startsWith("--llm-parts=")) {
|
|
87
105
|
llmPartsFile = a.slice("--llm-parts=".length);
|
|
88
106
|
} else {
|
|
89
107
|
extraPaths.push(a);
|
|
90
108
|
}
|
|
91
109
|
}
|
|
110
|
+
if (shardsOpt !== null || routesCmd) {
|
|
111
|
+
const { apiBase: apiBase2, websiteName: configWebsiteName2 } = await resolveApiBase(mode);
|
|
112
|
+
const websiteName2 = process.env.WEBSITE_NAME ?? configWebsiteName2 ?? "default";
|
|
113
|
+
const { runSharded } = await Promise.resolve().then(() => require("../shard-runner-CZnX4qnn.cjs"));
|
|
114
|
+
const code = await runSharded({
|
|
115
|
+
shards: shardsOpt ?? "auto",
|
|
116
|
+
mode,
|
|
117
|
+
apiBase: apiBase2,
|
|
118
|
+
websiteName: websiteName2,
|
|
119
|
+
extraPaths,
|
|
120
|
+
passthroughArgs,
|
|
121
|
+
routesCmd: routesCmd || void 0
|
|
122
|
+
});
|
|
123
|
+
process.exitCode = code;
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
92
126
|
const { apiBase, websiteName: configWebsiteName } = await resolveApiBase(mode);
|
|
93
|
-
await
|
|
127
|
+
await cmsRoutes.polyfillBloxSsgGlobals({ pageUrl: apiBase });
|
|
94
128
|
const websiteName = process.env.WEBSITE_NAME ?? configWebsiteName ?? "default";
|
|
95
129
|
console.log(`Fetching routes from ${apiBase} (mode: ${mode}, site: ${websiteName})…`);
|
|
96
130
|
let paths = [];
|
|
@@ -99,7 +133,7 @@ Environment:
|
|
|
99
133
|
let redirects = [];
|
|
100
134
|
const collections = {};
|
|
101
135
|
try {
|
|
102
|
-
const siteData = await
|
|
136
|
+
const siteData = await cmsRoutes.fetchCmsSiteData(apiBase, websiteName);
|
|
103
137
|
paths = siteData.paths;
|
|
104
138
|
website = siteData.website;
|
|
105
139
|
websiteId = siteData.websiteId;
|
|
@@ -148,7 +182,7 @@ Environment:
|
|
|
148
182
|
}
|
|
149
183
|
const startTime = Date.now();
|
|
150
184
|
try {
|
|
151
|
-
const result = await
|
|
185
|
+
const result = await cmsRoutes.prerender({
|
|
152
186
|
root: process.cwd(),
|
|
153
187
|
clientOutDir: "dist",
|
|
154
188
|
serverEntry: "dist/server/main.server.js",
|
package/dist/ssg/cli.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import process from "node:process";
|
|
3
|
-
import { p as polyfillBloxSsgGlobals, f as fetchCmsSiteData, a as prerender } from "../prerender-
|
|
3
|
+
import { p as polyfillBloxSsgGlobals, f as fetchCmsSiteData, a as prerender } from "../prerender-BCRyYSNg.js";
|
|
4
4
|
function installFetchTimeout() {
|
|
5
5
|
const ms = Number(process.env.BLOX_SSG_FETCH_TIMEOUT_MS) || 2e4;
|
|
6
6
|
const original = globalThis.fetch;
|
|
@@ -47,12 +47,21 @@ async function main() {
|
|
|
47
47
|
let shardIndex = 0;
|
|
48
48
|
let shardTotal = 1;
|
|
49
49
|
let llmPartsFile = "";
|
|
50
|
+
let shardsOpt = null;
|
|
51
|
+
let routesCmd = "";
|
|
52
|
+
const passthroughArgs = [];
|
|
50
53
|
const excludePaths = ["/_blox_preview"];
|
|
51
54
|
for (const a of argv) {
|
|
52
55
|
if (a === "--crawl") {
|
|
53
56
|
crawl = true;
|
|
57
|
+
passthroughArgs.push(a);
|
|
54
58
|
} else if (a === "--no-crawl") {
|
|
55
59
|
crawl = false;
|
|
60
|
+
} else if (a.startsWith("--shards=")) {
|
|
61
|
+
const v = a.slice("--shards=".length);
|
|
62
|
+
shardsOpt = v === "auto" ? "auto" : Number.isInteger(Number(v)) ? Number(v) : null;
|
|
63
|
+
} else if (a.startsWith("--routes-cmd=")) {
|
|
64
|
+
routesCmd = a.slice("--routes-cmd=".length);
|
|
56
65
|
} else if (a.startsWith("--shard=")) {
|
|
57
66
|
const [i, n] = a.slice("--shard=".length).split("/").map(Number);
|
|
58
67
|
if (Number.isInteger(i) && Number.isInteger(n) && n > 0 && i >= 0 && i < n) {
|
|
@@ -67,27 +76,52 @@ Usage:
|
|
|
67
76
|
blox-ssg [options] [extra-paths...]
|
|
68
77
|
|
|
69
78
|
Options:
|
|
70
|
-
--mode=<env>
|
|
71
|
-
--crawl
|
|
72
|
-
--no-crawl
|
|
73
|
-
--exclude=<path>
|
|
74
|
-
--
|
|
79
|
+
--mode=<env> Environment from bgl.config.ts (default: production)
|
|
80
|
+
--crawl Discover internal links from rendered HTML
|
|
81
|
+
--no-crawl Disable crawling (default)
|
|
82
|
+
--exclude=<path> Exclude path prefix from rendering
|
|
83
|
+
--shards=<n|auto> Render in N parallel child processes (auto = min(cores, cap)).
|
|
84
|
+
Avoids the single-process happy-dom leak on large sites and
|
|
85
|
+
parallelizes CPU-bound SSR. Merges each shard's llms parts.
|
|
86
|
+
--shard=<i/n> (internal) Render only this shard's slice — set by --shards.
|
|
87
|
+
--routes-cmd=<cmd> Shell command whose stdout (whitespace-separated) is the
|
|
88
|
+
route list. For router-driven sites with no CMS pages.
|
|
89
|
+
--help, -h Show this help
|
|
75
90
|
|
|
76
91
|
Environment:
|
|
77
|
-
BAGELINK_API_URL
|
|
78
|
-
WEBSITE_NAME
|
|
92
|
+
BAGELINK_API_URL Override API base URL
|
|
93
|
+
WEBSITE_NAME Override CMS website name
|
|
94
|
+
BLOX_SSG_SHARD_CAP Cap for --shards=auto (default 6)
|
|
95
|
+
BLOX_SSG_PREFETCH_BUDGET Global /resolve-path prefetch budget across shards (24)
|
|
79
96
|
`);
|
|
80
97
|
process.exit(0);
|
|
81
98
|
} else if (a.startsWith("--mode=")) {
|
|
82
99
|
mode = a.slice("--mode=".length);
|
|
83
100
|
} else if (a.startsWith("--exclude=")) {
|
|
84
101
|
excludePaths.push(a.slice("--exclude=".length));
|
|
102
|
+
passthroughArgs.push(a);
|
|
85
103
|
} else if (a.startsWith("--llm-parts=")) {
|
|
86
104
|
llmPartsFile = a.slice("--llm-parts=".length);
|
|
87
105
|
} else {
|
|
88
106
|
extraPaths.push(a);
|
|
89
107
|
}
|
|
90
108
|
}
|
|
109
|
+
if (shardsOpt !== null || routesCmd) {
|
|
110
|
+
const { apiBase: apiBase2, websiteName: configWebsiteName2 } = await resolveApiBase(mode);
|
|
111
|
+
const websiteName2 = process.env.WEBSITE_NAME ?? configWebsiteName2 ?? "default";
|
|
112
|
+
const { runSharded } = await import("../shard-runner-D4gCfCi3.js");
|
|
113
|
+
const code = await runSharded({
|
|
114
|
+
shards: shardsOpt ?? "auto",
|
|
115
|
+
mode,
|
|
116
|
+
apiBase: apiBase2,
|
|
117
|
+
websiteName: websiteName2,
|
|
118
|
+
extraPaths,
|
|
119
|
+
passthroughArgs,
|
|
120
|
+
routesCmd: routesCmd || void 0
|
|
121
|
+
});
|
|
122
|
+
process.exitCode = code;
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
91
125
|
const { apiBase, websiteName: configWebsiteName } = await resolveApiBase(mode);
|
|
92
126
|
await polyfillBloxSsgGlobals({ pageUrl: apiBase });
|
|
93
127
|
const websiteName = process.env.WEBSITE_NAME ?? configWebsiteName ?? "default";
|
package/dist/ssg/index.cjs
CHANGED
|
@@ -22,7 +22,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
22
22
|
mod
|
|
23
23
|
));
|
|
24
24
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
25
|
-
const
|
|
25
|
+
const cmsRoutes = require("../prerender-DQ7D6auC.cjs");
|
|
26
26
|
const ssg_client = require("./client.cjs");
|
|
27
27
|
const pinia = require("pinia");
|
|
28
28
|
const vue = require("vue");
|
|
@@ -61,7 +61,7 @@ async function renderBloxSsgPage(options) {
|
|
|
61
61
|
const seoOverride = seoHead && Object.keys(seoHead).length > 0 ? seoHead : void 0;
|
|
62
62
|
const bloxData = dataRegistry && collectData ? collectData(dataRegistry) : {};
|
|
63
63
|
const dataScript = Object.keys(bloxData).length > 0 ? `<script>window[${JSON.stringify(routes.BLOX_DATA_WINDOW_KEY)}]=${safeJson(bloxData)};${"<"}/script>` : "";
|
|
64
|
-
let head =
|
|
64
|
+
let head = cmsRoutes.buildPageHead({
|
|
65
65
|
url,
|
|
66
66
|
resolvedData,
|
|
67
67
|
website,
|
|
@@ -70,9 +70,9 @@ async function renderBloxSsgPage(options) {
|
|
|
70
70
|
stateScript: [stateScript, collectionsScript, websiteIdScript, dataScript].filter(Boolean).join("\n")
|
|
71
71
|
});
|
|
72
72
|
const seoData = resolvedData;
|
|
73
|
-
const primary =
|
|
73
|
+
const primary = cmsRoutes.primaryContextEntry(seoData == null ? void 0 : seoData.contexts);
|
|
74
74
|
if ((llm == null ? void 0 : llm.jsonLd) !== false) {
|
|
75
|
-
const jsonLd =
|
|
75
|
+
const jsonLd = cmsRoutes.buildJsonLd({
|
|
76
76
|
url,
|
|
77
77
|
page: seoData == null ? void 0 : seoData.page,
|
|
78
78
|
website,
|
|
@@ -92,7 +92,7 @@ ${jsonLd}`;
|
|
|
92
92
|
url,
|
|
93
93
|
title,
|
|
94
94
|
description,
|
|
95
|
-
text: (llm == null ? void 0 : llm.fullText) !== false ?
|
|
95
|
+
text: (llm == null ? void 0 : llm.fullText) !== false ? cmsRoutes.extractPageText(html) : void 0
|
|
96
96
|
};
|
|
97
97
|
}
|
|
98
98
|
const lang = locale || (website == null ? void 0 : website.default_locale) || "en";
|
|
@@ -234,21 +234,21 @@ function createBloxSSREntry(options) {
|
|
|
234
234
|
}
|
|
235
235
|
return { render, prefetchResolve };
|
|
236
236
|
}
|
|
237
|
-
exports.buildJsonLd =
|
|
238
|
-
exports.buildPageHead =
|
|
239
|
-
exports.buildSiteHead =
|
|
240
|
-
exports.extractPageText =
|
|
241
|
-
exports.fetchCmsPrerenderPaths =
|
|
242
|
-
exports.fetchCmsSiteData =
|
|
243
|
-
exports.generateLlmsFullTxt =
|
|
244
|
-
exports.generateLlmsTxt =
|
|
245
|
-
exports.generateNetlifyRedirects =
|
|
246
|
-
exports.generateRobotsTxt =
|
|
247
|
-
exports.generateSitemapXml =
|
|
248
|
-
exports.getSiteIconFlags =
|
|
249
|
-
exports.mergeLlmParts =
|
|
250
|
-
exports.polyfillBloxSsgGlobals =
|
|
251
|
-
exports.prerender =
|
|
237
|
+
exports.buildJsonLd = cmsRoutes.buildJsonLd;
|
|
238
|
+
exports.buildPageHead = cmsRoutes.buildPageHead;
|
|
239
|
+
exports.buildSiteHead = cmsRoutes.buildSiteHead;
|
|
240
|
+
exports.extractPageText = cmsRoutes.extractPageText;
|
|
241
|
+
exports.fetchCmsPrerenderPaths = cmsRoutes.fetchCmsPrerenderPaths;
|
|
242
|
+
exports.fetchCmsSiteData = cmsRoutes.fetchCmsSiteData;
|
|
243
|
+
exports.generateLlmsFullTxt = cmsRoutes.generateLlmsFullTxt;
|
|
244
|
+
exports.generateLlmsTxt = cmsRoutes.generateLlmsTxt;
|
|
245
|
+
exports.generateNetlifyRedirects = cmsRoutes.generateNetlifyRedirects;
|
|
246
|
+
exports.generateRobotsTxt = cmsRoutes.generateRobotsTxt;
|
|
247
|
+
exports.generateSitemapXml = cmsRoutes.generateSitemapXml;
|
|
248
|
+
exports.getSiteIconFlags = cmsRoutes.getSiteIconFlags;
|
|
249
|
+
exports.mergeLlmParts = cmsRoutes.mergeLlmParts;
|
|
250
|
+
exports.polyfillBloxSsgGlobals = cmsRoutes.polyfillBloxSsgGlobals;
|
|
251
|
+
exports.prerender = cmsRoutes.prerender;
|
|
252
252
|
exports.BLOX_COLLECTIONS_WINDOW_KEY = ssg_client.BLOX_COLLECTIONS_WINDOW_KEY;
|
|
253
253
|
exports.BLOX_DATA_WINDOW_KEY = ssg_client.BLOX_DATA_WINDOW_KEY;
|
|
254
254
|
exports.BLOX_PAGE_SEO_WINDOW_KEY = ssg_client.BLOX_PAGE_SEO_WINDOW_KEY;
|
package/dist/ssg/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { b as buildPageHead, c as primaryContextEntry, d as buildJsonLd, e as extractPageText } from "../prerender-
|
|
2
|
-
import { g, h, f, i, j, k, l, m, n, o, p, a } from "../prerender-
|
|
1
|
+
import { b as buildPageHead, c as primaryContextEntry, d as buildJsonLd, e as extractPageText } from "../prerender-BCRyYSNg.js";
|
|
2
|
+
import { g, h, f, i, j, k, l, m, n, o, p, a } from "../prerender-BCRyYSNg.js";
|
|
3
3
|
import { BLOX_STATE_WINDOW_KEY, BLOX_COLLECTIONS_WINDOW_KEY, BLOX_WEBSITE_ID_WINDOW_KEY } from "./client.mjs";
|
|
4
4
|
import { BLOX_DATA_WINDOW_KEY, BLOX_PAGE_SEO_WINDOW_KEY, getCollectionCache, getEmbeddedWebsiteId, installBloxStateCache, installCollectionCache } from "./client.mjs";
|
|
5
5
|
import { createPinia } from "pinia";
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in sharded SSG runner (coordinator).
|
|
3
|
+
*
|
|
4
|
+
* Vue SSR (`renderToString`) is CPU-bound and single-threaded, and a single
|
|
5
|
+
* long-lived `blox-ssg` process also accumulates happy-dom/window/timer state
|
|
6
|
+
* under bun — on large sites (thousands of pages) that leak eventually crashes
|
|
7
|
+
* the process silently. Splitting the work across N short-lived child processes
|
|
8
|
+
* (each rendering a disjoint `--shard=i/n` slice into the same `dist`) both
|
|
9
|
+
* parallelizes the CPU-bound render AND keeps each process well under the leak
|
|
10
|
+
* threshold.
|
|
11
|
+
*
|
|
12
|
+
* This module is the coordinator: it enumerates the route list once (optionally
|
|
13
|
+
* via an external `--routes-cmd` for router-driven sites that don't use CMS page
|
|
14
|
+
* discovery), then spawns the shards, divides the resolve-prefetch budget across
|
|
15
|
+
* them, and merges each shard's llms part file into the final llms.txt.
|
|
16
|
+
*
|
|
17
|
+
* Activated by `--shards=<n|auto>` on the CLI. Without it, the CLI runs a single
|
|
18
|
+
* in-process render exactly as before (full backward compat).
|
|
19
|
+
*/
|
|
20
|
+
export interface ShardRunnerOptions {
|
|
21
|
+
/** Number of shards, or 'auto' → min(cores, cap). */
|
|
22
|
+
shards: number | 'auto';
|
|
23
|
+
/** CLI mode (passed through to each shard as --mode=). */
|
|
24
|
+
mode: string;
|
|
25
|
+
/** Resolved API base (for the post-run llms merge). */
|
|
26
|
+
apiBase: string;
|
|
27
|
+
/** CMS website name (for the post-run llms merge). */
|
|
28
|
+
websiteName: string;
|
|
29
|
+
/** Extra paths already parsed from argv (router-driven sites, static routes). */
|
|
30
|
+
extraPaths: string[];
|
|
31
|
+
/** Other pass-through flags to forward verbatim to each shard (e.g. --exclude=…, --crawl). */
|
|
32
|
+
passthroughArgs: string[];
|
|
33
|
+
/**
|
|
34
|
+
* Optional shell command whose stdout (whitespace-separated) is a list of
|
|
35
|
+
* routes to render. Run once by the coordinator; the paths are forwarded to
|
|
36
|
+
* every shard. Lets router-driven sites (no CMS page discovery) enumerate
|
|
37
|
+
* their own routes without a bespoke runner script.
|
|
38
|
+
*/
|
|
39
|
+
routesCmd?: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Coordinate a sharded prerender. Returns the process exit code.
|
|
43
|
+
*/
|
|
44
|
+
export declare function runSharded(opts: ShardRunnerOptions): Promise<number>;
|
|
45
|
+
//# sourceMappingURL=shard-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard-runner.d.ts","sourceRoot":"","sources":["../../src/ssg/shard-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AASH,MAAM,WAAW,kBAAkB;IAClC,qDAAqD;IACrD,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;IACvB,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAA;IACZ,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAA;IACf,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAA;IACnB,iFAAiF;IACjF,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,8FAA8F;IAC9F,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAyDD;;GAEG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0D1E"}
|
package/package.json
CHANGED