@johnhenry/sitefile 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/LICENSE +21 -0
- package/README.md +475 -2
- package/lib/src/archive-writers.d.ts +31 -0
- package/lib/src/archive-writers.js +197 -0
- package/lib/src/attest.d.ts +126 -0
- package/lib/src/attest.js +220 -0
- package/lib/src/bake.d.ts +74 -0
- package/lib/src/bake.js +372 -0
- package/lib/src/base-image.d.ts +76 -0
- package/lib/src/base-image.js +138 -0
- package/lib/src/build-args.d.ts +26 -0
- package/lib/src/build-args.js +164 -0
- package/lib/src/cache-limit.d.ts +54 -0
- package/lib/src/cache-limit.js +239 -0
- package/lib/src/casefold-data.d.ts +2 -0
- package/lib/src/casefold-data.js +217 -0
- package/lib/src/cbor.d.ts +39 -0
- package/lib/src/cbor.js +174 -0
- package/lib/src/checklinks.d.ts +42 -0
- package/lib/src/checklinks.js +118 -0
- package/lib/src/cli/bake.d.ts +2 -0
- package/lib/src/cli/bake.js +179 -0
- package/lib/src/cli/build-image.d.ts +21 -0
- package/lib/src/cli/build-image.js +42 -0
- package/lib/src/cli/cache.d.ts +8 -0
- package/lib/src/cli/cache.js +109 -0
- package/lib/src/cli/common.d.ts +29 -0
- package/lib/src/cli/common.js +100 -0
- package/lib/src/cli/git-context.d.ts +18 -0
- package/lib/src/cli/git-context.js +48 -0
- package/lib/src/cli/images.d.ts +6 -0
- package/lib/src/cli/images.js +285 -0
- package/lib/src/cli/index.d.ts +10 -0
- package/lib/src/cli/index.js +95 -0
- package/lib/src/cli/outdated.d.ts +5 -0
- package/lib/src/cli/outdated.js +94 -0
- package/lib/src/cli/registry.d.ts +2 -0
- package/lib/src/cli/registry.js +122 -0
- package/lib/src/cli/serve-worker.d.ts +2 -0
- package/lib/src/cli/serve-worker.js +42 -0
- package/lib/src/cli/servers.d.ts +15 -0
- package/lib/src/cli/servers.js +451 -0
- package/lib/src/cli/vendor.d.ts +2 -0
- package/lib/src/cli/vendor.js +92 -0
- package/lib/src/cli/watch.d.ts +26 -0
- package/lib/src/cli/watch.js +81 -0
- package/lib/src/cli.d.ts +12 -0
- package/lib/src/cli.js +486 -0
- package/lib/src/csv.d.ts +27 -0
- package/lib/src/csv.js +117 -0
- package/lib/src/deflate.d.ts +76 -0
- package/lib/src/deflate.js +604 -0
- package/lib/src/digest.d.ts +3 -0
- package/lib/src/digest.js +7 -0
- package/lib/src/errors.d.ts +34 -0
- package/lib/src/errors.js +20 -0
- package/lib/src/executor.d.ts +386 -0
- package/lib/src/executor.js +2193 -0
- package/lib/src/explain.d.ts +64 -0
- package/lib/src/explain.js +445 -0
- package/lib/src/export-formats.d.ts +16 -0
- package/lib/src/export-formats.js +52 -0
- package/lib/src/exporter.d.ts +103 -0
- package/lib/src/exporter.js +676 -0
- package/lib/src/fingerprint.d.ts +40 -0
- package/lib/src/fingerprint.js +168 -0
- package/lib/src/frontmatter.d.ts +22 -0
- package/lib/src/frontmatter.js +45 -0
- package/lib/src/index.d.ts +12 -0
- package/lib/src/index.js +12 -0
- package/lib/src/language/ast.d.ts +353 -0
- package/lib/src/language/ast.js +57 -0
- package/lib/src/language/parser.d.ts +3 -0
- package/lib/src/language/parser.js +612 -0
- package/lib/src/language/scanner.d.ts +110 -0
- package/lib/src/language/scanner.js +449 -0
- package/lib/src/language/validate.d.ts +13 -0
- package/lib/src/language/validate.js +367 -0
- package/lib/src/links.d.ts +79 -0
- package/lib/src/links.js +583 -0
- package/lib/src/lint.d.ts +10 -0
- package/lib/src/lint.js +223 -0
- package/lib/src/markdown.d.ts +29 -0
- package/lib/src/markdown.js +612 -0
- package/lib/src/media.d.ts +82 -0
- package/lib/src/media.js +650 -0
- package/lib/src/memo.d.ts +57 -0
- package/lib/src/memo.js +93 -0
- package/lib/src/oci.d.ts +178 -0
- package/lib/src/oci.js +380 -0
- package/lib/src/outdated.d.ts +46 -0
- package/lib/src/outdated.js +195 -0
- package/lib/src/pages.d.ts +10 -0
- package/lib/src/pages.js +54 -0
- package/lib/src/paths.d.ts +39 -0
- package/lib/src/paths.js +101 -0
- package/lib/src/planner.d.ts +297 -0
- package/lib/src/planner.js +410 -0
- package/lib/src/plugins/cache.d.ts +70 -0
- package/lib/src/plugins/cache.js +226 -0
- package/lib/src/plugins/exec.d.ts +51 -0
- package/lib/src/plugins/exec.js +144 -0
- package/lib/src/plugins/host.d.ts +75 -0
- package/lib/src/plugins/host.js +277 -0
- package/lib/src/plugins/wasm.d.ts +23 -0
- package/lib/src/plugins/wasm.js +249 -0
- package/lib/src/profile.d.ts +42 -0
- package/lib/src/profile.js +68 -0
- package/lib/src/ref.d.ts +40 -0
- package/lib/src/ref.js +79 -0
- package/lib/src/registry-auth.d.ts +32 -0
- package/lib/src/registry-auth.js +151 -0
- package/lib/src/registry-cache.d.ts +71 -0
- package/lib/src/registry-cache.js +333 -0
- package/lib/src/registry-ops.d.ts +51 -0
- package/lib/src/registry-ops.js +248 -0
- package/lib/src/registry.d.ts +109 -0
- package/lib/src/registry.js +505 -0
- package/lib/src/render.d.ts +134 -0
- package/lib/src/render.js +576 -0
- package/lib/src/resources/archive.d.ts +11 -0
- package/lib/src/resources/archive.js +306 -0
- package/lib/src/resources/bundle-data.d.ts +16 -0
- package/lib/src/resources/bundle-data.js +50 -0
- package/lib/src/resources/convert.d.ts +12 -0
- package/lib/src/resources/convert.js +63 -0
- package/lib/src/resources/documents.d.ts +35 -0
- package/lib/src/resources/documents.js +122 -0
- package/lib/src/resources/git.d.ts +103 -0
- package/lib/src/resources/git.js +720 -0
- package/lib/src/resources/https.d.ts +40 -0
- package/lib/src/resources/https.js +256 -0
- package/lib/src/resources/ignore.d.ts +33 -0
- package/lib/src/resources/ignore.js +119 -0
- package/lib/src/resources/local.d.ts +64 -0
- package/lib/src/resources/local.js +248 -0
- package/lib/src/resources/lockfile.d.ts +52 -0
- package/lib/src/resources/lockfile.js +172 -0
- package/lib/src/resources/netpolicy.d.ts +6 -0
- package/lib/src/resources/netpolicy.js +82 -0
- package/lib/src/resources/types.d.ts +63 -0
- package/lib/src/resources/types.js +13 -0
- package/lib/src/resources/vendor.d.ts +94 -0
- package/lib/src/resources/vendor.js +296 -0
- package/lib/src/resources/wbn-reader.d.ts +44 -0
- package/lib/src/resources/wbn-reader.js +302 -0
- package/lib/src/schema.d.ts +25 -0
- package/lib/src/schema.js +169 -0
- package/lib/src/secrets.d.ts +21 -0
- package/lib/src/secrets.js +103 -0
- package/lib/src/serve-rules.d.ts +29 -0
- package/lib/src/serve-rules.js +122 -0
- package/lib/src/server.d.ts +45 -0
- package/lib/src/server.js +246 -0
- package/lib/src/servers.d.ts +71 -0
- package/lib/src/servers.js +139 -0
- package/lib/src/site.d.ts +60 -0
- package/lib/src/site.js +169 -0
- package/lib/src/store.d.ts +151 -0
- package/lib/src/store.js +596 -0
- package/lib/src/tree.d.ts +50 -0
- package/lib/src/tree.js +134 -0
- package/lib/src/values.d.ts +7 -0
- package/lib/src/values.js +37 -0
- package/lib/src/vendor.d.ts +19 -0
- package/lib/src/vendor.js +145 -0
- package/lib/src/watch.d.ts +44 -0
- package/lib/src/watch.js +137 -0
- package/lib/src/wbn.d.ts +5 -0
- package/lib/src/wbn.js +147 -0
- package/lib/src/yaml.d.ts +34 -0
- package/lib/src/yaml.js +562 -0
- package/package.json +58 -4
|
@@ -0,0 +1,2193 @@
|
|
|
1
|
+
import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { statSync } from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { SitefileError } from './errors.js';
|
|
6
|
+
import { sha256 } from './digest.js';
|
|
7
|
+
import { FEED_ITEM } from './language/ast.js';
|
|
8
|
+
import { fragmentText, parseProgram } from './language/validate.js';
|
|
9
|
+
import { joinVirtual, basenameVirtual, normalizeVirtual } from './paths.js';
|
|
10
|
+
import { importedStageRefs, plan, targetPlan } from './planner.js';
|
|
11
|
+
import { checkWasmPlugin } from './plugins/wasm.js';
|
|
12
|
+
import { DEFAULT_PLUGIN_POLICY, PluginHost, TRANSFORM_JOBS } from './plugins/host.js';
|
|
13
|
+
import { cacheKey, digestFolder, TransformCache } from './plugins/cache.js';
|
|
14
|
+
import { DEFAULT_TOOL_POLICY, resolveTool, runTool } from './plugins/exec.js';
|
|
15
|
+
import { decodeUtf8, fileRefs, fragmentNotValue, parseRenderTemplate, renderTemplate, resolveJsonPath, walk } from './render.js';
|
|
16
|
+
import { dataCollection, markdownCollection, sortDocuments } from './resources/documents.js';
|
|
17
|
+
import { bytesToResource, verifyIntegrity } from './resources/convert.js';
|
|
18
|
+
import { fetchRemote } from './resources/https.js';
|
|
19
|
+
import { redactError, redactSecrets, secretHeaders } from './secrets.js';
|
|
20
|
+
import { fetchGitCommit, pinnedGitUrl, selectGitPath } from './resources/git.js';
|
|
21
|
+
import { extractArchive } from './resources/archive.js';
|
|
22
|
+
import { writeTar } from './archive-writers.js';
|
|
23
|
+
import { LocalContext } from './resources/local.js';
|
|
24
|
+
import { Lockfile } from './resources/lockfile.js';
|
|
25
|
+
import { findVendorDir, Vendor } from './resources/vendor.js';
|
|
26
|
+
import { DEFAULT_ARCHIVE_LIMITS, DEFAULT_FETCH_POLICY } from './resources/types.js';
|
|
27
|
+
import { makeFile, Tree } from './tree.js';
|
|
28
|
+
import { IgnoreRules } from './resources/ignore.js';
|
|
29
|
+
import { lint } from './lint.js';
|
|
30
|
+
import { exportDirectory, exportFile, isExporterTempName } from './exporter.js';
|
|
31
|
+
import { sourceDateEpoch } from './archive-writers.js';
|
|
32
|
+
import { encodeTree, isArchiveFormat } from './export-formats.js';
|
|
33
|
+
import { buildFacts, makeAttestations } from './attest.js';
|
|
34
|
+
import { ANN, buildOciImage, imageMeta, layoutTree, ociArchive, parseConfig } from './oci.js';
|
|
35
|
+
import { baseRef as parseBaseRef, readLayerBlobs, resolveBase, resolveFromImage } from './base-image.js';
|
|
36
|
+
import { Store } from './store.js';
|
|
37
|
+
import { begin, reserve } from './profile.js';
|
|
38
|
+
import { BuildMemo, memoKey, valueKey } from './memo.js';
|
|
39
|
+
import { checkSchema, compileSchema } from './schema.js';
|
|
40
|
+
import { groupBy, paginate } from './pages.js';
|
|
41
|
+
import { parseYaml } from './yaml.js';
|
|
42
|
+
import { brokenLinks, reportBrokenLinks } from './checklinks.js';
|
|
43
|
+
import { composeRenames, fingerprint } from './fingerprint.js';
|
|
44
|
+
import { atomXml, headersFile, pageUrl, parseFeedDate, redirectsFile, sitemapXml } from './site.js';
|
|
45
|
+
/** Read, parse and validate a Sitefile. */
|
|
46
|
+
export async function loadProgram(options) {
|
|
47
|
+
const sitefilePath = path.resolve(options.sitefile ?? path.join(options.contextDir, 'Sitefile'));
|
|
48
|
+
let source;
|
|
49
|
+
try {
|
|
50
|
+
source = await readFile(sitefilePath, 'utf8');
|
|
51
|
+
}
|
|
52
|
+
catch (e) {
|
|
53
|
+
// A Git context names its Sitefile by the label (the checkout is a temporary directory).
|
|
54
|
+
throw new SitefileError('E_NOT_FOUND', `cannot read Sitefile '${options.label ?? sitefilePath}'`, undefined, { cause: e });
|
|
55
|
+
}
|
|
56
|
+
const label = options.label ?? (path.relative(process.cwd(), sitefilePath) || path.basename(sitefilePath));
|
|
57
|
+
return { program: parseProgram(source, label), sitefilePath, label, source };
|
|
58
|
+
}
|
|
59
|
+
/** Parse, validate and evaluate variables, without reading any sources. */
|
|
60
|
+
export async function planBuild(options) {
|
|
61
|
+
const { program } = await loadProgram(options);
|
|
62
|
+
return planFor(program, options);
|
|
63
|
+
}
|
|
64
|
+
/** `validate`'s lint warnings (src/lint.ts) for the whole Sitefile (a `--target` does not narrow them). */
|
|
65
|
+
export async function lintBuild(options, disabled = []) {
|
|
66
|
+
const { program, source } = await loadProgram(options);
|
|
67
|
+
return lint(program, plan(program, options.buildArgs ?? {}), source, disabled);
|
|
68
|
+
}
|
|
69
|
+
/** The plan of a program for these options (`target` applied). */
|
|
70
|
+
function planFor(program, options) {
|
|
71
|
+
const p = plan(program, options.buildArgs ?? {});
|
|
72
|
+
return options.target === undefined ? p : targetPlan(p, options.target);
|
|
73
|
+
}
|
|
74
|
+
/** The lockfile a build would use: `Sitefile.lock` next to the Sitefile, unless disabled. */
|
|
75
|
+
function lockPathFor(options, sitefilePath) {
|
|
76
|
+
return options.lockfile === false ? undefined : (options.lockfile ?? path.join(path.dirname(sitefilePath), 'Sitefile.lock'));
|
|
77
|
+
}
|
|
78
|
+
/** `validate --strict-pins`: check the pins of `p` against the lockfile without building or fetching. */
|
|
79
|
+
export async function checkPlanPins(options, p) {
|
|
80
|
+
const sitefilePath = path.resolve(options.sitefile ?? path.join(options.contextDir, 'Sitefile'));
|
|
81
|
+
const lockPath = lockPathFor(options, sitefilePath);
|
|
82
|
+
const lock = lockPath ? await Lockfile.load(lockPath) : undefined;
|
|
83
|
+
void checkPins(p, lock, options.skipBase ? undefined : (options.baseImage ?? (p.exportFormat === 'oci' ? p.exportBase : undefined)));
|
|
84
|
+
}
|
|
85
|
+
/** Run a build in memory. */
|
|
86
|
+
export async function build(options) {
|
|
87
|
+
// Build secrets never appear in a warning or an error, even when a server echoes one back.
|
|
88
|
+
if (options.secrets?.size) {
|
|
89
|
+
const secrets = options.secrets;
|
|
90
|
+
const onWarning = options.onWarning;
|
|
91
|
+
const redacted = { ...options, ...(onWarning ? { onWarning: (m) => onWarning(redactSecrets(m, secrets)) } : {}) };
|
|
92
|
+
try {
|
|
93
|
+
const r = await buildUnredacted(redacted);
|
|
94
|
+
return { ...r, warnings: r.warnings.map((w) => redactSecrets(w, secrets)) };
|
|
95
|
+
}
|
|
96
|
+
catch (e) {
|
|
97
|
+
throw redactError(e, secrets);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return buildUnredacted(options);
|
|
101
|
+
}
|
|
102
|
+
async function buildUnredacted(options) {
|
|
103
|
+
const memo = options.memo;
|
|
104
|
+
if (!memo)
|
|
105
|
+
return buildOnce(options);
|
|
106
|
+
memo.begin();
|
|
107
|
+
let ok = false;
|
|
108
|
+
try {
|
|
109
|
+
const r = await buildOnce(options);
|
|
110
|
+
ok = true;
|
|
111
|
+
return { ...r, memoStats: { ...memo.stats } };
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
memo.finish(ok);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
async function buildOnce(options) {
|
|
118
|
+
// Git imports AS sitefile are checked out into temporary directories, removed when the build ends.
|
|
119
|
+
const tempDirs = [];
|
|
120
|
+
try {
|
|
121
|
+
return await buildWith(options, tempDirs);
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
await Promise.all(tempDirs.map((d) => rm(d, { recursive: true, force: true })));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
async function buildWith(options, tempDirs) {
|
|
128
|
+
const prof = options.profiler;
|
|
129
|
+
const parseSpan = begin(prof, 'parse', 'build');
|
|
130
|
+
const { program, sitefilePath, label } = await loadProgram(options);
|
|
131
|
+
const full = plan(program, options.buildArgs ?? {});
|
|
132
|
+
const p = options.target === undefined ? full : targetPlan(full, options.target);
|
|
133
|
+
parseSpan.end({ imports: p.imports.length, stages: p.stages.length, ...(p.target !== undefined ? { target: p.target } : {}) });
|
|
134
|
+
const warnings = [];
|
|
135
|
+
const warn = (m) => {
|
|
136
|
+
warnings.push(m);
|
|
137
|
+
options.onWarning?.(m);
|
|
138
|
+
};
|
|
139
|
+
p.warnings.forEach(warn);
|
|
140
|
+
warnUnusedSecrets(p, options.secrets, warn);
|
|
141
|
+
const tools = await resolveTools(p, options, warn);
|
|
142
|
+
const pluginAllowed = checkPlugins(p, options, warn);
|
|
143
|
+
const contextReal = await realpath(options.contextDir).catch((e) => {
|
|
144
|
+
throw new SitefileError('E_NOT_FOUND', `build context '${options.contextDir}' does not exist`, undefined, { cause: e });
|
|
145
|
+
});
|
|
146
|
+
const ignore = ['.git'];
|
|
147
|
+
/** The same paths, absolute: an imported Sitefile's context inside this one skips them too. */
|
|
148
|
+
const skip = [];
|
|
149
|
+
if (options.outDir !== undefined) {
|
|
150
|
+
const outReal = await realpathLoose(path.resolve(options.outDir));
|
|
151
|
+
const rel = path.relative(contextReal, outReal);
|
|
152
|
+
if (rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel)) {
|
|
153
|
+
ignore.push(rel.split(path.sep).join('/'));
|
|
154
|
+
skip.push(outReal);
|
|
155
|
+
// Also skip temp/old siblings an interrupted export may have left behind.
|
|
156
|
+
const parent = path.dirname(outReal);
|
|
157
|
+
const siblings = await readdir(parent).catch(() => []);
|
|
158
|
+
for (const name of siblings) {
|
|
159
|
+
if (isExporterTempName(name, path.basename(outReal))) {
|
|
160
|
+
ignore.push(path.relative(contextReal, path.join(parent, name)).split(path.sep).join('/'));
|
|
161
|
+
skip.push(path.join(parent, name));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const vendorDir = await findVendorDir(contextReal, options.vendorDir);
|
|
167
|
+
if (vendorDir !== undefined) {
|
|
168
|
+
ignore.push(vendorDir);
|
|
169
|
+
skip.push(path.join(contextReal, ...vendorDir.split('/')));
|
|
170
|
+
}
|
|
171
|
+
for (const w of options.alsoWrites ?? []) {
|
|
172
|
+
const real = await realpathLoose(path.resolve(w));
|
|
173
|
+
const rel = path.relative(contextReal, real);
|
|
174
|
+
if (rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel)) {
|
|
175
|
+
ignore.push(rel.split(path.sep).join('/'));
|
|
176
|
+
skip.push(real);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const local = await LocalContext.open(contextReal, { ignore, allowHardlinks: options.allowHardlinks ?? false });
|
|
180
|
+
const vendor = vendorDir !== undefined ? await Vendor.open(local, vendorDir) : undefined;
|
|
181
|
+
const policy = { ...DEFAULT_FETCH_POLICY, ...options.fetchPolicy };
|
|
182
|
+
const archiveLimits = { ...DEFAULT_ARCHIVE_LIMITS, ...options.archiveLimits };
|
|
183
|
+
const lockPath = lockPathFor(options, sitefilePath);
|
|
184
|
+
const lock = lockPath ? await Lockfile.load(lockPath) : undefined;
|
|
185
|
+
// BASE: the image to build on (resolved after the files, below).
|
|
186
|
+
const baseRef = options.skipBase ? undefined : (options.baseImage ?? (p.exportFormat === 'oci' ? p.exportBase : undefined));
|
|
187
|
+
const pins = options.strictPins ? checkPins(p, lock, baseRef) : undefined;
|
|
188
|
+
// A --target build skips later stages; their FROM image lockfile entries are kept all the same.
|
|
189
|
+
for (const st of full.stages.slice(p.stages.length))
|
|
190
|
+
if (st.base.kind === 'image' && !st.base.local)
|
|
191
|
+
lock?.markImageUsed(imageLockKey(st.base));
|
|
192
|
+
const memo = options.memo;
|
|
193
|
+
const env = {
|
|
194
|
+
local,
|
|
195
|
+
policy,
|
|
196
|
+
archiveLimits,
|
|
197
|
+
lock,
|
|
198
|
+
label,
|
|
199
|
+
warn,
|
|
200
|
+
frozen: !!options.frozenLockfile,
|
|
201
|
+
update: !!options.updateLock,
|
|
202
|
+
vendor,
|
|
203
|
+
...(options.secrets ? { secrets: options.secrets } : {}),
|
|
204
|
+
...(memo ? { memo } : {}),
|
|
205
|
+
};
|
|
206
|
+
// Imports resolve concurrently: each gets its own trace thread, reserved in Sitefile order.
|
|
207
|
+
const importSpans = p.imports.map((spec, i) => {
|
|
208
|
+
prof?.nameThread(i + 2, `import ${spec.name}`);
|
|
209
|
+
return prof ? prof.reserve(`import ${spec.name}`, 'import', { uri: spec.uri, as: spec.as ?? 'file' }, i + 2) : undefined;
|
|
210
|
+
});
|
|
211
|
+
// Sitefile imports (AS sitefile) load after the others, one after the other, in Sitefile order.
|
|
212
|
+
const indexed = p.imports.map((spec, i) => [spec, i]);
|
|
213
|
+
const plainImports = indexed.filter(([s]) => s.as !== 'sitefile');
|
|
214
|
+
const plainResolved = await mapLimit(plainImports, options.concurrency ?? 4, async ([spec, i]) => {
|
|
215
|
+
importSpans[i]?.start();
|
|
216
|
+
const r = await resolveImport(spec, env);
|
|
217
|
+
importSpans[i]?.end({ kind: r.res.kind });
|
|
218
|
+
return r;
|
|
219
|
+
});
|
|
220
|
+
// DRAFTS false drops drafts; then SCHEMA checks run in Sitefile order, on the documents in path order; then SORT BY.
|
|
221
|
+
const resolved = new Map(plainImports.map(([s], j) => [s.name, { ...plainResolved[j], res: withoutDrafts(s, plainResolved[j].res) }]));
|
|
222
|
+
for (const spec of p.imports) {
|
|
223
|
+
const r = resolved.get(spec.name);
|
|
224
|
+
if (r && spec.schema)
|
|
225
|
+
await checkImportSchema(spec, r.res, local);
|
|
226
|
+
}
|
|
227
|
+
const images = [];
|
|
228
|
+
let imageStore;
|
|
229
|
+
const fromImage = async (b, unit) => {
|
|
230
|
+
imageStore ??= options.store ?? (await Store.open());
|
|
231
|
+
const r = await resolveFromImage(b.ref, b.at, b.local, {
|
|
232
|
+
store: imageStore,
|
|
233
|
+
platform: options.platform,
|
|
234
|
+
lock,
|
|
235
|
+
frozen: !!options.frozenLockfile,
|
|
236
|
+
update: !!options.updateLock,
|
|
237
|
+
vendor,
|
|
238
|
+
registry: {
|
|
239
|
+
...options.registry,
|
|
240
|
+
allowPrivateNetwork: options.registry?.allowPrivateNetwork ?? policy.allowPrivateNetwork,
|
|
241
|
+
},
|
|
242
|
+
warn,
|
|
243
|
+
span: b.span,
|
|
244
|
+
lead: `${b.lead} image`,
|
|
245
|
+
});
|
|
246
|
+
images.push({ name: r.name, digest: r.digest, line: b.span.line, local: b.local, ...(unit !== undefined ? { import: unit } : {}) });
|
|
247
|
+
return r;
|
|
248
|
+
};
|
|
249
|
+
const plugins = new PluginHost({ ...DEFAULT_PLUGIN_POLICY, ...options.pluginPolicy }, warn);
|
|
250
|
+
const cache = options.cacheDir !== undefined ? new TransformCache(options.cacheDir, options.remoteCache) : undefined;
|
|
251
|
+
const stats = { hits: 0, misses: 0, writes: 0, corrupt: 0, pluginsSpawned: 0, pluginCalls: 0, toolRuns: 0 };
|
|
252
|
+
const toolPolicy = { ...DEFAULT_TOOL_POLICY, ...options.toolPolicy };
|
|
253
|
+
const unitsEnv = {
|
|
254
|
+
remote: env,
|
|
255
|
+
archiveLimits,
|
|
256
|
+
...(memo ? { memo } : {}),
|
|
257
|
+
concurrency: options.concurrency ?? 4,
|
|
258
|
+
allowHardlinks: options.allowHardlinks ?? false,
|
|
259
|
+
strictPins: !!options.strictPins,
|
|
260
|
+
skip,
|
|
261
|
+
tempDirs,
|
|
262
|
+
recordSteps: !!options.recordSteps,
|
|
263
|
+
pins: {},
|
|
264
|
+
exec: { plugins, toolPolicy, stats, fetchPolicy: policy },
|
|
265
|
+
fromImage,
|
|
266
|
+
};
|
|
267
|
+
const root = {
|
|
268
|
+
name: undefined,
|
|
269
|
+
label,
|
|
270
|
+
local,
|
|
271
|
+
scope: rootScope(options, label),
|
|
272
|
+
chain: [{ id: options.sitefile === undefined ? contextReal : await realpathLoose(sitefilePath), label }],
|
|
273
|
+
};
|
|
274
|
+
const fullRefs = importedStageRefs(full.stages);
|
|
275
|
+
const resources = new Map();
|
|
276
|
+
// Incremental rebuilds: what each import is (its spec and the digest of what it read), and the build's ARG/LET values.
|
|
277
|
+
const resourceKeys = new Map();
|
|
278
|
+
const units = new Map();
|
|
279
|
+
for (const [spec, i] of indexed) {
|
|
280
|
+
const r = resolved.get(spec.name);
|
|
281
|
+
if (r) {
|
|
282
|
+
resources.set(spec.name, sorted(spec, r.res));
|
|
283
|
+
if (memo)
|
|
284
|
+
resourceKeys.set(spec.name, r.key);
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
importSpans[i]?.start();
|
|
288
|
+
const unit = await loadSitefileImport(spec, root, fullRefs, unitsEnv);
|
|
289
|
+
importSpans[i]?.end({ kind: 'tree' });
|
|
290
|
+
units.set(spec.name, unit);
|
|
291
|
+
resources.set(spec.name, unitResource(unit));
|
|
292
|
+
if (memo)
|
|
293
|
+
resourceKeys.set(spec.name, memoKey('sitefile', label, valueKey(spec), unit.digest));
|
|
294
|
+
}
|
|
295
|
+
if (pins)
|
|
296
|
+
Object.assign(pins.imports, unitsEnv.pins);
|
|
297
|
+
const envKey = memo ? memoKey('env', label, valueKey([...p.vars]), valueKey([...p.fragments.values()].map((f) => [f.name, f.text]))) : '';
|
|
298
|
+
const stageTrees = new Map();
|
|
299
|
+
const execEnv = {
|
|
300
|
+
local,
|
|
301
|
+
resources,
|
|
302
|
+
stages: stageTrees,
|
|
303
|
+
vars: p.vars,
|
|
304
|
+
fragments: p.fragments,
|
|
305
|
+
partials: new Map(),
|
|
306
|
+
label,
|
|
307
|
+
warn,
|
|
308
|
+
plugins,
|
|
309
|
+
tools: new Map(tools.map((t) => [t.name, t])),
|
|
310
|
+
toolPolicy,
|
|
311
|
+
pluginDigests: new Map(),
|
|
312
|
+
stats,
|
|
313
|
+
...(cache ? { cache } : {}),
|
|
314
|
+
...(prof ? { profiler: prof } : {}),
|
|
315
|
+
...(memo ? { memo } : {}),
|
|
316
|
+
resourceKeys,
|
|
317
|
+
envKey,
|
|
318
|
+
stageKeys: new Map(),
|
|
319
|
+
fromImage,
|
|
320
|
+
fetchPolicy: policy,
|
|
321
|
+
};
|
|
322
|
+
let tree = Tree.empty;
|
|
323
|
+
const steps = options.recordSteps ? [] : undefined;
|
|
324
|
+
// FINGERPRINT renames: per operation (by line), and per stage (composed, inherited by FROM <stage>).
|
|
325
|
+
const renames = new Map();
|
|
326
|
+
const stageRenames = new Map();
|
|
327
|
+
let currentRenames = new Map();
|
|
328
|
+
try {
|
|
329
|
+
for (const spec of p.plugins) {
|
|
330
|
+
const s = begin(prof, `PLUGIN ${spec.name}`, 'plugin', { path: './' + spec.path });
|
|
331
|
+
await registerPlugin(spec, local, plugins, pluginAllowed);
|
|
332
|
+
s.end();
|
|
333
|
+
}
|
|
334
|
+
// The stages of imported Sitefiles this build reads (`FROM theme+assets`, `--from=theme+assets`).
|
|
335
|
+
await provideImportedStages(units, importedStageRefs(p.stages), stageTrees, execEnv.stageKeys, stageRenames, unitsEnv);
|
|
336
|
+
for (const [i, stage] of p.stages.entries()) {
|
|
337
|
+
const s = begin(prof, `stage ${stage.name ?? '#' + (i + 1)}`, 'stage', { from: stageFrom(stage) });
|
|
338
|
+
const start = await startStage(stage, execEnv);
|
|
339
|
+
tree = start.tree;
|
|
340
|
+
let key = start.key;
|
|
341
|
+
currentRenames = stage.base.kind === 'stage' ? (stageRenames.get(stage.base.name) ?? new Map()) : new Map();
|
|
342
|
+
const record = steps ? [tree] : undefined;
|
|
343
|
+
steps?.push(record);
|
|
344
|
+
for (const op of stage.operations) {
|
|
345
|
+
const o = begin(prof, `${label}:${op.span.line} ${op.kind}`, 'operation', { line: op.span.line });
|
|
346
|
+
const r = await executeOp(op, tree, key, currentRenames.size ? { ...execEnv, renames: currentRenames } : execEnv);
|
|
347
|
+
tree = r.tree;
|
|
348
|
+
key = r.key;
|
|
349
|
+
if (r.renames) {
|
|
350
|
+
renames.set(op.span.line, r.renames);
|
|
351
|
+
currentRenames = composeRenames(currentRenames, r.renames);
|
|
352
|
+
}
|
|
353
|
+
record?.push(tree);
|
|
354
|
+
o.end({ files: tree.size, ...(r.memo ? { memo: r.memo } : {}) });
|
|
355
|
+
}
|
|
356
|
+
if (stage.name !== undefined) {
|
|
357
|
+
stageRenames.set(stage.name, currentRenames);
|
|
358
|
+
stageTrees.set(stage.name, tree);
|
|
359
|
+
execEnv.stageKeys.set(stage.name, key);
|
|
360
|
+
}
|
|
361
|
+
s.end({ files: tree.size });
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
finally {
|
|
365
|
+
plugins.close();
|
|
366
|
+
stats.pluginsSpawned = plugins.stats.spawned;
|
|
367
|
+
stats.pluginCalls = plugins.stats.calls;
|
|
368
|
+
if (cache)
|
|
369
|
+
Object.assign(stats, cache.stats);
|
|
370
|
+
}
|
|
371
|
+
if (tree.isFile(p.exportPath)) {
|
|
372
|
+
throw new SitefileError('E_EXPORT', `EXPORT path '${p.exportPath}' is a file; export a directory`);
|
|
373
|
+
}
|
|
374
|
+
tree = withSiteRules(tree, p, label, currentRenames);
|
|
375
|
+
const output = tree.subtree(p.exportPath);
|
|
376
|
+
if (output.size === 0)
|
|
377
|
+
warn(p.target === undefined ? `EXPORT ${p.exportPath} contains no files` : `--target ${p.target}: stage '${p.target}' contains no files`);
|
|
378
|
+
// BASE: the image to build on. Resolved after the files so a failing build never pulls anything; its
|
|
379
|
+
// lockfile entry is saved together with the imports'.
|
|
380
|
+
const baseSpan = baseRef ? begin(prof, `BASE ${baseRef}`, 'import') : undefined;
|
|
381
|
+
const base = baseRef
|
|
382
|
+
? await resolveBase(baseRef, {
|
|
383
|
+
store: options.store ?? (await Store.open()),
|
|
384
|
+
platform: options.platform,
|
|
385
|
+
lock,
|
|
386
|
+
frozen: !!options.frozenLockfile,
|
|
387
|
+
update: !!options.updateLock,
|
|
388
|
+
vendor,
|
|
389
|
+
registry: {
|
|
390
|
+
...options.registry,
|
|
391
|
+
allowPrivateNetwork: options.registry?.allowPrivateNetwork ?? policy.allowPrivateNetwork,
|
|
392
|
+
},
|
|
393
|
+
warn,
|
|
394
|
+
})
|
|
395
|
+
: undefined;
|
|
396
|
+
baseSpan?.end();
|
|
397
|
+
const lockfileWritten = lock && !options.frozenLockfile ? await lock.save() : false;
|
|
398
|
+
return {
|
|
399
|
+
plan: p,
|
|
400
|
+
tree,
|
|
401
|
+
output,
|
|
402
|
+
// Imported Sitefiles' own imports follow, by qualified name (`theme+data`), for the provenance.
|
|
403
|
+
resources: units.size ? new Map([...resources, ...nestedResources(units)]) : resources,
|
|
404
|
+
warnings,
|
|
405
|
+
lockfileWritten,
|
|
406
|
+
tools,
|
|
407
|
+
transformStats: stats,
|
|
408
|
+
...(base ? { base } : {}),
|
|
409
|
+
...(images.length ? { images } : {}),
|
|
410
|
+
...(steps ? { steps } : {}),
|
|
411
|
+
...(pins ? { pins } : {}),
|
|
412
|
+
...(units.size ? { imported: units } : {}),
|
|
413
|
+
...(renames.size ? { renames } : {}),
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
/** The Sitefile.lock `images` key of a registry `FROM image` reference. */
|
|
417
|
+
export function imageLockKey(b) {
|
|
418
|
+
return parseBaseRef(b.ref, b.span, `${b.lead} image`).lockKey;
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* `--strict-pins`: every remote input must be pinned, so the build is reproducible. A remote IMPORT
|
|
422
|
+
* needs INTEGRITY or a `Sitefile.lock` entry; a BASE image needs `@sha256:` or a lockfile `images`
|
|
423
|
+
* entry. `lock` is undefined with --no-lockfile (only the Sitefile's own pins count then). Checked in
|
|
424
|
+
* source order, before anything is fetched; the first unpinned input fails with E_UNPINNED.
|
|
425
|
+
*/
|
|
426
|
+
export function checkPins(p, lock, baseRef) {
|
|
427
|
+
const imports = {};
|
|
428
|
+
for (const spec of p.imports) {
|
|
429
|
+
if (spec.location.kind === 'git') {
|
|
430
|
+
if (spec.location.commit !== undefined)
|
|
431
|
+
imports[spec.name] = 'commit';
|
|
432
|
+
else if (lock?.getGit(spec.location.url))
|
|
433
|
+
imports[spec.name] = 'lockfile';
|
|
434
|
+
if (imports[spec.name])
|
|
435
|
+
continue;
|
|
436
|
+
throw new SitefileError('E_UNPINNED', `IMPORT ${spec.name}: '${spec.location.url}' is not pinned (--strict-pins); name a commit (#<40 hex>[:path]), or record it in Sitefile.lock with a build without --strict-pins`, spec.span);
|
|
437
|
+
}
|
|
438
|
+
if (spec.location.kind !== 'remote')
|
|
439
|
+
continue;
|
|
440
|
+
if (spec.integrity)
|
|
441
|
+
imports[spec.name] = 'integrity';
|
|
442
|
+
else if (lock?.get(spec.location.url))
|
|
443
|
+
imports[spec.name] = 'lockfile';
|
|
444
|
+
if (imports[spec.name])
|
|
445
|
+
continue;
|
|
446
|
+
throw new SitefileError('E_UNPINNED', `IMPORT ${spec.name}: '${spec.location.url}' is not pinned (--strict-pins); add INTEGRITY sha256:<hex>, or record it in Sitefile.lock with a build without --strict-pins`, spec.span);
|
|
447
|
+
}
|
|
448
|
+
const images = {};
|
|
449
|
+
for (const st of p.stages) {
|
|
450
|
+
if (st.base.kind !== 'image' || st.base.local)
|
|
451
|
+
continue;
|
|
452
|
+
const { ref, lockKey } = parseBaseRef(st.base.ref, st.base.span, `${st.base.lead} image`);
|
|
453
|
+
if (ref.digest !== undefined)
|
|
454
|
+
images[st.span.line] = 'digest';
|
|
455
|
+
else if (lock?.getImage(lockKey))
|
|
456
|
+
images[st.span.line] = 'lockfile';
|
|
457
|
+
else {
|
|
458
|
+
throw new SitefileError('E_UNPINNED', `${st.base.lead} image ${st.base.ref}: '${lockKey}' is not pinned (--strict-pins); pin it with @sha256:..., or record it in Sitefile.lock with a build without --strict-pins`, st.base.span);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
const withImages = Object.keys(images).length ? { imports, images } : { imports };
|
|
462
|
+
if (baseRef === undefined)
|
|
463
|
+
return withImages;
|
|
464
|
+
const { ref, lockKey } = parseBaseRef(baseRef);
|
|
465
|
+
if (ref.digest !== undefined)
|
|
466
|
+
return { ...withImages, base: 'digest' };
|
|
467
|
+
if (lock?.getImage(lockKey))
|
|
468
|
+
return { ...withImages, base: 'lockfile' };
|
|
469
|
+
throw new SitefileError('E_UNPINNED', `BASE ${baseRef}: '${lockKey}' is not pinned (--strict-pins); pin it with @sha256:..., or record it in Sitefile.lock with a build without --strict-pins`);
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Build, then write the exported tree to `outDir`: a directory for the
|
|
473
|
+
* `directory` format, a single file for zip, tar, tgz and wbn.
|
|
474
|
+
*/
|
|
475
|
+
export async function buildToDirectory(options) {
|
|
476
|
+
const result = await build(options);
|
|
477
|
+
const format = options.format ?? result.plan.exportFormat;
|
|
478
|
+
const span = begin(options.profiler, 'export', 'export', { format });
|
|
479
|
+
try {
|
|
480
|
+
return await exportBuild(result, format, options);
|
|
481
|
+
}
|
|
482
|
+
finally {
|
|
483
|
+
span.end();
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
async function exportBuild(result, format, options) {
|
|
487
|
+
const exportOptions = { contextDir: options.contextDir, ...(options.force ? { force: true } : {}) };
|
|
488
|
+
// Watch mode: a directory export writes only what changed since the session's last one.
|
|
489
|
+
const exportState = options.memo?.exportState;
|
|
490
|
+
if (format !== 'directory')
|
|
491
|
+
exportState?.clear();
|
|
492
|
+
if (format === 'oci') {
|
|
493
|
+
const img = buildOciImage(result.output, {
|
|
494
|
+
root: options.ociRoot ?? result.plan.exportRoot,
|
|
495
|
+
compression: options.ociCompression ?? result.plan.exportCompression,
|
|
496
|
+
platform: options.platform,
|
|
497
|
+
epoch: sourceDateEpoch(),
|
|
498
|
+
base: result.base,
|
|
499
|
+
...(await imageMeta(options.contextDir, { git: !!options.gitLabels, ...(options.imageLabels ? { labels: options.imageLabels } : {}) })),
|
|
500
|
+
});
|
|
501
|
+
// An image built on a BASE carries the base's layers: copy them in so the export is self-contained.
|
|
502
|
+
const baseBlobs = result.base ? await readLayerBlobs(options.store ?? (await Store.open()), img.baseLayers) : undefined;
|
|
503
|
+
const referrers = options.attest ? await imageAttestations(result, img, options, options.attest) : [];
|
|
504
|
+
// A .tar destination gets an OCI archive; anything else a layout directory.
|
|
505
|
+
if (options.outDir.toLowerCase().endsWith('.tar')) {
|
|
506
|
+
const data = ociArchive(img, options.ociRefName, sourceDateEpoch(), baseBlobs, referrers);
|
|
507
|
+
const written = await exportFile(data, options.outDir, exportOptions);
|
|
508
|
+
return { ...result, format, outDir: written.outFile, bytes: written.bytes, fileDigest: sha256(data), manifestDigest: img.manifestDigest };
|
|
509
|
+
}
|
|
510
|
+
const written = await exportDirectory(layoutTree(img, options.ociRefName, baseBlobs, referrers), options.outDir, exportOptions);
|
|
511
|
+
return { ...result, format, outDir: written.outDir, manifestDigest: img.manifestDigest };
|
|
512
|
+
}
|
|
513
|
+
if (!isArchiveFormat(format)) {
|
|
514
|
+
const written = await exportDirectory(result.output, options.outDir, { ...exportOptions, ...(exportState ? { incremental: exportState } : {}) });
|
|
515
|
+
return { ...result, format, outDir: written.outDir, ...(written.incremental ? { incremental: written.incremental } : {}) };
|
|
516
|
+
}
|
|
517
|
+
const data = encodeTree(result.output, format, {
|
|
518
|
+
epoch: sourceDateEpoch(),
|
|
519
|
+
baseUrl: options.baseUrl ?? result.plan.exportBase,
|
|
520
|
+
});
|
|
521
|
+
const written = await exportFile(data, options.outDir, exportOptions);
|
|
522
|
+
return { ...result, format, outDir: written.outFile, bytes: written.bytes, fileDigest: sha256(data) };
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* The attestations (`--provenance`, `--sbom`) of an image built from `result`: in-toto statements
|
|
526
|
+
* whose subject is the image manifest. They never change the image itself.
|
|
527
|
+
*/
|
|
528
|
+
export async function imageAttestations(result, img, options, request) {
|
|
529
|
+
if (!request.provenance && !request.sbom)
|
|
530
|
+
return [];
|
|
531
|
+
const facts = await buildFacts(result, options);
|
|
532
|
+
const title = parseConfig(img.configBytes).labels[ANN.title];
|
|
533
|
+
return makeAttestations({ digest: img.manifestDigest, size: img.manifestBytes.byteLength, files: result.output, root: img.root, title }, facts, request, sourceDateEpoch());
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* Read an import. `key` identifies the result for incremental rebuilds: the import's spec
|
|
537
|
+
* (with its Sitefile position) and the digest of the bytes or tree it read. With a memo,
|
|
538
|
+
* conversions (parsing Markdown, data and archives) whose key is unchanged are reused.
|
|
539
|
+
*/
|
|
540
|
+
async function resolveImport(spec, env) {
|
|
541
|
+
// `AS sitefile` imports are loaded by loadSitefileImport, one after the other.
|
|
542
|
+
if (spec.as === 'sitefile')
|
|
543
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: a Sitefile import is not a resource`, spec.span);
|
|
544
|
+
const origin = `${env.label}:${spec.span.line} IMPORT ${spec.name}`;
|
|
545
|
+
const id = env.memo ? valueKey(spec) : '';
|
|
546
|
+
const keyOf = (raw) => (env.memo ? memoKey('import', env.label, id, raw) : '');
|
|
547
|
+
// A collection's documents are memoized one by one, as children of the import's entry.
|
|
548
|
+
const convert = (key, compute) => env.memo ? env.memo.memoize(key, (children) => compute(env.memo.documents(children))) : compute();
|
|
549
|
+
const convertBytes = async (key, compute) => {
|
|
550
|
+
const hit = env.memo?.get(key);
|
|
551
|
+
if (hit)
|
|
552
|
+
return hit;
|
|
553
|
+
const r = await compute();
|
|
554
|
+
env.memo?.set(key, r);
|
|
555
|
+
return r;
|
|
556
|
+
};
|
|
557
|
+
if (spec.location.kind === 'local') {
|
|
558
|
+
const rel = spec.location.path;
|
|
559
|
+
const kind = await env.local.stat(rel, spec.span);
|
|
560
|
+
if (kind === 'directory') {
|
|
561
|
+
if (spec.as === 'markdown') {
|
|
562
|
+
if (spec.integrity)
|
|
563
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: INTEGRITY applies to files, not directories`, spec.span);
|
|
564
|
+
const tree = await env.local.readTree(rel, origin, spec.span);
|
|
565
|
+
const key = keyOf(tree.provenance.digest);
|
|
566
|
+
return {
|
|
567
|
+
key,
|
|
568
|
+
res: convert(key, (docs) => {
|
|
569
|
+
const value = markdownCollection(tree.tree, spec.uri.replace(/\/+$/, ''), spec.span, docs);
|
|
570
|
+
return { kind: 'value', format: 'markdown', value, provenance: tree.provenance };
|
|
571
|
+
}),
|
|
572
|
+
};
|
|
573
|
+
}
|
|
574
|
+
if (spec.as === 'json' || spec.as === 'yaml') {
|
|
575
|
+
if (spec.integrity)
|
|
576
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: INTEGRITY applies to files, not directories`, spec.span);
|
|
577
|
+
const format = spec.as;
|
|
578
|
+
const tree = await env.local.readTree(rel, origin, spec.span);
|
|
579
|
+
const key = keyOf(tree.provenance.digest);
|
|
580
|
+
return {
|
|
581
|
+
key,
|
|
582
|
+
res: convert(key, (docs) => {
|
|
583
|
+
const value = dataCollection(tree.tree, format, spec.uri.replace(/\/+$/, ''), spec.span, docs);
|
|
584
|
+
return { kind: 'value', format, value, provenance: tree.provenance };
|
|
585
|
+
}),
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
if (spec.as && spec.as !== 'tree') {
|
|
589
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: '${spec.uri}' is a directory and cannot be imported AS ${spec.as}`, spec.span);
|
|
590
|
+
}
|
|
591
|
+
if (spec.integrity)
|
|
592
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: INTEGRITY applies to files, not directories`, spec.span);
|
|
593
|
+
const tree = await env.local.readTree(rel, origin, spec.span);
|
|
594
|
+
return { key: keyOf(tree.provenance.digest), res: tree };
|
|
595
|
+
}
|
|
596
|
+
const file = await env.local.readFile(rel, spec.span);
|
|
597
|
+
if (spec.integrity)
|
|
598
|
+
verifyIntegrity(file.bytes, spec.integrity, spec.uri, spec.span);
|
|
599
|
+
const key = keyOf(`${file.provenance.digest}\0${file.mode}\0${file.basename}`);
|
|
600
|
+
if (!spec.as || spec.as === 'file')
|
|
601
|
+
return { key, res: file };
|
|
602
|
+
const as = spec.as;
|
|
603
|
+
return { key, res: await convertBytes(key, () => bytesToResource(file.bytes, as, { uri: spec.uri, basename: file.basename, mode: file.mode }, spec.span, env.archiveLimits)) };
|
|
604
|
+
}
|
|
605
|
+
if (spec.location.kind === 'git') {
|
|
606
|
+
const loc = spec.location;
|
|
607
|
+
const { commit, selection } = await fetchGitImport(spec, loc, env);
|
|
608
|
+
const resolvedUrl = pinnedGitUrl(loc, commit);
|
|
609
|
+
if (selection.kind === 'file') {
|
|
610
|
+
const digest = sha256(selection.bytes);
|
|
611
|
+
const key = keyOf(`${commit}\0${digest}\0${selection.mode}`);
|
|
612
|
+
const as = spec.as ?? 'file';
|
|
613
|
+
const res = await convertBytes(key, () => bytesToResource(selection.bytes, as, { uri: spec.uri, resolvedUrl, basename: selection.basename, mode: selection.mode }, spec.span, env.archiveLimits));
|
|
614
|
+
return { key, res: { ...res, provenance: { ...res.provenance, gitCommit: commit } } };
|
|
615
|
+
}
|
|
616
|
+
const tree = Tree.fromFiles(selection.files.map(([p, f]) => [p, makeFile(f.bytes, origin, f.mode)]));
|
|
617
|
+
const provenance = { uri: spec.uri, resolvedUrl, digest: tree.digest(), gitCommit: commit };
|
|
618
|
+
const key = keyOf(`${commit}\0${provenance.digest}`);
|
|
619
|
+
const base = spec.uri.replace(/\/+$/, '');
|
|
620
|
+
if (spec.as === 'markdown') {
|
|
621
|
+
return { key, res: convert(key, (docs) => ({ kind: 'value', format: 'markdown', value: markdownCollection(tree, base, spec.span, docs), provenance })) };
|
|
622
|
+
}
|
|
623
|
+
if (spec.as === 'json' || spec.as === 'yaml') {
|
|
624
|
+
const format = spec.as;
|
|
625
|
+
return { key, res: convert(key, (docs) => ({ kind: 'value', format, value: dataCollection(tree, format, base, spec.span, docs), provenance })) };
|
|
626
|
+
}
|
|
627
|
+
if (spec.as && spec.as !== 'tree') {
|
|
628
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: '${spec.uri}' is a directory and cannot be imported AS ${spec.as}`, spec.span);
|
|
629
|
+
}
|
|
630
|
+
return { key, res: { kind: 'tree', tree, provenance } };
|
|
631
|
+
}
|
|
632
|
+
const url = spec.location.url;
|
|
633
|
+
const as = spec.as ?? 'file';
|
|
634
|
+
const { bytes, finalUrl } = await fetchRemoteImport(spec, url, env);
|
|
635
|
+
const basename = path.posix.basename(new URL(finalUrl).pathname) || spec.name;
|
|
636
|
+
const key = keyOf(`${sha256(bytes)}\0${finalUrl}`);
|
|
637
|
+
return { key, res: await convertBytes(key, () => bytesToResource(bytes, as, { uri: url, resolvedUrl: finalUrl, basename }, spec.span, env.archiveLimits)) };
|
|
638
|
+
}
|
|
639
|
+
/** `--secret` ids that no IMPORT … SECRET names (likely a typo). */
|
|
640
|
+
export function warnUnusedSecrets(p, secrets, warn) {
|
|
641
|
+
if (!secrets)
|
|
642
|
+
return;
|
|
643
|
+
const used = new Set(p.imports.flatMap((s) => (s.secrets ?? []).map((x) => x.id)));
|
|
644
|
+
for (const id of [...secrets.keys()].sort())
|
|
645
|
+
if (!used.has(id))
|
|
646
|
+
warn(`--secret ${id} is not used by any IMPORT … SECRET`);
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* The bytes of a remote IMPORT, checked against its INTEGRITY and the lockfile (and recorded there):
|
|
650
|
+
* read from the vendor directory when it has the URL, otherwise fetched.
|
|
651
|
+
*/
|
|
652
|
+
export async function fetchRemoteImport(spec, url, env) {
|
|
653
|
+
const entry = env.lock?.get(url);
|
|
654
|
+
if (env.frozen && !entry) {
|
|
655
|
+
throw new SitefileError('E_LOCK_MISSING', `IMPORT ${spec.name}: '${url}' is not in the lockfile (--frozen-lockfile)`, spec.span);
|
|
656
|
+
}
|
|
657
|
+
const vendored = env.vendor?.resource(url);
|
|
658
|
+
if (env.vendor && !vendored) {
|
|
659
|
+
env.warn(`${env.label}:${spec.span.line}: IMPORT ${spec.name}: '${url}' is not vendored in ${env.vendor.dir}; fetching it (run 'sitefile vendor' to add it)`);
|
|
660
|
+
}
|
|
661
|
+
const fetchIt = () => {
|
|
662
|
+
const originHeaders = secretHeaders(spec, env.secrets);
|
|
663
|
+
return fetchRemote(url, env.policy, spec.span, originHeaders ? { originHeaders } : {});
|
|
664
|
+
};
|
|
665
|
+
const { bytes, finalUrl } = vendored && env.vendor ? await env.vendor.readResource(url, vendored, spec.span) : await fetchIt();
|
|
666
|
+
const digest = sha256(bytes);
|
|
667
|
+
if (spec.integrity) {
|
|
668
|
+
verifyIntegrity(bytes, spec.integrity, url, spec.span);
|
|
669
|
+
}
|
|
670
|
+
else if (entry && !env.update && entry.integrity !== digest) {
|
|
671
|
+
throw new SitefileError('E_LOCK_MISMATCH', `IMPORT ${spec.name}: '${url}' changed since it was locked (locked ${entry.integrity}, got ${digest}); rerun with --update-lock to accept it`, spec.span);
|
|
672
|
+
}
|
|
673
|
+
else if (!entry) {
|
|
674
|
+
env.warn(`${env.label}:${spec.span.line}: IMPORT ${spec.name} is not pinned; add INTEGRITY ${digest} for a reproducible build`);
|
|
675
|
+
}
|
|
676
|
+
if (env.frozen && entry && entry.integrity !== digest) {
|
|
677
|
+
throw new SitefileError('E_LOCK_MISMATCH', `IMPORT ${spec.name}: '${url}' does not match the lockfile (--frozen-lockfile)`, spec.span);
|
|
678
|
+
}
|
|
679
|
+
env.lock?.record(url, { integrity: digest, size: bytes.byteLength, resolved: finalUrl });
|
|
680
|
+
return { bytes, finalUrl };
|
|
681
|
+
}
|
|
682
|
+
/** The canonical bytes `sitefile vendor` stores for a Git selection: a file's bytes, or a tree as a deterministic tar. */
|
|
683
|
+
export function gitSelectionBytes(selection) {
|
|
684
|
+
if (selection.kind === 'file')
|
|
685
|
+
return selection.bytes;
|
|
686
|
+
return writeTar(Tree.fromFiles(selection.files.map(([p, f]) => [p, makeFile(f.bytes, 'vendor', f.mode)])), { epoch: 0 });
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* The content of a Git IMPORT, checked against the lockfile (and recorded there). The commit is the
|
|
690
|
+
* one in the URL, else the locked one (unless --update-lock), else what the ref names now. A vendored
|
|
691
|
+
* copy of that commit is read instead of the network (a ref with no lockfile entry uses the vendored
|
|
692
|
+
* commit, as a BASE tag uses the vendored digest).
|
|
693
|
+
*/
|
|
694
|
+
export async function fetchGitImport(spec, loc, env) {
|
|
695
|
+
const url = loc.url;
|
|
696
|
+
const entry = env.lock?.getGit(url);
|
|
697
|
+
if (env.frozen && !entry) {
|
|
698
|
+
throw new SitefileError('E_LOCK_MISSING', `IMPORT ${spec.name}: '${url}' is not in the lockfile (--frozen-lockfile)`, spec.span);
|
|
699
|
+
}
|
|
700
|
+
if (entry && loc.commit !== undefined && entry.commit !== loc.commit) {
|
|
701
|
+
throw new SitefileError('E_LOCK_MISMATCH', `IMPORT ${spec.name}: '${url}' is locked to commit ${entry.commit}; rerun with --update-lock to accept ${loc.commit}`, spec.span);
|
|
702
|
+
}
|
|
703
|
+
let want = loc.commit ?? (entry && !env.update ? entry.commit : undefined);
|
|
704
|
+
const vendored = env.vendor?.git(url);
|
|
705
|
+
let selection;
|
|
706
|
+
if (vendored && env.vendor && (want === undefined ? !env.update : vendored.commit === want)) {
|
|
707
|
+
want = vendored.commit;
|
|
708
|
+
selection = await env.vendor.readGit(url, vendored, loc.path.slice(loc.path.lastIndexOf('/') + 1), env.archiveLimits ?? DEFAULT_ARCHIVE_LIMITS, spec.span);
|
|
709
|
+
}
|
|
710
|
+
else {
|
|
711
|
+
if (env.vendor) {
|
|
712
|
+
env.warn(`${env.label}:${spec.span.line}: IMPORT ${spec.name}: '${url}' is not vendored in ${env.vendor.dir}${vendored ? ` at commit ${want ?? 'the latest'}` : ''}; fetching it (run 'sitefile vendor' to add it)`);
|
|
713
|
+
}
|
|
714
|
+
const limits = env.archiveLimits ?? DEFAULT_ARCHIVE_LIMITS;
|
|
715
|
+
const originHeaders = secretHeaders(spec, env.secrets);
|
|
716
|
+
const objs = await fetchGitCommit(loc.repo, loc.ref, want, { policy: env.policy, span: spec.span, ...(originHeaders ? { originHeaders } : {}) }, limits);
|
|
717
|
+
want = objs.commit;
|
|
718
|
+
selection = selectGitPath(objs, loc, limits, spec.span);
|
|
719
|
+
}
|
|
720
|
+
if (loc.commit === undefined && !entry) {
|
|
721
|
+
env.warn(`${env.label}:${spec.span.line}: IMPORT ${spec.name} is not pinned; pin commit ${want} (#${want}${loc.path === '' ? '' : ':' + loc.path}) for a reproducible build`);
|
|
722
|
+
}
|
|
723
|
+
env.lock?.recordGit(url, { commit: want });
|
|
724
|
+
return { commit: want, selection };
|
|
725
|
+
}
|
|
726
|
+
// ---------------------------------------------------------------- Sitefile imports (IMPORT … AS sitefile)
|
|
727
|
+
/** How deep Sitefile imports may nest (an imported Sitefile importing another counts one more level). */
|
|
728
|
+
export const MAX_SITEFILE_IMPORT_DEPTH = 8;
|
|
729
|
+
/** The label an imported Sitefile's messages and file origins use. */
|
|
730
|
+
export function sitefileLabel(scope) {
|
|
731
|
+
return scope.prefix + path.posix.join(scope.dir, 'Sitefile');
|
|
732
|
+
}
|
|
733
|
+
/** The label scope of the build's own Sitefile. */
|
|
734
|
+
function rootScope(options, label) {
|
|
735
|
+
if (options.sitefile === undefined)
|
|
736
|
+
return { prefix: '', dir: path.posix.dirname(label) };
|
|
737
|
+
return { prefix: '', dir: path.relative(process.cwd(), path.resolve(options.contextDir)).split(path.sep).join('/') || '.' };
|
|
738
|
+
}
|
|
739
|
+
/** The ARG values an `IMPORT … AS sitefile` passes, checked against the imported Sitefile's ARGs. */
|
|
740
|
+
export function importedArgs(spec, program) {
|
|
741
|
+
const declared = new Map(program.instructions.flatMap((i) => (i.kind === 'ARG' ? [[i.name, i]] : [])));
|
|
742
|
+
const args = {};
|
|
743
|
+
for (const a of spec.args ?? []) {
|
|
744
|
+
if (!declared.has(a.name))
|
|
745
|
+
throw new SitefileError('E_UNDEFINED', `IMPORT ${spec.name}: ${program.file} has no ARG ${a.name}`, a.span);
|
|
746
|
+
args[a.name] = a.value;
|
|
747
|
+
}
|
|
748
|
+
for (const [name, ins] of declared) {
|
|
749
|
+
if (ins.defaultValue === undefined && !(name in args)) {
|
|
750
|
+
throw new SitefileError('E_MISSING_ARG', `IMPORT ${spec.name}: ARG ${name} of ${program.file} has no default; set it with ARG ${name}=<value>`, spec.span);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
return args;
|
|
754
|
+
}
|
|
755
|
+
/**
|
|
756
|
+
* Parse, check and plan an imported Sitefile: no PLUGIN or TOOL, the IMPORT's ARG clauses, and the
|
|
757
|
+
* stages `refs` (references from the importing Sitefile) must exist.
|
|
758
|
+
*/
|
|
759
|
+
export function planImportedSitefile(spec, source, label, refs) {
|
|
760
|
+
const program = parseProgram(source, label);
|
|
761
|
+
for (const ins of program.instructions) {
|
|
762
|
+
if (ins.kind === 'PLUGIN' || ins.kind === 'TOOL') {
|
|
763
|
+
throw new SitefileError('E_UNSUPPORTED', `${ins.kind} ${ins.name}: a Sitefile imported AS sitefile cannot run plugins or tools`, ins.span);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
const p = plan(program, importedArgs(spec, program), (n) => `IMPORT ${spec.name} ARG ${n}`);
|
|
767
|
+
const names = p.stages.flatMap((s) => (s.name === undefined ? [] : [s.name]));
|
|
768
|
+
for (const ref of refs) {
|
|
769
|
+
if (ref.import !== spec.name || names.includes(ref.stage))
|
|
770
|
+
continue;
|
|
771
|
+
throw new SitefileError('E_UNDEFINED', `${ref.name}: ${label} has no stage '${ref.stage}' (${names.length ? `stages: ${names.join(', ')}` : 'it has no named stages'})`, ref.span);
|
|
772
|
+
}
|
|
773
|
+
return { program, plan: p };
|
|
774
|
+
}
|
|
775
|
+
/** Write a tree to a new temporary directory (a Git import's checkout). */
|
|
776
|
+
async function materializeTree(tree, env) {
|
|
777
|
+
const dir = await mkdtemp(path.join(os.tmpdir(), 'sitefile-import-'));
|
|
778
|
+
env.tempDirs.push(dir);
|
|
779
|
+
try {
|
|
780
|
+
for (const [p, n] of tree.entries()) {
|
|
781
|
+
const dest = path.join(dir, ...p.slice(1).split('/'));
|
|
782
|
+
await mkdir(path.dirname(dest), { recursive: true });
|
|
783
|
+
await writeFile(dest, n.bytes, { mode: n.mode });
|
|
784
|
+
await chmod(dest, n.mode);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
catch (e) {
|
|
788
|
+
throw new SitefileError('E_EXPORT', `cannot write a Git checkout to a temporary directory: ${e.message}`, undefined, { cause: e });
|
|
789
|
+
}
|
|
790
|
+
return dir;
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* Load an `IMPORT name FROM <dir|git+https://…> AS sitefile`: find its context (a directory of the
|
|
794
|
+
* importer's context, or a Git tree written to a temporary directory), read and plan its Sitefile with the
|
|
795
|
+
* IMPORT's ARGs, check the stages `refs` name, and resolve its imports exactly as a build would.
|
|
796
|
+
*/
|
|
797
|
+
async function loadSitefileImport(spec, importer, refs, env) {
|
|
798
|
+
const qualified = importer.name === undefined ? spec.name : `${importer.name}+${spec.name}`;
|
|
799
|
+
const origin = `${importer.label}:${spec.span.line} IMPORT ${spec.name}`;
|
|
800
|
+
let hostDir;
|
|
801
|
+
let scope;
|
|
802
|
+
let id;
|
|
803
|
+
let display;
|
|
804
|
+
let provenance;
|
|
805
|
+
let gitTree;
|
|
806
|
+
if (spec.location.kind === 'git') {
|
|
807
|
+
const loc = spec.location;
|
|
808
|
+
const { commit, selection } = await fetchGitImport(spec, loc, { ...env.remote, label: importer.label });
|
|
809
|
+
display = `'${loc.url}'`;
|
|
810
|
+
if (selection.kind === 'file') {
|
|
811
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: ${display} is a file; AS sitefile needs the directory that holds the Sitefile`, spec.span);
|
|
812
|
+
}
|
|
813
|
+
gitTree = Tree.fromFiles(selection.files.map(([p, f]) => [p, makeFile(f.bytes, origin, f.mode)]));
|
|
814
|
+
scope = { prefix: `git+${loc.repo}#${commit}:`, dir: loc.path };
|
|
815
|
+
id = `${scope.prefix}${loc.path}`;
|
|
816
|
+
provenance = { uri: spec.uri, resolvedUrl: pinnedGitUrl(loc, commit), gitCommit: commit };
|
|
817
|
+
hostDir = await materializeTree(gitTree, env);
|
|
818
|
+
}
|
|
819
|
+
else if (spec.location.kind === 'local') {
|
|
820
|
+
const rel = spec.location.path;
|
|
821
|
+
display = rel === '' ? `'./'` : `'./${rel}'`;
|
|
822
|
+
if ((await importer.local.stat(rel, spec.span)) !== 'directory') {
|
|
823
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: ${display} is a file; AS sitefile needs the directory that holds the Sitefile`, spec.span);
|
|
824
|
+
}
|
|
825
|
+
hostDir = await importer.local.realPath(rel, spec.span);
|
|
826
|
+
scope = { prefix: importer.scope.prefix, dir: path.posix.join(importer.scope.dir, rel) };
|
|
827
|
+
id = hostDir;
|
|
828
|
+
provenance = { uri: spec.uri };
|
|
829
|
+
}
|
|
830
|
+
else {
|
|
831
|
+
throw new SitefileError('E_PROTOCOL', `IMPORT ${spec.name}: AS sitefile reads a context directory or a git+https:// repository, not a URL`, spec.span);
|
|
832
|
+
}
|
|
833
|
+
const label = sitefileLabel(scope);
|
|
834
|
+
const seen = importer.chain.findIndex((c) => c.id === id);
|
|
835
|
+
if (seen !== -1) {
|
|
836
|
+
const cycle = [...importer.chain.slice(seen).map((c) => c.label), label];
|
|
837
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: Sitefile import cycle: ${cycle.join(' -> ')}`, spec.span);
|
|
838
|
+
}
|
|
839
|
+
if (importer.chain.length > MAX_SITEFILE_IMPORT_DEPTH) {
|
|
840
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: Sitefile imports nest more than ${MAX_SITEFILE_IMPORT_DEPTH} levels deep`, spec.span);
|
|
841
|
+
}
|
|
842
|
+
// Its context: the directory, sandboxed (its own .sitefileignore applies; what the build writes is skipped).
|
|
843
|
+
const ignore = ['.git'];
|
|
844
|
+
for (const s of env.skip) {
|
|
845
|
+
const rel = path.relative(hostDir, s);
|
|
846
|
+
if (rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel))
|
|
847
|
+
ignore.push(rel.split(path.sep).join('/'));
|
|
848
|
+
}
|
|
849
|
+
const local = await LocalContext.open(hostDir, { ignore, allowHardlinks: env.allowHardlinks });
|
|
850
|
+
let sitefileBytes;
|
|
851
|
+
try {
|
|
852
|
+
sitefileBytes = (await local.readFile('Sitefile', spec.span)).bytes;
|
|
853
|
+
}
|
|
854
|
+
catch (e) {
|
|
855
|
+
if (e instanceof SitefileError && e.code === 'E_NOT_FOUND') {
|
|
856
|
+
throw new SitefileError('E_NOT_FOUND', `IMPORT ${spec.name}: ${display} has no Sitefile`, spec.span, { cause: e });
|
|
857
|
+
}
|
|
858
|
+
throw e;
|
|
859
|
+
}
|
|
860
|
+
const { plan: p } = planImportedSitefile(spec, new TextDecoder().decode(sitefileBytes), label, refs);
|
|
861
|
+
p.warnings.forEach(env.remote.warn);
|
|
862
|
+
const lock = env.remote.lock;
|
|
863
|
+
if (env.strictPins) {
|
|
864
|
+
const pins = checkPins(p, lock, undefined);
|
|
865
|
+
for (const [n, how] of Object.entries(pins.imports))
|
|
866
|
+
env.pins[`${qualified}+${n}`] = how;
|
|
867
|
+
}
|
|
868
|
+
// Its FROM image lockfile entries are kept even when its stages are not built.
|
|
869
|
+
for (const st of p.stages)
|
|
870
|
+
if (st.base.kind === 'image' && !st.base.local)
|
|
871
|
+
lock?.markImageUsed(imageLockKey(st.base));
|
|
872
|
+
// Its imports, as a build of it would resolve them (every one, as for a --target build).
|
|
873
|
+
const ienv = { ...env.remote, label, local, archiveLimits: env.archiveLimits, ...(env.memo ? { memo: env.memo } : {}) };
|
|
874
|
+
const plain = p.imports.filter((s) => s.as !== 'sitefile');
|
|
875
|
+
const resolved = await mapLimit(plain, env.concurrency, (s) => resolveImport(s, ienv));
|
|
876
|
+
const byName = new Map(plain.map((s, i) => [s.name, { ...resolved[i], res: withoutDrafts(s, resolved[i].res) }]));
|
|
877
|
+
// SCHEMA files are read from the imported Sitefile's own context.
|
|
878
|
+
for (const s of plain) {
|
|
879
|
+
const r = byName.get(s.name);
|
|
880
|
+
if (r && s.schema)
|
|
881
|
+
await checkImportSchema(s, r.res, local);
|
|
882
|
+
}
|
|
883
|
+
const resources = new Map();
|
|
884
|
+
const resourceKeys = new Map();
|
|
885
|
+
const imported = new Map();
|
|
886
|
+
const nestedRefs = importedStageRefs(p.stages);
|
|
887
|
+
const self = { name: qualified, label, local, scope, chain: [...importer.chain, { id, label }] };
|
|
888
|
+
for (const s of p.imports) {
|
|
889
|
+
const r = byName.get(s.name);
|
|
890
|
+
if (r) {
|
|
891
|
+
resources.set(s.name, sorted(s, r.res));
|
|
892
|
+
resourceKeys.set(s.name, r.key);
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
895
|
+
const unit = await loadSitefileImport(s, self, nestedRefs, env);
|
|
896
|
+
imported.set(s.name, unit);
|
|
897
|
+
resources.set(s.name, unitResource(unit));
|
|
898
|
+
resourceKeys.set(s.name, env.memo ? memoKey('sitefile', label, valueKey(s), unit.digest) : '');
|
|
899
|
+
}
|
|
900
|
+
const unit = {
|
|
901
|
+
name: qualified,
|
|
902
|
+
label,
|
|
903
|
+
plan: p,
|
|
904
|
+
digest: gitTree ? gitTree.digest() : sha256(sitefileBytes),
|
|
905
|
+
isDirectory: gitTree ? (rel) => rel === '' || gitTree.isDirectory('/' + rel) : (rel) => isHostDirectory(path.join(hostDir, ...rel.split('/'))),
|
|
906
|
+
sitefileBytes,
|
|
907
|
+
resources,
|
|
908
|
+
imported,
|
|
909
|
+
local,
|
|
910
|
+
scope,
|
|
911
|
+
resourceKeys,
|
|
912
|
+
envKey: env.memo ? memoKey('env', label, valueKey([...p.vars]), valueKey([...p.fragments.values()].map((f) => [f.name, f.text]))) : '',
|
|
913
|
+
stageTrees: new Map(),
|
|
914
|
+
stageKeys: new Map(),
|
|
915
|
+
renames: new Map(),
|
|
916
|
+
stageRenames: new Map(),
|
|
917
|
+
...(env.recordSteps ? { steps: [] } : {}),
|
|
918
|
+
built: 0,
|
|
919
|
+
provenance,
|
|
920
|
+
};
|
|
921
|
+
return unit;
|
|
922
|
+
}
|
|
923
|
+
/** The resource an `AS sitefile` import stands for (provenance): its Sitefile, with the unit's digest. */
|
|
924
|
+
function unitResource(unit) {
|
|
925
|
+
return { kind: 'file', bytes: unit.sitefileBytes, mode: 0o644, basename: 'Sitefile', provenance: { ...unit.provenance, digest: unit.digest } };
|
|
926
|
+
}
|
|
927
|
+
function isHostDirectory(p) {
|
|
928
|
+
try {
|
|
929
|
+
return statSync(p).isDirectory();
|
|
930
|
+
}
|
|
931
|
+
catch {
|
|
932
|
+
return false;
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
/** The source of the build's own Sitefile, for {@link readImportedSitefile}. */
|
|
936
|
+
export async function rootSitefileSource(options, label, sitefilePath) {
|
|
937
|
+
const contextReal = await realpath(options.contextDir).catch((e) => {
|
|
938
|
+
throw new SitefileError('E_NOT_FOUND', `build context '${options.contextDir}' does not exist`, undefined, { cause: e });
|
|
939
|
+
});
|
|
940
|
+
const local = await LocalContext.open(contextReal, { ignore: ['.git'], allowHardlinks: options.allowHardlinks ?? false });
|
|
941
|
+
const id = options.sitefile === undefined ? contextReal : await realpathLoose(sitefilePath);
|
|
942
|
+
return { kind: 'local', local, scope: rootScope(options, label), label, chain: [{ id, label }] };
|
|
943
|
+
}
|
|
944
|
+
/**
|
|
945
|
+
* Read and plan the Sitefile an `IMPORT … AS sitefile` of `from` names, without building it (for `vendor`
|
|
946
|
+
* and `outdated`): the same location, cycle, depth and ARG rules as a build. A Git one is fetched with
|
|
947
|
+
* `fetchGit` (the commit a build would use). Returns its plan and the source its own imports read from.
|
|
948
|
+
*/
|
|
949
|
+
export async function readImportedSitefile(spec, from, refs, fetchGit) {
|
|
950
|
+
let scope;
|
|
951
|
+
let id;
|
|
952
|
+
let display;
|
|
953
|
+
let bytes;
|
|
954
|
+
let make;
|
|
955
|
+
if (spec.location.kind === 'git') {
|
|
956
|
+
const loc = spec.location;
|
|
957
|
+
const { commit, selection } = await fetchGit(spec, loc);
|
|
958
|
+
display = `'${loc.url}'`;
|
|
959
|
+
if (selection.kind === 'file') {
|
|
960
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: ${display} is a file; AS sitefile needs the directory that holds the Sitefile`, spec.span);
|
|
961
|
+
}
|
|
962
|
+
const tree = Tree.fromFiles(selection.files.map(([p, f]) => [p, makeFile(f.bytes, 'git', f.mode)]));
|
|
963
|
+
scope = { prefix: `git+${loc.repo}#${commit}:`, dir: loc.path };
|
|
964
|
+
id = `${scope.prefix}${loc.path}`;
|
|
965
|
+
bytes = tree.get('/Sitefile')?.bytes;
|
|
966
|
+
make = (label, chain) => ({ kind: 'tree', tree, dir: '', scope, label, chain });
|
|
967
|
+
}
|
|
968
|
+
else if (spec.location.kind === 'local') {
|
|
969
|
+
const rel = spec.location.path;
|
|
970
|
+
display = rel === '' ? `'./'` : `'./${rel}'`;
|
|
971
|
+
scope = { prefix: from.scope.prefix, dir: path.posix.join(from.scope.dir, rel) };
|
|
972
|
+
if (from.kind === 'local') {
|
|
973
|
+
if ((await from.local.stat(rel, spec.span)) !== 'directory') {
|
|
974
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: ${display} is a file; AS sitefile needs the directory that holds the Sitefile`, spec.span);
|
|
975
|
+
}
|
|
976
|
+
const real = await from.local.realPath(rel, spec.span);
|
|
977
|
+
id = real;
|
|
978
|
+
const local = await LocalContext.open(real, { ignore: ['.git'] });
|
|
979
|
+
bytes = await local.readFile('Sitefile', spec.span).then((f) => f.bytes, (e) => {
|
|
980
|
+
if (e instanceof SitefileError && e.code === 'E_NOT_FOUND')
|
|
981
|
+
return undefined;
|
|
982
|
+
throw e;
|
|
983
|
+
});
|
|
984
|
+
make = (label, chain) => ({ kind: 'local', local, scope, label, chain });
|
|
985
|
+
}
|
|
986
|
+
else {
|
|
987
|
+
const dir = path.posix.join(from.dir, rel).replace(/^\.$/, '');
|
|
988
|
+
const at = dir === '' ? '/' : '/' + dir;
|
|
989
|
+
if (from.tree.isFile(at)) {
|
|
990
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: ${display} is a file; AS sitefile needs the directory that holds the Sitefile`, spec.span);
|
|
991
|
+
}
|
|
992
|
+
if (!from.tree.isDirectory(at))
|
|
993
|
+
throw new SitefileError('E_NOT_FOUND', `'${display.slice(1, -1)}' does not exist in the build context`, spec.span);
|
|
994
|
+
id = `${from.scope.prefix}${scope.dir}`;
|
|
995
|
+
bytes = from.tree.get(at === '/' ? '/Sitefile' : `${at}/Sitefile`)?.bytes;
|
|
996
|
+
const tree = from.tree;
|
|
997
|
+
make = (label, chain) => ({ kind: 'tree', tree, dir, scope, label, chain });
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
else {
|
|
1001
|
+
throw new SitefileError('E_PROTOCOL', `IMPORT ${spec.name}: AS sitefile reads a context directory or a git+https:// repository, not a URL`, spec.span);
|
|
1002
|
+
}
|
|
1003
|
+
const label = sitefileLabel(scope);
|
|
1004
|
+
const seen = from.chain.findIndex((c) => c.id === id);
|
|
1005
|
+
if (seen !== -1) {
|
|
1006
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: Sitefile import cycle: ${[...from.chain.slice(seen).map((c) => c.label), label].join(' -> ')}`, spec.span);
|
|
1007
|
+
}
|
|
1008
|
+
if (from.chain.length > MAX_SITEFILE_IMPORT_DEPTH) {
|
|
1009
|
+
throw new SitefileError('E_VALIDATION', `IMPORT ${spec.name}: Sitefile imports nest more than ${MAX_SITEFILE_IMPORT_DEPTH} levels deep`, spec.span);
|
|
1010
|
+
}
|
|
1011
|
+
if (bytes === undefined)
|
|
1012
|
+
throw new SitefileError('E_NOT_FOUND', `IMPORT ${spec.name}: ${display} has no Sitefile`, spec.span);
|
|
1013
|
+
const { plan: p } = planImportedSitefile(spec, new TextDecoder().decode(bytes), label, refs);
|
|
1014
|
+
return { plan: p, source: make(label, [...from.chain, { id, label }]) };
|
|
1015
|
+
}
|
|
1016
|
+
/** The resources of the Sitefiles a build imports, by qualified name (`theme+data`), depth first in IMPORT order. */
|
|
1017
|
+
function nestedResources(units, out = []) {
|
|
1018
|
+
for (const unit of units.values()) {
|
|
1019
|
+
for (const [name, res] of unit.resources)
|
|
1020
|
+
out.push([`${unit.name}+${name}`, res]);
|
|
1021
|
+
nestedResources(unit.imported, out);
|
|
1022
|
+
}
|
|
1023
|
+
return out;
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Make the imported stages `refs` names available in `stages`/`keys` as `<import>+<stage>`: each imported
|
|
1027
|
+
* Sitefile builds its stages up to the last one referenced (like a --target build).
|
|
1028
|
+
*/
|
|
1029
|
+
async function provideImportedStages(units, refs, stages, keys, renames, env) {
|
|
1030
|
+
for (const [name, unit] of units) {
|
|
1031
|
+
const wanted = refs.filter((r) => r.import === name);
|
|
1032
|
+
if (!wanted.length)
|
|
1033
|
+
continue;
|
|
1034
|
+
const upto = Math.max(...wanted.map((r) => unit.plan.stages.findIndex((s) => s.name === r.stage)));
|
|
1035
|
+
await buildUnitStages(unit, upto, env);
|
|
1036
|
+
for (const st of unit.plan.stages.slice(0, unit.built)) {
|
|
1037
|
+
if (st.name === undefined)
|
|
1038
|
+
continue;
|
|
1039
|
+
stages.set(`${name}+${st.name}`, unit.stageTrees.get(st.name));
|
|
1040
|
+
keys.set(`${name}+${st.name}`, unit.stageKeys.get(st.name) ?? '');
|
|
1041
|
+
renames.set(`${name}+${st.name}`, unit.stageRenames.get(st.name) ?? new Map());
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
/** Build an imported Sitefile's stages up to index `upto` (those already built are kept). */
|
|
1046
|
+
async function buildUnitStages(unit, upto, env) {
|
|
1047
|
+
if (upto < unit.built)
|
|
1048
|
+
return;
|
|
1049
|
+
await provideImportedStages(unit.imported, importedStageRefs(unit.plan.stages.slice(0, upto + 1)), unit.stageTrees, unit.stageKeys, unit.stageRenames, env);
|
|
1050
|
+
const execEnv = {
|
|
1051
|
+
local: unit.local,
|
|
1052
|
+
resources: unit.resources,
|
|
1053
|
+
stages: unit.stageTrees,
|
|
1054
|
+
vars: unit.plan.vars,
|
|
1055
|
+
fragments: unit.plan.fragments,
|
|
1056
|
+
partials: new Map(),
|
|
1057
|
+
label: unit.label,
|
|
1058
|
+
warn: env.remote.warn,
|
|
1059
|
+
...env.exec,
|
|
1060
|
+
tools: new Map(),
|
|
1061
|
+
pluginDigests: new Map(),
|
|
1062
|
+
...(env.memo ? { memo: env.memo } : {}),
|
|
1063
|
+
resourceKeys: unit.resourceKeys,
|
|
1064
|
+
envKey: unit.envKey,
|
|
1065
|
+
stageKeys: unit.stageKeys,
|
|
1066
|
+
fromImage: (b) => env.fromImage(b, unit.name),
|
|
1067
|
+
};
|
|
1068
|
+
for (let i = unit.built; i <= upto; i++) {
|
|
1069
|
+
const stage = unit.plan.stages[i];
|
|
1070
|
+
const start = await startStage(stage, execEnv);
|
|
1071
|
+
let tree = start.tree;
|
|
1072
|
+
let key = start.key;
|
|
1073
|
+
let current = stage.base.kind === 'stage' ? (unit.stageRenames.get(stage.base.name) ?? new Map()) : new Map();
|
|
1074
|
+
const record = unit.steps ? [tree] : undefined;
|
|
1075
|
+
for (const op of stage.operations) {
|
|
1076
|
+
const r = await executeOp(op, tree, key, current.size ? { ...execEnv, renames: current } : execEnv);
|
|
1077
|
+
tree = r.tree;
|
|
1078
|
+
key = r.key;
|
|
1079
|
+
if (r.renames) {
|
|
1080
|
+
unit.renames.set(op.span.line, r.renames);
|
|
1081
|
+
current = composeRenames(current, r.renames);
|
|
1082
|
+
}
|
|
1083
|
+
record?.push(tree);
|
|
1084
|
+
}
|
|
1085
|
+
if (record)
|
|
1086
|
+
unit.steps.push(record);
|
|
1087
|
+
if (stage.name !== undefined) {
|
|
1088
|
+
unit.stageRenames.set(stage.name, current);
|
|
1089
|
+
unit.stageTrees.set(stage.name, tree);
|
|
1090
|
+
unit.stageKeys.set(stage.name, key);
|
|
1091
|
+
}
|
|
1092
|
+
unit.built = i + 1;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
/** The tree a stage starts from, and its key. */
|
|
1096
|
+
async function startStage(stage, env) {
|
|
1097
|
+
const base = stage.base;
|
|
1098
|
+
switch (base.kind) {
|
|
1099
|
+
case 'scratch':
|
|
1100
|
+
return { tree: Tree.empty, key: env.memo ? memoKey('scratch', env.envKey) : '' };
|
|
1101
|
+
case 'stage':
|
|
1102
|
+
return { tree: env.stages.get(base.name) ?? fail(base.name, stage.span), key: env.stageKeys.get(base.name) ?? '' };
|
|
1103
|
+
case 'local': {
|
|
1104
|
+
const display = base.path === '' ? './' : './' + base.path;
|
|
1105
|
+
if ((await env.local.stat(base.path, stage.span)) !== 'directory') {
|
|
1106
|
+
throw new SitefileError('E_TYPE', `FROM '${display}' is a file; FROM needs a directory`, stage.span);
|
|
1107
|
+
}
|
|
1108
|
+
const t = await env.local.readTree(base.path, `${env.label}:${stage.span.line} FROM`, stage.span);
|
|
1109
|
+
return { tree: t.tree, key: env.memo ? memoKey('from', env.envKey, String(stage.span.line), base.path, t.provenance.digest) : '' };
|
|
1110
|
+
}
|
|
1111
|
+
case 'image': {
|
|
1112
|
+
const img = await env.fromImage(base);
|
|
1113
|
+
return { tree: img.tree, key: env.memo ? memoKey('image', env.envKey, String(stage.span.line), base.ref, base.at ?? '', img.digest) : '' };
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
/** What a stage starts from, for the trace. */
|
|
1118
|
+
function stageFrom(stage) {
|
|
1119
|
+
const b = stage.base;
|
|
1120
|
+
if (b.kind === 'image')
|
|
1121
|
+
return `image ${b.ref}${b.at !== undefined ? ` AT ${b.at}` : ''}`;
|
|
1122
|
+
return b.kind === 'scratch' ? 'scratch' : b.kind === 'stage' ? b.name : b.path === '' ? './' : './' + b.path;
|
|
1123
|
+
}
|
|
1124
|
+
function fail(name, span) {
|
|
1125
|
+
throw new SitefileError('E_UNDEFINED', `unknown stage '${name}'`, span);
|
|
1126
|
+
}
|
|
1127
|
+
function recording(env) {
|
|
1128
|
+
const warnings = [];
|
|
1129
|
+
return {
|
|
1130
|
+
...env,
|
|
1131
|
+
warnings,
|
|
1132
|
+
unrecorded: env.warn,
|
|
1133
|
+
warn: (m) => {
|
|
1134
|
+
warnings.push(m);
|
|
1135
|
+
env.warn(m);
|
|
1136
|
+
},
|
|
1137
|
+
};
|
|
1138
|
+
}
|
|
1139
|
+
/**
|
|
1140
|
+
* Run `compute` unless `key` is memoized. A hit replays the warnings the step printed
|
|
1141
|
+
* after those this run already printed before the lookup (the same code ran up to here,
|
|
1142
|
+
* so they are a prefix). `store` says whether the result may be reused (a plugin can opt out).
|
|
1143
|
+
*/
|
|
1144
|
+
async function memoFiles(env, key, compute, store = () => true) {
|
|
1145
|
+
if (!env.memo || key === undefined)
|
|
1146
|
+
return { files: await compute([]) };
|
|
1147
|
+
const hit = env.memo.get(key);
|
|
1148
|
+
if (hit) {
|
|
1149
|
+
for (const w of hit.warnings.slice(env.warnings.length))
|
|
1150
|
+
env.warn(w);
|
|
1151
|
+
return { files: hit.files, key, hit: true };
|
|
1152
|
+
}
|
|
1153
|
+
const children = [];
|
|
1154
|
+
const files = await compute(children);
|
|
1155
|
+
if (!store())
|
|
1156
|
+
return { files };
|
|
1157
|
+
env.memo.set(key, { files, warnings: [...env.warnings] }, children);
|
|
1158
|
+
return { files, key };
|
|
1159
|
+
}
|
|
1160
|
+
/** What a source reference reads, for memo keys (the operation's own text covers inline bodies and paths). */
|
|
1161
|
+
function sourceKey(ref, src, env) {
|
|
1162
|
+
switch (ref.kind) {
|
|
1163
|
+
case 'inline':
|
|
1164
|
+
return 'inline';
|
|
1165
|
+
case 'local':
|
|
1166
|
+
return `local\0${src?.digest ?? ''}`;
|
|
1167
|
+
case 'stage':
|
|
1168
|
+
return `stage\0${env.stageKeys.get(ref.name) ?? ''}`;
|
|
1169
|
+
case 'import':
|
|
1170
|
+
return `import\0${env.resourceKeys.get(ref.name) ?? ''}`;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
/** Names an operand interpolates. */
|
|
1174
|
+
function operandNames(ops, out = new Set()) {
|
|
1175
|
+
for (const op of ops)
|
|
1176
|
+
for (const part of op?.parts ?? [])
|
|
1177
|
+
if (part.kind !== 'text')
|
|
1178
|
+
out.add(part.name);
|
|
1179
|
+
return out;
|
|
1180
|
+
}
|
|
1181
|
+
/**
|
|
1182
|
+
* The keys of the imports an operation may read, by name (ARG/LET values are in its opId).
|
|
1183
|
+
* `except`: a loop variable, which hides an import of the same name.
|
|
1184
|
+
*/
|
|
1185
|
+
function namesKey(env, names, except) {
|
|
1186
|
+
const list = [...new Set(names)].filter((n) => n !== except).sort();
|
|
1187
|
+
return memoKey('names', ...list.flatMap((n) => [n, env.resourceKeys.get(n) ?? '']));
|
|
1188
|
+
}
|
|
1189
|
+
/** The html fragments a template context resolves (LETs from the plan, `AS html` imports parsed on first use). */
|
|
1190
|
+
function templateContext(env) {
|
|
1191
|
+
const partial = (name) => {
|
|
1192
|
+
const res = env.resources.get(name);
|
|
1193
|
+
if (res?.kind !== 'value' || res.format !== 'html')
|
|
1194
|
+
return undefined;
|
|
1195
|
+
let nodes = env.partials.get(name);
|
|
1196
|
+
if (!nodes) {
|
|
1197
|
+
nodes = parseRenderTemplate(fragmentText(res.value), res.provenance.uri);
|
|
1198
|
+
env.partials.set(name, nodes);
|
|
1199
|
+
}
|
|
1200
|
+
return nodes;
|
|
1201
|
+
};
|
|
1202
|
+
return {
|
|
1203
|
+
isFragment: (name) => {
|
|
1204
|
+
if (env.loop?.name === name)
|
|
1205
|
+
return false;
|
|
1206
|
+
if (env.fragments.has(name))
|
|
1207
|
+
return true;
|
|
1208
|
+
const res = env.resources.get(name);
|
|
1209
|
+
return res?.kind === 'value' && res.format === 'html';
|
|
1210
|
+
},
|
|
1211
|
+
fragment: (name) => env.fragments.get(name)?.nodes ?? partial(name) ?? [],
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
/** Whether a template has a `${file …}` (it then reads the tree at its RENDER). */
|
|
1215
|
+
function usesFiles(nodes) {
|
|
1216
|
+
return nodes.some((n) => n.kind === 'file' || (n.kind === 'each' && (usesFiles(n.body) || usesFiles(n.empty))) || (n.kind === 'if' && (usesFiles(n.then) || usesFiles(n.otherwise))));
|
|
1217
|
+
}
|
|
1218
|
+
/**
|
|
1219
|
+
* The names a template reads, with those of the html fragments it uses (and theirs), for memo keys,
|
|
1220
|
+
* and whether it or one of those fragments reads files. A fragment that does not parse adds nothing:
|
|
1221
|
+
* rendering fails on it anyway.
|
|
1222
|
+
*/
|
|
1223
|
+
function renderDeps(nodes, env) {
|
|
1224
|
+
const out = templateNames(nodes);
|
|
1225
|
+
let files = usesFiles(nodes);
|
|
1226
|
+
const ctx = templateContext(env);
|
|
1227
|
+
const seen = new Set();
|
|
1228
|
+
for (let grew = true; grew;) {
|
|
1229
|
+
grew = false;
|
|
1230
|
+
for (const name of [...out]) {
|
|
1231
|
+
if (seen.has(name) || !ctx.isFragment(name))
|
|
1232
|
+
continue;
|
|
1233
|
+
seen.add(name);
|
|
1234
|
+
try {
|
|
1235
|
+
const before = out.size;
|
|
1236
|
+
const f = ctx.fragment(name);
|
|
1237
|
+
templateNames(f, out);
|
|
1238
|
+
files ||= usesFiles(f);
|
|
1239
|
+
grew ||= out.size > before;
|
|
1240
|
+
}
|
|
1241
|
+
catch {
|
|
1242
|
+
// E_TEMPLATE in an imported partial: reported when the RENDER runs.
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
return { names: out, files };
|
|
1247
|
+
}
|
|
1248
|
+
/** Top-level names a template reads (variables, #each lists, #if conditions). */
|
|
1249
|
+
export function templateNames(nodes, out = new Set()) {
|
|
1250
|
+
for (const n of nodes) {
|
|
1251
|
+
if (n.kind === 'var')
|
|
1252
|
+
out.add(n.name);
|
|
1253
|
+
else if (n.kind === 'file')
|
|
1254
|
+
for (const r of fileRefs(n))
|
|
1255
|
+
out.add(r.name);
|
|
1256
|
+
else if (n.kind === 'each') {
|
|
1257
|
+
out.add(n.list.name);
|
|
1258
|
+
templateNames(n.body, out);
|
|
1259
|
+
templateNames(n.empty, out);
|
|
1260
|
+
}
|
|
1261
|
+
else if (n.kind === 'if') {
|
|
1262
|
+
out.add(n.cond.name);
|
|
1263
|
+
templateNames(n.then, out);
|
|
1264
|
+
templateNames(n.otherwise, out);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
return out;
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* Run one operation: produce its files, then add them to the tree. With a memo, a step whose
|
|
1271
|
+
* key (the operation and everything it reads) is unchanged reuses its files, and one whose
|
|
1272
|
+
* input tree is also unchanged reuses the resulting tree. The key of the result chains the
|
|
1273
|
+
* input tree's key with the step's, like image layers.
|
|
1274
|
+
*/
|
|
1275
|
+
async function executeOp(op, tree, prevKey, env) {
|
|
1276
|
+
const policy = 'overwrite' in op && op.overwrite ? 'overwrite' : 'error';
|
|
1277
|
+
const memo = env.memo;
|
|
1278
|
+
if (!memo) {
|
|
1279
|
+
if (op.kind === 'SITEMAP')
|
|
1280
|
+
return { tree: sitemap(op, tree, env), key: '' };
|
|
1281
|
+
if (op.kind === 'FINGERPRINT')
|
|
1282
|
+
return { ...fingerprintOp(op, tree, env), key: '' };
|
|
1283
|
+
if (op.kind === 'CHECKLINKS') {
|
|
1284
|
+
await checkLinks(op, tree, env, env.warn);
|
|
1285
|
+
return { tree, key: '' };
|
|
1286
|
+
}
|
|
1287
|
+
return { tree: tree.withFiles((await produce(op, tree, recording(env), '', '')).files, policy, op.span), key: '' };
|
|
1288
|
+
}
|
|
1289
|
+
const opId = memoKey('op', env.envKey, valueKey(op));
|
|
1290
|
+
if (op.kind === 'SITEMAP' || op.kind === 'FINGERPRINT' || op.kind === 'CHECKLINKS') {
|
|
1291
|
+
// Reads the whole tree: only the chain key can say it is unchanged.
|
|
1292
|
+
const key = memoKey('chain', prevKey, opId);
|
|
1293
|
+
if (op.kind === 'CHECKLINKS') {
|
|
1294
|
+
// The tree is unchanged; a hit skips the check (and its requests) and prints its warnings again.
|
|
1295
|
+
const hit = memo.get(key);
|
|
1296
|
+
if (hit) {
|
|
1297
|
+
hit.warnings.forEach(env.warn);
|
|
1298
|
+
return { tree, key, memo: 'hit' };
|
|
1299
|
+
}
|
|
1300
|
+
const warnings = [];
|
|
1301
|
+
await checkLinks(op, tree, env, (m) => (warnings.push(m), env.warn(m)));
|
|
1302
|
+
memo.set(key, { warnings });
|
|
1303
|
+
return { tree, key, memo: 'miss' };
|
|
1304
|
+
}
|
|
1305
|
+
if (op.kind === 'FINGERPRINT') {
|
|
1306
|
+
const hit = memo.get(key);
|
|
1307
|
+
if (hit) {
|
|
1308
|
+
hit.warnings.forEach(env.warn);
|
|
1309
|
+
return { tree: hit.tree, renames: hit.renames, key, memo: 'hit' };
|
|
1310
|
+
}
|
|
1311
|
+
const warnings = [];
|
|
1312
|
+
const out = fingerprintOp(op, tree, { ...env, warn: (m) => (warnings.push(m), env.warn(m)) });
|
|
1313
|
+
memo.set(key, { ...out, warnings });
|
|
1314
|
+
return { ...out, key, memo: 'miss' };
|
|
1315
|
+
}
|
|
1316
|
+
const hit = memo.get(key);
|
|
1317
|
+
if (hit)
|
|
1318
|
+
return { tree: hit, key, memo: 'hit' };
|
|
1319
|
+
const out = sitemap(op, tree, env);
|
|
1320
|
+
memo.set(key, out);
|
|
1321
|
+
return { tree: out, key, memo: 'miss' };
|
|
1322
|
+
}
|
|
1323
|
+
const p = await produce(op, tree, recording(env), opId, prevKey);
|
|
1324
|
+
// Not memoizable (e.g. a plugin that opted out of caching): chain on what it produced instead.
|
|
1325
|
+
const stepKey = p.key ?? memoKey('files', ...p.files.flatMap(([f, n]) => [f, String(n.mode), n.digest, n.origin]));
|
|
1326
|
+
const key = memoKey('chain', prevKey, stepKey);
|
|
1327
|
+
const hit = memo.get(key);
|
|
1328
|
+
if (hit)
|
|
1329
|
+
return { tree: hit, key, memo: 'hit' };
|
|
1330
|
+
const out = tree.withFiles(p.files, policy, op.span);
|
|
1331
|
+
memo.set(key, out);
|
|
1332
|
+
return { tree: out, key, memo: p.hit ? 'files' : 'miss' };
|
|
1333
|
+
}
|
|
1334
|
+
async function checkLinks(op, tree, env, warn) {
|
|
1335
|
+
const broken = await brokenLinks(tree, op, env.fetchPolicy);
|
|
1336
|
+
reportBrokenLinks(broken, op, `${env.label}:${op.span.line} CHECKLINKS`, warn);
|
|
1337
|
+
}
|
|
1338
|
+
function fingerprintOp(op, tree, env) {
|
|
1339
|
+
return fingerprint(tree, op, `${env.label}:${op.span.line} FINGERPRINT`, env.warn);
|
|
1340
|
+
}
|
|
1341
|
+
function sitemap(op, tree, env) {
|
|
1342
|
+
const origin = `${env.label}:${op.span.line} SITEMAP`;
|
|
1343
|
+
return tree.withFiles([[op.dest, makeFile(new TextEncoder().encode(sitemapXml(tree, op.base)), origin)]], op.overwrite ? 'overwrite' : 'error', op.span);
|
|
1344
|
+
}
|
|
1345
|
+
/** Check a plugin file (sandbox, own folder, integrity) and start its process. */
|
|
1346
|
+
async function registerPlugin(spec, local, host, allowed) {
|
|
1347
|
+
const { real, bytes } = await local.realFile(spec.path, spec.span);
|
|
1348
|
+
if (spec.integrity)
|
|
1349
|
+
verifyIntegrity(bytes, spec.integrity, `./${spec.path}`, spec.span);
|
|
1350
|
+
if (real.endsWith('.wasm')) {
|
|
1351
|
+
// A wasm plugin has no imports, so it reads nothing: no own-folder rule needed.
|
|
1352
|
+
checkWasmPlugin(spec.name, bytes, spec.span);
|
|
1353
|
+
host.register(spec.name, real, spec.span, bytes);
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
// It runs as JavaScript (also a '.wasm' path that is a symlink to a JS file), so it needs the opt-in.
|
|
1357
|
+
allowed(spec);
|
|
1358
|
+
// A JS plugin may read its whole folder, so that folder must not be the context itself.
|
|
1359
|
+
if (path.dirname(real) === local.root) {
|
|
1360
|
+
throw new SitefileError('E_VALIDATION', `PLUGIN ${spec.name}: put the plugin in its own folder (e.g. ./plugins/${path.basename(real)}); a plugin can read every file next to it`, spec.span);
|
|
1361
|
+
}
|
|
1362
|
+
host.register(spec.name, real, spec.span);
|
|
1363
|
+
}
|
|
1364
|
+
/**
|
|
1365
|
+
* Check every JavaScript PLUGIN against `--allow-plugin` / `--allow-plugins` before
|
|
1366
|
+
* anything is read or started. A wasm plugin (a module that imports nothing) runs
|
|
1367
|
+
* without permission. Returns the check, which registerPlugin repeats on the real path.
|
|
1368
|
+
*/
|
|
1369
|
+
function checkPlugins(p, options, warn) {
|
|
1370
|
+
const names = new Set();
|
|
1371
|
+
for (const name of options.allowPlugin ?? []) {
|
|
1372
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name))
|
|
1373
|
+
throw new SitefileError('E_USAGE', `--allow-plugin expects a PLUGIN name (an identifier), got '${name}'`);
|
|
1374
|
+
names.add(name);
|
|
1375
|
+
}
|
|
1376
|
+
const all = options.allowAllPlugins ?? false;
|
|
1377
|
+
const allowed = (spec) => {
|
|
1378
|
+
if (all || names.has(spec.name))
|
|
1379
|
+
return;
|
|
1380
|
+
throw new SitefileError('E_UNSUPPORTED', `PLUGIN ${spec.name} runs JavaScript, which must be allowed explicitly: add --allow-plugin ${spec.name} (or --allow-plugins to allow every PLUGIN); WebAssembly plugins need no flag`, spec.span);
|
|
1381
|
+
};
|
|
1382
|
+
for (const spec of p.plugins)
|
|
1383
|
+
if (!spec.path.endsWith('.wasm'))
|
|
1384
|
+
allowed(spec);
|
|
1385
|
+
for (const name of names)
|
|
1386
|
+
if (!p.plugins.some((s) => s.name === name))
|
|
1387
|
+
warn(`--allow-plugin ${name} does not match any PLUGIN and was ignored`);
|
|
1388
|
+
return allowed;
|
|
1389
|
+
}
|
|
1390
|
+
/** Match each declared TOOL to a `--allow-exec` entry and resolve it to a real path. */
|
|
1391
|
+
async function resolveTools(p, options, warn) {
|
|
1392
|
+
const allowed = new Map();
|
|
1393
|
+
for (const spec of options.allowExec ?? []) {
|
|
1394
|
+
const eq = spec.indexOf('=');
|
|
1395
|
+
const name = eq < 0 ? spec : spec.slice(0, eq);
|
|
1396
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || (eq >= 0 && eq === spec.length - 1)) {
|
|
1397
|
+
throw new SitefileError('E_USAGE', `--allow-exec expects NAME or NAME=/absolute/path (NAME is an identifier), got '${spec}'`);
|
|
1398
|
+
}
|
|
1399
|
+
allowed.set(name, eq < 0 ? undefined : spec.slice(eq + 1));
|
|
1400
|
+
}
|
|
1401
|
+
const out = [];
|
|
1402
|
+
for (const t of p.tools) {
|
|
1403
|
+
if (!allowed.has(t.name)) {
|
|
1404
|
+
throw new SitefileError('E_UNSUPPORTED', `TOOL ${t.name} runs an external program, which must be allowed explicitly: add --allow-exec ${t.name} (or --allow-exec ${t.name}=/absolute/path)`, t.span);
|
|
1405
|
+
}
|
|
1406
|
+
const tool = await resolveTool(t.name, allowed.get(t.name), process.env['PATH'], t.span);
|
|
1407
|
+
options.onNotice?.(`tool ${t.name} -> ${tool.path}`);
|
|
1408
|
+
out.push(tool);
|
|
1409
|
+
}
|
|
1410
|
+
for (const name of allowed.keys())
|
|
1411
|
+
if (!p.tools.some((t) => t.name === name))
|
|
1412
|
+
warn(`--allow-exec ${name} does not match any TOOL and was ignored`);
|
|
1413
|
+
return out;
|
|
1414
|
+
}
|
|
1415
|
+
/** Interpolate an operand with ARG/LET values and, inside FOR, the loop element. */
|
|
1416
|
+
function interpolate(op, env) {
|
|
1417
|
+
let out = '';
|
|
1418
|
+
for (const part of op.parts) {
|
|
1419
|
+
if (part.kind === 'text')
|
|
1420
|
+
out += part.value;
|
|
1421
|
+
else
|
|
1422
|
+
out += String(lookup(part.name, part.path, part.span, env));
|
|
1423
|
+
}
|
|
1424
|
+
return out;
|
|
1425
|
+
}
|
|
1426
|
+
async function produceTransform(op, env, opId) {
|
|
1427
|
+
const origin = `${env.label}:${op.span.line} TRANSFORM`;
|
|
1428
|
+
let items = [undefined];
|
|
1429
|
+
if (op.each) {
|
|
1430
|
+
const list = valueOf(op.each.list, [], op.span, false, env);
|
|
1431
|
+
if (!Array.isArray(list)) {
|
|
1432
|
+
throw new SitefileError('E_TYPE', `FOR ${op.each.as} IN ${op.each.list}: '${op.each.list}' is not a list`, op.span);
|
|
1433
|
+
}
|
|
1434
|
+
if (list.length === 0)
|
|
1435
|
+
env.warn(`${origin}: FOR ${op.each.as} IN ${op.each.list} produces nothing: the list is empty`);
|
|
1436
|
+
items = list;
|
|
1437
|
+
}
|
|
1438
|
+
// The source is the same for every element; read it once.
|
|
1439
|
+
let input;
|
|
1440
|
+
let data;
|
|
1441
|
+
let srcKey = '';
|
|
1442
|
+
// A directory source: the plugin runs once per file (--include/--exclude pick them).
|
|
1443
|
+
let dir;
|
|
1444
|
+
const filtered = op.include !== undefined || op.exclude !== undefined;
|
|
1445
|
+
if (op.source) {
|
|
1446
|
+
const ref = op.source;
|
|
1447
|
+
const res = ref.kind === 'import' ? env.resources.get(ref.name) : undefined;
|
|
1448
|
+
if (ref.kind === 'import' && res?.kind === 'value') {
|
|
1449
|
+
if (ref.path !== '/')
|
|
1450
|
+
throw new SitefileError('E_PATH', `import '${ref.name}' is a value; use '${ref.name}:'`, ref.span);
|
|
1451
|
+
if (filtered)
|
|
1452
|
+
throw new SitefileError('E_TYPE', `--include and --exclude filter a directory, but '${ref.name}:' is a value import`, ref.span);
|
|
1453
|
+
data = res.value;
|
|
1454
|
+
}
|
|
1455
|
+
else {
|
|
1456
|
+
const src = await readSource(op.source, origin, env, op.exclude ? IgnoreRules.fromPatterns(op.exclude) : undefined);
|
|
1457
|
+
if (src.kind !== 'file') {
|
|
1458
|
+
if (op.each) {
|
|
1459
|
+
throw new SitefileError('E_TYPE', `TRANSFORM source '${src.label}' is a directory, which runs the plugin once per file and cannot be combined with FOR`, op.source.span);
|
|
1460
|
+
}
|
|
1461
|
+
dir = { tree: including(src.tree, op.include), label: src.label };
|
|
1462
|
+
}
|
|
1463
|
+
else {
|
|
1464
|
+
if (filtered)
|
|
1465
|
+
throw new SitefileError('E_TYPE', `--include and --exclude filter a directory, but '${src.label}' is a file`, op.source.span);
|
|
1466
|
+
input = { path: src.label, name: src.basename || src.label, bytes: src.bytes, ...textOf(src.bytes) };
|
|
1467
|
+
}
|
|
1468
|
+
if (env.memo)
|
|
1469
|
+
srcKey = sourceKey(ref, src, env);
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
const tool = env.tools.get(op.plugin);
|
|
1473
|
+
if (tool && !input && !dir && op.source) {
|
|
1474
|
+
throw new SitefileError('E_TYPE', `TRANSFORM … WITH ${op.plugin}: a TOOL needs a file source, not a value import`, op.source.span);
|
|
1475
|
+
}
|
|
1476
|
+
const inputDigest = input ? sha256(input.bytes) : undefined;
|
|
1477
|
+
let pluginDigest;
|
|
1478
|
+
if ((env.cache || env.memo) && !tool) {
|
|
1479
|
+
if (!env.pluginDigests.has(op.plugin)) {
|
|
1480
|
+
// A wasm plugin is just its module; a JS plugin can read its whole folder.
|
|
1481
|
+
const wasm = env.plugins.wasmBytes(op.plugin);
|
|
1482
|
+
const dir = env.plugins.folderOf(op.plugin);
|
|
1483
|
+
const digest = wasm ? sha256(wasm) : dir ? await digestFolder(dir) : undefined;
|
|
1484
|
+
env.pluginDigests.set(op.plugin, digest);
|
|
1485
|
+
if (digest === undefined && env.cache)
|
|
1486
|
+
env.warn(`${origin}: PLUGIN ${op.plugin}'s folder is too large to hash; its results are not cached`);
|
|
1487
|
+
}
|
|
1488
|
+
pluginDigest = env.pluginDigests.get(op.plugin);
|
|
1489
|
+
}
|
|
1490
|
+
// Incremental rebuilds: the same code (plugin digest, or tool binary), source, data, list and interpolated names.
|
|
1491
|
+
const code = tool ? `tool\0${tool.path}\0${tool.size}\0${tool.mtimeMs}` : pluginDigest !== undefined ? `plugin\0${pluginDigest}` : undefined;
|
|
1492
|
+
const memo = env.memo && code !== undefined ? env.memo : undefined;
|
|
1493
|
+
const operandRefs = operandNames([op.dest, ...op.options.map((o) => o.value), ...(op.args ?? [])], new Set(data !== undefined && op.source?.kind === 'import' ? [op.source.name] : []));
|
|
1494
|
+
const key = memo ? memoKey('transform', opId, srcKey, code, namesKey(env, op.each ? new Set([...operandRefs, op.each.list]) : operandRefs)) : undefined;
|
|
1495
|
+
const storable = () => tool !== undefined || env.plugins.cacheable(op.plugin);
|
|
1496
|
+
if (dir) {
|
|
1497
|
+
const files = { tree: dir.tree, label: dir.label, fileBase: memo ? memoKey('transform-file', opId, code, namesKey(env, operandRefs)) : undefined };
|
|
1498
|
+
return memoFiles(env, key, (children) => transformFiles(op, env, files, { tool, pluginDigest, memo, children }), storable);
|
|
1499
|
+
}
|
|
1500
|
+
// One FOR element's files: the same request gives the same result (the list itself is not read).
|
|
1501
|
+
const itemBase = memo && op.each ? memoKey('transform-item', opId, srcKey, code, namesKey(env, operandRefs, op.each.as)) : undefined;
|
|
1502
|
+
return memoFiles(env, key, async (children) => {
|
|
1503
|
+
const files = [];
|
|
1504
|
+
for (const item of items) {
|
|
1505
|
+
const itemKey = itemBase !== undefined ? memoKey(itemBase, valueKey(item)) : undefined;
|
|
1506
|
+
const reused = itemKey !== undefined ? memo.get(itemKey, children) : undefined;
|
|
1507
|
+
if (reused) {
|
|
1508
|
+
files.push(...reused);
|
|
1509
|
+
continue;
|
|
1510
|
+
}
|
|
1511
|
+
const itemFiles = [];
|
|
1512
|
+
const itemEnv = op.each ? { ...env, loop: { name: op.each.as, value: item } } : env;
|
|
1513
|
+
const rawDest = interpolate(op.dest, itemEnv);
|
|
1514
|
+
if (starName(rawDest) !== undefined) {
|
|
1515
|
+
throw new SitefileError('E_TYPE', `TRANSFORM destination '${rawDest}': a '*' name stands for each file of a directory source, and this TRANSFORM's source is not a directory`, op.dest.span);
|
|
1516
|
+
}
|
|
1517
|
+
const dest = normalizeVirtual(rawDest, op.dest.span, 'TRANSFORM destination');
|
|
1518
|
+
const options = Object.create(null);
|
|
1519
|
+
for (const o of op.options)
|
|
1520
|
+
options[o.key] = interpolate(o.value, itemEnv);
|
|
1521
|
+
const request = {
|
|
1522
|
+
apiVersion: 1,
|
|
1523
|
+
dest: dest.path,
|
|
1524
|
+
options,
|
|
1525
|
+
...(input ? { input } : {}),
|
|
1526
|
+
...(data !== undefined ? { data } : {}),
|
|
1527
|
+
...(op.each ? { item } : {}),
|
|
1528
|
+
};
|
|
1529
|
+
let args;
|
|
1530
|
+
if (tool) {
|
|
1531
|
+
if (dest.trailingSlash)
|
|
1532
|
+
throw new SitefileError('E_PLUGIN', `TOOL ${op.plugin} writes one file, but '${rawDest}' is a directory destination`, op.dest.span);
|
|
1533
|
+
args = (op.args ?? []).map((a) => interpolate(a, itemEnv));
|
|
1534
|
+
}
|
|
1535
|
+
// Everything the result can depend on: the code (plugin folder digest, or tool binary + argv) and the request.
|
|
1536
|
+
const cacheable = env.cache !== undefined && (tool !== undefined || pluginDigest !== undefined);
|
|
1537
|
+
const cached = cacheable
|
|
1538
|
+
? cacheKey({
|
|
1539
|
+
code: tool ? { tool: { path: tool.path, size: tool.size, mtimeMs: tool.mtimeMs }, args } : { plugin: pluginDigest },
|
|
1540
|
+
request: {
|
|
1541
|
+
dest: dest.path,
|
|
1542
|
+
options,
|
|
1543
|
+
input: input && { path: input.path, name: input.name, digest: inputDigest },
|
|
1544
|
+
data,
|
|
1545
|
+
item,
|
|
1546
|
+
},
|
|
1547
|
+
})
|
|
1548
|
+
: undefined;
|
|
1549
|
+
const call = begin(env.profiler, `TRANSFORM ${op.plugin} ${dest.path}`, 'transform', { plugin: op.plugin, tool: tool !== undefined, dest: dest.path });
|
|
1550
|
+
let result = cached ? await env.cache.get(cached) : undefined;
|
|
1551
|
+
const cacheState = !cached ? 'off' : result ? 'hit' : 'miss';
|
|
1552
|
+
if (!result) {
|
|
1553
|
+
if (tool) {
|
|
1554
|
+
env.stats.toolRuns++;
|
|
1555
|
+
result = await runTool(tool, { args: args, dest: dest.path, options, ...(input ? { input: { name: input.name, bytes: input.bytes } } : {}) }, env.toolPolicy,
|
|
1556
|
+
// The tool's stderr, like a TRANSFORM cache hit, is not replayed by a memo hit.
|
|
1557
|
+
(l) => env.unrecorded(l), op.span);
|
|
1558
|
+
}
|
|
1559
|
+
else {
|
|
1560
|
+
result = await env.plugins.call(op.plugin, request, op.span);
|
|
1561
|
+
}
|
|
1562
|
+
if (cached && (tool || env.plugins.cacheable(op.plugin)))
|
|
1563
|
+
await env.cache.put(cached, result);
|
|
1564
|
+
}
|
|
1565
|
+
call.end({ cache: cacheState });
|
|
1566
|
+
if (result.kind === 'file') {
|
|
1567
|
+
if (dest.trailingSlash) {
|
|
1568
|
+
throw new SitefileError('E_PLUGIN', `PLUGIN ${op.plugin} returned one file, but '${rawDest}' is a directory destination`, op.dest.span);
|
|
1569
|
+
}
|
|
1570
|
+
itemFiles.push([dest.path, makeFile(result.bytes, origin)]);
|
|
1571
|
+
}
|
|
1572
|
+
else {
|
|
1573
|
+
// { files }: placed under the destination directory.
|
|
1574
|
+
for (const [rel, bytes] of result.files) {
|
|
1575
|
+
const p = normalizeVirtual(`${dest.path === '/' ? '' : dest.path}/${rel}`, op.dest.span, `file '${rel}' returned by PLUGIN ${op.plugin}`).path;
|
|
1576
|
+
if (rel.includes('..') || rel.startsWith('/')) {
|
|
1577
|
+
throw new SitefileError('E_PLUGIN', `PLUGIN ${op.plugin} returned an unsafe path '${rel}'`, op.span);
|
|
1578
|
+
}
|
|
1579
|
+
itemFiles.push([p, makeFile(bytes, origin)]);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
if (itemKey !== undefined && storable())
|
|
1583
|
+
memo.set(itemKey, itemFiles);
|
|
1584
|
+
files.push(...itemFiles);
|
|
1585
|
+
}
|
|
1586
|
+
return files;
|
|
1587
|
+
},
|
|
1588
|
+
// A plugin that answered `cache: false` is not reused either.
|
|
1589
|
+
storable);
|
|
1590
|
+
}
|
|
1591
|
+
/** Decoded UTF-8 text for a plugin input, when the bytes are valid UTF-8. */
|
|
1592
|
+
function textOf(bytes) {
|
|
1593
|
+
try {
|
|
1594
|
+
return { text: new TextDecoder('utf-8', { fatal: true }).decode(bytes) };
|
|
1595
|
+
}
|
|
1596
|
+
catch {
|
|
1597
|
+
return {};
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
/** A directory source with only the files an `--include` pattern matches (relative to its root). */
|
|
1601
|
+
function including(tree, include) {
|
|
1602
|
+
if (!include)
|
|
1603
|
+
return tree;
|
|
1604
|
+
const rules = IgnoreRules.fromPatterns(include);
|
|
1605
|
+
return Tree.fromFiles(tree.entries().filter(([p]) => rules.excludes(p.slice(1))));
|
|
1606
|
+
}
|
|
1607
|
+
/**
|
|
1608
|
+
* The last segment of a TRANSFORM destination when it names each file of a directory source:
|
|
1609
|
+
* `*` (the file's own name) or `*.ext` (its name with the extension replaced). Undefined otherwise.
|
|
1610
|
+
*/
|
|
1611
|
+
export function starName(dest) {
|
|
1612
|
+
const last = dest.slice(dest.lastIndexOf('/') + 1);
|
|
1613
|
+
return last === '*' || /^\*\.[^*]+$/.test(last) ? last : undefined;
|
|
1614
|
+
}
|
|
1615
|
+
/**
|
|
1616
|
+
* Where one file of a directory TRANSFORM goes: `rel` is its path inside the source directory.
|
|
1617
|
+
* `TO /out/*` keeps its relative path, `TO /out/*.ext` replaces its extension, and any other
|
|
1618
|
+
* destination is the directory `<dest>/<relative dir>/` (the plugin names the files).
|
|
1619
|
+
*/
|
|
1620
|
+
export function fileDest(dest, rel) {
|
|
1621
|
+
const base = dest.slice(0, dest.lastIndexOf('/') + 1);
|
|
1622
|
+
const star = starName(dest);
|
|
1623
|
+
const cut = rel.lastIndexOf('/');
|
|
1624
|
+
const relDir = rel.slice(0, cut + 1);
|
|
1625
|
+
const name = rel.slice(cut + 1);
|
|
1626
|
+
if (star === '*')
|
|
1627
|
+
return base + rel;
|
|
1628
|
+
if (star !== undefined) {
|
|
1629
|
+
const dot = name.lastIndexOf('.');
|
|
1630
|
+
return base + relDir + (dot > 0 ? name.slice(0, dot) : name) + star.slice(1);
|
|
1631
|
+
}
|
|
1632
|
+
return (dest.endsWith('/') ? dest : dest + '/') + relDir;
|
|
1633
|
+
}
|
|
1634
|
+
/**
|
|
1635
|
+
* Run `fn(0..n)` at most `limit` at a time, starting them in index order. After a failure no
|
|
1636
|
+
* later index starts, so the lowest failing index is always among the results (and every index
|
|
1637
|
+
* below it has run): callers report that one, whatever order the calls finished in.
|
|
1638
|
+
*/
|
|
1639
|
+
export async function runBounded(n, limit, fn) {
|
|
1640
|
+
const out = new Array(n).fill(undefined);
|
|
1641
|
+
let next = 0;
|
|
1642
|
+
let failedAt = Infinity;
|
|
1643
|
+
const lane = async () => {
|
|
1644
|
+
for (;;) {
|
|
1645
|
+
const i = next++;
|
|
1646
|
+
if (i >= n || i > failedAt)
|
|
1647
|
+
return;
|
|
1648
|
+
try {
|
|
1649
|
+
out[i] = { ok: true, value: await fn(i) };
|
|
1650
|
+
}
|
|
1651
|
+
catch (error) {
|
|
1652
|
+
out[i] = { ok: false, error };
|
|
1653
|
+
failedAt = Math.min(failedAt, i);
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
};
|
|
1657
|
+
await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), n) }, lane));
|
|
1658
|
+
return out;
|
|
1659
|
+
}
|
|
1660
|
+
/** A failure for one file of a directory TRANSFORM: the same error, naming the file. */
|
|
1661
|
+
function namingFile(e, file) {
|
|
1662
|
+
return e instanceof SitefileError ? new SitefileError(e.code, `${file}: ${e.detail}`, e.span) : e;
|
|
1663
|
+
}
|
|
1664
|
+
/**
|
|
1665
|
+
* TRANSFORM over a directory: one plugin (or tool) call per file, in path order. Each call gets
|
|
1666
|
+
* its own `input` and destination (see `fileDest`), is cached and memoized per file, and calls
|
|
1667
|
+
* run up to TRANSFORM_JOBS at a time (a JavaScript plugin, one process, takes one at a time).
|
|
1668
|
+
* Files, tool output, profile events and errors come in path order whatever order calls finish in.
|
|
1669
|
+
*/
|
|
1670
|
+
async function transformFiles(op, env, dir, ctx) {
|
|
1671
|
+
const { tool, pluginDigest, memo, children } = ctx;
|
|
1672
|
+
const line = `${env.label}:${op.span.line} TRANSFORM`;
|
|
1673
|
+
const entries = dir.tree.entries();
|
|
1674
|
+
if (entries.length === 0) {
|
|
1675
|
+
env.warn(op.include || op.exclude
|
|
1676
|
+
? `${line}: --include/--exclude leave no files of '${dir.label}' to transform`
|
|
1677
|
+
: `${line}: '${dir.label}' contains no files`);
|
|
1678
|
+
}
|
|
1679
|
+
const rawDest = interpolate(op.dest, env);
|
|
1680
|
+
normalizeVirtual(rawDest, op.dest.span, 'TRANSFORM destination');
|
|
1681
|
+
const dirDest = rawDest.endsWith('/') ? rawDest : rawDest + '/';
|
|
1682
|
+
if (tool && starName(rawDest) === undefined) {
|
|
1683
|
+
throw new SitefileError('E_PLUGIN', `TOOL ${op.plugin} writes one file per input, but '${rawDest}' is a directory destination: use '${dirDest}*' to keep each file's name, or '${dirDest}*.ext' to change its extension`, op.dest.span);
|
|
1684
|
+
}
|
|
1685
|
+
const options = Object.create(null);
|
|
1686
|
+
for (const o of op.options)
|
|
1687
|
+
options[o.key] = interpolate(o.value, env);
|
|
1688
|
+
const args = tool ? (op.args ?? []).map((a) => interpolate(a, env)) : undefined;
|
|
1689
|
+
const store = () => tool !== undefined || env.plugins.cacheable(op.plugin);
|
|
1690
|
+
const done = [];
|
|
1691
|
+
const jobs = [];
|
|
1692
|
+
for (const [index, [p, node]] of entries.entries()) {
|
|
1693
|
+
const rel = p.slice(1);
|
|
1694
|
+
const label = (dir.label.endsWith('/') ? dir.label : dir.label + '/') + rel;
|
|
1695
|
+
const fileKey = dir.fileBase !== undefined ? memoKey(dir.fileBase, rel, node.digest) : undefined;
|
|
1696
|
+
const reused = fileKey !== undefined ? memo.get(fileKey, children) : undefined;
|
|
1697
|
+
if (reused) {
|
|
1698
|
+
done[index] = reused;
|
|
1699
|
+
continue;
|
|
1700
|
+
}
|
|
1701
|
+
const perFile = fileDest(rawDest, rel);
|
|
1702
|
+
const dest = normalizeVirtual(perFile, op.dest.span, 'TRANSFORM destination');
|
|
1703
|
+
const input = { path: label, name: basenameVirtual(p), bytes: node.bytes, ...textOf(node.bytes) };
|
|
1704
|
+
const cached = env.cache !== undefined && (tool !== undefined || pluginDigest !== undefined)
|
|
1705
|
+
? cacheKey({
|
|
1706
|
+
code: tool ? { tool: { path: tool.path, size: tool.size, mtimeMs: tool.mtimeMs }, args } : { plugin: pluginDigest },
|
|
1707
|
+
request: { dest: dest.path, options, input: { path: input.path, name: input.name, digest: sha256(input.bytes) }, data: undefined, item: undefined },
|
|
1708
|
+
})
|
|
1709
|
+
: undefined;
|
|
1710
|
+
const span = reserve(env.profiler, `TRANSFORM ${op.plugin} ${dest.path}`, 'transform', { plugin: op.plugin, tool: tool !== undefined, dest: dest.path, input: label });
|
|
1711
|
+
const job = {
|
|
1712
|
+
index,
|
|
1713
|
+
label,
|
|
1714
|
+
origin: `${line} ${label}`,
|
|
1715
|
+
rawDest: perFile,
|
|
1716
|
+
dest,
|
|
1717
|
+
input,
|
|
1718
|
+
...(cached !== undefined ? { cached } : {}),
|
|
1719
|
+
...(fileKey !== undefined ? { memoKey: fileKey } : {}),
|
|
1720
|
+
span,
|
|
1721
|
+
logs: [],
|
|
1722
|
+
cache: cached !== undefined ? 'miss' : 'off',
|
|
1723
|
+
};
|
|
1724
|
+
if (cached !== undefined) {
|
|
1725
|
+
const hit = await env.cache.get(cached);
|
|
1726
|
+
if (hit) {
|
|
1727
|
+
span.start();
|
|
1728
|
+
span.end({ cache: 'hit' });
|
|
1729
|
+
job.result = hit;
|
|
1730
|
+
job.cache = 'hit';
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
jobs.push(job);
|
|
1734
|
+
}
|
|
1735
|
+
// The misses, a few at a time (a JavaScript plugin's one process takes one call at a time).
|
|
1736
|
+
const misses = jobs.filter((j) => !j.result);
|
|
1737
|
+
const limit = tool || env.plugins.isWasm(op.plugin) ? TRANSFORM_JOBS : 1;
|
|
1738
|
+
const settled = await runBounded(misses.length, limit, async (i) => {
|
|
1739
|
+
const job = misses[i];
|
|
1740
|
+
job.span.start();
|
|
1741
|
+
let result;
|
|
1742
|
+
if (tool) {
|
|
1743
|
+
env.stats.toolRuns++;
|
|
1744
|
+
const call = { args: args, dest: job.dest.path, options, input: { name: job.input.name, bytes: job.input.bytes } };
|
|
1745
|
+
result = await runTool(tool, call, env.toolPolicy, (l) => job.logs.push(l), op.span);
|
|
1746
|
+
}
|
|
1747
|
+
else {
|
|
1748
|
+
result = await env.plugins.call(op.plugin, { apiVersion: 1, dest: job.dest.path, options, input: job.input }, op.span);
|
|
1749
|
+
}
|
|
1750
|
+
job.span.end({ cache: job.cache });
|
|
1751
|
+
return result;
|
|
1752
|
+
});
|
|
1753
|
+
misses.forEach((job, i) => {
|
|
1754
|
+
const r = settled[i];
|
|
1755
|
+
if (r?.ok)
|
|
1756
|
+
job.result = r.value;
|
|
1757
|
+
else if (r)
|
|
1758
|
+
job.error = { error: r.error };
|
|
1759
|
+
});
|
|
1760
|
+
// In path order: tool output, the first failure, cache writes and the files.
|
|
1761
|
+
for (const job of jobs) {
|
|
1762
|
+
// The tool's stderr, like a TRANSFORM cache hit, is not replayed by a memo hit.
|
|
1763
|
+
for (const l of job.logs)
|
|
1764
|
+
env.unrecorded(l);
|
|
1765
|
+
if (job.error)
|
|
1766
|
+
throw namingFile(job.error.error, job.label);
|
|
1767
|
+
const result = job.result;
|
|
1768
|
+
if (job.cache === 'miss' && store())
|
|
1769
|
+
await env.cache.put(job.cached, result);
|
|
1770
|
+
const files = [];
|
|
1771
|
+
if (result.kind === 'file') {
|
|
1772
|
+
if (job.dest.trailingSlash) {
|
|
1773
|
+
throw new SitefileError('E_PLUGIN', `${job.label}: PLUGIN ${op.plugin} returned one file, but '${job.rawDest}' is a directory destination (use '${dirDest}*' or '${dirDest}*.ext' for one file per input)`, op.dest.span);
|
|
1774
|
+
}
|
|
1775
|
+
files.push([job.dest.path, makeFile(result.bytes, job.origin)]);
|
|
1776
|
+
}
|
|
1777
|
+
else {
|
|
1778
|
+
for (const [rel, bytes] of result.files) {
|
|
1779
|
+
const p = normalizeVirtual(`${job.dest.path === '/' ? '' : job.dest.path}/${rel}`, op.dest.span, `file '${rel}' returned by PLUGIN ${op.plugin}`).path;
|
|
1780
|
+
if (rel.includes('..') || rel.startsWith('/')) {
|
|
1781
|
+
throw new SitefileError('E_PLUGIN', `${job.label}: PLUGIN ${op.plugin} returned an unsafe path '${rel}'`, op.span);
|
|
1782
|
+
}
|
|
1783
|
+
files.push([p, makeFile(bytes, job.origin)]);
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
if (job.memoKey !== undefined && store())
|
|
1787
|
+
memo.set(job.memoKey, files);
|
|
1788
|
+
done[job.index] = files;
|
|
1789
|
+
}
|
|
1790
|
+
// Two inputs may write the same path only with the same content.
|
|
1791
|
+
const out = [];
|
|
1792
|
+
const owner = new Map();
|
|
1793
|
+
for (const files of done) {
|
|
1794
|
+
for (const [p, node] of files ?? []) {
|
|
1795
|
+
const prior = owner.get(p);
|
|
1796
|
+
if (prior) {
|
|
1797
|
+
if (prior.digest === node.digest)
|
|
1798
|
+
continue;
|
|
1799
|
+
throw new SitefileError('E_CONFLICT', `'${p}' is written for two inputs with different content (${prior.origin} and ${node.origin}); rename one, or leave one out with --include/--exclude`, op.span);
|
|
1800
|
+
}
|
|
1801
|
+
owner.set(p, node);
|
|
1802
|
+
out.push([p, node]);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
return out;
|
|
1806
|
+
}
|
|
1807
|
+
/** REDIRECT and HEADER lines become `/_redirects` and `/_headers` at the root of the exported directory. */
|
|
1808
|
+
/**
|
|
1809
|
+
* REDIRECT and HEADER lines become `/_redirects` and `/_headers` at the root of the exported directory.
|
|
1810
|
+
* A HEADER pattern or REDIRECT target that is the URL of a file FINGERPRINT renamed in the exported stage
|
|
1811
|
+
* names its new URL.
|
|
1812
|
+
*/
|
|
1813
|
+
function withSiteRules(tree, p, label, renames) {
|
|
1814
|
+
const url = new Map();
|
|
1815
|
+
for (const [from, to] of renames) {
|
|
1816
|
+
const a = urlBelow(p.exportPath, from);
|
|
1817
|
+
const b = urlBelow(p.exportPath, to);
|
|
1818
|
+
if (a !== undefined && b !== undefined)
|
|
1819
|
+
url.set(a, b);
|
|
1820
|
+
}
|
|
1821
|
+
const redirects = p.redirects.map((r) => ({ ...r, to: url.get(r.to) ?? r.to }));
|
|
1822
|
+
const headers = p.headers.map((h) => ({ ...h, pattern: url.get(h.pattern) ?? h.pattern }));
|
|
1823
|
+
const files = [
|
|
1824
|
+
{ name: '_redirects', text: redirectsFile(redirects), rules: redirects, kind: 'REDIRECT' },
|
|
1825
|
+
{ name: '_headers', text: headersFile(headers), rules: headers, kind: 'HEADER' },
|
|
1826
|
+
];
|
|
1827
|
+
for (const f of files) {
|
|
1828
|
+
const first = f.rules[0];
|
|
1829
|
+
if (!first)
|
|
1830
|
+
continue;
|
|
1831
|
+
const policy = f.rules.some((r) => r.overwrite) ? 'overwrite' : 'error';
|
|
1832
|
+
const node = makeFile(new TextEncoder().encode(f.text), `${label}:${first.span.line} ${f.kind}`);
|
|
1833
|
+
tree = tree.withFiles([[joinVirtual(p.exportPath, f.name), node]], policy, first.span);
|
|
1834
|
+
}
|
|
1835
|
+
return tree;
|
|
1836
|
+
}
|
|
1837
|
+
/** The URL path of a tree path below the exported directory, or undefined outside it. */
|
|
1838
|
+
function urlBelow(dir, path) {
|
|
1839
|
+
if (dir === '/')
|
|
1840
|
+
return path;
|
|
1841
|
+
return path.startsWith(dir + '/') ? path.slice(dir.length) : undefined;
|
|
1842
|
+
}
|
|
1843
|
+
function feedFiles(op, env) {
|
|
1844
|
+
const origin = `${env.label}:${op.span.line} FEED`;
|
|
1845
|
+
const list = valueOf(op.list, [], op.span, false, env);
|
|
1846
|
+
if (!Array.isArray(list)) {
|
|
1847
|
+
throw new SitefileError('E_TYPE', `FEED ${op.list}: '${op.list}' is not a list (import a directory AS markdown)`, op.span);
|
|
1848
|
+
}
|
|
1849
|
+
if (list.length === 0)
|
|
1850
|
+
env.warn(`${origin}: '${op.list}' is empty, so the feed has no entries`);
|
|
1851
|
+
const entries = [];
|
|
1852
|
+
list.forEach((item, i) => {
|
|
1853
|
+
if (item === null || typeof item !== 'object' || Array.isArray(item)) {
|
|
1854
|
+
throw new SitefileError('E_TYPE', `FEED ${op.list}: element ${i + 1} is not a document (an object with title and date)`, op.span);
|
|
1855
|
+
}
|
|
1856
|
+
const doc = item;
|
|
1857
|
+
const name = typeof doc['path'] === 'string' ? doc['path'] : `#${i + 1}`;
|
|
1858
|
+
const text = (key) => {
|
|
1859
|
+
const v = doc[key];
|
|
1860
|
+
if (v === undefined || v === null)
|
|
1861
|
+
return undefined;
|
|
1862
|
+
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean')
|
|
1863
|
+
return String(v);
|
|
1864
|
+
throw new SitefileError('E_TYPE', `FEED ${op.list}: '${key}' of document '${name}' is not text`, op.span);
|
|
1865
|
+
};
|
|
1866
|
+
const required = (key) => {
|
|
1867
|
+
const v = text(key);
|
|
1868
|
+
if (v === undefined)
|
|
1869
|
+
throw new SitefileError('E_VALIDATION', `FEED ${op.list}: document '${name}' has no '${key}' (add it to its frontmatter)`, op.span);
|
|
1870
|
+
return v;
|
|
1871
|
+
};
|
|
1872
|
+
const title = required('title');
|
|
1873
|
+
const rawDate = required('date');
|
|
1874
|
+
const date = parseFeedDate(rawDate);
|
|
1875
|
+
if (!date) {
|
|
1876
|
+
throw new SitefileError('E_VALIDATION', `FEED ${op.list}: document '${name}' has date '${rawDate}'; expected YYYY-MM-DD or an RFC 3339 date-time`, op.span);
|
|
1877
|
+
}
|
|
1878
|
+
const raw = interpolate(op.link, { ...env, loop: { name: FEED_ITEM, value: item } });
|
|
1879
|
+
const link = normalizeVirtual(raw, op.link.span, 'FEED LINK');
|
|
1880
|
+
const author = text('author');
|
|
1881
|
+
const summary = text('excerpt');
|
|
1882
|
+
const content = text('content');
|
|
1883
|
+
entries.push({
|
|
1884
|
+
title,
|
|
1885
|
+
url: pageUrl(op.base, link.path === '/' || !link.trailingSlash ? link.path : link.path + '/'),
|
|
1886
|
+
date,
|
|
1887
|
+
...(author !== undefined && author !== '' ? { author } : {}),
|
|
1888
|
+
...(summary !== undefined && summary !== '' ? { summary } : {}),
|
|
1889
|
+
...(content !== undefined ? { content } : {}),
|
|
1890
|
+
});
|
|
1891
|
+
});
|
|
1892
|
+
const xml = atomXml({ title: op.title, siteUrl: op.base + '/', selfUrl: pageUrl(op.base, op.dest), entries });
|
|
1893
|
+
return [[op.dest, makeFile(new TextEncoder().encode(xml), origin)]];
|
|
1894
|
+
}
|
|
1895
|
+
/**
|
|
1896
|
+
* The files an operation (other than SITEMAP, FINGERPRINT and CHECKLINKS) adds; `opId` is its memo identity
|
|
1897
|
+
* ('' without a memo), `treeKey` the key of the input tree (a RENDER with `${file …}` reads it).
|
|
1898
|
+
*/
|
|
1899
|
+
async function produce(op, tree, env, opId, treeKey) {
|
|
1900
|
+
if (op.kind === 'TRANSFORM')
|
|
1901
|
+
return produceTransform(op, env, opId);
|
|
1902
|
+
// A feed reads its list and what its LINK names.
|
|
1903
|
+
if (op.kind === 'FEED') {
|
|
1904
|
+
const key = env.memo ? memoKey('feed', opId, namesKey(env, operandNames([op.link], new Set([op.list])))) : undefined;
|
|
1905
|
+
return memoFiles(env, key, async () => feedFiles(op, env));
|
|
1906
|
+
}
|
|
1907
|
+
const origin = `${env.label}:${op.span.line} ${op.kind}`;
|
|
1908
|
+
if (op.kind === 'COPY') {
|
|
1909
|
+
const exclude = op.exclude ? IgnoreRules.fromPatterns(op.exclude) : undefined;
|
|
1910
|
+
const src = await readSource(op.source, origin, env, exclude);
|
|
1911
|
+
// --parents: the source's own path (inside the context, import or stage) is kept under the destination.
|
|
1912
|
+
const kept = op.parents ? sourcePath(op.source) : '';
|
|
1913
|
+
if (src.kind === 'file') {
|
|
1914
|
+
if (exclude) {
|
|
1915
|
+
throw new SitefileError('E_TYPE', `--exclude filters a directory, but '${src.label}' is a file`, op.source.span);
|
|
1916
|
+
}
|
|
1917
|
+
const intoDir = op.parents || op.destIsDir || tree.isDirectory(op.dest);
|
|
1918
|
+
const key = env.memo ? memoKey('copy', opId, sourceKey(op.source, src, env), String(intoDir)) : undefined;
|
|
1919
|
+
return memoFiles(env, key, async () => {
|
|
1920
|
+
if (intoDir && src.inline) {
|
|
1921
|
+
throw new SitefileError('E_PATH', `'${op.dest}' is a directory; an inline ${src.label} body needs a file destination`, op.span);
|
|
1922
|
+
}
|
|
1923
|
+
const dest = intoDir ? joinVirtual(op.dest, kept || src.basename) : op.dest;
|
|
1924
|
+
return [[dest, makeFile(src.bytes, origin, op.chmod ?? src.mode)]];
|
|
1925
|
+
});
|
|
1926
|
+
}
|
|
1927
|
+
const key = env.memo ? memoKey('copy', opId, sourceKey(op.source, src, env), 'tree') : undefined;
|
|
1928
|
+
return memoFiles(env, key, async () => {
|
|
1929
|
+
if (src.tree.size === 0)
|
|
1930
|
+
env.warn(exclude ? `${origin}: --exclude leaves no files of '${src.label}' to copy` : `${origin}: '${src.label}' contains no files`);
|
|
1931
|
+
return src.tree.mountedAt(kept ? joinVirtual(op.dest, kept) : op.dest).map(([p, n]) => [p, { ...n, mode: op.chmod ?? n.mode, origin }]);
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
const src = await readSource(op.template, origin, env);
|
|
1935
|
+
if (src.kind !== 'file') {
|
|
1936
|
+
throw new SitefileError('E_TYPE', `RENDER template '${src.label}' is a directory, not a file`, op.template.span);
|
|
1937
|
+
}
|
|
1938
|
+
const text = decodeUtf8(src.bytes, `template '${src.label}'`, op.template.span);
|
|
1939
|
+
const parts = src.inline
|
|
1940
|
+
? parseRenderTemplate(text, src.inline.file, src.inline.firstLine)
|
|
1941
|
+
: parseRenderTemplate(text, src.label);
|
|
1942
|
+
// A template with `${file …}` (itself or in a fragment) reads the tree, so the input tree is part of its key.
|
|
1943
|
+
const deps = env.memo ? renderDeps(parts, env) : undefined;
|
|
1944
|
+
const templateKey = env.memo ? memoKey('template', sourceKey(op.template, src, env), deps.files ? treeKey : '') : '';
|
|
1945
|
+
// What the template names, the FOR list and the destination's variables (ARG/LET values are in opId).
|
|
1946
|
+
const key = env.memo
|
|
1947
|
+
? memoKey('render', opId, templateKey, namesKey(env, op.each ? operandNames([op.each.dest], new Set(deps.names).add(op.each.list)) : deps.names))
|
|
1948
|
+
: undefined;
|
|
1949
|
+
const fileCtx = { tree: { get: (p) => tree.get(p)?.bytes, isDirectory: (p) => tree.isDirectory(p), renamed: (p) => env.renames?.get(p) }, inlineSvg: op.inlineSvg ?? false };
|
|
1950
|
+
return memoFiles(env, key, async (children) => {
|
|
1951
|
+
if (!op.each) {
|
|
1952
|
+
const out = renderTemplate(parts, (name, p, span) => lookup(name, p, span, env), op.escape, (name, p, span, loose) => valueOf(name, p, span, loose, env), { ...templateContext(env), ...fileCtx });
|
|
1953
|
+
return [[op.dest, makeFile(new TextEncoder().encode(out), origin)]];
|
|
1954
|
+
}
|
|
1955
|
+
// RENDER … FOR item IN list: one file per element, the destination interpolated per element.
|
|
1956
|
+
const { as, list, dest } = op.each;
|
|
1957
|
+
const items = valueOf(list, [], op.span, false, env);
|
|
1958
|
+
if (!Array.isArray(items)) {
|
|
1959
|
+
throw new SitefileError('E_TYPE', `FOR ${as} IN ${list}: '${list}' is not a list (import a directory AS markdown, or a JSON array)`, op.span);
|
|
1960
|
+
}
|
|
1961
|
+
// BY / PAGE: the loop runs over groups or pages instead of the elements.
|
|
1962
|
+
const what = `FOR ${as} IN ${list}${op.each.by !== undefined ? ` BY ${op.each.by}` : ''}`;
|
|
1963
|
+
const groups = op.each.by !== undefined ? groupBy(items, op.each.by, what, op.span) : undefined;
|
|
1964
|
+
if (items.length === 0 && !(op.each.page && !groups))
|
|
1965
|
+
env.warn(`${origin}: FOR ${as} IN ${list} renders nothing: '${list}' is empty`);
|
|
1966
|
+
else if (groups?.length === 0)
|
|
1967
|
+
env.warn(`${origin}: ${what} renders nothing: no element has '${op.each.by}'`);
|
|
1968
|
+
const destOf = (item, op_) => {
|
|
1969
|
+
let raw = '';
|
|
1970
|
+
for (const part of op_.parts) {
|
|
1971
|
+
if (part.kind === 'text')
|
|
1972
|
+
raw += part.value;
|
|
1973
|
+
else
|
|
1974
|
+
raw += String(part.name === as ? resolveJsonPath(item, part.path, as, part.span) : lookup(part.name, part.path, part.span, env));
|
|
1975
|
+
}
|
|
1976
|
+
const target = normalizeVirtual(raw, op_.span, `RENDER destination for ${as}`);
|
|
1977
|
+
if (target.trailingSlash) {
|
|
1978
|
+
throw new SitefileError('E_PATH', `RENDER destination '${raw}' must be a file path, not a directory`, op_.span);
|
|
1979
|
+
}
|
|
1980
|
+
return target.path;
|
|
1981
|
+
};
|
|
1982
|
+
// Each element with its destination. Pages get `url`, `prev` and `next` once every page's destination is known.
|
|
1983
|
+
const elements = [];
|
|
1984
|
+
if (op.each.page) {
|
|
1985
|
+
const { size, first } = op.each.page;
|
|
1986
|
+
const runs = groups ? groups.map((g) => ({ key: g['key'], items: g['items'] })) : [{ items }];
|
|
1987
|
+
for (const pages of paginate(runs, size)) {
|
|
1988
|
+
const paths = pages.map((pg) => destOf(pg, pg['number'] === 1 && first ? first : dest));
|
|
1989
|
+
const urls = paths.map((p) => pageUrl('', p));
|
|
1990
|
+
pages.forEach((pg, i) => {
|
|
1991
|
+
elements.push({ item: { ...pg, url: urls[i], prev: urls[i - 1] ?? '', next: urls[i + 1] ?? '' }, path: paths[i] });
|
|
1992
|
+
});
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
else {
|
|
1996
|
+
for (const item of groups ?? items)
|
|
1997
|
+
elements.push({ item, path: destOf(item, dest) });
|
|
1998
|
+
}
|
|
1999
|
+
// Incremental rebuilds: an element's page depends on the template, the element and the imports the
|
|
2000
|
+
// template names (other than the loop variable), so editing one document re-renders one page.
|
|
2001
|
+
const memo = env.memo;
|
|
2002
|
+
const refsKey = memo ? namesKey(env, deps.names, as) : '';
|
|
2003
|
+
const files = [];
|
|
2004
|
+
for (const { item, path: targetPath } of elements) {
|
|
2005
|
+
const itemEnv = { ...env, loop: { name: as, value: item } };
|
|
2006
|
+
const target = { path: targetPath };
|
|
2007
|
+
const page = () => {
|
|
2008
|
+
const out = renderTemplate(parts, (name, p, span) => lookup(name, p, span, itemEnv), op.escape, (name, p, span, loose) => valueOf(name, p, span, loose, itemEnv), { ...templateContext(itemEnv), ...fileCtx });
|
|
2009
|
+
return makeFile(new TextEncoder().encode(out), origin);
|
|
2010
|
+
};
|
|
2011
|
+
files.push([target.path, memo ? memo.memoize(memoKey('item', opId, templateKey, refsKey, valueKey(item)), page, children) : page()]);
|
|
2012
|
+
}
|
|
2013
|
+
return files;
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
/** `IMPORT … DRAFTS false`: leave out the elements whose `draft` is `true` (or the text `true`, as in a CSV cell). */
|
|
2017
|
+
function withoutDrafts(spec, res) {
|
|
2018
|
+
if (spec.drafts === undefined || res.kind !== 'value')
|
|
2019
|
+
return res;
|
|
2020
|
+
if (!Array.isArray(res.value)) {
|
|
2021
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: DRAFTS needs a list (a directory of documents, a CSV file or a JSON/YAML array), not a single value`, spec.span);
|
|
2022
|
+
}
|
|
2023
|
+
if (spec.drafts)
|
|
2024
|
+
return res;
|
|
2025
|
+
const isDraft = (d) => d !== null && typeof d === 'object' && !Array.isArray(d) && (d['draft'] === true || d['draft'] === 'true');
|
|
2026
|
+
return { ...res, value: res.value.filter((d) => !isDraft(d)) };
|
|
2027
|
+
}
|
|
2028
|
+
/** `IMPORT … SCHEMA <path>`: read and compile the schema file, then check the import's documents. */
|
|
2029
|
+
async function checkImportSchema(spec, res, local) {
|
|
2030
|
+
const schema = spec.schema;
|
|
2031
|
+
const what = `IMPORT ${spec.name}`;
|
|
2032
|
+
if (res.kind !== 'value') {
|
|
2033
|
+
throw new SitefileError('E_TYPE', `${what}: SCHEMA checks documents, but the import is a ${res.kind}`, spec.span);
|
|
2034
|
+
}
|
|
2035
|
+
const file = await local.readFile(schema.path, schema.span);
|
|
2036
|
+
const text = decodeUtf8(file.bytes, `schema '${schema.uri}'`, schema.span);
|
|
2037
|
+
let value;
|
|
2038
|
+
if (/\.json$/i.test(schema.path)) {
|
|
2039
|
+
try {
|
|
2040
|
+
value = JSON.parse(text);
|
|
2041
|
+
}
|
|
2042
|
+
catch (e) {
|
|
2043
|
+
throw new SitefileError('E_SCHEMA', `${what}: schema '${schema.uri}' is not valid JSON: ${e.message}`, schema.span, { cause: e });
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
else
|
|
2047
|
+
value = parseYaml(text, schema.uri);
|
|
2048
|
+
checkSchema(res.value, compileSchema(value, schema.uri, what, schema.span), what, res.format === 'csv', spec.span);
|
|
2049
|
+
}
|
|
2050
|
+
/** Apply an import's SORT BY to a markdown collection. */
|
|
2051
|
+
function sorted(spec, res) {
|
|
2052
|
+
if (!spec.sort || res.kind !== 'value')
|
|
2053
|
+
return res;
|
|
2054
|
+
if (!Array.isArray(res.value)) {
|
|
2055
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: SORT BY needs a list (a directory of documents, a CSV file or a JSON/YAML array), not a single value`, spec.span);
|
|
2056
|
+
}
|
|
2057
|
+
if (!res.value.every((d) => d !== null && typeof d === 'object' && !Array.isArray(d))) {
|
|
2058
|
+
throw new SitefileError('E_TYPE', `IMPORT ${spec.name}: SORT BY needs a list of objects with a '${spec.sort.key}' field`, spec.span);
|
|
2059
|
+
}
|
|
2060
|
+
return { ...res, value: sortDocuments(res.value, spec.sort.key, spec.sort.descending) };
|
|
2061
|
+
}
|
|
2062
|
+
/** The path `--parents` keeps: context-relative for a local source, inside the import or stage otherwise ('' for a root). */
|
|
2063
|
+
function sourcePath(ref) {
|
|
2064
|
+
if (ref.kind === 'local')
|
|
2065
|
+
return ref.path;
|
|
2066
|
+
if (ref.kind === 'inline')
|
|
2067
|
+
return '';
|
|
2068
|
+
return ref.path.replace(/^\/+/, '');
|
|
2069
|
+
}
|
|
2070
|
+
/** A directory source without the files `exclude` (COPY --exclude) matches, relative to its root. */
|
|
2071
|
+
function excluding(tree, exclude) {
|
|
2072
|
+
if (!exclude)
|
|
2073
|
+
return tree;
|
|
2074
|
+
return Tree.fromFiles(tree.entries().filter(([p]) => !exclude.excludes(p.slice(1))));
|
|
2075
|
+
}
|
|
2076
|
+
async function readSource(ref, origin, env, exclude) {
|
|
2077
|
+
if (ref.kind === 'inline') {
|
|
2078
|
+
const bytes = new TextEncoder().encode(ref.text);
|
|
2079
|
+
const inline = { file: env.label, firstLine: ref.bodyLine };
|
|
2080
|
+
return { kind: 'file', bytes, mode: 0o644, basename: '', label: `<<${ref.delimiter}`, inline };
|
|
2081
|
+
}
|
|
2082
|
+
if (ref.kind === 'local') {
|
|
2083
|
+
const label = './' + ref.path;
|
|
2084
|
+
if ((await env.local.stat(ref.path, ref.span)) === 'directory') {
|
|
2085
|
+
const t = await env.local.readTree(ref.path, origin, ref.span, exclude);
|
|
2086
|
+
return { kind: 'tree', tree: t.tree, label, digest: t.provenance.digest };
|
|
2087
|
+
}
|
|
2088
|
+
const f = await env.local.readFile(ref.path, ref.span);
|
|
2089
|
+
return { kind: 'file', bytes: f.bytes, mode: f.mode, basename: f.basename, label, digest: `${f.provenance.digest}\0${f.mode}\0${f.basename}` };
|
|
2090
|
+
}
|
|
2091
|
+
if (ref.kind === 'stage') {
|
|
2092
|
+
const tree = env.stages.get(ref.name) ?? fail(ref.name, ref.span);
|
|
2093
|
+
const label = `--from=${ref.name} ${ref.path}`;
|
|
2094
|
+
const node = tree.get(ref.path);
|
|
2095
|
+
if (node)
|
|
2096
|
+
return { kind: 'file', bytes: node.bytes, mode: node.mode, basename: basenameVirtual(ref.path), label };
|
|
2097
|
+
if (tree.isDirectory(ref.path))
|
|
2098
|
+
return { kind: 'tree', tree: excluding(tree.subtree(ref.path), exclude), label };
|
|
2099
|
+
throw new SitefileError('E_NOT_FOUND', `'${ref.path}' does not exist in stage '${ref.name}'`, ref.span);
|
|
2100
|
+
}
|
|
2101
|
+
const label = ref.path === '/' ? `${ref.name}:` : `${ref.name}:${ref.path}`;
|
|
2102
|
+
const res = env.resources.get(ref.name);
|
|
2103
|
+
if (!res)
|
|
2104
|
+
throw new SitefileError('E_UNDEFINED', `unknown import '${ref.name}'`, ref.span);
|
|
2105
|
+
switch (res.kind) {
|
|
2106
|
+
case 'value':
|
|
2107
|
+
throw new SitefileError('E_TYPE', `import '${ref.name}' is a ${res.format} value: RENDER templates can read it, but it cannot be copied (import it AS file to copy it)`, ref.span);
|
|
2108
|
+
case 'file':
|
|
2109
|
+
if (ref.path !== '/')
|
|
2110
|
+
throw new SitefileError('E_PATH', `import '${ref.name}' is a single file; use '${ref.name}:'`, ref.span);
|
|
2111
|
+
return { kind: 'file', bytes: res.bytes, mode: res.mode, basename: res.basename, label };
|
|
2112
|
+
case 'tree': {
|
|
2113
|
+
const node = res.tree.get(ref.path);
|
|
2114
|
+
if (node)
|
|
2115
|
+
return { kind: 'file', bytes: node.bytes, mode: node.mode, basename: basenameVirtual(ref.path), label };
|
|
2116
|
+
if (res.tree.isDirectory(ref.path))
|
|
2117
|
+
return { kind: 'tree', tree: excluding(res.tree.subtree(ref.path), exclude), label };
|
|
2118
|
+
throw new SitefileError('E_NOT_FOUND', `'${ref.path}' does not exist in import '${ref.name}'`, ref.span);
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
function lookup(name, p, span, env) {
|
|
2123
|
+
if (env.loop && env.loop.name === name)
|
|
2124
|
+
return resolveJsonPath(env.loop.value, p, name, span);
|
|
2125
|
+
const v = env.vars.get(name);
|
|
2126
|
+
if (v !== undefined) {
|
|
2127
|
+
if (p.length)
|
|
2128
|
+
throw new SitefileError('E_TYPE', `'${name}' is an ARG/LET value; dotted access only works on JSON imports`, span);
|
|
2129
|
+
return v;
|
|
2130
|
+
}
|
|
2131
|
+
const res = env.resources.get(name);
|
|
2132
|
+
if (!res)
|
|
2133
|
+
throw new SitefileError('E_UNDEFINED', `'${name}' is not an ARG, LET or IMPORT`, span);
|
|
2134
|
+
if (res.kind !== 'value') {
|
|
2135
|
+
throw new SitefileError('E_TYPE', `import '${name}' is a ${res.kind}; templates can only read value imports (text, json, yaml, csv, markdown, wbn)`, span);
|
|
2136
|
+
}
|
|
2137
|
+
if (res.format === 'html')
|
|
2138
|
+
throw fragmentNotValue(name, span);
|
|
2139
|
+
if (res.format !== 'text')
|
|
2140
|
+
return resolveJsonPath(res.value, p, name, span);
|
|
2141
|
+
if (p.length)
|
|
2142
|
+
throw new SitefileError('E_TYPE', `import '${name}' is text; dotted access only works on structured imports (json, yaml, csv, markdown, wbn)`, span);
|
|
2143
|
+
return res.value;
|
|
2144
|
+
}
|
|
2145
|
+
/** Any value (lists, documents, scalars) for template blocks and FOR. `loose`: missing → undefined. */
|
|
2146
|
+
function valueOf(name, p, span, loose, env) {
|
|
2147
|
+
if (env.loop && env.loop.name === name)
|
|
2148
|
+
return walk(env.loop.value, p, name, span, loose);
|
|
2149
|
+
const v = env.vars.get(name);
|
|
2150
|
+
if (v !== undefined)
|
|
2151
|
+
return walk(v, p, name, span, loose);
|
|
2152
|
+
const res = env.resources.get(name);
|
|
2153
|
+
if (!res) {
|
|
2154
|
+
if (loose)
|
|
2155
|
+
return undefined;
|
|
2156
|
+
throw new SitefileError('E_UNDEFINED', `'${name}' is not an ARG, LET or IMPORT`, span);
|
|
2157
|
+
}
|
|
2158
|
+
if (res.kind !== 'value') {
|
|
2159
|
+
throw new SitefileError('E_TYPE', `import '${name}' is a ${res.kind}; templates can only read value imports (text, json, yaml, csv, markdown, wbn)`, span);
|
|
2160
|
+
}
|
|
2161
|
+
return walk(res.value, p, name, span, loose);
|
|
2162
|
+
}
|
|
2163
|
+
// ---------------------------------------------------------------- helpers
|
|
2164
|
+
async function mapLimit(items, limit, fn) {
|
|
2165
|
+
const out = new Array(items.length);
|
|
2166
|
+
let next = 0;
|
|
2167
|
+
const worker = async () => {
|
|
2168
|
+
while (next < items.length) {
|
|
2169
|
+
const i = next++;
|
|
2170
|
+
out[i] = await fn(items[i], i);
|
|
2171
|
+
}
|
|
2172
|
+
};
|
|
2173
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
|
|
2174
|
+
return out;
|
|
2175
|
+
}
|
|
2176
|
+
/** realpath of the nearest existing ancestor, with the missing tail re-joined. */
|
|
2177
|
+
async function realpathLoose(p) {
|
|
2178
|
+
const tail = [];
|
|
2179
|
+
let cur = p;
|
|
2180
|
+
for (;;) {
|
|
2181
|
+
try {
|
|
2182
|
+
return path.join(await realpath(cur), ...tail.reverse());
|
|
2183
|
+
}
|
|
2184
|
+
catch {
|
|
2185
|
+
const parent = path.dirname(cur);
|
|
2186
|
+
if (parent === cur)
|
|
2187
|
+
return p;
|
|
2188
|
+
tail.push(path.basename(cur));
|
|
2189
|
+
cur = parent;
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
//# sourceMappingURL=executor.js.map
|