@johnhenry/sitefile 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/CHANGELOG.md +152 -0
  2. package/LICENSE +21 -0
  3. package/README.md +513 -2
  4. package/lib/src/archive-writers.d.ts +31 -0
  5. package/lib/src/archive-writers.js +197 -0
  6. package/lib/src/attest.d.ts +126 -0
  7. package/lib/src/attest.js +220 -0
  8. package/lib/src/bake.d.ts +74 -0
  9. package/lib/src/bake.js +372 -0
  10. package/lib/src/base-image.d.ts +76 -0
  11. package/lib/src/base-image.js +138 -0
  12. package/lib/src/build-args.d.ts +26 -0
  13. package/lib/src/build-args.js +164 -0
  14. package/lib/src/cache-limit.d.ts +54 -0
  15. package/lib/src/cache-limit.js +239 -0
  16. package/lib/src/casefold-data.d.ts +2 -0
  17. package/lib/src/casefold-data.js +217 -0
  18. package/lib/src/cbor.d.ts +39 -0
  19. package/lib/src/cbor.js +174 -0
  20. package/lib/src/checklinks.d.ts +42 -0
  21. package/lib/src/checklinks.js +120 -0
  22. package/lib/src/cli/bake.d.ts +2 -0
  23. package/lib/src/cli/bake.js +179 -0
  24. package/lib/src/cli/build-image.d.ts +21 -0
  25. package/lib/src/cli/build-image.js +42 -0
  26. package/lib/src/cli/cache.d.ts +8 -0
  27. package/lib/src/cli/cache.js +119 -0
  28. package/lib/src/cli/common.d.ts +29 -0
  29. package/lib/src/cli/common.js +100 -0
  30. package/lib/src/cli/git-context.d.ts +18 -0
  31. package/lib/src/cli/git-context.js +48 -0
  32. package/lib/src/cli/images.d.ts +6 -0
  33. package/lib/src/cli/images.js +285 -0
  34. package/lib/src/cli/index.d.ts +10 -0
  35. package/lib/src/cli/index.js +95 -0
  36. package/lib/src/cli/network.d.ts +34 -0
  37. package/lib/src/cli/network.js +56 -0
  38. package/lib/src/cli/outdated.d.ts +5 -0
  39. package/lib/src/cli/outdated.js +99 -0
  40. package/lib/src/cli/registry.d.ts +2 -0
  41. package/lib/src/cli/registry.js +127 -0
  42. package/lib/src/cli/serve-worker.d.ts +2 -0
  43. package/lib/src/cli/serve-worker.js +42 -0
  44. package/lib/src/cli/servers.d.ts +15 -0
  45. package/lib/src/cli/servers.js +467 -0
  46. package/lib/src/cli/vendor.d.ts +2 -0
  47. package/lib/src/cli/vendor.js +97 -0
  48. package/lib/src/cli/watch.d.ts +26 -0
  49. package/lib/src/cli/watch.js +81 -0
  50. package/lib/src/cli.d.ts +12 -0
  51. package/lib/src/cli.js +522 -0
  52. package/lib/src/csv.d.ts +27 -0
  53. package/lib/src/csv.js +117 -0
  54. package/lib/src/deflate.d.ts +76 -0
  55. package/lib/src/deflate.js +604 -0
  56. package/lib/src/digest.d.ts +3 -0
  57. package/lib/src/digest.js +7 -0
  58. package/lib/src/errors.d.ts +56 -0
  59. package/lib/src/errors.js +29 -0
  60. package/lib/src/executor.d.ts +393 -0
  61. package/lib/src/executor.js +2201 -0
  62. package/lib/src/explain.d.ts +64 -0
  63. package/lib/src/explain.js +445 -0
  64. package/lib/src/export-formats.d.ts +23 -0
  65. package/lib/src/export-formats.js +63 -0
  66. package/lib/src/exporter.d.ts +103 -0
  67. package/lib/src/exporter.js +676 -0
  68. package/lib/src/fingerprint.d.ts +40 -0
  69. package/lib/src/fingerprint.js +168 -0
  70. package/lib/src/frontmatter.d.ts +22 -0
  71. package/lib/src/frontmatter.js +45 -0
  72. package/lib/src/index.d.ts +13 -0
  73. package/lib/src/index.js +13 -0
  74. package/lib/src/language/ast.d.ts +360 -0
  75. package/lib/src/language/ast.js +57 -0
  76. package/lib/src/language/parser.d.ts +3 -0
  77. package/lib/src/language/parser.js +612 -0
  78. package/lib/src/language/scanner.d.ts +110 -0
  79. package/lib/src/language/scanner.js +449 -0
  80. package/lib/src/language/validate.d.ts +13 -0
  81. package/lib/src/language/validate.js +367 -0
  82. package/lib/src/links.d.ts +79 -0
  83. package/lib/src/links.js +583 -0
  84. package/lib/src/lint.d.ts +10 -0
  85. package/lib/src/lint.js +223 -0
  86. package/lib/src/markdown.d.ts +29 -0
  87. package/lib/src/markdown.js +612 -0
  88. package/lib/src/media.d.ts +82 -0
  89. package/lib/src/media.js +650 -0
  90. package/lib/src/memo.d.ts +57 -0
  91. package/lib/src/memo.js +93 -0
  92. package/lib/src/oci.d.ts +178 -0
  93. package/lib/src/oci.js +380 -0
  94. package/lib/src/outdated.d.ts +46 -0
  95. package/lib/src/outdated.js +195 -0
  96. package/lib/src/pages.d.ts +10 -0
  97. package/lib/src/pages.js +54 -0
  98. package/lib/src/paths.d.ts +39 -0
  99. package/lib/src/paths.js +101 -0
  100. package/lib/src/planner.d.ts +297 -0
  101. package/lib/src/planner.js +410 -0
  102. package/lib/src/plugins/cache.d.ts +96 -0
  103. package/lib/src/plugins/cache.js +325 -0
  104. package/lib/src/plugins/exec.d.ts +51 -0
  105. package/lib/src/plugins/exec.js +144 -0
  106. package/lib/src/plugins/host.d.ts +75 -0
  107. package/lib/src/plugins/host.js +277 -0
  108. package/lib/src/plugins/wasm.d.ts +23 -0
  109. package/lib/src/plugins/wasm.js +249 -0
  110. package/lib/src/profile.d.ts +42 -0
  111. package/lib/src/profile.js +68 -0
  112. package/lib/src/ref.d.ts +40 -0
  113. package/lib/src/ref.js +79 -0
  114. package/lib/src/registry-auth.d.ts +32 -0
  115. package/lib/src/registry-auth.js +151 -0
  116. package/lib/src/registry-cache.d.ts +71 -0
  117. package/lib/src/registry-cache.js +333 -0
  118. package/lib/src/registry-ops.d.ts +51 -0
  119. package/lib/src/registry-ops.js +248 -0
  120. package/lib/src/registry.d.ts +115 -0
  121. package/lib/src/registry.js +524 -0
  122. package/lib/src/render.d.ts +134 -0
  123. package/lib/src/render.js +576 -0
  124. package/lib/src/resources/archive.d.ts +11 -0
  125. package/lib/src/resources/archive.js +306 -0
  126. package/lib/src/resources/bundle-data.d.ts +16 -0
  127. package/lib/src/resources/bundle-data.js +50 -0
  128. package/lib/src/resources/convert.d.ts +12 -0
  129. package/lib/src/resources/convert.js +63 -0
  130. package/lib/src/resources/documents.d.ts +35 -0
  131. package/lib/src/resources/documents.js +122 -0
  132. package/lib/src/resources/git.d.ts +103 -0
  133. package/lib/src/resources/git.js +720 -0
  134. package/lib/src/resources/https.d.ts +74 -0
  135. package/lib/src/resources/https.js +413 -0
  136. package/lib/src/resources/ignore.d.ts +33 -0
  137. package/lib/src/resources/ignore.js +119 -0
  138. package/lib/src/resources/local.d.ts +64 -0
  139. package/lib/src/resources/local.js +248 -0
  140. package/lib/src/resources/lockfile.d.ts +52 -0
  141. package/lib/src/resources/lockfile.js +172 -0
  142. package/lib/src/resources/netpolicy.d.ts +6 -0
  143. package/lib/src/resources/netpolicy.js +82 -0
  144. package/lib/src/resources/network.d.ts +70 -0
  145. package/lib/src/resources/network.js +304 -0
  146. package/lib/src/resources/types.d.ts +69 -0
  147. package/lib/src/resources/types.js +13 -0
  148. package/lib/src/resources/vendor.d.ts +94 -0
  149. package/lib/src/resources/vendor.js +296 -0
  150. package/lib/src/resources/wbn-reader.d.ts +44 -0
  151. package/lib/src/resources/wbn-reader.js +302 -0
  152. package/lib/src/schema.d.ts +25 -0
  153. package/lib/src/schema.js +169 -0
  154. package/lib/src/secrets.d.ts +21 -0
  155. package/lib/src/secrets.js +103 -0
  156. package/lib/src/serve-rules.d.ts +29 -0
  157. package/lib/src/serve-rules.js +123 -0
  158. package/lib/src/server.d.ts +45 -0
  159. package/lib/src/server.js +246 -0
  160. package/lib/src/servers.d.ts +85 -0
  161. package/lib/src/servers.js +225 -0
  162. package/lib/src/site.d.ts +60 -0
  163. package/lib/src/site.js +169 -0
  164. package/lib/src/store.d.ts +151 -0
  165. package/lib/src/store.js +596 -0
  166. package/lib/src/tree.d.ts +50 -0
  167. package/lib/src/tree.js +134 -0
  168. package/lib/src/values.d.ts +7 -0
  169. package/lib/src/values.js +37 -0
  170. package/lib/src/vendor.d.ts +19 -0
  171. package/lib/src/vendor.js +145 -0
  172. package/lib/src/watch.d.ts +44 -0
  173. package/lib/src/watch.js +137 -0
  174. package/lib/src/wbn.d.ts +26 -0
  175. package/lib/src/wbn.js +241 -0
  176. package/lib/src/yaml.d.ts +34 -0
  177. package/lib/src/yaml.js +562 -0
  178. package/package.json +59 -4
@@ -0,0 +1,277 @@
1
+ import { spawn } from 'node:child_process';
2
+ import os from 'node:os';
3
+ import path from 'node:path';
4
+ import { SitefileError } from '../errors.js';
5
+ import { WasmPlugin } from './wasm.js';
6
+ export const DEFAULT_PLUGIN_POLICY = { timeoutMs: 60_000, maxOutputBytes: 64 * 1024 * 1024 };
7
+ /** A directory TRANSFORM runs at most this many wasm or tool calls at a time (a JS plugin, one). */
8
+ export const TRANSFORM_JOBS = Math.max(1, Math.min(4, os.availableParallelism()));
9
+ /** Prefix that marks protocol lines on the child's stdout; anything else is plugin output. */
10
+ const MARK = '\u0000SITEFILE:';
11
+ /** Runs inside the sandboxed process (passed with -e, so it needs no file access). */
12
+ const RUNNER = String.raw `
13
+ const MARK = '\u0000SITEFILE:';
14
+ const send = (msg) => process.stdout.write(MARK + JSON.stringify(msg) + '\n');
15
+ // Plugin console output goes to stderr so stdout stays a clean channel.
16
+ for (const k of ['log', 'info', 'debug']) console[k] = (...a) => process.stderr.write(a.map(String).join(' ') + '\n');
17
+ const toB64 = (v) => Buffer.from(typeof v === 'string' ? Buffer.from(v, 'utf8') : v).toString('base64');
18
+ const isBytes = (v) => typeof v === 'string' || v instanceof Uint8Array;
19
+ let transform;
20
+ let buf = '';
21
+ process.stdin.setEncoding('utf8');
22
+ process.stdin.on('data', (chunk) => {
23
+ buf += chunk;
24
+ let nl;
25
+ while ((nl = buf.indexOf('\n')) !== -1) {
26
+ const line = buf.slice(0, nl);
27
+ buf = buf.slice(nl + 1);
28
+ handle(JSON.parse(line)).catch((e) => send({ type: 'fatal', message: String(e && e.stack || e) }));
29
+ }
30
+ });
31
+ async function handle(msg) {
32
+ if (msg.type === 'load') {
33
+ try {
34
+ const mod = await import(msg.url);
35
+ const api = mod.default && typeof mod.default.transform === 'function' ? mod.default : mod;
36
+ const version = api.apiVersion ?? mod.apiVersion ?? 1;
37
+ if (typeof api.transform !== 'function') throw new Error('the module must export transform(request), or a default object with transform');
38
+ if (version !== 1) throw new Error('unsupported plugin apiVersion ' + JSON.stringify(version) + ' (this Sitefile supports 1)');
39
+ transform = api.transform;
40
+ // 'export const cache = false' opts a (non-deterministic) plugin out of the result cache.
41
+ const cache = api.cache ?? mod.cache;
42
+ send({ type: 'ready', cache: cache !== false });
43
+ } catch (e) {
44
+ send({ type: 'load-error', message: e && e.message ? e.message : String(e) });
45
+ }
46
+ return;
47
+ }
48
+ if (msg.type === 'transform') {
49
+ const req = msg.request;
50
+ if (req.input) {
51
+ req.input.bytes = new Uint8Array(Buffer.from(req.input.bytes, 'base64'));
52
+ }
53
+ try {
54
+ let out = await transform(req);
55
+ if (isBytes(out)) out = { bytes: out };
56
+ if (out && isBytes(out.bytes)) return send({ type: 'result', id: msg.id, bytes: toB64(out.bytes) });
57
+ if (out && out.files && typeof out.files === 'object') {
58
+ const files = {};
59
+ for (const [k, v] of Object.entries(out.files)) {
60
+ if (!isBytes(v)) throw new Error('files[' + JSON.stringify(k) + '] must be a string or Uint8Array');
61
+ files[k] = toB64(v);
62
+ }
63
+ return send({ type: 'result', id: msg.id, files });
64
+ }
65
+ throw new Error('transform() must return a string, a Uint8Array, { bytes } or { files }');
66
+ } catch (e) {
67
+ send({ type: 'error', id: msg.id, message: e && e.message ? e.message : String(e) });
68
+ }
69
+ }
70
+ }
71
+ `;
72
+ class PluginProcess {
73
+ name;
74
+ policy;
75
+ log;
76
+ child;
77
+ buf = '';
78
+ stderr = '';
79
+ nextId = 1;
80
+ pending = new Map();
81
+ /** False when the module exports `cache = false`; known once it has loaded. */
82
+ cacheable = true;
83
+ ready;
84
+ exited;
85
+ constructor(name, file, policy, log, span) {
86
+ this.name = name;
87
+ this.policy = policy;
88
+ this.log = log;
89
+ const dir = path.dirname(file);
90
+ this.child = spawn(process.execPath, ['--permission', `--allow-fs-read=${dir}`, '--disable-warning=ExperimentalWarning', '--input-type=module', '-e', RUNNER], { cwd: dir, env: {}, stdio: ['pipe', 'pipe', 'pipe'] });
91
+ this.child.stdout.setEncoding('utf8');
92
+ this.child.stderr.setEncoding('utf8');
93
+ this.child.stdout.on('data', (chunk) => this.onStdout(chunk));
94
+ this.child.stderr.on('data', (chunk) => {
95
+ this.stderr = (this.stderr + chunk).slice(-8192);
96
+ for (const line of chunk.split('\n'))
97
+ if (line.trim())
98
+ this.log(line);
99
+ });
100
+ let onReady = () => { };
101
+ let onLoadError = () => { };
102
+ this.ready = new Promise((res, rej) => {
103
+ onReady = res;
104
+ onLoadError = rej;
105
+ });
106
+ // Awaited by call(); marking it handled keeps an unused failed plugin from crashing the process.
107
+ this.ready.catch(() => { });
108
+ this.onReadyMsg = onReady;
109
+ this.onLoadError = (m) => onLoadError(new SitefileError('E_PLUGIN', `PLUGIN ${name}: ${m}`, span));
110
+ this.child.on('error', (e) => this.fail(`could not start: ${e.message}`));
111
+ this.child.on('exit', (code, signal) => this.fail(signal ? `stopped by ${signal}` : `exited with code ${code}`));
112
+ this.write({ type: 'load', url: new URL(`file://${file}`).href });
113
+ }
114
+ onReadyMsg;
115
+ onLoadError;
116
+ write(msg) {
117
+ this.child.stdin.write(JSON.stringify(msg) + '\n');
118
+ }
119
+ onStdout(chunk) {
120
+ this.buf += chunk;
121
+ let nl;
122
+ while ((nl = this.buf.indexOf('\n')) !== -1) {
123
+ const line = this.buf.slice(0, nl);
124
+ this.buf = this.buf.slice(nl + 1);
125
+ if (!line.startsWith(MARK)) {
126
+ if (line.trim())
127
+ this.log(line);
128
+ continue;
129
+ }
130
+ if (line.length > this.policy.maxOutputBytes * 1.4 + 1024) {
131
+ this.fail(`a result is larger than ${this.policy.maxOutputBytes} bytes`);
132
+ return;
133
+ }
134
+ this.onMessage(JSON.parse(line.slice(MARK.length)));
135
+ }
136
+ if (this.buf.length > this.policy.maxOutputBytes * 1.4 + 1024)
137
+ this.fail(`a result is larger than ${this.policy.maxOutputBytes} bytes`);
138
+ }
139
+ onMessage(msg) {
140
+ switch (msg['type']) {
141
+ case 'ready':
142
+ this.cacheable = msg['cache'] !== false;
143
+ this.onReadyMsg();
144
+ return;
145
+ case 'load-error':
146
+ this.onLoadError(String(msg['message']));
147
+ return;
148
+ case 'fatal':
149
+ this.fail(String(msg['message']));
150
+ return;
151
+ }
152
+ const p = this.pending.get(msg['id']);
153
+ if (!p)
154
+ return;
155
+ this.pending.delete(msg['id']);
156
+ if (msg['type'] === 'error')
157
+ return p.reject(new Error(String(msg['message'])));
158
+ try {
159
+ if (typeof msg['bytes'] === 'string')
160
+ return p.resolve({ kind: 'file', bytes: this.decode(msg['bytes']) });
161
+ const files = Object.entries(msg['files']).map(([k, v]) => [k, this.decode(v)]);
162
+ p.resolve({ kind: 'files', files });
163
+ }
164
+ catch (e) {
165
+ p.reject(e);
166
+ }
167
+ }
168
+ decode(b64) {
169
+ const bytes = new Uint8Array(Buffer.from(b64, 'base64'));
170
+ if (bytes.byteLength > this.policy.maxOutputBytes)
171
+ throw new Error(`result is larger than ${this.policy.maxOutputBytes} bytes`);
172
+ return bytes;
173
+ }
174
+ fail(reason) {
175
+ if (this.exited)
176
+ return;
177
+ this.exited = reason;
178
+ const tail = this.stderr.trim() ? `\n${this.stderr.trim().split('\n').slice(-5).join('\n')}` : '';
179
+ const err = new Error(`${reason}${tail}`);
180
+ this.onLoadError(err.message);
181
+ for (const p of this.pending.values())
182
+ p.reject(err);
183
+ this.pending.clear();
184
+ if (this.child.exitCode === null)
185
+ this.child.kill('SIGKILL');
186
+ }
187
+ async call(request, span) {
188
+ await this.ready;
189
+ if (this.exited)
190
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${this.name} ${this.exited}`, span);
191
+ const id = this.nextId++;
192
+ const wire = {
193
+ ...request,
194
+ ...(request.input ? { input: { ...request.input, bytes: Buffer.from(request.input.bytes).toString('base64') } } : {}),
195
+ };
196
+ return new Promise((resolve, reject) => {
197
+ const timer = setTimeout(() => {
198
+ this.fail(`timed out after ${this.policy.timeoutMs} ms`);
199
+ }, this.policy.timeoutMs);
200
+ this.pending.set(id, {
201
+ resolve: (r) => {
202
+ clearTimeout(timer);
203
+ resolve(r);
204
+ },
205
+ reject: (e) => {
206
+ clearTimeout(timer);
207
+ reject(new SitefileError('E_PLUGIN', `PLUGIN ${this.name}: ${e.message}`, span));
208
+ },
209
+ });
210
+ this.write({ type: 'transform', id, request: wire });
211
+ });
212
+ }
213
+ close() {
214
+ this.exited ??= 'closed';
215
+ this.child.stdin.end();
216
+ if (this.child.exitCode === null)
217
+ this.child.kill();
218
+ }
219
+ }
220
+ export class PluginHost {
221
+ policy;
222
+ log;
223
+ registered = new Map();
224
+ procs = new Map();
225
+ stats = { spawned: 0, calls: 0 };
226
+ constructor(policy = DEFAULT_PLUGIN_POLICY, log = () => { }) {
227
+ this.policy = policy;
228
+ this.log = log;
229
+ }
230
+ /**
231
+ * `file` is the plugin's absolute real path (already sandbox- and integrity-checked).
232
+ * For a `.wasm` plugin, pass the checked module bytes. Runners start lazily.
233
+ */
234
+ register(name, file, span, wasm) {
235
+ if (!this.registered.has(name))
236
+ this.registered.set(name, { file, span, ...(wasm ? { wasm } : {}) });
237
+ }
238
+ /** Whether a registered plugin is a WebAssembly module. */
239
+ isWasm(name) {
240
+ return this.registered.get(name)?.wasm !== undefined;
241
+ }
242
+ /** A registered wasm plugin's module bytes. */
243
+ wasmBytes(name) {
244
+ return this.registered.get(name)?.wasm;
245
+ }
246
+ async call(name, request, span) {
247
+ let p = this.procs.get(name);
248
+ if (!p) {
249
+ const reg = this.registered.get(name);
250
+ if (!reg)
251
+ throw new SitefileError('E_UNDEFINED', `unknown plugin '${name}'`, span);
252
+ p = reg.wasm
253
+ ? new WasmPlugin(name, reg.wasm, this.policy, TRANSFORM_JOBS)
254
+ : new PluginProcess(name, reg.file, this.policy, (l) => this.log(`[plugin ${name}] ${l}`), reg.span);
255
+ this.procs.set(name, p);
256
+ this.stats.spawned++;
257
+ }
258
+ this.stats.calls++;
259
+ return p.call(request, span);
260
+ }
261
+ /** The folder a registered plugin lives in. */
262
+ folderOf(name) {
263
+ const f = this.registered.get(name)?.file;
264
+ return f === undefined ? undefined : path.dirname(f);
265
+ }
266
+ /** Whether results of this plugin may be cached (false after it exported `cache = false`). */
267
+ cacheable(name) {
268
+ return this.procs.get(name)?.cacheable ?? true;
269
+ }
270
+ close() {
271
+ for (const p of this.procs.values())
272
+ p.close();
273
+ this.procs.clear();
274
+ this.registered.clear();
275
+ }
276
+ }
277
+ //# sourceMappingURL=host.js.map
@@ -0,0 +1,23 @@
1
+ import { type SourceSpan } from '../errors.js';
2
+ import type { PluginPolicy, PluginRequest, PluginResult } from './host.js';
3
+ /** Compile and check a module against ABI v1; throws E_PLUGIN with a precise reason. */
4
+ export declare function checkWasmPlugin(name: string, bytes: Uint8Array, span?: SourceSpan): void;
5
+ /**
6
+ * A wasm plugin: up to `lanes` workers, so a directory TRANSFORM can run several calls at once.
7
+ * A call waits for a free worker, and its time limit starts when that worker takes it.
8
+ */
9
+ export declare class WasmPlugin {
10
+ readonly name: string;
11
+ private readonly bytes;
12
+ private readonly policy;
13
+ private readonly lanes;
14
+ /** No imports means no clock or randomness: results depend only on the request. */
15
+ readonly cacheable = true;
16
+ private readonly idle;
17
+ private readonly all;
18
+ private readonly waiting;
19
+ constructor(name: string, bytes: Uint8Array, policy: PluginPolicy, lanes?: number);
20
+ private lane;
21
+ call(request: PluginRequest, span?: SourceSpan): Promise<PluginResult>;
22
+ close(): void;
23
+ }
@@ -0,0 +1,249 @@
1
+ import { Worker } from 'node:worker_threads';
2
+ import { SitefileError } from '../errors.js';
3
+ /**
4
+ * WebAssembly renderer plugins (ABI v1).
5
+ *
6
+ * A plugin is a `wasm32-unknown-unknown` module with **no imports**, so it has
7
+ * no way to reach files, the network, the clock or randomness: it can only
8
+ * compute. It exports:
9
+ *
10
+ * memory its linear memory
11
+ * sitefile_alloc(len: i32) -> i32 reserve `len` bytes, return a pointer
12
+ * sitefile_transform(ptr: i32, len: i32) -> i64
13
+ * read the request at ptr..ptr+len and
14
+ * return (resultPtr << 32) | resultLen
15
+ * sitefile_api_version() -> i32 optional; must return 1
16
+ *
17
+ * Request and result are UTF-8 JSON. The request matches the JS plugin
18
+ * request (docs/PLUGINS.md) with `input.bytes` base64-encoded. The result is
19
+ * one of `{ "bytes": <base64> }`, `{ "text": "…" }`,
20
+ * `{ "files": { "a.txt": <base64>, … } }` or `{ "error": "message" }`.
21
+ *
22
+ * Each call gets a fresh instance (no state carries over between calls), in a
23
+ * worker thread that is terminated when a call exceeds the time limit.
24
+ */
25
+ const REQUIRED_EXPORTS = [
26
+ ['memory', 'memory'],
27
+ ['sitefile_alloc', 'function'],
28
+ ['sitefile_transform', 'function'],
29
+ ];
30
+ /** Compile and check a module against ABI v1; throws E_PLUGIN with a precise reason. */
31
+ export function checkWasmPlugin(name, bytes, span) {
32
+ let mod;
33
+ try {
34
+ mod = new WebAssembly.Module(bytes);
35
+ }
36
+ catch (e) {
37
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${name}: not a valid WebAssembly module (${e.message})`, span);
38
+ }
39
+ const imports = WebAssembly.Module.imports(mod);
40
+ if (imports.length > 0) {
41
+ const list = imports.map((i) => `${i.module}.${i.name} (${i.kind})`).join(', ');
42
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${name}: WebAssembly plugins can't import anything, so they can't reach files, the network or the clock; this module imports ${list}`, span);
43
+ }
44
+ const exports = new Map(WebAssembly.Module.exports(mod).map((e) => [e.name, e.kind]));
45
+ for (const [ename, kind] of REQUIRED_EXPORTS) {
46
+ if (exports.get(ename) !== kind) {
47
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${name}: a WebAssembly plugin must export ${kind} '${ename}' (see docs/PLUGINS.md)`, span);
48
+ }
49
+ }
50
+ }
51
+ /** Runs inside the worker; plain JS (eval'd), so it can't use TypeScript. */
52
+ const WORKER = String.raw `
53
+ const { parentPort, workerData } = require('node:worker_threads');
54
+ const mod = new WebAssembly.Module(workerData.bytes);
55
+ parentPort.on('message', ({ id, request }) => {
56
+ try {
57
+ const { exports: ex } = new WebAssembly.Instance(mod, {});
58
+ if (typeof ex.sitefile_api_version === 'function' && ex.sitefile_api_version() !== 1) {
59
+ throw new Error('sitefile_api_version() must return 1');
60
+ }
61
+ const ptr = ex.sitefile_alloc(request.length) >>> 0;
62
+ if (ptr + request.length > ex.memory.buffer.byteLength) throw new Error('sitefile_alloc returned a pointer outside memory');
63
+ new Uint8Array(ex.memory.buffer, ptr, request.length).set(request);
64
+ const packed = BigInt.asUintN(64, BigInt(ex.sitefile_transform(ptr, request.length)));
65
+ const outPtr = Number(packed >> 32n);
66
+ const outLen = Number(packed & 0xffffffffn);
67
+ const size = ex.memory.buffer.byteLength;
68
+ if (size > workerData.maxMemory) throw new Error('used ' + size + ' bytes of memory, more than the ' + workerData.maxMemory + ' allowed');
69
+ if (outPtr + outLen > size) throw new Error('sitefile_transform returned a result outside memory');
70
+ const out = new Uint8Array(ex.memory.buffer, outPtr, outLen).slice();
71
+ parentPort.postMessage({ id, out }, [out.buffer]);
72
+ } catch (e) {
73
+ parentPort.postMessage({ id, error: e && e.message ? e.message : String(e) });
74
+ }
75
+ });
76
+ `;
77
+ /** Largest linear memory a wasm plugin may use (default 1 GiB). */
78
+ const DEFAULT_MAX_MEMORY = 1024 * 1024 * 1024;
79
+ /**
80
+ * A wasm plugin: up to `lanes` workers, so a directory TRANSFORM can run several calls at once.
81
+ * A call waits for a free worker, and its time limit starts when that worker takes it.
82
+ */
83
+ export class WasmPlugin {
84
+ name;
85
+ bytes;
86
+ policy;
87
+ lanes;
88
+ /** No imports means no clock or randomness: results depend only on the request. */
89
+ cacheable = true;
90
+ idle = [];
91
+ all = [];
92
+ waiting = [];
93
+ constructor(name, bytes, policy, lanes = 1) {
94
+ this.name = name;
95
+ this.bytes = bytes;
96
+ this.policy = policy;
97
+ this.lanes = lanes;
98
+ }
99
+ async lane() {
100
+ const free = this.idle.pop();
101
+ if (free)
102
+ return free;
103
+ if (this.all.length < Math.max(1, this.lanes)) {
104
+ const lane = new WasmLane(this.name, this.bytes, this.policy);
105
+ this.all.push(lane);
106
+ return lane;
107
+ }
108
+ return new Promise((resolve) => this.waiting.push(resolve));
109
+ }
110
+ async call(request, span) {
111
+ const lane = await this.lane();
112
+ try {
113
+ return await lane.call(request, span);
114
+ }
115
+ finally {
116
+ const next = this.waiting.shift();
117
+ if (next)
118
+ next(lane);
119
+ else
120
+ this.idle.push(lane);
121
+ }
122
+ }
123
+ close() {
124
+ for (const lane of this.all)
125
+ lane.close();
126
+ this.all.length = 0;
127
+ this.idle.length = 0;
128
+ }
129
+ }
130
+ /** One worker thread running a wasm plugin's calls one at a time. */
131
+ class WasmLane {
132
+ name;
133
+ bytes;
134
+ policy;
135
+ worker;
136
+ nextId = 1;
137
+ pending = new Map();
138
+ failed;
139
+ constructor(name, bytes, policy) {
140
+ this.name = name;
141
+ this.bytes = bytes;
142
+ this.policy = policy;
143
+ }
144
+ start() {
145
+ if (this.worker)
146
+ return this.worker;
147
+ const w = new Worker(WORKER, {
148
+ eval: true,
149
+ workerData: { bytes: this.bytes, maxMemory: DEFAULT_MAX_MEMORY },
150
+ // The worker's own JS heap; wasm memory is checked after each call.
151
+ resourceLimits: { maxOldGenerationSizeMb: 256 },
152
+ env: {},
153
+ stdout: true,
154
+ stderr: true,
155
+ });
156
+ w.on('message', (m) => {
157
+ const p = this.pending.get(m.id);
158
+ if (!p)
159
+ return;
160
+ this.pending.delete(m.id);
161
+ if (m.error !== undefined)
162
+ p.reject(new Error(m.error));
163
+ else
164
+ p.resolve(m.out);
165
+ });
166
+ w.on('error', (e) => this.fail(e instanceof Error ? e.message : String(e)));
167
+ w.on('exit', (code) => this.fail(`stopped (exit code ${code})`));
168
+ this.worker = w;
169
+ return w;
170
+ }
171
+ fail(reason) {
172
+ for (const p of this.pending.values())
173
+ p.reject(new Error(reason));
174
+ this.pending.clear();
175
+ this.worker = undefined;
176
+ this.failed = reason;
177
+ }
178
+ async call(request, span) {
179
+ const wire = {
180
+ ...request,
181
+ ...(request.input ? { input: { ...request.input, bytes: Buffer.from(request.input.bytes).toString('base64') } } : {}),
182
+ };
183
+ const json = new TextEncoder().encode(JSON.stringify(wire));
184
+ const worker = this.start();
185
+ const id = this.nextId++;
186
+ const out = await new Promise((resolve, reject) => {
187
+ const timer = setTimeout(() => {
188
+ this.pending.delete(id);
189
+ void worker.terminate();
190
+ this.worker = undefined;
191
+ reject(new Error(`timed out after ${this.policy.timeoutMs} ms`));
192
+ }, this.policy.timeoutMs);
193
+ this.pending.set(id, {
194
+ resolve: (o) => {
195
+ clearTimeout(timer);
196
+ resolve(o);
197
+ },
198
+ reject: (e) => {
199
+ clearTimeout(timer);
200
+ reject(e);
201
+ },
202
+ });
203
+ worker.postMessage({ id, request: json }, [json.buffer]);
204
+ }).catch((e) => {
205
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${this.name}: ${e.message}`, span);
206
+ });
207
+ this.failed = undefined;
208
+ return this.parse(out, span);
209
+ }
210
+ parse(out, span) {
211
+ if (out.byteLength > this.policy.maxOutputBytes) {
212
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${this.name}: result is larger than ${this.policy.maxOutputBytes} bytes`, span);
213
+ }
214
+ let msg;
215
+ try {
216
+ msg = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(out));
217
+ }
218
+ catch (e) {
219
+ throw new SitefileError('E_PLUGIN', `PLUGIN ${this.name}: the result is not UTF-8 JSON (${e.message})`, span);
220
+ }
221
+ const bad = (why) => new SitefileError('E_PLUGIN', `PLUGIN ${this.name}: ${why}`, span);
222
+ if (msg === null || typeof msg !== 'object' || Array.isArray(msg))
223
+ throw bad('the result must be a JSON object');
224
+ const r = msg;
225
+ if (typeof r['error'] === 'string')
226
+ throw bad(r['error']);
227
+ const b64 = (v, what) => {
228
+ if (typeof v !== 'string' || !/^[A-Za-z0-9+/]*={0,2}$/.test(v))
229
+ throw bad(`${what} must be a base64 string`);
230
+ return new Uint8Array(Buffer.from(v, 'base64'));
231
+ };
232
+ if ('bytes' in r)
233
+ return { kind: 'file', bytes: b64(r['bytes'], 'bytes') };
234
+ if (typeof r['text'] === 'string')
235
+ return { kind: 'file', bytes: new TextEncoder().encode(r['text']) };
236
+ if (r['files'] !== null && typeof r['files'] === 'object' && !Array.isArray(r['files'])) {
237
+ const files = Object.entries(r['files']).map(([k, v]) => [k, b64(v, `files["${k}"]`)]);
238
+ return { kind: 'files', files };
239
+ }
240
+ throw bad('the result must have "bytes", "text", "files" or "error"');
241
+ }
242
+ close() {
243
+ const w = this.worker;
244
+ this.worker = undefined;
245
+ if (w)
246
+ void w.terminate();
247
+ }
248
+ }
249
+ //# sourceMappingURL=wasm.js.map
@@ -0,0 +1,42 @@
1
+ /**
2
+ * `--profile <file>`: a timing trace of one build in the Chrome trace event format
3
+ * (open it in chrome://tracing, https://ui.perfetto.dev or speedscope).
4
+ *
5
+ * Every event is a complete event (`ph: "X"`, microseconds) on process 1. The
6
+ * build runs on thread 1; import N (0-based, in Sitefile order) on thread N+2,
7
+ * because imports resolve concurrently. Events appear in the file in the order
8
+ * they started in the Sitefile (imports in declaration order), so two builds of
9
+ * the same Sitefile list the same events in the same order with the same names,
10
+ * categories and args; only `ts` and `dur` differ. The Rust implementation
11
+ * (rust/src/profile.rs) writes the same events.
12
+ */
13
+ export type ProfileArgs = Record<string, string | number | boolean>;
14
+ export interface ProfileSpan {
15
+ /** Start the clock (for reserved spans; `begin` starts it at once). */
16
+ start(): void;
17
+ /** Stop the clock, adding args. */
18
+ end(args?: ProfileArgs): void;
19
+ }
20
+ export declare class Profiler {
21
+ readonly command: string;
22
+ private readonly t0;
23
+ private readonly slots;
24
+ private readonly threads;
25
+ constructor(command: string);
26
+ private now;
27
+ /** Reserve an event's place in the file now; its clock starts with `start()`. */
28
+ reserve(name: string, cat: string, args?: ProfileArgs, tid?: number): ProfileSpan;
29
+ /** Start an event now. */
30
+ begin(name: string, cat: string, args?: ProfileArgs, tid?: number): ProfileSpan;
31
+ nameThread(tid: number, name: string): void;
32
+ /**
33
+ * The trace as JSON text, one event per line. Events still open (the build failed
34
+ * inside them) end now with `incomplete: true`; reserved events that never started
35
+ * are left out.
36
+ */
37
+ toJSON(): string;
38
+ }
39
+ /** The span helpers the executor uses: a no-op without a profiler. */
40
+ export declare function begin(p: Profiler | undefined, name: string, cat: string, args?: ProfileArgs): ProfileSpan;
41
+ /** Reserve an event's place now (its clock starts with `start()`): a no-op without a profiler. */
42
+ export declare function reserve(p: Profiler | undefined, name: string, cat: string, args?: ProfileArgs): ProfileSpan;
@@ -0,0 +1,68 @@
1
+ import { performance } from 'node:perf_hooks';
2
+ const NOOP = { start() { }, end() { } };
3
+ export class Profiler {
4
+ command;
5
+ t0 = performance.now();
6
+ slots = [];
7
+ threads = new Map([[1, 'build']]);
8
+ constructor(command) {
9
+ this.command = command;
10
+ }
11
+ now() {
12
+ return Math.round((performance.now() - this.t0) * 1000);
13
+ }
14
+ /** Reserve an event's place in the file now; its clock starts with `start()`. */
15
+ reserve(name, cat, args = {}, tid = 1) {
16
+ const slot = { name, cat, tid, args: { ...args } };
17
+ this.slots.push(slot);
18
+ return {
19
+ start: () => {
20
+ slot.ts = this.now();
21
+ },
22
+ end: (more) => {
23
+ if (slot.ts === undefined || slot.dur !== undefined)
24
+ return;
25
+ slot.dur = this.now() - slot.ts;
26
+ if (more)
27
+ Object.assign(slot.args, more);
28
+ },
29
+ };
30
+ }
31
+ /** Start an event now. */
32
+ begin(name, cat, args = {}, tid = 1) {
33
+ const s = this.reserve(name, cat, args, tid);
34
+ s.start();
35
+ return s;
36
+ }
37
+ nameThread(tid, name) {
38
+ this.threads.set(tid, name);
39
+ }
40
+ /**
41
+ * The trace as JSON text, one event per line. Events still open (the build failed
42
+ * inside them) end now with `incomplete: true`; reserved events that never started
43
+ * are left out.
44
+ */
45
+ toJSON() {
46
+ const now = this.now();
47
+ const lines = [JSON.stringify({ name: 'process_name', ph: 'M', pid: 1, tid: 1, args: { name: `sitefile ${this.command}` } })];
48
+ for (const [tid, name] of [...this.threads].sort((a, b) => a[0] - b[0])) {
49
+ lines.push(JSON.stringify({ name: 'thread_name', ph: 'M', pid: 1, tid, args: { name } }));
50
+ }
51
+ for (const s of this.slots) {
52
+ if (s.ts === undefined)
53
+ continue;
54
+ const args = s.dur === undefined ? { ...s.args, incomplete: true } : s.args;
55
+ lines.push(JSON.stringify({ name: s.name, cat: s.cat, ph: 'X', ts: s.ts, dur: s.dur ?? now - s.ts, pid: 1, tid: s.tid, args }));
56
+ }
57
+ return `{"traceEvents":[\n${lines.join(',\n')}\n],\n"displayTimeUnit":"ms"}\n`;
58
+ }
59
+ }
60
+ /** The span helpers the executor uses: a no-op without a profiler. */
61
+ export function begin(p, name, cat, args) {
62
+ return p ? p.begin(name, cat, args) : NOOP;
63
+ }
64
+ /** Reserve an event's place now (its clock starts with `start()`): a no-op without a profiler. */
65
+ export function reserve(p, name, cat, args) {
66
+ return p ? p.reserve(name, cat, args) : NOOP;
67
+ }
68
+ //# sourceMappingURL=profile.js.map