@genroc/eval-node 0.0.0-edge.f85ea85 → 0.0.0-edge.fb8c06a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -33
- package/dist/eval.js +85 -0
- package/dist/import.js +397 -0
- package/dist/realm.js +113 -0
- package/dist/worker.js +305 -0
- package/eval.ts +31 -5
- package/import.ts +190 -80
- package/package.json +6 -4
- package/realm.ts +61 -77
- package/worker.ts +44 -12
- package/bin/import.mjs +0 -4
- package/bin/worker.mjs +0 -2
package/README.md
CHANGED
|
@@ -24,6 +24,16 @@ on the cwd:
|
|
|
24
24
|
`genctl apply` and `genctl types` now resolve `$import` directives. Typechecking is the
|
|
25
25
|
resolver's exit code, so a stored definition cannot hold code that failed to typecheck.
|
|
26
26
|
|
|
27
|
+
## Building
|
|
28
|
+
|
|
29
|
+
npm run build # tsc -p tsconfig.build.json -> dist/
|
|
30
|
+
|
|
31
|
+
The published package is JavaScript, not TypeScript. **Node refuses to strip types for files
|
|
32
|
+
under `node_modules`**, so shipping `.ts` works from a checkout and fails for every consumer —
|
|
33
|
+
which is why `prepublishOnly` builds and CI installs the packed tarball rather than only running
|
|
34
|
+
from source. The source keeps its `.ts` import extensions; `rewriteRelativeImportExtensions`
|
|
35
|
+
turns them into `.js` on the way out.
|
|
36
|
+
|
|
27
37
|
## Running the worker
|
|
28
38
|
|
|
29
39
|
docker run -e GENROC_SERVER=http://host:8448 ghcr.io/genroc/eval-node:preview
|
|
@@ -80,21 +90,41 @@ queue worker rather than the HTTP sidecar it used to be:
|
|
|
80
90
|
that genroc had to be able to reach. A worker only needs outbound access, so it can live
|
|
81
91
|
anywhere — behind NAT, in another trust zone.
|
|
82
92
|
|
|
93
|
+
The cost of that inversion is that genroc cannot reach a running worker, so anything it needs
|
|
94
|
+
to say has to ride the renewal the worker already makes. **Renewing is therefore mandatory,
|
|
95
|
+
not an optimisation** — a worker that stops renewing is indistinguishable from one that died.
|
|
96
|
+
The response answers per token:
|
|
97
|
+
|
|
98
|
+
| list | what it means | what this worker does |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `renewed` | still yours | carry on |
|
|
101
|
+
| `lost` | already someone else's | stop; do **not** release — that would bump the new holder's claim |
|
|
102
|
+
| `cancelled` | still yours, nobody wants it | abort the script and release the claim |
|
|
103
|
+
|
|
104
|
+
`cancelled` is how an operator's `genctl cancel` reaches work already running: the evaluation
|
|
105
|
+
is aborted through an `AbortSignal`, its realm is terminated, and no outcome is submitted —
|
|
106
|
+
the process is stopping, so there is nothing left to answer. `renew_before_ms` on the claim
|
|
107
|
+
says how long the worker may wait before renewing again, so the interval is a value it reads
|
|
108
|
+
rather than one it guesses.
|
|
109
|
+
|
|
83
110
|
## The task input
|
|
84
111
|
|
|
85
112
|
The `input` of the external task IS the evaluation request:
|
|
86
113
|
|
|
87
114
|
```jsonc
|
|
88
115
|
{
|
|
89
|
-
"code": "
|
|
90
|
-
"input": { "amount": 250 },
|
|
91
|
-
"timeout_ms": 5000
|
|
116
|
+
"code": "export default (input) => ({ fee: input.amount * 0.1 });", // required — a module
|
|
117
|
+
"input": { "amount": 250 }, // optional — its argument
|
|
118
|
+
"timeout_ms": 5000 // optional — default 5000
|
|
92
119
|
}
|
|
93
120
|
```
|
|
94
121
|
|
|
95
|
-
`code` is
|
|
96
|
-
|
|
97
|
-
|
|
122
|
+
`code` is an **ES module**, and the evaluator imports it and calls its **default export** with
|
|
123
|
+
`input`. The value it returns reaches genroc; the function may be async, and so may the module's
|
|
124
|
+
top level. A module exporting anything else — or nothing — is a `compile_error`: there is
|
|
125
|
+
nothing to call, and so is one that will not parse; a throw from its top level is a `threw`
|
|
126
|
+
like any other. `import` of a node builtin resolves as it does anywhere else, and that is what
|
|
127
|
+
a bundled builtin lands on.
|
|
98
128
|
|
|
99
129
|
## The answer — the failure kind IS the error code
|
|
100
130
|
|
|
@@ -130,12 +160,14 @@ no retry policy to write.
|
|
|
130
160
|
type: external
|
|
131
161
|
input:
|
|
132
162
|
code: |
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
163
|
+
export default function (input) {
|
|
164
|
+
if (input.amount > 100) {
|
|
165
|
+
const e = new Error('amount over the limit');
|
|
166
|
+
e.name = 'LimitExceeded';
|
|
167
|
+
throw e;
|
|
168
|
+
}
|
|
169
|
+
return { fee: input.amount * 0.1 };
|
|
137
170
|
}
|
|
138
|
-
return { fee: input.amount * 0.1 };
|
|
139
171
|
input: "$: input"
|
|
140
172
|
result_schema: { type: object, properties: { fee: { type: number } }, required: [fee] }
|
|
141
173
|
raises:
|
|
@@ -154,7 +186,7 @@ no retry policy to write.
|
|
|
154
186
|
|
|
155
187
|
- id: script_failed
|
|
156
188
|
switch:
|
|
157
|
-
- case: '
|
|
189
|
+
- case: 'last_error.data.name == "LimitExceeded"'
|
|
158
190
|
raise: { code: limit_exceeded, message: "the script rejected the amount" }
|
|
159
191
|
- raise: { code: script_failed, message: "the script failed" }
|
|
160
192
|
```
|
|
@@ -217,12 +249,14 @@ the inferred type of what the definition passes; `Output` is what it declares
|
|
|
217
249
|
`tsc --noEmit`, and bundles — so **a type error is a failed apply**, and a stored definition
|
|
218
250
|
cannot hold code that failed to typecheck.
|
|
219
251
|
|
|
220
|
-
The bundle is
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
252
|
+
The bundle is one self-contained ES module whose default export is the author's own — nothing
|
|
253
|
+
wraps or rewrites it, so what the editor checks and what the realm runs are the same module, and
|
|
254
|
+
**a missing default export is a failed apply** rather than a fault at run time. Imports resolve
|
|
255
|
+
through TypeScript under the same config the check ran with, so a `paths` alias that typechecks
|
|
256
|
+
also bundles. They are inlined at build time, so the string a definition version stores is
|
|
257
|
+
self-contained forever — with one exception: **node builtins stay as imports**, which the realm
|
|
258
|
+
resolves. A package is frozen into the definition; `node:fs` is resolved by whatever runner
|
|
259
|
+
executes it.
|
|
226
260
|
|
|
227
261
|
### Your tsconfig, your types
|
|
228
262
|
|
|
@@ -248,8 +282,9 @@ never read by genroc, because genctl doubles every `$` on splice.
|
|
|
248
282
|
|
|
249
283
|
## The realm — one Worker per execution
|
|
250
284
|
|
|
251
|
-
`evaluate()` starts a Worker (`realm.ts`), posts the code into it
|
|
252
|
-
`
|
|
285
|
+
`evaluate()` starts a Worker (`realm.ts`), posts the code into it — where a module loader hook
|
|
286
|
+
serves it to `import()`, so the engine numbers the script's frames from the author's own source —
|
|
287
|
+
and races the reply against the budget; `terminate()` runs on every path. That thread is what the contract rests on, and it buys
|
|
253
288
|
exactly three things the previous in-process evaluator could not:
|
|
254
289
|
|
|
255
290
|
- **The budget is enforced, not merely reported.** A synchronous `while(true){}` never
|
|
@@ -261,16 +296,15 @@ exactly three things the previous in-process evaluator could not:
|
|
|
261
296
|
- **The script's mistakes stay the script's.** An uncaught throw, and `process.exit()`, end
|
|
262
297
|
the realm and come back as a `422` — neither reaches the runner.
|
|
263
298
|
|
|
264
|
-
It costs about **
|
|
265
|
-
laptop), a 200 KiB body about
|
|
266
|
-
types on every realm — precompiling it to JavaScript would buy that back, and
|
|
267
|
-
not done: a build artefact that goes stale against its source fails silently,
|
|
268
|
-
is the one where a wrong line number is invisible.
|
|
299
|
+
It costs about **27ms per execution** end to end for a trivial script (Node 24, M-series
|
|
300
|
+
laptop), a 200 KiB body about 30ms. Roughly 11ms of that is Node re-stripping `realm.ts`'s
|
|
301
|
+
types on every realm — precompiling it to JavaScript measures 17ms and would buy that back, and
|
|
302
|
+
is deliberately not done: a build artefact that goes stale against its source fails silently,
|
|
303
|
+
and this file is the one where a wrong line number is invisible.
|
|
269
304
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
(below), so it is the live upgrade path rather than a theoretical one.
|
|
305
|
+
A subprocess per execution measures ~55ms here, twice the thread rather than the ten times it
|
|
306
|
+
cost on the previous runtime. It contains the two things a thread cannot (below), so the upgrade
|
|
307
|
+
path is a live one at a price worth naming rather than a theoretical one.
|
|
274
308
|
|
|
275
309
|
## What this is not
|
|
276
310
|
|
|
@@ -281,7 +315,7 @@ thread no longer wins on price, and a subprocess contains the two things a threa
|
|
|
281
315
|
it needed was surface with nothing behind it. A value that must survive a retry belongs in
|
|
282
316
|
the definition, passed through `input`.
|
|
283
317
|
- **Not a sandbox.** The realm isolates *execution*, not *authority*: a script gets the
|
|
284
|
-
worker's filesystem, network and environment, and
|
|
318
|
+
worker's filesystem, network and environment, and any node builtin it imports. That is
|
|
285
319
|
deliberate — a script task is meant to do real work — but the trust boundary stays the
|
|
286
320
|
same-trust-domain one (your genroc, your worker host). It is not the multi-tenant story, and
|
|
287
321
|
nothing here should be mistaken for one. Pulling does move the boundary in one useful way:
|
|
@@ -295,9 +329,9 @@ thread no longer wins on price, and a subprocess contains the two things a threa
|
|
|
295
329
|
- **Concurrency is capped by this worker, not by genroc.** Each evaluation is a thread, and
|
|
296
330
|
`CONCURRENCY` is how many it will claim at once. Raising it past what the host can run turns
|
|
297
331
|
a queue back into threads fighting over a core.
|
|
298
|
-
- **Not where imports and type checking happen.** The evaluator
|
|
299
|
-
|
|
300
|
-
|
|
332
|
+
- **Not where imports and type checking happen.** The evaluator takes one self-contained
|
|
333
|
+
module and knows nothing about TypeScript; `import.ts` is what bundles an author's script and
|
|
334
|
+
its dependencies into that module, at author time. See above.
|
|
301
335
|
- **`eval.ts` and `realm.ts` know nothing about genroc.** `worker.ts` is the entire
|
|
302
336
|
queue-facing half, which is what keeps the containment strategy swappable — and what lets
|
|
303
337
|
the realm's own properties be tested by calling `evaluate()` directly.
|
package/dist/eval.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// Evaluation core: import a script module in its OWN realm, call its default export, and
|
|
2
|
+
// classify every outcome into one of the failure kinds in README.md. Nothing here knows about
|
|
3
|
+
// genroc — worker.ts is the only thing that talks to the queue — so this stays testable and
|
|
4
|
+
// the containment stays swappable.
|
|
5
|
+
//
|
|
6
|
+
// The containment is a Worker per execution (realm.ts). It is what makes the budget real:
|
|
7
|
+
// a synchronous busy loop never yields, so no in-process timer can interrupt it, and only a
|
|
8
|
+
// thread the host can kill bounds it.
|
|
9
|
+
import { Worker } from "node:worker_threads";
|
|
10
|
+
const DEFAULT_TIMEOUT_MS = 5_000;
|
|
11
|
+
// Resolved from THIS file's own extension: run from a checkout it is `.ts`, and from the
|
|
12
|
+
// published package `.js`, because Node will not strip types under node_modules.
|
|
13
|
+
const REALM_URL = new URL(import.meta.url.endsWith(".ts") ? "./realm.ts" : "./realm.js", import.meta.url);
|
|
14
|
+
/** Thrown, not returned: a realm that fails to start is the RUNNER faulting, which worker.ts
|
|
15
|
+
* answers by releasing the claim rather than by reporting an outcome. A script fault is a
|
|
16
|
+
* return value. */
|
|
17
|
+
class RealmFault extends Error {
|
|
18
|
+
constructor(message) {
|
|
19
|
+
super(message);
|
|
20
|
+
this.name = "RealmFault";
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
24
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
25
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
26
|
+
export class Cancelled extends Error {
|
|
27
|
+
constructor() {
|
|
28
|
+
super("cancelled");
|
|
29
|
+
this.name = "Cancelled";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export async function evaluate(req, signal) {
|
|
33
|
+
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
34
|
+
const worker = new Worker(REALM_URL);
|
|
35
|
+
let timer;
|
|
36
|
+
let onAbort;
|
|
37
|
+
try {
|
|
38
|
+
return await new Promise((resolve, reject) => {
|
|
39
|
+
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
40
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
41
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
42
|
+
if (signal) {
|
|
43
|
+
if (signal.aborted)
|
|
44
|
+
reject(new Cancelled());
|
|
45
|
+
onAbort = () => reject(new Cancelled());
|
|
46
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
47
|
+
}
|
|
48
|
+
worker.once("message", (reply) => resolve(reply));
|
|
49
|
+
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
50
|
+
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
51
|
+
// this too, by which time the promise has settled and the first result stands.
|
|
52
|
+
worker.once("exit", (code) => resolve(exited(code)));
|
|
53
|
+
worker.once("error", (err) => reject(new RealmFault(errorText(err))));
|
|
54
|
+
worker.postMessage({ code: req.code, input: req.input });
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
clearTimeout(timer);
|
|
59
|
+
if (signal && onAbort)
|
|
60
|
+
signal.removeEventListener("abort", onAbort);
|
|
61
|
+
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
62
|
+
// resolving before it is gone would report an evaluation the machine is still running.
|
|
63
|
+
await worker.terminate();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function timedOut(ms) {
|
|
67
|
+
return {
|
|
68
|
+
ok: false,
|
|
69
|
+
failure: { kind: "timeout", name: "TimeoutError", message: `script exceeded its ${ms}ms budget` },
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function exited(code) {
|
|
73
|
+
return {
|
|
74
|
+
ok: false,
|
|
75
|
+
failure: {
|
|
76
|
+
kind: "exited",
|
|
77
|
+
name: "RealmExited",
|
|
78
|
+
message: `the script ended its own realm with code ${code} instead of returning`,
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
function errorText(e) {
|
|
83
|
+
const message = e?.message;
|
|
84
|
+
return typeof message === "string" && message !== "" ? message : "the evaluation realm failed to start";
|
|
85
|
+
}
|
package/dist/import.js
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The code-phase resolver: manifest on stdin, `{"code": [...]}` on stdout, non-zero exit
|
|
3
|
+
// with the diagnostic on stderr. genctl never parses TypeScript and this never parses YAML
|
|
4
|
+
// — the manifest is the whole contract. See specs/source-resolution.md.
|
|
5
|
+
//
|
|
6
|
+
// Two modes, one binary: "types" writes the declarations an editor needs and returns no
|
|
7
|
+
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
|
+
// over the same project.
|
|
9
|
+
import { spawn } from "node:child_process";
|
|
10
|
+
import { existsSync } from "node:fs";
|
|
11
|
+
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
12
|
+
import { builtinModules } from "node:module";
|
|
13
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
14
|
+
import { fileURLToPath } from "node:url";
|
|
15
|
+
import commonjs from "@rollup/plugin-commonjs";
|
|
16
|
+
import json from "@rollup/plugin-json";
|
|
17
|
+
import { nodeResolve } from "@rollup/plugin-node-resolve";
|
|
18
|
+
import { rollup } from "rollup";
|
|
19
|
+
import ts from "typescript";
|
|
20
|
+
function die(message) {
|
|
21
|
+
console.error(message);
|
|
22
|
+
process.exit(1);
|
|
23
|
+
}
|
|
24
|
+
async function exists(path) {
|
|
25
|
+
try {
|
|
26
|
+
await access(path);
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/** Creates the parent directory, which `.genroc-cache/` relies on: nothing else makes it. */
|
|
34
|
+
async function write(path, content) {
|
|
35
|
+
await mkdir(dirname(path), { recursive: true });
|
|
36
|
+
await writeFile(path, content);
|
|
37
|
+
}
|
|
38
|
+
// ── JSON Schema → TypeScript ───────────────────────────────────────────────────
|
|
39
|
+
/** Only genroc's keyword set is handled; anything else its strict decoder would have
|
|
40
|
+
* refused before this ran (internal/schema, allowedKeywords). */
|
|
41
|
+
function tsType(s, used) {
|
|
42
|
+
if (s === undefined || s === null)
|
|
43
|
+
return "unknown";
|
|
44
|
+
if (typeof s.$ref === "string") {
|
|
45
|
+
const name = s.$ref.replace(/^#\/\$defs\//, "");
|
|
46
|
+
used.add(name);
|
|
47
|
+
return identifier(name);
|
|
48
|
+
}
|
|
49
|
+
if (Array.isArray(s.enum)) {
|
|
50
|
+
return s.enum.map((v) => JSON.stringify(v)).join(" | ") || "never";
|
|
51
|
+
}
|
|
52
|
+
if (Array.isArray(s.anyOf))
|
|
53
|
+
return union(s.anyOf.map((a) => tsType(a, used)));
|
|
54
|
+
if (Array.isArray(s.oneOf))
|
|
55
|
+
return union(s.oneOf.map((a) => tsType(a, used)));
|
|
56
|
+
if (Array.isArray(s.allOf)) {
|
|
57
|
+
return s.allOf.map((a) => tsType(a, used)).join(" & ") || "unknown";
|
|
58
|
+
}
|
|
59
|
+
const types = s.type === undefined ? [] : Array.isArray(s.type) ? s.type : [s.type];
|
|
60
|
+
if (types.length === 0) {
|
|
61
|
+
// The top type: `{}` means unknown, not "an empty object". specs/unknown-type.md.
|
|
62
|
+
return s.properties ? objectType(s, used) : "unknown";
|
|
63
|
+
}
|
|
64
|
+
return union(types.map((t) => scalarType(t, s, used)));
|
|
65
|
+
}
|
|
66
|
+
function scalarType(t, s, used) {
|
|
67
|
+
switch (t) {
|
|
68
|
+
case "object":
|
|
69
|
+
return objectType(s, used);
|
|
70
|
+
case "array":
|
|
71
|
+
return s.items ? `Array<${tsType(s.items, used)}>` : "unknown[]";
|
|
72
|
+
case "string":
|
|
73
|
+
return "string";
|
|
74
|
+
case "number":
|
|
75
|
+
case "integer":
|
|
76
|
+
return "number";
|
|
77
|
+
case "boolean":
|
|
78
|
+
return "boolean";
|
|
79
|
+
case "null":
|
|
80
|
+
return "null";
|
|
81
|
+
default:
|
|
82
|
+
return "unknown";
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function objectType(s, used) {
|
|
86
|
+
const props = s.properties ?? {};
|
|
87
|
+
const required = new Set(s.required ?? []);
|
|
88
|
+
const lines = [];
|
|
89
|
+
for (const [key, sub] of Object.entries(props)) {
|
|
90
|
+
const doc = typeof sub.description === "string"
|
|
91
|
+
? ` /** ${sub.description} */\n`
|
|
92
|
+
: "";
|
|
93
|
+
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`);
|
|
94
|
+
}
|
|
95
|
+
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
96
|
+
lines.push(` [key: string]: ${tsType(s.additionalProperties, used)};`);
|
|
97
|
+
}
|
|
98
|
+
if (lines.length === 0)
|
|
99
|
+
return "Record<string, unknown>";
|
|
100
|
+
return `{\n${lines.join("\n")}\n}`;
|
|
101
|
+
}
|
|
102
|
+
function union(parts) {
|
|
103
|
+
const seen = [...new Set(parts)];
|
|
104
|
+
return seen.length === 0 ? "unknown" : seen.join(" | ");
|
|
105
|
+
}
|
|
106
|
+
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
107
|
+
const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
108
|
+
const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
109
|
+
function deref(s, defs) {
|
|
110
|
+
let cur = s;
|
|
111
|
+
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
112
|
+
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
113
|
+
}
|
|
114
|
+
return cur;
|
|
115
|
+
}
|
|
116
|
+
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
117
|
+
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
118
|
+
function declarations(at) {
|
|
119
|
+
const { site, where } = at;
|
|
120
|
+
const defs = where.$defs ?? {};
|
|
121
|
+
const used = new Set();
|
|
122
|
+
const input = tsType(site.types?.Input, used);
|
|
123
|
+
const output = tsType(site.types?.Output, used);
|
|
124
|
+
const emitted = [];
|
|
125
|
+
const done = new Set();
|
|
126
|
+
while (true) {
|
|
127
|
+
const next = [...used].find((n) => !done.has(n));
|
|
128
|
+
if (next === undefined)
|
|
129
|
+
break;
|
|
130
|
+
done.add(next);
|
|
131
|
+
const body = tsType(defs[next], used);
|
|
132
|
+
emitted.push(`export type ${identifier(next)} = ${body};`);
|
|
133
|
+
}
|
|
134
|
+
return [
|
|
135
|
+
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
136
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
137
|
+
"",
|
|
138
|
+
...emitted,
|
|
139
|
+
emitted.length ? "" : "",
|
|
140
|
+
`export type Input = ${input};`,
|
|
141
|
+
"",
|
|
142
|
+
`export type Output = ${output};`,
|
|
143
|
+
"",
|
|
144
|
+
].join("\n");
|
|
145
|
+
}
|
|
146
|
+
/** Keyed by the script's PATH, not the task id: keyed by task, renaming a task would break
|
|
147
|
+
* the author's `import type` line with the error landing nowhere near the rename. */
|
|
148
|
+
function typesPathFor(scriptPath) {
|
|
149
|
+
return scriptPath.replace(/\.[^.\/]+$/, "") + ".genroc.d.ts";
|
|
150
|
+
}
|
|
151
|
+
// ── typecheck ──────────────────────────────────────────────────────────────────
|
|
152
|
+
/** The nearest tsconfig above the script — the one the author's editor already reads. Two
|
|
153
|
+
* different configs mean a red editor over a clean apply, or the reverse. The walk stops at
|
|
154
|
+
* the project root: above it is not this project. */
|
|
155
|
+
async function nearestTsconfig(from, root) {
|
|
156
|
+
for (let dir = from;; dir = dirname(dir)) {
|
|
157
|
+
const candidate = join(dir, "tsconfig.json");
|
|
158
|
+
if (await exists(candidate))
|
|
159
|
+
return candidate;
|
|
160
|
+
if (dir === root || dirname(dir) === dir)
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
async function typecheck(sites) {
|
|
165
|
+
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
166
|
+
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
167
|
+
const dir = join(root, ".genroc-cache");
|
|
168
|
+
await write(join(dir, ".gitignore"), "*\n");
|
|
169
|
+
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
170
|
+
// check each script under the other author's options.
|
|
171
|
+
const groups = new Map();
|
|
172
|
+
for (const at of sites) {
|
|
173
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
174
|
+
const group = groups.get(base);
|
|
175
|
+
if (group)
|
|
176
|
+
group.push(at);
|
|
177
|
+
else
|
|
178
|
+
groups.set(base, [at]);
|
|
179
|
+
}
|
|
180
|
+
let n = 0;
|
|
181
|
+
for (const [base, group] of groups) {
|
|
182
|
+
const config = {
|
|
183
|
+
...(base ? { extends: relative(dir, base) } : {}),
|
|
184
|
+
compilerOptions: {
|
|
185
|
+
noEmit: true,
|
|
186
|
+
strict: true,
|
|
187
|
+
skipLibCheck: true,
|
|
188
|
+
moduleDetection: "force",
|
|
189
|
+
module: "preserve",
|
|
190
|
+
target: "esnext",
|
|
191
|
+
// `lib` DESCRIBES the realm and is written after `extends` so a base cannot widen it:
|
|
192
|
+
// a worker thread has no document, whatever an author's config claims.
|
|
193
|
+
lib: ["esnext", "webworker"],
|
|
194
|
+
// `types` is the author's, and it is how a script opts into the node globals —
|
|
195
|
+
// the worker realm has them, so refusing the declarations would only lie. With no
|
|
196
|
+
// base config there is nothing to opt in with, so the default stays none.
|
|
197
|
+
...(base ? {} : { types: [] }),
|
|
198
|
+
},
|
|
199
|
+
files: group.flatMap((s) => [
|
|
200
|
+
relative(dir, s.file),
|
|
201
|
+
relative(dir, typesPathFor(s.file)),
|
|
202
|
+
]),
|
|
203
|
+
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
204
|
+
// drag the author's whole tree in, to be checked under the worker lib.
|
|
205
|
+
include: [],
|
|
206
|
+
};
|
|
207
|
+
const configPath = join(dir, groups.size === 1 ? "tsconfig.json" : `tsconfig.${n++}.json`);
|
|
208
|
+
await write(configPath, JSON.stringify(config, null, 2));
|
|
209
|
+
await runTsc(root, configPath);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
async function runTsc(root, configPath) {
|
|
213
|
+
const tsc = fileURLToPath(import.meta.resolve("typescript/bin/tsc"));
|
|
214
|
+
const proc = spawn(process.execPath, [tsc, "--noEmit", "-p", configPath], {
|
|
215
|
+
cwd: root,
|
|
216
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
217
|
+
});
|
|
218
|
+
let out = "";
|
|
219
|
+
let err = "";
|
|
220
|
+
proc.stdout.on("data", (c) => (out += c));
|
|
221
|
+
proc.stderr.on("data", (c) => (err += c));
|
|
222
|
+
const code = await new Promise((resolve, reject) => {
|
|
223
|
+
proc.on("error", reject);
|
|
224
|
+
proc.on("close", (c) => resolve(c ?? 1));
|
|
225
|
+
});
|
|
226
|
+
if (code !== 0) {
|
|
227
|
+
// tsc reports on stdout; the exit code IS the type check, so this is the diagnostic
|
|
228
|
+
// genctl surfaces and the reason a failed import never produces a string.
|
|
229
|
+
die([out, err].filter(Boolean).join("\n").trimEnd());
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
// ── bundle ─────────────────────────────────────────────────────────────────────
|
|
233
|
+
/** Transpiles only. The typecheck above already ran over the author's OWN tsconfig, and a
|
|
234
|
+
* second opinion from a config they do not control could fail a build they cannot fix. */
|
|
235
|
+
const transpile = {
|
|
236
|
+
name: "genroc-transpile",
|
|
237
|
+
transform(code, id) {
|
|
238
|
+
if (!id.endsWith(".ts") && !id.endsWith(".tsx"))
|
|
239
|
+
return null;
|
|
240
|
+
const out = ts.transpileModule(code, {
|
|
241
|
+
fileName: id,
|
|
242
|
+
compilerOptions: {
|
|
243
|
+
target: ts.ScriptTarget.ESNext,
|
|
244
|
+
module: ts.ModuleKind.ESNext,
|
|
245
|
+
verbatimModuleSyntax: false,
|
|
246
|
+
jsx: id.endsWith(".tsx") ? ts.JsxEmit.ReactJSX : undefined,
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
return { code: out.outputText, map: out.sourceMapText ?? null };
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
const BUILTIN = new Set([
|
|
253
|
+
...builtinModules,
|
|
254
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
255
|
+
]);
|
|
256
|
+
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
257
|
+
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
258
|
+
* resolver to keep in agreement with tsc; this one cannot disagree.
|
|
259
|
+
* A package resolving to a `.d.ts` is declined — that is a type, not the implementation —
|
|
260
|
+
* which leaves node_modules to nodeResolve. */
|
|
261
|
+
function tsResolve(configPath) {
|
|
262
|
+
let options = {};
|
|
263
|
+
if (configPath) {
|
|
264
|
+
const read = ts.readConfigFile(configPath, ts.sys.readFile);
|
|
265
|
+
options = ts.parseJsonConfigFileContent(read.config ?? {}, ts.sys, dirname(configPath)).options;
|
|
266
|
+
}
|
|
267
|
+
return {
|
|
268
|
+
name: "genroc-ts-resolve",
|
|
269
|
+
resolveId(source, importer) {
|
|
270
|
+
if (!importer || BUILTIN.has(source))
|
|
271
|
+
return null;
|
|
272
|
+
const { resolvedModule } = ts.resolveModuleName(source, importer, options, ts.sys);
|
|
273
|
+
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
274
|
+
return null;
|
|
275
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
276
|
+
? null
|
|
277
|
+
: resolvedModule.resolvedFileName;
|
|
278
|
+
},
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
282
|
+
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
283
|
+
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
284
|
+
async function bundle(at) {
|
|
285
|
+
const site = at.site;
|
|
286
|
+
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
287
|
+
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
288
|
+
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
289
|
+
const built = await rollup({
|
|
290
|
+
input: at.file,
|
|
291
|
+
external: (id) => BUILTIN.has(id),
|
|
292
|
+
plugins: [
|
|
293
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
294
|
+
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
295
|
+
commonjs(),
|
|
296
|
+
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
297
|
+
// did natively; without it rollup hands the JSON to the JS parser.
|
|
298
|
+
json(),
|
|
299
|
+
transpile,
|
|
300
|
+
],
|
|
301
|
+
onwarn(warning) {
|
|
302
|
+
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
303
|
+
die(`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`);
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
}).catch((e) => die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`));
|
|
307
|
+
const { output } = await built.generate({
|
|
308
|
+
format: "es",
|
|
309
|
+
inlineDynamicImports: true,
|
|
310
|
+
});
|
|
311
|
+
await built.close();
|
|
312
|
+
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
313
|
+
// instance, and the file it names is on this machine.
|
|
314
|
+
if (!output[0].exports.includes("default")) {
|
|
315
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
316
|
+
}
|
|
317
|
+
return output[0].code;
|
|
318
|
+
}
|
|
319
|
+
// ── main ───────────────────────────────────────────────────────────────────────
|
|
320
|
+
const stdin = await new Promise((resolve, reject) => {
|
|
321
|
+
let raw = "";
|
|
322
|
+
process.stdin.setEncoding("utf8");
|
|
323
|
+
process.stdin.on("data", (c) => (raw += c));
|
|
324
|
+
process.stdin.on("end", () => resolve(raw));
|
|
325
|
+
process.stdin.on("error", reject);
|
|
326
|
+
});
|
|
327
|
+
const manifest = JSON.parse(stdin);
|
|
328
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
329
|
+
const root = process.cwd();
|
|
330
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
331
|
+
die("stdin is not a genroc resolver manifest");
|
|
332
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
333
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
334
|
+
function address(pointer) {
|
|
335
|
+
return pointer
|
|
336
|
+
.map((seg) => typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
337
|
+
? `.${seg}`
|
|
338
|
+
: `[${JSON.stringify(seg)}]`)
|
|
339
|
+
.join("")
|
|
340
|
+
.replace(/^\./, "");
|
|
341
|
+
}
|
|
342
|
+
const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
|
|
343
|
+
site,
|
|
344
|
+
where,
|
|
345
|
+
file: resolve(where.dir, site.argument),
|
|
346
|
+
})));
|
|
347
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
348
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
349
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
350
|
+
// and `external` is the same request made directly.
|
|
351
|
+
for (const at of located) {
|
|
352
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
353
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
354
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
355
|
+
// directly.
|
|
356
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
357
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
358
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
359
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
360
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
361
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
365
|
+
// does is the one that must say when it is not there.
|
|
366
|
+
for (const at of located) {
|
|
367
|
+
if (!existsSync(at.file)) {
|
|
368
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
369
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
// One script at two sites with different input types is a refusal, not a union: the union
|
|
373
|
+
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
374
|
+
const byPath = new Map();
|
|
375
|
+
for (const at of located) {
|
|
376
|
+
const seen = byPath.get(at.file);
|
|
377
|
+
if (seen &&
|
|
378
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)) {
|
|
379
|
+
die(`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
380
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
381
|
+
"Split it into two scripts, or make the two call sites pass the same shape.");
|
|
382
|
+
}
|
|
383
|
+
byPath.set(at.file, at);
|
|
384
|
+
}
|
|
385
|
+
for (const at of byPath.values()) {
|
|
386
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
387
|
+
}
|
|
388
|
+
if (manifest.mode === "types") {
|
|
389
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
390
|
+
process.exit(0);
|
|
391
|
+
}
|
|
392
|
+
await typecheck([...byPath.values()]);
|
|
393
|
+
const code = [];
|
|
394
|
+
for (const at of located) {
|
|
395
|
+
code.push(await bundle(at));
|
|
396
|
+
}
|
|
397
|
+
process.stdout.write(JSON.stringify({ code }));
|