@genroc/eval-node 0.0.0-edge.ee19083 → 0.0.0-edge.f16dfcef
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +19 -2
- package/dist/eval.js +21 -1
- package/dist/import.js +98 -54
- package/dist/worker.js +30 -10
- package/eval.ts +20 -1
- package/import.ts +181 -74
- package/package.json +1 -1
- package/worker.ts +36 -11
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ resolver's exit code, so a stored definition cannot hold code that failed to typ
|
|
|
26
26
|
|
|
27
27
|
## Building
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
pnpm run build # tsc -p tsconfig.build.json -> dist/
|
|
30
30
|
|
|
31
31
|
The published package is JavaScript, not TypeScript. **Node refuses to strip types for files
|
|
32
32
|
under `node_modules`**, so shipping `.ts` works from a checkout and fails for every consumer —
|
|
@@ -90,6 +90,23 @@ queue worker rather than the HTTP sidecar it used to be:
|
|
|
90
90
|
that genroc had to be able to reach. A worker only needs outbound access, so it can live
|
|
91
91
|
anywhere — behind NAT, in another trust zone.
|
|
92
92
|
|
|
93
|
+
The cost of that inversion is that genroc cannot reach a running worker, so anything it needs
|
|
94
|
+
to say has to ride the renewal the worker already makes. **Renewing is therefore mandatory,
|
|
95
|
+
not an optimisation** — a worker that stops renewing is indistinguishable from one that died.
|
|
96
|
+
The response answers per token:
|
|
97
|
+
|
|
98
|
+
| list | what it means | what this worker does |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `renewed` | still yours | carry on |
|
|
101
|
+
| `lost` | already someone else's | stop; do **not** release — that would bump the new holder's claim |
|
|
102
|
+
| `cancelled` | still yours, nobody wants it | abort the script and release the claim |
|
|
103
|
+
|
|
104
|
+
`cancelled` is how an operator's `genctl cancel` reaches work already running: the evaluation
|
|
105
|
+
is aborted through an `AbortSignal`, its realm is terminated, and no outcome is submitted —
|
|
106
|
+
the process is stopping, so there is nothing left to answer. `renew_before_ms` on the claim
|
|
107
|
+
says how long the worker may wait before renewing again, so the interval is a value it reads
|
|
108
|
+
rather than one it guesses.
|
|
109
|
+
|
|
93
110
|
## The task input
|
|
94
111
|
|
|
95
112
|
The `input` of the external task IS the evaluation request:
|
|
@@ -169,7 +186,7 @@ no retry policy to write.
|
|
|
169
186
|
|
|
170
187
|
- id: script_failed
|
|
171
188
|
switch:
|
|
172
|
-
- case: '
|
|
189
|
+
- case: 'last_error.data.name == "LimitExceeded"'
|
|
173
190
|
raise: { code: limit_exceeded, message: "the script rejected the amount" }
|
|
174
191
|
- raise: { code: script_failed, message: "the script failed" }
|
|
175
192
|
```
|
package/dist/eval.js
CHANGED
|
@@ -20,13 +20,31 @@ class RealmFault extends Error {
|
|
|
20
20
|
this.name = "RealmFault";
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
24
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
25
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
26
|
+
export class Cancelled extends Error {
|
|
27
|
+
constructor() {
|
|
28
|
+
super("cancelled");
|
|
29
|
+
this.name = "Cancelled";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export async function evaluate(req, signal) {
|
|
24
33
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
25
34
|
const worker = new Worker(REALM_URL);
|
|
26
35
|
let timer;
|
|
36
|
+
let onAbort;
|
|
27
37
|
try {
|
|
28
38
|
return await new Promise((resolve, reject) => {
|
|
29
39
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
40
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
41
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
42
|
+
if (signal) {
|
|
43
|
+
if (signal.aborted)
|
|
44
|
+
reject(new Cancelled());
|
|
45
|
+
onAbort = () => reject(new Cancelled());
|
|
46
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
47
|
+
}
|
|
30
48
|
worker.once("message", (reply) => resolve(reply));
|
|
31
49
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
32
50
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -38,6 +56,8 @@ export async function evaluate(req) {
|
|
|
38
56
|
}
|
|
39
57
|
finally {
|
|
40
58
|
clearTimeout(timer);
|
|
59
|
+
if (signal && onAbort)
|
|
60
|
+
signal.removeEventListener("abort", onAbort);
|
|
41
61
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
42
62
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
43
63
|
await worker.terminate();
|
package/dist/import.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
8
|
// over the same project.
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
|
+
import { existsSync } from "node:fs";
|
|
10
11
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
11
12
|
import { builtinModules } from "node:module";
|
|
12
|
-
import { dirname, join, relative } from "node:path";
|
|
13
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
13
14
|
import { fileURLToPath } from "node:url";
|
|
14
15
|
import commonjs from "@rollup/plugin-commonjs";
|
|
15
16
|
import json from "@rollup/plugin-json";
|
|
@@ -86,45 +87,35 @@ function objectType(s, used) {
|
|
|
86
87
|
const required = new Set(s.required ?? []);
|
|
87
88
|
const lines = [];
|
|
88
89
|
for (const [key, sub] of Object.entries(props)) {
|
|
89
|
-
const doc = typeof sub.description === "string"
|
|
90
|
-
|
|
90
|
+
const doc = typeof sub.description === "string"
|
|
91
|
+
? ` /** ${sub.description} */\n`
|
|
92
|
+
: "";
|
|
93
|
+
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${nest(tsType(sub, used))};`);
|
|
91
94
|
}
|
|
92
95
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
93
|
-
lines.push(` [key: string]: ${tsType(s.additionalProperties, used)};`);
|
|
96
|
+
lines.push(` [key: string]: ${nest(tsType(s.additionalProperties, used))};`);
|
|
94
97
|
}
|
|
95
98
|
if (lines.length === 0)
|
|
96
99
|
return "Record<string, unknown>";
|
|
97
100
|
return `{\n${lines.join("\n")}\n}`;
|
|
98
101
|
}
|
|
102
|
+
/** Every line but the first, which is already placed by the property that opens it. */
|
|
103
|
+
const nest = (t) => t.replace(/\n/g, "\n ");
|
|
99
104
|
function union(parts) {
|
|
100
105
|
const seen = [...new Set(parts)];
|
|
101
106
|
return seen.length === 0 ? "unknown" : seen.join(" | ");
|
|
102
107
|
}
|
|
103
108
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
104
109
|
const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
105
|
-
const identifier = (n) =>
|
|
106
|
-
function deref(s, defs) {
|
|
107
|
-
let cur = s;
|
|
108
|
-
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
109
|
-
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
110
|
-
}
|
|
111
|
-
return cur;
|
|
112
|
-
}
|
|
113
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
114
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
115
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
116
|
-
* the navigation belongs here rather than in genctl. */
|
|
117
|
-
function scriptInput(site, defs) {
|
|
118
|
-
const action = deref(site.input, defs);
|
|
119
|
-
const bound = action?.properties?.input;
|
|
120
|
-
return bound ?? site.input;
|
|
121
|
-
}
|
|
110
|
+
const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
122
111
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
123
112
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
124
|
-
function declarations(
|
|
113
|
+
function declarations(at) {
|
|
114
|
+
const { site, where } = at;
|
|
115
|
+
const defs = where.$defs ?? {};
|
|
125
116
|
const used = new Set();
|
|
126
|
-
const input = tsType(
|
|
127
|
-
const output = tsType(site.
|
|
117
|
+
const input = tsType(site.types?.Input, used);
|
|
118
|
+
const output = tsType(site.types?.Output, used);
|
|
128
119
|
const emitted = [];
|
|
129
120
|
const done = new Set();
|
|
130
121
|
while (true) {
|
|
@@ -137,7 +128,7 @@ function declarations(site, defs) {
|
|
|
137
128
|
}
|
|
138
129
|
return [
|
|
139
130
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
140
|
-
`// ${
|
|
131
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
141
132
|
"",
|
|
142
133
|
...emitted,
|
|
143
134
|
emitted.length ? "" : "",
|
|
@@ -165,7 +156,7 @@ async function nearestTsconfig(from, root) {
|
|
|
165
156
|
return null;
|
|
166
157
|
}
|
|
167
158
|
}
|
|
168
|
-
async function typecheck(
|
|
159
|
+
async function typecheck(sites) {
|
|
169
160
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
170
161
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
171
162
|
const dir = join(root, ".genroc-cache");
|
|
@@ -173,13 +164,13 @@ async function typecheck(root, sites) {
|
|
|
173
164
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
174
165
|
// check each script under the other author's options.
|
|
175
166
|
const groups = new Map();
|
|
176
|
-
for (const
|
|
177
|
-
const base = (await nearestTsconfig(dirname(
|
|
167
|
+
for (const at of sites) {
|
|
168
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
178
169
|
const group = groups.get(base);
|
|
179
170
|
if (group)
|
|
180
|
-
group.push(
|
|
171
|
+
group.push(at);
|
|
181
172
|
else
|
|
182
|
-
groups.set(base, [
|
|
173
|
+
groups.set(base, [at]);
|
|
183
174
|
}
|
|
184
175
|
let n = 0;
|
|
185
176
|
for (const [base, group] of groups) {
|
|
@@ -200,7 +191,10 @@ async function typecheck(root, sites) {
|
|
|
200
191
|
// base config there is nothing to opt in with, so the default stays none.
|
|
201
192
|
...(base ? {} : { types: [] }),
|
|
202
193
|
},
|
|
203
|
-
files: group.flatMap((s) => [
|
|
194
|
+
files: group.flatMap((s) => [
|
|
195
|
+
relative(dir, s.file),
|
|
196
|
+
relative(dir, typesPathFor(s.file)),
|
|
197
|
+
]),
|
|
204
198
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
205
199
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
206
200
|
include: [],
|
|
@@ -250,7 +244,10 @@ const transpile = {
|
|
|
250
244
|
return { code: out.outputText, map: out.sourceMapText ?? null };
|
|
251
245
|
},
|
|
252
246
|
};
|
|
253
|
-
const BUILTIN = new Set([
|
|
247
|
+
const BUILTIN = new Set([
|
|
248
|
+
...builtinModules,
|
|
249
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
250
|
+
]);
|
|
254
251
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
255
252
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
256
253
|
* resolver to keep in agreement with tsc; this one cannot disagree.
|
|
@@ -270,22 +267,25 @@ function tsResolve(configPath) {
|
|
|
270
267
|
const { resolvedModule } = ts.resolveModuleName(source, importer, options, ts.sys);
|
|
271
268
|
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
272
269
|
return null;
|
|
273
|
-
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
270
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
271
|
+
? null
|
|
272
|
+
: resolvedModule.resolvedFileName;
|
|
274
273
|
},
|
|
275
274
|
};
|
|
276
275
|
}
|
|
277
276
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
278
277
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
279
278
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
280
|
-
async function bundle(
|
|
279
|
+
async function bundle(at) {
|
|
280
|
+
const site = at.site;
|
|
281
281
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
282
282
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
283
283
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
284
284
|
const built = await rollup({
|
|
285
|
-
input:
|
|
285
|
+
input: at.file,
|
|
286
286
|
external: (id) => BUILTIN.has(id),
|
|
287
287
|
plugins: [
|
|
288
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
288
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
289
289
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
290
290
|
commonjs(),
|
|
291
291
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -295,16 +295,19 @@ async function bundle(site, root) {
|
|
|
295
295
|
],
|
|
296
296
|
onwarn(warning) {
|
|
297
297
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
298
|
-
die(`${
|
|
298
|
+
die(`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`);
|
|
299
299
|
}
|
|
300
300
|
},
|
|
301
|
-
}).catch((e) => die(`${
|
|
302
|
-
const { output } = await built.generate({
|
|
301
|
+
}).catch((e) => die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`));
|
|
302
|
+
const { output } = await built.generate({
|
|
303
|
+
format: "es",
|
|
304
|
+
inlineDynamicImports: true,
|
|
305
|
+
});
|
|
303
306
|
await built.close();
|
|
304
307
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
305
308
|
// instance, and the file it names is on this machine.
|
|
306
309
|
if (!output[0].exports.includes("default")) {
|
|
307
|
-
die(`${
|
|
310
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
308
311
|
}
|
|
309
312
|
return output[0].code;
|
|
310
313
|
}
|
|
@@ -317,32 +320,73 @@ const stdin = await new Promise((resolve, reject) => {
|
|
|
317
320
|
process.stdin.on("error", reject);
|
|
318
321
|
});
|
|
319
322
|
const manifest = JSON.parse(stdin);
|
|
320
|
-
|
|
323
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
324
|
+
const root = process.cwd();
|
|
325
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
321
326
|
die("stdin is not a genroc resolver manifest");
|
|
327
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
328
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
329
|
+
function address(pointer) {
|
|
330
|
+
return pointer
|
|
331
|
+
.map((seg) => typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
332
|
+
? `.${seg}`
|
|
333
|
+
: `[${JSON.stringify(seg)}]`)
|
|
334
|
+
.join("")
|
|
335
|
+
.replace(/^\./, "");
|
|
336
|
+
}
|
|
337
|
+
const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
|
|
338
|
+
site,
|
|
339
|
+
where,
|
|
340
|
+
file: resolve(where.dir, site.argument),
|
|
341
|
+
})));
|
|
342
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
343
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
344
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
345
|
+
// and `external` is the same request made directly.
|
|
346
|
+
for (const at of located) {
|
|
347
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
348
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
349
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
350
|
+
// directly.
|
|
351
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
352
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
353
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
354
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
355
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
356
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
360
|
+
// does is the one that must say when it is not there.
|
|
361
|
+
for (const at of located) {
|
|
362
|
+
if (!existsSync(at.file)) {
|
|
363
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
364
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
322
367
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
323
368
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
324
369
|
const byPath = new Map();
|
|
325
|
-
for (const
|
|
326
|
-
const seen = byPath.get(
|
|
327
|
-
const defsOf = (x) => (manifest.schemas[x.process]?.$defs ?? {});
|
|
370
|
+
for (const at of located) {
|
|
371
|
+
const seen = byPath.get(at.file);
|
|
328
372
|
if (seen &&
|
|
329
|
-
JSON.stringify(
|
|
330
|
-
|
|
331
|
-
|
|
373
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)) {
|
|
374
|
+
die(`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
375
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
332
376
|
"Split it into two scripts, or make the two call sites pass the same shape.");
|
|
333
377
|
}
|
|
334
|
-
byPath.set(
|
|
378
|
+
byPath.set(at.file, at);
|
|
335
379
|
}
|
|
336
|
-
for (const
|
|
337
|
-
|
|
338
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
380
|
+
for (const at of byPath.values()) {
|
|
381
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
339
382
|
}
|
|
340
383
|
if (manifest.mode === "types") {
|
|
384
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
341
385
|
process.exit(0);
|
|
342
386
|
}
|
|
343
|
-
await typecheck(
|
|
387
|
+
await typecheck([...byPath.values()]);
|
|
344
388
|
const code = [];
|
|
345
|
-
for (const
|
|
346
|
-
code.push(await bundle(
|
|
389
|
+
for (const at of located) {
|
|
390
|
+
code.push(await bundle(at));
|
|
347
391
|
}
|
|
348
392
|
process.stdout.write(JSON.stringify({ code }));
|
package/dist/worker.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
//
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
import { readFileSync } from "node:fs";
|
|
8
|
-
import { evaluate } from "./eval.js";
|
|
8
|
+
import { Cancelled, evaluate } from "./eval.js";
|
|
9
9
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
10
10
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
11
11
|
// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
|
|
@@ -185,7 +185,7 @@ async function answer(token, outcome) {
|
|
|
185
185
|
console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
|
|
186
186
|
await release(token);
|
|
187
187
|
}
|
|
188
|
-
async function run(job) {
|
|
188
|
+
async function run(job, signal) {
|
|
189
189
|
let resolved;
|
|
190
190
|
try {
|
|
191
191
|
resolved = await resolveObjects(job);
|
|
@@ -206,9 +206,17 @@ async function run(job) {
|
|
|
206
206
|
}
|
|
207
207
|
let result;
|
|
208
208
|
try {
|
|
209
|
-
result = await evaluate(req);
|
|
209
|
+
result = await evaluate(req, signal);
|
|
210
210
|
}
|
|
211
211
|
catch (err) {
|
|
212
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
213
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
214
|
+
// waiting out a lease nobody is serving.
|
|
215
|
+
if (err instanceof Cancelled) {
|
|
216
|
+
console.log(`cancelled: ${job.token}`);
|
|
217
|
+
await release(job.token);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
212
220
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
213
221
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
214
222
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -245,11 +253,22 @@ async function renewLoop() {
|
|
|
245
253
|
tokens,
|
|
246
254
|
lease_ms: LEASE_MS,
|
|
247
255
|
});
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
256
|
+
if (!ok)
|
|
257
|
+
continue;
|
|
258
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
259
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
260
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
261
|
+
// own catch, so nothing is released twice.
|
|
262
|
+
for (const token of (data?.cancelled ?? [])) {
|
|
263
|
+
inFlight.get(token)?.abort.abort();
|
|
264
|
+
}
|
|
265
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
266
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
267
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
268
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
269
|
+
const lost = (data?.lost ?? []);
|
|
270
|
+
if (lost.length) {
|
|
271
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
}
|
|
@@ -258,8 +277,9 @@ async function pollLoop() {
|
|
|
258
277
|
const free = CONCURRENCY - inFlight.size;
|
|
259
278
|
const jobs = free > 0 ? await claim(free) : [];
|
|
260
279
|
for (const job of jobs) {
|
|
261
|
-
|
|
262
|
-
|
|
280
|
+
const abort = new AbortController();
|
|
281
|
+
inFlight.set(job.token, { job, abort });
|
|
282
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
263
283
|
}
|
|
264
284
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
265
285
|
// realms allow, not at the poll interval.
|
package/eval.ts
CHANGED
|
@@ -57,14 +57,32 @@ class RealmFault extends Error {
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
61
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
62
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
63
|
+
export class Cancelled extends Error {
|
|
64
|
+
constructor() {
|
|
65
|
+
super("cancelled");
|
|
66
|
+
this.name = "Cancelled";
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
|
|
61
71
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
62
72
|
|
|
63
73
|
const worker = new Worker(REALM_URL);
|
|
64
74
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
75
|
+
let onAbort: (() => void) | undefined;
|
|
65
76
|
try {
|
|
66
77
|
return await new Promise<EvalResult>((resolve, reject) => {
|
|
67
78
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
79
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
80
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
81
|
+
if (signal) {
|
|
82
|
+
if (signal.aborted) reject(new Cancelled());
|
|
83
|
+
onAbort = () => reject(new Cancelled());
|
|
84
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
85
|
+
}
|
|
68
86
|
worker.once("message", (reply: WorkerReply) => resolve(reply));
|
|
69
87
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
70
88
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -75,6 +93,7 @@ export async function evaluate(req: EvalRequest): Promise<EvalResult> {
|
|
|
75
93
|
});
|
|
76
94
|
} finally {
|
|
77
95
|
clearTimeout(timer);
|
|
96
|
+
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
78
97
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
79
98
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
80
99
|
await worker.terminate();
|
package/import.ts
CHANGED
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
// over the same project.
|
|
9
9
|
|
|
10
10
|
import { spawn } from "node:child_process";
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
11
12
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
12
13
|
import { builtinModules } from "node:module";
|
|
13
|
-
import { dirname, join, relative } from "node:path";
|
|
14
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
14
15
|
import { fileURLToPath } from "node:url";
|
|
15
16
|
|
|
16
17
|
import commonjs from "@rollup/plugin-commonjs";
|
|
@@ -22,20 +23,37 @@ import ts from "typescript";
|
|
|
22
23
|
type Schema = Record<string, any>;
|
|
23
24
|
|
|
24
25
|
type Site = {
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
// Which namespace the directive sits in: `process`, `task` or `action`.
|
|
27
|
+
level: "process" | "task" | "action";
|
|
27
28
|
task?: string;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
// What the site IS, at action level: the action's type, and the process a child call names.
|
|
30
|
+
action?: string;
|
|
31
|
+
child?: string;
|
|
32
|
+
// The slot as a path of keys and indices, so nothing has to unescape a JSON Pointer.
|
|
33
|
+
pointer: (string | number)[];
|
|
34
|
+
// Everything after `$<resolver>:`, verbatim. genctl does not read it as a path — that is
|
|
35
|
+
// this resolver's reading of it.
|
|
36
|
+
argument: string;
|
|
37
|
+
// What .genroc asked genctl to type, keyed by the name it chose. This resolver reads Input
|
|
38
|
+
// and Output; which ADDRESS each came from is the config's business, not ours.
|
|
39
|
+
types?: Record<string, Schema>;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
type ManifestProcess = {
|
|
43
|
+
name: string;
|
|
44
|
+
// The definition's own location, split because a relative argument is relative to the
|
|
45
|
+
// DIRECTORY — joining is ours to do, and this is the base.
|
|
46
|
+
dir: string;
|
|
47
|
+
file: string;
|
|
48
|
+
sites: Site[];
|
|
49
|
+
// Only what this process's fragments reach — a `$ref` in `types` points here.
|
|
50
|
+
$defs?: Record<string, Schema>;
|
|
32
51
|
};
|
|
33
52
|
|
|
34
53
|
type Manifest = {
|
|
35
54
|
mode: "types" | "build";
|
|
36
55
|
root: string;
|
|
37
|
-
|
|
38
|
-
sites: Site[];
|
|
56
|
+
processes: ManifestProcess[];
|
|
39
57
|
};
|
|
40
58
|
|
|
41
59
|
function die(message: string): never {
|
|
@@ -72,13 +90,16 @@ function tsType(s: Schema | undefined, used: Set<string>): string {
|
|
|
72
90
|
if (Array.isArray(s.enum)) {
|
|
73
91
|
return s.enum.map((v: unknown) => JSON.stringify(v)).join(" | ") || "never";
|
|
74
92
|
}
|
|
75
|
-
if (Array.isArray(s.anyOf))
|
|
76
|
-
|
|
93
|
+
if (Array.isArray(s.anyOf))
|
|
94
|
+
return union(s.anyOf.map((a: Schema) => tsType(a, used)));
|
|
95
|
+
if (Array.isArray(s.oneOf))
|
|
96
|
+
return union(s.oneOf.map((a: Schema) => tsType(a, used)));
|
|
77
97
|
if (Array.isArray(s.allOf)) {
|
|
78
98
|
return s.allOf.map((a: Schema) => tsType(a, used)).join(" & ") || "unknown";
|
|
79
99
|
}
|
|
80
100
|
|
|
81
|
-
const types: string[] =
|
|
101
|
+
const types: string[] =
|
|
102
|
+
s.type === undefined ? [] : Array.isArray(s.type) ? s.type : [s.type];
|
|
82
103
|
if (types.length === 0) {
|
|
83
104
|
// The top type: `{}` means unknown, not "an empty object". specs/unknown-type.md.
|
|
84
105
|
return s.properties ? objectType(s, used) : "unknown";
|
|
@@ -111,16 +132,26 @@ function objectType(s: Schema, used: Set<string>): string {
|
|
|
111
132
|
const required = new Set<string>(s.required ?? []);
|
|
112
133
|
const lines: string[] = [];
|
|
113
134
|
for (const [key, sub] of Object.entries(props)) {
|
|
114
|
-
const doc =
|
|
115
|
-
|
|
135
|
+
const doc =
|
|
136
|
+
typeof sub.description === "string"
|
|
137
|
+
? ` /** ${sub.description} */\n`
|
|
138
|
+
: "";
|
|
139
|
+
lines.push(
|
|
140
|
+
`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${nest(tsType(sub, used))};`,
|
|
141
|
+
);
|
|
116
142
|
}
|
|
117
143
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
118
|
-
lines.push(
|
|
144
|
+
lines.push(
|
|
145
|
+
` [key: string]: ${nest(tsType(s.additionalProperties, used))};`,
|
|
146
|
+
);
|
|
119
147
|
}
|
|
120
148
|
if (lines.length === 0) return "Record<string, unknown>";
|
|
121
149
|
return `{\n${lines.join("\n")}\n}`;
|
|
122
150
|
}
|
|
123
151
|
|
|
152
|
+
/** Every line but the first, which is already placed by the property that opens it. */
|
|
153
|
+
const nest = (t: string) => t.replace(/\n/g, "\n ");
|
|
154
|
+
|
|
124
155
|
function union(parts: string[]): string {
|
|
125
156
|
const seen = [...new Set(parts)];
|
|
126
157
|
return seen.length === 0 ? "unknown" : seen.join(" | ");
|
|
@@ -128,32 +159,17 @@ function union(parts: string[]): string {
|
|
|
128
159
|
|
|
129
160
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
130
161
|
const propKey = (k: string) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
131
|
-
const identifier = (n: string) =>
|
|
132
|
-
|
|
133
|
-
function deref(s: Schema | undefined, defs: Record<string, Schema>): Schema | undefined {
|
|
134
|
-
let cur = s;
|
|
135
|
-
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
136
|
-
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
137
|
-
}
|
|
138
|
-
return cur;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
142
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
143
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
144
|
-
* the navigation belongs here rather than in genctl. */
|
|
145
|
-
function scriptInput(site: Site, defs: Record<string, Schema>): Schema | undefined {
|
|
146
|
-
const action = deref(site.input, defs);
|
|
147
|
-
const bound = action?.properties?.input;
|
|
148
|
-
return bound ?? site.input;
|
|
149
|
-
}
|
|
162
|
+
const identifier = (n: string) =>
|
|
163
|
+
IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
150
164
|
|
|
151
165
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
152
166
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
153
|
-
function declarations(
|
|
167
|
+
function declarations(at: Located): string {
|
|
168
|
+
const { site, where } = at;
|
|
169
|
+
const defs = where.$defs ?? {};
|
|
154
170
|
const used = new Set<string>();
|
|
155
|
-
const input = tsType(
|
|
156
|
-
const output = tsType(site.
|
|
171
|
+
const input = tsType(site.types?.Input, used);
|
|
172
|
+
const output = tsType(site.types?.Output, used);
|
|
157
173
|
|
|
158
174
|
const emitted: string[] = [];
|
|
159
175
|
const done = new Set<string>();
|
|
@@ -167,7 +183,7 @@ function declarations(site: Site, defs: Record<string, Schema>): string {
|
|
|
167
183
|
|
|
168
184
|
return [
|
|
169
185
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
170
|
-
`// ${
|
|
186
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
171
187
|
"",
|
|
172
188
|
...emitted,
|
|
173
189
|
emitted.length ? "" : "",
|
|
@@ -189,7 +205,10 @@ function typesPathFor(scriptPath: string): string {
|
|
|
189
205
|
/** The nearest tsconfig above the script — the one the author's editor already reads. Two
|
|
190
206
|
* different configs mean a red editor over a clean apply, or the reverse. The walk stops at
|
|
191
207
|
* the project root: above it is not this project. */
|
|
192
|
-
async function nearestTsconfig(
|
|
208
|
+
async function nearestTsconfig(
|
|
209
|
+
from: string,
|
|
210
|
+
root: string,
|
|
211
|
+
): Promise<string | null> {
|
|
193
212
|
for (let dir = from; ; dir = dirname(dir)) {
|
|
194
213
|
const candidate = join(dir, "tsconfig.json");
|
|
195
214
|
if (await exists(candidate)) return candidate;
|
|
@@ -197,7 +216,7 @@ async function nearestTsconfig(from: string, root: string): Promise<string | nul
|
|
|
197
216
|
}
|
|
198
217
|
}
|
|
199
218
|
|
|
200
|
-
async function typecheck(
|
|
219
|
+
async function typecheck(sites: Located[]): Promise<void> {
|
|
201
220
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
202
221
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
203
222
|
const dir = join(root, ".genroc-cache");
|
|
@@ -205,12 +224,12 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
205
224
|
|
|
206
225
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
207
226
|
// check each script under the other author's options.
|
|
208
|
-
const groups = new Map<string,
|
|
209
|
-
for (const
|
|
210
|
-
const base = (await nearestTsconfig(dirname(
|
|
227
|
+
const groups = new Map<string, Located[]>();
|
|
228
|
+
for (const at of sites) {
|
|
229
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
211
230
|
const group = groups.get(base);
|
|
212
|
-
if (group) group.push(
|
|
213
|
-
else groups.set(base, [
|
|
231
|
+
if (group) group.push(at);
|
|
232
|
+
else groups.set(base, [at]);
|
|
214
233
|
}
|
|
215
234
|
|
|
216
235
|
let n = 0;
|
|
@@ -232,12 +251,18 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
232
251
|
// base config there is nothing to opt in with, so the default stays none.
|
|
233
252
|
...(base ? {} : { types: [] }),
|
|
234
253
|
},
|
|
235
|
-
files: group.flatMap((s) => [
|
|
254
|
+
files: group.flatMap((s) => [
|
|
255
|
+
relative(dir, s.file),
|
|
256
|
+
relative(dir, typesPathFor(s.file)),
|
|
257
|
+
]),
|
|
236
258
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
237
259
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
238
260
|
include: [],
|
|
239
261
|
};
|
|
240
|
-
const configPath = join(
|
|
262
|
+
const configPath = join(
|
|
263
|
+
dir,
|
|
264
|
+
groups.size === 1 ? "tsconfig.json" : `tsconfig.${n++}.json`,
|
|
265
|
+
);
|
|
241
266
|
await write(configPath, JSON.stringify(config, null, 2));
|
|
242
267
|
await runTsc(root, configPath);
|
|
243
268
|
}
|
|
@@ -285,7 +310,10 @@ const transpile: Plugin = {
|
|
|
285
310
|
},
|
|
286
311
|
};
|
|
287
312
|
|
|
288
|
-
const BUILTIN = new Set([
|
|
313
|
+
const BUILTIN = new Set([
|
|
314
|
+
...builtinModules,
|
|
315
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
316
|
+
]);
|
|
289
317
|
|
|
290
318
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
291
319
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
@@ -296,15 +324,27 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
296
324
|
let options: ts.CompilerOptions = {};
|
|
297
325
|
if (configPath) {
|
|
298
326
|
const read = ts.readConfigFile(configPath, ts.sys.readFile);
|
|
299
|
-
options = ts.parseJsonConfigFileContent(
|
|
327
|
+
options = ts.parseJsonConfigFileContent(
|
|
328
|
+
read.config ?? {},
|
|
329
|
+
ts.sys,
|
|
330
|
+
dirname(configPath),
|
|
331
|
+
).options;
|
|
300
332
|
}
|
|
301
333
|
return {
|
|
302
334
|
name: "genroc-ts-resolve",
|
|
303
335
|
resolveId(source, importer) {
|
|
304
336
|
if (!importer || BUILTIN.has(source)) return null;
|
|
305
|
-
const { resolvedModule } = ts.resolveModuleName(
|
|
306
|
-
|
|
307
|
-
|
|
337
|
+
const { resolvedModule } = ts.resolveModuleName(
|
|
338
|
+
source,
|
|
339
|
+
importer,
|
|
340
|
+
options,
|
|
341
|
+
ts.sys,
|
|
342
|
+
);
|
|
343
|
+
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
344
|
+
return null;
|
|
345
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
346
|
+
? null
|
|
347
|
+
: resolvedModule.resolvedFileName;
|
|
308
348
|
},
|
|
309
349
|
};
|
|
310
350
|
}
|
|
@@ -312,15 +352,16 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
312
352
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
313
353
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
314
354
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
315
|
-
async function bundle(
|
|
355
|
+
async function bundle(at: Located): Promise<string> {
|
|
356
|
+
const site = at.site;
|
|
316
357
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
317
358
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
318
359
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
319
360
|
const built = await rollup({
|
|
320
|
-
input:
|
|
361
|
+
input: at.file,
|
|
321
362
|
external: (id) => BUILTIN.has(id),
|
|
322
363
|
plugins: [
|
|
323
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
364
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
324
365
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
325
366
|
commonjs(),
|
|
326
367
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -330,17 +371,24 @@ async function bundle(site: Site, root: string): Promise<string> {
|
|
|
330
371
|
],
|
|
331
372
|
onwarn(warning) {
|
|
332
373
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
333
|
-
die(
|
|
374
|
+
die(
|
|
375
|
+
`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`,
|
|
376
|
+
);
|
|
334
377
|
}
|
|
335
378
|
},
|
|
336
|
-
}).catch((e: unknown) =>
|
|
379
|
+
}).catch((e: unknown) =>
|
|
380
|
+
die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`),
|
|
381
|
+
);
|
|
337
382
|
|
|
338
|
-
const { output } = await built.generate({
|
|
383
|
+
const { output } = await built.generate({
|
|
384
|
+
format: "es",
|
|
385
|
+
inlineDynamicImports: true,
|
|
386
|
+
});
|
|
339
387
|
await built.close();
|
|
340
388
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
341
389
|
// instance, and the file it names is on this machine.
|
|
342
390
|
if (!output[0].exports.includes("default")) {
|
|
343
|
-
die(`${
|
|
391
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
344
392
|
}
|
|
345
393
|
return output[0].code;
|
|
346
394
|
}
|
|
@@ -355,40 +403,99 @@ const stdin: string = await new Promise((resolve, reject) => {
|
|
|
355
403
|
process.stdin.on("error", reject);
|
|
356
404
|
});
|
|
357
405
|
const manifest = JSON.parse(stdin) as Manifest;
|
|
358
|
-
|
|
406
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
407
|
+
const root = process.cwd();
|
|
408
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
409
|
+
die("stdin is not a genroc resolver manifest");
|
|
410
|
+
|
|
411
|
+
/** Every site with the process it sits in and the file its argument names, in the manifest's own
|
|
412
|
+
* order — which is the order `code` must answer in. genctl passes the argument verbatim, so
|
|
413
|
+
* joining it to the definition's directory is this resolver's business: only it knows the
|
|
414
|
+
* argument is a path at all. */
|
|
415
|
+
type Located = { site: Site; where: ManifestProcess; file: string };
|
|
416
|
+
|
|
417
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
418
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
419
|
+
function address(pointer: (string | number)[]): string {
|
|
420
|
+
return pointer
|
|
421
|
+
.map((seg) =>
|
|
422
|
+
typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
423
|
+
? `.${seg}`
|
|
424
|
+
: `[${JSON.stringify(seg)}]`,
|
|
425
|
+
)
|
|
426
|
+
.join("")
|
|
427
|
+
.replace(/^\./, "");
|
|
428
|
+
}
|
|
429
|
+
const located: Located[] = manifest.processes.flatMap((where) =>
|
|
430
|
+
where.sites.map((site) => ({
|
|
431
|
+
site,
|
|
432
|
+
where,
|
|
433
|
+
file: resolve(where.dir, site.argument),
|
|
434
|
+
})),
|
|
435
|
+
);
|
|
436
|
+
|
|
437
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
438
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
439
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
440
|
+
// and `external` is the same request made directly.
|
|
441
|
+
for (const at of located) {
|
|
442
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
443
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
444
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
445
|
+
// directly.
|
|
446
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
447
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
448
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
449
|
+
die(
|
|
450
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
451
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
452
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`,
|
|
453
|
+
);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
458
|
+
// does is the one that must say when it is not there.
|
|
459
|
+
for (const at of located) {
|
|
460
|
+
if (!existsSync(at.file)) {
|
|
461
|
+
die(
|
|
462
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
463
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`,
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
359
467
|
|
|
360
468
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
361
469
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
362
|
-
const byPath = new Map<string,
|
|
363
|
-
for (const
|
|
364
|
-
const seen = byPath.get(
|
|
365
|
-
const defsOf = (x: Site) => (manifest.schemas[x.process]?.$defs ?? {}) as Record<string, Schema>;
|
|
470
|
+
const byPath = new Map<string, Located>();
|
|
471
|
+
for (const at of located) {
|
|
472
|
+
const seen = byPath.get(at.file);
|
|
366
473
|
if (
|
|
367
474
|
seen &&
|
|
368
|
-
JSON.stringify(
|
|
369
|
-
JSON.stringify(scriptInput(site, defsOf(site)))
|
|
475
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)
|
|
370
476
|
) {
|
|
371
477
|
die(
|
|
372
|
-
`${
|
|
478
|
+
`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
479
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
373
480
|
"Split it into two scripts, or make the two call sites pass the same shape.",
|
|
374
481
|
);
|
|
375
482
|
}
|
|
376
|
-
byPath.set(
|
|
483
|
+
byPath.set(at.file, at);
|
|
377
484
|
}
|
|
378
485
|
|
|
379
|
-
for (const
|
|
380
|
-
|
|
381
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
486
|
+
for (const at of byPath.values()) {
|
|
487
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
382
488
|
}
|
|
383
489
|
|
|
384
490
|
if (manifest.mode === "types") {
|
|
491
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
385
492
|
process.exit(0);
|
|
386
493
|
}
|
|
387
494
|
|
|
388
|
-
await typecheck(
|
|
495
|
+
await typecheck([...byPath.values()]);
|
|
389
496
|
|
|
390
497
|
const code: string[] = [];
|
|
391
|
-
for (const
|
|
392
|
-
code.push(await bundle(
|
|
498
|
+
for (const at of located) {
|
|
499
|
+
code.push(await bundle(at));
|
|
393
500
|
}
|
|
394
501
|
process.stdout.write(JSON.stringify({ code }));
|
package/package.json
CHANGED
package/worker.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
|
|
8
8
|
import { readFileSync } from "node:fs";
|
|
9
|
-
import { evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
9
|
+
import { Cancelled, evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
10
10
|
|
|
11
11
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
12
12
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
@@ -135,7 +135,12 @@ function asEvalRequest(input: unknown): EvalRequest | string {
|
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
|
|
138
|
+
/** A claim this worker is currently serving. The controller is how the renewal loop reaches
|
|
139
|
+
* the evaluation: genroc cannot call us, so a cancellation arrives as an answer to our own
|
|
140
|
+
* heartbeat and has to be delivered inward from there. */
|
|
141
|
+
type Running = { job: QueueTask; abort: AbortController };
|
|
142
|
+
|
|
143
|
+
const inFlight = new Map<string, Running>();
|
|
139
144
|
let running = true;
|
|
140
145
|
|
|
141
146
|
// Whether the last claim reached genroc. A worker polls several times a second, so an
|
|
@@ -203,7 +208,7 @@ async function answer(token: string, outcome: Record<string, unknown>): Promise<
|
|
|
203
208
|
await release(token);
|
|
204
209
|
}
|
|
205
210
|
|
|
206
|
-
async function run(job: QueueTask): Promise<void> {
|
|
211
|
+
async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
207
212
|
let resolved: unknown;
|
|
208
213
|
try {
|
|
209
214
|
resolved = await resolveObjects(job);
|
|
@@ -224,8 +229,16 @@ async function run(job: QueueTask): Promise<void> {
|
|
|
224
229
|
|
|
225
230
|
let result;
|
|
226
231
|
try {
|
|
227
|
-
result = await evaluate(req);
|
|
232
|
+
result = await evaluate(req, signal);
|
|
228
233
|
} catch (err) {
|
|
234
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
235
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
236
|
+
// waiting out a lease nobody is serving.
|
|
237
|
+
if (err instanceof Cancelled) {
|
|
238
|
+
console.log(`cancelled: ${job.token}`);
|
|
239
|
+
await release(job.token);
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
229
242
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
230
243
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
231
244
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -263,11 +276,22 @@ async function renewLoop(): Promise<void> {
|
|
|
263
276
|
tokens,
|
|
264
277
|
lease_ms: LEASE_MS,
|
|
265
278
|
});
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
279
|
+
if (!ok) continue;
|
|
280
|
+
|
|
281
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
282
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
283
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
284
|
+
// own catch, so nothing is released twice.
|
|
285
|
+
for (const token of (data?.cancelled ?? []) as string[]) {
|
|
286
|
+
inFlight.get(token)?.abort.abort();
|
|
287
|
+
}
|
|
288
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
289
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
290
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
291
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
292
|
+
const lost = (data?.lost ?? []) as string[];
|
|
293
|
+
if (lost.length) {
|
|
294
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
271
295
|
}
|
|
272
296
|
}
|
|
273
297
|
}
|
|
@@ -277,8 +301,9 @@ async function pollLoop(): Promise<void> {
|
|
|
277
301
|
const free = CONCURRENCY - inFlight.size;
|
|
278
302
|
const jobs = free > 0 ? await claim(free) : [];
|
|
279
303
|
for (const job of jobs) {
|
|
280
|
-
|
|
281
|
-
|
|
304
|
+
const abort = new AbortController();
|
|
305
|
+
inFlight.set(job.token, { job, abort });
|
|
306
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
282
307
|
}
|
|
283
308
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
284
309
|
// realms allow, not at the poll interval.
|