@genroc/eval-node 0.0.0-edge.ee19083 → 0.0.0-edge.f163e4e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +19 -2
- package/dist/eval.js +21 -1
- package/dist/import.js +94 -52
- package/dist/worker.js +30 -10
- package/eval.ts +20 -1
- package/import.ts +175 -73
- package/package.json +1 -1
- package/worker.ts +36 -11
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ resolver's exit code, so a stored definition cannot hold code that failed to typ
|
|
|
26
26
|
|
|
27
27
|
## Building
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
pnpm run build # tsc -p tsconfig.build.json -> dist/
|
|
30
30
|
|
|
31
31
|
The published package is JavaScript, not TypeScript. **Node refuses to strip types for files
|
|
32
32
|
under `node_modules`**, so shipping `.ts` works from a checkout and fails for every consumer —
|
|
@@ -90,6 +90,23 @@ queue worker rather than the HTTP sidecar it used to be:
|
|
|
90
90
|
that genroc had to be able to reach. A worker only needs outbound access, so it can live
|
|
91
91
|
anywhere — behind NAT, in another trust zone.
|
|
92
92
|
|
|
93
|
+
The cost of that inversion is that genroc cannot reach a running worker, so anything it needs
|
|
94
|
+
to say has to ride the renewal the worker already makes. **Renewing is therefore mandatory,
|
|
95
|
+
not an optimisation** — a worker that stops renewing is indistinguishable from one that died.
|
|
96
|
+
The response answers per token:
|
|
97
|
+
|
|
98
|
+
| list | what it means | what this worker does |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `renewed` | still yours | carry on |
|
|
101
|
+
| `lost` | already someone else's | stop; do **not** release — that would bump the new holder's claim |
|
|
102
|
+
| `cancelled` | still yours, nobody wants it | abort the script and release the claim |
|
|
103
|
+
|
|
104
|
+
`cancelled` is how an operator's `genctl cancel` reaches work already running: the evaluation
|
|
105
|
+
is aborted through an `AbortSignal`, its realm is terminated, and no outcome is submitted —
|
|
106
|
+
the process is stopping, so there is nothing left to answer. `renew_before_ms` on the claim
|
|
107
|
+
says how long the worker may wait before renewing again, so the interval is a value it reads
|
|
108
|
+
rather than one it guesses.
|
|
109
|
+
|
|
93
110
|
## The task input
|
|
94
111
|
|
|
95
112
|
The `input` of the external task IS the evaluation request:
|
|
@@ -169,7 +186,7 @@ no retry policy to write.
|
|
|
169
186
|
|
|
170
187
|
- id: script_failed
|
|
171
188
|
switch:
|
|
172
|
-
- case: '
|
|
189
|
+
- case: 'last_error.data.name == "LimitExceeded"'
|
|
173
190
|
raise: { code: limit_exceeded, message: "the script rejected the amount" }
|
|
174
191
|
- raise: { code: script_failed, message: "the script failed" }
|
|
175
192
|
```
|
package/dist/eval.js
CHANGED
|
@@ -20,13 +20,31 @@ class RealmFault extends Error {
|
|
|
20
20
|
this.name = "RealmFault";
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
24
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
25
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
26
|
+
export class Cancelled extends Error {
|
|
27
|
+
constructor() {
|
|
28
|
+
super("cancelled");
|
|
29
|
+
this.name = "Cancelled";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export async function evaluate(req, signal) {
|
|
24
33
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
25
34
|
const worker = new Worker(REALM_URL);
|
|
26
35
|
let timer;
|
|
36
|
+
let onAbort;
|
|
27
37
|
try {
|
|
28
38
|
return await new Promise((resolve, reject) => {
|
|
29
39
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
40
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
41
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
42
|
+
if (signal) {
|
|
43
|
+
if (signal.aborted)
|
|
44
|
+
reject(new Cancelled());
|
|
45
|
+
onAbort = () => reject(new Cancelled());
|
|
46
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
47
|
+
}
|
|
30
48
|
worker.once("message", (reply) => resolve(reply));
|
|
31
49
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
32
50
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -38,6 +56,8 @@ export async function evaluate(req) {
|
|
|
38
56
|
}
|
|
39
57
|
finally {
|
|
40
58
|
clearTimeout(timer);
|
|
59
|
+
if (signal && onAbort)
|
|
60
|
+
signal.removeEventListener("abort", onAbort);
|
|
41
61
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
42
62
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
43
63
|
await worker.terminate();
|
package/dist/import.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
8
|
// over the same project.
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
|
+
import { existsSync } from "node:fs";
|
|
10
11
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
11
12
|
import { builtinModules } from "node:module";
|
|
12
|
-
import { dirname, join, relative } from "node:path";
|
|
13
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
13
14
|
import { fileURLToPath } from "node:url";
|
|
14
15
|
import commonjs from "@rollup/plugin-commonjs";
|
|
15
16
|
import json from "@rollup/plugin-json";
|
|
@@ -86,7 +87,9 @@ function objectType(s, used) {
|
|
|
86
87
|
const required = new Set(s.required ?? []);
|
|
87
88
|
const lines = [];
|
|
88
89
|
for (const [key, sub] of Object.entries(props)) {
|
|
89
|
-
const doc = typeof sub.description === "string"
|
|
90
|
+
const doc = typeof sub.description === "string"
|
|
91
|
+
? ` /** ${sub.description} */\n`
|
|
92
|
+
: "";
|
|
90
93
|
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`);
|
|
91
94
|
}
|
|
92
95
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
@@ -102,29 +105,15 @@ function union(parts) {
|
|
|
102
105
|
}
|
|
103
106
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
104
107
|
const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
105
|
-
const identifier = (n) =>
|
|
106
|
-
function deref(s, defs) {
|
|
107
|
-
let cur = s;
|
|
108
|
-
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
109
|
-
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
110
|
-
}
|
|
111
|
-
return cur;
|
|
112
|
-
}
|
|
113
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
114
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
115
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
116
|
-
* the navigation belongs here rather than in genctl. */
|
|
117
|
-
function scriptInput(site, defs) {
|
|
118
|
-
const action = deref(site.input, defs);
|
|
119
|
-
const bound = action?.properties?.input;
|
|
120
|
-
return bound ?? site.input;
|
|
121
|
-
}
|
|
108
|
+
const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
122
109
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
123
110
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
124
|
-
function declarations(
|
|
111
|
+
function declarations(at) {
|
|
112
|
+
const { site, where } = at;
|
|
113
|
+
const defs = where.$defs ?? {};
|
|
125
114
|
const used = new Set();
|
|
126
|
-
const input = tsType(
|
|
127
|
-
const output = tsType(site.
|
|
115
|
+
const input = tsType(site.types?.Input, used);
|
|
116
|
+
const output = tsType(site.types?.Output, used);
|
|
128
117
|
const emitted = [];
|
|
129
118
|
const done = new Set();
|
|
130
119
|
while (true) {
|
|
@@ -137,7 +126,7 @@ function declarations(site, defs) {
|
|
|
137
126
|
}
|
|
138
127
|
return [
|
|
139
128
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
140
|
-
`// ${
|
|
129
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
141
130
|
"",
|
|
142
131
|
...emitted,
|
|
143
132
|
emitted.length ? "" : "",
|
|
@@ -165,7 +154,7 @@ async function nearestTsconfig(from, root) {
|
|
|
165
154
|
return null;
|
|
166
155
|
}
|
|
167
156
|
}
|
|
168
|
-
async function typecheck(
|
|
157
|
+
async function typecheck(sites) {
|
|
169
158
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
170
159
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
171
160
|
const dir = join(root, ".genroc-cache");
|
|
@@ -173,13 +162,13 @@ async function typecheck(root, sites) {
|
|
|
173
162
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
174
163
|
// check each script under the other author's options.
|
|
175
164
|
const groups = new Map();
|
|
176
|
-
for (const
|
|
177
|
-
const base = (await nearestTsconfig(dirname(
|
|
165
|
+
for (const at of sites) {
|
|
166
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
178
167
|
const group = groups.get(base);
|
|
179
168
|
if (group)
|
|
180
|
-
group.push(
|
|
169
|
+
group.push(at);
|
|
181
170
|
else
|
|
182
|
-
groups.set(base, [
|
|
171
|
+
groups.set(base, [at]);
|
|
183
172
|
}
|
|
184
173
|
let n = 0;
|
|
185
174
|
for (const [base, group] of groups) {
|
|
@@ -200,7 +189,10 @@ async function typecheck(root, sites) {
|
|
|
200
189
|
// base config there is nothing to opt in with, so the default stays none.
|
|
201
190
|
...(base ? {} : { types: [] }),
|
|
202
191
|
},
|
|
203
|
-
files: group.flatMap((s) => [
|
|
192
|
+
files: group.flatMap((s) => [
|
|
193
|
+
relative(dir, s.file),
|
|
194
|
+
relative(dir, typesPathFor(s.file)),
|
|
195
|
+
]),
|
|
204
196
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
205
197
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
206
198
|
include: [],
|
|
@@ -250,7 +242,10 @@ const transpile = {
|
|
|
250
242
|
return { code: out.outputText, map: out.sourceMapText ?? null };
|
|
251
243
|
},
|
|
252
244
|
};
|
|
253
|
-
const BUILTIN = new Set([
|
|
245
|
+
const BUILTIN = new Set([
|
|
246
|
+
...builtinModules,
|
|
247
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
248
|
+
]);
|
|
254
249
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
255
250
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
256
251
|
* resolver to keep in agreement with tsc; this one cannot disagree.
|
|
@@ -270,22 +265,25 @@ function tsResolve(configPath) {
|
|
|
270
265
|
const { resolvedModule } = ts.resolveModuleName(source, importer, options, ts.sys);
|
|
271
266
|
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
272
267
|
return null;
|
|
273
|
-
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
268
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
269
|
+
? null
|
|
270
|
+
: resolvedModule.resolvedFileName;
|
|
274
271
|
},
|
|
275
272
|
};
|
|
276
273
|
}
|
|
277
274
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
278
275
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
279
276
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
280
|
-
async function bundle(
|
|
277
|
+
async function bundle(at) {
|
|
278
|
+
const site = at.site;
|
|
281
279
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
282
280
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
283
281
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
284
282
|
const built = await rollup({
|
|
285
|
-
input:
|
|
283
|
+
input: at.file,
|
|
286
284
|
external: (id) => BUILTIN.has(id),
|
|
287
285
|
plugins: [
|
|
288
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
286
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
289
287
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
290
288
|
commonjs(),
|
|
291
289
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -295,16 +293,19 @@ async function bundle(site, root) {
|
|
|
295
293
|
],
|
|
296
294
|
onwarn(warning) {
|
|
297
295
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
298
|
-
die(`${
|
|
296
|
+
die(`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`);
|
|
299
297
|
}
|
|
300
298
|
},
|
|
301
|
-
}).catch((e) => die(`${
|
|
302
|
-
const { output } = await built.generate({
|
|
299
|
+
}).catch((e) => die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`));
|
|
300
|
+
const { output } = await built.generate({
|
|
301
|
+
format: "es",
|
|
302
|
+
inlineDynamicImports: true,
|
|
303
|
+
});
|
|
303
304
|
await built.close();
|
|
304
305
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
305
306
|
// instance, and the file it names is on this machine.
|
|
306
307
|
if (!output[0].exports.includes("default")) {
|
|
307
|
-
die(`${
|
|
308
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
308
309
|
}
|
|
309
310
|
return output[0].code;
|
|
310
311
|
}
|
|
@@ -317,32 +318,73 @@ const stdin = await new Promise((resolve, reject) => {
|
|
|
317
318
|
process.stdin.on("error", reject);
|
|
318
319
|
});
|
|
319
320
|
const manifest = JSON.parse(stdin);
|
|
320
|
-
|
|
321
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
322
|
+
const root = process.cwd();
|
|
323
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
321
324
|
die("stdin is not a genroc resolver manifest");
|
|
325
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
326
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
327
|
+
function address(pointer) {
|
|
328
|
+
return pointer
|
|
329
|
+
.map((seg) => typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
330
|
+
? `.${seg}`
|
|
331
|
+
: `[${JSON.stringify(seg)}]`)
|
|
332
|
+
.join("")
|
|
333
|
+
.replace(/^\./, "");
|
|
334
|
+
}
|
|
335
|
+
const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
|
|
336
|
+
site,
|
|
337
|
+
where,
|
|
338
|
+
file: resolve(where.dir, site.argument),
|
|
339
|
+
})));
|
|
340
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
341
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
342
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
343
|
+
// and `external` is the same request made directly.
|
|
344
|
+
for (const at of located) {
|
|
345
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
346
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
347
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
348
|
+
// directly.
|
|
349
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
350
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
351
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
352
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
353
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
354
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
358
|
+
// does is the one that must say when it is not there.
|
|
359
|
+
for (const at of located) {
|
|
360
|
+
if (!existsSync(at.file)) {
|
|
361
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
362
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
322
365
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
323
366
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
324
367
|
const byPath = new Map();
|
|
325
|
-
for (const
|
|
326
|
-
const seen = byPath.get(
|
|
327
|
-
const defsOf = (x) => (manifest.schemas[x.process]?.$defs ?? {});
|
|
368
|
+
for (const at of located) {
|
|
369
|
+
const seen = byPath.get(at.file);
|
|
328
370
|
if (seen &&
|
|
329
|
-
JSON.stringify(
|
|
330
|
-
|
|
331
|
-
|
|
371
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)) {
|
|
372
|
+
die(`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
373
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
332
374
|
"Split it into two scripts, or make the two call sites pass the same shape.");
|
|
333
375
|
}
|
|
334
|
-
byPath.set(
|
|
376
|
+
byPath.set(at.file, at);
|
|
335
377
|
}
|
|
336
|
-
for (const
|
|
337
|
-
|
|
338
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
378
|
+
for (const at of byPath.values()) {
|
|
379
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
339
380
|
}
|
|
340
381
|
if (manifest.mode === "types") {
|
|
382
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
341
383
|
process.exit(0);
|
|
342
384
|
}
|
|
343
|
-
await typecheck(
|
|
385
|
+
await typecheck([...byPath.values()]);
|
|
344
386
|
const code = [];
|
|
345
|
-
for (const
|
|
346
|
-
code.push(await bundle(
|
|
387
|
+
for (const at of located) {
|
|
388
|
+
code.push(await bundle(at));
|
|
347
389
|
}
|
|
348
390
|
process.stdout.write(JSON.stringify({ code }));
|
package/dist/worker.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
//
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
import { readFileSync } from "node:fs";
|
|
8
|
-
import { evaluate } from "./eval.js";
|
|
8
|
+
import { Cancelled, evaluate } from "./eval.js";
|
|
9
9
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
10
10
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
11
11
|
// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
|
|
@@ -185,7 +185,7 @@ async function answer(token, outcome) {
|
|
|
185
185
|
console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
|
|
186
186
|
await release(token);
|
|
187
187
|
}
|
|
188
|
-
async function run(job) {
|
|
188
|
+
async function run(job, signal) {
|
|
189
189
|
let resolved;
|
|
190
190
|
try {
|
|
191
191
|
resolved = await resolveObjects(job);
|
|
@@ -206,9 +206,17 @@ async function run(job) {
|
|
|
206
206
|
}
|
|
207
207
|
let result;
|
|
208
208
|
try {
|
|
209
|
-
result = await evaluate(req);
|
|
209
|
+
result = await evaluate(req, signal);
|
|
210
210
|
}
|
|
211
211
|
catch (err) {
|
|
212
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
213
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
214
|
+
// waiting out a lease nobody is serving.
|
|
215
|
+
if (err instanceof Cancelled) {
|
|
216
|
+
console.log(`cancelled: ${job.token}`);
|
|
217
|
+
await release(job.token);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
212
220
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
213
221
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
214
222
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -245,11 +253,22 @@ async function renewLoop() {
|
|
|
245
253
|
tokens,
|
|
246
254
|
lease_ms: LEASE_MS,
|
|
247
255
|
});
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
256
|
+
if (!ok)
|
|
257
|
+
continue;
|
|
258
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
259
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
260
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
261
|
+
// own catch, so nothing is released twice.
|
|
262
|
+
for (const token of (data?.cancelled ?? [])) {
|
|
263
|
+
inFlight.get(token)?.abort.abort();
|
|
264
|
+
}
|
|
265
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
266
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
267
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
268
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
269
|
+
const lost = (data?.lost ?? []);
|
|
270
|
+
if (lost.length) {
|
|
271
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
}
|
|
@@ -258,8 +277,9 @@ async function pollLoop() {
|
|
|
258
277
|
const free = CONCURRENCY - inFlight.size;
|
|
259
278
|
const jobs = free > 0 ? await claim(free) : [];
|
|
260
279
|
for (const job of jobs) {
|
|
261
|
-
|
|
262
|
-
|
|
280
|
+
const abort = new AbortController();
|
|
281
|
+
inFlight.set(job.token, { job, abort });
|
|
282
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
263
283
|
}
|
|
264
284
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
265
285
|
// realms allow, not at the poll interval.
|
package/eval.ts
CHANGED
|
@@ -57,14 +57,32 @@ class RealmFault extends Error {
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
61
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
62
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
63
|
+
export class Cancelled extends Error {
|
|
64
|
+
constructor() {
|
|
65
|
+
super("cancelled");
|
|
66
|
+
this.name = "Cancelled";
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
|
|
61
71
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
62
72
|
|
|
63
73
|
const worker = new Worker(REALM_URL);
|
|
64
74
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
75
|
+
let onAbort: (() => void) | undefined;
|
|
65
76
|
try {
|
|
66
77
|
return await new Promise<EvalResult>((resolve, reject) => {
|
|
67
78
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
79
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
80
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
81
|
+
if (signal) {
|
|
82
|
+
if (signal.aborted) reject(new Cancelled());
|
|
83
|
+
onAbort = () => reject(new Cancelled());
|
|
84
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
85
|
+
}
|
|
68
86
|
worker.once("message", (reply: WorkerReply) => resolve(reply));
|
|
69
87
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
70
88
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -75,6 +93,7 @@ export async function evaluate(req: EvalRequest): Promise<EvalResult> {
|
|
|
75
93
|
});
|
|
76
94
|
} finally {
|
|
77
95
|
clearTimeout(timer);
|
|
96
|
+
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
78
97
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
79
98
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
80
99
|
await worker.terminate();
|
package/import.ts
CHANGED
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
// over the same project.
|
|
9
9
|
|
|
10
10
|
import { spawn } from "node:child_process";
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
11
12
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
12
13
|
import { builtinModules } from "node:module";
|
|
13
|
-
import { dirname, join, relative } from "node:path";
|
|
14
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
14
15
|
import { fileURLToPath } from "node:url";
|
|
15
16
|
|
|
16
17
|
import commonjs from "@rollup/plugin-commonjs";
|
|
@@ -22,20 +23,37 @@ import ts from "typescript";
|
|
|
22
23
|
type Schema = Record<string, any>;
|
|
23
24
|
|
|
24
25
|
type Site = {
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
// Which namespace the directive sits in: `process`, `task` or `action`.
|
|
27
|
+
level: "process" | "task" | "action";
|
|
27
28
|
task?: string;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
// What the site IS, at action level: the action's type, and the process a child call names.
|
|
30
|
+
action?: string;
|
|
31
|
+
child?: string;
|
|
32
|
+
// The slot as a path of keys and indices, so nothing has to unescape a JSON Pointer.
|
|
33
|
+
pointer: (string | number)[];
|
|
34
|
+
// Everything after `$<resolver>:`, verbatim. genctl does not read it as a path — that is
|
|
35
|
+
// this resolver's reading of it.
|
|
36
|
+
argument: string;
|
|
37
|
+
// What .genroc asked genctl to type, keyed by the name it chose. This resolver reads Input
|
|
38
|
+
// and Output; which ADDRESS each came from is the config's business, not ours.
|
|
39
|
+
types?: Record<string, Schema>;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
type ManifestProcess = {
|
|
43
|
+
name: string;
|
|
44
|
+
// The definition's own location, split because a relative argument is relative to the
|
|
45
|
+
// DIRECTORY — joining is ours to do, and this is the base.
|
|
46
|
+
dir: string;
|
|
47
|
+
file: string;
|
|
48
|
+
sites: Site[];
|
|
49
|
+
// Only what this process's fragments reach — a `$ref` in `types` points here.
|
|
50
|
+
$defs?: Record<string, Schema>;
|
|
32
51
|
};
|
|
33
52
|
|
|
34
53
|
type Manifest = {
|
|
35
54
|
mode: "types" | "build";
|
|
36
55
|
root: string;
|
|
37
|
-
|
|
38
|
-
sites: Site[];
|
|
56
|
+
processes: ManifestProcess[];
|
|
39
57
|
};
|
|
40
58
|
|
|
41
59
|
function die(message: string): never {
|
|
@@ -72,13 +90,16 @@ function tsType(s: Schema | undefined, used: Set<string>): string {
|
|
|
72
90
|
if (Array.isArray(s.enum)) {
|
|
73
91
|
return s.enum.map((v: unknown) => JSON.stringify(v)).join(" | ") || "never";
|
|
74
92
|
}
|
|
75
|
-
if (Array.isArray(s.anyOf))
|
|
76
|
-
|
|
93
|
+
if (Array.isArray(s.anyOf))
|
|
94
|
+
return union(s.anyOf.map((a: Schema) => tsType(a, used)));
|
|
95
|
+
if (Array.isArray(s.oneOf))
|
|
96
|
+
return union(s.oneOf.map((a: Schema) => tsType(a, used)));
|
|
77
97
|
if (Array.isArray(s.allOf)) {
|
|
78
98
|
return s.allOf.map((a: Schema) => tsType(a, used)).join(" & ") || "unknown";
|
|
79
99
|
}
|
|
80
100
|
|
|
81
|
-
const types: string[] =
|
|
101
|
+
const types: string[] =
|
|
102
|
+
s.type === undefined ? [] : Array.isArray(s.type) ? s.type : [s.type];
|
|
82
103
|
if (types.length === 0) {
|
|
83
104
|
// The top type: `{}` means unknown, not "an empty object". specs/unknown-type.md.
|
|
84
105
|
return s.properties ? objectType(s, used) : "unknown";
|
|
@@ -111,8 +132,13 @@ function objectType(s: Schema, used: Set<string>): string {
|
|
|
111
132
|
const required = new Set<string>(s.required ?? []);
|
|
112
133
|
const lines: string[] = [];
|
|
113
134
|
for (const [key, sub] of Object.entries(props)) {
|
|
114
|
-
const doc =
|
|
115
|
-
|
|
135
|
+
const doc =
|
|
136
|
+
typeof sub.description === "string"
|
|
137
|
+
? ` /** ${sub.description} */\n`
|
|
138
|
+
: "";
|
|
139
|
+
lines.push(
|
|
140
|
+
`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`,
|
|
141
|
+
);
|
|
116
142
|
}
|
|
117
143
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
118
144
|
lines.push(` [key: string]: ${tsType(s.additionalProperties, used)};`);
|
|
@@ -128,32 +154,17 @@ function union(parts: string[]): string {
|
|
|
128
154
|
|
|
129
155
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
130
156
|
const propKey = (k: string) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
131
|
-
const identifier = (n: string) =>
|
|
132
|
-
|
|
133
|
-
function deref(s: Schema | undefined, defs: Record<string, Schema>): Schema | undefined {
|
|
134
|
-
let cur = s;
|
|
135
|
-
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
136
|
-
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
137
|
-
}
|
|
138
|
-
return cur;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
142
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
143
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
144
|
-
* the navigation belongs here rather than in genctl. */
|
|
145
|
-
function scriptInput(site: Site, defs: Record<string, Schema>): Schema | undefined {
|
|
146
|
-
const action = deref(site.input, defs);
|
|
147
|
-
const bound = action?.properties?.input;
|
|
148
|
-
return bound ?? site.input;
|
|
149
|
-
}
|
|
157
|
+
const identifier = (n: string) =>
|
|
158
|
+
IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
150
159
|
|
|
151
160
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
152
161
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
153
|
-
function declarations(
|
|
162
|
+
function declarations(at: Located): string {
|
|
163
|
+
const { site, where } = at;
|
|
164
|
+
const defs = where.$defs ?? {};
|
|
154
165
|
const used = new Set<string>();
|
|
155
|
-
const input = tsType(
|
|
156
|
-
const output = tsType(site.
|
|
166
|
+
const input = tsType(site.types?.Input, used);
|
|
167
|
+
const output = tsType(site.types?.Output, used);
|
|
157
168
|
|
|
158
169
|
const emitted: string[] = [];
|
|
159
170
|
const done = new Set<string>();
|
|
@@ -167,7 +178,7 @@ function declarations(site: Site, defs: Record<string, Schema>): string {
|
|
|
167
178
|
|
|
168
179
|
return [
|
|
169
180
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
170
|
-
`// ${
|
|
181
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
171
182
|
"",
|
|
172
183
|
...emitted,
|
|
173
184
|
emitted.length ? "" : "",
|
|
@@ -189,7 +200,10 @@ function typesPathFor(scriptPath: string): string {
|
|
|
189
200
|
/** The nearest tsconfig above the script — the one the author's editor already reads. Two
|
|
190
201
|
* different configs mean a red editor over a clean apply, or the reverse. The walk stops at
|
|
191
202
|
* the project root: above it is not this project. */
|
|
192
|
-
async function nearestTsconfig(
|
|
203
|
+
async function nearestTsconfig(
|
|
204
|
+
from: string,
|
|
205
|
+
root: string,
|
|
206
|
+
): Promise<string | null> {
|
|
193
207
|
for (let dir = from; ; dir = dirname(dir)) {
|
|
194
208
|
const candidate = join(dir, "tsconfig.json");
|
|
195
209
|
if (await exists(candidate)) return candidate;
|
|
@@ -197,7 +211,7 @@ async function nearestTsconfig(from: string, root: string): Promise<string | nul
|
|
|
197
211
|
}
|
|
198
212
|
}
|
|
199
213
|
|
|
200
|
-
async function typecheck(
|
|
214
|
+
async function typecheck(sites: Located[]): Promise<void> {
|
|
201
215
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
202
216
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
203
217
|
const dir = join(root, ".genroc-cache");
|
|
@@ -205,12 +219,12 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
205
219
|
|
|
206
220
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
207
221
|
// check each script under the other author's options.
|
|
208
|
-
const groups = new Map<string,
|
|
209
|
-
for (const
|
|
210
|
-
const base = (await nearestTsconfig(dirname(
|
|
222
|
+
const groups = new Map<string, Located[]>();
|
|
223
|
+
for (const at of sites) {
|
|
224
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
211
225
|
const group = groups.get(base);
|
|
212
|
-
if (group) group.push(
|
|
213
|
-
else groups.set(base, [
|
|
226
|
+
if (group) group.push(at);
|
|
227
|
+
else groups.set(base, [at]);
|
|
214
228
|
}
|
|
215
229
|
|
|
216
230
|
let n = 0;
|
|
@@ -232,12 +246,18 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
232
246
|
// base config there is nothing to opt in with, so the default stays none.
|
|
233
247
|
...(base ? {} : { types: [] }),
|
|
234
248
|
},
|
|
235
|
-
files: group.flatMap((s) => [
|
|
249
|
+
files: group.flatMap((s) => [
|
|
250
|
+
relative(dir, s.file),
|
|
251
|
+
relative(dir, typesPathFor(s.file)),
|
|
252
|
+
]),
|
|
236
253
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
237
254
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
238
255
|
include: [],
|
|
239
256
|
};
|
|
240
|
-
const configPath = join(
|
|
257
|
+
const configPath = join(
|
|
258
|
+
dir,
|
|
259
|
+
groups.size === 1 ? "tsconfig.json" : `tsconfig.${n++}.json`,
|
|
260
|
+
);
|
|
241
261
|
await write(configPath, JSON.stringify(config, null, 2));
|
|
242
262
|
await runTsc(root, configPath);
|
|
243
263
|
}
|
|
@@ -285,7 +305,10 @@ const transpile: Plugin = {
|
|
|
285
305
|
},
|
|
286
306
|
};
|
|
287
307
|
|
|
288
|
-
const BUILTIN = new Set([
|
|
308
|
+
const BUILTIN = new Set([
|
|
309
|
+
...builtinModules,
|
|
310
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
311
|
+
]);
|
|
289
312
|
|
|
290
313
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
291
314
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
@@ -296,15 +319,27 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
296
319
|
let options: ts.CompilerOptions = {};
|
|
297
320
|
if (configPath) {
|
|
298
321
|
const read = ts.readConfigFile(configPath, ts.sys.readFile);
|
|
299
|
-
options = ts.parseJsonConfigFileContent(
|
|
322
|
+
options = ts.parseJsonConfigFileContent(
|
|
323
|
+
read.config ?? {},
|
|
324
|
+
ts.sys,
|
|
325
|
+
dirname(configPath),
|
|
326
|
+
).options;
|
|
300
327
|
}
|
|
301
328
|
return {
|
|
302
329
|
name: "genroc-ts-resolve",
|
|
303
330
|
resolveId(source, importer) {
|
|
304
331
|
if (!importer || BUILTIN.has(source)) return null;
|
|
305
|
-
const { resolvedModule } = ts.resolveModuleName(
|
|
306
|
-
|
|
307
|
-
|
|
332
|
+
const { resolvedModule } = ts.resolveModuleName(
|
|
333
|
+
source,
|
|
334
|
+
importer,
|
|
335
|
+
options,
|
|
336
|
+
ts.sys,
|
|
337
|
+
);
|
|
338
|
+
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
339
|
+
return null;
|
|
340
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
341
|
+
? null
|
|
342
|
+
: resolvedModule.resolvedFileName;
|
|
308
343
|
},
|
|
309
344
|
};
|
|
310
345
|
}
|
|
@@ -312,15 +347,16 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
312
347
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
313
348
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
314
349
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
315
|
-
async function bundle(
|
|
350
|
+
async function bundle(at: Located): Promise<string> {
|
|
351
|
+
const site = at.site;
|
|
316
352
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
317
353
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
318
354
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
319
355
|
const built = await rollup({
|
|
320
|
-
input:
|
|
356
|
+
input: at.file,
|
|
321
357
|
external: (id) => BUILTIN.has(id),
|
|
322
358
|
plugins: [
|
|
323
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
359
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
324
360
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
325
361
|
commonjs(),
|
|
326
362
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -330,17 +366,24 @@ async function bundle(site: Site, root: string): Promise<string> {
|
|
|
330
366
|
],
|
|
331
367
|
onwarn(warning) {
|
|
332
368
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
333
|
-
die(
|
|
369
|
+
die(
|
|
370
|
+
`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`,
|
|
371
|
+
);
|
|
334
372
|
}
|
|
335
373
|
},
|
|
336
|
-
}).catch((e: unknown) =>
|
|
374
|
+
}).catch((e: unknown) =>
|
|
375
|
+
die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`),
|
|
376
|
+
);
|
|
337
377
|
|
|
338
|
-
const { output } = await built.generate({
|
|
378
|
+
const { output } = await built.generate({
|
|
379
|
+
format: "es",
|
|
380
|
+
inlineDynamicImports: true,
|
|
381
|
+
});
|
|
339
382
|
await built.close();
|
|
340
383
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
341
384
|
// instance, and the file it names is on this machine.
|
|
342
385
|
if (!output[0].exports.includes("default")) {
|
|
343
|
-
die(`${
|
|
386
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
344
387
|
}
|
|
345
388
|
return output[0].code;
|
|
346
389
|
}
|
|
@@ -355,40 +398,99 @@ const stdin: string = await new Promise((resolve, reject) => {
|
|
|
355
398
|
process.stdin.on("error", reject);
|
|
356
399
|
});
|
|
357
400
|
const manifest = JSON.parse(stdin) as Manifest;
|
|
358
|
-
|
|
401
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
402
|
+
const root = process.cwd();
|
|
403
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
404
|
+
die("stdin is not a genroc resolver manifest");
|
|
405
|
+
|
|
406
|
+
/** Every site with the process it sits in and the file its argument names, in the manifest's own
|
|
407
|
+
* order — which is the order `code` must answer in. genctl passes the argument verbatim, so
|
|
408
|
+
* joining it to the definition's directory is this resolver's business: only it knows the
|
|
409
|
+
* argument is a path at all. */
|
|
410
|
+
type Located = { site: Site; where: ManifestProcess; file: string };
|
|
411
|
+
|
|
412
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
413
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
414
|
+
function address(pointer: (string | number)[]): string {
|
|
415
|
+
return pointer
|
|
416
|
+
.map((seg) =>
|
|
417
|
+
typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
418
|
+
? `.${seg}`
|
|
419
|
+
: `[${JSON.stringify(seg)}]`,
|
|
420
|
+
)
|
|
421
|
+
.join("")
|
|
422
|
+
.replace(/^\./, "");
|
|
423
|
+
}
|
|
424
|
+
const located: Located[] = manifest.processes.flatMap((where) =>
|
|
425
|
+
where.sites.map((site) => ({
|
|
426
|
+
site,
|
|
427
|
+
where,
|
|
428
|
+
file: resolve(where.dir, site.argument),
|
|
429
|
+
})),
|
|
430
|
+
);
|
|
431
|
+
|
|
432
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
433
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
434
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
435
|
+
// and `external` is the same request made directly.
|
|
436
|
+
for (const at of located) {
|
|
437
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
438
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
439
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
440
|
+
// directly.
|
|
441
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
442
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
443
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
444
|
+
die(
|
|
445
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
446
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
447
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`,
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
453
|
+
// does is the one that must say when it is not there.
|
|
454
|
+
for (const at of located) {
|
|
455
|
+
if (!existsSync(at.file)) {
|
|
456
|
+
die(
|
|
457
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
458
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`,
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
359
462
|
|
|
360
463
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
361
464
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
362
|
-
const byPath = new Map<string,
|
|
363
|
-
for (const
|
|
364
|
-
const seen = byPath.get(
|
|
365
|
-
const defsOf = (x: Site) => (manifest.schemas[x.process]?.$defs ?? {}) as Record<string, Schema>;
|
|
465
|
+
const byPath = new Map<string, Located>();
|
|
466
|
+
for (const at of located) {
|
|
467
|
+
const seen = byPath.get(at.file);
|
|
366
468
|
if (
|
|
367
469
|
seen &&
|
|
368
|
-
JSON.stringify(
|
|
369
|
-
JSON.stringify(scriptInput(site, defsOf(site)))
|
|
470
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)
|
|
370
471
|
) {
|
|
371
472
|
die(
|
|
372
|
-
`${
|
|
473
|
+
`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
474
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
373
475
|
"Split it into two scripts, or make the two call sites pass the same shape.",
|
|
374
476
|
);
|
|
375
477
|
}
|
|
376
|
-
byPath.set(
|
|
478
|
+
byPath.set(at.file, at);
|
|
377
479
|
}
|
|
378
480
|
|
|
379
|
-
for (const
|
|
380
|
-
|
|
381
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
481
|
+
for (const at of byPath.values()) {
|
|
482
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
382
483
|
}
|
|
383
484
|
|
|
384
485
|
if (manifest.mode === "types") {
|
|
486
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
385
487
|
process.exit(0);
|
|
386
488
|
}
|
|
387
489
|
|
|
388
|
-
await typecheck(
|
|
490
|
+
await typecheck([...byPath.values()]);
|
|
389
491
|
|
|
390
492
|
const code: string[] = [];
|
|
391
|
-
for (const
|
|
392
|
-
code.push(await bundle(
|
|
493
|
+
for (const at of located) {
|
|
494
|
+
code.push(await bundle(at));
|
|
393
495
|
}
|
|
394
496
|
process.stdout.write(JSON.stringify({ code }));
|
package/package.json
CHANGED
package/worker.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
|
|
8
8
|
import { readFileSync } from "node:fs";
|
|
9
|
-
import { evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
9
|
+
import { Cancelled, evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
10
10
|
|
|
11
11
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
12
12
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
@@ -135,7 +135,12 @@ function asEvalRequest(input: unknown): EvalRequest | string {
|
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
|
|
138
|
+
/** A claim this worker is currently serving. The controller is how the renewal loop reaches
|
|
139
|
+
* the evaluation: genroc cannot call us, so a cancellation arrives as an answer to our own
|
|
140
|
+
* heartbeat and has to be delivered inward from there. */
|
|
141
|
+
type Running = { job: QueueTask; abort: AbortController };
|
|
142
|
+
|
|
143
|
+
const inFlight = new Map<string, Running>();
|
|
139
144
|
let running = true;
|
|
140
145
|
|
|
141
146
|
// Whether the last claim reached genroc. A worker polls several times a second, so an
|
|
@@ -203,7 +208,7 @@ async function answer(token: string, outcome: Record<string, unknown>): Promise<
|
|
|
203
208
|
await release(token);
|
|
204
209
|
}
|
|
205
210
|
|
|
206
|
-
async function run(job: QueueTask): Promise<void> {
|
|
211
|
+
async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
207
212
|
let resolved: unknown;
|
|
208
213
|
try {
|
|
209
214
|
resolved = await resolveObjects(job);
|
|
@@ -224,8 +229,16 @@ async function run(job: QueueTask): Promise<void> {
|
|
|
224
229
|
|
|
225
230
|
let result;
|
|
226
231
|
try {
|
|
227
|
-
result = await evaluate(req);
|
|
232
|
+
result = await evaluate(req, signal);
|
|
228
233
|
} catch (err) {
|
|
234
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
235
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
236
|
+
// waiting out a lease nobody is serving.
|
|
237
|
+
if (err instanceof Cancelled) {
|
|
238
|
+
console.log(`cancelled: ${job.token}`);
|
|
239
|
+
await release(job.token);
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
229
242
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
230
243
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
231
244
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -263,11 +276,22 @@ async function renewLoop(): Promise<void> {
|
|
|
263
276
|
tokens,
|
|
264
277
|
lease_ms: LEASE_MS,
|
|
265
278
|
});
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
279
|
+
if (!ok) continue;
|
|
280
|
+
|
|
281
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
282
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
283
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
284
|
+
// own catch, so nothing is released twice.
|
|
285
|
+
for (const token of (data?.cancelled ?? []) as string[]) {
|
|
286
|
+
inFlight.get(token)?.abort.abort();
|
|
287
|
+
}
|
|
288
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
289
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
290
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
291
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
292
|
+
const lost = (data?.lost ?? []) as string[];
|
|
293
|
+
if (lost.length) {
|
|
294
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
271
295
|
}
|
|
272
296
|
}
|
|
273
297
|
}
|
|
@@ -277,8 +301,9 @@ async function pollLoop(): Promise<void> {
|
|
|
277
301
|
const free = CONCURRENCY - inFlight.size;
|
|
278
302
|
const jobs = free > 0 ? await claim(free) : [];
|
|
279
303
|
for (const job of jobs) {
|
|
280
|
-
|
|
281
|
-
|
|
304
|
+
const abort = new AbortController();
|
|
305
|
+
inFlight.set(job.token, { job, abort });
|
|
306
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
282
307
|
}
|
|
283
308
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
284
309
|
// realms allow, not at the poll interval.
|