@genroc/eval-node 0.0.0-edge.3efef64 → 0.0.0-edge.3fd9037f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +18 -1
- package/dist/eval.js +21 -1
- package/dist/import.js +105 -89
- package/dist/worker.js +33 -13
- package/eval.ts +20 -1
- package/import.ts +159 -115
- package/package.json +1 -1
- package/worker.ts +41 -16
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ resolver's exit code, so a stored definition cannot hold code that failed to typ
|
|
|
26
26
|
|
|
27
27
|
## Building
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
pnpm run build # tsc -p tsconfig.build.json -> dist/
|
|
30
30
|
|
|
31
31
|
The published package is JavaScript, not TypeScript. **Node refuses to strip types for files
|
|
32
32
|
under `node_modules`**, so shipping `.ts` works from a checkout and fails for every consumer —
|
|
@@ -90,6 +90,23 @@ queue worker rather than the HTTP sidecar it used to be:
|
|
|
90
90
|
that genroc had to be able to reach. A worker only needs outbound access, so it can live
|
|
91
91
|
anywhere — behind NAT, in another trust zone.
|
|
92
92
|
|
|
93
|
+
The cost of that inversion is that genroc cannot reach a running worker, so anything it needs
|
|
94
|
+
to say has to ride the renewal the worker already makes. **Renewing is therefore mandatory,
|
|
95
|
+
not an optimisation** — a worker that stops renewing is indistinguishable from one that died.
|
|
96
|
+
The response answers per token:
|
|
97
|
+
|
|
98
|
+
| list | what it means | what this worker does |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `renewed` | still yours | carry on |
|
|
101
|
+
| `lost` | already someone else's | stop; do **not** release — that would bump the new holder's claim |
|
|
102
|
+
| `cancelled` | still yours, nobody wants it | abort the script and release the claim |
|
|
103
|
+
|
|
104
|
+
`cancelled` is how an operator's `genctl cancel` reaches work already running: the evaluation
|
|
105
|
+
is aborted through an `AbortSignal`, its realm is terminated, and no outcome is submitted —
|
|
106
|
+
the process is stopping, so there is nothing left to answer. `renew_before_ms` on the claim
|
|
107
|
+
says how long the worker may wait before renewing again, so the interval is a value it reads
|
|
108
|
+
rather than one it guesses.
|
|
109
|
+
|
|
93
110
|
## The task input
|
|
94
111
|
|
|
95
112
|
The `input` of the external task IS the evaluation request:
|
package/dist/eval.js
CHANGED
|
@@ -20,13 +20,31 @@ class RealmFault extends Error {
|
|
|
20
20
|
this.name = "RealmFault";
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
24
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
25
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
26
|
+
export class Cancelled extends Error {
|
|
27
|
+
constructor() {
|
|
28
|
+
super("cancelled");
|
|
29
|
+
this.name = "Cancelled";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export async function evaluate(req, signal) {
|
|
24
33
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
25
34
|
const worker = new Worker(REALM_URL);
|
|
26
35
|
let timer;
|
|
36
|
+
let onAbort;
|
|
27
37
|
try {
|
|
28
38
|
return await new Promise((resolve, reject) => {
|
|
29
39
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
40
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
41
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
42
|
+
if (signal) {
|
|
43
|
+
if (signal.aborted)
|
|
44
|
+
reject(new Cancelled());
|
|
45
|
+
onAbort = () => reject(new Cancelled());
|
|
46
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
47
|
+
}
|
|
30
48
|
worker.once("message", (reply) => resolve(reply));
|
|
31
49
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
32
50
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -38,6 +56,8 @@ export async function evaluate(req) {
|
|
|
38
56
|
}
|
|
39
57
|
finally {
|
|
40
58
|
clearTimeout(timer);
|
|
59
|
+
if (signal && onAbort)
|
|
60
|
+
signal.removeEventListener("abort", onAbort);
|
|
41
61
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
42
62
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
43
63
|
await worker.terminate();
|
package/dist/import.js
CHANGED
|
@@ -6,11 +6,10 @@
|
|
|
6
6
|
// Two modes, one binary: "types" writes the declarations an editor needs and returns no
|
|
7
7
|
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
8
|
// over the same project.
|
|
9
|
-
import {
|
|
10
|
-
import { access,
|
|
9
|
+
import { existsSync } from "node:fs";
|
|
10
|
+
import { access, writeFile } from "node:fs/promises";
|
|
11
11
|
import { builtinModules } from "node:module";
|
|
12
|
-
import { dirname, join,
|
|
13
|
-
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { dirname, join, resolve } from "node:path";
|
|
14
13
|
import commonjs from "@rollup/plugin-commonjs";
|
|
15
14
|
import json from "@rollup/plugin-json";
|
|
16
15
|
import { nodeResolve } from "@rollup/plugin-node-resolve";
|
|
@@ -29,11 +28,6 @@ async function exists(path) {
|
|
|
29
28
|
return false;
|
|
30
29
|
}
|
|
31
30
|
}
|
|
32
|
-
/** Creates the parent directory, which `.genroc-cache/` relies on: nothing else makes it. */
|
|
33
|
-
async function write(path, content) {
|
|
34
|
-
await mkdir(dirname(path), { recursive: true });
|
|
35
|
-
await writeFile(path, content);
|
|
36
|
-
}
|
|
37
31
|
// ── JSON Schema → TypeScript ───────────────────────────────────────────────────
|
|
38
32
|
/** Only genroc's keyword set is handled; anything else its strict decoder would have
|
|
39
33
|
* refused before this ran (internal/schema, allowedKeywords). */
|
|
@@ -89,15 +83,17 @@ function objectType(s, used) {
|
|
|
89
83
|
const doc = typeof sub.description === "string"
|
|
90
84
|
? ` /** ${sub.description} */\n`
|
|
91
85
|
: "";
|
|
92
|
-
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`);
|
|
86
|
+
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${nest(tsType(sub, used))};`);
|
|
93
87
|
}
|
|
94
88
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
95
|
-
lines.push(` [key: string]: ${tsType(s.additionalProperties, used)};`);
|
|
89
|
+
lines.push(` [key: string]: ${nest(tsType(s.additionalProperties, used))};`);
|
|
96
90
|
}
|
|
97
91
|
if (lines.length === 0)
|
|
98
92
|
return "Record<string, unknown>";
|
|
99
93
|
return `{\n${lines.join("\n")}\n}`;
|
|
100
94
|
}
|
|
95
|
+
/** Every line but the first, which is already placed by the property that opens it. */
|
|
96
|
+
const nest = (t) => t.replace(/\n/g, "\n ");
|
|
101
97
|
function union(parts) {
|
|
102
98
|
const seen = [...new Set(parts)];
|
|
103
99
|
return seen.length === 0 ? "unknown" : seen.join(" | ");
|
|
@@ -105,28 +101,14 @@ function union(parts) {
|
|
|
105
101
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
106
102
|
const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
107
103
|
const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
108
|
-
function deref(s, defs) {
|
|
109
|
-
let cur = s;
|
|
110
|
-
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
111
|
-
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
112
|
-
}
|
|
113
|
-
return cur;
|
|
114
|
-
}
|
|
115
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
116
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
117
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
118
|
-
* the navigation belongs here rather than in genctl. */
|
|
119
|
-
function scriptInput(site, defs) {
|
|
120
|
-
const action = deref(site.input, defs);
|
|
121
|
-
const bound = action?.properties?.input;
|
|
122
|
-
return bound ?? site.input;
|
|
123
|
-
}
|
|
124
104
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
125
105
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
126
|
-
function declarations(
|
|
106
|
+
function declarations(at) {
|
|
107
|
+
const { site, where } = at;
|
|
108
|
+
const defs = where.$defs ?? {};
|
|
127
109
|
const used = new Set();
|
|
128
|
-
const input = tsType(
|
|
129
|
-
const output = tsType(site.
|
|
110
|
+
const input = tsType(site.types?.Input, used);
|
|
111
|
+
const output = tsType(site.types?.Output, used);
|
|
130
112
|
const emitted = [];
|
|
131
113
|
const done = new Set();
|
|
132
114
|
while (true) {
|
|
@@ -139,7 +121,7 @@ function declarations(site, defs) {
|
|
|
139
121
|
}
|
|
140
122
|
return [
|
|
141
123
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
142
|
-
`// ${
|
|
124
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
143
125
|
"",
|
|
144
126
|
...emitted,
|
|
145
127
|
emitted.length ? "" : "",
|
|
@@ -167,26 +149,22 @@ async function nearestTsconfig(from, root) {
|
|
|
167
149
|
return null;
|
|
168
150
|
}
|
|
169
151
|
}
|
|
170
|
-
async function typecheck(
|
|
171
|
-
//
|
|
172
|
-
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
173
|
-
const dir = join(root, ".genroc-cache");
|
|
174
|
-
await write(join(dir, ".gitignore"), "*\n");
|
|
175
|
-
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
152
|
+
async function typecheck(sites) {
|
|
153
|
+
// One program per distinct base config: `extends` takes a single base, so merging two would
|
|
176
154
|
// check each script under the other author's options.
|
|
177
155
|
const groups = new Map();
|
|
178
|
-
for (const
|
|
179
|
-
const base = (await nearestTsconfig(dirname(
|
|
156
|
+
for (const at of sites) {
|
|
157
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
180
158
|
const group = groups.get(base);
|
|
181
159
|
if (group)
|
|
182
|
-
group.push(
|
|
160
|
+
group.push(at);
|
|
183
161
|
else
|
|
184
|
-
groups.set(base, [
|
|
162
|
+
groups.set(base, [at]);
|
|
185
163
|
}
|
|
186
|
-
|
|
164
|
+
const diagnostics = [];
|
|
187
165
|
for (const [base, group] of groups) {
|
|
188
166
|
const config = {
|
|
189
|
-
...(base ? { extends:
|
|
167
|
+
...(base ? { extends: base } : {}),
|
|
190
168
|
compilerOptions: {
|
|
191
169
|
noEmit: true,
|
|
192
170
|
strict: true,
|
|
@@ -202,37 +180,33 @@ async function typecheck(root, sites) {
|
|
|
202
180
|
// base config there is nothing to opt in with, so the default stays none.
|
|
203
181
|
...(base ? {} : { types: [] }),
|
|
204
182
|
},
|
|
205
|
-
files: group.flatMap((s) => [
|
|
206
|
-
relative(dir, s.path),
|
|
207
|
-
relative(dir, typesPathFor(s.path)),
|
|
208
|
-
]),
|
|
183
|
+
files: group.flatMap((s) => [s.file, typesPathFor(s.file)]),
|
|
209
184
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
210
185
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
211
186
|
include: [],
|
|
212
187
|
};
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
188
|
+
// Never written: the name only anchors `@types` lookup at the root. It must not be
|
|
189
|
+
// `tsconfig.json`: that is the base the config `extends` whenever the author keeps one at
|
|
190
|
+
// the root, and tsc refuses a config that extends its own path as circular.
|
|
191
|
+
const parsed = ts.parseJsonConfigFileContent(config, ts.sys, root, undefined, join(root, "tsconfig.genroc.json"));
|
|
192
|
+
const program = ts.createProgram({
|
|
193
|
+
rootNames: parsed.fileNames,
|
|
194
|
+
options: parsed.options,
|
|
195
|
+
projectReferences: parsed.projectReferences,
|
|
196
|
+
configFileParsingDiagnostics: parsed.errors,
|
|
197
|
+
});
|
|
198
|
+
diagnostics.push(...parsed.errors, ...ts.getPreEmitDiagnostics(program));
|
|
216
199
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
proc.stderr.on("data", (c) => (err += c));
|
|
228
|
-
const code = await new Promise((resolve, reject) => {
|
|
229
|
-
proc.on("error", reject);
|
|
230
|
-
proc.on("close", (c) => resolve(c ?? 1));
|
|
231
|
-
});
|
|
232
|
-
if (code !== 0) {
|
|
233
|
-
// tsc reports on stdout; the exit code IS the type check, so this is the diagnostic
|
|
234
|
-
// genctl surfaces and the reason a failed import never produces a string.
|
|
235
|
-
die([out, err].filter(Boolean).join("\n").trimEnd());
|
|
200
|
+
if (diagnostics.length > 0) {
|
|
201
|
+
// The diagnostics are the whole type check: this is what genctl surfaces, and the reason a
|
|
202
|
+
// failed import never produces a string.
|
|
203
|
+
die(ts
|
|
204
|
+
.formatDiagnostics(diagnostics, {
|
|
205
|
+
getCanonicalFileName: (f) => f,
|
|
206
|
+
getCurrentDirectory: () => root,
|
|
207
|
+
getNewLine: () => "\n",
|
|
208
|
+
})
|
|
209
|
+
.trimEnd());
|
|
236
210
|
}
|
|
237
211
|
}
|
|
238
212
|
// ── bundle ─────────────────────────────────────────────────────────────────────
|
|
@@ -287,15 +261,16 @@ function tsResolve(configPath) {
|
|
|
287
261
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
288
262
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
289
263
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
290
|
-
async function bundle(
|
|
264
|
+
async function bundle(at) {
|
|
265
|
+
const site = at.site;
|
|
291
266
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
292
267
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
293
268
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
294
269
|
const built = await rollup({
|
|
295
|
-
input:
|
|
270
|
+
input: at.file,
|
|
296
271
|
external: (id) => BUILTIN.has(id),
|
|
297
272
|
plugins: [
|
|
298
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
273
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
299
274
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
300
275
|
commonjs(),
|
|
301
276
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -305,10 +280,10 @@ async function bundle(site, root) {
|
|
|
305
280
|
],
|
|
306
281
|
onwarn(warning) {
|
|
307
282
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
308
|
-
die(`${
|
|
283
|
+
die(`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`);
|
|
309
284
|
}
|
|
310
285
|
},
|
|
311
|
-
}).catch((e) => die(`${
|
|
286
|
+
}).catch((e) => die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`));
|
|
312
287
|
const { output } = await built.generate({
|
|
313
288
|
format: "es",
|
|
314
289
|
inlineDynamicImports: true,
|
|
@@ -317,7 +292,7 @@ async function bundle(site, root) {
|
|
|
317
292
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
318
293
|
// instance, and the file it names is on this machine.
|
|
319
294
|
if (!output[0].exports.includes("default")) {
|
|
320
|
-
die(`${
|
|
295
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
321
296
|
}
|
|
322
297
|
return output[0].code;
|
|
323
298
|
}
|
|
@@ -330,32 +305,73 @@ const stdin = await new Promise((resolve, reject) => {
|
|
|
330
305
|
process.stdin.on("error", reject);
|
|
331
306
|
});
|
|
332
307
|
const manifest = JSON.parse(stdin);
|
|
333
|
-
|
|
308
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
309
|
+
const root = process.cwd();
|
|
310
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
334
311
|
die("stdin is not a genroc resolver manifest");
|
|
312
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
313
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
314
|
+
function address(pointer) {
|
|
315
|
+
return pointer
|
|
316
|
+
.map((seg) => typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
317
|
+
? `.${seg}`
|
|
318
|
+
: `[${JSON.stringify(seg)}]`)
|
|
319
|
+
.join("")
|
|
320
|
+
.replace(/^\./, "");
|
|
321
|
+
}
|
|
322
|
+
const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
|
|
323
|
+
site,
|
|
324
|
+
where,
|
|
325
|
+
file: resolve(where.dir, site.argument),
|
|
326
|
+
})));
|
|
327
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
328
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
329
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
330
|
+
// and `external` is the same request made directly.
|
|
331
|
+
for (const at of located) {
|
|
332
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
333
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
334
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
335
|
+
// directly.
|
|
336
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
337
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
338
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
339
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
340
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
341
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
345
|
+
// does is the one that must say when it is not there.
|
|
346
|
+
for (const at of located) {
|
|
347
|
+
if (!existsSync(at.file)) {
|
|
348
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
349
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
335
352
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
336
353
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
337
354
|
const byPath = new Map();
|
|
338
|
-
for (const
|
|
339
|
-
const seen = byPath.get(
|
|
340
|
-
const defsOf = (x) => (manifest.schemas[x.process]?.$defs ?? {});
|
|
355
|
+
for (const at of located) {
|
|
356
|
+
const seen = byPath.get(at.file);
|
|
341
357
|
if (seen &&
|
|
342
|
-
JSON.stringify(
|
|
343
|
-
|
|
344
|
-
|
|
358
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)) {
|
|
359
|
+
die(`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
360
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
345
361
|
"Split it into two scripts, or make the two call sites pass the same shape.");
|
|
346
362
|
}
|
|
347
|
-
byPath.set(
|
|
363
|
+
byPath.set(at.file, at);
|
|
348
364
|
}
|
|
349
|
-
for (const
|
|
350
|
-
|
|
351
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
365
|
+
for (const at of byPath.values()) {
|
|
366
|
+
await writeFile(typesPathFor(at.file), declarations(at));
|
|
352
367
|
}
|
|
353
368
|
if (manifest.mode === "types") {
|
|
369
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
354
370
|
process.exit(0);
|
|
355
371
|
}
|
|
356
|
-
await typecheck(
|
|
372
|
+
await typecheck([...byPath.values()]);
|
|
357
373
|
const code = [];
|
|
358
|
-
for (const
|
|
359
|
-
code.push(await bundle(
|
|
374
|
+
for (const at of located) {
|
|
375
|
+
code.push(await bundle(at));
|
|
360
376
|
}
|
|
361
377
|
process.stdout.write(JSON.stringify({ code }));
|
package/dist/worker.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
//
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
import { readFileSync } from "node:fs";
|
|
8
|
-
import { evaluate } from "./eval.js";
|
|
8
|
+
import { Cancelled, evaluate } from "./eval.js";
|
|
9
9
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
10
10
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
11
11
|
// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
|
|
@@ -62,15 +62,15 @@ async function fetchObject(ref) {
|
|
|
62
62
|
/** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
|
|
63
63
|
* needs no parser: the whole reason they are not JSON Pointer strings. */
|
|
64
64
|
async function resolveObjects(job) {
|
|
65
|
-
let input = job.
|
|
65
|
+
let input = job.external_input;
|
|
66
66
|
for (const e of job.objects ?? []) {
|
|
67
67
|
const value = await fetchObject(e.ref);
|
|
68
68
|
if (e.path.length === 0) {
|
|
69
69
|
input = value;
|
|
70
70
|
continue;
|
|
71
71
|
}
|
|
72
|
-
// The path is rooted at the entry and starts with "
|
|
73
|
-
const rest = e.path[0] === "
|
|
72
|
+
// The path is rooted at the entry and starts with "external_input", the value being rebuilt.
|
|
73
|
+
const rest = e.path[0] === "external_input" ? e.path.slice(1) : e.path;
|
|
74
74
|
if (rest.length === 0) {
|
|
75
75
|
input = value;
|
|
76
76
|
continue;
|
|
@@ -185,7 +185,7 @@ async function answer(token, outcome) {
|
|
|
185
185
|
console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
|
|
186
186
|
await release(token);
|
|
187
187
|
}
|
|
188
|
-
async function run(job) {
|
|
188
|
+
async function run(job, signal) {
|
|
189
189
|
let resolved;
|
|
190
190
|
try {
|
|
191
191
|
resolved = await resolveObjects(job);
|
|
@@ -206,9 +206,17 @@ async function run(job) {
|
|
|
206
206
|
}
|
|
207
207
|
let result;
|
|
208
208
|
try {
|
|
209
|
-
result = await evaluate(req);
|
|
209
|
+
result = await evaluate(req, signal);
|
|
210
210
|
}
|
|
211
211
|
catch (err) {
|
|
212
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
213
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
214
|
+
// waiting out a lease nobody is serving.
|
|
215
|
+
if (err instanceof Cancelled) {
|
|
216
|
+
console.log(`cancelled: ${job.token}`);
|
|
217
|
+
await release(job.token);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
212
220
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
213
221
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
214
222
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -245,11 +253,22 @@ async function renewLoop() {
|
|
|
245
253
|
tokens,
|
|
246
254
|
lease_ms: LEASE_MS,
|
|
247
255
|
});
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
256
|
+
if (!ok)
|
|
257
|
+
continue;
|
|
258
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
259
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
260
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
261
|
+
// own catch, so nothing is released twice.
|
|
262
|
+
for (const token of (data?.cancelled ?? [])) {
|
|
263
|
+
inFlight.get(token)?.abort.abort();
|
|
264
|
+
}
|
|
265
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
266
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
267
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
268
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
269
|
+
const lost = (data?.lost ?? []);
|
|
270
|
+
if (lost.length) {
|
|
271
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
}
|
|
@@ -258,8 +277,9 @@ async function pollLoop() {
|
|
|
258
277
|
const free = CONCURRENCY - inFlight.size;
|
|
259
278
|
const jobs = free > 0 ? await claim(free) : [];
|
|
260
279
|
for (const job of jobs) {
|
|
261
|
-
|
|
262
|
-
|
|
280
|
+
const abort = new AbortController();
|
|
281
|
+
inFlight.set(job.token, { job, abort });
|
|
282
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
263
283
|
}
|
|
264
284
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
265
285
|
// realms allow, not at the poll interval.
|
package/eval.ts
CHANGED
|
@@ -57,14 +57,32 @@ class RealmFault extends Error {
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
61
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
62
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
63
|
+
export class Cancelled extends Error {
|
|
64
|
+
constructor() {
|
|
65
|
+
super("cancelled");
|
|
66
|
+
this.name = "Cancelled";
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
|
|
61
71
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
62
72
|
|
|
63
73
|
const worker = new Worker(REALM_URL);
|
|
64
74
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
75
|
+
let onAbort: (() => void) | undefined;
|
|
65
76
|
try {
|
|
66
77
|
return await new Promise<EvalResult>((resolve, reject) => {
|
|
67
78
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
79
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
80
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
81
|
+
if (signal) {
|
|
82
|
+
if (signal.aborted) reject(new Cancelled());
|
|
83
|
+
onAbort = () => reject(new Cancelled());
|
|
84
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
85
|
+
}
|
|
68
86
|
worker.once("message", (reply: WorkerReply) => resolve(reply));
|
|
69
87
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
70
88
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -75,6 +93,7 @@ export async function evaluate(req: EvalRequest): Promise<EvalResult> {
|
|
|
75
93
|
});
|
|
76
94
|
} finally {
|
|
77
95
|
clearTimeout(timer);
|
|
96
|
+
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
78
97
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
79
98
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
80
99
|
await worker.terminate();
|
package/import.ts
CHANGED
|
@@ -7,11 +7,10 @@
|
|
|
7
7
|
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
8
|
// over the same project.
|
|
9
9
|
|
|
10
|
-
import {
|
|
11
|
-
import { access,
|
|
10
|
+
import { existsSync } from "node:fs";
|
|
11
|
+
import { access, writeFile } from "node:fs/promises";
|
|
12
12
|
import { builtinModules } from "node:module";
|
|
13
|
-
import { dirname, join,
|
|
14
|
-
import { fileURLToPath } from "node:url";
|
|
13
|
+
import { dirname, join, resolve } from "node:path";
|
|
15
14
|
|
|
16
15
|
import commonjs from "@rollup/plugin-commonjs";
|
|
17
16
|
import json from "@rollup/plugin-json";
|
|
@@ -22,20 +21,37 @@ import ts from "typescript";
|
|
|
22
21
|
type Schema = Record<string, any>;
|
|
23
22
|
|
|
24
23
|
type Site = {
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
// Which namespace the directive sits in: `process`, `task` or `action`.
|
|
25
|
+
level: "process" | "task" | "action";
|
|
27
26
|
task?: string;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
// What the site IS, at action level: the action's type, and the process a child call names.
|
|
28
|
+
action?: string;
|
|
29
|
+
child?: string;
|
|
30
|
+
// The slot as a path of keys and indices, so nothing has to unescape a JSON Pointer.
|
|
31
|
+
pointer: (string | number)[];
|
|
32
|
+
// Everything after `$<resolver>:`, verbatim. genctl does not read it as a path — that is
|
|
33
|
+
// this resolver's reading of it.
|
|
34
|
+
argument: string;
|
|
35
|
+
// What .genroc asked genctl to type, keyed by the name it chose. This resolver reads Input
|
|
36
|
+
// and Output; which ADDRESS each came from is the config's business, not ours.
|
|
37
|
+
types?: Record<string, Schema>;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
type ManifestProcess = {
|
|
41
|
+
name: string;
|
|
42
|
+
// The definition's own location, split because a relative argument is relative to the
|
|
43
|
+
// DIRECTORY — joining is ours to do, and this is the base.
|
|
44
|
+
dir: string;
|
|
45
|
+
file: string;
|
|
46
|
+
sites: Site[];
|
|
47
|
+
// Only what this process's fragments reach — a `$ref` in `types` points here.
|
|
48
|
+
$defs?: Record<string, Schema>;
|
|
32
49
|
};
|
|
33
50
|
|
|
34
51
|
type Manifest = {
|
|
35
52
|
mode: "types" | "build";
|
|
36
53
|
root: string;
|
|
37
|
-
|
|
38
|
-
sites: Site[];
|
|
54
|
+
processes: ManifestProcess[];
|
|
39
55
|
};
|
|
40
56
|
|
|
41
57
|
function die(message: string): never {
|
|
@@ -52,12 +68,6 @@ async function exists(path: string): Promise<boolean> {
|
|
|
52
68
|
}
|
|
53
69
|
}
|
|
54
70
|
|
|
55
|
-
/** Creates the parent directory, which `.genroc-cache/` relies on: nothing else makes it. */
|
|
56
|
-
async function write(path: string, content: string): Promise<void> {
|
|
57
|
-
await mkdir(dirname(path), { recursive: true });
|
|
58
|
-
await writeFile(path, content);
|
|
59
|
-
}
|
|
60
|
-
|
|
61
71
|
// ── JSON Schema → TypeScript ───────────────────────────────────────────────────
|
|
62
72
|
|
|
63
73
|
/** Only genroc's keyword set is handled; anything else its strict decoder would have
|
|
@@ -119,16 +129,21 @@ function objectType(s: Schema, used: Set<string>): string {
|
|
|
119
129
|
? ` /** ${sub.description} */\n`
|
|
120
130
|
: "";
|
|
121
131
|
lines.push(
|
|
122
|
-
`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`,
|
|
132
|
+
`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${nest(tsType(sub, used))};`,
|
|
123
133
|
);
|
|
124
134
|
}
|
|
125
135
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
126
|
-
lines.push(
|
|
136
|
+
lines.push(
|
|
137
|
+
` [key: string]: ${nest(tsType(s.additionalProperties, used))};`,
|
|
138
|
+
);
|
|
127
139
|
}
|
|
128
140
|
if (lines.length === 0) return "Record<string, unknown>";
|
|
129
141
|
return `{\n${lines.join("\n")}\n}`;
|
|
130
142
|
}
|
|
131
143
|
|
|
144
|
+
/** Every line but the first, which is already placed by the property that opens it. */
|
|
145
|
+
const nest = (t: string) => t.replace(/\n/g, "\n ");
|
|
146
|
+
|
|
132
147
|
function union(parts: string[]): string {
|
|
133
148
|
const seen = [...new Set(parts)];
|
|
134
149
|
return seen.length === 0 ? "unknown" : seen.join(" | ");
|
|
@@ -139,36 +154,14 @@ const propKey = (k: string) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
|
139
154
|
const identifier = (n: string) =>
|
|
140
155
|
IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
141
156
|
|
|
142
|
-
function deref(
|
|
143
|
-
s: Schema | undefined,
|
|
144
|
-
defs: Record<string, Schema>,
|
|
145
|
-
): Schema | undefined {
|
|
146
|
-
let cur = s;
|
|
147
|
-
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
148
|
-
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
149
|
-
}
|
|
150
|
-
return cur;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
154
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
155
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
156
|
-
* the navigation belongs here rather than in genctl. */
|
|
157
|
-
function scriptInput(
|
|
158
|
-
site: Site,
|
|
159
|
-
defs: Record<string, Schema>,
|
|
160
|
-
): Schema | undefined {
|
|
161
|
-
const action = deref(site.input, defs);
|
|
162
|
-
const bound = action?.properties?.input;
|
|
163
|
-
return bound ?? site.input;
|
|
164
|
-
}
|
|
165
|
-
|
|
166
157
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
167
158
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
168
|
-
function declarations(
|
|
159
|
+
function declarations(at: Located): string {
|
|
160
|
+
const { site, where } = at;
|
|
161
|
+
const defs = where.$defs ?? {};
|
|
169
162
|
const used = new Set<string>();
|
|
170
|
-
const input = tsType(
|
|
171
|
-
const output = tsType(site.
|
|
163
|
+
const input = tsType(site.types?.Input, used);
|
|
164
|
+
const output = tsType(site.types?.Output, used);
|
|
172
165
|
|
|
173
166
|
const emitted: string[] = [];
|
|
174
167
|
const done = new Set<string>();
|
|
@@ -182,7 +175,7 @@ function declarations(site: Site, defs: Record<string, Schema>): string {
|
|
|
182
175
|
|
|
183
176
|
return [
|
|
184
177
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
185
|
-
`// ${
|
|
178
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
186
179
|
"",
|
|
187
180
|
...emitted,
|
|
188
181
|
emitted.length ? "" : "",
|
|
@@ -215,26 +208,21 @@ async function nearestTsconfig(
|
|
|
215
208
|
}
|
|
216
209
|
}
|
|
217
210
|
|
|
218
|
-
async function typecheck(
|
|
219
|
-
//
|
|
220
|
-
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
221
|
-
const dir = join(root, ".genroc-cache");
|
|
222
|
-
await write(join(dir, ".gitignore"), "*\n");
|
|
223
|
-
|
|
224
|
-
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
211
|
+
async function typecheck(sites: Located[]): Promise<void> {
|
|
212
|
+
// One program per distinct base config: `extends` takes a single base, so merging two would
|
|
225
213
|
// check each script under the other author's options.
|
|
226
|
-
const groups = new Map<string,
|
|
227
|
-
for (const
|
|
228
|
-
const base = (await nearestTsconfig(dirname(
|
|
214
|
+
const groups = new Map<string, Located[]>();
|
|
215
|
+
for (const at of sites) {
|
|
216
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
229
217
|
const group = groups.get(base);
|
|
230
|
-
if (group) group.push(
|
|
231
|
-
else groups.set(base, [
|
|
218
|
+
if (group) group.push(at);
|
|
219
|
+
else groups.set(base, [at]);
|
|
232
220
|
}
|
|
233
221
|
|
|
234
|
-
|
|
222
|
+
const diagnostics: ts.Diagnostic[] = [];
|
|
235
223
|
for (const [base, group] of groups) {
|
|
236
224
|
const config: Record<string, unknown> = {
|
|
237
|
-
...(base ? { extends:
|
|
225
|
+
...(base ? { extends: base } : {}),
|
|
238
226
|
compilerOptions: {
|
|
239
227
|
noEmit: true,
|
|
240
228
|
strict: true,
|
|
@@ -250,41 +238,42 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
250
238
|
// base config there is nothing to opt in with, so the default stays none.
|
|
251
239
|
...(base ? {} : { types: [] }),
|
|
252
240
|
},
|
|
253
|
-
files: group.flatMap((s) => [
|
|
254
|
-
relative(dir, s.path),
|
|
255
|
-
relative(dir, typesPathFor(s.path)),
|
|
256
|
-
]),
|
|
241
|
+
files: group.flatMap((s) => [s.file, typesPathFor(s.file)]),
|
|
257
242
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
258
243
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
259
244
|
include: [],
|
|
260
245
|
};
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
246
|
+
// Never written: the name only anchors `@types` lookup at the root. It must not be
|
|
247
|
+
// `tsconfig.json`: that is the base the config `extends` whenever the author keeps one at
|
|
248
|
+
// the root, and tsc refuses a config that extends its own path as circular.
|
|
249
|
+
const parsed = ts.parseJsonConfigFileContent(
|
|
250
|
+
config,
|
|
251
|
+
ts.sys,
|
|
252
|
+
root,
|
|
253
|
+
undefined,
|
|
254
|
+
join(root, "tsconfig.genroc.json"),
|
|
264
255
|
);
|
|
265
|
-
|
|
266
|
-
|
|
256
|
+
const program = ts.createProgram({
|
|
257
|
+
rootNames: parsed.fileNames,
|
|
258
|
+
options: parsed.options,
|
|
259
|
+
projectReferences: parsed.projectReferences,
|
|
260
|
+
configFileParsingDiagnostics: parsed.errors,
|
|
261
|
+
});
|
|
262
|
+
diagnostics.push(...parsed.errors, ...ts.getPreEmitDiagnostics(program));
|
|
267
263
|
}
|
|
268
|
-
}
|
|
269
264
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
proc.on("close", (c) => resolve(c ?? 1));
|
|
283
|
-
});
|
|
284
|
-
if (code !== 0) {
|
|
285
|
-
// tsc reports on stdout; the exit code IS the type check, so this is the diagnostic
|
|
286
|
-
// genctl surfaces and the reason a failed import never produces a string.
|
|
287
|
-
die([out, err].filter(Boolean).join("\n").trimEnd());
|
|
265
|
+
if (diagnostics.length > 0) {
|
|
266
|
+
// The diagnostics are the whole type check: this is what genctl surfaces, and the reason a
|
|
267
|
+
// failed import never produces a string.
|
|
268
|
+
die(
|
|
269
|
+
ts
|
|
270
|
+
.formatDiagnostics(diagnostics, {
|
|
271
|
+
getCanonicalFileName: (f) => f,
|
|
272
|
+
getCurrentDirectory: () => root,
|
|
273
|
+
getNewLine: () => "\n",
|
|
274
|
+
})
|
|
275
|
+
.trimEnd(),
|
|
276
|
+
);
|
|
288
277
|
}
|
|
289
278
|
}
|
|
290
279
|
|
|
@@ -351,15 +340,16 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
351
340
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
352
341
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
353
342
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
354
|
-
async function bundle(
|
|
343
|
+
async function bundle(at: Located): Promise<string> {
|
|
344
|
+
const site = at.site;
|
|
355
345
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
356
346
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
357
347
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
358
348
|
const built = await rollup({
|
|
359
|
-
input:
|
|
349
|
+
input: at.file,
|
|
360
350
|
external: (id) => BUILTIN.has(id),
|
|
361
351
|
plugins: [
|
|
362
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
352
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
363
353
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
364
354
|
commonjs(),
|
|
365
355
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -370,12 +360,12 @@ async function bundle(site: Site, root: string): Promise<string> {
|
|
|
370
360
|
onwarn(warning) {
|
|
371
361
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
372
362
|
die(
|
|
373
|
-
`${
|
|
363
|
+
`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`,
|
|
374
364
|
);
|
|
375
365
|
}
|
|
376
366
|
},
|
|
377
367
|
}).catch((e: unknown) =>
|
|
378
|
-
die(`${
|
|
368
|
+
die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`),
|
|
379
369
|
);
|
|
380
370
|
|
|
381
371
|
const { output } = await built.generate({
|
|
@@ -386,7 +376,7 @@ async function bundle(site: Site, root: string): Promise<string> {
|
|
|
386
376
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
387
377
|
// instance, and the file it names is on this machine.
|
|
388
378
|
if (!output[0].exports.includes("default")) {
|
|
389
|
-
die(`${
|
|
379
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
390
380
|
}
|
|
391
381
|
return output[0].code;
|
|
392
382
|
}
|
|
@@ -401,45 +391,99 @@ const stdin: string = await new Promise((resolve, reject) => {
|
|
|
401
391
|
process.stdin.on("error", reject);
|
|
402
392
|
});
|
|
403
393
|
const manifest = JSON.parse(stdin) as Manifest;
|
|
404
|
-
|
|
394
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
395
|
+
const root = process.cwd();
|
|
396
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
405
397
|
die("stdin is not a genroc resolver manifest");
|
|
406
398
|
|
|
399
|
+
/** Every site with the process it sits in and the file its argument names, in the manifest's own
|
|
400
|
+
* order — which is the order `code` must answer in. genctl passes the argument verbatim, so
|
|
401
|
+
* joining it to the definition's directory is this resolver's business: only it knows the
|
|
402
|
+
* argument is a path at all. */
|
|
403
|
+
type Located = { site: Site; where: ManifestProcess; file: string };
|
|
404
|
+
|
|
405
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
406
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
407
|
+
function address(pointer: (string | number)[]): string {
|
|
408
|
+
return pointer
|
|
409
|
+
.map((seg) =>
|
|
410
|
+
typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
411
|
+
? `.${seg}`
|
|
412
|
+
: `[${JSON.stringify(seg)}]`,
|
|
413
|
+
)
|
|
414
|
+
.join("")
|
|
415
|
+
.replace(/^\./, "");
|
|
416
|
+
}
|
|
417
|
+
const located: Located[] = manifest.processes.flatMap((where) =>
|
|
418
|
+
where.sites.map((site) => ({
|
|
419
|
+
site,
|
|
420
|
+
where,
|
|
421
|
+
file: resolve(where.dir, site.argument),
|
|
422
|
+
})),
|
|
423
|
+
);
|
|
424
|
+
|
|
425
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
426
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
427
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
428
|
+
// and `external` is the same request made directly.
|
|
429
|
+
for (const at of located) {
|
|
430
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
431
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
432
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
433
|
+
// directly.
|
|
434
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
435
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
436
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
437
|
+
die(
|
|
438
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
439
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
440
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`,
|
|
441
|
+
);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
446
|
+
// does is the one that must say when it is not there.
|
|
447
|
+
for (const at of located) {
|
|
448
|
+
if (!existsSync(at.file)) {
|
|
449
|
+
die(
|
|
450
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
451
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`,
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
407
456
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
408
457
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
409
|
-
const byPath = new Map<string,
|
|
410
|
-
for (const
|
|
411
|
-
const seen = byPath.get(
|
|
412
|
-
const defsOf = (x: Site) =>
|
|
413
|
-
(manifest.schemas[x.process]?.$defs ?? {}) as Record<string, Schema>;
|
|
458
|
+
const byPath = new Map<string, Located>();
|
|
459
|
+
for (const at of located) {
|
|
460
|
+
const seen = byPath.get(at.file);
|
|
414
461
|
if (
|
|
415
462
|
seen &&
|
|
416
|
-
JSON.stringify(
|
|
417
|
-
JSON.stringify(scriptInput(site, defsOf(site)))
|
|
463
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)
|
|
418
464
|
) {
|
|
419
465
|
die(
|
|
420
|
-
`${
|
|
466
|
+
`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
467
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
421
468
|
"Split it into two scripts, or make the two call sites pass the same shape.",
|
|
422
469
|
);
|
|
423
470
|
}
|
|
424
|
-
byPath.set(
|
|
471
|
+
byPath.set(at.file, at);
|
|
425
472
|
}
|
|
426
473
|
|
|
427
|
-
for (const
|
|
428
|
-
|
|
429
|
-
string,
|
|
430
|
-
Schema
|
|
431
|
-
>;
|
|
432
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
474
|
+
for (const at of byPath.values()) {
|
|
475
|
+
await writeFile(typesPathFor(at.file), declarations(at));
|
|
433
476
|
}
|
|
434
477
|
|
|
435
478
|
if (manifest.mode === "types") {
|
|
479
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
436
480
|
process.exit(0);
|
|
437
481
|
}
|
|
438
482
|
|
|
439
|
-
await typecheck(
|
|
483
|
+
await typecheck([...byPath.values()]);
|
|
440
484
|
|
|
441
485
|
const code: string[] = [];
|
|
442
|
-
for (const
|
|
443
|
-
code.push(await bundle(
|
|
486
|
+
for (const at of located) {
|
|
487
|
+
code.push(await bundle(at));
|
|
444
488
|
}
|
|
445
489
|
process.stdout.write(JSON.stringify({ code }));
|
package/package.json
CHANGED
package/worker.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
|
|
8
8
|
import { readFileSync } from "node:fs";
|
|
9
|
-
import { evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
9
|
+
import { Cancelled, evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
10
10
|
|
|
11
11
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
12
12
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
@@ -43,8 +43,8 @@ type ObjectEntry = { path: (string | number)[]; ref: string; size: number };
|
|
|
43
43
|
type QueueTask = {
|
|
44
44
|
token: string;
|
|
45
45
|
process: string;
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
task: string;
|
|
47
|
+
external_input: unknown;
|
|
48
48
|
objects?: ObjectEntry[];
|
|
49
49
|
raises?: Record<string, unknown>;
|
|
50
50
|
};
|
|
@@ -76,15 +76,15 @@ async function fetchObject(ref: string): Promise<unknown> {
|
|
|
76
76
|
/** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
|
|
77
77
|
* needs no parser: the whole reason they are not JSON Pointer strings. */
|
|
78
78
|
async function resolveObjects(job: QueueTask): Promise<unknown> {
|
|
79
|
-
let input: any = job.
|
|
79
|
+
let input: any = job.external_input;
|
|
80
80
|
for (const e of job.objects ?? []) {
|
|
81
81
|
const value = await fetchObject(e.ref);
|
|
82
82
|
if (e.path.length === 0) {
|
|
83
83
|
input = value;
|
|
84
84
|
continue;
|
|
85
85
|
}
|
|
86
|
-
// The path is rooted at the entry and starts with "
|
|
87
|
-
const rest = e.path[0] === "
|
|
86
|
+
// The path is rooted at the entry and starts with "external_input", the value being rebuilt.
|
|
87
|
+
const rest = e.path[0] === "external_input" ? e.path.slice(1) : e.path;
|
|
88
88
|
if (rest.length === 0) {
|
|
89
89
|
input = value;
|
|
90
90
|
continue;
|
|
@@ -135,7 +135,12 @@ function asEvalRequest(input: unknown): EvalRequest | string {
|
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
|
|
138
|
+
/** A claim this worker is currently serving. The controller is how the renewal loop reaches
|
|
139
|
+
* the evaluation: genroc cannot call us, so a cancellation arrives as an answer to our own
|
|
140
|
+
* heartbeat and has to be delivered inward from there. */
|
|
141
|
+
type Running = { job: QueueTask; abort: AbortController };
|
|
142
|
+
|
|
143
|
+
const inFlight = new Map<string, Running>();
|
|
139
144
|
let running = true;
|
|
140
145
|
|
|
141
146
|
// Whether the last claim reached genroc. A worker polls several times a second, so an
|
|
@@ -203,7 +208,7 @@ async function answer(token: string, outcome: Record<string, unknown>): Promise<
|
|
|
203
208
|
await release(token);
|
|
204
209
|
}
|
|
205
210
|
|
|
206
|
-
async function run(job: QueueTask): Promise<void> {
|
|
211
|
+
async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
207
212
|
let resolved: unknown;
|
|
208
213
|
try {
|
|
209
214
|
resolved = await resolveObjects(job);
|
|
@@ -224,8 +229,16 @@ async function run(job: QueueTask): Promise<void> {
|
|
|
224
229
|
|
|
225
230
|
let result;
|
|
226
231
|
try {
|
|
227
|
-
result = await evaluate(req);
|
|
232
|
+
result = await evaluate(req, signal);
|
|
228
233
|
} catch (err) {
|
|
234
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
235
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
236
|
+
// waiting out a lease nobody is serving.
|
|
237
|
+
if (err instanceof Cancelled) {
|
|
238
|
+
console.log(`cancelled: ${job.token}`);
|
|
239
|
+
await release(job.token);
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
229
242
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
230
243
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
231
244
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -263,11 +276,22 @@ async function renewLoop(): Promise<void> {
|
|
|
263
276
|
tokens,
|
|
264
277
|
lease_ms: LEASE_MS,
|
|
265
278
|
});
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
279
|
+
if (!ok) continue;
|
|
280
|
+
|
|
281
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
282
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
283
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
284
|
+
// own catch, so nothing is released twice.
|
|
285
|
+
for (const token of (data?.cancelled ?? []) as string[]) {
|
|
286
|
+
inFlight.get(token)?.abort.abort();
|
|
287
|
+
}
|
|
288
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
289
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
290
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
291
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
292
|
+
const lost = (data?.lost ?? []) as string[];
|
|
293
|
+
if (lost.length) {
|
|
294
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
271
295
|
}
|
|
272
296
|
}
|
|
273
297
|
}
|
|
@@ -277,8 +301,9 @@ async function pollLoop(): Promise<void> {
|
|
|
277
301
|
const free = CONCURRENCY - inFlight.size;
|
|
278
302
|
const jobs = free > 0 ? await claim(free) : [];
|
|
279
303
|
for (const job of jobs) {
|
|
280
|
-
|
|
281
|
-
|
|
304
|
+
const abort = new AbortController();
|
|
305
|
+
inFlight.set(job.token, { job, abort });
|
|
306
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
282
307
|
}
|
|
283
308
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
284
309
|
// realms allow, not at the poll interval.
|