@genroc/eval-node 0.0.0-edge.ac9249c → 0.0.0-edge.afdb7a4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +19 -2
- package/dist/eval.js +21 -1
- package/dist/import.js +94 -45
- package/dist/realm.js +10 -1
- package/dist/worker.js +30 -10
- package/eval.ts +20 -1
- package/import.ts +179 -66
- package/package.json +1 -1
- package/realm.ts +11 -1
- package/worker.ts +36 -11
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ resolver's exit code, so a stored definition cannot hold code that failed to typ
|
|
|
26
26
|
|
|
27
27
|
## Building
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
pnpm run build # tsc -p tsconfig.build.json -> dist/
|
|
30
30
|
|
|
31
31
|
The published package is JavaScript, not TypeScript. **Node refuses to strip types for files
|
|
32
32
|
under `node_modules`**, so shipping `.ts` works from a checkout and fails for every consumer —
|
|
@@ -90,6 +90,23 @@ queue worker rather than the HTTP sidecar it used to be:
|
|
|
90
90
|
that genroc had to be able to reach. A worker only needs outbound access, so it can live
|
|
91
91
|
anywhere — behind NAT, in another trust zone.
|
|
92
92
|
|
|
93
|
+
The cost of that inversion is that genroc cannot reach a running worker, so anything it needs
|
|
94
|
+
to say has to ride the renewal the worker already makes. **Renewing is therefore mandatory,
|
|
95
|
+
not an optimisation** — a worker that stops renewing is indistinguishable from one that died.
|
|
96
|
+
The response answers per token:
|
|
97
|
+
|
|
98
|
+
| list | what it means | what this worker does |
|
|
99
|
+
|---|---|---|
|
|
100
|
+
| `renewed` | still yours | carry on |
|
|
101
|
+
| `lost` | already someone else's | stop; do **not** release — that would bump the new holder's claim |
|
|
102
|
+
| `cancelled` | still yours, nobody wants it | abort the script and release the claim |
|
|
103
|
+
|
|
104
|
+
`cancelled` is how an operator's `genctl cancel` reaches work already running: the evaluation
|
|
105
|
+
is aborted through an `AbortSignal`, its realm is terminated, and no outcome is submitted —
|
|
106
|
+
the process is stopping, so there is nothing left to answer. `renew_before_ms` on the claim
|
|
107
|
+
says how long the worker may wait before renewing again, so the interval is a value it reads
|
|
108
|
+
rather than one it guesses.
|
|
109
|
+
|
|
93
110
|
## The task input
|
|
94
111
|
|
|
95
112
|
The `input` of the external task IS the evaluation request:
|
|
@@ -169,7 +186,7 @@ no retry policy to write.
|
|
|
169
186
|
|
|
170
187
|
- id: script_failed
|
|
171
188
|
switch:
|
|
172
|
-
- case: '
|
|
189
|
+
- case: 'last_error.data.name == "LimitExceeded"'
|
|
173
190
|
raise: { code: limit_exceeded, message: "the script rejected the amount" }
|
|
174
191
|
- raise: { code: script_failed, message: "the script failed" }
|
|
175
192
|
```
|
package/dist/eval.js
CHANGED
|
@@ -20,13 +20,31 @@ class RealmFault extends Error {
|
|
|
20
20
|
this.name = "RealmFault";
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
24
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
25
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
26
|
+
export class Cancelled extends Error {
|
|
27
|
+
constructor() {
|
|
28
|
+
super("cancelled");
|
|
29
|
+
this.name = "Cancelled";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export async function evaluate(req, signal) {
|
|
24
33
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
25
34
|
const worker = new Worker(REALM_URL);
|
|
26
35
|
let timer;
|
|
36
|
+
let onAbort;
|
|
27
37
|
try {
|
|
28
38
|
return await new Promise((resolve, reject) => {
|
|
29
39
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
40
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
41
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
42
|
+
if (signal) {
|
|
43
|
+
if (signal.aborted)
|
|
44
|
+
reject(new Cancelled());
|
|
45
|
+
onAbort = () => reject(new Cancelled());
|
|
46
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
47
|
+
}
|
|
30
48
|
worker.once("message", (reply) => resolve(reply));
|
|
31
49
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
32
50
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -38,6 +56,8 @@ export async function evaluate(req) {
|
|
|
38
56
|
}
|
|
39
57
|
finally {
|
|
40
58
|
clearTimeout(timer);
|
|
59
|
+
if (signal && onAbort)
|
|
60
|
+
signal.removeEventListener("abort", onAbort);
|
|
41
61
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
42
62
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
43
63
|
await worker.terminate();
|
package/dist/import.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
8
|
// over the same project.
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
|
+
import { existsSync } from "node:fs";
|
|
10
11
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
11
12
|
import { builtinModules } from "node:module";
|
|
12
|
-
import { dirname, join, relative } from "node:path";
|
|
13
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
13
14
|
import { fileURLToPath } from "node:url";
|
|
14
15
|
import commonjs from "@rollup/plugin-commonjs";
|
|
15
16
|
import json from "@rollup/plugin-json";
|
|
@@ -86,7 +87,9 @@ function objectType(s, used) {
|
|
|
86
87
|
const required = new Set(s.required ?? []);
|
|
87
88
|
const lines = [];
|
|
88
89
|
for (const [key, sub] of Object.entries(props)) {
|
|
89
|
-
const doc = typeof sub.description === "string"
|
|
90
|
+
const doc = typeof sub.description === "string"
|
|
91
|
+
? ` /** ${sub.description} */\n`
|
|
92
|
+
: "";
|
|
90
93
|
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`);
|
|
91
94
|
}
|
|
92
95
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
@@ -102,7 +105,7 @@ function union(parts) {
|
|
|
102
105
|
}
|
|
103
106
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
104
107
|
const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
105
|
-
const identifier = (n) =>
|
|
108
|
+
const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
106
109
|
function deref(s, defs) {
|
|
107
110
|
let cur = s;
|
|
108
111
|
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
@@ -110,21 +113,14 @@ function deref(s, defs) {
|
|
|
110
113
|
}
|
|
111
114
|
return cur;
|
|
112
115
|
}
|
|
113
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
114
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
115
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
116
|
-
* the navigation belongs here rather than in genctl. */
|
|
117
|
-
function scriptInput(site, defs) {
|
|
118
|
-
const action = deref(site.input, defs);
|
|
119
|
-
const bound = action?.properties?.input;
|
|
120
|
-
return bound ?? site.input;
|
|
121
|
-
}
|
|
122
116
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
123
117
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
124
|
-
function declarations(
|
|
118
|
+
function declarations(at) {
|
|
119
|
+
const { site, where } = at;
|
|
120
|
+
const defs = where.$defs ?? {};
|
|
125
121
|
const used = new Set();
|
|
126
|
-
const input = tsType(
|
|
127
|
-
const output = tsType(site.
|
|
122
|
+
const input = tsType(site.types?.Input, used);
|
|
123
|
+
const output = tsType(site.types?.Output, used);
|
|
128
124
|
const emitted = [];
|
|
129
125
|
const done = new Set();
|
|
130
126
|
while (true) {
|
|
@@ -137,7 +133,7 @@ function declarations(site, defs) {
|
|
|
137
133
|
}
|
|
138
134
|
return [
|
|
139
135
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
140
|
-
`// ${
|
|
136
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
141
137
|
"",
|
|
142
138
|
...emitted,
|
|
143
139
|
emitted.length ? "" : "",
|
|
@@ -165,7 +161,7 @@ async function nearestTsconfig(from, root) {
|
|
|
165
161
|
return null;
|
|
166
162
|
}
|
|
167
163
|
}
|
|
168
|
-
async function typecheck(
|
|
164
|
+
async function typecheck(sites) {
|
|
169
165
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
170
166
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
171
167
|
const dir = join(root, ".genroc-cache");
|
|
@@ -173,13 +169,13 @@ async function typecheck(root, sites) {
|
|
|
173
169
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
174
170
|
// check each script under the other author's options.
|
|
175
171
|
const groups = new Map();
|
|
176
|
-
for (const
|
|
177
|
-
const base = (await nearestTsconfig(dirname(
|
|
172
|
+
for (const at of sites) {
|
|
173
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
178
174
|
const group = groups.get(base);
|
|
179
175
|
if (group)
|
|
180
|
-
group.push(
|
|
176
|
+
group.push(at);
|
|
181
177
|
else
|
|
182
|
-
groups.set(base, [
|
|
178
|
+
groups.set(base, [at]);
|
|
183
179
|
}
|
|
184
180
|
let n = 0;
|
|
185
181
|
for (const [base, group] of groups) {
|
|
@@ -200,7 +196,10 @@ async function typecheck(root, sites) {
|
|
|
200
196
|
// base config there is nothing to opt in with, so the default stays none.
|
|
201
197
|
...(base ? {} : { types: [] }),
|
|
202
198
|
},
|
|
203
|
-
files: group.flatMap((s) => [
|
|
199
|
+
files: group.flatMap((s) => [
|
|
200
|
+
relative(dir, s.file),
|
|
201
|
+
relative(dir, typesPathFor(s.file)),
|
|
202
|
+
]),
|
|
204
203
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
205
204
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
206
205
|
include: [],
|
|
@@ -250,7 +249,10 @@ const transpile = {
|
|
|
250
249
|
return { code: out.outputText, map: out.sourceMapText ?? null };
|
|
251
250
|
},
|
|
252
251
|
};
|
|
253
|
-
const BUILTIN = new Set([
|
|
252
|
+
const BUILTIN = new Set([
|
|
253
|
+
...builtinModules,
|
|
254
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
255
|
+
]);
|
|
254
256
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
255
257
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
256
258
|
* resolver to keep in agreement with tsc; this one cannot disagree.
|
|
@@ -270,22 +272,25 @@ function tsResolve(configPath) {
|
|
|
270
272
|
const { resolvedModule } = ts.resolveModuleName(source, importer, options, ts.sys);
|
|
271
273
|
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
272
274
|
return null;
|
|
273
|
-
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
275
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
276
|
+
? null
|
|
277
|
+
: resolvedModule.resolvedFileName;
|
|
274
278
|
},
|
|
275
279
|
};
|
|
276
280
|
}
|
|
277
281
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
278
282
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
279
283
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
280
|
-
async function bundle(
|
|
284
|
+
async function bundle(at) {
|
|
285
|
+
const site = at.site;
|
|
281
286
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
282
287
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
283
288
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
284
289
|
const built = await rollup({
|
|
285
|
-
input:
|
|
290
|
+
input: at.file,
|
|
286
291
|
external: (id) => BUILTIN.has(id),
|
|
287
292
|
plugins: [
|
|
288
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
293
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
289
294
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
290
295
|
commonjs(),
|
|
291
296
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -295,16 +300,19 @@ async function bundle(site, root) {
|
|
|
295
300
|
],
|
|
296
301
|
onwarn(warning) {
|
|
297
302
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
298
|
-
die(`${
|
|
303
|
+
die(`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`);
|
|
299
304
|
}
|
|
300
305
|
},
|
|
301
|
-
}).catch((e) => die(`${
|
|
302
|
-
const { output } = await built.generate({
|
|
306
|
+
}).catch((e) => die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`));
|
|
307
|
+
const { output } = await built.generate({
|
|
308
|
+
format: "es",
|
|
309
|
+
inlineDynamicImports: true,
|
|
310
|
+
});
|
|
303
311
|
await built.close();
|
|
304
312
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
305
313
|
// instance, and the file it names is on this machine.
|
|
306
314
|
if (!output[0].exports.includes("default")) {
|
|
307
|
-
die(`${
|
|
315
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
308
316
|
}
|
|
309
317
|
return output[0].code;
|
|
310
318
|
}
|
|
@@ -317,32 +325,73 @@ const stdin = await new Promise((resolve, reject) => {
|
|
|
317
325
|
process.stdin.on("error", reject);
|
|
318
326
|
});
|
|
319
327
|
const manifest = JSON.parse(stdin);
|
|
320
|
-
|
|
328
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
329
|
+
const root = process.cwd();
|
|
330
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
321
331
|
die("stdin is not a genroc resolver manifest");
|
|
332
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
333
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
334
|
+
function address(pointer) {
|
|
335
|
+
return pointer
|
|
336
|
+
.map((seg) => typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
337
|
+
? `.${seg}`
|
|
338
|
+
: `[${JSON.stringify(seg)}]`)
|
|
339
|
+
.join("")
|
|
340
|
+
.replace(/^\./, "");
|
|
341
|
+
}
|
|
342
|
+
const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
|
|
343
|
+
site,
|
|
344
|
+
where,
|
|
345
|
+
file: resolve(where.dir, site.argument),
|
|
346
|
+
})));
|
|
347
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
348
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
349
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
350
|
+
// and `external` is the same request made directly.
|
|
351
|
+
for (const at of located) {
|
|
352
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
353
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
354
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
355
|
+
// directly.
|
|
356
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
357
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
358
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
359
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
360
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
361
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
365
|
+
// does is the one that must say when it is not there.
|
|
366
|
+
for (const at of located) {
|
|
367
|
+
if (!existsSync(at.file)) {
|
|
368
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
369
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
322
372
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
323
373
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
324
374
|
const byPath = new Map();
|
|
325
|
-
for (const
|
|
326
|
-
const seen = byPath.get(
|
|
327
|
-
const defsOf = (x) => (manifest.schemas[x.process]?.$defs ?? {});
|
|
375
|
+
for (const at of located) {
|
|
376
|
+
const seen = byPath.get(at.file);
|
|
328
377
|
if (seen &&
|
|
329
|
-
JSON.stringify(
|
|
330
|
-
|
|
331
|
-
|
|
378
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)) {
|
|
379
|
+
die(`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
380
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
332
381
|
"Split it into two scripts, or make the two call sites pass the same shape.");
|
|
333
382
|
}
|
|
334
|
-
byPath.set(
|
|
383
|
+
byPath.set(at.file, at);
|
|
335
384
|
}
|
|
336
|
-
for (const
|
|
337
|
-
|
|
338
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
385
|
+
for (const at of byPath.values()) {
|
|
386
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
339
387
|
}
|
|
340
388
|
if (manifest.mode === "types") {
|
|
389
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
341
390
|
process.exit(0);
|
|
342
391
|
}
|
|
343
|
-
await typecheck(
|
|
392
|
+
await typecheck([...byPath.values()]);
|
|
344
393
|
const code = [];
|
|
345
|
-
for (const
|
|
346
|
-
code.push(await bundle(
|
|
394
|
+
for (const at of located) {
|
|
395
|
+
code.push(await bundle(at));
|
|
347
396
|
}
|
|
348
397
|
process.stdout.write(JSON.stringify({ code }));
|
package/dist/realm.js
CHANGED
|
@@ -97,8 +97,17 @@ async function run(req) {
|
|
|
97
97
|
return { ok: false, failure: describe(err, "nonserializable") };
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
|
+
// The realm's stdio is a pipe to the host thread, and eval.ts terminates this thread the moment
|
|
101
|
+
// the reply lands — so whatever a script wrote last is still in the pipe when it dies. An empty
|
|
102
|
+
// write's callback fires once the queue ahead of it has drained, which makes the reply a barrier
|
|
103
|
+
// for `console` and a direct process.stdout.write alike: both are this stream.
|
|
104
|
+
function flush(stream) {
|
|
105
|
+
return new Promise((resolve) => stream.write("", () => resolve()));
|
|
106
|
+
}
|
|
100
107
|
// Non-null because this module only ever runs as a worker entry point; a null port here
|
|
101
108
|
// would mean eval.ts loaded it as a plain module, which nothing does.
|
|
102
109
|
parentPort.on("message", async (req) => {
|
|
103
|
-
|
|
110
|
+
const reply = await run(req);
|
|
111
|
+
await Promise.all([flush(process.stdout), flush(process.stderr)]);
|
|
112
|
+
parentPort.postMessage(reply);
|
|
104
113
|
});
|
package/dist/worker.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
//
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
import { readFileSync } from "node:fs";
|
|
8
|
-
import { evaluate } from "./eval.js";
|
|
8
|
+
import { Cancelled, evaluate } from "./eval.js";
|
|
9
9
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
10
10
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
11
11
|
// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
|
|
@@ -185,7 +185,7 @@ async function answer(token, outcome) {
|
|
|
185
185
|
console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
|
|
186
186
|
await release(token);
|
|
187
187
|
}
|
|
188
|
-
async function run(job) {
|
|
188
|
+
async function run(job, signal) {
|
|
189
189
|
let resolved;
|
|
190
190
|
try {
|
|
191
191
|
resolved = await resolveObjects(job);
|
|
@@ -206,9 +206,17 @@ async function run(job) {
|
|
|
206
206
|
}
|
|
207
207
|
let result;
|
|
208
208
|
try {
|
|
209
|
-
result = await evaluate(req);
|
|
209
|
+
result = await evaluate(req, signal);
|
|
210
210
|
}
|
|
211
211
|
catch (err) {
|
|
212
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
213
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
214
|
+
// waiting out a lease nobody is serving.
|
|
215
|
+
if (err instanceof Cancelled) {
|
|
216
|
+
console.log(`cancelled: ${job.token}`);
|
|
217
|
+
await release(job.token);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
212
220
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
213
221
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
214
222
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -245,11 +253,22 @@ async function renewLoop() {
|
|
|
245
253
|
tokens,
|
|
246
254
|
lease_ms: LEASE_MS,
|
|
247
255
|
});
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
256
|
+
if (!ok)
|
|
257
|
+
continue;
|
|
258
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
259
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
260
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
261
|
+
// own catch, so nothing is released twice.
|
|
262
|
+
for (const token of (data?.cancelled ?? [])) {
|
|
263
|
+
inFlight.get(token)?.abort.abort();
|
|
264
|
+
}
|
|
265
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
266
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
267
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
268
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
269
|
+
const lost = (data?.lost ?? []);
|
|
270
|
+
if (lost.length) {
|
|
271
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
}
|
|
@@ -258,8 +277,9 @@ async function pollLoop() {
|
|
|
258
277
|
const free = CONCURRENCY - inFlight.size;
|
|
259
278
|
const jobs = free > 0 ? await claim(free) : [];
|
|
260
279
|
for (const job of jobs) {
|
|
261
|
-
|
|
262
|
-
|
|
280
|
+
const abort = new AbortController();
|
|
281
|
+
inFlight.set(job.token, { job, abort });
|
|
282
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
263
283
|
}
|
|
264
284
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
265
285
|
// realms allow, not at the poll interval.
|
package/eval.ts
CHANGED
|
@@ -57,14 +57,32 @@ class RealmFault extends Error {
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
61
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
62
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
63
|
+
export class Cancelled extends Error {
|
|
64
|
+
constructor() {
|
|
65
|
+
super("cancelled");
|
|
66
|
+
this.name = "Cancelled";
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
|
|
61
71
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
62
72
|
|
|
63
73
|
const worker = new Worker(REALM_URL);
|
|
64
74
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
75
|
+
let onAbort: (() => void) | undefined;
|
|
65
76
|
try {
|
|
66
77
|
return await new Promise<EvalResult>((resolve, reject) => {
|
|
67
78
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
79
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
80
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
81
|
+
if (signal) {
|
|
82
|
+
if (signal.aborted) reject(new Cancelled());
|
|
83
|
+
onAbort = () => reject(new Cancelled());
|
|
84
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
85
|
+
}
|
|
68
86
|
worker.once("message", (reply: WorkerReply) => resolve(reply));
|
|
69
87
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
70
88
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -75,6 +93,7 @@ export async function evaluate(req: EvalRequest): Promise<EvalResult> {
|
|
|
75
93
|
});
|
|
76
94
|
} finally {
|
|
77
95
|
clearTimeout(timer);
|
|
96
|
+
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
78
97
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
79
98
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
80
99
|
await worker.terminate();
|
package/import.ts
CHANGED
|
@@ -8,9 +8,10 @@
|
|
|
8
8
|
// over the same project.
|
|
9
9
|
|
|
10
10
|
import { spawn } from "node:child_process";
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
11
12
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
12
13
|
import { builtinModules } from "node:module";
|
|
13
|
-
import { dirname, join, relative } from "node:path";
|
|
14
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
14
15
|
import { fileURLToPath } from "node:url";
|
|
15
16
|
|
|
16
17
|
import commonjs from "@rollup/plugin-commonjs";
|
|
@@ -22,20 +23,37 @@ import ts from "typescript";
|
|
|
22
23
|
type Schema = Record<string, any>;
|
|
23
24
|
|
|
24
25
|
type Site = {
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
// Which namespace the directive sits in: `process`, `task` or `action`.
|
|
27
|
+
level: "process" | "task" | "action";
|
|
27
28
|
task?: string;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
// What the site IS, at action level: the action's type, and the process a child call names.
|
|
30
|
+
action?: string;
|
|
31
|
+
child?: string;
|
|
32
|
+
// The slot as a path of keys and indices, so nothing has to unescape a JSON Pointer.
|
|
33
|
+
pointer: (string | number)[];
|
|
34
|
+
// Everything after `$<resolver>:`, verbatim. genctl does not read it as a path — that is
|
|
35
|
+
// this resolver's reading of it.
|
|
36
|
+
argument: string;
|
|
37
|
+
// What .genroc asked genctl to type, keyed by the name it chose. This resolver reads Input
|
|
38
|
+
// and Output; which ADDRESS each came from is the config's business, not ours.
|
|
39
|
+
types?: Record<string, Schema>;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
type ManifestProcess = {
|
|
43
|
+
name: string;
|
|
44
|
+
// The definition's own location, split because a relative argument is relative to the
|
|
45
|
+
// DIRECTORY — joining is ours to do, and this is the base.
|
|
46
|
+
dir: string;
|
|
47
|
+
file: string;
|
|
48
|
+
sites: Site[];
|
|
49
|
+
// Only what this process's fragments reach — a `$ref` in `types` points here.
|
|
50
|
+
$defs?: Record<string, Schema>;
|
|
32
51
|
};
|
|
33
52
|
|
|
34
53
|
type Manifest = {
|
|
35
54
|
mode: "types" | "build";
|
|
36
55
|
root: string;
|
|
37
|
-
|
|
38
|
-
sites: Site[];
|
|
56
|
+
processes: ManifestProcess[];
|
|
39
57
|
};
|
|
40
58
|
|
|
41
59
|
function die(message: string): never {
|
|
@@ -72,13 +90,16 @@ function tsType(s: Schema | undefined, used: Set<string>): string {
|
|
|
72
90
|
if (Array.isArray(s.enum)) {
|
|
73
91
|
return s.enum.map((v: unknown) => JSON.stringify(v)).join(" | ") || "never";
|
|
74
92
|
}
|
|
75
|
-
if (Array.isArray(s.anyOf))
|
|
76
|
-
|
|
93
|
+
if (Array.isArray(s.anyOf))
|
|
94
|
+
return union(s.anyOf.map((a: Schema) => tsType(a, used)));
|
|
95
|
+
if (Array.isArray(s.oneOf))
|
|
96
|
+
return union(s.oneOf.map((a: Schema) => tsType(a, used)));
|
|
77
97
|
if (Array.isArray(s.allOf)) {
|
|
78
98
|
return s.allOf.map((a: Schema) => tsType(a, used)).join(" & ") || "unknown";
|
|
79
99
|
}
|
|
80
100
|
|
|
81
|
-
const types: string[] =
|
|
101
|
+
const types: string[] =
|
|
102
|
+
s.type === undefined ? [] : Array.isArray(s.type) ? s.type : [s.type];
|
|
82
103
|
if (types.length === 0) {
|
|
83
104
|
// The top type: `{}` means unknown, not "an empty object". specs/unknown-type.md.
|
|
84
105
|
return s.properties ? objectType(s, used) : "unknown";
|
|
@@ -111,8 +132,13 @@ function objectType(s: Schema, used: Set<string>): string {
|
|
|
111
132
|
const required = new Set<string>(s.required ?? []);
|
|
112
133
|
const lines: string[] = [];
|
|
113
134
|
for (const [key, sub] of Object.entries(props)) {
|
|
114
|
-
const doc =
|
|
115
|
-
|
|
135
|
+
const doc =
|
|
136
|
+
typeof sub.description === "string"
|
|
137
|
+
? ` /** ${sub.description} */\n`
|
|
138
|
+
: "";
|
|
139
|
+
lines.push(
|
|
140
|
+
`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`,
|
|
141
|
+
);
|
|
116
142
|
}
|
|
117
143
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
118
144
|
lines.push(` [key: string]: ${tsType(s.additionalProperties, used)};`);
|
|
@@ -128,9 +154,13 @@ function union(parts: string[]): string {
|
|
|
128
154
|
|
|
129
155
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
130
156
|
const propKey = (k: string) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
131
|
-
const identifier = (n: string) =>
|
|
157
|
+
const identifier = (n: string) =>
|
|
158
|
+
IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
132
159
|
|
|
133
|
-
function deref(
|
|
160
|
+
function deref(
|
|
161
|
+
s: Schema | undefined,
|
|
162
|
+
defs: Record<string, Schema>,
|
|
163
|
+
): Schema | undefined {
|
|
134
164
|
let cur = s;
|
|
135
165
|
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
136
166
|
cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
|
|
@@ -138,22 +168,14 @@ function deref(s: Schema | undefined, defs: Record<string, Schema>): Schema | un
|
|
|
138
168
|
return cur;
|
|
139
169
|
}
|
|
140
170
|
|
|
141
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
142
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
143
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
144
|
-
* the navigation belongs here rather than in genctl. */
|
|
145
|
-
function scriptInput(site: Site, defs: Record<string, Schema>): Schema | undefined {
|
|
146
|
-
const action = deref(site.input, defs);
|
|
147
|
-
const bound = action?.properties?.input;
|
|
148
|
-
return bound ?? site.input;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
171
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
152
172
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
153
|
-
function declarations(
|
|
173
|
+
function declarations(at: Located): string {
|
|
174
|
+
const { site, where } = at;
|
|
175
|
+
const defs = where.$defs ?? {};
|
|
154
176
|
const used = new Set<string>();
|
|
155
|
-
const input = tsType(
|
|
156
|
-
const output = tsType(site.
|
|
177
|
+
const input = tsType(site.types?.Input, used);
|
|
178
|
+
const output = tsType(site.types?.Output, used);
|
|
157
179
|
|
|
158
180
|
const emitted: string[] = [];
|
|
159
181
|
const done = new Set<string>();
|
|
@@ -167,7 +189,7 @@ function declarations(site: Site, defs: Record<string, Schema>): string {
|
|
|
167
189
|
|
|
168
190
|
return [
|
|
169
191
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
170
|
-
`// ${
|
|
192
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
171
193
|
"",
|
|
172
194
|
...emitted,
|
|
173
195
|
emitted.length ? "" : "",
|
|
@@ -189,7 +211,10 @@ function typesPathFor(scriptPath: string): string {
|
|
|
189
211
|
/** The nearest tsconfig above the script — the one the author's editor already reads. Two
|
|
190
212
|
* different configs mean a red editor over a clean apply, or the reverse. The walk stops at
|
|
191
213
|
* the project root: above it is not this project. */
|
|
192
|
-
async function nearestTsconfig(
|
|
214
|
+
async function nearestTsconfig(
|
|
215
|
+
from: string,
|
|
216
|
+
root: string,
|
|
217
|
+
): Promise<string | null> {
|
|
193
218
|
for (let dir = from; ; dir = dirname(dir)) {
|
|
194
219
|
const candidate = join(dir, "tsconfig.json");
|
|
195
220
|
if (await exists(candidate)) return candidate;
|
|
@@ -197,7 +222,7 @@ async function nearestTsconfig(from: string, root: string): Promise<string | nul
|
|
|
197
222
|
}
|
|
198
223
|
}
|
|
199
224
|
|
|
200
|
-
async function typecheck(
|
|
225
|
+
async function typecheck(sites: Located[]): Promise<void> {
|
|
201
226
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
202
227
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
203
228
|
const dir = join(root, ".genroc-cache");
|
|
@@ -205,12 +230,12 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
205
230
|
|
|
206
231
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
207
232
|
// check each script under the other author's options.
|
|
208
|
-
const groups = new Map<string,
|
|
209
|
-
for (const
|
|
210
|
-
const base = (await nearestTsconfig(dirname(
|
|
233
|
+
const groups = new Map<string, Located[]>();
|
|
234
|
+
for (const at of sites) {
|
|
235
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
211
236
|
const group = groups.get(base);
|
|
212
|
-
if (group) group.push(
|
|
213
|
-
else groups.set(base, [
|
|
237
|
+
if (group) group.push(at);
|
|
238
|
+
else groups.set(base, [at]);
|
|
214
239
|
}
|
|
215
240
|
|
|
216
241
|
let n = 0;
|
|
@@ -232,12 +257,18 @@ async function typecheck(root: string, sites: Site[]): Promise<void> {
|
|
|
232
257
|
// base config there is nothing to opt in with, so the default stays none.
|
|
233
258
|
...(base ? {} : { types: [] }),
|
|
234
259
|
},
|
|
235
|
-
files: group.flatMap((s) => [
|
|
260
|
+
files: group.flatMap((s) => [
|
|
261
|
+
relative(dir, s.file),
|
|
262
|
+
relative(dir, typesPathFor(s.file)),
|
|
263
|
+
]),
|
|
236
264
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
237
265
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
238
266
|
include: [],
|
|
239
267
|
};
|
|
240
|
-
const configPath = join(
|
|
268
|
+
const configPath = join(
|
|
269
|
+
dir,
|
|
270
|
+
groups.size === 1 ? "tsconfig.json" : `tsconfig.${n++}.json`,
|
|
271
|
+
);
|
|
241
272
|
await write(configPath, JSON.stringify(config, null, 2));
|
|
242
273
|
await runTsc(root, configPath);
|
|
243
274
|
}
|
|
@@ -285,7 +316,10 @@ const transpile: Plugin = {
|
|
|
285
316
|
},
|
|
286
317
|
};
|
|
287
318
|
|
|
288
|
-
const BUILTIN = new Set([
|
|
319
|
+
const BUILTIN = new Set([
|
|
320
|
+
...builtinModules,
|
|
321
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
322
|
+
]);
|
|
289
323
|
|
|
290
324
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
291
325
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
@@ -296,15 +330,27 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
296
330
|
let options: ts.CompilerOptions = {};
|
|
297
331
|
if (configPath) {
|
|
298
332
|
const read = ts.readConfigFile(configPath, ts.sys.readFile);
|
|
299
|
-
options = ts.parseJsonConfigFileContent(
|
|
333
|
+
options = ts.parseJsonConfigFileContent(
|
|
334
|
+
read.config ?? {},
|
|
335
|
+
ts.sys,
|
|
336
|
+
dirname(configPath),
|
|
337
|
+
).options;
|
|
300
338
|
}
|
|
301
339
|
return {
|
|
302
340
|
name: "genroc-ts-resolve",
|
|
303
341
|
resolveId(source, importer) {
|
|
304
342
|
if (!importer || BUILTIN.has(source)) return null;
|
|
305
|
-
const { resolvedModule } = ts.resolveModuleName(
|
|
306
|
-
|
|
307
|
-
|
|
343
|
+
const { resolvedModule } = ts.resolveModuleName(
|
|
344
|
+
source,
|
|
345
|
+
importer,
|
|
346
|
+
options,
|
|
347
|
+
ts.sys,
|
|
348
|
+
);
|
|
349
|
+
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
350
|
+
return null;
|
|
351
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
352
|
+
? null
|
|
353
|
+
: resolvedModule.resolvedFileName;
|
|
308
354
|
},
|
|
309
355
|
};
|
|
310
356
|
}
|
|
@@ -312,15 +358,16 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
312
358
|
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
313
359
|
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
314
360
|
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
315
|
-
async function bundle(
|
|
361
|
+
async function bundle(at: Located): Promise<string> {
|
|
362
|
+
const site = at.site;
|
|
316
363
|
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
317
364
|
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
318
365
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
319
366
|
const built = await rollup({
|
|
320
|
-
input:
|
|
367
|
+
input: at.file,
|
|
321
368
|
external: (id) => BUILTIN.has(id),
|
|
322
369
|
plugins: [
|
|
323
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
370
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
324
371
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
325
372
|
commonjs(),
|
|
326
373
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -330,17 +377,24 @@ async function bundle(site: Site, root: string): Promise<string> {
|
|
|
330
377
|
],
|
|
331
378
|
onwarn(warning) {
|
|
332
379
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
333
|
-
die(
|
|
380
|
+
die(
|
|
381
|
+
`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`,
|
|
382
|
+
);
|
|
334
383
|
}
|
|
335
384
|
},
|
|
336
|
-
}).catch((e: unknown) =>
|
|
385
|
+
}).catch((e: unknown) =>
|
|
386
|
+
die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`),
|
|
387
|
+
);
|
|
337
388
|
|
|
338
|
-
const { output } = await built.generate({
|
|
389
|
+
const { output } = await built.generate({
|
|
390
|
+
format: "es",
|
|
391
|
+
inlineDynamicImports: true,
|
|
392
|
+
});
|
|
339
393
|
await built.close();
|
|
340
394
|
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
341
395
|
// instance, and the file it names is on this machine.
|
|
342
396
|
if (!output[0].exports.includes("default")) {
|
|
343
|
-
die(`${
|
|
397
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
344
398
|
}
|
|
345
399
|
return output[0].code;
|
|
346
400
|
}
|
|
@@ -355,40 +409,99 @@ const stdin: string = await new Promise((resolve, reject) => {
|
|
|
355
409
|
process.stdin.on("error", reject);
|
|
356
410
|
});
|
|
357
411
|
const manifest = JSON.parse(stdin) as Manifest;
|
|
358
|
-
|
|
412
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
413
|
+
const root = process.cwd();
|
|
414
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
415
|
+
die("stdin is not a genroc resolver manifest");
|
|
416
|
+
|
|
417
|
+
/** Every site with the process it sits in and the file its argument names, in the manifest's own
|
|
418
|
+
* order — which is the order `code` must answer in. genctl passes the argument verbatim, so
|
|
419
|
+
* joining it to the definition's directory is this resolver's business: only it knows the
|
|
420
|
+
* argument is a path at all. */
|
|
421
|
+
type Located = { site: Site; where: ManifestProcess; file: string };
|
|
422
|
+
|
|
423
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
424
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
425
|
+
function address(pointer: (string | number)[]): string {
|
|
426
|
+
return pointer
|
|
427
|
+
.map((seg) =>
|
|
428
|
+
typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
429
|
+
? `.${seg}`
|
|
430
|
+
: `[${JSON.stringify(seg)}]`,
|
|
431
|
+
)
|
|
432
|
+
.join("")
|
|
433
|
+
.replace(/^\./, "");
|
|
434
|
+
}
|
|
435
|
+
const located: Located[] = manifest.processes.flatMap((where) =>
|
|
436
|
+
where.sites.map((site) => ({
|
|
437
|
+
site,
|
|
438
|
+
where,
|
|
439
|
+
file: resolve(where.dir, site.argument),
|
|
440
|
+
})),
|
|
441
|
+
);
|
|
442
|
+
|
|
443
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
444
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
445
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
446
|
+
// and `external` is the same request made directly.
|
|
447
|
+
for (const at of located) {
|
|
448
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
449
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
450
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
451
|
+
// directly.
|
|
452
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
453
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
454
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
455
|
+
die(
|
|
456
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
457
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
458
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`,
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
464
|
+
// does is the one that must say when it is not there.
|
|
465
|
+
for (const at of located) {
|
|
466
|
+
if (!existsSync(at.file)) {
|
|
467
|
+
die(
|
|
468
|
+
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
469
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`,
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
359
473
|
|
|
360
474
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
361
475
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
362
|
-
const byPath = new Map<string,
|
|
363
|
-
for (const
|
|
364
|
-
const seen = byPath.get(
|
|
365
|
-
const defsOf = (x: Site) => (manifest.schemas[x.process]?.$defs ?? {}) as Record<string, Schema>;
|
|
476
|
+
const byPath = new Map<string, Located>();
|
|
477
|
+
for (const at of located) {
|
|
478
|
+
const seen = byPath.get(at.file);
|
|
366
479
|
if (
|
|
367
480
|
seen &&
|
|
368
|
-
JSON.stringify(
|
|
369
|
-
JSON.stringify(scriptInput(site, defsOf(site)))
|
|
481
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)
|
|
370
482
|
) {
|
|
371
483
|
die(
|
|
372
|
-
`${
|
|
484
|
+
`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
485
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
373
486
|
"Split it into two scripts, or make the two call sites pass the same shape.",
|
|
374
487
|
);
|
|
375
488
|
}
|
|
376
|
-
byPath.set(
|
|
489
|
+
byPath.set(at.file, at);
|
|
377
490
|
}
|
|
378
491
|
|
|
379
|
-
for (const
|
|
380
|
-
|
|
381
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
492
|
+
for (const at of byPath.values()) {
|
|
493
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
382
494
|
}
|
|
383
495
|
|
|
384
496
|
if (manifest.mode === "types") {
|
|
497
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
385
498
|
process.exit(0);
|
|
386
499
|
}
|
|
387
500
|
|
|
388
|
-
await typecheck(
|
|
501
|
+
await typecheck([...byPath.values()]);
|
|
389
502
|
|
|
390
503
|
const code: string[] = [];
|
|
391
|
-
for (const
|
|
392
|
-
code.push(await bundle(
|
|
504
|
+
for (const at of located) {
|
|
505
|
+
code.push(await bundle(at));
|
|
393
506
|
}
|
|
394
507
|
process.stdout.write(JSON.stringify({ code }));
|
package/package.json
CHANGED
package/realm.ts
CHANGED
|
@@ -99,8 +99,18 @@ async function run(req: WorkerRequest): Promise<WorkerReply> {
|
|
|
99
99
|
}
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
// The realm's stdio is a pipe to the host thread, and eval.ts terminates this thread the moment
|
|
103
|
+
// the reply lands — so whatever a script wrote last is still in the pipe when it dies. An empty
|
|
104
|
+
// write's callback fires once the queue ahead of it has drained, which makes the reply a barrier
|
|
105
|
+
// for `console` and a direct process.stdout.write alike: both are this stream.
|
|
106
|
+
function flush(stream: NodeJS.WriteStream): Promise<void> {
|
|
107
|
+
return new Promise((resolve) => stream.write("", () => resolve()));
|
|
108
|
+
}
|
|
109
|
+
|
|
102
110
|
// Non-null because this module only ever runs as a worker entry point; a null port here
|
|
103
111
|
// would mean eval.ts loaded it as a plain module, which nothing does.
|
|
104
112
|
parentPort!.on("message", async (req: WorkerRequest) => {
|
|
105
|
-
|
|
113
|
+
const reply = await run(req);
|
|
114
|
+
await Promise.all([flush(process.stdout), flush(process.stderr)]);
|
|
115
|
+
parentPort!.postMessage(reply);
|
|
106
116
|
});
|
package/worker.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
7
|
|
|
8
8
|
import { readFileSync } from "node:fs";
|
|
9
|
-
import { evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
9
|
+
import { Cancelled, evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
10
10
|
|
|
11
11
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
12
12
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
@@ -135,7 +135,12 @@ function asEvalRequest(input: unknown): EvalRequest | string {
|
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
|
|
138
|
+
/** A claim this worker is currently serving. The controller is how the renewal loop reaches
|
|
139
|
+
* the evaluation: genroc cannot call us, so a cancellation arrives as an answer to our own
|
|
140
|
+
* heartbeat and has to be delivered inward from there. */
|
|
141
|
+
type Running = { job: QueueTask; abort: AbortController };
|
|
142
|
+
|
|
143
|
+
const inFlight = new Map<string, Running>();
|
|
139
144
|
let running = true;
|
|
140
145
|
|
|
141
146
|
// Whether the last claim reached genroc. A worker polls several times a second, so an
|
|
@@ -203,7 +208,7 @@ async function answer(token: string, outcome: Record<string, unknown>): Promise<
|
|
|
203
208
|
await release(token);
|
|
204
209
|
}
|
|
205
210
|
|
|
206
|
-
async function run(job: QueueTask): Promise<void> {
|
|
211
|
+
async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
207
212
|
let resolved: unknown;
|
|
208
213
|
try {
|
|
209
214
|
resolved = await resolveObjects(job);
|
|
@@ -224,8 +229,16 @@ async function run(job: QueueTask): Promise<void> {
|
|
|
224
229
|
|
|
225
230
|
let result;
|
|
226
231
|
try {
|
|
227
|
-
result = await evaluate(req);
|
|
232
|
+
result = await evaluate(req, signal);
|
|
228
233
|
} catch (err) {
|
|
234
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
235
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
236
|
+
// waiting out a lease nobody is serving.
|
|
237
|
+
if (err instanceof Cancelled) {
|
|
238
|
+
console.log(`cancelled: ${job.token}`);
|
|
239
|
+
await release(job.token);
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
229
242
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
230
243
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
231
244
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -263,11 +276,22 @@ async function renewLoop(): Promise<void> {
|
|
|
263
276
|
tokens,
|
|
264
277
|
lease_ms: LEASE_MS,
|
|
265
278
|
});
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
279
|
+
if (!ok) continue;
|
|
280
|
+
|
|
281
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
282
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
283
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
284
|
+
// own catch, so nothing is released twice.
|
|
285
|
+
for (const token of (data?.cancelled ?? []) as string[]) {
|
|
286
|
+
inFlight.get(token)?.abort.abort();
|
|
287
|
+
}
|
|
288
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
289
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
290
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
291
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
292
|
+
const lost = (data?.lost ?? []) as string[];
|
|
293
|
+
if (lost.length) {
|
|
294
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
271
295
|
}
|
|
272
296
|
}
|
|
273
297
|
}
|
|
@@ -277,8 +301,9 @@ async function pollLoop(): Promise<void> {
|
|
|
277
301
|
const free = CONCURRENCY - inFlight.size;
|
|
278
302
|
const jobs = free > 0 ? await claim(free) : [];
|
|
279
303
|
for (const job of jobs) {
|
|
280
|
-
|
|
281
|
-
|
|
304
|
+
const abort = new AbortController();
|
|
305
|
+
inFlight.set(job.token, { job, abort });
|
|
306
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
282
307
|
}
|
|
283
308
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
284
309
|
// realms allow, not at the poll interval.
|