@edgehero/pi-dispatch 1.10.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +300 -148
- package/README.md +50 -0
- package/deploy/com.pi-dispatch.worker.plist +9 -3
- package/deploy/docker-compose.yml +49 -16
- package/deploy/egress-proxy.conf +32 -2
- package/deploy/nssm-install.cmd +12 -6
- package/deploy/pi-dispatch-egress-out.network +10 -0
- package/deploy/pi-dispatch-egress-proxy.container +50 -0
- package/deploy/pi-dispatch-netns-keeper.container +80 -0
- package/deploy/pi-dispatch-netns-keeper.network +18 -0
- package/deploy/pi-dispatch-valkey.container +51 -0
- package/deploy/pi-dispatch-valkey.network +16 -0
- package/deploy/receiver.service +6 -0
- package/deploy/worker-env-wrapper.cmd +11 -0
- package/deploy/worker-env-wrapper.sh +60 -34
- package/deploy/worker.service +18 -8
- package/package.json +14 -4
- package/src/azure-host.mjs +19 -0
- package/src/azure-identity.mjs +18 -2
- package/src/backend-conformance.mjs +71 -18
- package/src/backend-local.mjs +637 -21
- package/src/backend-podman.mjs +1168 -0
- package/src/backend-registry.mjs +86 -3
- package/src/backends.mjs +489 -37
- package/src/branch.mjs +7 -2
- package/src/cancel-cli.mjs +174 -0
- package/src/cancel-state.mjs +125 -0
- package/src/cli.mjs +188 -90
- package/src/config.mjs +503 -43
- package/src/connection.mjs +374 -8
- package/src/container-spec.mjs +102 -7
- package/src/daemon-facts.mjs +167 -0
- package/src/deployment-venue.mjs +158 -0
- package/src/docker-run.mjs +146 -15
- package/src/doctor.mjs +4701 -414
- package/src/egress-conf-copy.mjs +166 -0
- package/src/egress-proxy-state.mjs +151 -0
- package/src/egress.mjs +455 -25
- package/src/entry.mjs +27 -0
- package/src/env-allowlist.mjs +222 -40
- package/src/env-file.mjs +1869 -33
- package/src/exit-code.mjs +15 -0
- package/src/flow-gate.mjs +5 -3
- package/src/forgejo-host.mjs +19 -0
- package/src/forgejo-identity.mjs +21 -2
- package/src/get-token.mjs +67 -18
- package/src/git-dirty.mjs +9 -1
- package/src/git-hardening.mjs +33 -0
- package/src/github-app-setup.mjs +29 -12
- package/src/github-prompt.mjs +4 -1
- package/src/gitlab-host.mjs +19 -0
- package/src/gitlab-identity.mjs +19 -2
- package/src/host-registry.mjs +32 -5
- package/src/identity.mjs +29 -4
- package/src/image-preflight.mjs +46 -11
- package/src/image-ref.mjs +21 -0
- package/src/index.mjs +387 -17
- package/src/init.mjs +197 -38
- package/src/job-user.mjs +252 -0
- package/src/json-duplicates.mjs +204 -0
- package/src/live-probes.mjs +1020 -0
- package/src/materialize.mjs +4 -11
- package/src/netns-keeper.mjs +264 -0
- package/src/on-failure.mjs +119 -0
- package/src/outbox.mjs +7 -0
- package/src/podman-stack.mjs +1304 -0
- package/src/prepare-github.mjs +6 -6
- package/src/prepare-local.mjs +51 -17
- package/src/prepare.mjs +27 -6
- package/src/processor.mjs +505 -26
- package/src/provider-key.mjs +41 -0
- package/src/provider-steering.mjs +144 -0
- package/src/queue.mjs +35 -8
- package/src/redact.mjs +84 -0
- package/src/reserved-env.mjs +7 -3
- package/src/retention-sweep.mjs +178 -0
- package/src/run-container.mjs +181 -14
- package/src/run-history.mjs +105 -16
- package/src/runtime-observations.mjs +1152 -0
- package/src/runtime-settings.mjs +13 -8
- package/src/sandbox-cli.mjs +100 -95
- package/src/sandbox-store.mjs +612 -45
- package/src/sandbox.mjs +1459 -37
- package/src/schedules.mjs +16 -3
- package/src/secret-profiles.mjs +2 -1
- package/src/secrets.mjs +23 -6
- package/src/service-env.mjs +247 -0
- package/src/service.mjs +618 -28
- package/src/session-store.mjs +678 -53
- package/src/start.mjs +1395 -268
- package/src/transient.mjs +240 -0
- package/src/triggers-file.mjs +71 -15
- package/src/triggers.mjs +176 -19
- package/src/up.mjs +1399 -85
- package/src/valkey-auth.mjs +529 -0
- package/src/valkey-endpoint.mjs +367 -0
- package/src/watch-closer.mjs +158 -0
package/src/init.mjs
CHANGED
|
@@ -7,12 +7,22 @@
|
|
|
7
7
|
* scoped pauses, an empty packages list stages nothing, an empty subscriptions list declares no plan
|
|
8
8
|
* prices — so a fresh deployment starts inert and is opted into feature by feature.
|
|
9
9
|
*/
|
|
10
|
-
import { existsSync, copyFileSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { existsSync, copyFileSync, lstatSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { fileURLToPath } from "node:url";
|
|
12
12
|
import { join } from "node:path";
|
|
13
|
+
import { parseBackendList, venuesOf } from "./backends.mjs";
|
|
14
|
+
import { deploymentVenueEnv } from "./deployment-venue.mjs";
|
|
15
|
+
import { setEnvKeyIfEmpty } from "./env-file.mjs";
|
|
16
|
+
import { PACKAGED_EGRESS_PROXY_CONF } from "./egress-conf-copy.mjs";
|
|
17
|
+
import { VALKEY_PASSWORD_KEY, newValkeyPassword } from "./valkey-auth.mjs";
|
|
13
18
|
|
|
14
19
|
const EMPTY_TRIGGERS = `${JSON.stringify({ triggers: [] }, null, 2)}\n`;
|
|
15
|
-
|
|
20
|
+
/**
|
|
21
|
+
* EXPORTED so a test scaffolds what `init` scaffolds (issue #384). Doctor now asks the worker's own loader
|
|
22
|
+
* whether a configured file loads, and the fixtures wrote `[]` and `{}`, which those loaders refuse: a test
|
|
23
|
+
* that scaffolds content the product never writes measures the wrong deployment.
|
|
24
|
+
*/
|
|
25
|
+
export const EMPTY_PAUSE_WINDOWS = `${JSON.stringify({ windows: [] }, null, 2)}\n`;
|
|
16
26
|
// Pinned third-party pi packages staged into the global overlay (issue #58). Empty by default: staging
|
|
17
27
|
// runs third-party code inside jobs, so it is opted into package by package, never scaffolded populated.
|
|
18
28
|
const EMPTY_PACKAGES = `${JSON.stringify({ packages: [] }, null, 2)}\n`;
|
|
@@ -23,14 +33,15 @@ const EMPTY_SUBSCRIPTIONS = `${JSON.stringify({ version: 1, subscriptions: [] },
|
|
|
23
33
|
// one-job-per-folder mutex for local jobs is code, not configuration, so it needs no scaffold line.
|
|
24
34
|
// Versioned for the subscriptions reason, sharpened: this is enforcement config, and a silently
|
|
25
35
|
// down-read newer file would be a silently widened spend limit.
|
|
26
|
-
const EMPTY_SCOPED_LIMITS = `${JSON.stringify({ version: 1, limits: [] }, null, 2)}\n`;
|
|
36
|
+
export const EMPTY_SCOPED_LIMITS = `${JSON.stringify({ version: 1, limits: [] }, null, 2)}\n`;
|
|
27
37
|
/**
|
|
28
38
|
* The egress allowlist (REQ-EGRESS-ALLOWLIST): the hosts a job container may reach, one bare hostname per
|
|
29
39
|
* line. Scaffolded with the three a job cannot work without, and NOT empty -- unlike every other scaffold
|
|
30
40
|
* in this file, whose empty form is inert. An empty allowlist is not inert, it is a deployment where every
|
|
31
41
|
* job dies at its first turn, so the safe default here is the working minimum rather than nothing.
|
|
32
42
|
*
|
|
33
|
-
* The provider is an ordinary entry.
|
|
43
|
+
* The provider is an ordinary entry. No address-based rule allows it (the proxy's one address rule only denies this
|
|
44
|
+
* host's loopback and link-local addresses, issue #428), and nothing is special about it: the
|
|
34
45
|
* proxy carries provider traffic like everything else, because the runner's own `fetch` follows the proxy
|
|
35
46
|
* once NODE_USE_ENV_PROXY is set, which the worker sets (worker/src/egress.mjs).
|
|
36
47
|
*/
|
|
@@ -54,56 +65,180 @@ api.anthropic.com
|
|
|
54
65
|
registry.npmjs.org
|
|
55
66
|
`;
|
|
56
67
|
|
|
68
|
+
// The package's own copy of the proxy's rules, which init scaffolds (issue #480). It lives in egress-conf-copy.mjs since
|
|
69
|
+
// issue #484, beside the comparison doctor and `up` make against it, and is re-exported here for init's callers.
|
|
70
|
+
export { PACKAGED_EGRESS_PROXY_CONF };
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* `deps.env` is the caller's environment (the CLI's and doctor's pass theirs), and `deps.venues` a venue set the caller
|
|
74
|
+
* already decided (`up` passes its own, so the two never disagree). Otherwise the venue is decided as `up` decides it
|
|
75
|
+
* (`deploymentVenueEnv`): this shell where it sets a key, else the deployment `.env`, and a disagreement between the two
|
|
76
|
+
* is said instead of any next steps. The default `env` is `{}` and not `process.env` so a test is never steered by the
|
|
77
|
+
* shell it runs in. `deps.steps: false` prints the file list and no "Next:" at all: `up` passes it (issue #480), since
|
|
78
|
+
* `up` performs that ladder itself and printing it mid-pass told a reader to do by hand what was being done for them.
|
|
79
|
+
*/
|
|
57
80
|
export function runInit(cwd = process.cwd(), deps = {}) {
|
|
58
|
-
const { fs = { existsSync, copyFileSync, writeFileSync }, out = (s) => process.stdout.write(s) } = deps;
|
|
81
|
+
const { fs = { existsSync, copyFileSync, lstatSync, mkdirSync, readFileSync, statSync, writeFileSync }, readPackageFile = readFileSync, out = (s) => process.stdout.write(s), env = {}, platform = process.platform, newPassword = newValkeyPassword } = deps;
|
|
59
82
|
const results = [];
|
|
60
|
-
|
|
61
|
-
//
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
83
|
+
// The list prints even when a write throws part way (PR #488's final review: a read-only deploy/ lost the seven lines
|
|
84
|
+
// of what init had already created, leaving one EACCES line), so the operator always sees what now exists.
|
|
85
|
+
const printList = () => {
|
|
86
|
+
// The name column is as wide as the longest name (issue #480): a fixed 20 put egress-allowlist.conf's note out of line.
|
|
87
|
+
const width = Math.max(...results.map(([, name]) => name.length));
|
|
88
|
+
for (const [verb, name, note] of results) {
|
|
89
|
+
out(`${verb.padEnd(7)} ${name.padEnd(width)} ${note}\n`);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
65
92
|
const envPath = join(cwd, ".env");
|
|
66
|
-
|
|
67
|
-
|
|
93
|
+
try {
|
|
94
|
+
// .env from the example. Prefer the copy in cwd (the clone's repo root); fall back to the copy
|
|
95
|
+
// SHIPPED with the worker package (worker/.env.example, kept byte-identical to the root example by
|
|
96
|
+
// worker/test/publish.test.mjs) so init works both from elsewhere in a checkout and from an npm
|
|
97
|
+
// install, where the repo root does not exist.
|
|
98
|
+
if (fs.existsSync(envPath)) {
|
|
99
|
+
results.push(["kept", ".env", KEPT]);
|
|
100
|
+
} else {
|
|
101
|
+
const cwdExample = join(cwd, ".env.example");
|
|
102
|
+
const source = fs.existsSync(cwdExample)
|
|
103
|
+
? cwdExample
|
|
104
|
+
: fileURLToPath(new URL("../.env.example", import.meta.url));
|
|
105
|
+
// Issue #468: the new file carries this deployment's own Valkey password (the example's `# VALKEY_PASSWORD=` line
|
|
106
|
+
// filled in, never shown) and is created readable by this account alone: it holds that password and, soon, the
|
|
107
|
+
// provider key. `wx`: a file that appeared since the check above is never overwritten (init's contract).
|
|
108
|
+
const text = setEnvKeyIfEmpty(String(fs.readFileSync(source, "utf8")), VALKEY_PASSWORD_KEY, newPassword(), { platform });
|
|
109
|
+
if (createOnly(fs, envPath, text, { mode: 0o600 })) results.push(["created", ".env", `from .env.example, mode 0600, with a generated ${VALKEY_PASSWORD_KEY} (value not shown): set your provider key next`]);
|
|
110
|
+
else results.push(["kept", ".env", KEPT]);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
scaffold(fs, results, join(cwd, "triggers.json"), EMPTY_TRIGGERS, "empty triggers list");
|
|
114
|
+
scaffold(fs, results, join(cwd, "pause-windows.json"), EMPTY_PAUSE_WINDOWS, "empty pause-windows list");
|
|
115
|
+
scaffold(fs, results, join(cwd, "pi-packages.json"), EMPTY_PACKAGES, "empty pi package list (stage with import-pi --with-packages)");
|
|
116
|
+
scaffold(fs, results, join(cwd, "subscriptions.json"), EMPTY_SUBSCRIPTIONS, "empty subscription list (declare plan prices for the admin's cost analytics)");
|
|
117
|
+
scaffold(fs, results, join(cwd, "scoped-limits.json"), EMPTY_SCOPED_LIMITS, "empty scoped-limits list (per repo/folder caps; the folder mutex needs no file)");
|
|
118
|
+
scaffold(fs, results, join(cwd, "egress-allowlist.conf"), DEFAULT_EGRESS_ALLOWLIST, "egress allowlist (provider + forge + registry; the egress policy is on unless PI_EGRESS=0)");
|
|
119
|
+
// Issue #480: the proxy's rules, the file beside the allowlist that the docker proxy mounts. Create-only like every
|
|
120
|
+
// scaffold here, so a clone's own deploy/egress-proxy.conf is reported and kept. The one scaffold whose content is
|
|
121
|
+
// not this module's: it is the package's file verbatim, read from the package and never through `fs` (the
|
|
122
|
+
// deployment folder's seam), as `up` reads its Quadlet templates. The podman venue does not read this copy (its unit
|
|
123
|
+
// mounts an account-owned copy `service install` writes), and it costs nothing there.
|
|
124
|
+
//
|
|
125
|
+
// Only into a deploy/ that is a real directory of this folder (PR #488's review): a symlinked deploy/ would put the
|
|
126
|
+
// file wherever the link points, so init refuses there and writes nothing, and says so. Asked again after the mkdir,
|
|
127
|
+
// so a link that appeared in between is refused too.
|
|
128
|
+
const deployDir = join(cwd, "deploy");
|
|
129
|
+
const notOurDir = () => {
|
|
130
|
+
const st = lstatOrNull(fs, deployDir);
|
|
131
|
+
return st && (st.isSymbolicLink() || !st.isDirectory()) ? (st.isSymbolicLink() ? "a symlink" : "not a directory") : null;
|
|
132
|
+
};
|
|
133
|
+
let deployRefusal = notOurDir();
|
|
134
|
+
if (!deployRefusal && !lstatOrNull(fs, deployDir)) {
|
|
135
|
+
fs.mkdirSync(deployDir, { recursive: true });
|
|
136
|
+
deployRefusal = notOurDir();
|
|
137
|
+
}
|
|
138
|
+
if (deployRefusal) {
|
|
139
|
+
results.push(["refused", "deploy/egress-proxy.conf", `deploy/ here is ${deployRefusal}, so init writes nothing into it: make deploy/ a directory of this folder, then run \`pi-dispatch init\` again`]);
|
|
140
|
+
} else {
|
|
141
|
+
scaffold(fs, results, join(deployDir, "egress-proxy.conf"), () => readPackageFile(PACKAGED_EGRESS_PROXY_CONF), "the egress proxy's rules, the package's copy (shipped, not edited; the hosts go in the allowlist)", "deploy/egress-proxy.conf");
|
|
142
|
+
}
|
|
143
|
+
} catch (err) {
|
|
144
|
+
printList();
|
|
145
|
+
throw err;
|
|
146
|
+
}
|
|
147
|
+
printList();
|
|
148
|
+
// A refused scaffold is a failed init (fail loudly): the line above says which, and the steps below still print.
|
|
149
|
+
const code = results.some(([verb]) => verb === "refused") ? 1 : 0;
|
|
150
|
+
if (deps.steps === false) return code;
|
|
151
|
+
if (deps.venues) {
|
|
152
|
+
out(nextSteps(deps.venues, { platform }));
|
|
153
|
+
return code;
|
|
154
|
+
}
|
|
155
|
+
// Decided exactly as `up` decides it (issue #453, gate round 1): a next-steps ladder for a venue `up` would refuse to
|
|
156
|
+
// guess is a ladder for the wrong venue. The files above are scaffolded either way; only the steps wait.
|
|
157
|
+
const venue = deploymentVenueEnv({ env, fs, envPath, platform, command: "init" });
|
|
158
|
+
if (venue.error) {
|
|
159
|
+
out(`\nNext: which venue this deployment runs is unknown, so no steps are shown: ${venue.error}. Then run \`pi-dispatch init\` again for them (it keeps every file above).\n`);
|
|
160
|
+
return code;
|
|
161
|
+
}
|
|
162
|
+
for (const note of venue.notes) out(`⚠ ${note}\n`);
|
|
163
|
+
try {
|
|
164
|
+
parseBackendList(venue.env.PI_BACKENDS);
|
|
165
|
+
} catch (err) {
|
|
166
|
+
out(`\nNext: which venue this deployment runs is unknown, so no steps are shown: ${err.message}. Fix PI_BACKENDS, then run \`pi-dispatch init\` again for them (it keeps every file above).\n`);
|
|
167
|
+
return code;
|
|
168
|
+
}
|
|
169
|
+
out(nextSteps(venuesOf(venue.env), { platform }));
|
|
170
|
+
return code;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const KEPT = "already exists, left untouched";
|
|
174
|
+
|
|
175
|
+
// `name` is what the list shows, the file's own name unless it sits below the folder; `content` may be a function, so a
|
|
176
|
+
// file read from the package is read only when it is about to be written.
|
|
177
|
+
function scaffold(fs, results, path, content, note, name = path.split(/[\\/]/).pop()) {
|
|
178
|
+
// A DIRECTORY where the file belongs (PR #488's review) is not "kept": every reader of these files wants a file, and
|
|
179
|
+
// docker mounts a directory at deploy/egress-proxy.conf where squid reads its config. Followed through a link, too.
|
|
180
|
+
if (isDirectory(fs, path)) {
|
|
181
|
+
results.push(["refused", name, `${name} here is a directory, not a file: remove it, then run \`pi-dispatch init\` again`]);
|
|
182
|
+
} else if (fs.existsSync(path)) {
|
|
183
|
+
results.push(["kept", name, KEPT]);
|
|
184
|
+
} else if (createOnly(fs, path, typeof content === "function" ? content() : content)) {
|
|
185
|
+
results.push(["created", name, note]);
|
|
68
186
|
} else {
|
|
69
|
-
|
|
70
|
-
const source = fs.existsSync(cwdExample)
|
|
71
|
-
? cwdExample
|
|
72
|
-
: fileURLToPath(new URL("../.env.example", import.meta.url));
|
|
73
|
-
fs.copyFileSync(source, envPath);
|
|
74
|
-
results.push(["created", ".env", "from .env.example — set your provider key next"]);
|
|
187
|
+
results.push(["kept", name, KEPT]);
|
|
75
188
|
}
|
|
189
|
+
}
|
|
76
190
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
191
|
+
/**
|
|
192
|
+
* Every scaffold's write (PR #488's review): `wx` (O_CREAT|O_EXCL), so nothing that exists is written through, not even
|
|
193
|
+
* a dangling symlink, which `existsSync` reports absent and a plain write follows out of the folder. EEXIST is "kept",
|
|
194
|
+
* as init's contract says; any other failure is thrown as it was.
|
|
195
|
+
*/
|
|
196
|
+
function createOnly(fs, path, content, opts = {}) {
|
|
197
|
+
try {
|
|
198
|
+
fs.writeFileSync(path, content, { ...opts, flag: "wx" });
|
|
199
|
+
return true;
|
|
200
|
+
} catch (err) {
|
|
201
|
+
if (err?.code === "EEXIST") return false;
|
|
202
|
+
throw err;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
83
205
|
|
|
84
|
-
|
|
85
|
-
|
|
206
|
+
function isDirectory(fs, path) {
|
|
207
|
+
const st = lstatOrNull(fs, path);
|
|
208
|
+
if (!st) return false;
|
|
209
|
+
if (!st.isSymbolicLink()) return st.isDirectory();
|
|
210
|
+
try {
|
|
211
|
+
return typeof fs.statSync === "function" && fs.statSync(path).isDirectory();
|
|
212
|
+
} catch {
|
|
213
|
+
return false;
|
|
86
214
|
}
|
|
87
|
-
out(nextSteps());
|
|
88
|
-
return 0;
|
|
89
215
|
}
|
|
90
216
|
|
|
91
|
-
function
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
results.push(["created", name, note]);
|
|
217
|
+
function lstatOrNull(fs, path) {
|
|
218
|
+
try {
|
|
219
|
+
return fs.lstatSync(path);
|
|
220
|
+
} catch (err) {
|
|
221
|
+
if (err?.code === "ENOENT") return null;
|
|
222
|
+
throw err;
|
|
98
223
|
}
|
|
99
224
|
}
|
|
100
225
|
|
|
101
|
-
|
|
226
|
+
/**
|
|
227
|
+
* The docker text is the same for every set that includes `local`. Its step 2 was a compose command naming
|
|
228
|
+
* deploy/docker-compose.yml, a file a folder made by init alone does not have (issue #480), and it started no egress
|
|
229
|
+
* proxy either; it is now `pi-dispatch up`, which starts Valkey and the proxy itself. A podman-only deployment (issue
|
|
230
|
+
* #453) gets the podman ladder instead, the steps of docs/podman.md "Setup" in the order a fresh folder needs them
|
|
231
|
+
* (named by its URL since #480, because a folder made without a clone has no docs/):
|
|
232
|
+
* the job image into this account's own store, the venue key and provider key in `.env` (what `up` and `service
|
|
233
|
+
* install` read), the stack as Quadlet units, doctor, and the worker as a user service.
|
|
234
|
+
*/
|
|
235
|
+
export function nextSteps(venues = { localUsed: true, podmanUsed: false }, { platform = "linux" } = {}) {
|
|
236
|
+
if (venues.podmanUsed && !venues.localUsed) return platform === "linux" ? PODMAN_NEXT_STEPS : PODMAN_OFF_LINUX;
|
|
102
237
|
return `
|
|
103
238
|
Next:
|
|
104
239
|
1. docker pull ghcr.io/edgehero/pi-job:latest && docker tag ghcr.io/edgehero/pi-job:latest pi-job:latest
|
|
105
|
-
# the prebuilt job image (or build
|
|
106
|
-
2.
|
|
240
|
+
# the prebuilt job image (or build your own from a clone)
|
|
241
|
+
2. pi-dispatch up # Valkey and the egress proxy (unless PI_EGRESS=0)
|
|
107
242
|
3. edit .env # set ANTHROPIC_API_KEY (or your provider's key)
|
|
108
243
|
4. pi-dispatch doctor # verify Docker, Valkey, image, and key
|
|
109
244
|
5. pi-dispatch worker # drain the queue
|
|
@@ -112,3 +247,27 @@ Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /
|
|
|
112
247
|
(or let the panel do all of the above: /dispatch setup walks these steps with a consent per action)
|
|
113
248
|
`;
|
|
114
249
|
}
|
|
250
|
+
|
|
251
|
+
// Off Linux the podman venue refuses the host (`podman-platform`: Podman machine on macOS and Windows was never
|
|
252
|
+
// measured), so its ladder would walk an operator into Quadlet steps that cannot work here.
|
|
253
|
+
const PODMAN_OFF_LINUX = `
|
|
254
|
+
Next: PI_BACKENDS lists only the podman venue, which runs on Linux alone: on this host the worker refuses it
|
|
255
|
+
(podman-platform), so there are no podman steps to run here. Run this deployment on a Linux host, or add \`local\` to
|
|
256
|
+
PI_BACKENDS to run jobs on Docker here (then run \`pi-dispatch init\` again for those steps).
|
|
257
|
+
`;
|
|
258
|
+
|
|
259
|
+
const PODMAN_NEXT_STEPS = `
|
|
260
|
+
Next (the podman venue; run these as the worker's own account. First set the account up as the Podman guide's "Setup"
|
|
261
|
+
steps 1-4 say (https://github.com/edgehero/pi-dispatch/blob/main/docs/podman.md), linger included:
|
|
262
|
+
sudo loginctl enable-linger <account>, without which a job gets no bounds):
|
|
263
|
+
1. podman pull ghcr.io/edgehero/pi-job:latest && podman tag ghcr.io/edgehero/pi-job:latest pi-job:latest
|
|
264
|
+
# the prebuilt job image, in this account's own store
|
|
265
|
+
2. edit .env # PI_BACKENDS=podman, and ANTHROPIC_API_KEY (or your provider's key)
|
|
266
|
+
3. pi-dispatch up # Valkey and the egress proxy as Quadlet units in your user manager
|
|
267
|
+
4. pi-dispatch doctor # verify Podman, Valkey, image, and key
|
|
268
|
+
5. pi-dispatch service install # the worker as a user service, after those units (also installs them)
|
|
269
|
+
6. pi-dispatch doctor --live # read the bounds, egress and job user back off real containers
|
|
270
|
+
|
|
271
|
+
Operator panel (optional): pi install npm:@edgehero/pi-dispatch-admin then /dispatch
|
|
272
|
+
(or let the panel do all of the above: /dispatch setup walks these steps with a consent per action)
|
|
273
|
+
`;
|
package/src/job-user.mjs
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WHICH uid a job container runs as, decided from facts rather than probed (issue #341,
|
|
3
|
+
* `DES-JOB-USER-INFERRED-READ-BACK-ON-REQUEST`).
|
|
4
|
+
*
|
|
5
|
+
* On a daemon that enforces bind-mount ownership, a job can read its 0700 job dir, write its mounts and leave files
|
|
6
|
+
* the host can remove ONLY as the uid that owns them. The image runs as uid 1001, the worker as whoever started it,
|
|
7
|
+
* and those differ on almost every native Linux host: measured, every job then fails. Docker Desktop maps ownership,
|
|
8
|
+
* so there the image's own user works and nothing may change.
|
|
9
|
+
*
|
|
10
|
+
* Decided WITHOUT starting a container. `CONST-ISOLATION-CONTAINER-PER-JOB` rejects "probing at worker boot or per
|
|
11
|
+
* job", so the decision reads what is already cheap to ask: the platform, the worker's own ids, the endpoint the
|
|
12
|
+
* docker CLI resolves, one `docker info`, and the owner of a local socket. A wrong inference is not silent: the
|
|
13
|
+
* runner refuses a job whose inputs it cannot read before any spend (`job-inputs-unreadable`), and `pi-dispatch
|
|
14
|
+
* doctor --live` reads the mounts back on request.
|
|
15
|
+
*
|
|
16
|
+
* Pure parts (`parseDaemonFacts`, `decideJobUser`, `resolveImageUser`) take values, apart from `decideJobUser`'s
|
|
17
|
+
* `os.release()` default; the two readers take seams.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { statSync } from "node:fs";
|
|
21
|
+
import { release as osRelease } from "node:os";
|
|
22
|
+
import { execDockerBounded } from "./backend-local.mjs";
|
|
23
|
+
import { CONTAINER_HOME, SHIPPED_IMAGE_UID } from "./container-spec.mjs";
|
|
24
|
+
import { DAEMON_FACTS_ARGS, DAEMON_FACTS_MAX_BUFFER, DAEMON_FACTS_TIMEOUT_MS, PODMAN_PRODUCT_LICENSE, displayVersion, parseDaemonFacts } from "./daemon-facts.mjs";
|
|
25
|
+
|
|
26
|
+
// Moved to the leaf `daemon-facts.mjs` (issue #452, gate round 3) and re-exported, so every importer keeps its path.
|
|
27
|
+
export { DAEMON_FACTS_ARGS, DAEMON_FACTS_TIMEOUT_MS, PODMAN_PRODUCT_LICENSE, displayVersion, parseDaemonFacts };
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Whether a job's OWN bind mounts carry `:Z` (issue #355): `true` only for a Podman daemon that reports SELinux, reached
|
|
31
|
+
* through an endpoint on this host, from a Linux worker. Measured on an enforcing Fedora 44 host (rootful Podman 5.8.1,
|
|
32
|
+
* container-selinux 2.247.0): every unlabelled bind source is denied to the container, `ls` included, with or without
|
|
33
|
+
* `:ro`, so every job on the documented Podman route failed at `/job` before it spent anything.
|
|
34
|
+
*
|
|
35
|
+
* ONE function, read by the job path, doctor and the sandbox, so the three cannot disagree about when a mount is
|
|
36
|
+
* relabelled. Each condition is a measured boundary rather than caution:
|
|
37
|
+
* - Podman only. Docker Engine with `selinux-enabled` reports the same `name=selinux` and was not measured, so its
|
|
38
|
+
* argv stays exactly what it was (out of scope, named in the design entry).
|
|
39
|
+
* - `selinux === true` only: `null` (a shape that did not say) is not a reason to relabel anything.
|
|
40
|
+
* - `endpoint.local === true` only: bind sources are the DAEMON's paths, and on another machine this worker's job
|
|
41
|
+
* directories are not what gets relabelled.
|
|
42
|
+
* - a Linux worker only: a Podman machine on macOS or Windows reports its Linux VM's SELinux while the bind sources are
|
|
43
|
+
* the host's own files shared into that VM, a route nobody measured with `:Z`.
|
|
44
|
+
*
|
|
45
|
+
* WHICH mounts is `containerSpec`'s business, not this function's: only the directories the worker creates per job.
|
|
46
|
+
* `:Z` is PRIVATE (a per-container MCS pair), which is exactly right for a directory one container ever sees, and
|
|
47
|
+
* measured to lock every other container out of what it relabels. So an operator's local folder and the shared global
|
|
48
|
+
* overlay are NEVER relabelled; doctor names the `semanage fcontext` fix for those instead.
|
|
49
|
+
*/
|
|
50
|
+
export function relabelsPrivateMounts(facts, endpoint, platform = process.platform) {
|
|
51
|
+
return platform === "linux" && facts?.podman === true && facts?.selinux === true && endpoint?.local === true;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* `async () => ({ answered: true, facts } | { answered: false, reason, transient })`.
|
|
56
|
+
*
|
|
57
|
+
* A clean exit that parses to neither shape is DETERMINATE (`unparseable`): the CLI answered and said something no
|
|
58
|
+
* rule can read. Every other failure is TRANSIENT here, on purpose and unlike the endpoint read: a clean exit whose
|
|
59
|
+
* body says no daemon answered (`daemon-unreachable`, what a daemon down or still starting gives), a non-zero exit, a
|
|
60
|
+
* timeout, a signal, no docker binary. A worker unit with `RestartPreventExitStatus=2` must never be stranded by a
|
|
61
|
+
* daemon that is merely late. No CLI text is ever read or logged.
|
|
62
|
+
*/
|
|
63
|
+
export function makeDaemonFactsReader({ run = (args) => execDockerBounded(args, { timeoutMs: DAEMON_FACTS_TIMEOUT_MS, maxBuffer: DAEMON_FACTS_MAX_BUFFER }) } = {}) {
|
|
64
|
+
return async function readDaemonFacts() {
|
|
65
|
+
let result;
|
|
66
|
+
try {
|
|
67
|
+
result = await run(DAEMON_FACTS_ARGS);
|
|
68
|
+
} catch (err) {
|
|
69
|
+
result = { code: null, stdout: "", error: err };
|
|
70
|
+
}
|
|
71
|
+
if (result?.error || result?.code !== 0) {
|
|
72
|
+
const error = result?.error;
|
|
73
|
+
const reason = error?.timedOut || error?.killed ? "timeout"
|
|
74
|
+
: typeof error?.signal === "string" ? `signal-${error.signal.toLowerCase()}`
|
|
75
|
+
: error?.code === "ENOENT" ? "docker-not-found"
|
|
76
|
+
: typeof result?.code === "number" ? `exit-${result.code}`
|
|
77
|
+
: typeof error?.code === "number" ? `exit-${error.code}`
|
|
78
|
+
: "spawn-failed";
|
|
79
|
+
return { answered: false, reason, transient: true };
|
|
80
|
+
}
|
|
81
|
+
const parsed = parseDaemonFacts(result.stdout);
|
|
82
|
+
if (parsed?.unreachable) return { answered: false, reason: "daemon-unreachable", transient: true };
|
|
83
|
+
return parsed ? { answered: true, facts: parsed.facts } : { answered: false, reason: "unparseable", transient: false };
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* `{ uid, gid }` of a local unix socket, or `null`. Takes the docker endpoint's `unix://` path or Podman's
|
|
89
|
+
* `remoteSocket.path`, which is a bare path when the service is local and `unix://...` when it is remote (both
|
|
90
|
+
* measured). `stat`, never `lstat`: `/var/run/docker.sock` is a symlink on Docker Desktop, and a link pointing at a
|
|
91
|
+
* rootless socket would read as root-owned. Any failure (EACCES on a 0700 parent, ENOENT) is simply no fact.
|
|
92
|
+
*/
|
|
93
|
+
export function socketFacts(path, { stat = statSync } = {}) {
|
|
94
|
+
if (typeof path !== "string") return null;
|
|
95
|
+
const bare = path.startsWith("unix://") ? path.slice("unix://".length) : path;
|
|
96
|
+
if (!bare.startsWith("/")) return null;
|
|
97
|
+
try {
|
|
98
|
+
const s = stat(bare);
|
|
99
|
+
return typeof s?.uid === "number" && typeof s?.gid === "number" ? { uid: s.uid, gid: s.gid } : null;
|
|
100
|
+
} catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Is a Podman `remoteSocket.path` a unix socket (bare path or `unix://`), i.e. a local service? */
|
|
106
|
+
// The fixed operator texts, one per cause, for the boot refusal, the sandbox and doctor. The forge comments keep
|
|
107
|
+
// their own shorter texts (a comment's reader may not be the operator), and no text carries CLI output, an endpoint
|
|
108
|
+
// or a path: a refusal that repeats what docker printed can repeat a credential.
|
|
109
|
+
export const JOB_USER_FIX = Object.freeze({
|
|
110
|
+
rootless: "the container runtime runs rootless (a user namespace between the job and this worker), so no uid a job may run as can read the worker's 0700 job dir; run jobs on a rootful Docker or Podman daemon. If this was inferred from a socket this worker's uid owns on a rootful daemon (a systemd SocketUser= override), point the worker at the daemon's own socket",
|
|
111
|
+
"userns-remap": "the Docker daemon remaps container uids (userns-remap), so no uid a job may run as can read the worker's 0700 job dir; run jobs on a daemon without userns-remap",
|
|
112
|
+
"worker-is-root": "the worker runs as root, and a job must not run as root (nonRoot); run the worker as an unprivileged account, as deploy/worker.service's User= does",
|
|
113
|
+
"desktop-linux-userns": "Docker Desktop on Linux maps container uids like a rootless daemon, so no uid a job may run as can read the worker's 0700 job dir; use Docker Engine on this host (WSL2 is not affected)",
|
|
114
|
+
"runtime-unreadable": "the docker CLI answered `docker info` with something no rule can read, so which uid a job may run as is unknown; point the real docker CLI at a Docker or Podman daemon",
|
|
115
|
+
"root-group": "the worker's primary group is gid 0, and a job runs with that group; run the worker with an unprivileged primary group",
|
|
116
|
+
"docker-group": "the worker's primary group is the docker socket's group, and a job runs with that group; make docker a supplementary group (log out and back in rather than `newgrp docker`)",
|
|
117
|
+
// Not a daemon cause: the per-image rule's refusal (`job-image-any-uid-unsupported`), here so every surface shares one text.
|
|
118
|
+
"any-uid-unsupported": "the job image does not declare `anyUid` (`dev.pi-dispatch.capabilities`), so it cannot run as this worker's own uid, which this host's container runtime requires; rebuild it from a release that has this feature, or run the worker as uid 1001",
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The daemon half of the decision. Pure apart from the `release` default. Returns `{ mode, user, cause, reason }`,
|
|
123
|
+
* one of FOUR modes:
|
|
124
|
+
* - `image`: the image's own USER runs, argv byte-identical to before issue #341;
|
|
125
|
+
* - `worker`: the job runs as `user`, the worker's own "<euid>:<egid>" (the per-image rule may still decline);
|
|
126
|
+
* - `unmappable`: no uid works, `cause` names why;
|
|
127
|
+
* - `unknown`: not decidable now (`reason`), never cached and never a boot exit.
|
|
128
|
+
*
|
|
129
|
+
* `endpoint` is the endpoint resolver's answer; `daemon` is `readDaemonFacts()`'s; `socket` is `socketFacts(...)`.
|
|
130
|
+
* The rows are ORDERED, and the order is part of the contract (the design entry's table).
|
|
131
|
+
*/
|
|
132
|
+
export function decideJobUser({ platform, release = osRelease(), euid, egid, endpoint, daemon, socket = null }) {
|
|
133
|
+
const image = (cause) => ({ mode: "image", user: null, cause, reason: null });
|
|
134
|
+
const unmappable = (cause) => ({ mode: "unmappable", user: null, cause, reason: null });
|
|
135
|
+
const unknown = (reason) => ({ mode: "unknown", user: null, cause: null, reason });
|
|
136
|
+
|
|
137
|
+
// A daemon on these platforms runs in a Linux VM, and Docker Desktop's file sharing maps ownership (measured), so the
|
|
138
|
+
// image's own user works. The other VM-backed daemons there (OrbStack, Colima, Podman machine) are unmeasured.
|
|
139
|
+
if (platform === "darwin" || platform === "win32") return image("desktop-platform");
|
|
140
|
+
// Bind-mount sources are another machine's paths: nothing about this host's uids applies, and doctor already warns.
|
|
141
|
+
if (endpoint?.local === false) return image("endpoint-not-local");
|
|
142
|
+
if (!daemon?.answered) return daemon?.transient === false ? unmappable("runtime-unreadable") : unknown(daemon?.reason ?? "no-daemon-facts");
|
|
143
|
+
const facts = daemon.facts;
|
|
144
|
+
if (endpoint?.local !== true) {
|
|
145
|
+
// No docker endpoint resolved. Only Podman's own shape (its docker emulation) says enough to go on; a Docker
|
|
146
|
+
// shape with no endpoint leaves the socket rows below blind.
|
|
147
|
+
if (facts.shape !== "podman") return unknown("endpoint-unresolved");
|
|
148
|
+
// A client of a service that listens on TCP: its path did not survive parsing, so there is no socket on this host
|
|
149
|
+
// to read. A client reaching a unix-socket service over ssh reports that service's own unix path and is NOT
|
|
150
|
+
// caught here (a residual the design entry names).
|
|
151
|
+
if (facts.serviceIsRemote === true && facts.remoteSocketPath === null) return image("endpoint-not-local");
|
|
152
|
+
}
|
|
153
|
+
if (facts.os === "Docker Desktop") {
|
|
154
|
+
if (!/microsoft/i.test(String(release))) return unmappable("desktop-linux-userns");
|
|
155
|
+
// WSL2 bind mounts keep uids (unmeasured): the ordinary rows decide.
|
|
156
|
+
}
|
|
157
|
+
if (facts.rootless === true) return unmappable("rootless");
|
|
158
|
+
if (facts.userns === true) return unmappable("userns-remap");
|
|
159
|
+
if (typeof euid !== "number" || typeof egid !== "number") return unknown("no-process-ids");
|
|
160
|
+
// A rootless daemon's socket belongs to the user running it. Needed for Podman before its compat info carried
|
|
161
|
+
// name=rootless (absent at v4.3.1, present at v4.9.3); narrowed to THIS uid's socket, never any non-root owner.
|
|
162
|
+
if (euid !== 0 && socket && socket.uid === euid) return unmappable("rootless");
|
|
163
|
+
if (euid === 0) return unmappable("worker-is-root");
|
|
164
|
+
return { mode: "worker", user: `${euid}:${egid}`, cause: null, reason: null };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The per-image half, for a job about to run. Returns `{ user, home }` (`user` null means the image's own USER),
|
|
169
|
+
* `{ refused, cause }` or `{ unavailable, reason }`.
|
|
170
|
+
*
|
|
171
|
+
* ORDER MATTERS. uid 1001 comes first: the image already runs as it, so no `--user` and no group rule, whatever the
|
|
172
|
+
* image declares; a uid-1001 worker whose primary group is docker works today and must keep working once the shipped
|
|
173
|
+
* image carries `anyUid`. Only a `--user` puts the worker's gid into the container, so the two group refusals live on
|
|
174
|
+
* that path alone.
|
|
175
|
+
*/
|
|
176
|
+
export function resolveImageUser(decision, { capabilities = [], euid, egid, socket = null } = {}) {
|
|
177
|
+
if (decision?.mode === "image") return { user: null, home: null };
|
|
178
|
+
if (decision?.mode === "unmappable") return { refused: "job-user-unmappable", cause: decision.cause };
|
|
179
|
+
if (decision?.mode !== "worker") return { unavailable: true, reason: decision?.reason ?? "unknown" };
|
|
180
|
+
if (euid === SHIPPED_IMAGE_UID) return { user: null, home: null };
|
|
181
|
+
if (!Array.isArray(capabilities) || !capabilities.includes("anyUid")) return { refused: "job-image-any-uid-unsupported", cause: "any-uid-unsupported" };
|
|
182
|
+
if (egid === 0) return { refused: "job-user-unmappable", cause: "root-group" };
|
|
183
|
+
if (socket && egid === socket.gid) return { refused: "job-user-unmappable", cause: "docker-group" };
|
|
184
|
+
return { user: decision.user, home: CONTAINER_HOME };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The causes that stop a worker whose default venue is `local` from booting: facts about the daemon or the worker's
|
|
189
|
+
* own identity, which no job on that venue can get past. ONE set, read by the worker's boot and by doctor's severity,
|
|
190
|
+
* so doctor cannot show a warning for a cause the worker refuses to boot on. The rest refuse per job: the group rows
|
|
191
|
+
* apply only to a job whose image needs `--user`, and `runtime-unreadable` describes one answer.
|
|
192
|
+
*/
|
|
193
|
+
// A plain Set (read with `.has` only); `Object.freeze` would not stop `.add`, so it is not pretended.
|
|
194
|
+
export const BOOT_REFUSING_JOB_USER_CAUSES = new Set(["rootless", "userns-remap", "worker-is-root", "desktop-linux-userns"]);
|
|
195
|
+
|
|
196
|
+
/** The operator-facing refusal for an unmappable decision or cause. */
|
|
197
|
+
export function jobUserRefusal(causeOrDecision) {
|
|
198
|
+
const cause = typeof causeOrDecision === "string" ? causeOrDecision : causeOrDecision?.cause;
|
|
199
|
+
return `Refused: ${JOB_USER_FIX[cause] ?? "the job user could not be decided"} (issue #341).`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* `async ({ endpoint, key }) => ({ decision, facts, daemon, socket })`, cached by `key` (the endpoint state string the
|
|
204
|
+
* caller already keeps). Concurrent callers for one key share one read. `daemon` is the raw read result (`{ answered,
|
|
205
|
+
* facts }` or `{ answered: false, reason, transient }`), which the runtime observations read (issue #345).
|
|
206
|
+
*
|
|
207
|
+
* NOT cached: `unknown`, `unmappable` `runtime-unreadable`, and any decision made without an answered read (an `image`
|
|
208
|
+
* decided on macOS while the daemon was still starting). Each describes an answer rather than a daemon, and a cached one
|
|
209
|
+
* would retry or refuse every later job on that endpoint, or leave the observations unread, until the worker restarted.
|
|
210
|
+
* A cached decision is otherwise kept until the endpoint state changes: a daemon reconfigured behind an unchanged
|
|
211
|
+
* endpoint (rootful to rootless on one socket path) is read again only after a restart, a residual the design entry names.
|
|
212
|
+
*/
|
|
213
|
+
export function makeJobUserResolver({
|
|
214
|
+
readFacts,
|
|
215
|
+
platform = process.platform,
|
|
216
|
+
release = osRelease(),
|
|
217
|
+
euid = process.geteuid?.(),
|
|
218
|
+
egid = process.getegid?.(),
|
|
219
|
+
stat = statSync,
|
|
220
|
+
} = {}) {
|
|
221
|
+
let cached = null;
|
|
222
|
+
const inFlight = new Map();
|
|
223
|
+
return resolveJobUser;
|
|
224
|
+
async function resolveJobUser({ endpoint, key }) {
|
|
225
|
+
if (cached && cached.key === key) return cached.value;
|
|
226
|
+
if (inFlight.has(key)) return inFlight.get(key);
|
|
227
|
+
const work = (async () => {
|
|
228
|
+
// Asked on EVERY platform and endpoint, although a VM-backed platform or an endpoint on another machine decides
|
|
229
|
+
// `image` before any daemon fact is read: the same read is where the runtime observations come from (issue
|
|
230
|
+
// #345), and a Docker Desktop host skipped here would never be credited with the bounds its daemon applies.
|
|
231
|
+
const daemon = await readFacts();
|
|
232
|
+
// A local docker endpoint's display form IS its unix path (credentials never ride a unix URL); with no endpoint,
|
|
233
|
+
// Podman's own shape names the service socket.
|
|
234
|
+
const socketPath = endpoint?.local === true && typeof endpoint.endpoint === "string" && endpoint.endpoint.startsWith("unix://")
|
|
235
|
+
? endpoint.endpoint
|
|
236
|
+
: daemon?.answered ? daemon.facts.remoteSocketPath : null;
|
|
237
|
+
const socket = socketFacts(socketPath, { stat });
|
|
238
|
+
const decision = decideJobUser({ platform, release, euid, egid, endpoint, daemon, socket });
|
|
239
|
+
const value = { decision, facts: daemon?.answered ? daemon.facts : null, daemon, socket };
|
|
240
|
+
// Cached only for an ANSWERED read: an `image` decided on a VM-backed platform while its daemon was still starting
|
|
241
|
+
// must not pin "not read" for the runtime observations until the endpoint changes.
|
|
242
|
+
if (decision.mode !== "unknown" && decision.cause !== "runtime-unreadable" && daemon?.answered === true) cached = { key, value };
|
|
243
|
+
return value;
|
|
244
|
+
})();
|
|
245
|
+
inFlight.set(key, work);
|
|
246
|
+
try {
|
|
247
|
+
return await work;
|
|
248
|
+
} finally {
|
|
249
|
+
inFlight.delete(key);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|