@edgehero/pi-dispatch 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +94 -4
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +199 -30
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1663 -198
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +103 -9
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-auth.mjs +4 -0
- package/src/forgejo-host.mjs +127 -31
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +73 -15
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +740 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1554 -167
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +296 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +782 -40
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/service.mjs
CHANGED
|
@@ -23,6 +23,9 @@
|
|
|
23
23
|
* package's own exported entry, wherever npm (or the workspace symlink) put it.
|
|
24
24
|
* null when the package is not installed — receiver renders refuse loudly instead
|
|
25
25
|
* of writing a unit that would crash-loop at boot.
|
|
26
|
+
* - consoleEntry the admin package's bin, dist/console.mjs (issue #708, resolveConsoleEntry): the
|
|
27
|
+
* same resolution, else the `pi-dispatch-console` a global install put on PATH. null
|
|
28
|
+
* refuses the console renders, for the receiver's reason.
|
|
26
29
|
*
|
|
27
30
|
* Scope doctrine:
|
|
28
31
|
* - User-level by default, everywhere. macOS REFUSES root outright (a LaunchAgent is per-user, and a
|
|
@@ -36,24 +39,31 @@
|
|
|
36
39
|
* from the cross-scope check (a second receiver is pointless, not destructive) but still refuse
|
|
37
40
|
* same-scope duplicates.
|
|
38
41
|
*
|
|
39
|
-
* `restart --drain` composes the README's manual ritual (pause → poll active → restart → resume)
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
+
* `restart --drain` composes the README's manual ritual (pause → poll active → restart → resume) over this
|
|
43
|
+
* host's queues (issue #804): the shared one and, when the deployment declares PI_WORKER_NAME, the host's own,
|
|
44
|
+
* which is where its cron, chained and manual runs wait. It reads VALKEY_URL and PI_WORKER_NAME alone, as cli.mjs's
|
|
45
|
+
* verbs read them, so the drain works even when the rest of the config is broken. It is not the fleet's pause: a
|
|
46
|
+
* peer's own queue is neither paused nor waited for.
|
|
42
47
|
*/
|
|
43
48
|
import { spawn as nodeSpawn } from "node:child_process";
|
|
44
|
-
import { existsSync, mkdirSync, readFileSync, realpathSync, unlinkSync, writeFileSync } from "node:fs";
|
|
49
|
+
import { accessSync, constants as fsConstants, existsSync, mkdirSync, readFileSync, realpathSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
45
50
|
import { lookup as dnsLookup } from "node:dns/promises";
|
|
46
51
|
import { connect as netConnect } from "node:net";
|
|
47
52
|
import { homedir, networkInterfaces, tmpdir, userInfo } from "node:os";
|
|
48
|
-
import { dirname, join, resolve } from "node:path";
|
|
53
|
+
import { basename, delimiter, dirname, join, resolve } from "node:path";
|
|
49
54
|
import { fileURLToPath } from "node:url";
|
|
50
55
|
import { parseArgs } from "node:util";
|
|
51
|
-
import { parseBackendList, venuesOf } from "./backends.mjs";
|
|
56
|
+
import { openshellSetupOf, parseBackendList, venuesOf } from "./backends.mjs";
|
|
57
|
+
import { OPENSHELL_BREW_INSTALL, OPENSHELL_GATEWAY_UNIT, OPENSHELL_NO_WINDOWS } from "./openshell-cli.mjs";
|
|
52
58
|
import { sharedShellIgnored } from "./deployment-venue.mjs";
|
|
53
59
|
import { egressArmed, egressProxyName } from "./egress.mjs";
|
|
54
60
|
import { ENV_WRITER_FS, updateEnvFile } from "./env-file.mjs";
|
|
61
|
+
import { operatorCount, readTokensFile, tokensFilePath } from "./console-tokens.mjs";
|
|
62
|
+
import { resolveSetupWorkerEntry, setupCachePath } from "./setup-host-entry.mjs";
|
|
63
|
+
import { cliDeploymentEnv, deploymentServiceEnv } from "./service-env.mjs";
|
|
64
|
+
import * as nodeFs from "node:fs";
|
|
55
65
|
import { MODEL_ENDPOINTS_INCLUDE_NAME } from "./model-endpoints.mjs";
|
|
56
|
-
import { VALKEY_HEALTH_SCRIPT, VALKEY_PASSWORD_KEY, VALKEY_START_SCRIPT, dollarsDoubled, newValkeyPassword, valkeyPasswordDecision } from "./valkey-auth.mjs";
|
|
66
|
+
import { VALKEY_HEALTH_SCRIPT, VALKEY_IMAGE, VALKEY_PASSWORD_KEY, VALKEY_START_SCRIPT, dollarsDoubled, newValkeyPassword, valkeyPasswordDecision } from "./valkey-auth.mjs";
|
|
57
67
|
import { ALL_QUADLET_FILES, ALLOWLIST_PLACEHOLDER, MODEL_ENDPOINTS_PLACEHOLDER, NETNS_KEEPER, NETNS_KEEPER_FORMAT, judgeNetnsKeeper, keeperUnderRunningProxyHint, managerEnvRefusal, PROXY_CONF_PLACEHOLDER, QUADLET_FILES, applyStack, decideValkey, describeAction, passwdNameFrom, readSubuidRanges, readValkeyKeys, valkeySharedOn, VALKEY_SHARED_KEY, describeRollBack, journalWrite, rollBackWrites, foreignContainerRefusal, foreignContainers, lingerNote, planStack, proxyConfCopyPath, proxyRestartWarning, quadletDir, readLinger, readStackKeys, stackComponents, unknownContainerRefusal, userBusRefusal, valkeyEnvPath, valkeyPasswordRestartWarning, workerUnitDeps } from "./podman-stack.mjs";
|
|
58
68
|
|
|
59
69
|
/** This command's default fs seam: its own calls, and every call the `.env` writer makes (`ENV_WRITER_FS`, issue #522). */
|
|
@@ -80,6 +90,70 @@ function resolveReceiverStart() {
|
|
|
80
90
|
}
|
|
81
91
|
}
|
|
82
92
|
|
|
93
|
+
/** The admin package, whose bin is the console (`pi-dispatch-console` -> ./dist/console.mjs, in its published `files`). */
|
|
94
|
+
export const CONSOLE_PACKAGE = "@edgehero/pi-dispatch-admin";
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The console's entry point (issue #708): the admin package's dist/console.mjs as module resolution finds it from here,
|
|
98
|
+
* else the real path of the `pi-dispatch-console` on this shell's PATH. null when neither exists.
|
|
99
|
+
*
|
|
100
|
+
* Two lookups because the admin package is NOT a dependency of this one: it is a pi extension an operator installs
|
|
101
|
+
* with `pi install` or `npm i -g`. Resolution from here finds it when it sits in the same node_modules tree as this
|
|
102
|
+
* worker (both global, or both in the deployment folder); the PATH lookup covers the rest. The package has no
|
|
103
|
+
* `exports` map, so the subpath resolves as a plain file, and `import.meta.resolve` does not check that the file
|
|
104
|
+
* exists (measured on node 23.5: a checkout without `node admin/build.mjs` resolves to a dist/ that is not there),
|
|
105
|
+
* hence the existence check.
|
|
106
|
+
*
|
|
107
|
+
* The PATH hit is accepted only when its real path IS the package's file: it ends in dist/console.mjs and the
|
|
108
|
+
* package.json two levels up names this package (round 1 review of #708). npm's bin is a symlink to exactly that file,
|
|
109
|
+
* but pnpm and Volta put a SHIM on PATH (a shell script, or a binary that picks a version at run time), and a unit that
|
|
110
|
+
* ran `node <shim>` would crash at every start. The real path, not the PATH name, because a service manager's PATH is
|
|
111
|
+
* not this shell's and launchd's cannot see an nvm or Homebrew prefix at all. `findOnPath` reads only absolute POSIX
|
|
112
|
+
* directories, so on Windows this lookup does not run and resolution is the only way (`refuseMissingConsole` says so).
|
|
113
|
+
*/
|
|
114
|
+
export function resolveConsoleEntry({ findBin, realpath, exists = existsSync, readFile = (p) => readFileSync(p, "utf8"), resolveModule = (specifier) => import.meta.resolve(specifier) }) {
|
|
115
|
+
try {
|
|
116
|
+
const resolved = fileURLToPath(resolveModule(`${CONSOLE_PACKAGE}/dist/console.mjs`));
|
|
117
|
+
if (exists(resolved)) return resolved;
|
|
118
|
+
} catch {}
|
|
119
|
+
const found = findBin("pi-dispatch-console");
|
|
120
|
+
if (!found) return null;
|
|
121
|
+
try {
|
|
122
|
+
const real = realpath(found);
|
|
123
|
+
const dist = dirname(real);
|
|
124
|
+
if (basename(real) !== "console.mjs" || basename(dist) !== "dist") return null;
|
|
125
|
+
const pkg = JSON.parse(readFile(join(dirname(dist), "package.json")));
|
|
126
|
+
return pkg?.name === CONSOLE_PACKAGE ? real : null;
|
|
127
|
+
} catch {
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The console tokens file the SERVICE's console reads for the deployment in `deployDir` (issue #708): PI_CONSOLE_TOKENS_FILE
|
|
134
|
+
* from the deployment `.env` (through the reader the console itself uses), else console-tokens.json there. Never this
|
|
135
|
+
* shell's export: the unit inherits no shell, loads no EnvironmentFile and its wrapper sources no `.env`, so `serve`
|
|
136
|
+
* finds only the file's value. One helper for `service install --console` and `up`'s next step, so the two cannot
|
|
137
|
+
* name different files.
|
|
138
|
+
*/
|
|
139
|
+
export function consoleTokensPathFor(deployDir, platform) {
|
|
140
|
+
const { env } = deploymentServiceEnv({ env: {}, dir: deployDir, keys: ["PI_CONSOLE_TOKENS_FILE"], platform, fs: nodeFs });
|
|
141
|
+
return tokensFilePath(env, deployDir);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* How many operator tokens the console this unit starts would find (issue #708): `{ path, operators }` or
|
|
146
|
+
* `{ path, refusal }`. The tokens file is where `serve --deployment <deployDir>` looks: PI_CONSOLE_TOKENS_FILE as the
|
|
147
|
+
* SERVICE sees it (the deployment `.env`, through the reader the console itself uses), never this shell's export,
|
|
148
|
+
* which the unit does not inherit. Read with the real filesystem on purpose: `readTokensFile` judges the file the way
|
|
149
|
+
* `serve` does (owner-only, regular, no symlink), and a fake that answered differently would prove nothing.
|
|
150
|
+
*/
|
|
151
|
+
export function readConsoleOperators(deployDir, platform) {
|
|
152
|
+
const path = consoleTokensPathFor(deployDir, platform);
|
|
153
|
+
const read = readTokensFile({ path });
|
|
154
|
+
return read.refusal ? { path, refusal: read.refusal } : { path, operators: operatorCount(read.file) };
|
|
155
|
+
}
|
|
156
|
+
|
|
83
157
|
/**
|
|
84
158
|
* The whole substitution surface, template by template. The render replaces ONLY these literals (plus
|
|
85
159
|
* the scope-dependent `User=` / `WantedBy=` rewrites called out below); the pin test in
|
|
@@ -89,7 +163,8 @@ function resolveReceiverStart() {
|
|
|
89
163
|
// The two worker-template literals a podman render rewrites (issue #430), spelled once for the pin table and the render.
|
|
90
164
|
const WORKER_DESCRIPTION = "Description=pi-dispatch worker (drains the job queue on the host; launches job containers via docker)";
|
|
91
165
|
const WORKER_VALKEY_COMMENT = "# Valkey must be reachable (on this host, `pi-dispatch up` in the deployment folder starts it), but it is a separate\n# unit/container -- not ordered here since it may be remote.\n";
|
|
92
|
-
|
|
166
|
+
// The template's ordering line, which an openshell render on Podman alone rewrites without docker.service (issue #707).
|
|
167
|
+
const WORKER_AFTER = "After=network-online.target docker.service";
|
|
93
168
|
export const TEMPLATE_PINS = {
|
|
94
169
|
"worker.service": [
|
|
95
170
|
"ExecStart=/usr/bin/node worker/src/cli.mjs worker", // the WHOLE line → `<execPath> <cliPath> worker` (cli.mjs sits beside this module in src/ in both layouts)
|
|
@@ -100,6 +175,7 @@ export const TEMPLATE_PINS = {
|
|
|
100
175
|
"\nWants=network-online.target\n", // the anchor the podman venue's Wants=/After= on its Quadlet units go after (issue #430), user scope only
|
|
101
176
|
"Description=pi-dispatch worker (drains the job queue on the host; launches job containers via docker)", // → names rootless podman on a podman deployment (issue #430)
|
|
102
177
|
"# Valkey must be reachable (on this host, `pi-dispatch up` in the deployment folder starts it), but it is a separate\n# unit/container -- not ordered here since it may be remote.\n", // → says it IS ordered, when the Quadlet Valkey is installed
|
|
178
|
+
"After=network-online.target docker.service", // → without docker.service on an openshell deployment whose engine is podman and that blesses no `local` (issue #707); the openshell gateway's Wants=/After= go at the \nWants= anchor above
|
|
103
179
|
// Byte-for-byte survivors — semantics the render must not lose:
|
|
104
180
|
"RestartPreventExitStatus=2", // EXIT_POLICY is never restarted (a retry loop is a bill)
|
|
105
181
|
"StartLimitIntervalSec=60",
|
|
@@ -123,8 +199,21 @@ export const TEMPLATE_PINS = {
|
|
|
123
199
|
"KillSignal=SIGTERM",
|
|
124
200
|
"TimeoutStopSec=30",
|
|
125
201
|
],
|
|
202
|
+
// Issue #708: the console, the receiver's shape. Its ExecStart names the console package's entry and the deployment
|
|
203
|
+
// (`serve --deployment`), so the folder is in the line twice and the WHOLE line is replaced, as for the other two.
|
|
204
|
+
"console.service": [
|
|
205
|
+
"ExecStart=/usr/bin/node /usr/lib/node_modules/@edgehero/pi-dispatch-admin/dist/console.mjs serve --deployment /opt/pi-dispatch", // the WHOLE line → `<execPath> <consoleEntry> serve --deployment <deployDir>`
|
|
206
|
+
"WorkingDirectory=/opt/pi-dispatch",
|
|
207
|
+
"\nUser=pi\n",
|
|
208
|
+
"WantedBy=multi-user.target",
|
|
209
|
+
"RestartPreventExitStatus=2", // serve's refusals (no operator token, the lock held, an unusable listen) exit 2, and the next start would refuse the same way
|
|
210
|
+
"StartLimitIntervalSec=60",
|
|
211
|
+
"StartLimitBurst=5",
|
|
212
|
+
"KillSignal=SIGTERM",
|
|
213
|
+
"TimeoutStopSec=30",
|
|
214
|
+
],
|
|
126
215
|
"com.pi-dispatch.worker.plist": [
|
|
127
|
-
"<string>com.pi-dispatch.worker</string>", // → com.pi-dispatch.receiver for --receiver
|
|
216
|
+
"<string>com.pi-dispatch.worker</string>", // → com.pi-dispatch.receiver for --receiver, com.pi-dispatch.console for --console
|
|
128
217
|
"<string>/opt/pi-dispatch/deploy/worker-env-wrapper.sh</string>", // → the PACKAGE's wrapper copy, followed by the exec argv (<node> <script> …) the wrapper now runs verbatim
|
|
129
218
|
"<key>WorkingDirectory</key>\n\t<string>/opt/pi-dispatch</string>", // → the deployment folder (load-bearing: the wrapper sources ./.env there); also the anchor for the PATH injection below
|
|
130
219
|
"<string>/opt/pi-dispatch/logs/worker.out.log</string>", // → <deployDir>/logs/… (install creates the dir; launchd will not)
|
|
@@ -149,14 +238,18 @@ export const TEMPLATE_PINS = {
|
|
|
149
238
|
// they read back (`--env-setup`, issue #209 — the plist's EnvironmentVariables dict on macOS, nssm's
|
|
150
239
|
// AppEnvironmentExtra on Windows), and a rename on one side with silence on the other would produce
|
|
151
240
|
// a unit that starts fine and ignores the operator's secrets manager.
|
|
152
|
-
|
|
153
|
-
|
|
241
|
+
// Since issue #708 also PI_WRAPPER_NO_DOTENV, which the console's plist and nssm entry set and the wrappers test
|
|
242
|
+
// before they load ./.env: the same rename hazard, and here a rename would hand the console every worker secret.
|
|
243
|
+
"worker-env-wrapper.sh": ["PI_ENV_SETUP", "PI_WRAPPER_NO_DOTENV"],
|
|
244
|
+
"worker-env-wrapper.cmd": ["PI_ENV_SETUP", "PI_WRAPPER_NO_DOTENV"],
|
|
154
245
|
// The podman venue's stack as Quadlet units (issue #430, podman-stack.mjs). Three are copied verbatim, so what is
|
|
155
246
|
// pinned is what the worker and the installer rely on by NAME: the container and network names the worker attaches
|
|
156
247
|
// by, the unit names the worker unit Wants=, the loopback-only port, and an explicit Network= in every .container
|
|
157
248
|
// (a containers.conf `netns = "host"` would otherwise put it in the host namespace, measured).
|
|
158
249
|
"pi-dispatch-valkey.network": ["NetworkName=pi-dispatch-valkey"],
|
|
159
250
|
"pi-dispatch-valkey.container": [
|
|
251
|
+
// Issue #761: the pinned Valkey, fully qualified for Podman's short-name rule, from the one constant.
|
|
252
|
+
`Image=docker.io/${VALKEY_IMAGE}`,
|
|
160
253
|
"ContainerName=pi-dispatch-valkey",
|
|
161
254
|
"Network=pi-dispatch-valkey.network",
|
|
162
255
|
"PublishPort=127.0.0.1:6379:6379",
|
|
@@ -222,7 +315,7 @@ export const TEMPLATE_PINS = {
|
|
|
222
315
|
* composed form quotes.
|
|
223
316
|
*/
|
|
224
317
|
const ENV_SETUP_REFUSALS = {
|
|
225
|
-
linux: { re: /['"
|
|
318
|
+
linux: { re: /['"$%\\\x00-\x1f\x7f]/, why: "the path lands inside a single-quoted systemd Exec word, and systemd expands $VAR and %-specifiers there whatever the quoting" },
|
|
226
319
|
darwin: { re: /[<>&"\x00-\x1f\x7f]/, why: "the launchd unit is a plist, and the path lands in an XML <string>" },
|
|
227
320
|
win32: { re: /[%"\x00-\x1f\x7f]/, why: "cmd expands %VAR% when the wrapper runs the script" },
|
|
228
321
|
};
|
|
@@ -278,6 +371,23 @@ const ENV_SETUP_READERS = {
|
|
|
278
371
|
},
|
|
279
372
|
};
|
|
280
373
|
|
|
374
|
+
/**
|
|
375
|
+
* `text` as XML character data: `&`, `<` and `>` only, `&` first. No string this render writes lands in an attribute,
|
|
376
|
+
* so `'` and `"` need no entity, and leaving them literal keeps every plist that held one byte-identical to before
|
|
377
|
+
* (final review of #708). `>` is escaped although character data allows it, so a path can never write `-->`.
|
|
378
|
+
*/
|
|
379
|
+
function xmlText(text) {
|
|
380
|
+
return String(text).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* The reverse, for what readUnitSeam reads back out of a plist: the three entities `xmlText` writes plus the two it does
|
|
385
|
+
* not (a hand-edited plist may carry them). `&` last, so `&lt;` stays `<`.
|
|
386
|
+
*/
|
|
387
|
+
function xmlUntext(text) {
|
|
388
|
+
return text.replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
389
|
+
}
|
|
390
|
+
|
|
281
391
|
/**
|
|
282
392
|
* What an installed unit records: the configured `--env-setup` path and the deployment folder it
|
|
283
393
|
* serves. Both null for a unit rendered without the flag, which is every unit that predates issue #209
|
|
@@ -294,7 +404,12 @@ export function readUnitSeam(text, platform) {
|
|
|
294
404
|
// capture already ends exactly where the renderer's delimiter does, and a path is allowed to hold a
|
|
295
405
|
// space anywhere in it. An empty value means the same as no value at all.
|
|
296
406
|
const clip = (v) => (v === null ? null : v.replace(/\r$/, "") || null);
|
|
297
|
-
|
|
407
|
+
// The folder as the renderer escaped it (round 1 review of #708): `%` doubled in a systemd unit (WorkingDirectory=
|
|
408
|
+
// expands specifiers), XML entities in a plist. Undone here so a folder holding either still reads as itself.
|
|
409
|
+
const folder = clip(grab(readers.deployDir));
|
|
410
|
+
const deployDir = folder === null ? null : platform === "linux" ? folder.replace(/%%/g, "%") : platform === "darwin" ? xmlUntext(folder) : folder;
|
|
411
|
+
const setup = clip(grab(readers.setup));
|
|
412
|
+
return { setup: setup !== null && platform === "darwin" ? xmlUntext(setup) : setup, deployDir };
|
|
298
413
|
}
|
|
299
414
|
|
|
300
415
|
/**
|
|
@@ -312,7 +427,7 @@ export function readUnitUser(text, platform) {
|
|
|
312
427
|
|
|
313
428
|
const SUBCOMMANDS = new Set(["render", "install", "uninstall", "status", "start", "stop", "restart"]);
|
|
314
429
|
|
|
315
|
-
const SERVICE_USAGE = `pi-dispatch service — run the worker (or --receiver) as an OS service, rendered for THIS host
|
|
430
|
+
const SERVICE_USAGE = `pi-dispatch service — run the worker (or --receiver, or --console) as an OS service, rendered for THIS host
|
|
316
431
|
|
|
317
432
|
pi-dispatch service render the unit(s) with this host's real node + repo paths
|
|
318
433
|
pi-dispatch service install [--force] write + enable the user-level unit
|
|
@@ -320,10 +435,14 @@ const SERVICE_USAGE = `pi-dispatch service — run the worker (or --receiver) as
|
|
|
320
435
|
pi-dispatch service uninstall stop, disable and remove the installed unit
|
|
321
436
|
pi-dispatch service status which unit exists in which scope, and whether it is active
|
|
322
437
|
pi-dispatch service start|stop thin launchctl / systemctl --user / nssm wrappers
|
|
323
|
-
pi-dispatch service restart [--drain] restart; --drain pauses
|
|
324
|
-
|
|
438
|
+
pi-dispatch service restart [--drain] restart; --drain pauses this host's queues (the shared one, and
|
|
439
|
+
its own when PI_WORKER_NAME is set), waits for their active jobs to
|
|
440
|
+
finish, restarts, resumes the ones it paused [--drain-timeout <s>, default 600]
|
|
325
441
|
|
|
326
442
|
flags: --receiver the webhook receiver instead of the worker
|
|
443
|
+
--console the operator console (pi-dispatch-console serve) instead of the worker;
|
|
444
|
+
install needs an operator token (pi-dispatch-console token add)
|
|
445
|
+
--host-setup render|install --console: grant bounded local setup, Linux/macOS user scope
|
|
327
446
|
--user | --system linux scope (default --user; --system never executes root commands)
|
|
328
447
|
--force replace an existing unit in the same scope
|
|
329
448
|
--print also print the rendered unit before installing
|
|
@@ -343,6 +462,10 @@ export async function runService(argv = [], deps = {}) {
|
|
|
343
462
|
// Injectable so tests can render as if from an npm install without installing anything.
|
|
344
463
|
moduleDir = MODULE_DIR,
|
|
345
464
|
resolveReceiver = resolveReceiverStart,
|
|
465
|
+
// Issue #708: the console package's entry (see resolveConsoleEntry); a test injects a path or null.
|
|
466
|
+
resolveConsole = null,
|
|
467
|
+
// Issue #708: the operator tokens the console unit would start with (see readConsoleOperators).
|
|
468
|
+
consoleOperators = readConsoleOperators,
|
|
346
469
|
home = homedir(),
|
|
347
470
|
// env-internal USER: whose account a rendered unit runs as, taken from the login already running
|
|
348
471
|
// this command. An operator changes it by running the command as someone else, not by declaring it.
|
|
@@ -354,7 +477,9 @@ export async function runService(argv = [], deps = {}) {
|
|
|
354
477
|
err = (s) => process.stderr.write(s),
|
|
355
478
|
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
356
479
|
now = () => Date.now(),
|
|
357
|
-
queue
|
|
480
|
+
// Test seam: `(name) => queue`, one handle per queue the drain covers. Production builds them lazily in doRestart
|
|
481
|
+
// from VALKEY_URL.
|
|
482
|
+
queueFor = null,
|
|
358
483
|
// Is anything on 127.0.0.1:6379? Asked only on a podman deployment, to decide whether the Quadlet Valkey is
|
|
359
484
|
// wanted (issue #430); the same plain TCP connect `up` uses, for up's reason. Issue #464: asked of every address
|
|
360
485
|
// VALKEY_URL's host resolves to (`lookup`, as the worker's client resolves it), and `interfaces` says which of
|
|
@@ -368,6 +493,9 @@ export async function runService(argv = [], deps = {}) {
|
|
|
368
493
|
realpath = (p) => realpathSync(p),
|
|
369
494
|
// Issue #468: how a new Valkey password is made (a test's seam; 32 random bytes, hex, never shown).
|
|
370
495
|
newPassword = newValkeyPassword,
|
|
496
|
+
// Issue #707: where this shell finds a CLI by name (its PATH, the first executable file), for the plist's PATH.
|
|
497
|
+
// env-internal PATH: this shell's own search path, read only to find the openshell CLI's directory for the plist.
|
|
498
|
+
findBin = (name) => findOnPath(name, env.PATH),
|
|
371
499
|
} = deps;
|
|
372
500
|
|
|
373
501
|
let values, positionals;
|
|
@@ -377,6 +505,8 @@ export async function runService(argv = [], deps = {}) {
|
|
|
377
505
|
allowPositionals: true,
|
|
378
506
|
options: {
|
|
379
507
|
receiver: { type: "boolean", default: false },
|
|
508
|
+
console: { type: "boolean", default: false },
|
|
509
|
+
"host-setup": { type: "boolean", default: false },
|
|
380
510
|
user: { type: "boolean", default: false },
|
|
381
511
|
system: { type: "boolean", default: false },
|
|
382
512
|
force: { type: "boolean", default: false },
|
|
@@ -396,9 +526,17 @@ export async function runService(argv = [], deps = {}) {
|
|
|
396
526
|
return cmd ? 1 : 0; // bare `service` is a usage view, an unknown subcommand is an error
|
|
397
527
|
}
|
|
398
528
|
if (values.user && values.system) return fail(err, "--user and --system are mutually exclusive");
|
|
529
|
+
// One component per invocation: every subcommand addresses exactly one unit, and guessing which of two was meant
|
|
530
|
+
// would install, stop or remove the other one.
|
|
531
|
+
if (values.receiver && values.console) return fail(err, "--receiver and --console are mutually exclusive: each names one service; run the command once for each");
|
|
532
|
+
if (values["host-setup"] && (!values.console || !["render", "install"].includes(cmd) || values.system || !["linux", "darwin"].includes(platform))) return fail(err, "--host-setup is only for render/install --console in a Linux or macOS user service");
|
|
399
533
|
if (platform === "darwin" && values.system) {
|
|
400
|
-
|
|
534
|
+
// The Docker Desktop reason is the worker's; the console and the receiver start no container (round 1 review of #708).
|
|
535
|
+
return fail(err, values.console || values.receiver ? "macOS is user-scope only here (a LaunchAgent, in your login session, where this command installs every unit). Drop --system." : "macOS is user-scope only (a LaunchAgent): Docker Desktop lives in the login session, so a system daemon would wait on a docker socket that only exists once you log in. Drop --system.");
|
|
401
536
|
}
|
|
537
|
+
// `--drain` pauses the QUEUES and waits for their jobs, which only the worker runs (round 1 review of #708): on the
|
|
538
|
+
// receiver or the console it would pause every job for a restart that drains nothing.
|
|
539
|
+
if (values.drain && (values.console || values.receiver)) return fail(err, "--drain applies to the worker alone (it pauses the queues and waits for their running jobs); restart the receiver or the console without it");
|
|
402
540
|
if (platform === "win32" && (values.user || values.system)) {
|
|
403
541
|
return fail(err, "Windows services via nssm are machine-scoped; --user/--system do not apply here");
|
|
404
542
|
}
|
|
@@ -420,6 +558,9 @@ export async function runService(argv = [], deps = {}) {
|
|
|
420
558
|
// Resolved only when asked for: worker-only invocations must not care whether the receiver
|
|
421
559
|
// package exists here at all.
|
|
422
560
|
receiverStart: values.receiver ? resolveReceiver() : null,
|
|
561
|
+
// Issue #708: the same rule for the console, resolved only for --console.
|
|
562
|
+
consoleEntry: values.console ? (resolveConsole ?? (() => resolveConsoleEntry({ findBin, realpath })))() : null,
|
|
563
|
+
consoleOperators,
|
|
423
564
|
home,
|
|
424
565
|
user,
|
|
425
566
|
tmp,
|
|
@@ -429,16 +570,21 @@ export async function runService(argv = [], deps = {}) {
|
|
|
429
570
|
err,
|
|
430
571
|
sleep,
|
|
431
572
|
now,
|
|
432
|
-
|
|
573
|
+
queueFor,
|
|
433
574
|
probeTcp,
|
|
434
575
|
lookup,
|
|
435
576
|
interfaces,
|
|
436
577
|
runSync,
|
|
437
578
|
realpath,
|
|
438
579
|
newPassword,
|
|
439
|
-
|
|
580
|
+
findBin,
|
|
581
|
+
which: values.receiver ? "receiver" : values.console ? "console" : "worker",
|
|
440
582
|
scope: platform === "linux" && values.system ? "system" : "user",
|
|
441
583
|
force: values.force,
|
|
584
|
+
// The terminal bootstrap reuses this exact planner for an existing granted console.
|
|
585
|
+
// This dependency is internal; no environment variable or start flag grants authority.
|
|
586
|
+
hostSetup: values["host-setup"] || (values.console && deps.setupConsoleGrant === true),
|
|
587
|
+
setupExpected: deps.setupExpected,
|
|
442
588
|
// The operator's env-setup script, or null for the shipped default. Null is load-bearing: with no
|
|
443
589
|
// seam configured every render below is byte-identical to what it produced before issue #209.
|
|
444
590
|
envSetup: null,
|
|
@@ -455,7 +601,13 @@ export async function runService(argv = [], deps = {}) {
|
|
|
455
601
|
if (resolved.error) return fail(err, resolved.error);
|
|
456
602
|
ctx.envSetup = resolved.path;
|
|
457
603
|
}
|
|
604
|
+
if (ctx.hostSetup && (!(deps.setupEntry ?? resolveSetupWorkerEntry)().entry || setupCachePath(ctx.consoleEntry))) return fail(err, "--host-setup needs durable installed worker and console entries, never npm execution-cache paths");
|
|
458
605
|
|
|
606
|
+
if (deps.setupPlan === true) return setupServicePlan(ctx, cmd);
|
|
607
|
+
if (deps.setupExpected !== undefined) {
|
|
608
|
+
const current = await setupServicePlan(ctx, cmd);
|
|
609
|
+
if (JSON.stringify(current) !== JSON.stringify(deps.setupExpected)) return fail(err, "the setup service plan changed before dispatch; preview it again");
|
|
610
|
+
}
|
|
459
611
|
switch (cmd) {
|
|
460
612
|
case "render":
|
|
461
613
|
return doRender(ctx);
|
|
@@ -476,6 +628,107 @@ export async function runService(argv = [], deps = {}) {
|
|
|
476
628
|
}
|
|
477
629
|
}
|
|
478
630
|
|
|
631
|
+
/**
|
|
632
|
+
* The services this command installs, by `ctx.which`. Each has ONE systemd template, and the launchd and nssm forms are
|
|
633
|
+
* derived from the worker's (the receiver's precedent): the same KeepAlive/ExitTimeOut shape and the same AppExit pair,
|
|
634
|
+
* which is what keeps every component's exit 2 a stop rather than a restart loop.
|
|
635
|
+
*
|
|
636
|
+
* template the deploy/ file renderLinuxUnit reads (a TEMPLATE_PINS key)
|
|
637
|
+
* templateExec that template's ExecStart line, replaced WHOLE by execArgvFor's argv
|
|
638
|
+
*/
|
|
639
|
+
const COMPONENTS = Object.freeze(["worker", "receiver", "console"]);
|
|
640
|
+
const COMPONENT_TEMPLATES = Object.freeze({
|
|
641
|
+
worker: { template: "worker.service", templateExec: "ExecStart=/usr/bin/node worker/src/cli.mjs worker" },
|
|
642
|
+
receiver: { template: "receiver.service", templateExec: "ExecStart=/usr/bin/node receiver/src/start.mjs" },
|
|
643
|
+
console: { template: "console.service", templateExec: "ExecStart=/usr/bin/node /usr/lib/node_modules/@edgehero/pi-dispatch-admin/dist/console.mjs serve --deployment /opt/pi-dispatch" },
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
/**
|
|
647
|
+
* The command a unit runs, for every service manager: the systemd ExecStart, the argv the launchd wrapper execs and
|
|
648
|
+
* nssm's AppParameters. One function because there were three copies of it, one per manager, and a component added
|
|
649
|
+
* to two of them would install a unit on the third that runs the worker. Absolute node, absolute script: nothing here
|
|
650
|
+
* depends on a PATH or a working directory. The console names its deployment itself (`serve --deployment`), so it
|
|
651
|
+
* reads THIS deployment's `.env` and tokens even where the manager's working directory were not honoured.
|
|
652
|
+
*/
|
|
653
|
+
function execArgvFor(ctx) {
|
|
654
|
+
if (ctx.which === "receiver") return [ctx.execPath, ctx.receiverStart];
|
|
655
|
+
if (ctx.which === "console") return [ctx.execPath, ctx.consoleEntry, "serve", "--deployment", ctx.deployDir, ...(ctx.hostSetup ? ["--host-setup"] : [])];
|
|
656
|
+
return [ctx.execPath, ctx.cliPath, "worker"];
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/** The browser's planner uses the installer's own render and stack plan, never a second unit template (#774). */
|
|
660
|
+
export async function planServiceSetup(argv, deps = {}) {
|
|
661
|
+
return runService(argv, { ...deps, setupPlan: true, out: () => {}, err: () => {} });
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
async function setupServicePlan(ctx, operation) {
|
|
665
|
+
const no = (reason) => ({ refusal: reason });
|
|
666
|
+
if (!["linux", "darwin"].includes(ctx.platform) || ctx.scope !== "user" || (ctx.platform === "darwin" && ctx.euid === 0)) return no("unsupported");
|
|
667
|
+
if (ctx.which === "receiver" && !ctx.receiverStart) return no("missing-receiver");
|
|
668
|
+
if (ctx.which === "console" && !ctx.consoleEntry) return no("unsafe-entry");
|
|
669
|
+
if (renderRefusal(ctx) !== null) return no("unsafe-deployment");
|
|
670
|
+
const paths = unitPaths(ctx);
|
|
671
|
+
if (ctx.which === "worker" && paths.otherScopeWorkerPaths.some((p) => ctx.fs.existsSync(p))) return no("foreign-service");
|
|
672
|
+
const venue = deploymentVenue(ctx);
|
|
673
|
+
if (venue.error) return no("unsafe-deployment");
|
|
674
|
+
let stack = null;
|
|
675
|
+
if (ctx.platform === "linux" && ctx.which === "worker") {
|
|
676
|
+
stack = await stackFor(ctx, venue, { readKeeper: true });
|
|
677
|
+
if (stack?.error || stack?.valkeyRefusal || stack?.pendingPassword || (stack?.plan.restart?.length ?? 0) > 0 || (stack?.plan.clientsRestarted?.length ?? 0) > 0 || stack?.plan.files.some((f) => f.state === "changed")) return no("unsafe-dependencies");
|
|
678
|
+
if (stack) {
|
|
679
|
+
const foreign = await foreignContainers(stack.plan, (cmd, args) => runQuery(ctx, cmd, args));
|
|
680
|
+
if (foreign.found.length || foreign.unknown.length) return no("unsafe-dependencies");
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
const text = ctx.platform === "linux" ? renderLinuxUnit(ctx, stack?.plan.start ?? [], venue.venues ?? null, venue.openshell) : renderPlist(ctx, { pathDirs: openshellPathDirs(ctx, venue) });
|
|
684
|
+
let before = null;
|
|
685
|
+
if (ctx.fs.existsSync(paths.installPath)) {
|
|
686
|
+
try {
|
|
687
|
+
const st = ctx.fs.lstatSync?.(paths.installPath);
|
|
688
|
+
if (st && (st.isSymbolicLink() || !st.isFile() || st.uid !== ctx.euid || (st.mode & 0o022))) return no("foreign-service");
|
|
689
|
+
before = ctx.fs.readFileSync(paths.installPath, "utf8");
|
|
690
|
+
} catch { return no("foreign-service"); }
|
|
691
|
+
// A hand-edited unit can launch anything this account can launch; it is not a setup-owned service.
|
|
692
|
+
if (before !== text) return no("foreign-service");
|
|
693
|
+
}
|
|
694
|
+
if (operation === "install" && before !== null) return no("service-exists");
|
|
695
|
+
if (operation !== "install" && before === null) return no("missing-service");
|
|
696
|
+
let active = false;
|
|
697
|
+
let loaded = false;
|
|
698
|
+
if (ctx.platform === "linux") {
|
|
699
|
+
const manager = await runQuery(ctx, "systemctl", ["--user", "show-environment"]);
|
|
700
|
+
if (manager.code !== 0 || managerEnvRefusal(manager, { home: ctx.home, euid: ctx.euid, user: ctx.user, realpath: ctx.realpath })) return no("manager-unavailable");
|
|
701
|
+
const status = await runQuery(ctx, "systemctl", ["--user", "show", "--property=LoadState,ActiveState,FragmentPath,DropInPaths,WorkingDirectory,ExecStart", paths.name]);
|
|
702
|
+
if (status.code !== 0) return no("manager-unavailable");
|
|
703
|
+
const properties = Object.fromEntries(String(status.stdout).split("\n").map((line) => { const at = line.indexOf("="); return [line.slice(0, at), line.slice(at + 1)]; }));
|
|
704
|
+
if (before !== null) {
|
|
705
|
+
try {
|
|
706
|
+
if (properties.LoadState !== "loaded" || !properties.FragmentPath || ctx.realpath(properties.FragmentPath) !== ctx.realpath(paths.installPath) || properties.DropInPaths !== "" || !properties.WorkingDirectory || ctx.realpath(properties.WorkingDirectory) !== ctx.realpath(ctx.deployDir)) return no("foreign-service");
|
|
707
|
+
const argv = /(?:^|[ ;])argv\[\]=([^;]*);/.exec(properties.ExecStart ?? "")?.[1]?.trim();
|
|
708
|
+
if (argv !== execArgvFor(ctx).join(" ")) return no("foreign-service");
|
|
709
|
+
} catch { return no("foreign-service"); }
|
|
710
|
+
} else if (properties.LoadState && properties.LoadState !== "not-found") return no("foreign-service");
|
|
711
|
+
loaded = properties.LoadState === "loaded";
|
|
712
|
+
active = /^ActiveState=active$/m.test(status.stdout);
|
|
713
|
+
} else {
|
|
714
|
+
const manager = await runQuery(ctx, "launchctl", ["print", `gui/${ctx.euid}`]);
|
|
715
|
+
if (manager.code !== 0) return no("manager-unavailable");
|
|
716
|
+
const status = await runQuery(ctx, "launchctl", ["print", `gui/${ctx.euid}/${paths.name}`]);
|
|
717
|
+
loaded = status.code === 0;
|
|
718
|
+
if (loaded) {
|
|
719
|
+
try {
|
|
720
|
+
const source = /^\s*path = (.+)$/m.exec(status.stdout)?.[1];
|
|
721
|
+
const directory = /^\s*working directory = (.+)$/m.exec(status.stdout)?.[1];
|
|
722
|
+
const argumentsText = /arguments = \{\s*\n([\s\S]*?)\n\s*\}/.exec(status.stdout)?.[1];
|
|
723
|
+
const argv = argumentsText?.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
724
|
+
if (!before || !source || ctx.realpath(source) !== ctx.realpath(paths.installPath) || !directory || ctx.realpath(directory) !== ctx.realpath(ctx.deployDir) || JSON.stringify(argv) !== JSON.stringify(["/bin/sh", ctx.wrapperSh, ...execArgvFor(ctx)])) return no("foreign-service");
|
|
725
|
+
} catch { return no("foreign-service"); }
|
|
726
|
+
}
|
|
727
|
+
active = status.code === 0 && /state = running/.test(status.stdout);
|
|
728
|
+
}
|
|
729
|
+
return { operation, which: ctx.which, path: paths.installPath, text, before, active, loaded, stack: stack?.plan ?? null, effects: [...(operation === "install" ? [`write-${ctx.which}-unit`] : []), `${operation === "install" ? "start" : operation}-${ctx.which}`, ...(stack?.plan.files.some((f) => f.state === "new") ? ["write-stack-files"] : []), ...(stack?.plan.start ?? []).flatMap((unit) => unit.includes("valkey") ? ["start-valkey"] : unit.includes("proxy") ? ["start-proxy"] : unit.includes("keeper") ? ["start-keeper"] : [])] };
|
|
730
|
+
}
|
|
731
|
+
|
|
479
732
|
/**
|
|
480
733
|
* Where each unit lives (or is looked for) per platform. `otherScopeWorkerPaths` exists only to
|
|
481
734
|
* enforce DES-CONCURRENCY-3 at install time — those locations are READ, never written. On Linux the
|
|
@@ -513,7 +766,8 @@ function unitPaths(ctx) {
|
|
|
513
766
|
}
|
|
514
767
|
|
|
515
768
|
/**
|
|
516
|
-
* Every unit FILE this tool could have installed on this host:
|
|
769
|
+
* Every unit FILE this tool could have installed on this host: every component (the worker, the receiver and, since
|
|
770
|
+
* issue #708, the console), both scopes, plus the
|
|
517
771
|
* `worker.service` name the README's manual `sudo cp` instructions produce (doStatus already counts
|
|
518
772
|
* that one as a real worker unit, so a preflight reading units must too). Derived from unitPaths
|
|
519
773
|
* rather than restated, so the two cannot disagree about where a unit lives.
|
|
@@ -523,7 +777,7 @@ function unitPaths(ctx) {
|
|
|
523
777
|
export function installedUnitPaths(platform, home) {
|
|
524
778
|
if (platform !== "linux" && platform !== "darwin") return [];
|
|
525
779
|
const found = [];
|
|
526
|
-
for (const which of
|
|
780
|
+
for (const which of COMPONENTS) {
|
|
527
781
|
const paths = unitPaths({ platform, home, which, scope: "user" });
|
|
528
782
|
found.push({ which, scope: "user", path: platform === "linux" ? paths.userPath : paths.installPath });
|
|
529
783
|
found.push({ which, scope: "system", path: paths.systemPath });
|
|
@@ -532,6 +786,21 @@ export function installedUnitPaths(platform, home) {
|
|
|
532
786
|
return found;
|
|
533
787
|
}
|
|
534
788
|
|
|
789
|
+
/**
|
|
790
|
+
* The console unit FILE installed for the deployment in `cwd` (issue #708), or null: one of `installedUnitPaths`'
|
|
791
|
+
* console entries whose WorkingDirectory (`readUnitSeam`) is this folder. Shared by doctor and `up`, so the two agree
|
|
792
|
+
* on whether a console service exists. Always null on win32, where the service is nssm's and is asked by name.
|
|
793
|
+
*/
|
|
794
|
+
export function installedConsoleUnitPath({ platform, home, cwd, fileExists = existsSync, readUnit = (path) => readFileSync(path, "utf8") }) {
|
|
795
|
+
for (const { path, which } of installedUnitPaths(platform, home)) {
|
|
796
|
+
if (which !== "console" || !fileExists(path)) continue;
|
|
797
|
+
try {
|
|
798
|
+
if (readUnitSeam(readUnit(path), platform).deployDir === cwd) return path;
|
|
799
|
+
} catch {}
|
|
800
|
+
}
|
|
801
|
+
return null;
|
|
802
|
+
}
|
|
803
|
+
|
|
535
804
|
function readTemplate(ctx, name) {
|
|
536
805
|
return ctx.fs.readFileSync(join(ctx.templatesDir, name), "utf8");
|
|
537
806
|
}
|
|
@@ -552,7 +821,10 @@ function subDeployDir(ctx, text) {
|
|
|
552
821
|
// Function replacement, here and everywhere below a computed path is the REPLACEMENT: String.replace
|
|
553
822
|
// and replaceAll both read `$&` and its relatives out of a replacement STRING, so a deployment folder
|
|
554
823
|
// holding one would splice itself into the unit. A function replacement is taken literally.
|
|
555
|
-
|
|
824
|
+
// `%` doubled (round 1 review of #708): WorkingDirectory= and EnvironmentFile= expand systemd specifiers, so a
|
|
825
|
+
// folder named `a%hb` would run in `a<home>b`. readUnitSeam undoes it. Linux units only: the plist has its own
|
|
826
|
+
// escaping (xmlText), and nssm takes the folder as an argv word.
|
|
827
|
+
return text.replaceAll("/opt/pi-dispatch", () => ctx.deployDir.replace(/%/g, "%%"));
|
|
556
828
|
}
|
|
557
829
|
|
|
558
830
|
/**
|
|
@@ -578,6 +850,68 @@ function refuseMissingReceiver(ctx) {
|
|
|
578
850
|
);
|
|
579
851
|
}
|
|
580
852
|
|
|
853
|
+
/**
|
|
854
|
+
* The console refusals (issue #708), shared by render and install as the receiver's is: the console package's entry
|
|
855
|
+
* not found, so a unit would name a file that is not there.
|
|
856
|
+
*/
|
|
857
|
+
function refuseMissingConsole(ctx) {
|
|
858
|
+
// Worded per platform (round 1 review of #708): `findOnPath` reads no Windows directory, so there it did not look.
|
|
859
|
+
const looked =
|
|
860
|
+
ctx.platform === "win32"
|
|
861
|
+
? `${CONSOLE_PACKAGE}/dist/console.mjs does not resolve from this worker's install (on Windows that is the only place looked: the PATH lookup does not run there)`
|
|
862
|
+
: `neither does ${CONSOLE_PACKAGE}/dist/console.mjs resolve from this worker's install, nor is a \`pi-dispatch-console\` on this shell's PATH that is that package's own file (a pnpm or Volta shim is not)`;
|
|
863
|
+
return fail(
|
|
864
|
+
ctx.err,
|
|
865
|
+
`the operator console is not installed where this command can find it: ${looked}. Install it in the same place as this worker, so it resolves from here: npm install -g ${CONSOLE_PACKAGE} when the worker is installed globally, or npm install ${CONSOLE_PACKAGE} in the folder whose node_modules holds this worker (an admin package installed anywhere else is not found), then re-run`,
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
/**
|
|
870
|
+
* Paths a unit cannot carry (round 1 review of #708), refused before anything is rendered, on every component: a control
|
|
871
|
+
* character in the deployment folder (it would end or corrupt the unit line, the XML or the nssm property), surrounding
|
|
872
|
+
* whitespace on Linux (systemd strips it from WorkingDirectory=, so the unit would run in another folder), and on
|
|
873
|
+
* Windows a command word holding a character cmd reads (the wrapper runs its argv as `%*`, where `"` cannot be
|
|
874
|
+
* quoted and `&`, `|`, `<`, `>`, `^` and `%` outside quotes change the command). Everything else is quoted or escaped
|
|
875
|
+
* per form: systemdWord, xmlText, nssm's argv. null when the render can go ahead.
|
|
876
|
+
*/
|
|
877
|
+
function renderRefusal(ctx) {
|
|
878
|
+
const shown = (c) => (c.charCodeAt(0) < 0x20 || c.charCodeAt(0) === 0x7f ? `\\x${c.charCodeAt(0).toString(16).padStart(2, "0")}` : c);
|
|
879
|
+
const control = /[\x00-\x1f\x7f]/.exec(ctx.deployDir);
|
|
880
|
+
if (control) return fail(ctx.err, `the deployment folder ${JSON.stringify(ctx.deployDir)} holds ${JSON.stringify(shown(control[0]))}, a control character no service unit can carry; run this from a folder without one`);
|
|
881
|
+
if (ctx.platform === "linux" && ctx.deployDir !== ctx.deployDir.trim()) return fail(ctx.err, `the deployment folder ${JSON.stringify(ctx.deployDir)} starts or ends with whitespace, which systemd strips from WorkingDirectory=, so the unit would run in another folder; rename it`);
|
|
882
|
+
// NOT refused (final review of #708): a `--` in the folder, which the plist template also names inside XML comments.
|
|
883
|
+
// plutil accepts it there, and `xmlText` escapes `>`, so a folder can never write the `-->` that would end one;
|
|
884
|
+
// refusing it broke worker installs that had always worked in a folder like ~/my--deploy.
|
|
885
|
+
if (ctx.platform === "win32") {
|
|
886
|
+
for (const word of execArgvFor(ctx)) {
|
|
887
|
+
const bad = /["%&|<>^\x00-\x1f\x7f]/.exec(word);
|
|
888
|
+
if (bad) return fail(ctx.err, `${JSON.stringify(word)} holds ${JSON.stringify(shown(bad[0]))}, which the service's wrapper cannot pass to the command it runs (cmd reads it); move it to a path without one`);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
return null;
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
/**
|
|
895
|
+
* Install only: a console unit started with no operator token in its tokens file would exit 2 on every start (`serve`
|
|
896
|
+
* refuses, so nobody could change anything from it), and RestartPreventExitStatus=2 would then leave it stopped. An
|
|
897
|
+
* install that "succeeds" into a service that never runs is the silent no-op this project refuses, so it is refused
|
|
898
|
+
* here, with the fix. A tokens file that cannot be used at all (missing, too open, invalid) is refused the same way and
|
|
899
|
+
* with its own sentence. `render` does not refuse: it writes nothing, and the unit it shows is right once a token exists.
|
|
900
|
+
*/
|
|
901
|
+
function refuseConsoleWithoutOperator(ctx) {
|
|
902
|
+
let found;
|
|
903
|
+
try {
|
|
904
|
+
found = ctx.consoleOperators(ctx.deployDir, ctx.platform);
|
|
905
|
+
} catch (err) {
|
|
906
|
+
return fail(ctx.err, `the console's tokens file could not be read (${err?.message ?? err}), so the console would refuse to start; nothing was installed`);
|
|
907
|
+
}
|
|
908
|
+
if (found.refusal) return fail(ctx.err, `${found.refusal}. The console refuses to start without it, so nothing was installed`);
|
|
909
|
+
if (found.operators === 0) {
|
|
910
|
+
return fail(ctx.err, `${found.path} holds no operator token, so \`pi-dispatch-console serve\` would refuse to start (exit 2) and the unit would stay stopped. Add one first: pi-dispatch-console token add --name <you> --role operator --deployment ${ctx.deployDir}`);
|
|
911
|
+
}
|
|
912
|
+
return null;
|
|
913
|
+
}
|
|
914
|
+
|
|
581
915
|
/**
|
|
582
916
|
* The systemd form of the env-setup seam. Three pieces, each load-bearing:
|
|
583
917
|
*
|
|
@@ -600,8 +934,29 @@ function refuseMissingReceiver(ctx) {
|
|
|
600
934
|
* whole script as one word, and a worker exiting 2 under it reports ExecMainStatus=2 with NRestarts=0.
|
|
601
935
|
*/
|
|
602
936
|
function composeEnvSetupExec(setup, argv) {
|
|
603
|
-
|
|
604
|
-
|
|
937
|
+
// The argv inside the script, in sh's double quotes, as it always was, while every word is one that form carries
|
|
938
|
+
// as itself. Otherwise (round 1 review of #708: a deployment folder in the console's argv may hold a space, a `'`,
|
|
939
|
+
// a `$` or a `%`) the argv goes AFTER the script as its own systemd words (`systemdWord`), and the script runs
|
|
940
|
+
// `exec "$@"` (written `$$@`: systemd turns `$$` into `$` whatever the quoting), so nothing typed into a path is
|
|
941
|
+
// ever shell text. The prefix readUnitSeam reads is the same in both forms.
|
|
942
|
+
if (argv.every((a) => !/['"$`\\%\x00-\x1f\x7f]/.test(a))) {
|
|
943
|
+
const command = argv.map((a) => `"${a}"`).join(" ");
|
|
944
|
+
return `/bin/sh -c 'set -a; . "${setup}" || exit 1; set +a; exec ${command}'`;
|
|
945
|
+
}
|
|
946
|
+
return `/bin/sh -c 'set -a; . "${setup}" || exit 1; set +a; exec "$$@"' sh ${argv.map(systemdWord).join(" ")}`;
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* One word of a systemd Exec line (systemd.service(5), "Command lines"; systemd.unit(5), "Specifiers"): as it is when it
|
|
951
|
+
* holds only characters systemd reads literally, else in double quotes with `\\` and `"` escaped, `%` written `%%` (a
|
|
952
|
+
* specifier) and `$` written `$$` (variable expansion, which happens inside quotes too). Round 1 review of #708: the
|
|
953
|
+
* console's argv carries the deployment folder, and an unquoted one split at a space and expanded `%h` and `$HOME`.
|
|
954
|
+
* Every path a render held before is a bare word, so those units are byte-identical. A control character cannot be
|
|
955
|
+
* written on one unit line at all; `renderRefusal` refuses it before anything is rendered.
|
|
956
|
+
*/
|
|
957
|
+
export function systemdWord(word) {
|
|
958
|
+
if (/^[A-Za-z0-9_@+=:,./-]+$/.test(word)) return word;
|
|
959
|
+
return `"${word.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/%/g, "%%").replace(/\$/g, "$$$$")}"`;
|
|
605
960
|
}
|
|
606
961
|
|
|
607
962
|
/**
|
|
@@ -609,16 +964,14 @@ function composeEnvSetupExec(setup, argv) {
|
|
|
609
964
|
* known literals (see TEMPLATE_PINS); everything else — RestartPreventExitStatus=2, the StartLimit
|
|
610
965
|
* crash-loop bound, KillSignal, TimeoutStopSec — passes through byte-for-byte.
|
|
611
966
|
*/
|
|
612
|
-
function renderLinuxUnit(ctx, stackUnits = [], venues = null) {
|
|
613
|
-
const template = ctx.which
|
|
967
|
+
function renderLinuxUnit(ctx, stackUnits = [], venues = null, openshell = null) {
|
|
968
|
+
const { template, templateExec: templateLine } = COMPONENT_TEMPLATES[ctx.which];
|
|
614
969
|
// The ExecStart line is replaced WHOLE, not path-by-path: the template's script path is relative
|
|
615
970
|
// to a repo-root WorkingDirectory that only a checkout has. The rendered unit points at absolute
|
|
616
971
|
// entries that exist in both layouts — cliPath beside this module; the receiver package's ./start
|
|
617
|
-
// export — so ExecStart works no matter what WorkingDirectory is.
|
|
618
|
-
const argv = ctx
|
|
619
|
-
const
|
|
620
|
-
ctx.which === "receiver" ? "ExecStart=/usr/bin/node receiver/src/start.mjs" : "ExecStart=/usr/bin/node worker/src/cli.mjs worker";
|
|
621
|
-
const execStart = `ExecStart=${ctx.envSetup ? composeEnvSetupExec(ctx.envSetup, argv) : argv.join(" ")}`;
|
|
972
|
+
// export; the console package's dist/console.mjs — so ExecStart works no matter what WorkingDirectory is.
|
|
973
|
+
const argv = execArgvFor(ctx);
|
|
974
|
+
const execStart = `ExecStart=${ctx.envSetup ? composeEnvSetupExec(ctx.envSetup, argv) : argv.map(systemdWord).join(" ")}`;
|
|
622
975
|
// The banner outranks the template's own "TEMPLATE/UNTESTED EXAMPLE — set the PLACEHOLDERs" header,
|
|
623
976
|
// which renders through below (the no-markers design keeps templates byte-usable, so their prose
|
|
624
977
|
// survives): a reader of the rendered unit should know the placeholders are already substituted.
|
|
@@ -649,16 +1002,29 @@ function renderLinuxUnit(ctx, stackUnits = [], venues = null) {
|
|
|
649
1002
|
unit = unit.replace("WantedBy=multi-user.target", "WantedBy=default.target");
|
|
650
1003
|
// The podman venue's Quadlet units (issue #430), which live in this same user manager, so the worker can be
|
|
651
1004
|
// ordered after them. Nothing is added when there are none, which keeps every other render byte-identical.
|
|
652
|
-
const deps = workerUnitDeps(stackUnits);
|
|
653
|
-
|
|
1005
|
+
const deps = workerUnitDeps(stackUnits, venues?.podmanUsed || !openshell?.used ? {} : { forWhat: "the openshell venue's Valkey on Podman (issue #707)" });
|
|
1006
|
+
// Issue #707: the openshell venues' gateway is the `openshell-gateway` USER unit OpenShell's package ships, in this
|
|
1007
|
+
// same user manager, so the worker pulls it in at boot. `Wants=`, never `Requires=`, for workerUnitDeps' reason: a
|
|
1008
|
+
// gateway that fails to start must not stop the worker, whose boot says why (and whose other venues still run).
|
|
1009
|
+
// NO `After=`, the obvious companion, REJECTED (review round 1 of #715, measured with `systemd-analyze --user verify
|
|
1010
|
+
// default.target`): the 0.1.4 deb's unit has `After=default.target` and `WantedBy=default.target`, and the worker is
|
|
1011
|
+
// WantedBy default.target too, so default.target -> worker -> gateway -> default.target is an ordering cycle, and
|
|
1012
|
+
// systemd breaks it by deleting the worker's start job. Unordered is safe: a gateway down at boot is only logged
|
|
1013
|
+
// (`openshell_gateway_down_at_boot`), and every job reads it again before it spends.
|
|
1014
|
+
const gatewayDeps = openshell?.used ? `# Added by \`pi-dispatch service install\` for the openshell venue (issue #707): the gateway's own user unit, pulled\n# in but NOT ordered (its unit is After=default.target, so an After= here is an ordering cycle).\nWants=${OPENSHELL_GATEWAY_UNIT}\n` : "";
|
|
1015
|
+
if (deps || gatewayDeps) unit = unit.replace("\nWants=network-online.target\n", () => `\nWants=network-online.target\n${deps}${gatewayDeps}`);
|
|
654
1016
|
// The template's own words, made true for a podman deployment (round 2 nit): its Description says jobs launch
|
|
655
|
-
// via docker, and its comment says Valkey is not ordered here. Both anchors are TEMPLATE_PINS.
|
|
656
|
-
|
|
657
|
-
|
|
1017
|
+
// via docker, and its comment says Valkey is not ordered here. Both anchors are TEMPLATE_PINS. Issue #707: and for
|
|
1018
|
+
// an openshell one, whose jobs run in the gateway's sandboxes on its engine.
|
|
1019
|
+
if (venues?.podmanUsed || openshell?.used) {
|
|
1020
|
+
const runtimes = [venues?.localUsed ? "docker" : null, venues?.podmanUsed ? "rootless podman" : null, openshell?.used ? `an OpenShell gateway on ${openshell.engine}` : null].filter(Boolean);
|
|
1021
|
+
const runtime = runtimes.length > 1 ? `${runtimes.slice(0, -1).join(", ")} and ${runtimes.at(-1)}` : runtimes[0];
|
|
658
1022
|
unit = unit.replace(WORKER_DESCRIPTION, () => `Description=pi-dispatch worker (drains the job queue on the host; launches job containers via ${runtime})`);
|
|
659
1023
|
if (stackUnits.includes(QUADLET_FILES.valkey.unit)) {
|
|
660
|
-
unit = unit.replace(WORKER_VALKEY_COMMENT, () => "# Valkey is the podman venue's Quadlet unit in this same user manager, so the worker is ordered after it\n# (the Wants=/After= lines `pi-dispatch service install` added below).\n");
|
|
1024
|
+
unit = unit.replace(WORKER_VALKEY_COMMENT, () => (venues?.podmanUsed ? "# Valkey is the podman venue's Quadlet unit in this same user manager, so the worker is ordered after it\n# (the Wants=/After= lines `pi-dispatch service install` added below).\n" : "# Valkey is a Quadlet unit in this same user manager (the openshell venue's engine is Podman), so the worker is\n# ordered after it (the Wants=/After= lines `pi-dispatch service install` added below).\n"));
|
|
661
1025
|
}
|
|
1026
|
+
// Nothing of this deployment runs on docker: no `local` venue, and the gateway's engine is Podman.
|
|
1027
|
+
if (openshell?.used && openshell.engine === "podman" && !venues?.localUsed) unit = unit.replace(WORKER_AFTER, () => "After=network-online.target");
|
|
662
1028
|
}
|
|
663
1029
|
} else {
|
|
664
1030
|
unit = unit.replace(/^User=pi$/m, () => `User=${ctx.user}`);
|
|
@@ -675,31 +1041,36 @@ function renderLinuxUnit(ctx, stackUnits = [], venues = null) {
|
|
|
675
1041
|
* the wrapper's conversion to a clean 0 is exactly what KeepAlive/SuccessfulExit=false reads as
|
|
676
1042
|
* "leave it stopped" -- launchd's only way to spell RestartPreventExitStatus=2.
|
|
677
1043
|
*/
|
|
678
|
-
function renderPlist(ctx) {
|
|
1044
|
+
function renderPlist(ctx, { pathDirs = [] } = {}) {
|
|
679
1045
|
// Substituted before anything composed goes in (subDeployDir): the two anchors below are therefore
|
|
680
1046
|
// written in their post-substitution form, and PI_ENV_SETUP can no longer be rewritten into a path
|
|
681
1047
|
// the operator never typed. Only subDeployDir here — the template's /usr/bin/node is example prose.
|
|
682
|
-
|
|
1048
|
+
// Every string this render puts into the XML is escaped (`xmlText`, round 1 review of #708): a deployment folder
|
|
1049
|
+
// or a package path holding `&` or `<` produced a plist launchd refuses to load. No-op for every other path, so
|
|
1050
|
+
// those renders are byte-identical to before.
|
|
1051
|
+
let plist = readTemplate(ctx, "com.pi-dispatch.worker.plist").replaceAll("/opt/pi-dispatch", () => xmlText(ctx.deployDir));
|
|
683
1052
|
// One wrapper, two daemons: the exec argv IS the difference now. The wrapper sources ./.env in the
|
|
684
1053
|
// unit's WorkingDirectory and runs exactly these arguments — no `receiver` selector flag, no paths
|
|
685
1054
|
// guessed inside the wrapper (issue #96: the wrapper's self-relative guess broke under npm install).
|
|
686
|
-
const execArgv = ctx
|
|
687
|
-
if (ctx.which
|
|
1055
|
+
const execArgv = execArgvFor(ctx);
|
|
1056
|
+
if (ctx.which !== "worker") {
|
|
1057
|
+
// The console (issue #708) is derived exactly as the receiver is: `serve`'s refusals exit 2, which the wrapper
|
|
1058
|
+
// turns into the clean exit KeepAlive leaves stopped.
|
|
688
1059
|
plist = plist
|
|
689
|
-
.replace("<string>com.pi-dispatch.worker</string>",
|
|
690
|
-
.replaceAll("worker.out.log",
|
|
691
|
-
.replaceAll("worker.err.log",
|
|
1060
|
+
.replace("<string>com.pi-dispatch.worker</string>", () => `<string>com.pi-dispatch.${ctx.which}</string>`)
|
|
1061
|
+
.replaceAll("worker.out.log", () => `${ctx.which}.out.log`)
|
|
1062
|
+
.replaceAll("worker.err.log", () => `${ctx.which}.err.log`);
|
|
692
1063
|
}
|
|
693
1064
|
// The template's two-element ProgramArguments (sh + wrapper) becomes sh + the PACKAGE's wrapper +
|
|
694
1065
|
// the command: absolute node, absolute script. The wrapper path is module-relative (templatesDir),
|
|
695
1066
|
// so it exists in a checkout AND under node_modules — unlike the old repo-root guess.
|
|
696
1067
|
plist = plist.replace(
|
|
697
|
-
`<string>${ctx.deployDir}/deploy/worker-env-wrapper.sh</string>`,
|
|
1068
|
+
`<string>${xmlText(ctx.deployDir)}/deploy/worker-env-wrapper.sh</string>`,
|
|
698
1069
|
() =>
|
|
699
1070
|
[
|
|
700
|
-
`<string>${ctx.wrapperSh}</string>`,
|
|
701
|
-
"<!-- the command the wrapper execs after sourcing ./.env in WorkingDirectory - absolute paths, nothing guessed -->",
|
|
702
|
-
...execArgv.map((a) => `<string>${a}</string>`),
|
|
1071
|
+
`<string>${xmlText(ctx.wrapperSh)}</string>`,
|
|
1072
|
+
ctx.which === "console" ? "<!-- the command the wrapper execs in WorkingDirectory (it sources no .env for the console: PI_WRAPPER_NO_DOTENV below) - absolute paths, nothing guessed -->" : "<!-- the command the wrapper execs after sourcing ./.env in WorkingDirectory - absolute paths, nothing guessed -->",
|
|
1073
|
+
...execArgv.map((a) => `<string>${xmlText(a)}</string>`),
|
|
703
1074
|
].join("\n\t\t"),
|
|
704
1075
|
);
|
|
705
1076
|
// launchd's default PATH is /usr/bin:/bin — an nvm or Homebrew node is invisible to it. The exec
|
|
@@ -709,7 +1080,19 @@ function renderPlist(ctx) {
|
|
|
709
1080
|
// no-EnvironmentVariables stance is about credentials, which still live only in .env.
|
|
710
1081
|
// The dict is built as a list so the seam below can extend it without a second anchor. Everything
|
|
711
1082
|
// in it is a PATH; nothing in it is a credential, which is what keeps this file committable.
|
|
712
|
-
|
|
1083
|
+
// Issue #707: and the directories of the CLIs a venue spawns by name (the openshell CLI, resolved from this shell's
|
|
1084
|
+
// PATH at install time), which launchd's PATH cannot see either: a Homebrew `openshell` is in /opt/homebrew/bin.
|
|
1085
|
+
// With none, the line is byte-identical to the one before (review round 1 of #715), node in /usr/bin included.
|
|
1086
|
+
const fixed = ["/usr/bin", "/bin", "/usr/sbin", "/sbin"];
|
|
1087
|
+
const extra = [...new Set(pathDirs)].filter((d) => d !== dirname(ctx.execPath) && !fixed.includes(d));
|
|
1088
|
+
const envEntries = ["\t\t<key>PATH</key>", `\t\t<string>${xmlText([dirname(ctx.execPath), ...extra, ...fixed].join(":"))}</string>`];
|
|
1089
|
+
if (ctx.which === "console") {
|
|
1090
|
+
// Issue #708 (round 1 review): the console reads the deployment's .env itself (`serve --deployment`), only its own
|
|
1091
|
+
// keys and only from a file no other account can change, so the wrapper must not source the whole file into its
|
|
1092
|
+
// environment (every worker secret, each read as this "shell's" and so winning over the file). The wrapper tests
|
|
1093
|
+
// this before its load; PI_ENV_SETUP below still runs when configured.
|
|
1094
|
+
envEntries.push("\t\t<key>PI_WRAPPER_NO_DOTENV</key>", "\t\t<string>1</string>");
|
|
1095
|
+
}
|
|
713
1096
|
if (ctx.envSetup) {
|
|
714
1097
|
// launchd has no EnvironmentFile and no shell in front of ProgramArguments, so the seam is a
|
|
715
1098
|
// VARIABLE the wrapper reads rather than a composed command line: nothing here has to be quoted
|
|
@@ -718,10 +1101,10 @@ function renderPlist(ctx) {
|
|
|
718
1101
|
envEntries.push(
|
|
719
1102
|
"\t\t<!-- `pi-dispatch service --env-setup`: worker-env-wrapper.sh sources this file after ./.env.\n\t\t A path, not a credential. Re-render to change it. -->",
|
|
720
1103
|
"\t\t<key>PI_ENV_SETUP</key>",
|
|
721
|
-
`\t\t<string>${ctx.envSetup}</string>`,
|
|
1104
|
+
`\t\t<string>${xmlText(ctx.envSetup)}</string>`,
|
|
722
1105
|
);
|
|
723
1106
|
}
|
|
724
|
-
const workingDir = `<key>WorkingDirectory</key>\n\t<string>${ctx.deployDir}</string>`;
|
|
1107
|
+
const workingDir = `<key>WorkingDirectory</key>\n\t<string>${xmlText(ctx.deployDir)}</string>`;
|
|
725
1108
|
plist = plist.replace(
|
|
726
1109
|
workingDir,
|
|
727
1110
|
() =>
|
|
@@ -744,7 +1127,7 @@ function renderPlist(ctx) {
|
|
|
744
1127
|
function nssmSequence(ctx) {
|
|
745
1128
|
const service = `pi-dispatch-${ctx.which}`;
|
|
746
1129
|
const logDir = `${ctx.deployDir}\\logs`;
|
|
747
|
-
const execArgv = ctx
|
|
1130
|
+
const execArgv = execArgvFor(ctx);
|
|
748
1131
|
return {
|
|
749
1132
|
service,
|
|
750
1133
|
commands: [
|
|
@@ -753,7 +1136,9 @@ function nssmSequence(ctx) {
|
|
|
753
1136
|
// Same seam as the plist's EnvironmentVariables dict, for the same reason: nssm takes a
|
|
754
1137
|
// NAME=VALUE with no shell between it and the wrapper, so the path needs no quoting and
|
|
755
1138
|
// AppParameters stays byte-identical to the default render.
|
|
756
|
-
|
|
1139
|
+
// The console's PI_WRAPPER_NO_DOTENV (issue #708) travels the same way, in the same one property (nssm takes
|
|
1140
|
+
// several NAME=VALUE words there).
|
|
1141
|
+
...(ctx.envSetup || ctx.which === "console" ? [["set", service, "AppEnvironmentExtra", ...(ctx.envSetup ? [`PI_ENV_SETUP=${ctx.envSetup}`] : []), ...(ctx.which === "console" ? ["PI_WRAPPER_NO_DOTENV=1"] : [])]] : []),
|
|
757
1142
|
["set", service, "AppStdout", `${logDir}\\${ctx.which}.out.log`],
|
|
758
1143
|
["set", service, "AppStderr", `${logDir}\\${ctx.which}.err.log`],
|
|
759
1144
|
["set", service, "AppStopMethodConsole", "15000"],
|
|
@@ -779,8 +1164,10 @@ function nssmSequence(ctx) {
|
|
|
779
1164
|
* splits it) and a file it cannot read there is not a refusal: refusing an install over a key that could change
|
|
780
1165
|
* nothing about it would be a refusal with no remedy.
|
|
781
1166
|
*/
|
|
782
|
-
|
|
783
|
-
|
|
1167
|
+
const NO_OPENSHELL = Object.freeze(openshellSetupOf(null));
|
|
1168
|
+
|
|
1169
|
+
function deploymentVenue(ctx) {
|
|
1170
|
+
if (ctx.which !== "worker") return { used: false, openshell: NO_OPENSHELL };
|
|
784
1171
|
const envPath = join(ctx.deployDir, ".env");
|
|
785
1172
|
const linux = ctx.platform === "linux";
|
|
786
1173
|
let fileKeys = {};
|
|
@@ -790,18 +1177,18 @@ function podmanVenue(ctx) {
|
|
|
790
1177
|
// Bytes, not text: `readStackKeys` checks what systemd refuses to load before it decodes (issue #447).
|
|
791
1178
|
text = ctx.fs.readFileSync(envPath);
|
|
792
1179
|
} catch (err) {
|
|
793
|
-
if (!linux) return { used: false };
|
|
1180
|
+
if (!linux) return { used: false, openshell: NO_OPENSHELL };
|
|
794
1181
|
return { error: `cannot read ${envPath} to learn whether this deployment runs the podman venue: ${err?.message}` };
|
|
795
1182
|
}
|
|
796
1183
|
const loader = linux ? "systemd" : ctx.platform === "darwin" ? "shell" : "cmd";
|
|
797
1184
|
const read = readStackKeys(text, { loader, path: envPath });
|
|
798
|
-
if (read.error) return linux ? { error: read.error } : { used: false };
|
|
1185
|
+
if (read.error) return linux ? { error: read.error } : { used: false, openshell: NO_OPENSHELL };
|
|
799
1186
|
fileKeys = read.keys;
|
|
800
1187
|
// Issue #464: where the worker's queue is, and whether a Valkey another uid holds may be it (PI_VALKEY_SHARED), as
|
|
801
1188
|
// the service reads them, for the Valkey rule (`decideValkey`). A line the loaders read differently is refused,
|
|
802
1189
|
// naming what in it is the problem, rather than guessed at.
|
|
803
1190
|
const valkey = readValkeyKeys(text, { loader, path: envPath });
|
|
804
|
-
if (valkey.error) return linux ? { error: valkey.error } : { used: false };
|
|
1191
|
+
if (valkey.error) return linux ? { error: valkey.error } : { used: false, openshell: NO_OPENSHELL };
|
|
805
1192
|
fileKeys = { ...fileKeys, ...valkey.keys };
|
|
806
1193
|
}
|
|
807
1194
|
// Refused here, unlike doctor's venuesOf, which reads an unparseable list as `local`: doctor then reports the parse,
|
|
@@ -809,11 +1196,15 @@ function podmanVenue(ctx) {
|
|
|
809
1196
|
try {
|
|
810
1197
|
parseBackendList(fileKeys.PI_BACKENDS);
|
|
811
1198
|
} catch (err) {
|
|
812
|
-
return linux ? { error: `${envPath}: ${err.message}` } : { used: false };
|
|
1199
|
+
return linux ? { error: `${envPath}: ${err.message}` } : { used: false, openshell: NO_OPENSHELL };
|
|
813
1200
|
}
|
|
814
1201
|
const venues = venuesOf(fileKeys);
|
|
815
|
-
|
|
816
|
-
|
|
1202
|
+
// Issue #707: the openshell venues' setup by the one rule `up` follows (`openshellSetupOf`): which engine, and whether
|
|
1203
|
+
// this deployment's Valkey is a Quadlet unit on it. An engine the worker refuses at boot is refused here too, on Linux,
|
|
1204
|
+
// rather than installing a unit that starts into that refusal.
|
|
1205
|
+
const openshell = openshellSetupOf(venues, fileKeys);
|
|
1206
|
+
if (openshell.engineError && linux) return { error: `${envPath}: ${openshell.engineError}` };
|
|
1207
|
+
return { used: venues.podmanUsed, venues, env: fileKeys, openshell };
|
|
817
1208
|
}
|
|
818
1209
|
|
|
819
1210
|
/**
|
|
@@ -823,6 +1214,18 @@ function podmanVenue(ctx) {
|
|
|
823
1214
|
* the worker would race its own queue at every boot; and installing them means writing into a user's config from a
|
|
824
1215
|
* command whose system-scope doctrine is to write nothing. User scope with linger does both properly.
|
|
825
1216
|
*/
|
|
1217
|
+
/**
|
|
1218
|
+
* The refusal for an openshell venue in SYSTEM scope (issue #707), as podman's: the gateway is the `openshell-gateway` USER
|
|
1219
|
+
* unit, which a system unit cannot order itself after, and the CLI the worker drives keeps its gateway registration under
|
|
1220
|
+
* that user's own config directory, which a system unit's environment does not point at.
|
|
1221
|
+
*/
|
|
1222
|
+
function refuseOpenshellSystemScope(ctx) {
|
|
1223
|
+
return fail(
|
|
1224
|
+
ctx.err,
|
|
1225
|
+
`PI_BACKENDS in ${join(ctx.deployDir, ".env")} lists an openshell venue, and its gateway runs as ${OPENSHELL_GATEWAY_UNIT}, a unit of the worker account's OWN user manager, which a system unit cannot order itself after; the openshell CLI also keeps its gateway registration under that account's home. Install in user scope (drop --system) with linger on: sudo loginctl enable-linger ${ctx.user}`,
|
|
1226
|
+
);
|
|
1227
|
+
}
|
|
1228
|
+
|
|
826
1229
|
function refusePodmanSystemScope(ctx) {
|
|
827
1230
|
return fail(
|
|
828
1231
|
ctx.err,
|
|
@@ -836,10 +1239,11 @@ function refusePodmanSystemScope(ctx) {
|
|
|
836
1239
|
* `up` starts what is not running, while an install also keeps (and orders the worker after) a Valkey unit an
|
|
837
1240
|
* earlier run already installed, even though that Valkey is now the thing listening.
|
|
838
1241
|
*/
|
|
839
|
-
async function podmanStackFor(ctx, venue, { readKeeper = false } = {}) {
|
|
1242
|
+
async function podmanStackFor(ctx, venue, { readKeeper = false, egress = true } = {}) {
|
|
840
1243
|
let armed;
|
|
841
1244
|
try {
|
|
842
|
-
|
|
1245
|
+
// `egress: false` (issue #707): the openshell venue's Valkey alone, with no proxy and no keeper, whatever PI_EGRESS says.
|
|
1246
|
+
armed = egress ? egressArmed(venue.env) : false;
|
|
843
1247
|
} catch (err) {
|
|
844
1248
|
return { error: `${join(ctx.deployDir, ".env")}: ${err.message}` };
|
|
845
1249
|
}
|
|
@@ -910,12 +1314,17 @@ async function podmanStackFor(ctx, venue, { readKeeper = false } = {}) {
|
|
|
910
1314
|
|
|
911
1315
|
async function doRender(ctx) {
|
|
912
1316
|
if (ctx.which === "receiver" && !ctx.receiverStart) return refuseMissingReceiver(ctx);
|
|
1317
|
+
if (ctx.which === "console" && !ctx.consoleEntry) return refuseMissingConsole(ctx);
|
|
1318
|
+
const unrenderable = renderRefusal(ctx);
|
|
1319
|
+
if (unrenderable !== null) return unrenderable;
|
|
913
1320
|
const paths = unitPaths(ctx);
|
|
914
1321
|
if (ctx.platform === "darwin") {
|
|
915
1322
|
ctx.out(`# → ${paths.installPath}\n`);
|
|
916
|
-
ctx.out(renderPlist(ctx));
|
|
1323
|
+
ctx.out(renderPlist(ctx, { pathDirs: openshellPathDirs(ctx, deploymentVenue(ctx)) }));
|
|
917
1324
|
ctx.out(
|
|
918
|
-
|
|
1325
|
+
ctx.which === "console"
|
|
1326
|
+
? "\n# note: ProgramArguments runs the package's worker-env-wrapper.sh, which (PI_WRAPPER_NO_DOTENV=1) sources NO\n# .env for the console: `serve --deployment` reads the console's own keys from it. The wrapper converts serve's\n# refusal (exit 2) into a clean exit, so KeepAlive leaves it stopped instead of relaunching it.\n"
|
|
1327
|
+
: "\n# note: ProgramArguments runs the package's worker-env-wrapper.sh, which sources ./.env in the\n# WorkingDirectory above and then runs the argv that follows it — launchd has no EnvironmentFile.\n# The wrapper also converts a policy refusal (exit 2, EXIT_POLICY) into a clean exit, so KeepAlive\n# never relaunch-loops a refusal into a provider bill.\n",
|
|
919
1328
|
);
|
|
920
1329
|
if (ctx.envSetup) {
|
|
921
1330
|
ctx.out(
|
|
@@ -925,16 +1334,14 @@ async function doRender(ctx) {
|
|
|
925
1334
|
return 0;
|
|
926
1335
|
}
|
|
927
1336
|
if (ctx.platform === "linux") {
|
|
928
|
-
const venue =
|
|
1337
|
+
const venue = deploymentVenue(ctx);
|
|
929
1338
|
if (venue.error) return fail(ctx.err, venue.error);
|
|
930
|
-
|
|
931
|
-
if (venue.used)
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
if (stack.error) return fail(ctx.err, stack.error);
|
|
935
|
-
}
|
|
1339
|
+
if (ctx.scope === "system" && venue.used) return refusePodmanSystemScope(ctx);
|
|
1340
|
+
if (ctx.scope === "system" && venue.openshell.used) return refuseOpenshellSystemScope(ctx);
|
|
1341
|
+
const stack = await stackFor(ctx, venue);
|
|
1342
|
+
if (stack?.error) return fail(ctx.err, stack.error);
|
|
936
1343
|
ctx.out(`# → ${paths.installPath}\n`);
|
|
937
|
-
ctx.out(renderLinuxUnit(ctx, stack?.plan.start ?? [],
|
|
1344
|
+
ctx.out(renderLinuxUnit(ctx, stack?.plan.start ?? [], venue.venues ?? null, venue.openshell));
|
|
938
1345
|
// The proxy's rules copy is named, not printed (round 3 nit): it is the package's egress-proxy.conf verbatim, and
|
|
939
1346
|
// 4.7 KB of squid configuration under a "Quadlet" heading read as a unit file.
|
|
940
1347
|
for (const f of stack?.plan.files ?? []) {
|
|
@@ -956,6 +1363,13 @@ async function doRender(ctx) {
|
|
|
956
1363
|
|
|
957
1364
|
async function doInstall(ctx) {
|
|
958
1365
|
if (ctx.which === "receiver" && !ctx.receiverStart) return refuseMissingReceiver(ctx);
|
|
1366
|
+
if (ctx.which === "console" && !ctx.consoleEntry) return refuseMissingConsole(ctx);
|
|
1367
|
+
const unrenderable = renderRefusal(ctx);
|
|
1368
|
+
if (unrenderable !== null) return unrenderable;
|
|
1369
|
+
if (ctx.which === "console") {
|
|
1370
|
+
const refused = refuseConsoleWithoutOperator(ctx);
|
|
1371
|
+
if (refused !== null) return refused;
|
|
1372
|
+
}
|
|
959
1373
|
const paths = unitPaths(ctx);
|
|
960
1374
|
|
|
961
1375
|
// THE refusal, worker only: one worker per docker daemon (DES-CONCURRENCY-3). The worker's boot
|
|
@@ -973,30 +1387,56 @@ async function doInstall(ctx) {
|
|
|
973
1387
|
}
|
|
974
1388
|
}
|
|
975
1389
|
|
|
976
|
-
const venue =
|
|
1390
|
+
const venue = deploymentVenue(ctx);
|
|
977
1391
|
if (venue.error) return fail(ctx.err, venue.error);
|
|
1392
|
+
// Issue #707: no OpenShell gateway exists for Windows (the pinned release's assets were checked), so a worker unit for
|
|
1393
|
+
// this deployment would start into a venue that cannot run here. Refused before anything is written.
|
|
1394
|
+
if (ctx.platform === "win32" && venue.openshell.used) return fail(ctx.err, OPENSHELL_NO_WINDOWS);
|
|
978
1395
|
if (ctx.platform === "darwin" || ctx.platform === "win32") {
|
|
979
|
-
const code = ctx.platform === "darwin" ? await installDarwin(ctx, paths) : await installWindows(ctx);
|
|
1396
|
+
const code = ctx.platform === "darwin" ? await installDarwin(ctx, paths, venue) : await installWindows(ctx);
|
|
980
1397
|
// Said, never silently skipped: the venue refuses every host that is not Linux (podman-platform), so there is
|
|
981
1398
|
// no stack to install here, and a deployment listing it should know why nothing about Podman happened.
|
|
982
1399
|
if (code === 0 && venue.used) ctx.out("note: PI_BACKENDS lists podman, and the podman venue runs only on Linux (it refuses this host as podman-platform), so no podman stack was installed here\n");
|
|
983
1400
|
return code;
|
|
984
1401
|
}
|
|
985
|
-
if (ctx.scope === "system") return venue.used ? refusePodmanSystemScope(ctx) : installLinuxSystem(ctx, paths);
|
|
986
|
-
let stack = null;
|
|
1402
|
+
if (ctx.scope === "system") return venue.used ? refusePodmanSystemScope(ctx) : venue.openshell.used ? refuseOpenshellSystemScope(ctx) : installLinuxSystem(ctx, paths);
|
|
987
1403
|
let foreign = [];
|
|
988
|
-
|
|
989
|
-
|
|
1404
|
+
const stack = await stackFor(ctx, venue, { readKeeper: true });
|
|
1405
|
+
if (stack) {
|
|
990
1406
|
if (stack.error) return fail(ctx.err, stack.error);
|
|
991
1407
|
// Every stack refusal first, the unit's own among them, so installLinuxUser's unit check never fires alone.
|
|
992
1408
|
const refused = await stackRefusal(ctx, paths, stack);
|
|
993
1409
|
if (refused.error) return fail(ctx.err, refused.error);
|
|
994
1410
|
foreign = refused.foreign;
|
|
995
1411
|
}
|
|
996
|
-
return installLinuxUser(ctx, paths, stack, foreign);
|
|
1412
|
+
return installLinuxUser(ctx, paths, stack, foreign, venue);
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
/**
|
|
1416
|
+
* The Quadlet stack this deployment's worker unit is installed with, or null: the podman venue's (Valkey, the proxy and
|
|
1417
|
+
* the keeper, by their own rules), or, for an openshell venue on Podman that no `local` or `podman` venue decides the
|
|
1418
|
+
* queue for, its Valkey alone (issue #707, `openshellSetupOf`'s `valkeyEngine`), with no proxy and no keeper.
|
|
1419
|
+
*/
|
|
1420
|
+
async function stackFor(ctx, venue, opts = {}) {
|
|
1421
|
+
if (venue.used) return podmanStackFor(ctx, venue, opts);
|
|
1422
|
+
if (venue.openshell?.valkeyEngine === "podman") return podmanStackFor(ctx, venue, { ...opts, egress: false });
|
|
1423
|
+
return null;
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
/**
|
|
1427
|
+
* The directory of the `openshell` CLI as this shell finds it (issue #707), for the plist's PATH: launchd's own PATH
|
|
1428
|
+
* cannot see a Homebrew install, and the worker spawns the CLI by name. Empty where no openshell venue is blessed, or
|
|
1429
|
+
* where it is not found (then `installDarwin` says so).
|
|
1430
|
+
*/
|
|
1431
|
+
function openshellPathDirs(ctx, venue) {
|
|
1432
|
+
if (!venue?.openshell?.used) return [];
|
|
1433
|
+
const found = ctx.findBin("openshell");
|
|
1434
|
+
// A directory the plist cannot carry as written (review round 1 of #715): `&`, `<`, `>`, `"` or a control byte would
|
|
1435
|
+
// break or change the XML, the refusal `--env-setup` makes for the same file. Left out, and said by installDarwin.
|
|
1436
|
+
return found && !ENV_SETUP_REFUSALS.darwin.re.test(dirname(found)) ? [dirname(found)] : [];
|
|
997
1437
|
}
|
|
998
1438
|
|
|
999
|
-
async function installDarwin(ctx, paths) {
|
|
1439
|
+
async function installDarwin(ctx, paths, venue = null) {
|
|
1000
1440
|
// Root refusal (the svc.sh darwin precedent): sudo would bootstrap into ROOT's gui domain and
|
|
1001
1441
|
// write root's LaunchAgents — a unit the operator's own session neither sees nor controls, running
|
|
1002
1442
|
// outside the login session that owns Docker Desktop.
|
|
@@ -1023,7 +1463,7 @@ async function installDarwin(ctx, paths) {
|
|
|
1023
1463
|
// launchd creates the StandardOutPath FILES but not their parent directory: without this the job
|
|
1024
1464
|
// spawns and dies with its error unwritable. Done at install, not render: render stays read-only.
|
|
1025
1465
|
ctx.fs.mkdirSync(join(ctx.deployDir, "logs"), { recursive: true });
|
|
1026
|
-
journalWrite(ctx.fs, journal, paths.installPath, renderPlist(ctx));
|
|
1466
|
+
journalWrite(ctx.fs, journal, paths.installPath, renderPlist(ctx, { pathDirs: openshellPathDirs(ctx, venue) }));
|
|
1027
1467
|
} catch (err) {
|
|
1028
1468
|
return fail(ctx.err, `could not write ${paths.installPath} (${err?.message ?? err}), so nothing was installed: ${describeRollBack(rollBackWrites(ctx.fs, journal))}`);
|
|
1029
1469
|
}
|
|
@@ -1035,9 +1475,20 @@ async function installDarwin(ctx, paths) {
|
|
|
1035
1475
|
if (enable !== 0) return fail(ctx.err, `launchctl enable gui/${ctx.euid}/${paths.name} failed (exit ${enable})`);
|
|
1036
1476
|
ctx.out(`installed ${paths.name} → ${paths.installPath} (bootstrapped into gui/${ctx.euid}; RunAtLoad starts it now and on login)\n`);
|
|
1037
1477
|
// The honest note: no pretending a LaunchAgent is a boot daemon. It is the right fit anyway.
|
|
1478
|
+
// The Docker Desktop reason is the worker's (issue #708): the console starts no container, and telling its operator
|
|
1479
|
+
// about a docker socket would explain a choice by a fact that does not apply to it.
|
|
1038
1480
|
ctx.out(
|
|
1039
|
-
|
|
1481
|
+
ctx.which === "console"
|
|
1482
|
+
? "note: a LaunchAgent is LOGIN-scoped — it runs while you are logged in, not from boot, which is when a browser on this Mac can reach it.\n"
|
|
1483
|
+
: "note: a LaunchAgent is LOGIN-scoped — it runs while you are logged in, not from boot. That is the honest fit here: Docker Desktop is itself login-scoped, so a boot-time daemon would only wait on a docker socket that appears at login anyway.\n",
|
|
1040
1484
|
);
|
|
1485
|
+
// Issue #707: the openshell venue's gateway, which a LaunchAgent cannot order itself after; Homebrew's service runs it
|
|
1486
|
+
// in the same login session. And the CLI the worker spawns by name, whose directory is now on the plist's PATH.
|
|
1487
|
+
if (venue?.openshell?.used) {
|
|
1488
|
+
const found = ctx.findBin("openshell");
|
|
1489
|
+
if (found && ENV_SETUP_REFUSALS.darwin.re.test(dirname(found))) ctx.out(`⚠ the openshell venue: this shell finds \`openshell\` in ${JSON.stringify(dirname(found))}, which the plist cannot carry (${ENV_SETUP_REFUSALS.darwin.why}), so its PATH does not name it and the worker will not find the CLI. Install it somewhere without those characters, then re-run with --force\n`);
|
|
1490
|
+
else ctx.out(found ? `note: the openshell venue: the plist's PATH carries ${dirname(found)}, where this shell finds \`openshell\`. Keep its gateway running in this login session: \`brew services start openshell\` (the service of OpenShell's release formula, \`${OPENSHELL_BREW_INSTALL}\`; homebrew-core's formula defines none)\n` : `⚠ the openshell venue: \`openshell\` is not on this shell's PATH, so the plist's PATH carries no directory for it and the worker will not find the CLI. Install OpenShell's release formula (\`${OPENSHELL_BREW_INSTALL}\`), then re-run with --force; its service keeps the gateway running in this login session: \`brew services start openshell\`\n`);
|
|
1491
|
+
}
|
|
1041
1492
|
return 0;
|
|
1042
1493
|
}
|
|
1043
1494
|
|
|
@@ -1091,7 +1542,7 @@ async function stackRefusal(ctx, paths, stack) {
|
|
|
1091
1542
|
return { foreign };
|
|
1092
1543
|
}
|
|
1093
1544
|
|
|
1094
|
-
async function installLinuxUser(ctx, paths, stack = null, foreign = []) {
|
|
1545
|
+
async function installLinuxUser(ctx, paths, stack = null, foreign = [], venue = null) {
|
|
1095
1546
|
if (ctx.fs.existsSync(paths.installPath) && !ctx.force) {
|
|
1096
1547
|
return fail(ctx.err, `${paths.installPath} already exists — pass --force to replace it (same non-clobber contract as init)`);
|
|
1097
1548
|
}
|
|
@@ -1104,7 +1555,7 @@ async function installLinuxUser(ctx, paths, stack = null, foreign = []) {
|
|
|
1104
1555
|
const writeUnit = () => {
|
|
1105
1556
|
try {
|
|
1106
1557
|
ctx.fs.mkdirSync(dirname(paths.installPath), { recursive: true });
|
|
1107
|
-
journalWrite(ctx.fs, journal, paths.installPath, renderLinuxUnit(ctx, stack?.plan.start ?? [], stack?.venues ?? null));
|
|
1558
|
+
journalWrite(ctx.fs, journal, paths.installPath, renderLinuxUnit(ctx, stack?.plan.start ?? [], venue?.venues ?? stack?.venues ?? null, venue?.openshell ?? null));
|
|
1108
1559
|
return { ok: true };
|
|
1109
1560
|
} catch (err) {
|
|
1110
1561
|
return { ok: false, failed: `write ${paths.installPath}`, code: null, message: err?.message };
|
|
@@ -1130,7 +1581,7 @@ async function installLinuxUser(ctx, paths, stack = null, foreign = []) {
|
|
|
1130
1581
|
if (passwordWarning) ctx.out(`⚠ ${passwordWarning}\n`);
|
|
1131
1582
|
if (stack.plan.actions.length > 0) {
|
|
1132
1583
|
// Shown, then done: the same lines `up` asks consent for, carried out by the same function.
|
|
1133
|
-
ctx.out(
|
|
1584
|
+
ctx.out(`${venue?.used === false ? "openshell venue on Podman (PI_BACKENDS and PI_OPENSHELL_ENGINE in .env): its Valkey" : "podman venue (PI_BACKENDS in .env): its stack"}, as Quadlet units in ${stack.plan.dir}:\n`);
|
|
1134
1585
|
for (const action of stack.plan.actions) ctx.out(` ${describeAction(action)}\n`);
|
|
1135
1586
|
const applied = await applyStack(stack.plan, { fs: ctx.fs, run: (cmd, args) => run(ctx, cmd, args), journal, beforeRuns: writeUnit });
|
|
1136
1587
|
if (!applied.ok) {
|
|
@@ -1162,6 +1613,7 @@ async function installLinuxUser(ctx, paths, stack = null, foreign = []) {
|
|
|
1162
1613
|
return fail(ctx.err, `systemctl --user enable --now ${paths.name} failed (exit ${enable}); the unit is written at ${paths.installPath}; \`systemctl --user status ${paths.name}\` has the details (files this run wrote, which remain: ${writtenList()})`);
|
|
1163
1614
|
}
|
|
1164
1615
|
ctx.out(`installed ${paths.name} → ${paths.installPath} (enabled and started in your user manager)\n`);
|
|
1616
|
+
if (venue?.openshell?.used) ctx.out(`${paths.name} Wants= ${OPENSHELL_GATEWAY_UNIT} (not After=, whose ordering would be a cycle through default.target), the OpenShell gateway's own user unit (\`systemctl --user enable --now ${OPENSHELL_GATEWAY_UNIT}\` starts it at boot too)\n`);
|
|
1165
1617
|
if (stack) {
|
|
1166
1618
|
// Said as it happened: `start` on a unit already active is a no-op, so a unit whose file this run replaced was
|
|
1167
1619
|
// RESTARTED (planStack), and the line names which is which rather than calling everything "started".
|
|
@@ -1482,6 +1934,10 @@ async function startStop(ctx, verb) {
|
|
|
1482
1934
|
}
|
|
1483
1935
|
let code;
|
|
1484
1936
|
if (ctx.platform === "darwin") {
|
|
1937
|
+
if (verb === "start" && ctx.setupExpected?.loaded === false) {
|
|
1938
|
+
const loaded = await run(ctx, "launchctl", ["bootstrap", `gui/${ctx.euid}`, paths.installPath]);
|
|
1939
|
+
if (loaded !== 0) return fail(ctx.err, "the previewed LaunchAgent could not be loaded");
|
|
1940
|
+
}
|
|
1485
1941
|
code =
|
|
1486
1942
|
verb === "start"
|
|
1487
1943
|
? await run(ctx, "launchctl", ["kickstart", `gui/${ctx.euid}/${paths.name}`])
|
|
@@ -1507,9 +1963,28 @@ async function doRestart(ctx, values) {
|
|
|
1507
1963
|
if (!Number.isFinite(timeoutS) || timeoutS <= 0) {
|
|
1508
1964
|
return fail(ctx.err, `--drain-timeout must be a positive number of seconds, got: ${values["drain-timeout"]}`);
|
|
1509
1965
|
}
|
|
1510
|
-
|
|
1966
|
+
// Which queues (issue #804): the ones this host's worker drains, as `createWorker` builds them, the shared queue and,
|
|
1967
|
+
// when the deployment declares PI_WORKER_NAME, `pi-jobs@<name>`. The shared queue alone let the restart go ahead while
|
|
1968
|
+
// a cron, chained or manual run from the host queue still ran, and the stop aborted it. The name is read as
|
|
1969
|
+
// `pi-dispatch run` reads it, this shell's else the deployment `.env`'s, and a disagreement or a line the service's
|
|
1970
|
+
// loader may read differently refuses: guessing would drain a queue the worker does not run, or miss the one it does.
|
|
1971
|
+
const deployment = cliDeploymentEnv(ctx.env, ["PI_WORKER_NAME"], { cwd: ctx.deployDir, platform: ctx.platform, readFile: (p) => ctx.fs.readFileSync(p) });
|
|
1972
|
+
if (deployment.problem) return fail(ctx.err, `${deployment.problem}. Nothing was paused.`);
|
|
1973
|
+
// Lazy, as below: `service` subcommands that never touch the queue must not load bullmq/ioredis.
|
|
1974
|
+
const { QUEUE, hostQueueName } = await import("./queue.mjs");
|
|
1975
|
+
let names = [QUEUE];
|
|
1976
|
+
if (deployment.env.PI_WORKER_NAME) {
|
|
1977
|
+
// The worker's own rule (`workerName`), so a name it refuses at boot is refused here, never drained as typed.
|
|
1978
|
+
const { workerName } = await import("./config.mjs");
|
|
1979
|
+
try {
|
|
1980
|
+
names = [QUEUE, hostQueueName(workerName(deployment.env))];
|
|
1981
|
+
} catch (error) {
|
|
1982
|
+
return fail(ctx.err, `${error.message}. Nothing was paused.`);
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
let queueFor = ctx.queueFor;
|
|
1511
1986
|
let url;
|
|
1512
|
-
if (!
|
|
1987
|
+
if (!queueFor) {
|
|
1513
1988
|
// VALKEY_URL only, exactly like cli.mjs's pause/resume: the drain must work even when the rest
|
|
1514
1989
|
// of the config (forge auth …) is broken, and failFast keeps a down Valkey an error in seconds
|
|
1515
1990
|
// instead of a hung restart. Lazy imports for the same reason cli.mjs uses them: `service`
|
|
@@ -1518,54 +1993,111 @@ async function doRestart(ctx, values) {
|
|
|
1518
1993
|
const { cliValkeyUrl, parseConnection } = await import("./connection.mjs");
|
|
1519
1994
|
url = cliValkeyUrl(ctx.env, { cwd: ctx.deployDir, warn: (line) => ctx.err(line) });
|
|
1520
1995
|
const { makeQueue } = await import("./queue.mjs");
|
|
1521
|
-
|
|
1522
|
-
}
|
|
1996
|
+
queueFor = (name) => makeQueue(parseConnection(url, { failFast: true }), { name });
|
|
1997
|
+
}
|
|
1998
|
+
const queues = [];
|
|
1999
|
+
// The queues THIS drain paused and has not resumed yet. Only these are resumed at the end: a queue an operator paused
|
|
2000
|
+
// before the drain (`pi-dispatch pause`, say) stays paused after it, where the drain used to resume it regardless.
|
|
2001
|
+
const held = [];
|
|
2002
|
+
// The queues paused before this drain, which it leaves as it found them.
|
|
2003
|
+
const kept = [];
|
|
2004
|
+
const and = (list) => list.join(" and ");
|
|
2005
|
+
const keptWords = () => `${and(kept)} ${kept.length === 1 ? "was" : "were"} paused before the drain and ${kept.length === 1 ? "stays" : "stay"} paused`;
|
|
2006
|
+
// The words an operator acts on, naming the queues this drain left paused.
|
|
2007
|
+
const staysPaused = (list) => (list.length === 1 ? `the queue ${list[0]} STAYS PAUSED` : `each queue (${list.join(", ")}) STAYS PAUSED`);
|
|
2008
|
+
// The way back from a drain that stopped short. `pi-dispatch resume` is the deployment's resume and takes every queue
|
|
2009
|
+
// at once, so the advice says so: told only to run it, an operator would also resume a queue paused before the drain,
|
|
2010
|
+
// which the drain itself leaves as it found it.
|
|
2011
|
+
const resumeNote = () => `\`pi-dispatch resume\` resumes every queue of the deployment${kept.length > 0 ? ` (${and(kept)} too, though ${kept.length === 1 ? "it was" : "they were"} paused before the drain)` : ""}`;
|
|
2012
|
+
// `then` is what the operator does first. With nothing paused by this drain there is nothing of its own to resume.
|
|
2013
|
+
const leftPaused = (then) => (held.length > 0 ? `${staysPaused(held)}; ${then}, then resume: ${resumeNote()}` : `this drain paused no queue (${keptWords()}); ${then}`);
|
|
1523
2014
|
try {
|
|
1524
|
-
|
|
1525
|
-
|
|
2015
|
+
// One queue at a time, each opened only once the one before it answered, and INSIDE the try, so `finally` closes
|
|
2016
|
+
// every handle opened (a throw while opening the second cannot leak the first). Opening them all first left the
|
|
2017
|
+
// second still connecting when the first could not reach Valkey, and closing a BullMQ queue mid-connect makes it
|
|
2018
|
+
// emit an `error` nothing listens for: an unhandled rejection printed after the right message.
|
|
2019
|
+
const before = [];
|
|
2020
|
+
for (const name of names) {
|
|
2021
|
+
const q = queueFor(name);
|
|
2022
|
+
queues.push(q);
|
|
2023
|
+
before.push((await q.isPaused()) === true);
|
|
2024
|
+
}
|
|
2025
|
+
kept.push(...names.filter((_, i) => before[i]));
|
|
2026
|
+
for (const [i, q] of queues.entries()) {
|
|
2027
|
+
if (before[i]) continue;
|
|
2028
|
+
await q.pause();
|
|
2029
|
+
held.push(names[i]);
|
|
2030
|
+
}
|
|
2031
|
+
if (kept.length === 0) ctx.out(`paused ${and(held)}: no new jobs will start; waiting for their active jobs to finish\n`);
|
|
2032
|
+
else ctx.out(`${held.length > 0 ? `paused ${and(held)}; ` : ""}${keptWords()}: no new jobs will start; waiting for the active jobs on ${and(names)} to finish\n`);
|
|
1526
2033
|
const deadline = ctx.now() + timeoutS * 1000;
|
|
1527
|
-
|
|
2034
|
+
// Every queue's active count, summed: a job running from either one is a job the restart would abort.
|
|
2035
|
+
const activeNow = async () => {
|
|
2036
|
+
const per = await Promise.all(queues.map((q) => q.getJobCounts("active").then((c) => Number(c?.active ?? 0))));
|
|
2037
|
+
return { active: per.reduce((a, b) => a + b, 0), shown: names.length > 1 ? ` (${names.map((n, i) => `${n} ${per[i]}`).join(", ")})` : "" };
|
|
2038
|
+
};
|
|
2039
|
+
let { active, shown } = await activeNow();
|
|
1528
2040
|
while (active > 0) {
|
|
1529
2041
|
if (ctx.now() >= deadline) {
|
|
1530
2042
|
// Deliberately NO resume and NO restart: a job is still running. Restarting would abort
|
|
1531
2043
|
// it; resuming would feed new jobs toward a restart that is still owed. Paused is the
|
|
1532
2044
|
// safe durable state (it survives the restart the operator will now do by hand).
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
);
|
|
2045
|
+
const stays = held.length > 0 ? `${staysPaused(held)} so the running job can finish undisturbed. Investigate (pi-dispatch status), then restart and resume yourself: ${resumeNote()}.` : `this drain paused no queue (${keptWords()}). Investigate (pi-dispatch status), then restart yourself.`;
|
|
2046
|
+
ctx.out(`drain timed out after ${timeoutS}s with ${active} job(s) still active${shown}: NOT restarting.\n${stays[0].toUpperCase()}${stays.slice(1)}\n`);
|
|
1536
2047
|
return 1;
|
|
1537
2048
|
}
|
|
1538
|
-
ctx.out(` ${active} active
|
|
2049
|
+
ctx.out(` ${active} active${shown}, waiting\n`);
|
|
1539
2050
|
await ctx.sleep(2000);
|
|
1540
|
-
({ active
|
|
2051
|
+
({ active, shown } = await activeNow());
|
|
1541
2052
|
}
|
|
1542
2053
|
// A HELD job is neither active nor waiting, so the loop above has just reported a drained queue with
|
|
1543
2054
|
// however many jobs still parked on `run.waitFor` (issue #230). They are safe -- a hold spends
|
|
1544
2055
|
// nothing, survives a restart and reserves no slot -- but the operator is upgrading, and those jobs
|
|
1545
2056
|
// will wake against the new version. Said plainly rather than left to be discovered, which is what
|
|
1546
2057
|
// this command would otherwise be doing: reporting a drained queue it cannot see all of.
|
|
1547
|
-
const delayed = await
|
|
2058
|
+
const delayed = (await Promise.all(queues.map((q) => q.getJobCounts("delayed").then((c) => Number(c?.delayed ?? 0), () => 0)))).reduce((a, b) => a + b, 0);
|
|
1548
2059
|
if (delayed > 0) {
|
|
1549
2060
|
ctx.out(`note: ${delayed} job(s) sit in the delayed set (cron next-occurrences, retry backoff, quiet hours, scope/host deferrals, or jobs held on run.waitFor). None is active, so none blocked this drain; they will wake against the new version.\n`);
|
|
1550
2061
|
}
|
|
1551
2062
|
const stopped = await doStop(ctx);
|
|
1552
2063
|
if (stopped !== 0) {
|
|
1553
|
-
ctx.out(
|
|
2064
|
+
ctx.out(`restart did not happen: ${leftPaused("fix the service")}.\n`);
|
|
1554
2065
|
return 1;
|
|
1555
2066
|
}
|
|
1556
2067
|
const started = await doStart(ctx);
|
|
1557
2068
|
if (started !== 0) {
|
|
1558
|
-
ctx.out(
|
|
2069
|
+
ctx.out(`the service did not come back: ${leftPaused("fix the service")}.\n`);
|
|
1559
2070
|
return 1;
|
|
1560
2071
|
}
|
|
1561
|
-
|
|
1562
|
-
|
|
2072
|
+
const ours = [...held];
|
|
2073
|
+
for (const name of ours) {
|
|
2074
|
+
await queues[names.indexOf(name)].resume();
|
|
2075
|
+
held.shift();
|
|
2076
|
+
}
|
|
2077
|
+
ctx.out(`${ours.length > 0 ? `resumed ${and(ours)}: ` : ""}drained restart complete${kept.length > 0 ? `; ${keptWords()}` : ""}\n`);
|
|
1563
2078
|
return 0;
|
|
1564
2079
|
} catch (error) {
|
|
1565
|
-
|
|
2080
|
+
// Name what this drain left paused: a failure after the first pause (the second pause refused, Valkey gone mid-wait,
|
|
2081
|
+
// a resume refused partway) leaves the deployment half switched, and "could not reach Valkey" alone reads as
|
|
2082
|
+
// "nothing happened".
|
|
2083
|
+
const left = held.length > 0 ? `${leftPaused("wait until Valkey answers")}.` : "Nothing was paused.";
|
|
2084
|
+
return fail(ctx.err, `could not reach Valkey: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n ${error.message}\n ${left}`);
|
|
1566
2085
|
} finally {
|
|
1567
|
-
await
|
|
2086
|
+
for (const q of queues) await q.close().catch(() => {});
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
/** The first executable regular file named `name` in a PATH string, or null (issue #707's lookup seam's default). */
|
|
2091
|
+
export function findOnPath(name, path, { access = (p) => accessSync(p, fsConstants.X_OK), isFile = (p) => statSync(p).isFile() } = {}) {
|
|
2092
|
+
for (const dir of String(path ?? "").split(delimiter)) {
|
|
2093
|
+
if (dir === "" || !dir.startsWith("/")) continue;
|
|
2094
|
+
const candidate = join(dir, name);
|
|
2095
|
+
try {
|
|
2096
|
+
access(candidate);
|
|
2097
|
+
if (isFile(candidate)) return candidate;
|
|
2098
|
+
} catch {}
|
|
1568
2099
|
}
|
|
2100
|
+
return null;
|
|
1569
2101
|
}
|
|
1570
2102
|
|
|
1571
2103
|
function fail(err, message) {
|