@edgehero/pi-dispatch 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +94 -4
- package/README.md +20 -7
- package/deploy/console.service +52 -0
- package/deploy/docker-compose.yml +3 -1
- package/deploy/pi-dispatch-valkey.container +4 -2
- package/deploy/sandbox-gateway.service +23 -0
- package/deploy/worker-env-wrapper.cmd +6 -0
- package/deploy/worker-env-wrapper.sh +10 -1
- package/package.json +117 -19
- package/src/agent-snapshot.mjs +49 -0
- package/src/answerer-runtime.mjs +194 -0
- package/src/answerer.mjs +178 -0
- package/src/approval-policy.mjs +76 -0
- package/src/approval-runtime.mjs +188 -0
- package/src/approval-session.mjs +137 -0
- package/src/approval-state.mjs +170 -0
- package/src/auth-cli.mjs +86 -0
- package/src/auth-connections.mjs +145 -0
- package/src/auth-policy.mjs +32 -0
- package/src/auth-store.mjs +97 -0
- package/src/azure-host.mjs +66 -4
- package/src/azure-prompt.mjs +5 -4
- package/src/backend-conformance.mjs +34 -2
- package/src/backend-local.mjs +103 -34
- package/src/backend-openshell.mjs +1540 -0
- package/src/backend-podman.mjs +63 -2
- package/src/backend-registry.mjs +61 -4
- package/src/backends.mjs +395 -6
- package/src/bullmq-internals.mjs +208 -0
- package/src/canary-cli.mjs +28 -0
- package/src/canary-lock.mjs +41 -0
- package/src/canary-runtime.mjs +237 -0
- package/src/canary.mjs +53 -0
- package/src/cancel-cli.mjs +47 -34
- package/src/cancel-state.mjs +213 -15
- package/src/capabilities.mjs +36 -1
- package/src/capacity-cli.mjs +28 -8
- package/src/capacity-records.mjs +75 -16
- package/src/capacity.mjs +65 -0
- package/src/chat-cli.mjs +288 -0
- package/src/chat-fork.mjs +547 -0
- package/src/chat-mode.mjs +156 -0
- package/src/chat-view.mjs +720 -0
- package/src/checkpoint-paths.mjs +113 -0
- package/src/cli.mjs +128 -41
- package/src/comment-tables.mjs +322 -0
- package/src/community-auth-adapters.mjs +87 -0
- package/src/community-auth-helper.mjs +76 -0
- package/src/community-auth-metadata.mjs +6 -0
- package/src/community-auth-output-pins.mjs +38 -0
- package/src/community-auth-patches.mjs +168 -0
- package/src/community-auth-pins.mjs +38 -0
- package/src/config.mjs +199 -30
- package/src/connection.mjs +83 -7
- package/src/console-answer.mjs +113 -0
- package/src/console-audit.mjs +604 -0
- package/src/console-bootstrap.mjs +159 -0
- package/src/console-client.mjs +848 -0
- package/src/console-probe.mjs +106 -0
- package/src/console-tokens.mjs +459 -0
- package/src/container-names.mjs +51 -0
- package/src/container-spec.mjs +20 -1
- package/src/cpu-reserve.mjs +4 -1
- package/src/defer-reasons.mjs +200 -0
- package/src/defer-write.mjs +208 -0
- package/src/deployment-venue.mjs +23 -6
- package/src/docker-run.mjs +11 -0
- package/src/doctor-bundles.mjs +130 -0
- package/src/doctor-model-proposals.mjs +109 -0
- package/src/doctor-openshell.mjs +238 -0
- package/src/doctor-results.mjs +602 -0
- package/src/doctor-spawner.mjs +215 -0
- package/src/doctor.mjs +1663 -198
- package/src/dollar-budget.mjs +212 -3
- package/src/endpoint-status.mjs +100 -0
- package/src/env-allowlist.mjs +103 -9
- package/src/env-file.mjs +6 -4
- package/src/file-expect.mjs +47 -0
- package/src/forgejo-auth.mjs +4 -0
- package/src/forgejo-host.mjs +127 -31
- package/src/forgejo-prompt.mjs +5 -4
- package/src/forges.mjs +73 -15
- package/src/fork-job.mjs +58 -0
- package/src/fork-policy.mjs +23 -0
- package/src/fork-store.mjs +338 -0
- package/src/get-token.mjs +1 -1
- package/src/git-dirty.mjs +119 -14
- package/src/git-hardening.mjs +85 -0
- package/src/github-host.mjs +31 -6
- package/src/github-prompt.mjs +7 -16
- package/src/gitlab-host.mjs +42 -3
- package/src/gitlab-prompt.mjs +5 -4
- package/src/held-document.mjs +23 -0
- package/src/held-replies.mjs +325 -0
- package/src/held-settle.mjs +266 -0
- package/src/hook-runs.mjs +188 -0
- package/src/host-budget.mjs +8 -5
- package/src/host-clone.mjs +72 -0
- package/src/host-loads.mjs +557 -0
- package/src/host-registry.mjs +66 -6
- package/src/host-telemetry.mjs +92 -0
- package/src/http-body.mjs +78 -0
- package/src/image-preflight.mjs +156 -13
- package/src/import-pi.mjs +60 -172
- package/src/inbox-relay.mjs +482 -0
- package/src/inbox-settle.mjs +196 -0
- package/src/index.mjs +740 -129
- package/src/init.mjs +35 -5
- package/src/job-data-version.mjs +75 -0
- package/src/job-id.mjs +30 -15
- package/src/job-layout.mjs +22 -0
- package/src/job-size.mjs +16 -0
- package/src/live-jobs.mjs +14 -6
- package/src/live-probes.mjs +6 -2
- package/src/live-transcript.mjs +872 -0
- package/src/loaded-record.mjs +609 -0
- package/src/mcp-config.mjs +1 -1
- package/src/mcp-preflight.mjs +36 -5
- package/src/model-attempts.mjs +84 -0
- package/src/model-catalog.mjs +83 -0
- package/src/model-cooldowns.mjs +22 -0
- package/src/model-failover-protocol.mjs +11 -0
- package/src/model-failover.mjs +134 -0
- package/src/model-observation.mjs +50 -0
- package/src/model-proposal-protocol.mjs +21 -0
- package/src/model-proposals.mjs +75 -0
- package/src/model-ref.mjs +67 -10
- package/src/model-roles-file.mjs +24 -0
- package/src/model-roles.mjs +73 -0
- package/src/model-status.mjs +34 -0
- package/src/models-json.mjs +4 -1
- package/src/money.mjs +1 -1
- package/src/oauth-openai.mjs +121 -0
- package/src/older-state.mjs +249 -0
- package/src/on-failure.mjs +17 -13
- package/src/open-browser.mjs +31 -2
- package/src/opener-cli.mjs +135 -0
- package/src/opener-protocol.mjs +27 -0
- package/src/opener-terminal.mjs +128 -0
- package/src/openshell-cli.mjs +155 -0
- package/src/openshell-policy.mjs +163 -0
- package/src/outbox.mjs +14 -0
- package/src/output-cap.mjs +1 -0
- package/src/package-canary-record.mjs +26 -0
- package/src/package-canary.mjs +135 -0
- package/src/package-names.mjs +9 -0
- package/src/package-stage.mjs +418 -0
- package/src/packages.mjs +24 -16
- package/src/parked.mjs +2 -1
- package/src/pause-windows.mjs +136 -17
- package/src/podman-stack.mjs +40 -14
- package/src/portfolio-snapshot.mjs +26 -7
- package/src/prepare-github.mjs +41 -77
- package/src/prepare-local.mjs +32 -2
- package/src/prepare.mjs +45 -11
- package/src/private-store-boundary.mjs +70 -0
- package/src/processor.mjs +1554 -167
- package/src/projects.mjs +23 -0
- package/src/prompt-fence.mjs +18 -0
- package/src/provider-key.mjs +7 -0
- package/src/provider-steering.mjs +1 -0
- package/src/question-context.mjs +35 -0
- package/src/question-group-runtime.mjs +91 -0
- package/src/question-groups.mjs +171 -0
- package/src/queue.mjs +296 -55
- package/src/reason-catalog.mjs +448 -0
- package/src/receiver-status.mjs +230 -0
- package/src/record-reason.mjs +37 -0
- package/src/recovery-queue.mjs +566 -0
- package/src/redact-secrets.mjs +219 -0
- package/src/reserve-estimate.mjs +101 -0
- package/src/reserved-env.mjs +36 -1
- package/src/retention.mjs +67 -0
- package/src/run-budget-ledger.mjs +283 -0
- package/src/run-checkpoint.mjs +627 -0
- package/src/run-container.mjs +749 -33
- package/src/run-history.mjs +791 -45
- package/src/run-mirror.mjs +75 -7
- package/src/run-progress.mjs +751 -0
- package/src/runtime-observations.mjs +3 -1
- package/src/runtime-settings.mjs +272 -37
- package/src/sandbox-cli.mjs +78 -2
- package/src/sandbox-deployment.mjs +51 -0
- package/src/sandbox-gateway-protocol.mjs +378 -0
- package/src/sandbox-hosts.mjs +100 -0
- package/src/sandbox-inventory.mjs +83 -0
- package/src/sandbox-policy.mjs +225 -0
- package/src/sandbox-prepare-requests.mjs +145 -0
- package/src/sandbox-prepare.mjs +285 -0
- package/src/sandbox-refs.mjs +70 -0
- package/src/sandbox-session-monitor.mjs +4 -0
- package/src/sandbox-session-watch.mjs +124 -0
- package/src/sandbox-sessions.mjs +161 -0
- package/src/sandbox-store.mjs +63 -6
- package/src/sandbox-workspace-guard.mjs +39 -0
- package/src/sandbox.mjs +27 -200
- package/src/scheduler-stall-guard.mjs +14 -8
- package/src/schedules.mjs +19 -6
- package/src/secret-bundles.mjs +245 -0
- package/src/secret-profiles.mjs +103 -3
- package/src/secrets.mjs +19 -9
- package/src/service-env.mjs +38 -5
- package/src/service.mjs +638 -106
- package/src/session-key.mjs +60 -0
- package/src/session-store.mjs +168 -10
- package/src/setup-contract.mjs +40 -0
- package/src/setup-egress.mjs +113 -0
- package/src/setup-host-entry.mjs +33 -0
- package/src/setup-host.mjs +242 -0
- package/src/setup-lock.mjs +41 -0
- package/src/size-records.mjs +15 -3
- package/src/start.mjs +1353 -135
- package/src/status-comment.mjs +415 -0
- package/src/stderr-transcript.mjs +318 -0
- package/src/steer-ack.mjs +122 -0
- package/src/steer-cli.mjs +213 -0
- package/src/steer-framing.mjs +185 -0
- package/src/steer-inbox.mjs +693 -0
- package/src/store-version.mjs +139 -0
- package/src/subscription-config.mjs +56 -0
- package/src/subscription-limits.mjs +3 -0
- package/src/subscription-quota.mjs +259 -0
- package/src/subscription-runtime.mjs +123 -0
- package/src/subscription-status.mjs +42 -0
- package/src/subscription-wait.mjs +11 -0
- package/src/subscriptions.mjs +7 -1
- package/src/thinking-level.mjs +19 -0
- package/src/touch-outbox.mjs +61 -0
- package/src/touch-state.mjs +133 -0
- package/src/triggers-file.mjs +299 -39
- package/src/triggers.mjs +782 -40
- package/src/trust-paths.mjs +171 -0
- package/src/up.mjs +398 -51
- package/src/updates-cli.mjs +61 -0
- package/src/updates-look.mjs +127 -0
- package/src/updates-reconcile.mjs +182 -0
- package/src/updates-subscription-adapters.mjs +8 -0
- package/src/updates.mjs +252 -0
- package/src/valkey-auth.mjs +32 -9
- package/src/valkey-endpoint.mjs +14 -4
- package/src/wait-for.mjs +1 -1
- package/src/wait-state.mjs +62 -3
- package/src/watch-board.mjs +38 -0
- package/src/watch-closer.mjs +1 -1
- package/src/watch-contract.mjs +102 -0
- package/src/watch-decision.mjs +78 -0
- package/src/watch-drainer.mjs +14 -0
- package/src/watch-issue-adapter.mjs +44 -0
- package/src/watch-issue.mjs +111 -0
- package/src/watch-record.mjs +10 -0
- package/src/watch-runtime.mjs +167 -0
- package/src/watch-store.mjs +183 -0
- package/src/watch-trace.mjs +34 -0
- package/src/workspace-checkpoint.mjs +1750 -0
- package/src/zero-touch-fixes.mjs +99 -0
- package/src/zero-touch.mjs +225 -0
- package/src/repeat-slot.mjs +0 -13
package/src/model-ref.mjs
CHANGED
|
@@ -31,8 +31,39 @@ export const MODEL_REF_PATTERN = /^(?=.{1,64}$)[~@]?[a-z0-9][a-z0-9._:/@_-]*$/;
|
|
|
31
31
|
* never be listed. Applied after lowercasing, like `MODEL_REF_PATTERN`.
|
|
32
32
|
*/
|
|
33
33
|
export const PROVIDER_REF_PATTERN = /^(?=.{1,64}$)[a-z0-9][a-z0-9._-]*$/;
|
|
34
|
+
/** What a model id may be, in words: the sentence every source's refusal ends with. */
|
|
35
|
+
const MODEL_ID_SHAPE = "a model id of 1 to 64 characters: letters, digits and . _ - : / @, starting with a letter or digit (or ~ or @ then one)";
|
|
36
|
+
/** What a provider id may be, in words. */
|
|
37
|
+
const PROVIDER_ID_SHAPE = "a provider id of 1 to 64 characters: letters, digits, dot, dash and underscore, starting with a letter or digit";
|
|
38
|
+
|
|
34
39
|
/**
|
|
35
|
-
*
|
|
40
|
+
* Is `value` a model id a record may carry (issue #673)? Null when it is, else the shape it must have, as a
|
|
41
|
+
* sentence ("must be a model id of ..."). ONE rule for every source of a job's model: a trigger's `run.model`
|
|
42
|
+
* (`validateModelRef`), `PI_MODEL` (`loadConfig`), the settings overlay's `model` (`validateOverlay`) and
|
|
43
|
+
* `pi-dispatch run --model`. Held to one rule because the run record writes whichever of them won, and the
|
|
44
|
+
* console's runs filter (`RUN_MODEL_RE`, admin) accepts exactly the ids this rule does and more, so a run on a model
|
|
45
|
+
* the worker accepted is always one the console can find.
|
|
46
|
+
*
|
|
47
|
+
* ASCII is checked on the ORIGINAL before lowercasing, because lowercasing is not closed over ASCII: the Kelvin
|
|
48
|
+
* sign (U+212A) lowercases to a plain `k`, so a lowercase-then-match check alone would pass a homoglyph that then
|
|
49
|
+
* reaches the job verbatim and names a model nobody listed. Only the CHECK lowercases: the caller keeps the
|
|
50
|
+
* original case, because pi resolves model ids exactly and case-sensitively, and the pinned catalog holds
|
|
51
|
+
* uppercase ids. The sentence never carries the value.
|
|
52
|
+
*/
|
|
53
|
+
export function modelIdProblem(value) {
|
|
54
|
+
if (typeof value === "string" && value.startsWith("role:") && !/^role:[a-z][a-z0-9_-]{0,31}$/.test(value)) return "must name role:<lowercase-role>, with a role of 1 to 32 letters, digits, underscores or dashes";
|
|
55
|
+
if (typeof value !== "string" || !/^[\x21-\x7E]+$/.test(value) || !MODEL_REF_PATTERN.test(value.toLowerCase())) return `must be ${MODEL_ID_SHAPE}`;
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** `modelIdProblem` for a provider id (issue #673), against `PROVIDER_REF_PATTERN`, by the same rule. */
|
|
60
|
+
export function providerIdProblem(value) {
|
|
61
|
+
if (typeof value !== "string" || !/^[\x21-\x7E]+$/.test(value) || !PROVIDER_REF_PATTERN.test(value.toLowerCase())) return `must be ${PROVIDER_ID_SHAPE}`;
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Validate the optional `run.provider`, `run.model`, `run.maxTurns`, `run.models` and `run.fallbackModels` of one trigger (issue #502) and return
|
|
36
67
|
* the fields that were present, so a normalizer can spread them and an absent field stays absent. Shared by
|
|
37
68
|
* the trigger loader and the console, so both refuse with the same words.
|
|
38
69
|
*
|
|
@@ -48,24 +79,24 @@ export function validateModelRef(run, at, path) {
|
|
|
48
79
|
// import-free: run-history.mjs imports it too, and must not drag config.mjs's fs and os in with it.
|
|
49
80
|
const configError = (message) => Object.assign(new Error(message), { piDispatchConfig: true });
|
|
50
81
|
const out = {};
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
// a homoglyph that then reaches the job verbatim and names a model nobody listed.
|
|
54
|
-
const ascii = (value) => /^[\x21-\x7E]+$/.test(value);
|
|
82
|
+
// The shape checks (ASCII on the original, then the pattern on the lowercased value) are `providerIdProblem`
|
|
83
|
+
// and `modelIdProblem`, shared with PI_MODEL, PI_PROVIDER, the settings overlay and `pi-dispatch run` (#673).
|
|
55
84
|
// NULL IS ABSENT, for all three. Before #502 a hand-written cron entry's `"model": null` loaded and
|
|
56
85
|
// meant the deployment default (the worker fills `job.data ?? overlay ?? env`, and `??` passes over
|
|
57
86
|
// null), so refusing it now would turn a working file into one that refuses to load for no gain.
|
|
58
87
|
if (run?.provider != null) {
|
|
59
88
|
const provider = run.provider;
|
|
60
|
-
|
|
61
|
-
|
|
89
|
+
const problem = providerIdProblem(provider);
|
|
90
|
+
if (problem !== null) {
|
|
91
|
+
throw configError(`${at}: run.provider ${problem}: ${path}`);
|
|
62
92
|
}
|
|
63
93
|
out.provider = provider;
|
|
64
94
|
}
|
|
65
95
|
if (run?.model != null) {
|
|
66
96
|
const model = run.model;
|
|
67
|
-
|
|
68
|
-
|
|
97
|
+
const problem = modelIdProblem(model);
|
|
98
|
+
if (problem !== null) {
|
|
99
|
+
throw configError(`${at}: run.model ${problem}: ${path}`);
|
|
69
100
|
}
|
|
70
101
|
out.model = model;
|
|
71
102
|
}
|
|
@@ -82,14 +113,40 @@ export function validateModelRef(run, at, path) {
|
|
|
82
113
|
// The main model must be on its own list, refused HERE when the trigger names all three, because the
|
|
83
114
|
// file alone answers it. When the provider or the model comes from the deployment default instead,
|
|
84
115
|
// the worker answers it at job start (`model-not-allowed`, pre-spend).
|
|
85
|
-
if (out.provider !== undefined && out.model !== undefined && !modelOnList(run.models, out.provider, out.model)) {
|
|
116
|
+
if (out.provider !== undefined && out.model !== undefined && !out.model.startsWith("role:") && !modelOnList(run.models, out.provider, out.model)) {
|
|
86
117
|
throw configError(`${at}: run.models does not list this trigger's own run.provider/run.model, so every job it starts would be refused: ${path}`);
|
|
87
118
|
}
|
|
88
119
|
out.models = [...run.models];
|
|
89
120
|
}
|
|
121
|
+
// Fallbacks are operator-reviewed candidates, never an implicit widening of the allowed list.
|
|
122
|
+
// The deployment list is checked at pickup when this pure loader cannot see it.
|
|
123
|
+
if (run?.fallbackModels != null) {
|
|
124
|
+
const problem = modelListProblem(run.fallbackModels);
|
|
125
|
+
if (problem !== null) throw configError(`${at}: run.fallbackModels ${problem}: ${path}`);
|
|
126
|
+
for (const entry of run.fallbackModels) {
|
|
127
|
+
const ref = splitModelEntry(entry);
|
|
128
|
+
if (out.models !== undefined && !modelOnList(out.models, ref.provider, ref.model)) {
|
|
129
|
+
throw configError(`${at}: run.models does not list a run.fallbackModels entry: ${path}`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
out.fallbackModels = [...run.fallbackModels];
|
|
133
|
+
}
|
|
90
134
|
return out;
|
|
91
135
|
}
|
|
92
136
|
|
|
137
|
+
/**
|
|
138
|
+
* Whether a job's fallbacks can be tried at all: each must be on its allowed list, the trigger's or else the deployment's, and
|
|
139
|
+
* with no list none is. Without this a job with fallbacks is refused as `model-failover-no-candidate`, before it spends. One
|
|
140
|
+
* rule for the failover's selection and the console's preview.
|
|
141
|
+
*/
|
|
142
|
+
export function failoverListUsable(job) {
|
|
143
|
+
if (modelListProblem(job?.fallbackModels) !== null || modelListProblem(job?.models) !== null) return false;
|
|
144
|
+
return job.fallbackModels.every((ref) => {
|
|
145
|
+
const split = splitModelEntry(ref);
|
|
146
|
+
return split !== null && modelOnList(job.models, split.provider, split.model);
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
93
150
|
/** The most entries an allowed-model list may carry (issue #502). Past this a list is a catalog, not a policy. */
|
|
94
151
|
export const MAX_ALLOWED_MODELS = 16;
|
|
95
152
|
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import * as nodeFs from "node:fs";
|
|
2
|
+
import { parseTriggersDocument } from "./triggers.mjs";
|
|
3
|
+
|
|
4
|
+
/** Same live/last-good posture as project bundles. A broken first read cannot silently remove model policy. */
|
|
5
|
+
export function makeReadModelRoles({ triggersPath, fs = nodeFs, log = () => {} }) {
|
|
6
|
+
let last = null;
|
|
7
|
+
let failed = false;
|
|
8
|
+
return () => {
|
|
9
|
+
if (!triggersPath) return {};
|
|
10
|
+
try {
|
|
11
|
+
const text = fs.readFileSync(triggersPath, "utf8");
|
|
12
|
+
last = parseTriggersDocument(text, triggersPath).modelRoles;
|
|
13
|
+
failed = false;
|
|
14
|
+
return last;
|
|
15
|
+
} catch (error) {
|
|
16
|
+
if (error?.code === "ENOENT" && last === null) return {};
|
|
17
|
+
if (!failed) log("model_roles_last_good", { kept: last !== null });
|
|
18
|
+
failed = true;
|
|
19
|
+
if (last !== null) return last;
|
|
20
|
+
if (error?.piDispatchConfig === true) throw Object.assign(new Error("the model roles file is invalid and no validated snapshot exists"), { piDispatchConfig: true, modelSelectionReason: "model-roles-invalid" });
|
|
21
|
+
throw new Error("modelRoles could not be read and no validated snapshot exists");
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { modelListProblem, splitModelEntry } from "./model-ref.mjs";
|
|
2
|
+
import { isProjectId } from "./project-id.mjs";
|
|
3
|
+
|
|
4
|
+
export const MODEL_ROLE_RE = /^[a-z][a-z0-9_-]{0,31}$/;
|
|
5
|
+
const object = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
6
|
+
const scopeOf = (roles, id) => object(roles) && typeof id === "string" && Object.hasOwn(roles, id) ? roles[id] : {};
|
|
7
|
+
const failure = (reason, message) => Object.assign(new Error(message), { piDispatchConfig: true, modelSelectionReason: reason });
|
|
8
|
+
|
|
9
|
+
/** A misspelt policy block must refuse rather than silently removing the reviewed ceiling. */
|
|
10
|
+
export function isModelRolesField(key) {
|
|
11
|
+
const text = String(key);
|
|
12
|
+
const normalized = text.normalize("NFKC").replace(/[^a-z]/gi, "").toLowerCase();
|
|
13
|
+
if (["modelroles", "modelrole"].includes(normalized)) return true;
|
|
14
|
+
if (/^[\x20-\x7E]*$/.test(text) || normalized.length < 4) return false;
|
|
15
|
+
let cursor = 0;
|
|
16
|
+
for (const letter of "modelroles") if (normalized[cursor] === letter) cursor++;
|
|
17
|
+
return cursor === normalized.length;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Reviewed policy, never job data. Bounded maps keep the effective allowlist a policy rather than a catalog. */
|
|
21
|
+
export function validateModelRoles(value) {
|
|
22
|
+
if (value === undefined) return {};
|
|
23
|
+
const bad = () => { throw failure("model-roles-invalid", "modelRoles must map default/project ids to at most 16 roles, each a provider/model or {model, fallbackModels}"); };
|
|
24
|
+
if (!object(value) || Object.keys(value).length > 128) bad();
|
|
25
|
+
const out = {};
|
|
26
|
+
for (const [project, roles] of Object.entries(value)) {
|
|
27
|
+
if (!isProjectId(project) || !object(roles) || Object.keys(roles).length > 16) bad();
|
|
28
|
+
const row = {};
|
|
29
|
+
for (const [role, entry] of Object.entries(roles)) {
|
|
30
|
+
if (!MODEL_ROLE_RE.test(role) || ["inherit", "constructor", "prototype"].includes(role)) bad();
|
|
31
|
+
const ref = typeof entry === "string" ? entry : entry?.model;
|
|
32
|
+
if (!splitModelEntry(ref) || splitModelEntry(ref).model.startsWith("role:")) bad();
|
|
33
|
+
if (typeof entry === "string") row[role] = entry;
|
|
34
|
+
else {
|
|
35
|
+
if (!object(entry) || Object.keys(entry).some((k) => !["model", "fallbackModels"].includes(k))) bad();
|
|
36
|
+
if (entry.fallbackModels !== undefined && modelListProblem(entry.fallbackModels) !== null) bad();
|
|
37
|
+
row[role] = { model: ref, ...(entry.fallbackModels !== undefined ? { fallbackModels: [...entry.fallbackModels] } : {}) };
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
out[project] = row;
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function roleMapFor(roles, project = null) {
|
|
46
|
+
const entries = { ...scopeOf(roles, "default"), ...(project && project !== "default" ? scopeOf(roles, project) : {}) };
|
|
47
|
+
return Object.fromEntries(Object.entries(entries).map(([role, entry]) => [role, typeof entry === "string" ? entry : entry.model]));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** One snapshot is used for admission, endpoint slots and the runner. Failover may narrow it afterwards. */
|
|
51
|
+
export function resolveModelRoles(job, roles, project = null, { authoredProvider = undefined } = {}) {
|
|
52
|
+
// A receiver or an old queued delivery cannot carry authority over the live map.
|
|
53
|
+
const { modelRoles: _queuedMap, modelRole: _queuedRole, ...authored } = job;
|
|
54
|
+
job = authored;
|
|
55
|
+
const map = roleMapFor(roles, project);
|
|
56
|
+
const requested = typeof job.model === "string" && job.model.startsWith("role:") ? job.model.slice(5) : null;
|
|
57
|
+
if (requested !== null && (!MODEL_ROLE_RE.test(requested) || !Object.hasOwn(map, requested))) throw failure("model-role-unknown", "the requested model role is not declared in modelRoles");
|
|
58
|
+
if (requested === null && Object.keys(map).length === 0) return job;
|
|
59
|
+
let resolved = { ...job };
|
|
60
|
+
if (requested !== null) {
|
|
61
|
+
const ref = splitModelEntry(map[requested]);
|
|
62
|
+
if (authoredProvider !== undefined && authoredProvider !== null && authoredProvider !== ref.provider) throw failure("model-role-provider-conflict", "run.provider conflicts with its model role provider");
|
|
63
|
+
const entry = scopeOf(roles, project)[requested] ?? scopeOf(roles, "default")[requested];
|
|
64
|
+
resolved = { ...resolved, ...ref, modelRole: requested };
|
|
65
|
+
if (job.fallbackModels !== undefined && typeof entry !== "string" && entry.fallbackModels !== undefined) throw failure("model-roles-invalid", "declare fallbackModels on the role or the trigger, not both");
|
|
66
|
+
if (typeof entry !== "string" && entry.fallbackModels !== undefined) resolved.fallbackModels = [...entry.fallbackModels];
|
|
67
|
+
}
|
|
68
|
+
const refs = [...(job.models ?? [`${resolved.provider}/${resolved.model}`]), ...Object.values(map)];
|
|
69
|
+
for (const entry of Object.values({ ...scopeOf(roles, "default"), ...(project && project !== "default" ? scopeOf(roles, project) : {}) })) if (typeof entry !== "string") refs.push(...(entry.fallbackModels ?? []));
|
|
70
|
+
const models = [...new Set(refs)];
|
|
71
|
+
if (modelListProblem(models) !== null) throw failure("model-roles-invalid", "the role-expanded allowed-model list is invalid or exceeds 16 models");
|
|
72
|
+
return { ...resolved, models, modelRoles: map };
|
|
73
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** The failover price verdict shared with operator displays. Looking never reserves or calls a provider. */
|
|
2
|
+
import { builtinChatModels, builtinModel, checkModelsKnown } from "./model-catalog.mjs";
|
|
3
|
+
import { modelEntryOf } from "./model-endpoints.mjs";
|
|
4
|
+
import { splitModelEntry } from "./model-ref.mjs";
|
|
5
|
+
import { outputCapView, outputUnboundable } from "./output-cap.mjs";
|
|
6
|
+
|
|
7
|
+
// This copy is pinned to the runner's PRICED_APIS by model-failover.test.mjs. The published worker cannot import
|
|
8
|
+
// the image workspace, and a new runner API must not silently become eligible before its free bound is verified.
|
|
9
|
+
export const FAILOVER_PRICED_APIS = Object.freeze(["anthropic-messages", "azure-openai-responses", "bedrock-converse-stream", "cloudflare-workers-ai-system-one", "google-generative-ai", "google-vertex", "mistral-conversations", "openai-codex-responses", "openai-completions", "openai-decisions", "openai-responses", "typesafe-system-one"]);
|
|
10
|
+
/** Catalog/overlay composition is shared with the existing output-cap gate, rather than guessed from a URL or id. */
|
|
11
|
+
export function failoverModelView(ref, { models = null, lookup = builtinModel } = {}) {
|
|
12
|
+
const split = splitModelEntry(ref);
|
|
13
|
+
if (!split) return null;
|
|
14
|
+
const { provider, model } = split;
|
|
15
|
+
const view = outputCapView({ models, provider, modelId: model, builtinModel: lookup, builtinChatModels });
|
|
16
|
+
const { defined, override } = modelEntryOf(models, provider, model);
|
|
17
|
+
// pi provider-composer.js modelFromJson defaults, held to the pinned artifact by the test.
|
|
18
|
+
const base = defined ? { ...defined, contextWindow: defined.contextWindow ?? 128000, maxTokens: defined.maxTokens ?? 16384 } : lookup(provider, model);
|
|
19
|
+
return view ? { ...view, contextWindow: override?.contextWindow ?? base?.contextWindow, maxTokens: override?.maxTokens ?? base?.maxTokens } : null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function failoverPriceEligible(view) {
|
|
23
|
+
return Boolean(view && FAILOVER_PRICED_APIS.includes(view.api) && !outputUnboundable(view) && Number.isFinite(view.maxTokens) && view.maxTokens > 0 && ["input", "output", "cacheRead", "cacheWrite"].every((key) => Number.isFinite(view.cost?.[key]) && view.cost[key] >= 0));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function failoverModelStatus(ref, { models = null, lookup = builtinModel } = {}) {
|
|
27
|
+
const split = splitModelEntry(ref);
|
|
28
|
+
if (!split) return { known: false, priced: false, reason: "model-unknown", api: null };
|
|
29
|
+
const refs = [{ provider: split.provider, id: split.model, main: true }, { provider: split.provider, id: split.model }];
|
|
30
|
+
const known = checkModelsKnown(refs, { readOverlay: () => models });
|
|
31
|
+
const view = failoverModelView(ref, { models, lookup });
|
|
32
|
+
const reason = known.unknown || known.unavailable ? "model-unknown" : known.fallbackUnlisted ? "model-not-allowed" : !failoverPriceEligible(view) ? "model-unpriced" : null;
|
|
33
|
+
return { known: !known.unknown && !known.unavailable, priced: reason === null, reason, api: typeof view?.api === "string" ? view.api : null };
|
|
34
|
+
}
|
package/src/models-json.mjs
CHANGED
|
@@ -163,7 +163,7 @@ const posInt = T.int({ minimum: 1 });
|
|
|
163
163
|
const ImageResize = T.obj({ maxWidth: opt(posInt), maxHeight: opt(posInt), maxBytes: opt(posInt), jpegQuality: opt(T.int({ minimum: 1, maximum: 100 })) });
|
|
164
164
|
const ModelInputLimits = T.obj({ maxRequestBytes: opt(posInt), images: opt(T.obj({ resize: opt(ImageResize), maxPerMessage: opt(posInt), maxPerRequest: opt(posInt) })) });
|
|
165
165
|
// compat-schema.js's ProviderCompatSchema: the merged properties of the OpenAI completions, OpenAI responses, Anthropic
|
|
166
|
-
// messages, Bedrock and
|
|
166
|
+
// messages, Bedrock, Mistral and System One schemas. A key two of them share takes pi's generic override, which for every shared
|
|
167
167
|
// key here is the same type as each api's own (a boolean, or the three-way affinity format), so the superset is exact.
|
|
168
168
|
const bool = opt(T.bool());
|
|
169
169
|
const ProviderCompat = T.obj({
|
|
@@ -209,6 +209,9 @@ const ProviderCompat = T.obj({
|
|
|
209
209
|
supportsStrictTools: bool,
|
|
210
210
|
supportsMidConvoEffort: bool,
|
|
211
211
|
supportsMidConvoToolChanges: bool,
|
|
212
|
+
// System One classifier image transport, checked before pi reads the overlay.
|
|
213
|
+
imageInput: opt(T.union(T.lit("assets"), T.lit("images"), T.lit("state-image-urls"))),
|
|
214
|
+
sendImageOptInHeader: bool,
|
|
212
215
|
allowedFallbackModels: opt(T.arr(T.obj({ provider: req(T.str({ minLength: 1 })), model: req(T.str({ minLength: 1 })), cost: req(ModelCost) }), { maxItems: 3 })),
|
|
213
216
|
});
|
|
214
217
|
const nonEmpty = T.str({ minLength: 1 });
|
package/src/money.mjs
CHANGED
|
@@ -75,7 +75,7 @@ export function optionalUsdMicros(value, key) {
|
|
|
75
75
|
|
|
76
76
|
/**
|
|
77
77
|
* Integer micro-dollars as a plain dollar decimal, at least two decimals and no trailing zeros past them:
|
|
78
|
-
* 2500000 is "2.50", 1 is "0.000001". For messages and
|
|
78
|
+
* 2500000 is "2.50", 1 is "0.000001". For messages, displays and exact managed-fork cap serialization; no floating-point conversion is needed.
|
|
79
79
|
*/
|
|
80
80
|
export function formatMicros(micros) {
|
|
81
81
|
if (!Number.isSafeInteger(micros) || micros < 0) throw new TypeError("formatMicros: want a non-negative safe integer");
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/** Official ChatGPT direct OAuth. Registration identity and renewal secrets stay on the owner host. */
|
|
2
|
+
import { randomBytes, createHash } from "node:crypto";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { authError, validOpenAIAccessToken } from "./auth-store.mjs";
|
|
7
|
+
import { readAuthJson } from "./community-auth-helper.mjs";
|
|
8
|
+
|
|
9
|
+
const ISSUER = "https://auth.openai.com";
|
|
10
|
+
const RESOURCE = "https://api.openai.com/v1";
|
|
11
|
+
export const OPENAI_REDIRECT_URI = "http://127.0.0.1:1455/auth/callback";
|
|
12
|
+
export const OPENAI_LOGIN_INSTRUCTIONS = `For sign-in from another machine, forward port 1455 to 127.0.0.1:1455 on this host, or paste the final ${OPENAI_REDIRECT_URI} URL into this terminal.`;
|
|
13
|
+
const SCOPES = "openid profile email offline_access resource.invoke chatgpt.tokens.use.direct";
|
|
14
|
+
const TERMINAL = new Set(["invalid_grant", "invalid_refresh_token", "token_expired", "refresh_token_expired", "refresh_token_invalidated", "refresh_token_reused"]);
|
|
15
|
+
export const PINNED_OPENAI_OAUTH_SHA256 = "4cf9fc758a07ffe4ee022326404b5dcc86767ee4154945c0417c7f95eaecdcc2";
|
|
16
|
+
export function verifyPinnedOpenAIOAuth({ resolve = import.meta.resolve, readFile = readFileSync } = {}) {
|
|
17
|
+
const path = fileURLToPath(new URL("../auth/oauth/openai-chatgpt.js", resolve("@earendil-works/pi-ai/providers/openai")));
|
|
18
|
+
if (createHash("sha256").update(readFile(path)).digest("hex") !== PINNED_OPENAI_OAUTH_SHA256) throw authError("auth-adapter-source-mismatch");
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
export function parseOpenAIToken(data, { now, previous = null } = {}) {
|
|
22
|
+
if (!data || !validOpenAIAccessToken(data.access_token) || typeof data.refresh_token !== "string" || !data.refresh_token || !Number.isFinite(data.expires_in) || data.expires_in <= 0 || data.expires_in > 31 * 24 * 3600 || typeof data.scope !== "string") throw authError("auth-response-invalid", { uncertain: true });
|
|
23
|
+
const scopes = data.scope.split(/\s+/).filter(Boolean);
|
|
24
|
+
if (!scopes.includes("chatgpt.tokens.use.direct")) throw authError("auth-permission-missing");
|
|
25
|
+
const earliest = data.earliest_refresh_at;
|
|
26
|
+
const earliestRefreshAt = typeof earliest === "number" && Number.isFinite(earliest) ? Math.ceil(earliest < 1e12 ? earliest * 1000 : earliest) : typeof earliest === "string" && Number.isFinite(Date.parse(earliest)) ? Date.parse(earliest) : null;
|
|
27
|
+
return { type: "oauth", access: data.access_token, refresh: data.refresh_token, expires: Math.floor(now() + data.expires_in * 1000), scopes, ...(earliestRefreshAt === null ? {} : { earliestRefreshAt }) };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function verifyOpenAIIdentity({ token, clientId, nonce, fetch = globalThis.fetch, now = () => Date.now(), verify = null }) {
|
|
31
|
+
let claims;
|
|
32
|
+
try {
|
|
33
|
+
if (verify) claims = await verify({ token, clientId, nonce, issuer: ISSUER });
|
|
34
|
+
else {
|
|
35
|
+
const { createRemoteJWKSet, jwtVerify, customFetch } = await import("jose");
|
|
36
|
+
const response = await fetch(`${ISSUER}/.well-known/openid-configuration`, { signal: AbortSignal.timeout(15_000) });
|
|
37
|
+
if (!response.ok) throw new Error();
|
|
38
|
+
const discovery = await readAuthJson(response);
|
|
39
|
+
if (discovery.issuer !== ISSUER || typeof discovery.jwks_uri !== "string" || new URL(discovery.jwks_uri).origin !== ISSUER) throw new Error();
|
|
40
|
+
const jwks = createRemoteJWKSet(new URL(discovery.jwks_uri), { [customFetch]: fetch });
|
|
41
|
+
claims = (await jwtVerify(token, jwks, { issuer: ISSUER, audience: clientId, currentDate: new Date(now()), clockTolerance: 30, algorithms: ["RS256", "ES256"] })).payload;
|
|
42
|
+
}
|
|
43
|
+
if (typeof claims?.sub !== "string" || !claims.sub || claims.sub.length > 512 || claims.nonce !== nonce || claims.iss !== ISSUER || !(claims.aud === clientId || Array.isArray(claims.aud) && claims.aud.includes(clientId)) || !Number.isFinite(claims.exp) || claims.exp * 1000 <= now()) throw new Error();
|
|
44
|
+
return { subject: claims.sub };
|
|
45
|
+
} catch { throw authError("auth-identity-invalid"); }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function authorizationCallback({ url, state, interaction, signal, serverFactory = createServer }) {
|
|
49
|
+
signal?.throwIfAborted();
|
|
50
|
+
const manualAbort = new AbortController();
|
|
51
|
+
let resolveCallback;
|
|
52
|
+
const callback = new Promise((resolve) => { resolveCallback = resolve; });
|
|
53
|
+
const valid = (received) => received?.origin === new URL(OPENAI_REDIRECT_URI).origin && received.pathname === "/auth/callback" && received.searchParams.getAll("state").length === 1 && received.searchParams.get("state") === state && received.searchParams.getAll("code").length === 1 && Boolean(received.searchParams.get("code"));
|
|
54
|
+
const server = serverFactory((request, response) => {
|
|
55
|
+
let received;
|
|
56
|
+
try { received = new URL(request.url, OPENAI_REDIRECT_URI); } catch { /* Invalid input cannot end the pending sign-in. */ }
|
|
57
|
+
if (!valid(received)) { response.writeHead(400); response.end("Invalid authorization callback"); return; }
|
|
58
|
+
response.writeHead(200, { "Content-Type": "text/plain", "Cache-Control": "no-store" }); response.end("Sign-in complete. Return to pi-dispatch.");
|
|
59
|
+
resolveCallback(received);
|
|
60
|
+
});
|
|
61
|
+
const abort = () => resolveCallback(null);
|
|
62
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
63
|
+
try {
|
|
64
|
+
await new Promise((resolve) => { server.once("error", resolve); server.listen(1455, "127.0.0.1", resolve); });
|
|
65
|
+
interaction.notify({ type: "auth_url", url, instructions: OPENAI_LOGIN_INSTRUCTIONS });
|
|
66
|
+
const manual = Promise.resolve(interaction.prompt({ type: "manual_code", message: "Paste the final callback URL", signal: AbortSignal.any([manualAbort.signal, ...(signal ? [signal] : [])]) })).then((value) => { try { return new URL(value); } catch { throw authError("auth-callback-invalid"); } });
|
|
67
|
+
const received = await Promise.race([callback, manual]);
|
|
68
|
+
signal?.throwIfAborted();
|
|
69
|
+
if (!valid(received)) throw authError("auth-callback-invalid");
|
|
70
|
+
return received;
|
|
71
|
+
} finally { manualAbort.abort(); signal?.removeEventListener("abort", abort); server.closeAllConnections?.(); if (server.listening) await new Promise((resolve) => server.close(resolve)); }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function createOpenAIAuthAdapter({ fetch = globalThis.fetch, now = () => Date.now(), verifyIdentity = verifyOpenAIIdentity, callback = authorizationCallback } = {}) {
|
|
75
|
+
async function tokenRequest(body, signal) {
|
|
76
|
+
let response;
|
|
77
|
+
try { response = await fetch(`${ISSUER}/api/accounts/oauth/token`, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams(body), signal: AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(15_000)]) }); }
|
|
78
|
+
catch { throw authError("auth-network-unavailable", { transient: true, uncertain: true }); }
|
|
79
|
+
const data = await readAuthJson(response);
|
|
80
|
+
if (!response.ok) {
|
|
81
|
+
const code = typeof data?.error === "string" ? data.error : data?.error?.code;
|
|
82
|
+
if (TERMINAL.has(code)) throw authError("auth-reauth-required");
|
|
83
|
+
if (code === "invalid_client") throw authError("auth-client-invalid");
|
|
84
|
+
throw authError("auth-provider-refused", { transient: response.status === 429 || response.status >= 500 });
|
|
85
|
+
}
|
|
86
|
+
return data;
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
async validate() { verifyPinnedOpenAIOAuth(); },
|
|
90
|
+
async login({ interaction, signal, previous }) {
|
|
91
|
+
if (typeof interaction?.prompt !== "function" || typeof interaction?.notify !== "function") throw authError("auth-interaction-required");
|
|
92
|
+
const verifier = randomBytes(32).toString("base64url");
|
|
93
|
+
const state = randomBytes(32).toString("base64url");
|
|
94
|
+
const nonce = randomBytes(32).toString("base64url");
|
|
95
|
+
const hostId = previous.registration.hostId;
|
|
96
|
+
const savedClient = previous.registration.clientId;
|
|
97
|
+
const url = new URL(`${ISSUER}/oauth/authorize`);
|
|
98
|
+
url.search = new URLSearchParams({ client_id: savedClient ?? "dynamic_agent_client", ...(savedClient ? {} : { agent_name_hint: "pi-dispatch" }), ext_agent_host_id: `urn:uuid:${hostId}`, response_type: "code", redirect_uri: OPENAI_REDIRECT_URI, resource: RESOURCE, scope: SCOPES, state, nonce, code_challenge: createHash("sha256").update(verifier).digest("base64url"), code_challenge_method: "S256" }).toString();
|
|
99
|
+
const returned = await callback({ url: url.toString(), state, interaction, signal });
|
|
100
|
+
const clientId = savedClient ?? returned.searchParams.get("client_id");
|
|
101
|
+
if (typeof clientId !== "string" || !clientId || clientId.length > 1024) throw authError("auth-client-invalid");
|
|
102
|
+
const data = await tokenRequest({ grant_type: "authorization_code", client_id: clientId, code: returned.searchParams.get("code"), code_verifier: verifier, redirect_uri: OPENAI_REDIRECT_URI, resource: RESOURCE }, signal);
|
|
103
|
+
const identity = await verifyIdentity({ token: data.id_token, clientId, nonce, fetch, now });
|
|
104
|
+
if (previous.registration.subject && previous.registration.subject !== identity.subject) throw authError("auth-account-changed");
|
|
105
|
+
return { credential: { ...parseOpenAIToken(data, { now }), clientId }, registration: { hostId, clientId, subject: identity.subject } };
|
|
106
|
+
},
|
|
107
|
+
async refresh({ credential, signal }) {
|
|
108
|
+
if (credential.earliestRefreshAt > now()) throw authError("auth-refresh-too-early", { transient: true });
|
|
109
|
+
if (typeof credential.clientId !== "string" || !credential.clientId) throw authError("auth-client-invalid");
|
|
110
|
+
return { ...parseOpenAIToken(await tokenRequest({ grant_type: "refresh_token", client_id: credential.clientId, refresh_token: credential.refresh, resource: RESOURCE }, signal), { now, previous: credential }), clientId: credential.clientId };
|
|
111
|
+
},
|
|
112
|
+
async revoke({ credential, signal }) {
|
|
113
|
+
const discovery = await fetch(`${ISSUER}/.well-known/openid-configuration`, { signal });
|
|
114
|
+
if (!discovery.ok) throw authError("auth-revocation-unconfirmed");
|
|
115
|
+
const config = await readAuthJson(discovery);
|
|
116
|
+
if (typeof config.revocation_endpoint !== "string" || new URL(config.revocation_endpoint).origin !== ISSUER) throw authError("auth-revocation-unconfirmed");
|
|
117
|
+
const response = await fetch(config.revocation_endpoint, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams({ token: credential.refresh, token_type_hint: "refresh_token", client_id: credential.clientId }), signal });
|
|
118
|
+
return { confirmed: response.status === 200 };
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
}
|