@skrr-ai/cli 0.1.27 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/dedicated-machines.js +3 -1
- package/dist/lib/sky-code.js +11 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/cjs/index.d.ts +1 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/cjs/index.js +3 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/cjs/loopback-server.js +64 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/cjs/managed-inference-broker.d.ts +30 -0
- package/dist/node_modules/@skrr-ai/inference-broker/dist/cjs/managed-inference-broker.js +43 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/esm/index.d.ts +1 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/esm/index.js +1 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/esm/loopback-server.js +64 -1
- package/dist/node_modules/@skrr-ai/inference-broker/dist/esm/managed-inference-broker.d.ts +30 -0
- package/dist/node_modules/@skrr-ai/inference-broker/dist/esm/managed-inference-broker.js +41 -0
- package/dist/node_modules/@skrr-ai/inference-broker/package.json +1 -1
- package/oclif.manifest.json +2650 -2650
- package/package.json +1 -1
|
@@ -318,7 +318,9 @@ function describeDedicatedLease(lease, harnesses = []) {
|
|
|
318
318
|
const row = (label, value) => {
|
|
319
319
|
if (value === undefined || value === null || value === '')
|
|
320
320
|
return;
|
|
321
|
-
|
|
321
|
+
// At least one space: `restored from` is as wide as the column and printed
|
|
322
|
+
// as `restored fromrtl_…`.
|
|
323
|
+
lines.push(` ${`${label} `.padEnd(13)}${value}`);
|
|
322
324
|
};
|
|
323
325
|
row('id', dedicatedLeaseId(lease));
|
|
324
326
|
row('state', lease.stateReason
|
package/dist/lib/sky-code.js
CHANGED
|
@@ -36,6 +36,7 @@ const node_fs_1 = require("node:fs");
|
|
|
36
36
|
const node_os_1 = require("node:os");
|
|
37
37
|
const node_path_1 = __importDefault(require("node:path"));
|
|
38
38
|
const auth_core_1 = require("@skrr-ai/auth-core");
|
|
39
|
+
const inference_broker_1 = require("@skrr-ai/inference-broker");
|
|
39
40
|
const config_1 = require("./config");
|
|
40
41
|
// Deliberately does NOT import the managed marker any more. This module used to
|
|
41
42
|
// read it to decide whether a run was managed; that decision now arrives as a
|
|
@@ -468,11 +469,20 @@ function engineSpawnEnv(env, extraEnv,
|
|
|
468
469
|
// Required, not defaulted: the safe value differs per call and a parameter that
|
|
469
470
|
// can be forgotten on a security boundary will be.
|
|
470
471
|
mode) {
|
|
471
|
-
|
|
472
|
+
const spawnEnv = {
|
|
472
473
|
...(0, auth_core_1.sanitizeSpawnEnv)(env),
|
|
473
474
|
...(mode === 'managed' ? {} : inheritedProviderCredentials(env)),
|
|
474
475
|
...extraEnv,
|
|
475
476
|
};
|
|
477
|
+
if (mode !== 'managed')
|
|
478
|
+
return spawnEnv;
|
|
479
|
+
// OSK-8758 — a managed engine reaches the broker at a synthetic host, and must
|
|
480
|
+
// never address that host to a proxy. Behind an egress proxy (a Dedicated
|
|
481
|
+
// Runtime guest) Bun wrote every broker request in proxy form, and a runtime
|
|
482
|
+
// that honoured the proxy over the socket would send the prompt to it. Layered
|
|
483
|
+
// over the env built above, so it extends whatever list the engine would
|
|
484
|
+
// otherwise have received rather than replacing it.
|
|
485
|
+
return { ...spawnEnv, ...(0, inference_broker_1.inferenceBrokerNoProxyEnv)(spawnEnv) };
|
|
476
486
|
}
|
|
477
487
|
async function execEngine(args, env = process.env, options = {}) {
|
|
478
488
|
// Resolve the binary FIRST, before `prepare` runs.
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
* ESM and CJS are both emitted because the daemon is ESM and the CLI is CJS, and
|
|
52
52
|
* this package exists precisely so both can host the broker.
|
|
53
53
|
*/
|
|
54
|
-
export { INFERENCE_BROKER_ORIGIN, INFERENCE_BROKER_SOCKET_ENV, MANAGED_SESSION_ENV, MANAGED_PROMPT_CACHING_ENV, MANAGED_STATUS_PATH, type ManagedSessionStatus, buildManagedInferenceRelayCredential, startManagedInferenceBroker, isManagedInferenceModelScope, type GatewayCredential, type ManagedCapabilityRefresh, type ManagedCapabilityRefreshResult, type ManagedInferenceBrokerOptions, type ManagedInferenceBrokerSession, type ManagedInferenceInputTokenAccounting, type ManagedInferenceModelScope, type ManagedInferenceRequestEnvelope, type ManagedReasoningPolicy, } from './managed-inference-broker.js';
|
|
54
|
+
export { INFERENCE_BROKER_ORIGIN, INFERENCE_BROKER_SOCKET_ENV, MANAGED_SESSION_ENV, MANAGED_PROMPT_CACHING_ENV, MANAGED_STATUS_PATH, INFERENCE_BROKER_HOST, inferenceBrokerNoProxyEnv, type ManagedSessionStatus, buildManagedInferenceRelayCredential, startManagedInferenceBroker, isManagedInferenceModelScope, type GatewayCredential, type ManagedCapabilityRefresh, type ManagedCapabilityRefreshResult, type ManagedInferenceBrokerOptions, type ManagedInferenceBrokerSession, type ManagedInferenceInputTokenAccounting, type ManagedInferenceModelScope, type ManagedInferenceRequestEnvelope, type ManagedReasoningPolicy, } from './managed-inference-broker.js';
|
|
55
55
|
export { minimumCapabilityWindowMs, scheduleRenewalDelay, startRenewalLoop, type CapabilityRefreshOutcome, type RenewalLoop, type RenewalLoopOptions, type RenewalOutcome, type RenewalStopReason, } from './managed-inference-renewal.js';
|
|
56
56
|
export { ModelProxyLoopbackServer, type ModelProxyForwarder } from './loopback-server.js';
|
|
57
57
|
export * from './contract.js';
|
|
@@ -67,13 +67,15 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
67
67
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
68
68
|
};
|
|
69
69
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
70
|
-
exports.ModelProxyLoopbackServer = exports.startRenewalLoop = exports.scheduleRenewalDelay = exports.minimumCapabilityWindowMs = exports.isManagedInferenceModelScope = exports.startManagedInferenceBroker = exports.buildManagedInferenceRelayCredential = exports.MANAGED_STATUS_PATH = exports.MANAGED_PROMPT_CACHING_ENV = exports.MANAGED_SESSION_ENV = exports.INFERENCE_BROKER_SOCKET_ENV = exports.INFERENCE_BROKER_ORIGIN = void 0;
|
|
70
|
+
exports.ModelProxyLoopbackServer = exports.startRenewalLoop = exports.scheduleRenewalDelay = exports.minimumCapabilityWindowMs = exports.isManagedInferenceModelScope = exports.startManagedInferenceBroker = exports.buildManagedInferenceRelayCredential = exports.inferenceBrokerNoProxyEnv = exports.INFERENCE_BROKER_HOST = exports.MANAGED_STATUS_PATH = exports.MANAGED_PROMPT_CACHING_ENV = exports.MANAGED_SESSION_ENV = exports.INFERENCE_BROKER_SOCKET_ENV = exports.INFERENCE_BROKER_ORIGIN = void 0;
|
|
71
71
|
var managed_inference_broker_js_1 = require("./managed-inference-broker.js");
|
|
72
72
|
Object.defineProperty(exports, "INFERENCE_BROKER_ORIGIN", { enumerable: true, get: function () { return managed_inference_broker_js_1.INFERENCE_BROKER_ORIGIN; } });
|
|
73
73
|
Object.defineProperty(exports, "INFERENCE_BROKER_SOCKET_ENV", { enumerable: true, get: function () { return managed_inference_broker_js_1.INFERENCE_BROKER_SOCKET_ENV; } });
|
|
74
74
|
Object.defineProperty(exports, "MANAGED_SESSION_ENV", { enumerable: true, get: function () { return managed_inference_broker_js_1.MANAGED_SESSION_ENV; } });
|
|
75
75
|
Object.defineProperty(exports, "MANAGED_PROMPT_CACHING_ENV", { enumerable: true, get: function () { return managed_inference_broker_js_1.MANAGED_PROMPT_CACHING_ENV; } });
|
|
76
76
|
Object.defineProperty(exports, "MANAGED_STATUS_PATH", { enumerable: true, get: function () { return managed_inference_broker_js_1.MANAGED_STATUS_PATH; } });
|
|
77
|
+
Object.defineProperty(exports, "INFERENCE_BROKER_HOST", { enumerable: true, get: function () { return managed_inference_broker_js_1.INFERENCE_BROKER_HOST; } });
|
|
78
|
+
Object.defineProperty(exports, "inferenceBrokerNoProxyEnv", { enumerable: true, get: function () { return managed_inference_broker_js_1.inferenceBrokerNoProxyEnv; } });
|
|
77
79
|
Object.defineProperty(exports, "buildManagedInferenceRelayCredential", { enumerable: true, get: function () { return managed_inference_broker_js_1.buildManagedInferenceRelayCredential; } });
|
|
78
80
|
Object.defineProperty(exports, "startManagedInferenceBroker", { enumerable: true, get: function () { return managed_inference_broker_js_1.startManagedInferenceBroker; } });
|
|
79
81
|
Object.defineProperty(exports, "isManagedInferenceModelScope", { enumerable: true, get: function () { return managed_inference_broker_js_1.isManagedInferenceModelScope; } });
|
|
@@ -131,6 +131,14 @@ class ModelProxyLoopbackServer {
|
|
|
131
131
|
writeError(res, 401, contract_js_1.MODEL_PROXY_ERROR.UNAUTHORIZED, 'Invalid relay token.');
|
|
132
132
|
return;
|
|
133
133
|
}
|
|
134
|
+
// Judged after authentication, so an unauthenticated caller learns nothing
|
|
135
|
+
// about which request forms this relay accepts; and before the body is read,
|
|
136
|
+
// so a refused request is never buffered.
|
|
137
|
+
const path = originFormPath(req.url ?? '/', new URL(this.baseUrl()).origin);
|
|
138
|
+
if (path === null) {
|
|
139
|
+
req.resume();
|
|
140
|
+
throw new contract_js_1.ModelProxyError(contract_js_1.MODEL_PROXY_ERROR.REQUEST_REJECTED, 'model proxy loopback accepts only requests addressed to its own origin');
|
|
141
|
+
}
|
|
134
142
|
const body = await readBody(req, this.opts.maxBodyBytes);
|
|
135
143
|
const controller = new AbortController();
|
|
136
144
|
this.activeRequests.add(controller);
|
|
@@ -143,7 +151,7 @@ class ModelProxyLoopbackServer {
|
|
|
143
151
|
});
|
|
144
152
|
const headers = normalizeIncomingHeaders(req.headers);
|
|
145
153
|
try {
|
|
146
|
-
const upstream = await this.opts.forwarder.forward({ method: req.method ?? 'POST', path
|
|
154
|
+
const upstream = await this.opts.forwarder.forward({ method: req.method ?? 'POST', path, headers, body }, { signal: controller.signal });
|
|
147
155
|
res.writeHead(upstream.status, upstream.headers);
|
|
148
156
|
for await (const chunk of upstream.body) {
|
|
149
157
|
if (!res.write(chunk)) {
|
|
@@ -179,6 +187,61 @@ function constantTimeEqual(a, b) {
|
|
|
179
187
|
return false;
|
|
180
188
|
return (0, node_crypto_1.timingSafeEqual)(bufA, bufB);
|
|
181
189
|
}
|
|
190
|
+
/**
|
|
191
|
+
* The request-target as the ORIGIN-FORM path the forwarder contract carries, or
|
|
192
|
+
* null when the request was addressed somewhere else.
|
|
193
|
+
*
|
|
194
|
+
* HTTP/1.1 gives a POST two request-target forms. Origin-form (`/v1/messages`) is
|
|
195
|
+
* what a client sends an origin server; absolute-form
|
|
196
|
+
* (`http://host/v1/messages`) is what it sends a forward PROXY — and RFC 9112
|
|
197
|
+
* §3.2.2 still requires a server to accept it. Node passes either through as
|
|
198
|
+
* `req.url`, verbatim.
|
|
199
|
+
*
|
|
200
|
+
* OSK-8758 is why this matters here. A Dedicated Runtime guest exports
|
|
201
|
+
* HTTP(S)_PROXY for its egress proxy, and the broker's synthetic
|
|
202
|
+
* `oversky-inference-broker.invalid` is in no NO_PROXY. Bun keeps dialing the Unix
|
|
203
|
+
* socket named by `fetch({ unix })` but writes the request line in proxy form, so
|
|
204
|
+
* every managed turn reached the broker with an absolute URL for a "path", failed
|
|
205
|
+
* its exact route allowlist, and was refused as an unapproved upstream route —
|
|
206
|
+
* while a laptop, with no proxy configured, sent origin-form and worked.
|
|
207
|
+
*
|
|
208
|
+
* Accepted only when the target denotes EXACTLY the request origin-form would
|
|
209
|
+
* have, so this is a parsing fix and never a widening:
|
|
210
|
+
*
|
|
211
|
+
* - the origin must be this relay's own advertised origin. This loopback is not a
|
|
212
|
+
* forward proxy: a foreign origin is refused here, even one a downstream egress
|
|
213
|
+
* allowlist would accept — naming the upstream is not a way to reach it;
|
|
214
|
+
* - no userinfo, which a request-target may not carry;
|
|
215
|
+
* - the path and query must survive URL parsing byte-for-byte. The WHATWG parser
|
|
216
|
+
* resolves dot segments and rewrites `\` to `/`, and origin-form bytes reach the
|
|
217
|
+
* forwarder unnormalized, so a spelling the forwarder would refuse in one form
|
|
218
|
+
* cannot become an approved route in the other.
|
|
219
|
+
*
|
|
220
|
+
* Anything that is not absolute-form (origin-form and the odd `*`) is returned
|
|
221
|
+
* untouched, exactly as before, for the forwarder to judge.
|
|
222
|
+
*/
|
|
223
|
+
function originFormPath(rawTarget, ownOrigin) {
|
|
224
|
+
if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(rawTarget))
|
|
225
|
+
return rawTarget;
|
|
226
|
+
let target;
|
|
227
|
+
try {
|
|
228
|
+
target = new URL(rawTarget);
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
// `origin` is the string "null" for every non-special scheme, so two unrelated
|
|
234
|
+
// opaque origins would otherwise compare equal.
|
|
235
|
+
if (target.origin === 'null' || target.origin !== ownOrigin)
|
|
236
|
+
return null;
|
|
237
|
+
if (target.username || target.password)
|
|
238
|
+
return null;
|
|
239
|
+
const authorityEnd = rawTarget.indexOf('/', rawTarget.indexOf('://') + 3);
|
|
240
|
+
if (authorityEnd < 0)
|
|
241
|
+
return null;
|
|
242
|
+
const path = rawTarget.slice(authorityEnd);
|
|
243
|
+
return path === `${target.pathname}${target.search}` ? path : null;
|
|
244
|
+
}
|
|
182
245
|
function normalizeIncomingHeaders(headers) {
|
|
183
246
|
const out = {};
|
|
184
247
|
for (const [key, value] of Object.entries(headers)) {
|
|
@@ -177,7 +177,37 @@ export interface ManagedSessionStatus {
|
|
|
177
177
|
};
|
|
178
178
|
}
|
|
179
179
|
export declare const INFERENCE_BROKER_ORIGIN = "http://oversky-inference-broker.invalid";
|
|
180
|
+
/** The synthetic host inside {@link INFERENCE_BROKER_ORIGIN}. Never resolved. */
|
|
181
|
+
export declare const INFERENCE_BROKER_HOST: string;
|
|
180
182
|
export declare const INFERENCE_BROKER_SOCKET_ENV = "OVERSKY_INFERENCE_BROKER_SOCKET";
|
|
183
|
+
/**
|
|
184
|
+
* OSK-8758 — the proxy exemption a process holding the broker environment needs.
|
|
185
|
+
*
|
|
186
|
+
* The broker is reached over a Unix socket, but its base URLs name
|
|
187
|
+
* {@link INFERENCE_BROKER_HOST}, and an engine that honours HTTP(S)_PROXY treats
|
|
188
|
+
* that name like any other host. On a Dedicated Runtime guest, which exports an
|
|
189
|
+
* egress proxy, Bun kept dialing the socket but wrote the request line in proxy
|
|
190
|
+
* form, and the broker refused every turn. The loopback server now accepts that
|
|
191
|
+
* form. This makes sure it is never produced: engine traffic to the synthetic host
|
|
192
|
+
* is not addressed to a proxy at all. A runtime that one day honours the proxy
|
|
193
|
+
* over `fetch({ unix })` would otherwise send the prompt to the egress proxy.
|
|
194
|
+
*
|
|
195
|
+
* Returns BOTH spellings, for layering LAST over the environment the child would
|
|
196
|
+
* otherwise receive (`env`), because tools disagree on which one they read and
|
|
197
|
+
* whether they fall back to the other:
|
|
198
|
+
*
|
|
199
|
+
* - existing entries keep their order, and the host is appended once;
|
|
200
|
+
* - a spelling that is unset (or empty) is seeded from the other one before the
|
|
201
|
+
* host is appended. Seeding only the host would silently drop every existing
|
|
202
|
+
* exemption for a tool that prefers the unset spelling, which today reads the
|
|
203
|
+
* other through its fallback;
|
|
204
|
+
* - a list that is `*` already exempts everything and is left alone, because
|
|
205
|
+
* some clients honour `*` only as the entire value.
|
|
206
|
+
*/
|
|
207
|
+
export declare function inferenceBrokerNoProxyEnv(env: Readonly<Record<string, string | undefined>>): {
|
|
208
|
+
NO_PROXY: string;
|
|
209
|
+
no_proxy: string;
|
|
210
|
+
};
|
|
181
211
|
export declare const MANAGED_SESSION_ENV = "OVERSKY_MANAGED_SESSION";
|
|
182
212
|
export declare const MANAGED_PROMPT_CACHING_ENV = "OVERSKY_MANAGED_PROMPT_CACHING";
|
|
183
213
|
export declare function managedClientModelId(scope: ManagedInferenceModelScope): string;
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
* endpoint while it is alive.
|
|
12
12
|
*/
|
|
13
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
-
exports.MANAGED_PROMPT_CACHING_ENV = exports.MANAGED_SESSION_ENV = exports.INFERENCE_BROKER_SOCKET_ENV = exports.INFERENCE_BROKER_ORIGIN = exports.MANAGED_STATUS_PATH = void 0;
|
|
14
|
+
exports.MANAGED_PROMPT_CACHING_ENV = exports.MANAGED_SESSION_ENV = exports.INFERENCE_BROKER_SOCKET_ENV = exports.INFERENCE_BROKER_HOST = exports.INFERENCE_BROKER_ORIGIN = exports.MANAGED_STATUS_PATH = void 0;
|
|
15
|
+
exports.inferenceBrokerNoProxyEnv = inferenceBrokerNoProxyEnv;
|
|
15
16
|
exports.managedClientModelId = managedClientModelId;
|
|
16
17
|
exports.isGatewayCredential = isGatewayCredential;
|
|
17
18
|
exports.isManagedInferenceModelScope = isManagedInferenceModelScope;
|
|
@@ -37,7 +38,48 @@ const managed_inference_renewal_js_1 = require("./managed-inference-renewal.js")
|
|
|
37
38
|
*/
|
|
38
39
|
exports.MANAGED_STATUS_PATH = '/oversky/managed-status';
|
|
39
40
|
exports.INFERENCE_BROKER_ORIGIN = 'http://oversky-inference-broker.invalid';
|
|
41
|
+
/** The synthetic host inside {@link INFERENCE_BROKER_ORIGIN}. Never resolved. */
|
|
42
|
+
exports.INFERENCE_BROKER_HOST = new URL(exports.INFERENCE_BROKER_ORIGIN).hostname;
|
|
40
43
|
exports.INFERENCE_BROKER_SOCKET_ENV = 'OVERSKY_INFERENCE_BROKER_SOCKET';
|
|
44
|
+
/**
|
|
45
|
+
* OSK-8758 — the proxy exemption a process holding the broker environment needs.
|
|
46
|
+
*
|
|
47
|
+
* The broker is reached over a Unix socket, but its base URLs name
|
|
48
|
+
* {@link INFERENCE_BROKER_HOST}, and an engine that honours HTTP(S)_PROXY treats
|
|
49
|
+
* that name like any other host. On a Dedicated Runtime guest, which exports an
|
|
50
|
+
* egress proxy, Bun kept dialing the socket but wrote the request line in proxy
|
|
51
|
+
* form, and the broker refused every turn. The loopback server now accepts that
|
|
52
|
+
* form. This makes sure it is never produced: engine traffic to the synthetic host
|
|
53
|
+
* is not addressed to a proxy at all. A runtime that one day honours the proxy
|
|
54
|
+
* over `fetch({ unix })` would otherwise send the prompt to the egress proxy.
|
|
55
|
+
*
|
|
56
|
+
* Returns BOTH spellings, for layering LAST over the environment the child would
|
|
57
|
+
* otherwise receive (`env`), because tools disagree on which one they read and
|
|
58
|
+
* whether they fall back to the other:
|
|
59
|
+
*
|
|
60
|
+
* - existing entries keep their order, and the host is appended once;
|
|
61
|
+
* - a spelling that is unset (or empty) is seeded from the other one before the
|
|
62
|
+
* host is appended. Seeding only the host would silently drop every existing
|
|
63
|
+
* exemption for a tool that prefers the unset spelling, which today reads the
|
|
64
|
+
* other through its fallback;
|
|
65
|
+
* - a list that is `*` already exempts everything and is left alone, because
|
|
66
|
+
* some clients honour `*` only as the entire value.
|
|
67
|
+
*/
|
|
68
|
+
function inferenceBrokerNoProxyEnv(env) {
|
|
69
|
+
const upper = nonEmpty(env.NO_PROXY);
|
|
70
|
+
const lower = nonEmpty(env.no_proxy);
|
|
71
|
+
return {
|
|
72
|
+
NO_PROXY: withInferenceBrokerHost(upper ?? lower ?? ''),
|
|
73
|
+
no_proxy: withInferenceBrokerHost(lower ?? upper ?? ''),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function withInferenceBrokerHost(list) {
|
|
77
|
+
const entries = list.split(',').map((entry) => entry.trim().toLowerCase());
|
|
78
|
+
if (entries.includes('*') || entries.includes(exports.INFERENCE_BROKER_HOST))
|
|
79
|
+
return list;
|
|
80
|
+
const kept = list.replace(/[\s,]+$/, '');
|
|
81
|
+
return kept ? `${kept},${exports.INFERENCE_BROKER_HOST}` : exports.INFERENCE_BROKER_HOST;
|
|
82
|
+
}
|
|
41
83
|
exports.MANAGED_SESSION_ENV = 'OVERSKY_MANAGED_SESSION';
|
|
42
84
|
exports.MANAGED_PROMPT_CACHING_ENV = 'OVERSKY_MANAGED_PROMPT_CACHING';
|
|
43
85
|
const ALLOWED_PATHS = new Set([
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
* ESM and CJS are both emitted because the daemon is ESM and the CLI is CJS, and
|
|
52
52
|
* this package exists precisely so both can host the broker.
|
|
53
53
|
*/
|
|
54
|
-
export { INFERENCE_BROKER_ORIGIN, INFERENCE_BROKER_SOCKET_ENV, MANAGED_SESSION_ENV, MANAGED_PROMPT_CACHING_ENV, MANAGED_STATUS_PATH, type ManagedSessionStatus, buildManagedInferenceRelayCredential, startManagedInferenceBroker, isManagedInferenceModelScope, type GatewayCredential, type ManagedCapabilityRefresh, type ManagedCapabilityRefreshResult, type ManagedInferenceBrokerOptions, type ManagedInferenceBrokerSession, type ManagedInferenceInputTokenAccounting, type ManagedInferenceModelScope, type ManagedInferenceRequestEnvelope, type ManagedReasoningPolicy, } from './managed-inference-broker.js';
|
|
54
|
+
export { INFERENCE_BROKER_ORIGIN, INFERENCE_BROKER_SOCKET_ENV, MANAGED_SESSION_ENV, MANAGED_PROMPT_CACHING_ENV, MANAGED_STATUS_PATH, INFERENCE_BROKER_HOST, inferenceBrokerNoProxyEnv, type ManagedSessionStatus, buildManagedInferenceRelayCredential, startManagedInferenceBroker, isManagedInferenceModelScope, type GatewayCredential, type ManagedCapabilityRefresh, type ManagedCapabilityRefreshResult, type ManagedInferenceBrokerOptions, type ManagedInferenceBrokerSession, type ManagedInferenceInputTokenAccounting, type ManagedInferenceModelScope, type ManagedInferenceRequestEnvelope, type ManagedReasoningPolicy, } from './managed-inference-broker.js';
|
|
55
55
|
export { minimumCapabilityWindowMs, scheduleRenewalDelay, startRenewalLoop, type CapabilityRefreshOutcome, type RenewalLoop, type RenewalLoopOptions, type RenewalOutcome, type RenewalStopReason, } from './managed-inference-renewal.js';
|
|
56
56
|
export { ModelProxyLoopbackServer, type ModelProxyForwarder } from './loopback-server.js';
|
|
57
57
|
export * from './contract.js';
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
* ESM and CJS are both emitted because the daemon is ESM and the CLI is CJS, and
|
|
52
52
|
* this package exists precisely so both can host the broker.
|
|
53
53
|
*/
|
|
54
|
-
export { INFERENCE_BROKER_ORIGIN, INFERENCE_BROKER_SOCKET_ENV, MANAGED_SESSION_ENV, MANAGED_PROMPT_CACHING_ENV, MANAGED_STATUS_PATH, buildManagedInferenceRelayCredential, startManagedInferenceBroker, isManagedInferenceModelScope, } from './managed-inference-broker.js';
|
|
54
|
+
export { INFERENCE_BROKER_ORIGIN, INFERENCE_BROKER_SOCKET_ENV, MANAGED_SESSION_ENV, MANAGED_PROMPT_CACHING_ENV, MANAGED_STATUS_PATH, INFERENCE_BROKER_HOST, inferenceBrokerNoProxyEnv, buildManagedInferenceRelayCredential, startManagedInferenceBroker, isManagedInferenceModelScope, } from './managed-inference-broker.js';
|
|
55
55
|
export { minimumCapabilityWindowMs, scheduleRenewalDelay, startRenewalLoop, } from './managed-inference-renewal.js';
|
|
56
56
|
export { ModelProxyLoopbackServer } from './loopback-server.js';
|
|
57
57
|
// Re-exported in full because `daemon/src/model-proxy/contract.ts` becomes a
|
|
@@ -128,6 +128,14 @@ export class ModelProxyLoopbackServer {
|
|
|
128
128
|
writeError(res, 401, MODEL_PROXY_ERROR.UNAUTHORIZED, 'Invalid relay token.');
|
|
129
129
|
return;
|
|
130
130
|
}
|
|
131
|
+
// Judged after authentication, so an unauthenticated caller learns nothing
|
|
132
|
+
// about which request forms this relay accepts; and before the body is read,
|
|
133
|
+
// so a refused request is never buffered.
|
|
134
|
+
const path = originFormPath(req.url ?? '/', new URL(this.baseUrl()).origin);
|
|
135
|
+
if (path === null) {
|
|
136
|
+
req.resume();
|
|
137
|
+
throw new ModelProxyError(MODEL_PROXY_ERROR.REQUEST_REJECTED, 'model proxy loopback accepts only requests addressed to its own origin');
|
|
138
|
+
}
|
|
131
139
|
const body = await readBody(req, this.opts.maxBodyBytes);
|
|
132
140
|
const controller = new AbortController();
|
|
133
141
|
this.activeRequests.add(controller);
|
|
@@ -140,7 +148,7 @@ export class ModelProxyLoopbackServer {
|
|
|
140
148
|
});
|
|
141
149
|
const headers = normalizeIncomingHeaders(req.headers);
|
|
142
150
|
try {
|
|
143
|
-
const upstream = await this.opts.forwarder.forward({ method: req.method ?? 'POST', path
|
|
151
|
+
const upstream = await this.opts.forwarder.forward({ method: req.method ?? 'POST', path, headers, body }, { signal: controller.signal });
|
|
144
152
|
res.writeHead(upstream.status, upstream.headers);
|
|
145
153
|
for await (const chunk of upstream.body) {
|
|
146
154
|
if (!res.write(chunk)) {
|
|
@@ -175,6 +183,61 @@ function constantTimeEqual(a, b) {
|
|
|
175
183
|
return false;
|
|
176
184
|
return timingSafeEqual(bufA, bufB);
|
|
177
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* The request-target as the ORIGIN-FORM path the forwarder contract carries, or
|
|
188
|
+
* null when the request was addressed somewhere else.
|
|
189
|
+
*
|
|
190
|
+
* HTTP/1.1 gives a POST two request-target forms. Origin-form (`/v1/messages`) is
|
|
191
|
+
* what a client sends an origin server; absolute-form
|
|
192
|
+
* (`http://host/v1/messages`) is what it sends a forward PROXY — and RFC 9112
|
|
193
|
+
* §3.2.2 still requires a server to accept it. Node passes either through as
|
|
194
|
+
* `req.url`, verbatim.
|
|
195
|
+
*
|
|
196
|
+
* OSK-8758 is why this matters here. A Dedicated Runtime guest exports
|
|
197
|
+
* HTTP(S)_PROXY for its egress proxy, and the broker's synthetic
|
|
198
|
+
* `oversky-inference-broker.invalid` is in no NO_PROXY. Bun keeps dialing the Unix
|
|
199
|
+
* socket named by `fetch({ unix })` but writes the request line in proxy form, so
|
|
200
|
+
* every managed turn reached the broker with an absolute URL for a "path", failed
|
|
201
|
+
* its exact route allowlist, and was refused as an unapproved upstream route —
|
|
202
|
+
* while a laptop, with no proxy configured, sent origin-form and worked.
|
|
203
|
+
*
|
|
204
|
+
* Accepted only when the target denotes EXACTLY the request origin-form would
|
|
205
|
+
* have, so this is a parsing fix and never a widening:
|
|
206
|
+
*
|
|
207
|
+
* - the origin must be this relay's own advertised origin. This loopback is not a
|
|
208
|
+
* forward proxy: a foreign origin is refused here, even one a downstream egress
|
|
209
|
+
* allowlist would accept — naming the upstream is not a way to reach it;
|
|
210
|
+
* - no userinfo, which a request-target may not carry;
|
|
211
|
+
* - the path and query must survive URL parsing byte-for-byte. The WHATWG parser
|
|
212
|
+
* resolves dot segments and rewrites `\` to `/`, and origin-form bytes reach the
|
|
213
|
+
* forwarder unnormalized, so a spelling the forwarder would refuse in one form
|
|
214
|
+
* cannot become an approved route in the other.
|
|
215
|
+
*
|
|
216
|
+
* Anything that is not absolute-form (origin-form and the odd `*`) is returned
|
|
217
|
+
* untouched, exactly as before, for the forwarder to judge.
|
|
218
|
+
*/
|
|
219
|
+
function originFormPath(rawTarget, ownOrigin) {
|
|
220
|
+
if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(rawTarget))
|
|
221
|
+
return rawTarget;
|
|
222
|
+
let target;
|
|
223
|
+
try {
|
|
224
|
+
target = new URL(rawTarget);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
// `origin` is the string "null" for every non-special scheme, so two unrelated
|
|
230
|
+
// opaque origins would otherwise compare equal.
|
|
231
|
+
if (target.origin === 'null' || target.origin !== ownOrigin)
|
|
232
|
+
return null;
|
|
233
|
+
if (target.username || target.password)
|
|
234
|
+
return null;
|
|
235
|
+
const authorityEnd = rawTarget.indexOf('/', rawTarget.indexOf('://') + 3);
|
|
236
|
+
if (authorityEnd < 0)
|
|
237
|
+
return null;
|
|
238
|
+
const path = rawTarget.slice(authorityEnd);
|
|
239
|
+
return path === `${target.pathname}${target.search}` ? path : null;
|
|
240
|
+
}
|
|
178
241
|
function normalizeIncomingHeaders(headers) {
|
|
179
242
|
const out = {};
|
|
180
243
|
for (const [key, value] of Object.entries(headers)) {
|
|
@@ -177,7 +177,37 @@ export interface ManagedSessionStatus {
|
|
|
177
177
|
};
|
|
178
178
|
}
|
|
179
179
|
export declare const INFERENCE_BROKER_ORIGIN = "http://oversky-inference-broker.invalid";
|
|
180
|
+
/** The synthetic host inside {@link INFERENCE_BROKER_ORIGIN}. Never resolved. */
|
|
181
|
+
export declare const INFERENCE_BROKER_HOST: string;
|
|
180
182
|
export declare const INFERENCE_BROKER_SOCKET_ENV = "OVERSKY_INFERENCE_BROKER_SOCKET";
|
|
183
|
+
/**
|
|
184
|
+
* OSK-8758 — the proxy exemption a process holding the broker environment needs.
|
|
185
|
+
*
|
|
186
|
+
* The broker is reached over a Unix socket, but its base URLs name
|
|
187
|
+
* {@link INFERENCE_BROKER_HOST}, and an engine that honours HTTP(S)_PROXY treats
|
|
188
|
+
* that name like any other host. On a Dedicated Runtime guest, which exports an
|
|
189
|
+
* egress proxy, Bun kept dialing the socket but wrote the request line in proxy
|
|
190
|
+
* form, and the broker refused every turn. The loopback server now accepts that
|
|
191
|
+
* form. This makes sure it is never produced: engine traffic to the synthetic host
|
|
192
|
+
* is not addressed to a proxy at all. A runtime that one day honours the proxy
|
|
193
|
+
* over `fetch({ unix })` would otherwise send the prompt to the egress proxy.
|
|
194
|
+
*
|
|
195
|
+
* Returns BOTH spellings, for layering LAST over the environment the child would
|
|
196
|
+
* otherwise receive (`env`), because tools disagree on which one they read and
|
|
197
|
+
* whether they fall back to the other:
|
|
198
|
+
*
|
|
199
|
+
* - existing entries keep their order, and the host is appended once;
|
|
200
|
+
* - a spelling that is unset (or empty) is seeded from the other one before the
|
|
201
|
+
* host is appended. Seeding only the host would silently drop every existing
|
|
202
|
+
* exemption for a tool that prefers the unset spelling, which today reads the
|
|
203
|
+
* other through its fallback;
|
|
204
|
+
* - a list that is `*` already exempts everything and is left alone, because
|
|
205
|
+
* some clients honour `*` only as the entire value.
|
|
206
|
+
*/
|
|
207
|
+
export declare function inferenceBrokerNoProxyEnv(env: Readonly<Record<string, string | undefined>>): {
|
|
208
|
+
NO_PROXY: string;
|
|
209
|
+
no_proxy: string;
|
|
210
|
+
};
|
|
181
211
|
export declare const MANAGED_SESSION_ENV = "OVERSKY_MANAGED_SESSION";
|
|
182
212
|
export declare const MANAGED_PROMPT_CACHING_ENV = "OVERSKY_MANAGED_PROMPT_CACHING";
|
|
183
213
|
export declare function managedClientModelId(scope: ManagedInferenceModelScope): string;
|
|
@@ -28,7 +28,48 @@ import { startRenewalLoop, } from './managed-inference-renewal.js';
|
|
|
28
28
|
*/
|
|
29
29
|
export const MANAGED_STATUS_PATH = '/oversky/managed-status';
|
|
30
30
|
export const INFERENCE_BROKER_ORIGIN = 'http://oversky-inference-broker.invalid';
|
|
31
|
+
/** The synthetic host inside {@link INFERENCE_BROKER_ORIGIN}. Never resolved. */
|
|
32
|
+
export const INFERENCE_BROKER_HOST = new URL(INFERENCE_BROKER_ORIGIN).hostname;
|
|
31
33
|
export const INFERENCE_BROKER_SOCKET_ENV = 'OVERSKY_INFERENCE_BROKER_SOCKET';
|
|
34
|
+
/**
|
|
35
|
+
* OSK-8758 — the proxy exemption a process holding the broker environment needs.
|
|
36
|
+
*
|
|
37
|
+
* The broker is reached over a Unix socket, but its base URLs name
|
|
38
|
+
* {@link INFERENCE_BROKER_HOST}, and an engine that honours HTTP(S)_PROXY treats
|
|
39
|
+
* that name like any other host. On a Dedicated Runtime guest, which exports an
|
|
40
|
+
* egress proxy, Bun kept dialing the socket but wrote the request line in proxy
|
|
41
|
+
* form, and the broker refused every turn. The loopback server now accepts that
|
|
42
|
+
* form. This makes sure it is never produced: engine traffic to the synthetic host
|
|
43
|
+
* is not addressed to a proxy at all. A runtime that one day honours the proxy
|
|
44
|
+
* over `fetch({ unix })` would otherwise send the prompt to the egress proxy.
|
|
45
|
+
*
|
|
46
|
+
* Returns BOTH spellings, for layering LAST over the environment the child would
|
|
47
|
+
* otherwise receive (`env`), because tools disagree on which one they read and
|
|
48
|
+
* whether they fall back to the other:
|
|
49
|
+
*
|
|
50
|
+
* - existing entries keep their order, and the host is appended once;
|
|
51
|
+
* - a spelling that is unset (or empty) is seeded from the other one before the
|
|
52
|
+
* host is appended. Seeding only the host would silently drop every existing
|
|
53
|
+
* exemption for a tool that prefers the unset spelling, which today reads the
|
|
54
|
+
* other through its fallback;
|
|
55
|
+
* - a list that is `*` already exempts everything and is left alone, because
|
|
56
|
+
* some clients honour `*` only as the entire value.
|
|
57
|
+
*/
|
|
58
|
+
export function inferenceBrokerNoProxyEnv(env) {
|
|
59
|
+
const upper = nonEmpty(env.NO_PROXY);
|
|
60
|
+
const lower = nonEmpty(env.no_proxy);
|
|
61
|
+
return {
|
|
62
|
+
NO_PROXY: withInferenceBrokerHost(upper ?? lower ?? ''),
|
|
63
|
+
no_proxy: withInferenceBrokerHost(lower ?? upper ?? ''),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function withInferenceBrokerHost(list) {
|
|
67
|
+
const entries = list.split(',').map((entry) => entry.trim().toLowerCase());
|
|
68
|
+
if (entries.includes('*') || entries.includes(INFERENCE_BROKER_HOST))
|
|
69
|
+
return list;
|
|
70
|
+
const kept = list.replace(/[\s,]+$/, '');
|
|
71
|
+
return kept ? `${kept},${INFERENCE_BROKER_HOST}` : INFERENCE_BROKER_HOST;
|
|
72
|
+
}
|
|
32
73
|
export const MANAGED_SESSION_ENV = 'OVERSKY_MANAGED_SESSION';
|
|
33
74
|
export const MANAGED_PROMPT_CACHING_ENV = 'OVERSKY_MANAGED_PROMPT_CACHING';
|
|
34
75
|
const ALLOWED_PATHS = new Set([
|