@wardby/cli 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/coding/local-git.js +9 -1
- package/dist/coding/registry/pypi.d.ts +25 -1
- package/dist/coding/registry/pypi.js +140 -16
- package/dist/coding/registry/types.d.ts +17 -3
- package/dist/config/providers.d.ts +3 -2
- package/dist/core/http-runtime.js +13 -6
- package/dist/help-index.json +207 -15
- package/dist/mcp/auth/ownership.d.ts +1 -1
- package/dist/mcp/tools/agents.js +1 -1
- package/dist/providers/coding-proxy/mock-upstream.d.ts +33 -0
- package/dist/providers/coding-proxy/mock-upstream.js +116 -0
- package/dist/providers/coding-proxy/registry/prisma-store.d.ts +3 -0
- package/dist/providers/coding-proxy/registry/prisma-store.js +17 -0
- package/dist/providers/coding-proxy/registry/service.d.ts +4 -0
- package/dist/providers/coding-proxy/registry/service.js +20 -3
- package/dist/providers/coding-proxy/registry/store.d.ts +18 -1
- package/dist/providers/coding-proxy/registry/store.js +33 -0
- package/dist/providers/coding-proxy/runtime.js +12 -1
- package/dist/providers/coding-proxy/types.d.ts +2 -0
- package/dist/providers/jobs/kubernetes-isolation.d.ts +13 -0
- package/dist/providers/jobs/kubernetes-isolation.js +52 -5
- package/dist/providers/jobs/kubernetes.d.ts +20 -6
- package/dist/providers/jobs/kubernetes.js +43 -32
- package/dist/providers/llm/openai.js +0 -1
- package/dist/quickstart/coding-db.js +14 -2
- package/dist/quickstart/coding-doctor.d.ts +3 -0
- package/dist/quickstart/coding-doctor.js +3 -0
- package/dist/quickstart/coding-images.d.ts +9 -0
- package/dist/quickstart/coding-images.js +39 -0
- package/dist/quickstart/coding-seed.d.ts +13 -1
- package/dist/quickstart/coding-seed.js +25 -7
- package/dist/quickstart/coding.d.ts +2 -0
- package/dist/quickstart/coding.js +75 -3
- package/dist/quickstart/config.d.ts +2 -0
- package/dist/quickstart/config.js +4 -0
- package/dist/quickstart/images.d.ts +11 -0
- package/dist/quickstart/images.js +74 -12
- package/dist/quickstart/index.d.ts +0 -2
- package/dist/quickstart/index.js +10 -14
- package/dist/quickstart/manifests.d.ts +56 -0
- package/dist/quickstart/manifests.js +523 -0
- package/dist/quickstart/python-detect.d.ts +10 -0
- package/dist/quickstart/python-detect.js +32 -0
- package/dist/quickstart/repo-packages.d.ts +10 -0
- package/dist/quickstart/repo-packages.js +132 -0
- package/dist/quickstart/reviewer-prompt.d.ts +12 -0
- package/dist/quickstart/reviewer-prompt.js +84 -0
- package/dist/quickstart/scenarios.d.ts +19 -0
- package/dist/quickstart/scenarios.js +28 -0
- package/dist/quickstart/sync-reviewer-prompt.d.ts +1 -0
- package/dist/quickstart/sync-reviewer-prompt.js +10 -0
- package/dist/quickstart-images.json +1 -1
- package/dist/viewer/api-schema.d.ts +4 -4
- package/docs/coding-agent-setup.md +17 -0
- package/docs/coding-packages.md +101 -3
- package/docs/coding-worker-byo-images.md +47 -2
- package/docs/coding-worker-isolation.md +20 -9
- package/docs/getting-started.md +139 -11
- package/docs/knowledge.md +11 -0
- package/help/architecture-agent.md +136 -0
- package/help/build-worker-image.md +198 -0
- package/help/coding-packages.md +31 -2
- package/help/creating-agents.md +14 -0
- package/help/deploy-gke.md +29 -5
- package/help/deployment-targets.md +3 -3
- package/help/getting-started.md +19 -0
- package/help/local-repositories.md +36 -5
- package/package.json +3 -2
|
@@ -72,9 +72,9 @@ export declare function assertStrictlyOwnedTool(tool: Tool | null, id: string, p
|
|
|
72
72
|
* triggered them. No canRead/null=public path exists here on purpose.
|
|
73
73
|
*/
|
|
74
74
|
export declare function requireOwnedTask(db: PrismaClient, taskId: string, principalId: string): Promise<{
|
|
75
|
+
error: import("@prisma/client/runtime/client").JsonValue | null;
|
|
75
76
|
status: import("#prisma").TaskStatus;
|
|
76
77
|
runId: string | null;
|
|
77
|
-
error: import("@prisma/client/runtime/client").JsonValue | null;
|
|
78
78
|
createdAt: Date;
|
|
79
79
|
updatedAt: Date;
|
|
80
80
|
kind: import("#prisma").TaskKind;
|
package/dist/mcp/tools/agents.js
CHANGED
|
@@ -225,7 +225,7 @@ const profileJsonSchema = {
|
|
|
225
225
|
},
|
|
226
226
|
packageAllowlist: {
|
|
227
227
|
type: "object",
|
|
228
|
-
description: "Approved top-level packages per ecosystem (npm, pypi). Needs packages:approve or agents:admin.",
|
|
228
|
+
description: "Approved top-level packages per ecosystem (npm, pypi). A PyPI entry may name extras, e.g. psycopg[binary]. Needs packages:approve or agents:admin.",
|
|
229
229
|
additionalProperties: { type: "array", maxItems: 256, items: { type: "string" } },
|
|
230
230
|
},
|
|
231
231
|
packagePolicy: {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A fake model upstream for load tests: a `fetch` that never opens a socket.
|
|
3
|
+
* It answers an OpenAI Responses streaming request with one assistant message
|
|
4
|
+
* whose text is the coding worker's final JSON (`outcome: "no_changes"`), so a
|
|
5
|
+
* run finishes without tool calls, changes or a pull request. Enabled only by
|
|
6
|
+
* `mockUpstreamFromEnv` (see its guard). Never use outside a load test.
|
|
7
|
+
*
|
|
8
|
+
* Codex/OpenAI Responses only (ruling R1): any other host, including the
|
|
9
|
+
* Anthropic Messages endpoint, gets a 501 rather than a fabricated reply.
|
|
10
|
+
*/
|
|
11
|
+
export interface MockUpstreamConfig {
|
|
12
|
+
latencyMs: number;
|
|
13
|
+
jitterMs: number;
|
|
14
|
+
inputTokens: number;
|
|
15
|
+
outputTokens: number;
|
|
16
|
+
}
|
|
17
|
+
export declare const DEFAULT_MOCK_UPSTREAM: MockUpstreamConfig;
|
|
18
|
+
/** The runId the coding worker's prompt names, or null when the request carries none. */
|
|
19
|
+
export declare function extractRunId(body: string): string | null;
|
|
20
|
+
/**
|
|
21
|
+
* Builds a `fetch` that simulates the OpenAI Responses streaming endpoint for
|
|
22
|
+
* load tests: no socket, no credential forwarding, deterministic latency and
|
|
23
|
+
* token counts. `sleep` is injectable so tests can assert on the delay without
|
|
24
|
+
* waiting it out.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createMockUpstream(overrides?: Partial<MockUpstreamConfig>, sleep?: (ms: number) => Promise<void>): typeof globalThis.fetch;
|
|
27
|
+
/**
|
|
28
|
+
* The mock model upstream is a load-test seam inside a production binary, so
|
|
29
|
+
* it needs two deliberate switches: WARDBY_LOAD_TEST=1 and
|
|
30
|
+
* WARDBY_CODING_PROXY_MOCK_UPSTREAM=1. One without the other, or any other
|
|
31
|
+
* value, refuses to start the proxy rather than silently picking a side.
|
|
32
|
+
*/
|
|
33
|
+
export declare function mockUpstreamFromEnv(env: NodeJS.ProcessEnv): MockUpstreamConfig | null;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
export const DEFAULT_MOCK_UPSTREAM = {
|
|
2
|
+
latencyMs: 5000,
|
|
3
|
+
jitterMs: 0,
|
|
4
|
+
inputTokens: 1200,
|
|
5
|
+
outputTokens: 80,
|
|
6
|
+
};
|
|
7
|
+
const RUN_ID = /The final JSON runId must be ([A-Za-z0-9][A-Za-z0-9_-]*)\./;
|
|
8
|
+
/** The runId the coding worker's prompt names, or null when the request carries none. */
|
|
9
|
+
export function extractRunId(body) {
|
|
10
|
+
return RUN_ID.exec(body)?.[1] ?? null;
|
|
11
|
+
}
|
|
12
|
+
const sse = (event, data) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
|
|
13
|
+
async function bodyText(input, init) {
|
|
14
|
+
const raw = init?.body;
|
|
15
|
+
if (typeof raw === "string")
|
|
16
|
+
return raw;
|
|
17
|
+
if (raw instanceof Uint8Array)
|
|
18
|
+
return Buffer.from(raw).toString("utf8");
|
|
19
|
+
if (input instanceof Request)
|
|
20
|
+
return input.text();
|
|
21
|
+
return "";
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Builds a `fetch` that simulates the OpenAI Responses streaming endpoint for
|
|
25
|
+
* load tests: no socket, no credential forwarding, deterministic latency and
|
|
26
|
+
* token counts. `sleep` is injectable so tests can assert on the delay without
|
|
27
|
+
* waiting it out.
|
|
28
|
+
*/
|
|
29
|
+
export function createMockUpstream(overrides = {}, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
|
|
30
|
+
const config = { ...DEFAULT_MOCK_UPSTREAM, ...overrides };
|
|
31
|
+
return async (input, init) => {
|
|
32
|
+
const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url);
|
|
33
|
+
if (url.hostname !== "api.openai.com") {
|
|
34
|
+
return Response.json({ error: { code: "mock_upstream_protocol_unsupported" } }, { status: 501 });
|
|
35
|
+
}
|
|
36
|
+
const runId = extractRunId(await bodyText(input, init));
|
|
37
|
+
if (!runId)
|
|
38
|
+
return Response.json({ error: { code: "mock_upstream_no_run_id" } }, { status: 400 });
|
|
39
|
+
await sleep(config.latencyMs + Math.floor(Math.random() * (config.jitterMs + 1)));
|
|
40
|
+
const text = JSON.stringify({
|
|
41
|
+
schemaVersion: 1,
|
|
42
|
+
runId,
|
|
43
|
+
outcome: "no_changes",
|
|
44
|
+
summary: "Load test: no changes.",
|
|
45
|
+
tag: null,
|
|
46
|
+
tests: [],
|
|
47
|
+
});
|
|
48
|
+
const id = `resp_mock_${runId}`;
|
|
49
|
+
const item = {
|
|
50
|
+
id: `msg_${runId}`,
|
|
51
|
+
type: "message",
|
|
52
|
+
role: "assistant",
|
|
53
|
+
status: "completed",
|
|
54
|
+
content: [{ type: "output_text", text, annotations: [] }],
|
|
55
|
+
};
|
|
56
|
+
const usage = {
|
|
57
|
+
input_tokens: config.inputTokens,
|
|
58
|
+
input_tokens_details: { cached_tokens: 0 },
|
|
59
|
+
output_tokens: config.outputTokens,
|
|
60
|
+
output_tokens_details: { reasoning_tokens: 0 },
|
|
61
|
+
total_tokens: config.inputTokens + config.outputTokens,
|
|
62
|
+
};
|
|
63
|
+
const stream = sse("response.created", { type: "response.created", response: { id, status: "in_progress", output: [] } }) +
|
|
64
|
+
sse("response.output_item.added", {
|
|
65
|
+
type: "response.output_item.added",
|
|
66
|
+
output_index: 0,
|
|
67
|
+
item: { ...item, status: "in_progress", content: [] },
|
|
68
|
+
}) +
|
|
69
|
+
sse("response.output_text.delta", {
|
|
70
|
+
type: "response.output_text.delta",
|
|
71
|
+
output_index: 0,
|
|
72
|
+
content_index: 0,
|
|
73
|
+
item_id: item.id,
|
|
74
|
+
delta: text,
|
|
75
|
+
}) +
|
|
76
|
+
sse("response.output_item.done", { type: "response.output_item.done", output_index: 0, item }) +
|
|
77
|
+
sse("response.completed", {
|
|
78
|
+
type: "response.completed",
|
|
79
|
+
response: { id, status: "completed", output: [item], usage },
|
|
80
|
+
});
|
|
81
|
+
return new Response(stream, { status: 200, headers: { "content-type": "text/event-stream" } });
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
const FLAG_LOAD = "WARDBY_LOAD_TEST";
|
|
85
|
+
const FLAG_MOCK = "WARDBY_CODING_PROXY_MOCK_UPSTREAM";
|
|
86
|
+
function nonNegativeInt(env, name, fallback) {
|
|
87
|
+
const raw = env[name];
|
|
88
|
+
if (raw === undefined || raw === "")
|
|
89
|
+
return fallback;
|
|
90
|
+
// Plain decimal digits only: no sign, hex, exponent, or whitespace.
|
|
91
|
+
const n = /^\d+$/.test(raw) ? Number(raw) : NaN;
|
|
92
|
+
if (!Number.isSafeInteger(n) || n < 0) {
|
|
93
|
+
throw new Error(`mock_upstream_guard: ${name} must be a non-negative integer`);
|
|
94
|
+
}
|
|
95
|
+
return n;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The mock model upstream is a load-test seam inside a production binary, so
|
|
99
|
+
* it needs two deliberate switches: WARDBY_LOAD_TEST=1 and
|
|
100
|
+
* WARDBY_CODING_PROXY_MOCK_UPSTREAM=1. One without the other, or any other
|
|
101
|
+
* value, refuses to start the proxy rather than silently picking a side.
|
|
102
|
+
*/
|
|
103
|
+
export function mockUpstreamFromEnv(env) {
|
|
104
|
+
const load = env[FLAG_LOAD];
|
|
105
|
+
const mock = env[FLAG_MOCK];
|
|
106
|
+
if (load === undefined && mock === undefined)
|
|
107
|
+
return null;
|
|
108
|
+
if (load !== "1" || mock !== "1") {
|
|
109
|
+
throw new Error(`mock_upstream_guard: set both ${FLAG_LOAD}=1 and ${FLAG_MOCK}=1 to use the mock upstream, or neither`);
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
...DEFAULT_MOCK_UPSTREAM,
|
|
113
|
+
latencyMs: nonNegativeInt(env, "WARDBY_LOAD_MOCK_LATENCY_MS", DEFAULT_MOCK_UPSTREAM.latencyMs),
|
|
114
|
+
jitterMs: nonNegativeInt(env, "WARDBY_LOAD_MOCK_JITTER_MS", DEFAULT_MOCK_UPSTREAM.jitterMs),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { PrismaClient } from "#prisma";
|
|
2
|
+
import type { FileDependency } from "../../../coding/registry/types.js";
|
|
2
3
|
import type { ApprovedVersion, PlanRefusedVersion, RegistryFetchRecord, RegistryRunContext, RegistryStore, StoredVersionFact } from "./store.js";
|
|
3
4
|
export declare class PrismaRegistryStore implements RegistryStore {
|
|
4
5
|
private readonly db;
|
|
@@ -6,6 +7,8 @@ export declare class PrismaRegistryStore implements RegistryStore {
|
|
|
6
7
|
findRunByRegistryTokenHash(hash: string, now: Date): Promise<RegistryRunContext | null>;
|
|
7
8
|
isAllowedDependency(runId: string, ecosystem: string, name: string): Promise<boolean>;
|
|
8
9
|
addAllowances(runId: string, ecosystem: string, names: readonly string[]): Promise<void>;
|
|
10
|
+
allowedExtras(runId: string, ecosystem: string, name: string): Promise<string[]>;
|
|
11
|
+
addExtraAllowances(runId: string, ecosystem: string, dependencies: readonly FileDependency[]): Promise<void>;
|
|
9
12
|
recordFetch(record: RegistryFetchRecord): Promise<void>;
|
|
10
13
|
usage(runId: string): Promise<{
|
|
11
14
|
files: number;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { extraAllowanceName, extraOfAllowanceName } from "./store.js";
|
|
1
2
|
/** Versions looked up per query, so a 5000-entry lockfile is a handful of
|
|
2
3
|
* bounded queries rather than one enormous OR. */
|
|
3
4
|
const FACT_QUERY_CHUNK = 500;
|
|
@@ -26,6 +27,9 @@ export class PrismaRegistryStore {
|
|
|
26
27
|
};
|
|
27
28
|
}
|
|
28
29
|
async isAllowedDependency(runId, ecosystem, name) {
|
|
30
|
+
// An extra's row (`name[extra]`) is never a package allowance.
|
|
31
|
+
if (name.includes("["))
|
|
32
|
+
return false;
|
|
29
33
|
const row = await this.db.registryAllowance.findUnique({
|
|
30
34
|
where: { runId_ecosystem_name: { runId, ecosystem, name } },
|
|
31
35
|
select: { runId: true },
|
|
@@ -40,6 +44,19 @@ export class PrismaRegistryStore {
|
|
|
40
44
|
skipDuplicates: true,
|
|
41
45
|
});
|
|
42
46
|
}
|
|
47
|
+
async allowedExtras(runId, ecosystem, name) {
|
|
48
|
+
const rows = await this.db.registryAllowance.findMany({
|
|
49
|
+
where: { runId, ecosystem, name: { startsWith: `${name}[` } },
|
|
50
|
+
select: { name: true },
|
|
51
|
+
});
|
|
52
|
+
return rows
|
|
53
|
+
.map((row) => extraOfAllowanceName(row.name, name))
|
|
54
|
+
.filter((extra) => extra !== null)
|
|
55
|
+
.sort();
|
|
56
|
+
}
|
|
57
|
+
async addExtraAllowances(runId, ecosystem, dependencies) {
|
|
58
|
+
await this.addAllowances(runId, ecosystem, dependencies.flatMap((dependency) => dependency.extras.map((extra) => extraAllowanceName(dependency.name, extra))));
|
|
59
|
+
}
|
|
43
60
|
async recordFetch(record) {
|
|
44
61
|
await this.db.registryFetch.create({ data: record });
|
|
45
62
|
}
|
|
@@ -209,5 +209,9 @@ export declare class RegistryService {
|
|
|
209
209
|
* exactly what was reserved: the declared size when known, or the bytes
|
|
210
210
|
* actually streamed when it was not. */
|
|
211
211
|
private releaseDownload;
|
|
212
|
+
/** The extras the run allows for `name`: those its allowlist entries name
|
|
213
|
+
* plus those a served parent's dependency line asked of it. They decide
|
|
214
|
+
* which of the package's extra-gated dependencies its files allow. */
|
|
215
|
+
private allowedExtras;
|
|
212
216
|
private download;
|
|
213
217
|
}
|
|
@@ -168,7 +168,7 @@ function versionDependencies(adapter, meta, version) {
|
|
|
168
168
|
function isPackageName(adapter, name) {
|
|
169
169
|
try {
|
|
170
170
|
const entry = adapter.parseAllowlistEntry(name);
|
|
171
|
-
return !entry.wildcard && entry.range === undefined && entry.name === name;
|
|
171
|
+
return !entry.wildcard && entry.range === undefined && entry.extras === undefined && entry.name === name;
|
|
172
172
|
}
|
|
173
173
|
catch {
|
|
174
174
|
return false;
|
|
@@ -962,6 +962,16 @@ export class RegistryService {
|
|
|
962
962
|
else
|
|
963
963
|
this.inFlight.set(runId, slot);
|
|
964
964
|
}
|
|
965
|
+
/** The extras the run allows for `name`: those its allowlist entries name
|
|
966
|
+
* plus those a served parent's dependency line asked of it. They decide
|
|
967
|
+
* which of the package's extra-gated dependencies its files allow. */
|
|
968
|
+
async allowedExtras(adapter, context, name) {
|
|
969
|
+
const entries = parseAllowlist(context.allowlist, this.options.adapters).get(adapter.id) ?? [];
|
|
970
|
+
const extras = new Set(entries.flatMap((entry) => (!entry.wildcard && entry.name === name ? (entry.extras ?? []) : [])));
|
|
971
|
+
for (const extra of await this.options.store.allowedExtras(context.runId, adapter.id, name))
|
|
972
|
+
extras.add(extra);
|
|
973
|
+
return [...extras].sort();
|
|
974
|
+
}
|
|
965
975
|
async download(adapter, context, name, route, file, signal) {
|
|
966
976
|
const { limits, store } = this.options;
|
|
967
977
|
const runId = context.runId;
|
|
@@ -1097,8 +1107,15 @@ export class RegistryService {
|
|
|
1097
1107
|
served.bytes += bytes;
|
|
1098
1108
|
if (adapter.dependenciesFromFile && buffered.length > 0) {
|
|
1099
1109
|
const body = Buffer.concat(buffered);
|
|
1100
|
-
const
|
|
1101
|
-
await
|
|
1110
|
+
const extras = await this.allowedExtras(adapter, context, name);
|
|
1111
|
+
const dependencies = (await adapter.dependenciesFromFile(route, body, extras).catch(() => [])).map((dependency) => ({ name: adapter.normalizeName(dependency.name), extras: dependency.extras }));
|
|
1112
|
+
await store.addAllowances(context.runId, adapter.id, dependencies.map((dependency) => dependency.name));
|
|
1113
|
+
// Dependency extras are followed only below a package the run
|
|
1114
|
+
// allows with extras (an entry naming extras, or a dependency
|
|
1115
|
+
// such a package asked for with extras): a plain entry follows
|
|
1116
|
+
// no extras, its own or its dependencies'.
|
|
1117
|
+
if (extras.length > 0)
|
|
1118
|
+
await store.addExtraAllowances(context.runId, adapter.id, dependencies);
|
|
1102
1119
|
}
|
|
1103
1120
|
controller.close();
|
|
1104
1121
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { PackageAllowlist } from "../../../coding/registry/allowlist.js";
|
|
2
|
-
import type { DeclaredDependency } from "../../../coding/registry/types.js";
|
|
2
|
+
import type { DeclaredDependency, FileDependency } from "../../../coding/registry/types.js";
|
|
3
3
|
export interface RegistryRunContext {
|
|
4
4
|
runId: string;
|
|
5
5
|
deadlineAt: Date;
|
|
@@ -41,6 +41,17 @@ export interface PlanRefusedVersion {
|
|
|
41
41
|
/** Set for a too-new refusal: it lapses once the version is old enough. */
|
|
42
42
|
publishedAt?: Date | null;
|
|
43
43
|
}
|
|
44
|
+
/**
|
|
45
|
+
* An extra a run allows for a package (PyPI `uvicorn[standard]`) is kept as
|
|
46
|
+
* an allowance row named `name[extra]`, beside the package's own row. No
|
|
47
|
+
* registry route can ask for that name (routes accept only valid package
|
|
48
|
+
* names, which never contain `[`), so the row never allows a fetch itself:
|
|
49
|
+
* it only tells the proxy which of the package's extra-gated dependencies
|
|
50
|
+
* to follow when it serves the package's files.
|
|
51
|
+
*/
|
|
52
|
+
export declare function extraAllowanceName(name: string, extra: string): string;
|
|
53
|
+
/** The extra in an allowance row name for `name`, or null when the row is not one of its extras. */
|
|
54
|
+
export declare function extraOfAllowanceName(rowName: string, name: string): string | null;
|
|
44
55
|
/** Run-scoped store over the registry-only token session lookup and the
|
|
45
56
|
* RegistryAllowance/RegistryFetch tables (Task 5). Backs the registry proxy:
|
|
46
57
|
* it never sees the model capability, only the derived registry token. */
|
|
@@ -48,6 +59,10 @@ export interface RegistryStore {
|
|
|
48
59
|
findRunByRegistryTokenHash(hash: string, now: Date): Promise<RegistryRunContext | null>;
|
|
49
60
|
isAllowedDependency(runId: string, ecosystem: string, name: string): Promise<boolean>;
|
|
50
61
|
addAllowances(runId: string, ecosystem: string, names: readonly string[]): Promise<void>;
|
|
62
|
+
/** The extras the run allows for `name` (normalized, sorted). */
|
|
63
|
+
allowedExtras(runId: string, ecosystem: string, name: string): Promise<string[]>;
|
|
64
|
+
/** Records extras the run allows for each package (an existing one is kept). */
|
|
65
|
+
addExtraAllowances(runId: string, ecosystem: string, dependencies: readonly FileDependency[]): Promise<void>;
|
|
51
66
|
recordFetch(record: RegistryFetchRecord): Promise<void>;
|
|
52
67
|
usage(runId: string): Promise<{
|
|
53
68
|
files: number;
|
|
@@ -99,6 +114,8 @@ export declare class MemoryRegistryStore implements RegistryStore {
|
|
|
99
114
|
findRunByRegistryTokenHash(hash: string, now: Date): Promise<RegistryRunContext | null>;
|
|
100
115
|
isAllowedDependency(runId: string, ecosystem: string, name: string): Promise<boolean>;
|
|
101
116
|
addAllowances(runId: string, ecosystem: string, names: readonly string[]): Promise<void>;
|
|
117
|
+
allowedExtras(runId: string, ecosystem: string, name: string): Promise<string[]>;
|
|
118
|
+
addExtraAllowances(runId: string, ecosystem: string, dependencies: readonly FileDependency[]): Promise<void>;
|
|
102
119
|
recordFetch(record: RegistryFetchRecord): Promise<void>;
|
|
103
120
|
usage(runId: string): Promise<{
|
|
104
121
|
files: number;
|
|
@@ -1,3 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An extra a run allows for a package (PyPI `uvicorn[standard]`) is kept as
|
|
3
|
+
* an allowance row named `name[extra]`, beside the package's own row. No
|
|
4
|
+
* registry route can ask for that name (routes accept only valid package
|
|
5
|
+
* names, which never contain `[`), so the row never allows a fetch itself:
|
|
6
|
+
* it only tells the proxy which of the package's extra-gated dependencies
|
|
7
|
+
* to follow when it serves the package's files.
|
|
8
|
+
*/
|
|
9
|
+
export function extraAllowanceName(name, extra) {
|
|
10
|
+
return `${name}[${extra}]`;
|
|
11
|
+
}
|
|
12
|
+
/** The extra in an allowance row name for `name`, or null when the row is not one of its extras. */
|
|
13
|
+
export function extraOfAllowanceName(rowName, name) {
|
|
14
|
+
const prefix = `${name}[`;
|
|
15
|
+
if (!rowName.startsWith(prefix) || !rowName.endsWith("]"))
|
|
16
|
+
return null;
|
|
17
|
+
const extra = rowName.slice(prefix.length, -1);
|
|
18
|
+
return /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(extra) ? extra : null;
|
|
19
|
+
}
|
|
1
20
|
/** In-memory adapter for tests. `contexts` is settable directly so tests can
|
|
2
21
|
* seed a run without going through a full proxy session. */
|
|
3
22
|
export class MemoryRegistryStore {
|
|
@@ -15,12 +34,26 @@ export class MemoryRegistryStore {
|
|
|
15
34
|
return context && context.deadlineAt > now ? context : null;
|
|
16
35
|
}
|
|
17
36
|
async isAllowedDependency(runId, ecosystem, name) {
|
|
37
|
+
// An extra's row (`name[extra]`) is never a package allowance.
|
|
38
|
+
if (name.includes("["))
|
|
39
|
+
return false;
|
|
18
40
|
return this.allowances.has(`${runId}\0${ecosystem}\0${name}`);
|
|
19
41
|
}
|
|
20
42
|
async addAllowances(runId, ecosystem, names) {
|
|
21
43
|
for (const name of names)
|
|
22
44
|
this.allowances.add(`${runId}\0${ecosystem}\0${name}`);
|
|
23
45
|
}
|
|
46
|
+
async allowedExtras(runId, ecosystem, name) {
|
|
47
|
+
const prefix = `${runId}\0${ecosystem}\0`;
|
|
48
|
+
return [...this.allowances]
|
|
49
|
+
.filter((key) => key.startsWith(prefix))
|
|
50
|
+
.map((key) => extraOfAllowanceName(key.slice(prefix.length), name))
|
|
51
|
+
.filter((extra) => extra !== null)
|
|
52
|
+
.sort();
|
|
53
|
+
}
|
|
54
|
+
async addExtraAllowances(runId, ecosystem, dependencies) {
|
|
55
|
+
await this.addAllowances(runId, ecosystem, dependencies.flatMap((dependency) => dependency.extras.map((extra) => extraAllowanceName(dependency.name, extra))));
|
|
56
|
+
}
|
|
24
57
|
async recordFetch(record) {
|
|
25
58
|
this.fetches.push({ ...record, createdAt: new Date() });
|
|
26
59
|
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { logger } from "../../core/logger.js";
|
|
1
2
|
import { REGISTRY_ADAPTERS } from "../../coding/registry/adapters.js";
|
|
2
3
|
import { CODING_PROXY_ALIAS, CODING_PROXY_DENY_PORT, CODING_PROXY_PORT } from "../jobs/docker-isolation.js";
|
|
3
4
|
import { startDenyPortListener } from "./deny-port.js";
|
|
4
5
|
import { EnvironmentCredentialResolver } from "./environment-credentials.js";
|
|
6
|
+
import { createMockUpstream, mockUpstreamFromEnv } from "./mock-upstream.js";
|
|
5
7
|
import { CodingProxy } from "./proxy.js";
|
|
6
8
|
import { PrismaProxyLedger } from "./prisma-ledger.js";
|
|
7
9
|
import { OsvAudit } from "./registry/audit.js";
|
|
@@ -38,10 +40,19 @@ export function createRegistryUpstream(pinned) {
|
|
|
38
40
|
/** Starts the trusted proxy with the fixed worker-only Docker endpoint, plus the deny port the enforcement witness probes. */
|
|
39
41
|
export async function startConfiguredCodingProxy(options) {
|
|
40
42
|
const env = options.env ?? process.env;
|
|
43
|
+
// Throws (mock_upstream_guard) on a partial or malformed setting, before anything listens.
|
|
44
|
+
const mock = mockUpstreamFromEnv(env);
|
|
45
|
+
if (mock) {
|
|
46
|
+
logger.warn({ module: "coding-proxy-runtime", event: "proxy.mock_upstream_enabled" }, "coding proxy: MOCK MODEL UPSTREAM ENABLED (load test). No request reaches a model provider; never use this on a real deployment.");
|
|
47
|
+
}
|
|
48
|
+
const audit = mock
|
|
49
|
+
? (event) => options.audit?.({ ...event, mockUpstream: true })
|
|
50
|
+
: options.audit;
|
|
41
51
|
const proxy = new CodingProxy({
|
|
42
52
|
ledger: new PrismaProxyLedger(options.db),
|
|
43
53
|
credentials: new EnvironmentCredentialResolver(env),
|
|
44
|
-
audit
|
|
54
|
+
audit,
|
|
55
|
+
...(mock ? { fetch: createMockUpstream(mock) } : {}),
|
|
45
56
|
});
|
|
46
57
|
const buildPinnedFetch = options.createPinnedFetch ?? createPinnedProxyFetch;
|
|
47
58
|
const upstreamHosts = [...new Set([...REGISTRY_ADAPTERS.values()].flatMap((adapter) => adapter.upstreamHosts))];
|
|
@@ -124,6 +124,8 @@ export interface ProxyAuditEvent {
|
|
|
124
124
|
/** For a failed stream: the upstream response's content type and encoding headers, for diagnosis. */
|
|
125
125
|
contentType?: string;
|
|
126
126
|
contentEncoding?: string;
|
|
127
|
+
/** Set on every event while the load-test mock upstream is enabled. */
|
|
128
|
+
mockUpstream?: boolean;
|
|
127
129
|
}
|
|
128
130
|
export type ProxyAuditSink = (event: ProxyAuditEvent) => void;
|
|
129
131
|
export interface CredentialResolver {
|
|
@@ -167,3 +167,16 @@ export declare function assertRunNetworkPolicyMatches(actual: V1NetworkPolicy, e
|
|
|
167
167
|
* The IP is validated and embedded as a JSON string literal.
|
|
168
168
|
*/
|
|
169
169
|
export declare function enforcementProbeScript(proxyIp: string): string;
|
|
170
|
+
/**
|
|
171
|
+
* A whole enforcement streak in one process: up to `streak` probes, each exactly
|
|
172
|
+
* `enforcementProbeScript`'s measurement and classification (the body is shared), `intervalMs`
|
|
173
|
+
* apart. Exits ENFORCEMENT_PROBE_PROVEN only after `streak` consecutive proven probes; at the
|
|
174
|
+
* first probe that is not proven it exits at once with that probe's own code, so the caller's
|
|
175
|
+
* verdict is still the last probe's. Running the streak in one exec (rather than one exec per
|
|
176
|
+
* probe) pays the keeper's exec + interpreter start-up once per streak instead of once per probe;
|
|
177
|
+
* the evidence required is unchanged.
|
|
178
|
+
*
|
|
179
|
+
* The IP is validated and embedded as a JSON string literal; `streak` and `intervalMs` must be
|
|
180
|
+
* positive safe integers and are embedded as numeric literals.
|
|
181
|
+
*/
|
|
182
|
+
export declare function enforcementStreakScript(proxyIp: string, streak: number, intervalMs: number): string;
|
|
@@ -770,9 +770,24 @@ export function assertRunNetworkPolicyMatches(actual, expected) {
|
|
|
770
770
|
* The IP is validated and embedded as a JSON string literal.
|
|
771
771
|
*/
|
|
772
772
|
export function enforcementProbeScript(proxyIp) {
|
|
773
|
+
return [...enforcementProbeBody(proxyIp), "(async () => {", " process.exit(await probe());", "})();"].join("\n");
|
|
774
|
+
}
|
|
775
|
+
/**
|
|
776
|
+
* The one measurement both enforcement scripts make, as script lines defining `probe()`: connect to
|
|
777
|
+
* CODING_PROXY_PORT, then to CODING_PROXY_DENY_PORT (both with a 3 s connect timeout, always both,
|
|
778
|
+
* in that order), and resolve to the exit code classifying that pair. Its first statement presets
|
|
779
|
+
* `process.exitCode = 1` so that a natural exit is never mistaken for PROVEN. Shared verbatim so the
|
|
780
|
+
* single probe and the streak can never disagree about what "proven" means.
|
|
781
|
+
*/
|
|
782
|
+
function enforcementProbeBody(proxyIp) {
|
|
773
783
|
if (isIP(proxyIp) === 0)
|
|
774
784
|
throw isolationError();
|
|
775
785
|
return [
|
|
786
|
+
// Preset a code no probe produces (it reads as "did not run"), so the only way to exit 0 is
|
|
787
|
+
// the explicit proven exit. Without it, a process that drains its event loop without ever
|
|
788
|
+
// calling process.exit — a probe promise that never settles, or a rejection under a
|
|
789
|
+
// non-throwing --unhandled-rejections mode — would exit 0 and read as PROVEN.
|
|
790
|
+
"process.exitCode = 1;",
|
|
776
791
|
'const net = require("node:net");',
|
|
777
792
|
"const tcp = (port) =>",
|
|
778
793
|
" new Promise((done) => {",
|
|
@@ -781,13 +796,45 @@ export function enforcementProbeScript(proxyIp) {
|
|
|
781
796
|
' socket.once("timeout", () => { socket.destroy(); done("timeout"); });',
|
|
782
797
|
' socket.once("error", () => done("error"));',
|
|
783
798
|
" });",
|
|
784
|
-
"
|
|
799
|
+
"const probe = async () => {",
|
|
785
800
|
` const allowed = await tcp(${CODING_PROXY_PORT});`,
|
|
786
801
|
` const denied = await tcp(${CODING_PROXY_DENY_PORT});`,
|
|
787
|
-
` if (allowed !== "connect")
|
|
788
|
-
` if (denied === "connect")
|
|
789
|
-
` if (denied === "error")
|
|
790
|
-
`
|
|
802
|
+
` if (allowed !== "connect") return ${ENFORCEMENT_PROBE_PROXY_UNREACHABLE};`,
|
|
803
|
+
` if (denied === "connect") return ${ENFORCEMENT_PROBE_DENY_REACHABLE};`,
|
|
804
|
+
` if (denied === "error") return ${ENFORCEMENT_PROBE_DENY_REFUSED};`,
|
|
805
|
+
` return ${ENFORCEMENT_PROBE_PROVEN};`,
|
|
806
|
+
"};",
|
|
807
|
+
];
|
|
808
|
+
}
|
|
809
|
+
/**
|
|
810
|
+
* A whole enforcement streak in one process: up to `streak` probes, each exactly
|
|
811
|
+
* `enforcementProbeScript`'s measurement and classification (the body is shared), `intervalMs`
|
|
812
|
+
* apart. Exits ENFORCEMENT_PROBE_PROVEN only after `streak` consecutive proven probes; at the
|
|
813
|
+
* first probe that is not proven it exits at once with that probe's own code, so the caller's
|
|
814
|
+
* verdict is still the last probe's. Running the streak in one exec (rather than one exec per
|
|
815
|
+
* probe) pays the keeper's exec + interpreter start-up once per streak instead of once per probe;
|
|
816
|
+
* the evidence required is unchanged.
|
|
817
|
+
*
|
|
818
|
+
* The IP is validated and embedded as a JSON string literal; `streak` and `intervalMs` must be
|
|
819
|
+
* positive safe integers and are embedded as numeric literals.
|
|
820
|
+
*/
|
|
821
|
+
export function enforcementStreakScript(proxyIp, streak, intervalMs) {
|
|
822
|
+
const body = enforcementProbeBody(proxyIp);
|
|
823
|
+
if (!isPositiveSafeInteger(streak) || !isPositiveSafeInteger(intervalMs))
|
|
824
|
+
throw isolationError();
|
|
825
|
+
return [
|
|
826
|
+
...body,
|
|
827
|
+
"const pause = (ms) => new Promise((done) => setTimeout(done, ms));",
|
|
828
|
+
"(async () => {",
|
|
829
|
+
` for (let index = 0; index < ${JSON.stringify(streak)}; index += 1) {`,
|
|
830
|
+
` if (index > 0) await pause(${JSON.stringify(intervalMs)});`,
|
|
831
|
+
" const code = await probe();",
|
|
832
|
+
` if (code !== ${ENFORCEMENT_PROBE_PROVEN}) return process.exit(code);`,
|
|
833
|
+
" }",
|
|
834
|
+
` return process.exit(${ENFORCEMENT_PROBE_PROVEN});`,
|
|
791
835
|
"})();",
|
|
792
836
|
].join("\n");
|
|
793
837
|
}
|
|
838
|
+
function isPositiveSafeInteger(value) {
|
|
839
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
|
|
840
|
+
}
|
|
@@ -24,15 +24,19 @@ export interface KubernetesJobLauncherOptions {
|
|
|
24
24
|
readyTimeoutMs?: number;
|
|
25
25
|
/**
|
|
26
26
|
* How long to wait for the run's NetworkPolicy to be enforced before seeding. Default 30_000,
|
|
27
|
-
* but never *less* than
|
|
28
|
-
*
|
|
29
|
-
*
|
|
27
|
+
* but never *less* than one full streak exec's timeout
|
|
28
|
+
* (`enforcementExecTimeoutMs * ENFORCEMENT_BLOCKED_STREAK + ENFORCEMENT_POLL_MS * (ENFORCEMENT_BLOCKED_STREAK - 1)`,
|
|
29
|
+
* 31_000 at the default exec timeout): see the constructor. An explicit override here is a floor
|
|
30
|
+
* raise, not an escape hatch — the gate still needs room for a full streak of probes at whatever
|
|
31
|
+
* the effective per-probe budget is.
|
|
30
32
|
*/
|
|
31
33
|
enforcementTimeoutMs?: number;
|
|
32
34
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
35
|
+
* Time budget for a single enforcement probe. Default 10_000 (DEFAULT_ENFORCEMENT_EXEC_TIMEOUT_MS).
|
|
36
|
+
* A streak's probes share one exec, whose timeout is this times ENFORCEMENT_BLOCKED_STREAK plus
|
|
37
|
+
* the poll gaps between them. Raise this on a resource-constrained cluster (e.g. `kind` on a
|
|
38
|
+
* laptop) where a healthy probe can legitimately take longer than 10 s to run inside the
|
|
39
|
+
* keeper's CPU/memory limits.
|
|
36
40
|
*/
|
|
37
41
|
enforcementExecTimeoutMs?: number;
|
|
38
42
|
createArchive?: (directory: string) => {
|
|
@@ -60,6 +64,8 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
|
|
|
60
64
|
private readonly warn;
|
|
61
65
|
private readonly enforcementTimeoutMs;
|
|
62
66
|
private readonly enforcementExecTimeoutMs;
|
|
67
|
+
/** Timeout of one streak exec: ENFORCEMENT_BLOCKED_STREAK probes plus the gaps between them. */
|
|
68
|
+
private readonly streakExecTimeoutMs;
|
|
63
69
|
private preflightResult?;
|
|
64
70
|
constructor(options: KubernetesJobLauncherOptions);
|
|
65
71
|
/**
|
|
@@ -96,6 +102,14 @@ export declare class KubernetesJobLauncher implements WorkspaceJobLauncher {
|
|
|
96
102
|
* proxy on CODING_PROXY_PORT and fail to reach it on CODING_PROXY_DENY_PORT. Anything else resets
|
|
97
103
|
* the streak; the wall-clock bound still applies.
|
|
98
104
|
*
|
|
105
|
+
* The whole streak runs in ONE keeper exec (`enforcementStreakScript`), so exec and interpreter
|
|
106
|
+
* start-up is paid once per streak rather than once per probe. That exec exits 0 only after the
|
|
107
|
+
* full streak was proven; otherwise it stops at the first non-proven probe and exits with that
|
|
108
|
+
* probe's code. Any non-zero exit is a broken streak: if the bound has been reached it is the
|
|
109
|
+
* verdict, else the launcher waits ENFORCEMENT_POLL_MS and starts a fresh streak from zero. The
|
|
110
|
+
* exec is bounded by `streakExecTimeoutMs` (the per-probe budget times the streak, plus the gaps);
|
|
111
|
+
* an exec that times out or fails throws, failing the launch exactly as a failed probe exec did.
|
|
112
|
+
*
|
|
99
113
|
* "Fail to reach" means the connect *timed out* — the packet was dropped. A refusal (RST) is not
|
|
100
114
|
* a denial: it proves the SYN reached the destination host, so the deny port is merely unserved
|
|
101
115
|
* and witnesses nothing. That is its own verdict, below.
|