@vellumai/cli 0.8.12-dev.202606152248.70317d3 → 0.8.12-dev.202606152340.7efde97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@vellumai/local-mode/src/__tests__/status.test.ts +224 -0
- package/node_modules/@vellumai/local-mode/src/index.ts +5 -0
- package/node_modules/@vellumai/local-mode/src/lockfile-contract.ts +8 -1
- package/node_modules/@vellumai/local-mode/src/status.ts +342 -0
- package/package.json +1 -1
- package/src/__tests__/nginx-ingress.test.ts +129 -0
- package/src/__tests__/tunnel.test.ts +35 -0
- package/src/commands/nginx-ingress.ts +22 -6
- package/src/commands/tunnel.ts +43 -10
- package/src/lib/assistant-client.ts +2 -0
- package/src/lib/feature-flags.test.ts +45 -0
- package/src/lib/feature-flags.ts +5 -1
- package/src/lib/nginx-ingress.ts +229 -10
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
2
|
+
import { mkdirSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { createServer, type Server } from "node:http";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
|
|
7
|
+
import { getLocalAssistantStatus } from "../status";
|
|
8
|
+
|
|
9
|
+
let tempDir: string;
|
|
10
|
+
let lockfilePath: string;
|
|
11
|
+
let instanceDir: string;
|
|
12
|
+
|
|
13
|
+
function writeLockfile(entry: Record<string, unknown>): void {
|
|
14
|
+
writeFileSync(
|
|
15
|
+
lockfilePath,
|
|
16
|
+
JSON.stringify({
|
|
17
|
+
assistants: [entry],
|
|
18
|
+
activeAssistant: entry.assistantId,
|
|
19
|
+
}),
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function writeLocalLockfile(overrides: Record<string, unknown> = {}): void {
|
|
24
|
+
writeLockfile({
|
|
25
|
+
assistantId: "local-1",
|
|
26
|
+
cloud: "local",
|
|
27
|
+
resources: {
|
|
28
|
+
instanceDir,
|
|
29
|
+
daemonPort: 30101,
|
|
30
|
+
gatewayPort: 30102,
|
|
31
|
+
qdrantPort: 30103,
|
|
32
|
+
},
|
|
33
|
+
...overrides,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function writeAssistantPid(value: string): string {
|
|
38
|
+
const pidDir = path.join(instanceDir, ".vellum", "workspace");
|
|
39
|
+
mkdirSync(pidDir, { recursive: true });
|
|
40
|
+
const pidPath = path.join(pidDir, "vellum.pid");
|
|
41
|
+
writeFileSync(pidPath, value);
|
|
42
|
+
return pidPath;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function markStale(filePath: string): void {
|
|
46
|
+
const stale = new Date(Date.now() - 120_000);
|
|
47
|
+
utimesSync(filePath, stale, stale);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function listen(server: Server, port = 0): Promise<number> {
|
|
51
|
+
return new Promise((resolve) => {
|
|
52
|
+
server.listen(port, "127.0.0.1", () => {
|
|
53
|
+
const address = server.address();
|
|
54
|
+
if (!address || typeof address === "string") {
|
|
55
|
+
throw new Error("expected TCP server address");
|
|
56
|
+
}
|
|
57
|
+
resolve(address.port);
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function close(server: Server): Promise<void> {
|
|
63
|
+
return new Promise((resolve, reject) => {
|
|
64
|
+
server.close((err) => (err ? reject(err) : resolve()));
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function unusedPort(): Promise<number> {
|
|
69
|
+
const server = createServer();
|
|
70
|
+
const port = await listen(server);
|
|
71
|
+
await close(server);
|
|
72
|
+
return port;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
beforeEach(() => {
|
|
76
|
+
tempDir = path.join(
|
|
77
|
+
tmpdir(),
|
|
78
|
+
`vellum-local-status-test-${Date.now()}-${Math.random()}`,
|
|
79
|
+
);
|
|
80
|
+
mkdirSync(tempDir, { recursive: true });
|
|
81
|
+
lockfilePath = path.join(tempDir, "lockfile.json");
|
|
82
|
+
instanceDir = path.join(tempDir, "instance");
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
afterEach(() => {
|
|
86
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
describe("getLocalAssistantStatus", () => {
|
|
90
|
+
test("returns sleeping when the assistant PID file is absent", async () => {
|
|
91
|
+
writeLocalLockfile();
|
|
92
|
+
|
|
93
|
+
expect(await getLocalAssistantStatus([lockfilePath], "local-1")).toEqual({
|
|
94
|
+
ok: true,
|
|
95
|
+
state: "sleeping",
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("returns sleeping for legacy local entries without cloud/resources", async () => {
|
|
100
|
+
writeLockfile({
|
|
101
|
+
assistantId: "legacy-local",
|
|
102
|
+
baseDataDir: instanceDir,
|
|
103
|
+
runtimeUrl: "http://127.0.0.1:30102",
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
expect(
|
|
107
|
+
await getLocalAssistantStatus([lockfilePath], "legacy-local"),
|
|
108
|
+
).toEqual({
|
|
109
|
+
ok: true,
|
|
110
|
+
state: "sleeping",
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("returns sleeping when the assistant PID file points at a dead process", async () => {
|
|
115
|
+
writeLocalLockfile();
|
|
116
|
+
writeAssistantPid("999999999");
|
|
117
|
+
|
|
118
|
+
const result = await getLocalAssistantStatus([lockfilePath], "local-1");
|
|
119
|
+
|
|
120
|
+
expect(result.ok).toBe(true);
|
|
121
|
+
if (result.ok) {
|
|
122
|
+
expect(result.state).toBe("sleeping");
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("returns crashed when the assistant PID file is invalid", async () => {
|
|
127
|
+
writeLocalLockfile();
|
|
128
|
+
writeAssistantPid("not-a-pid");
|
|
129
|
+
|
|
130
|
+
const result = await getLocalAssistantStatus([lockfilePath], "local-1");
|
|
131
|
+
|
|
132
|
+
expect(result.ok).toBe(true);
|
|
133
|
+
if (result.ok) {
|
|
134
|
+
expect(result.state).toBe("crashed");
|
|
135
|
+
expect(result.detail).toContain("PID file");
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("returns starting when a fresh assistant PID is alive but health is not ready yet", async () => {
|
|
140
|
+
writeLocalLockfile({
|
|
141
|
+
resources: {
|
|
142
|
+
instanceDir,
|
|
143
|
+
daemonPort: await unusedPort(),
|
|
144
|
+
gatewayPort: 30102,
|
|
145
|
+
qdrantPort: 30103,
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
writeAssistantPid(String(process.pid));
|
|
149
|
+
|
|
150
|
+
const result = await getLocalAssistantStatus([lockfilePath], "local-1");
|
|
151
|
+
|
|
152
|
+
expect(result.ok).toBe(true);
|
|
153
|
+
if (result.ok) {
|
|
154
|
+
expect(result.state).toBe("starting");
|
|
155
|
+
expect(result.pid).toBe(process.pid);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test("returns crashed when an old live assistant PID is still not responding", async () => {
|
|
160
|
+
writeLocalLockfile({
|
|
161
|
+
resources: {
|
|
162
|
+
instanceDir,
|
|
163
|
+
daemonPort: await unusedPort(),
|
|
164
|
+
gatewayPort: 30102,
|
|
165
|
+
qdrantPort: 30103,
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
const pidPath = writeAssistantPid(String(process.pid));
|
|
169
|
+
markStale(pidPath);
|
|
170
|
+
|
|
171
|
+
const result = await getLocalAssistantStatus([lockfilePath], "local-1");
|
|
172
|
+
|
|
173
|
+
expect(result.ok).toBe(true);
|
|
174
|
+
if (result.ok) {
|
|
175
|
+
expect(result.state).toBe("crashed");
|
|
176
|
+
expect(result.detail).toContain("not responding");
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("returns starting while the gateway is coming up for a freshly started assistant", async () => {
|
|
181
|
+
const server = createServer((_req, res) => {
|
|
182
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
183
|
+
res.end(JSON.stringify({ status: "healthy" }));
|
|
184
|
+
});
|
|
185
|
+
const daemonPort = await listen(server);
|
|
186
|
+
try {
|
|
187
|
+
writeLocalLockfile({
|
|
188
|
+
resources: {
|
|
189
|
+
instanceDir,
|
|
190
|
+
daemonPort,
|
|
191
|
+
gatewayPort: await unusedPort(),
|
|
192
|
+
qdrantPort: 30103,
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
writeAssistantPid(String(process.pid));
|
|
196
|
+
|
|
197
|
+
const result = await getLocalAssistantStatus([lockfilePath], "local-1");
|
|
198
|
+
|
|
199
|
+
expect(result.ok).toBe(true);
|
|
200
|
+
if (result.ok) {
|
|
201
|
+
expect(result.state).toBe("starting");
|
|
202
|
+
expect(result.pid).toBe(process.pid);
|
|
203
|
+
}
|
|
204
|
+
} finally {
|
|
205
|
+
await close(server);
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
test("rejects non-local assistants", async () => {
|
|
210
|
+
writeLockfile({
|
|
211
|
+
assistantId: "platform-1",
|
|
212
|
+
cloud: "vellum",
|
|
213
|
+
runtimeUrl: "https://example.com",
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
expect(await getLocalAssistantStatus([lockfilePath], "platform-1")).toEqual(
|
|
217
|
+
{
|
|
218
|
+
ok: false,
|
|
219
|
+
status: 404,
|
|
220
|
+
error: "Local assistant not found",
|
|
221
|
+
},
|
|
222
|
+
);
|
|
223
|
+
});
|
|
224
|
+
});
|
|
@@ -37,6 +37,11 @@ export { runRetire } from "./retire";
|
|
|
37
37
|
export type { RetireResult } from "./retire";
|
|
38
38
|
export { runWake } from "./wake";
|
|
39
39
|
export type { WakeOptions, WakeResult } from "./wake";
|
|
40
|
+
export { getLocalAssistantStatus } from "./status";
|
|
41
|
+
export type {
|
|
42
|
+
LocalAssistantRuntimeState,
|
|
43
|
+
LocalAssistantStatusResult,
|
|
44
|
+
} from "./status";
|
|
40
45
|
export { getGuardianAccessToken } from "./guardian-token";
|
|
41
46
|
export type { TokenResult } from "./guardian-token";
|
|
42
47
|
export {
|
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
33
|
export interface LocalAssistantResources {
|
|
34
|
+
instanceDir?: string;
|
|
34
35
|
gatewayPort: number;
|
|
35
36
|
daemonPort: number;
|
|
36
37
|
}
|
|
@@ -69,7 +70,13 @@ function parseResources(value: unknown): LocalAssistantResources | undefined {
|
|
|
69
70
|
if (!isRecord(value)) return undefined;
|
|
70
71
|
if (typeof value.gatewayPort !== "number") return undefined;
|
|
71
72
|
if (typeof value.daemonPort !== "number") return undefined;
|
|
72
|
-
return {
|
|
73
|
+
return {
|
|
74
|
+
...(typeof value.instanceDir === "string"
|
|
75
|
+
? { instanceDir: value.instanceDir }
|
|
76
|
+
: {}),
|
|
77
|
+
gatewayPort: value.gatewayPort,
|
|
78
|
+
daemonPort: value.daemonPort,
|
|
79
|
+
};
|
|
73
80
|
}
|
|
74
81
|
|
|
75
82
|
/**
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import http from "node:http";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { SEEDS } from "@vellumai/environments";
|
|
7
|
+
|
|
8
|
+
import type {
|
|
9
|
+
LockfileAssistant,
|
|
10
|
+
} from "./lockfile-contract";
|
|
11
|
+
import { getLockfileData } from "./lockfile";
|
|
12
|
+
|
|
13
|
+
const HEALTH_TIMEOUT_MS = 1_500;
|
|
14
|
+
const STARTING_GRACE_MS = 60_000;
|
|
15
|
+
const PRODUCTION_ENVIRONMENT_NAME = "production";
|
|
16
|
+
const DEFAULT_PORTS = {
|
|
17
|
+
daemon: 7821,
|
|
18
|
+
gateway: 7830,
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type LocalAssistantRuntimeState =
|
|
22
|
+
| "healthy"
|
|
23
|
+
| "sleeping"
|
|
24
|
+
| "starting"
|
|
25
|
+
| "crashed"
|
|
26
|
+
| "unknown";
|
|
27
|
+
|
|
28
|
+
export type LocalAssistantStatusResult =
|
|
29
|
+
| {
|
|
30
|
+
ok: true;
|
|
31
|
+
state: LocalAssistantRuntimeState;
|
|
32
|
+
detail?: string;
|
|
33
|
+
pid?: number;
|
|
34
|
+
}
|
|
35
|
+
| { ok: false; status: number; error: string };
|
|
36
|
+
|
|
37
|
+
type PidState =
|
|
38
|
+
| { state: "missing" }
|
|
39
|
+
| { state: "starting"; updatedAtMs: number }
|
|
40
|
+
| { state: "alive"; pid: number; updatedAtMs: number }
|
|
41
|
+
| { state: "dead"; pid: number; updatedAtMs: number }
|
|
42
|
+
| { state: "invalid"; value: string; updatedAtMs: number };
|
|
43
|
+
|
|
44
|
+
interface StatusResources {
|
|
45
|
+
instanceDir: string;
|
|
46
|
+
gatewayPort: number;
|
|
47
|
+
daemonPort: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function getDaemonPidPath(instanceDir: string): string {
|
|
51
|
+
return path.join(instanceDir, ".vellum", "workspace", "vellum.pid");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function getGatewayPidPath(instanceDir: string): string {
|
|
55
|
+
return path.join(instanceDir, ".vellum", "gateway.pid");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function readPidState(pidFile: string): PidState {
|
|
59
|
+
if (!existsSync(pidFile)) return { state: "missing" };
|
|
60
|
+
|
|
61
|
+
const updatedAtMs = statSync(pidFile).mtimeMs;
|
|
62
|
+
const value = readFileSync(pidFile, "utf-8").trim();
|
|
63
|
+
if (!value) return { state: "missing" };
|
|
64
|
+
if (value === "starting") return { state: "starting", updatedAtMs };
|
|
65
|
+
|
|
66
|
+
const pid = Number(value);
|
|
67
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
68
|
+
return { state: "invalid", value, updatedAtMs };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
try {
|
|
72
|
+
process.kill(pid, 0);
|
|
73
|
+
return { state: "alive", pid, updatedAtMs };
|
|
74
|
+
} catch {
|
|
75
|
+
return { state: "dead", pid, updatedAtMs };
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function isFreshPidState(
|
|
80
|
+
pidState: PidState,
|
|
81
|
+
observedAtMs: number,
|
|
82
|
+
): boolean {
|
|
83
|
+
return (
|
|
84
|
+
"updatedAtMs" in pidState &&
|
|
85
|
+
observedAtMs - pidState.updatedAtMs <= STARTING_GRACE_MS
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function httpHealthCheck(port: number): Promise<boolean> {
|
|
90
|
+
return new Promise((resolve) => {
|
|
91
|
+
const req = http.get(
|
|
92
|
+
{
|
|
93
|
+
hostname: "127.0.0.1",
|
|
94
|
+
port,
|
|
95
|
+
path: "/healthz",
|
|
96
|
+
timeout: HEALTH_TIMEOUT_MS,
|
|
97
|
+
},
|
|
98
|
+
(res) => {
|
|
99
|
+
const chunks: Buffer[] = [];
|
|
100
|
+
res.on("data", (chunk: Buffer) => chunks.push(chunk));
|
|
101
|
+
res.on("end", () => {
|
|
102
|
+
if (res.statusCode !== 200) {
|
|
103
|
+
resolve(false);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
try {
|
|
108
|
+
const body = JSON.parse(Buffer.concat(chunks).toString()) as {
|
|
109
|
+
status?: string;
|
|
110
|
+
};
|
|
111
|
+
resolve(
|
|
112
|
+
body.status === undefined ||
|
|
113
|
+
body.status === "healthy" ||
|
|
114
|
+
body.status === "ok",
|
|
115
|
+
);
|
|
116
|
+
} catch {
|
|
117
|
+
resolve(true);
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
},
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
req.on("timeout", () => {
|
|
124
|
+
req.destroy();
|
|
125
|
+
resolve(false);
|
|
126
|
+
});
|
|
127
|
+
req.on("error", () => resolve(false));
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function localOnlyEntry(
|
|
132
|
+
entry: LockfileAssistant | undefined,
|
|
133
|
+
): LockfileAssistant | null {
|
|
134
|
+
if (!entry || (entry.cloud != null && entry.cloud !== "local")) return null;
|
|
135
|
+
return entry;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
139
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function parsePortFromUrl(url: unknown): number | undefined {
|
|
143
|
+
if (typeof url !== "string") return undefined;
|
|
144
|
+
try {
|
|
145
|
+
const parsed = new URL(url);
|
|
146
|
+
const port = Number(parsed.port);
|
|
147
|
+
return Number.isInteger(port) && port > 0 ? port : undefined;
|
|
148
|
+
} catch {
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function defaultPorts(env: Record<string, string | undefined>): {
|
|
154
|
+
daemon: number;
|
|
155
|
+
gateway: number;
|
|
156
|
+
} {
|
|
157
|
+
const envName = env.VELLUM_ENVIRONMENT?.trim() || PRODUCTION_ENVIRONMENT_NAME;
|
|
158
|
+
const seed = SEEDS[envName] ?? SEEDS[PRODUCTION_ENVIRONMENT_NAME];
|
|
159
|
+
return {
|
|
160
|
+
daemon: seed?.portsOverride?.daemon ?? DEFAULT_PORTS.daemon,
|
|
161
|
+
gateway: seed?.portsOverride?.gateway ?? DEFAULT_PORTS.gateway,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function defaultInstanceDir(
|
|
166
|
+
env: Record<string, string | undefined>,
|
|
167
|
+
assistantId: string,
|
|
168
|
+
): string {
|
|
169
|
+
const envName = env.VELLUM_ENVIRONMENT?.trim() || PRODUCTION_ENVIRONMENT_NAME;
|
|
170
|
+
const xdgDataHome =
|
|
171
|
+
env.XDG_DATA_HOME?.trim() || path.join(os.homedir(), ".local", "share");
|
|
172
|
+
const dataRoot =
|
|
173
|
+
envName === PRODUCTION_ENVIRONMENT_NAME ? "vellum" : `vellum-${envName}`;
|
|
174
|
+
return path.join(xdgDataHome, dataRoot, "assistants", assistantId);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function firstString(...values: unknown[]): string | undefined {
|
|
178
|
+
for (const value of values) {
|
|
179
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
180
|
+
}
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function firstNumber(...values: unknown[]): number | undefined {
|
|
185
|
+
for (const value of values) {
|
|
186
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
187
|
+
}
|
|
188
|
+
return undefined;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function findRawAssistant(
|
|
192
|
+
lockfilePaths: string[],
|
|
193
|
+
assistantId: string,
|
|
194
|
+
): Record<string, unknown> | null {
|
|
195
|
+
for (const candidate of lockfilePaths) {
|
|
196
|
+
let data: unknown;
|
|
197
|
+
try {
|
|
198
|
+
data = JSON.parse(readFileSync(candidate, "utf-8"));
|
|
199
|
+
} catch {
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if (!isRecord(data) || !Array.isArray(data.assistants)) return null;
|
|
203
|
+
const entry = data.assistants.find(
|
|
204
|
+
(assistant) =>
|
|
205
|
+
isRecord(assistant) && assistant.assistantId === assistantId,
|
|
206
|
+
);
|
|
207
|
+
return isRecord(entry) ? entry : null;
|
|
208
|
+
}
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function resolveStatusResources(
|
|
213
|
+
entry: LockfileAssistant,
|
|
214
|
+
rawEntry: Record<string, unknown> | null,
|
|
215
|
+
env: Record<string, string | undefined>,
|
|
216
|
+
): StatusResources {
|
|
217
|
+
const rawResources = isRecord(rawEntry?.resources)
|
|
218
|
+
? rawEntry.resources
|
|
219
|
+
: undefined;
|
|
220
|
+
const ports = defaultPorts(env);
|
|
221
|
+
const instanceDir =
|
|
222
|
+
firstString(
|
|
223
|
+
entry.resources?.instanceDir,
|
|
224
|
+
rawResources?.instanceDir,
|
|
225
|
+
rawEntry?.baseDataDir,
|
|
226
|
+
) ?? defaultInstanceDir(env, entry.assistantId);
|
|
227
|
+
return {
|
|
228
|
+
instanceDir,
|
|
229
|
+
daemonPort:
|
|
230
|
+
firstNumber(entry.resources?.daemonPort, rawResources?.daemonPort) ??
|
|
231
|
+
ports.daemon,
|
|
232
|
+
gatewayPort:
|
|
233
|
+
firstNumber(entry.resources?.gatewayPort, rawResources?.gatewayPort) ??
|
|
234
|
+
parsePortFromUrl(rawEntry?.localUrl) ??
|
|
235
|
+
parsePortFromUrl(rawEntry?.runtimeUrl ?? entry.runtimeUrl) ??
|
|
236
|
+
ports.gateway,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function runtimeStatusForEntry(
|
|
241
|
+
entry: LockfileAssistant,
|
|
242
|
+
rawEntry: Record<string, unknown> | null,
|
|
243
|
+
env: Record<string, string | undefined>,
|
|
244
|
+
): Promise<LocalAssistantStatusResult> {
|
|
245
|
+
const resources = resolveStatusResources(entry, rawEntry, env);
|
|
246
|
+
const observedAtMs = Date.now();
|
|
247
|
+
|
|
248
|
+
const assistantPid = readPidState(getDaemonPidPath(resources.instanceDir));
|
|
249
|
+
if (assistantPid.state === "missing") {
|
|
250
|
+
return { ok: true, state: "sleeping" };
|
|
251
|
+
}
|
|
252
|
+
if (assistantPid.state === "starting") {
|
|
253
|
+
return { ok: true, state: "starting" };
|
|
254
|
+
}
|
|
255
|
+
if (assistantPid.state === "dead") {
|
|
256
|
+
return { ok: true, state: "sleeping", pid: assistantPid.pid };
|
|
257
|
+
}
|
|
258
|
+
if (assistantPid.state === "invalid") {
|
|
259
|
+
return {
|
|
260
|
+
ok: true,
|
|
261
|
+
state: "crashed",
|
|
262
|
+
detail: "assistant PID file is invalid",
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const assistantHealthy = await httpHealthCheck(resources.daemonPort);
|
|
267
|
+
if (!assistantHealthy) {
|
|
268
|
+
if (isFreshPidState(assistantPid, observedAtMs)) {
|
|
269
|
+
return { ok: true, state: "starting", pid: assistantPid.pid };
|
|
270
|
+
}
|
|
271
|
+
return {
|
|
272
|
+
ok: true,
|
|
273
|
+
state: "crashed",
|
|
274
|
+
pid: assistantPid.pid,
|
|
275
|
+
detail: "assistant process is not responding",
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const gatewayPid = readPidState(getGatewayPidPath(resources.instanceDir));
|
|
280
|
+
if (gatewayPid.state === "starting") {
|
|
281
|
+
return { ok: true, state: "starting", pid: assistantPid.pid };
|
|
282
|
+
}
|
|
283
|
+
if (gatewayPid.state !== "alive") {
|
|
284
|
+
if (
|
|
285
|
+
isFreshPidState(assistantPid, observedAtMs) ||
|
|
286
|
+
isFreshPidState(gatewayPid, observedAtMs)
|
|
287
|
+
) {
|
|
288
|
+
return { ok: true, state: "starting", pid: assistantPid.pid };
|
|
289
|
+
}
|
|
290
|
+
return {
|
|
291
|
+
ok: true,
|
|
292
|
+
state: "crashed",
|
|
293
|
+
pid: assistantPid.pid,
|
|
294
|
+
detail: "gateway process is not running",
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const gatewayHealthy = await httpHealthCheck(resources.gatewayPort);
|
|
299
|
+
if (!gatewayHealthy) {
|
|
300
|
+
if (isFreshPidState(gatewayPid, observedAtMs)) {
|
|
301
|
+
return { ok: true, state: "starting", pid: gatewayPid.pid };
|
|
302
|
+
}
|
|
303
|
+
return {
|
|
304
|
+
ok: true,
|
|
305
|
+
state: "crashed",
|
|
306
|
+
pid: gatewayPid.pid,
|
|
307
|
+
detail: "gateway process is not responding",
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
return { ok: true, state: "healthy", pid: assistantPid.pid };
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
export async function getLocalAssistantStatus(
|
|
315
|
+
lockfilePaths: string[],
|
|
316
|
+
assistantId: string,
|
|
317
|
+
env: Record<string, string | undefined> = process.env,
|
|
318
|
+
): Promise<LocalAssistantStatusResult> {
|
|
319
|
+
const result = getLockfileData(lockfilePaths);
|
|
320
|
+
if (!result.ok) {
|
|
321
|
+
return {
|
|
322
|
+
ok: false,
|
|
323
|
+
status: result.status,
|
|
324
|
+
error: result.error ?? "Failed to read lockfile",
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const entry = localOnlyEntry(
|
|
329
|
+
result.data.assistants.find(
|
|
330
|
+
(assistant) => assistant.assistantId === assistantId,
|
|
331
|
+
),
|
|
332
|
+
);
|
|
333
|
+
if (!entry) {
|
|
334
|
+
return { ok: false, status: 404, error: "Local assistant not found" };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
return runtimeStatusForEntry(
|
|
338
|
+
entry,
|
|
339
|
+
findRawAssistant(lockfilePaths, assistantId),
|
|
340
|
+
env,
|
|
341
|
+
);
|
|
342
|
+
}
|
package/package.json
CHANGED
|
@@ -21,6 +21,7 @@ mock.module("node:child_process", () => ({
|
|
|
21
21
|
|
|
22
22
|
import {
|
|
23
23
|
buildIngressNginxConfig,
|
|
24
|
+
buildRemoteWebIndexHtml,
|
|
24
25
|
resolveTunnelTargetPort,
|
|
25
26
|
stopIngressNginx,
|
|
26
27
|
} from "../lib/nginx-ingress.js";
|
|
@@ -29,6 +30,18 @@ const originalKill = process.kill;
|
|
|
29
30
|
|
|
30
31
|
describe("buildIngressNginxConfig", () => {
|
|
31
32
|
const conf = buildIngressNginxConfig({ gatewayPort: 7830, listenPort: 7840 });
|
|
33
|
+
const remoteConf = buildIngressNginxConfig({
|
|
34
|
+
gatewayPort: 7830,
|
|
35
|
+
listenPort: 7840,
|
|
36
|
+
remoteWebIngress: {
|
|
37
|
+
webDistDir: "/tmp/vellum web/dist",
|
|
38
|
+
config: {
|
|
39
|
+
mode: "remote-gateway",
|
|
40
|
+
apiBaseUrl: "/v1",
|
|
41
|
+
platformDisabled: true,
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
});
|
|
32
45
|
|
|
33
46
|
test("listens on loopback only", () => {
|
|
34
47
|
expect(conf).toContain("listen 127.0.0.1:7840;");
|
|
@@ -42,12 +55,97 @@ describe("buildIngressNginxConfig", () => {
|
|
|
42
55
|
test("proxies requests to the gateway", () => {
|
|
43
56
|
expect(conf).toContain("location / {");
|
|
44
57
|
expect(conf).toContain("proxy_pass http://127.0.0.1:7830;");
|
|
58
|
+
expect(conf).toContain('proxy_set_header X-Vellum-Edge-Forwarded "1";');
|
|
45
59
|
expect(conf).not.toContain("return 404;");
|
|
46
60
|
expect(conf).not.toContain("return 403;");
|
|
47
61
|
expect(conf).not.toContain("location =");
|
|
48
62
|
expect(conf).not.toContain("location ~");
|
|
49
63
|
});
|
|
50
64
|
|
|
65
|
+
test("declares static MIME types needed by the SPA", () => {
|
|
66
|
+
expect(remoteConf).toContain("default_type application/octet-stream;");
|
|
67
|
+
expect(remoteConf).toContain("types {");
|
|
68
|
+
expect(remoteConf).toContain("application/javascript js mjs;");
|
|
69
|
+
expect(remoteConf).toContain("text/css css;");
|
|
70
|
+
expect(remoteConf).toContain("text/html html htm;");
|
|
71
|
+
expect(remoteConf).toContain("font/woff2 woff2;");
|
|
72
|
+
expect(remoteConf).toContain("image/svg+xml svg svgz;");
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("serves the remote web SPA from /assistant when configured", () => {
|
|
76
|
+
expect(remoteConf).toContain("location = / {");
|
|
77
|
+
expect(remoteConf).toContain("return 302 /assistant/;");
|
|
78
|
+
expect(remoteConf.indexOf("location = / {")).toBeLessThan(
|
|
79
|
+
remoteConf.indexOf("location / {"),
|
|
80
|
+
);
|
|
81
|
+
expect(remoteConf).toContain("location = /assistant {");
|
|
82
|
+
expect(remoteConf).toContain("return 302 /assistant/;");
|
|
83
|
+
expect(remoteConf).toContain("location ^~ /assistant/assets/ {");
|
|
84
|
+
expect(remoteConf).toContain('alias "/tmp/vellum web/dist/assets/";');
|
|
85
|
+
expect(remoteConf).toContain("try_files $uri =404;");
|
|
86
|
+
expect(remoteConf).toContain("location = /assistant/ {");
|
|
87
|
+
expect(remoteConf).toContain(
|
|
88
|
+
"rewrite ^ /assistant/__remote-index.html last;",
|
|
89
|
+
);
|
|
90
|
+
expect(remoteConf).toContain("location = /assistant/index.html {");
|
|
91
|
+
expect(remoteConf).toContain("location = /assistant/__remote-index.html {");
|
|
92
|
+
expect(remoteConf).toContain("internal;");
|
|
93
|
+
expect(remoteConf).toContain('alias "/tmp/vellum web/dist/index.html";');
|
|
94
|
+
expect(remoteConf).toContain("location ^~ /assistant/ {");
|
|
95
|
+
expect(remoteConf).toContain('alias "/tmp/vellum web/dist/";');
|
|
96
|
+
expect(remoteConf).toContain(
|
|
97
|
+
"try_files $uri $uri/ /assistant/__remote-index.html;",
|
|
98
|
+
);
|
|
99
|
+
expect(remoteConf).toContain("location / {\n return 404;\n }");
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("serves remote web config for the SPA", () => {
|
|
103
|
+
expect(remoteConf).toContain("location = /assistant/__config {");
|
|
104
|
+
expect(remoteConf).toContain("default_type application/json;");
|
|
105
|
+
expect(remoteConf).toContain('add_header Cache-Control "no-store";');
|
|
106
|
+
expect(remoteConf).toContain(
|
|
107
|
+
'return 200 "{\\"mode\\":\\"remote-gateway\\",\\"apiBaseUrl\\":\\"/v1\\",\\"platformDisabled\\":true,\\"disablePlatform\\":true}";',
|
|
108
|
+
);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test("proxies health and public API traffic to the gateway in remote web mode", () => {
|
|
112
|
+
expect(remoteConf).toContain("location = /healthz {");
|
|
113
|
+
expect(remoteConf).toContain("location ^~ /v1/ {");
|
|
114
|
+
expect(remoteConf).toContain("proxy_pass http://127.0.0.1:7830;");
|
|
115
|
+
expect(remoteConf).toContain("proxy_request_buffering off;");
|
|
116
|
+
expect(remoteConf).toContain("proxy_buffering off;");
|
|
117
|
+
expect(remoteConf).toContain(
|
|
118
|
+
'proxy_set_header X-Vellum-Edge-Forwarded "1";',
|
|
119
|
+
);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("blocks local-only bootstrap helpers before generic API proxying", () => {
|
|
123
|
+
const deniedLocations = [
|
|
124
|
+
"location = /auth/token { return 404; }",
|
|
125
|
+
"location = /auth/token/ { return 404; }",
|
|
126
|
+
"location = /v1/pair { return 404; }",
|
|
127
|
+
"location = /v1/pair/ { return 404; }",
|
|
128
|
+
"location = /v1/pair/web-init { return 404; }",
|
|
129
|
+
"location = /v1/pair/web-init/ { return 404; }",
|
|
130
|
+
"location = /v1/devices { return 404; }",
|
|
131
|
+
"location = /v1/devices/ { return 404; }",
|
|
132
|
+
"location = /v1/devices/revoke { return 404; }",
|
|
133
|
+
"location = /v1/devices/revoke/ { return 404; }",
|
|
134
|
+
"location = /v1/guardian/init { return 404; }",
|
|
135
|
+
"location = /v1/guardian/init/ { return 404; }",
|
|
136
|
+
"location = /v1/guardian/reset-bootstrap { return 404; }",
|
|
137
|
+
"location = /v1/guardian/reset-bootstrap/ { return 404; }",
|
|
138
|
+
"location ^~ /assistant/__local/ { return 404; }",
|
|
139
|
+
"location ^~ /assistant/__gateway/ { return 404; }",
|
|
140
|
+
];
|
|
141
|
+
for (const location of deniedLocations) {
|
|
142
|
+
expect(remoteConf).toContain(location);
|
|
143
|
+
expect(remoteConf.indexOf(location)).toBeLessThan(
|
|
144
|
+
remoteConf.indexOf("location ^~ /v1/ {"),
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
|
|
51
149
|
test("supports websockets and SSE streaming", () => {
|
|
52
150
|
expect(conf).toContain("map $http_upgrade $connection_upgrade");
|
|
53
151
|
expect(conf).toContain("proxy_http_version 1.1;");
|
|
@@ -59,6 +157,37 @@ describe("buildIngressNginxConfig", () => {
|
|
|
59
157
|
});
|
|
60
158
|
});
|
|
61
159
|
|
|
160
|
+
describe("buildRemoteWebIndexHtml", () => {
|
|
161
|
+
test("injects the remote gateway config after any bundled local config", () => {
|
|
162
|
+
const html =
|
|
163
|
+
'<html><head><script>window.__VELLUM_CONFIG__={"webUrl":"https://www.vellum.ai"}</script></head><body></body></html>';
|
|
164
|
+
const result = buildRemoteWebIndexHtml(html, {
|
|
165
|
+
mode: "remote-gateway",
|
|
166
|
+
apiBaseUrl: "/v1",
|
|
167
|
+
disablePlatform: true,
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
expect(result).toContain(
|
|
171
|
+
'window.__VELLUM_CONFIG__={"webUrl":"https://www.vellum.ai"}',
|
|
172
|
+
);
|
|
173
|
+
expect(result).toContain(
|
|
174
|
+
'window.__VELLUM_CONFIG__={"mode":"remote-gateway","apiBaseUrl":"/v1","disablePlatform":true}',
|
|
175
|
+
);
|
|
176
|
+
expect(result.indexOf('"webUrl"')).toBeLessThan(
|
|
177
|
+
result.indexOf('"remote-gateway"'),
|
|
178
|
+
);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("escapes config JSON before embedding it in a script tag", () => {
|
|
182
|
+
const result = buildRemoteWebIndexHtml("</head>", {
|
|
183
|
+
value: "</script><script>alert(1)</script>",
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
expect(result).not.toContain("</script><script>alert(1)</script>");
|
|
187
|
+
expect(result).toContain("\\u003c/script\\u003e");
|
|
188
|
+
});
|
|
189
|
+
});
|
|
190
|
+
|
|
62
191
|
describe("nginx ingress process state", () => {
|
|
63
192
|
const workspaces: string[] = [];
|
|
64
193
|
|
|
@@ -74,6 +74,19 @@ function writeLockfile(entry: AssistantEntry): void {
|
|
|
74
74
|
);
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
function mockEnabledFlagFetch() {
|
|
78
|
+
const fetchMock = mock(async (_input: string, _init?: RequestInit) => {
|
|
79
|
+
return new Response(
|
|
80
|
+
JSON.stringify({
|
|
81
|
+
flags: [{ key: "web-remote-ingress", enabled: true }],
|
|
82
|
+
}),
|
|
83
|
+
{ status: 200, headers: { "Content-Type": "application/json" } },
|
|
84
|
+
);
|
|
85
|
+
});
|
|
86
|
+
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
|
|
87
|
+
return fetchMock;
|
|
88
|
+
}
|
|
89
|
+
|
|
77
90
|
describe("tunnel nginx ingress feature flag", () => {
|
|
78
91
|
beforeEach(() => {
|
|
79
92
|
process.argv = ["bun", "vellum", "tunnel"];
|
|
@@ -116,6 +129,28 @@ describe("tunnel nginx ingress feature flag", () => {
|
|
|
116
129
|
expect(runCloudflareTunnelMock).not.toHaveBeenCalled();
|
|
117
130
|
});
|
|
118
131
|
|
|
132
|
+
test("checks the nginx flag through the local gateway for ngrok", async () => {
|
|
133
|
+
const entry = makeLocalEntry();
|
|
134
|
+
entry.runtimeUrl = "https://stale-tunnel.ngrok-free.dev";
|
|
135
|
+
writeLockfile(entry);
|
|
136
|
+
process.argv = ["bun", "vellum", "tunnel", "--provider", "ngrok"];
|
|
137
|
+
const fetchMock = mockEnabledFlagFetch();
|
|
138
|
+
|
|
139
|
+
await tunnel();
|
|
140
|
+
|
|
141
|
+
const [url, init] = fetchMock.mock.calls[0];
|
|
142
|
+
expect(url).toBe(
|
|
143
|
+
"http://127.0.0.1:7830/v1/assistants/assistant-1/feature-flags",
|
|
144
|
+
);
|
|
145
|
+
expect(init?.method).toBe("GET");
|
|
146
|
+
expect(runNgrokTunnelMock).toHaveBeenCalledWith({
|
|
147
|
+
port: 7830,
|
|
148
|
+
workspaceDir: join(entry.resources!.instanceDir, ".vellum", "workspace"),
|
|
149
|
+
preferNginxIngress: true,
|
|
150
|
+
});
|
|
151
|
+
expect(runCloudflareTunnelMock).not.toHaveBeenCalled();
|
|
152
|
+
});
|
|
153
|
+
|
|
119
154
|
test("does not start cloudflared when the flag lookup fails", async () => {
|
|
120
155
|
process.argv = ["bun", "vellum", "tunnel", "--provider", "cloudflare"];
|
|
121
156
|
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
import { waitForDaemonReady } from "../lib/http-client.js";
|
|
18
18
|
import {
|
|
19
19
|
DEFAULT_NGINX_INGRESS_PORT,
|
|
20
|
+
findWebDistDir,
|
|
20
21
|
getIngressPaths,
|
|
21
22
|
getIngressPid,
|
|
22
23
|
getNginxIngressPort,
|
|
@@ -33,14 +34,12 @@ function printHelp(): void {
|
|
|
33
34
|
console.log("Usage: vellum nginx-ingress <subcommand> [<name>] [options]");
|
|
34
35
|
console.log("");
|
|
35
36
|
console.log(
|
|
36
|
-
"Manage the nginx
|
|
37
|
+
"Manage the nginx web edge that serves the SPA and fronts the gateway",
|
|
37
38
|
);
|
|
38
39
|
console.log(
|
|
39
|
-
"access: browser → tunnel (TLS) → nginx@127.0.0.1
|
|
40
|
-
);
|
|
41
|
-
console.log(
|
|
42
|
-
"nginx ingress is running, `vellum tunnel` targets it instead of the gateway.",
|
|
40
|
+
"for remote web access: browser → tunnel (TLS) → nginx@127.0.0.1.",
|
|
43
41
|
);
|
|
42
|
+
console.log("While nginx ingress is running, `vellum tunnel` targets it.");
|
|
44
43
|
console.log("");
|
|
45
44
|
console.log("Subcommands:");
|
|
46
45
|
console.log(" up Generate the nginx config and start the proxy");
|
|
@@ -148,6 +147,7 @@ async function assertWebRemoteIngressEnabled(
|
|
|
148
147
|
enabled = await isAssistantFeatureFlagEnabled(
|
|
149
148
|
target.assistantId,
|
|
150
149
|
WEB_REMOTE_INGRESS_FLAG,
|
|
150
|
+
{ runtimeUrl: `http://127.0.0.1:${target.gatewayPort}` },
|
|
151
151
|
);
|
|
152
152
|
} catch (err) {
|
|
153
153
|
throw new Error(
|
|
@@ -188,15 +188,31 @@ async function up(target: NginxIngressTarget): Promise<void> {
|
|
|
188
188
|
return;
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
+
const webDistDir = findWebDistDir();
|
|
192
|
+
if (!webDistDir) {
|
|
193
|
+
console.error(
|
|
194
|
+
"Error: unable to locate built web assets for remote web ingress.",
|
|
195
|
+
);
|
|
196
|
+
console.error("");
|
|
197
|
+
console.error("Build the SPA first:");
|
|
198
|
+
console.error(" cd apps/web && VITE_PLATFORM_MODE=false bun run build");
|
|
199
|
+
console.error("");
|
|
200
|
+
console.error(
|
|
201
|
+
"Or install @vellumai/web so its packaged dist directory is available.",
|
|
202
|
+
);
|
|
203
|
+
process.exit(1);
|
|
204
|
+
}
|
|
205
|
+
|
|
191
206
|
console.log(`Using ${version}`);
|
|
192
207
|
console.log(
|
|
193
|
-
`Starting nginx ingress on 127.0.0.1:${listenPort} → gateway 127.0.0.1:${gatewayPort}...`,
|
|
208
|
+
`Starting nginx ingress on 127.0.0.1:${listenPort} → web ${webDistDir} + gateway 127.0.0.1:${gatewayPort}...`,
|
|
194
209
|
);
|
|
195
210
|
|
|
196
211
|
const child = startIngressNginx({
|
|
197
212
|
workspaceDir,
|
|
198
213
|
gatewayPort,
|
|
199
214
|
listenPort,
|
|
215
|
+
remoteWebIngress: { webDistDir },
|
|
200
216
|
});
|
|
201
217
|
child.unref();
|
|
202
218
|
|
package/src/commands/tunnel.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { join } from "path";
|
|
2
2
|
|
|
3
|
-
import { resolveAssistant } from "../lib/assistant-config";
|
|
3
|
+
import { resolveAssistant, type AssistantEntry } from "../lib/assistant-config";
|
|
4
4
|
import { runCloudflareTunnel } from "../lib/cloudflare-tunnel.js";
|
|
5
|
+
import { GATEWAY_PORT } from "../lib/constants.js";
|
|
5
6
|
import {
|
|
6
7
|
isAssistantFeatureFlagEnabled,
|
|
7
8
|
WEB_REMOTE_INGRESS_FLAG,
|
|
@@ -92,11 +93,36 @@ function parseArgs(): TunnelArgs {
|
|
|
92
93
|
return { assistantName, provider };
|
|
93
94
|
}
|
|
94
95
|
|
|
95
|
-
|
|
96
|
+
function parsePortFromUrl(url: unknown): number | undefined {
|
|
97
|
+
if (typeof url !== "string" || !url.trim()) return undefined;
|
|
98
|
+
try {
|
|
99
|
+
const port = Number(new URL(url).port);
|
|
100
|
+
return Number.isInteger(port) && port > 0 && port <= 65535
|
|
101
|
+
? port
|
|
102
|
+
: undefined;
|
|
103
|
+
} catch {
|
|
104
|
+
return undefined;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function resolveEntryGatewayPort(entry: AssistantEntry): number {
|
|
109
|
+
return (
|
|
110
|
+
entry.resources?.gatewayPort ??
|
|
111
|
+
parsePortFromUrl(entry.localUrl) ??
|
|
112
|
+
parsePortFromUrl(entry.runtimeUrl) ??
|
|
113
|
+
GATEWAY_PORT
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function shouldPreferNginxIngress(
|
|
118
|
+
assistantId: string,
|
|
119
|
+
gatewayPort: number,
|
|
120
|
+
): Promise<boolean> {
|
|
96
121
|
try {
|
|
97
122
|
return await isAssistantFeatureFlagEnabled(
|
|
98
123
|
assistantId,
|
|
99
124
|
WEB_REMOTE_INGRESS_FLAG,
|
|
125
|
+
{ runtimeUrl: `http://127.0.0.1:${gatewayPort}` },
|
|
100
126
|
);
|
|
101
127
|
} catch (err) {
|
|
102
128
|
throw new Error(
|
|
@@ -124,17 +150,21 @@ export async function tunnel(): Promise<void> {
|
|
|
124
150
|
}
|
|
125
151
|
|
|
126
152
|
const resources = entry.resources;
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
153
|
+
const gatewayPort = resolveEntryGatewayPort(entry);
|
|
154
|
+
const baseTunnelOpts = {
|
|
155
|
+
port: gatewayPort,
|
|
156
|
+
...(resources
|
|
157
|
+
? { workspaceDir: join(resources.instanceDir, ".vellum", "workspace") }
|
|
158
|
+
: {}),
|
|
159
|
+
};
|
|
133
160
|
|
|
134
161
|
if (provider === "ngrok") {
|
|
135
162
|
await runNgrokTunnel({
|
|
136
163
|
...baseTunnelOpts,
|
|
137
|
-
preferNginxIngress: await shouldPreferNginxIngress(
|
|
164
|
+
preferNginxIngress: await shouldPreferNginxIngress(
|
|
165
|
+
entry.assistantId,
|
|
166
|
+
gatewayPort,
|
|
167
|
+
),
|
|
138
168
|
});
|
|
139
169
|
return;
|
|
140
170
|
}
|
|
@@ -142,7 +172,10 @@ export async function tunnel(): Promise<void> {
|
|
|
142
172
|
if (provider === "cloudflare") {
|
|
143
173
|
await runCloudflareTunnel({
|
|
144
174
|
...baseTunnelOpts,
|
|
145
|
-
preferNginxIngress: await shouldPreferNginxIngress(
|
|
175
|
+
preferNginxIngress: await shouldPreferNginxIngress(
|
|
176
|
+
entry.assistantId,
|
|
177
|
+
gatewayPort,
|
|
178
|
+
),
|
|
146
179
|
});
|
|
147
180
|
return;
|
|
148
181
|
}
|
|
@@ -26,6 +26,7 @@ const FALLBACK_RUNTIME_URL = `http://127.0.0.1:${GATEWAY_PORT}`;
|
|
|
26
26
|
|
|
27
27
|
export interface AssistantClientOpts {
|
|
28
28
|
assistantId?: string;
|
|
29
|
+
runtimeUrl?: string;
|
|
29
30
|
/**
|
|
30
31
|
* When provided alongside `orgId`, the client authenticates with a
|
|
31
32
|
* session token instead of a guardian token. The session token is
|
|
@@ -73,6 +74,7 @@ export class AssistantClient {
|
|
|
73
74
|
}
|
|
74
75
|
|
|
75
76
|
this.runtimeUrl = (
|
|
77
|
+
opts?.runtimeUrl ||
|
|
76
78
|
entry.localUrl ||
|
|
77
79
|
entry.runtimeUrl ||
|
|
78
80
|
FALLBACK_RUNTIME_URL
|
|
@@ -52,6 +52,16 @@ function mockFetch(response: Response): void {
|
|
|
52
52
|
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
function mockFetchWithUrls(response: Response): string[] {
|
|
56
|
+
const urls: string[] = [];
|
|
57
|
+
const fetchMock = async (input: RequestInfo | URL) => {
|
|
58
|
+
urls.push(String(input));
|
|
59
|
+
return response;
|
|
60
|
+
};
|
|
61
|
+
globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch;
|
|
62
|
+
return urls;
|
|
63
|
+
}
|
|
64
|
+
|
|
55
65
|
describe("isAssistantFeatureFlagEnabled", () => {
|
|
56
66
|
beforeEach(() => {
|
|
57
67
|
process.env.VELLUM_LOCKFILE_DIR = testDir;
|
|
@@ -84,6 +94,41 @@ describe("isAssistantFeatureFlagEnabled", () => {
|
|
|
84
94
|
).resolves.toBe(true);
|
|
85
95
|
});
|
|
86
96
|
|
|
97
|
+
test("uses the supplied runtime URL instead of a stale lockfile runtimeUrl", async () => {
|
|
98
|
+
writeFileSync(
|
|
99
|
+
join(testDir, ".vellum.lock.json"),
|
|
100
|
+
JSON.stringify(
|
|
101
|
+
{
|
|
102
|
+
activeAssistant: "assistant-1",
|
|
103
|
+
assistants: [
|
|
104
|
+
{
|
|
105
|
+
assistantId: "assistant-1",
|
|
106
|
+
runtimeUrl: "https://stale-tunnel.ngrok-free.dev",
|
|
107
|
+
cloud: "local",
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
},
|
|
111
|
+
null,
|
|
112
|
+
2,
|
|
113
|
+
),
|
|
114
|
+
);
|
|
115
|
+
const urls = mockFetchWithUrls(
|
|
116
|
+
jsonResponse({
|
|
117
|
+
flags: [{ key: WEB_REMOTE_INGRESS_FLAG, enabled: true }],
|
|
118
|
+
}),
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
await expect(
|
|
122
|
+
isAssistantFeatureFlagEnabled("assistant-1", WEB_REMOTE_INGRESS_FLAG, {
|
|
123
|
+
runtimeUrl: "http://127.0.0.1:9123",
|
|
124
|
+
}),
|
|
125
|
+
).resolves.toBe(true);
|
|
126
|
+
|
|
127
|
+
expect(urls).toEqual([
|
|
128
|
+
"http://127.0.0.1:9123/v1/assistants/assistant-1/feature-flags",
|
|
129
|
+
]);
|
|
130
|
+
});
|
|
131
|
+
|
|
87
132
|
test("returns false when the assistant flag is disabled or missing", async () => {
|
|
88
133
|
mockFetch(
|
|
89
134
|
jsonResponse({
|
package/src/lib/feature-flags.ts
CHANGED
|
@@ -14,8 +14,12 @@ type FeatureFlagsResponse = {
|
|
|
14
14
|
export async function isAssistantFeatureFlagEnabled(
|
|
15
15
|
assistantId: string,
|
|
16
16
|
key: string,
|
|
17
|
+
opts: { runtimeUrl?: string } = {},
|
|
17
18
|
): Promise<boolean> {
|
|
18
|
-
const client = new AssistantClient({
|
|
19
|
+
const client = new AssistantClient({
|
|
20
|
+
assistantId,
|
|
21
|
+
runtimeUrl: opts.runtimeUrl,
|
|
22
|
+
});
|
|
19
23
|
const res = await client.get("/feature-flags");
|
|
20
24
|
if (!res.ok) {
|
|
21
25
|
const body = await res.text().catch(() => "");
|
package/src/lib/nginx-ingress.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
rmSync,
|
|
14
14
|
writeFileSync,
|
|
15
15
|
} from "node:fs";
|
|
16
|
+
import { createRequire } from "node:module";
|
|
16
17
|
import { dirname, join } from "node:path";
|
|
17
18
|
|
|
18
19
|
import { GATEWAY_PORT } from "./constants.js";
|
|
@@ -26,6 +27,7 @@ import { GATEWAY_PORT } from "./constants.js";
|
|
|
26
27
|
*/
|
|
27
28
|
|
|
28
29
|
export const DEFAULT_NGINX_INGRESS_PORT = 7840;
|
|
30
|
+
const _require = createRequire(import.meta.url);
|
|
29
31
|
|
|
30
32
|
/** Listen port for nginx ingress, from VELLUM_NGINX_INGRESS_PORT. */
|
|
31
33
|
export function getNginxIngressPort(): number {
|
|
@@ -78,13 +80,119 @@ function saveRawConfig(
|
|
|
78
80
|
writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");
|
|
79
81
|
}
|
|
80
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Locate the pre-built @vellumai/web dist directory.
|
|
85
|
+
*
|
|
86
|
+
* Resolution order:
|
|
87
|
+
* 1. npm-installed package — require.resolve('@vellumai/web/package.json')
|
|
88
|
+
* 2. Source checkout — walk up from cli/ to find apps/web/dist/
|
|
89
|
+
*/
|
|
90
|
+
export function findWebDistDir(): string | null {
|
|
91
|
+
try {
|
|
92
|
+
const pkgPath = _require.resolve("@vellumai/web/package.json");
|
|
93
|
+
const distDir = join(dirname(pkgPath), "dist");
|
|
94
|
+
if (existsSync(join(distDir, "index.html"))) {
|
|
95
|
+
return distDir;
|
|
96
|
+
}
|
|
97
|
+
} catch {
|
|
98
|
+
// Package not installed; try source checkout.
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
let dir = import.meta.dir;
|
|
102
|
+
for (let depth = 0; depth < 8; depth++) {
|
|
103
|
+
const candidate = join(dir, "apps", "web", "dist", "index.html");
|
|
104
|
+
if (existsSync(candidate)) {
|
|
105
|
+
return dirname(candidate);
|
|
106
|
+
}
|
|
107
|
+
const parent = dirname(dir);
|
|
108
|
+
if (parent === dir) break;
|
|
109
|
+
dir = parent;
|
|
110
|
+
}
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function nginxQuoted(value: string, label: string): string {
|
|
115
|
+
if (/[\u0000-\u001f\u007f]/.test(value)) {
|
|
116
|
+
throw new Error(`${label} contains a control character`);
|
|
117
|
+
}
|
|
118
|
+
return `"${value
|
|
119
|
+
.replace(/\\/g, "\\\\")
|
|
120
|
+
.replace(/"/g, '\\"')
|
|
121
|
+
.replace(/\$/g, "\\$")}"`;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function nginxDirPath(dir: string): string {
|
|
125
|
+
return dir.endsWith("/") ? dir : `${dir}/`;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function gatewayProxyBlock(gatewayPort: number): string {
|
|
129
|
+
return ` proxy_pass http://127.0.0.1:${gatewayPort};
|
|
130
|
+
proxy_http_version 1.1;
|
|
131
|
+
proxy_request_buffering off;
|
|
132
|
+
proxy_buffering off;
|
|
133
|
+
proxy_read_timeout 1h;
|
|
134
|
+
proxy_set_header Host $host;
|
|
135
|
+
proxy_set_header X-Vellum-Edge-Forwarded "1";
|
|
136
|
+
proxy_set_header Upgrade $http_upgrade;
|
|
137
|
+
proxy_set_header Connection $connection_upgrade;`;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export interface RemoteWebIngressOptions {
|
|
141
|
+
webDistDir: string;
|
|
142
|
+
indexHtmlPath?: string;
|
|
143
|
+
config?: Record<string, unknown>;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function remoteWebIngressConfig(
|
|
147
|
+
config: Record<string, unknown> | undefined,
|
|
148
|
+
): Record<string, unknown> {
|
|
149
|
+
return {
|
|
150
|
+
mode: "remote-gateway",
|
|
151
|
+
apiBaseUrl: "/v1",
|
|
152
|
+
platformDisabled: true,
|
|
153
|
+
disablePlatform: true,
|
|
154
|
+
...config,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function safeScriptJson(value: unknown): string {
|
|
159
|
+
return JSON.stringify(value)
|
|
160
|
+
.replace(/</g, "\\u003c")
|
|
161
|
+
.replace(/>/g, "\\u003e");
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export function buildRemoteWebIndexHtml(
|
|
165
|
+
rawHtml: string,
|
|
166
|
+
config: Record<string, unknown>,
|
|
167
|
+
): string {
|
|
168
|
+
const script = `<script>window.__VELLUM_CONFIG__=${safeScriptJson(config)}</script>`;
|
|
169
|
+
if (rawHtml.includes("</head>")) {
|
|
170
|
+
return rawHtml.replace("</head>", `${script}</head>`);
|
|
171
|
+
}
|
|
172
|
+
return `${script}${rawHtml}`;
|
|
173
|
+
}
|
|
174
|
+
|
|
81
175
|
/**
|
|
82
176
|
* Build the nginx config that forwards tunnel web traffic to the gateway.
|
|
83
177
|
*/
|
|
84
178
|
export function buildIngressNginxConfig(opts: {
|
|
85
179
|
gatewayPort: number;
|
|
86
180
|
listenPort: number;
|
|
181
|
+
remoteWebIngress?: RemoteWebIngressOptions;
|
|
87
182
|
}): string {
|
|
183
|
+
const proxyBlock = gatewayProxyBlock(opts.gatewayPort);
|
|
184
|
+
const remoteWebIngress = opts.remoteWebIngress;
|
|
185
|
+
const serverLocations = remoteWebIngress
|
|
186
|
+
? buildRemoteWebIngressLocations({
|
|
187
|
+
gatewayPort: opts.gatewayPort,
|
|
188
|
+
webDistDir: remoteWebIngress.webDistDir,
|
|
189
|
+
indexHtmlPath: remoteWebIngress.indexHtmlPath,
|
|
190
|
+
config: remoteWebIngressConfig(remoteWebIngress.config),
|
|
191
|
+
})
|
|
192
|
+
: ` location / {
|
|
193
|
+
${proxyBlock}
|
|
194
|
+
}`;
|
|
195
|
+
|
|
88
196
|
return `
|
|
89
197
|
worker_processes 1;
|
|
90
198
|
error_log stderr;
|
|
@@ -94,6 +202,24 @@ events {}
|
|
|
94
202
|
|
|
95
203
|
http {
|
|
96
204
|
access_log off;
|
|
205
|
+
default_type application/octet-stream;
|
|
206
|
+
|
|
207
|
+
types {
|
|
208
|
+
application/javascript js mjs;
|
|
209
|
+
application/json json map;
|
|
210
|
+
application/wasm wasm;
|
|
211
|
+
font/woff woff;
|
|
212
|
+
font/woff2 woff2;
|
|
213
|
+
image/gif gif;
|
|
214
|
+
image/jpeg jpeg jpg;
|
|
215
|
+
image/png png;
|
|
216
|
+
image/svg+xml svg svgz;
|
|
217
|
+
image/webp webp;
|
|
218
|
+
image/x-icon ico;
|
|
219
|
+
text/css css;
|
|
220
|
+
text/html html htm;
|
|
221
|
+
text/plain txt;
|
|
222
|
+
}
|
|
97
223
|
|
|
98
224
|
map $http_upgrade $connection_upgrade {
|
|
99
225
|
default upgrade;
|
|
@@ -104,21 +230,95 @@ http {
|
|
|
104
230
|
listen 127.0.0.1:${opts.listenPort};
|
|
105
231
|
client_max_body_size 512m;
|
|
106
232
|
|
|
107
|
-
|
|
108
|
-
proxy_pass http://127.0.0.1:${opts.gatewayPort};
|
|
109
|
-
proxy_http_version 1.1;
|
|
110
|
-
proxy_request_buffering off;
|
|
111
|
-
proxy_buffering off;
|
|
112
|
-
proxy_read_timeout 1h;
|
|
113
|
-
proxy_set_header Host $host;
|
|
114
|
-
proxy_set_header Upgrade $http_upgrade;
|
|
115
|
-
proxy_set_header Connection $connection_upgrade;
|
|
116
|
-
}
|
|
233
|
+
${serverLocations}
|
|
117
234
|
}
|
|
118
235
|
}
|
|
119
236
|
`;
|
|
120
237
|
}
|
|
121
238
|
|
|
239
|
+
function buildRemoteWebIngressLocations(opts: {
|
|
240
|
+
gatewayPort: number;
|
|
241
|
+
webDistDir: string;
|
|
242
|
+
indexHtmlPath?: string;
|
|
243
|
+
config: Record<string, unknown>;
|
|
244
|
+
}): string {
|
|
245
|
+
const proxyBlock = gatewayProxyBlock(opts.gatewayPort);
|
|
246
|
+
const webDistDir = nginxDirPath(opts.webDistDir);
|
|
247
|
+
const webAssetsDir = join(opts.webDistDir, "assets");
|
|
248
|
+
const indexHtmlPath =
|
|
249
|
+
opts.indexHtmlPath ?? join(opts.webDistDir, "index.html");
|
|
250
|
+
const configJson = JSON.stringify(opts.config);
|
|
251
|
+
|
|
252
|
+
return ` location = /auth/token { return 404; }
|
|
253
|
+
location = /auth/token/ { return 404; }
|
|
254
|
+
location = /v1/pair { return 404; }
|
|
255
|
+
location = /v1/pair/ { return 404; }
|
|
256
|
+
location = /v1/pair/web-init { return 404; }
|
|
257
|
+
location = /v1/pair/web-init/ { return 404; }
|
|
258
|
+
location = /v1/devices { return 404; }
|
|
259
|
+
location = /v1/devices/ { return 404; }
|
|
260
|
+
location = /v1/devices/revoke { return 404; }
|
|
261
|
+
location = /v1/devices/revoke/ { return 404; }
|
|
262
|
+
location = /v1/guardian/init { return 404; }
|
|
263
|
+
location = /v1/guardian/init/ { return 404; }
|
|
264
|
+
location = /v1/guardian/reset-bootstrap { return 404; }
|
|
265
|
+
location = /v1/guardian/reset-bootstrap/ { return 404; }
|
|
266
|
+
location ^~ /assistant/__local/ { return 404; }
|
|
267
|
+
location ^~ /assistant/__gateway/ { return 404; }
|
|
268
|
+
|
|
269
|
+
location = /healthz {
|
|
270
|
+
${proxyBlock}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
location ^~ /v1/ {
|
|
274
|
+
${proxyBlock}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
location = /assistant {
|
|
278
|
+
return 302 /assistant/;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
location = /assistant/ {
|
|
282
|
+
rewrite ^ /assistant/__remote-index.html last;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
location = /assistant/index.html {
|
|
286
|
+
rewrite ^ /assistant/__remote-index.html last;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
location = /assistant/__remote-index.html {
|
|
290
|
+
internal;
|
|
291
|
+
alias ${nginxQuoted(indexHtmlPath, "remote web ingress index path")};
|
|
292
|
+
add_header Cache-Control "no-store";
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
location = /assistant/__config {
|
|
296
|
+
default_type application/json;
|
|
297
|
+
add_header Cache-Control "no-store";
|
|
298
|
+
return 200 ${nginxQuoted(configJson, "remote web ingress config")};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
location ^~ /assistant/assets/ {
|
|
302
|
+
alias ${nginxQuoted(nginxDirPath(webAssetsDir), "web assets path")};
|
|
303
|
+
try_files $uri =404;
|
|
304
|
+
add_header Cache-Control "public, max-age=31536000, immutable";
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
location ^~ /assistant/ {
|
|
308
|
+
alias ${nginxQuoted(webDistDir, "web dist path")};
|
|
309
|
+
try_files $uri $uri/ /assistant/__remote-index.html;
|
|
310
|
+
add_header Cache-Control "no-store";
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
location = / {
|
|
314
|
+
return 302 /assistant/;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
location / {
|
|
318
|
+
return 404;
|
|
319
|
+
}`;
|
|
320
|
+
}
|
|
321
|
+
|
|
122
322
|
function nginxBin(): string {
|
|
123
323
|
return process.env.NGINX_BIN || "nginx";
|
|
124
324
|
}
|
|
@@ -250,15 +450,34 @@ export function startIngressNginx(opts: {
|
|
|
250
450
|
workspaceDir: string;
|
|
251
451
|
gatewayPort: number;
|
|
252
452
|
listenPort: number;
|
|
453
|
+
remoteWebIngress?: RemoteWebIngressOptions;
|
|
253
454
|
}): ChildProcess {
|
|
254
455
|
const paths = getIngressPaths(opts.workspaceDir);
|
|
255
456
|
mkdirSync(paths.dir, { recursive: true });
|
|
256
457
|
mkdirSync(join(opts.workspaceDir, "data", "logs"), { recursive: true });
|
|
458
|
+
const remoteWebIngress = opts.remoteWebIngress
|
|
459
|
+
? {
|
|
460
|
+
...opts.remoteWebIngress,
|
|
461
|
+
config: remoteWebIngressConfig(opts.remoteWebIngress.config),
|
|
462
|
+
indexHtmlPath: join(paths.dir, "assistant-index.html"),
|
|
463
|
+
}
|
|
464
|
+
: undefined;
|
|
465
|
+
if (remoteWebIngress) {
|
|
466
|
+
const rawIndexHtml = readFileSync(
|
|
467
|
+
join(remoteWebIngress.webDistDir, "index.html"),
|
|
468
|
+
"utf-8",
|
|
469
|
+
);
|
|
470
|
+
writeFileSync(
|
|
471
|
+
remoteWebIngress.indexHtmlPath,
|
|
472
|
+
buildRemoteWebIndexHtml(rawIndexHtml, remoteWebIngress.config),
|
|
473
|
+
);
|
|
474
|
+
}
|
|
257
475
|
writeFileSync(
|
|
258
476
|
paths.confPath,
|
|
259
477
|
buildIngressNginxConfig({
|
|
260
478
|
gatewayPort: opts.gatewayPort,
|
|
261
479
|
listenPort: opts.listenPort,
|
|
480
|
+
remoteWebIngress,
|
|
262
481
|
}),
|
|
263
482
|
);
|
|
264
483
|
|