@vellumai/cli 0.12.1 → 0.12.2-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@vellumai/local-mode/package.json +2 -1
- package/node_modules/@vellumai/local-mode/src/teleport-backup-policy.test.ts +88 -0
- package/node_modules/@vellumai/local-mode/src/teleport-backup-policy.ts +74 -0
- package/package.json +1 -1
- package/src/__tests__/backup-ops-matching.test.ts +48 -0
- package/src/__tests__/teleport-backup.test.ts +208 -0
- package/src/__tests__/teleport.test.ts +297 -0
- package/src/commands/teleport.ts +95 -0
- package/src/lib/backup-ops.ts +75 -8
- package/src/lib/teleport-backup.ts +139 -0
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
"type": "module",
|
|
7
7
|
"exports": {
|
|
8
8
|
".": "./src/index.ts",
|
|
9
|
-
"./contract": "./src/lockfile-contract.ts"
|
|
9
|
+
"./contract": "./src/lockfile-contract.ts",
|
|
10
|
+
"./teleport-backup-policy": "./src/teleport-backup-policy.ts"
|
|
10
11
|
},
|
|
11
12
|
"scripts": {
|
|
12
13
|
"typecheck": "bunx tsc --noEmit",
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
hasRecentBackup,
|
|
5
|
+
managedBackupIsReady,
|
|
6
|
+
readyManagedBackupCreatedAts,
|
|
7
|
+
RECENT_BACKUP_MAX_AGE_MS,
|
|
8
|
+
} from "./teleport-backup-policy";
|
|
9
|
+
|
|
10
|
+
const NOW = Date.parse("2026-09-15T12:00:00Z");
|
|
11
|
+
|
|
12
|
+
function ago(ms: number): string {
|
|
13
|
+
return new Date(NOW - ms).toISOString();
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
describe("hasRecentBackup", () => {
|
|
17
|
+
test("true when a backup is within the max age", () => {
|
|
18
|
+
expect(hasRecentBackup([ago(5 * 60_000)], NOW)).toBe(true);
|
|
19
|
+
expect(hasRecentBackup([ago(RECENT_BACKUP_MAX_AGE_MS)], NOW)).toBe(true);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("false when every backup is older than the max age", () => {
|
|
23
|
+
expect(hasRecentBackup([ago(RECENT_BACKUP_MAX_AGE_MS + 1)], NOW)).toBe(
|
|
24
|
+
false,
|
|
25
|
+
);
|
|
26
|
+
expect(hasRecentBackup([ago(2 * 24 * 60 * 60_000)], NOW)).toBe(false);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("false with no backups", () => {
|
|
30
|
+
expect(hasRecentBackup([], NOW)).toBe(false);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("ignores empty, missing, unparseable and future timestamps", () => {
|
|
34
|
+
expect(
|
|
35
|
+
hasRecentBackup(["", undefined, null, "garbage", ago(-60_000)], NOW),
|
|
36
|
+
).toBe(false);
|
|
37
|
+
expect(hasRecentBackup(["", "garbage", ago(60_000)], NOW)).toBe(true);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("finds a recent backup anywhere in the list", () => {
|
|
41
|
+
expect(
|
|
42
|
+
hasRecentBackup(
|
|
43
|
+
[ago(3 * RECENT_BACKUP_MAX_AGE_MS), ago(30 * 60_000)],
|
|
44
|
+
NOW,
|
|
45
|
+
),
|
|
46
|
+
).toBe(true);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("honors a custom max age", () => {
|
|
50
|
+
expect(hasRecentBackup([ago(10 * 60_000)], NOW, 5 * 60_000)).toBe(false);
|
|
51
|
+
expect(hasRecentBackup([ago(10 * 60_000)], NOW, 15 * 60_000)).toBe(true);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
describe("readyManagedBackupCreatedAts", () => {
|
|
56
|
+
test("keeps ready snapshots and drops pending ones", () => {
|
|
57
|
+
expect(
|
|
58
|
+
readyManagedBackupCreatedAts([
|
|
59
|
+
{
|
|
60
|
+
snapshot_name: "a",
|
|
61
|
+
created_at: "2026-09-15T11:00:00Z",
|
|
62
|
+
ready_to_use: true,
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
snapshot_name: "b",
|
|
66
|
+
created_at: "2026-09-15T11:30:00Z",
|
|
67
|
+
ready_to_use: false,
|
|
68
|
+
},
|
|
69
|
+
{ snapshot_name: "c", created_at: "2026-09-15T10:00:00Z" },
|
|
70
|
+
]),
|
|
71
|
+
).toEqual(["2026-09-15T11:00:00Z", "2026-09-15T10:00:00Z"]);
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe("managedBackupIsReady", () => {
|
|
76
|
+
const backups = [
|
|
77
|
+
{ snapshot_name: "ready", ready_to_use: true },
|
|
78
|
+
{ snapshot_name: "pending", ready_to_use: false },
|
|
79
|
+
{ snapshot_name: "unknown" },
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
test("true only for a snapshot explicitly marked ready", () => {
|
|
83
|
+
expect(managedBackupIsReady(backups, "ready")).toBe(true);
|
|
84
|
+
expect(managedBackupIsReady(backups, "pending")).toBe(false);
|
|
85
|
+
expect(managedBackupIsReady(backups, "unknown")).toBe(false);
|
|
86
|
+
expect(managedBackupIsReady(backups, "missing")).toBe(false);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pre-teleport backup policy shared by every teleport surface (CLI and web).
|
|
3
|
+
*
|
|
4
|
+
* A teleport ends with the source assistant being retired, so each surface
|
|
5
|
+
* snapshots the source before any data leaves it. This module holds the
|
|
6
|
+
* decisions those surfaces must agree on: how fresh an existing backup has to
|
|
7
|
+
* be to stand in for a new one, and how a platform (PVC) snapshot is judged
|
|
8
|
+
* restorable. Transport lives with each client; nothing here touches the
|
|
9
|
+
* network or the filesystem.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/** A backup at most this old satisfies the pre-teleport backup requirement. */
|
|
13
|
+
export const RECENT_BACKUP_MAX_AGE_MS = 60 * 60 * 1000;
|
|
14
|
+
|
|
15
|
+
/** How long to wait for a freshly cut platform snapshot to become restorable. */
|
|
16
|
+
export const MANAGED_BACKUP_READY_TIMEOUT_MS = 10 * 60 * 1000;
|
|
17
|
+
|
|
18
|
+
/** How often to re-list platform snapshots while waiting for readiness. */
|
|
19
|
+
export const MANAGED_BACKUP_READY_POLL_INTERVAL_MS = 5_000;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Whether any of `createdAts` (ISO-8601 timestamps; empty, missing and
|
|
23
|
+
* unparseable values are ignored) falls within `maxAgeMs` of `now`.
|
|
24
|
+
*/
|
|
25
|
+
export function hasRecentBackup(
|
|
26
|
+
createdAts: Iterable<string | null | undefined>,
|
|
27
|
+
now: number = Date.now(),
|
|
28
|
+
maxAgeMs: number = RECENT_BACKUP_MAX_AGE_MS,
|
|
29
|
+
): boolean {
|
|
30
|
+
for (const createdAt of createdAts) {
|
|
31
|
+
if (!createdAt) {
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
const createdMs = Date.parse(createdAt);
|
|
35
|
+
if (Number.isNaN(createdMs)) {
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const age = now - createdMs;
|
|
39
|
+
if (age >= 0 && age <= maxAgeMs) {
|
|
40
|
+
return true;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The fields of a platform backup listing entry the policy reads. */
|
|
47
|
+
export interface ManagedBackupEntry {
|
|
48
|
+
snapshot_name?: string;
|
|
49
|
+
created_at?: string;
|
|
50
|
+
ready_to_use?: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* `created_at` of every restorable platform snapshot. A snapshot still being
|
|
55
|
+
* cut (`ready_to_use: false`) never counts as a usable restore point.
|
|
56
|
+
*/
|
|
57
|
+
export function readyManagedBackupCreatedAts(
|
|
58
|
+
backups: readonly ManagedBackupEntry[],
|
|
59
|
+
): Array<string | undefined> {
|
|
60
|
+
return backups
|
|
61
|
+
.filter((backup) => backup.ready_to_use !== false)
|
|
62
|
+
.map((backup) => backup.created_at);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Whether the listing shows `snapshotName` as restorable. */
|
|
66
|
+
export function managedBackupIsReady(
|
|
67
|
+
backups: readonly ManagedBackupEntry[],
|
|
68
|
+
snapshotName: string,
|
|
69
|
+
): boolean {
|
|
70
|
+
return backups.some(
|
|
71
|
+
(backup) =>
|
|
72
|
+
backup.snapshot_name === snapshotName && backup.ready_to_use === true,
|
|
73
|
+
);
|
|
74
|
+
}
|
package/package.json
CHANGED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { assistantBackupFilenamePattern } from "../lib/backup-ops.js";
|
|
4
|
+
|
|
5
|
+
const STAMP = "2026-09-15T14-27-28-123Z";
|
|
6
|
+
|
|
7
|
+
describe("assistantBackupFilenamePattern", () => {
|
|
8
|
+
test("matches this CLI's backups for exactly the given assistant", () => {
|
|
9
|
+
const pattern = assistantBackupFilenamePattern("alpha");
|
|
10
|
+
expect(pattern.test(`alpha-pre-upgrade-${STAMP}.vbundle`)).toBe(true);
|
|
11
|
+
expect(pattern.test(`alpha-pre-teleport-${STAMP}.vbundle`)).toBe(true);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test("does not match an assistant whose id extends this one", () => {
|
|
15
|
+
const pattern = assistantBackupFilenamePattern("alpha");
|
|
16
|
+
expect(pattern.test(`alpha-prod-pre-upgrade-${STAMP}.vbundle`)).toBe(false);
|
|
17
|
+
expect(pattern.test(`alpha-prod-pre-teleport-${STAMP}.vbundle`)).toBe(
|
|
18
|
+
false,
|
|
19
|
+
);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("does not match a shorter id's backups from a longer id", () => {
|
|
23
|
+
const pattern = assistantBackupFilenamePattern("alpha-prod");
|
|
24
|
+
expect(pattern.test(`alpha-pre-upgrade-${STAMP}.vbundle`)).toBe(false);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("ignores user-named vellum backup files and other kinds", () => {
|
|
28
|
+
const pattern = assistantBackupFilenamePattern("alpha");
|
|
29
|
+
expect(pattern.test(`alpha-${STAMP}.vbundle`)).toBe(false);
|
|
30
|
+
expect(pattern.test(`alpha-manual-${STAMP}.vbundle`)).toBe(false);
|
|
31
|
+
expect(pattern.test(`alpha-pre-upgrade-${STAMP}.vbundle.enc`)).toBe(false);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("restricts to the requested kinds", () => {
|
|
35
|
+
const pattern = assistantBackupFilenamePattern("alpha", ["pre-teleport"]);
|
|
36
|
+
expect(pattern.test(`alpha-pre-teleport-${STAMP}.vbundle`)).toBe(true);
|
|
37
|
+
expect(pattern.test(`alpha-pre-upgrade-${STAMP}.vbundle`)).toBe(false);
|
|
38
|
+
expect(
|
|
39
|
+
pattern.test(`alpha-pre-teleport-prod-pre-teleport-${STAMP}.vbundle`),
|
|
40
|
+
).toBe(false);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("escapes regex metacharacters in ids", () => {
|
|
44
|
+
const pattern = assistantBackupFilenamePattern("a.b");
|
|
45
|
+
expect(pattern.test(`a.b-pre-upgrade-${STAMP}.vbundle`)).toBe(true);
|
|
46
|
+
expect(pattern.test(`axb-pre-upgrade-${STAMP}.vbundle`)).toBe(false);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import {
|
|
2
|
+
afterAll,
|
|
3
|
+
afterEach,
|
|
4
|
+
beforeEach,
|
|
5
|
+
describe,
|
|
6
|
+
expect,
|
|
7
|
+
mock,
|
|
8
|
+
spyOn,
|
|
9
|
+
test,
|
|
10
|
+
} from "bun:test";
|
|
11
|
+
|
|
12
|
+
import * as platformClient from "../lib/platform-client.js";
|
|
13
|
+
import {
|
|
14
|
+
createPlatformBackup,
|
|
15
|
+
listPlatformBackups,
|
|
16
|
+
} from "../lib/teleport-backup.js";
|
|
17
|
+
|
|
18
|
+
const authHeadersSpy = spyOn(platformClient, "authHeaders").mockResolvedValue({
|
|
19
|
+
"Content-Type": "application/json",
|
|
20
|
+
"X-Session-Token": "platform-token",
|
|
21
|
+
"Vellum-Organization-Id": "org-1",
|
|
22
|
+
});
|
|
23
|
+
const invalidateOrgIdCacheSpy = spyOn(
|
|
24
|
+
platformClient,
|
|
25
|
+
"invalidateOrgIdCache",
|
|
26
|
+
).mockImplementation(() => {});
|
|
27
|
+
|
|
28
|
+
afterAll(() => {
|
|
29
|
+
authHeadersSpy.mockRestore();
|
|
30
|
+
invalidateOrgIdCacheSpy.mockRestore();
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
let originalFetch: typeof globalThis.fetch;
|
|
34
|
+
let fetchCalls: Array<{ url: string; method: string }>;
|
|
35
|
+
let responses: Array<() => Response>;
|
|
36
|
+
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
originalFetch = globalThis.fetch;
|
|
39
|
+
fetchCalls = [];
|
|
40
|
+
responses = [];
|
|
41
|
+
globalThis.fetch = mock(
|
|
42
|
+
async (url: string | URL | Request, init?: RequestInit) => {
|
|
43
|
+
fetchCalls.push({
|
|
44
|
+
url: typeof url === "string" ? url : url.toString(),
|
|
45
|
+
method: init?.method ?? "GET",
|
|
46
|
+
});
|
|
47
|
+
const next = responses.shift();
|
|
48
|
+
if (!next) {
|
|
49
|
+
throw new Error("unexpected fetch");
|
|
50
|
+
}
|
|
51
|
+
return next();
|
|
52
|
+
},
|
|
53
|
+
) as unknown as typeof globalThis.fetch;
|
|
54
|
+
authHeadersSpy.mockClear();
|
|
55
|
+
invalidateOrgIdCacheSpy.mockClear();
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
afterEach(() => {
|
|
59
|
+
globalThis.fetch = originalFetch;
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const ENTRY = {
|
|
63
|
+
runtimeUrl: "https://platform.vellum.ai",
|
|
64
|
+
assistantId: "11111111-2222-3333-4444-555555555555",
|
|
65
|
+
};
|
|
66
|
+
const BACKUPS_URL = `https://platform.vellum.ai/v1/assistants/${ENTRY.assistantId}/backups/`;
|
|
67
|
+
const FAST_POLL = { pollIntervalMs: 1, timeoutMs: 200 };
|
|
68
|
+
|
|
69
|
+
function json(body: unknown, status = 200): () => Response {
|
|
70
|
+
return () => Response.json(body, { status });
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
describe("listPlatformBackups", () => {
|
|
74
|
+
test("returns created_at of ready snapshots only", async () => {
|
|
75
|
+
responses.push(
|
|
76
|
+
json({
|
|
77
|
+
backups: [
|
|
78
|
+
{ created_at: "2026-09-15T11:00:00Z", ready_to_use: true },
|
|
79
|
+
{ created_at: "2026-09-15T11:30:00Z", ready_to_use: false },
|
|
80
|
+
{ created_at: "2026-09-15T10:00:00Z" },
|
|
81
|
+
],
|
|
82
|
+
}),
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
const result = await listPlatformBackups(ENTRY, "platform-token");
|
|
86
|
+
|
|
87
|
+
expect(result).toEqual(["2026-09-15T11:00:00Z", "2026-09-15T10:00:00Z"]);
|
|
88
|
+
expect(fetchCalls[0]).toEqual({ url: BACKUPS_URL, method: "GET" });
|
|
89
|
+
expect(authHeadersSpy).toHaveBeenCalledWith(
|
|
90
|
+
"platform-token",
|
|
91
|
+
"https://platform.vellum.ai",
|
|
92
|
+
);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("refreshes the org id cache and retries once on 401", async () => {
|
|
96
|
+
responses.push(() => new Response("unauthorized", { status: 401 }));
|
|
97
|
+
responses.push(json({ backups: [] }));
|
|
98
|
+
|
|
99
|
+
const result = await listPlatformBackups(ENTRY, "platform-token");
|
|
100
|
+
|
|
101
|
+
expect(result).toEqual([]);
|
|
102
|
+
expect(fetchCalls).toHaveLength(2);
|
|
103
|
+
expect(invalidateOrgIdCacheSpy).toHaveBeenCalledWith(
|
|
104
|
+
"platform-token",
|
|
105
|
+
"https://platform.vellum.ai",
|
|
106
|
+
);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("throws on a persistent error", async () => {
|
|
110
|
+
responses.push(() => new Response("Bad gateway", { status: 502 }));
|
|
111
|
+
|
|
112
|
+
await expect(listPlatformBackups(ENTRY, "platform-token")).rejects.toThrow(
|
|
113
|
+
"Platform backup list failed (502): Bad gateway",
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe("createPlatformBackup", () => {
|
|
119
|
+
test("returns immediately when the POST reports the snapshot ready", async () => {
|
|
120
|
+
responses.push(json({ snapshot_name: "snap-1", ready_to_use: true }, 201));
|
|
121
|
+
|
|
122
|
+
await createPlatformBackup(ENTRY, "platform-token", FAST_POLL);
|
|
123
|
+
|
|
124
|
+
expect(fetchCalls).toEqual([{ url: BACKUPS_URL, method: "POST" }]);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("polls the listing until the new snapshot is ready", async () => {
|
|
128
|
+
responses.push(json({ snapshot_name: "snap-1", ready_to_use: false }, 201));
|
|
129
|
+
responses.push(
|
|
130
|
+
json({ backups: [{ snapshot_name: "snap-1", ready_to_use: false }] }),
|
|
131
|
+
);
|
|
132
|
+
responses.push(
|
|
133
|
+
json({
|
|
134
|
+
backups: [
|
|
135
|
+
{ snapshot_name: "older", ready_to_use: true },
|
|
136
|
+
{ snapshot_name: "snap-1", ready_to_use: true },
|
|
137
|
+
],
|
|
138
|
+
}),
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
await createPlatformBackup(ENTRY, "platform-token", FAST_POLL);
|
|
142
|
+
|
|
143
|
+
expect(fetchCalls.map((call) => call.method)).toEqual([
|
|
144
|
+
"POST",
|
|
145
|
+
"GET",
|
|
146
|
+
"GET",
|
|
147
|
+
]);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("another ready snapshot does not satisfy the wait", async () => {
|
|
151
|
+
responses.push(json({ snapshot_name: "snap-1" }, 201));
|
|
152
|
+
responses.push(
|
|
153
|
+
json({ backups: [{ snapshot_name: "older", ready_to_use: true }] }),
|
|
154
|
+
);
|
|
155
|
+
responses.push(
|
|
156
|
+
json({ backups: [{ snapshot_name: "snap-1", ready_to_use: true }] }),
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
await createPlatformBackup(ENTRY, "platform-token", FAST_POLL);
|
|
160
|
+
|
|
161
|
+
expect(fetchCalls).toHaveLength(3);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("throws when the snapshot is not ready before the timeout", async () => {
|
|
165
|
+
responses.push(json({ snapshot_name: "snap-1", ready_to_use: false }, 201));
|
|
166
|
+
const pending = json({
|
|
167
|
+
backups: [{ snapshot_name: "snap-1", ready_to_use: false }],
|
|
168
|
+
});
|
|
169
|
+
for (let i = 0; i < 1000; i++) {
|
|
170
|
+
responses.push(pending);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
await expect(
|
|
174
|
+
createPlatformBackup(ENTRY, "platform-token", {
|
|
175
|
+
pollIntervalMs: 1,
|
|
176
|
+
timeoutMs: 30,
|
|
177
|
+
}),
|
|
178
|
+
).rejects.toThrow("Platform backup snap-1 was not ready after 0s");
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("throws when the POST fails", async () => {
|
|
182
|
+
responses.push(
|
|
183
|
+
() => new Response('{"detail":"Bad gateway"}', { status: 502 }),
|
|
184
|
+
);
|
|
185
|
+
|
|
186
|
+
await expect(
|
|
187
|
+
createPlatformBackup(ENTRY, "platform-token", FAST_POLL),
|
|
188
|
+
).rejects.toThrow("Platform backup create failed (502)");
|
|
189
|
+
expect(fetchCalls).toHaveLength(1);
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test("throws when the POST returns no snapshot name", async () => {
|
|
193
|
+
responses.push(json({}, 201));
|
|
194
|
+
|
|
195
|
+
await expect(
|
|
196
|
+
createPlatformBackup(ENTRY, "platform-token", FAST_POLL),
|
|
197
|
+
).rejects.toThrow("no snapshot name");
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("a listing failure while waiting is fatal", async () => {
|
|
201
|
+
responses.push(json({ snapshot_name: "snap-1" }, 201));
|
|
202
|
+
responses.push(() => new Response("Bad gateway", { status: 502 }));
|
|
203
|
+
|
|
204
|
+
await expect(
|
|
205
|
+
createPlatformBackup(ENTRY, "platform-token", FAST_POLL),
|
|
206
|
+
).rejects.toThrow("Platform backup list failed (502)");
|
|
207
|
+
});
|
|
208
|
+
});
|
|
@@ -39,6 +39,8 @@ import * as assistantConfig from "../lib/assistant-config.js";
|
|
|
39
39
|
import * as guardianToken from "../lib/guardian-token.js";
|
|
40
40
|
import * as platformClient from "../lib/platform-client.js";
|
|
41
41
|
import * as localRuntimeClient from "../lib/local-runtime-client.js";
|
|
42
|
+
import * as backupOps from "../lib/backup-ops.js";
|
|
43
|
+
import * as teleportBackup from "../lib/teleport-backup.js";
|
|
42
44
|
|
|
43
45
|
// Snapshot the real exports before any `mock.module()` call so we can
|
|
44
46
|
// reliably restore them after this file's tests complete, regardless of
|
|
@@ -47,6 +49,8 @@ const realAssistantConfig = { ...assistantConfig };
|
|
|
47
49
|
const realGuardianToken = { ...guardianToken };
|
|
48
50
|
const realPlatformClient = { ...platformClient };
|
|
49
51
|
const realLocalRuntimeClient = { ...localRuntimeClient };
|
|
52
|
+
const realBackupOps = { ...backupOps };
|
|
53
|
+
const realTeleportBackup = { ...teleportBackup };
|
|
50
54
|
|
|
51
55
|
const findAssistantByNameMock = mock<
|
|
52
56
|
typeof assistantConfig.findAssistantByName
|
|
@@ -282,6 +286,36 @@ mock.module("../lib/local-runtime-client.js", () => ({
|
|
|
282
286
|
localRuntimePreflightFromGcs: localRuntimePreflightFromGcsMock,
|
|
283
287
|
}));
|
|
284
288
|
|
|
289
|
+
// Pre-export source backup. Default to "no backups yet" so the create path
|
|
290
|
+
// runs; individual tests override to exercise reuse and failure.
|
|
291
|
+
const listAssistantBackupTimesMock = mock<
|
|
292
|
+
typeof backupOps.listAssistantBackupTimes
|
|
293
|
+
>(() => []);
|
|
294
|
+
const createBackupMock = mock<typeof backupOps.createBackup>(
|
|
295
|
+
async () => "/tmp/backups/my-local-pre-teleport-2026.vbundle",
|
|
296
|
+
);
|
|
297
|
+
const pruneOldBackupsMock = mock<typeof backupOps.pruneOldBackups>(() => {});
|
|
298
|
+
|
|
299
|
+
mock.module("../lib/backup-ops.js", () => ({
|
|
300
|
+
...realBackupOps,
|
|
301
|
+
listAssistantBackupTimes: listAssistantBackupTimesMock,
|
|
302
|
+
createBackup: createBackupMock,
|
|
303
|
+
pruneOldBackups: pruneOldBackupsMock,
|
|
304
|
+
}));
|
|
305
|
+
|
|
306
|
+
const listPlatformBackupsMock = mock<typeof teleportBackup.listPlatformBackups>(
|
|
307
|
+
async () => [],
|
|
308
|
+
);
|
|
309
|
+
const createPlatformBackupMock = mock<
|
|
310
|
+
typeof teleportBackup.createPlatformBackup
|
|
311
|
+
>(async () => {});
|
|
312
|
+
|
|
313
|
+
mock.module("../lib/teleport-backup.js", () => ({
|
|
314
|
+
...realTeleportBackup,
|
|
315
|
+
listPlatformBackups: listPlatformBackupsMock,
|
|
316
|
+
createPlatformBackup: createPlatformBackupMock,
|
|
317
|
+
}));
|
|
318
|
+
|
|
285
319
|
// Snapshot the remaining real modules before mocking so `afterAll` can
|
|
286
320
|
// restore them too — otherwise these mocks leak into sibling test files that
|
|
287
321
|
// import the same modules in the same `bun test` run.
|
|
@@ -362,6 +396,8 @@ afterAll(() => {
|
|
|
362
396
|
mock.module("../lib/guardian-token.js", () => realGuardianToken);
|
|
363
397
|
mock.module("../lib/platform-client.js", () => realPlatformClient);
|
|
364
398
|
mock.module("../lib/local-runtime-client.js", () => realLocalRuntimeClient);
|
|
399
|
+
mock.module("../lib/backup-ops.js", () => realBackupOps);
|
|
400
|
+
mock.module("../lib/teleport-backup.js", () => realTeleportBackup);
|
|
365
401
|
mock.module("../lib/hatch-local.js", () => realHatchLocal);
|
|
366
402
|
mock.module("../lib/docker.js", () => realDocker);
|
|
367
403
|
mock.module("../lib/process.js", () => realProcess);
|
|
@@ -532,6 +568,19 @@ beforeEach(() => {
|
|
|
532
568
|
localRuntimePollJobStatusMock.mockReset();
|
|
533
569
|
localRuntimePollJobStatusMock.mockImplementation(defaultLocalRuntimePollImpl);
|
|
534
570
|
|
|
571
|
+
listAssistantBackupTimesMock.mockReset();
|
|
572
|
+
listAssistantBackupTimesMock.mockReturnValue([]);
|
|
573
|
+
createBackupMock.mockReset();
|
|
574
|
+
createBackupMock.mockResolvedValue(
|
|
575
|
+
"/tmp/backups/my-local-pre-teleport-2026.vbundle",
|
|
576
|
+
);
|
|
577
|
+
pruneOldBackupsMock.mockReset();
|
|
578
|
+
pruneOldBackupsMock.mockImplementation(() => {});
|
|
579
|
+
listPlatformBackupsMock.mockReset();
|
|
580
|
+
listPlatformBackupsMock.mockResolvedValue([]);
|
|
581
|
+
createPlatformBackupMock.mockReset();
|
|
582
|
+
createPlatformBackupMock.mockResolvedValue(undefined);
|
|
583
|
+
|
|
535
584
|
hatchLocalMock.mockReset();
|
|
536
585
|
hatchLocalMock.mockResolvedValue(undefined);
|
|
537
586
|
hatchDockerMock.mockReset();
|
|
@@ -2019,6 +2068,254 @@ describe("dry-run", () => {
|
|
|
2019
2068
|
// Pre-check: block teleport to platform when existing assistant detected
|
|
2020
2069
|
// ---------------------------------------------------------------------------
|
|
2021
2070
|
|
|
2071
|
+
describe("pre-export source backup", () => {
|
|
2072
|
+
test("local source with no recent backup: exports a host-side backup before exporting", async () => {
|
|
2073
|
+
setArgv("--from", "my-local", "--platform");
|
|
2074
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2075
|
+
findAssistantByNameMock.mockImplementation((name: string) =>
|
|
2076
|
+
name === "my-local" ? localEntry : null,
|
|
2077
|
+
);
|
|
2078
|
+
|
|
2079
|
+
const order: string[] = [];
|
|
2080
|
+
createBackupMock.mockImplementation(async () => {
|
|
2081
|
+
order.push("backup");
|
|
2082
|
+
return "/tmp/backups/my-local-pre-teleport-2026.vbundle";
|
|
2083
|
+
});
|
|
2084
|
+
localRuntimeExportToGcsMock.mockImplementation(async () => {
|
|
2085
|
+
order.push("export");
|
|
2086
|
+
return { jobId: "local-export-job-1" };
|
|
2087
|
+
});
|
|
2088
|
+
|
|
2089
|
+
const restoreFetch = installTrackingFetch();
|
|
2090
|
+
try {
|
|
2091
|
+
await teleport();
|
|
2092
|
+
} finally {
|
|
2093
|
+
restoreFetch();
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
expect(listAssistantBackupTimesMock).toHaveBeenCalledWith("my-local");
|
|
2097
|
+
expect(createBackupMock).toHaveBeenCalledWith(
|
|
2098
|
+
"http://localhost:7821",
|
|
2099
|
+
"my-local",
|
|
2100
|
+
expect.objectContaining({ prefix: "my-local-pre-teleport" }),
|
|
2101
|
+
);
|
|
2102
|
+
expect(pruneOldBackupsMock).toHaveBeenCalledWith(
|
|
2103
|
+
"my-local",
|
|
2104
|
+
3,
|
|
2105
|
+
"pre-teleport",
|
|
2106
|
+
);
|
|
2107
|
+
expect(createPlatformBackupMock).not.toHaveBeenCalled();
|
|
2108
|
+
expect(order).toEqual(["backup", "export"]);
|
|
2109
|
+
});
|
|
2110
|
+
|
|
2111
|
+
test("local source with a recent backup: reuses it and skips the export", async () => {
|
|
2112
|
+
setArgv("--from", "my-local", "--platform");
|
|
2113
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2114
|
+
findAssistantByNameMock.mockImplementation((name: string) =>
|
|
2115
|
+
name === "my-local" ? localEntry : null,
|
|
2116
|
+
);
|
|
2117
|
+
listAssistantBackupTimesMock.mockReturnValue([
|
|
2118
|
+
new Date(Date.now() - 5 * 60_000).toISOString(),
|
|
2119
|
+
]);
|
|
2120
|
+
|
|
2121
|
+
const restoreFetch = installTrackingFetch();
|
|
2122
|
+
try {
|
|
2123
|
+
await teleport();
|
|
2124
|
+
} finally {
|
|
2125
|
+
restoreFetch();
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
expect(createBackupMock).not.toHaveBeenCalled();
|
|
2129
|
+
expect(localRuntimeExportToGcsMock).toHaveBeenCalled();
|
|
2130
|
+
});
|
|
2131
|
+
|
|
2132
|
+
test("local source with only stale backups: takes a fresh one", async () => {
|
|
2133
|
+
setArgv("--from", "my-local", "--platform");
|
|
2134
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2135
|
+
findAssistantByNameMock.mockImplementation((name: string) =>
|
|
2136
|
+
name === "my-local" ? localEntry : null,
|
|
2137
|
+
);
|
|
2138
|
+
listAssistantBackupTimesMock.mockReturnValue([
|
|
2139
|
+
new Date(Date.now() - 3 * 60 * 60_000).toISOString(),
|
|
2140
|
+
]);
|
|
2141
|
+
|
|
2142
|
+
const restoreFetch = installTrackingFetch();
|
|
2143
|
+
try {
|
|
2144
|
+
await teleport();
|
|
2145
|
+
} finally {
|
|
2146
|
+
restoreFetch();
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
expect(createBackupMock).toHaveBeenCalledTimes(1);
|
|
2150
|
+
});
|
|
2151
|
+
|
|
2152
|
+
test("docker source: backs up host-side through its runtime URL", async () => {
|
|
2153
|
+
setArgv("--from", "my-docker", "--local");
|
|
2154
|
+
const dockerEntry = makeEntry("my-docker", {
|
|
2155
|
+
cloud: "docker",
|
|
2156
|
+
runtimeUrl: "http://localhost:9821",
|
|
2157
|
+
});
|
|
2158
|
+
findAssistantByNameMock.mockImplementation((name: string) =>
|
|
2159
|
+
name === "my-docker" ? dockerEntry : null,
|
|
2160
|
+
);
|
|
2161
|
+
const freshLocal = makeEntry("fresh-local", { cloud: "local" });
|
|
2162
|
+
loadAllAssistantsMock.mockImplementation(() =>
|
|
2163
|
+
hatchLocalMock.mock.calls.length > 0
|
|
2164
|
+
? [dockerEntry, freshLocal]
|
|
2165
|
+
: [dockerEntry],
|
|
2166
|
+
);
|
|
2167
|
+
|
|
2168
|
+
const restoreFetch = installTrackingFetch();
|
|
2169
|
+
try {
|
|
2170
|
+
await teleport();
|
|
2171
|
+
} finally {
|
|
2172
|
+
restoreFetch();
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
expect(createBackupMock).toHaveBeenCalledWith(
|
|
2176
|
+
"http://localhost:9821",
|
|
2177
|
+
"my-docker",
|
|
2178
|
+
expect.objectContaining({ prefix: "my-docker-pre-teleport" }),
|
|
2179
|
+
);
|
|
2180
|
+
});
|
|
2181
|
+
|
|
2182
|
+
test("platform source: takes a PVC snapshot through the source platform before exporting", async () => {
|
|
2183
|
+
setArgv("--from", "my-platform", "--local", "my-local");
|
|
2184
|
+
const platformEntry = makeEntry("my-platform", {
|
|
2185
|
+
cloud: "vellum",
|
|
2186
|
+
runtimeUrl: "https://platform.vellum.ai",
|
|
2187
|
+
});
|
|
2188
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2189
|
+
findAssistantByNameMock.mockImplementation((name: string) => {
|
|
2190
|
+
if (name === "my-platform") return platformEntry;
|
|
2191
|
+
if (name === "my-local") return localEntry;
|
|
2192
|
+
return null;
|
|
2193
|
+
});
|
|
2194
|
+
|
|
2195
|
+
const order: string[] = [];
|
|
2196
|
+
createPlatformBackupMock.mockImplementation(async () => {
|
|
2197
|
+
order.push("backup");
|
|
2198
|
+
});
|
|
2199
|
+
localRuntimeExportToGcsMock.mockImplementation(async () => {
|
|
2200
|
+
order.push("export");
|
|
2201
|
+
return { jobId: "local-export-job-1" };
|
|
2202
|
+
});
|
|
2203
|
+
|
|
2204
|
+
const restoreFetch = installTrackingFetch();
|
|
2205
|
+
try {
|
|
2206
|
+
await teleport();
|
|
2207
|
+
} finally {
|
|
2208
|
+
restoreFetch();
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
expect(listPlatformBackupsMock).toHaveBeenCalledWith(
|
|
2212
|
+
expect.objectContaining({
|
|
2213
|
+
assistantId: "my-platform",
|
|
2214
|
+
runtimeUrl: "https://platform.vellum.ai",
|
|
2215
|
+
}),
|
|
2216
|
+
"platform-token",
|
|
2217
|
+
);
|
|
2218
|
+
expect(createPlatformBackupMock).toHaveBeenCalledWith(
|
|
2219
|
+
expect.objectContaining({ assistantId: "my-platform" }),
|
|
2220
|
+
"platform-token",
|
|
2221
|
+
);
|
|
2222
|
+
expect(createBackupMock).not.toHaveBeenCalled();
|
|
2223
|
+
expect(order).toEqual(["backup", "export"]);
|
|
2224
|
+
});
|
|
2225
|
+
|
|
2226
|
+
test("platform source with a recent ready snapshot: reuses it", async () => {
|
|
2227
|
+
setArgv("--from", "my-platform", "--local", "my-local");
|
|
2228
|
+
const platformEntry = makeEntry("my-platform", {
|
|
2229
|
+
cloud: "vellum",
|
|
2230
|
+
runtimeUrl: "https://platform.vellum.ai",
|
|
2231
|
+
});
|
|
2232
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2233
|
+
findAssistantByNameMock.mockImplementation((name: string) => {
|
|
2234
|
+
if (name === "my-platform") return platformEntry;
|
|
2235
|
+
if (name === "my-local") return localEntry;
|
|
2236
|
+
return null;
|
|
2237
|
+
});
|
|
2238
|
+
listPlatformBackupsMock.mockResolvedValue([
|
|
2239
|
+
new Date(Date.now() - 20 * 60_000).toISOString(),
|
|
2240
|
+
]);
|
|
2241
|
+
|
|
2242
|
+
const restoreFetch = installTrackingFetch();
|
|
2243
|
+
try {
|
|
2244
|
+
await teleport();
|
|
2245
|
+
} finally {
|
|
2246
|
+
restoreFetch();
|
|
2247
|
+
}
|
|
2248
|
+
|
|
2249
|
+
expect(createPlatformBackupMock).not.toHaveBeenCalled();
|
|
2250
|
+
expect(localRuntimeExportToGcsMock).toHaveBeenCalled();
|
|
2251
|
+
});
|
|
2252
|
+
|
|
2253
|
+
test("host-side backup failure aborts before export and before hatching", async () => {
|
|
2254
|
+
setArgv("--from", "my-local", "--platform");
|
|
2255
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2256
|
+
findAssistantByNameMock.mockImplementation((name: string) =>
|
|
2257
|
+
name === "my-local" ? localEntry : null,
|
|
2258
|
+
);
|
|
2259
|
+
createBackupMock.mockResolvedValue(null);
|
|
2260
|
+
|
|
2261
|
+
const restoreFetch = installTrackingFetch();
|
|
2262
|
+
try {
|
|
2263
|
+
await expect(teleport()).rejects.toThrow("process.exit:1");
|
|
2264
|
+
} finally {
|
|
2265
|
+
restoreFetch();
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
expect(localRuntimeExportToGcsMock).not.toHaveBeenCalled();
|
|
2269
|
+
expect(hatchAssistantMock).not.toHaveBeenCalled();
|
|
2270
|
+
expect(platformImportBundleFromGcsMock).not.toHaveBeenCalled();
|
|
2271
|
+
expect(consoleErrorSpy).toHaveBeenCalledWith(
|
|
2272
|
+
expect.stringContaining("Could not back up 'my-local'"),
|
|
2273
|
+
);
|
|
2274
|
+
});
|
|
2275
|
+
|
|
2276
|
+
test("platform snapshot failure (never ready) aborts too", async () => {
|
|
2277
|
+
setArgv("--from", "my-platform", "--local", "my-local");
|
|
2278
|
+
const platformEntry = makeEntry("my-platform", {
|
|
2279
|
+
cloud: "vellum",
|
|
2280
|
+
runtimeUrl: "https://platform.vellum.ai",
|
|
2281
|
+
});
|
|
2282
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2283
|
+
findAssistantByNameMock.mockImplementation((name: string) => {
|
|
2284
|
+
if (name === "my-platform") return platformEntry;
|
|
2285
|
+
if (name === "my-local") return localEntry;
|
|
2286
|
+
return null;
|
|
2287
|
+
});
|
|
2288
|
+
createPlatformBackupMock.mockRejectedValue(
|
|
2289
|
+
new Error("Platform backup snap-1 was not ready after 600s"),
|
|
2290
|
+
);
|
|
2291
|
+
|
|
2292
|
+
const restoreFetch = installTrackingFetch();
|
|
2293
|
+
try {
|
|
2294
|
+
await expect(teleport()).rejects.toThrow("process.exit:1");
|
|
2295
|
+
} finally {
|
|
2296
|
+
restoreFetch();
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
expect(localRuntimeExportToGcsMock).not.toHaveBeenCalled();
|
|
2300
|
+
expect(consoleErrorSpy).toHaveBeenCalledWith(
|
|
2301
|
+
expect.stringContaining("was not ready"),
|
|
2302
|
+
);
|
|
2303
|
+
});
|
|
2304
|
+
|
|
2305
|
+
test("dry-run never backs up the source", async () => {
|
|
2306
|
+
setArgv("--from", "my-local", "--platform", "--dry-run");
|
|
2307
|
+
const localEntry = makeEntry("my-local", { cloud: "local" });
|
|
2308
|
+
findAssistantByNameMock.mockImplementation((name: string) =>
|
|
2309
|
+
name === "my-local" ? localEntry : null,
|
|
2310
|
+
);
|
|
2311
|
+
|
|
2312
|
+
await teleport();
|
|
2313
|
+
|
|
2314
|
+
expect(listAssistantBackupTimesMock).not.toHaveBeenCalled();
|
|
2315
|
+
expect(createBackupMock).not.toHaveBeenCalled();
|
|
2316
|
+
});
|
|
2317
|
+
});
|
|
2318
|
+
|
|
2022
2319
|
describe("pre-check: existing platform assistant", () => {
|
|
2023
2320
|
test("blocks before any work when pre-check finds existing assistant", async () => {
|
|
2024
2321
|
setArgv("--from", "my-local", "--platform");
|
package/src/commands/teleport.ts
CHANGED
|
@@ -37,6 +37,15 @@ import {
|
|
|
37
37
|
MigrationInProgressError,
|
|
38
38
|
} from "../lib/local-runtime-client.js";
|
|
39
39
|
import { pollJobUntilDone } from "../lib/job-polling.js";
|
|
40
|
+
import {
|
|
41
|
+
createBackup,
|
|
42
|
+
listAssistantBackupTimes,
|
|
43
|
+
pruneOldBackups,
|
|
44
|
+
} from "../lib/backup-ops.js";
|
|
45
|
+
import {
|
|
46
|
+
createPlatformBackup,
|
|
47
|
+
listPlatformBackups,
|
|
48
|
+
} from "../lib/teleport-backup.js";
|
|
40
49
|
import {
|
|
41
50
|
hatchDocker,
|
|
42
51
|
retireDocker,
|
|
@@ -56,6 +65,10 @@ import {
|
|
|
56
65
|
} from "../lib/upgrade-lifecycle.js";
|
|
57
66
|
import { compareVersions } from "../lib/version-compat.js";
|
|
58
67
|
import { join } from "node:path";
|
|
68
|
+
import {
|
|
69
|
+
hasRecentBackup,
|
|
70
|
+
RECENT_BACKUP_MAX_AGE_MS,
|
|
71
|
+
} from "@vellumai/local-mode/teleport-backup-policy";
|
|
59
72
|
|
|
60
73
|
function printHelp(): void {
|
|
61
74
|
console.log(
|
|
@@ -88,6 +101,20 @@ function printHelp(): void {
|
|
|
88
101
|
);
|
|
89
102
|
console.log("transfers (e.g. local to local) are not supported.");
|
|
90
103
|
console.log("");
|
|
104
|
+
console.log(
|
|
105
|
+
"Before exporting, the source assistant is backed up. Platform sources",
|
|
106
|
+
);
|
|
107
|
+
console.log(
|
|
108
|
+
"get a PVC snapshot (waited on until restorable); local and docker",
|
|
109
|
+
);
|
|
110
|
+
console.log(
|
|
111
|
+
"sources get a .vbundle written to ~/.local/share/vellum/backups/ on this",
|
|
112
|
+
);
|
|
113
|
+
console.log(
|
|
114
|
+
`machine. A backup of the source taken within the last ${RECENT_BACKUP_MAX_AGE_MS / 60_000} minutes is`,
|
|
115
|
+
);
|
|
116
|
+
console.log("reused. If the backup fails, the teleport is aborted.");
|
|
117
|
+
console.log("");
|
|
91
118
|
console.log(
|
|
92
119
|
"For local-to-docker and docker-to-local transfers, the source assistant",
|
|
93
120
|
);
|
|
@@ -363,6 +390,70 @@ interface ImportResponse {
|
|
|
363
390
|
};
|
|
364
391
|
}
|
|
365
392
|
|
|
393
|
+
// ---------------------------------------------------------------------------
|
|
394
|
+
// Pre-export safety backup of the source
|
|
395
|
+
//
|
|
396
|
+
// Runs before any data leaves the source. Reuses a backup of this assistant
|
|
397
|
+
// younger than RECENT_BACKUP_MAX_AGE_MS; otherwise takes one and blocks until
|
|
398
|
+
// it is usable. Any failure aborts the teleport: the source is retired at the
|
|
399
|
+
// end of a successful teleport, and that is only safe with a restore point
|
|
400
|
+
// behind it.
|
|
401
|
+
//
|
|
402
|
+
// Local and docker sources are exported to the host's CLI backup directory
|
|
403
|
+
// rather than snapshotted by their own gateway: docker retirement removes
|
|
404
|
+
// every source volume (including the gateway's backup pool), and bare-metal
|
|
405
|
+
// gateways share one unlabelled pool across assistants, so neither would
|
|
406
|
+
// give this assistant a restore point that outlives the teleport.
|
|
407
|
+
// ---------------------------------------------------------------------------
|
|
408
|
+
|
|
409
|
+
/** Export timeout for the host-side pre-teleport backup of a local/docker source. */
|
|
410
|
+
const LOCAL_SOURCE_BACKUP_TIMEOUT_MS = 30 * 60 * 1000;
|
|
411
|
+
|
|
412
|
+
async function backupSourceBeforeTeleport(
|
|
413
|
+
entry: AssistantEntry,
|
|
414
|
+
cloud: string,
|
|
415
|
+
displayName: string,
|
|
416
|
+
): Promise<void> {
|
|
417
|
+
console.log(`Backing up ${displayName} (${cloud})...`);
|
|
418
|
+
try {
|
|
419
|
+
if (cloud === "vellum") {
|
|
420
|
+
const platformToken = readPlatformToken();
|
|
421
|
+
if (!platformToken) {
|
|
422
|
+
console.error("Not logged in. Run 'vellum login' first.");
|
|
423
|
+
process.exit(1);
|
|
424
|
+
}
|
|
425
|
+
const createdAts = await listPlatformBackups(entry, platformToken);
|
|
426
|
+
if (hasRecentBackup(createdAts)) {
|
|
427
|
+
console.log("Recent backup found, reusing it.");
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
await createPlatformBackup(entry, platformToken);
|
|
431
|
+
console.log("Backup complete.");
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
if (hasRecentBackup(listAssistantBackupTimes(entry.assistantId))) {
|
|
436
|
+
console.log("Recent backup found, reusing it.");
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
const backupPath = await createBackup(entry.runtimeUrl, entry.assistantId, {
|
|
440
|
+
prefix: `${entry.assistantId}-pre-teleport`,
|
|
441
|
+
description: `Pre-teleport snapshot of ${displayName} (${cloud})`,
|
|
442
|
+
timeoutMs: LOCAL_SOURCE_BACKUP_TIMEOUT_MS,
|
|
443
|
+
});
|
|
444
|
+
if (!backupPath) {
|
|
445
|
+
throw new Error("backup export failed (see warning above)");
|
|
446
|
+
}
|
|
447
|
+
pruneOldBackups(entry.assistantId, 3, "pre-teleport");
|
|
448
|
+
console.log(`Backup saved: ${backupPath}`);
|
|
449
|
+
} catch (err) {
|
|
450
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
451
|
+
console.error(`Error: Could not back up '${displayName}': ${msg}`);
|
|
452
|
+
console.error("Teleport aborted; the source assistant was not modified.");
|
|
453
|
+
process.exit(1);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
366
457
|
// ---------------------------------------------------------------------------
|
|
367
458
|
// Export from source — unified GCS flow
|
|
368
459
|
//
|
|
@@ -1399,6 +1490,8 @@ export async function teleport(): Promise<void> {
|
|
|
1399
1490
|
// where the import will run. For existing targets that's the lockfile's
|
|
1400
1491
|
// runtimeUrl; for fresh hatches it's getPlatformUrl() (which is what
|
|
1401
1492
|
// resolveOrHatchTarget writes to the new entry).
|
|
1493
|
+
await backupSourceBeforeTeleport(fromEntry, fromCloud, from);
|
|
1494
|
+
|
|
1402
1495
|
console.log(`Exporting from ${from} (${fromCloud})...`);
|
|
1403
1496
|
const bundlePlatformUrl = targetPlatformUrl ?? getPlatformUrl();
|
|
1404
1497
|
const { bundleKey } = await exportFromAssistant(
|
|
@@ -1469,6 +1562,8 @@ export async function teleport(): Promise<void> {
|
|
|
1469
1562
|
const bundlePlatformUrl =
|
|
1470
1563
|
fromCloud === "vellum" ? fromEntry.runtimeUrl : getPlatformUrl();
|
|
1471
1564
|
|
|
1565
|
+
await backupSourceBeforeTeleport(fromEntry, fromCloud, from);
|
|
1566
|
+
|
|
1472
1567
|
// Export from source (bundle lives in GCS after this returns).
|
|
1473
1568
|
console.log(`Exporting from ${from} (${fromCloud})...`);
|
|
1474
1569
|
const { bundleKey } = await exportFromAssistant(
|
package/src/lib/backup-ops.ts
CHANGED
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
mkdirSync,
|
|
4
4
|
readdirSync,
|
|
5
5
|
readFileSync,
|
|
6
|
+
statSync,
|
|
6
7
|
unlinkSync,
|
|
7
8
|
writeFileSync,
|
|
8
9
|
} from "fs";
|
|
@@ -60,8 +61,9 @@ async function getGuardianAccessToken(
|
|
|
60
61
|
export async function createBackup(
|
|
61
62
|
runtimeUrl: string,
|
|
62
63
|
assistantId: string,
|
|
63
|
-
options?: { prefix?: string; description?: string },
|
|
64
|
+
options?: { prefix?: string; description?: string; timeoutMs?: number },
|
|
64
65
|
): Promise<string | null> {
|
|
66
|
+
const timeoutMs = options?.timeoutMs ?? 120_000;
|
|
65
67
|
try {
|
|
66
68
|
let accessToken = await getGuardianAccessToken(runtimeUrl, assistantId);
|
|
67
69
|
if (!accessToken) {
|
|
@@ -82,7 +84,7 @@ export async function createBackup(
|
|
|
82
84
|
body: JSON.stringify({
|
|
83
85
|
description: options?.description ?? "CLI backup",
|
|
84
86
|
}),
|
|
85
|
-
signal: AbortSignal.timeout(
|
|
87
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
86
88
|
},
|
|
87
89
|
);
|
|
88
90
|
|
|
@@ -106,7 +108,7 @@ export async function createBackup(
|
|
|
106
108
|
body: JSON.stringify({
|
|
107
109
|
description: options?.description ?? "CLI backup",
|
|
108
110
|
}),
|
|
109
|
-
signal: AbortSignal.timeout(
|
|
111
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
110
112
|
});
|
|
111
113
|
}
|
|
112
114
|
|
|
@@ -283,19 +285,84 @@ export async function restoreBackup(
|
|
|
283
285
|
}
|
|
284
286
|
}
|
|
285
287
|
|
|
288
|
+
/** Filename kinds this CLI writes: `<assistantId>-<kind>-<timestamp>.vbundle`. */
|
|
289
|
+
export const CLI_BACKUP_KINDS = ["pre-upgrade", "pre-teleport"] as const;
|
|
290
|
+
export type CliBackupKind = (typeof CLI_BACKUP_KINDS)[number];
|
|
291
|
+
|
|
292
|
+
/** `new Date().toISOString().replace(/[:.]/g, "-")`, as used in every CLI backup filename. */
|
|
293
|
+
const BACKUP_TIMESTAMP_PATTERN =
|
|
294
|
+
"\\d{4}-\\d{2}-\\d{2}T\\d{2}-\\d{2}-\\d{2}-\\d{3}Z";
|
|
295
|
+
|
|
296
|
+
function escapeRegExp(value: string): string {
|
|
297
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Matches the backup filenames this CLI wrote for exactly `assistantId` and
|
|
302
|
+
* one of `kinds`. The kind and timestamp segments are matched in full, so an
|
|
303
|
+
* assistant whose id is a prefix of another's (`alpha` vs `alpha-prod`, or
|
|
304
|
+
* `alpha` vs `alpha-pre-teleport-prod`) never matches the other's files.
|
|
305
|
+
*/
|
|
306
|
+
export function assistantBackupFilenamePattern(
|
|
307
|
+
assistantId: string,
|
|
308
|
+
kinds: readonly CliBackupKind[] = CLI_BACKUP_KINDS,
|
|
309
|
+
): RegExp {
|
|
310
|
+
return new RegExp(
|
|
311
|
+
`^${escapeRegExp(assistantId)}-(?:${kinds.join("|")})-${BACKUP_TIMESTAMP_PATTERN}\\.vbundle$`,
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Modification times of the `.vbundle` backups this CLI has written for
|
|
317
|
+
* `assistantId` (every kind in `CLI_BACKUP_KINDS`), as ISO timestamps.
|
|
318
|
+
* Missing directory yields `[]`.
|
|
319
|
+
*/
|
|
320
|
+
export function listAssistantBackupTimes(assistantId: string): string[] {
|
|
321
|
+
const backupsDir = getBackupsDir();
|
|
322
|
+
let names: string[];
|
|
323
|
+
try {
|
|
324
|
+
names = readdirSync(backupsDir);
|
|
325
|
+
} catch (err) {
|
|
326
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
|
327
|
+
return [];
|
|
328
|
+
}
|
|
329
|
+
throw err;
|
|
330
|
+
}
|
|
331
|
+
const pattern = assistantBackupFilenamePattern(assistantId);
|
|
332
|
+
const times: string[] = [];
|
|
333
|
+
for (const name of names) {
|
|
334
|
+
if (!pattern.test(name)) {
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
try {
|
|
338
|
+
times.push(statSync(join(backupsDir, name)).mtime.toISOString());
|
|
339
|
+
} catch (err) {
|
|
340
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
341
|
+
throw err;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
return times;
|
|
346
|
+
}
|
|
347
|
+
|
|
286
348
|
/**
|
|
287
|
-
* Keep only the N most recent
|
|
288
|
-
* deleting older ones.
|
|
349
|
+
* Keep only the N most recent backups of one `kind` for an assistant,
|
|
350
|
+
* deleting older ones. Filenames are matched exactly (id, kind and
|
|
351
|
+
* timestamp), never by prefix. Default: keep 3 pre-upgrade backups.
|
|
289
352
|
* Never throws — failures are silently ignored.
|
|
290
353
|
*/
|
|
291
|
-
export function pruneOldBackups(
|
|
354
|
+
export function pruneOldBackups(
|
|
355
|
+
assistantId: string,
|
|
356
|
+
keep: number = 3,
|
|
357
|
+
kind: CliBackupKind = "pre-upgrade",
|
|
358
|
+
): void {
|
|
292
359
|
try {
|
|
293
360
|
const backupsDir = getBackupsDir();
|
|
294
361
|
if (!existsSync(backupsDir)) return;
|
|
295
362
|
|
|
296
|
-
const
|
|
363
|
+
const pattern = assistantBackupFilenamePattern(assistantId, [kind]);
|
|
297
364
|
const entries = readdirSync(backupsDir)
|
|
298
|
-
.filter((f) =>
|
|
365
|
+
.filter((f) => pattern.test(f))
|
|
299
366
|
.sort();
|
|
300
367
|
|
|
301
368
|
if (entries.length <= keep) return;
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Platform (cloud="vellum") half of the pre-teleport safety backup.
|
|
3
|
+
*
|
|
4
|
+
* A platform source is snapshotted through Django's user-facing backup
|
|
5
|
+
* endpoint, which cuts a PVC VolumeSnapshot via vembda. The POST returns as
|
|
6
|
+
* soon as the snapshot object exists; it becomes restorable asynchronously,
|
|
7
|
+
* so {@link createPlatformBackup} polls the listing until the new snapshot
|
|
8
|
+
* reports `ready_to_use` before returning. A snapshot that never becomes
|
|
9
|
+
* ready within the timeout is a backup failure, not a success.
|
|
10
|
+
*
|
|
11
|
+
* Local and docker sources are backed up host-side by `backup-ops` instead,
|
|
12
|
+
* so the restore point survives the source's retirement. Policy shared with
|
|
13
|
+
* the web teleport lives in `@vellumai/local-mode/teleport-backup-policy`.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import {
|
|
17
|
+
MANAGED_BACKUP_READY_POLL_INTERVAL_MS,
|
|
18
|
+
MANAGED_BACKUP_READY_TIMEOUT_MS,
|
|
19
|
+
managedBackupIsReady,
|
|
20
|
+
readyManagedBackupCreatedAts,
|
|
21
|
+
type ManagedBackupEntry,
|
|
22
|
+
} from "@vellumai/local-mode/teleport-backup-policy";
|
|
23
|
+
|
|
24
|
+
import type { AssistantEntry } from "./assistant-config.js";
|
|
25
|
+
import { loopbackSafeFetch } from "./loopback-fetch.js";
|
|
26
|
+
import { authHeaders, invalidateOrgIdCache } from "./platform-client.js";
|
|
27
|
+
|
|
28
|
+
type PlatformEntry = Pick<AssistantEntry, "runtimeUrl" | "assistantId">;
|
|
29
|
+
|
|
30
|
+
function platformBackupsUrl(entry: PlatformEntry): string {
|
|
31
|
+
return `${entry.runtimeUrl}/v1/assistants/${entry.assistantId}/backups/`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function errorSuffix(response: Response): Promise<string> {
|
|
35
|
+
const body = await response.text().catch(() => "");
|
|
36
|
+
return `(${response.status})${body ? `: ${body}` : ""}`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Send a platform backup request with user-session auth, refreshing the
|
|
41
|
+
* cached organization id once on 401 (the same recovery the runtime identity
|
|
42
|
+
* probe applies).
|
|
43
|
+
*/
|
|
44
|
+
async function platformBackupRequest(
|
|
45
|
+
entry: PlatformEntry,
|
|
46
|
+
token: string,
|
|
47
|
+
method: "GET" | "POST",
|
|
48
|
+
): Promise<Response> {
|
|
49
|
+
const doRequest = async (): Promise<Response> =>
|
|
50
|
+
loopbackSafeFetch(platformBackupsUrl(entry), {
|
|
51
|
+
method,
|
|
52
|
+
headers: {
|
|
53
|
+
...(await authHeaders(token, entry.runtimeUrl)),
|
|
54
|
+
Accept: "application/json",
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
let response = await doRequest();
|
|
58
|
+
if (response.status === 401) {
|
|
59
|
+
invalidateOrgIdCache(token, entry.runtimeUrl);
|
|
60
|
+
response = await doRequest();
|
|
61
|
+
}
|
|
62
|
+
return response;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function fetchPlatformBackups(
|
|
66
|
+
entry: PlatformEntry,
|
|
67
|
+
token: string,
|
|
68
|
+
): Promise<ManagedBackupEntry[]> {
|
|
69
|
+
const response = await platformBackupRequest(entry, token, "GET");
|
|
70
|
+
if (!response.ok) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`Platform backup list failed ${await errorSuffix(response)}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
const body = (await response.json()) as { backups?: ManagedBackupEntry[] };
|
|
76
|
+
return body.backups ?? [];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* `created_at` of every restorable PVC snapshot of a platform assistant:
|
|
81
|
+
* `GET /v1/assistants/<id>/backups/`.
|
|
82
|
+
*/
|
|
83
|
+
export async function listPlatformBackups(
|
|
84
|
+
entry: PlatformEntry,
|
|
85
|
+
token: string,
|
|
86
|
+
): Promise<Array<string | undefined>> {
|
|
87
|
+
return readyManagedBackupCreatedAts(await fetchPlatformBackups(entry, token));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface CreatePlatformBackupOptions {
|
|
91
|
+
timeoutMs?: number;
|
|
92
|
+
pollIntervalMs?: number;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Take a PVC snapshot of a platform assistant and wait until it is
|
|
97
|
+
* restorable: `POST /v1/assistants/<id>/backups/`, then re-list until the
|
|
98
|
+
* returned `snapshot_name` reports `ready_to_use`. Throws if the POST fails
|
|
99
|
+
* or the snapshot is not ready within `timeoutMs`.
|
|
100
|
+
*/
|
|
101
|
+
export async function createPlatformBackup(
|
|
102
|
+
entry: PlatformEntry,
|
|
103
|
+
token: string,
|
|
104
|
+
options: CreatePlatformBackupOptions = {},
|
|
105
|
+
): Promise<void> {
|
|
106
|
+
const timeoutMs = options.timeoutMs ?? MANAGED_BACKUP_READY_TIMEOUT_MS;
|
|
107
|
+
const pollIntervalMs =
|
|
108
|
+
options.pollIntervalMs ?? MANAGED_BACKUP_READY_POLL_INTERVAL_MS;
|
|
109
|
+
|
|
110
|
+
const response = await platformBackupRequest(entry, token, "POST");
|
|
111
|
+
if (!response.ok) {
|
|
112
|
+
throw new Error(
|
|
113
|
+
`Platform backup create failed ${await errorSuffix(response)}`,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
const created = (await response.json().catch(() => null)) as {
|
|
117
|
+
snapshot_name?: string;
|
|
118
|
+
ready_to_use?: boolean;
|
|
119
|
+
} | null;
|
|
120
|
+
const snapshotName = created?.snapshot_name;
|
|
121
|
+
if (!snapshotName) {
|
|
122
|
+
throw new Error("Platform backup create returned no snapshot name");
|
|
123
|
+
}
|
|
124
|
+
if (created?.ready_to_use === true) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const deadline = Date.now() + timeoutMs;
|
|
129
|
+
while (Date.now() < deadline) {
|
|
130
|
+
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
|
131
|
+
const backups = await fetchPlatformBackups(entry, token);
|
|
132
|
+
if (managedBackupIsReady(backups, snapshotName)) {
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
throw new Error(
|
|
137
|
+
`Platform backup ${snapshotName} was not ready after ${Math.round(timeoutMs / 1000)}s`,
|
|
138
|
+
);
|
|
139
|
+
}
|