@stigmer/server 3.22.1 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifactstorage/artifact-storage.d.ts +29 -31
- package/dist/artifactstorage/artifact-storage.d.ts.map +1 -1
- package/dist/artifactstorage/artifact-storage.js +5 -7
- package/dist/artifactstorage/artifact-storage.js.map +1 -1
- package/dist/artifactstorage/r2-storage.d.ts +12 -7
- package/dist/artifactstorage/r2-storage.d.ts.map +1 -1
- package/dist/artifactstorage/r2-storage.js +6 -14
- package/dist/artifactstorage/r2-storage.js.map +1 -1
- package/dist/boot/compose.d.ts.map +1 -1
- package/dist/boot/compose.js +49 -24
- package/dist/boot/compose.js.map +1 -1
- package/dist/boot/config.d.ts +4 -3
- package/dist/boot/config.d.ts.map +1 -1
- package/dist/boot/config.js.map +1 -1
- package/dist/boot/skill-transfer-origin.d.ts +52 -0
- package/dist/boot/skill-transfer-origin.d.ts.map +1 -0
- package/dist/boot/skill-transfer-origin.js +54 -0
- package/dist/boot/skill-transfer-origin.js.map +1 -0
- package/dist/domain/plugin/controller.d.ts +3 -7
- package/dist/domain/plugin/controller.d.ts.map +1 -1
- package/dist/domain/plugin/controller.js +8 -8
- package/dist/domain/plugin/controller.js.map +1 -1
- package/dist/domain/skill/constants.d.ts +26 -3
- package/dist/domain/skill/constants.d.ts.map +1 -1
- package/dist/domain/skill/constants.js +26 -3
- package/dist/domain/skill/constants.js.map +1 -1
- package/dist/domain/skill/controller.d.ts +6 -11
- package/dist/domain/skill/controller.d.ts.map +1 -1
- package/dist/domain/skill/controller.js +33 -22
- package/dist/domain/skill/controller.js.map +1 -1
- package/dist/domain/skill/push.d.ts +2 -2
- package/dist/domain/skill/push.d.ts.map +1 -1
- package/dist/domain/skill/push.js +2 -2
- package/dist/domain/skill/push.js.map +1 -1
- package/dist/domain/skill/transfer/handler.d.ts +17 -9
- package/dist/domain/skill/transfer/handler.d.ts.map +1 -1
- package/dist/domain/skill/transfer/handler.js +19 -8
- package/dist/domain/skill/transfer/handler.js.map +1 -1
- package/dist/domain/skill/transfer/slots.d.ts +26 -41
- package/dist/domain/skill/transfer/slots.d.ts.map +1 -1
- package/dist/domain/skill/transfer/slots.js +80 -87
- package/dist/domain/skill/transfer/slots.js.map +1 -1
- package/dist/domain/skill/transfer/staging.d.ts +48 -0
- package/dist/domain/skill/transfer/staging.d.ts.map +1 -0
- package/dist/domain/skill/transfer/staging.js +127 -0
- package/dist/domain/skill/transfer/staging.js.map +1 -0
- package/dist/pipeline/steps/resolve-artifact-source.d.ts +11 -8
- package/dist/pipeline/steps/resolve-artifact-source.d.ts.map +1 -1
- package/dist/pipeline/steps/resolve-artifact-source.js +8 -4
- package/dist/pipeline/steps/resolve-artifact-source.js.map +1 -1
- package/package.json +6 -6
- package/src/artifactstorage/__tests__/artifact-storage.test.ts +36 -33
- package/src/artifactstorage/__tests__/r2-storage.test.ts +11 -13
- package/src/artifactstorage/artifact-storage.ts +37 -33
- package/src/artifactstorage/r2-storage.ts +6 -15
- package/src/boot/__tests__/skill-transfer-origin.test.ts +72 -0
- package/src/boot/compose.ts +62 -25
- package/src/boot/config.ts +4 -3
- package/src/boot/skill-transfer-origin.ts +76 -0
- package/src/domain/plugin/controller.ts +15 -17
- package/src/domain/skill/__tests__/skill.test.ts +5 -3
- package/src/domain/skill/__tests__/slots.test.ts +113 -60
- package/src/domain/skill/__tests__/staging.test.ts +304 -0
- package/src/domain/skill/constants.ts +28 -3
- package/src/domain/skill/controller.ts +44 -35
- package/src/domain/skill/push.ts +3 -3
- package/src/domain/skill/transfer/handler.ts +33 -15
- package/src/domain/skill/transfer/slots.ts +89 -100
- package/src/domain/skill/transfer/staging.ts +191 -0
- package/src/pipeline/steps/resolve-artifact-source.ts +20 -13
- package/src/transport/registry/__tests__/lanes.test.ts +1 -1
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pins the archive staging port (transfer/staging.ts): the reference ⇄
|
|
3
|
+
* staging-key mapping and its refusals; mint handing the driver the
|
|
4
|
+
* domain's key and the slot TTL; consume reading then deleting, with the
|
|
5
|
+
* delete best-effort and loud; the one "unknown or expired" condition for
|
|
6
|
+
* a malformed, missing or already-consumed reference; a driver fault
|
|
7
|
+
* propagating untouched; the download capability's floor TTL. First over
|
|
8
|
+
* a recording fake driver (what any bucket sees), then over the real local
|
|
9
|
+
* driver with its slot registry, wired exactly as boot/compose.ts wires it
|
|
10
|
+
* — the two drivers must be indistinguishable above this module.
|
|
11
|
+
*/
|
|
12
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
import { Readable } from "node:stream";
|
|
16
|
+
|
|
17
|
+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
18
|
+
|
|
19
|
+
import {
|
|
20
|
+
ArtifactStorageNotFoundError,
|
|
21
|
+
LocalArtifactStorage,
|
|
22
|
+
} from "../../../artifactstorage/artifact-storage.js";
|
|
23
|
+
import type { ArtifactStorage } from "../../../artifactstorage/artifact-storage.js";
|
|
24
|
+
import { createLogger } from "../../../boot/logger.js";
|
|
25
|
+
import { SKILL_ARTIFACTS_PATH_PREFIX } from "../../../transport/constants.js";
|
|
26
|
+
import {
|
|
27
|
+
DEFAULT_SLOT_TTL_MS,
|
|
28
|
+
DOWNLOAD_URL_TTL_MS,
|
|
29
|
+
MAX_ZIP_SIZE,
|
|
30
|
+
STAGING_KEY_PREFIX,
|
|
31
|
+
UPLOADS_SEGMENT,
|
|
32
|
+
} from "../constants.js";
|
|
33
|
+
import { transferServeUrl, uploadUrl } from "../transfer/handler.js";
|
|
34
|
+
import { SlotUnknownError, UploadSlots } from "../transfer/slots.js";
|
|
35
|
+
import {
|
|
36
|
+
newArchiveStaging,
|
|
37
|
+
newUploadRef,
|
|
38
|
+
stagingKeyOf,
|
|
39
|
+
uploadRefOf,
|
|
40
|
+
} from "../transfer/staging.js";
|
|
41
|
+
import type { ArchiveStaging } from "../transfer/staging.js";
|
|
42
|
+
|
|
43
|
+
const logLines: string[] = [];
|
|
44
|
+
const logger = createLogger({
|
|
45
|
+
level: "error",
|
|
46
|
+
pretty: false,
|
|
47
|
+
write: (line) => {
|
|
48
|
+
logLines.push(line);
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
describe("the reference ⇄ staging key mapping", () => {
|
|
53
|
+
it("mints sau_-prefixed 128-bit hex references whose key sits under the staging prefix", () => {
|
|
54
|
+
const ref = newUploadRef();
|
|
55
|
+
expect(ref).toMatch(/^sau_[0-9a-f]{32}$/);
|
|
56
|
+
const key = stagingKeyOf(ref);
|
|
57
|
+
expect(key).toBe(`${STAGING_KEY_PREFIX}${ref.slice(4)}.zip`);
|
|
58
|
+
expect(uploadRefOf(key!)).toBe(ref);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("answers no key for a reference this server could not have minted", () => {
|
|
62
|
+
for (const bad of [
|
|
63
|
+
"",
|
|
64
|
+
"sau_",
|
|
65
|
+
"sau_nope",
|
|
66
|
+
"sau_" + "g".repeat(32),
|
|
67
|
+
"sau_" + "0".repeat(31),
|
|
68
|
+
"skl_" + "0".repeat(32),
|
|
69
|
+
"sau_" + "A".repeat(32),
|
|
70
|
+
]) {
|
|
71
|
+
expect(stagingKeyOf(bad), bad).toBeUndefined();
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("refuses to derive a reference from a foreign key", () => {
|
|
76
|
+
expect(() => uploadRefOf("skills/deadbeef.zip")).toThrow(
|
|
77
|
+
"is not a staging key this domain minted",
|
|
78
|
+
);
|
|
79
|
+
expect(() => uploadRefOf(`${STAGING_KEY_PREFIX}notes.txt`)).toThrow(
|
|
80
|
+
"is not a staging key this domain minted",
|
|
81
|
+
);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
// ─── Over a recording fake: what a bucket driver is asked to do ──────────
|
|
86
|
+
|
|
87
|
+
interface Recorded {
|
|
88
|
+
presignPut: Array<{ key: string; declaredSizeBytes: number; ttlMs: number }>;
|
|
89
|
+
downloads: string[];
|
|
90
|
+
deletes: string[];
|
|
91
|
+
signed: Array<{ key: string; expiresInMs: number; downloadFilename: string }>;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function fakeDriver(
|
|
95
|
+
options: { failDelete?: boolean; failDownload?: Error } = {},
|
|
96
|
+
): {
|
|
97
|
+
driver: ArtifactStorage;
|
|
98
|
+
blobs: Map<string, Uint8Array>;
|
|
99
|
+
recorded: Recorded;
|
|
100
|
+
} {
|
|
101
|
+
const blobs = new Map<string, Uint8Array>();
|
|
102
|
+
const recorded: Recorded = {
|
|
103
|
+
presignPut: [],
|
|
104
|
+
downloads: [],
|
|
105
|
+
deletes: [],
|
|
106
|
+
signed: [],
|
|
107
|
+
};
|
|
108
|
+
const driver: ArtifactStorage = {
|
|
109
|
+
upload: (key, data) => {
|
|
110
|
+
blobs.set(key, data);
|
|
111
|
+
return Promise.resolve();
|
|
112
|
+
},
|
|
113
|
+
download: (key) => {
|
|
114
|
+
recorded.downloads.push(key);
|
|
115
|
+
if (options.failDownload) return Promise.reject(options.failDownload);
|
|
116
|
+
const data = blobs.get(key);
|
|
117
|
+
return data === undefined
|
|
118
|
+
? Promise.reject(new ArtifactStorageNotFoundError(key))
|
|
119
|
+
: Promise.resolve(data);
|
|
120
|
+
},
|
|
121
|
+
size: (key) => Promise.resolve(blobs.get(key)?.length ?? 0),
|
|
122
|
+
presignPut: (key, declaredSizeBytes, ttlMs) => {
|
|
123
|
+
recorded.presignPut.push({ key, declaredSizeBytes, ttlMs });
|
|
124
|
+
return Promise.resolve({
|
|
125
|
+
url: `https://bucket.test/${key}?sig=put`,
|
|
126
|
+
ttlMs: ttlMs / 2,
|
|
127
|
+
});
|
|
128
|
+
},
|
|
129
|
+
getSignedUrl: (key, expiresInMs, downloadFilename) => {
|
|
130
|
+
recorded.signed.push({ key, expiresInMs, downloadFilename });
|
|
131
|
+
return Promise.resolve(`https://bucket.test/${key}?sig=get`);
|
|
132
|
+
},
|
|
133
|
+
delete: (key) => {
|
|
134
|
+
recorded.deletes.push(key);
|
|
135
|
+
if (options.failDelete)
|
|
136
|
+
return Promise.reject(new Error("bucket refused the delete"));
|
|
137
|
+
blobs.delete(key);
|
|
138
|
+
return Promise.resolve();
|
|
139
|
+
},
|
|
140
|
+
exists: (key) => Promise.resolve(blobs.has(key)),
|
|
141
|
+
health: () => Promise.resolve(),
|
|
142
|
+
};
|
|
143
|
+
return { driver, blobs, recorded };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
describe("newArchiveStaging over a bucket-shaped driver", () => {
|
|
147
|
+
beforeEach(() => {
|
|
148
|
+
logLines.length = 0;
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it("mint hands the driver the domain's key and the slot TTL, and returns the driver's URL and granted TTL", async () => {
|
|
152
|
+
const { driver, recorded } = fakeDriver();
|
|
153
|
+
const staging = newArchiveStaging(driver, logger);
|
|
154
|
+
const minted = await staging.mint(1234);
|
|
155
|
+
|
|
156
|
+
expect(recorded.presignPut).toEqual([
|
|
157
|
+
{
|
|
158
|
+
key: stagingKeyOf(minted.ref),
|
|
159
|
+
declaredSizeBytes: 1234,
|
|
160
|
+
ttlMs: DEFAULT_SLOT_TTL_MS,
|
|
161
|
+
},
|
|
162
|
+
]);
|
|
163
|
+
expect(minted.url).toBe(
|
|
164
|
+
`https://bucket.test/${stagingKeyOf(minted.ref)}?sig=put`,
|
|
165
|
+
);
|
|
166
|
+
expect(minted.ttlMs).toBe(DEFAULT_SLOT_TTL_MS / 2);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
it("mint refuses a declaration outside the archive ceiling before any driver call", async () => {
|
|
170
|
+
const { driver, recorded } = fakeDriver();
|
|
171
|
+
const staging = newArchiveStaging(driver, logger);
|
|
172
|
+
await expect(staging.mint(0)).rejects.toThrow(
|
|
173
|
+
`declared size 0 outside (0, ${MAX_ZIP_SIZE}]`,
|
|
174
|
+
);
|
|
175
|
+
await expect(staging.mint(MAX_ZIP_SIZE + 1)).rejects.toThrow("outside");
|
|
176
|
+
expect(recorded.presignPut).toEqual([]);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it("consume reads the staged key, deletes it, and a replay is unknown", async () => {
|
|
180
|
+
const { driver, blobs, recorded } = fakeDriver();
|
|
181
|
+
const staging = newArchiveStaging(driver, logger);
|
|
182
|
+
const minted = await staging.mint(5);
|
|
183
|
+
const key = stagingKeyOf(minted.ref)!;
|
|
184
|
+
blobs.set(key, Buffer.from("hello"));
|
|
185
|
+
|
|
186
|
+
expect(Buffer.from(await staging.consume(minted.ref)).toString()).toBe(
|
|
187
|
+
"hello",
|
|
188
|
+
);
|
|
189
|
+
expect(recorded.downloads).toEqual([key]);
|
|
190
|
+
expect(recorded.deletes).toEqual([key]);
|
|
191
|
+
await expect(staging.consume(minted.ref)).rejects.toThrow(SlotUnknownError);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it("a malformed or never-uploaded reference is one condition: unknown or expired", async () => {
|
|
195
|
+
const { driver, recorded } = fakeDriver();
|
|
196
|
+
const staging = newArchiveStaging(driver, logger);
|
|
197
|
+
await expect(staging.consume("not-a-ref")).rejects.toThrow(
|
|
198
|
+
"upload reference unknown or expired",
|
|
199
|
+
);
|
|
200
|
+
// Nothing shaped like a key reaches the driver for a malformed ref.
|
|
201
|
+
expect(recorded.downloads).toEqual([]);
|
|
202
|
+
|
|
203
|
+
const minted = await staging.mint(5);
|
|
204
|
+
await expect(staging.consume(minted.ref)).rejects.toThrow(SlotUnknownError);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("a failed delete is loud and does not fail the consume", async () => {
|
|
208
|
+
const { driver, blobs } = fakeDriver({ failDelete: true });
|
|
209
|
+
const staging = newArchiveStaging(driver, logger);
|
|
210
|
+
const minted = await staging.mint(5);
|
|
211
|
+
blobs.set(stagingKeyOf(minted.ref)!, Buffer.from("hello"));
|
|
212
|
+
|
|
213
|
+
expect(Buffer.from(await staging.consume(minted.ref)).toString()).toBe(
|
|
214
|
+
"hello",
|
|
215
|
+
);
|
|
216
|
+
expect(
|
|
217
|
+
logLines.some((line) =>
|
|
218
|
+
line.includes("failed to delete consumed staged artifact"),
|
|
219
|
+
),
|
|
220
|
+
).toBe(true);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("a driver fault on download propagates as itself, never as an unusable reference", async () => {
|
|
224
|
+
const fault = new Error("bucket unreachable");
|
|
225
|
+
const { driver } = fakeDriver({ failDownload: fault });
|
|
226
|
+
const staging = newArchiveStaging(driver, logger);
|
|
227
|
+
const minted = await staging.mint(5);
|
|
228
|
+
await expect(staging.consume(minted.ref)).rejects.toBe(fault);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it("downloadUrl signs the stored key for the floor TTL and reports it", async () => {
|
|
232
|
+
const { driver, recorded } = fakeDriver();
|
|
233
|
+
const staging = newArchiveStaging(driver, logger);
|
|
234
|
+
const capability = await staging.downloadUrl("skills/abc.zip");
|
|
235
|
+
expect(recorded.signed).toEqual([
|
|
236
|
+
{
|
|
237
|
+
key: "skills/abc.zip",
|
|
238
|
+
expiresInMs: DOWNLOAD_URL_TTL_MS,
|
|
239
|
+
downloadFilename: "",
|
|
240
|
+
},
|
|
241
|
+
]);
|
|
242
|
+
expect(capability).toEqual({
|
|
243
|
+
url: "https://bucket.test/skills/abc.zip?sig=get",
|
|
244
|
+
ttlMs: DOWNLOAD_URL_TTL_MS,
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
// ─── Over the real local driver: the slot registry behind the same port ──
|
|
250
|
+
|
|
251
|
+
describe("newArchiveStaging over LocalArtifactStorage and its slot registry", () => {
|
|
252
|
+
const BASE_URL = "http://localhost:8080";
|
|
253
|
+
let root: string;
|
|
254
|
+
let slots: UploadSlots;
|
|
255
|
+
let staging: ArchiveStaging;
|
|
256
|
+
|
|
257
|
+
beforeEach(() => {
|
|
258
|
+
root = mkdtempSync(path.join(tmpdir(), "archive-staging-local-"));
|
|
259
|
+
slots = new UploadSlots(root, STAGING_KEY_PREFIX, 60_000, 1024 * 1024);
|
|
260
|
+
const driver = new LocalArtifactStorage(root, transferServeUrl(BASE_URL), {
|
|
261
|
+
reserve: (key, declaredSizeBytes) => {
|
|
262
|
+
const { ttlMs } = slots.reserve(key, declaredSizeBytes);
|
|
263
|
+
return { url: uploadUrl(BASE_URL, uploadRefOf(key)), ttlMs };
|
|
264
|
+
},
|
|
265
|
+
});
|
|
266
|
+
staging = newArchiveStaging(driver, logger);
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
afterEach(() => {
|
|
270
|
+
rmSync(root, { recursive: true, force: true });
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
it("mint → lane receive → consume round-trips the bytes and retires the reference", async () => {
|
|
274
|
+
const bytes = Buffer.from("staged artifact bytes");
|
|
275
|
+
const minted = await staging.mint(bytes.length);
|
|
276
|
+
|
|
277
|
+
// The URL is the lane's wire shape and the ref is its last segment —
|
|
278
|
+
// exactly how the lane handler dispatches a PUT.
|
|
279
|
+
expect(minted.url).toBe(
|
|
280
|
+
`${BASE_URL}${SKILL_ARTIFACTS_PATH_PREFIX}${UPLOADS_SEGMENT}${minted.ref}`,
|
|
281
|
+
);
|
|
282
|
+
// The registry's slot TTL governs, not the domain's ask.
|
|
283
|
+
expect(minted.ttlMs).toBe(60_000);
|
|
284
|
+
|
|
285
|
+
await slots.receive(stagingKeyOf(minted.ref)!, Readable.from(bytes));
|
|
286
|
+
expect(Buffer.from(await staging.consume(minted.ref))).toEqual(bytes);
|
|
287
|
+
await expect(staging.consume(minted.ref)).rejects.toThrow(SlotUnknownError);
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
it("a reserved-but-never-uploaded reference is unknown, exactly as on a bucket", async () => {
|
|
291
|
+
const minted = await staging.mint(4);
|
|
292
|
+
await expect(staging.consume(minted.ref)).rejects.toThrow(
|
|
293
|
+
"upload reference unknown or expired",
|
|
294
|
+
);
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
it("downloadUrl renders the lane's own GET route with the floor TTL", async () => {
|
|
298
|
+
const capability = await staging.downloadUrl("skills/abc.zip");
|
|
299
|
+
expect(capability).toEqual({
|
|
300
|
+
url: `${BASE_URL}${SKILL_ARTIFACTS_PATH_PREFIX}/skills/abc.zip`,
|
|
301
|
+
ttlMs: DOWNLOAD_URL_TTL_MS,
|
|
302
|
+
});
|
|
303
|
+
});
|
|
304
|
+
});
|
|
@@ -27,10 +27,34 @@ export const MAX_SKILL_MD_SIZE = 1 * 1024 * 1024;
|
|
|
27
27
|
/**
|
|
28
28
|
* Upload-slot lifetime — generous enough for a 100MB upload on a slow
|
|
29
29
|
* link, short enough that abandoned slots don't accumulate. Go:
|
|
30
|
-
* transfer.DefaultSlotTTL (15 minutes).
|
|
30
|
+
* transfer.DefaultSlotTTL (15 minutes). The same lifetime is asked of a
|
|
31
|
+
* bucket driver's presigned PUT, so the reference and the URL it names
|
|
32
|
+
* expire together whichever driver serves them.
|
|
31
33
|
*/
|
|
32
34
|
export const DEFAULT_SLOT_TTL_MS = 15 * 60 * 1000;
|
|
33
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Download capability lifetime — the floor of a minted download URL's
|
|
38
|
+
* validity, reported as ttl_seconds. A bucket driver signs its GET for
|
|
39
|
+
* exactly this long, the lifetime the cloud edition already grants
|
|
40
|
+
* execution-artifact downloads; the local lane's URL is a content-hash
|
|
41
|
+
* capability that never expires, so the floor holds there trivially. An
|
|
42
|
+
* hour outlives any mount the runner performs while staying short enough
|
|
43
|
+
* that a leaked URL is not a standing grant.
|
|
44
|
+
*/
|
|
45
|
+
export const DOWNLOAD_URL_TTL_MS = 60 * 60 * 1000;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Where staged uploads live in the blob store, under the skill store's own
|
|
49
|
+
* prefix: `skills/staging/<hex>.zip`. The retired cloud service staged
|
|
50
|
+
* under this exact prefix, so the bucket lifecycle rule that sweeps
|
|
51
|
+
* abandoned uploads is written for it, and on the local driver the boot
|
|
52
|
+
* wipe targets the same directory. Every staged object is a random
|
|
53
|
+
* 128-bit key, so its exposure through the download lane is the same
|
|
54
|
+
* capability the upload reference already grants.
|
|
55
|
+
*/
|
|
56
|
+
export const STAGING_KEY_PREFIX = "skills/staging/";
|
|
57
|
+
|
|
34
58
|
/**
|
|
35
59
|
* Random capability-token size: 16 bytes = 128 bits of entropy, matching
|
|
36
60
|
* the unguessability of the content-hash download keys. Go: refByteLen.
|
|
@@ -53,8 +77,9 @@ export const UPLOADS_SEGMENT = "/uploads/";
|
|
|
53
77
|
|
|
54
78
|
/**
|
|
55
79
|
* Downloads are restricted to the skill artifact store's own keys
|
|
56
|
-
* ("skills/<hash>.zip"
|
|
57
|
-
*
|
|
80
|
+
* ("skills/<hash>.zip", and the staged uploads under STAGING_KEY_PREFIX);
|
|
81
|
+
* everything else under the storage root — should the two ever share one
|
|
82
|
+
* — stays unreachable from this lane.
|
|
58
83
|
*/
|
|
59
84
|
export const DOWNLOAD_KEY_PREFIX = "skills/";
|
|
60
85
|
|
|
@@ -13,10 +13,12 @@
|
|
|
13
13
|
* optional deps; every absent-surface answer is a deliberate arm, not an
|
|
14
14
|
* accident.
|
|
15
15
|
*
|
|
16
|
-
* Versus Stigmer Cloud, OSS excludes the
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* Versus Stigmer Cloud, OSS excludes the TransformResponse/SendResponse
|
|
17
|
+
* steps (no response transformation). The transfer lane's capability
|
|
18
|
+
* URLs come from the blob driver through the staging port
|
|
19
|
+
* (transfer/staging.ts): the local driver's point at this server's own
|
|
20
|
+
* HTTP lane, a bucket driver's straight at the bucket — same
|
|
21
|
+
* capability-URL trust model, same client semantics, one code path.
|
|
20
22
|
*
|
|
21
23
|
* Proven by skill.conformance.test.ts (CONFORMANCE_TARGET=local) and
|
|
22
24
|
* the co-located __tests__/ suites.
|
|
@@ -119,8 +121,11 @@ import {
|
|
|
119
121
|
import { skillSearchExtractor } from "./search-extractor.js";
|
|
120
122
|
import { ArtifactNotFoundError } from "./storage/artifact-storage.js";
|
|
121
123
|
import type { SkillArtifactStorage } from "./storage/artifact-storage.js";
|
|
122
|
-
import {
|
|
123
|
-
|
|
124
|
+
import type {
|
|
125
|
+
ArchiveStaging,
|
|
126
|
+
DownloadCapability,
|
|
127
|
+
StagedUpload,
|
|
128
|
+
} from "./transfer/staging.js";
|
|
124
129
|
import {
|
|
125
130
|
LIST_VERSIONS_RESPONSE_KEY,
|
|
126
131
|
LIST_VERSIONS_SKILL_ID_KEY,
|
|
@@ -136,13 +141,6 @@ import {
|
|
|
136
141
|
*/
|
|
137
142
|
const EXECUTION_ARTIFACT_DOWNLOAD_TIMEOUT_MS = 60_000;
|
|
138
143
|
|
|
139
|
-
/** The transfer lane's controller-side pair (Go SetTransferLane). */
|
|
140
|
-
export interface SkillTransferLaneDeps {
|
|
141
|
-
readonly slots: UploadSlots;
|
|
142
|
-
/** Externally-reachable base minted into upload and download URLs. */
|
|
143
|
-
readonly baseUrl: string;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
144
|
export interface SkillControllerDeps {
|
|
147
145
|
readonly store: Store;
|
|
148
146
|
readonly logger: Logger;
|
|
@@ -158,11 +156,12 @@ export interface SkillControllerDeps {
|
|
|
158
156
|
*/
|
|
159
157
|
readonly executionArtifactStorage?: ArtifactStorage;
|
|
160
158
|
/**
|
|
161
|
-
* The
|
|
162
|
-
* createArtifactUploadUrl and
|
|
163
|
-
* FailedPrecondition and push accepts
|
|
159
|
+
* The transfer lane's staging port (Go SetTransferLane) over the skill
|
|
160
|
+
* blob driver. Optional — absent, createArtifactUploadUrl and
|
|
161
|
+
* getArtifactDownloadUrl answer FailedPrecondition and push accepts
|
|
162
|
+
* inline bytes only.
|
|
164
163
|
*/
|
|
165
|
-
readonly
|
|
164
|
+
readonly staging?: ArchiveStaging;
|
|
166
165
|
}
|
|
167
166
|
|
|
168
167
|
/** Registers both skill services on the router (routes stage). */
|
|
@@ -222,7 +221,7 @@ async function push(
|
|
|
222
221
|
await newPipeline<typeof PushSkillRequestSchema>("skill-push", deps.logger)
|
|
223
222
|
.addStep(newAuthorizeStep(method, deps.authorizer))
|
|
224
223
|
.addStep(newValidateProtoStep())
|
|
225
|
-
.addStep(newResolveArtifactSourceStep(deps.
|
|
224
|
+
.addStep(newResolveArtifactSourceStep(deps.staging))
|
|
226
225
|
.addStep(newBuildInitialSkillStep())
|
|
227
226
|
.addStep(newExtractAndHashArtifactStep())
|
|
228
227
|
.addStep(newResolveSlugForPushStep())
|
|
@@ -266,19 +265,20 @@ async function push(
|
|
|
266
265
|
}
|
|
267
266
|
|
|
268
267
|
/**
|
|
269
|
-
* CreateArtifactUploadUrl — mints a short-lived, single-use
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
*
|
|
273
|
-
* surfacing as a
|
|
268
|
+
* CreateArtifactUploadUrl — mints a short-lived, single-use upload URL for
|
|
269
|
+
* an artifact exceeding the gRPC message cap (#675): the blob driver's own
|
|
270
|
+
* PUT surface, through the staging port. The size gate is the fail-loud
|
|
271
|
+
* half of the contract: an over-limit artifact is refused with the actual
|
|
272
|
+
* limit in the message BEFORE any bytes move, instead of surfacing as a
|
|
273
|
+
* transport error mid-upload.
|
|
274
274
|
*/
|
|
275
275
|
async function createArtifactUploadUrl(
|
|
276
276
|
deps: SkillControllerDeps,
|
|
277
277
|
req: CreateSkillArtifactUploadUrlRequest,
|
|
278
278
|
ctx: HandlerContext,
|
|
279
279
|
): Promise<SkillArtifactUploadUrl> {
|
|
280
|
-
const
|
|
281
|
-
if (
|
|
280
|
+
const staging = deps.staging;
|
|
281
|
+
if (staging === undefined) {
|
|
282
282
|
throw failedPreconditionError(TRANSFER_LANE_NOT_CONFIGURED);
|
|
283
283
|
}
|
|
284
284
|
|
|
@@ -307,15 +307,15 @@ async function createArtifactUploadUrl(
|
|
|
307
307
|
);
|
|
308
308
|
}
|
|
309
309
|
|
|
310
|
-
let minted:
|
|
310
|
+
let minted: StagedUpload;
|
|
311
311
|
try {
|
|
312
|
-
minted =
|
|
312
|
+
minted = await staging.mint(Number(req.sizeBytes));
|
|
313
313
|
} catch (error) {
|
|
314
314
|
throw internalError(error, "failed to mint upload reference");
|
|
315
315
|
}
|
|
316
316
|
|
|
317
317
|
return create(SkillArtifactUploadUrlSchema, {
|
|
318
|
-
url:
|
|
318
|
+
url: minted.url,
|
|
319
319
|
artifactUploadRef: minted.ref,
|
|
320
320
|
ttlSeconds: Math.trunc(minted.ttlMs / 1000),
|
|
321
321
|
});
|
|
@@ -738,17 +738,19 @@ async function getArtifact(
|
|
|
738
738
|
|
|
739
739
|
/**
|
|
740
740
|
* GetArtifactDownloadUrl — the transfer-lane twin of GetArtifact (#675):
|
|
741
|
-
* stats (never loads) the artifact
|
|
742
|
-
*
|
|
743
|
-
*
|
|
741
|
+
* stats (never loads) the artifact, then asks the staging port for its
|
|
742
|
+
* capability URL. ttl_seconds is the floor of the URL's validity: a bucket
|
|
743
|
+
* driver signs its GET for exactly that long, while the local lane's URL
|
|
744
|
+
* embeds the same content-hash capability the storage key does and never
|
|
745
|
+
* expires, so the floor holds there trivially.
|
|
744
746
|
*/
|
|
745
747
|
async function getArtifactDownloadUrl(
|
|
746
748
|
deps: SkillControllerDeps,
|
|
747
749
|
req: GetArtifactRequest,
|
|
748
750
|
ctx: HandlerContext,
|
|
749
751
|
): Promise<SkillArtifactDownloadUrl> {
|
|
750
|
-
const
|
|
751
|
-
if (
|
|
752
|
+
const staging = deps.staging;
|
|
753
|
+
if (staging === undefined) {
|
|
752
754
|
throw failedPreconditionError(TRANSFER_LANE_NOT_CONFIGURED);
|
|
753
755
|
}
|
|
754
756
|
|
|
@@ -784,9 +786,16 @@ async function getArtifactDownloadUrl(
|
|
|
784
786
|
throw internalError(error, "failed to stat skill artifact");
|
|
785
787
|
}
|
|
786
788
|
|
|
789
|
+
let capability: DownloadCapability;
|
|
790
|
+
try {
|
|
791
|
+
capability = await staging.downloadUrl(req.artifactStorageKey);
|
|
792
|
+
} catch (error) {
|
|
793
|
+
throw internalError(error, "failed to mint skill artifact download URL");
|
|
794
|
+
}
|
|
795
|
+
|
|
787
796
|
return create(SkillArtifactDownloadUrlSchema, {
|
|
788
|
-
url:
|
|
789
|
-
ttlSeconds:
|
|
797
|
+
url: capability.url,
|
|
798
|
+
ttlSeconds: Math.trunc(capability.ttlMs / 1000),
|
|
790
799
|
sizeBytes: BigInt(size),
|
|
791
800
|
});
|
|
792
801
|
}
|
package/src/domain/skill/push.ts
CHANGED
|
@@ -81,7 +81,7 @@ import { skillSearchExtractor } from "./search-extractor.js";
|
|
|
81
81
|
import { extractSkillMd } from "./storage/zip-gate.js";
|
|
82
82
|
import type { ExtractSkillMdResult } from "./storage/zip-gate.js";
|
|
83
83
|
import type { SkillArtifactStorage } from "./storage/artifact-storage.js";
|
|
84
|
-
import type {
|
|
84
|
+
import type { ArchiveStaging } from "./transfer/staging.js";
|
|
85
85
|
|
|
86
86
|
type PushDesc = typeof PushSkillRequestSchema;
|
|
87
87
|
|
|
@@ -99,9 +99,9 @@ export const SHOULD_CREATE_SKILL_KEY = "shouldCreateSkill";
|
|
|
99
99
|
* pipeline/steps/resolve-artifact-source.ts for the semantics).
|
|
100
100
|
*/
|
|
101
101
|
export function newResolveArtifactSourceStep(
|
|
102
|
-
|
|
102
|
+
staging: ArchiveStaging | undefined,
|
|
103
103
|
): PipelineStep<PushDesc> {
|
|
104
|
-
return newSharedResolveArtifactSourceStep<PushDesc>(
|
|
104
|
+
return newSharedResolveArtifactSourceStep<PushDesc>(staging, {
|
|
105
105
|
source: (req) => ({
|
|
106
106
|
artifact: req.artifact,
|
|
107
107
|
artifactUploadRef: req.artifactUploadRef,
|
|
@@ -1,24 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Skill artifact transfer lane — ports pkg/domain/skill/transfer/handler.go:
|
|
3
3
|
* the HTTP upload/download surface that carries skill artifact bytes
|
|
4
|
-
* OUTSIDE the gRPC control plane (#675)
|
|
5
|
-
* 10MB while the skill layer permits
|
|
6
|
-
* physically cannot carry every valid
|
|
4
|
+
* OUTSIDE the gRPC control plane (#675) when the blob driver is the local
|
|
5
|
+
* one. The gRPC server caps messages at 10MB while the skill layer permits
|
|
6
|
+
* 100MB artifacts, so inline bytes physically cannot carry every valid
|
|
7
|
+
* skill; a bucket driver signs URLs straight to its bucket and this lane
|
|
8
|
+
* carries nothing for it.
|
|
7
9
|
*
|
|
8
10
|
* PUT {prefix}/uploads/{ref} — stage artifact bytes (capability: ref)
|
|
9
11
|
* GET {prefix}/{storage_key} — serve artifact bytes (capability: key)
|
|
10
12
|
*
|
|
11
13
|
* Neither route carries bearer auth by design: the URL is the credential,
|
|
12
|
-
* mirroring
|
|
14
|
+
* mirroring a bucket's pre-signed URLs. Minting an upload URL requires the
|
|
13
15
|
* same gRPC authorization as push; download keys are unguessable content
|
|
14
16
|
* hashes handed out by authorized skill reads.
|
|
15
17
|
*
|
|
16
18
|
* The lane slots into the transport's existing skillTransferLane seam
|
|
17
19
|
* (transport/server.ts lane 3); URL renderers live next to the route
|
|
18
20
|
* dispatch so the URL shape and its handler cannot drift apart (Go's
|
|
19
|
-
* stated invariant)
|
|
20
|
-
*
|
|
21
|
-
*
|
|
21
|
+
* stated invariant): uploadUrl is what the local driver's staged-upload
|
|
22
|
+
* lane mints, and transferServeUrl is the serve URL the composition hands
|
|
23
|
+
* the local skill driver so its signed GETs land on the download route
|
|
24
|
+
* below. Proven by __tests__/skill.test.ts's transfer-lane block (both
|
|
25
|
+
* protocol stacks) and the conformance suite's transfer-lane tests.
|
|
22
26
|
*/
|
|
23
27
|
import type { Logger } from "../../../boot/logger.js";
|
|
24
28
|
import { SKILL_ARTIFACTS_PATH_PREFIX } from "../../../transport/constants.js";
|
|
@@ -32,6 +36,7 @@ import {
|
|
|
32
36
|
SlotUnknownError,
|
|
33
37
|
} from "./slots.js";
|
|
34
38
|
import type { UploadSlots } from "./slots.js";
|
|
39
|
+
import { stagingKeyOf } from "./staging.js";
|
|
35
40
|
|
|
36
41
|
/**
|
|
37
42
|
* Renders the capability URL for a minted reference against the lane's
|
|
@@ -41,9 +46,13 @@ export function uploadUrl(baseUrl: string, ref: string): string {
|
|
|
41
46
|
return `${trimTrailingSlash(baseUrl)}${SKILL_ARTIFACTS_PATH_PREFIX}${UPLOADS_SEGMENT}${ref}`;
|
|
42
47
|
}
|
|
43
48
|
|
|
44
|
-
/**
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
/**
|
|
50
|
+
* The lane's download origin: the local skill driver's serve URL, so that
|
|
51
|
+
* `${serveUrl}/${storageKey}` (LocalArtifactStorage.getSignedUrl's shape)
|
|
52
|
+
* is exactly the GET route this handler dispatches.
|
|
53
|
+
*/
|
|
54
|
+
export function transferServeUrl(baseUrl: string): string {
|
|
55
|
+
return `${trimTrailingSlash(baseUrl)}${SKILL_ARTIFACTS_PATH_PREFIX}`;
|
|
47
56
|
}
|
|
48
57
|
|
|
49
58
|
function trimTrailingSlash(url: string): string {
|
|
@@ -108,16 +117,25 @@ async function handleUpload(
|
|
|
108
117
|
return;
|
|
109
118
|
}
|
|
110
119
|
|
|
120
|
+
// Deliberately one shape for "malformed", "never existed" and "expired":
|
|
121
|
+
// distinguishing them would let an unauthorized caller probe which
|
|
122
|
+
// tokens were once valid.
|
|
123
|
+
const unknown = () =>
|
|
124
|
+
writeText(response, 404, "upload reference unknown or expired — request a new upload URL");
|
|
125
|
+
|
|
126
|
+
const key = stagingKeyOf(ref);
|
|
127
|
+
if (key === undefined) {
|
|
128
|
+
unknown();
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
|
|
111
132
|
try {
|
|
112
|
-
await slots.receive(
|
|
133
|
+
await slots.receive(key, request);
|
|
113
134
|
response.statusCode = 204;
|
|
114
135
|
response.end();
|
|
115
136
|
} catch (error) {
|
|
116
137
|
if (error instanceof SlotUnknownError) {
|
|
117
|
-
|
|
118
|
-
// distinguishing them would let an unauthorized caller probe which
|
|
119
|
-
// tokens were once valid.
|
|
120
|
-
writeText(response, 404, "upload reference unknown or expired — request a new upload URL");
|
|
138
|
+
unknown();
|
|
121
139
|
} else if (error instanceof SlotConsumedError) {
|
|
122
140
|
writeText(response, 409, "upload reference already used — request a new upload URL");
|
|
123
141
|
} else if (error instanceof SizeMismatchError) {
|