@stigmer/server 3.22.1 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifactstorage/artifact-storage.d.ts +29 -31
- package/dist/artifactstorage/artifact-storage.d.ts.map +1 -1
- package/dist/artifactstorage/artifact-storage.js +5 -7
- package/dist/artifactstorage/artifact-storage.js.map +1 -1
- package/dist/artifactstorage/r2-storage.d.ts +12 -7
- package/dist/artifactstorage/r2-storage.d.ts.map +1 -1
- package/dist/artifactstorage/r2-storage.js +6 -14
- package/dist/artifactstorage/r2-storage.js.map +1 -1
- package/dist/boot/compose.d.ts.map +1 -1
- package/dist/boot/compose.js +49 -24
- package/dist/boot/compose.js.map +1 -1
- package/dist/boot/config.d.ts +4 -3
- package/dist/boot/config.d.ts.map +1 -1
- package/dist/boot/config.js.map +1 -1
- package/dist/boot/skill-transfer-origin.d.ts +52 -0
- package/dist/boot/skill-transfer-origin.d.ts.map +1 -0
- package/dist/boot/skill-transfer-origin.js +54 -0
- package/dist/boot/skill-transfer-origin.js.map +1 -0
- package/dist/domain/plugin/controller.d.ts +3 -7
- package/dist/domain/plugin/controller.d.ts.map +1 -1
- package/dist/domain/plugin/controller.js +8 -8
- package/dist/domain/plugin/controller.js.map +1 -1
- package/dist/domain/skill/constants.d.ts +26 -3
- package/dist/domain/skill/constants.d.ts.map +1 -1
- package/dist/domain/skill/constants.js +26 -3
- package/dist/domain/skill/constants.js.map +1 -1
- package/dist/domain/skill/controller.d.ts +6 -11
- package/dist/domain/skill/controller.d.ts.map +1 -1
- package/dist/domain/skill/controller.js +33 -22
- package/dist/domain/skill/controller.js.map +1 -1
- package/dist/domain/skill/push.d.ts +2 -2
- package/dist/domain/skill/push.d.ts.map +1 -1
- package/dist/domain/skill/push.js +2 -2
- package/dist/domain/skill/push.js.map +1 -1
- package/dist/domain/skill/transfer/handler.d.ts +17 -9
- package/dist/domain/skill/transfer/handler.d.ts.map +1 -1
- package/dist/domain/skill/transfer/handler.js +19 -8
- package/dist/domain/skill/transfer/handler.js.map +1 -1
- package/dist/domain/skill/transfer/slots.d.ts +26 -41
- package/dist/domain/skill/transfer/slots.d.ts.map +1 -1
- package/dist/domain/skill/transfer/slots.js +80 -87
- package/dist/domain/skill/transfer/slots.js.map +1 -1
- package/dist/domain/skill/transfer/staging.d.ts +48 -0
- package/dist/domain/skill/transfer/staging.d.ts.map +1 -0
- package/dist/domain/skill/transfer/staging.js +127 -0
- package/dist/domain/skill/transfer/staging.js.map +1 -0
- package/dist/pipeline/steps/resolve-artifact-source.d.ts +11 -8
- package/dist/pipeline/steps/resolve-artifact-source.d.ts.map +1 -1
- package/dist/pipeline/steps/resolve-artifact-source.js +8 -4
- package/dist/pipeline/steps/resolve-artifact-source.js.map +1 -1
- package/package.json +6 -6
- package/src/artifactstorage/__tests__/artifact-storage.test.ts +36 -33
- package/src/artifactstorage/__tests__/r2-storage.test.ts +11 -13
- package/src/artifactstorage/artifact-storage.ts +37 -33
- package/src/artifactstorage/r2-storage.ts +6 -15
- package/src/boot/__tests__/skill-transfer-origin.test.ts +72 -0
- package/src/boot/compose.ts +62 -25
- package/src/boot/config.ts +4 -3
- package/src/boot/skill-transfer-origin.ts +76 -0
- package/src/domain/plugin/controller.ts +15 -17
- package/src/domain/skill/__tests__/skill.test.ts +5 -3
- package/src/domain/skill/__tests__/slots.test.ts +113 -60
- package/src/domain/skill/__tests__/staging.test.ts +304 -0
- package/src/domain/skill/constants.ts +28 -3
- package/src/domain/skill/controller.ts +44 -35
- package/src/domain/skill/push.ts +3 -3
- package/src/domain/skill/transfer/handler.ts +33 -15
- package/src/domain/skill/transfer/slots.ts +89 -100
- package/src/domain/skill/transfer/staging.ts +191 -0
- package/src/pipeline/steps/resolve-artifact-source.ts +20 -13
- package/src/transport/registry/__tests__/lanes.test.ts +1 -1
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { SlotUnknownError } from "../../domain/skill/transfer/slots.js";
|
|
1
2
|
import { failedPreconditionError, invalidArgumentError } from "../errors.js";
|
|
2
3
|
/** Where the resolved bytes ride for every downstream step. */
|
|
3
4
|
export const ARTIFACT_BYTES_KEY = "pushArtifactBytes";
|
|
4
|
-
export function newResolveArtifactSourceStep(
|
|
5
|
+
export function newResolveArtifactSourceStep(staging, binding) {
|
|
5
6
|
return {
|
|
6
7
|
name: "ResolveArtifactSource",
|
|
7
8
|
async execute(ctx) {
|
|
@@ -10,15 +11,18 @@ export function newResolveArtifactSourceStep(slots, binding) {
|
|
|
10
11
|
ctx.set(ARTIFACT_BYTES_KEY, artifact);
|
|
11
12
|
return;
|
|
12
13
|
}
|
|
13
|
-
if (
|
|
14
|
+
if (staging === undefined) {
|
|
14
15
|
throw failedPreconditionError(binding.laneNotConfigured);
|
|
15
16
|
}
|
|
16
17
|
let data;
|
|
17
18
|
try {
|
|
18
|
-
data = await
|
|
19
|
+
data = await staging.consume(artifactUploadRef);
|
|
19
20
|
}
|
|
20
21
|
catch (error) {
|
|
21
|
-
|
|
22
|
+
if (error instanceof SlotUnknownError) {
|
|
23
|
+
throw invalidArgumentError(`artifact_upload_ref not usable: ${error.message} — request a new upload URL via createArtifactUploadUrl`);
|
|
24
|
+
}
|
|
25
|
+
throw error;
|
|
22
26
|
}
|
|
23
27
|
ctx.set(ARTIFACT_BYTES_KEY, data);
|
|
24
28
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resolve-artifact-source.js","sourceRoot":"","sources":["../../../src/pipeline/steps/resolve-artifact-source.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"resolve-artifact-source.js","sourceRoot":"","sources":["../../../src/pipeline/steps/resolve-artifact-source.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AAExE,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAI7E,+DAA+D;AAC/D,MAAM,CAAC,MAAM,kBAAkB,GAAG,mBAAmB,CAAC;AAYtD,MAAM,UAAU,4BAA4B,CAC1C,OAAmC,EACnC,OAAoC;IAEpC,OAAO;QACL,IAAI,EAAE,uBAAuB;QAC7B,KAAK,CAAC,OAAO,CAAC,GAAyB;YACrC,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClE,IAAI,iBAAiB,KAAK,EAAE,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC;gBACtC,OAAO;YACT,CAAC;YACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,uBAAuB,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YAC3D,CAAC;YACD,IAAI,IAAgB,CAAC;YACrB,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YAClD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;oBACtC,MAAM,oBAAoB,CACxB,mCAAmC,KAAK,CAAC,OAAO,yDAAyD,CAC1G,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;QACpC,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stigmer/server",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.23.0",
|
|
4
4
|
"description": "The Stigmer OSS control plane as a composable library: composeServer, the extension registry and the driver seams a composition builds on. The self-contained deployable of the same server is @stigmer/server-slim.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -62,11 +62,11 @@
|
|
|
62
62
|
"@connectrpc/connect-node": "^2.0.0",
|
|
63
63
|
"@kubernetes/client-node": "2.0.0",
|
|
64
64
|
"@opentelemetry/api": "^1.9.0",
|
|
65
|
-
"@stigmer/outbound": "3.
|
|
66
|
-
"@stigmer/plugin-package": "3.
|
|
67
|
-
"@stigmer/protos": "3.
|
|
68
|
-
"@stigmer/temporal-codecs": "3.
|
|
69
|
-
"@stigmer/zip-structure": "3.
|
|
65
|
+
"@stigmer/outbound": "3.23.0",
|
|
66
|
+
"@stigmer/plugin-package": "3.23.0",
|
|
67
|
+
"@stigmer/protos": "3.23.0",
|
|
68
|
+
"@stigmer/temporal-codecs": "3.23.0",
|
|
69
|
+
"@stigmer/zip-structure": "3.23.0",
|
|
70
70
|
"@temporalio/activity": "1.16.2",
|
|
71
71
|
"@temporalio/client": "1.16.2",
|
|
72
72
|
"@temporalio/common": "1.16.2",
|
|
@@ -14,9 +14,13 @@ import path from "node:path";
|
|
|
14
14
|
import { Readable } from "node:stream";
|
|
15
15
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
STAGING_KEY_PREFIX,
|
|
19
|
+
UPLOADS_SEGMENT,
|
|
20
|
+
} from "../../domain/skill/constants.js";
|
|
18
21
|
import { UploadSlots } from "../../domain/skill/transfer/slots.js";
|
|
19
22
|
import { uploadUrl } from "../../domain/skill/transfer/handler.js";
|
|
23
|
+
import { uploadRefOf } from "../../domain/skill/transfer/staging.js";
|
|
20
24
|
import { SKILL_ARTIFACTS_PATH_PREFIX } from "../../transport/constants.js";
|
|
21
25
|
import {
|
|
22
26
|
ArtifactStorageNotFoundError,
|
|
@@ -184,33 +188,32 @@ describe("LocalArtifactStorage", () => {
|
|
|
184
188
|
});
|
|
185
189
|
|
|
186
190
|
it("presignPut without a wired lane is the explicit not-configured throw, never a silent no-op", async () => {
|
|
187
|
-
await expect(
|
|
188
|
-
"
|
|
189
|
-
);
|
|
191
|
+
await expect(
|
|
192
|
+
storage.presignPut("skills/staging/deadbeef.zip", 16, 60_000),
|
|
193
|
+
).rejects.toThrow("local presigned uploads not configured");
|
|
190
194
|
});
|
|
191
195
|
});
|
|
192
196
|
});
|
|
193
197
|
|
|
194
198
|
// The local presigned-PUT arm over the REAL skill-transfer-lane slot
|
|
195
199
|
// mechanism (§6b, the Q1 ruling: one upload surface, no new lane) — the
|
|
196
|
-
// adapter below mirrors boot/compose.ts's wiring exactly.
|
|
200
|
+
// adapter below mirrors boot/compose.ts's wiring exactly. The caller names
|
|
201
|
+
// the key; the lane stages under it inside this driver's root.
|
|
197
202
|
describe("LocalArtifactStorage presignPut over the transfer-lane slots", () => {
|
|
198
203
|
const BASE_URL = "http://localhost:8080";
|
|
204
|
+
const KEY = `${STAGING_KEY_PREFIX}${"ab".repeat(16)}.zip`;
|
|
199
205
|
let root: string;
|
|
200
206
|
let slots: UploadSlots;
|
|
201
207
|
let driver: ArtifactStorage;
|
|
202
208
|
|
|
203
209
|
beforeEach(() => {
|
|
204
210
|
root = mkdtempSync(path.join(tmpdir(), "artifact-presign-"));
|
|
205
|
-
slots = new UploadSlots(
|
|
206
|
-
path.join(root, "skills-staging"),
|
|
207
|
-
60_000,
|
|
208
|
-
1024 * 1024,
|
|
209
|
-
);
|
|
211
|
+
slots = new UploadSlots(root, STAGING_KEY_PREFIX, 60_000, 1024 * 1024);
|
|
210
212
|
driver = new LocalArtifactStorage(root, "", {
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
213
|
+
reserve: (key, declaredSizeBytes) => {
|
|
214
|
+
const { ttlMs } = slots.reserve(key, declaredSizeBytes);
|
|
215
|
+
return { url: uploadUrl(BASE_URL, uploadRefOf(key)), ttlMs };
|
|
216
|
+
},
|
|
214
217
|
});
|
|
215
218
|
});
|
|
216
219
|
|
|
@@ -218,36 +221,36 @@ describe("LocalArtifactStorage presignPut over the transfer-lane slots", () => {
|
|
|
218
221
|
rmSync(root, { recursive: true, force: true });
|
|
219
222
|
});
|
|
220
223
|
|
|
221
|
-
it("
|
|
224
|
+
it("presignPut(key) → PUT (lane receive) → download(key) round-trips the bytes", async () => {
|
|
222
225
|
const bytes = Buffer.from("staged artifact bytes");
|
|
223
|
-
const upload = await driver.presignPut(bytes.length, 999_999_999);
|
|
226
|
+
const upload = await driver.presignPut(KEY, bytes.length, 999_999_999);
|
|
224
227
|
|
|
225
228
|
// The URL is the lane's wire shape and the ref is its last segment —
|
|
226
229
|
// exactly how the lane handler dispatches a PUT.
|
|
227
|
-
expect(
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
),
|
|
231
|
-
).toBe(true);
|
|
230
|
+
expect(upload.url).toBe(
|
|
231
|
+
`${BASE_URL}${SKILL_ARTIFACTS_PATH_PREFIX}${UPLOADS_SEGMENT}${uploadRefOf(KEY)}`,
|
|
232
|
+
);
|
|
232
233
|
// The lane's slot TTL governs, not the caller's larger ask.
|
|
233
234
|
expect(upload.ttlMs).toBe(60_000);
|
|
234
235
|
|
|
235
|
-
|
|
236
|
-
await slots.receive(ref, Readable.from(bytes));
|
|
236
|
+
await slots.receive(KEY, Readable.from(bytes));
|
|
237
237
|
|
|
238
|
-
expect(Buffer.from(await driver.download(
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
238
|
+
expect(Buffer.from(await driver.download(KEY))).toEqual(bytes);
|
|
239
|
+
expect(await driver.size(KEY)).toBe(bytes.length);
|
|
240
|
+
// The driver's own delete retires the staged bytes — the consume path.
|
|
241
|
+
await driver.delete(KEY);
|
|
242
|
+
await expect(driver.download(KEY)).rejects.toThrow(ArtifactStorageNotFoundError);
|
|
242
243
|
});
|
|
243
244
|
|
|
244
|
-
it("
|
|
245
|
-
|
|
246
|
-
expect(
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
245
|
+
it("an unreceived key is honest about absence", async () => {
|
|
246
|
+
await driver.presignPut(KEY, 4, 60_000);
|
|
247
|
+
await expect(driver.download(KEY)).rejects.toThrow(ArtifactStorageNotFoundError);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
it("refuses a key that would stage outside the driver root before the lane sees it", async () => {
|
|
251
|
+
await expect(
|
|
252
|
+
driver.presignPut("../outside.zip", 4, 60_000),
|
|
253
|
+
).rejects.toThrow("resolves outside the artifact storage root");
|
|
251
254
|
});
|
|
252
255
|
});
|
|
253
256
|
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* ships r2_storage.go untested; these pins are what the port adds:
|
|
5
5
|
* required-config copy, the 7-day presign clamp, the signed
|
|
6
6
|
* Content-Disposition, the not-found mapping, and the O5 widened surface
|
|
7
|
-
* (typed not-found on download/size, presigned-PUT under the
|
|
8
|
-
*
|
|
7
|
+
* (typed not-found on download/size, presigned-PUT under the caller's key
|
|
8
|
+
* with the signed Content-Length).
|
|
9
9
|
*/
|
|
10
10
|
import { describe, expect, it } from "vitest";
|
|
11
11
|
|
|
@@ -13,7 +13,6 @@ import { ArtifactStorageNotFoundError } from "../artifact-storage.js";
|
|
|
13
13
|
import {
|
|
14
14
|
R2ArtifactStorage,
|
|
15
15
|
R2_MAX_EXPIRATION_MS,
|
|
16
|
-
R2_STAGING_PREFIX,
|
|
17
16
|
isNotFoundError,
|
|
18
17
|
} from "../r2-storage.js";
|
|
19
18
|
|
|
@@ -64,16 +63,14 @@ describe("R2ArtifactStorage presigned URLs", () => {
|
|
|
64
63
|
});
|
|
65
64
|
|
|
66
65
|
describe("R2ArtifactStorage O5 widened surface", () => {
|
|
67
|
-
it("presignPut
|
|
66
|
+
it("presignPut signs a PUT under the caller's key, clamps the TTL, and signs the declared size", async () => {
|
|
68
67
|
const storage = new R2ArtifactStorage(VALID);
|
|
69
|
-
const
|
|
68
|
+
const key = "skills/staging/0123456789abcdef0123456789abcdef.zip";
|
|
69
|
+
const upload = await storage.presignPut(key, 1234, R2_MAX_EXPIRATION_MS * 5);
|
|
70
70
|
|
|
71
|
-
expect(upload.stagingKey.startsWith(R2_STAGING_PREFIX)).toBe(true);
|
|
72
71
|
expect(upload.ttlMs).toBe(R2_MAX_EXPIRATION_MS);
|
|
73
72
|
expect(
|
|
74
|
-
upload.url.startsWith(
|
|
75
|
-
`${VALID.endpoint}/${VALID.bucket}/${upload.stagingKey}?`,
|
|
76
|
-
),
|
|
73
|
+
upload.url.startsWith(`${VALID.endpoint}/${VALID.bucket}/${key}?`),
|
|
77
74
|
).toBe(true);
|
|
78
75
|
expect(upload.url).toContain("X-Amz-Expires=604800");
|
|
79
76
|
// Content-Length rides the signature: a body of a different size
|
|
@@ -81,11 +78,12 @@ describe("R2ArtifactStorage O5 widened surface", () => {
|
|
|
81
78
|
expect(upload.url.toLowerCase()).toContain("content-length");
|
|
82
79
|
});
|
|
83
80
|
|
|
84
|
-
it("
|
|
81
|
+
it("the key is the caller's verbatim — the driver invents none of its own", async () => {
|
|
85
82
|
const storage = new R2ArtifactStorage(VALID);
|
|
86
|
-
const a = await storage.presignPut(1, 60_000);
|
|
87
|
-
const b = await storage.presignPut(1, 60_000);
|
|
88
|
-
expect(a.
|
|
83
|
+
const a = await storage.presignPut("staging/a.zip", 1, 60_000);
|
|
84
|
+
const b = await storage.presignPut("staging/b.zip", 1, 60_000);
|
|
85
|
+
expect(a.url).toContain(`/${VALID.bucket}/staging/a.zip?`);
|
|
86
|
+
expect(b.url).toContain(`/${VALID.bucket}/staging/b.zip?`);
|
|
89
87
|
});
|
|
90
88
|
|
|
91
89
|
it("download and size map the SDK's not-found onto the typed class; real faults stay wrapped", async () => {
|
|
@@ -58,17 +58,15 @@ export class ArtifactStorageNotFoundError extends Error {
|
|
|
58
58
|
}
|
|
59
59
|
|
|
60
60
|
/**
|
|
61
|
-
* A staged upload minted by presignPut: the URL receives the bytes,
|
|
62
|
-
*
|
|
63
|
-
* short-lived by contract — the local lane sweeps them on TTL
|
|
64
|
-
* boot;
|
|
65
|
-
* prefix (the §6b driver-side sweep expectation).
|
|
61
|
+
* A staged upload minted by presignPut: the URL receives the bytes, which
|
|
62
|
+
* the caller reads back through download() under the key IT named. Staged
|
|
63
|
+
* blobs are short-lived by contract — the local lane sweeps them on TTL
|
|
64
|
+
* expiry and boot; a bucket deployment configures a lifecycle rule on the
|
|
65
|
+
* domain's staging prefix (the §6b driver-side sweep expectation).
|
|
66
66
|
*/
|
|
67
67
|
export interface PresignedUpload {
|
|
68
|
-
/** Accepts
|
|
68
|
+
/** Accepts an HTTP PUT of exactly the declared byte count. */
|
|
69
69
|
readonly url: string;
|
|
70
|
-
/** Driver key where the staged bytes land, readable via download(). */
|
|
71
|
-
readonly stagingKey: string;
|
|
72
70
|
/** The TTL actually granted, after per-driver clamping. */
|
|
73
71
|
readonly ttlMs: number;
|
|
74
72
|
}
|
|
@@ -98,18 +96,25 @@ export interface ArtifactStorage {
|
|
|
98
96
|
downloadFilename: string,
|
|
99
97
|
): Promise<string>;
|
|
100
98
|
/**
|
|
101
|
-
* A time-limited upload URL for a
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
99
|
+
* A time-limited upload URL for a blob of exactly declaredSizeBytes at
|
|
100
|
+
* the key the CALLER names (§6b). The key is the domain's: it chooses
|
|
101
|
+
* the staging prefix its sweep targets and the mapping from its wire
|
|
102
|
+
* reference, and reads the bytes back through download(key) — the
|
|
103
|
+
* driver never invents a key of its own. Per-driver semantics differ
|
|
104
|
+
* DELIBERATELY and are contract, not accident: the local backend rides
|
|
105
|
+
* the skill transfer lane's URL-as-credential slot mechanism
|
|
106
|
+
* (single-use, exact-size enforced by the lane, TTL clamped to the
|
|
107
|
+
* lane's slot TTL) and answers only on instances with a staged-upload
|
|
108
|
+
* lane wired — unwired instances throw the explicit not-configured
|
|
109
|
+
* error, never a silent no-op. The R2 backend presigns a PUT
|
|
110
|
+
* (repeatable within its TTL, size enforced by the signed
|
|
111
|
+
* Content-Length header, TTL clamped to the 7-day R2 maximum).
|
|
111
112
|
*/
|
|
112
|
-
presignPut(
|
|
113
|
+
presignPut(
|
|
114
|
+
key: string,
|
|
115
|
+
declaredSizeBytes: number,
|
|
116
|
+
ttlMs: number,
|
|
117
|
+
): Promise<PresignedUpload>;
|
|
113
118
|
/** Removes the artifact; missing is not an error. */
|
|
114
119
|
delete(key: string): Promise<void>;
|
|
115
120
|
/** Whether an artifact with the key exists. */
|
|
@@ -136,16 +141,16 @@ export type ArtifactStorageDriverFactory = () => ArtifactStorage;
|
|
|
136
141
|
* the shape it needs and stays ignorant of who provides it.
|
|
137
142
|
*/
|
|
138
143
|
export interface StagedUploadLane {
|
|
139
|
-
/** Reserves a single-use upload slot; returns its reference + TTL. */
|
|
140
|
-
mint(declaredSizeBytes: number): { ref: string; ttlMs: number };
|
|
141
|
-
/** The externally-reachable PUT URL for a minted reference. */
|
|
142
|
-
uploadUrl(ref: string): string;
|
|
143
144
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
145
|
+
* Reserves a single-use upload slot whose bytes land at `key` under this
|
|
146
|
+
* driver instance's root — the lane stages into the driver's own tree,
|
|
147
|
+
* so download(key) reads exactly what the lane received. Returns the
|
|
148
|
+
* externally-reachable PUT URL and the TTL the lane grants.
|
|
147
149
|
*/
|
|
148
|
-
|
|
150
|
+
reserve(
|
|
151
|
+
key: string,
|
|
152
|
+
declaredSizeBytes: number,
|
|
153
|
+
): { url: string; ttlMs: number };
|
|
149
154
|
}
|
|
150
155
|
|
|
151
156
|
export interface ArtifactStorageConfig {
|
|
@@ -259,6 +264,7 @@ export class LocalArtifactStorage implements ArtifactStorage {
|
|
|
259
264
|
}
|
|
260
265
|
|
|
261
266
|
async presignPut(
|
|
267
|
+
key: string,
|
|
262
268
|
declaredSizeBytes: number,
|
|
263
269
|
_ttlMs: number,
|
|
264
270
|
): Promise<PresignedUpload> {
|
|
@@ -267,15 +273,13 @@ export class LocalArtifactStorage implements ArtifactStorage {
|
|
|
267
273
|
"local presigned uploads not configured - no staged-upload lane is wired to this store instance",
|
|
268
274
|
);
|
|
269
275
|
}
|
|
276
|
+
// The key must be one this instance can read back, or the lane would
|
|
277
|
+
// stage bytes download(key) can never reach.
|
|
278
|
+
this.resolveWithinRoot(key);
|
|
270
279
|
// The lane's slot TTL governs, not the caller's ask — the slot
|
|
271
280
|
// registry sweeps on ITS clock, and a URL outliving its slot would be
|
|
272
281
|
// a credential for nothing.
|
|
273
|
-
|
|
274
|
-
return {
|
|
275
|
-
url: this.stagedUploadLane.uploadUrl(ref),
|
|
276
|
-
stagingKey: this.stagedUploadLane.stagedKey(ref),
|
|
277
|
-
ttlMs,
|
|
278
|
-
};
|
|
282
|
+
return this.stagedUploadLane.reserve(key, declaredSizeBytes);
|
|
279
283
|
}
|
|
280
284
|
|
|
281
285
|
async getSignedUrl(
|
|
@@ -9,8 +9,6 @@
|
|
|
9
9
|
* that have logic: config validation copy, the 7-day presign clamp, the
|
|
10
10
|
* signed Content-Disposition, and the not-found mapping.
|
|
11
11
|
*/
|
|
12
|
-
import { randomBytes } from "node:crypto";
|
|
13
|
-
|
|
14
12
|
import {
|
|
15
13
|
DeleteObjectCommand,
|
|
16
14
|
GetObjectCommand,
|
|
@@ -30,13 +28,6 @@ import {
|
|
|
30
28
|
/** Go: "R2 has a maximum expiration of 7 days" — presigns clamp here. */
|
|
31
29
|
export const R2_MAX_EXPIRATION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
32
30
|
|
|
33
|
-
/**
|
|
34
|
-
* Key prefix for presignPut staging blobs (§6b). Pinned as the surface a
|
|
35
|
-
* deployment's bucket lifecycle rule targets: the driver mints under it,
|
|
36
|
-
* operations sweeps it — changing it orphans staged blobs from the sweep.
|
|
37
|
-
*/
|
|
38
|
-
export const R2_STAGING_PREFIX = "staging/";
|
|
39
|
-
|
|
40
31
|
/**
|
|
41
32
|
* Go get_content/get_download_url wrap their storage calls in a 30s
|
|
42
33
|
* context timeout. The TS ArtifactStorage interface carries no signal, so
|
|
@@ -167,17 +158,17 @@ export class R2ArtifactStorage implements ArtifactStorage {
|
|
|
167
158
|
}
|
|
168
159
|
|
|
169
160
|
async presignPut(
|
|
161
|
+
key: string,
|
|
170
162
|
declaredSizeBytes: number,
|
|
171
163
|
ttlMs: number,
|
|
172
164
|
): Promise<PresignedUpload> {
|
|
173
165
|
const clampedMs = Math.min(ttlMs, R2_MAX_EXPIRATION_MS);
|
|
174
|
-
//
|
|
175
|
-
//
|
|
176
|
-
//
|
|
177
|
-
const stagingKey = `${R2_STAGING_PREFIX}${randomBytes(16).toString("hex")}`;
|
|
166
|
+
// The key is the caller's: the domain names its staging prefix (the
|
|
167
|
+
// surface a bucket lifecycle rule sweeps) and keeps the unguessable
|
|
168
|
+
// part of the URL-as-credential in the reference it hands the client.
|
|
178
169
|
const command = new PutObjectCommand({
|
|
179
170
|
Bucket: this.bucket,
|
|
180
|
-
Key:
|
|
171
|
+
Key: key,
|
|
181
172
|
// Signed into the URL: a PUT whose body disagrees with the declared
|
|
182
173
|
// size fails the signature check — R2's arm of the exact-size
|
|
183
174
|
// contract the local lane enforces by counting bytes.
|
|
@@ -187,7 +178,7 @@ export class R2ArtifactStorage implements ArtifactStorage {
|
|
|
187
178
|
const url = await this.presign(this.client, command, {
|
|
188
179
|
expiresIn: Math.floor(clampedMs / 1000),
|
|
189
180
|
});
|
|
190
|
-
return { url,
|
|
181
|
+
return { url, ttlMs: clampedMs };
|
|
191
182
|
} catch (error) {
|
|
192
183
|
throw new Error(`r2 presign failed: ${errorText(error)}`, { cause: error });
|
|
193
184
|
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pins the skill transfer origin verdict (boot/skill-transfer-origin.ts):
|
|
3
|
+
* loopback on the relay is fine for a single-operator install and a
|
|
4
|
+
* warning under the require-authentication posture; a public origin is
|
|
5
|
+
* reachable whatever the posture; a bucket driver makes the base URL
|
|
6
|
+
* unused; an unparseable base URL is not mistaken for loopback.
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from "vitest";
|
|
9
|
+
|
|
10
|
+
import { assessSkillTransferOrigin } from "../skill-transfer-origin.js";
|
|
11
|
+
|
|
12
|
+
describe("assessSkillTransferOrigin", () => {
|
|
13
|
+
it("the default loopback on a single-operator install is correct", () => {
|
|
14
|
+
expect(
|
|
15
|
+
assessSkillTransferOrigin({
|
|
16
|
+
baseUrl: "http://localhost:7234",
|
|
17
|
+
relaysThroughServer: true,
|
|
18
|
+
requireAuthentication: false,
|
|
19
|
+
}),
|
|
20
|
+
).toEqual({ kind: "loopback-local" });
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it.each([
|
|
24
|
+
"http://localhost:7234",
|
|
25
|
+
"http://127.0.0.1:7234",
|
|
26
|
+
"http://[::1]:7234",
|
|
27
|
+
"http://0.0.0.0:7234",
|
|
28
|
+
])(
|
|
29
|
+
"loopback (%s) under the require-authentication posture is the warning, naming the URL",
|
|
30
|
+
(baseUrl) => {
|
|
31
|
+
expect(
|
|
32
|
+
assessSkillTransferOrigin({
|
|
33
|
+
baseUrl,
|
|
34
|
+
relaysThroughServer: true,
|
|
35
|
+
requireAuthentication: true,
|
|
36
|
+
}),
|
|
37
|
+
).toEqual({ kind: "loopback-hosted", baseUrl });
|
|
38
|
+
},
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
it("a public origin is reachable under either posture", () => {
|
|
42
|
+
for (const requireAuthentication of [true, false]) {
|
|
43
|
+
expect(
|
|
44
|
+
assessSkillTransferOrigin({
|
|
45
|
+
baseUrl: "https://api.example.com",
|
|
46
|
+
relaysThroughServer: true,
|
|
47
|
+
requireAuthentication,
|
|
48
|
+
}),
|
|
49
|
+
).toEqual({ kind: "reachable" });
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("a driver that signs its own URLs makes the base URL unused, loopback or not", () => {
|
|
54
|
+
expect(
|
|
55
|
+
assessSkillTransferOrigin({
|
|
56
|
+
baseUrl: "http://localhost:7234",
|
|
57
|
+
relaysThroughServer: false,
|
|
58
|
+
requireAuthentication: true,
|
|
59
|
+
}),
|
|
60
|
+
).toEqual({ kind: "unused" });
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("an unparseable base URL is not loopback; its failure is loud on the first mint instead", () => {
|
|
64
|
+
expect(
|
|
65
|
+
assessSkillTransferOrigin({
|
|
66
|
+
baseUrl: "not a url",
|
|
67
|
+
relaysThroughServer: true,
|
|
68
|
+
requireAuthentication: true,
|
|
69
|
+
}),
|
|
70
|
+
).toEqual({ kind: "reachable" });
|
|
71
|
+
});
|
|
72
|
+
});
|
package/src/boot/compose.ts
CHANGED
|
@@ -119,14 +119,20 @@ import { registerSkillServices } from "../domain/skill/controller.js";
|
|
|
119
119
|
import {
|
|
120
120
|
DEFAULT_SLOT_TTL_MS,
|
|
121
121
|
MAX_ZIP_SIZE,
|
|
122
|
+
STAGING_KEY_PREFIX,
|
|
122
123
|
} from "../domain/skill/constants.js";
|
|
123
124
|
import { newSkillArtifactStorage } from "../domain/skill/storage/artifact-storage.js";
|
|
124
125
|
import { newContentAddressedArchiveStore } from "../archive/content-store.js";
|
|
125
126
|
import {
|
|
126
127
|
newSkillTransferLane,
|
|
128
|
+
transferServeUrl,
|
|
127
129
|
uploadUrl as skillUploadUrl,
|
|
128
130
|
} from "../domain/skill/transfer/handler.js";
|
|
129
131
|
import { UploadSlots } from "../domain/skill/transfer/slots.js";
|
|
132
|
+
import {
|
|
133
|
+
newArchiveStaging,
|
|
134
|
+
uploadRefOf,
|
|
135
|
+
} from "../domain/skill/transfer/staging.js";
|
|
130
136
|
import {
|
|
131
137
|
DEFAULT_WRITE_VERSION,
|
|
132
138
|
ENCRYPTION_KEY_ENV_VAR,
|
|
@@ -166,6 +172,7 @@ import { SqliteStore } from "../store/sqlite/store.js";
|
|
|
166
172
|
import type { Store } from "../store/interface.js";
|
|
167
173
|
import { resolveExtensions } from "../extensions/registry.js";
|
|
168
174
|
import { resolveOAuthRedirectUri } from "./oauth-redirect-uri.js";
|
|
175
|
+
import { assessSkillTransferOrigin } from "./skill-transfer-origin.js";
|
|
169
176
|
import type { ServerExtension } from "../extensions/registry.js";
|
|
170
177
|
import { registerWorkflowServices } from "../domain/workflow/controller.js";
|
|
171
178
|
import {
|
|
@@ -926,7 +933,7 @@ export async function composeServer(
|
|
|
926
933
|
: undefined;
|
|
927
934
|
// The skill artifact store + transfer lane (Go server.go:318-340). The
|
|
928
935
|
// store is content-addressed and never-GC at {storagePath}/skills/;
|
|
929
|
-
// the slots registry wipes {storagePath}/skills
|
|
936
|
+
// the slots registry wipes {storagePath}/skills/staging at boot (orphans
|
|
930
937
|
// from a dead process are useless — the registry died with it). On OSS
|
|
931
938
|
// the lane is ALWAYS configured (Go wires it unconditionally too): the
|
|
932
939
|
// FailedPrecondition lane-absent arms exist for construction-order
|
|
@@ -938,16 +945,25 @@ export async function composeServer(
|
|
|
938
945
|
// deployment changes nothing for skill artifacts until it opts in
|
|
939
946
|
// explicitly, and the Go-written skills/ directory keeps serving in
|
|
940
947
|
// place on the default arm.
|
|
948
|
+
//
|
|
949
|
+
// Capability URLs come from the driver through the staging port
|
|
950
|
+
// (domain/skill/transfer/staging.ts, stigmer#1219): the local driver's
|
|
951
|
+
// presignPut reserves a slot here and renders the lane URL below, a
|
|
952
|
+
// bucket driver's signs a PUT straight to the bucket, and the
|
|
953
|
+
// controllers cannot tell which. The slots are constructed on every
|
|
954
|
+
// arm so the HTTP lane always has a registry to consult; on a bucket
|
|
955
|
+
// arm nothing is ever reserved in it.
|
|
941
956
|
const skillUploadSlots = new UploadSlots(
|
|
942
|
-
|
|
957
|
+
config.storagePath,
|
|
958
|
+
STAGING_KEY_PREFIX,
|
|
943
959
|
DEFAULT_SLOT_TTL_MS,
|
|
944
960
|
MAX_ZIP_SIZE,
|
|
945
961
|
);
|
|
962
|
+
const skillType =
|
|
963
|
+
config.skillArtifactStorageType === ""
|
|
964
|
+
? "local"
|
|
965
|
+
: config.skillArtifactStorageType;
|
|
946
966
|
const skillStorageDriver = ((): ArtifactStorage => {
|
|
947
|
-
const skillType =
|
|
948
|
-
config.skillArtifactStorageType === ""
|
|
949
|
-
? "local"
|
|
950
|
-
: config.skillArtifactStorageType;
|
|
951
967
|
if (skillType === "local") {
|
|
952
968
|
// Go layout invariant: {storagePath}/skills exists from boot with
|
|
953
969
|
// 0755 (the retired LocalFileStorage constructor's mkdir; the
|
|
@@ -958,16 +974,23 @@ export async function composeServer(
|
|
|
958
974
|
});
|
|
959
975
|
// The local presigned-PUT arm rides the skill transfer lane's slot
|
|
960
976
|
// mechanism (§6b Q1 ruling: one upload surface, no new lane). The
|
|
961
|
-
// slots stage inside THIS driver's root
|
|
962
|
-
// reads back
|
|
963
|
-
//
|
|
964
|
-
//
|
|
965
|
-
return new LocalArtifactStorage(
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
977
|
+
// slots stage inside THIS driver's root under the domain's staging
|
|
978
|
+
// key, so download(key) reads back what the lane received; the
|
|
979
|
+
// serve URL is the lane's download route, so getSignedUrl renders
|
|
980
|
+
// the very URL the lane dispatches.
|
|
981
|
+
return new LocalArtifactStorage(
|
|
982
|
+
config.storagePath,
|
|
983
|
+
transferServeUrl(config.skillTransferBaseUrl),
|
|
984
|
+
{
|
|
985
|
+
reserve: (key, declaredSizeBytes) => {
|
|
986
|
+
const { ttlMs } = skillUploadSlots.reserve(key, declaredSizeBytes);
|
|
987
|
+
return {
|
|
988
|
+
url: skillUploadUrl(config.skillTransferBaseUrl, uploadRefOf(key)),
|
|
989
|
+
ttlMs,
|
|
990
|
+
};
|
|
991
|
+
},
|
|
992
|
+
},
|
|
993
|
+
);
|
|
971
994
|
}
|
|
972
995
|
// Non-local skill backends share the artifact store's R2 settings for
|
|
973
996
|
// the built-in arm and the composition's registered drivers beyond it.
|
|
@@ -994,11 +1017,31 @@ export async function composeServer(
|
|
|
994
1017
|
skillStorageDriver,
|
|
995
1018
|
PLUGIN_ARTIFACT_KEY_PREFIX,
|
|
996
1019
|
);
|
|
1020
|
+
// One staging port for every archive kind, over the one skill blob
|
|
1021
|
+
// driver: the controllers mint, consume and sign through it and never
|
|
1022
|
+
// meet the slots or the bucket.
|
|
1023
|
+
const skillArchiveStaging = newArchiveStaging(skillStorageDriver, logger);
|
|
997
1024
|
const skillTransferLane = newSkillTransferLane(
|
|
998
1025
|
skillUploadSlots,
|
|
999
1026
|
skillArtifactStorage,
|
|
1000
1027
|
logger,
|
|
1001
1028
|
);
|
|
1029
|
+
// A hosted server relaying bytes through itself from its loopback
|
|
1030
|
+
// default mints URLs no caller can reach (stigmer#1219); nothing inside
|
|
1031
|
+
// the process can observe that, so the boot says it. Never fatal: the
|
|
1032
|
+
// inline push and every other RPC work, and a self-host behind a tunnel
|
|
1033
|
+
// it forgot to name deserves a pointer, not an outage.
|
|
1034
|
+
const transferOrigin = assessSkillTransferOrigin({
|
|
1035
|
+
baseUrl: config.skillTransferBaseUrl,
|
|
1036
|
+
relaysThroughServer: skillType === "local",
|
|
1037
|
+
requireAuthentication,
|
|
1038
|
+
});
|
|
1039
|
+
if (transferOrigin.kind === "loopback-hosted") {
|
|
1040
|
+
logger.warn(
|
|
1041
|
+
"skill artifact transfer lane mints capability URLs on the server's own loopback while the server requires authentication; pushes above the gRPC message cap and sandbox skill downloads will fail — set SKILL_TRANSFER_BASE_URL to the origin callers reach this server on",
|
|
1042
|
+
{ baseUrl: transferOrigin.baseUrl },
|
|
1043
|
+
);
|
|
1044
|
+
}
|
|
1002
1045
|
// The console lane (lane 4, DD-012): present only when a static export
|
|
1003
1046
|
// is bundled (slim artifacts) or configured (STIGMER_CONSOLE_DIR) —
|
|
1004
1047
|
// "not bundled" is a modeled state, logged once, with the router
|
|
@@ -1463,14 +1506,11 @@ export async function composeServer(
|
|
|
1463
1506
|
// The agentexecution blob store (server.go:362-363) — read side of
|
|
1464
1507
|
// pushFromExecutionArtifact.
|
|
1465
1508
|
executionArtifactStorage: artifactStorage,
|
|
1466
|
-
|
|
1467
|
-
slots: skillUploadSlots,
|
|
1468
|
-
baseUrl: config.skillTransferBaseUrl,
|
|
1469
|
-
},
|
|
1509
|
+
staging: skillArchiveStaging,
|
|
1470
1510
|
});
|
|
1471
1511
|
// Plugins materialise their members through the in-process lane as
|
|
1472
1512
|
// the installing caller; the lazy provider resolves the clients built
|
|
1473
|
-
// from these same routes after boot. The
|
|
1513
|
+
// from these same routes after boot. The staging port is the skill
|
|
1474
1514
|
// lane's: one upload surface for every archive.
|
|
1475
1515
|
registerPluginServices(router, {
|
|
1476
1516
|
store,
|
|
@@ -1479,10 +1519,7 @@ export async function composeServer(
|
|
|
1479
1519
|
authorizationLifecycle,
|
|
1480
1520
|
artifactStorage: pluginArtifactStorage,
|
|
1481
1521
|
materializerProvider: () => requireInProcess().pluginMaterializer,
|
|
1482
|
-
|
|
1483
|
-
slots: skillUploadSlots,
|
|
1484
|
-
baseUrl: config.skillTransferBaseUrl,
|
|
1485
|
-
},
|
|
1522
|
+
staging: skillArchiveStaging,
|
|
1486
1523
|
});
|
|
1487
1524
|
// Artifact CRUD shares the ONE blob store with agentexecution's
|
|
1488
1525
|
// attachment lanes (Go server.go 347–374).
|