@vellumai/assistant 0.12.6 → 0.12.7-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/ARCHITECTURE.md +14 -14
- package/docs/architecture/integrations.md +5 -5
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
- package/node_modules/@vellumai/gateway-client/src/__tests__/route-trust-class.test.ts +111 -0
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +5 -2
- package/node_modules/@vellumai/gateway-client/src/index.ts +9 -0
- package/node_modules/@vellumai/gateway-client/src/route-trust-class.ts +94 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +2 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +8 -4
- package/openapi.yaml +153 -5
- package/package.json +1 -1
- package/src/__tests__/channel-policy.test.ts +7 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +141 -2
- package/src/__tests__/identity-routes.test.ts +1 -0
- package/src/__tests__/list-messages-client-message-id.test.ts +43 -0
- package/src/__tests__/openai-provider.test.ts +27 -0
- package/src/__tests__/skill-execute-input.test.ts +11 -0
- package/src/__tests__/tool-approval-handler.test.ts +4 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +26 -0
- package/src/agent/loop-unparseable-tool-retry.test.ts +333 -0
- package/src/agent/loop.ts +133 -12
- package/src/calls/__tests__/call-controller.test.ts +273 -1
- package/src/calls/__tests__/media-stream-server-integration.test.ts +40 -0
- package/src/calls/__tests__/media-stream-stt-session.test.ts +33 -0
- package/src/calls/call-constants.ts +10 -0
- package/src/calls/call-controller.ts +359 -25
- package/src/calls/media-stream-server.ts +18 -0
- package/src/calls/media-stream-stt-session.ts +30 -9
- package/src/channels/config.ts +8 -0
- package/src/cli/lib/bundled-marketplace.json +39 -0
- package/src/config/composer-preferences.test.ts +97 -0
- package/src/config/composer-preferences.ts +95 -0
- package/src/config/feature-flag-registry.json +16 -8
- package/src/config/schemas/llm.ts +1 -0
- package/src/contacts/types.ts +1 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +18 -2
- package/src/daemon/conversation-agent-loop.ts +3 -0
- package/src/daemon/conversation-messaging.ts +4 -0
- package/src/daemon/conversation-process.ts +15 -0
- package/src/daemon/conversation-queue-manager.ts +2 -0
- package/src/daemon/conversation.ts +2 -0
- package/src/daemon/handlers/shared.ts +13 -2
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/process-message.ts +5 -0
- package/src/i18n/__tests__/i18n.test.ts +25 -0
- package/src/i18n/assistant-text.ts +18 -0
- package/src/i18n/index.ts +1 -0
- package/src/i18n/messages.ts +11 -0
- package/src/ipc/routes/__tests__/route-adapter.test.ts +40 -0
- package/src/ipc/routes/route-adapter.ts +9 -0
- package/src/notifications/README.md +1 -1
- package/src/notifications/signal.ts +4 -0
- package/src/persistence/conversation-key-store.ts +12 -11
- package/src/providers/__tests__/unparseable-tool-args.test.ts +25 -0
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/ionet/client.test.ts +143 -0
- package/src/providers/ionet/client.ts +37 -0
- package/src/providers/model-catalog.ts +40 -0
- package/src/providers/openai/chat-completions-provider.ts +33 -2
- package/src/providers/retry.ts +10 -4
- package/src/providers/unparseable-tool-args.ts +29 -5
- package/src/runtime/__tests__/http-router-trust.test.ts +360 -0
- package/src/runtime/__tests__/shared-principal-lookup.test.ts +346 -0
- package/src/runtime/auth/__tests__/guard-tests.test.ts +51 -1
- package/src/runtime/auth/__tests__/route-policy.test.ts +85 -1
- package/src/runtime/auth/__tests__/scopes.test.ts +36 -0
- package/src/runtime/auth/route-policy.ts +44 -0
- package/src/runtime/auth/scopes.ts +15 -0
- package/src/runtime/auth/types.ts +1 -0
- package/src/runtime/channel-reply-delivery.ts +2 -0
- package/src/runtime/http-router.ts +61 -1
- package/src/runtime/http-types.ts +3 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +35 -10
- package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +48 -0
- package/src/runtime/migrations/vbundle-builder.ts +19 -3
- package/src/runtime/migrations/vbundle-streaming-importer.ts +14 -19
- package/src/runtime/migrations/vbundle-streaming-validator.ts +14 -7
- package/src/runtime/migrations/vbundle-validator.ts +54 -1
- package/src/runtime/routes/composer-settings-routes.test.ts +71 -0
- package/src/runtime/routes/composer-settings-routes.ts +76 -0
- package/src/runtime/routes/conversation-routes.ts +54 -14
- package/src/runtime/routes/identity-routes.ts +2 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +4 -0
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/migration-routes.ts +3 -1
- package/src/runtime/routes/types.ts +4 -1
- package/src/runtime/shared-principal-lookup.ts +235 -0
- package/src/tools/host-terminal/host-shell.ts +4 -2
- package/src/tools/shared/shell-output.test.ts +42 -0
- package/src/tools/shared/shell-output.ts +27 -4
- package/src/tools/skills/execute.ts +10 -7
- package/src/tools/terminal/shell.ts +2 -0
- package/src/tools/tool-approval-handler.ts +9 -1
|
@@ -3,6 +3,7 @@ import type { MessageAudience } from "@vellumai/gateway-client";
|
|
|
3
3
|
import { stripVellumLinks } from "../daemon/assistant-attachments.js";
|
|
4
4
|
import type { RenderedHistoryContent } from "../daemon/handlers/shared.js";
|
|
5
5
|
import { renderHistoryContent } from "../daemon/handlers/shared.js";
|
|
6
|
+
import { DEFAULT_LOCALE } from "../i18n/index.js";
|
|
6
7
|
import { editChannelMessage } from "../messaging/providers/index.js";
|
|
7
8
|
import { getAttachmentMetadataForMessage } from "../persistence/attachments-store.js";
|
|
8
9
|
import {
|
|
@@ -344,6 +345,7 @@ function readPersistedAssistantReply(msg: PersistedMessage): {
|
|
|
344
345
|
undefined,
|
|
345
346
|
undefined,
|
|
346
347
|
msg.metadata,
|
|
348
|
+
DEFAULT_LOCALE,
|
|
347
349
|
);
|
|
348
350
|
|
|
349
351
|
const linked = getAttachmentMetadataForMessage(msg.id);
|
|
@@ -10,7 +10,10 @@
|
|
|
10
10
|
* `normalizeEndpointForPolicy`.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import {
|
|
13
|
+
import { tokenMayReachRoute } from "@vellumai/gateway-client";
|
|
14
|
+
|
|
15
|
+
import { getLogger } from "../util/logger.js";
|
|
16
|
+
import { enforcePolicy, type RoutePolicy } from "./auth/route-policy.js";
|
|
14
17
|
import type { AuthContext } from "./auth/types.js";
|
|
15
18
|
import { httpError } from "./http-errors.js";
|
|
16
19
|
import type { HTTPRouteDefinition, RouteParams } from "./http-router-types.js";
|
|
@@ -18,6 +21,9 @@ import { withErrorHandling } from "./middleware/error-handler.js";
|
|
|
18
21
|
import { routeDefinitionsToHTTPRoutes } from "./routes/http-adapter.js";
|
|
19
22
|
import { ROUTES } from "./routes/index.js";
|
|
20
23
|
import type { RouteLoggingConfig } from "./routes/types.js";
|
|
24
|
+
import { resolveSharedPrincipalFresh } from "./shared-principal-lookup.js";
|
|
25
|
+
|
|
26
|
+
const log = getLogger("http-router");
|
|
21
27
|
|
|
22
28
|
// ---------------------------------------------------------------------------
|
|
23
29
|
// Compiled route — internal representation with pre-built regex
|
|
@@ -106,6 +112,15 @@ export class HttpRouter {
|
|
|
106
112
|
continue;
|
|
107
113
|
}
|
|
108
114
|
|
|
115
|
+
const trustDenied = await enforceTrustClass(
|
|
116
|
+
compiled.def.endpoint,
|
|
117
|
+
compiled.def.policy,
|
|
118
|
+
authContext,
|
|
119
|
+
);
|
|
120
|
+
if (trustDenied) {
|
|
121
|
+
return trustDenied;
|
|
122
|
+
}
|
|
123
|
+
|
|
109
124
|
// Extract named params
|
|
110
125
|
const params: RouteParams = {};
|
|
111
126
|
for (let i = 0; i < compiled.paramNames.length; i++) {
|
|
@@ -143,6 +158,51 @@ export class HttpRouter {
|
|
|
143
158
|
}
|
|
144
159
|
}
|
|
145
160
|
|
|
161
|
+
// ---------------------------------------------------------------------------
|
|
162
|
+
// Trust class
|
|
163
|
+
// ---------------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Refuses a caller on a route that does not admit its trust class, with the
|
|
167
|
+
* same 404 an unmatched path gets so it cannot probe which routes exist. It
|
|
168
|
+
* runs ahead of path decoding and the scope check so neither can answer 400 or
|
|
169
|
+
* 403 first, and under the dev auth bypass too, because a trust-checked
|
|
170
|
+
* context exists only when its bearer was verified.
|
|
171
|
+
*
|
|
172
|
+
* A trust-exempt profile (see {@link isTrustCheckedScopeProfile}) counts as
|
|
173
|
+
* the guardian with no lookup, so guardian, service and local callers never
|
|
174
|
+
* wait on the gateway here. A trust-checked one is read fresh, matching the
|
|
175
|
+
* gateway's per-request ACL read on the IPC path, so a revoked contact is
|
|
176
|
+
* refused on its next request.
|
|
177
|
+
*/
|
|
178
|
+
async function enforceTrustClass(
|
|
179
|
+
endpoint: string,
|
|
180
|
+
policy: RoutePolicy | null,
|
|
181
|
+
authContext: AuthContext,
|
|
182
|
+
): Promise<Response | null> {
|
|
183
|
+
const principalId = authContext.actorPrincipalId;
|
|
184
|
+
const admitted = await tokenMayReachRoute(
|
|
185
|
+
authContext.scopeProfile,
|
|
186
|
+
policy?.allowedTrustClasses,
|
|
187
|
+
async () =>
|
|
188
|
+
principalId
|
|
189
|
+
? (await resolveSharedPrincipalFresh(principalId)).trustClass
|
|
190
|
+
: undefined,
|
|
191
|
+
);
|
|
192
|
+
if (admitted) {
|
|
193
|
+
return null;
|
|
194
|
+
}
|
|
195
|
+
log.warn(
|
|
196
|
+
{
|
|
197
|
+
endpoint,
|
|
198
|
+
scopeProfile: authContext.scopeProfile,
|
|
199
|
+
actorPrincipalId: principalId,
|
|
200
|
+
},
|
|
201
|
+
"Route policy denied: trust class not admitted",
|
|
202
|
+
);
|
|
203
|
+
return httpError("NOT_FOUND", "Not found", 404);
|
|
204
|
+
}
|
|
205
|
+
|
|
146
206
|
// ---------------------------------------------------------------------------
|
|
147
207
|
// Compilation helpers
|
|
148
208
|
// ---------------------------------------------------------------------------
|
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
ConversationCreateOptions,
|
|
13
13
|
SlackInboundMessageMetadata,
|
|
14
14
|
} from "../daemon/handlers/shared.js";
|
|
15
|
+
import type { SupportedLocale } from "../i18n/index.js";
|
|
15
16
|
|
|
16
17
|
// Re-export so route modules (background-dispatch, etc.) can pull the type
|
|
17
18
|
// from the runtime barrel without reaching into daemon internals.
|
|
@@ -110,6 +111,8 @@ export interface RuntimeMessageConversationOptions {
|
|
|
110
111
|
sourceChannel?: ChannelId;
|
|
111
112
|
/** Originating interface (e.g. "cli", "web"). Defaults to "web". */
|
|
112
113
|
sourceInterface?: InterfaceId;
|
|
114
|
+
/** Locale for daemon-authored text emitted during this turn. */
|
|
115
|
+
locale?: SupportedLocale;
|
|
113
116
|
}
|
|
114
117
|
|
|
115
118
|
export type MessageProcessor = (
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
import {
|
|
31
31
|
computeLegacyManifestSha256,
|
|
32
32
|
computeManifestChecksum,
|
|
33
|
+
MANIFEST_MAX_BYTES,
|
|
33
34
|
} from "../vbundle-validator.js";
|
|
34
35
|
import { defaultV1Options } from "./v1-test-helpers.js";
|
|
35
36
|
|
|
@@ -202,15 +203,16 @@ describe("readAndValidateManifest — negative paths", () => {
|
|
|
202
203
|
});
|
|
203
204
|
|
|
204
205
|
test("throws manifest_too_large and fails fast before draining the whole body", async () => {
|
|
205
|
-
// Fake a tar entry whose body would emit
|
|
206
|
-
// validator must destroy() the stream and throw the moment the
|
|
207
|
-
// byte count crosses
|
|
206
|
+
// Fake a tar entry whose body would emit 10x the cap if fully drained.
|
|
207
|
+
// The validator must destroy() the stream and throw the moment the
|
|
208
|
+
// running byte count crosses MANIFEST_MAX_BYTES; it must NOT keep
|
|
209
|
+
// pulling chunks.
|
|
208
210
|
//
|
|
209
211
|
// We count bytes emitted via a _read implementation, and after the
|
|
210
|
-
// throw assert both that destroy() fired and that far
|
|
211
|
-
//
|
|
212
|
-
const CHUNK =
|
|
213
|
-
const TOTAL_CHUNKS =
|
|
212
|
+
// throw assert both that destroy() fired and that far less than the
|
|
213
|
+
// full body was ever pulled out of the stream.
|
|
214
|
+
const CHUNK = MANIFEST_MAX_BYTES / 2;
|
|
215
|
+
const TOTAL_CHUNKS = 20;
|
|
214
216
|
let chunksEmitted = 0;
|
|
215
217
|
let bytesEmitted = 0;
|
|
216
218
|
const body = new Readable({
|
|
@@ -245,11 +247,34 @@ describe("readAndValidateManifest — negative paths", () => {
|
|
|
245
247
|
expect(err).toBeInstanceOf(StreamingValidationError);
|
|
246
248
|
expect(err?.code).toBe("manifest_too_large");
|
|
247
249
|
// Fail-fast assertions: destroy() was called, and we didn't drain past
|
|
248
|
-
//
|
|
249
|
-
//
|
|
250
|
+
// the cap + one chunk. If the validator had drained to EOF we'd see all
|
|
251
|
+
// 20 chunks here.
|
|
250
252
|
expect(body.destroyed).toBe(true);
|
|
251
253
|
expect(chunksEmitted).toBeLessThanOrEqual(3);
|
|
252
|
-
expect(bytesEmitted).toBeLessThan(2 *
|
|
254
|
+
expect(bytesEmitted).toBeLessThan(2 * MANIFEST_MAX_BYTES);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
test("throws bundle_too_many_entries when manifest.contents exceeds maxEntries", async () => {
|
|
258
|
+
const { archive } = buildVBundle({
|
|
259
|
+
files: [
|
|
260
|
+
{ path: "data/db/assistant.db", data: new Uint8Array() },
|
|
261
|
+
{ path: "workspace/a.txt", data: new TextEncoder().encode("a") },
|
|
262
|
+
{ path: "workspace/b.txt", data: new TextEncoder().encode("b") },
|
|
263
|
+
],
|
|
264
|
+
...defaultV1Options(),
|
|
265
|
+
});
|
|
266
|
+
const { entry, drainRest } = await firstEntryOf(archive);
|
|
267
|
+
|
|
268
|
+
let err: StreamingValidationError | null = null;
|
|
269
|
+
try {
|
|
270
|
+
await readAndValidateManifest(entry, { maxEntries: 2 });
|
|
271
|
+
} catch (e) {
|
|
272
|
+
err = e as StreamingValidationError;
|
|
273
|
+
}
|
|
274
|
+
await drainRest();
|
|
275
|
+
|
|
276
|
+
expect(err).toBeInstanceOf(StreamingValidationError);
|
|
277
|
+
expect(err?.code).toBe("bundle_too_many_entries");
|
|
253
278
|
});
|
|
254
279
|
|
|
255
280
|
test("throws manifest_malformed when manifest body is not valid JSON", async () => {
|
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
canonicalizeJson,
|
|
20
20
|
computeLegacyManifestSha256,
|
|
21
21
|
computeManifestChecksum,
|
|
22
|
+
MANIFEST_MAX_BYTES,
|
|
23
|
+
MANIFEST_MAX_ENTRIES,
|
|
22
24
|
ManifestSchema,
|
|
23
25
|
validateVBundle,
|
|
24
26
|
} from "../vbundle-validator.js";
|
|
@@ -174,6 +176,52 @@ describe("ManifestSchema — v1 acceptance", () => {
|
|
|
174
176
|
});
|
|
175
177
|
});
|
|
176
178
|
|
|
179
|
+
describe("validateVBundle: streaming manifest limits", () => {
|
|
180
|
+
test("rejects a manifest over MANIFEST_MAX_BYTES with MANIFEST_TOO_LARGE", () => {
|
|
181
|
+
const oversized = new Uint8Array(MANIFEST_MAX_BYTES + 1).fill(0x20);
|
|
182
|
+
const archive = gzipSync(
|
|
183
|
+
tarArchive([{ name: "manifest.json", data: oversized }]),
|
|
184
|
+
);
|
|
185
|
+
const result = validateVBundle(archive, { enforceStreamingLimits: true });
|
|
186
|
+
expect(result.is_valid).toBe(false);
|
|
187
|
+
expect(result.errors.map((e) => e.code)).toEqual(["MANIFEST_TOO_LARGE"]);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("rejects a manifest declaring more than MANIFEST_MAX_ENTRIES with MANIFEST_TOO_MANY_ENTRIES", () => {
|
|
191
|
+
const skeleton = v1Skeleton();
|
|
192
|
+
const dbEntry = (skeleton.contents as Array<Record<string, unknown>>)[0]!;
|
|
193
|
+
const extra = Array.from({ length: MANIFEST_MAX_ENTRIES }, (_, i) => ({
|
|
194
|
+
...dbEntry,
|
|
195
|
+
path: `workspace/${i}`,
|
|
196
|
+
}));
|
|
197
|
+
const archive = gzipTarOf(
|
|
198
|
+
withChecksum({ ...skeleton, contents: [dbEntry, ...extra] }),
|
|
199
|
+
);
|
|
200
|
+
const result = validateVBundle(archive, { enforceStreamingLimits: true });
|
|
201
|
+
expect(result.is_valid).toBe(false);
|
|
202
|
+
expect(result.errors.map((e) => e.code)).toEqual([
|
|
203
|
+
"MANIFEST_TOO_MANY_ENTRIES",
|
|
204
|
+
]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
test("does not apply the limits unless enforceStreamingLimits is set", () => {
|
|
208
|
+
const skeleton = v1Skeleton();
|
|
209
|
+
const dbEntry = (skeleton.contents as Array<Record<string, unknown>>)[0]!;
|
|
210
|
+
const extra = Array.from({ length: MANIFEST_MAX_ENTRIES }, (_, i) => ({
|
|
211
|
+
...dbEntry,
|
|
212
|
+
path: `workspace/${i}`,
|
|
213
|
+
}));
|
|
214
|
+
const archive = gzipTarOf(
|
|
215
|
+
withChecksum({ ...skeleton, contents: [dbEntry, ...extra] }),
|
|
216
|
+
[{ name: "data/db/assistant.db", data: DB_BYTES }],
|
|
217
|
+
);
|
|
218
|
+
const result = validateVBundle(archive);
|
|
219
|
+
expect(result.errors.map((e) => e.code)).not.toContain(
|
|
220
|
+
"MANIFEST_TOO_MANY_ENTRIES",
|
|
221
|
+
);
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
|
|
177
225
|
describe("ManifestSchema — v1 rejection", () => {
|
|
178
226
|
test("rejects schema_version !== 1", () => {
|
|
179
227
|
const manifest = withChecksum({ ...v1Skeleton(), schema_version: 2 });
|
|
@@ -31,9 +31,11 @@ import { sanitizeConfigForTransfer } from "../../config/sanitize-for-transfer.js
|
|
|
31
31
|
import { trackDaemonActivity } from "../../daemon/activity-trail.js";
|
|
32
32
|
import { getLogger } from "../../util/logger.js";
|
|
33
33
|
import type { VBundleOriginMode } from "./origin-mode.js";
|
|
34
|
-
import
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
import {
|
|
35
|
+
MANIFEST_MAX_BYTES,
|
|
36
|
+
MANIFEST_MAX_ENTRIES,
|
|
37
|
+
type ManifestFileEntryType,
|
|
38
|
+
type ManifestType,
|
|
37
39
|
} from "./vbundle-validator.js";
|
|
38
40
|
|
|
39
41
|
// ---------------------------------------------------------------------------
|
|
@@ -400,6 +402,20 @@ function buildManifestObject(input: {
|
|
|
400
402
|
const checksum = sha256Hex(canonicalizeJson(manifestWithEmptyChecksum));
|
|
401
403
|
const manifest: ManifestType = { ...manifestWithEmptyChecksum, checksum };
|
|
402
404
|
const manifestData = new TextEncoder().encode(JSON.stringify(manifest));
|
|
405
|
+
if (
|
|
406
|
+
manifest.contents.length > MANIFEST_MAX_ENTRIES ||
|
|
407
|
+
manifestData.length > MANIFEST_MAX_BYTES
|
|
408
|
+
) {
|
|
409
|
+
getLogger("vbundle-builder").warn(
|
|
410
|
+
{
|
|
411
|
+
entries: manifest.contents.length,
|
|
412
|
+
manifestBytes: manifestData.length,
|
|
413
|
+
maxEntries: MANIFEST_MAX_ENTRIES,
|
|
414
|
+
maxBytes: MANIFEST_MAX_BYTES,
|
|
415
|
+
},
|
|
416
|
+
"Bundle manifest exceeds the streaming import limits; teleport and staged-path imports will reject this bundle",
|
|
417
|
+
);
|
|
418
|
+
}
|
|
403
419
|
return { manifest, manifestData };
|
|
404
420
|
}
|
|
405
421
|
|
|
@@ -69,7 +69,10 @@ import {
|
|
|
69
69
|
verifySymlinkEntry,
|
|
70
70
|
} from "./vbundle-streaming-validator.js";
|
|
71
71
|
import { parseVBundleStream } from "./vbundle-tar-stream.js";
|
|
72
|
-
import
|
|
72
|
+
import {
|
|
73
|
+
MANIFEST_MAX_ENTRIES,
|
|
74
|
+
type ManifestType,
|
|
75
|
+
} from "./vbundle-validator.js";
|
|
73
76
|
|
|
74
77
|
const log = getLogger("vbundle-streaming-importer");
|
|
75
78
|
|
|
@@ -91,11 +94,11 @@ const log = getLogger("vbundle-streaming-importer");
|
|
|
91
94
|
const DEFAULT_MAX_BUNDLE_BYTES = 16 * 1024 * 1024 * 1024;
|
|
92
95
|
|
|
93
96
|
/**
|
|
94
|
-
* Entry-count ceiling for the bundle
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
+
* Entry-count ceiling for the bundle, shared with manifest validation so
|
|
98
|
+
* preflight and import agree. Also applied to tar-level entries so padding
|
|
99
|
+
* extras cannot exceed what the manifest declared.
|
|
97
100
|
*/
|
|
98
|
-
const DEFAULT_MAX_BUNDLE_ENTRIES =
|
|
101
|
+
const DEFAULT_MAX_BUNDLE_ENTRIES = MANIFEST_MAX_ENTRIES;
|
|
99
102
|
|
|
100
103
|
/**
|
|
101
104
|
* Prefixes used for scratch dirs the streaming importer creates INSIDE the
|
|
@@ -153,7 +156,7 @@ export interface StreamCommitArgs {
|
|
|
153
156
|
maxBundleBytes?: number;
|
|
154
157
|
/**
|
|
155
158
|
* Test-only override for the entry-count ceiling. Production callers
|
|
156
|
-
* should omit this and rely on the
|
|
159
|
+
* should omit this and rely on the `MANIFEST_MAX_ENTRIES` default.
|
|
157
160
|
*/
|
|
158
161
|
maxBundleEntries?: number;
|
|
159
162
|
}
|
|
@@ -313,7 +316,9 @@ export async function streamCommitImport(
|
|
|
313
316
|
if (entryIndex === 0) {
|
|
314
317
|
// First entry MUST be manifest.json — readAndValidateManifest
|
|
315
318
|
// enforces that and throws StreamingValidationError otherwise.
|
|
316
|
-
const manifestResult = await readAndValidateManifest(entry
|
|
319
|
+
const manifestResult = await readAndValidateManifest(entry, {
|
|
320
|
+
maxEntries: bundleEntryCap,
|
|
321
|
+
});
|
|
317
322
|
manifest = manifestResult.manifest;
|
|
318
323
|
expected = manifestResult.expected;
|
|
319
324
|
|
|
@@ -340,18 +345,8 @@ export async function streamCommitImport(
|
|
|
340
345
|
);
|
|
341
346
|
}
|
|
342
347
|
|
|
343
|
-
//
|
|
344
|
-
//
|
|
345
|
-
// importer is willing to do for this bundle.
|
|
346
|
-
if (manifest.contents.length > bundleEntryCap) {
|
|
347
|
-
throw new StreamingValidationError(
|
|
348
|
-
"bundle_too_many_entries",
|
|
349
|
-
`bundle contains more than ${bundleEntryCap} entries (declared: ${manifest.contents.length})`,
|
|
350
|
-
);
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
// Only NOW — after the manifest is parsed, the version gate passes,
|
|
354
|
-
// and the entry-count ceiling is enforced — do we materialize the
|
|
348
|
+
// Only NOW, after the manifest is parsed (including its entry-count
|
|
349
|
+
// ceiling) and the version gate passes, do we materialize the
|
|
355
350
|
// temp staging dir on disk. Doing this lazily preserves the plan
|
|
356
351
|
// invariant that importers gate on runtime-version compat BEFORE
|
|
357
352
|
// any state mutation. If this throws, the outer catch runs
|
|
@@ -28,6 +28,8 @@ import {
|
|
|
28
28
|
computeLegacyManifestSha256,
|
|
29
29
|
computeManifestChecksum,
|
|
30
30
|
LegacyManifestSchema,
|
|
31
|
+
MANIFEST_MAX_BYTES,
|
|
32
|
+
MANIFEST_MAX_ENTRIES,
|
|
31
33
|
ManifestSchema,
|
|
32
34
|
type ManifestType,
|
|
33
35
|
translateLegacyManifest,
|
|
@@ -74,21 +76,17 @@ export class StreamingValidationError extends Error {
|
|
|
74
76
|
// Manifest validation
|
|
75
77
|
// ---------------------------------------------------------------------------
|
|
76
78
|
|
|
77
|
-
// Manifests are metadata only — typically tens to hundreds of KB even for
|
|
78
|
-
// huge bundles. A 1 MiB cap is comfortably above realistic sizes and
|
|
79
|
-
// protects against a malicious archive whose "manifest" is actually a
|
|
80
|
-
// multi-GB stream intended to OOM the validator.
|
|
81
|
-
const MANIFEST_MAX_BYTES = 1 * 1024 * 1024;
|
|
82
|
-
|
|
83
79
|
/**
|
|
84
80
|
* Drain the first tar entry — which MUST be `manifest.json` — and run the
|
|
85
81
|
* full manifest-level validation pipeline:
|
|
86
82
|
* 1. Entry name check.
|
|
87
|
-
* 2. Size cap (
|
|
83
|
+
* 2. Size cap (`MANIFEST_MAX_BYTES`).
|
|
88
84
|
* 3. JSON parse.
|
|
89
85
|
* 4. Zod schema validation.
|
|
90
86
|
* 5. Self-referencing `checksum` verification against the
|
|
91
87
|
* canonicalized JSON (minus that field).
|
|
88
|
+
* 6. Entry-count ceiling on `contents` (`maxEntries`, default
|
|
89
|
+
* `MANIFEST_MAX_ENTRIES`).
|
|
92
90
|
*
|
|
93
91
|
* On success, returns the parsed manifest plus a `Map` keyed by archive
|
|
94
92
|
* path that callers consult as each subsequent entry streams past.
|
|
@@ -98,7 +96,9 @@ const MANIFEST_MAX_BYTES = 1 * 1024 * 1024;
|
|
|
98
96
|
*/
|
|
99
97
|
export async function readAndValidateManifest(
|
|
100
98
|
first: StreamedTarEntry,
|
|
99
|
+
options: { maxEntries?: number } = {},
|
|
101
100
|
): Promise<ManifestReadResult> {
|
|
101
|
+
const maxEntries = options.maxEntries ?? MANIFEST_MAX_ENTRIES;
|
|
102
102
|
if (first.header.name !== "manifest.json") {
|
|
103
103
|
// Drain the body so the underlying tar extractor isn't left dangling
|
|
104
104
|
// on backpressure before the caller reports the error.
|
|
@@ -186,6 +186,13 @@ export async function readAndValidateManifest(
|
|
|
186
186
|
manifest = translateLegacyManifest(legacy);
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
+
if (manifest.contents.length > maxEntries) {
|
|
190
|
+
throw new StreamingValidationError(
|
|
191
|
+
"bundle_too_many_entries",
|
|
192
|
+
`bundle contains more than ${maxEntries} entries (declared: ${manifest.contents.length})`,
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
189
196
|
const expected = new Map<
|
|
190
197
|
string,
|
|
191
198
|
{ sha256: string; size: number; linkTarget: string | null }
|
|
@@ -381,6 +381,33 @@ export function computeManifestChecksum(manifest: unknown): string {
|
|
|
381
381
|
// Core validation
|
|
382
382
|
// ---------------------------------------------------------------------------
|
|
383
383
|
|
|
384
|
+
/**
|
|
385
|
+
* Hard caps on `manifest.json`, shared by the buffered and streaming
|
|
386
|
+
* validators so preflight and import agree on what is acceptable. The two
|
|
387
|
+
* are sized together: the manifest carries one `contents` entry (path,
|
|
388
|
+
* sha256, size) per bundled file at roughly 150 bytes each, so 200,000
|
|
389
|
+
* entries serialize to about 32 MiB.
|
|
390
|
+
*
|
|
391
|
+
* That absorbs a workspace with a stray dependency install or checked-out
|
|
392
|
+
* repo while still bounding the transient memory a hostile "manifest" can
|
|
393
|
+
* force the importer to hold (buffer, decoded string, parsed object,
|
|
394
|
+
* canonical re-serialization for the self-checksum) and the per-entry work
|
|
395
|
+
* it commits to, on a memory-constrained pod.
|
|
396
|
+
*/
|
|
397
|
+
export const MANIFEST_MAX_BYTES = 32 * 1024 * 1024;
|
|
398
|
+
export const MANIFEST_MAX_ENTRIES = 200_000;
|
|
399
|
+
|
|
400
|
+
export interface ValidateVBundleOptions {
|
|
401
|
+
/**
|
|
402
|
+
* Reject manifests over `MANIFEST_MAX_BYTES` or `MANIFEST_MAX_ENTRIES`.
|
|
403
|
+
* Preflights that precede a streaming import set this so they never
|
|
404
|
+
* approve a bundle the import will refuse. The buffered import path
|
|
405
|
+
* (`commitImport`) has no such ceilings, so it stays off there to keep
|
|
406
|
+
* existing large backups restorable.
|
|
407
|
+
*/
|
|
408
|
+
enforceStreamingLimits?: boolean;
|
|
409
|
+
}
|
|
410
|
+
|
|
384
411
|
// Only manifest.json is structurally required. The DB and config live under
|
|
385
412
|
// workspace/ (new format) or data/db/ + config/ (old format) — both are valid.
|
|
386
413
|
const REQUIRED_ENTRIES = ["manifest.json"];
|
|
@@ -397,7 +424,10 @@ const MAX_DECOMPRESSED_SIZE = 2 * 1024 * 1024 * 1024;
|
|
|
397
424
|
* 3. Manifest checksum (SHA-256 of canonicalized JSON with the `checksum` field set to empty string)
|
|
398
425
|
* 4. Per-file content integrity (SHA-256 of each file vs manifest declaration)
|
|
399
426
|
*/
|
|
400
|
-
export function validateVBundle(
|
|
427
|
+
export function validateVBundle(
|
|
428
|
+
data: Uint8Array,
|
|
429
|
+
{ enforceStreamingLimits = false }: ValidateVBundleOptions = {},
|
|
430
|
+
): VBundleValidationResult {
|
|
401
431
|
const errors: ValidationError[] = [];
|
|
402
432
|
|
|
403
433
|
// Step 1: Decompress gzip with size cap to prevent zip-bomb DoS
|
|
@@ -455,6 +485,17 @@ export function validateVBundle(data: Uint8Array): VBundleValidationResult {
|
|
|
455
485
|
}
|
|
456
486
|
|
|
457
487
|
// Step 4: Parse and validate manifest schema
|
|
488
|
+
if (
|
|
489
|
+
enforceStreamingLimits &&
|
|
490
|
+
manifestEntry.data.length > MANIFEST_MAX_BYTES
|
|
491
|
+
) {
|
|
492
|
+
errors.push({
|
|
493
|
+
code: "MANIFEST_TOO_LARGE",
|
|
494
|
+
message: `manifest.json exceeds ${MANIFEST_MAX_BYTES} byte limit (${manifestEntry.data.length} bytes)`,
|
|
495
|
+
path: "manifest.json",
|
|
496
|
+
});
|
|
497
|
+
return { is_valid: false, errors };
|
|
498
|
+
}
|
|
458
499
|
let manifestRaw: unknown;
|
|
459
500
|
try {
|
|
460
501
|
manifestRaw = JSON.parse(new TextDecoder().decode(manifestEntry.data));
|
|
@@ -523,6 +564,18 @@ export function validateVBundle(data: Uint8Array): VBundleValidationResult {
|
|
|
523
564
|
manifest = translateLegacyManifest(legacy);
|
|
524
565
|
}
|
|
525
566
|
|
|
567
|
+
if (
|
|
568
|
+
enforceStreamingLimits &&
|
|
569
|
+
manifest.contents.length > MANIFEST_MAX_ENTRIES
|
|
570
|
+
) {
|
|
571
|
+
errors.push({
|
|
572
|
+
code: "MANIFEST_TOO_MANY_ENTRIES",
|
|
573
|
+
message: `manifest.json declares more than ${MANIFEST_MAX_ENTRIES} entries (${manifest.contents.length})`,
|
|
574
|
+
path: "manifest.json",
|
|
575
|
+
});
|
|
576
|
+
return { is_valid: false, errors };
|
|
577
|
+
}
|
|
578
|
+
|
|
526
579
|
// Step 6: Verify per-file content integrity
|
|
527
580
|
const manifestFilePaths = new Set(manifest.contents.map((f) => f.path));
|
|
528
581
|
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { afterEach, beforeEach, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { ComposerPreferencesSchema } from "../../config/composer-preferences.js";
|
|
4
|
+
|
|
5
|
+
const preferences = new Map<
|
|
6
|
+
string,
|
|
7
|
+
ReturnType<typeof ComposerPreferencesSchema.parse>
|
|
8
|
+
>();
|
|
9
|
+
const resolvePrincipal = mock(async (id?: string) => id);
|
|
10
|
+
const publish = mock(async (_tags: string[], _origin?: string) => {});
|
|
11
|
+
mock.module("../local-actor-identity.js", () => ({
|
|
12
|
+
resolveActorPrincipalIdForLocalGuardian: resolvePrincipal,
|
|
13
|
+
}));
|
|
14
|
+
mock.module("../sync/sync-publisher.js", () => ({
|
|
15
|
+
publishSyncInvalidation: publish,
|
|
16
|
+
}));
|
|
17
|
+
mock.module("../../config/composer-preferences.js", () => ({
|
|
18
|
+
readComposerPreferences: (id: string) =>
|
|
19
|
+
preferences.get(id) ?? ComposerPreferencesSchema.parse({}),
|
|
20
|
+
updateComposerPreferences: (id: string, patch: object) => {
|
|
21
|
+
const value = ComposerPreferencesSchema.parse({
|
|
22
|
+
...preferences.get(id),
|
|
23
|
+
...patch,
|
|
24
|
+
});
|
|
25
|
+
preferences.set(id, value);
|
|
26
|
+
return value;
|
|
27
|
+
},
|
|
28
|
+
}));
|
|
29
|
+
const { ROUTES } = await import("./composer-settings-routes.js");
|
|
30
|
+
const get = ROUTES.find((route) => route.method === "GET")!;
|
|
31
|
+
const patch = ROUTES.find((route) => route.method === "PATCH")!;
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
preferences.clear();
|
|
34
|
+
resolvePrincipal.mockClear();
|
|
35
|
+
publish.mockClear();
|
|
36
|
+
});
|
|
37
|
+
afterEach(() => {
|
|
38
|
+
mock.restore();
|
|
39
|
+
});
|
|
40
|
+
test("uses the caller identity and emits value-free invalidation after saving", async () => {
|
|
41
|
+
await patch.handler({
|
|
42
|
+
headers: {
|
|
43
|
+
"x-vellum-actor-principal-id": "user-1",
|
|
44
|
+
"x-vellum-client-id": "client-1",
|
|
45
|
+
},
|
|
46
|
+
body: { lastAutonomy: "low" },
|
|
47
|
+
});
|
|
48
|
+
expect(preferences.get("user-1")?.lastAutonomy).toBe("low");
|
|
49
|
+
expect(publish).toHaveBeenCalledWith(
|
|
50
|
+
["assistant:self:composerPreferences"],
|
|
51
|
+
"client-1",
|
|
52
|
+
);
|
|
53
|
+
const result = await get.handler({
|
|
54
|
+
headers: { "x-vellum-actor-principal-id": "user-2" },
|
|
55
|
+
});
|
|
56
|
+
expect(result).toMatchObject({ preferences: { lastAutonomy: null } });
|
|
57
|
+
});
|
|
58
|
+
test("refuses preferences with no resolved identity", async () => {
|
|
59
|
+
await expect(get.handler({})).rejects.toThrow("signed-in user");
|
|
60
|
+
expect(publish).not.toHaveBeenCalled();
|
|
61
|
+
});
|
|
62
|
+
test("the body cannot select another user's preferences", async () => {
|
|
63
|
+
await expect(
|
|
64
|
+
patch.handler({
|
|
65
|
+
headers: { "x-vellum-actor-principal-id": "user-1" },
|
|
66
|
+
body: { principalId: "user-2", lastModeId: "balanced" },
|
|
67
|
+
}),
|
|
68
|
+
).rejects.toThrow();
|
|
69
|
+
expect(preferences.size).toBe(0);
|
|
70
|
+
expect(publish).not.toHaveBeenCalled();
|
|
71
|
+
});
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
ComposerPreferencesPatchSchema,
|
|
5
|
+
ComposerPreferencesSchema,
|
|
6
|
+
readComposerPreferences,
|
|
7
|
+
updateComposerPreferences,
|
|
8
|
+
} from "../../config/composer-preferences.js";
|
|
9
|
+
import { SYNC_TAGS } from "../../daemon/message-types/sync.js";
|
|
10
|
+
import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
|
|
11
|
+
import { resolveActorPrincipalIdForLocalGuardian } from "../local-actor-identity.js";
|
|
12
|
+
import { publishSyncInvalidation } from "../sync/sync-publisher.js";
|
|
13
|
+
import { BadRequestError, UnauthorizedError } from "./errors.js";
|
|
14
|
+
import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
|
|
15
|
+
|
|
16
|
+
const ResponseSchema = z.object({
|
|
17
|
+
preferences: ComposerPreferencesSchema,
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
async function principalFor(args: RouteHandlerArgs): Promise<string> {
|
|
21
|
+
const id = await resolveActorPrincipalIdForLocalGuardian(
|
|
22
|
+
args.headers?.["x-vellum-actor-principal-id"]?.trim() || undefined,
|
|
23
|
+
);
|
|
24
|
+
if (!id) {
|
|
25
|
+
throw new UnauthorizedError(
|
|
26
|
+
"A signed-in user is required for composer preferences",
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
return id;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function responseFor(principalId: string) {
|
|
33
|
+
return { preferences: readComposerPreferences(principalId) };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export const ROUTES: RouteDefinition[] = [
|
|
37
|
+
{
|
|
38
|
+
operationId: "composer_settings_get",
|
|
39
|
+
endpoint: "composer/settings",
|
|
40
|
+
method: "GET",
|
|
41
|
+
policy: {
|
|
42
|
+
requiredScopes: ["settings.read"],
|
|
43
|
+
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
44
|
+
},
|
|
45
|
+
summary: "Get the current user's composer preferences",
|
|
46
|
+
tags: ["settings"],
|
|
47
|
+
responseBody: ResponseSchema,
|
|
48
|
+
handler: async (args) => responseFor(await principalFor(args)),
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
operationId: "composer_settings_patch",
|
|
52
|
+
endpoint: "composer/settings",
|
|
53
|
+
method: "PATCH",
|
|
54
|
+
policy: {
|
|
55
|
+
requiredScopes: ["settings.write"],
|
|
56
|
+
allowedPrincipalTypes: ACTOR_PRINCIPALS,
|
|
57
|
+
},
|
|
58
|
+
summary: "Update the current user's composer preferences",
|
|
59
|
+
tags: ["settings"],
|
|
60
|
+
requestBody: ComposerPreferencesPatchSchema,
|
|
61
|
+
responseBody: ResponseSchema,
|
|
62
|
+
handler: async (args) => {
|
|
63
|
+
const patch = ComposerPreferencesPatchSchema.safeParse(args.body);
|
|
64
|
+
if (!patch.success) {
|
|
65
|
+
throw new BadRequestError(patch.error.message);
|
|
66
|
+
}
|
|
67
|
+
const principalId = await principalFor(args);
|
|
68
|
+
updateComposerPreferences(principalId, patch.data);
|
|
69
|
+
await publishSyncInvalidation(
|
|
70
|
+
[SYNC_TAGS.assistantComposerPreferences],
|
|
71
|
+
args.headers?.["x-vellum-client-id"],
|
|
72
|
+
);
|
|
73
|
+
return responseFor(principalId);
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
];
|