@vellumai/assistant 0.12.4-staging.1 → 0.12.4-staging.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openapi.yaml +3 -1
- package/package.json +1 -1
- package/src/__tests__/config-loader-backfill.test.ts +1 -0
- package/src/__tests__/default-profile-catalog-fallback.test.ts +1 -0
- package/src/__tests__/inference-profile-validation-jev.test.ts +17 -0
- package/src/__tests__/llm-catalog-parity.test.ts +4 -0
- package/src/__tests__/llm-resolver-override-or-default.test.ts +43 -0
- package/src/__tests__/openai-responses-prompt-cache.test.ts +3 -1
- package/src/__tests__/plugin-api-model-profiles.test.ts +7 -0
- package/src/__tests__/pricing.test.ts +33 -0
- package/src/__tests__/skill-load-tool.test.ts +42 -11
- package/src/__tests__/skill-tool-factory.test.ts +50 -2
- package/src/__tests__/validate-input.test.ts +73 -0
- package/src/__tests__/workspace-migration-159-rename-colliding-jev-profile-name.test.ts +197 -0
- package/src/api/requests/dictation.ts +2 -2
- package/src/calls/__tests__/typesafe-noul.test.ts +66 -0
- package/src/calls/typesafe-noul.ts +8 -2
- package/src/calls/voice-escalation-judge.ts +5 -4
- package/src/config/__tests__/default-profile-catalog.test.ts +33 -1
- package/src/config/__tests__/jev-managed-profile.test.ts +82 -0
- package/src/config/call-site-defaults.ts +11 -11
- package/src/config/default-profile-catalog.ts +88 -18
- package/src/config/default-profile-names.ts +35 -0
- package/src/config/inference-profile-validation.ts +7 -6
- package/src/config/llm-resolver.ts +16 -0
- package/src/config/profile-text-generation.test.ts +9 -0
- package/src/config/schemas/call-site-catalog.ts +1 -1
- package/src/config/schemas/llm.ts +54 -9
- package/src/daemon/conversation-slash.ts +2 -2
- package/src/daemon/dictation-text-processing.ts +4 -4
- package/src/plugin-api/model-profiles.ts +5 -3
- package/src/plugins/defaults/memory/substrate/skill-content.ts +1 -1
- package/src/plugins/defaults/memory/v3/__tests__/render-injection.test.ts +3 -0
- package/src/providers/__tests__/connection-model-compat.test.ts +2 -0
- package/src/providers/anthropic/client.ts +2 -2
- package/src/providers/model-catalog.ts +257 -4
- package/src/providers/openai/codex-models.ts +2 -0
- package/src/runtime/routes/__tests__/dictation-processing.test.ts +174 -0
- package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +16 -1
- package/src/runtime/routes/conversation-routes.ts +4 -2
- package/src/runtime/routes/diagnostics-routes.ts +66 -229
- package/src/runtime/routes/llm-call-sites-routes.ts +19 -10
- package/src/skills/validate-input.ts +75 -11
- package/src/tools/skills/execute.ts +1 -1
- package/src/tools/skills/format-input-schema.ts +15 -0
- package/src/tools/skills/load.ts +2 -26
- package/src/tools/skills/skill-tool-factory.ts +2 -1
- package/src/tools/workflows/manage-workflows.ts +4 -2
- package/src/workspace/migrations/159-rename-colliding-jev-profile-name.ts +344 -0
- package/src/workspace/migrations/registry.ts +2 -0
- package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +0 -107
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { afterAll, describe, expect, mock, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type { Provider } from "../../providers/types.js";
|
|
4
|
+
|
|
5
|
+
let configured: {
|
|
6
|
+
provider: Provider;
|
|
7
|
+
configuredProviderName: string;
|
|
8
|
+
entryRouted: boolean;
|
|
9
|
+
} | null = null;
|
|
10
|
+
|
|
11
|
+
const actual = await import("../../providers/provider-send-message.js");
|
|
12
|
+
mock.module("../../providers/provider-send-message.js", () => ({
|
|
13
|
+
...actual,
|
|
14
|
+
resolveConfiguredProvider: async () => configured,
|
|
15
|
+
}));
|
|
16
|
+
|
|
17
|
+
const { resolveTypesafeProvider } = await import("../typesafe-noul.js");
|
|
18
|
+
|
|
19
|
+
afterAll(() => {
|
|
20
|
+
mock.restore();
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
function stub(name: string): Provider {
|
|
24
|
+
return {
|
|
25
|
+
name,
|
|
26
|
+
defaultModel: "jev-latest",
|
|
27
|
+
sendMessage: async () => {
|
|
28
|
+
throw new Error("not dispatched in this test");
|
|
29
|
+
},
|
|
30
|
+
} as unknown as Provider;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe("resolveTypesafeProvider", () => {
|
|
34
|
+
test("a BYOK TypeSafe profile resolves", async () => {
|
|
35
|
+
configured = {
|
|
36
|
+
provider: stub("typesafe"),
|
|
37
|
+
configuredProviderName: "typesafe",
|
|
38
|
+
entryRouted: false,
|
|
39
|
+
};
|
|
40
|
+
expect((await resolveTypesafeProvider("voiceEscalationJudge"))?.name).toBe(
|
|
41
|
+
"typesafe",
|
|
42
|
+
);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("a managed Jev profile resolves although it is configured as vellum", async () => {
|
|
46
|
+
configured = {
|
|
47
|
+
provider: stub("typesafe"),
|
|
48
|
+
configuredProviderName: "vellum",
|
|
49
|
+
entryRouted: false,
|
|
50
|
+
};
|
|
51
|
+
expect((await resolveTypesafeProvider("voiceEscalationJudge"))?.name).toBe(
|
|
52
|
+
"typesafe",
|
|
53
|
+
);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("any other dispatched provider is unavailable", async () => {
|
|
57
|
+
configured = {
|
|
58
|
+
provider: stub("anthropic"),
|
|
59
|
+
configuredProviderName: "vellum",
|
|
60
|
+
entryRouted: false,
|
|
61
|
+
};
|
|
62
|
+
expect(await resolveTypesafeProvider("voiceEscalationJudge")).toBeNull();
|
|
63
|
+
configured = null;
|
|
64
|
+
expect(await resolveTypesafeProvider("voiceEscalationJudge")).toBeNull();
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -28,12 +28,18 @@ export interface TypesafeNoulResult {
|
|
|
28
28
|
latencyMs: number;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* The call site's provider when it resolves to TypeSafe, else null. Judged
|
|
33
|
+
* on the dispatched adapter's name rather than the configured provider: a
|
|
34
|
+
* managed profile is configured as `vellum` and only substitutes the
|
|
35
|
+
* TypeSafe upstream at dispatch, so the configured name alone would report
|
|
36
|
+
* the managed Jev profile as unavailable.
|
|
37
|
+
*/
|
|
32
38
|
export async function resolveTypesafeProvider(
|
|
33
39
|
callSite: LLMCallSite,
|
|
34
40
|
): Promise<Provider | null> {
|
|
35
41
|
const resolved = await resolveConfiguredProvider(callSite);
|
|
36
|
-
return resolved?.
|
|
42
|
+
return resolved?.provider.name === TYPESAFE_PROVIDER_NAME
|
|
37
43
|
? resolved.provider
|
|
38
44
|
: null;
|
|
39
45
|
}
|
|
@@ -9,10 +9,11 @@
|
|
|
9
9
|
* correct reply need something the front door cannot do from here? A
|
|
10
10
|
* confident yes overrules a front-door answer and hands the turn off.
|
|
11
11
|
*
|
|
12
|
-
* The judge
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* The judge uses managed Jev by default when Vellum-managed inference is
|
|
13
|
+
* available. An explicit TypeSafe call-site override can use a user-owned
|
|
14
|
+
* connection. Every other outcome (unavailable route, provider error,
|
|
15
|
+
* timeout, unparseable answer) is a "clear" verdict, so the front door's own
|
|
16
|
+
* decision stands exactly as it would without the judge.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
import type { Message, Provider } from "../providers/types.js";
|
|
@@ -5,6 +5,7 @@ import { resolveModelIntent } from "../../providers/model-intents.js";
|
|
|
5
5
|
import { CALL_SITE_DEFAULTS } from "../call-site-defaults.js";
|
|
6
6
|
import {
|
|
7
7
|
CODE_DEFAULT_PROFILE_ENTRIES,
|
|
8
|
+
getConversationProfilesForProvider,
|
|
8
9
|
getEffectiveProfile,
|
|
9
10
|
getEffectiveProfiles,
|
|
10
11
|
getEffectiveProfilesForProvider,
|
|
@@ -59,6 +60,37 @@ describe("getEffectiveProfiles", () => {
|
|
|
59
60
|
}
|
|
60
61
|
});
|
|
61
62
|
|
|
63
|
+
test("the managed Jev profile routes jev-latest through TypeSafe and is managed-only", () => {
|
|
64
|
+
const jev = CODE_DEFAULT_PROFILE_ENTRIES["jev-managed"];
|
|
65
|
+
expect(jev.model).toBe("jev-latest");
|
|
66
|
+
expect(jev.provider_connection).toBeUndefined();
|
|
67
|
+
expect(resolveRoutingIdentity(jev.provider, jev.model)).toEqual({
|
|
68
|
+
connectionName: "vellum",
|
|
69
|
+
expectedProvider: "typesafe",
|
|
70
|
+
});
|
|
71
|
+
expect(
|
|
72
|
+
resolveDefaultProfileForProvider(undefined, "jev-managed", null),
|
|
73
|
+
).toBeDefined();
|
|
74
|
+
expect(
|
|
75
|
+
resolveDefaultProfileForProvider(undefined, "jev-managed", {
|
|
76
|
+
provider: "anthropic",
|
|
77
|
+
connectionName: "anthropic-personal",
|
|
78
|
+
}),
|
|
79
|
+
).toBeUndefined();
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("the conversation view drops the managed Jev profile but keeps it selectable for call sites", () => {
|
|
83
|
+
expect(
|
|
84
|
+
Object.keys(getConversationProfilesForProvider(undefined, null)),
|
|
85
|
+
).not.toContain("jev-managed");
|
|
86
|
+
expect(
|
|
87
|
+
Object.keys(getUserSelectableProfilesForProvider(undefined, null)),
|
|
88
|
+
).toContain("jev-managed");
|
|
89
|
+
expect(
|
|
90
|
+
Object.keys(getConversationProfilesForProvider(undefined, null)),
|
|
91
|
+
).toContain("balanced");
|
|
92
|
+
});
|
|
93
|
+
|
|
62
94
|
test("the managed Balanced profile routes GLM 5.3 Flash through Fireworks", () => {
|
|
63
95
|
const balanced = CODE_DEFAULT_PROFILE_ENTRIES.balanced;
|
|
64
96
|
expect(balanced.model).toBe("accounts/fireworks/models/glm-5p3-flash");
|
|
@@ -80,7 +112,7 @@ describe("getEffectiveProfiles", () => {
|
|
|
80
112
|
test("defaults absent from the workspace resolve from the catalog; os-beta stays flag-gated", () => {
|
|
81
113
|
const effective = getEffectiveProfiles(undefined);
|
|
82
114
|
expect(Object.keys(effective).sort()).toEqual(
|
|
83
|
-
[...DEFAULT_PROFILE_KEYS, ...BACKUP_PROFILE_KEYS].sort(),
|
|
115
|
+
[...DEFAULT_PROFILE_KEYS, ...BACKUP_PROFILE_KEYS, "jev-managed"].sort(),
|
|
84
116
|
);
|
|
85
117
|
expect(getEffectiveProfile({}, "balanced")?.model).toBe(
|
|
86
118
|
CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string,
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { LLMSchema } from "../schemas/llm.js";
|
|
4
|
+
|
|
5
|
+
const PIN = { callSites: { voiceEscalationJudge: { profile: "jev-managed" } } };
|
|
6
|
+
|
|
7
|
+
describe("jev-managed as a call-site pin target", () => {
|
|
8
|
+
test("resolves under the managed default provider", () => {
|
|
9
|
+
expect(LLMSchema.safeParse(PIN).success).toBe(true);
|
|
10
|
+
expect(
|
|
11
|
+
LLMSchema.safeParse({
|
|
12
|
+
...PIN,
|
|
13
|
+
defaultProvider: { provider: "vellum" },
|
|
14
|
+
}).success,
|
|
15
|
+
).toBe(true);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("can be pinned only to the verdict-consuming call sites", () => {
|
|
19
|
+
for (const site of [
|
|
20
|
+
"voiceEscalationJudge",
|
|
21
|
+
"voiceContinuationJudge",
|
|
22
|
+
"memoryV3SelectL2",
|
|
23
|
+
]) {
|
|
24
|
+
expect(
|
|
25
|
+
LLMSchema.safeParse({
|
|
26
|
+
callSites: { [site]: { profile: "jev-managed" } },
|
|
27
|
+
}).success,
|
|
28
|
+
).toBe(true);
|
|
29
|
+
}
|
|
30
|
+
const result = LLMSchema.safeParse({
|
|
31
|
+
callSites: { conversationTitle: { profile: "jev-managed" } },
|
|
32
|
+
});
|
|
33
|
+
expect(result.success).toBe(false);
|
|
34
|
+
expect(JSON.stringify(result.error?.issues)).toContain(
|
|
35
|
+
"can only be pinned to",
|
|
36
|
+
);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("cannot be a mix constituent", () => {
|
|
40
|
+
const result = LLMSchema.safeParse({
|
|
41
|
+
profiles: {
|
|
42
|
+
blend: {
|
|
43
|
+
mix: [
|
|
44
|
+
{ profile: "jev-managed", weight: 1 },
|
|
45
|
+
{ profile: "balanced", weight: 1 },
|
|
46
|
+
],
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
expect(result.success).toBe(false);
|
|
51
|
+
expect(JSON.stringify(result.error?.issues)).toContain(
|
|
52
|
+
"cannot be a mix constituent",
|
|
53
|
+
);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("is rejected as the active or advisor profile even on the managed column", () => {
|
|
57
|
+
for (const field of ["activeProfile", "advisorProfile"] as const) {
|
|
58
|
+
const result = LLMSchema.safeParse({ [field]: "jev-managed" });
|
|
59
|
+
expect(result.success).toBe(false);
|
|
60
|
+
expect(JSON.stringify(result.error?.issues)).toContain(
|
|
61
|
+
"structured answers rather than chat text",
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
expect(LLMSchema.safeParse({ activeProfile: "balanced" }).success).toBe(
|
|
65
|
+
true,
|
|
66
|
+
);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("is rejected under a BYOK default provider, where it has no body", () => {
|
|
70
|
+
const result = LLMSchema.safeParse({
|
|
71
|
+
...PIN,
|
|
72
|
+
defaultProvider: {
|
|
73
|
+
provider: "anthropic",
|
|
74
|
+
connectionName: "anthropic-personal",
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
expect(result.success).toBe(false);
|
|
78
|
+
expect(JSON.stringify(result.error?.issues)).toContain(
|
|
79
|
+
"managed Jev profile",
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { JEV_MANAGED_PROFILE_KEY } from "./default-profile-names.js";
|
|
1
2
|
import { type LLMCallSite } from "./schemas/llm.js";
|
|
2
3
|
|
|
3
4
|
type CallSiteDefaultConfig = {
|
|
@@ -123,9 +124,8 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
|
|
|
123
124
|
effort: "low",
|
|
124
125
|
thinking: { enabled: false },
|
|
125
126
|
},
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
// answers, so it is priced as a latency call.
|
|
127
|
+
// Voice intent classification and explicit selected-text edits are
|
|
128
|
+
// latency-sensitive: the caller is waiting for insertion or dispatch.
|
|
129
129
|
interactionClassifier: {
|
|
130
130
|
profile: "latency-optimized",
|
|
131
131
|
effort: "low",
|
|
@@ -153,14 +153,14 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
|
|
|
153
153
|
effort: "low",
|
|
154
154
|
thinking: { enabled: false },
|
|
155
155
|
},
|
|
156
|
-
// The
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
//
|
|
163
|
-
// pinned here.
|
|
156
|
+
// The managed-only profile resolves through the Vellum connection. On a
|
|
157
|
+
// BYOK install it has no body, so the judge is unavailable and the front
|
|
158
|
+
// door's own decision stands.
|
|
159
|
+
voiceEscalationJudge: {
|
|
160
|
+
profile: JEV_MANAGED_PROFILE_KEY,
|
|
161
|
+
},
|
|
162
|
+
// The barge-in continuation judge remains off until a TypeSafe profile is
|
|
163
|
+
// pinned here (see live-voice/continuation-judge.ts).
|
|
164
164
|
voiceContinuationJudge: { profile: "cost-optimized" },
|
|
165
165
|
// Names the background continuation a barge-in spawns, from the interrupted
|
|
166
166
|
// transcript. The label is fixed at spawn, so the call runs under a short
|
|
@@ -15,12 +15,14 @@ import {
|
|
|
15
15
|
type DefaultProfileKey,
|
|
16
16
|
type DefaultProfileProvider,
|
|
17
17
|
FALLBACK_PROFILE_BY_KEY,
|
|
18
|
-
isBackupProfileKey,
|
|
19
18
|
isDefaultProfileKey,
|
|
20
19
|
isDefaultProfileProvider,
|
|
20
|
+
isManagedOnlyProfileKey,
|
|
21
|
+
JEV_MANAGED_PROFILE_KEY,
|
|
21
22
|
OS_BETA_PROFILE_KEY,
|
|
22
23
|
} from "./default-profile-names.js";
|
|
23
24
|
import { resolveDefaultConnectionName } from "./default-provider-resolution.js";
|
|
25
|
+
import { profileSupportsTextGeneration } from "./profile-text-generation.js";
|
|
24
26
|
import {
|
|
25
27
|
backupProfilesResolveUnderDefaultProvider,
|
|
26
28
|
DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS,
|
|
@@ -385,18 +387,39 @@ export const PROFILE_IMPLS: Record<
|
|
|
385
387
|
Record<DefaultProfileProvider, DefaultProfileTemplate>
|
|
386
388
|
>;
|
|
387
389
|
|
|
390
|
+
/**
|
|
391
|
+
* The managed Jev profile: TypeSafe's System One decision model through the
|
|
392
|
+
* platform proxy. Managed column only, because the platform is the only
|
|
393
|
+
* route the catalog offers for it here, and never a conversation profile,
|
|
394
|
+
* because `jev-latest` returns structured answers rather than chat text
|
|
395
|
+
* (`supportsText: false` on the `typesafe` catalog entry). Its budget is
|
|
396
|
+
* TypeSafe's published request limit rather than the default context window.
|
|
397
|
+
*/
|
|
398
|
+
export const JEV_MANAGED_PROFILE_TEMPLATE: DefaultProfileTemplate = {
|
|
399
|
+
model: "jev-latest",
|
|
400
|
+
provider: "vellum",
|
|
401
|
+
source: "managed",
|
|
402
|
+
label: "Jev",
|
|
403
|
+
description:
|
|
404
|
+
"TypeSafe decision model for yes/no verdicts, for the voice judges and memory selection",
|
|
405
|
+
maxTokens: 4096,
|
|
406
|
+
thinking: { enabled: false, streamThinking: false },
|
|
407
|
+
contextWindow: { maxInputTokens: 32_000 },
|
|
408
|
+
};
|
|
409
|
+
|
|
388
410
|
/**
|
|
389
411
|
* Managed profiles, i.e. the `vellum` column keyed by profile name, plus the
|
|
390
|
-
* managed backup profiles. Backups come after the
|
|
391
|
-
* places them after the primaries in the seeded
|
|
392
|
-
* inserts missing managed keys in this record's
|
|
393
|
-
* user-facing defaults only: an internal profile is
|
|
394
|
-
* listed or ordered.
|
|
412
|
+
* managed backup profiles and the managed Jev profile. Backups come after the
|
|
413
|
+
* primaries, which is what places them after the primaries in the seeded
|
|
414
|
+
* `profileOrder`: the seeder inserts missing managed keys in this record's
|
|
415
|
+
* order. Keyed by the user-facing defaults only: an internal profile is
|
|
416
|
+
* code-resolved and never listed or ordered.
|
|
395
417
|
*/
|
|
396
418
|
export const MANAGED_PROFILE_TEMPLATES: Record<string, DefaultProfileTemplate> =
|
|
397
419
|
Object.fromEntries([
|
|
398
420
|
...DEFAULT_PROFILE_KEYS.map((key) => [key, PROFILE_IMPLS[key].vellum]),
|
|
399
421
|
...BACKUP_PROFILE_KEYS.map((key) => [key, BACKUP_PROFILE_IMPLS[key]]),
|
|
422
|
+
[JEV_MANAGED_PROFILE_KEY, JEV_MANAGED_PROFILE_TEMPLATE],
|
|
400
423
|
]);
|
|
401
424
|
|
|
402
425
|
/**
|
|
@@ -484,6 +507,9 @@ export const OS_BETA_PROFILE_TEMPLATE: DefaultProfileTemplate = {
|
|
|
484
507
|
export const CODE_OWNED_PROFILE_NAMES = new Set<string>([
|
|
485
508
|
"latency-optimized",
|
|
486
509
|
...BACKUP_PROFILE_KEYS,
|
|
510
|
+
// A shadow could repoint the judges at a chat model, which answers a
|
|
511
|
+
// yes/no question with prose the verdict parser cannot read.
|
|
512
|
+
JEV_MANAGED_PROFILE_KEY,
|
|
487
513
|
]);
|
|
488
514
|
|
|
489
515
|
// All managed profiles, including the flag-gated os-beta, are invariant:
|
|
@@ -494,6 +520,7 @@ export const CODE_OWNED_PROFILE_NAMES = new Set<string>([
|
|
|
494
520
|
export const INVARIANT_PROFILE_NAMES = new Set<string>([
|
|
495
521
|
...DEFAULT_PROFILE_KEYS,
|
|
496
522
|
...BACKUP_PROFILE_KEYS,
|
|
523
|
+
JEV_MANAGED_PROFILE_KEY,
|
|
497
524
|
OS_BETA_PROFILE_KEY,
|
|
498
525
|
]);
|
|
499
526
|
|
|
@@ -507,6 +534,7 @@ export const INVARIANT_PROFILE_NAMES = new Set<string>([
|
|
|
507
534
|
export const MANAGED_PROFILE_NAMES = new Set<string>([
|
|
508
535
|
...DEFAULT_PROFILE_KEYS,
|
|
509
536
|
...BACKUP_PROFILE_KEYS,
|
|
537
|
+
JEV_MANAGED_PROFILE_KEY,
|
|
510
538
|
OS_BETA_PROFILE_KEY,
|
|
511
539
|
]);
|
|
512
540
|
|
|
@@ -590,6 +618,19 @@ for (const key of BACKUP_PROFILE_KEYS) {
|
|
|
590
618
|
}
|
|
591
619
|
}
|
|
592
620
|
|
|
621
|
+
// The managed Jev profile pins a model the platform proxy must front, and the
|
|
622
|
+
// judges depend on it being the non-text TypeSafe model rather than a chat
|
|
623
|
+
// model that happens to share the id.
|
|
624
|
+
if (
|
|
625
|
+
getManagedUpstream(JEV_MANAGED_PROFILE_TEMPLATE.model ?? "") !== "typesafe"
|
|
626
|
+
) {
|
|
627
|
+
throw new Error(
|
|
628
|
+
`JEV_MANAGED_PROFILE_TEMPLATE references model "${JEV_MANAGED_PROFILE_TEMPLATE.model ?? ""}" ` +
|
|
629
|
+
`which the managed route does not serve through typesafe. ` +
|
|
630
|
+
`Update model-catalog.ts, platform-proxy/constants.ts, or default-profile-catalog.ts.`,
|
|
631
|
+
);
|
|
632
|
+
}
|
|
633
|
+
|
|
593
634
|
// Provider choices without a named column materialize from the shared BYOK
|
|
594
635
|
// templates; verify each one's resolved model lands in the catalog.
|
|
595
636
|
for (const provider of DEFAULT_PROVIDER_CHOICES) {
|
|
@@ -621,6 +662,10 @@ function buildDefaultProfileEntries(): Record<string, ProfileEntry> {
|
|
|
621
662
|
const impl = BACKUP_PROFILE_IMPLS[key];
|
|
622
663
|
entries[key] = materializeProfile(impl, impl.provider);
|
|
623
664
|
}
|
|
665
|
+
entries[JEV_MANAGED_PROFILE_KEY] = materializeProfile(
|
|
666
|
+
JEV_MANAGED_PROFILE_TEMPLATE,
|
|
667
|
+
JEV_MANAGED_PROFILE_TEMPLATE.provider,
|
|
668
|
+
);
|
|
624
669
|
entries[OS_BETA_PROFILE_KEY] = materializeProfile(
|
|
625
670
|
OS_BETA_PROFILE_TEMPLATE,
|
|
626
671
|
OS_BETA_PROFILE_TEMPLATE.provider,
|
|
@@ -697,12 +742,13 @@ function resolveAgainstBody(
|
|
|
697
742
|
if (body == null) {
|
|
698
743
|
// A managed stub is the workspace's slot for a code-owned profile, never
|
|
699
744
|
// a profile of its own, so with no body behind it there is nothing to
|
|
700
|
-
// resolve. Only a
|
|
701
|
-
// default keys always have a body), and it
|
|
702
|
-
//
|
|
703
|
-
// profile with no provider or model and let a
|
|
704
|
-
// valid, which is precisely what `LLMSchema` rejects
|
|
705
|
-
|
|
745
|
+
// resolve. Only a managed-only name (a backup, the Jev profile) reaches
|
|
746
|
+
// this branch with a stub (the default keys always have a body), and it
|
|
747
|
+
// reaches it on the columns where those profiles do not exist. Resolving
|
|
748
|
+
// to the stub would list a profile with no provider or model and let a
|
|
749
|
+
// reference to it look valid, which is precisely what `LLMSchema` rejects
|
|
750
|
+
// on those columns.
|
|
751
|
+
if (isManagedOnlyProfileKey(name) && workspace?.source === "managed") {
|
|
706
752
|
return undefined;
|
|
707
753
|
}
|
|
708
754
|
return workspace;
|
|
@@ -787,14 +833,14 @@ function defaultProfileBodyForProvider(
|
|
|
787
833
|
defaultProvider: DefaultProviderConfig | null,
|
|
788
834
|
): ProfileEntry | undefined {
|
|
789
835
|
if (!isDefaultProfileKey(name)) {
|
|
790
|
-
// Backup profiles
|
|
791
|
-
// under a BYOK or chatgpt default provider the
|
|
792
|
-
// `fallbackProfile` pointers, and
|
|
793
|
-
//
|
|
836
|
+
// Backup profiles and the managed Jev profile exist on the managed
|
|
837
|
+
// (`vellum`) column only: under a BYOK or chatgpt default provider the
|
|
838
|
+
// primaries carry no `fallbackProfile` pointers, and neither may
|
|
839
|
+
// materialize, since the install may hold no credential for their
|
|
794
840
|
// provider. A null defaultProvider predates `llm.defaultProvider` and
|
|
795
|
-
// resolves to the managed column, so it keeps
|
|
841
|
+
// resolves to the managed column, so it keeps them.
|
|
796
842
|
if (
|
|
797
|
-
|
|
843
|
+
isManagedOnlyProfileKey(name) &&
|
|
798
844
|
!backupProfilesResolveUnderDefaultProvider(defaultProvider)
|
|
799
845
|
) {
|
|
800
846
|
return undefined;
|
|
@@ -923,3 +969,27 @@ export function getUserSelectableProfilesForProvider(
|
|
|
923
969
|
}
|
|
924
970
|
return selectable;
|
|
925
971
|
}
|
|
972
|
+
|
|
973
|
+
/**
|
|
974
|
+
* The profiles a conversation can run on: the user-selectable view minus any
|
|
975
|
+
* profile whose model returns structured answers rather than chat text (the
|
|
976
|
+
* managed Jev profile). Every surface that picks or validates a conversation
|
|
977
|
+
* profile (the `/model` command, a per-conversation pin, the plugin and
|
|
978
|
+
* workflow listings) reads this view; the call-site override pickers keep
|
|
979
|
+
* the wider selectable view, since the verdict sites are what Jev is for.
|
|
980
|
+
*/
|
|
981
|
+
export function getConversationProfilesForProvider(
|
|
982
|
+
workspaceProfiles: Record<string, ProfileEntry> | undefined,
|
|
983
|
+
defaultProvider: DefaultProviderConfig | null,
|
|
984
|
+
): Record<string, ProfileEntry> {
|
|
985
|
+
const selectable = getUserSelectableProfilesForProvider(
|
|
986
|
+
workspaceProfiles,
|
|
987
|
+
defaultProvider,
|
|
988
|
+
);
|
|
989
|
+
for (const [name, entry] of Object.entries(selectable)) {
|
|
990
|
+
if (!profileSupportsTextGeneration(entry, selectable)) {
|
|
991
|
+
delete selectable[name];
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
return selectable;
|
|
995
|
+
}
|
|
@@ -67,6 +67,41 @@ export function isBackupProfileKey(value: string): value is BackupProfileKey {
|
|
|
67
67
|
return (BACKUP_PROFILE_KEYS as readonly string[]).includes(value);
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
/**
|
|
71
|
+
* Managed profile for the TypeSafe decision model (Jev). Deliberately NOT
|
|
72
|
+
* part of `DEFAULT_PROFILE_KEYS`: it has no BYOK column, because the
|
|
73
|
+
* platform serves Jev only through the managed route, and its model returns
|
|
74
|
+
* structured answers rather than chat text, so it can never be the active
|
|
75
|
+
* or conversation profile. It exists as a pin target for the call sites
|
|
76
|
+
* that consume a yes/no verdict (the voice judges, the memory selector).
|
|
77
|
+
*/
|
|
78
|
+
export const JEV_MANAGED_PROFILE_KEY = "jev-managed";
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The call sites that consume a structured verdict and so may be pinned to
|
|
82
|
+
* the Jev profile. Every other site expects chat text or a tool call, which
|
|
83
|
+
* the decision model never produces.
|
|
84
|
+
*/
|
|
85
|
+
export const JEV_MANAGED_PROFILE_CALL_SITES = [
|
|
86
|
+
"voiceEscalationJudge",
|
|
87
|
+
"voiceContinuationJudge",
|
|
88
|
+
"memoryV3SelectL2",
|
|
89
|
+
] as const;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Keys whose profile exists on the managed (`vellum`) column only: the
|
|
93
|
+
* backups and the Jev profile. Under a BYOK or ChatGPT default provider
|
|
94
|
+
* these names have no body to resolve to.
|
|
95
|
+
*/
|
|
96
|
+
export const MANAGED_ONLY_PROFILE_KEYS = [
|
|
97
|
+
...BACKUP_PROFILE_KEYS,
|
|
98
|
+
JEV_MANAGED_PROFILE_KEY,
|
|
99
|
+
] as const;
|
|
100
|
+
|
|
101
|
+
export function isManagedOnlyProfileKey(value: string): boolean {
|
|
102
|
+
return (MANAGED_ONLY_PROFILE_KEYS as readonly string[]).includes(value);
|
|
103
|
+
}
|
|
104
|
+
|
|
70
105
|
/**
|
|
71
106
|
* Flag-gated default profile: only available while the `os-beta` feature
|
|
72
107
|
* flag has reconciled it into the workspace (see `sync-gated-profiles.ts`).
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getConversationProfilesForProvider } from "./default-profile-catalog.js";
|
|
2
2
|
import { getConfigReadOnly } from "./loader.js";
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -6,17 +6,18 @@ import { getConfigReadOnly } from "./loader.js";
|
|
|
6
6
|
* (code-defined defaults + workspace `llm.profiles`). Returns a user-facing
|
|
7
7
|
* error message when the key is empty or unknown, or `null` when valid.
|
|
8
8
|
*
|
|
9
|
-
* Resolved through the `llm.defaultProvider`-aware
|
|
10
|
-
*
|
|
11
|
-
* routes remain available to automatic
|
|
12
|
-
*
|
|
9
|
+
* Resolved through the `llm.defaultProvider`-aware conversation view, since
|
|
10
|
+
* every caller (schedules, subagent spawns, per-conversation pins) runs chat
|
|
11
|
+
* turns on the result. Managed backup routes remain available to automatic
|
|
12
|
+
* fallback resolution, and the managed Jev profile to the verdict call
|
|
13
|
+
* sites, but neither is reported as a selectable profile here.
|
|
13
14
|
*/
|
|
14
15
|
export function validateInferenceProfileKey(profile: string): string | null {
|
|
15
16
|
if (!profile.trim()) {
|
|
16
17
|
return "inferenceProfile must be a non-empty string";
|
|
17
18
|
}
|
|
18
19
|
const llm = getConfigReadOnly().llm;
|
|
19
|
-
const profiles =
|
|
20
|
+
const profiles = getConversationProfilesForProvider(
|
|
20
21
|
llm?.profiles,
|
|
21
22
|
llm?.defaultProvider ?? null,
|
|
22
23
|
);
|
|
@@ -627,6 +627,22 @@ export function resolveDefaultProfileKey(
|
|
|
627
627
|
return selectWinningProfile(callSite, llm, {}).profileName ?? undefined;
|
|
628
628
|
}
|
|
629
629
|
|
|
630
|
+
/**
|
|
631
|
+
* Returns the code-owned profile tier for catalog display when that tier is
|
|
632
|
+
* available through the workspace's default provider. Workspace call-site
|
|
633
|
+
* pins do not affect this value. A managed-only tier on a BYOK install returns
|
|
634
|
+
* undefined so clients do not report a profile that cannot run there.
|
|
635
|
+
*/
|
|
636
|
+
export function resolveShippedDefaultProfileKey(
|
|
637
|
+
callSite: LLMCallSite,
|
|
638
|
+
llm: z.infer<typeof LLMSchema>,
|
|
639
|
+
): string | undefined {
|
|
640
|
+
const intent = CALL_SITE_DEFAULTS[callSite]?.profile ?? "balanced";
|
|
641
|
+
return usableDefaultIntent(intent, llm, {}, () => undefined) == null
|
|
642
|
+
? undefined
|
|
643
|
+
: intent;
|
|
644
|
+
}
|
|
645
|
+
|
|
630
646
|
/**
|
|
631
647
|
* Returns the profile key that `resolveCallSiteConfig` treats as the winning
|
|
632
648
|
* (highest-precedence) profile for a turn — the profile whose fragment supplies
|
|
@@ -15,6 +15,15 @@ describe("profileSupportsTextGeneration", () => {
|
|
|
15
15
|
).toBe(false);
|
|
16
16
|
});
|
|
17
17
|
|
|
18
|
+
test("false for the managed Jev profile shape", () => {
|
|
19
|
+
expect(
|
|
20
|
+
profileSupportsTextGeneration(
|
|
21
|
+
{ provider: "vellum", model: "jev-latest" },
|
|
22
|
+
{},
|
|
23
|
+
),
|
|
24
|
+
).toBe(false);
|
|
25
|
+
});
|
|
26
|
+
|
|
18
27
|
test("true for a chat model and for unlisted custom ids", () => {
|
|
19
28
|
expect(
|
|
20
29
|
profileSupportsTextGeneration(
|
|
@@ -307,7 +307,7 @@ const CATALOG_RECORD: CatalogRecord = {
|
|
|
307
307
|
id: "voiceEscalationJudge",
|
|
308
308
|
displayName: "Voice Escalation Judge",
|
|
309
309
|
description:
|
|
310
|
-
"Second opinion on the voice front door's escalate decision.
|
|
310
|
+
"Second opinion on the voice front door's escalate decision. Uses managed Jev when Vellum-managed inference is available, or an explicitly configured TypeSafe profile; a confident 'needs tools' verdict hands the turn to the escalated leg.",
|
|
311
311
|
domain: "agentLoop",
|
|
312
312
|
},
|
|
313
313
|
voiceContinuationJudge: {
|