@vellumai/assistant 0.12.4-staging.1 → 0.12.4-staging.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/openapi.yaml +3 -1
  2. package/package.json +1 -1
  3. package/src/__tests__/config-loader-backfill.test.ts +1 -0
  4. package/src/__tests__/default-profile-catalog-fallback.test.ts +1 -0
  5. package/src/__tests__/inference-profile-validation-jev.test.ts +17 -0
  6. package/src/__tests__/llm-catalog-parity.test.ts +4 -0
  7. package/src/__tests__/llm-resolver-override-or-default.test.ts +43 -0
  8. package/src/__tests__/openai-responses-prompt-cache.test.ts +3 -1
  9. package/src/__tests__/plugin-api-model-profiles.test.ts +7 -0
  10. package/src/__tests__/pricing.test.ts +33 -0
  11. package/src/__tests__/skill-load-tool.test.ts +42 -11
  12. package/src/__tests__/skill-tool-factory.test.ts +50 -2
  13. package/src/__tests__/validate-input.test.ts +73 -0
  14. package/src/__tests__/workspace-migration-159-rename-colliding-jev-profile-name.test.ts +197 -0
  15. package/src/api/requests/dictation.ts +2 -2
  16. package/src/calls/__tests__/typesafe-noul.test.ts +66 -0
  17. package/src/calls/typesafe-noul.ts +8 -2
  18. package/src/calls/voice-escalation-judge.ts +5 -4
  19. package/src/config/__tests__/default-profile-catalog.test.ts +33 -1
  20. package/src/config/__tests__/jev-managed-profile.test.ts +82 -0
  21. package/src/config/call-site-defaults.ts +11 -11
  22. package/src/config/default-profile-catalog.ts +88 -18
  23. package/src/config/default-profile-names.ts +35 -0
  24. package/src/config/inference-profile-validation.ts +7 -6
  25. package/src/config/llm-resolver.ts +16 -0
  26. package/src/config/profile-text-generation.test.ts +9 -0
  27. package/src/config/schemas/call-site-catalog.ts +1 -1
  28. package/src/config/schemas/llm.ts +54 -9
  29. package/src/daemon/conversation-slash.ts +2 -2
  30. package/src/daemon/dictation-text-processing.ts +4 -4
  31. package/src/plugin-api/model-profiles.ts +5 -3
  32. package/src/plugins/defaults/memory/substrate/skill-content.ts +1 -1
  33. package/src/plugins/defaults/memory/v3/__tests__/render-injection.test.ts +3 -0
  34. package/src/providers/__tests__/connection-model-compat.test.ts +2 -0
  35. package/src/providers/anthropic/client.ts +2 -2
  36. package/src/providers/model-catalog.ts +257 -4
  37. package/src/providers/openai/codex-models.ts +2 -0
  38. package/src/runtime/routes/__tests__/dictation-processing.test.ts +174 -0
  39. package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +16 -1
  40. package/src/runtime/routes/conversation-routes.ts +4 -2
  41. package/src/runtime/routes/diagnostics-routes.ts +66 -229
  42. package/src/runtime/routes/llm-call-sites-routes.ts +19 -10
  43. package/src/skills/validate-input.ts +75 -11
  44. package/src/tools/skills/execute.ts +1 -1
  45. package/src/tools/skills/format-input-schema.ts +15 -0
  46. package/src/tools/skills/load.ts +2 -26
  47. package/src/tools/skills/skill-tool-factory.ts +2 -1
  48. package/src/tools/workflows/manage-workflows.ts +4 -2
  49. package/src/workspace/migrations/159-rename-colliding-jev-profile-name.ts +344 -0
  50. package/src/workspace/migrations/registry.ts +2 -0
  51. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +0 -107
@@ -0,0 +1,66 @@
1
+ import { afterAll, describe, expect, mock, test } from "bun:test";
2
+
3
+ import type { Provider } from "../../providers/types.js";
4
+
5
+ let configured: {
6
+ provider: Provider;
7
+ configuredProviderName: string;
8
+ entryRouted: boolean;
9
+ } | null = null;
10
+
11
+ const actual = await import("../../providers/provider-send-message.js");
12
+ mock.module("../../providers/provider-send-message.js", () => ({
13
+ ...actual,
14
+ resolveConfiguredProvider: async () => configured,
15
+ }));
16
+
17
+ const { resolveTypesafeProvider } = await import("../typesafe-noul.js");
18
+
19
+ afterAll(() => {
20
+ mock.restore();
21
+ });
22
+
23
+ function stub(name: string): Provider {
24
+ return {
25
+ name,
26
+ defaultModel: "jev-latest",
27
+ sendMessage: async () => {
28
+ throw new Error("not dispatched in this test");
29
+ },
30
+ } as unknown as Provider;
31
+ }
32
+
33
+ describe("resolveTypesafeProvider", () => {
34
+ test("a BYOK TypeSafe profile resolves", async () => {
35
+ configured = {
36
+ provider: stub("typesafe"),
37
+ configuredProviderName: "typesafe",
38
+ entryRouted: false,
39
+ };
40
+ expect((await resolveTypesafeProvider("voiceEscalationJudge"))?.name).toBe(
41
+ "typesafe",
42
+ );
43
+ });
44
+
45
+ test("a managed Jev profile resolves although it is configured as vellum", async () => {
46
+ configured = {
47
+ provider: stub("typesafe"),
48
+ configuredProviderName: "vellum",
49
+ entryRouted: false,
50
+ };
51
+ expect((await resolveTypesafeProvider("voiceEscalationJudge"))?.name).toBe(
52
+ "typesafe",
53
+ );
54
+ });
55
+
56
+ test("any other dispatched provider is unavailable", async () => {
57
+ configured = {
58
+ provider: stub("anthropic"),
59
+ configuredProviderName: "vellum",
60
+ entryRouted: false,
61
+ };
62
+ expect(await resolveTypesafeProvider("voiceEscalationJudge")).toBeNull();
63
+ configured = null;
64
+ expect(await resolveTypesafeProvider("voiceEscalationJudge")).toBeNull();
65
+ });
66
+ });
@@ -28,12 +28,18 @@ export interface TypesafeNoulResult {
28
28
  latencyMs: number;
29
29
  }
30
30
 
31
- /** The call site's provider when it resolves to TypeSafe, else null. */
31
+ /**
32
+ * The call site's provider when it resolves to TypeSafe, else null. Judged
33
+ * on the dispatched adapter's name rather than the configured provider: a
34
+ * managed profile is configured as `vellum` and only substitutes the
35
+ * TypeSafe upstream at dispatch, so the configured name alone would report
36
+ * the managed Jev profile as unavailable.
37
+ */
32
38
  export async function resolveTypesafeProvider(
33
39
  callSite: LLMCallSite,
34
40
  ): Promise<Provider | null> {
35
41
  const resolved = await resolveConfiguredProvider(callSite);
36
- return resolved?.configuredProviderName === TYPESAFE_PROVIDER_NAME
42
+ return resolved?.provider.name === TYPESAFE_PROVIDER_NAME
37
43
  ? resolved.provider
38
44
  : null;
39
45
  }
@@ -9,10 +9,11 @@
9
9
  * correct reply need something the front door cannot do from here? A
10
10
  * confident yes overrules a front-door answer and hands the turn off.
11
11
  *
12
- * The judge runs only when the `voiceEscalationJudge` call site resolves to
13
- * the TypeSafe provider. Every other outcome (no such profile, provider
14
- * error, timeout, unparseable answer) is a "clear" verdict, so the front
15
- * door's own decision stands exactly as it would without the judge.
12
+ * The judge uses managed Jev by default when Vellum-managed inference is
13
+ * available. An explicit TypeSafe call-site override can use a user-owned
14
+ * connection. Every other outcome (unavailable route, provider error,
15
+ * timeout, unparseable answer) is a "clear" verdict, so the front door's own
16
+ * decision stands exactly as it would without the judge.
16
17
  */
17
18
 
18
19
  import type { Message, Provider } from "../providers/types.js";
@@ -5,6 +5,7 @@ import { resolveModelIntent } from "../../providers/model-intents.js";
5
5
  import { CALL_SITE_DEFAULTS } from "../call-site-defaults.js";
6
6
  import {
7
7
  CODE_DEFAULT_PROFILE_ENTRIES,
8
+ getConversationProfilesForProvider,
8
9
  getEffectiveProfile,
9
10
  getEffectiveProfiles,
10
11
  getEffectiveProfilesForProvider,
@@ -59,6 +60,37 @@ describe("getEffectiveProfiles", () => {
59
60
  }
60
61
  });
61
62
 
63
+ test("the managed Jev profile routes jev-latest through TypeSafe and is managed-only", () => {
64
+ const jev = CODE_DEFAULT_PROFILE_ENTRIES["jev-managed"];
65
+ expect(jev.model).toBe("jev-latest");
66
+ expect(jev.provider_connection).toBeUndefined();
67
+ expect(resolveRoutingIdentity(jev.provider, jev.model)).toEqual({
68
+ connectionName: "vellum",
69
+ expectedProvider: "typesafe",
70
+ });
71
+ expect(
72
+ resolveDefaultProfileForProvider(undefined, "jev-managed", null),
73
+ ).toBeDefined();
74
+ expect(
75
+ resolveDefaultProfileForProvider(undefined, "jev-managed", {
76
+ provider: "anthropic",
77
+ connectionName: "anthropic-personal",
78
+ }),
79
+ ).toBeUndefined();
80
+ });
81
+
82
+ test("the conversation view drops the managed Jev profile but keeps it selectable for call sites", () => {
83
+ expect(
84
+ Object.keys(getConversationProfilesForProvider(undefined, null)),
85
+ ).not.toContain("jev-managed");
86
+ expect(
87
+ Object.keys(getUserSelectableProfilesForProvider(undefined, null)),
88
+ ).toContain("jev-managed");
89
+ expect(
90
+ Object.keys(getConversationProfilesForProvider(undefined, null)),
91
+ ).toContain("balanced");
92
+ });
93
+
62
94
  test("the managed Balanced profile routes GLM 5.3 Flash through Fireworks", () => {
63
95
  const balanced = CODE_DEFAULT_PROFILE_ENTRIES.balanced;
64
96
  expect(balanced.model).toBe("accounts/fireworks/models/glm-5p3-flash");
@@ -80,7 +112,7 @@ describe("getEffectiveProfiles", () => {
80
112
  test("defaults absent from the workspace resolve from the catalog; os-beta stays flag-gated", () => {
81
113
  const effective = getEffectiveProfiles(undefined);
82
114
  expect(Object.keys(effective).sort()).toEqual(
83
- [...DEFAULT_PROFILE_KEYS, ...BACKUP_PROFILE_KEYS].sort(),
115
+ [...DEFAULT_PROFILE_KEYS, ...BACKUP_PROFILE_KEYS, "jev-managed"].sort(),
84
116
  );
85
117
  expect(getEffectiveProfile({}, "balanced")?.model).toBe(
86
118
  CODE_DEFAULT_PROFILE_ENTRIES.balanced.model as string,
@@ -0,0 +1,82 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { LLMSchema } from "../schemas/llm.js";
4
+
5
+ const PIN = { callSites: { voiceEscalationJudge: { profile: "jev-managed" } } };
6
+
7
+ describe("jev-managed as a call-site pin target", () => {
8
+ test("resolves under the managed default provider", () => {
9
+ expect(LLMSchema.safeParse(PIN).success).toBe(true);
10
+ expect(
11
+ LLMSchema.safeParse({
12
+ ...PIN,
13
+ defaultProvider: { provider: "vellum" },
14
+ }).success,
15
+ ).toBe(true);
16
+ });
17
+
18
+ test("can be pinned only to the verdict-consuming call sites", () => {
19
+ for (const site of [
20
+ "voiceEscalationJudge",
21
+ "voiceContinuationJudge",
22
+ "memoryV3SelectL2",
23
+ ]) {
24
+ expect(
25
+ LLMSchema.safeParse({
26
+ callSites: { [site]: { profile: "jev-managed" } },
27
+ }).success,
28
+ ).toBe(true);
29
+ }
30
+ const result = LLMSchema.safeParse({
31
+ callSites: { conversationTitle: { profile: "jev-managed" } },
32
+ });
33
+ expect(result.success).toBe(false);
34
+ expect(JSON.stringify(result.error?.issues)).toContain(
35
+ "can only be pinned to",
36
+ );
37
+ });
38
+
39
+ test("cannot be a mix constituent", () => {
40
+ const result = LLMSchema.safeParse({
41
+ profiles: {
42
+ blend: {
43
+ mix: [
44
+ { profile: "jev-managed", weight: 1 },
45
+ { profile: "balanced", weight: 1 },
46
+ ],
47
+ },
48
+ },
49
+ });
50
+ expect(result.success).toBe(false);
51
+ expect(JSON.stringify(result.error?.issues)).toContain(
52
+ "cannot be a mix constituent",
53
+ );
54
+ });
55
+
56
+ test("is rejected as the active or advisor profile even on the managed column", () => {
57
+ for (const field of ["activeProfile", "advisorProfile"] as const) {
58
+ const result = LLMSchema.safeParse({ [field]: "jev-managed" });
59
+ expect(result.success).toBe(false);
60
+ expect(JSON.stringify(result.error?.issues)).toContain(
61
+ "structured answers rather than chat text",
62
+ );
63
+ }
64
+ expect(LLMSchema.safeParse({ activeProfile: "balanced" }).success).toBe(
65
+ true,
66
+ );
67
+ });
68
+
69
+ test("is rejected under a BYOK default provider, where it has no body", () => {
70
+ const result = LLMSchema.safeParse({
71
+ ...PIN,
72
+ defaultProvider: {
73
+ provider: "anthropic",
74
+ connectionName: "anthropic-personal",
75
+ },
76
+ });
77
+ expect(result.success).toBe(false);
78
+ expect(JSON.stringify(result.error?.issues)).toContain(
79
+ "managed Jev profile",
80
+ );
81
+ });
82
+ });
@@ -1,3 +1,4 @@
1
+ import { JEV_MANAGED_PROFILE_KEY } from "./default-profile-names.js";
1
2
  import { type LLMCallSite } from "./schemas/llm.js";
2
3
 
3
4
  type CallSiteDefaultConfig = {
@@ -123,9 +124,8 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
123
124
  effort: "low",
124
125
  thinking: { enabled: false },
125
126
  },
126
- // The dictation cleanup pass, and nothing else runs on this site. The
127
- // caller has already stopped speaking and has nothing on screen until this
128
- // answers, so it is priced as a latency call.
127
+ // Voice intent classification and explicit selected-text edits are
128
+ // latency-sensitive: the caller is waiting for insertion or dispatch.
129
129
  interactionClassifier: {
130
130
  profile: "latency-optimized",
131
131
  effort: "low",
@@ -153,14 +153,14 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
153
153
  effort: "low",
154
154
  thinking: { enabled: false },
155
155
  },
156
- // The escalation judge only runs when this site resolves to the TypeSafe
157
- // provider (see calls/voice-escalation-judge.ts), so the shipped default is
158
- // any ordinary profile: the judge stays off until a user pins a TypeSafe
159
- // profile here.
160
- voiceEscalationJudge: { profile: "cost-optimized" },
161
- // Same arrangement for the barge-in continuation judge
162
- // (live-voice/continuation-judge.ts): off until a TypeSafe profile is
163
- // pinned here.
156
+ // The managed-only profile resolves through the Vellum connection. On a
157
+ // BYOK install it has no body, so the judge is unavailable and the front
158
+ // door's own decision stands.
159
+ voiceEscalationJudge: {
160
+ profile: JEV_MANAGED_PROFILE_KEY,
161
+ },
162
+ // The barge-in continuation judge remains off until a TypeSafe profile is
163
+ // pinned here (see live-voice/continuation-judge.ts).
164
164
  voiceContinuationJudge: { profile: "cost-optimized" },
165
165
  // Names the background continuation a barge-in spawns, from the interrupted
166
166
  // transcript. The label is fixed at spawn, so the call runs under a short
@@ -15,12 +15,14 @@ import {
15
15
  type DefaultProfileKey,
16
16
  type DefaultProfileProvider,
17
17
  FALLBACK_PROFILE_BY_KEY,
18
- isBackupProfileKey,
19
18
  isDefaultProfileKey,
20
19
  isDefaultProfileProvider,
20
+ isManagedOnlyProfileKey,
21
+ JEV_MANAGED_PROFILE_KEY,
21
22
  OS_BETA_PROFILE_KEY,
22
23
  } from "./default-profile-names.js";
23
24
  import { resolveDefaultConnectionName } from "./default-provider-resolution.js";
25
+ import { profileSupportsTextGeneration } from "./profile-text-generation.js";
24
26
  import {
25
27
  backupProfilesResolveUnderDefaultProvider,
26
28
  DEFAULT_CONTEXT_WINDOW_MAX_INPUT_TOKENS,
@@ -385,18 +387,39 @@ export const PROFILE_IMPLS: Record<
385
387
  Record<DefaultProfileProvider, DefaultProfileTemplate>
386
388
  >;
387
389
 
390
+ /**
391
+ * The managed Jev profile: TypeSafe's System One decision model through the
392
+ * platform proxy. Managed column only, because the platform is the only
393
+ * route the catalog offers for it here, and never a conversation profile,
394
+ * because `jev-latest` returns structured answers rather than chat text
395
+ * (`supportsText: false` on the `typesafe` catalog entry). Its budget is
396
+ * TypeSafe's published request limit rather than the default context window.
397
+ */
398
+ export const JEV_MANAGED_PROFILE_TEMPLATE: DefaultProfileTemplate = {
399
+ model: "jev-latest",
400
+ provider: "vellum",
401
+ source: "managed",
402
+ label: "Jev",
403
+ description:
404
+ "TypeSafe decision model for yes/no verdicts, for the voice judges and memory selection",
405
+ maxTokens: 4096,
406
+ thinking: { enabled: false, streamThinking: false },
407
+ contextWindow: { maxInputTokens: 32_000 },
408
+ };
409
+
388
410
  /**
389
411
  * Managed profiles, i.e. the `vellum` column keyed by profile name, plus the
390
- * managed backup profiles. Backups come after the primaries, which is what
391
- * places them after the primaries in the seeded `profileOrder`: the seeder
392
- * inserts missing managed keys in this record's order. Keyed by the
393
- * user-facing defaults only: an internal profile is code-resolved and never
394
- * listed or ordered.
412
+ * managed backup profiles and the managed Jev profile. Backups come after the
413
+ * primaries, which is what places them after the primaries in the seeded
414
+ * `profileOrder`: the seeder inserts missing managed keys in this record's
415
+ * order. Keyed by the user-facing defaults only: an internal profile is
416
+ * code-resolved and never listed or ordered.
395
417
  */
396
418
  export const MANAGED_PROFILE_TEMPLATES: Record<string, DefaultProfileTemplate> =
397
419
  Object.fromEntries([
398
420
  ...DEFAULT_PROFILE_KEYS.map((key) => [key, PROFILE_IMPLS[key].vellum]),
399
421
  ...BACKUP_PROFILE_KEYS.map((key) => [key, BACKUP_PROFILE_IMPLS[key]]),
422
+ [JEV_MANAGED_PROFILE_KEY, JEV_MANAGED_PROFILE_TEMPLATE],
400
423
  ]);
401
424
 
402
425
  /**
@@ -484,6 +507,9 @@ export const OS_BETA_PROFILE_TEMPLATE: DefaultProfileTemplate = {
484
507
  export const CODE_OWNED_PROFILE_NAMES = new Set<string>([
485
508
  "latency-optimized",
486
509
  ...BACKUP_PROFILE_KEYS,
510
+ // A shadow could repoint the judges at a chat model, which answers a
511
+ // yes/no question with prose the verdict parser cannot read.
512
+ JEV_MANAGED_PROFILE_KEY,
487
513
  ]);
488
514
 
489
515
  // All managed profiles, including the flag-gated os-beta, are invariant:
@@ -494,6 +520,7 @@ export const CODE_OWNED_PROFILE_NAMES = new Set<string>([
494
520
  export const INVARIANT_PROFILE_NAMES = new Set<string>([
495
521
  ...DEFAULT_PROFILE_KEYS,
496
522
  ...BACKUP_PROFILE_KEYS,
523
+ JEV_MANAGED_PROFILE_KEY,
497
524
  OS_BETA_PROFILE_KEY,
498
525
  ]);
499
526
 
@@ -507,6 +534,7 @@ export const INVARIANT_PROFILE_NAMES = new Set<string>([
507
534
  export const MANAGED_PROFILE_NAMES = new Set<string>([
508
535
  ...DEFAULT_PROFILE_KEYS,
509
536
  ...BACKUP_PROFILE_KEYS,
537
+ JEV_MANAGED_PROFILE_KEY,
510
538
  OS_BETA_PROFILE_KEY,
511
539
  ]);
512
540
 
@@ -590,6 +618,19 @@ for (const key of BACKUP_PROFILE_KEYS) {
590
618
  }
591
619
  }
592
620
 
621
+ // The managed Jev profile pins a model the platform proxy must front, and the
622
+ // judges depend on it being the non-text TypeSafe model rather than a chat
623
+ // model that happens to share the id.
624
+ if (
625
+ getManagedUpstream(JEV_MANAGED_PROFILE_TEMPLATE.model ?? "") !== "typesafe"
626
+ ) {
627
+ throw new Error(
628
+ `JEV_MANAGED_PROFILE_TEMPLATE references model "${JEV_MANAGED_PROFILE_TEMPLATE.model ?? ""}" ` +
629
+ `which the managed route does not serve through typesafe. ` +
630
+ `Update model-catalog.ts, platform-proxy/constants.ts, or default-profile-catalog.ts.`,
631
+ );
632
+ }
633
+
593
634
  // Provider choices without a named column materialize from the shared BYOK
594
635
  // templates; verify each one's resolved model lands in the catalog.
595
636
  for (const provider of DEFAULT_PROVIDER_CHOICES) {
@@ -621,6 +662,10 @@ function buildDefaultProfileEntries(): Record<string, ProfileEntry> {
621
662
  const impl = BACKUP_PROFILE_IMPLS[key];
622
663
  entries[key] = materializeProfile(impl, impl.provider);
623
664
  }
665
+ entries[JEV_MANAGED_PROFILE_KEY] = materializeProfile(
666
+ JEV_MANAGED_PROFILE_TEMPLATE,
667
+ JEV_MANAGED_PROFILE_TEMPLATE.provider,
668
+ );
624
669
  entries[OS_BETA_PROFILE_KEY] = materializeProfile(
625
670
  OS_BETA_PROFILE_TEMPLATE,
626
671
  OS_BETA_PROFILE_TEMPLATE.provider,
@@ -697,12 +742,13 @@ function resolveAgainstBody(
697
742
  if (body == null) {
698
743
  // A managed stub is the workspace's slot for a code-owned profile, never
699
744
  // a profile of its own, so with no body behind it there is nothing to
700
- // resolve. Only a backup name reaches this branch with a stub (the
701
- // default keys always have a body), and it reaches it on the columns
702
- // where the backups do not exist. Resolving to the stub would list a
703
- // profile with no provider or model and let a reference to it look
704
- // valid, which is precisely what `LLMSchema` rejects on those columns.
705
- if (isBackupProfileKey(name) && workspace?.source === "managed") {
745
+ // resolve. Only a managed-only name (a backup, the Jev profile) reaches
746
+ // this branch with a stub (the default keys always have a body), and it
747
+ // reaches it on the columns where those profiles do not exist. Resolving
748
+ // to the stub would list a profile with no provider or model and let a
749
+ // reference to it look valid, which is precisely what `LLMSchema` rejects
750
+ // on those columns.
751
+ if (isManagedOnlyProfileKey(name) && workspace?.source === "managed") {
706
752
  return undefined;
707
753
  }
708
754
  return workspace;
@@ -787,14 +833,14 @@ function defaultProfileBodyForProvider(
787
833
  defaultProvider: DefaultProviderConfig | null,
788
834
  ): ProfileEntry | undefined {
789
835
  if (!isDefaultProfileKey(name)) {
790
- // Backup profiles are companions of the managed (`vellum`) column only:
791
- // under a BYOK or chatgpt default provider the primaries carry no
792
- // `fallbackProfile` pointers, and the backups must not materialize
793
- // either, since the install may hold no credential for the backup's
836
+ // Backup profiles and the managed Jev profile exist on the managed
837
+ // (`vellum`) column only: under a BYOK or chatgpt default provider the
838
+ // primaries carry no `fallbackProfile` pointers, and neither may
839
+ // materialize, since the install may hold no credential for their
794
840
  // provider. A null defaultProvider predates `llm.defaultProvider` and
795
- // resolves to the managed column, so it keeps its backups.
841
+ // resolves to the managed column, so it keeps them.
796
842
  if (
797
- isBackupProfileKey(name) &&
843
+ isManagedOnlyProfileKey(name) &&
798
844
  !backupProfilesResolveUnderDefaultProvider(defaultProvider)
799
845
  ) {
800
846
  return undefined;
@@ -923,3 +969,27 @@ export function getUserSelectableProfilesForProvider(
923
969
  }
924
970
  return selectable;
925
971
  }
972
+
973
+ /**
974
+ * The profiles a conversation can run on: the user-selectable view minus any
975
+ * profile whose model returns structured answers rather than chat text (the
976
+ * managed Jev profile). Every surface that picks or validates a conversation
977
+ * profile (the `/model` command, a per-conversation pin, the plugin and
978
+ * workflow listings) reads this view; the call-site override pickers keep
979
+ * the wider selectable view, since the verdict sites are what Jev is for.
980
+ */
981
+ export function getConversationProfilesForProvider(
982
+ workspaceProfiles: Record<string, ProfileEntry> | undefined,
983
+ defaultProvider: DefaultProviderConfig | null,
984
+ ): Record<string, ProfileEntry> {
985
+ const selectable = getUserSelectableProfilesForProvider(
986
+ workspaceProfiles,
987
+ defaultProvider,
988
+ );
989
+ for (const [name, entry] of Object.entries(selectable)) {
990
+ if (!profileSupportsTextGeneration(entry, selectable)) {
991
+ delete selectable[name];
992
+ }
993
+ }
994
+ return selectable;
995
+ }
@@ -67,6 +67,41 @@ export function isBackupProfileKey(value: string): value is BackupProfileKey {
67
67
  return (BACKUP_PROFILE_KEYS as readonly string[]).includes(value);
68
68
  }
69
69
 
70
+ /**
71
+ * Managed profile for the TypeSafe decision model (Jev). Deliberately NOT
72
+ * part of `DEFAULT_PROFILE_KEYS`: it has no BYOK column, because the
73
+ * platform serves Jev only through the managed route, and its model returns
74
+ * structured answers rather than chat text, so it can never be the active
75
+ * or conversation profile. It exists as a pin target for the call sites
76
+ * that consume a yes/no verdict (the voice judges, the memory selector).
77
+ */
78
+ export const JEV_MANAGED_PROFILE_KEY = "jev-managed";
79
+
80
+ /**
81
+ * The call sites that consume a structured verdict and so may be pinned to
82
+ * the Jev profile. Every other site expects chat text or a tool call, which
83
+ * the decision model never produces.
84
+ */
85
+ export const JEV_MANAGED_PROFILE_CALL_SITES = [
86
+ "voiceEscalationJudge",
87
+ "voiceContinuationJudge",
88
+ "memoryV3SelectL2",
89
+ ] as const;
90
+
91
+ /**
92
+ * Keys whose profile exists on the managed (`vellum`) column only: the
93
+ * backups and the Jev profile. Under a BYOK or ChatGPT default provider
94
+ * these names have no body to resolve to.
95
+ */
96
+ export const MANAGED_ONLY_PROFILE_KEYS = [
97
+ ...BACKUP_PROFILE_KEYS,
98
+ JEV_MANAGED_PROFILE_KEY,
99
+ ] as const;
100
+
101
+ export function isManagedOnlyProfileKey(value: string): boolean {
102
+ return (MANAGED_ONLY_PROFILE_KEYS as readonly string[]).includes(value);
103
+ }
104
+
70
105
  /**
71
106
  * Flag-gated default profile: only available while the `os-beta` feature
72
107
  * flag has reconciled it into the workspace (see `sync-gated-profiles.ts`).
@@ -1,4 +1,4 @@
1
- import { getUserSelectableProfilesForProvider } from "./default-profile-catalog.js";
1
+ import { getConversationProfilesForProvider } from "./default-profile-catalog.js";
2
2
  import { getConfigReadOnly } from "./loader.js";
3
3
 
4
4
  /**
@@ -6,17 +6,18 @@ import { getConfigReadOnly } from "./loader.js";
6
6
  * (code-defined defaults + workspace `llm.profiles`). Returns a user-facing
7
7
  * error message when the key is empty or unknown, or `null` when valid.
8
8
  *
9
- * Resolved through the `llm.defaultProvider`-aware user-selectable view so
10
- * availability matches what a direct profile pin can choose. Managed backup
11
- * routes remain available to automatic fallback resolution but are not
12
- * reported as selectable profiles.
9
+ * Resolved through the `llm.defaultProvider`-aware conversation view, since
10
+ * every caller (schedules, subagent spawns, per-conversation pins) runs chat
11
+ * turns on the result. Managed backup routes remain available to automatic
12
+ * fallback resolution, and the managed Jev profile to the verdict call
13
+ * sites, but neither is reported as a selectable profile here.
13
14
  */
14
15
  export function validateInferenceProfileKey(profile: string): string | null {
15
16
  if (!profile.trim()) {
16
17
  return "inferenceProfile must be a non-empty string";
17
18
  }
18
19
  const llm = getConfigReadOnly().llm;
19
- const profiles = getUserSelectableProfilesForProvider(
20
+ const profiles = getConversationProfilesForProvider(
20
21
  llm?.profiles,
21
22
  llm?.defaultProvider ?? null,
22
23
  );
@@ -627,6 +627,22 @@ export function resolveDefaultProfileKey(
627
627
  return selectWinningProfile(callSite, llm, {}).profileName ?? undefined;
628
628
  }
629
629
 
630
+ /**
631
+ * Returns the code-owned profile tier for catalog display when that tier is
632
+ * available through the workspace's default provider. Workspace call-site
633
+ * pins do not affect this value. A managed-only tier on a BYOK install returns
634
+ * undefined so clients do not report a profile that cannot run there.
635
+ */
636
+ export function resolveShippedDefaultProfileKey(
637
+ callSite: LLMCallSite,
638
+ llm: z.infer<typeof LLMSchema>,
639
+ ): string | undefined {
640
+ const intent = CALL_SITE_DEFAULTS[callSite]?.profile ?? "balanced";
641
+ return usableDefaultIntent(intent, llm, {}, () => undefined) == null
642
+ ? undefined
643
+ : intent;
644
+ }
645
+
630
646
  /**
631
647
  * Returns the profile key that `resolveCallSiteConfig` treats as the winning
632
648
  * (highest-precedence) profile for a turn — the profile whose fragment supplies
@@ -15,6 +15,15 @@ describe("profileSupportsTextGeneration", () => {
15
15
  ).toBe(false);
16
16
  });
17
17
 
18
+ test("false for the managed Jev profile shape", () => {
19
+ expect(
20
+ profileSupportsTextGeneration(
21
+ { provider: "vellum", model: "jev-latest" },
22
+ {},
23
+ ),
24
+ ).toBe(false);
25
+ });
26
+
18
27
  test("true for a chat model and for unlisted custom ids", () => {
19
28
  expect(
20
29
  profileSupportsTextGeneration(
@@ -307,7 +307,7 @@ const CATALOG_RECORD: CatalogRecord = {
307
307
  id: "voiceEscalationJudge",
308
308
  displayName: "Voice Escalation Judge",
309
309
  description:
310
- "Second opinion on the voice front door's escalate decision. Runs only when pinned to a TypeSafe profile; a confident 'needs tools' verdict hands the turn to the escalated leg.",
310
+ "Second opinion on the voice front door's escalate decision. Uses managed Jev when Vellum-managed inference is available, or an explicitly configured TypeSafe profile; a confident 'needs tools' verdict hands the turn to the escalated leg.",
311
311
  domain: "agentLoop",
312
312
  },
313
313
  voiceContinuationJudge: {