@vellumai/assistant 0.12.2-dev.202609172017.3da279e → 0.12.2-dev.202609172115.0b9be8f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openapi.yaml +3 -3
- package/package.json +1 -1
- package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
- package/src/__tests__/llm-catalog-parity.test.ts +2 -2
- package/src/config/__tests__/default-provider.test.ts +3 -1
- package/src/config/profile-text-generation.test.ts +2 -2
- package/src/config/schemas/llm.ts +1 -1
- package/src/mcp/__tests__/manager-tool-caps.test.ts +32 -20
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +119 -1
- package/src/plugins/defaults/memory/v3/orchestrate.ts +3 -1
- package/src/plugins/defaults/memory/v3/pool-select.ts +263 -6
- package/src/providers/inference/adapter-factory.ts +1 -1
- package/src/providers/jev/client.test.ts +10 -0
- package/src/providers/jev/client.ts +16 -1
- package/src/providers/model-catalog.ts +4 -3
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
- package/src/runtime/routes/secret-routes.ts +1 -1
package/openapi.yaml
CHANGED
|
@@ -16556,7 +16556,7 @@ paths:
|
|
|
16556
16556
|
type: string
|
|
16557
16557
|
description:
|
|
16558
16558
|
"Filter by provider id. One of: anthropic, openai, gemini, ollama, fireworks, together, openrouter,
|
|
16559
|
-
vercel-ai-gateway, litellm, opencode, openai-compatible, minimax, atlascloud, baseten, poolside,
|
|
16559
|
+
vercel-ai-gateway, litellm, opencode, openai-compatible, minimax, atlascloud, baseten, poolside, typesafe,
|
|
16560
16560
|
vellum"
|
|
16561
16561
|
responses:
|
|
16562
16562
|
"200":
|
|
@@ -16829,7 +16829,7 @@ paths:
|
|
|
16829
16829
|
type: string
|
|
16830
16830
|
description:
|
|
16831
16831
|
"Filter by provider. One of: anthropic, openai, gemini, ollama, fireworks, together, openrouter,
|
|
16832
|
-
vercel-ai-gateway, litellm, opencode, openai-compatible, minimax, atlascloud, baseten, poolside,
|
|
16832
|
+
vercel-ai-gateway, litellm, opencode, openai-compatible, minimax, atlascloud, baseten, poolside, typesafe,
|
|
16833
16833
|
vellum, chatgpt"
|
|
16834
16834
|
responses:
|
|
16835
16835
|
"200":
|
|
@@ -38400,7 +38400,7 @@ components:
|
|
|
38400
38400
|
- atlascloud
|
|
38401
38401
|
- baseten
|
|
38402
38402
|
- poolside
|
|
38403
|
-
-
|
|
38403
|
+
- typesafe
|
|
38404
38404
|
- vellum
|
|
38405
38405
|
- chatgpt
|
|
38406
38406
|
Auth:
|
package/package.json
CHANGED
|
@@ -234,12 +234,12 @@ describe("LLM catalog parity: daemon vs client", () => {
|
|
|
234
234
|
});
|
|
235
235
|
|
|
236
236
|
test("jev-latest opts out of chat text generation", () => {
|
|
237
|
-
expect(catalogModelSupportsText("
|
|
237
|
+
expect(catalogModelSupportsText("typesafe", "jev-latest")).toBe(false);
|
|
238
238
|
expect(catalogModelSupportsText("anthropic", "claude-opus-4-8")).toBe(true);
|
|
239
239
|
expect(catalogModelSupportsText("openai-compatible", "local-model")).toBe(
|
|
240
240
|
true,
|
|
241
241
|
);
|
|
242
|
-
expect(DEFAULT_PROVIDER_CHOICES).not.toContain("
|
|
242
|
+
expect(DEFAULT_PROVIDER_CHOICES).not.toContain("typesafe");
|
|
243
243
|
expect(DEFAULT_PROVIDER_CHOICES).toContain("poolside");
|
|
244
244
|
});
|
|
245
245
|
|
|
@@ -73,7 +73,9 @@ describe("LLMSchema.defaultProvider", () => {
|
|
|
73
73
|
});
|
|
74
74
|
|
|
75
75
|
test("rejects a structured-decision catalog provider", () => {
|
|
76
|
-
expect(() =>
|
|
76
|
+
expect(() =>
|
|
77
|
+
DefaultProviderSchema.parse({ provider: "typesafe" }),
|
|
78
|
+
).toThrow();
|
|
77
79
|
});
|
|
78
80
|
|
|
79
81
|
test("rejects an empty connectionName", () => {
|
|
@@ -9,7 +9,7 @@ describe("profileSupportsTextGeneration", () => {
|
|
|
9
9
|
test("false for a structured-decision catalog model", () => {
|
|
10
10
|
expect(
|
|
11
11
|
profileSupportsTextGeneration(
|
|
12
|
-
{ provider: "
|
|
12
|
+
{ provider: "typesafe", model: "jev-latest" },
|
|
13
13
|
{},
|
|
14
14
|
),
|
|
15
15
|
).toBe(false);
|
|
@@ -35,7 +35,7 @@ describe("profileSupportsTextGeneration", () => {
|
|
|
35
35
|
profileSupportsTextGeneration(
|
|
36
36
|
{ mix: [{ profile: "jev" }, { profile: "balanced" }] },
|
|
37
37
|
{
|
|
38
|
-
jev: { provider: "
|
|
38
|
+
jev: { provider: "typesafe", model: "jev-latest" },
|
|
39
39
|
balanced: { provider: "anthropic", model: "claude-opus-4-8" },
|
|
40
40
|
},
|
|
41
41
|
),
|
|
@@ -62,7 +62,7 @@ export const KNOWN_LLM_PROVIDERS = [
|
|
|
62
62
|
"opencode",
|
|
63
63
|
"baseten",
|
|
64
64
|
"poolside",
|
|
65
|
-
"
|
|
65
|
+
"typesafe",
|
|
66
66
|
// Routing identities: "vellum" = the platform-managed route (upstream
|
|
67
67
|
// derived from the model at dispatch) and the catalog owner of
|
|
68
68
|
// Vellum-hosted GPU models; "chatgpt" = the subscription route to OpenAI.
|
|
@@ -10,7 +10,8 @@ import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
|
10
10
|
import type { ResolvedMcpConfig } from "../../config/schemas/mcp.js";
|
|
11
11
|
|
|
12
12
|
const toolsByServer = new Map<string, Array<{ name: string }>>();
|
|
13
|
-
const
|
|
13
|
+
const connectGates = new Map<string, Promise<void>>();
|
|
14
|
+
const listedServers = new Set<string>();
|
|
14
15
|
let mcpGlobalMaxTools: number | undefined;
|
|
15
16
|
|
|
16
17
|
mock.module("../../config/loader.js", () => ({
|
|
@@ -29,12 +30,13 @@ mock.module("../client.js", () => ({
|
|
|
29
30
|
return null;
|
|
30
31
|
}
|
|
31
32
|
async connect() {
|
|
32
|
-
const
|
|
33
|
-
if (
|
|
34
|
-
await
|
|
33
|
+
const gate = connectGates.get(this.serverId);
|
|
34
|
+
if (gate) {
|
|
35
|
+
await gate;
|
|
35
36
|
}
|
|
36
37
|
}
|
|
37
38
|
async listTools() {
|
|
39
|
+
listedServers.add(this.serverId);
|
|
38
40
|
return (toolsByServer.get(this.serverId) ?? []).map((tool) => ({
|
|
39
41
|
name: tool.name,
|
|
40
42
|
description: `${this.serverId} ${tool.name}`,
|
|
@@ -66,7 +68,8 @@ function configWith(ids: string[]): ResolvedMcpConfig {
|
|
|
66
68
|
describe("McpServerManager tool selection", () => {
|
|
67
69
|
beforeEach(() => {
|
|
68
70
|
toolsByServer.clear();
|
|
69
|
-
|
|
71
|
+
connectGates.clear();
|
|
72
|
+
listedServers.clear();
|
|
70
73
|
mcpGlobalMaxTools = undefined;
|
|
71
74
|
});
|
|
72
75
|
|
|
@@ -98,23 +101,32 @@ describe("McpServerManager tool selection", () => {
|
|
|
98
101
|
test("a slow earlier server does not prevent later servers from connecting", async () => {
|
|
99
102
|
toolsByServer.set("slow", [{ name: "slow_tool" }]);
|
|
100
103
|
toolsByServer.set("fast", [{ name: "fast_tool" }]);
|
|
101
|
-
|
|
104
|
+
let releaseSlow!: () => void;
|
|
105
|
+
connectGates.set(
|
|
106
|
+
"slow",
|
|
107
|
+
new Promise<void>((resolve) => {
|
|
108
|
+
releaseSlow = resolve;
|
|
109
|
+
}),
|
|
110
|
+
);
|
|
102
111
|
|
|
103
112
|
const manager = new McpServerManager();
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
113
|
+
const starting = manager.start(configWith(["slow", "fast"]));
|
|
114
|
+
try {
|
|
115
|
+
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
116
|
+
expect([...listedServers]).toEqual(["fast"]);
|
|
117
|
+
|
|
118
|
+
releaseSlow();
|
|
119
|
+
const started = await starting;
|
|
120
|
+
expect(started.connectedServerCount).toBe(2);
|
|
121
|
+
expect(started.servers.map((server) => server.serverId).sort()).toEqual([
|
|
122
|
+
"fast",
|
|
123
|
+
"slow",
|
|
124
|
+
]);
|
|
125
|
+
} finally {
|
|
126
|
+
releaseSlow();
|
|
127
|
+
await starting;
|
|
128
|
+
await manager.stop();
|
|
129
|
+
}
|
|
118
130
|
});
|
|
119
131
|
|
|
120
132
|
test("a workspace global-max override raises how many tools are kept", async () => {
|
|
@@ -85,7 +85,7 @@ mock.module("../../../../../util/logger.js", () => ({
|
|
|
85
85
|
}),
|
|
86
86
|
}));
|
|
87
87
|
|
|
88
|
-
const { selectPool, MemoryV3RetrievalUnavailableError } =
|
|
88
|
+
const { selectPool, MemoryV3RetrievalUnavailableError, TYPE_SAFE_POOL_KEEP_NOUL } =
|
|
89
89
|
await import("../pool-select.js");
|
|
90
90
|
type SelectorPool = Parameters<typeof selectPool>[0];
|
|
91
91
|
|
|
@@ -968,3 +968,121 @@ describe("selectPool: cataloged thinking and forced-tool compatibility", () => {
|
|
|
968
968
|
expect(selection.pages).toEqual([{ slug: "topic-x", sections: [] }]);
|
|
969
969
|
});
|
|
970
970
|
});
|
|
971
|
+
|
|
972
|
+
// ---------------------------------------------------------------------------
|
|
973
|
+
// selectPool: TypeSafe System One noul-per-candidate path.
|
|
974
|
+
// ---------------------------------------------------------------------------
|
|
975
|
+
|
|
976
|
+
function typesafeResponse(answers: Record<string, unknown>): ProviderResponse {
|
|
977
|
+
return {
|
|
978
|
+
model: "jev-latest",
|
|
979
|
+
stopReason: "end_turn",
|
|
980
|
+
usage: { inputTokens: 0, outputTokens: 0 },
|
|
981
|
+
content: [{ type: "text", text: JSON.stringify(answers, null, 2) }],
|
|
982
|
+
rawResponse: { answers },
|
|
983
|
+
};
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
function makeTypesafeProvider(response: ProviderResponse): Provider {
|
|
987
|
+
return {
|
|
988
|
+
name: "typesafe",
|
|
989
|
+
sendMessage: async (messages, options) => {
|
|
990
|
+
providerCalls.push({ messages, options });
|
|
991
|
+
return response;
|
|
992
|
+
},
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
function noulAnswer(noul: number): { type: "noul"; noul: number } {
|
|
997
|
+
return { type: "noul", noul };
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
describe("selectPool: TypeSafe System One", () => {
|
|
1001
|
+
test("sends one noul per candidate and no select_pages tool", async () => {
|
|
1002
|
+
providerStub = makeTypesafeProvider(
|
|
1003
|
+
typesafeResponse({
|
|
1004
|
+
"1": noulAnswer(0.9),
|
|
1005
|
+
"2": noulAnswer(0.1),
|
|
1006
|
+
"3": noulAnswer(0.8),
|
|
1007
|
+
"4": noulAnswer(0.2),
|
|
1008
|
+
}),
|
|
1009
|
+
);
|
|
1010
|
+
|
|
1011
|
+
await selectPool(makePool(), makeTurn("rollout?"));
|
|
1012
|
+
|
|
1013
|
+
expect(providerCalls).toHaveLength(1);
|
|
1014
|
+
const [call] = providerCalls;
|
|
1015
|
+
expect(call.options?.tools).toBeUndefined();
|
|
1016
|
+
expect(
|
|
1017
|
+
(call.options?.config as Record<string, unknown> | undefined)?.tool_choice,
|
|
1018
|
+
).toBeUndefined();
|
|
1019
|
+
expect(
|
|
1020
|
+
(call.options?.config as Record<string, unknown> | undefined)?.callSite,
|
|
1021
|
+
).toBe("memoryV3SelectL2");
|
|
1022
|
+
|
|
1023
|
+
const payload = JSON.parse(
|
|
1024
|
+
(call.messages[0]!.content[0] as { text: string }).text,
|
|
1025
|
+
) as {
|
|
1026
|
+
state: {
|
|
1027
|
+
candidates: Record<string, { slug: string; text: string }>;
|
|
1028
|
+
current_message: string;
|
|
1029
|
+
selector_instructions: string;
|
|
1030
|
+
};
|
|
1031
|
+
questions: Record<string, { type: string; instructions: string }>;
|
|
1032
|
+
};
|
|
1033
|
+
expect(Object.keys(payload.questions)).toEqual(["1", "2", "3", "4"]);
|
|
1034
|
+
expect(payload.questions["1"]?.type).toBe("noul");
|
|
1035
|
+
expect(payload.questions["1"]?.instructions).toContain("`candidates.1`");
|
|
1036
|
+
expect(payload.state.candidates["1"]?.slug).toBe("page-a");
|
|
1037
|
+
expect(payload.state.candidates["1"]?.text).toBe(CARD_A);
|
|
1038
|
+
expect(payload.state.candidates["3"]?.slug).toBe("topic-x");
|
|
1039
|
+
expect(payload.state.current_message).toBe("rollout?");
|
|
1040
|
+
expect(payload.state.selector_instructions.length).toBeGreaterThan(0);
|
|
1041
|
+
});
|
|
1042
|
+
|
|
1043
|
+
test("keeps candidates at or above the inclusive noul threshold", async () => {
|
|
1044
|
+
providerStub = makeTypesafeProvider(
|
|
1045
|
+
typesafeResponse({
|
|
1046
|
+
"1": noulAnswer(TYPE_SAFE_POOL_KEEP_NOUL),
|
|
1047
|
+
"2": noulAnswer(TYPE_SAFE_POOL_KEEP_NOUL - 0.01),
|
|
1048
|
+
"3": noulAnswer(0.91),
|
|
1049
|
+
"4": noulAnswer(0.12),
|
|
1050
|
+
}),
|
|
1051
|
+
);
|
|
1052
|
+
|
|
1053
|
+
const result = await selectPool(makePool(), makeTurn("rollout?"));
|
|
1054
|
+
expect(result.keptAll).toBe(false);
|
|
1055
|
+
expect(result.pages).toEqual([
|
|
1056
|
+
{ slug: "page-a", sections: [] },
|
|
1057
|
+
{ slug: "topic-x", sections: [] },
|
|
1058
|
+
]);
|
|
1059
|
+
});
|
|
1060
|
+
|
|
1061
|
+
test("an all-below-threshold pool is a deliberate empty selection", async () => {
|
|
1062
|
+
providerStub = makeTypesafeProvider(
|
|
1063
|
+
typesafeResponse({
|
|
1064
|
+
"1": noulAnswer(0.1),
|
|
1065
|
+
"2": noulAnswer(0.2),
|
|
1066
|
+
"3": noulAnswer(0.05),
|
|
1067
|
+
"4": noulAnswer(0.3),
|
|
1068
|
+
}),
|
|
1069
|
+
);
|
|
1070
|
+
|
|
1071
|
+
const result = await selectPool(makePool(), makeTurn("nothing relevant"));
|
|
1072
|
+
expect(result).toEqual({ pages: [], keptAll: false });
|
|
1073
|
+
});
|
|
1074
|
+
|
|
1075
|
+
test("unusable answers throw after the re-prompt retry", async () => {
|
|
1076
|
+
providerStub = makeTypesafeProvider({
|
|
1077
|
+
model: "jev-latest",
|
|
1078
|
+
stopReason: "end_turn",
|
|
1079
|
+
usage: { inputTokens: 0, outputTokens: 0 },
|
|
1080
|
+
content: [{ type: "text", text: "not-json" }],
|
|
1081
|
+
});
|
|
1082
|
+
|
|
1083
|
+
await expect(selectPool(makePool(), makeTurn("x"))).rejects.toThrow(
|
|
1084
|
+
MemoryV3RetrievalUnavailableError,
|
|
1085
|
+
);
|
|
1086
|
+
expect(providerCalls).toHaveLength(3);
|
|
1087
|
+
});
|
|
1088
|
+
});
|
|
@@ -67,7 +67,9 @@
|
|
|
67
67
|
* 3. A SINGLE forced-tool select (`selectPool`) over the whole pool. The
|
|
68
68
|
* result is this turn's selections — current turn only. Cross-turn
|
|
69
69
|
* persistence is the injector's job (net-new blocks frozen into history),
|
|
70
|
-
* not a per-turn re-rendered carry set.
|
|
70
|
+
* not a per-turn re-rendered carry set. When the call site resolves to
|
|
71
|
+
* TypeSafe, `selectPool` asks one System One noul per numbered candidate
|
|
72
|
+
* instead of forcing `select_pages`.
|
|
71
73
|
*/
|
|
72
74
|
|
|
73
75
|
import type { AssistantConfig } from "../../../../config/schema.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Memory v3 — single pool selector.
|
|
3
3
|
*
|
|
4
|
-
* Runs a SINGLE
|
|
4
|
+
* Runs a SINGLE selector call over one unified candidate pool rendered in
|
|
5
5
|
* two segments that share one numbering:
|
|
6
6
|
*
|
|
7
7
|
* 1. STABLE PREFIX — the core+hot lane pages as FULL CARDS (head section +
|
|
@@ -52,6 +52,8 @@
|
|
|
52
52
|
import type {
|
|
53
53
|
ContentBlock,
|
|
54
54
|
Message,
|
|
55
|
+
Provider,
|
|
56
|
+
ProviderResponse,
|
|
55
57
|
ToolUseContent,
|
|
56
58
|
} from "@vellumai/plugin-api";
|
|
57
59
|
import { getConfiguredProvider, safeStringSlice } from "@vellumai/plugin-api";
|
|
@@ -179,7 +181,8 @@ type PoolSelectorAttemptFailureReason =
|
|
|
179
181
|
| "provider_error"
|
|
180
182
|
| "missing_tool_use"
|
|
181
183
|
| "unexpected_tool_name"
|
|
182
|
-
| "schema_mismatch"
|
|
184
|
+
| "schema_mismatch"
|
|
185
|
+
| "unusable_answers";
|
|
183
186
|
|
|
184
187
|
interface PoolSelectorAttemptFailure {
|
|
185
188
|
attempt: number;
|
|
@@ -493,6 +496,248 @@ export function selectAllPoolCandidates(pool: SelectorPool): SelectedPage[] {
|
|
|
493
496
|
);
|
|
494
497
|
}
|
|
495
498
|
|
|
499
|
+
/**
|
|
500
|
+
* Inclusive noul threshold for TypeSafe pool selection. 0.5 is calibrated
|
|
501
|
+
* equal yes/no. This sits slightly below that so a plausible candidate is
|
|
502
|
+
* kept, matching the selector's recall-heavy rule.
|
|
503
|
+
*/
|
|
504
|
+
export const TYPE_SAFE_POOL_KEEP_NOUL = 0.4;
|
|
505
|
+
|
|
506
|
+
/** Catalog id of the TypeSafe System One provider. */
|
|
507
|
+
const TYPE_SAFE_PROVIDER_ID = "typesafe";
|
|
508
|
+
|
|
509
|
+
type TypesafeNoulQuestion = {
|
|
510
|
+
type: "noul";
|
|
511
|
+
instructions: string;
|
|
512
|
+
criteria: { yes: string; no: string };
|
|
513
|
+
};
|
|
514
|
+
|
|
515
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
516
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function noulFromAnswer(answer: unknown): number | undefined {
|
|
520
|
+
if (typeof answer === "number" && Number.isFinite(answer)) {
|
|
521
|
+
return answer;
|
|
522
|
+
}
|
|
523
|
+
if (
|
|
524
|
+
isRecord(answer) &&
|
|
525
|
+
typeof answer.noul === "number" &&
|
|
526
|
+
Number.isFinite(answer.noul)
|
|
527
|
+
) {
|
|
528
|
+
return answer.noul;
|
|
529
|
+
}
|
|
530
|
+
return undefined;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function typesafeCandidateEntries(
|
|
534
|
+
pool: SelectorPool,
|
|
535
|
+
): Record<string, { slug: Slug; text: string }> {
|
|
536
|
+
const entries: Record<string, { slug: Slug; text: string }> = {};
|
|
537
|
+
pool.stable.forEach((candidate, index) => {
|
|
538
|
+
entries[String(index + 1)] = {
|
|
539
|
+
slug: candidate.slug,
|
|
540
|
+
text: candidate.card,
|
|
541
|
+
};
|
|
542
|
+
});
|
|
543
|
+
pool.finder.forEach((candidate, index) => {
|
|
544
|
+
entries[String(pool.stable.length + index + 1)] = {
|
|
545
|
+
slug: candidate.slug,
|
|
546
|
+
text: renderFinderLine(candidate),
|
|
547
|
+
};
|
|
548
|
+
});
|
|
549
|
+
return entries;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
function typesafeKeepQuestion(id: string): TypesafeNoulQuestion {
|
|
553
|
+
return {
|
|
554
|
+
type: "noul",
|
|
555
|
+
instructions:
|
|
556
|
+
`Would the upcoming assistant reply draw on \`candidates.${id}\`? ` +
|
|
557
|
+
"Lean inclusive. Facts, current task and event state, register, " +
|
|
558
|
+
"framing, calibration, and relationship texture all count. True when " +
|
|
559
|
+
"the candidate could plausibly inform the reply.",
|
|
560
|
+
criteria: {
|
|
561
|
+
yes: "The reply would draw on this candidate.",
|
|
562
|
+
no: "The reply would not draw on this candidate.",
|
|
563
|
+
},
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
function answersFromSelectorResponse(
|
|
568
|
+
response: ProviderResponse,
|
|
569
|
+
): Record<string, unknown> | null {
|
|
570
|
+
const raw = response.rawResponse;
|
|
571
|
+
if (isRecord(raw) && isRecord(raw.answers)) {
|
|
572
|
+
return raw.answers;
|
|
573
|
+
}
|
|
574
|
+
const textBlock = response.content.find((block) => block.type === "text");
|
|
575
|
+
if (!textBlock || textBlock.type !== "text") {
|
|
576
|
+
return null;
|
|
577
|
+
}
|
|
578
|
+
try {
|
|
579
|
+
const parsed: unknown = JSON.parse(textBlock.text);
|
|
580
|
+
return isRecord(parsed) ? parsed : null;
|
|
581
|
+
} catch {
|
|
582
|
+
return null;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
async function selectPoolWithTypesafe(
|
|
587
|
+
pool: SelectorPool,
|
|
588
|
+
turn: MemoryRoutingTurn,
|
|
589
|
+
ordered: PoolLine[],
|
|
590
|
+
systemPrompt: string,
|
|
591
|
+
provider: Provider,
|
|
592
|
+
): Promise<PoolSelection> {
|
|
593
|
+
const candidates = typesafeCandidateEntries(pool);
|
|
594
|
+
const questions: Record<string, TypesafeNoulQuestion> = {};
|
|
595
|
+
for (const id of Object.keys(candidates)) {
|
|
596
|
+
questions[id] = typesafeKeepQuestion(id);
|
|
597
|
+
}
|
|
598
|
+
const state = {
|
|
599
|
+
selector_instructions: systemPrompt,
|
|
600
|
+
candidates,
|
|
601
|
+
...(turn.situationalContext
|
|
602
|
+
? { situation: turn.situationalContext }
|
|
603
|
+
: {}),
|
|
604
|
+
recent_context: turn.recentContext,
|
|
605
|
+
current_message: turn.currentMessage,
|
|
606
|
+
};
|
|
607
|
+
const userMsg: Message = {
|
|
608
|
+
role: "user",
|
|
609
|
+
content: [
|
|
610
|
+
{
|
|
611
|
+
type: "text",
|
|
612
|
+
text: JSON.stringify({ state, questions }),
|
|
613
|
+
},
|
|
614
|
+
],
|
|
615
|
+
};
|
|
616
|
+
|
|
617
|
+
const failures: PoolSelectorAttemptFailure[] = [];
|
|
618
|
+
let attempt = 0;
|
|
619
|
+
const recordFailure = (
|
|
620
|
+
failure: Omit<
|
|
621
|
+
PoolSelectorAttemptFailure,
|
|
622
|
+
| "callSite"
|
|
623
|
+
| "providerName"
|
|
624
|
+
| "candidateCount"
|
|
625
|
+
| "stableCount"
|
|
626
|
+
| "finderCount"
|
|
627
|
+
>,
|
|
628
|
+
): void => {
|
|
629
|
+
const diagnostic: PoolSelectorAttemptFailure = {
|
|
630
|
+
...failure,
|
|
631
|
+
callSite: MEMORY_V3_SELECT_CALL_SITE,
|
|
632
|
+
providerName: provider.name,
|
|
633
|
+
candidateCount: ordered.length,
|
|
634
|
+
stableCount: pool.stable.length,
|
|
635
|
+
finderCount: pool.finder.length,
|
|
636
|
+
};
|
|
637
|
+
failures.push(diagnostic);
|
|
638
|
+
log.warn(diagnostic, "pool selector attempt failed");
|
|
639
|
+
};
|
|
640
|
+
|
|
641
|
+
let lastError: unknown = null;
|
|
642
|
+
const parsed = await retryForResult(async () => {
|
|
643
|
+
attempt += 1;
|
|
644
|
+
let response: Awaited<ReturnType<typeof provider.sendMessage>>;
|
|
645
|
+
try {
|
|
646
|
+
response = await provider.sendMessage([userMsg], {
|
|
647
|
+
config: {
|
|
648
|
+
callSite: MEMORY_V3_SELECT_CALL_SITE,
|
|
649
|
+
conversationId: turn.conversationId,
|
|
650
|
+
disableTurnStartCache: true,
|
|
651
|
+
},
|
|
652
|
+
});
|
|
653
|
+
lastError = null;
|
|
654
|
+
} catch (error) {
|
|
655
|
+
lastError = error;
|
|
656
|
+
recordFailure({
|
|
657
|
+
attempt,
|
|
658
|
+
reason: "provider_error",
|
|
659
|
+
error: summarizeError(error),
|
|
660
|
+
});
|
|
661
|
+
throw error;
|
|
662
|
+
}
|
|
663
|
+
const answers = answersFromSelectorResponse(response);
|
|
664
|
+
if (!answers) {
|
|
665
|
+
recordFailure({
|
|
666
|
+
attempt,
|
|
667
|
+
reason: "unusable_answers",
|
|
668
|
+
response: summarizeResponse(response),
|
|
669
|
+
});
|
|
670
|
+
return null;
|
|
671
|
+
}
|
|
672
|
+
const picked: number[] = [];
|
|
673
|
+
let parsedCount = 0;
|
|
674
|
+
for (let index = 0; index < ordered.length; index++) {
|
|
675
|
+
const noul = noulFromAnswer(answers[String(index + 1)]);
|
|
676
|
+
if (noul === undefined) {
|
|
677
|
+
continue;
|
|
678
|
+
}
|
|
679
|
+
parsedCount += 1;
|
|
680
|
+
if (noul >= TYPE_SAFE_POOL_KEEP_NOUL) {
|
|
681
|
+
picked.push(index);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
if (parsedCount === 0) {
|
|
685
|
+
recordFailure({
|
|
686
|
+
attempt,
|
|
687
|
+
reason: "unusable_answers",
|
|
688
|
+
response: summarizeResponse(response),
|
|
689
|
+
});
|
|
690
|
+
return null;
|
|
691
|
+
}
|
|
692
|
+
return { pages: mergeSelectedLines(ordered, picked), keptAll: false };
|
|
693
|
+
});
|
|
694
|
+
|
|
695
|
+
if (parsed === null) {
|
|
696
|
+
if (lastError !== null) {
|
|
697
|
+
const detail =
|
|
698
|
+
lastError instanceof Error ? lastError.message : String(lastError);
|
|
699
|
+
const redactedDetail = truncate(
|
|
700
|
+
redactLogString(detail),
|
|
701
|
+
ERROR_MESSAGE_MAX_CHARS,
|
|
702
|
+
);
|
|
703
|
+
log.warn(
|
|
704
|
+
{
|
|
705
|
+
candidateCount: ordered.length,
|
|
706
|
+
stableCount: pool.stable.length,
|
|
707
|
+
finderCount: pool.finder.length,
|
|
708
|
+
callSite: MEMORY_V3_SELECT_CALL_SITE,
|
|
709
|
+
providerName: provider.name,
|
|
710
|
+
failures,
|
|
711
|
+
},
|
|
712
|
+
"pool selector provider call failed after retries",
|
|
713
|
+
);
|
|
714
|
+
throw new MemoryV3RetrievalUnavailableError(
|
|
715
|
+
`memory-v3 pool selector provider call failed after retries: ${redactedDetail}`,
|
|
716
|
+
{
|
|
717
|
+
cause: lastError,
|
|
718
|
+
conversationNotice: providerBillingNoticeFromError(lastError),
|
|
719
|
+
},
|
|
720
|
+
);
|
|
721
|
+
}
|
|
722
|
+
log.warn(
|
|
723
|
+
{
|
|
724
|
+
candidateCount: ordered.length,
|
|
725
|
+
stableCount: pool.stable.length,
|
|
726
|
+
finderCount: pool.finder.length,
|
|
727
|
+
callSite: MEMORY_V3_SELECT_CALL_SITE,
|
|
728
|
+
providerName: provider.name,
|
|
729
|
+
failures,
|
|
730
|
+
},
|
|
731
|
+
"pool selector returned no usable TypeSafe answers after retries",
|
|
732
|
+
);
|
|
733
|
+
throw new MemoryV3RetrievalUnavailableError(
|
|
734
|
+
"memory-v3 pool selector returned no usable selection after retries",
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
return parsed;
|
|
739
|
+
}
|
|
740
|
+
|
|
496
741
|
/** A selection plus whether it came from the recall-safe keep-all fallback. */
|
|
497
742
|
export interface PoolSelection {
|
|
498
743
|
pages: SelectedPage[];
|
|
@@ -504,14 +749,16 @@ export interface PoolSelection {
|
|
|
504
749
|
}
|
|
505
750
|
|
|
506
751
|
/**
|
|
507
|
-
* Run the single
|
|
752
|
+
* Run the single selector over the unified candidate pool. Returns
|
|
508
753
|
* the pages to inject, merged per slug (a page selected as a card and on
|
|
509
754
|
* finder lines yields one entry carrying every selected section), plus a
|
|
510
755
|
* `keptAll` flag marking the recall-safe fallback.
|
|
511
756
|
*
|
|
512
|
-
*
|
|
513
|
-
* relevant" signal, `keptAll: true`); an explicit `[]`
|
|
514
|
-
*
|
|
757
|
+
* On a chat model, an omitted `ids` keeps ALL candidates (the recall-safe
|
|
758
|
+
* "all of these are relevant" signal, `keptAll: true`); an explicit `[]`
|
|
759
|
+
* keeps none. TypeSafe answers one noul per candidate and never omits ids,
|
|
760
|
+
* so `keptAll` is always false on that path. An infrastructure failure
|
|
761
|
+
* (after a short re-prompt retry) throws
|
|
515
762
|
* {@link MemoryV3RetrievalUnavailableError}, and the orchestrator keeps the
|
|
516
763
|
* stable prefix unjudged in its place.
|
|
517
764
|
*
|
|
@@ -545,6 +792,16 @@ export async function selectPool(
|
|
|
545
792
|
);
|
|
546
793
|
}
|
|
547
794
|
|
|
795
|
+
if (provider.name === TYPE_SAFE_PROVIDER_ID) {
|
|
796
|
+
return selectPoolWithTypesafe(
|
|
797
|
+
pool,
|
|
798
|
+
turn,
|
|
799
|
+
ordered,
|
|
800
|
+
systemPrompt,
|
|
801
|
+
provider,
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
|
|
548
805
|
// Two content blocks: the stable prefix (cards) carries the cache
|
|
549
806
|
// breakpoint; the dynamic tail (finder lines + per-turn context) does not.
|
|
550
807
|
// See the module doc for the cache contract.
|
|
@@ -206,7 +206,7 @@ const ADAPTER_FACTORIES: Record<string, AdapterFactory> = {
|
|
|
206
206
|
streamTimeoutMs,
|
|
207
207
|
...(baseURL ? { baseURL } : {}),
|
|
208
208
|
}),
|
|
209
|
-
|
|
209
|
+
typesafe: ({ apiKey, model, streamTimeoutMs, baseURL }) =>
|
|
210
210
|
new JevProvider(apiKey, model, {
|
|
211
211
|
streamTimeoutMs,
|
|
212
212
|
...(baseURL ? { baseURL } : {}),
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
conversationToState,
|
|
6
6
|
DEFAULT_JEV_MODEL,
|
|
7
7
|
JevProvider,
|
|
8
|
+
noulFromAnswer,
|
|
8
9
|
parseSystemOneOverride,
|
|
9
10
|
validateJevApiKey,
|
|
10
11
|
} from "./client.js";
|
|
@@ -66,6 +67,15 @@ describe("parseSystemOneOverride", () => {
|
|
|
66
67
|
});
|
|
67
68
|
});
|
|
68
69
|
|
|
70
|
+
describe("noulFromAnswer", () => {
|
|
71
|
+
test("reads a typed noul object or a bare number", () => {
|
|
72
|
+
expect(noulFromAnswer({ type: "noul", noul: 0.91 })).toBe(0.91);
|
|
73
|
+
expect(noulFromAnswer(0.4)).toBe(0.4);
|
|
74
|
+
expect(noulFromAnswer({ type: "choice", choice: "keep" })).toBeUndefined();
|
|
75
|
+
expect(noulFromAnswer(null)).toBeUndefined();
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
|
|
69
79
|
describe("conversationToState", () => {
|
|
70
80
|
test("flattens system prompt and turns into a string", () => {
|
|
71
81
|
const state = conversationToState(
|
|
@@ -15,7 +15,7 @@ import type {
|
|
|
15
15
|
|
|
16
16
|
const log = getLogger("jev-client");
|
|
17
17
|
|
|
18
|
-
export const JEV_PROVIDER_ID = "
|
|
18
|
+
export const JEV_PROVIDER_ID = "typesafe";
|
|
19
19
|
export const DEFAULT_JEV_BASE_URL = "https://api.typesafe.ai";
|
|
20
20
|
export const DEFAULT_JEV_MODEL = "jev-latest";
|
|
21
21
|
|
|
@@ -115,6 +115,21 @@ function isJevQuestions(value: unknown): value is JevQuestions {
|
|
|
115
115
|
return entries.every(([, question]) => isJevQuestion(question));
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
+
/** Calibrated P(yes) from a System One noul answer, or undefined when absent. */
|
|
119
|
+
export function noulFromAnswer(answer: unknown): number | undefined {
|
|
120
|
+
if (typeof answer === "number" && Number.isFinite(answer)) {
|
|
121
|
+
return answer;
|
|
122
|
+
}
|
|
123
|
+
if (
|
|
124
|
+
isRecord(answer) &&
|
|
125
|
+
typeof answer.noul === "number" &&
|
|
126
|
+
Number.isFinite(answer.noul)
|
|
127
|
+
) {
|
|
128
|
+
return answer.noul;
|
|
129
|
+
}
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
|
|
118
133
|
/**
|
|
119
134
|
* If the last user message is a TypeSafe System One payload, use it as the
|
|
120
135
|
* evaluation request. `state` is optional; callers fall back to the
|
|
@@ -77,7 +77,8 @@ export interface CatalogModel {
|
|
|
77
77
|
* Whether the model produces free-form chat text. Omit (or true) for
|
|
78
78
|
* ordinary chat models. False for structured-decision models that return
|
|
79
79
|
* answers rather than generated text; those stay out of conversation
|
|
80
|
-
* pickers and cannot be the conversation model.
|
|
80
|
+
* pickers and cannot be the conversation model. They can still back a
|
|
81
|
+
* saved profile and a call-site pin.
|
|
81
82
|
*/
|
|
82
83
|
supportsText?: boolean;
|
|
83
84
|
supportsEffort?: boolean;
|
|
@@ -2532,8 +2533,8 @@ const RAW_PROVIDER_CATALOG: ProviderCatalogEntry[] = [
|
|
|
2532
2533
|
apiKeyPlaceholder: "Your Poolside API key",
|
|
2533
2534
|
},
|
|
2534
2535
|
{
|
|
2535
|
-
id: "
|
|
2536
|
-
displayName: "
|
|
2536
|
+
id: "typesafe",
|
|
2537
|
+
displayName: "TypeSafe",
|
|
2537
2538
|
subtitle:
|
|
2538
2539
|
"TypeSafe System One decision model. Returns structured answers, not generated text. Requires a TypeSafe API key.",
|
|
2539
2540
|
setupMode: "api-key",
|
|
@@ -303,7 +303,7 @@ async function handleAddSecret({ body }: RouteHandlerArgs) {
|
|
|
303
303
|
);
|
|
304
304
|
return { success: false, error: validation.reason };
|
|
305
305
|
}
|
|
306
|
-
} else if (name === "
|
|
306
|
+
} else if (name === "typesafe") {
|
|
307
307
|
const validation = await validateJevApiKey(value);
|
|
308
308
|
if (!validation.valid) {
|
|
309
309
|
log.warn(
|