@llm4ts/core 3.3.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/ConnectorConfig.d.ts +3 -1
  2. package/dist/ConnectorConfig.d.ts.map +1 -1
  3. package/dist/ConnectorConfig.js +4 -17
  4. package/dist/ConnectorConfig.js.map +1 -1
  5. package/dist/Models.d.ts +177 -22
  6. package/dist/Models.d.ts.map +1 -1
  7. package/dist/Models.js +144 -99
  8. package/dist/Models.js.map +1 -1
  9. package/dist/UsageLimits.d.ts.map +1 -1
  10. package/dist/UsageLimits.js +1 -0
  11. package/dist/UsageLimits.js.map +1 -1
  12. package/dist/eval/Checks.d.ts +1 -1
  13. package/dist/eval/Checks.d.ts.map +1 -1
  14. package/dist/eval/Checks.js.map +1 -1
  15. package/dist/eval/Eval.d.ts +4 -1
  16. package/dist/eval/Eval.d.ts.map +1 -1
  17. package/dist/eval/Eval.js.map +1 -1
  18. package/dist/eval/EvalSuite.d.ts +1 -2
  19. package/dist/eval/EvalSuite.d.ts.map +1 -1
  20. package/dist/eval/EvalSuite.js.map +1 -1
  21. package/dist/eval/Evaluator.d.ts +1 -6
  22. package/dist/eval/Evaluator.d.ts.map +1 -1
  23. package/dist/eval/Evaluator.js.map +1 -1
  24. package/dist/eval/Judge.d.ts +1 -2
  25. package/dist/eval/Judge.d.ts.map +1 -1
  26. package/dist/eval/Judge.js.map +1 -1
  27. package/dist/providers/AntigravityConnector.d.ts.map +1 -1
  28. package/dist/providers/AntigravityConnector.js +5 -4
  29. package/dist/providers/AntigravityConnector.js.map +1 -1
  30. package/dist/providers/ClaudeCliConnector.d.ts.map +1 -1
  31. package/dist/providers/ClaudeCliConnector.js +7 -6
  32. package/dist/providers/ClaudeCliConnector.js.map +1 -1
  33. package/dist/providers/CodexConnector.d.ts.map +1 -1
  34. package/dist/providers/CodexConnector.js +8 -7
  35. package/dist/providers/CodexConnector.js.map +1 -1
  36. package/dist/providers/ConnectorFactories.d.ts.map +1 -1
  37. package/dist/providers/ConnectorFactories.js +26 -20
  38. package/dist/providers/ConnectorFactories.js.map +1 -1
  39. package/dist/providers/CopilotConnector.d.ts.map +1 -1
  40. package/dist/providers/CopilotConnector.js +5 -4
  41. package/dist/providers/CopilotConnector.js.map +1 -1
  42. package/dist/providers/CursorConnector.d.ts.map +1 -1
  43. package/dist/providers/CursorConnector.js +6 -5
  44. package/dist/providers/CursorConnector.js.map +1 -1
  45. package/dist/providers/GeminiAcpSession.d.ts.map +1 -1
  46. package/dist/providers/GeminiAcpSession.js +3 -1
  47. package/dist/providers/GeminiAcpSession.js.map +1 -1
  48. package/dist/providers/GeminiCliProvider.d.ts.map +1 -1
  49. package/dist/providers/GeminiCliProvider.js +4 -3
  50. package/dist/providers/GeminiCliProvider.js.map +1 -1
  51. package/dist/providers/GrokCliConnector.d.ts.map +1 -1
  52. package/dist/providers/GrokCliConnector.js +11 -5
  53. package/dist/providers/GrokCliConnector.js.map +1 -1
  54. package/dist/providers/OpenCodeCliConnector.d.ts.map +1 -1
  55. package/dist/providers/OpenCodeCliConnector.js +6 -5
  56. package/dist/providers/OpenCodeCliConnector.js.map +1 -1
  57. package/dist/providers/PiConnector.d.ts.map +1 -1
  58. package/dist/providers/PiConnector.js +8 -7
  59. package/dist/providers/PiConnector.js.map +1 -1
  60. package/package.json +17 -20
  61. package/src/ConnectorConfig.ts +18 -17
  62. package/src/Models.ts +238 -101
  63. package/src/UsageLimits.ts +1 -0
  64. package/src/eval/Checks.ts +2 -2
  65. package/src/eval/Eval.ts +4 -1
  66. package/src/eval/EvalSuite.ts +1 -2
  67. package/src/eval/Evaluator.ts +1 -6
  68. package/src/eval/Judge.ts +2 -2
  69. package/src/providers/AntigravityConnector.ts +12 -4
  70. package/src/providers/ClaudeCliConnector.ts +9 -6
  71. package/src/providers/CodexConnector.ts +10 -7
  72. package/src/providers/ConnectorFactories.ts +53 -51
  73. package/src/providers/CopilotConnector.ts +6 -4
  74. package/src/providers/CursorConnector.ts +7 -5
  75. package/src/providers/GeminiAcpSession.ts +4 -1
  76. package/src/providers/GeminiCliProvider.ts +6 -3
  77. package/src/providers/GrokCliConnector.ts +12 -5
  78. package/src/providers/OpenCodeCliConnector.ts +7 -5
  79. package/src/providers/PiConnector.ts +15 -7
  80. package/dist/AgentSession.d.ts +0 -66
  81. package/dist/AgentSession.d.ts.map +0 -1
  82. package/dist/AgentSession.js +0 -53
  83. package/dist/AgentSession.js.map +0 -1
  84. package/dist/providers/ClaudeAgentSession.d.ts +0 -11
  85. package/dist/providers/ClaudeAgentSession.d.ts.map +0 -1
  86. package/dist/providers/ClaudeAgentSession.js +0 -161
  87. package/dist/providers/ClaudeAgentSession.js.map +0 -1
  88. package/src/AgentSession.ts +0 -69
  89. package/src/providers/ClaudeAgentSession.ts +0 -230
package/src/Models.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import * as Duration from "effect/Duration"
2
2
  import * as Effect from "effect/Effect"
3
+ import * as Record from "effect/Record"
3
4
  import * as Schema from "effect/Schema"
4
5
 
5
6
  export const LlmProvider = Schema.Literals([
@@ -10,7 +11,6 @@ export const LlmProvider = Schema.Literals([
10
11
  "LmStudio",
11
12
  "Ollama",
12
13
  "MlxLm",
13
- "OpenCode",
14
14
  "Mock"
15
15
  ])
16
16
  export type LlmProvider = typeof LlmProvider.Type
@@ -19,119 +19,256 @@ export class ConnectorId extends Schema.Class<ConnectorId>("ConnectorId")({
19
19
  value: Schema.String
20
20
  }) {}
21
21
 
22
- export const ConnectorIds = Object.freeze({
23
- OpenAI: new ConnectorId({ value: "openai" }),
24
- Anthropic: new ConnectorId({ value: "anthropic" }),
25
- GeminiApi: new ConnectorId({ value: "gemini-api" }),
26
- LmStudio: new ConnectorId({ value: "lm-studio" }),
27
- Ollama: new ConnectorId({ value: "ollama" }),
28
- MlxLm: new ConnectorId({ value: "mlx-lm" }),
29
- ClaudeCli: new ConnectorId({ value: "claude-cli" }),
30
- GeminiCli: new ConnectorId({ value: "gemini-cli" }),
31
- OpenCode: new ConnectorId({ value: "opencode" }),
32
- Codex: new ConnectorId({ value: "codex" }),
33
- Copilot: new ConnectorId({ value: "copilot" }),
34
- Pi: new ConnectorId({ value: "pi" }),
35
- AntigravityCli: new ConnectorId({ value: "antigravity-cli" }),
36
- Grok: new ConnectorId({ value: "grok" }),
37
- Cursor: new ConnectorId({ value: "cursor" }),
38
- Mock: new ConnectorId({ value: "mock" })
39
- })
40
-
41
- export const apiConnectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
42
- ConnectorIds.OpenAI,
43
- ConnectorIds.Anthropic,
44
- ConnectorIds.GeminiApi,
45
- ConnectorIds.LmStudio,
46
- ConnectorIds.Ollama,
47
- ConnectorIds.MlxLm
48
- ])
22
+ export const ConnectorKind = Schema.Literals(["Api", "Cli"])
23
+ export type ConnectorKind = typeof ConnectorKind.Type
49
24
 
50
- export const cliConnectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
51
- ConnectorIds.ClaudeCli,
52
- ConnectorIds.GeminiCli,
53
- ConnectorIds.OpenCode,
54
- ConnectorIds.Codex,
55
- ConnectorIds.Copilot,
56
- ConnectorIds.Pi,
57
- ConnectorIds.AntigravityCli,
58
- ConnectorIds.Grok,
59
- ConnectorIds.Cursor
60
- ])
25
+ interface ConnectorIdentity {
26
+ readonly id: string
27
+ /** The `LlmConfig` provider it serves. Such a row sits under the provider's own name. */
28
+ readonly provider?: LlmProvider
29
+ readonly baseUrl?: string
30
+ /**
31
+ * What an operator may write for it: a coder name on a CLI row
32
+ * (`LLM4TS_CODER`, `LLM4TS_REASONER`), a provider name on an API row
33
+ * (`LLM4TS_PROVIDER`). The first is the one help and menus show. Empty
34
+ * means it is not offered under any name.
35
+ */
36
+ readonly names: ReadonlyArray<string>
37
+ }
38
+
39
+ export interface ApiConnectorRow extends ConnectorIdentity {
40
+ readonly kind: "Api"
41
+ /** Environment variables that may hold its key; the first one set wins. */
42
+ readonly credentialVariables: ReadonlyArray<string>
43
+ /** Answers without a backend, so it is no part of `apiConnectorIds`. */
44
+ readonly simulated?: true
45
+ }
46
+
47
+ export interface CliConnectorRow extends ConnectorIdentity {
48
+ readonly kind: "Cli"
49
+ /** The program it spawns, and the one looked for on `PATH`. */
50
+ readonly executable: string
51
+ }
52
+
53
+ export type ConnectorRow = ApiConnectorRow | CliConnectorRow
54
+
55
+ /**
56
+ * Every connector's identity, one row each. The id lists, the provider
57
+ * mappings and the names an operator may write are all read from here; the
58
+ * factories (`providers/ConnectorFactories.ts`) and the runner's presets are
59
+ * records keyed by these names, so a row without either does not compile.
60
+ */
61
+ export const connectorTable = {
62
+ OpenAI: {
63
+ id: "openai",
64
+ kind: "Api",
65
+ provider: "OpenAI",
66
+ baseUrl: "https://api.openai.com/v1",
67
+ credentialVariables: ["OPENAI_API_KEY"],
68
+ names: ["openai"]
69
+ },
70
+ Anthropic: {
71
+ id: "anthropic",
72
+ kind: "Api",
73
+ provider: "Anthropic",
74
+ baseUrl: "https://api.anthropic.com",
75
+ credentialVariables: ["ANTHROPIC_API_KEY"],
76
+ names: ["anthropic"]
77
+ },
78
+ GeminiApi: {
79
+ id: "gemini-api",
80
+ kind: "Api",
81
+ provider: "GeminiApi",
82
+ baseUrl: "https://generativelanguage.googleapis.com",
83
+ credentialVariables: ["GEMINI_API_KEY", "GOOGLE_API_KEY"],
84
+ names: ["gemini", "gemini-api"]
85
+ },
86
+ LmStudio: {
87
+ id: "lm-studio",
88
+ kind: "Api",
89
+ provider: "LmStudio",
90
+ baseUrl: "http://localhost:1234/v1",
91
+ credentialVariables: [],
92
+ names: ["lm-studio", "lmstudio"]
93
+ },
94
+ Ollama: {
95
+ id: "ollama",
96
+ kind: "Api",
97
+ provider: "Ollama",
98
+ baseUrl: "http://localhost:11434",
99
+ credentialVariables: [],
100
+ names: ["ollama"]
101
+ },
102
+ MlxLm: {
103
+ id: "mlx-lm",
104
+ kind: "Api",
105
+ provider: "MlxLm",
106
+ baseUrl: "http://localhost:8080",
107
+ credentialVariables: [],
108
+ names: ["mlx-lm", "mlxlm", "mlx"]
109
+ },
110
+ ClaudeCli: {
111
+ id: "claude-cli",
112
+ kind: "Cli",
113
+ executable: "claude",
114
+ names: ["claude", "claude-cli"]
115
+ },
116
+ GeminiCli: {
117
+ id: "gemini-cli",
118
+ kind: "Cli",
119
+ provider: "GeminiCli",
120
+ executable: "gemini",
121
+ names: ["gemini", "gemini-cli"]
122
+ },
123
+ OpenCode: { id: "opencode", kind: "Cli", executable: "opencode", names: ["opencode"] },
124
+ Codex: { id: "codex", kind: "Cli", executable: "codex", names: ["codex"] },
125
+ Copilot: { id: "copilot", kind: "Cli", executable: "gh", names: [] },
126
+ Pi: { id: "pi", kind: "Cli", executable: "pi", names: ["pi"] },
127
+ AntigravityCli: {
128
+ id: "antigravity-cli",
129
+ kind: "Cli",
130
+ executable: "agy",
131
+ names: ["agy", "antigravity", "antigravity-cli"]
132
+ },
133
+ Grok: { id: "grok", kind: "Cli", executable: "grok", names: ["grok"] },
134
+ Cursor: { id: "cursor", kind: "Cli", executable: "cursor-agent", names: ["cursor"] },
135
+ Mock: {
136
+ id: "mock",
137
+ kind: "Api",
138
+ provider: "Mock",
139
+ credentialVariables: [],
140
+ simulated: true,
141
+ names: ["mock"]
142
+ }
143
+ } as const satisfies Readonly<Record<string, ConnectorRow>>
144
+
145
+ export type ConnectorName = keyof typeof connectorTable
146
+ type RowOf<K extends ConnectorName> = (typeof connectorTable)[K]
147
+
148
+ export type CliConnectorName = {
149
+ [K in ConnectorName]: RowOf<K>["kind"] extends "Cli" ? K : never
150
+ }[ConnectorName]
151
+ export type ApiConnectorName = Exclude<ConnectorName, CliConnectorName>
152
+
153
+ /** Every name `LLM4TS_CODER` and `LLM4TS_REASONER` read. */
154
+ export type CoderName = RowOf<CliConnectorName>["names"][number]
155
+ /** Every name `LLM4TS_PROVIDER` and `LLM4TS_JUDGMENT_PROVIDER` read. */
156
+ export type ApiProviderName = RowOf<ApiConnectorName>["names"][number]
157
+
158
+ type OfferedCoder = {
159
+ [K in CliConnectorName]: RowOf<K>["names"] extends readonly [] ? never : K
160
+ }[CliConnectorName]
161
+
162
+ const rows: Readonly<Record<ConnectorName, ConnectorRow>> = connectorTable
163
+ const providerRows: { readonly [P in LlmProvider]: ConnectorRow & { readonly provider: P } } =
164
+ connectorTable
165
+ const names = Record.keys(connectorTable)
166
+
167
+ export const isCliConnectorName = (name: ConnectorName): name is CliConnectorName =>
168
+ rows[name].kind === "Cli"
169
+
170
+ export const ConnectorIds: Readonly<Record<ConnectorName, ConnectorId>> = Object.freeze(
171
+ Record.map(connectorTable, (row) => new ConnectorId({ value: row.id }))
172
+ )
173
+
174
+ const idsWhere = (keep: (row: ConnectorRow) => boolean): ReadonlyArray<ConnectorId> =>
175
+ Object.freeze(names.filter((name) => keep(rows[name])).map((name) => ConnectorIds[name]))
176
+
177
+ export const apiConnectorIds: ReadonlyArray<ConnectorId> = idsWhere(
178
+ (row) => row.kind === "Api" && row.simulated !== true
179
+ )
180
+
181
+ export const cliConnectorIds: ReadonlyArray<ConnectorId> = idsWhere((row) => row.kind === "Cli")
61
182
 
62
183
  export const connectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
63
184
  ...apiConnectorIds,
64
185
  ...cliConnectorIds,
65
- ConnectorIds.Mock
186
+ ...idsWhere((row) => row.kind === "Api" && row.simulated === true)
66
187
  ])
67
188
 
68
- export const defaultBaseUrl = (provider: LlmProvider): string | undefined => {
69
- switch (provider) {
70
- case "GeminiCli":
71
- case "Mock":
72
- return undefined
73
- case "GeminiApi":
74
- return "https://generativelanguage.googleapis.com"
75
- case "OpenAI":
76
- return "https://api.openai.com/v1"
77
- case "Anthropic":
78
- return "https://api.anthropic.com"
79
- case "LmStudio":
80
- return "http://localhost:1234/v1"
81
- case "Ollama":
82
- return "http://localhost:11434"
83
- case "MlxLm":
84
- return "http://localhost:8080"
85
- case "OpenCode":
86
- return "http://localhost:4096"
87
- }
88
- }
189
+ const rowsById = new Map<string, ConnectorRow>(names.map((name) => [rows[name].id, rows[name]]))
89
190
 
90
- const connectorProviderTable: Readonly<Record<string, LlmProvider>> = {
91
- openai: "OpenAI",
92
- anthropic: "Anthropic",
93
- "gemini-api": "GeminiApi",
94
- "lm-studio": "LmStudio",
95
- ollama: "Ollama",
96
- "mlx-lm": "MlxLm",
97
- opencode: "OpenCode",
98
- "gemini-cli": "GeminiCli",
99
- mock: "Mock"
100
- }
191
+ export const connectorRow = (id: ConnectorId): ConnectorRow | undefined => rowsById.get(id.value)
192
+
193
+ export const defaultBaseUrl = (provider: LlmProvider): string | undefined =>
194
+ providerRows[provider].baseUrl
101
195
 
102
196
  export const connectorProvider = (id: ConnectorId): LlmProvider | undefined =>
103
- connectorProviderTable[id.value]
197
+ connectorRow(id)?.provider
104
198
 
105
- export const connectorDefaultBaseUrl = (id: ConnectorId): string | undefined => {
106
- const provider = connectorProvider(id)
107
- return provider === undefined ? undefined : defaultBaseUrl(provider)
108
- }
199
+ export const connectorDefaultBaseUrl = (id: ConnectorId): string | undefined =>
200
+ connectorRow(id)?.baseUrl
109
201
 
110
- export const providerConnectorId = (provider: LlmProvider): ConnectorId => {
111
- switch (provider) {
112
- case "GeminiCli":
113
- return ConnectorIds.GeminiCli
114
- case "GeminiApi":
115
- return ConnectorIds.GeminiApi
116
- case "OpenAI":
117
- return ConnectorIds.OpenAI
118
- case "Anthropic":
119
- return ConnectorIds.Anthropic
120
- case "LmStudio":
121
- return ConnectorIds.LmStudio
122
- case "Ollama":
123
- return ConnectorIds.Ollama
124
- case "MlxLm":
125
- return ConnectorIds.MlxLm
126
- case "OpenCode":
127
- return ConnectorIds.OpenCode
128
- case "Mock":
129
- return ConnectorIds.Mock
130
- }
131
- }
202
+ export const providerConnectorId = (provider: LlmProvider): ConnectorId => ConnectorIds[provider]
132
203
 
133
- export const ConnectorKind = Schema.Literals(["Api", "Cli"])
134
- export type ConnectorKind = typeof ConnectorKind.Type
204
+ const namedConnectors = <K extends ConnectorName>(keys: ReadonlyArray<K>): ReadonlyMap<string, K> =>
205
+ new Map(keys.flatMap((key) => rows[key].names.map((name) => [name, key] as const)))
206
+
207
+ const cliNames = names.filter(isCliConnectorName)
208
+ const apiNames = names.filter((name) => !isCliConnectorName(name))
209
+ const coderConnectors = namedConnectors(cliNames)
210
+ const apiProviderConnectors = namedConnectors(apiNames)
211
+
212
+ /** The CLI connector a coder name stands for; the name as written, lower case. */
213
+ export const coderConnector = (name: string): CliConnectorName | undefined =>
214
+ coderConnectors.get(name)
215
+
216
+ /** The API connector a provider name stands for; the name as written, lower case. */
217
+ export const apiProviderConnector = (name: string): ApiConnectorName | undefined =>
218
+ apiProviderConnectors.get(name)
219
+
220
+ export const coderNames: ReadonlyArray<CoderName> = cliNames.flatMap(
221
+ (name) => connectorTable[name].names
222
+ )
223
+
224
+ export const apiProviderNames: ReadonlyArray<ApiProviderName> = apiNames.flatMap(
225
+ (name) => connectorTable[name].names
226
+ )
227
+
228
+ /** Every variable an API connector reads its key from. Names only, never a value. */
229
+ export const credentialVariables: ReadonlyArray<string> = apiNames.flatMap(
230
+ (name) => connectorTable[name].credentialVariables
231
+ )
232
+
233
+ export const cliExecutable = <K extends CliConnectorName>(name: K): RowOf<K>["executable"] =>
234
+ connectorTable[name].executable
235
+
236
+ const inOrder =
237
+ <All extends string>() =>
238
+ <const L extends ReadonlyArray<All>>(
239
+ list: L,
240
+ ..._missing: [Exclude<All, L[number]>] extends [never] ? [] : [missing: Exclude<All, L[number]>]
241
+ ): L =>
242
+ list
243
+
244
+ /** The coding agents help and menus offer, in the order they show them. */
245
+ export const offeredCoders = inOrder<OfferedCoder>()([
246
+ "ClaudeCli",
247
+ "Codex",
248
+ "GeminiCli",
249
+ "Pi",
250
+ "AntigravityCli",
251
+ "Grok",
252
+ "Cursor",
253
+ "OpenCode"
254
+ ])
255
+
256
+ /** The API providers help and errors offer, in the order they show them. */
257
+ export const offeredApiProviders = inOrder<ApiConnectorName>()([
258
+ "Mock",
259
+ "OpenAI",
260
+ "Anthropic",
261
+ "GeminiApi",
262
+ "LmStudio",
263
+ "Ollama",
264
+ "MlxLm"
265
+ ])
266
+
267
+ export const offeredCoderNames = offeredCoders.map((name) => connectorTable[name].names[0])
268
+
269
+ export const offeredApiProviderNames = offeredApiProviders.map(
270
+ (name) => connectorTable[name].names[0]
271
+ )
135
272
 
136
273
  export const MessageRole = Schema.Literals(["System", "User", "Assistant", "Tool"])
137
274
  export type MessageRole = typeof MessageRole.Type
@@ -1,3 +1,4 @@
1
+ // @internal: usage-limit classification for the CLI connectors
1
2
  import * as DateTime from "effect/DateTime"
2
3
  import * as Duration from "effect/Duration"
3
4
  import { RateLimitError, UsageLimitError, type LlmError } from "./Errors.ts"
@@ -1,6 +1,6 @@
1
1
  import * as Effect from "effect/Effect"
2
- import { DimensionScore, EvalResult } from "./Eval.ts"
3
- import { makeEvaluator, type Evaluator } from "./Evaluator.ts"
2
+ import { DimensionScore, EvalResult, type Evaluator } from "./Eval.ts"
3
+ import { makeEvaluator } from "./Evaluator.ts"
4
4
 
5
5
  const piiPatterns: ReadonlyArray<readonly [label: string, pattern: RegExp]> = [
6
6
  ["email", /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/],
package/src/eval/Eval.ts CHANGED
@@ -1,7 +1,6 @@
1
1
  import * as Effect from "effect/Effect"
2
2
  import * as Schema from "effect/Schema"
3
3
  import type { LlmError } from "../Errors.ts"
4
- import type { Evaluator } from "./Evaluator.ts"
5
4
 
6
5
  export class Dimension extends Schema.Class<Dimension>("Dimension")({
7
6
  name: Schema.String,
@@ -87,6 +86,10 @@ export class RepeatedEval extends Schema.Class<RepeatedEval>("RepeatedEval")({
87
86
  const median = (sorted: ReadonlyArray<number>): number =>
88
87
  sorted.length === 0 ? 0 : (sorted[Math.floor((sorted.length - 1) / 2)] ?? 0)
89
88
 
89
+ export interface Evaluator<in In> {
90
+ readonly evaluate: (input: In) => Effect.Effect<EvalResult, LlmError>
91
+ }
92
+
90
93
  export const repeat = Effect.fn("@llm4ts/core/eval/Eval.repeat")(function* <In>(
91
94
  evaluator: Evaluator<In>,
92
95
  input: In,
@@ -1,8 +1,7 @@
1
1
  import * as Effect from "effect/Effect"
2
2
  import type { LlmError } from "../Errors.ts"
3
3
  import { repeat } from "./Eval.ts"
4
- import type { EvalResult, RepeatedEval } from "./Eval.ts"
5
- import type { Evaluator } from "./Evaluator.ts"
4
+ import type { EvalResult, Evaluator, RepeatedEval } from "./Eval.ts"
6
5
 
7
6
  export interface EvalCase<In> {
8
7
  readonly name: string
@@ -1,10 +1,5 @@
1
1
  import * as Effect from "effect/Effect"
2
- import type { LlmError } from "../Errors.ts"
3
- import { EvalResult } from "./Eval.ts"
4
-
5
- export interface Evaluator<in In> {
6
- readonly evaluate: (input: In) => Effect.Effect<EvalResult, LlmError>
7
- }
2
+ import { EvalResult, type Evaluator } from "./Eval.ts"
8
3
 
9
4
  export const makeEvaluator = <In>(evaluate: Evaluator<In>["evaluate"]): Evaluator<In> => ({
10
5
  evaluate
package/src/eval/Judge.ts CHANGED
@@ -6,8 +6,8 @@ import { score, type ScoreQuestion, type State } from "../judgment/Schemas.ts"
6
6
  import type { LlmServiceShape } from "../LlmService.ts"
7
7
  import type { JsonSchema } from "../Models.ts"
8
8
  import { DimensionScore, EvalResult } from "./Eval.ts"
9
- import type { Dimension, Sample } from "./Eval.ts"
10
- import { makeEvaluator, type Evaluator } from "./Evaluator.ts"
9
+ import type { Dimension, Evaluator, Sample } from "./Eval.ts"
10
+ import { makeEvaluator } from "./Evaluator.ts"
11
11
 
12
12
  export class JudgeResponse extends Schema.Class<JudgeResponse>("JudgeResponse")({
13
13
  scores: Schema.Array(DimensionScore)
@@ -2,10 +2,18 @@ import * as Effect from "effect/Effect"
2
2
  import * as Stream from "effect/Stream"
3
3
  import { makeCliConnector, type CliConnectorShape } from "../Connector.ts"
4
4
  import type { CliConnectorConfig } from "../ConnectorConfig.ts"
5
- import { ConnectorCapabilities, ConnectorIds, HealthStatus, LlmChunk } from "../Models.ts"
5
+ import {
6
+ ConnectorCapabilities,
7
+ ConnectorIds,
8
+ HealthStatus,
9
+ LlmChunk,
10
+ cliExecutable
11
+ } from "../Models.ts"
6
12
  import type { ProcessExecutorShape } from "../ProcessExecutor.ts"
7
13
  import { failClassifiedCliError, optionalModelArgs, sortedFlagArgs } from "./CliSupport.ts"
8
14
 
15
+ const executable = cliExecutable("AntigravityCli")
16
+
9
17
  // `agy --version` proves the install, not the login: the CLI offers no offline
10
18
  // login check, so the auth status stays Unknown rather than claiming a session.
11
19
  const healthy = HealthStatus.make({
@@ -33,14 +41,14 @@ export const antigravityExtraArgs = (config: CliConnectorConfig): ReadonlyArray<
33
41
  export const buildAntigravityArgv = (
34
42
  config: CliConnectorConfig,
35
43
  prompt: string
36
- ): ReadonlyArray<string> => ["agy", ...antigravityExtraArgs(config), "--print", prompt]
44
+ ): ReadonlyArray<string> => [executable, ...antigravityExtraArgs(config), "--print", prompt]
37
45
 
38
46
  export const makeAntigravityConnector = (
39
47
  config: CliConnectorConfig,
40
48
  executor: ProcessExecutorShape
41
49
  ): CliConnectorShape => {
42
50
  const cwd = config.workingDir ?? "."
43
- const healthCheck = executor.run(["agy", "--version"], cwd, {}).pipe(
51
+ const healthCheck = executor.run([executable, "--version"], cwd, {}).pipe(
44
52
  Effect.map((result) => (result.exitCode === 0 ? healthy : unhealthy)),
45
53
  Effect.catch(() => Effect.succeed(unhealthy))
46
54
  )
@@ -75,7 +83,7 @@ export const makeAntigravityConnector = (
75
83
  // therefore cost summaries and budgets) cannot include it.
76
84
  capabilities: ConnectorCapabilities.make({ interactiveSessions: true, usageReporting: false }),
77
85
  buildArgv: (prompt, _context) => buildAntigravityArgv(config, prompt),
78
- buildInteractiveArgv: (_context) => ["agy", ...antigravityExtraArgs(config)],
86
+ buildInteractiveArgv: (_context) => [executable, ...antigravityExtraArgs(config)],
79
87
  complete,
80
88
  completeStream,
81
89
  healthCheck
@@ -11,7 +11,8 @@ import {
11
11
  ConnectorIds,
12
12
  LlmChunk,
13
13
  TokenUsage,
14
- type JsonSchema
14
+ type JsonSchema,
15
+ cliExecutable
15
16
  } from "../Models.ts"
16
17
  import type { ProcessExecutorShape } from "../ProcessExecutor.ts"
17
18
  import { collect } from "../Streaming.ts"
@@ -37,6 +38,8 @@ import {
37
38
  jsonObjectEntries
38
39
  } from "./CliSupport.ts"
39
40
 
41
+ const executable = cliExecutable("ClaudeCli")
42
+
40
43
  /**
41
44
  * `claude auth status --json` reads the credential store and answers
42
45
  * `{"loggedIn": true|false, ...}` without a request to Anthropic. A claude
@@ -243,7 +246,7 @@ export const makeClaudeCliConnector = (
243
246
  prompt: string
244
247
  ) {
245
248
  const result = yield* executor.runWithStdin(
246
- ["claude", "--print", ...extraArgs],
249
+ [executable, "--print", ...extraArgs],
247
250
  cwd,
248
251
  config.envVars,
249
252
  prompt
@@ -269,7 +272,7 @@ export const makeClaudeCliConnector = (
269
272
  ([model, said]) =>
270
273
  executor
271
274
  .runStreamingWithStdin(
272
- ["claude", "--print", "--output-format", "stream-json", "--verbose", ...extraArgs],
275
+ [executable, "--print", "--output-format", "stream-json", "--verbose", ...extraArgs],
273
276
  cwd,
274
277
  config.envVars,
275
278
  prompt
@@ -341,11 +344,11 @@ export const makeClaudeCliConnector = (
341
344
  // another way are not), so the grade is partial.
342
345
  pathDeny: "partial"
343
346
  }),
344
- buildArgv: (prompt, _context) => ["claude", "--print", ...extraArgs, prompt],
345
- buildInteractiveArgv: (_context) => ["claude", ...extraArgs],
347
+ buildArgv: (prompt, _context) => [executable, "--print", ...extraArgs, prompt],
348
+ buildInteractiveArgv: (_context) => [executable, ...extraArgs],
346
349
  complete,
347
350
  completeStream,
348
- versionProbe: { executor, binary: "claude", cwd, authProbe: claudeAuthProbe }
351
+ versionProbe: { executor, binary: executable, cwd, authProbe: claudeAuthProbe }
349
352
  })
350
353
 
351
354
  const executeStructuredWithUsage = <A, E, RD, RE>(
@@ -10,7 +10,8 @@ import {
10
10
  ConnectorIds,
11
11
  LlmChunk,
12
12
  TokenUsage,
13
- type JsonSchema
13
+ type JsonSchema,
14
+ cliExecutable
14
15
  } from "../Models.ts"
15
16
  import type { ProcessExecutorShape } from "../ProcessExecutor.ts"
16
17
  import { collect } from "../Streaming.ts"
@@ -33,6 +34,8 @@ import {
33
34
  type JsonValue
34
35
  } from "./CliSupport.ts"
35
36
 
37
+ const executable = cliExecutable("Codex")
38
+
36
39
  export const codexExtraArgs = (config: CliConnectorConfig): ReadonlyArray<string> => {
37
40
  const effectiveFlags = config.readOnly
38
41
  ? {
@@ -251,7 +254,7 @@ export const makeCodexConnector = (
251
254
  prompt: string
252
255
  ) {
253
256
  const result = yield* executor.runWithStdin(
254
- ["codex", "exec", ...extraArgs],
257
+ [executable, "exec", ...extraArgs],
255
258
  cwd,
256
259
  config.envVars,
257
260
  prompt
@@ -265,7 +268,7 @@ export const makeCodexConnector = (
265
268
  })
266
269
 
267
270
  const completeStream = (prompt: string) =>
268
- rawStream(["codex", "exec", "--json", ...extraArgs], prompt).pipe(
271
+ rawStream([executable, "exec", "--json", ...extraArgs], prompt).pipe(
269
272
  Stream.mapEffect(failCodexChunk)
270
273
  )
271
274
 
@@ -279,11 +282,11 @@ export const makeCodexConnector = (
279
282
  effort: "mapped",
280
283
  isolatedHeadless: "partial"
281
284
  }),
282
- buildArgv: (prompt, _context) => ["codex", "exec", ...extraArgs, prompt],
283
- buildInteractiveArgv: (_context) => ["codex", ...extraArgs],
285
+ buildArgv: (prompt, _context) => [executable, "exec", ...extraArgs, prompt],
286
+ buildInteractiveArgv: (_context) => [executable, ...extraArgs],
284
287
  complete,
285
288
  completeStream,
286
- versionProbe: { executor, binary: "codex", cwd, authProbe: codexAuthProbe }
289
+ versionProbe: { executor, binary: executable, cwd, authProbe: codexAuthProbe }
287
290
  })
288
291
 
289
292
  const executeStructuredWithUsage = <A, E, RD, RE>(
@@ -308,7 +311,7 @@ export const makeCodexConnector = (
308
311
  jsonText(codexStrictSchema(jsonSchema))
309
312
  )
310
313
  const reply = yield* collect(
311
- rawStream(["codex", "exec", "--json", "--output-schema", path, ...extraArgs], prompt)
314
+ rawStream([executable, "exec", "--json", "--output-schema", path, ...extraArgs], prompt)
312
315
  )
313
316
  if (reply.metadata.codexError !== undefined) {
314
317
  return yield* failClassifiedCliError("codex", "codex error", reply.metadata.codexError)