@llm4ts/core 3.3.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ConnectorConfig.d.ts +3 -1
- package/dist/ConnectorConfig.d.ts.map +1 -1
- package/dist/ConnectorConfig.js +4 -17
- package/dist/ConnectorConfig.js.map +1 -1
- package/dist/Models.d.ts +177 -22
- package/dist/Models.d.ts.map +1 -1
- package/dist/Models.js +144 -99
- package/dist/Models.js.map +1 -1
- package/dist/UsageLimits.d.ts.map +1 -1
- package/dist/UsageLimits.js +1 -0
- package/dist/UsageLimits.js.map +1 -1
- package/dist/eval/Checks.d.ts +1 -1
- package/dist/eval/Checks.d.ts.map +1 -1
- package/dist/eval/Checks.js.map +1 -1
- package/dist/eval/Eval.d.ts +4 -1
- package/dist/eval/Eval.d.ts.map +1 -1
- package/dist/eval/Eval.js.map +1 -1
- package/dist/eval/EvalSuite.d.ts +1 -2
- package/dist/eval/EvalSuite.d.ts.map +1 -1
- package/dist/eval/EvalSuite.js.map +1 -1
- package/dist/eval/Evaluator.d.ts +1 -6
- package/dist/eval/Evaluator.d.ts.map +1 -1
- package/dist/eval/Evaluator.js.map +1 -1
- package/dist/eval/Judge.d.ts +1 -2
- package/dist/eval/Judge.d.ts.map +1 -1
- package/dist/eval/Judge.js.map +1 -1
- package/dist/providers/AntigravityConnector.d.ts.map +1 -1
- package/dist/providers/AntigravityConnector.js +5 -4
- package/dist/providers/AntigravityConnector.js.map +1 -1
- package/dist/providers/ClaudeCliConnector.d.ts.map +1 -1
- package/dist/providers/ClaudeCliConnector.js +7 -6
- package/dist/providers/ClaudeCliConnector.js.map +1 -1
- package/dist/providers/CodexConnector.d.ts.map +1 -1
- package/dist/providers/CodexConnector.js +8 -7
- package/dist/providers/CodexConnector.js.map +1 -1
- package/dist/providers/ConnectorFactories.d.ts.map +1 -1
- package/dist/providers/ConnectorFactories.js +26 -20
- package/dist/providers/ConnectorFactories.js.map +1 -1
- package/dist/providers/CopilotConnector.d.ts.map +1 -1
- package/dist/providers/CopilotConnector.js +5 -4
- package/dist/providers/CopilotConnector.js.map +1 -1
- package/dist/providers/CursorConnector.d.ts.map +1 -1
- package/dist/providers/CursorConnector.js +6 -5
- package/dist/providers/CursorConnector.js.map +1 -1
- package/dist/providers/GeminiAcpSession.d.ts.map +1 -1
- package/dist/providers/GeminiAcpSession.js +3 -1
- package/dist/providers/GeminiAcpSession.js.map +1 -1
- package/dist/providers/GeminiCliProvider.d.ts.map +1 -1
- package/dist/providers/GeminiCliProvider.js +4 -3
- package/dist/providers/GeminiCliProvider.js.map +1 -1
- package/dist/providers/GrokCliConnector.d.ts.map +1 -1
- package/dist/providers/GrokCliConnector.js +11 -5
- package/dist/providers/GrokCliConnector.js.map +1 -1
- package/dist/providers/OpenCodeCliConnector.d.ts.map +1 -1
- package/dist/providers/OpenCodeCliConnector.js +6 -5
- package/dist/providers/OpenCodeCliConnector.js.map +1 -1
- package/dist/providers/PiConnector.d.ts.map +1 -1
- package/dist/providers/PiConnector.js +8 -7
- package/dist/providers/PiConnector.js.map +1 -1
- package/package.json +17 -20
- package/src/ConnectorConfig.ts +18 -17
- package/src/Models.ts +238 -101
- package/src/UsageLimits.ts +1 -0
- package/src/eval/Checks.ts +2 -2
- package/src/eval/Eval.ts +4 -1
- package/src/eval/EvalSuite.ts +1 -2
- package/src/eval/Evaluator.ts +1 -6
- package/src/eval/Judge.ts +2 -2
- package/src/providers/AntigravityConnector.ts +12 -4
- package/src/providers/ClaudeCliConnector.ts +9 -6
- package/src/providers/CodexConnector.ts +10 -7
- package/src/providers/ConnectorFactories.ts +53 -51
- package/src/providers/CopilotConnector.ts +6 -4
- package/src/providers/CursorConnector.ts +7 -5
- package/src/providers/GeminiAcpSession.ts +4 -1
- package/src/providers/GeminiCliProvider.ts +6 -3
- package/src/providers/GrokCliConnector.ts +12 -5
- package/src/providers/OpenCodeCliConnector.ts +7 -5
- package/src/providers/PiConnector.ts +15 -7
- package/dist/AgentSession.d.ts +0 -66
- package/dist/AgentSession.d.ts.map +0 -1
- package/dist/AgentSession.js +0 -53
- package/dist/AgentSession.js.map +0 -1
- package/dist/providers/ClaudeAgentSession.d.ts +0 -11
- package/dist/providers/ClaudeAgentSession.d.ts.map +0 -1
- package/dist/providers/ClaudeAgentSession.js +0 -161
- package/dist/providers/ClaudeAgentSession.js.map +0 -1
- package/src/AgentSession.ts +0 -69
- package/src/providers/ClaudeAgentSession.ts +0 -230
package/src/Models.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as Duration from "effect/Duration"
|
|
2
2
|
import * as Effect from "effect/Effect"
|
|
3
|
+
import * as Record from "effect/Record"
|
|
3
4
|
import * as Schema from "effect/Schema"
|
|
4
5
|
|
|
5
6
|
export const LlmProvider = Schema.Literals([
|
|
@@ -10,7 +11,6 @@ export const LlmProvider = Schema.Literals([
|
|
|
10
11
|
"LmStudio",
|
|
11
12
|
"Ollama",
|
|
12
13
|
"MlxLm",
|
|
13
|
-
"OpenCode",
|
|
14
14
|
"Mock"
|
|
15
15
|
])
|
|
16
16
|
export type LlmProvider = typeof LlmProvider.Type
|
|
@@ -19,119 +19,256 @@ export class ConnectorId extends Schema.Class<ConnectorId>("ConnectorId")({
|
|
|
19
19
|
value: Schema.String
|
|
20
20
|
}) {}
|
|
21
21
|
|
|
22
|
-
export const
|
|
23
|
-
|
|
24
|
-
Anthropic: new ConnectorId({ value: "anthropic" }),
|
|
25
|
-
GeminiApi: new ConnectorId({ value: "gemini-api" }),
|
|
26
|
-
LmStudio: new ConnectorId({ value: "lm-studio" }),
|
|
27
|
-
Ollama: new ConnectorId({ value: "ollama" }),
|
|
28
|
-
MlxLm: new ConnectorId({ value: "mlx-lm" }),
|
|
29
|
-
ClaudeCli: new ConnectorId({ value: "claude-cli" }),
|
|
30
|
-
GeminiCli: new ConnectorId({ value: "gemini-cli" }),
|
|
31
|
-
OpenCode: new ConnectorId({ value: "opencode" }),
|
|
32
|
-
Codex: new ConnectorId({ value: "codex" }),
|
|
33
|
-
Copilot: new ConnectorId({ value: "copilot" }),
|
|
34
|
-
Pi: new ConnectorId({ value: "pi" }),
|
|
35
|
-
AntigravityCli: new ConnectorId({ value: "antigravity-cli" }),
|
|
36
|
-
Grok: new ConnectorId({ value: "grok" }),
|
|
37
|
-
Cursor: new ConnectorId({ value: "cursor" }),
|
|
38
|
-
Mock: new ConnectorId({ value: "mock" })
|
|
39
|
-
})
|
|
40
|
-
|
|
41
|
-
export const apiConnectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
|
|
42
|
-
ConnectorIds.OpenAI,
|
|
43
|
-
ConnectorIds.Anthropic,
|
|
44
|
-
ConnectorIds.GeminiApi,
|
|
45
|
-
ConnectorIds.LmStudio,
|
|
46
|
-
ConnectorIds.Ollama,
|
|
47
|
-
ConnectorIds.MlxLm
|
|
48
|
-
])
|
|
22
|
+
export const ConnectorKind = Schema.Literals(["Api", "Cli"])
|
|
23
|
+
export type ConnectorKind = typeof ConnectorKind.Type
|
|
49
24
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
25
|
+
interface ConnectorIdentity {
|
|
26
|
+
readonly id: string
|
|
27
|
+
/** The `LlmConfig` provider it serves. Such a row sits under the provider's own name. */
|
|
28
|
+
readonly provider?: LlmProvider
|
|
29
|
+
readonly baseUrl?: string
|
|
30
|
+
/**
|
|
31
|
+
* What an operator may write for it: a coder name on a CLI row
|
|
32
|
+
* (`LLM4TS_CODER`, `LLM4TS_REASONER`), a provider name on an API row
|
|
33
|
+
* (`LLM4TS_PROVIDER`). The first is the one help and menus show. Empty
|
|
34
|
+
* means it is not offered under any name.
|
|
35
|
+
*/
|
|
36
|
+
readonly names: ReadonlyArray<string>
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ApiConnectorRow extends ConnectorIdentity {
|
|
40
|
+
readonly kind: "Api"
|
|
41
|
+
/** Environment variables that may hold its key; the first one set wins. */
|
|
42
|
+
readonly credentialVariables: ReadonlyArray<string>
|
|
43
|
+
/** Answers without a backend, so it is no part of `apiConnectorIds`. */
|
|
44
|
+
readonly simulated?: true
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface CliConnectorRow extends ConnectorIdentity {
|
|
48
|
+
readonly kind: "Cli"
|
|
49
|
+
/** The program it spawns, and the one looked for on `PATH`. */
|
|
50
|
+
readonly executable: string
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type ConnectorRow = ApiConnectorRow | CliConnectorRow
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Every connector's identity, one row each. The id lists, the provider
|
|
57
|
+
* mappings and the names an operator may write are all read from here; the
|
|
58
|
+
* factories (`providers/ConnectorFactories.ts`) and the runner's presets are
|
|
59
|
+
* records keyed by these names, so a row without either does not compile.
|
|
60
|
+
*/
|
|
61
|
+
export const connectorTable = {
|
|
62
|
+
OpenAI: {
|
|
63
|
+
id: "openai",
|
|
64
|
+
kind: "Api",
|
|
65
|
+
provider: "OpenAI",
|
|
66
|
+
baseUrl: "https://api.openai.com/v1",
|
|
67
|
+
credentialVariables: ["OPENAI_API_KEY"],
|
|
68
|
+
names: ["openai"]
|
|
69
|
+
},
|
|
70
|
+
Anthropic: {
|
|
71
|
+
id: "anthropic",
|
|
72
|
+
kind: "Api",
|
|
73
|
+
provider: "Anthropic",
|
|
74
|
+
baseUrl: "https://api.anthropic.com",
|
|
75
|
+
credentialVariables: ["ANTHROPIC_API_KEY"],
|
|
76
|
+
names: ["anthropic"]
|
|
77
|
+
},
|
|
78
|
+
GeminiApi: {
|
|
79
|
+
id: "gemini-api",
|
|
80
|
+
kind: "Api",
|
|
81
|
+
provider: "GeminiApi",
|
|
82
|
+
baseUrl: "https://generativelanguage.googleapis.com",
|
|
83
|
+
credentialVariables: ["GEMINI_API_KEY", "GOOGLE_API_KEY"],
|
|
84
|
+
names: ["gemini", "gemini-api"]
|
|
85
|
+
},
|
|
86
|
+
LmStudio: {
|
|
87
|
+
id: "lm-studio",
|
|
88
|
+
kind: "Api",
|
|
89
|
+
provider: "LmStudio",
|
|
90
|
+
baseUrl: "http://localhost:1234/v1",
|
|
91
|
+
credentialVariables: [],
|
|
92
|
+
names: ["lm-studio", "lmstudio"]
|
|
93
|
+
},
|
|
94
|
+
Ollama: {
|
|
95
|
+
id: "ollama",
|
|
96
|
+
kind: "Api",
|
|
97
|
+
provider: "Ollama",
|
|
98
|
+
baseUrl: "http://localhost:11434",
|
|
99
|
+
credentialVariables: [],
|
|
100
|
+
names: ["ollama"]
|
|
101
|
+
},
|
|
102
|
+
MlxLm: {
|
|
103
|
+
id: "mlx-lm",
|
|
104
|
+
kind: "Api",
|
|
105
|
+
provider: "MlxLm",
|
|
106
|
+
baseUrl: "http://localhost:8080",
|
|
107
|
+
credentialVariables: [],
|
|
108
|
+
names: ["mlx-lm", "mlxlm", "mlx"]
|
|
109
|
+
},
|
|
110
|
+
ClaudeCli: {
|
|
111
|
+
id: "claude-cli",
|
|
112
|
+
kind: "Cli",
|
|
113
|
+
executable: "claude",
|
|
114
|
+
names: ["claude", "claude-cli"]
|
|
115
|
+
},
|
|
116
|
+
GeminiCli: {
|
|
117
|
+
id: "gemini-cli",
|
|
118
|
+
kind: "Cli",
|
|
119
|
+
provider: "GeminiCli",
|
|
120
|
+
executable: "gemini",
|
|
121
|
+
names: ["gemini", "gemini-cli"]
|
|
122
|
+
},
|
|
123
|
+
OpenCode: { id: "opencode", kind: "Cli", executable: "opencode", names: ["opencode"] },
|
|
124
|
+
Codex: { id: "codex", kind: "Cli", executable: "codex", names: ["codex"] },
|
|
125
|
+
Copilot: { id: "copilot", kind: "Cli", executable: "gh", names: [] },
|
|
126
|
+
Pi: { id: "pi", kind: "Cli", executable: "pi", names: ["pi"] },
|
|
127
|
+
AntigravityCli: {
|
|
128
|
+
id: "antigravity-cli",
|
|
129
|
+
kind: "Cli",
|
|
130
|
+
executable: "agy",
|
|
131
|
+
names: ["agy", "antigravity", "antigravity-cli"]
|
|
132
|
+
},
|
|
133
|
+
Grok: { id: "grok", kind: "Cli", executable: "grok", names: ["grok"] },
|
|
134
|
+
Cursor: { id: "cursor", kind: "Cli", executable: "cursor-agent", names: ["cursor"] },
|
|
135
|
+
Mock: {
|
|
136
|
+
id: "mock",
|
|
137
|
+
kind: "Api",
|
|
138
|
+
provider: "Mock",
|
|
139
|
+
credentialVariables: [],
|
|
140
|
+
simulated: true,
|
|
141
|
+
names: ["mock"]
|
|
142
|
+
}
|
|
143
|
+
} as const satisfies Readonly<Record<string, ConnectorRow>>
|
|
144
|
+
|
|
145
|
+
export type ConnectorName = keyof typeof connectorTable
|
|
146
|
+
type RowOf<K extends ConnectorName> = (typeof connectorTable)[K]
|
|
147
|
+
|
|
148
|
+
export type CliConnectorName = {
|
|
149
|
+
[K in ConnectorName]: RowOf<K>["kind"] extends "Cli" ? K : never
|
|
150
|
+
}[ConnectorName]
|
|
151
|
+
export type ApiConnectorName = Exclude<ConnectorName, CliConnectorName>
|
|
152
|
+
|
|
153
|
+
/** Every name `LLM4TS_CODER` and `LLM4TS_REASONER` read. */
|
|
154
|
+
export type CoderName = RowOf<CliConnectorName>["names"][number]
|
|
155
|
+
/** Every name `LLM4TS_PROVIDER` and `LLM4TS_JUDGMENT_PROVIDER` read. */
|
|
156
|
+
export type ApiProviderName = RowOf<ApiConnectorName>["names"][number]
|
|
157
|
+
|
|
158
|
+
type OfferedCoder = {
|
|
159
|
+
[K in CliConnectorName]: RowOf<K>["names"] extends readonly [] ? never : K
|
|
160
|
+
}[CliConnectorName]
|
|
161
|
+
|
|
162
|
+
const rows: Readonly<Record<ConnectorName, ConnectorRow>> = connectorTable
|
|
163
|
+
const providerRows: { readonly [P in LlmProvider]: ConnectorRow & { readonly provider: P } } =
|
|
164
|
+
connectorTable
|
|
165
|
+
const names = Record.keys(connectorTable)
|
|
166
|
+
|
|
167
|
+
export const isCliConnectorName = (name: ConnectorName): name is CliConnectorName =>
|
|
168
|
+
rows[name].kind === "Cli"
|
|
169
|
+
|
|
170
|
+
export const ConnectorIds: Readonly<Record<ConnectorName, ConnectorId>> = Object.freeze(
|
|
171
|
+
Record.map(connectorTable, (row) => new ConnectorId({ value: row.id }))
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
const idsWhere = (keep: (row: ConnectorRow) => boolean): ReadonlyArray<ConnectorId> =>
|
|
175
|
+
Object.freeze(names.filter((name) => keep(rows[name])).map((name) => ConnectorIds[name]))
|
|
176
|
+
|
|
177
|
+
export const apiConnectorIds: ReadonlyArray<ConnectorId> = idsWhere(
|
|
178
|
+
(row) => row.kind === "Api" && row.simulated !== true
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
export const cliConnectorIds: ReadonlyArray<ConnectorId> = idsWhere((row) => row.kind === "Cli")
|
|
61
182
|
|
|
62
183
|
export const connectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
|
|
63
184
|
...apiConnectorIds,
|
|
64
185
|
...cliConnectorIds,
|
|
65
|
-
|
|
186
|
+
...idsWhere((row) => row.kind === "Api" && row.simulated === true)
|
|
66
187
|
])
|
|
67
188
|
|
|
68
|
-
|
|
69
|
-
switch (provider) {
|
|
70
|
-
case "GeminiCli":
|
|
71
|
-
case "Mock":
|
|
72
|
-
return undefined
|
|
73
|
-
case "GeminiApi":
|
|
74
|
-
return "https://generativelanguage.googleapis.com"
|
|
75
|
-
case "OpenAI":
|
|
76
|
-
return "https://api.openai.com/v1"
|
|
77
|
-
case "Anthropic":
|
|
78
|
-
return "https://api.anthropic.com"
|
|
79
|
-
case "LmStudio":
|
|
80
|
-
return "http://localhost:1234/v1"
|
|
81
|
-
case "Ollama":
|
|
82
|
-
return "http://localhost:11434"
|
|
83
|
-
case "MlxLm":
|
|
84
|
-
return "http://localhost:8080"
|
|
85
|
-
case "OpenCode":
|
|
86
|
-
return "http://localhost:4096"
|
|
87
|
-
}
|
|
88
|
-
}
|
|
189
|
+
const rowsById = new Map<string, ConnectorRow>(names.map((name) => [rows[name].id, rows[name]]))
|
|
89
190
|
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
"lm-studio": "LmStudio",
|
|
95
|
-
ollama: "Ollama",
|
|
96
|
-
"mlx-lm": "MlxLm",
|
|
97
|
-
opencode: "OpenCode",
|
|
98
|
-
"gemini-cli": "GeminiCli",
|
|
99
|
-
mock: "Mock"
|
|
100
|
-
}
|
|
191
|
+
export const connectorRow = (id: ConnectorId): ConnectorRow | undefined => rowsById.get(id.value)
|
|
192
|
+
|
|
193
|
+
export const defaultBaseUrl = (provider: LlmProvider): string | undefined =>
|
|
194
|
+
providerRows[provider].baseUrl
|
|
101
195
|
|
|
102
196
|
export const connectorProvider = (id: ConnectorId): LlmProvider | undefined =>
|
|
103
|
-
|
|
197
|
+
connectorRow(id)?.provider
|
|
104
198
|
|
|
105
|
-
export const connectorDefaultBaseUrl = (id: ConnectorId): string | undefined =>
|
|
106
|
-
|
|
107
|
-
return provider === undefined ? undefined : defaultBaseUrl(provider)
|
|
108
|
-
}
|
|
199
|
+
export const connectorDefaultBaseUrl = (id: ConnectorId): string | undefined =>
|
|
200
|
+
connectorRow(id)?.baseUrl
|
|
109
201
|
|
|
110
|
-
export const providerConnectorId = (provider: LlmProvider): ConnectorId =>
|
|
111
|
-
switch (provider) {
|
|
112
|
-
case "GeminiCli":
|
|
113
|
-
return ConnectorIds.GeminiCli
|
|
114
|
-
case "GeminiApi":
|
|
115
|
-
return ConnectorIds.GeminiApi
|
|
116
|
-
case "OpenAI":
|
|
117
|
-
return ConnectorIds.OpenAI
|
|
118
|
-
case "Anthropic":
|
|
119
|
-
return ConnectorIds.Anthropic
|
|
120
|
-
case "LmStudio":
|
|
121
|
-
return ConnectorIds.LmStudio
|
|
122
|
-
case "Ollama":
|
|
123
|
-
return ConnectorIds.Ollama
|
|
124
|
-
case "MlxLm":
|
|
125
|
-
return ConnectorIds.MlxLm
|
|
126
|
-
case "OpenCode":
|
|
127
|
-
return ConnectorIds.OpenCode
|
|
128
|
-
case "Mock":
|
|
129
|
-
return ConnectorIds.Mock
|
|
130
|
-
}
|
|
131
|
-
}
|
|
202
|
+
export const providerConnectorId = (provider: LlmProvider): ConnectorId => ConnectorIds[provider]
|
|
132
203
|
|
|
133
|
-
|
|
134
|
-
|
|
204
|
+
const namedConnectors = <K extends ConnectorName>(keys: ReadonlyArray<K>): ReadonlyMap<string, K> =>
|
|
205
|
+
new Map(keys.flatMap((key) => rows[key].names.map((name) => [name, key] as const)))
|
|
206
|
+
|
|
207
|
+
const cliNames = names.filter(isCliConnectorName)
|
|
208
|
+
const apiNames = names.filter((name) => !isCliConnectorName(name))
|
|
209
|
+
const coderConnectors = namedConnectors(cliNames)
|
|
210
|
+
const apiProviderConnectors = namedConnectors(apiNames)
|
|
211
|
+
|
|
212
|
+
/** The CLI connector a coder name stands for; the name as written, lower case. */
|
|
213
|
+
export const coderConnector = (name: string): CliConnectorName | undefined =>
|
|
214
|
+
coderConnectors.get(name)
|
|
215
|
+
|
|
216
|
+
/** The API connector a provider name stands for; the name as written, lower case. */
|
|
217
|
+
export const apiProviderConnector = (name: string): ApiConnectorName | undefined =>
|
|
218
|
+
apiProviderConnectors.get(name)
|
|
219
|
+
|
|
220
|
+
export const coderNames: ReadonlyArray<CoderName> = cliNames.flatMap(
|
|
221
|
+
(name) => connectorTable[name].names
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
export const apiProviderNames: ReadonlyArray<ApiProviderName> = apiNames.flatMap(
|
|
225
|
+
(name) => connectorTable[name].names
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
/** Every variable an API connector reads its key from. Names only, never a value. */
|
|
229
|
+
export const credentialVariables: ReadonlyArray<string> = apiNames.flatMap(
|
|
230
|
+
(name) => connectorTable[name].credentialVariables
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
export const cliExecutable = <K extends CliConnectorName>(name: K): RowOf<K>["executable"] =>
|
|
234
|
+
connectorTable[name].executable
|
|
235
|
+
|
|
236
|
+
const inOrder =
|
|
237
|
+
<All extends string>() =>
|
|
238
|
+
<const L extends ReadonlyArray<All>>(
|
|
239
|
+
list: L,
|
|
240
|
+
..._missing: [Exclude<All, L[number]>] extends [never] ? [] : [missing: Exclude<All, L[number]>]
|
|
241
|
+
): L =>
|
|
242
|
+
list
|
|
243
|
+
|
|
244
|
+
/** The coding agents help and menus offer, in the order they show them. */
|
|
245
|
+
export const offeredCoders = inOrder<OfferedCoder>()([
|
|
246
|
+
"ClaudeCli",
|
|
247
|
+
"Codex",
|
|
248
|
+
"GeminiCli",
|
|
249
|
+
"Pi",
|
|
250
|
+
"AntigravityCli",
|
|
251
|
+
"Grok",
|
|
252
|
+
"Cursor",
|
|
253
|
+
"OpenCode"
|
|
254
|
+
])
|
|
255
|
+
|
|
256
|
+
/** The API providers help and errors offer, in the order they show them. */
|
|
257
|
+
export const offeredApiProviders = inOrder<ApiConnectorName>()([
|
|
258
|
+
"Mock",
|
|
259
|
+
"OpenAI",
|
|
260
|
+
"Anthropic",
|
|
261
|
+
"GeminiApi",
|
|
262
|
+
"LmStudio",
|
|
263
|
+
"Ollama",
|
|
264
|
+
"MlxLm"
|
|
265
|
+
])
|
|
266
|
+
|
|
267
|
+
export const offeredCoderNames = offeredCoders.map((name) => connectorTable[name].names[0])
|
|
268
|
+
|
|
269
|
+
export const offeredApiProviderNames = offeredApiProviders.map(
|
|
270
|
+
(name) => connectorTable[name].names[0]
|
|
271
|
+
)
|
|
135
272
|
|
|
136
273
|
export const MessageRole = Schema.Literals(["System", "User", "Assistant", "Tool"])
|
|
137
274
|
export type MessageRole = typeof MessageRole.Type
|
package/src/UsageLimits.ts
CHANGED
package/src/eval/Checks.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect"
|
|
2
|
-
import { DimensionScore, EvalResult } from "./Eval.ts"
|
|
3
|
-
import { makeEvaluator
|
|
2
|
+
import { DimensionScore, EvalResult, type Evaluator } from "./Eval.ts"
|
|
3
|
+
import { makeEvaluator } from "./Evaluator.ts"
|
|
4
4
|
|
|
5
5
|
const piiPatterns: ReadonlyArray<readonly [label: string, pattern: RegExp]> = [
|
|
6
6
|
["email", /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/],
|
package/src/eval/Eval.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect"
|
|
2
2
|
import * as Schema from "effect/Schema"
|
|
3
3
|
import type { LlmError } from "../Errors.ts"
|
|
4
|
-
import type { Evaluator } from "./Evaluator.ts"
|
|
5
4
|
|
|
6
5
|
export class Dimension extends Schema.Class<Dimension>("Dimension")({
|
|
7
6
|
name: Schema.String,
|
|
@@ -87,6 +86,10 @@ export class RepeatedEval extends Schema.Class<RepeatedEval>("RepeatedEval")({
|
|
|
87
86
|
const median = (sorted: ReadonlyArray<number>): number =>
|
|
88
87
|
sorted.length === 0 ? 0 : (sorted[Math.floor((sorted.length - 1) / 2)] ?? 0)
|
|
89
88
|
|
|
89
|
+
export interface Evaluator<in In> {
|
|
90
|
+
readonly evaluate: (input: In) => Effect.Effect<EvalResult, LlmError>
|
|
91
|
+
}
|
|
92
|
+
|
|
90
93
|
export const repeat = Effect.fn("@llm4ts/core/eval/Eval.repeat")(function* <In>(
|
|
91
94
|
evaluator: Evaluator<In>,
|
|
92
95
|
input: In,
|
package/src/eval/EvalSuite.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect"
|
|
2
2
|
import type { LlmError } from "../Errors.ts"
|
|
3
3
|
import { repeat } from "./Eval.ts"
|
|
4
|
-
import type { EvalResult, RepeatedEval } from "./Eval.ts"
|
|
5
|
-
import type { Evaluator } from "./Evaluator.ts"
|
|
4
|
+
import type { EvalResult, Evaluator, RepeatedEval } from "./Eval.ts"
|
|
6
5
|
|
|
7
6
|
export interface EvalCase<In> {
|
|
8
7
|
readonly name: string
|
package/src/eval/Evaluator.ts
CHANGED
|
@@ -1,10 +1,5 @@
|
|
|
1
1
|
import * as Effect from "effect/Effect"
|
|
2
|
-
import type
|
|
3
|
-
import { EvalResult } from "./Eval.ts"
|
|
4
|
-
|
|
5
|
-
export interface Evaluator<in In> {
|
|
6
|
-
readonly evaluate: (input: In) => Effect.Effect<EvalResult, LlmError>
|
|
7
|
-
}
|
|
2
|
+
import { EvalResult, type Evaluator } from "./Eval.ts"
|
|
8
3
|
|
|
9
4
|
export const makeEvaluator = <In>(evaluate: Evaluator<In>["evaluate"]): Evaluator<In> => ({
|
|
10
5
|
evaluate
|
package/src/eval/Judge.ts
CHANGED
|
@@ -6,8 +6,8 @@ import { score, type ScoreQuestion, type State } from "../judgment/Schemas.ts"
|
|
|
6
6
|
import type { LlmServiceShape } from "../LlmService.ts"
|
|
7
7
|
import type { JsonSchema } from "../Models.ts"
|
|
8
8
|
import { DimensionScore, EvalResult } from "./Eval.ts"
|
|
9
|
-
import type { Dimension, Sample } from "./Eval.ts"
|
|
10
|
-
import { makeEvaluator
|
|
9
|
+
import type { Dimension, Evaluator, Sample } from "./Eval.ts"
|
|
10
|
+
import { makeEvaluator } from "./Evaluator.ts"
|
|
11
11
|
|
|
12
12
|
export class JudgeResponse extends Schema.Class<JudgeResponse>("JudgeResponse")({
|
|
13
13
|
scores: Schema.Array(DimensionScore)
|
|
@@ -2,10 +2,18 @@ import * as Effect from "effect/Effect"
|
|
|
2
2
|
import * as Stream from "effect/Stream"
|
|
3
3
|
import { makeCliConnector, type CliConnectorShape } from "../Connector.ts"
|
|
4
4
|
import type { CliConnectorConfig } from "../ConnectorConfig.ts"
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
ConnectorCapabilities,
|
|
7
|
+
ConnectorIds,
|
|
8
|
+
HealthStatus,
|
|
9
|
+
LlmChunk,
|
|
10
|
+
cliExecutable
|
|
11
|
+
} from "../Models.ts"
|
|
6
12
|
import type { ProcessExecutorShape } from "../ProcessExecutor.ts"
|
|
7
13
|
import { failClassifiedCliError, optionalModelArgs, sortedFlagArgs } from "./CliSupport.ts"
|
|
8
14
|
|
|
15
|
+
const executable = cliExecutable("AntigravityCli")
|
|
16
|
+
|
|
9
17
|
// `agy --version` proves the install, not the login: the CLI offers no offline
|
|
10
18
|
// login check, so the auth status stays Unknown rather than claiming a session.
|
|
11
19
|
const healthy = HealthStatus.make({
|
|
@@ -33,14 +41,14 @@ export const antigravityExtraArgs = (config: CliConnectorConfig): ReadonlyArray<
|
|
|
33
41
|
export const buildAntigravityArgv = (
|
|
34
42
|
config: CliConnectorConfig,
|
|
35
43
|
prompt: string
|
|
36
|
-
): ReadonlyArray<string> => [
|
|
44
|
+
): ReadonlyArray<string> => [executable, ...antigravityExtraArgs(config), "--print", prompt]
|
|
37
45
|
|
|
38
46
|
export const makeAntigravityConnector = (
|
|
39
47
|
config: CliConnectorConfig,
|
|
40
48
|
executor: ProcessExecutorShape
|
|
41
49
|
): CliConnectorShape => {
|
|
42
50
|
const cwd = config.workingDir ?? "."
|
|
43
|
-
const healthCheck = executor.run([
|
|
51
|
+
const healthCheck = executor.run([executable, "--version"], cwd, {}).pipe(
|
|
44
52
|
Effect.map((result) => (result.exitCode === 0 ? healthy : unhealthy)),
|
|
45
53
|
Effect.catch(() => Effect.succeed(unhealthy))
|
|
46
54
|
)
|
|
@@ -75,7 +83,7 @@ export const makeAntigravityConnector = (
|
|
|
75
83
|
// therefore cost summaries and budgets) cannot include it.
|
|
76
84
|
capabilities: ConnectorCapabilities.make({ interactiveSessions: true, usageReporting: false }),
|
|
77
85
|
buildArgv: (prompt, _context) => buildAntigravityArgv(config, prompt),
|
|
78
|
-
buildInteractiveArgv: (_context) => [
|
|
86
|
+
buildInteractiveArgv: (_context) => [executable, ...antigravityExtraArgs(config)],
|
|
79
87
|
complete,
|
|
80
88
|
completeStream,
|
|
81
89
|
healthCheck
|
|
@@ -11,7 +11,8 @@ import {
|
|
|
11
11
|
ConnectorIds,
|
|
12
12
|
LlmChunk,
|
|
13
13
|
TokenUsage,
|
|
14
|
-
type JsonSchema
|
|
14
|
+
type JsonSchema,
|
|
15
|
+
cliExecutable
|
|
15
16
|
} from "../Models.ts"
|
|
16
17
|
import type { ProcessExecutorShape } from "../ProcessExecutor.ts"
|
|
17
18
|
import { collect } from "../Streaming.ts"
|
|
@@ -37,6 +38,8 @@ import {
|
|
|
37
38
|
jsonObjectEntries
|
|
38
39
|
} from "./CliSupport.ts"
|
|
39
40
|
|
|
41
|
+
const executable = cliExecutable("ClaudeCli")
|
|
42
|
+
|
|
40
43
|
/**
|
|
41
44
|
* `claude auth status --json` reads the credential store and answers
|
|
42
45
|
* `{"loggedIn": true|false, ...}` without a request to Anthropic. A claude
|
|
@@ -243,7 +246,7 @@ export const makeClaudeCliConnector = (
|
|
|
243
246
|
prompt: string
|
|
244
247
|
) {
|
|
245
248
|
const result = yield* executor.runWithStdin(
|
|
246
|
-
[
|
|
249
|
+
[executable, "--print", ...extraArgs],
|
|
247
250
|
cwd,
|
|
248
251
|
config.envVars,
|
|
249
252
|
prompt
|
|
@@ -269,7 +272,7 @@ export const makeClaudeCliConnector = (
|
|
|
269
272
|
([model, said]) =>
|
|
270
273
|
executor
|
|
271
274
|
.runStreamingWithStdin(
|
|
272
|
-
[
|
|
275
|
+
[executable, "--print", "--output-format", "stream-json", "--verbose", ...extraArgs],
|
|
273
276
|
cwd,
|
|
274
277
|
config.envVars,
|
|
275
278
|
prompt
|
|
@@ -341,11 +344,11 @@ export const makeClaudeCliConnector = (
|
|
|
341
344
|
// another way are not), so the grade is partial.
|
|
342
345
|
pathDeny: "partial"
|
|
343
346
|
}),
|
|
344
|
-
buildArgv: (prompt, _context) => [
|
|
345
|
-
buildInteractiveArgv: (_context) => [
|
|
347
|
+
buildArgv: (prompt, _context) => [executable, "--print", ...extraArgs, prompt],
|
|
348
|
+
buildInteractiveArgv: (_context) => [executable, ...extraArgs],
|
|
346
349
|
complete,
|
|
347
350
|
completeStream,
|
|
348
|
-
versionProbe: { executor, binary:
|
|
351
|
+
versionProbe: { executor, binary: executable, cwd, authProbe: claudeAuthProbe }
|
|
349
352
|
})
|
|
350
353
|
|
|
351
354
|
const executeStructuredWithUsage = <A, E, RD, RE>(
|
|
@@ -10,7 +10,8 @@ import {
|
|
|
10
10
|
ConnectorIds,
|
|
11
11
|
LlmChunk,
|
|
12
12
|
TokenUsage,
|
|
13
|
-
type JsonSchema
|
|
13
|
+
type JsonSchema,
|
|
14
|
+
cliExecutable
|
|
14
15
|
} from "../Models.ts"
|
|
15
16
|
import type { ProcessExecutorShape } from "../ProcessExecutor.ts"
|
|
16
17
|
import { collect } from "../Streaming.ts"
|
|
@@ -33,6 +34,8 @@ import {
|
|
|
33
34
|
type JsonValue
|
|
34
35
|
} from "./CliSupport.ts"
|
|
35
36
|
|
|
37
|
+
const executable = cliExecutable("Codex")
|
|
38
|
+
|
|
36
39
|
export const codexExtraArgs = (config: CliConnectorConfig): ReadonlyArray<string> => {
|
|
37
40
|
const effectiveFlags = config.readOnly
|
|
38
41
|
? {
|
|
@@ -251,7 +254,7 @@ export const makeCodexConnector = (
|
|
|
251
254
|
prompt: string
|
|
252
255
|
) {
|
|
253
256
|
const result = yield* executor.runWithStdin(
|
|
254
|
-
[
|
|
257
|
+
[executable, "exec", ...extraArgs],
|
|
255
258
|
cwd,
|
|
256
259
|
config.envVars,
|
|
257
260
|
prompt
|
|
@@ -265,7 +268,7 @@ export const makeCodexConnector = (
|
|
|
265
268
|
})
|
|
266
269
|
|
|
267
270
|
const completeStream = (prompt: string) =>
|
|
268
|
-
rawStream([
|
|
271
|
+
rawStream([executable, "exec", "--json", ...extraArgs], prompt).pipe(
|
|
269
272
|
Stream.mapEffect(failCodexChunk)
|
|
270
273
|
)
|
|
271
274
|
|
|
@@ -279,11 +282,11 @@ export const makeCodexConnector = (
|
|
|
279
282
|
effort: "mapped",
|
|
280
283
|
isolatedHeadless: "partial"
|
|
281
284
|
}),
|
|
282
|
-
buildArgv: (prompt, _context) => [
|
|
283
|
-
buildInteractiveArgv: (_context) => [
|
|
285
|
+
buildArgv: (prompt, _context) => [executable, "exec", ...extraArgs, prompt],
|
|
286
|
+
buildInteractiveArgv: (_context) => [executable, ...extraArgs],
|
|
284
287
|
complete,
|
|
285
288
|
completeStream,
|
|
286
|
-
versionProbe: { executor, binary:
|
|
289
|
+
versionProbe: { executor, binary: executable, cwd, authProbe: codexAuthProbe }
|
|
287
290
|
})
|
|
288
291
|
|
|
289
292
|
const executeStructuredWithUsage = <A, E, RD, RE>(
|
|
@@ -308,7 +311,7 @@ export const makeCodexConnector = (
|
|
|
308
311
|
jsonText(codexStrictSchema(jsonSchema))
|
|
309
312
|
)
|
|
310
313
|
const reply = yield* collect(
|
|
311
|
-
rawStream([
|
|
314
|
+
rawStream([executable, "exec", "--json", "--output-schema", path, ...extraArgs], prompt)
|
|
312
315
|
)
|
|
313
316
|
if (reply.metadata.codexError !== undefined) {
|
|
314
317
|
return yield* failClassifiedCliError("codex", "codex error", reply.metadata.codexError)
|