@opencode/ai 2.0.14 → 2.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -56
- package/dist/experimental/evaluation-client.d.ts +1 -1
- package/dist/experimental/evaluation-client.js +39 -3
- package/dist/experimental/evaluation.d.ts +4 -4
- package/dist/experimental/evaluation.js +2 -2
- package/dist/experimental/system-one.d.ts +3 -3
- package/dist/experimental/system-one.js +40 -51
- package/dist/generation.d.ts +69 -0
- package/dist/generation.js +84 -0
- package/dist/image-client.d.ts +6 -4
- package/dist/image-client.js +11 -11
- package/dist/image.d.ts +1040 -62
- package/dist/image.js +80 -52
- package/dist/index.d.ts +5 -2
- package/dist/index.js +3 -1
- package/dist/llm.d.ts +9 -1
- package/dist/media.d.ts +212 -0
- package/dist/media.js +228 -0
- package/dist/promise.d.ts +554 -0
- package/dist/promise.js +44 -0
- package/dist/protocols/anthropic-messages.js +7 -17
- package/dist/protocols/bedrock-converse.d.ts +4 -4
- package/dist/protocols/bedrock-converse.js +1 -6
- package/dist/protocols/gemini.d.ts +21 -0
- package/dist/protocols/gemini.js +42 -6
- package/dist/protocols/google-images.d.ts +9 -21
- package/dist/protocols/google-images.js +169 -132
- package/dist/protocols/meta-images.d.ts +7 -12
- package/dist/protocols/meta-images.js +92 -66
- package/dist/protocols/mistral-chat.js +7 -6
- package/dist/protocols/open-responses.d.ts +11 -3
- package/dist/protocols/open-responses.js +23 -12
- package/dist/protocols/openai-chat.d.ts +34 -1
- package/dist/protocols/openai-chat.js +100 -31
- package/dist/protocols/openai-images.d.ts +9 -20
- package/dist/protocols/openai-images.js +112 -142
- package/dist/protocols/shared.d.ts +17 -17
- package/dist/protocols/shared.js +32 -35
- package/dist/protocols/utils/bedrock-media.d.ts +2 -3
- package/dist/protocols/utils/bedrock-media.js +4 -4
- package/dist/protocols/utils/media-input.d.ts +10 -0
- package/dist/protocols/utils/media-input.js +17 -0
- package/dist/protocols/utils/responses-compaction.js +6 -5
- package/dist/protocols/utils/tool-stream.d.ts +27 -3
- package/dist/protocols/xai-images.d.ts +9 -15
- package/dist/protocols/xai-images.js +85 -83
- package/dist/protocols/zai-images.d.ts +9 -13
- package/dist/protocols/zai-images.js +59 -57
- package/dist/providers/amazon-bedrock.d.ts +2 -2
- package/dist/providers/cerebras.js +6 -1
- package/dist/providers/deepinfra.js +6 -1
- package/dist/providers/google-vertex.d.ts +7 -0
- package/dist/providers/google.d.ts +7 -0
- package/dist/providers/index.d.ts +1 -0
- package/dist/providers/index.js +1 -0
- package/dist/providers/openrouter.d.ts +19 -0
- package/dist/providers/openrouter.js +13 -1
- package/dist/providers/vercel-ai-gateway.d.ts +41 -0
- package/dist/providers/vercel-ai-gateway.js +85 -0
- package/dist/route/client.d.ts +9 -1
- package/dist/route/endpoint.d.ts +10 -10
- package/dist/route/executor-service.d.ts +12 -0
- package/dist/route/executor-service.js +3 -0
- package/dist/route/executor.d.ts +4 -9
- package/dist/route/executor.js +3 -3
- package/dist/route/index.d.ts +2 -0
- package/dist/route/index.js +2 -0
- package/dist/route/media-protocol.d.ts +45 -0
- package/dist/route/media-protocol.js +40 -0
- package/dist/route/media.d.ts +46 -0
- package/dist/route/media.js +64 -0
- package/dist/schema/errors.d.ts +13 -3
- package/dist/schema/errors.js +7 -0
- package/dist/schema/events.d.ts +563 -40
- package/dist/schema/events.js +35 -2
- package/dist/schema/messages.d.ts +98 -8
- package/dist/schema/messages.js +8 -6
- package/dist/schema/options.d.ts +2 -0
- package/dist/schema/options.js +3 -0
- package/dist/testing.d.ts +72 -8
- package/dist/utils/media-type.d.ts +6 -0
- package/dist/utils/media-type.js +49 -0
- package/dist/utils/sanitize.js +3 -1
- package/package.json +7 -3
- package/dist/protocols/utils/image-input.d.ts +0 -21
- package/dist/protocols/utils/image-input.js +0 -20
package/README.md
CHANGED
|
@@ -8,10 +8,10 @@ import { LLM, LLMClient } from "@opencode/ai"
|
|
|
8
8
|
import { RequestExecutor } from "@opencode/ai/route"
|
|
9
9
|
import { OpenAI } from "@opencode/ai/providers"
|
|
10
10
|
|
|
11
|
-
const
|
|
11
|
+
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
|
12
12
|
|
|
13
13
|
const request = LLM.request({
|
|
14
|
-
model,
|
|
14
|
+
model: openai.responses("gpt-4o-mini"), // `.chat(...)` selects the Chat Completions API instead
|
|
15
15
|
system: "You are concise.",
|
|
16
16
|
prompt: "Say hello in one short sentence.",
|
|
17
17
|
generation: { maxTokens: 40 },
|
|
@@ -29,6 +29,43 @@ await Effect.runPromise(program.pipe(Effect.provide(llmLayer)))
|
|
|
29
29
|
|
|
30
30
|
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
|
31
31
|
|
|
32
|
+
The same configured facade names image models. `Image.request` resolves the provider's image route from the ref and
|
|
33
|
+
returns `Media.Asset`s with lazily decoded bytes:
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
import { NodeFileSystem } from "@effect/platform-node"
|
|
37
|
+
import { Image, ImageClient, Media } from "@opencode/ai"
|
|
38
|
+
|
|
39
|
+
const image = Effect.gen(function* () {
|
|
40
|
+
const response = yield* Image.generate({
|
|
41
|
+
model: openai.image("gpt-image-2"),
|
|
42
|
+
prompt: "A robot tending a rooftop garden",
|
|
43
|
+
size: "1024x1024",
|
|
44
|
+
providerOptions: { quality: "high" }, // typed per image model
|
|
45
|
+
})
|
|
46
|
+
yield* Media.write(response.image, "./garden.png")
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
// `asset.bytes()` / `Media.write` also need the executor, so merge it into the environment instead of hiding it.
|
|
50
|
+
const imageLayer = ImageClient.layer.pipe(Layer.provideMerge(RequestExecutor.fetchLayer))
|
|
51
|
+
|
|
52
|
+
await Effect.runPromise(image.pipe(Effect.provide(imageLayer), Effect.provide(NodeFileSystem.layer)))
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Prefer promises? `@opencode/ai/promise` exposes the same LLM and image APIs over one managed runtime:
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
import { AI } from "@opencode/ai/promise"
|
|
59
|
+
|
|
60
|
+
const ai = AI.make()
|
|
61
|
+
const text = await ai.llm.generate({ model: openai.responses("gpt-4o-mini"), prompt: "Say hello." })
|
|
62
|
+
const generated = await ai.image.generate({ model: openai.image("gpt-image-2"), prompt: "A lighthouse" })
|
|
63
|
+
for await (const event of ai.llm.stream({ model: openai.responses("gpt-4o-mini"), prompt: "Stream hello." })) {
|
|
64
|
+
// LLMEvent
|
|
65
|
+
}
|
|
66
|
+
await ai.dispose()
|
|
67
|
+
```
|
|
68
|
+
|
|
32
69
|
## Experimental evaluation
|
|
33
70
|
|
|
34
71
|
Evaluation models compare shared state with typed choice, score, and boolean questions. The API is
|
|
@@ -41,7 +78,7 @@ import { TypeSafeAI } from "@opencode/ai/providers"
|
|
|
41
78
|
|
|
42
79
|
const model = TypeSafeAI.configure().experimental.evaluation("jev-latest")
|
|
43
80
|
|
|
44
|
-
const program = Evaluation.
|
|
81
|
+
const program = Evaluation.run({
|
|
45
82
|
model,
|
|
46
83
|
state: "I was charged twice. Please refund the duplicate payment.",
|
|
47
84
|
questions: {
|
|
@@ -66,7 +103,17 @@ console.log(response.answers.refund.probability)
|
|
|
66
103
|
```
|
|
67
104
|
|
|
68
105
|
`TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCodeZen` exposes the same selector and reads
|
|
69
|
-
`OPENCODE_API_KEY`.
|
|
106
|
+
`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape:
|
|
107
|
+
|
|
108
|
+
```ts
|
|
109
|
+
import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
|
|
110
|
+
|
|
111
|
+
OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13")
|
|
112
|
+
VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`.
|
|
116
|
+
The common API uses `boolean`; System One routes lower it to native `noul`.
|
|
70
117
|
Choice and score confidence plus score legends remain available in provider metadata, and the
|
|
71
118
|
provider's rounded probabilities are returned unchanged.
|
|
72
119
|
|
|
@@ -355,23 +402,25 @@ citations or separate result blocks. Retain `response.message` for either API's
|
|
|
355
402
|
Use `Image.generate` for one-off generation or editing:
|
|
356
403
|
|
|
357
404
|
```ts
|
|
358
|
-
import { Image,
|
|
405
|
+
import { Image, Media } from "@opencode/ai"
|
|
359
406
|
|
|
360
407
|
const generation = Image.generate({
|
|
361
|
-
model: meta
|
|
408
|
+
model: meta("muse-image-1.0"),
|
|
362
409
|
prompt: "A flat black square on a white background.",
|
|
363
|
-
|
|
410
|
+
n: 1,
|
|
411
|
+
providerOptions: { reasoningStrength: "low" },
|
|
364
412
|
})
|
|
365
413
|
|
|
366
414
|
const edit = Image.generate({
|
|
367
|
-
model: meta
|
|
415
|
+
model: meta("muse-image-1.0"),
|
|
368
416
|
prompt: "Make the square purple.",
|
|
369
|
-
images: [
|
|
370
|
-
|
|
417
|
+
images: [Media.bytes(imageBytes, "image/webp")],
|
|
418
|
+
format: "png",
|
|
419
|
+
providerOptions: { reasoningStrength: "low" },
|
|
371
420
|
})
|
|
372
421
|
```
|
|
373
422
|
|
|
374
|
-
The default image format is WEBP; `
|
|
423
|
+
The default image format is WEBP; `format` also accepts PNG/JPEG and `responseFormat: "url"`
|
|
375
424
|
returns a signed URL. `size` is an aspect-ratio hint. For conversational images, select
|
|
376
425
|
`meta.responses("muse-image-1.0")` with `tools: [Meta.imageGeneration({ reasoningStrength: "low" })]`.
|
|
377
426
|
Generated images are provider-executed tool results with file content. Retain `response.message` to
|
|
@@ -382,29 +431,40 @@ Meta Responses is explicitly HTTP/SSE-only and does not use WebSockets, even whe
|
|
|
382
431
|
|
|
383
432
|
## Image generation
|
|
384
433
|
|
|
385
|
-
Use `Image.generate` with an image model for direct asset generation
|
|
434
|
+
Use `Image.generate` with an image model for direct asset generation. `Image.request` mirrors `LLM.request`: the
|
|
435
|
+
model comes from the facade's `.image(...)` selector (mirroring `.responses(...)`), common fields
|
|
436
|
+
(`images`, `mask`, `n`, `size`, `aspectRatio`, `seed`, `format`) lower natively or fail typed, and
|
|
437
|
+
`providerOptions` is inferred from the selected model:
|
|
386
438
|
|
|
387
439
|
```ts
|
|
388
|
-
import { Image,
|
|
440
|
+
import { Image, Media } from "@opencode/ai"
|
|
389
441
|
import { OpenAI } from "@opencode/ai/providers"
|
|
390
442
|
|
|
443
|
+
const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY })
|
|
444
|
+
|
|
391
445
|
const program = Effect.gen(function* () {
|
|
392
446
|
const response = yield* Image.generate({
|
|
393
|
-
model:
|
|
447
|
+
model: openai.image("gpt-image-2"),
|
|
394
448
|
prompt: "A robot tending a rooftop garden",
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
449
|
+
n: 2,
|
|
450
|
+
size: "1024x1024",
|
|
451
|
+
format: "webp",
|
|
452
|
+
providerOptions: {
|
|
398
453
|
quality: "high", // inferred from the OpenAI image model
|
|
399
|
-
outputFormat: "webp",
|
|
400
454
|
future_option: true, // unknown native options pass through unchanged
|
|
401
455
|
},
|
|
402
456
|
})
|
|
403
457
|
|
|
404
|
-
return response.images //
|
|
458
|
+
return response.images // Media.Asset[] with owned bytes or a provider URL
|
|
405
459
|
})
|
|
406
460
|
```
|
|
407
461
|
|
|
462
|
+
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
|
|
463
|
+
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
|
|
464
|
+
`asset.base64()`, and `asset.dataUrl()` decode or download lazily and cache; `asset.materialize()` pulls a `url`
|
|
465
|
+
asset into owned bytes before the provider URL expires. Construct assets with `Media.bytes`, `Media.base64`,
|
|
466
|
+
`Media.url`, `Media.ref(provider, id)`, `Media.fromDataUrl`, or `Media.file(path)`.
|
|
467
|
+
|
|
408
468
|
Pass ordered image inputs to the same method for editing, composition, or image-conditioned generation:
|
|
409
469
|
|
|
410
470
|
```ts
|
|
@@ -414,49 +474,45 @@ const response =
|
|
|
414
474
|
model,
|
|
415
475
|
prompt: "Combine these product photos into one studio scene",
|
|
416
476
|
images: [
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
477
|
+
Media.bytes(firstBytes, "image/png"),
|
|
478
|
+
Media.url("https://example.com/second.webp"),
|
|
479
|
+
Media.ref("openai", "file_123"),
|
|
420
480
|
],
|
|
421
|
-
|
|
481
|
+
providerOptions,
|
|
422
482
|
http,
|
|
423
483
|
})
|
|
424
484
|
```
|
|
425
485
|
|
|
426
|
-
`
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
edits and its JSON reference body for URL or file-ID edits.
|
|
431
|
-
|
|
486
|
+
`Media.ref(provider, id)` represents provider file handles such as OpenAI file IDs or Gemini Files URIs; routes
|
|
487
|
+
only forward refs that belong to their own provider. Raw strings are not accepted as image inputs, avoiding
|
|
488
|
+
ambiguity between base64, URLs, and provider IDs. Empty or omitted `images` uses text-to-image generation; a
|
|
489
|
+
non-empty array selects the provider's edit behavior without enforcing provider image-count limits locally. OpenAI
|
|
490
|
+
uses multipart for byte/data-URL edits and its JSON reference body for URL or file-ID edits. The common `mask`
|
|
491
|
+
field selects inpainting; routes that cannot honor it fail with `UnsupportedOperation`:
|
|
432
492
|
|
|
433
493
|
```ts
|
|
434
494
|
yield *
|
|
435
495
|
Image.generate({
|
|
436
|
-
model:
|
|
496
|
+
model: openai.image("gpt-image-2"),
|
|
437
497
|
prompt,
|
|
438
|
-
images: [
|
|
439
|
-
|
|
498
|
+
images: [Media.bytes(sourceBytes, "image/png")],
|
|
499
|
+
mask: Media.bytes(maskBytes, "image/png"),
|
|
440
500
|
})
|
|
441
501
|
```
|
|
442
502
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
public HTTP URLs, and hosted Z.ai image generation does not accept image inputs. These cases fail with
|
|
448
|
-
`InvalidRequest` before network I/O.
|
|
503
|
+
On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
|
|
504
|
+
or `mask` fields, and the transport owns the multipart `Content-Type` boundary. For JSON requests, `http.body`
|
|
505
|
+
remains the final raw-native overlay. Gemini does not fetch public HTTP URLs, and hosted Z.ai image generation does
|
|
506
|
+
not accept image inputs. These cases fail with a typed `AIError` before network I/O.
|
|
449
507
|
|
|
450
508
|
Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
|
|
451
509
|
|
|
452
510
|
```ts
|
|
453
|
-
const model = OpenAI.configure({ apiKey }).image("gpt-image-2")
|
|
454
|
-
|
|
455
511
|
yield *
|
|
456
512
|
Image.generate({
|
|
457
|
-
model,
|
|
513
|
+
model: openai.image("gpt-image-2"),
|
|
458
514
|
prompt,
|
|
459
|
-
|
|
515
|
+
providerOptions: { quality: "medium" },
|
|
460
516
|
http,
|
|
461
517
|
})
|
|
462
518
|
```
|
|
@@ -466,11 +522,11 @@ xAI image models use the same request API with xAI-native controls:
|
|
|
466
522
|
```ts
|
|
467
523
|
yield *
|
|
468
524
|
Image.generate({
|
|
469
|
-
model: XAI.configure({ apiKey })
|
|
525
|
+
model: XAI.configure({ apiKey })("any-model-id"),
|
|
470
526
|
prompt,
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
527
|
+
n: 2,
|
|
528
|
+
aspectRatio: "16:9",
|
|
529
|
+
providerOptions: {
|
|
474
530
|
resolution: "1k",
|
|
475
531
|
responseFormat: "b64_json",
|
|
476
532
|
future_option: true,
|
|
@@ -486,12 +542,12 @@ import { Google } from "@opencode/ai/providers"
|
|
|
486
542
|
|
|
487
543
|
const googleProgram = Effect.gen(function* () {
|
|
488
544
|
const response = yield* Image.generate({
|
|
489
|
-
model: Google.configure({ apiKey })
|
|
545
|
+
model: Google.configure({ apiKey })("any-model-id"),
|
|
490
546
|
prompt: "A robot tending a rooftop garden",
|
|
491
|
-
|
|
492
|
-
|
|
547
|
+
aspectRatio: "16:9",
|
|
548
|
+
seed: 42,
|
|
549
|
+
providerOptions: {
|
|
493
550
|
imageSize: "2K",
|
|
494
|
-
seed: 42,
|
|
495
551
|
thinkingLevel: "HIGH",
|
|
496
552
|
includeThoughts: true,
|
|
497
553
|
futureOption: true,
|
|
@@ -513,9 +569,9 @@ Z.ai image models infer open Z.ai-native options from the selected model:
|
|
|
513
569
|
```ts
|
|
514
570
|
yield *
|
|
515
571
|
Image.generate({
|
|
516
|
-
model: ZAI.configure({ apiKey })
|
|
572
|
+
model: ZAI.configure({ apiKey })("any-model-id"),
|
|
517
573
|
prompt,
|
|
518
|
-
|
|
574
|
+
providerOptions: {
|
|
519
575
|
quality: "hd",
|
|
520
576
|
userID: "user-123",
|
|
521
577
|
future_option: true,
|
|
@@ -525,8 +581,8 @@ yield *
|
|
|
525
581
|
```
|
|
526
582
|
|
|
527
583
|
Z.ai does not include trustworthy MIME metadata for output URLs, so generated images use
|
|
528
|
-
`application/octet-stream
|
|
529
|
-
remain available.
|
|
584
|
+
`application/octet-stream` until materialized. Output URLs expire after 30 days; call `asset.materialize()` and
|
|
585
|
+
persist the bytes promptly if they must remain available.
|
|
530
586
|
|
|
531
587
|
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
|
|
532
588
|
|
|
@@ -544,7 +600,7 @@ const program = Effect.gen(function* () {
|
|
|
544
600
|
})
|
|
545
601
|
```
|
|
546
602
|
|
|
547
|
-
The hosted result is represented as a provider-executed tool call and tool result
|
|
603
|
+
The hosted result is represented as a provider-executed tool call and tool result, and the generated image is also emitted as a first-class `media` `LLMEvent` (`response.message` then carries a `media` part). Gemini image-capable models emit the same `media` event for inline image output. Retaining `response.message` preserves the generated image for continuation on both routes.
|
|
548
604
|
|
|
549
605
|
## Public API
|
|
550
606
|
|
|
@@ -553,8 +609,11 @@ The hosted result is represented as a provider-executed tool call and tool resul
|
|
|
553
609
|
- **`Message.user(...)` / `Message.assistant(...)` / `Message.tool(...)`** — message constructors from the canonical schema model.
|
|
554
610
|
- **`LanguageModel.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
|
|
555
611
|
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
|
|
556
|
-
- **`Image.generate
|
|
612
|
+
- **`Image.request` / `Image.generate` / `Image.stream`** — generate images through a provider-neutral image request and response model.
|
|
557
613
|
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
|
|
614
|
+
- **`Media`** — the shared asset type (`Media.Asset`, `Media.Source`) and constructors used by messages, tool results, and media requests.
|
|
615
|
+
- **`Generation`** — provider-neutral handle for an in-flight media generation (`await`, `refresh`, `cancel`, `events`) used by queued media routes.
|
|
616
|
+
- **`@opencode/ai/promise`** — `AI.make({ layer? })` and a default `ai` client exposing `llm` and `image` as Promise / `AsyncIterable` APIs.
|
|
558
617
|
|
|
559
618
|
## Testing
|
|
560
619
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Context, Effect, Layer } from "effect";
|
|
2
2
|
import { RequestExecutor } from "../route/executor.js";
|
|
3
|
-
import {
|
|
3
|
+
import { AIError } from "../schema/index.js";
|
|
4
4
|
import { type EvaluationOptions, type EvaluationQuestions, type EvaluationRequestFor, type EvaluationResponseFor } from "./evaluation.js";
|
|
5
5
|
export type Execute = RequestExecutor.Interface["execute"];
|
|
6
6
|
export interface Interface {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Context, Effect, Layer } from "effect";
|
|
2
2
|
import { RequestExecutor } from "../route/executor.js";
|
|
3
|
-
import { mergeHttpOptions } from "../schema/index.js";
|
|
3
|
+
import { AIError, InvalidProviderOutputError, mergeHttpOptions } from "../schema/index.js";
|
|
4
4
|
import { sanitizeSurrogates } from "../utils/sanitize.js";
|
|
5
5
|
import {} from "./evaluation.js";
|
|
6
6
|
export class Service extends Context.Service()("@opencode/AI/Experimental/EvaluationClient") {
|
|
@@ -9,14 +9,50 @@ export const evaluate = (request) => Effect.flatMap(Service, (client) => client.
|
|
|
9
9
|
export const layer = Layer.effect(Service, Effect.gen(function* () {
|
|
10
10
|
const executor = yield* RequestExecutor.Service;
|
|
11
11
|
return Service.of({
|
|
12
|
-
evaluate: (request) => request.model.route
|
|
12
|
+
evaluate: (request) => request.model.route
|
|
13
|
+
.evaluate({
|
|
13
14
|
...sanitizeSurrogates({
|
|
14
15
|
...request,
|
|
15
16
|
model: undefined,
|
|
16
17
|
http: mergeHttpOptions(request.model.http, request.http),
|
|
17
18
|
}),
|
|
18
19
|
model: request.model,
|
|
19
|
-
}, executor.execute)
|
|
20
|
+
}, executor.execute)
|
|
21
|
+
.pipe(Effect.flatMap((response) => {
|
|
22
|
+
const questions = Object.entries(request.questions);
|
|
23
|
+
if (questions.length === Object.keys(response.answers).length &&
|
|
24
|
+
questions.every(([id, question]) => {
|
|
25
|
+
const answer = response.answers[id];
|
|
26
|
+
if (question.type === "boolean")
|
|
27
|
+
return answer?.type === "boolean";
|
|
28
|
+
if (question.type === "choice") {
|
|
29
|
+
if (answer?.type !== "choice" || !Object.hasOwn(question.criteria, answer.choice))
|
|
30
|
+
return false;
|
|
31
|
+
if (answer.probabilities === undefined)
|
|
32
|
+
return true;
|
|
33
|
+
const keys = Object.keys(question.criteria);
|
|
34
|
+
const probabilities = answer.probabilities;
|
|
35
|
+
return (Object.keys(probabilities).length === keys.length &&
|
|
36
|
+
keys.every((key) => Object.hasOwn(probabilities, key)));
|
|
37
|
+
}
|
|
38
|
+
if (answer?.type !== "score" || answer.score < 0 || answer.score > question.criteria.length - 1)
|
|
39
|
+
return false;
|
|
40
|
+
if (answer.probabilities === undefined)
|
|
41
|
+
return true;
|
|
42
|
+
const keys = question.criteria.map((_, index) => String(index));
|
|
43
|
+
const probabilities = answer.probabilities;
|
|
44
|
+
return (Object.keys(probabilities).length === keys.length &&
|
|
45
|
+
keys.every((key) => Object.hasOwn(probabilities, key)));
|
|
46
|
+
}))
|
|
47
|
+
return Effect.succeed(response);
|
|
48
|
+
return Effect.fail(new AIError({
|
|
49
|
+
reason: new InvalidProviderOutputError({
|
|
50
|
+
route: request.model.route.id,
|
|
51
|
+
message: "Evaluation answers do not match the requested questions",
|
|
52
|
+
cause: response.answers,
|
|
53
|
+
}),
|
|
54
|
+
}));
|
|
55
|
+
})),
|
|
20
56
|
});
|
|
21
57
|
}));
|
|
22
58
|
export const fetchLayer = layer.pipe(Layer.provide(RequestExecutor.fetchLayer));
|
|
@@ -96,7 +96,7 @@ export type AnswersFor<Questions extends EvaluationQuestions> = {
|
|
|
96
96
|
export type EvaluationOptions = Record<string, unknown>;
|
|
97
97
|
export interface EvaluationRoute<Options extends EvaluationOptions = EvaluationOptions> {
|
|
98
98
|
readonly id: string;
|
|
99
|
-
readonly evaluate:
|
|
99
|
+
readonly evaluate: (request: EvaluationRequestFor<Options>, execute: Execute) => Effect.Effect<EvaluationResponse, AIError>;
|
|
100
100
|
}
|
|
101
101
|
export declare class EvaluationModel<Options extends EvaluationOptions = EvaluationOptions> {
|
|
102
102
|
protected readonly _Options: (options: Options) => Options;
|
|
@@ -225,10 +225,10 @@ export type EvaluationResponseFor<Questions extends EvaluationQuestions> = Omit<
|
|
|
225
225
|
};
|
|
226
226
|
export declare function request<const Model extends object, const Questions extends EvaluationQuestions>(input: EvaluationRequestInput<Model, Questions>): EvaluationRequestFor<EvaluationModelOptions<Model>, Questions>;
|
|
227
227
|
export declare function request(input: EvaluationRequest): EvaluationRequest;
|
|
228
|
-
export declare function
|
|
229
|
-
export declare function
|
|
228
|
+
export declare function run<const Model extends object, const Questions extends EvaluationQuestions>(input: EvaluationRequestInput<Model, Questions>): Effect.Effect<EvaluationResponseFor<Questions>, AIError, Service>;
|
|
229
|
+
export declare function run(input: EvaluationRequest): Effect.Effect<EvaluationResponse, AIError, Service>;
|
|
230
230
|
export declare const Evaluation: {
|
|
231
231
|
readonly request: typeof request;
|
|
232
|
-
readonly
|
|
232
|
+
readonly run: typeof run;
|
|
233
233
|
};
|
|
234
234
|
export {};
|
|
@@ -97,7 +97,7 @@ export function request(input) {
|
|
|
97
97
|
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
|
98
98
|
});
|
|
99
99
|
}
|
|
100
|
-
export function
|
|
100
|
+
export function run(input) {
|
|
101
101
|
return Effect.try({
|
|
102
102
|
try: () => (input instanceof EvaluationRequest ? input : request(input)),
|
|
103
103
|
catch: (cause) => new AIError({
|
|
@@ -110,5 +110,5 @@ export function evaluate(input) {
|
|
|
110
110
|
}
|
|
111
111
|
export const Evaluation = {
|
|
112
112
|
request,
|
|
113
|
-
|
|
113
|
+
run,
|
|
114
114
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EvaluationModel } from "./evaluation.js";
|
|
1
|
+
import { EvaluationModel, type EvaluationOptions } from "./evaluation.js";
|
|
2
2
|
import { type Definition as AuthDefinition } from "../route/auth.js";
|
|
3
3
|
import { HttpOptions, ModelID } from "../schema/index.js";
|
|
4
4
|
export interface ModelInput {
|
|
@@ -10,7 +10,7 @@ export interface ModelInput {
|
|
|
10
10
|
readonly headers?: Record<string, string>;
|
|
11
11
|
readonly http?: HttpOptions;
|
|
12
12
|
}
|
|
13
|
-
export declare const model: (cfg: ModelInput) => EvaluationModel<
|
|
13
|
+
export declare const model: <Options extends EvaluationOptions = EvaluationOptions>(cfg: ModelInput) => EvaluationModel<Options>;
|
|
14
14
|
export declare const SystemOne: {
|
|
15
|
-
readonly model: (cfg: ModelInput) => EvaluationModel<
|
|
15
|
+
readonly model: <Options extends EvaluationOptions = EvaluationOptions>(cfg: ModelInput) => EvaluationModel<Options>;
|
|
16
16
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Effect, Schema } from "effect";
|
|
2
2
|
import { Headers, HttpClientRequest } from "effect/unstable/http";
|
|
3
|
-
import {
|
|
3
|
+
import { ChoiceQuestion, EvaluationInput, EvaluationModel, EvaluationResponse, EvaluationRounding, ScoreQuestion, } from "./evaluation.js";
|
|
4
4
|
import { Auth } from "../route/auth.js";
|
|
5
5
|
import { AIError, HttpContext, HttpOptions, InvalidProviderOutputError, InvalidRequestError, ModelID, Usage, mergeJsonRecords, } from "../schema/index.js";
|
|
6
6
|
const Noul = Schema.Struct({
|
|
@@ -40,19 +40,29 @@ const Score = Schema.Struct({
|
|
|
40
40
|
legend: Schema.optional(Schema.Record(Schema.String, Schema.Json)),
|
|
41
41
|
confidence: Schema.optional(Probability),
|
|
42
42
|
});
|
|
43
|
-
const
|
|
43
|
+
const Answer = Schema.Union([NoulAnswer, Choice, Score]).pipe(Schema.toTaggedUnion("type"));
|
|
44
|
+
const NativeUsage = Schema.StructWithRest(Schema.Struct({
|
|
44
45
|
input_tokens: Schema.optional(Schema.Number),
|
|
45
46
|
output_tokens: Schema.optional(Schema.Number),
|
|
47
|
+
}), [Schema.Record(Schema.String, Schema.Unknown)]);
|
|
48
|
+
const Response = Schema.Struct({
|
|
49
|
+
model: Schema.String,
|
|
50
|
+
answers: Schema.Record(Schema.String, Answer),
|
|
51
|
+
usage: Schema.optional(NativeUsage),
|
|
52
|
+
id: Schema.optional(Schema.String),
|
|
53
|
+
provider: Schema.optional(Schema.String),
|
|
54
|
+
provider_metadata: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
|
|
46
55
|
});
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
56
|
+
export const model = (cfg) => EvaluationModel.make({
|
|
57
|
+
id: cfg.id,
|
|
58
|
+
provider: cfg.provider,
|
|
59
|
+
http: cfg.http,
|
|
60
|
+
route: {
|
|
51
61
|
id: "system-one",
|
|
52
62
|
evaluate: (req, send) => Effect.gen(function* () {
|
|
53
63
|
const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`);
|
|
54
64
|
Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value));
|
|
55
|
-
const body = yield*
|
|
65
|
+
const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({
|
|
56
66
|
...mergeJsonRecords(req.options, req.http?.body),
|
|
57
67
|
model: req.model.id,
|
|
58
68
|
state: req.state,
|
|
@@ -67,56 +77,36 @@ export const model = (cfg) => {
|
|
|
67
77
|
});
|
|
68
78
|
const res = yield* send(HttpClientRequest.post(url).pipe(HttpClientRequest.setHeaders(headers), HttpClientRequest.bodyText(body, "application/json")));
|
|
69
79
|
const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers });
|
|
70
|
-
const fail = (message, cause, body) => new AIError({ reason: new InvalidProviderOutputError({ route:
|
|
80
|
+
const fail = (message, cause, body) => new AIError({ reason: new InvalidProviderOutputError({ route: "system-one", message, body, http, cause }) });
|
|
71
81
|
const text = yield* res.text.pipe(Effect.mapError((cause) => fail("Failed to read the System One response", cause)));
|
|
72
|
-
const
|
|
73
|
-
const output = Schema.Struct({
|
|
74
|
-
model: Schema.String,
|
|
75
|
-
answers: Schema.Struct(Object.fromEntries(entries.map(([id, question]) => {
|
|
76
|
-
if (question.type === "boolean")
|
|
77
|
-
return [id, NoulAnswer];
|
|
78
|
-
if (question.type === "choice") {
|
|
79
|
-
const keys = Object.keys(question.criteria);
|
|
80
|
-
return [
|
|
81
|
-
id,
|
|
82
|
-
Choice.pipe(Schema.refine((x) => Object.hasOwn(question.criteria, x.choice) && exact(x.probabilities, keys), { message: `Question "${id}" returned an invalid choice answer` })),
|
|
83
|
-
];
|
|
84
|
-
}
|
|
85
|
-
const keys = question.criteria.map((_, index) => String(index));
|
|
86
|
-
return [
|
|
87
|
-
id,
|
|
88
|
-
Score.pipe(Schema.refine((x) => x.score >= 0 && x.score <= question.criteria.length - 1 && exact(x.probabilities, keys), { message: `Question "${id}" returned an invalid score answer` })),
|
|
89
|
-
];
|
|
90
|
-
}))),
|
|
91
|
-
usage: Schema.optional(NativeUsage),
|
|
92
|
-
});
|
|
93
|
-
const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(output))(text).pipe(Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)));
|
|
82
|
+
const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe(Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)));
|
|
94
83
|
const confidence = {};
|
|
95
84
|
const legend = {};
|
|
96
|
-
const answers = Object.fromEntries(entries.map(([id,
|
|
97
|
-
|
|
98
|
-
|
|
85
|
+
const answers = Object.fromEntries(Object.entries(data.answers).map(([id, answer]) => {
|
|
86
|
+
if (answer.type === "noul")
|
|
87
|
+
return [id, { type: "boolean", probability: answer.noul }];
|
|
88
|
+
if (answer.type === "choice") {
|
|
89
|
+
if (answer.confidence !== undefined)
|
|
90
|
+
confidence[id] = answer.confidence;
|
|
99
91
|
return [
|
|
100
92
|
id,
|
|
101
|
-
{
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
confidence[id] = value.confidence;
|
|
107
|
-
return [
|
|
108
|
-
id,
|
|
109
|
-
{ type: "choice", choice: value.choice, probabilities: value.probabilities },
|
|
93
|
+
{
|
|
94
|
+
type: "choice",
|
|
95
|
+
choice: answer.choice,
|
|
96
|
+
probabilities: answer.probabilities,
|
|
97
|
+
},
|
|
110
98
|
];
|
|
111
99
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
return [id, { type: "score", score: value.score, probabilities: value.probabilities }];
|
|
100
|
+
if (answer.confidence !== undefined)
|
|
101
|
+
confidence[id] = answer.confidence;
|
|
102
|
+
if (answer.legend !== undefined)
|
|
103
|
+
legend[id] = answer.legend;
|
|
104
|
+
return [id, { type: "score", score: answer.score, probabilities: answer.probabilities }];
|
|
118
105
|
}));
|
|
119
106
|
const meta = {
|
|
107
|
+
...(data.id === undefined ? {} : { responseId: data.id }),
|
|
108
|
+
...(data.provider === undefined ? {} : { provider: data.provider }),
|
|
109
|
+
...data.provider_metadata?.[cfg.providerMetadataKey],
|
|
120
110
|
...(Object.keys(confidence).length === 0 ? {} : { confidence }),
|
|
121
111
|
...(Object.keys(legend).length === 0 ? {} : { legend }),
|
|
122
112
|
};
|
|
@@ -137,7 +127,6 @@ export const model = (cfg) => {
|
|
|
137
127
|
providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta },
|
|
138
128
|
});
|
|
139
129
|
}),
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
};
|
|
130
|
+
},
|
|
131
|
+
});
|
|
143
132
|
export const SystemOne = { model };
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { Duration, Effect, Schedule, Schema, Stream } from "effect";
|
|
2
|
+
import { AIError } from "./schema/errors.js";
|
|
3
|
+
export declare const Status: Schema.Literals<readonly ["queued", "running", "completed", "failed", "cancelled", "expired"]>;
|
|
4
|
+
export type Status = Schema.Schema.Type<typeof Status>;
|
|
5
|
+
/** Provider-neutral view of one generation observation. */
|
|
6
|
+
export interface Snapshot {
|
|
7
|
+
readonly id: string;
|
|
8
|
+
readonly status: Status;
|
|
9
|
+
/** Normalized 0..1 when the provider reports progress. */
|
|
10
|
+
readonly progress?: number;
|
|
11
|
+
readonly position?: number;
|
|
12
|
+
readonly expiresAt?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Route-owned generation operations. `token` is the route's serializable handle (operation name, task id, response URL)
|
|
16
|
+
* so a generation can be resumed from another process; its shape is opaque to `Generation`.
|
|
17
|
+
*/
|
|
18
|
+
export interface Route<Response> {
|
|
19
|
+
readonly status: (token: unknown) => Effect.Effect<Snapshot, AIError>;
|
|
20
|
+
readonly result: (token: unknown) => Effect.Effect<Response, AIError>;
|
|
21
|
+
readonly cancel?: (token: unknown) => Effect.Effect<void, AIError>;
|
|
22
|
+
/** Provider polling hint (e.g. `openai-poll-after-ms`) that overrides the default interval for the next poll. */
|
|
23
|
+
readonly pollHint?: (snapshot: Snapshot) => Duration.Duration | undefined;
|
|
24
|
+
}
|
|
25
|
+
export interface Poll {
|
|
26
|
+
readonly interval?: Duration.Input;
|
|
27
|
+
readonly timeout?: Duration.Input;
|
|
28
|
+
/** Full override of the polling schedule; `interval` and `pollHint` are ignored when supplied. */
|
|
29
|
+
readonly schedule?: Schedule.Schedule<unknown, Snapshot>;
|
|
30
|
+
}
|
|
31
|
+
export declare const DEFAULT_POLL_INTERVAL: Duration.Duration;
|
|
32
|
+
export declare const DEFAULT_POLL_TIMEOUT: Duration.Duration;
|
|
33
|
+
export type Event = {
|
|
34
|
+
readonly type: "generation-queued";
|
|
35
|
+
readonly id: string;
|
|
36
|
+
readonly position?: number;
|
|
37
|
+
} | {
|
|
38
|
+
readonly type: "generation-progress";
|
|
39
|
+
readonly id: string;
|
|
40
|
+
readonly progress?: number;
|
|
41
|
+
} | {
|
|
42
|
+
readonly type: "generation-finished";
|
|
43
|
+
readonly id: string;
|
|
44
|
+
readonly status: Status;
|
|
45
|
+
};
|
|
46
|
+
export declare class Generation<Response> {
|
|
47
|
+
readonly route: Route<Response>;
|
|
48
|
+
readonly token: unknown;
|
|
49
|
+
readonly id: string;
|
|
50
|
+
readonly status: Status;
|
|
51
|
+
readonly progress?: number;
|
|
52
|
+
readonly position?: number;
|
|
53
|
+
readonly expiresAt?: number;
|
|
54
|
+
constructor(route: Route<Response>, token: unknown, snapshot: Snapshot);
|
|
55
|
+
get snapshot(): Snapshot;
|
|
56
|
+
get terminal(): boolean;
|
|
57
|
+
refresh(): Effect.Effect<Generation<Response>, AIError>;
|
|
58
|
+
/** Poll until the generation reaches a terminal status, then fetch the result. Fails with a `Timeout` reason on deadline. */
|
|
59
|
+
await(options?: {
|
|
60
|
+
readonly poll?: Poll;
|
|
61
|
+
}): Effect.Effect<Response, AIError>;
|
|
62
|
+
cancel(): Effect.Effect<void, AIError>;
|
|
63
|
+
/** Status observations as a stream, ending after the first terminal observation. */
|
|
64
|
+
events(options?: {
|
|
65
|
+
readonly poll?: Poll;
|
|
66
|
+
}): Stream.Stream<Event, AIError>;
|
|
67
|
+
private poll;
|
|
68
|
+
private schedule;
|
|
69
|
+
}
|