bios-sdk 0.1.1-dev.33 → 0.1.1-dev.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -107,6 +107,39 @@ for await (const chunk of client.inference.streamChatCompletions({
|
|
|
107
107
|
// controller.abort() cancels an unfinished generation.
|
|
108
108
|
```
|
|
109
109
|
|
|
110
|
+
#### Serverless catalog models
|
|
111
|
+
|
|
112
|
+
Call any catalog model by id on the unified `/v1` endpoint with a workspace
|
|
113
|
+
platform key that carries the serverless scope — no per-deployment key. The
|
|
114
|
+
gateway routes by `model`; dedicated deployments and serverless models share the
|
|
115
|
+
same endpoint. `reasoningEffort` is forwarded to the endpoint, and streaming
|
|
116
|
+
surfaces `content` and `reasoning_content` deltas incrementally.
|
|
117
|
+
|
|
118
|
+
```typescript
|
|
119
|
+
const client = new BiOS({
|
|
120
|
+
inferenceKey: 'bios-platform-key-with-serverless-scope',
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
for await (const chunk of client.inference.streamChatCompletions({
|
|
124
|
+
model: 'meta-llama/Llama-3.1-8B-Instruct', // serverless catalog id
|
|
125
|
+
messages: [{ role: 'user', content: 'Explain tensor parallelism briefly.' }],
|
|
126
|
+
reasoningEffort: 'low',
|
|
127
|
+
})) {
|
|
128
|
+
const delta = (chunk.choices as any)?.[0]?.delta ?? {};
|
|
129
|
+
if (delta.reasoning_content) process.stdout.write(delta.reasoning_content);
|
|
130
|
+
if (delta.content) process.stdout.write(delta.content);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Non-streaming; the final chunk carries `usage` when the model reports it.
|
|
134
|
+
const completion = await client.inference.chatCompletions({
|
|
135
|
+
model: 'meta-llama/Llama-3.1-8B-Instruct',
|
|
136
|
+
messages: [{ role: 'user', content: 'One sentence on GPUs.' }],
|
|
137
|
+
});
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Streaming billing is charged server-side on completed usage; the SDK only needs
|
|
141
|
+
to request `usage` where the endpoint exposes it (no client change).
|
|
142
|
+
|
|
110
143
|
### Models
|
|
111
144
|
|
|
112
145
|
The catalog lists only models hosted on BiOS (the platform's own verified
|