@withgateway/sdk 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1303 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +1290 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/{index-CTc8iOrx.d.ts → index-B5HUQQDE.d.mts} +9 -0
- package/dist/{index-CTc8iOrx.d.mts → index-Dpo-t8iO.d.ts} +9 -0
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +878 -32
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +845 -25
- package/dist/index.mjs.map +1 -1
- package/dist/scores/index.d.mts +165 -0
- package/dist/scores/index.d.ts +165 -0
- package/dist/scores/index.js +252 -0
- package/dist/scores/index.js.map +1 -0
- package/dist/scores/index.mjs +223 -0
- package/dist/scores/index.mjs.map +1 -0
- package/dist/tracing/index.d.mts +2 -1
- package/dist/tracing/index.d.ts +2 -1
- package/dist/tracing/index.js +262 -15
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +256 -9
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +377 -0
- package/dist/worlds/index.d.ts +377 -0
- package/dist/worlds/index.js +584 -0
- package/dist/worlds/index.js.map +1 -0
- package/dist/worlds/index.mjs +578 -0
- package/dist/worlds/index.mjs.map +1 -0
- package/package.json +32 -2
package/README.md
CHANGED
|
@@ -152,6 +152,57 @@ experiments.clearExperimentContext() // Clear context (call after each request)
|
|
|
152
152
|
experiments.shutdown() // Cleanup (call on exit)
|
|
153
153
|
```
|
|
154
154
|
|
|
155
|
+
### Scores & Success Signals
|
|
156
|
+
|
|
157
|
+
Record evaluation scores and ROI-shaped **success signals** against a project.
|
|
158
|
+
A success signal is a score tagged with `gateway.success.signal`, which the
|
|
159
|
+
platform's **Success Metrics** feature aggregates into a metric's headline
|
|
160
|
+
count / pass-rate (the `HOOK` metric source). The score shape is identical to
|
|
161
|
+
the Python SDK's `run.success()`, so signals from either SDK aggregate the same.
|
|
162
|
+
|
|
163
|
+
```typescript
|
|
164
|
+
import { ScoresClient, success } from "@withgateway/sdk/scores";
|
|
165
|
+
|
|
166
|
+
// Env-based (GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY):
|
|
167
|
+
await success("resolved_ticket", true, {
|
|
168
|
+
agentId: "support-triage",
|
|
169
|
+
sessionId,
|
|
170
|
+
});
|
|
171
|
+
await success("handle_time_s", 42, { agentId: "support-triage", sessionId });
|
|
172
|
+
|
|
173
|
+
// Explicit keys:
|
|
174
|
+
const scores = new ScoresClient({ host, publicKey, secretKey });
|
|
175
|
+
await scores.success("resolved_ticket", true, {
|
|
176
|
+
agentId: "support-triage",
|
|
177
|
+
sessionId,
|
|
178
|
+
});
|
|
179
|
+
await scores.score("rubric", 0.9, { sessionId }); // a plain (non-signal) score
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
A boolean `value` is recorded as a 1/0 **BOOLEAN** score (pass/fail); a number
|
|
183
|
+
is a **NUMERIC** score. `metricName` defaults to the signal `name`. Provide
|
|
184
|
+
exactly one target — `sessionId`, `traceId` (+ optional `observationId`), or
|
|
185
|
+
`datasetRunId`; when none is given the active OpenTelemetry trace id is used,
|
|
186
|
+
and if there is none the call is a fail-soft no-op. Every `score`/`success`
|
|
187
|
+
call is **fail-soft**: a bad key or transport failure warns and resolves, never
|
|
188
|
+
throwing into your run. (`postScore` is the raw variant that DOES throw.)
|
|
189
|
+
|
|
190
|
+
Under `tracing.initIsolated(...)`, the returned handle carries a `scores`
|
|
191
|
+
client bound to that provider's project keys:
|
|
192
|
+
|
|
193
|
+
```typescript
|
|
194
|
+
const handle = tracing.initIsolated({
|
|
195
|
+
host,
|
|
196
|
+
publicKey,
|
|
197
|
+
secretKey,
|
|
198
|
+
agent: "support-agent",
|
|
199
|
+
});
|
|
200
|
+
await handle.scores.success("resolved_ticket", true, {
|
|
201
|
+
agentId: "support-agent",
|
|
202
|
+
sessionId,
|
|
203
|
+
});
|
|
204
|
+
```
|
|
205
|
+
|
|
155
206
|
### Security Wrapper
|
|
156
207
|
|
|
157
208
|
Use the security wrapper when you want Gateway to control tool visibility,
|
|
@@ -215,6 +266,70 @@ const client = new ExperimentClient({
|
|
|
215
266
|
});
|
|
216
267
|
```
|
|
217
268
|
|
|
269
|
+
### Worlds & Live Sessions
|
|
270
|
+
|
|
271
|
+
A **world** is a versioned environment the platform hosts — tasks, tools,
|
|
272
|
+
seed data and a grader, addressed as `slug@ref`. A **session** boots one
|
|
273
|
+
task's container and holds it warm while _your_ agent, running wherever it
|
|
274
|
+
runs, calls its tools.
|
|
275
|
+
|
|
276
|
+
The distinction that matters: the agent is local, the world is not. Every
|
|
277
|
+
tool call executes inside the platform's container, so you are testing
|
|
278
|
+
against the environment as it actually is rather than a copy unwrapped on
|
|
279
|
+
your own disk.
|
|
280
|
+
|
|
281
|
+
```typescript
|
|
282
|
+
import { runSessions } from "@withgateway/sdk";
|
|
283
|
+
|
|
284
|
+
const report = await runSessions({
|
|
285
|
+
world: "revenue-ops-sim@main",
|
|
286
|
+
concurrency: 4, // one warm container per task
|
|
287
|
+
failUnder: 0.7, // report.passed reads the mean
|
|
288
|
+
agent: async ({ toolkit, instruction }) => {
|
|
289
|
+
// The world's own tool schemas and callable implementations — nothing
|
|
290
|
+
// to hand-write, nothing to keep in sync.
|
|
291
|
+
await myAgent(toolkit.schemas, toolkit.impls, instruction);
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
console.log(report.mean, report.passed);
|
|
296
|
+
process.exit(report.passed ? 0 : 1);
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
`runSessions` resolves the world **once**, so every task pins the same
|
|
300
|
+
version and a push mid-run cannot split results across two worlds. A task
|
|
301
|
+
that throws scores 0 and records why; the rest keep their results. Containers
|
|
302
|
+
are handed back warm, so the next run starts hot.
|
|
303
|
+
|
|
304
|
+
Driving a single session directly, when you want the control:
|
|
305
|
+
|
|
306
|
+
```typescript
|
|
307
|
+
import { openSession } from "@withgateway/sdk";
|
|
308
|
+
|
|
309
|
+
const session = await openSession(
|
|
310
|
+
"revenue-ops-sim",
|
|
311
|
+
"revops-mara-dedupe-contacts",
|
|
312
|
+
);
|
|
313
|
+
await session.ready(); // wait for the container
|
|
314
|
+
await session.call("crm_search", { entity: "deals" });
|
|
315
|
+
const { reward, rewards } = await session.grade(); // the task's own grader
|
|
316
|
+
await session.close({ keepWarm: true });
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
Sessions trace themselves from the platform — one trace per episode, one span
|
|
320
|
+
per tool call — so an episode is readable even if your process dies mid-run.
|
|
321
|
+
|
|
322
|
+
**Full worked example:** [`examples/world-suite.ts`](examples/world-suite.ts)
|
|
323
|
+
is a complete local production harness — a real tool-calling agent loop,
|
|
324
|
+
parallel execution, a pass gate, a JSON artifact and a CI exit code. The
|
|
325
|
+
Python twin is [`gatewaysdk/examples/worlds/world_suite.py`](../sdk/examples/worlds/world_suite.py).
|
|
326
|
+
|
|
327
|
+
There is also a CLI for pushing, pulling and running worlds:
|
|
328
|
+
|
|
329
|
+
```bash
|
|
330
|
+
npx gateway bench --help
|
|
331
|
+
```
|
|
332
|
+
|
|
218
333
|
## Cross-Language Compatibility
|
|
219
334
|
|
|
220
335
|
This SDK produces **identical assignments** to the Python SDK (`gatewaysdk`) for the same `(experimentKey, userId)` pair. You can run experiments across services in different languages.
|
package/dist/cli.d.mts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
#!/usr/bin/env node
|