@pipelex/sdk 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/dist/cli/script.d.ts +1 -1
- package/dist/client.d.ts +68 -3
- package/dist/client.js +163 -82
- package/dist/client.js.map +1 -1
- package/dist/error-verdicts.js +14 -3
- package/dist/error-verdicts.js.map +1 -1
- package/dist/errors.d.ts +30 -8
- package/dist/errors.js +13 -8
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js.map +1 -1
- package/dist/product-models.d.ts +46 -0
- package/dist/runs.d.ts +11 -0
- package/dist/runs.js +35 -0
- package/dist/runs.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +3 -2
- package/docs/errors.md +39 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -72,6 +72,23 @@ const saved = await client.start({ method_id: "mt_abc123@3", inputs: {} });
|
|
|
72
72
|
console.log(saved.method_version); // 3
|
|
73
73
|
```
|
|
74
74
|
|
|
75
|
+
### A start you can retry: `idempotencyKey`
|
|
76
|
+
|
|
77
|
+
`start` and `startAndWaitForResult` take an optional `idempotencyKey`, sent as the `Idempotency-Key` header of `POST /v1/start`. The hosted API binds the key to the run the first request starts, so the same start sent again under the same key is answered with that run and starts nothing: a start whose answer was lost can be sent again without paying for a second run. Generate one key per run, and keep it with the options, unchanged, across that start's retries:
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
import { runMayHaveStartedOf } from "@pipelex/sdk";
|
|
81
|
+
|
|
82
|
+
const options = { method_id: "mt_abc123", inputs, idempotencyKey: crypto.randomUUID() };
|
|
83
|
+
const ack = await client.start(options).catch((err: unknown) => {
|
|
84
|
+
// The run may exist: under the same key, the same start is answered with it.
|
|
85
|
+
if (runMayHaveStartedOf(err)) return client.start(options);
|
|
86
|
+
throw err;
|
|
87
|
+
});
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
A key is 1 to 255 visible ASCII characters; a key reused with other options is refused with a `409` `conflict`, and one whose earlier start is still being handled with a retryable `409` `idempotency_key_in_use` ([`docs/errors.md`](./docs/errors.md#a-starts-idempotency-key--idempotencykey)). A bare runner ignores the header.
|
|
91
|
+
|
|
75
92
|
### Checking a model reference
|
|
76
93
|
|
|
77
94
|
`checkModelReference` asks the runner whether a reference a method's `model` field could name resolves, as what kind and to which model (`GET /v1/models/check`, served by `pipelex-api` from pipelex 0.78.0). A reference that resolves nowhere is a verdict, not an error, and narrowing on `kind` types its `matches`:
|
|
@@ -102,6 +119,8 @@ const client = new PipelexApiClient({ apiKey: process.env.PIPELEX_API_KEY });
|
|
|
102
119
|
|
|
103
120
|
const me = await client.getMe(); // GET /v1/me
|
|
104
121
|
const page = await client.listMethods(); // GET /v1/methods — one page: { items, nextCursor }
|
|
122
|
+
// Only the methods never published. A filtered page can be empty and still carry a nextCursor.
|
|
123
|
+
const drafts = await client.listMethods({ published: false });
|
|
105
124
|
for await (const method of client.iterateMethods()) {
|
|
106
125
|
// follows the cursor for callers that genuinely want the whole catalog
|
|
107
126
|
}
|
|
@@ -134,7 +153,7 @@ try {
|
|
|
134
153
|
|
|
135
154
|
### Errors
|
|
136
155
|
|
|
137
|
-
Every error the SDK throws carries a verdict: `retryable`, whether asking again can succeed, and `errorDomain`, who can fix the failure (`input` for the caller, `config` for a change to the environment such as the credential, the plan or the base URL, `runtime` for nobody beforehand). Both are always decided, and `errorVerdictOf` reads them from anything a `catch` holds, returning `undefined` for what is not an SDK error, such as a bug in the calling code or your own abort. A failed `start` or blocking `execute` also says whether it may have created a run anyway, as `runMayHaveStarted`, read with `runMayHaveStartedOf`: when it is `true`, `retryable` is `false`, since asking again could start a second paid run, and the person should look the run up in the run history first ([`docs/errors.md`](./docs/errors.md#a-failed-run-start--runmayhavestarted)).
|
|
156
|
+
Every error the SDK throws carries a verdict: `retryable`, whether asking again can succeed, and `errorDomain`, who can fix the failure (`input` for the caller, `config` for a change to the environment such as the credential, the plan or the base URL, `runtime` for nobody beforehand). Both are always decided, and `errorVerdictOf` reads them from anything a `catch` holds, returning `undefined` for what is not an SDK error, such as a bug in the calling code or your own abort. A failed `start` or blocking `execute` also says whether it may have created a run anyway, as `runMayHaveStarted`, read with `runMayHaveStartedOf`: when it is `true`, `retryable` is `false`, since asking again could start a second paid run, and the person should look the run up in the run history first ([`docs/errors.md`](./docs/errors.md#a-failed-run-start--runmayhavestarted)), unless the start carried an `idempotencyKey`, in which case sending it again unchanged recovers the run ([above](#a-start-you-can-retry-idempotencykey)).
|
|
138
157
|
|
|
139
158
|
```ts
|
|
140
159
|
import { errorVerdictOf } from "@pipelex/sdk";
|
package/dist/cli/script.d.ts
CHANGED
|
@@ -15,7 +15,7 @@ import type { CommandIO } from "./io.js";
|
|
|
15
15
|
* update notice and npm 12's run notices off the script's stderr, which carries the command's
|
|
16
16
|
* own lines, while npm's warnings and errors still print.
|
|
17
17
|
*/
|
|
18
|
-
export declare const SCRIPT_RUNNER = "npx --yes --no-update-notifier --loglevel=warn @pipelex/sdk@0.
|
|
18
|
+
export declare const SCRIPT_RUNNER = "npx --yes --no-update-notifier --loglevel=warn @pipelex/sdk@0.42.0";
|
|
19
19
|
export declare const SCRIPT_WRITER = "@pipelex/sdk";
|
|
20
20
|
export declare const SCRIPT_NEEDS = "Node 22.12 or later";
|
|
21
21
|
/** Run `pipelex-sdk script` and return its exit code. */
|
package/dist/client.d.ts
CHANGED
|
@@ -105,8 +105,37 @@ export interface PipelexHostedRunExtensions {
|
|
|
105
105
|
}
|
|
106
106
|
/** `execute()` options — the protocol's run arguments plus the Pipelex API and hosted extensions. */
|
|
107
107
|
export type PipelexRunOptions = RunOptions & PipelexApiRunExtensions & PipelexHostedRunExtensions;
|
|
108
|
-
/**
|
|
109
|
-
|
|
108
|
+
/**
|
|
109
|
+
* What a run start sends beside its body: the request's own headers, which no run argument
|
|
110
|
+
* reaches. Each is camelCase, since none of them is a wire field of the run request.
|
|
111
|
+
*/
|
|
112
|
+
export interface PipelexStartRequestOptions {
|
|
113
|
+
/**
|
|
114
|
+
* The start's idempotency key, sent as the `Idempotency-Key` header of `POST /v1/start`, and
|
|
115
|
+
* not sent at all when absent or `null`. Generate one per run the caller means to start (a
|
|
116
|
+
* `crypto.randomUUID()` is a good key), and keep it, with every other option unchanged, across
|
|
117
|
+
* that start's retries.
|
|
118
|
+
*
|
|
119
|
+
* The hosted API binds the key to the run the first request starts, before it dispatches it, so
|
|
120
|
+
* the same start sent again under the same key is answered with that run's acknowledgement and
|
|
121
|
+
* starts nothing, whatever became of the first attempt, for 24 hours. A failed start whose
|
|
122
|
+
* `runMayHaveStarted` is true can therefore be sent again unchanged, and recovers the run instead
|
|
123
|
+
* of paying for a second one. The same key with a different body is refused (`409` `conflict`),
|
|
124
|
+
* as is a key whose earlier request is still being handled (`409` `idempotency_key_in_use`,
|
|
125
|
+
* retryable: send it again under the same key in a few seconds); see `docs/errors.md`. A bare
|
|
126
|
+
* runner ignores the header.
|
|
127
|
+
*
|
|
128
|
+
* A key is 1 to 255 visible ASCII characters (`!` to `~`, so no space); anything else, an empty
|
|
129
|
+
* string included, is a `RequestArgumentError` before any request is sent, since a start sent
|
|
130
|
+
* without the key the caller asked for would lose the protection silently.
|
|
131
|
+
*/
|
|
132
|
+
idempotencyKey?: string | null;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* `start()` / `startAndWaitForResult()` options — the same, for the durable path, plus the start
|
|
136
|
+
* request's own `idempotencyKey`.
|
|
137
|
+
*/
|
|
138
|
+
export type PipelexStartOptions = StartOptions & PipelexApiRunExtensions & PipelexHostedRunExtensions & PipelexStartRequestOptions;
|
|
110
139
|
/**
|
|
111
140
|
* The method selector `validate()` takes in place of inline contents — declared
|
|
112
141
|
* beside the other wire shapes in `./models.js` and re-exported here, the import
|
|
@@ -205,6 +234,8 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
205
234
|
* cleanly. Non-2xx interpretation is left to the caller. `url` is a fully
|
|
206
235
|
* resolved absolute URL. `mayStartRun` marks a request that may create a run,
|
|
207
236
|
* whose `ApiUnreachableError` then says whether it may have (see `RunCreation`).
|
|
237
|
+
* `idempotencyKey`, already checked by `idempotencyKeyOf`, is sent as the
|
|
238
|
+
* `Idempotency-Key` header; only `start` passes one.
|
|
208
239
|
*/
|
|
209
240
|
private requestRaw;
|
|
210
241
|
/**
|
|
@@ -337,6 +368,13 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
337
368
|
* durable — poll `getRunStatus` / `getRunResult`. A `method_ref` run's ack
|
|
338
369
|
* additionally carries `method_provenance` — the address, the tag, and the
|
|
339
370
|
* commit SHA that was actually fetched.
|
|
371
|
+
*
|
|
372
|
+
* `options.idempotencyKey` is sent as the `Idempotency-Key` header, and nothing
|
|
373
|
+
* is sent without it: the hosted API answers the same start sent again under the
|
|
374
|
+
* same key with the run the first one started, so a failure whose
|
|
375
|
+
* `runMayHaveStarted` is true can be retried unchanged (see
|
|
376
|
+
* `PipelexStartRequestOptions`). A malformed key is a `RequestArgumentError`
|
|
377
|
+
* before anything is sent.
|
|
340
378
|
*/
|
|
341
379
|
start(options: PipelexStartOptions): Promise<PipelexRunResultStart>;
|
|
342
380
|
/**
|
|
@@ -687,6 +725,12 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
687
725
|
* up in the run history first. An error from the wait, once `onStarted` has named the run,
|
|
688
726
|
* started nothing, so its `runMayHaveStarted` is false, and its `retryable` is about waiting
|
|
689
727
|
* again on that id with `waitForResult`, never about calling this method again.
|
|
728
|
+
*
|
|
729
|
+
* `options.idempotencyKey` rides the durable start, so a call that failed while its start may
|
|
730
|
+
* have created a run can be made again with the same options and the same key: the hosted API
|
|
731
|
+
* answers the start with the run the first call started, and the wait follows it. The blocking
|
|
732
|
+
* execute of a bare runner carries no key, a bare runner having no idempotency to honour it. A
|
|
733
|
+
* malformed key is refused before the version handshake.
|
|
690
734
|
*/
|
|
691
735
|
startAndWaitForResult(options: PipelexStartOptions, pollOptions?: StartAndWaitForResultOptions): Promise<RunResults>;
|
|
692
736
|
/** The authenticated user's profile — `GET /v1/me`. */
|
|
@@ -705,7 +749,11 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
705
749
|
* Rows are `MethodSummary`: no `mthds`, no `python`, no `updated_at`. Reach
|
|
706
750
|
* for `getMethod(id)` when you need a method's source.
|
|
707
751
|
*
|
|
708
|
-
* `q` is applied server-side over the whole catalog, not over the page
|
|
752
|
+
* `q` is applied server-side over the whole catalog, not over the page, and so is
|
|
753
|
+
* `published` (`true` for the methods with a published version, `false` for those never
|
|
754
|
+
* published). Each row carries the method's publish state (`latest_version`,
|
|
755
|
+
* `latest_published`, `draft_digest`). Both filters run after the index read, so a page can
|
|
756
|
+
* come back empty and still carry a `nextCursor`: keep following it until it is `null`.
|
|
709
757
|
*/
|
|
710
758
|
listMethods(query?: ListMethodsQuery): Promise<MethodPage>;
|
|
711
759
|
/**
|
|
@@ -726,6 +774,11 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
726
774
|
* TRUNCATED list with no error — precisely the bug paging was introduced to
|
|
727
775
|
* remove. If you truly want an array, `Array.fromAsync` makes that your
|
|
728
776
|
* explicit choice.
|
|
777
|
+
*
|
|
778
|
+
* It keeps going **through an empty page that carries a cursor**: under `q` or `published`
|
|
779
|
+
* such a page means "nothing matched in the slice just read", not "done". It stops on a
|
|
780
|
+
* `null` cursor or on a cursor the server failed to advance, and throws
|
|
781
|
+
* `PagingNotTerminatingError` past a page ceiling no real catalog reaches.
|
|
729
782
|
*/
|
|
730
783
|
iterateMethods(query?: Omit<ListMethodsQuery, "cursor">): AsyncGenerator<MethodSummary, void, undefined>;
|
|
731
784
|
/**
|
|
@@ -1065,6 +1118,13 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
1065
1118
|
* conditions, so a bounded page genuinely reads less. They are INSTANTS,
|
|
1066
1119
|
* not days — see `ListRunsQuery`.
|
|
1067
1120
|
*
|
|
1121
|
+
* `createdBy: "me"` keeps the caller's own runs and `status` the runs in any of the given
|
|
1122
|
+
* statuses, sent as `created_by=me` and one repeated `status=` parameter per status. Both are
|
|
1123
|
+
* filters applied after the index read, so a filtered page can hold fewer runs than `limit`,
|
|
1124
|
+
* or none, and still carry a `nextCursor`: an empty page is not the end, a `null` cursor is.
|
|
1125
|
+
* An empty `status` list, or a name that is no `RunStatus`, is a `RequestArgumentError`
|
|
1126
|
+
* before any request.
|
|
1127
|
+
*
|
|
1068
1128
|
* Takes a bare catalog id: the history files the runs of every version and
|
|
1069
1129
|
* of the draft together under it, so `mt_…@3` names no history of its own
|
|
1070
1130
|
* and the platform refuses it with a `400`. Strip a suffix with
|
|
@@ -1094,6 +1154,11 @@ export declare class PipelexApiClient implements MTHDSProtocol<DictPipeOutput> {
|
|
|
1094
1154
|
* when to stop, and only one page is ever in memory. If you truly want an
|
|
1095
1155
|
* array, `Array.fromAsync` makes that your explicit choice.
|
|
1096
1156
|
*
|
|
1157
|
+
* It keeps going **through an empty page that carries a cursor**: under `createdBy` or
|
|
1158
|
+
* `status` such a page means "nothing matched in the slice just read", not "done". It stops on
|
|
1159
|
+
* a `null` cursor or on a cursor the server failed to advance, and throws
|
|
1160
|
+
* `PagingNotTerminatingError` past a page ceiling no real history reaches.
|
|
1161
|
+
*
|
|
1097
1162
|
* Takes a bare catalog id, as `listRuns` does.
|
|
1098
1163
|
*/
|
|
1099
1164
|
iterateRuns(methodId: string, query?: Omit<ListRunsQuery, "cursor">): AsyncGenerator<RunHistoryItem, void, undefined>;
|
package/dist/client.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// source combinations are legal is an invariant of `RunRequest` itself, so this
|
|
3
3
|
// client and the MTHDS runners cannot disagree about what they reject.
|
|
4
4
|
import { assertExclusiveRunSources, hasBundlePayload, parseMethodFiles, serializeMethodFiles, } from "mthds/protocol";
|
|
5
|
-
import { assertArtifactSelection, assertWaitOptions, pollUntilResult, readRunDegradation, selectionIncludesMainStuff, throwIfAborted, } from "./runs.js";
|
|
5
|
+
import { assertArtifactSelection, assertWaitOptions, pollUntilResult, readRunDegradation, RUN_STATUSES, runStatusFilter, selectionIncludesMainStuff, throwIfAborted, } from "./runs.js";
|
|
6
6
|
import { ApiResponseError, ApiUnreachableError, EmptyMethodSourceError, MissingMainStuffError, PagingNotTerminatingError, PipelineExecuteTimeoutError, PipelineRequestError, RequestArgumentError, RunLifecycleUnavailableError, RunStillRunningError, } from "./errors.js";
|
|
7
7
|
import { isPlainObject, isValidationItem, readFieldError, readMigration, readProviderMetadata, readRunErrorReport, } from "./error-models.js";
|
|
8
8
|
import { methodSourceToContents } from "./method-source.js";
|
|
@@ -13,20 +13,69 @@ import { prepareInputs as prepareInputsImpl } from "./prepare-inputs.js";
|
|
|
13
13
|
import { downloadArtifacts as downloadArtifactsImpl, fetchArtifact as fetchArtifactImpl, resolveArtifacts as resolveArtifactsImpl, } from "./artifacts.js";
|
|
14
14
|
import { PipelexExecuteResult, resultsFromExecute } from "./execute-result.js";
|
|
15
15
|
import { MAX_TIMER_DELAY_MS, isTimerDelay } from "./timers.js";
|
|
16
|
-
// A pure RUNAWAY guard on `iterateMethods`,
|
|
16
|
+
// A pure RUNAWAY guard on the paged-list iterators (`iterateMethods`,
|
|
17
|
+
// `iterateRuns`), deliberately not a coverage limit.
|
|
17
18
|
//
|
|
18
19
|
// It counts TOTAL pages, not empty ones. An earlier version capped consecutive
|
|
19
20
|
// empty pages, which is the wrong axis: an empty page whose cursor ADVANCED is
|
|
20
|
-
// real progress through the index — the platform applies
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
21
|
+
// real progress through the index — the platform applies the list filters
|
|
22
|
+
// (`q` and `published` on methods, `createdBy` and `status` on runs) after a
|
|
23
|
+
// bounded read per request, so a sparse match legitimately yields runs of
|
|
24
|
+
// empty pages — and capping them turns a valid sparse search into a thrown
|
|
25
|
+
// error. The stuck case is already caught by the no-progress check (the
|
|
26
|
+
// server handing back the cursor we sent).
|
|
25
27
|
//
|
|
26
|
-
// At the platform's 200-row maximum page size this is 2,000,000
|
|
28
|
+
// At the platform's 200-row maximum page size this is 2,000,000 rows, so it
|
|
27
29
|
// cannot fire on real data; it exists only so a server minting fresh cursors
|
|
28
30
|
// forever cannot hang a caller indefinitely.
|
|
29
31
|
const MAX_PAGES = 10_000;
|
|
32
|
+
/**
|
|
33
|
+
* Follow a paged list's cursor until the server says there is no more, yielding every row —
|
|
34
|
+
* the one loop behind `iterateMethods` and `iterateRuns`, so the two cannot drift apart.
|
|
35
|
+
*
|
|
36
|
+
* It yields lazily: a caller that breaks early stops the requests. It stops when `nextCursor`
|
|
37
|
+
* is `null`, the normal end, and when the server hands back the cursor it was just sent, which
|
|
38
|
+
* is no progress and would repeat rows already yielded. It keeps going through an EMPTY page
|
|
39
|
+
* that carries a cursor, since a filtered page can hold no row and still not be the end; past
|
|
40
|
+
* `MAX_PAGES` pages it throws `PagingNotTerminatingError` rather than return a truncated list.
|
|
41
|
+
*/
|
|
42
|
+
async function* followCursor(route, readPage) {
|
|
43
|
+
let cursor;
|
|
44
|
+
let pages = 0;
|
|
45
|
+
for (;;) {
|
|
46
|
+
const page = await readPage(cursor);
|
|
47
|
+
// Checked BEFORE yielding: a server handing back the very cursor we sent
|
|
48
|
+
// has not advanced, so this page repeats rows already emitted — yielding
|
|
49
|
+
// first and stopping after would silently double-count them for anyone
|
|
50
|
+
// aggregating the stream (summing cost, counting runs). Cannot fire on the
|
|
51
|
+
// first request, where `cursor` is undefined.
|
|
52
|
+
if (cursor !== undefined && page.nextCursor === cursor)
|
|
53
|
+
return;
|
|
54
|
+
for (const item of page.items)
|
|
55
|
+
yield item;
|
|
56
|
+
// `nextCursor === null` is the NORMAL end of the list, and that page is
|
|
57
|
+
// real data, so it has to be emitted before this fires.
|
|
58
|
+
if (page.nextCursor === null)
|
|
59
|
+
return;
|
|
60
|
+
// An EMPTY page is NOT the end, and treating it as one was a bug. The
|
|
61
|
+
// platform filters after a bounded index read per request, so a sparse
|
|
62
|
+
// match legitimately returns `{items: [], nextCursor: "…"}` — "nothing
|
|
63
|
+
// matched in the slice I just read, keep going". Returning here dropped
|
|
64
|
+
// every later match silently, which is the exact truncation this
|
|
65
|
+
// pagination work exists to remove.
|
|
66
|
+
pages += 1;
|
|
67
|
+
if (pages >= MAX_PAGES) {
|
|
68
|
+
// THROW, do not return. A caller that asked for every row and got a
|
|
69
|
+
// partial answer with no error is back to the original bug one layer
|
|
70
|
+
// up; an error is the only honest response to a server that will not
|
|
71
|
+
// finish. Unreachable on real data — see MAX_PAGES. It also bounds a
|
|
72
|
+
// cursor that CYCLES (`c1 → c2 → c1`), which the adjacent check misses.
|
|
73
|
+
throw new PagingNotTerminatingError(`${route} did not terminate after ${MAX_PAGES} pages; refusing to keep paging. ` +
|
|
74
|
+
`This is a server-side fault, not a coverage limit.`, MAX_PAGES);
|
|
75
|
+
}
|
|
76
|
+
cursor = page.nextCursor;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
30
79
|
/** Decoders of an answer's bytes: the strict one tells a body that is not UTF-8. */
|
|
31
80
|
const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
|
|
32
81
|
const LENIENT_UTF8 = new TextDecoder("utf-8");
|
|
@@ -139,7 +188,7 @@ export class PipelexApiClient {
|
|
|
139
188
|
* no request path can miss the `User-Agent` or the bearer. Requests to third
|
|
140
189
|
* parties (presigned object-store URLs) never go through here.
|
|
141
190
|
*/
|
|
142
|
-
requestHeaders(hasBody) {
|
|
191
|
+
requestHeaders(hasBody, idempotencyKey) {
|
|
143
192
|
const headers = { Accept: "application/json" };
|
|
144
193
|
if (this.userAgent !== undefined) {
|
|
145
194
|
headers["User-Agent"] = this.userAgent;
|
|
@@ -150,6 +199,9 @@ export class PipelexApiClient {
|
|
|
150
199
|
if (hasBody) {
|
|
151
200
|
headers["Content-Type"] = "application/json";
|
|
152
201
|
}
|
|
202
|
+
if (idempotencyKey !== undefined) {
|
|
203
|
+
headers[IDEMPOTENCY_KEY_HEADER] = idempotencyKey;
|
|
204
|
+
}
|
|
153
205
|
return headers;
|
|
154
206
|
}
|
|
155
207
|
/**
|
|
@@ -159,10 +211,12 @@ export class PipelexApiClient {
|
|
|
159
211
|
* cleanly. Non-2xx interpretation is left to the caller. `url` is a fully
|
|
160
212
|
* resolved absolute URL. `mayStartRun` marks a request that may create a run,
|
|
161
213
|
* whose `ApiUnreachableError` then says whether it may have (see `RunCreation`).
|
|
214
|
+
* `idempotencyKey`, already checked by `idempotencyKeyOf`, is sent as the
|
|
215
|
+
* `Idempotency-Key` header; only `start` passes one.
|
|
162
216
|
*/
|
|
163
217
|
async requestRaw(method, url, options = {}) {
|
|
164
218
|
const hasBody = options.body !== undefined;
|
|
165
|
-
const headers = this.requestHeaders(hasBody);
|
|
219
|
+
const headers = this.requestHeaders(hasBody, options.idempotencyKey);
|
|
166
220
|
const timeoutMs = options.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
167
221
|
// A longer delay overflows the timer, which then fires at once as a false timeout.
|
|
168
222
|
if (!isTimerDelay(timeoutMs)) {
|
|
@@ -526,8 +580,16 @@ export class PipelexApiClient {
|
|
|
526
580
|
* durable — poll `getRunStatus` / `getRunResult`. A `method_ref` run's ack
|
|
527
581
|
* additionally carries `method_provenance` — the address, the tag, and the
|
|
528
582
|
* commit SHA that was actually fetched.
|
|
583
|
+
*
|
|
584
|
+
* `options.idempotencyKey` is sent as the `Idempotency-Key` header, and nothing
|
|
585
|
+
* is sent without it: the hosted API answers the same start sent again under the
|
|
586
|
+
* same key with the run the first one started, so a failure whose
|
|
587
|
+
* `runMayHaveStarted` is true can be retried unchanged (see
|
|
588
|
+
* `PipelexStartRequestOptions`). A malformed key is a `RequestArgumentError`
|
|
589
|
+
* before anything is sent.
|
|
529
590
|
*/
|
|
530
591
|
async start(options) {
|
|
592
|
+
const idempotencyKey = idempotencyKeyOf(options);
|
|
531
593
|
const extensions = buildExtensions(options.extra);
|
|
532
594
|
const api = buildApiRunExtensions(options);
|
|
533
595
|
const hosted = buildHostedRunExtensions(options);
|
|
@@ -578,6 +640,7 @@ export class PipelexApiClient {
|
|
|
578
640
|
const res = await this.requestRaw("POST", url, {
|
|
579
641
|
body: request,
|
|
580
642
|
timeoutMs: needsLongCeiling ? DEFAULT_REQUEST_TIMEOUT_MS : POLL_REQUEST_TIMEOUT_MS,
|
|
643
|
+
idempotencyKey,
|
|
581
644
|
...STARTS_RUN,
|
|
582
645
|
});
|
|
583
646
|
// A bare runner with no run store 404s here just as it does on the result
|
|
@@ -1178,10 +1241,18 @@ export class PipelexApiClient {
|
|
|
1178
1241
|
* up in the run history first. An error from the wait, once `onStarted` has named the run,
|
|
1179
1242
|
* started nothing, so its `runMayHaveStarted` is false, and its `retryable` is about waiting
|
|
1180
1243
|
* again on that id with `waitForResult`, never about calling this method again.
|
|
1244
|
+
*
|
|
1245
|
+
* `options.idempotencyKey` rides the durable start, so a call that failed while its start may
|
|
1246
|
+
* have created a run can be made again with the same options and the same key: the hosted API
|
|
1247
|
+
* answers the start with the run the first call started, and the wait follows it. The blocking
|
|
1248
|
+
* execute of a bare runner carries no key, a bare runner having no idempotency to honour it. A
|
|
1249
|
+
* malformed key is refused before the version handshake.
|
|
1181
1250
|
*/
|
|
1182
1251
|
async startAndWaitForResult(options, pollOptions) {
|
|
1183
1252
|
// Before the run starts: a refusal after it would carry no run id to re-poll by.
|
|
1184
1253
|
assertWaitOptions(pollOptions);
|
|
1254
|
+
// Before the handshake too: a key the start would refuse is the caller's to fix, whatever runs.
|
|
1255
|
+
idempotencyKeyOf(options);
|
|
1185
1256
|
const { onStarted, onStarting, ...waitOptions } = pollOptions ?? {};
|
|
1186
1257
|
const hosted = await this.supportsRunLifecycle();
|
|
1187
1258
|
// The handshake may have outlasted the caller's patience: an abort that landed during it
|
|
@@ -1237,16 +1308,23 @@ export class PipelexApiClient {
|
|
|
1237
1308
|
* Rows are `MethodSummary`: no `mthds`, no `python`, no `updated_at`. Reach
|
|
1238
1309
|
* for `getMethod(id)` when you need a method's source.
|
|
1239
1310
|
*
|
|
1240
|
-
* `q` is applied server-side over the whole catalog, not over the page
|
|
1311
|
+
* `q` is applied server-side over the whole catalog, not over the page, and so is
|
|
1312
|
+
* `published` (`true` for the methods with a published version, `false` for those never
|
|
1313
|
+
* published). Each row carries the method's publish state (`latest_version`,
|
|
1314
|
+
* `latest_published`, `draft_digest`). Both filters run after the index read, so a page can
|
|
1315
|
+
* come back empty and still carry a `nextCursor`: keep following it until it is `null`.
|
|
1241
1316
|
*/
|
|
1242
1317
|
async listMethods(query = {}) {
|
|
1243
1318
|
const params = new URLSearchParams();
|
|
1244
1319
|
// `!== undefined`, not truthiness — mirroring `listRuns`. Omission means
|
|
1245
1320
|
// "the caller did not ask"; an explicit empty string is bad input and the
|
|
1246
1321
|
// API should say so, rather than being silently dropped into an unfiltered
|
|
1247
|
-
// query that returns everything and reads as working.
|
|
1322
|
+
// query that returns everything and reads as working. `published: false`
|
|
1323
|
+
// is a filter of its own, never omission.
|
|
1248
1324
|
if (query.q !== undefined)
|
|
1249
1325
|
params.set("q", query.q);
|
|
1326
|
+
if (query.published !== undefined)
|
|
1327
|
+
params.set("published", String(query.published));
|
|
1250
1328
|
if (query.limit !== undefined)
|
|
1251
1329
|
params.set("limit", String(query.limit));
|
|
1252
1330
|
if (query.cursor !== undefined)
|
|
@@ -1272,43 +1350,14 @@ export class PipelexApiClient {
|
|
|
1272
1350
|
* TRUNCATED list with no error — precisely the bug paging was introduced to
|
|
1273
1351
|
* remove. If you truly want an array, `Array.fromAsync` makes that your
|
|
1274
1352
|
* explicit choice.
|
|
1353
|
+
*
|
|
1354
|
+
* It keeps going **through an empty page that carries a cursor**: under `q` or `published`
|
|
1355
|
+
* such a page means "nothing matched in the slice just read", not "done". It stops on a
|
|
1356
|
+
* `null` cursor or on a cursor the server failed to advance, and throws
|
|
1357
|
+
* `PagingNotTerminatingError` past a page ceiling no real catalog reaches.
|
|
1275
1358
|
*/
|
|
1276
1359
|
async *iterateMethods(query = {}) {
|
|
1277
|
-
|
|
1278
|
-
let pages = 0;
|
|
1279
|
-
for (;;) {
|
|
1280
|
-
const page = await this.listMethods({ ...query, cursor });
|
|
1281
|
-
// Checked BEFORE yielding: a server handing back the very cursor we sent
|
|
1282
|
-
// has not advanced, so this page repeats rows already emitted. Cannot
|
|
1283
|
-
// fire on the first request, where `cursor` is undefined.
|
|
1284
|
-
if (cursor !== undefined && page.nextCursor === cursor)
|
|
1285
|
-
return;
|
|
1286
|
-
for (const method of page.items)
|
|
1287
|
-
yield method;
|
|
1288
|
-
// `nextCursor === null` is the NORMAL end of the catalog, and that page
|
|
1289
|
-
// is real data, so it has to be emitted before this fires.
|
|
1290
|
-
if (page.nextCursor === null)
|
|
1291
|
-
return;
|
|
1292
|
-
// An EMPTY page is NOT the end, and treating it as one was a bug. The
|
|
1293
|
-
// platform applies `q` as a post-read filter over a bounded number of
|
|
1294
|
-
// index pages per request, so a sparse match legitimately returns
|
|
1295
|
-
// `{items: [], nextCursor: "…"}` — "nothing matched in the slice I just
|
|
1296
|
-
// read, keep going". Returning here dropped every later match silently,
|
|
1297
|
-
// which is the exact truncation this pagination work exists to remove.
|
|
1298
|
-
// (`iterateRuns` can stop on an empty page because its date bounds are
|
|
1299
|
-
// index KEY conditions, so a run page is never empty-with-a-cursor. The
|
|
1300
|
-
// difference is the server, not the client.)
|
|
1301
|
-
pages += 1;
|
|
1302
|
-
if (pages >= MAX_PAGES) {
|
|
1303
|
-
// THROW, do not return. A caller that asked for every method and got a
|
|
1304
|
-
// partial answer with no error is back to the original bug one layer
|
|
1305
|
-
// up; an error is the only honest response to a server that will not
|
|
1306
|
-
// finish. Unreachable on real data — see MAX_PAGES.
|
|
1307
|
-
throw new PagingNotTerminatingError(`listMethods did not terminate after ${MAX_PAGES} pages; refusing to keep paging. ` +
|
|
1308
|
-
`This is a server-side fault, not a coverage limit.`, MAX_PAGES);
|
|
1309
|
-
}
|
|
1310
|
-
cursor = page.nextCursor;
|
|
1311
|
-
}
|
|
1360
|
+
yield* followCursor("listMethods", (cursor) => this.listMethods({ ...query, cursor }));
|
|
1312
1361
|
}
|
|
1313
1362
|
/**
|
|
1314
1363
|
* Read one method — `GET /v1/methods/{id}`: its identity, its draft (`mthds`, `python`,
|
|
@@ -1739,6 +1788,13 @@ export class PipelexApiClient {
|
|
|
1739
1788
|
* conditions, so a bounded page genuinely reads less. They are INSTANTS,
|
|
1740
1789
|
* not days — see `ListRunsQuery`.
|
|
1741
1790
|
*
|
|
1791
|
+
* `createdBy: "me"` keeps the caller's own runs and `status` the runs in any of the given
|
|
1792
|
+
* statuses, sent as `created_by=me` and one repeated `status=` parameter per status. Both are
|
|
1793
|
+
* filters applied after the index read, so a filtered page can hold fewer runs than `limit`,
|
|
1794
|
+
* or none, and still carry a `nextCursor`: an empty page is not the end, a `null` cursor is.
|
|
1795
|
+
* An empty `status` list, or a name that is no `RunStatus`, is a `RequestArgumentError`
|
|
1796
|
+
* before any request.
|
|
1797
|
+
*
|
|
1742
1798
|
* Takes a bare catalog id: the history files the runs of every version and
|
|
1743
1799
|
* of the draft together under it, so `mt_…@3` names no history of its own
|
|
1744
1800
|
* and the platform refuses it with a `400`. Strip a suffix with
|
|
@@ -1756,6 +1812,10 @@ export class PipelexApiClient {
|
|
|
1756
1812
|
params.set("created_from", query.createdFrom);
|
|
1757
1813
|
if (query.createdTo !== undefined)
|
|
1758
1814
|
params.set("created_to", query.createdTo);
|
|
1815
|
+
if (query.createdBy !== undefined)
|
|
1816
|
+
params.set("created_by", query.createdBy);
|
|
1817
|
+
for (const status of runStatusFilter(query.status) ?? [])
|
|
1818
|
+
params.append("status", status);
|
|
1759
1819
|
if (query.limit !== undefined)
|
|
1760
1820
|
params.set("limit", String(query.limit));
|
|
1761
1821
|
if (query.cursor !== undefined)
|
|
@@ -1785,33 +1845,15 @@ export class PipelexApiClient {
|
|
|
1785
1845
|
* when to stop, and only one page is ever in memory. If you truly want an
|
|
1786
1846
|
* array, `Array.fromAsync` makes that your explicit choice.
|
|
1787
1847
|
*
|
|
1848
|
+
* It keeps going **through an empty page that carries a cursor**: under `createdBy` or
|
|
1849
|
+
* `status` such a page means "nothing matched in the slice just read", not "done". It stops on
|
|
1850
|
+
* a `null` cursor or on a cursor the server failed to advance, and throws
|
|
1851
|
+
* `PagingNotTerminatingError` past a page ceiling no real history reaches.
|
|
1852
|
+
*
|
|
1788
1853
|
* Takes a bare catalog id, as `listRuns` does.
|
|
1789
1854
|
*/
|
|
1790
1855
|
async *iterateRuns(methodId, query = {}) {
|
|
1791
|
-
|
|
1792
|
-
for (;;) {
|
|
1793
|
-
const page = await this.listRuns(methodId, { ...query, cursor });
|
|
1794
|
-
// Checked BEFORE yielding. A server handing back the very cursor we sent
|
|
1795
|
-
// has not advanced, so this page repeats rows already emitted — yielding
|
|
1796
|
-
// first and stopping after would silently double-count them for anyone
|
|
1797
|
-
// aggregating the stream (summing cost, counting runs). Cannot fire on
|
|
1798
|
-
// the first request, where `cursor` is undefined.
|
|
1799
|
-
if (cursor !== undefined && page.nextCursor === cursor)
|
|
1800
|
-
return;
|
|
1801
|
-
for (const run of page.items)
|
|
1802
|
-
yield run;
|
|
1803
|
-
// Checked AFTER: `nextCursor === null` is the NORMAL end of the history
|
|
1804
|
-
// and that page is real data, so it has to be emitted first. The empty
|
|
1805
|
-
// page catches a server that keeps minting fresh cursors while returning
|
|
1806
|
-
// nothing, which would otherwise spin forever yielding nothing.
|
|
1807
|
-
//
|
|
1808
|
-
// Neither of these is a page cap. Both fire only on a server that is not
|
|
1809
|
-
// making progress, so neither can truncate a healthy stream the way a
|
|
1810
|
-
// `maxPages` limit silently did.
|
|
1811
|
-
if (page.nextCursor === null || page.items.length === 0)
|
|
1812
|
-
return;
|
|
1813
|
-
cursor = page.nextCursor;
|
|
1814
|
-
}
|
|
1856
|
+
yield* followCursor("listRuns", (cursor) => this.listRuns(methodId, { ...query, cursor }));
|
|
1815
1857
|
}
|
|
1816
1858
|
/**
|
|
1817
1859
|
* The whole record for one run — `GET /v1/runs/{id}`.
|
|
@@ -2170,6 +2212,55 @@ function nonEmptyFiles(files) {
|
|
|
2170
2212
|
function nonEmptyString(value) {
|
|
2171
2213
|
return value != null && value.length > 0 ? value : undefined;
|
|
2172
2214
|
}
|
|
2215
|
+
/** The request header the hosted API reads a run start's idempotency key from. */
|
|
2216
|
+
const IDEMPOTENCY_KEY_HEADER = "Idempotency-Key";
|
|
2217
|
+
/**
|
|
2218
|
+
* The longest idempotency key a start sends. It is room for any generated key, and keeps the
|
|
2219
|
+
* hosted API's stored record of the key far inside its store's limit, which a longer key could
|
|
2220
|
+
* exceed, so that the start would run without the protection the key was sent for.
|
|
2221
|
+
*/
|
|
2222
|
+
const MAX_IDEMPOTENCY_KEY_LENGTH = 255;
|
|
2223
|
+
/**
|
|
2224
|
+
* Visible ASCII, `!` to `~`: what a header value carries unchanged through every proxy and both
|
|
2225
|
+
* SDKs' transports, where fetch refuses a line break, httpx a character outside ASCII, and a space
|
|
2226
|
+
* at either end is trimmed on the way.
|
|
2227
|
+
*/
|
|
2228
|
+
const VISIBLE_ASCII = /^[\x21-\x7E]+$/;
|
|
2229
|
+
/**
|
|
2230
|
+
* The idempotency key a start sends, or `undefined` when the options carry none (absent or
|
|
2231
|
+
* `null`). A key that is not a string of 1 to 255 visible ASCII characters is refused with a
|
|
2232
|
+
* `RequestArgumentError` rather than dropped, an empty one included, since a start sent without
|
|
2233
|
+
* the key its caller gave would lose the protection the caller asked for without a word. The
|
|
2234
|
+
* message never quotes the key, which a caller may derive from something private. The Python SDK
|
|
2235
|
+
* holds a key to the same rule, in the same words.
|
|
2236
|
+
*/
|
|
2237
|
+
function idempotencyKeyOf(options) {
|
|
2238
|
+
const key = options.idempotencyKey;
|
|
2239
|
+
if (key == null)
|
|
2240
|
+
return undefined;
|
|
2241
|
+
const fault = idempotencyKeyFault(key);
|
|
2242
|
+
if (fault !== undefined) {
|
|
2243
|
+
throw new RequestArgumentError(`The idempotency key must be 1 to ${MAX_IDEMPOTENCY_KEY_LENGTH} visible ASCII characters ` +
|
|
2244
|
+
`(\`!\` to \`~\`), and this one ${fault}. Leave it out to start the run without one.`);
|
|
2245
|
+
}
|
|
2246
|
+
return key;
|
|
2247
|
+
}
|
|
2248
|
+
/**
|
|
2249
|
+
* What is wrong with an idempotency key, or `undefined` when it is a valid one. The characters are
|
|
2250
|
+
* checked before the length, which then counts ASCII characters alone, so the two SDKs, which
|
|
2251
|
+
* count a character outside ASCII differently, word every refusal alike.
|
|
2252
|
+
*/
|
|
2253
|
+
function idempotencyKeyFault(key) {
|
|
2254
|
+
if (typeof key !== "string")
|
|
2255
|
+
return "is not a string";
|
|
2256
|
+
if (key.length === 0)
|
|
2257
|
+
return "is empty";
|
|
2258
|
+
if (!VISIBLE_ASCII.test(key))
|
|
2259
|
+
return "holds a character outside visible ASCII";
|
|
2260
|
+
if (key.length > MAX_IDEMPOTENCY_KEY_LENGTH)
|
|
2261
|
+
return `is ${key.length} characters long`;
|
|
2262
|
+
return undefined;
|
|
2263
|
+
}
|
|
2173
2264
|
function withValidateMarkdownRender(render) {
|
|
2174
2265
|
const formats = new Set(render ?? []);
|
|
2175
2266
|
formats.add(VALIDATE_MARKDOWN_RENDER_FORMAT);
|
|
@@ -2284,16 +2375,6 @@ function parseRetryAfter(headers) {
|
|
|
2284
2375
|
const seconds = Number(raw);
|
|
2285
2376
|
return Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
|
|
2286
2377
|
}
|
|
2287
|
-
const KNOWN_RUN_STATUSES = [
|
|
2288
|
-
"PENDING",
|
|
2289
|
-
"STARTED",
|
|
2290
|
-
"RUNNING",
|
|
2291
|
-
"COMPLETED",
|
|
2292
|
-
"FAILED",
|
|
2293
|
-
"CANCELLED",
|
|
2294
|
-
"TERMINATED",
|
|
2295
|
-
"TIMED_OUT",
|
|
2296
|
-
];
|
|
2297
2378
|
/** The response header the platform and the runner stamp the request's correlation id on. */
|
|
2298
2379
|
const REQUEST_ID_HEADER = "x-request-id";
|
|
2299
2380
|
/**
|
|
@@ -2342,7 +2423,7 @@ function withCheckedDegradation(run) {
|
|
|
2342
2423
|
return { ...run, degradation: readRunDegradation(run.degradation) };
|
|
2343
2424
|
}
|
|
2344
2425
|
function knownRunStatus(value) {
|
|
2345
|
-
return typeof value === "string" &&
|
|
2426
|
+
return typeof value === "string" && RUN_STATUSES.includes(value)
|
|
2346
2427
|
? value
|
|
2347
2428
|
: undefined;
|
|
2348
2429
|
}
|