@pipelex/sdk 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/bundle-resources.d.ts +62 -0
- package/dist/bundle-resources.js +165 -0
- package/dist/bundle-resources.js.map +1 -0
- package/dist/cli/script.d.ts +1 -1
- package/dist/cli/source.d.ts +2 -1
- package/dist/cli/source.js +61 -3
- package/dist/cli/source.js.map +1 -1
- package/dist/client.d.ts +107 -23
- package/dist/client.js +216 -113
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +16 -5
- package/dist/errors.js +15 -5
- package/dist/errors.js.map +1 -1
- package/dist/hooks/bundle-gather.d.ts +28 -1
- package/dist/hooks/bundle-gather.js +163 -3
- package/dist/hooks/bundle-gather.js.map +1 -1
- package/dist/hooks/claude-mthds-check.d.ts +4 -2
- package/dist/hooks/claude-mthds-check.js +5 -2
- package/dist/hooks/claude-mthds-check.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/models.d.ts +42 -4
- package/dist/prepare-inputs.d.ts +48 -7
- package/dist/prepare-inputs.js +236 -35
- package/dist/prepare-inputs.js.map +1 -1
- package/dist/product-models.d.ts +46 -0
- package/dist/runs.d.ts +11 -0
- package/dist/runs.js +35 -0
- package/dist/runs.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +5 -3
- package/docs/bundle-resources.md +107 -0
- package/docs/cli.md +1 -0
- package/docs/crate-routes.md +5 -1
- package/docs/errors.md +4 -3
- package/docs/hook-bundle.md +2 -0
- package/docs/input-preparation.md +9 -5
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -2,30 +2,80 @@
|
|
|
2
2
|
// source combinations are legal is an invariant of `RunRequest` itself, so this
|
|
3
3
|
// client and the MTHDS runners cannot disagree about what they reject.
|
|
4
4
|
import { assertExclusiveRunSources, hasBundlePayload, parseMethodFiles, serializeMethodFiles, } from "mthds/protocol";
|
|
5
|
-
import { assertArtifactSelection, assertWaitOptions, pollUntilResult, readRunDegradation, selectionIncludesMainStuff, throwIfAborted, } from "./runs.js";
|
|
5
|
+
import { assertArtifactSelection, assertWaitOptions, pollUntilResult, readRunDegradation, RUN_STATUSES, runStatusFilter, selectionIncludesMainStuff, throwIfAborted, } from "./runs.js";
|
|
6
6
|
import { ApiResponseError, ApiUnreachableError, EmptyMethodSourceError, MissingMainStuffError, PagingNotTerminatingError, PipelineExecuteTimeoutError, PipelineRequestError, RequestArgumentError, RunLifecycleUnavailableError, RunStillRunningError, } from "./errors.js";
|
|
7
7
|
import { isPlainObject, isValidationItem, readFieldError, readMigration, readProviderMetadata, readRunErrorReport, } from "./error-models.js";
|
|
8
8
|
import { methodSourceToContents } from "./method-source.js";
|
|
9
9
|
import { buildUserAgent } from "./user-agent.js";
|
|
10
10
|
import { uploadFile as uploadFileImpl } from "./upload.js";
|
|
11
|
+
import { encodeBundleResources, hasBundleResources } from "./bundle-resources.js";
|
|
11
12
|
import { prepareInputs as prepareInputsImpl } from "./prepare-inputs.js";
|
|
12
13
|
import { downloadArtifacts as downloadArtifactsImpl, fetchArtifact as fetchArtifactImpl, resolveArtifacts as resolveArtifactsImpl, } from "./artifacts.js";
|
|
13
14
|
import { PipelexExecuteResult, resultsFromExecute } from "./execute-result.js";
|
|
14
15
|
import { MAX_TIMER_DELAY_MS, isTimerDelay } from "./timers.js";
|
|
15
|
-
// A pure RUNAWAY guard on `iterateMethods`,
|
|
16
|
+
// A pure RUNAWAY guard on the paged-list iterators (`iterateMethods`,
|
|
17
|
+
// `iterateRuns`), deliberately not a coverage limit.
|
|
16
18
|
//
|
|
17
19
|
// It counts TOTAL pages, not empty ones. An earlier version capped consecutive
|
|
18
20
|
// empty pages, which is the wrong axis: an empty page whose cursor ADVANCED is
|
|
19
|
-
// real progress through the index — the platform applies
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
21
|
+
// real progress through the index — the platform applies the list filters
|
|
22
|
+
// (`q` and `published` on methods, `createdBy` and `status` on runs) after a
|
|
23
|
+
// bounded read per request, so a sparse match legitimately yields runs of
|
|
24
|
+
// empty pages — and capping them turns a valid sparse search into a thrown
|
|
25
|
+
// error. The stuck case is already caught by the no-progress check (the
|
|
26
|
+
// server handing back the cursor we sent).
|
|
24
27
|
//
|
|
25
|
-
// At the platform's 200-row maximum page size this is 2,000,000
|
|
28
|
+
// At the platform's 200-row maximum page size this is 2,000,000 rows, so it
|
|
26
29
|
// cannot fire on real data; it exists only so a server minting fresh cursors
|
|
27
30
|
// forever cannot hang a caller indefinitely.
|
|
28
31
|
const MAX_PAGES = 10_000;
|
|
32
|
+
/**
|
|
33
|
+
* Follow a paged list's cursor until the server says there is no more, yielding every row —
|
|
34
|
+
* the one loop behind `iterateMethods` and `iterateRuns`, so the two cannot drift apart.
|
|
35
|
+
*
|
|
36
|
+
* It yields lazily: a caller that breaks early stops the requests. It stops when `nextCursor`
|
|
37
|
+
* is `null`, the normal end, and when the server hands back the cursor it was just sent, which
|
|
38
|
+
* is no progress and would repeat rows already yielded. It keeps going through an EMPTY page
|
|
39
|
+
* that carries a cursor, since a filtered page can hold no row and still not be the end; past
|
|
40
|
+
* `MAX_PAGES` pages it throws `PagingNotTerminatingError` rather than return a truncated list.
|
|
41
|
+
*/
|
|
42
|
+
async function* followCursor(route, readPage) {
|
|
43
|
+
let cursor;
|
|
44
|
+
let pages = 0;
|
|
45
|
+
for (;;) {
|
|
46
|
+
const page = await readPage(cursor);
|
|
47
|
+
// Checked BEFORE yielding: a server handing back the very cursor we sent
|
|
48
|
+
// has not advanced, so this page repeats rows already emitted — yielding
|
|
49
|
+
// first and stopping after would silently double-count them for anyone
|
|
50
|
+
// aggregating the stream (summing cost, counting runs). Cannot fire on the
|
|
51
|
+
// first request, where `cursor` is undefined.
|
|
52
|
+
if (cursor !== undefined && page.nextCursor === cursor)
|
|
53
|
+
return;
|
|
54
|
+
for (const item of page.items)
|
|
55
|
+
yield item;
|
|
56
|
+
// `nextCursor === null` is the NORMAL end of the list, and that page is
|
|
57
|
+
// real data, so it has to be emitted before this fires.
|
|
58
|
+
if (page.nextCursor === null)
|
|
59
|
+
return;
|
|
60
|
+
// An EMPTY page is NOT the end, and treating it as one was a bug. The
|
|
61
|
+
// platform filters after a bounded index read per request, so a sparse
|
|
62
|
+
// match legitimately returns `{items: [], nextCursor: "…"}` — "nothing
|
|
63
|
+
// matched in the slice I just read, keep going". Returning here dropped
|
|
64
|
+
// every later match silently, which is the exact truncation this
|
|
65
|
+
// pagination work exists to remove.
|
|
66
|
+
pages += 1;
|
|
67
|
+
if (pages >= MAX_PAGES) {
|
|
68
|
+
// THROW, do not return. A caller that asked for every row and got a
|
|
69
|
+
// partial answer with no error is back to the original bug one layer
|
|
70
|
+
// up; an error is the only honest response to a server that will not
|
|
71
|
+
// finish. Unreachable on real data — see MAX_PAGES. It also bounds a
|
|
72
|
+
// cursor that CYCLES (`c1 → c2 → c1`), which the adjacent check misses.
|
|
73
|
+
throw new PagingNotTerminatingError(`${route} did not terminate after ${MAX_PAGES} pages; refusing to keep paging. ` +
|
|
74
|
+
`This is a server-side fault, not a coverage limit.`, MAX_PAGES);
|
|
75
|
+
}
|
|
76
|
+
cursor = page.nextCursor;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
29
79
|
/** Decoders of an answer's bytes: the strict one tells a body that is not UTF-8. */
|
|
30
80
|
const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
|
|
31
81
|
const LENIENT_UTF8 = new TextDecoder("utf-8");
|
|
@@ -464,6 +514,9 @@ export class PipelexApiClient {
|
|
|
464
514
|
assertRunSourcesExclusive(options);
|
|
465
515
|
assertMethodRefPairsWithNothing(options);
|
|
466
516
|
assertLinkageMethodIdIsBare(options);
|
|
517
|
+
assertResourcesAccompanyABundle(options);
|
|
518
|
+
// No await when there are none: a request without them leaves on the tick it always did.
|
|
519
|
+
const resources = options.resources == null ? undefined : await encodeBundleResources(options.resources);
|
|
467
520
|
const request = {
|
|
468
521
|
pipe_code: options.pipe_code,
|
|
469
522
|
mthds_contents: options.mthds_contents,
|
|
@@ -473,6 +526,7 @@ export class PipelexApiClient {
|
|
|
473
526
|
dynamic_output_concept_ref: options.dynamic_output_concept_ref,
|
|
474
527
|
files: nonEmptyFiles(options.files),
|
|
475
528
|
bundle_b64: nonEmptyString(options.bundle_b64),
|
|
529
|
+
resources,
|
|
476
530
|
...api,
|
|
477
531
|
...hosted,
|
|
478
532
|
...extensions,
|
|
@@ -537,6 +591,9 @@ export class PipelexApiClient {
|
|
|
537
591
|
assertRunSourcesExclusive(options);
|
|
538
592
|
assertMethodRefPairsWithNothing(options);
|
|
539
593
|
assertLinkageMethodIdIsBare(options);
|
|
594
|
+
assertResourcesAccompanyABundle(options);
|
|
595
|
+
// No await when there are none: a request without them leaves on the tick it always did.
|
|
596
|
+
const resources = options.resources == null ? undefined : await encodeBundleResources(options.resources);
|
|
540
597
|
// `?? undefined` so JSON.stringify drops absent fields from the wire body.
|
|
541
598
|
const request = {
|
|
542
599
|
pipe_code: options.pipe_code ?? undefined,
|
|
@@ -547,6 +604,7 @@ export class PipelexApiClient {
|
|
|
547
604
|
dynamic_output_concept_ref: options.dynamic_output_concept_ref ?? undefined,
|
|
548
605
|
files: nonEmptyFiles(options.files),
|
|
549
606
|
bundle_b64: nonEmptyString(options.bundle_b64),
|
|
607
|
+
resources,
|
|
550
608
|
...api,
|
|
551
609
|
...hosted,
|
|
552
610
|
...extensions,
|
|
@@ -554,16 +612,17 @@ export class PipelexApiClient {
|
|
|
554
612
|
const url = this.url("start");
|
|
555
613
|
// `start` returns a 202 fast, so the poll timeout normally fits — with
|
|
556
614
|
// exceptions that get the blocking-execute ceiling instead. A method bundle,
|
|
557
|
-
// inline as `mthds_contents` or in `files`/`bundle_b64`,
|
|
558
|
-
// request *body* multi-megabyte, and
|
|
559
|
-
// charged against this budget — the same payload must
|
|
560
|
-
// durable path yet succeed on the fallback. And a
|
|
561
|
-
// the server FETCH the package before the ack
|
|
562
|
-
// whose clone timeout runs well past 30s on a
|
|
563
|
-
// would surface as `ApiUnreachableError`,
|
|
564
|
-
// server that is still cloning.
|
|
615
|
+
// inline as `mthds_contents` or in `files`/`bundle_b64`, or the files its
|
|
616
|
+
// steps read (`resources`), can make the request *body* multi-megabyte, and
|
|
617
|
+
// the whole upload is charged against this budget — the same payload must
|
|
618
|
+
// not time out on the durable path yet succeed on the fallback. And a
|
|
619
|
+
// `method_ref` start makes the server FETCH the package before the ack
|
|
620
|
+
// (provenance rides the 202), whose clone timeout runs well past 30s on a
|
|
621
|
+
// cold cache — aborting it here would surface as `ApiUnreachableError`,
|
|
622
|
+
// blaming the network for a healthy server that is still cloning.
|
|
565
623
|
const needsLongCeiling = hasBundlePayload(options) ||
|
|
566
624
|
(options.mthds_contents?.length ?? 0) > 0 ||
|
|
625
|
+
resources !== undefined ||
|
|
567
626
|
nonEmptyString(options.method_ref) !== undefined;
|
|
568
627
|
const res = await this.requestRaw("POST", url, {
|
|
569
628
|
body: request,
|
|
@@ -645,6 +704,15 @@ export class PipelexApiClient {
|
|
|
645
704
|
* the field regardless — which is why `PipelexValidationReport.input_form` is typed
|
|
646
705
|
* optional rather than required.
|
|
647
706
|
*
|
|
707
|
+
* `options.resources` sends the files the bundle's steps read, such as a
|
|
708
|
+
* `PipeDocGen` step's `template_file`, keyed by the path the step names, each as
|
|
709
|
+
* its bytes (see {@link ValidateOptions}). A step whose template file was not sent
|
|
710
|
+
* is an `is_valid: false` verdict saying so, and a template check's findings in an
|
|
711
|
+
* office template file ride the valid arm's `warnings`. A key no step names, or
|
|
712
|
+
* files over the server's limits, are no-verdict refusals: a `422`
|
|
713
|
+
* `BundleResourceNotNamedError`, a `413` `PayloadTooLarge` or
|
|
714
|
+
* `BundleResourcesTooLargeError`.
|
|
715
|
+
*
|
|
648
716
|
* A `{ method_ref }` source makes the server clone a repository before it
|
|
649
717
|
* validates anything, and it needs no special budget here: this route already
|
|
650
718
|
* defaults to the 20-minute execute ceiling, which clears the internal
|
|
@@ -682,6 +750,9 @@ export class PipelexApiClient {
|
|
|
682
750
|
throw new RequestArgumentError("mthds_sources labels inline mthds_contents; a method_ref / method_id validation gets " +
|
|
683
751
|
"its source labels from the package's (or the stored method's) real file names.");
|
|
684
752
|
}
|
|
753
|
+
if (methodRef !== undefined && hasBundleResources(options.resources)) {
|
|
754
|
+
throw new RequestArgumentError("resources accompany inline mthds_contents; a method_ref's files are the package's own.");
|
|
755
|
+
}
|
|
685
756
|
if (methodRef !== undefined)
|
|
686
757
|
body.method_ref = methodRef;
|
|
687
758
|
if (methodId !== undefined)
|
|
@@ -691,6 +762,11 @@ export class PipelexApiClient {
|
|
|
691
762
|
if (views !== undefined) {
|
|
692
763
|
body.views = views;
|
|
693
764
|
}
|
|
765
|
+
// No await when there are none: a request without them leaves on the tick it always did.
|
|
766
|
+
const resources = options.resources == null ? undefined : await encodeBundleResources(options.resources);
|
|
767
|
+
if (resources !== undefined) {
|
|
768
|
+
body.resources = resources;
|
|
769
|
+
}
|
|
694
770
|
const res = await this.requestRaw("POST", this.url("validate"), {
|
|
695
771
|
body,
|
|
696
772
|
timeoutMs: options.timeoutMs,
|
|
@@ -719,6 +795,7 @@ export class PipelexApiClient {
|
|
|
719
795
|
? files.map((file, index) => file.uri ?? `inline://file-${index + 1}.mthds`)
|
|
720
796
|
: undefined;
|
|
721
797
|
return this.validate(mthdsContents, options.allowSignatures ?? false, mthdsSources, options.render, options.views, {
|
|
798
|
+
resources: options.resources,
|
|
722
799
|
timeoutMs: options.timeoutMs,
|
|
723
800
|
signal: options.signal,
|
|
724
801
|
});
|
|
@@ -793,10 +870,11 @@ export class PipelexApiClient {
|
|
|
793
870
|
* never on a message.
|
|
794
871
|
*
|
|
795
872
|
* All of these are static and inference-free, so they default to the management-call
|
|
796
|
-
* timeout. The
|
|
797
|
-
*
|
|
798
|
-
* repository before it answers
|
|
799
|
-
*
|
|
873
|
+
* timeout. The internal raises are `crateRequestTimeoutMs`'s: the fetch-sized budget a
|
|
874
|
+
* crate route gets when its closure is a `method_ref`, because the server may clone a
|
|
875
|
+
* repository before it answers, and the execute ceiling it gets when the request
|
|
876
|
+
* carries `resources`, whose upload alone can outlast the default. Each is matched to
|
|
877
|
+
* what the request costs, not a caller-facing parameter.
|
|
800
878
|
*
|
|
801
879
|
* **These routes deliberately expose no `timeoutMs` / `signal`, and that is not an
|
|
802
880
|
* oversight to be "fixed" per route.** Automated reviewers have proposed adding them
|
|
@@ -807,8 +885,11 @@ export class PipelexApiClient {
|
|
|
807
885
|
* ceiling and takes caller transport options; every extension route rides the
|
|
808
886
|
* static core. Giving one static route an override while its siblings lack one is
|
|
809
887
|
* the inconsistency.
|
|
810
|
-
* 2. The input is **bounded server-side** — the runner's `pipelex_api/limits.py` caps
|
|
811
|
-
*
|
|
888
|
+
* 2. The input is **bounded server-side** — the runner's `pipelex_api/limits.py` caps the
|
|
889
|
+
* `.mthds` files of a request and the files sent with them in `resources`, each by a
|
|
890
|
+
* count, a size per file and, for `resources`, a total, all configurable per
|
|
891
|
+
* deployment — and none of these routes runs inference. The upload those limits allow
|
|
892
|
+
* is what `crateRequestTimeoutMs` budgets for, not the server's work.
|
|
812
893
|
* 3. On the hosted path an override would be **inert**: the gateway caps responses at
|
|
813
894
|
* ~30s (see `POLL_REQUEST_TIMEOUT_MS` above), so raising `timeoutMs` would still be
|
|
814
895
|
* cut off upstream, just with a less honest error.
|
|
@@ -930,7 +1011,7 @@ export class PipelexApiClient {
|
|
|
930
1011
|
* see {@link PipelexHostedToolingExtensions}).
|
|
931
1012
|
*/
|
|
932
1013
|
async resolve(request) {
|
|
933
|
-
return this.requestExtension("resolve", request, {
|
|
1014
|
+
return this.requestExtension("resolve", await wireCrateRequest(request), {
|
|
934
1015
|
timeoutMs: crateRequestTimeoutMs(request),
|
|
935
1016
|
});
|
|
936
1017
|
}
|
|
@@ -950,7 +1031,7 @@ export class PipelexApiClient {
|
|
|
950
1031
|
* a malformed selector is a 422; a registry-form `method_ref` is a `501`.
|
|
951
1032
|
*/
|
|
952
1033
|
async codegen(request) {
|
|
953
|
-
return this.requestExtension("codegen", request, {
|
|
1034
|
+
return this.requestExtension("codegen", await wireCrateRequest(request), {
|
|
954
1035
|
timeoutMs: crateRequestTimeoutMs(request),
|
|
955
1036
|
});
|
|
956
1037
|
}
|
|
@@ -970,8 +1051,14 @@ export class PipelexApiClient {
|
|
|
970
1051
|
* describes every pipe instead, and never refuses for want of an entry pipe.
|
|
971
1052
|
* `include_files: true` echoes the resolved closure's `.mthds` files as `files`.
|
|
972
1053
|
*
|
|
1054
|
+
* Beside inline `files`, `resources` sends the files the closure's steps read, such as a
|
|
1055
|
+
* `PipeDocGen` step's `template_file`, keyed by the path the step names, each as its bytes;
|
|
1056
|
+
* a step whose template file was not sent makes the closure invalid. `resolve` and `codegen`
|
|
1057
|
+
* take it too (see {@link CrateRequestBase}).
|
|
1058
|
+
*
|
|
973
1059
|
* Same 200-verdict discipline and same three-form closure selector as {@link resolve}
|
|
974
|
-
* (the request is posted
|
|
1060
|
+
* (the request is posted as given, its `resources` encoded to base64, and the selector XOR
|
|
1061
|
+
* and the `resources` pairing are the server's to enforce).
|
|
975
1062
|
* Only a no-verdict condition throws `ApiResponseError`: a malformed selector and an
|
|
976
1063
|
* over-limit file are `ValidationError` `422`s; a selection the route refuses is a
|
|
977
1064
|
* `422` whose `errorType` is `EntryPipeNotFoundError` (an unknown `pipe_ref`, no entry
|
|
@@ -979,13 +1066,14 @@ export class PipelexApiClient {
|
|
|
979
1066
|
* candidates in its `serverMessage`; a registry-form `method_ref` is a `501`; a pipe
|
|
980
1067
|
* whose artifacts cannot be derived is a `500`.
|
|
981
1068
|
*
|
|
982
|
-
* A `method_ref` gets the fetch-sized budget, as on the other crate routes
|
|
983
|
-
*
|
|
984
|
-
*
|
|
985
|
-
*
|
|
1069
|
+
* A `method_ref` gets the fetch-sized budget, as on the other crate routes, and a
|
|
1070
|
+
* request carrying `resources` the execute ceiling, since their upload alone can
|
|
1071
|
+
* outlast the default. On the hosted API the gateway caps a request at about 30
|
|
1072
|
+
* seconds whatever the client allows, so a cold `method_ref` clone can answer a `502`
|
|
1073
|
+
* that a retry clears once the runner has cached the clone.
|
|
986
1074
|
*/
|
|
987
1075
|
async pipeIo(request) {
|
|
988
|
-
return this.requestExtension("pipe-io", request, {
|
|
1076
|
+
return this.requestExtension("pipe-io", await wireCrateRequest(request), {
|
|
989
1077
|
timeoutMs: crateRequestTimeoutMs(request),
|
|
990
1078
|
});
|
|
991
1079
|
}
|
|
@@ -1198,16 +1286,23 @@ export class PipelexApiClient {
|
|
|
1198
1286
|
* Rows are `MethodSummary`: no `mthds`, no `python`, no `updated_at`. Reach
|
|
1199
1287
|
* for `getMethod(id)` when you need a method's source.
|
|
1200
1288
|
*
|
|
1201
|
-
* `q` is applied server-side over the whole catalog, not over the page
|
|
1289
|
+
* `q` is applied server-side over the whole catalog, not over the page, and so is
|
|
1290
|
+
* `published` (`true` for the methods with a published version, `false` for those never
|
|
1291
|
+
* published). Each row carries the method's publish state (`latest_version`,
|
|
1292
|
+
* `latest_published`, `draft_digest`). Both filters run after the index read, so a page can
|
|
1293
|
+
* come back empty and still carry a `nextCursor`: keep following it until it is `null`.
|
|
1202
1294
|
*/
|
|
1203
1295
|
async listMethods(query = {}) {
|
|
1204
1296
|
const params = new URLSearchParams();
|
|
1205
1297
|
// `!== undefined`, not truthiness — mirroring `listRuns`. Omission means
|
|
1206
1298
|
// "the caller did not ask"; an explicit empty string is bad input and the
|
|
1207
1299
|
// API should say so, rather than being silently dropped into an unfiltered
|
|
1208
|
-
// query that returns everything and reads as working.
|
|
1300
|
+
// query that returns everything and reads as working. `published: false`
|
|
1301
|
+
// is a filter of its own, never omission.
|
|
1209
1302
|
if (query.q !== undefined)
|
|
1210
1303
|
params.set("q", query.q);
|
|
1304
|
+
if (query.published !== undefined)
|
|
1305
|
+
params.set("published", String(query.published));
|
|
1211
1306
|
if (query.limit !== undefined)
|
|
1212
1307
|
params.set("limit", String(query.limit));
|
|
1213
1308
|
if (query.cursor !== undefined)
|
|
@@ -1233,43 +1328,14 @@ export class PipelexApiClient {
|
|
|
1233
1328
|
* TRUNCATED list with no error — precisely the bug paging was introduced to
|
|
1234
1329
|
* remove. If you truly want an array, `Array.fromAsync` makes that your
|
|
1235
1330
|
* explicit choice.
|
|
1331
|
+
*
|
|
1332
|
+
* It keeps going **through an empty page that carries a cursor**: under `q` or `published`
|
|
1333
|
+
* such a page means "nothing matched in the slice just read", not "done". It stops on a
|
|
1334
|
+
* `null` cursor or on a cursor the server failed to advance, and throws
|
|
1335
|
+
* `PagingNotTerminatingError` past a page ceiling no real catalog reaches.
|
|
1236
1336
|
*/
|
|
1237
1337
|
async *iterateMethods(query = {}) {
|
|
1238
|
-
|
|
1239
|
-
let pages = 0;
|
|
1240
|
-
for (;;) {
|
|
1241
|
-
const page = await this.listMethods({ ...query, cursor });
|
|
1242
|
-
// Checked BEFORE yielding: a server handing back the very cursor we sent
|
|
1243
|
-
// has not advanced, so this page repeats rows already emitted. Cannot
|
|
1244
|
-
// fire on the first request, where `cursor` is undefined.
|
|
1245
|
-
if (cursor !== undefined && page.nextCursor === cursor)
|
|
1246
|
-
return;
|
|
1247
|
-
for (const method of page.items)
|
|
1248
|
-
yield method;
|
|
1249
|
-
// `nextCursor === null` is the NORMAL end of the catalog, and that page
|
|
1250
|
-
// is real data, so it has to be emitted before this fires.
|
|
1251
|
-
if (page.nextCursor === null)
|
|
1252
|
-
return;
|
|
1253
|
-
// An EMPTY page is NOT the end, and treating it as one was a bug. The
|
|
1254
|
-
// platform applies `q` as a post-read filter over a bounded number of
|
|
1255
|
-
// index pages per request, so a sparse match legitimately returns
|
|
1256
|
-
// `{items: [], nextCursor: "…"}` — "nothing matched in the slice I just
|
|
1257
|
-
// read, keep going". Returning here dropped every later match silently,
|
|
1258
|
-
// which is the exact truncation this pagination work exists to remove.
|
|
1259
|
-
// (`iterateRuns` can stop on an empty page because its date bounds are
|
|
1260
|
-
// index KEY conditions, so a run page is never empty-with-a-cursor. The
|
|
1261
|
-
// difference is the server, not the client.)
|
|
1262
|
-
pages += 1;
|
|
1263
|
-
if (pages >= MAX_PAGES) {
|
|
1264
|
-
// THROW, do not return. A caller that asked for every method and got a
|
|
1265
|
-
// partial answer with no error is back to the original bug one layer
|
|
1266
|
-
// up; an error is the only honest response to a server that will not
|
|
1267
|
-
// finish. Unreachable on real data — see MAX_PAGES.
|
|
1268
|
-
throw new PagingNotTerminatingError(`listMethods did not terminate after ${MAX_PAGES} pages; refusing to keep paging. ` +
|
|
1269
|
-
`This is a server-side fault, not a coverage limit.`, MAX_PAGES);
|
|
1270
|
-
}
|
|
1271
|
-
cursor = page.nextCursor;
|
|
1272
|
-
}
|
|
1338
|
+
yield* followCursor("listMethods", (cursor) => this.listMethods({ ...query, cursor }));
|
|
1273
1339
|
}
|
|
1274
1340
|
/**
|
|
1275
1341
|
* Read one method — `GET /v1/methods/{id}`: its identity, its draft (`mthds`, `python`,
|
|
@@ -1657,7 +1723,8 @@ export class PipelexApiClient {
|
|
|
1657
1723
|
* Name the method exactly one of three ways, all server-resolved through the
|
|
1658
1724
|
* one `pipeIo` call this composes:
|
|
1659
1725
|
*
|
|
1660
|
-
* - `files` — the inline MTHDS closure
|
|
1726
|
+
* - `files` — the inline MTHDS closure, with the `resources` its steps read
|
|
1727
|
+
* (a `PipeDocGen` step's `template_file`), keyed by the path a step names;
|
|
1661
1728
|
* - `method_ref` — a published method's address, fetched by the runner;
|
|
1662
1729
|
* - `method_id` — a stored method's catalog id, resolved by the platform
|
|
1663
1730
|
* (hosted only; requires an API key).
|
|
@@ -1668,8 +1735,11 @@ export class PipelexApiClient {
|
|
|
1668
1735
|
* position is a fact of the method, never a guess from the value's shape. It
|
|
1669
1736
|
* needs an API serving `POST /v1/pipe-io`. See `docs/input-preparation.md`.
|
|
1670
1737
|
*
|
|
1671
|
-
*
|
|
1672
|
-
*
|
|
1738
|
+
* A value at a file input that cannot be a file, such as a number or a
|
|
1739
|
+
* `data:` URL that does not decode, throws `InvalidInputValueError`, naming the
|
|
1740
|
+
* input. With `allowUploads: false` it uploads nothing and reads no file: a
|
|
1741
|
+
* value that would need an upload throws `UploadNotAllowedError`, naming the
|
|
1742
|
+
* input too.
|
|
1673
1743
|
*/
|
|
1674
1744
|
async prepareInputs(request) {
|
|
1675
1745
|
return prepareInputsImpl(this, request);
|
|
@@ -1696,6 +1766,13 @@ export class PipelexApiClient {
|
|
|
1696
1766
|
* conditions, so a bounded page genuinely reads less. They are INSTANTS,
|
|
1697
1767
|
* not days — see `ListRunsQuery`.
|
|
1698
1768
|
*
|
|
1769
|
+
* `createdBy: "me"` keeps the caller's own runs and `status` the runs in any of the given
|
|
1770
|
+
* statuses, sent as `created_by=me` and one repeated `status=` parameter per status. Both are
|
|
1771
|
+
* filters applied after the index read, so a filtered page can hold fewer runs than `limit`,
|
|
1772
|
+
* or none, and still carry a `nextCursor`: an empty page is not the end, a `null` cursor is.
|
|
1773
|
+
* An empty `status` list, or a name that is no `RunStatus`, is a `RequestArgumentError`
|
|
1774
|
+
* before any request.
|
|
1775
|
+
*
|
|
1699
1776
|
* Takes a bare catalog id: the history files the runs of every version and
|
|
1700
1777
|
* of the draft together under it, so `mt_…@3` names no history of its own
|
|
1701
1778
|
* and the platform refuses it with a `400`. Strip a suffix with
|
|
@@ -1713,6 +1790,10 @@ export class PipelexApiClient {
|
|
|
1713
1790
|
params.set("created_from", query.createdFrom);
|
|
1714
1791
|
if (query.createdTo !== undefined)
|
|
1715
1792
|
params.set("created_to", query.createdTo);
|
|
1793
|
+
if (query.createdBy !== undefined)
|
|
1794
|
+
params.set("created_by", query.createdBy);
|
|
1795
|
+
for (const status of runStatusFilter(query.status) ?? [])
|
|
1796
|
+
params.append("status", status);
|
|
1716
1797
|
if (query.limit !== undefined)
|
|
1717
1798
|
params.set("limit", String(query.limit));
|
|
1718
1799
|
if (query.cursor !== undefined)
|
|
@@ -1742,33 +1823,15 @@ export class PipelexApiClient {
|
|
|
1742
1823
|
* when to stop, and only one page is ever in memory. If you truly want an
|
|
1743
1824
|
* array, `Array.fromAsync` makes that your explicit choice.
|
|
1744
1825
|
*
|
|
1826
|
+
* It keeps going **through an empty page that carries a cursor**: under `createdBy` or
|
|
1827
|
+
* `status` such a page means "nothing matched in the slice just read", not "done". It stops on
|
|
1828
|
+
* a `null` cursor or on a cursor the server failed to advance, and throws
|
|
1829
|
+
* `PagingNotTerminatingError` past a page ceiling no real history reaches.
|
|
1830
|
+
*
|
|
1745
1831
|
* Takes a bare catalog id, as `listRuns` does.
|
|
1746
1832
|
*/
|
|
1747
1833
|
async *iterateRuns(methodId, query = {}) {
|
|
1748
|
-
|
|
1749
|
-
for (;;) {
|
|
1750
|
-
const page = await this.listRuns(methodId, { ...query, cursor });
|
|
1751
|
-
// Checked BEFORE yielding. A server handing back the very cursor we sent
|
|
1752
|
-
// has not advanced, so this page repeats rows already emitted — yielding
|
|
1753
|
-
// first and stopping after would silently double-count them for anyone
|
|
1754
|
-
// aggregating the stream (summing cost, counting runs). Cannot fire on
|
|
1755
|
-
// the first request, where `cursor` is undefined.
|
|
1756
|
-
if (cursor !== undefined && page.nextCursor === cursor)
|
|
1757
|
-
return;
|
|
1758
|
-
for (const run of page.items)
|
|
1759
|
-
yield run;
|
|
1760
|
-
// Checked AFTER: `nextCursor === null` is the NORMAL end of the history
|
|
1761
|
-
// and that page is real data, so it has to be emitted first. The empty
|
|
1762
|
-
// page catches a server that keeps minting fresh cursors while returning
|
|
1763
|
-
// nothing, which would otherwise spin forever yielding nothing.
|
|
1764
|
-
//
|
|
1765
|
-
// Neither of these is a page cap. Both fire only on a server that is not
|
|
1766
|
-
// making progress, so neither can truncate a healthy stream the way a
|
|
1767
|
-
// `maxPages` limit silently did.
|
|
1768
|
-
if (page.nextCursor === null || page.items.length === 0)
|
|
1769
|
-
return;
|
|
1770
|
-
cursor = page.nextCursor;
|
|
1771
|
-
}
|
|
1834
|
+
yield* followCursor("listRuns", (cursor) => this.listRuns(methodId, { ...query, cursor }));
|
|
1772
1835
|
}
|
|
1773
1836
|
/**
|
|
1774
1837
|
* The whole record for one run — `GET /v1/runs/{id}`.
|
|
@@ -1808,6 +1871,8 @@ export class PipelexApiClient {
|
|
|
1808
1871
|
// this path runs the same method as the durable one, or it runs nothing.
|
|
1809
1872
|
files: options.files ?? undefined,
|
|
1810
1873
|
bundle_b64: options.bundle_b64 ?? undefined,
|
|
1874
|
+
// And so must the files its steps read, or the same method fails to load here.
|
|
1875
|
+
resources: options.resources ?? undefined,
|
|
1811
1876
|
method_ref: options.method_ref ?? undefined,
|
|
1812
1877
|
method_id: options.method_id ?? undefined,
|
|
1813
1878
|
extra: options.extra ?? undefined,
|
|
@@ -1888,12 +1953,13 @@ const PROTOCOL_REQUEST_KEYS = [
|
|
|
1888
1953
|
"files",
|
|
1889
1954
|
"bundle_b64",
|
|
1890
1955
|
];
|
|
1891
|
-
// The PIPELEX API's own request fields — the layer-2
|
|
1892
|
-
//
|
|
1893
|
-
//
|
|
1894
|
-
//
|
|
1895
|
-
// selector-exclusivity
|
|
1896
|
-
|
|
1956
|
+
// The PIPELEX API's own request fields — the layer-2 extensions the runner
|
|
1957
|
+
// reads itself (`method_ref`, a run source in its own right, and `resources`,
|
|
1958
|
+
// the files the bundle's steps read). Reserved on `extra` for the same reason
|
|
1959
|
+
// the protocol fields are: `extra` merges last, so a smuggled copy would
|
|
1960
|
+
// overwrite the validated named option and bypass the selector-exclusivity and
|
|
1961
|
+
// path checks.
|
|
1962
|
+
const PIPELEX_API_REQUEST_KEYS = ["method_ref", "resources"];
|
|
1897
1963
|
// The HOSTED API's own request fields — the layer-3 extensions this client
|
|
1898
1964
|
// names itself. Reserved on `extra` for the same reason the protocol fields
|
|
1899
1965
|
// are: `extra` merges last, so a smuggled copy would overwrite the validated
|
|
@@ -1956,10 +2022,20 @@ function buildExtensions(extra) {
|
|
|
1956
2022
|
const METHOD_REF_FETCH_TIMEOUT_MS = 180_000; // 3 min — covers the server's clone + resolve
|
|
1957
2023
|
/**
|
|
1958
2024
|
* The internal request budget for a crate route's `CrateRequestBase` closure
|
|
1959
|
-
* (`resolve`, `codegen`, `pipeIo`): the static-route default, unless the
|
|
1960
|
-
* is
|
|
2025
|
+
* (`resolve`, `codegen`, `pipeIo`): the static-route default, unless the request
|
|
2026
|
+
* is one the default would cut short through no fault of the server's.
|
|
2027
|
+
*
|
|
2028
|
+
* - **`resources`** make the request *body* as large as the runner's limits allow,
|
|
2029
|
+
* about 13 MiB of base64 by default, and the whole upload is charged against
|
|
2030
|
+
* this budget. They get the execute ceiling `start` gives a request carrying
|
|
2031
|
+
* them, and `validate` carries them under by default, so the same bundle and
|
|
2032
|
+
* files never time out on one route yet go through on another.
|
|
2033
|
+
* - **A `method_ref`** may make the server clone a repository first, which gets
|
|
2034
|
+
* the fetch-sized budget.
|
|
1961
2035
|
*/
|
|
1962
2036
|
function crateRequestTimeoutMs(request) {
|
|
2037
|
+
if (hasBundleResources(request.resources))
|
|
2038
|
+
return DEFAULT_REQUEST_TIMEOUT_MS;
|
|
1963
2039
|
return nonEmptyString(request.method_ref) !== undefined ? METHOD_REF_FETCH_TIMEOUT_MS : undefined;
|
|
1964
2040
|
}
|
|
1965
2041
|
/**
|
|
@@ -2065,6 +2141,43 @@ function assertLinkageMethodIdIsBare(options) {
|
|
|
2065
2141
|
"that did not. Send the bare id (parseMethodSelector(...).method_id), or drop the inline " +
|
|
2066
2142
|
"source to run the version the selector names.");
|
|
2067
2143
|
}
|
|
2144
|
+
/**
|
|
2145
|
+
* Enforce the run routes' `resources` pairing, mirroring the runner's own `422`s
|
|
2146
|
+
* so an illegal pairing fails before anything is encoded or sent. The files a
|
|
2147
|
+
* bundle's steps read accompany that bundle: inline `mthds_contents` or a method
|
|
2148
|
+
* bundle. Beside a `method_ref` they are refused, a published package's files
|
|
2149
|
+
* being its own, and so they are with no source at all, where no step could read
|
|
2150
|
+
* them. A hosted `method_id` without an inline source is left to the server: the
|
|
2151
|
+
* platform resolves it into the bundle the runner then reads them with.
|
|
2152
|
+
*
|
|
2153
|
+
* Presence follows the other pairing checks: `resources` counts when it carries a
|
|
2154
|
+
* file, `mthds_contents` when non-empty, a bundle when `hasBundlePayload` says so.
|
|
2155
|
+
*/
|
|
2156
|
+
function assertResourcesAccompanyABundle(options) {
|
|
2157
|
+
if (!hasBundleResources(options.resources))
|
|
2158
|
+
return;
|
|
2159
|
+
if (nonEmptyString(options.method_ref) !== undefined) {
|
|
2160
|
+
throw new RequestArgumentError("resources accompany inline mthds_contents or a method bundle; a method_ref's files are the package's own.");
|
|
2161
|
+
}
|
|
2162
|
+
const hasInlineSource = (options.mthds_contents != null && options.mthds_contents.length > 0) ||
|
|
2163
|
+
hasBundlePayload(options);
|
|
2164
|
+
if (!hasInlineSource && nonEmptyString(options.method_id) === undefined) {
|
|
2165
|
+
throw new RequestArgumentError("resources accompany the bundle whose steps read them: send them with mthds_contents or a method bundle.");
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
/**
|
|
2169
|
+
* A crate request as the wire takes it: the request as given, its `resources`
|
|
2170
|
+
* encoded into the base64 map, the key left off when there is nothing to send.
|
|
2171
|
+
* A request that carries no `resources` key is posted untouched.
|
|
2172
|
+
*/
|
|
2173
|
+
async function wireCrateRequest(request) {
|
|
2174
|
+
if (typeof request !== "object" || request === null || !("resources" in request)) {
|
|
2175
|
+
return request;
|
|
2176
|
+
}
|
|
2177
|
+
const { resources, ...rest } = request;
|
|
2178
|
+
const encoded = await encodeBundleResources(resources);
|
|
2179
|
+
return encoded === undefined ? rest : { ...rest, resources: encoded };
|
|
2180
|
+
}
|
|
2068
2181
|
/**
|
|
2069
2182
|
* Normalize a bundle encoding for the wire: an empty map / string is NOT a
|
|
2070
2183
|
* runnable bundle, so it must not be sent (the runner rejects a zero-file
|
|
@@ -2191,16 +2304,6 @@ function parseRetryAfter(headers) {
|
|
|
2191
2304
|
const seconds = Number(raw);
|
|
2192
2305
|
return Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
|
|
2193
2306
|
}
|
|
2194
|
-
const KNOWN_RUN_STATUSES = [
|
|
2195
|
-
"PENDING",
|
|
2196
|
-
"STARTED",
|
|
2197
|
-
"RUNNING",
|
|
2198
|
-
"COMPLETED",
|
|
2199
|
-
"FAILED",
|
|
2200
|
-
"CANCELLED",
|
|
2201
|
-
"TERMINATED",
|
|
2202
|
-
"TIMED_OUT",
|
|
2203
|
-
];
|
|
2204
2307
|
/** The response header the platform and the runner stamp the request's correlation id on. */
|
|
2205
2308
|
const REQUEST_ID_HEADER = "x-request-id";
|
|
2206
2309
|
/**
|
|
@@ -2249,7 +2352,7 @@ function withCheckedDegradation(run) {
|
|
|
2249
2352
|
return { ...run, degradation: readRunDegradation(run.degradation) };
|
|
2250
2353
|
}
|
|
2251
2354
|
function knownRunStatus(value) {
|
|
2252
|
-
return typeof value === "string" &&
|
|
2355
|
+
return typeof value === "string" && RUN_STATUSES.includes(value)
|
|
2253
2356
|
? value
|
|
2254
2357
|
: undefined;
|
|
2255
2358
|
}
|