@pipelex/sdk 0.46.1 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -9
- package/dist/cli/script.d.ts +1 -1
- package/dist/cli/source.d.ts +8 -7
- package/dist/cli/source.js +11 -5
- package/dist/cli/source.js.map +1 -1
- package/dist/client.d.ts +455 -237
- package/dist/client.js +1067 -522
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +49 -19
- package/dist/errors.js +50 -21
- package/dist/errors.js.map +1 -1
- package/dist/hooks/bundle-gather.d.ts +28 -40
- package/dist/hooks/bundle-gather.js +55 -230
- package/dist/hooks/bundle-gather.js.map +1 -1
- package/dist/hooks/claude-mthds-check.d.ts +7 -5
- package/dist/hooks/claude-mthds-check.js +8 -53
- package/dist/hooks/claude-mthds-check.js.map +1 -1
- package/dist/hooks/validate-stage.d.ts +27 -0
- package/dist/hooks/validate-stage.js +57 -0
- package/dist/hooks/validate-stage.js.map +1 -0
- package/dist/index.d.ts +7 -7
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/method-trees.d.ts +344 -0
- package/dist/method-trees.js +876 -0
- package/dist/method-trees.js.map +1 -0
- package/dist/models.d.ts +162 -59
- package/dist/prepare-inputs.d.ts +46 -27
- package/dist/prepare-inputs.js +123 -42
- package/dist/prepare-inputs.js.map +1 -1
- package/dist/product-models.d.ts +97 -75
- package/dist/runs.d.ts +5 -3
- package/dist/runs.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +20 -21
- package/docs/cli.md +1 -1
- package/docs/crate-routes.md +42 -20
- package/docs/errors.md +7 -6
- package/docs/hook-bundle.md +1 -1
- package/docs/input-preparation.md +15 -40
- package/docs/method-trees.md +199 -0
- package/docs/run-results.md +5 -2
- package/package.json +2 -2
- package/dist/bundle-resources.d.ts +0 -62
- package/dist/bundle-resources.js +0 -165
- package/dist/bundle-resources.js.map +0 -1
- package/dist/method-source.d.ts +0 -19
- package/dist/method-source.js +0 -50
- package/dist/method-source.js.map +0 -1
- package/docs/bundle-resources.md +0 -107
package/dist/client.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
|
-
//
|
|
1
|
+
// Method-source predicates come from the standard, not a local restatement: which
|
|
2
2
|
// source combinations are legal is an invariant of `RunRequest` itself, so this
|
|
3
3
|
// client and the MTHDS runners cannot disagree about what they reject.
|
|
4
|
-
import {
|
|
4
|
+
import { assertOneMethodSource, parseJsonBody } from "mthds/protocol";
|
|
5
5
|
import { assertArtifactSelection, assertWaitOptions, pollUntilResult, readRunDegradation, RUN_STATUSES, runProgressFault, runStatusFilter, selectionIncludesMainStuff, throwIfAborted, withCheckedProgressDegradations, } from "./runs.js";
|
|
6
|
-
import { ApiResponseError, ApiUnreachableError,
|
|
6
|
+
import { ApiResponseError, ApiUnreachableError, MissingMainStuffError, PagingNotTerminatingError, PipelineExecuteTimeoutError, PipelineRequestError, RequestArgumentError, RunLifecycleUnavailableError, RunStillRunningError, } from "./errors.js";
|
|
7
7
|
import { isPlainObject, isValidationItem, readFieldError, readMigration, readProviderMetadata, readRunErrorReport, } from "./error-models.js";
|
|
8
|
-
import {
|
|
8
|
+
import { HANDSHAKE_TTL_MS, INLINE_TREE_MAX_BYTES, MAX_CONCURRENT_TRANSFERS, REQUEST_SENDING, SAVE_SENDING, announcedTreeLimits, contentsTreeSource, declaresBlobStore, eitherSignal, emptyTreeRefusal, fileByBlob, forEachConcurrently, hashMethodFiles, inlineFilesOf, inlineMapBytes, inlineThreshold, isMethodTreeTooLarge, listDirectoryTree, listerLimitsOf, missingBlobsOf, presentMap, readTreeBytesInto, redactSignedLinks, sha256Hex, treeShapeFault, treeTotalSize, uniqueBlobs, uploadBlob, writeMethodTreeTo, } from "./method-trees.js";
|
|
9
9
|
import { buildUserAgent } from "./user-agent.js";
|
|
10
10
|
import { uploadFile as uploadFileImpl } from "./upload.js";
|
|
11
|
-
import { encodeBundleResources, hasBundleResources } from "./bundle-resources.js";
|
|
12
11
|
import { prepareInputs as prepareInputsImpl } from "./prepare-inputs.js";
|
|
13
12
|
import { downloadArtifacts as downloadArtifactsImpl, fetchArtifact as fetchArtifactImpl, resolveArtifacts as resolveArtifactsImpl, } from "./artifacts.js";
|
|
14
13
|
import { PipelexExecuteResult, resultsFromExecute } from "./execute-result.js";
|
|
@@ -123,32 +122,30 @@ const VALIDATE_MARKDOWN_RENDER_FORMAT = "markdown";
|
|
|
123
122
|
*/
|
|
124
123
|
const BARE_RUNNER_IMPLEMENTATION = "pipelex-api";
|
|
125
124
|
/**
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
125
|
+
* Client for the Pipelex hosted API — and any MTHDS-compliant runner.
|
|
126
|
+
*
|
|
127
|
+
* One base URL (`PIPELEX_BASE_URL`); every endpoint is `<base>/v1/<endpoint>`:
|
|
128
|
+
* - **protocol** (`execute` / `start` / `validate` / `models` / `version`) — works
|
|
129
|
+
* against any MTHDS-compliant runner, hosted or bare.
|
|
130
|
+
* - **crate extensions** (`/v1/resolve`, `/v1/codegen`, `/v1/pipe-io`) — the normalized
|
|
131
|
+
* library crate, the stamped typed artifacts projected from it, and a method's three
|
|
132
|
+
* I/O artifacts derived with no dry run.
|
|
133
|
+
* - **tools extensions** (`lint` / `format`) — single-file static diagnostics and
|
|
134
|
+
* canonical formatting, served by any pipelex-api runner.
|
|
135
|
+
* - **run lifecycle** (`getRunStatus` / `getRunProgress` / `getRunResult` /
|
|
136
|
+
* `waitForResult`) — the durable polling extension that survives long runs and
|
|
137
|
+
* lets a caller resume by id. Served only by a deployment that includes the
|
|
138
|
+
* platform block (the Pipelex Hosted API); a bare `pipelex-api` runner 404s those
|
|
139
|
+
* routes, which the lifecycle methods translate into a clear
|
|
140
|
+
* `RunLifecycleUnavailableError`.
|
|
141
|
+
*
|
|
142
|
+
* Implements `MTHDSProtocol<DictPipeOutput>` so the protocol-execution methods
|
|
143
|
+
* stay shaped like the standard's wire surface (`mthds/protocol`), all but
|
|
144
|
+
* `validate`, whose verdict is the one the hosted plane answers: since protocol
|
|
145
|
+
* `2.0.0` the protocol's verdict lists its faults as problem items, which the
|
|
146
|
+
* hosted plane does not send yet (see `PipelexValidationResult`). The Pipelex
|
|
147
|
+
* extensions (the crate and tools routes, the run lifecycle) ride on top.
|
|
147
148
|
*/
|
|
148
|
-
function withWirePython(input) {
|
|
149
|
-
const { python, ...rest } = input;
|
|
150
|
-
return python === undefined ? rest : { ...rest, python: serializeMethodFiles(python) };
|
|
151
|
-
}
|
|
152
149
|
/**
|
|
153
150
|
* A request's body as the JSON text it is sent as, written by `stringifyExactJson` in one pass,
|
|
154
151
|
* so a `bigint` read from an answer carrying a person's data goes back as its exact digits, and
|
|
@@ -324,6 +321,16 @@ export class PipelexApiClient {
|
|
|
324
321
|
originUrl;
|
|
325
322
|
/** Cached `/v1/version` handshake outcome — whether the durable lifecycle is served. */
|
|
326
323
|
lifecycleAvailable;
|
|
324
|
+
/**
|
|
325
|
+
* The last `/v1/version` handshake that answered and when it was asked, kept for
|
|
326
|
+
* `HANDSHAKE_TTL_MS` as `mthds`'s `MthdsApiClient` keeps it: what sending a tree reads from it
|
|
327
|
+
* (`blob_store`, the tree limits) changes only with a deploy. A handshake that failed is not
|
|
328
|
+
* kept, so the next call asks again, and a refusal showing it stale drops it sooner
|
|
329
|
+
* (`forgetHandshake`).
|
|
330
|
+
*/
|
|
331
|
+
versionCache;
|
|
332
|
+
/** The client's request ceiling (`PipelexApiClientOptions.requestTimeoutMs`). */
|
|
333
|
+
requestTimeoutMs;
|
|
327
334
|
/**
|
|
328
335
|
* The `User-Agent` sent on every API request, computed once at construction;
|
|
329
336
|
* `undefined` in a browser, where no header may be set.
|
|
@@ -351,6 +358,13 @@ export class PipelexApiClient {
|
|
|
351
358
|
}
|
|
352
359
|
this.baseUrl = normalizedBaseUrl;
|
|
353
360
|
this.originUrl = new URL("/", this.baseUrl).origin;
|
|
361
|
+
const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
362
|
+
// A longer delay overflows the timer, which then fires at once as a false timeout.
|
|
363
|
+
if (!isTimerDelay(requestTimeoutMs)) {
|
|
364
|
+
throw new RangeError(`"requestTimeoutMs" must be a positive number no larger than ${MAX_TIMER_DELAY_MS}, got ` +
|
|
365
|
+
`${String(requestTimeoutMs)}.`);
|
|
366
|
+
}
|
|
367
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
354
368
|
}
|
|
355
369
|
// ── URL resolution ───────────────────────────────────────────────────
|
|
356
370
|
/** Build an API URL: `<base>/v1/<endpoint>`. */
|
|
@@ -395,7 +409,7 @@ export class PipelexApiClient {
|
|
|
395
409
|
// argument rather than caught below and reported as a network failure.
|
|
396
410
|
const bodyText = hasBody ? requestBodyText(method, url, options.body) : undefined;
|
|
397
411
|
const headers = this.requestHeaders(hasBody, options.idempotencyKey);
|
|
398
|
-
const timeoutMs = options.timeoutMs ??
|
|
412
|
+
const timeoutMs = options.timeoutMs ?? this.requestTimeoutMs;
|
|
399
413
|
// A longer delay overflows the timer, which then fires at once as a false timeout.
|
|
400
414
|
if (!isTimerDelay(timeoutMs)) {
|
|
401
415
|
throw new RangeError(`"timeoutMs" must be a positive number no larger than ${MAX_TIMER_DELAY_MS}, got ` +
|
|
@@ -486,28 +500,27 @@ export class PipelexApiClient {
|
|
|
486
500
|
return this.readObjectAnswer(method, endpoint, res);
|
|
487
501
|
}
|
|
488
502
|
/**
|
|
489
|
-
* A methods-catalog route answering one stored method (`
|
|
490
|
-
* `
|
|
503
|
+
* A methods-catalog route answering one stored method (`createMethod`, `writeDraft`,
|
|
504
|
+
* `renameMethod`), read as `readObjectAnswer` reads it.
|
|
491
505
|
*/
|
|
492
|
-
async requestMethodData(method, endpoint, body) {
|
|
493
|
-
const res = await this.requestProductAnswer(method, endpoint, body);
|
|
494
|
-
|
|
495
|
-
return this.readStoredSources(method, endpoint, res, wire, "a stored method");
|
|
506
|
+
async requestMethodData(method, endpoint, body, options = {}) {
|
|
507
|
+
const res = await this.requestProductAnswer(method, endpoint, body, options);
|
|
508
|
+
return this.readObjectAnswer(method, endpoint, res);
|
|
496
509
|
}
|
|
497
510
|
/**
|
|
498
|
-
*
|
|
499
|
-
*
|
|
500
|
-
*
|
|
501
|
-
*
|
|
502
|
-
*
|
|
503
|
-
* change to the call fixes it. `subject` names what carried them in the message.
|
|
511
|
+
* A route answering a saved tree's listing beside what names it (`getMethod`,
|
|
512
|
+
* `getMethodVersion`), read as `readObjectAnswer` reads it, then its `files` checked: a listing
|
|
513
|
+
* that is not an array of `{ path, sha256, size, url, expires_at }`, each a string but `size`, a
|
|
514
|
+
* non-negative integer, is an answer the SDK cannot read, since `writeMethodTree` would write
|
|
515
|
+
* from it. Throws the `ApiResponseError` `unreadableAnswer` builds, `subject` naming the answer.
|
|
504
516
|
*/
|
|
505
|
-
|
|
506
|
-
const
|
|
507
|
-
|
|
508
|
-
|
|
517
|
+
readSavedTree(method, endpoint, res, subject) {
|
|
518
|
+
const answer = this.readObjectAnswer(method, endpoint, res);
|
|
519
|
+
const fault = savedFilesFault(answer.files);
|
|
520
|
+
if (fault !== null) {
|
|
521
|
+
throw this.unreadableAnswer(method, `/${API_PREFIX}/${endpoint}`, res, `${subject} whose \`files\` ${fault}`);
|
|
509
522
|
}
|
|
510
|
-
return
|
|
523
|
+
return answer;
|
|
511
524
|
}
|
|
512
525
|
/**
|
|
513
526
|
* A verdict, the `200` a route answers discriminated on `outcome` with the object the verdict
|
|
@@ -592,11 +605,14 @@ export class PipelexApiClient {
|
|
|
592
605
|
}
|
|
593
606
|
return value;
|
|
594
607
|
}
|
|
595
|
-
/**
|
|
608
|
+
/**
|
|
609
|
+
* Issue a product request and return its 2xx answer, a non-2xx thrown as `ApiResponseError`.
|
|
610
|
+
* The request takes the poll timeout unless `options.timeoutMs` gives another.
|
|
611
|
+
*/
|
|
596
612
|
async requestProductAnswer(method, endpoint, body, options = {}) {
|
|
597
613
|
const res = await this.requestRaw(method, this.url(endpoint), {
|
|
598
614
|
body,
|
|
599
|
-
timeoutMs: POLL_REQUEST_TIMEOUT_MS,
|
|
615
|
+
timeoutMs: options.timeoutMs ?? POLL_REQUEST_TIMEOUT_MS,
|
|
600
616
|
signal: options.signal,
|
|
601
617
|
});
|
|
602
618
|
if (res.status < 200 || res.status >= 300) {
|
|
@@ -673,11 +689,13 @@ export class PipelexApiClient {
|
|
|
673
689
|
* `API <method> <path> answered <status> with <what>`: the answer's status, its raw text as
|
|
674
690
|
* `responseBody`, no hosted problem member but the `X-Request-ID` header, the failure that made it
|
|
675
691
|
* unreadable as `cause`, and the verdict the fallback gives a 2xx, `runtime` and not retryable.
|
|
676
|
-
*
|
|
692
|
+
* An answer that carries signed links (`RawResponse.carriesLinks`) is held with every link
|
|
693
|
+
* hidden, and with no `cause`, which could quote one. On a request that may create a run, the run may have started: the server accepted it.
|
|
677
694
|
*/
|
|
678
695
|
unreadableAnswer(method, path, res, what, cause, run = {}) {
|
|
679
|
-
|
|
680
|
-
|
|
696
|
+
const carriesLinks = res.carriesLinks === true;
|
|
697
|
+
return new ApiResponseError(`API ${method} ${path} answered ${res.status} with ${what}`, this.baseUrl, res.status, res.statusText, carriesLinks ? redactSignedLinks(res.body) : res.body, undefined, undefined, undefined, undefined, {
|
|
698
|
+
cause: carriesLinks ? undefined : cause,
|
|
681
699
|
details: { requestId: nonEmptyHeader(res.headers, REQUEST_ID_HEADER) },
|
|
682
700
|
headers: res.headers,
|
|
683
701
|
mayStartRun: run.mayStartRun === true,
|
|
@@ -773,46 +791,44 @@ export class PipelexApiClient {
|
|
|
773
791
|
*
|
|
774
792
|
* The answer is read with `parseExactJson`, so an integer past `Number.MAX_SAFE_INTEGER` in the
|
|
775
793
|
* working memory, the main output included, is a `bigint` holding its exact digits.
|
|
794
|
+
*
|
|
795
|
+
* The method's tree, given as a `directory`, `contents` or `tree`, is prepared before the
|
|
796
|
+
* request: listed, read, and sent inline or by hash after uploading the files the API lacks (see
|
|
797
|
+
* `PipelexRunOptions`). A tree this client refuses (two forms, no file, a directory it cannot
|
|
798
|
+
* list) is a `RequestArgumentError`, and a file that fails to upload a `MethodFileTransferError`,
|
|
799
|
+
* each before the run is asked.
|
|
776
800
|
*/
|
|
777
801
|
async execute(options) {
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
...extensions,
|
|
808
|
-
};
|
|
809
|
-
// Timed on the monotonic clock, which a change of the system's time does not move.
|
|
810
|
-
const startedAt = performance.now();
|
|
802
|
+
return this.executeRun(options);
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* `execute`, stopped by `signal` before each step of the tree's preparation and right before the
|
|
806
|
+
* request is sent, never during it, since a request sent may have started a run. `sending` is
|
|
807
|
+
* the tree already prepared, which the blocking fallback of `startAndWaitForResult` sends as the
|
|
808
|
+
* start sent it rather than preparing it again.
|
|
809
|
+
*/
|
|
810
|
+
async executeRun(options, signal, sending) {
|
|
811
|
+
const parts = this.runParts("execute", options);
|
|
812
|
+
const source = sending?.source ?? (await this.treeToSend("execute", parts.forms, REQUEST_SENDING, signal));
|
|
813
|
+
// Timed on the monotonic clock, which a change of the system's time does not move, from the
|
|
814
|
+
// request itself: the tree's preparation, uploads included, is no part of the gateway's cut-off.
|
|
815
|
+
// Set when the request is sent, which the answer read below always follows.
|
|
816
|
+
let sentAt = 0;
|
|
817
|
+
const res = await this.postMethodRequest(source, "execute", REQUEST_SENDING, { signal }, async (files) => {
|
|
818
|
+
const request = runRequestBody("execute", options, files, parts);
|
|
819
|
+
throwIfAborted(signal);
|
|
820
|
+
sentAt = performance.now();
|
|
821
|
+
try {
|
|
822
|
+
return await this.requestRaw("POST", this.url("execute"), {
|
|
823
|
+
body: request,
|
|
824
|
+
...STARTS_RUN,
|
|
825
|
+
});
|
|
826
|
+
}
|
|
827
|
+
catch (err) {
|
|
828
|
+
throw executeCutOff(err, sentAt);
|
|
829
|
+
}
|
|
830
|
+
}, sending?.initial);
|
|
811
831
|
try {
|
|
812
|
-
const res = await this.requestRaw("POST", this.url("execute"), {
|
|
813
|
-
body: request,
|
|
814
|
-
...STARTS_RUN,
|
|
815
|
-
});
|
|
816
832
|
this.throwIfExecuteDegraded(res);
|
|
817
833
|
if (res.status < 200 || res.status >= 300) {
|
|
818
834
|
this.throwApiResponseError("POST", "execute", res, STARTS_RUN);
|
|
@@ -827,16 +843,10 @@ export class PipelexApiClient {
|
|
|
827
843
|
return new PipelexExecuteResult(answer);
|
|
828
844
|
}
|
|
829
845
|
catch (err) {
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
//
|
|
833
|
-
|
|
834
|
-
// translate it into a clear, actionable error pointing at start+poll.
|
|
835
|
-
const elapsedMs = performance.now() - startedAt;
|
|
836
|
-
if (isGatewayCutOff(err, elapsedMs)) {
|
|
837
|
-
throw new PipelineExecuteTimeoutError(elapsedMs, { cause: err });
|
|
838
|
-
}
|
|
839
|
-
throw err;
|
|
846
|
+
// The hosted gateway terminates synchronous requests at ~30s. A run that exceeds that comes
|
|
847
|
+
// back as a gateway 503/504 (or a client abort) — translated into a clear, actionable error
|
|
848
|
+
// pointing at start+poll.
|
|
849
|
+
throw executeCutOff(err, sentAt);
|
|
840
850
|
}
|
|
841
851
|
}
|
|
842
852
|
/**
|
|
@@ -858,62 +868,48 @@ export class PipelexApiClient {
|
|
|
858
868
|
* `runMayHaveStarted` is true can be retried unchanged (see
|
|
859
869
|
* `PipelexStartRequestOptions`). A malformed key is a `RequestArgumentError`
|
|
860
870
|
* before anything is sent.
|
|
871
|
+
*
|
|
872
|
+
* The method's tree is prepared before the request as `execute` prepares it. The prepared body
|
|
873
|
+
* is the same for the same tree, so a start retried under its idempotency key sends the body the
|
|
874
|
+
* key was bound to. A start whose hash entries name a blob the API does not hold yet is refused
|
|
875
|
+
* with a `422` `missing_blobs`, which releases its key: send it again unchanged, which asks the
|
|
876
|
+
* blob route and uploads what is missing first.
|
|
861
877
|
*/
|
|
862
878
|
async start(options) {
|
|
879
|
+
return this.startRun(options);
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* `start`, stopped by `signal` as `executeRun` is, `sending` being the tree already prepared.
|
|
883
|
+
*/
|
|
884
|
+
async startRun(options, signal, sending) {
|
|
863
885
|
const idempotencyKey = idempotencyKeyOf(options);
|
|
864
|
-
const
|
|
865
|
-
const
|
|
866
|
-
const hosted = buildHostedRunExtensions(options);
|
|
867
|
-
if (!options.pipe_code &&
|
|
868
|
-
(!options.mthds_contents || options.mthds_contents.length === 0) &&
|
|
869
|
-
!hasBundlePayload(options) &&
|
|
870
|
-
Object.keys(api).length === 0 &&
|
|
871
|
-
Object.keys(hosted).length === 0 &&
|
|
872
|
-
Object.keys(extensions).length === 0) {
|
|
873
|
-
throw new RequestArgumentError("Either pipe_code, mthds_contents, a method bundle (files/bundle_b64), a method_ref, a hosted method_id or a server-specific extension arg (extra) must be provided to start().");
|
|
874
|
-
}
|
|
875
|
-
assertRunSourcesExclusive(options);
|
|
876
|
-
assertMethodRefPairsWithNothing(options);
|
|
877
|
-
assertLinkageMethodIdIsBare(options);
|
|
878
|
-
assertResourcesAccompanyABundle(options);
|
|
879
|
-
// No await when there are none: a request without them leaves on the tick it always did.
|
|
880
|
-
const resources = options.resources == null ? undefined : await encodeBundleResources(options.resources);
|
|
881
|
-
// `?? undefined` so JSON.stringify drops absent fields from the wire body.
|
|
882
|
-
const request = {
|
|
883
|
-
pipe_code: options.pipe_code ?? undefined,
|
|
884
|
-
mthds_contents: options.mthds_contents ?? undefined,
|
|
885
|
-
inputs: options.inputs ?? undefined,
|
|
886
|
-
output_name: options.output_name ?? undefined,
|
|
887
|
-
output_multiplicity: options.output_multiplicity ?? undefined,
|
|
888
|
-
dynamic_output_concept_ref: options.dynamic_output_concept_ref ?? undefined,
|
|
889
|
-
files: nonEmptyFiles(options.files),
|
|
890
|
-
bundle_b64: nonEmptyString(options.bundle_b64),
|
|
891
|
-
resources,
|
|
892
|
-
...api,
|
|
893
|
-
...hosted,
|
|
894
|
-
...extensions,
|
|
895
|
-
};
|
|
886
|
+
const parts = this.runParts("start", options);
|
|
887
|
+
const source = sending?.source ?? (await this.treeToSend("start", parts.forms, REQUEST_SENDING, signal));
|
|
896
888
|
const url = this.url("start");
|
|
897
|
-
|
|
898
|
-
//
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
889
|
+
const res = await this.postMethodRequest(source, "start", REQUEST_SENDING,
|
|
890
|
+
// A start under a key is bound to the refusal the platform stores, so it is not sent again.
|
|
891
|
+
{ signal, retryTooLarge: idempotencyKey === undefined }, async (files) => {
|
|
892
|
+
const request = runRequestBody("start", options, files, parts);
|
|
893
|
+
// `start` returns a 202 fast, so the poll timeout normally fits — with exceptions that get
|
|
894
|
+
// the blocking-execute ceiling instead. A method's tree sent inline can make the request
|
|
895
|
+
// *body* multi-megabyte when the API keeps no blob store, and so can a bundle riding
|
|
896
|
+
// `extra` as a runner's `mthds_contents` or `bundle_b64`, as the CLI sends one; the whole
|
|
897
|
+
// upload is charged against this budget — the same payload must not time out on the
|
|
898
|
+
// durable path yet succeed on the fallback. And a `method_ref` start makes the server FETCH
|
|
899
|
+
// the package before the ack (provenance rides the 202), whose clone timeout runs well past
|
|
900
|
+
// 30s on a cold cache — aborting it here would surface as `ApiUnreachableError`, blaming
|
|
901
|
+
// the network for a healthy server that is still cloning.
|
|
902
|
+
const needsLongCeiling = files !== undefined ||
|
|
903
|
+
parts.api.method_ref !== undefined ||
|
|
904
|
+
carriesExtraBundle(parts.extensions);
|
|
905
|
+
throwIfAborted(signal);
|
|
906
|
+
return this.requestRaw("POST", url, {
|
|
907
|
+
body: request,
|
|
908
|
+
timeoutMs: needsLongCeiling ? this.requestTimeoutMs : POLL_REQUEST_TIMEOUT_MS,
|
|
909
|
+
idempotencyKey,
|
|
910
|
+
...STARTS_RUN,
|
|
911
|
+
});
|
|
912
|
+
}, sending?.initial);
|
|
917
913
|
// A bare runner with no run store 404s here just as it does on the result
|
|
918
914
|
// routes — surface the same clear `RunLifecycleUnavailableError` (and let
|
|
919
915
|
// `startAndWaitForResult` fall back to the blocking `execute`).
|
|
@@ -931,30 +927,38 @@ export class PipelexApiClient {
|
|
|
931
927
|
return ack;
|
|
932
928
|
}
|
|
933
929
|
/**
|
|
934
|
-
* Parse, validate, and dry-run
|
|
930
|
+
* Parse, validate, and dry-run a method — `POST /v1/validate`.
|
|
935
931
|
*
|
|
936
932
|
* `/validate` is a diagnostic endpoint: every produced verdict rides a **200**,
|
|
937
933
|
* discriminated on `is_valid`. This returns the `PipelexValidationResult` union
|
|
938
934
|
* verbatim — `is_valid: true` ⇒ the typed `PipelexValidationReport` (structural
|
|
939
935
|
* artifacts), `is_valid: false` ⇒ a `PipelexInvalidReport` (`validation_errors[]`).
|
|
940
|
-
* An invalid
|
|
941
|
-
* *no-verdict* condition (a malformed request,
|
|
942
|
-
*
|
|
943
|
-
*
|
|
944
|
-
* `source`
|
|
945
|
-
*
|
|
946
|
-
*
|
|
947
|
-
*
|
|
948
|
-
* - **
|
|
936
|
+
* An invalid method is NOT thrown — the caller pattern-matches `is_valid`. Only a
|
|
937
|
+
* *no-verdict* condition (a malformed request, a tree the route refuses, auth, a
|
|
938
|
+
* server fault) is non-2xx and surfaces as `ApiResponseError`.
|
|
939
|
+
*
|
|
940
|
+
* `source` names WHAT is validated, in exactly one form — the strict tooling XOR
|
|
941
|
+
* (the routes are stateless, so there is no linkage exception; a second selector
|
|
942
|
+
* is a request-shape `422`, refused here first with a `RequestArgumentError`):
|
|
943
|
+
*
|
|
944
|
+
* - **the method's tree**, in one of the four forms of `MethodTreeForms`: a
|
|
945
|
+
* `directory` the `mthds` package's lister lists under the standard's
|
|
946
|
+
* membership rule (Node only), `contents` by tree path, a listed `tree`, or the
|
|
947
|
+
* protocol's `files` map, sent as given. A `directory`, `contents` or `tree` is
|
|
948
|
+
* sent as the `files` map, inline or by hash, exactly as a run sends it (see
|
|
949
|
+
* `PipelexRunOptions`). Every `.mthds` file of the tree is judged, named by its
|
|
950
|
+
* tree path in the diagnostics' `source`, and every other file is one a step may
|
|
951
|
+
* read, such as a `PipeDocGen` step's template file, whose template check's
|
|
952
|
+
* findings ride the valid arm's `warnings`;
|
|
949
953
|
* - **`{ method_ref }`** — a published method's address, resolved by the
|
|
950
954
|
* server (pipelex-api >= 0.21.0) through the same fetch path as a
|
|
951
|
-
* `method_ref` run, the package's real
|
|
955
|
+
* `method_ref` run, the package's real tree paths feeding the diagnostics'
|
|
952
956
|
* source labels;
|
|
953
957
|
* - **`{ method_id }`** — a stored method's catalog id, hosted-only: the
|
|
954
|
-
* platform resolves the version it names and
|
|
955
|
-
*
|
|
956
|
-
*
|
|
957
|
-
*
|
|
958
|
+
* platform resolves the version it names and sends that version's tree to the
|
|
959
|
+
* runner (a bare runner rejects the request as carrying no source it
|
|
960
|
+
* understands). A bare `mt_…` validates the latest published version,
|
|
961
|
+
* `mt_…@<n>` version `n` and `mt_…@draft` the draft.
|
|
958
962
|
*
|
|
959
963
|
* A selector-resolution failure (fetch failure, no package at the address, an
|
|
960
964
|
* unknown or foreign-org id, a bare id of a method never published, `409`
|
|
@@ -963,40 +967,23 @@ export class PipelexApiClient {
|
|
|
963
967
|
* `ApiResponseError` — never an `is_valid: false` verdict, which is reserved
|
|
964
968
|
* for actual MTHDS content.
|
|
965
969
|
*
|
|
966
|
-
* `
|
|
967
|
-
*
|
|
968
|
-
*
|
|
969
|
-
*
|
|
970
|
-
*
|
|
971
|
-
*
|
|
972
|
-
*
|
|
973
|
-
*
|
|
974
|
-
*
|
|
975
|
-
* `
|
|
976
|
-
*
|
|
977
|
-
*
|
|
978
|
-
*
|
|
979
|
-
*
|
|
980
|
-
*
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
* top-level field to the valid arm — `input_form` and `output_form`. Unlike `render`,
|
|
984
|
-
* this client sends `views` ONLY when the caller asks: the point of an opt-in view
|
|
985
|
-
* is that the default response stays byte-identical, and the highest-frequency
|
|
986
|
-
* consumers (hook pipelines, CI gates, agent loops) never pay for bytes they
|
|
987
|
-
* discard. `pipelex-api` gates `input_form` on this token as of 0.18.0; a 0.17.0
|
|
988
|
-
* runner resolved no token (the key is silently ignored, never a 422) and emitted
|
|
989
|
-
* the field regardless — which is why `PipelexValidationReport.input_form` is typed
|
|
990
|
-
* optional rather than required.
|
|
991
|
-
*
|
|
992
|
-
* `options.resources` sends the files the bundle's steps read, such as a
|
|
993
|
-
* `PipeDocGen` step's `template_file`, keyed by the path the step names, each as
|
|
994
|
-
* its bytes (see {@link ValidateOptions}). A step whose template file was not sent
|
|
995
|
-
* is an `is_valid: false` verdict saying so, and a template check's findings in an
|
|
996
|
-
* office template file ride the valid arm's `warnings`. A key no step names, or
|
|
997
|
-
* files over the server's limits, are no-verdict refusals: a `422`
|
|
998
|
-
* `BundleResourceNotNamedError`, a `413` `PayloadTooLarge` or
|
|
999
|
-
* `BundleResourcesTooLargeError`.
|
|
970
|
+
* `allowSignatures` tolerates unimplemented pipe signatures, as the protocol's
|
|
971
|
+
* `validate(source, allowSignatures)` does: they surface as `pending_signatures`
|
|
972
|
+
* and `is_runnable`, never as an error.
|
|
973
|
+
*
|
|
974
|
+
* `options.render` is the Pipelex-API presentation hint — a list of view-format
|
|
975
|
+
* tokens. This client always asks for Markdown so both valid results and produced
|
|
976
|
+
* validation-error verdicts carry `rendered_markdown`; callers may add more tokens.
|
|
977
|
+
* Unknown tokens are server-side lenient-ignored (never a 422).
|
|
978
|
+
*
|
|
979
|
+
* `options.views` is the sibling opt-in for *structured* views where `render`
|
|
980
|
+
* carries *rendered text*. The two lists are independent, each resolving its own
|
|
981
|
+
* tokens against its own supported set, and each supported token adds a same-named
|
|
982
|
+
* top-level field to the valid arm — `input_form` and `output_form`. Unlike
|
|
983
|
+
* `render`, this client sends `views` ONLY when the caller asks: the point of an
|
|
984
|
+
* opt-in view is that the default response stays byte-identical, and the
|
|
985
|
+
* highest-frequency consumers (hook pipelines, CI gates, agent loops) never pay for
|
|
986
|
+
* bytes they discard.
|
|
1000
987
|
*
|
|
1001
988
|
* A `{ method_ref }` source makes the server clone a repository before it
|
|
1002
989
|
* validates anything, and it needs no special budget here: this route already
|
|
@@ -1006,85 +993,52 @@ export class PipelexApiClient {
|
|
|
1006
993
|
* exists to RAISE routes whose default is the ~30s poll ceiling; applying it here
|
|
1007
994
|
* would lower this one.
|
|
1008
995
|
*/
|
|
1009
|
-
async validate(source, allowSignatures = false,
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
if (Array.isArray(source)) {
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
throw new RequestArgumentError("mthds_sources labels inline mthds_contents; a method_ref / method_id validation gets " +
|
|
1036
|
-
"its source labels from the package's (or the stored method's) real file names.");
|
|
1037
|
-
}
|
|
1038
|
-
if (methodRef !== undefined && hasBundleResources(options.resources)) {
|
|
1039
|
-
throw new RequestArgumentError("resources accompany inline mthds_contents; a method_ref's files are the package's own.");
|
|
1040
|
-
}
|
|
996
|
+
async validate(source, allowSignatures = false, options = {}) {
|
|
997
|
+
// The illegal shapes are partly compile errors for typed callers; the runtime checks back them
|
|
998
|
+
// for untyped (JS) callers — a typed `RequestArgumentError`, never a native TypeError off a
|
|
999
|
+
// null source — mirroring the server's strict tooling XOR instead of silently picking.
|
|
1000
|
+
if (source === null || typeof source !== "object" || Array.isArray(source)) {
|
|
1001
|
+
throw new RequestArgumentError("validate() takes the method as an object naming exactly one of directory, contents, " +
|
|
1002
|
+
"tree, files, method_ref or method_id.");
|
|
1003
|
+
}
|
|
1004
|
+
const forms = protocolTreeForms("validate", source);
|
|
1005
|
+
const methodRef = nonEmptyString(source.method_ref);
|
|
1006
|
+
const methodId = nonEmptyString(source.method_id);
|
|
1007
|
+
const given = treeFormsGiven(forms);
|
|
1008
|
+
if (methodRef !== undefined)
|
|
1009
|
+
given.push("method_ref");
|
|
1010
|
+
if (methodId !== undefined)
|
|
1011
|
+
given.push("method_id");
|
|
1012
|
+
if (given.length !== 1) {
|
|
1013
|
+
throw new RequestArgumentError("validate() takes exactly one of the method's tree (directory, contents, tree or files), " +
|
|
1014
|
+
`method_ref or method_id; got ${given.length === 0 ? "none" : given.join(" and ")}.`);
|
|
1015
|
+
}
|
|
1016
|
+
const signal = options.signal;
|
|
1017
|
+
const sent = await this.treeToSend("validate", forms, REQUEST_SENDING, signal);
|
|
1018
|
+
const res = await this.postMethodRequest(sent, "validate", REQUEST_SENDING, { signal }, (files) => {
|
|
1019
|
+
const body = { allow_signatures: allowSignatures };
|
|
1020
|
+
if (files != null)
|
|
1021
|
+
body.files = files;
|
|
1041
1022
|
if (methodRef !== undefined)
|
|
1042
1023
|
body.method_ref = methodRef;
|
|
1043
1024
|
if (methodId !== undefined)
|
|
1044
1025
|
body.method_id = methodId;
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
const res = await this.requestRaw("POST", this.url("validate"), {
|
|
1056
|
-
body,
|
|
1057
|
-
timeoutMs: options.timeoutMs,
|
|
1058
|
-
signal: options.signal,
|
|
1026
|
+
body.render = withValidateMarkdownRender(options.render);
|
|
1027
|
+
if (options.views !== undefined) {
|
|
1028
|
+
body.views = options.views;
|
|
1029
|
+
}
|
|
1030
|
+
throwIfAborted(signal);
|
|
1031
|
+
return this.requestRaw("POST", this.url("validate"), {
|
|
1032
|
+
body,
|
|
1033
|
+
timeoutMs: options.timeoutMs,
|
|
1034
|
+
signal,
|
|
1035
|
+
});
|
|
1059
1036
|
});
|
|
1060
1037
|
if (res.status < 200 || res.status >= 300) {
|
|
1061
1038
|
this.throwApiResponseError("POST", "validate", res);
|
|
1062
1039
|
}
|
|
1063
1040
|
return this.readObjectAnswer("POST", "validate", res);
|
|
1064
1041
|
}
|
|
1065
|
-
/**
|
|
1066
|
-
* Validate paired MTHDS files while preserving URI attribution for diagnostics.
|
|
1067
|
-
*
|
|
1068
|
-
* This adapter intentionally keeps the low-level `validate(...)` payload shape
|
|
1069
|
-
* intact for existing consumers. When any file has a URI, every content gets a
|
|
1070
|
-
* parallel source label; inline labels are deterministic so the server never
|
|
1071
|
-
* sees a length-mismatched `mthds_sources` array.
|
|
1072
|
-
*/
|
|
1073
|
-
async validateFiles(files, options = {}) {
|
|
1074
|
-
if (files.length === 0) {
|
|
1075
|
-
throw new RequestArgumentError("At least one MTHDS file must be provided to validateFiles().");
|
|
1076
|
-
}
|
|
1077
|
-
const mthdsContents = files.map((file) => file.content);
|
|
1078
|
-
const hasAnyUri = files.some((file) => file.uri !== undefined);
|
|
1079
|
-
const mthdsSources = hasAnyUri
|
|
1080
|
-
? files.map((file, index) => file.uri ?? `inline://file-${index + 1}.mthds`)
|
|
1081
|
-
: undefined;
|
|
1082
|
-
return this.validate(mthdsContents, options.allowSignatures ?? false, mthdsSources, options.render, options.views, {
|
|
1083
|
-
resources: options.resources,
|
|
1084
|
-
timeoutMs: options.timeoutMs,
|
|
1085
|
-
signal: options.signal,
|
|
1086
|
-
});
|
|
1087
|
-
}
|
|
1088
1042
|
// ── Tools extensions (Pipelex API — `/v1/lint`, `/v1/format`) ─────────
|
|
1089
1043
|
//
|
|
1090
1044
|
// NOT REACHABLE ON ANY HOSTED ENVIRONMENT. Both are served by any `pipelex-api`
|
|
@@ -1155,11 +1109,10 @@ export class PipelexApiClient {
|
|
|
1155
1109
|
* never on a message.
|
|
1156
1110
|
*
|
|
1157
1111
|
* All of these are static and inference-free, so they default to the management-call
|
|
1158
|
-
* timeout. The internal
|
|
1112
|
+
* timeout. The internal raise is `crateRequestTimeoutMs`'s: the fetch-sized budget a
|
|
1159
1113
|
* crate route gets when its closure is a `method_ref`, because the server may clone a
|
|
1160
|
-
* repository before it answers
|
|
1161
|
-
*
|
|
1162
|
-
* what the request costs, not a caller-facing parameter.
|
|
1114
|
+
* repository before it answers. It is matched to what the request costs, not a
|
|
1115
|
+
* caller-facing parameter.
|
|
1163
1116
|
*
|
|
1164
1117
|
* **These routes deliberately expose no `timeoutMs` / `signal`, and that is not an
|
|
1165
1118
|
* oversight to be "fixed" per route.** Automated reviewers have proposed adding them
|
|
@@ -1170,11 +1123,11 @@ export class PipelexApiClient {
|
|
|
1170
1123
|
* ceiling and takes caller transport options; every extension route rides the
|
|
1171
1124
|
* static core. Giving one static route an override while its siblings lack one is
|
|
1172
1125
|
* the inconsistency.
|
|
1173
|
-
* 2. The input is **bounded server-side** — the
|
|
1174
|
-
*
|
|
1175
|
-
*
|
|
1176
|
-
*
|
|
1177
|
-
*
|
|
1126
|
+
* 2. The input is **bounded server-side** — the deployment caps a request's tree by its
|
|
1127
|
+
* count of files, a size per file, a total and the bytes it carries inline, all
|
|
1128
|
+
* announced by its `/v1/version` handshake — and none of these routes runs inference.
|
|
1129
|
+
* The upload those limits allow is what `crateRequestTimeoutMs` budgets for, not the
|
|
1130
|
+
* server's work.
|
|
1178
1131
|
* 3. On the hosted path an override would be **inert**: the gateway caps responses at
|
|
1179
1132
|
* ~30s (see `POLL_REQUEST_TIMEOUT_MS` above), so raising `timeoutMs` would still be
|
|
1180
1133
|
* cut off upstream, just with a less honest error.
|
|
@@ -1182,6 +1135,12 @@ export class PipelexApiClient {
|
|
|
1182
1135
|
* If a static route ever genuinely needs longer, give the WHOLE static family one
|
|
1183
1136
|
* uniform transport-options parameter in a single pass — do not bolt it onto one
|
|
1184
1137
|
* route.
|
|
1138
|
+
*
|
|
1139
|
+
* The one exception is the crate family's `signal` (`CrateRouteOptions`), given to all three
|
|
1140
|
+
* crate routes at once: a `directory`, `tree` or `contents` can cost the listing, the version
|
|
1141
|
+
* handshake, the blob route and uploads before the request, client-side work a caller such as
|
|
1142
|
+
* `prepareInputs` must be able to stop. It is read between those steps and stops nothing in flight, so it is no
|
|
1143
|
+
* transport option and the reasons above still hold.
|
|
1185
1144
|
*/
|
|
1186
1145
|
async requestExtension(endpoint, body, options = {}) {
|
|
1187
1146
|
const res = await this.requestRaw("POST", this.url(endpoint), {
|
|
@@ -1242,8 +1201,13 @@ export class PipelexApiClient {
|
|
|
1242
1201
|
* The handshake for feature detection (hosted extensions or not).
|
|
1243
1202
|
*/
|
|
1244
1203
|
async version() {
|
|
1204
|
+
return this.requestVersion();
|
|
1205
|
+
}
|
|
1206
|
+
/** `GET /v1/version`, as `version` asks it, stopped by `signal`. */
|
|
1207
|
+
async requestVersion(signal) {
|
|
1245
1208
|
const res = await this.requestRaw("GET", this.url("version"), {
|
|
1246
1209
|
timeoutMs: POLL_REQUEST_TIMEOUT_MS,
|
|
1210
|
+
signal,
|
|
1247
1211
|
});
|
|
1248
1212
|
if (res.status < 200 || res.status >= 300) {
|
|
1249
1213
|
this.throwApiResponseError("GET", "version", res);
|
|
@@ -1270,8 +1234,10 @@ export class PipelexApiClient {
|
|
|
1270
1234
|
// `docs/crate-routes.md`).
|
|
1271
1235
|
//
|
|
1272
1236
|
// All three are STATIC routes (no dry-run sweep), so like every static sibling they
|
|
1273
|
-
// take no `timeoutMs
|
|
1274
|
-
//
|
|
1237
|
+
// take no `timeoutMs` — see the policy note on `requestExtension` before adding one
|
|
1238
|
+
// here. Their `signal` (`CrateRouteOptions`) is that note's one exception: it is read
|
|
1239
|
+
// between the steps that prepare a `directory`, `tree` or `contents`, and stops nothing in
|
|
1240
|
+
// flight.
|
|
1275
1241
|
/**
|
|
1276
1242
|
* Resolve a closure into its normalized library crate — `POST /v1/resolve`.
|
|
1277
1243
|
*
|
|
@@ -1290,15 +1256,23 @@ export class PipelexApiClient {
|
|
|
1290
1256
|
* (fetch failure, no package at the address, an unknown or foreign-org id) is
|
|
1291
1257
|
* non-2xx too — never an `is_valid: false` verdict.
|
|
1292
1258
|
*
|
|
1293
|
-
* The closure arrives in exactly one of three forms:
|
|
1294
|
-
*
|
|
1295
|
-
*
|
|
1296
|
-
*
|
|
1259
|
+
* The closure arrives in exactly one of three forms: the method's tree, as the
|
|
1260
|
+
* protocol's `files` map or as a `directory`, `tree` or `contents` the client sends
|
|
1261
|
+
* inline when small and by hash otherwise (see {@link CrateRequestBase}); an
|
|
1262
|
+
* address-form `method_ref` (server-resolved, pipelex-api >= 0.21.0; the registry form
|
|
1263
|
+
* stays a `501`); or a hosted `method_id` (platform-resolved — see
|
|
1264
|
+
* {@link PipelexHostedToolingExtensions}). An empty map or string is absent.
|
|
1265
|
+
*
|
|
1266
|
+
* `options.signal` stops the call between the steps a `directory`, `tree` or `contents`
|
|
1267
|
+
* costs before the request — the listing, the version handshake, the blob route, each
|
|
1268
|
+
* read and upload of a missing blob — and before the request itself, throwing the
|
|
1269
|
+
* abort; a request already sent runs to its end. A blob that fails to upload throws
|
|
1270
|
+
* `MethodFileTransferError`, and a tree this client refuses to send (no file, two
|
|
1271
|
+
* forms, a directory it cannot list, a `contents` value that is neither text nor
|
|
1272
|
+
* bytes) `RequestArgumentError`, each before the route is asked.
|
|
1297
1273
|
*/
|
|
1298
|
-
async resolve(request) {
|
|
1299
|
-
return this.
|
|
1300
|
-
timeoutMs: crateRequestTimeoutMs(request),
|
|
1301
|
-
});
|
|
1274
|
+
async resolve(request, options = {}) {
|
|
1275
|
+
return this.crateRequest("resolve", "resolve", request, options);
|
|
1302
1276
|
}
|
|
1303
1277
|
/**
|
|
1304
1278
|
* Project a closure's crate into stamped typed artifacts — `POST /v1/codegen`.
|
|
@@ -1315,10 +1289,8 @@ export class PipelexApiClient {
|
|
|
1315
1289
|
* unknown `kind`/`target`, a `pipe_ref` on the concept-set-wide `types` kind, or
|
|
1316
1290
|
* a malformed selector is a 422; a registry-form `method_ref` is a `501`.
|
|
1317
1291
|
*/
|
|
1318
|
-
async codegen(request) {
|
|
1319
|
-
return this.
|
|
1320
|
-
timeoutMs: crateRequestTimeoutMs(request),
|
|
1321
|
-
});
|
|
1292
|
+
async codegen(request, options = {}) {
|
|
1293
|
+
return this.crateRequest("codegen", "codegen", request, options);
|
|
1322
1294
|
}
|
|
1323
1295
|
/**
|
|
1324
1296
|
* Read a method's three I/O artifacts — `POST /v1/pipe-io`.
|
|
@@ -1331,19 +1303,20 @@ export class PipelexApiClient {
|
|
|
1331
1303
|
* dry-runs every pipe; a caller that shows a method, prepares its inputs or generates
|
|
1332
1304
|
* types for it reads this, and one that needs the dry-run verdict stays on `validate`.
|
|
1333
1305
|
*
|
|
1334
|
-
* Selection: the request's qualified `pipe_ref`, else
|
|
1335
|
-
* `
|
|
1336
|
-
*
|
|
1337
|
-
* `include_files: true` echoes the
|
|
1306
|
+
* Selection: the request's qualified `pipe_ref`, else the `main_pipe` of the
|
|
1307
|
+
* `METHODS.toml` at the tree's root, sent in the tree or fetched with a package, else the
|
|
1308
|
+
* closure's single `main_pipe` declaration. `all_pipes: true` describes every pipe
|
|
1309
|
+
* instead, and never refuses for want of an entry pipe. `include_files: true` echoes the
|
|
1310
|
+
* method's own `.mthds` bundles as `files`, a map from tree path to each bundle's text,
|
|
1311
|
+
* in the shape of the request's `files` map.
|
|
1338
1312
|
*
|
|
1339
|
-
*
|
|
1340
|
-
*
|
|
1341
|
-
*
|
|
1342
|
-
* take it too (see {@link CrateRequestBase}).
|
|
1313
|
+
* The files the closure's steps read, such as a `PipeDocGen` step's `template_file`, are
|
|
1314
|
+
* files of the method's tree like any other; a step whose template file is not in the tree
|
|
1315
|
+
* makes the closure invalid (see {@link CrateRequestBase}).
|
|
1343
1316
|
*
|
|
1344
|
-
* Same 200-verdict discipline
|
|
1345
|
-
* (the request
|
|
1346
|
-
*
|
|
1317
|
+
* Same 200-verdict discipline, same three-form closure selector and same `options` as
|
|
1318
|
+
* {@link resolve} (the request's members are posted as given, its `directory`, `tree` or
|
|
1319
|
+
* `contents` as the `files` map, and the selector XOR is the server's to enforce).
|
|
1347
1320
|
* Only a no-verdict condition throws `ApiResponseError`: a malformed selector and an
|
|
1348
1321
|
* over-limit file are `ValidationError` `422`s; a selection the route refuses is a
|
|
1349
1322
|
* `422` whose `errorType` is `EntryPipeNotFoundError` (an unknown `pipe_ref`, no entry
|
|
@@ -1351,16 +1324,70 @@ export class PipelexApiClient {
|
|
|
1351
1324
|
* candidates in its `serverMessage`; a registry-form `method_ref` is a `501`; a pipe
|
|
1352
1325
|
* whose artifacts cannot be derived is a `500`.
|
|
1353
1326
|
*
|
|
1354
|
-
* A `method_ref` gets the fetch-sized budget, as on the other crate routes
|
|
1355
|
-
*
|
|
1356
|
-
*
|
|
1357
|
-
*
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1327
|
+
* A `method_ref` gets the fetch-sized budget, as on the other crate routes. On the
|
|
1328
|
+
* hosted API the gateway caps a request at about 30 seconds whatever the client allows,
|
|
1329
|
+
* so a cold `method_ref` clone can answer a `502` that a retry clears once the runner
|
|
1330
|
+
* has cached the clone.
|
|
1331
|
+
*/
|
|
1332
|
+
async pipeIo(request, options = {}) {
|
|
1333
|
+
return this.crateRequest("pipe-io", "pipeIo", request, options);
|
|
1334
|
+
}
|
|
1335
|
+
/**
|
|
1336
|
+
* Post a crate request: its members as given, and its tree as the `files` map a save would
|
|
1337
|
+
* send, each key left off when there is nothing to send. An empty map or string is absent, as in
|
|
1338
|
+
* the Python SDK: `files: {}` or `null`, `contents: {}`, `directory: ""` and `dependencies: {}`
|
|
1339
|
+
* are not sent, so an empty one never reaches the wire as a second selector; a `directory` that
|
|
1340
|
+
* is not a string is refused, as on a run. A `directory`, `tree` or `contents` is prepared
|
|
1341
|
+
* through the path every request and save sends a tree by (`postMethodRequest`), inline when
|
|
1342
|
+
* small and by hash otherwise, uploading the files the API lacks first; the blob route is asked
|
|
1343
|
+
* under the request's `read_scope`, the scope its hash entries are read in. `signal` is read
|
|
1344
|
+
* before each step and right before the request (see `TreeSendOptions`). The request gets the
|
|
1345
|
+
* budget `crateRequestTimeoutMs` gives it for the `files` it carries.
|
|
1346
|
+
*/
|
|
1347
|
+
async crateRequest(endpoint, call, request, options) {
|
|
1348
|
+
const signal = options.signal;
|
|
1349
|
+
throwIfAborted(signal);
|
|
1350
|
+
if (typeof request !== "object" || request === null) {
|
|
1351
|
+
return this.requestExtension(endpoint, request);
|
|
1352
|
+
}
|
|
1353
|
+
const { directory, contents, tree, files, dependencies, ...rest } = request;
|
|
1354
|
+
const source = await this.treeToSend(call, {
|
|
1355
|
+
directory: typeof directory === "string" ? nonEmptyString(directory) : directory,
|
|
1356
|
+
contents: presentMap(contents),
|
|
1357
|
+
tree: tree ?? undefined,
|
|
1358
|
+
files: presentMap(files),
|
|
1359
|
+
}, REQUEST_SENDING, signal);
|
|
1360
|
+
const packages = presentMap(dependencies);
|
|
1361
|
+
const res = await this.postMethodRequest(source, call, REQUEST_SENDING, { signal, readScope: nonEmptyString(request.read_scope) }, (sent) => {
|
|
1362
|
+
const body = { ...rest };
|
|
1363
|
+
if (sent != null)
|
|
1364
|
+
body.files = sent;
|
|
1365
|
+
if (packages !== undefined)
|
|
1366
|
+
body.dependencies = packages;
|
|
1367
|
+
throwIfAborted(signal);
|
|
1368
|
+
return this.requestRaw("POST", this.url(endpoint), {
|
|
1369
|
+
body,
|
|
1370
|
+
timeoutMs: this.crateRequestTimeoutMs(request, sent),
|
|
1371
|
+
});
|
|
1363
1372
|
});
|
|
1373
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1374
|
+
this.throwApiResponseError("POST", endpoint, res);
|
|
1375
|
+
}
|
|
1376
|
+
return this.readObjectAnswer("POST", endpoint, res);
|
|
1377
|
+
}
|
|
1378
|
+
/**
|
|
1379
|
+
* The request budget for a crate route's closure (`resolve`, `codegen`, `pipeIo`): the
|
|
1380
|
+
* static-route default, unless the closure is a `method_ref`, which may make the server clone a
|
|
1381
|
+
* repository first and gets the fetch-sized budget, or its `files` carry more inline than
|
|
1382
|
+
* `INLINE_TREE_MAX_BYTES`, which gets the client's request ceiling (`requestTimeoutMs`): a tree
|
|
1383
|
+
* sent to an API keeping no blob store travels inline whatever its weight, and so does a `files`
|
|
1384
|
+
* map given as is, and the upload of the body is charged against the budget. A tree sent by
|
|
1385
|
+
* hash keeps the default, its files uploaded before the request under the client's own ceiling.
|
|
1386
|
+
*/
|
|
1387
|
+
crateRequestTimeoutMs(request, files) {
|
|
1388
|
+
if (nonEmptyString(request.method_ref) !== undefined)
|
|
1389
|
+
return METHOD_REF_FETCH_TIMEOUT_MS;
|
|
1390
|
+
return carriesLargeInlineTree(files) ? this.requestTimeoutMs : POLL_REQUEST_TIMEOUT_MS;
|
|
1364
1391
|
}
|
|
1365
1392
|
// ── Hosted extension: durable run lifecycle (NOT part of the protocol) ──
|
|
1366
1393
|
/**
|
|
@@ -1515,18 +1542,25 @@ export class PipelexApiClient {
|
|
|
1515
1542
|
* gzip stream, which `requestRaw` reports as an `ApiUnreachableError` carrying
|
|
1516
1543
|
* the decompressor's code (see `isUndecodableBody`).
|
|
1517
1544
|
*
|
|
1545
|
+
* `signal` stops the handshake: its abort is thrown as is and nothing is cached.
|
|
1546
|
+
*
|
|
1518
1547
|
* @throws {ApiUnreachableError} The handshake got no answer. Nothing is cached,
|
|
1519
1548
|
* so the next call asks again, and no start is sent to a host that did not
|
|
1520
1549
|
* answer: sending it would wait a second time for the same silence.
|
|
1521
1550
|
*/
|
|
1522
|
-
async supportsRunLifecycle() {
|
|
1551
|
+
async supportsRunLifecycle(signal) {
|
|
1523
1552
|
if (this.lifecycleAvailable === undefined) {
|
|
1524
1553
|
try {
|
|
1525
|
-
const
|
|
1554
|
+
const askedAt = Date.now();
|
|
1555
|
+
const info = await this.requestVersion(signal);
|
|
1556
|
+
this.versionCache = { info, askedAt };
|
|
1526
1557
|
const impl = info.implementation;
|
|
1527
1558
|
this.lifecycleAvailable = !(typeof impl === "string" && impl === BARE_RUNNER_IMPLEMENTATION);
|
|
1528
1559
|
}
|
|
1529
1560
|
catch (error) {
|
|
1561
|
+
// The caller's abort is no answer: it propagates, and nothing is cached.
|
|
1562
|
+
if (signal?.aborted === true)
|
|
1563
|
+
throw signal.reason;
|
|
1530
1564
|
// No answer at all propagates. A body that arrived and could not be decoded is an answer,
|
|
1531
1565
|
// and so is every other failure here: a non-2xx status, or a body that is no version.
|
|
1532
1566
|
if (error instanceof ApiUnreachableError && !isUndecodableBody(error))
|
|
@@ -1551,9 +1585,15 @@ export class PipelexApiClient {
|
|
|
1551
1585
|
* run, so the caller knows until then that none exists (see
|
|
1552
1586
|
* `StartAndWaitForResultOptions`).
|
|
1553
1587
|
*
|
|
1554
|
-
* `pollOptions.signal`
|
|
1555
|
-
*
|
|
1556
|
-
*
|
|
1588
|
+
* `pollOptions.signal` also stops everything before the run is created: the version handshake,
|
|
1589
|
+
* the listing of a `directory`, and the reads, the blob route and the uploads of a tree sent by
|
|
1590
|
+
* hash, each of them stopped in flight. It is read again right before the start and before the
|
|
1591
|
+
* blocking execute, so a caller that aborts before then gets its abort and no run is created.
|
|
1592
|
+
* Once one of them is sent, the abort stops only the wait.
|
|
1593
|
+
*
|
|
1594
|
+
* The method's tree is prepared once, before the start: when a runner turns out to lack the
|
|
1595
|
+
* durable routes, the blocking execute sends the same `files` map, its blobs already uploaded,
|
|
1596
|
+
* rather than listing, reading and hashing the tree again.
|
|
1557
1597
|
*
|
|
1558
1598
|
* An error from the start, or from the blocking execute, says whether that request may still
|
|
1559
1599
|
* have started a run whose id never reached the caller (`runMayHaveStarted`); when it may have,
|
|
@@ -1574,10 +1614,12 @@ export class PipelexApiClient {
|
|
|
1574
1614
|
// Before the handshake too: a key the start would refuse is the caller's to fix, whatever runs.
|
|
1575
1615
|
idempotencyKeyOf(options);
|
|
1576
1616
|
const { onStarted, onStarting, ...waitOptions } = pollOptions ?? {};
|
|
1577
|
-
const
|
|
1617
|
+
const signal = waitOptions.signal;
|
|
1618
|
+
const hosted = await this.supportsRunLifecycle(signal);
|
|
1578
1619
|
// The handshake may have outlasted the caller's patience: an abort that landed during it
|
|
1579
1620
|
// creates no run.
|
|
1580
|
-
throwIfAborted(
|
|
1621
|
+
throwIfAborted(signal);
|
|
1622
|
+
const sending = await this.prepareRunTree(hosted ? "start" : "execute", options, signal);
|
|
1581
1623
|
if (hosted) {
|
|
1582
1624
|
// A runner can look hosted yet lack the durable routes — `implementation`
|
|
1583
1625
|
// is an extension field, so a compliant bare runner that omits it is
|
|
@@ -1587,22 +1629,38 @@ export class PipelexApiClient {
|
|
|
1587
1629
|
// the durable attempt.
|
|
1588
1630
|
let ack;
|
|
1589
1631
|
try {
|
|
1632
|
+
throwIfAborted(signal);
|
|
1590
1633
|
onStarting?.();
|
|
1591
|
-
ack = await this.
|
|
1634
|
+
ack = await this.startRun(options, signal, sending);
|
|
1592
1635
|
}
|
|
1593
1636
|
catch (err) {
|
|
1594
1637
|
if (!(err instanceof RunLifecycleUnavailableError))
|
|
1595
1638
|
throw err;
|
|
1596
1639
|
this.lifecycleAvailable = false;
|
|
1597
|
-
throwIfAborted(
|
|
1640
|
+
throwIfAborted(signal);
|
|
1598
1641
|
onStarting?.();
|
|
1599
|
-
return this.executeBlocking(options);
|
|
1642
|
+
return this.executeBlocking(options, signal, sending);
|
|
1600
1643
|
}
|
|
1601
1644
|
onStarted?.(ack);
|
|
1602
1645
|
return this.waitForResult(ack.pipeline_run_id, waitOptions);
|
|
1603
1646
|
}
|
|
1647
|
+
throwIfAborted(signal);
|
|
1604
1648
|
onStarting?.();
|
|
1605
|
-
return this.executeBlocking(options);
|
|
1649
|
+
return this.executeBlocking(options, signal, sending);
|
|
1650
|
+
}
|
|
1651
|
+
/**
|
|
1652
|
+
* A run's tree prepared once for `startAndWaitForResult`, under `call`'s refusals: the run's
|
|
1653
|
+
* arguments checked, the tree listed or read, and its `files` prepared, uploads included, under
|
|
1654
|
+
* `signal`. A `files` map given as is, or no tree, needs no preparation.
|
|
1655
|
+
*/
|
|
1656
|
+
async prepareRunTree(call, options, signal) {
|
|
1657
|
+
const parts = this.runParts(call, options);
|
|
1658
|
+
const source = await this.treeToSend(call, parts.forms, REQUEST_SENDING, signal);
|
|
1659
|
+
if (source === undefined || !("tree" in source))
|
|
1660
|
+
return { source, initial: undefined };
|
|
1661
|
+
const read = new Map();
|
|
1662
|
+
const prepared = await this.prepareFiles(source, call, REQUEST_SENDING, { signal }, read);
|
|
1663
|
+
return { source, initial: { prepared, read } };
|
|
1606
1664
|
}
|
|
1607
1665
|
// ── Pipelex product surface (hosted management routes) ─────────────────
|
|
1608
1666
|
//
|
|
@@ -1671,10 +1729,13 @@ export class PipelexApiClient {
|
|
|
1671
1729
|
yield* followCursor("listMethods", (cursor) => this.listMethods({ ...query, cursor }));
|
|
1672
1730
|
}
|
|
1673
1731
|
/**
|
|
1674
|
-
* Read one method — `GET /v1/methods/{id}`: its identity, its draft
|
|
1675
|
-
* `input_data`) with the draft's token (`updated_at`) and digest (`draft_digest`),
|
|
1676
|
-
* latest published version's summary (`latest_version`, `latest_published`), whatever
|
|
1677
|
-
* publish state. A method never published reads with both `null`, never as an error.
|
|
1732
|
+
* Read one method with its draft's files — `GET /v1/methods/{id}`: its identity, its draft
|
|
1733
|
+
* (`files`, `input_data`) with the draft's token (`updated_at`) and digest (`draft_digest`),
|
|
1734
|
+
* and its latest published version's summary (`latest_version`, `latest_published`), whatever
|
|
1735
|
+
* the publish state. A method never published reads with both `null`, never as an error.
|
|
1736
|
+
* `files` lists the draft's tree in tree path order, each file with its SHA-256, its size and a
|
|
1737
|
+
* link to its bytes valid for about fifteen minutes, and is empty for an empty draft;
|
|
1738
|
+
* `writeMethodTree` brings the tree to disk.
|
|
1678
1739
|
*
|
|
1679
1740
|
* Takes a bare catalog id: the method routes address the method itself, never a version of
|
|
1680
1741
|
* it. A caller holding `mt_…@3` strips the suffix with `parseMethodSelector` and reads that
|
|
@@ -1682,54 +1743,47 @@ export class PipelexApiClient {
|
|
|
1682
1743
|
* suffixed id, before any request, rather than read back the `404` the platform would answer,
|
|
1683
1744
|
* and for an id that is empty, `.` or `..`, which the URL would resolve into another route's
|
|
1684
1745
|
* path.
|
|
1746
|
+
*
|
|
1747
|
+
* Throws `ApiResponseError`: `404` `not_found` for an unknown or foreign-org method, `409`
|
|
1748
|
+
* `method_being_deleted` while its erasure runs, and a `409` whose verdict is `runtime` for a
|
|
1749
|
+
* method whose draft names a tree the platform holds no record of. An answer whose `files` is
|
|
1750
|
+
* not a listing is an answer the SDK cannot read, thrown as an `ApiResponseError` too.
|
|
1685
1751
|
*/
|
|
1686
1752
|
async getMethod(methodId) {
|
|
1687
|
-
|
|
1753
|
+
const endpoint = methodPath(methodId, "getMethod");
|
|
1754
|
+
const res = await this.requestProductAnswer("GET", endpoint, undefined);
|
|
1755
|
+
return this.readSavedTree("GET", endpoint, withLinks(res), "a stored method");
|
|
1688
1756
|
}
|
|
1689
1757
|
/**
|
|
1690
|
-
*
|
|
1691
|
-
*
|
|
1692
|
-
*
|
|
1693
|
-
* `mthds` source with `methodSourceToContents`, and labels each resulting file
|
|
1694
|
-
* with the `method_id` as its `source` provenance.
|
|
1695
|
-
*
|
|
1696
|
-
* **This is the draft, not what a bare id runs.** A run, a validation or a pipe I/O
|
|
1697
|
-
* read by a bare `method_id` resolves to the latest PUBLISHED version, which the
|
|
1698
|
-
* draft may be ahead of. For the closure of a version, read it with
|
|
1699
|
-
* `getMethodVersion` and parse its `mthds` with `methodSourceToContents`.
|
|
1758
|
+
* Create a method — `POST /v1/methods`. The new method holds its tree as its draft and has no
|
|
1759
|
+
* published version, so its bare id answers `409 method_not_published` on the run and tooling
|
|
1760
|
+
* routes until its first `publishMethod`; its draft runs as `mt_…@draft` at once.
|
|
1700
1761
|
*
|
|
1701
|
-
*
|
|
1702
|
-
*
|
|
1703
|
-
*
|
|
1704
|
-
*
|
|
1705
|
-
*
|
|
1706
|
-
* pass-through instead; nothing in this client expands an id behind your back.
|
|
1762
|
+
* The tree is `files`, sent as given, or `tree` or `contents`, which the client sends inline
|
|
1763
|
+
* when its inline map weighs 256 KiB or less, or when the platform keeps no blob store, and
|
|
1764
|
+
* otherwise every file by hash, after asking `POST /v1/blobs/missing` which files the
|
|
1765
|
+
* organization lacks and uploading those: a file uploaded once is never uploaded again. With
|
|
1766
|
+
* none of the three, the draft is empty (see `docs/method-trees.md`).
|
|
1707
1767
|
*
|
|
1708
|
-
*
|
|
1709
|
-
*
|
|
1710
|
-
*
|
|
1711
|
-
*
|
|
1712
|
-
*
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
const contents = methodSourceToContents(method.mthds);
|
|
1717
|
-
if (contents.length === 0) {
|
|
1718
|
-
throw new EmptyMethodSourceError(methodId);
|
|
1719
|
-
}
|
|
1720
|
-
return contents.map((content) => ({ content, source: methodId }));
|
|
1721
|
-
}
|
|
1722
|
-
/**
|
|
1723
|
-
* Create a method — `POST /v1/methods`. The new method holds the input as its draft and has
|
|
1724
|
-
* no published version, so its bare id answers `409 method_not_published` on the run and
|
|
1725
|
-
* tooling routes until its first `publishMethod`; its draft runs as `mt_…@draft` at once.
|
|
1726
|
-
*
|
|
1727
|
-
* Throws `ApiResponseError`: `413` `payload_too_large` for a method that would leave no room
|
|
1728
|
-
* for a publish, measured as a draft write is, and `422` `validation_failed` for Python files
|
|
1729
|
-
* no run could import or for text holding a lone surrogate.
|
|
1768
|
+
* Throws `RequestArgumentError` for two forms of the tree, or `contents` holding no file, before
|
|
1769
|
+
* any request; `MethodFileTransferError` when a file the organization lacks could not be
|
|
1770
|
+
* uploaded, and nothing is saved. Throws `ApiResponseError`: `422` `invalid_method_files` for a
|
|
1771
|
+
* map the standard refuses, with its `tree_refusal`; `422` `method_tree_too_large` for a tree
|
|
1772
|
+
* past one of the deployment's limits, named in its `limit`; `422` `blob_mismatch` for a hash
|
|
1773
|
+
* stated at another size than the one held; `422` `missing_blobs` for a hash entry whose bytes
|
|
1774
|
+
* the organization does not hold; `413` `payload_too_large` for a method that would leave no
|
|
1775
|
+
* room for a publish.
|
|
1730
1776
|
*/
|
|
1731
1777
|
async createMethod(input) {
|
|
1732
|
-
|
|
1778
|
+
const source = await this.treeToSend("createMethod", input, SAVE_SENDING);
|
|
1779
|
+
return this.saveMethodTree("POST", "methods", "createMethod", source, (files) => {
|
|
1780
|
+
const body = { name: input.name };
|
|
1781
|
+
if (files != null)
|
|
1782
|
+
body.files = files;
|
|
1783
|
+
if (input.input_data !== undefined)
|
|
1784
|
+
body.input_data = input.input_data;
|
|
1785
|
+
return body;
|
|
1786
|
+
});
|
|
1733
1787
|
}
|
|
1734
1788
|
/**
|
|
1735
1789
|
* Replace a method's draft — `PUT /v1/methods/{id}/draft` — and return the method with the
|
|
@@ -1737,21 +1791,372 @@ export class PipelexApiClient {
|
|
|
1737
1791
|
*
|
|
1738
1792
|
* The draft is never validated on write, and writing it changes nothing for the callers of
|
|
1739
1793
|
* the method's bare id, who run the latest published version until the next `publishMethod`.
|
|
1740
|
-
*
|
|
1794
|
+
* The write replaces the draft's whole tree, given in exactly one form: `files`, sent as given,
|
|
1795
|
+
* `files: null` emptying the draft, or `tree` or `contents`, sent inline or by hash as
|
|
1796
|
+
* `createMethod` sends them. `input_data` is three-way (see `MethodDraftInput`). With
|
|
1741
1797
|
* `expected_updated_at`, the write is a compare-and-swap on the draft token: a draft that
|
|
1742
1798
|
* moved since is refused with a `409` whose `code` is `method_update_conflict`, and nothing is
|
|
1743
1799
|
* written. Without it, last writer wins.
|
|
1744
1800
|
*
|
|
1745
|
-
* Takes a bare catalog id, as `getMethod` does. Throws `
|
|
1746
|
-
*
|
|
1747
|
-
* `
|
|
1748
|
-
*
|
|
1749
|
-
*
|
|
1750
|
-
*
|
|
1751
|
-
*
|
|
1801
|
+
* Takes a bare catalog id, as `getMethod` does. Throws `RequestArgumentError` for a draft that
|
|
1802
|
+
* gives its tree in no form or in two, or `contents` holding no file, before any request;
|
|
1803
|
+
* `MethodFileTransferError` when a file the organization lacks could not be uploaded. Throws
|
|
1804
|
+
* `ApiResponseError`: `404` `not_found` for an unknown or foreign-org method, `409`
|
|
1805
|
+
* `method_being_deleted` while its erasure runs, the `422` refusals of a tree `createMethod`
|
|
1806
|
+
* lists, `413` `payload_too_large` for a draft that would grow the method past the room a
|
|
1807
|
+
* publish needs, `403` for a read-only key, and a `503` that wrote nothing, safe to retry,
|
|
1808
|
+
* when the method kept changing under the write.
|
|
1752
1809
|
*/
|
|
1753
1810
|
async writeDraft(methodId, input) {
|
|
1754
|
-
|
|
1811
|
+
const endpoint = `${methodPath(methodId, "writeDraft")}/draft`;
|
|
1812
|
+
if (treeFormsGiven(input).length === 0) {
|
|
1813
|
+
throw new RequestArgumentError("writeDraft() replaces the draft's whole tree: give it as directory, contents, tree or " +
|
|
1814
|
+
"files (files: null empties the draft).");
|
|
1815
|
+
}
|
|
1816
|
+
const source = await this.treeToSend("writeDraft", input, SAVE_SENDING);
|
|
1817
|
+
return this.saveMethodTree("PUT", endpoint, "writeDraft", source, (files) => {
|
|
1818
|
+
const body = { files: files ?? null };
|
|
1819
|
+
if (input.input_data !== undefined)
|
|
1820
|
+
body.input_data = input.input_data;
|
|
1821
|
+
if (input.expected_updated_at !== undefined) {
|
|
1822
|
+
body.expected_updated_at = input.expected_updated_at;
|
|
1823
|
+
}
|
|
1824
|
+
return body;
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* Send a save carrying a method's tree (`createMethod`, `writeDraft`) through the path every
|
|
1829
|
+
* request sends a tree by (`postMethodRequest`), its body built by `bodyOf` from the `files`
|
|
1830
|
+
* prepared, under `saveTimeoutMs`, and read the stored method it answers as
|
|
1831
|
+
* `requestMethodData` reads it.
|
|
1832
|
+
*/
|
|
1833
|
+
async saveMethodTree(method, endpoint, call, source, bodyOf) {
|
|
1834
|
+
const res = await this.postMethodRequest(source, call, SAVE_SENDING, {}, (files) => this.requestRaw(method, this.url(endpoint), {
|
|
1835
|
+
body: bodyOf(files),
|
|
1836
|
+
timeoutMs: this.saveTimeoutMs(files),
|
|
1837
|
+
}));
|
|
1838
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1839
|
+
this.throwApiResponseError(method, endpoint, res);
|
|
1840
|
+
}
|
|
1841
|
+
return this.readObjectAnswer(method, endpoint, res);
|
|
1842
|
+
}
|
|
1843
|
+
/**
|
|
1844
|
+
* The time limit of a save carrying `files`: the client's request ceiling, as for a `start`
|
|
1845
|
+
* carrying its method, since a tree sent inline to a deployment with no blob store weighs up to
|
|
1846
|
+
* its `max_inline_bytes`, a `files` map given as is whatever it weighs, and the whole upload is
|
|
1847
|
+
* charged against the limit; the poll timeout for a save sending no tree.
|
|
1848
|
+
*/
|
|
1849
|
+
saveTimeoutMs(files) {
|
|
1850
|
+
return files != null && Object.keys(files).length > 0
|
|
1851
|
+
? this.requestTimeoutMs
|
|
1852
|
+
: POLL_REQUEST_TIMEOUT_MS;
|
|
1853
|
+
}
|
|
1854
|
+
/**
|
|
1855
|
+
* The limits the deployment holds a method's tree to — `GET /v1/version`, read from
|
|
1856
|
+
* `limits.method_tree` of the public handshake — in the shape the `mthds` package's
|
|
1857
|
+
* `listMethodTree` takes as its `limits` option, so a tree the platform would refuse is refused
|
|
1858
|
+
* while it is listed, before any file is hashed. A `directory` given to any route is listed under
|
|
1859
|
+
* them already; they serve a caller listing a tree of its own:
|
|
1860
|
+
*
|
|
1861
|
+
* ```ts
|
|
1862
|
+
* import { listMethodTree } from "mthds";
|
|
1863
|
+
* const tree = await listMethodTree("methods/receipts", { limits: await client.methodTreeLimits() });
|
|
1864
|
+
* await client.createMethod({ name: "Receipts", tree });
|
|
1865
|
+
* ```
|
|
1866
|
+
*
|
|
1867
|
+
* A limit the server does not announce is absent, so a bare runner, which announces none,
|
|
1868
|
+
* yields no limit, and none is ever assumed. A value that is not a positive integer is left out
|
|
1869
|
+
* rather than trusted. The inline limit, `max_inline_bytes`, is no listing limit: the client
|
|
1870
|
+
* applies it itself when it chooses how a tree travels.
|
|
1871
|
+
*/
|
|
1872
|
+
async methodTreeLimits() {
|
|
1873
|
+
return listerLimitsOf(announcedTreeLimits(await this.cachedVersion()));
|
|
1874
|
+
}
|
|
1875
|
+
/**
|
|
1876
|
+
* The version handshake, asked once per `HANDSHAKE_TTL_MS`: an answer is kept that long (see
|
|
1877
|
+
* `versionCache`), and a failure is thrown and not kept, so the next call asks again. `signal`
|
|
1878
|
+
* stops the ask, its abort thrown as is.
|
|
1879
|
+
*/
|
|
1880
|
+
async cachedVersion(signal) {
|
|
1881
|
+
const cached = this.versionCache;
|
|
1882
|
+
if (cached !== undefined && Date.now() - cached.askedAt < HANDSHAKE_TTL_MS)
|
|
1883
|
+
return cached.info;
|
|
1884
|
+
const askedAt = Date.now();
|
|
1885
|
+
const info = await this.requestVersion(signal);
|
|
1886
|
+
this.versionCache = { info, askedAt };
|
|
1887
|
+
return info;
|
|
1888
|
+
}
|
|
1889
|
+
/** Drop the kept handshake, which a refusal has shown to be stale. */
|
|
1890
|
+
forgetHandshake() {
|
|
1891
|
+
this.versionCache = undefined;
|
|
1892
|
+
}
|
|
1893
|
+
/**
|
|
1894
|
+
* Write a saved method's tree to disk: each file of a listing `getMethod` or `getMethodVersion`
|
|
1895
|
+
* served, at its tree path under `options.destination`. **Node only.**
|
|
1896
|
+
*
|
|
1897
|
+
* Each file is downloaded through its link, at most eight at a time, with no credential of this
|
|
1898
|
+
* client, and checked against the listing: its size and its SHA-256 must be the ones listed, so
|
|
1899
|
+
* what lands on disk is the tree the method's digest names. The files are written into a fresh
|
|
1900
|
+
* hidden directory beside the destination, which takes its name only once every file is
|
|
1901
|
+
* written: an interrupted or failed write never leaves a partial tree under that name. The
|
|
1902
|
+
* destination must not exist, or be an empty directory; one holding anything is refused rather
|
|
1903
|
+
* than written into or replaced. An empty listing writes an empty directory.
|
|
1904
|
+
*
|
|
1905
|
+
* The links expire about fifteen minutes after the read that served them: write the tree soon
|
|
1906
|
+
* after reading the method, and read it again for fresh links when a download is refused.
|
|
1907
|
+
*
|
|
1908
|
+
* Throws `RequestArgumentError` before any download when the destination exists and is not an
|
|
1909
|
+
* empty directory, or the listing holds a path that is not a tree path, the same path twice, or
|
|
1910
|
+
* paths that collide once case-folded or where one names a directory of another;
|
|
1911
|
+
* `MethodFileTransferError` when a file could not be downloaded, or its bytes are not the file
|
|
1912
|
+
* listed, and nothing is left under the destination.
|
|
1913
|
+
*/
|
|
1914
|
+
async writeMethodTree(files, options) {
|
|
1915
|
+
await writeMethodTreeTo(files, options.destination, { timeoutMs: this.requestTimeoutMs });
|
|
1916
|
+
}
|
|
1917
|
+
/**
|
|
1918
|
+
* The method's tree a call was given, made ready to prepare: a `directory` listed, `contents`
|
|
1919
|
+
* turned into a tree, a `tree` checked, `files` kept as given; `undefined` when the input gives
|
|
1920
|
+
* none. Two forms are refused with `RequestArgumentError` before any request, naming `call`, and
|
|
1921
|
+
* so are a `directory` that is not a string and a `tree` that is not a listed tree. A
|
|
1922
|
+
* `directory` is listed by the `mthds` package's lister under the limits the deployment
|
|
1923
|
+
* announces, so a tree past one is refused while it is listed, before any file is hashed; the
|
|
1924
|
+
* handshake that announces them is asked under `signal`. It is listed once per call: a request
|
|
1925
|
+
* sent again sends the tree this listing read.
|
|
1926
|
+
*/
|
|
1927
|
+
async treeToSend(call, input, sending, signal) {
|
|
1928
|
+
const given = treeFormsGiven(input);
|
|
1929
|
+
if (given.length > 1) {
|
|
1930
|
+
throw new RequestArgumentError(`${call}() takes the method's tree in one form, directory, contents, tree or files; got ` +
|
|
1931
|
+
`${given.join(" and ")}.`);
|
|
1932
|
+
}
|
|
1933
|
+
if (given[0] === "directory") {
|
|
1934
|
+
const directory = input.directory;
|
|
1935
|
+
if (typeof directory !== "string")
|
|
1936
|
+
throw directoryRefusal(call);
|
|
1937
|
+
throwIfAborted(signal);
|
|
1938
|
+
const limits = listerLimitsOf(announcedTreeLimits(await this.cachedVersion(signal)));
|
|
1939
|
+
throwIfAborted(signal);
|
|
1940
|
+
return { tree: await listDirectoryTree(directory, limits, call) };
|
|
1941
|
+
}
|
|
1942
|
+
if (input.tree != null)
|
|
1943
|
+
return { tree: checkedTreeSource(call, input.tree) };
|
|
1944
|
+
if (input.contents != null) {
|
|
1945
|
+
return { tree: await contentsTreeSource(input.contents, call, sending) };
|
|
1946
|
+
}
|
|
1947
|
+
return input.files === undefined ? undefined : { files: input.files };
|
|
1948
|
+
}
|
|
1949
|
+
/**
|
|
1950
|
+
* Post a request carrying a method's tree, by `mthds`'s `MthdsApiClient` rule: prepare the
|
|
1951
|
+
* tree's `files` (`prepareFiles`), hand them to `post`, which builds the body and sends it, and
|
|
1952
|
+
* answer the two refusals that call for sending the same request once more, as `mthds` does.
|
|
1953
|
+
*
|
|
1954
|
+
* - A `422` coded `request.blobs_missing` whose `missing_blobs` the tree sent by hash holds, such
|
|
1955
|
+
* as a blob dropped since the check: the blob route is asked again for those, they are
|
|
1956
|
+
* uploaded, and the request is sent once more (`postWithBlobRetry`).
|
|
1957
|
+
* - A `422` refusing a tree the client prepared as past one of the receiver's limits
|
|
1958
|
+
* (`isMethodTreeTooLarge`): the handshake that announced the limits may be stale, so it is
|
|
1959
|
+
* forgotten, and the tree prepared again on a fresh one and sent once more. A `files` map
|
|
1960
|
+
* given as is was not prepared, so it is never sent again; nor is a request whose
|
|
1961
|
+
* `options.retryTooLarge` is `false`, a start under an idempotency key, which the platform
|
|
1962
|
+
* binds to the refusal it stores.
|
|
1963
|
+
*
|
|
1964
|
+
* A `422` created nothing, so the request sent again is no repeat of one whose outcome is
|
|
1965
|
+
* unknown. `initial`, when given, is the tree already prepared for the first send, with the
|
|
1966
|
+
* bytes it read, so a caller sending one tree on two routes prepares it once. `options.signal`
|
|
1967
|
+
* is read before each step of the preparation and before each send. Answers the last answer
|
|
1968
|
+
* whole, its status unread, with the `files` it carried.
|
|
1969
|
+
*/
|
|
1970
|
+
async postMethodRequest(source, call, sending, options, post, initial) {
|
|
1971
|
+
const read = initial?.read ?? new Map();
|
|
1972
|
+
let ready = initial?.prepared;
|
|
1973
|
+
const send = async () => {
|
|
1974
|
+
const prepared = ready ?? (await this.prepareFiles(source, call, sending, options, read));
|
|
1975
|
+
ready = undefined;
|
|
1976
|
+
throwIfAborted(options.signal);
|
|
1977
|
+
return this.postWithBlobRetry(prepared, read, post, sending, options);
|
|
1978
|
+
};
|
|
1979
|
+
const res = await send();
|
|
1980
|
+
if (source === undefined ||
|
|
1981
|
+
!("tree" in source) ||
|
|
1982
|
+
options.retryTooLarge === false ||
|
|
1983
|
+
!isMethodTreeTooLarge(res.status, res.body)) {
|
|
1984
|
+
return res;
|
|
1985
|
+
}
|
|
1986
|
+
this.forgetHandshake();
|
|
1987
|
+
return send();
|
|
1988
|
+
}
|
|
1989
|
+
/**
|
|
1990
|
+
* Send a request whose tree may have travelled by hash. When it did and the receiver answers a
|
|
1991
|
+
* `422` coded `request.blobs_missing` whose `missing_blobs` the tree holds, the client asks the
|
|
1992
|
+
* blob route where to upload them, uploads them and sends the same request once more, which the
|
|
1993
|
+
* refusal left uncreated; any other answer, or a list naming a blob the tree does not hold, is
|
|
1994
|
+
* answered as it came.
|
|
1995
|
+
*/
|
|
1996
|
+
async postWithBlobRetry(prepared, read, post, sending, options) {
|
|
1997
|
+
const res = await post(prepared.files);
|
|
1998
|
+
const tree = prepared.byHashTree;
|
|
1999
|
+
if (tree === undefined)
|
|
2000
|
+
return res;
|
|
2001
|
+
const missing = missingBlobsOf(res.status, res.body);
|
|
2002
|
+
if (missing === undefined || missing.length === 0)
|
|
2003
|
+
return res;
|
|
2004
|
+
const byBlob = fileByBlob(tree);
|
|
2005
|
+
const files = missing.map((sha256) => byBlob.get(sha256));
|
|
2006
|
+
if (files.some((file) => file === undefined))
|
|
2007
|
+
return res;
|
|
2008
|
+
throwIfAborted(options.signal);
|
|
2009
|
+
const asked = await this.findMissingBlobs({ blobs: files.map((file) => ({ sha256: file.sha256, size: file.size })) }, options.readScope, options.signal);
|
|
2010
|
+
await this.uploadMissing(tree, asked, read, sending, options.signal);
|
|
2011
|
+
throwIfAborted(options.signal);
|
|
2012
|
+
return post(prepared.files);
|
|
2013
|
+
}
|
|
2014
|
+
/**
|
|
2015
|
+
* The `files` a request or a save sends for `source`: a `files` map as given, a tree prepared
|
|
2016
|
+
* (`prepareTree`), or none.
|
|
2017
|
+
*/
|
|
2018
|
+
async prepareFiles(source, call, sending, options, read) {
|
|
2019
|
+
if (source === undefined)
|
|
2020
|
+
return { files: undefined, byHashTree: undefined };
|
|
2021
|
+
if ("files" in source)
|
|
2022
|
+
return { files: source.files, byHashTree: undefined };
|
|
2023
|
+
return this.prepareTree(source.tree, call, sending, options, read);
|
|
2024
|
+
}
|
|
2025
|
+
/**
|
|
2026
|
+
* The `files` map that sends `tree`: inline, or by hash once the receiver holds every file —
|
|
2027
|
+
* `mthds`'s `MthdsApiClient` rule, read from the version handshake the client keeps.
|
|
2028
|
+
*
|
|
2029
|
+
* When the handshake declares a blob store, a tree whose inline map weighs no more than the
|
|
2030
|
+
* inline threshold travels inline, measured as the request spends it, its JSON with no
|
|
2031
|
+
* whitespace in UTF-8; the threshold is the lower of 256 KiB (`INLINE_TREE_MAX_BYTES`) and the
|
|
2032
|
+
* `max_inline_bytes` the handshake announces, and a tree whose files alone pass it is not read
|
|
2033
|
+
* to be weighed. A heavier tree travels by hash: `POST /v1/blobs/missing` is asked which files
|
|
2034
|
+
* the receiver lacks, naming the tree's digest, each one it lacks is uploaded once, and every
|
|
2035
|
+
* file travels by hash. A `404` from the blob route says the handshake that declared the store
|
|
2036
|
+
* is stale, so it is forgotten and the tree prepared once more on a fresh one. When the
|
|
2037
|
+
* handshake declares no blob store, the whole tree travels inline whatever its weight, held to
|
|
2038
|
+
* the announced `max_inline_bytes` (`inlineWithin`). Each file is read at most once into `read`,
|
|
2039
|
+
* so a file read to be weighed is uploaded from memory. A tree holding no file is refused with
|
|
2040
|
+
* `RequestArgumentError` before any request, `call` naming the caller's method.
|
|
2041
|
+
* `options.readScope` is the scope the blob route is asked under, and `options.signal` is read
|
|
2042
|
+
* before each read, the handshake, the blob route and each upload, and is passed to each of
|
|
2043
|
+
* them, so an abort also stops a request or an upload under way.
|
|
2044
|
+
*/
|
|
2045
|
+
async prepareTree(tree, call, sending, options, read, retried = false) {
|
|
2046
|
+
const { signal, readScope } = options;
|
|
2047
|
+
if (tree.files.length === 0)
|
|
2048
|
+
throw emptyTreeRefusal(call, "a tree", sending);
|
|
2049
|
+
throwIfAborted(signal);
|
|
2050
|
+
const version = await this.cachedVersion(signal);
|
|
2051
|
+
throwIfAborted(signal);
|
|
2052
|
+
const announced = announcedTreeLimits(version);
|
|
2053
|
+
if (!declaresBlobStore(version)) {
|
|
2054
|
+
const files = await inlineWithin(tree, call, sending, announced.maxInlineBytes, read, signal);
|
|
2055
|
+
return { files, byHashTree: undefined };
|
|
2056
|
+
}
|
|
2057
|
+
const threshold = inlineThreshold(announced);
|
|
2058
|
+
// An inline map is never smaller than the bytes it carries, so a tree whose bytes alone pass
|
|
2059
|
+
// the threshold is not read in full just to be measured.
|
|
2060
|
+
if (treeTotalSize(tree) <= threshold) {
|
|
2061
|
+
const inline = inlineFilesOf(tree, await readTreeBytesInto(tree, read, signal));
|
|
2062
|
+
if (inlineMapBytes(inline) <= threshold)
|
|
2063
|
+
return { files: inline, byHashTree: undefined };
|
|
2064
|
+
}
|
|
2065
|
+
throwIfAborted(signal);
|
|
2066
|
+
let asked;
|
|
2067
|
+
try {
|
|
2068
|
+
asked = await this.findMissingBlobs({ blobs: uniqueBlobs(tree), digest: tree.digest }, readScope, signal);
|
|
2069
|
+
}
|
|
2070
|
+
catch (err) {
|
|
2071
|
+
if (!retried && err instanceof ApiResponseError && err.status === 404) {
|
|
2072
|
+
this.forgetHandshake();
|
|
2073
|
+
return this.prepareTree(tree, call, sending, options, read, true);
|
|
2074
|
+
}
|
|
2075
|
+
throw err;
|
|
2076
|
+
}
|
|
2077
|
+
await this.uploadMissing(tree, asked, read, sending, signal);
|
|
2078
|
+
return { files: hashMethodFiles(tree), byHashTree: tree };
|
|
2079
|
+
}
|
|
2080
|
+
/**
|
|
2081
|
+
* Which of these blobs the organization lacks, and where to upload each —
|
|
2082
|
+
* `POST /v1/blobs/missing` — with the answer it was read from, which carries signed links. A
|
|
2083
|
+
* blob held is left out, and an empty `missing` means nothing to upload. An answer whose
|
|
2084
|
+
* `missing` is not a list of blobs, each with its upload target, is an answer the SDK cannot
|
|
2085
|
+
* read, thrown as an `ApiResponseError` holding the answer with its links hidden.
|
|
2086
|
+
*
|
|
2087
|
+
* `readScope`, a crate request's `read_scope`, is sent as the route's `read_scope` query, the
|
|
2088
|
+
* Pipelex API's extension naming the blob store asked, so the blobs are uploaded into the store
|
|
2089
|
+
* the request's hash entries are read from; each upload target is signed for that scope. The
|
|
2090
|
+
* hosted platform's own route reads the caller's organization instead. `signal` stops the
|
|
2091
|
+
* request, its abort thrown as is.
|
|
2092
|
+
*/
|
|
2093
|
+
async findMissingBlobs(request, readScope, signal) {
|
|
2094
|
+
const endpoint = readScope === undefined
|
|
2095
|
+
? "blobs/missing"
|
|
2096
|
+
: `blobs/missing?${new URLSearchParams({ read_scope: readScope }).toString()}`;
|
|
2097
|
+
const res = withLinks(await this.requestProductAnswer("POST", endpoint, request, { signal }));
|
|
2098
|
+
const answer = this.readObjectAnswer("POST", endpoint, res);
|
|
2099
|
+
const missing = answer.missing;
|
|
2100
|
+
if (!Array.isArray(missing) || !missing.every(isMissingBlob)) {
|
|
2101
|
+
throw this.unreadableAnswer("POST", `/${API_PREFIX}/${endpoint}`, res, "a body whose `missing` is not a list of blobs, each with its upload target");
|
|
2102
|
+
}
|
|
2103
|
+
return { result: answer, res };
|
|
2104
|
+
}
|
|
2105
|
+
/**
|
|
2106
|
+
* Upload each missing blob from the tree's bytes, a few at a time, stopping at the first
|
|
2107
|
+
* failure. A blob the platform asks for that the tree does not hold, or holds at another size,
|
|
2108
|
+
* is the platform's fault, thrown before anything is sent as the `ApiResponseError` of an answer
|
|
2109
|
+
* the SDK cannot read. The bytes the tree reads for a blob are checked against it before they
|
|
2110
|
+
* are sent, since storage keeps a blob under the hash it is named by: bytes of another hash or
|
|
2111
|
+
* size are refused with `RequestArgumentError`, and nothing more is uploaded. `userSignal` is
|
|
2112
|
+
* read before each blob's read and before its upload, and reaches each upload under way: once it
|
|
2113
|
+
* has aborted, no further blob is read or sent, the uploads in flight are stopped, and the abort
|
|
2114
|
+
* is thrown rather than a transfer failure.
|
|
2115
|
+
*/
|
|
2116
|
+
async uploadMissing(tree, asked, read, sending, userSignal) {
|
|
2117
|
+
const missing = asked.result.missing;
|
|
2118
|
+
const byBlob = fileByBlob(tree);
|
|
2119
|
+
const strangers = missing
|
|
2120
|
+
.filter((blob) => byBlob.get(blob.sha256)?.size !== blob.size)
|
|
2121
|
+
.map((blob) => blob.sha256);
|
|
2122
|
+
if (strangers.length > 0) {
|
|
2123
|
+
throw this.unreadableAnswer("POST", `/${API_PREFIX}/blobs/missing`, asked.res, `a body asking for blobs this tree does not hold: ${[...new Set(strangers)].sort().join(", ")}`);
|
|
2124
|
+
}
|
|
2125
|
+
// The tasks share the one signal the run of uploads aborts on a failure, linked here once to
|
|
2126
|
+
// the caller's and detached when the uploads are over.
|
|
2127
|
+
let linked;
|
|
2128
|
+
try {
|
|
2129
|
+
await forEachConcurrently(missing, MAX_CONCURRENT_TRANSFERS, async (blob, signal) => {
|
|
2130
|
+
linked ??= eitherSignal(signal, userSignal);
|
|
2131
|
+
await this.uploadOne(tree, blob, byBlob, read, sending, linked.signal, userSignal);
|
|
2132
|
+
});
|
|
2133
|
+
}
|
|
2134
|
+
catch (err) {
|
|
2135
|
+
if (userSignal?.aborted === true)
|
|
2136
|
+
throw userSignal.reason;
|
|
2137
|
+
throw err;
|
|
2138
|
+
}
|
|
2139
|
+
finally {
|
|
2140
|
+
linked?.dispose();
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
/** Read, check and upload one missing blob (see `uploadMissing`). */
|
|
2144
|
+
async uploadOne(tree, blob, byBlob, read, sending, signal, userSignal) {
|
|
2145
|
+
const file = byBlob.get(blob.sha256);
|
|
2146
|
+
throwIfAborted(userSignal);
|
|
2147
|
+
const bytes = read?.get(file.path) ?? (await tree.readFile(file.path));
|
|
2148
|
+
const bytesSha256 = await sha256Hex(bytes);
|
|
2149
|
+
if (bytes.length !== blob.size || bytesSha256 !== blob.sha256) {
|
|
2150
|
+
throw new RequestArgumentError(`The tree read ${bytes.length} bytes hashing to ${bytesSha256} for '${file.path}', which ` +
|
|
2151
|
+
`its listing names as ${blob.size} bytes hashing to ${blob.sha256}, so they were not ` +
|
|
2152
|
+
`uploaded and ${sending.unsent}.`);
|
|
2153
|
+
}
|
|
2154
|
+
throwIfAborted(userSignal);
|
|
2155
|
+
await uploadBlob(blob, bytes, {
|
|
2156
|
+
timeoutMs: this.requestTimeoutMs,
|
|
2157
|
+
signal,
|
|
2158
|
+
unsent: sending.unsent,
|
|
2159
|
+
});
|
|
1755
2160
|
}
|
|
1756
2161
|
/**
|
|
1757
2162
|
* Rename a method — `PATCH /v1/methods/{id}` — and return it. The name belongs to the method,
|
|
@@ -1817,11 +2222,10 @@ export class PipelexApiClient {
|
|
|
1817
2222
|
outcomes: ["published", "unchanged", "refused"],
|
|
1818
2223
|
member: "method",
|
|
1819
2224
|
});
|
|
1820
|
-
|
|
1821
|
-
return { ...answer, outcome, method };
|
|
2225
|
+
return { ...answer, outcome, method: subject };
|
|
1822
2226
|
}
|
|
1823
2227
|
/**
|
|
1824
|
-
* One page of a method's published versions, newest first, without their
|
|
2228
|
+
* One page of a method's published versions, newest first, without their files —
|
|
1825
2229
|
* `GET /v1/methods/{id}/versions`.
|
|
1826
2230
|
*
|
|
1827
2231
|
* Pass the page's `nextCursor` back as `cursor` to continue; `null` is the last page. A page
|
|
@@ -1839,14 +2243,16 @@ export class PipelexApiClient {
|
|
|
1839
2243
|
return this.requestPage(`${endpoint}${search}`);
|
|
1840
2244
|
}
|
|
1841
2245
|
/**
|
|
1842
|
-
* One published version of a method, with its
|
|
1843
|
-
* `
|
|
1844
|
-
*
|
|
2246
|
+
* One published version of a method, with its files — `GET /v1/methods/{id}/versions/{n}`.
|
|
2247
|
+
* `files` lists the version's tree as `getMethod` lists the draft's, each file with a link to
|
|
2248
|
+
* its bytes valid for about fifteen minutes, which `writeMethodTree` writes to disk.
|
|
1845
2249
|
*
|
|
1846
2250
|
* Takes a bare catalog id, as `getMethod` does, and the version number. Throws
|
|
1847
2251
|
* `RequestArgumentError` for a `version` that is not a positive integer, before any request.
|
|
1848
2252
|
* Throws `ApiResponseError`: `404` `method_version_not_found` for a version the method never
|
|
1849
|
-
* published, `404` `not_found` for an unknown method, `409` `method_being_deleted
|
|
2253
|
+
* published, `404` `not_found` for an unknown method, `409` `method_being_deleted`, and `409`
|
|
2254
|
+
* `conflict`, whose verdict is `runtime`, for a version stored before methods were trees and
|
|
2255
|
+
* never rewritten as one, which names no tree to serve.
|
|
1850
2256
|
*/
|
|
1851
2257
|
async getMethodVersion(methodId, version) {
|
|
1852
2258
|
if (!Number.isSafeInteger(version) || version < 1) {
|
|
@@ -1854,8 +2260,7 @@ export class PipelexApiClient {
|
|
|
1854
2260
|
}
|
|
1855
2261
|
const endpoint = `${methodPath(methodId, "getMethodVersion")}/versions/${version}`;
|
|
1856
2262
|
const res = await this.requestProductAnswer("GET", endpoint, undefined);
|
|
1857
|
-
|
|
1858
|
-
return this.readStoredSources("GET", endpoint, res, wire, "a published version");
|
|
2263
|
+
return this.readSavedTree("GET", endpoint, withLinks(res), "a published version");
|
|
1859
2264
|
}
|
|
1860
2265
|
/**
|
|
1861
2266
|
* Erase a method and everything it produced — `DELETE /v1/methods/{id}`.
|
|
@@ -2247,8 +2652,9 @@ export class PipelexApiClient {
|
|
|
2247
2652
|
* Name the method exactly one of three ways, all server-resolved through the
|
|
2248
2653
|
* one `pipeIo` call this composes:
|
|
2249
2654
|
*
|
|
2250
|
-
* -
|
|
2251
|
-
*
|
|
2655
|
+
* - the method's tree — a `directory`, `contents`, a listed `tree` or the `files`
|
|
2656
|
+
* map, sent as `pipeIo` sends it, with the files its steps read (a `PipeDocGen`
|
|
2657
|
+
* step's `template_file`) among its files;
|
|
2252
2658
|
* - `method_ref` — a published method's address, fetched by the runner;
|
|
2253
2659
|
* - `method_id` — a stored method's catalog id, resolved by the platform
|
|
2254
2660
|
* (hosted only; requires an API key).
|
|
@@ -2353,9 +2759,8 @@ export class PipelexApiClient {
|
|
|
2353
2759
|
/**
|
|
2354
2760
|
* The whole record for one run — `GET /v1/runs/{id}`.
|
|
2355
2761
|
*
|
|
2356
|
-
* The ONLY call that returns
|
|
2357
|
-
*
|
|
2358
|
-
* few seconds. A failed run's stored report, `error`, is checked field by
|
|
2762
|
+
* The ONLY call that returns what the run actually executed and its
|
|
2763
|
+
* `inputs`. Kept off the status read, which pollers hit every few seconds. A failed run's stored report, `error`, is checked field by
|
|
2359
2764
|
* field (see `RunErrorReport`).
|
|
2360
2765
|
*
|
|
2361
2766
|
* Read with `parseExactJson`, so an integer past `Number.MAX_SAFE_INTEGER` in the inputs is a
|
|
@@ -2372,6 +2777,30 @@ export class PipelexApiClient {
|
|
|
2372
2777
|
const segment = idPathSegment(runId, "updateRun", RUN_ID_ARGUMENT);
|
|
2373
2778
|
await this.requestProduct("PUT", `runs/${segment}`, input);
|
|
2374
2779
|
}
|
|
2780
|
+
/**
|
|
2781
|
+
* A run's arguments checked before anything is listed, read or sent, `call` naming the caller's
|
|
2782
|
+
* method in each refusal: the generic `extra` passthrough, the Pipelex API's and the hosted
|
|
2783
|
+
* API's own selectors, and the method's tree in its forms as the standard checks them; then the
|
|
2784
|
+
* "something to run" precondition, the `method_ref` pairings and the linkage form's bare
|
|
2785
|
+
* `method_id`.
|
|
2786
|
+
*/
|
|
2787
|
+
runParts(call, options) {
|
|
2788
|
+
const extensions = buildExtensions(options.extra);
|
|
2789
|
+
const api = buildApiRunExtensions(options);
|
|
2790
|
+
const hosted = buildHostedRunExtensions(options);
|
|
2791
|
+
const forms = protocolTreeForms(call, options);
|
|
2792
|
+
const treeForms = treeFormsGiven(forms);
|
|
2793
|
+
if (!options.pipe_code &&
|
|
2794
|
+
treeForms.length === 0 &&
|
|
2795
|
+
Object.keys(api).length === 0 &&
|
|
2796
|
+
Object.keys(hosted).length === 0 &&
|
|
2797
|
+
Object.keys(extensions).length === 0) {
|
|
2798
|
+
throw new RequestArgumentError(`Either pipe_code, the method's tree (directory, contents, tree or files), a method_ref, a hosted method_id or a server-specific extension arg (extra) must be provided to ${call}().`);
|
|
2799
|
+
}
|
|
2800
|
+
assertMethodRefPairsWithNothing(options, treeForms);
|
|
2801
|
+
assertLinkageMethodIdIsBare(options, treeForms);
|
|
2802
|
+
return { forms, extensions, api, hosted };
|
|
2803
|
+
}
|
|
2375
2804
|
/**
|
|
2376
2805
|
* Blocking `POST /v1/execute` adapted onto `RunResults` — the bare-runner
|
|
2377
2806
|
* path. Forwards every protocol field PLUS every extension surface: the
|
|
@@ -2383,24 +2812,24 @@ export class PipelexApiClient {
|
|
|
2383
2812
|
* so a runner that cannot resolve it says so (Rule 4) instead of the client
|
|
2384
2813
|
* silently dropping it.
|
|
2385
2814
|
*/
|
|
2386
|
-
async executeBlocking(options) {
|
|
2387
|
-
const response = await this.
|
|
2815
|
+
async executeBlocking(options, signal, sending) {
|
|
2816
|
+
const response = await this.executeRun({
|
|
2388
2817
|
pipe_code: options.pipe_code ?? undefined,
|
|
2389
|
-
mthds_contents: options.mthds_contents ?? undefined,
|
|
2390
2818
|
inputs: options.inputs ?? undefined,
|
|
2391
2819
|
output_name: options.output_name ?? undefined,
|
|
2392
2820
|
output_multiplicity: options.output_multiplicity ?? undefined,
|
|
2393
2821
|
dynamic_output_concept_ref: options.dynamic_output_concept_ref ?? undefined,
|
|
2394
|
-
// The
|
|
2395
|
-
//
|
|
2822
|
+
// The method's tree must survive the fallback too — a bare runner reached through this path
|
|
2823
|
+
// runs the same method as the durable one, or it runs nothing. Its forms are checked here as
|
|
2824
|
+
// the start checked them, and `sending` carries the tree the start prepared, sent as is.
|
|
2825
|
+
directory: options.directory ?? undefined,
|
|
2826
|
+
contents: options.contents ?? undefined,
|
|
2827
|
+
tree: options.tree ?? undefined,
|
|
2396
2828
|
files: options.files ?? undefined,
|
|
2397
|
-
bundle_b64: options.bundle_b64 ?? undefined,
|
|
2398
|
-
// And so must the files its steps read, or the same method fails to load here.
|
|
2399
|
-
resources: options.resources ?? undefined,
|
|
2400
2829
|
method_ref: options.method_ref ?? undefined,
|
|
2401
2830
|
method_id: options.method_id ?? undefined,
|
|
2402
2831
|
extra: options.extra ?? undefined,
|
|
2403
|
-
});
|
|
2832
|
+
}, signal, sending);
|
|
2404
2833
|
return resultsFromExecute(response);
|
|
2405
2834
|
}
|
|
2406
2835
|
}
|
|
@@ -2461,26 +2890,23 @@ function describeRefusedBaseUrl(value) {
|
|
|
2461
2890
|
return `${shown} with ${inWords(parts, "and")} (not shown)`;
|
|
2462
2891
|
}
|
|
2463
2892
|
// The protocol's own request fields — `extra` is for extension args only.
|
|
2464
|
-
// `files`
|
|
2465
|
-
//
|
|
2466
|
-
//
|
|
2893
|
+
// `files` is reserved too: it is the method's tree, a named option, so smuggling
|
|
2894
|
+
// it through `extra` (which merges last into the body) would overwrite the
|
|
2895
|
+
// prepared tree and bypass the one-form check.
|
|
2467
2896
|
const PROTOCOL_REQUEST_KEYS = [
|
|
2468
2897
|
"pipe_code",
|
|
2469
|
-
"mthds_contents",
|
|
2470
2898
|
"inputs",
|
|
2471
2899
|
"output_name",
|
|
2472
2900
|
"output_multiplicity",
|
|
2473
2901
|
"dynamic_output_concept_ref",
|
|
2474
2902
|
"files",
|
|
2475
|
-
"bundle_b64",
|
|
2476
2903
|
];
|
|
2477
|
-
// The PIPELEX API's own request
|
|
2478
|
-
//
|
|
2479
|
-
// the
|
|
2480
|
-
//
|
|
2481
|
-
//
|
|
2482
|
-
|
|
2483
|
-
const PIPELEX_API_REQUEST_KEYS = ["method_ref", "resources"];
|
|
2904
|
+
// The PIPELEX API's own request field — the layer-2 extension the runner reads
|
|
2905
|
+
// itself (`method_ref`, a run source in its own right). Reserved on `extra` for
|
|
2906
|
+
// the same reason the protocol fields are: `extra` merges last, so a smuggled
|
|
2907
|
+
// copy would overwrite the validated named option and bypass the
|
|
2908
|
+
// selector-exclusivity check.
|
|
2909
|
+
const PIPELEX_API_REQUEST_KEYS = ["method_ref"];
|
|
2484
2910
|
// The HOSTED API's own request fields — the layer-3 extensions this client
|
|
2485
2911
|
// names itself. Reserved on `extra` for the same reason the protocol fields
|
|
2486
2912
|
// are: `extra` merges last, so a smuggled copy would overwrite the validated
|
|
@@ -2493,15 +2919,20 @@ const PIPELEX_API_REQUEST_KEYS = ["method_ref", "resources"];
|
|
|
2493
2919
|
// keep passing that vendor's `method_id` straight through. That per-layer
|
|
2494
2920
|
// split is normative for the whole layered client stack.
|
|
2495
2921
|
const HOSTED_REQUEST_KEYS = ["method_id"];
|
|
2922
|
+
// The client-only options, which are never serialized as such: the three forms of
|
|
2923
|
+
// the method's tree the client turns into `files` (`directory`, `contents`, `tree`)
|
|
2924
|
+
// and the `bundleMain` hint. Reserved on `extra` so none of them reaches the wire
|
|
2925
|
+
// through the passthrough under its own name.
|
|
2926
|
+
const CLIENT_ONLY_KEYS = ["directory", "contents", "tree", "bundleMain"];
|
|
2496
2927
|
// Keys that must never ride `extra`: the named request options above (which
|
|
2497
2928
|
// `extra` would overwrite — it merges last into the body) plus the client-only
|
|
2498
|
-
//
|
|
2499
|
-
//
|
|
2929
|
+
// options. A field the client no longer names, such as the runner's legacy
|
|
2930
|
+
// `mthds_contents`, is NOT reserved: `extra` is the passthrough for exactly that.
|
|
2500
2931
|
const RESERVED_EXTRA_KEYS = new Set([
|
|
2501
2932
|
...PROTOCOL_REQUEST_KEYS,
|
|
2502
2933
|
...PIPELEX_API_REQUEST_KEYS,
|
|
2503
2934
|
...HOSTED_REQUEST_KEYS,
|
|
2504
|
-
|
|
2935
|
+
...CLIENT_ONLY_KEYS,
|
|
2505
2936
|
]);
|
|
2506
2937
|
// Prototype-pollution vectors. An own `__proto__` (exactly what `JSON.parse`
|
|
2507
2938
|
// yields, and `extra` is the field most likely populated from untrusted JSON),
|
|
@@ -2542,22 +2973,110 @@ function buildExtensions(extra) {
|
|
|
2542
2973
|
// caps responses regardless.
|
|
2543
2974
|
const METHOD_REF_FETCH_TIMEOUT_MS = 180_000; // 3 min — covers the server's clone + resolve
|
|
2544
2975
|
/**
|
|
2545
|
-
*
|
|
2546
|
-
*
|
|
2547
|
-
* is
|
|
2548
|
-
*
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2976
|
+
* Whether a crate request's `files` carry more inline than `INLINE_TREE_MAX_BYTES`, weighed as the
|
|
2977
|
+
* request spends them: a tree sent inline to an API keeping no blob store, whatever its weight, or
|
|
2978
|
+
* a `files` map given as is. A map of hash entries weighs little, and a value that is not a map
|
|
2979
|
+
* is not weighed.
|
|
2980
|
+
*/
|
|
2981
|
+
function carriesLargeInlineTree(files) {
|
|
2982
|
+
if (files === null || files === undefined || !isPlainObject(files))
|
|
2983
|
+
return false;
|
|
2984
|
+
try {
|
|
2985
|
+
return inlineMapBytes(files) > INLINE_TREE_MAX_BYTES;
|
|
2986
|
+
}
|
|
2987
|
+
catch {
|
|
2988
|
+
// A map JSON cannot carry is refused as the caller's argument when the request is written.
|
|
2989
|
+
return false;
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
/**
|
|
2993
|
+
* A run's wire body: the protocol's arguments, the method's `files`, then the Pipelex API's and
|
|
2994
|
+
* the hosted API's selectors and the `extra` passthrough. A start drops a `null` argument, as it
|
|
2995
|
+
* always has; `execute` sends its arguments as given. `?? undefined` so `JSON.stringify` drops an
|
|
2996
|
+
* absent field from the wire body.
|
|
2556
2997
|
*/
|
|
2557
|
-
function
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2998
|
+
function runRequestBody(call, options, files, parts) {
|
|
2999
|
+
const dropNull = call === "start";
|
|
3000
|
+
return {
|
|
3001
|
+
pipe_code: dropNull ? (options.pipe_code ?? undefined) : options.pipe_code,
|
|
3002
|
+
inputs: dropNull ? (options.inputs ?? undefined) : options.inputs,
|
|
3003
|
+
output_name: dropNull ? (options.output_name ?? undefined) : options.output_name,
|
|
3004
|
+
output_multiplicity: dropNull
|
|
3005
|
+
? (options.output_multiplicity ?? undefined)
|
|
3006
|
+
: options.output_multiplicity,
|
|
3007
|
+
dynamic_output_concept_ref: dropNull
|
|
3008
|
+
? (options.dynamic_output_concept_ref ?? undefined)
|
|
3009
|
+
: options.dynamic_output_concept_ref,
|
|
3010
|
+
files: files ?? undefined,
|
|
3011
|
+
...parts.api,
|
|
3012
|
+
...parts.hosted,
|
|
3013
|
+
...parts.extensions,
|
|
3014
|
+
};
|
|
3015
|
+
}
|
|
3016
|
+
/**
|
|
3017
|
+
* The error a blocking execute throws for `err`: a `PipelineExecuteTimeoutError` when the hosted
|
|
3018
|
+
* gateway cut the request off (see `isGatewayCutOff`), timed from `sentAt`, and `err` otherwise.
|
|
3019
|
+
*/
|
|
3020
|
+
function executeCutOff(err, sentAt) {
|
|
3021
|
+
if (err instanceof RunStillRunningError)
|
|
3022
|
+
return err;
|
|
3023
|
+
const elapsedMs = performance.now() - sentAt;
|
|
3024
|
+
return isGatewayCutOff(err, elapsedMs)
|
|
3025
|
+
? new PipelineExecuteTimeoutError(elapsedMs, { cause: err })
|
|
3026
|
+
: err;
|
|
3027
|
+
}
|
|
3028
|
+
/**
|
|
3029
|
+
* Whether a run's `extra` carries a method bundle a runner reads, `mthds_contents` or
|
|
3030
|
+
* `bundle_b64`, as the `pipelex-sdk` command sends one: a start carrying one gets the request
|
|
3031
|
+
* ceiling, as the Python SDK gives it, since the bundle can make the body large.
|
|
3032
|
+
*/
|
|
3033
|
+
function carriesExtraBundle(extensions) {
|
|
3034
|
+
const contents = extensions["mthds_contents"];
|
|
3035
|
+
const carriesContents = Array.isArray(contents) ? contents.length > 0 : Boolean(contents);
|
|
3036
|
+
return carriesContents || Boolean(extensions["bundle_b64"]);
|
|
3037
|
+
}
|
|
3038
|
+
/** The refusal of a `directory` that is not a path, `call` naming the caller's method. */
|
|
3039
|
+
function directoryRefusal(call) {
|
|
3040
|
+
return new RequestArgumentError(`${call}() got a directory that is not a path: give the method's directory as a string.`);
|
|
3041
|
+
}
|
|
3042
|
+
/**
|
|
3043
|
+
* `value` as a listed tree, or the refusal of one that is not, `call` naming the caller's method:
|
|
3044
|
+
* a `tree` must have its `files`, its `digest` and a `readFile`, as the `MethodTree` the `mthds`
|
|
3045
|
+
* package's `listMethodTree` returns has, which the shape check `prepareInputs` applies too.
|
|
3046
|
+
*/
|
|
3047
|
+
function checkedTreeSource(call, value) {
|
|
3048
|
+
const fault = treeShapeFault(value);
|
|
3049
|
+
if (fault !== undefined) {
|
|
3050
|
+
throw new RequestArgumentError(`${call}() got a tree that is not a listed method tree — its files, its digest and a ` +
|
|
3051
|
+
`readFile — such as the MethodTree listMethodTree returns; got ${fault}.`);
|
|
3052
|
+
}
|
|
3053
|
+
return value;
|
|
3054
|
+
}
|
|
3055
|
+
/**
|
|
3056
|
+
* The whole tree inline, for a receiver keeping no blob store, held to the inline limit it
|
|
3057
|
+
* announces as `mthds`'s `MthdsApiClient` holds it: its files' declared sizes are checked before
|
|
3058
|
+
* anything is read, since the inline map weighs at least that much, then the encoded map, so a
|
|
3059
|
+
* tree the receiver would refuse is refused here with `RequestArgumentError`, naming the limit,
|
|
3060
|
+
* without being sent. Each file is read into `read`, under `signal`.
|
|
3061
|
+
*/
|
|
3062
|
+
async function inlineWithin(tree, call, sending, maxInlineBytes, read, signal) {
|
|
3063
|
+
if (maxInlineBytes !== undefined && treeTotalSize(tree) > maxInlineBytes) {
|
|
3064
|
+
throw inlineLimitRefusal(call, sending, "its files alone weigh", treeTotalSize(tree), maxInlineBytes);
|
|
3065
|
+
}
|
|
3066
|
+
const inline = inlineFilesOf(tree, await readTreeBytesInto(tree, read, signal));
|
|
3067
|
+
if (maxInlineBytes !== undefined) {
|
|
3068
|
+
const weight = inlineMapBytes(inline);
|
|
3069
|
+
if (weight > maxInlineBytes) {
|
|
3070
|
+
throw inlineLimitRefusal(call, sending, "its inline map weighs", weight, maxInlineBytes);
|
|
3071
|
+
}
|
|
3072
|
+
}
|
|
3073
|
+
return inline;
|
|
3074
|
+
}
|
|
3075
|
+
/** The refusal of a tree past the inline limit of a receiver keeping no blob store. */
|
|
3076
|
+
function inlineLimitRefusal(call, sending, measured, weight, limit) {
|
|
3077
|
+
return new RequestArgumentError(`${call}() cannot send the method's tree: the API keeps no blob store, so the tree travels ` +
|
|
3078
|
+
`inline, and ${measured} ${weight} bytes, past the ${limit} bytes it takes inline (its ` +
|
|
3079
|
+
`max_inline_bytes), so ${sending.unsent}.`);
|
|
2561
3080
|
}
|
|
2562
3081
|
/**
|
|
2563
3082
|
* Copy the Pipelex API's own run arguments onto the wire body. Named options,
|
|
@@ -2585,14 +3104,20 @@ function buildHostedRunExtensions(options) {
|
|
|
2585
3104
|
return methodId === undefined ? {} : { method_id: methodId };
|
|
2586
3105
|
}
|
|
2587
3106
|
/**
|
|
2588
|
-
* The
|
|
2589
|
-
*
|
|
3107
|
+
* The forms of the method's tree a protocol route (a run, `validate`) was given, checked as the
|
|
3108
|
+
* MTHDS standard checks them: its `assertOneMethodSource` on `files` and `tree` (both at once, or
|
|
3109
|
+
* either holding no file, refused whatever else the request carries), its refusal rethrown as a
|
|
2590
3110
|
* `RequestArgumentError` with the same message and the standard's error as `cause`, so it carries
|
|
2591
|
-
* a verdict like every other argument the client refuses
|
|
3111
|
+
* a verdict like every other argument the client refuses; and `contents` holding no file refused
|
|
3112
|
+
* the same way. A `null` member and an empty `directory` are absent.
|
|
2592
3113
|
*/
|
|
2593
|
-
function
|
|
3114
|
+
function protocolTreeForms(call, options) {
|
|
3115
|
+
// A `tree` the standard's check would read as a tree is checked as one first, so a value that
|
|
3116
|
+
// is not one is refused as the caller's argument rather than failing as a native TypeError.
|
|
3117
|
+
if (options.tree != null)
|
|
3118
|
+
checkedTreeSource(call, options.tree);
|
|
2594
3119
|
try {
|
|
2595
|
-
|
|
3120
|
+
assertOneMethodSource({ files: options.files ?? null, tree: options.tree ?? null });
|
|
2596
3121
|
}
|
|
2597
3122
|
catch (err) {
|
|
2598
3123
|
if (err instanceof PipelineRequestError) {
|
|
@@ -2600,32 +3125,59 @@ function assertRunSourcesExclusive(options) {
|
|
|
2600
3125
|
}
|
|
2601
3126
|
throw err;
|
|
2602
3127
|
}
|
|
3128
|
+
const contents = options.contents ?? undefined;
|
|
3129
|
+
if (contents !== undefined &&
|
|
3130
|
+
typeof contents === "object" &&
|
|
3131
|
+
Object.keys(contents).length === 0) {
|
|
3132
|
+
throw emptyTreeRefusal(call, "contents", REQUEST_SENDING);
|
|
3133
|
+
}
|
|
3134
|
+
return {
|
|
3135
|
+
directory: typeof options.directory === "string"
|
|
3136
|
+
? nonEmptyString(options.directory)
|
|
3137
|
+
: (options.directory ?? undefined),
|
|
3138
|
+
contents,
|
|
3139
|
+
tree: options.tree ?? undefined,
|
|
3140
|
+
files: options.files ?? undefined,
|
|
3141
|
+
};
|
|
3142
|
+
}
|
|
3143
|
+
/**
|
|
3144
|
+
* The names of the forms of the method's tree `input` gives, in the order the refusals name them.
|
|
3145
|
+
* `files` counts when it is not `undefined`, since a save's `files: null` empties a draft; the
|
|
3146
|
+
* other forms count when they are neither `undefined` nor `null`, and `directory` when it is not
|
|
3147
|
+
* empty either. A route that reads `files: null` as absent normalizes it before asking.
|
|
3148
|
+
*/
|
|
3149
|
+
function treeFormsGiven(input) {
|
|
3150
|
+
const given = [];
|
|
3151
|
+
if (input.directory != null && input.directory !== "")
|
|
3152
|
+
given.push("directory");
|
|
3153
|
+
if (input.contents != null)
|
|
3154
|
+
given.push("contents");
|
|
3155
|
+
if (input.tree != null)
|
|
3156
|
+
given.push("tree");
|
|
3157
|
+
if (input.files !== undefined)
|
|
3158
|
+
given.push("files");
|
|
3159
|
+
return given;
|
|
2603
3160
|
}
|
|
2604
3161
|
/**
|
|
2605
3162
|
* Enforce the run routes' `method_ref` exclusivity, mirroring the server's own
|
|
2606
|
-
* 422s so an illegal pairing fails before anything
|
|
2607
|
-
* `method_ref` is a complete run source (the fetched package carries its
|
|
2608
|
-
*
|
|
2609
|
-
*
|
|
2610
|
-
*
|
|
3163
|
+
* 422s so an illegal pairing fails before anything is listed, read or sent. A
|
|
3164
|
+
* `method_ref` is a complete run source (the fetched package carries its whole
|
|
3165
|
+
* tree and its entry pipe), so it pairs with NOTHING: not with the method's tree
|
|
3166
|
+
* in any form, and not with the hosted `method_id` — an address run has its own
|
|
3167
|
+
* provenance and needs no linkage id.
|
|
2611
3168
|
*
|
|
2612
|
-
* The one documented run-route exception is deliberately NOT here:
|
|
2613
|
-
*
|
|
2614
|
-
*
|
|
2615
|
-
* `
|
|
2616
|
-
*
|
|
2617
|
-
* mirrors the server's validator; presence semantics match it as well
|
|
2618
|
-
* (`mthds_contents` counts when non-empty, a bundle encoding counts when the
|
|
2619
|
-
* key is present, the selectors count when non-empty).
|
|
3169
|
+
* The one documented run-route exception is deliberately NOT here: a tree +
|
|
3170
|
+
* `method_id` stays legal (the tree runs; the id demotes to run-history linkage),
|
|
3171
|
+
* its one condition, a bare id, held by `assertLinkageMethodIdIsBare` below.
|
|
3172
|
+
* `pipe_code` beside a `method_ref` is legal too — it overrides the manifest's
|
|
3173
|
+
* `main_pipe`. The selectors count when non-empty.
|
|
2620
3174
|
*/
|
|
2621
|
-
function assertMethodRefPairsWithNothing(options) {
|
|
3175
|
+
function assertMethodRefPairsWithNothing(options, treeForms) {
|
|
2622
3176
|
if (nonEmptyString(options.method_ref) === undefined)
|
|
2623
3177
|
return;
|
|
2624
|
-
if (
|
|
2625
|
-
throw new RequestArgumentError(
|
|
2626
|
-
|
|
2627
|
-
if (options.files != null || options.bundle_b64 != null) {
|
|
2628
|
-
throw new RequestArgumentError("method_ref and a method bundle (bundle_b64 / files) are mutually exclusive; send one or the other.");
|
|
3178
|
+
if (treeForms.length > 0) {
|
|
3179
|
+
throw new RequestArgumentError(`method_ref and the method's tree (${treeForms.join(", ")}) are mutually exclusive: a ` +
|
|
3180
|
+
"published method's tree is its package's own. Send one or the other.");
|
|
2629
3181
|
}
|
|
2630
3182
|
if (nonEmptyString(options.method_id) !== undefined) {
|
|
2631
3183
|
throw new RequestArgumentError("method_ref and method_id are mutually exclusive: an address run carries its own provenance " +
|
|
@@ -2634,79 +3186,24 @@ function assertMethodRefPairsWithNothing(options) {
|
|
|
2634
3186
|
}
|
|
2635
3187
|
/**
|
|
2636
3188
|
* Enforce the run routes' linkage clause, mirroring the platform's own `422` so a
|
|
2637
|
-
* suffixed linkage id fails before anything
|
|
2638
|
-
*
|
|
2639
|
-
*
|
|
2640
|
-
*
|
|
2641
|
-
*
|
|
2642
|
-
*
|
|
2643
|
-
*
|
|
2644
|
-
* `mthds_contents` when non-empty and a bundle encoding when `hasBundlePayload` says
|
|
2645
|
-
* it carries one, since `nonEmptyFiles` and `nonEmptyString` drop an empty `files: {}`
|
|
2646
|
-
* or `bundle_b64: ""` from the body, and a suffixed id beside one then runs the version
|
|
2647
|
-
* it names. That differs on purpose from `assertMethodRefPairsWithNothing`, which
|
|
2648
|
-
* mirrors the platform's presence-based pairing validator. Only the suffix is checked,
|
|
2649
|
-
* by its `@`, since the catalog id's alphabet has none: the id itself stays a
|
|
2650
|
-
* pass-through the platform resolves.
|
|
3189
|
+
* suffixed linkage id fails before anything is listed, read or sent. Beside the
|
|
3190
|
+
* method's tree, in any form, the `method_id` is run-history linkage and must be a
|
|
3191
|
+
* bare catalog id: the tree is what runs, so a version suffix (`mt_…@3`,
|
|
3192
|
+
* `mt_…@draft`) would claim a version that did not. A `method_id` alone keeps its
|
|
3193
|
+
* suffix, which names the version to run. Only the suffix is checked, by its `@`,
|
|
3194
|
+
* since the catalog id's alphabet has none: the id itself stays a pass-through the
|
|
3195
|
+
* platform resolves.
|
|
2651
3196
|
*/
|
|
2652
|
-
function assertLinkageMethodIdIsBare(options) {
|
|
3197
|
+
function assertLinkageMethodIdIsBare(options, treeForms) {
|
|
2653
3198
|
const methodId = nonEmptyString(options.method_id);
|
|
2654
3199
|
if (methodId === undefined || !methodId.includes("@"))
|
|
2655
3200
|
return;
|
|
2656
|
-
|
|
2657
|
-
hasBundlePayload(options);
|
|
2658
|
-
if (!hasInlineSource)
|
|
3201
|
+
if (treeForms.length === 0)
|
|
2659
3202
|
return;
|
|
2660
|
-
throw new RequestArgumentError(`method_id "${methodId}" beside
|
|
2661
|
-
"catalog id: the
|
|
2662
|
-
"
|
|
2663
|
-
"
|
|
2664
|
-
}
|
|
2665
|
-
/**
|
|
2666
|
-
* Enforce the run routes' `resources` pairing, mirroring the runner's own `422`s
|
|
2667
|
-
* so an illegal pairing fails before anything is encoded or sent. The files a
|
|
2668
|
-
* bundle's steps read accompany that bundle: inline `mthds_contents` or a method
|
|
2669
|
-
* bundle. Beside a `method_ref` they are refused, a published package's files
|
|
2670
|
-
* being its own, and so they are with no source at all, where no step could read
|
|
2671
|
-
* them. A hosted `method_id` without an inline source is left to the server: the
|
|
2672
|
-
* platform resolves it into the bundle the runner then reads them with.
|
|
2673
|
-
*
|
|
2674
|
-
* Presence follows the other pairing checks: `resources` counts when it carries a
|
|
2675
|
-
* file, `mthds_contents` when non-empty, a bundle when `hasBundlePayload` says so.
|
|
2676
|
-
*/
|
|
2677
|
-
function assertResourcesAccompanyABundle(options) {
|
|
2678
|
-
if (!hasBundleResources(options.resources))
|
|
2679
|
-
return;
|
|
2680
|
-
if (nonEmptyString(options.method_ref) !== undefined) {
|
|
2681
|
-
throw new RequestArgumentError("resources accompany inline mthds_contents or a method bundle; a method_ref's files are the package's own.");
|
|
2682
|
-
}
|
|
2683
|
-
const hasInlineSource = (options.mthds_contents != null && options.mthds_contents.length > 0) ||
|
|
2684
|
-
hasBundlePayload(options);
|
|
2685
|
-
if (!hasInlineSource && nonEmptyString(options.method_id) === undefined) {
|
|
2686
|
-
throw new RequestArgumentError("resources accompany the bundle whose steps read them: send them with mthds_contents or a method bundle.");
|
|
2687
|
-
}
|
|
2688
|
-
}
|
|
2689
|
-
/**
|
|
2690
|
-
* A crate request as the wire takes it: the request as given, its `resources`
|
|
2691
|
-
* encoded into the base64 map, the key left off when there is nothing to send.
|
|
2692
|
-
* A request that carries no `resources` key is posted untouched.
|
|
2693
|
-
*/
|
|
2694
|
-
async function wireCrateRequest(request) {
|
|
2695
|
-
if (typeof request !== "object" || request === null || !("resources" in request)) {
|
|
2696
|
-
return request;
|
|
2697
|
-
}
|
|
2698
|
-
const { resources, ...rest } = request;
|
|
2699
|
-
const encoded = await encodeBundleResources(resources);
|
|
2700
|
-
return encoded === undefined ? rest : { ...rest, resources: encoded };
|
|
2701
|
-
}
|
|
2702
|
-
/**
|
|
2703
|
-
* Normalize a bundle encoding for the wire: an empty map / string is NOT a
|
|
2704
|
-
* runnable bundle, so it must not be sent (the runner rejects a zero-file
|
|
2705
|
-
* bundle). Exclusivity is still checked on presence upstream, so an empty
|
|
2706
|
-
* encoding supplied alongside another source has already been rejected.
|
|
2707
|
-
*/
|
|
2708
|
-
function nonEmptyFiles(files) {
|
|
2709
|
-
return files != null && Object.keys(files).length > 0 ? files : undefined;
|
|
3203
|
+
throw new RequestArgumentError(`method_id "${methodId}" beside the method's tree is run-history linkage and must be a bare ` +
|
|
3204
|
+
"catalog id: the tree is what runs, so a version suffix would claim a version that did " +
|
|
3205
|
+
"not. Send the bare id (parseMethodSelector(...).method_id), or drop the tree to run the " +
|
|
3206
|
+
"version the selector names.");
|
|
2710
3207
|
}
|
|
2711
3208
|
function nonEmptyString(value) {
|
|
2712
3209
|
return value != null && value.length > 0 ? value : undefined;
|
|
@@ -2813,6 +3310,54 @@ const UNDECODABLE_BODY_CODE = /^(Z_|ERR__ERROR_|ZSTD_error_)/;
|
|
|
2813
3310
|
function isUndecodableBody(error) {
|
|
2814
3311
|
return error.code !== undefined && UNDECODABLE_BODY_CODE.test(error.code);
|
|
2815
3312
|
}
|
|
3313
|
+
/** A SHA-256 as the platform names a blob: 64 lowercase hexadecimal digits. */
|
|
3314
|
+
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
|
3315
|
+
/** `res`, marked as an answer whose body carries signed links (`RawResponse.carriesLinks`). */
|
|
3316
|
+
function withLinks(res) {
|
|
3317
|
+
return { ...res, carriesLinks: true };
|
|
3318
|
+
}
|
|
3319
|
+
/** Whether one entry of a `POST /v1/blobs/missing` answer is a blob with its upload target. */
|
|
3320
|
+
function isMissingBlob(entry) {
|
|
3321
|
+
if (!isPlainObject(entry))
|
|
3322
|
+
return false;
|
|
3323
|
+
const { sha256, size, upload } = entry;
|
|
3324
|
+
if (typeof sha256 !== "string" || !SHA256_HEX.test(sha256))
|
|
3325
|
+
return false;
|
|
3326
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0)
|
|
3327
|
+
return false;
|
|
3328
|
+
if (!isPlainObject(upload))
|
|
3329
|
+
return false;
|
|
3330
|
+
if (typeof upload.method !== "string" || typeof upload.url !== "string")
|
|
3331
|
+
return false;
|
|
3332
|
+
const headers = upload.headers;
|
|
3333
|
+
return (isPlainObject(headers) && Object.values(headers).every((value) => typeof value === "string"));
|
|
3334
|
+
}
|
|
3335
|
+
/**
|
|
3336
|
+
* Why a saved tree's listing is not one the SDK reads, or `null` when it is: an array of objects,
|
|
3337
|
+
* each with a string `path`, `sha256`, `url` and `expires_at` and a non-negative integer `size`.
|
|
3338
|
+
*/
|
|
3339
|
+
function savedFilesFault(files) {
|
|
3340
|
+
if (!Array.isArray(files))
|
|
3341
|
+
return "is not an array";
|
|
3342
|
+
for (let index = 0; index < files.length; index += 1) {
|
|
3343
|
+
const file = files[index];
|
|
3344
|
+
if (!isPlainObject(file))
|
|
3345
|
+
return `holds an entry that is not an object, at [${index}]`;
|
|
3346
|
+
for (const member of ["path", "sha256", "url", "expires_at"]) {
|
|
3347
|
+
if (typeof file[member] !== "string") {
|
|
3348
|
+
return `holds an entry whose \`${member}\` is not a string, at [${index}]`;
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
if (!SHA256_HEX.test(file.sha256)) {
|
|
3352
|
+
return `holds an entry whose \`sha256\` is not 64 lowercase hex digits, at [${index}]`;
|
|
3353
|
+
}
|
|
3354
|
+
const size = file.size;
|
|
3355
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0) {
|
|
3356
|
+
return `holds an entry whose \`size\` is not a non-negative integer, at [${index}]`;
|
|
3357
|
+
}
|
|
3358
|
+
}
|
|
3359
|
+
return null;
|
|
3360
|
+
}
|
|
2816
3361
|
function extractNetworkErrorCode(err) {
|
|
2817
3362
|
if (err instanceof DOMException && err.name === "TimeoutError") {
|
|
2818
3363
|
return "ABORT_TIMEOUT";
|