@pipelex/sdk 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -4
- package/dist/cli/script.d.ts +1 -1
- package/dist/client.d.ts +168 -61
- package/dist/client.js +360 -120
- package/dist/client.js.map +1 -1
- package/dist/errors.d.ts +45 -18
- package/dist/errors.js +46 -20
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/method-trees.d.ts +209 -0
- package/dist/method-trees.js +689 -0
- package/dist/method-trees.js.map +1 -0
- package/dist/product-models.d.ts +97 -73
- package/dist/runs.d.ts +5 -3
- package/dist/runs.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/docs/architecture.md +5 -5
- package/docs/bundle-resources.md +1 -1
- package/docs/crate-routes.md +2 -2
- package/docs/errors.md +3 -2
- package/docs/input-preparation.md +5 -30
- package/docs/method-trees.md +101 -0
- package/package.json +2 -2
- package/dist/method-source.d.ts +0 -19
- package/dist/method-source.js +0 -50
- package/dist/method-source.js.map +0 -1
package/dist/client.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// Run-source predicates come from the standard, not a local restatement: which
|
|
2
2
|
// source combinations are legal is an invariant of `RunRequest` itself, so this
|
|
3
3
|
// client and the MTHDS runners cannot disagree about what they reject.
|
|
4
|
-
import { assertExclusiveRunSources, hasBundlePayload, parseJsonBody
|
|
4
|
+
import { assertExclusiveRunSources, hasBundlePayload, parseJsonBody } from "mthds/protocol";
|
|
5
5
|
import { assertArtifactSelection, assertWaitOptions, pollUntilResult, readRunDegradation, RUN_STATUSES, runProgressFault, runStatusFilter, selectionIncludesMainStuff, throwIfAborted, withCheckedProgressDegradations, } from "./runs.js";
|
|
6
|
-
import { ApiResponseError, ApiUnreachableError,
|
|
6
|
+
import { ApiResponseError, ApiUnreachableError, MissingMainStuffError, PagingNotTerminatingError, PipelineExecuteTimeoutError, PipelineRequestError, RequestArgumentError, RunLifecycleUnavailableError, RunStillRunningError, } from "./errors.js";
|
|
7
7
|
import { isPlainObject, isValidationItem, readFieldError, readMigration, readProviderMetadata, readRunErrorReport, } from "./error-models.js";
|
|
8
|
-
import {
|
|
8
|
+
import { INLINE_TREE_MAX_BYTES, MAX_CONCURRENT_TRANSFERS, announcedTreeLimits, contentsTreeSource, declaresBlobStore, fileByBlob, forEachConcurrently, hashMethodFiles, inlineFilesOf, inlineMapBytes, readTreeBytes, redactSignedLinks, sha256Hex, treeTotalSize, uniqueBlobs, uploadBlob, writeMethodTreeTo, } from "./method-trees.js";
|
|
9
9
|
import { buildUserAgent } from "./user-agent.js";
|
|
10
10
|
import { uploadFile as uploadFileImpl } from "./upload.js";
|
|
11
11
|
import { encodeBundleResources, hasBundleResources } from "./bundle-resources.js";
|
|
@@ -123,32 +123,30 @@ const VALIDATE_MARKDOWN_RENDER_FORMAT = "markdown";
|
|
|
123
123
|
*/
|
|
124
124
|
const BARE_RUNNER_IMPLEMENTATION = "pipelex-api";
|
|
125
125
|
/**
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
126
|
+
* Client for the Pipelex hosted API — and any MTHDS-compliant runner.
|
|
127
|
+
*
|
|
128
|
+
* One base URL (`PIPELEX_BASE_URL`); every endpoint is `<base>/v1/<endpoint>`:
|
|
129
|
+
* - **protocol** (`execute` / `start` / `validate` / `models` / `version`) — works
|
|
130
|
+
* against any MTHDS-compliant runner, hosted or bare.
|
|
131
|
+
* - **crate extensions** (`/v1/resolve`, `/v1/codegen`, `/v1/pipe-io`) — the normalized
|
|
132
|
+
* library crate, the stamped typed artifacts projected from it, and a method's three
|
|
133
|
+
* I/O artifacts derived with no dry run.
|
|
134
|
+
* - **tools extensions** (`lint` / `format`) — single-file static diagnostics and
|
|
135
|
+
* canonical formatting, served by any pipelex-api runner.
|
|
136
|
+
* - **run lifecycle** (`getRunStatus` / `getRunProgress` / `getRunResult` /
|
|
137
|
+
* `waitForResult`) — the durable polling extension that survives long runs and
|
|
138
|
+
* lets a caller resume by id. Served only by a deployment that includes the
|
|
139
|
+
* platform block (the Pipelex Hosted API); a bare `pipelex-api` runner 404s those
|
|
140
|
+
* routes, which the lifecycle methods translate into a clear
|
|
141
|
+
* `RunLifecycleUnavailableError`.
|
|
142
|
+
*
|
|
143
|
+
* Implements `MTHDSProtocol<DictPipeOutput>` so the protocol-execution methods
|
|
144
|
+
* stay shaped like the standard's wire surface (`mthds/protocol`), all but
|
|
145
|
+
* `validate`, whose verdict is the one the hosted plane answers: since protocol
|
|
146
|
+
* `2.0.0` the protocol's verdict lists its faults as problem items, which the
|
|
147
|
+
* hosted plane does not send yet (see `PipelexValidationResult`). The Pipelex
|
|
148
|
+
* extensions (the crate and tools routes, the run lifecycle) ride on top.
|
|
147
149
|
*/
|
|
148
|
-
function withWirePython(input) {
|
|
149
|
-
const { python, ...rest } = input;
|
|
150
|
-
return python === undefined ? rest : { ...rest, python: serializeMethodFiles(python) };
|
|
151
|
-
}
|
|
152
150
|
/**
|
|
153
151
|
* A request's body as the JSON text it is sent as, written by `stringifyExactJson` in one pass,
|
|
154
152
|
* so a `bigint` read from an answer carrying a person's data goes back as its exact digits, and
|
|
@@ -324,6 +322,14 @@ export class PipelexApiClient {
|
|
|
324
322
|
originUrl;
|
|
325
323
|
/** Cached `/v1/version` handshake outcome — whether the durable lifecycle is served. */
|
|
326
324
|
lifecycleAvailable;
|
|
325
|
+
/**
|
|
326
|
+
* The first `/v1/version` handshake that answered, kept for the client's lifetime: what a save
|
|
327
|
+
* reads from it (`blob_store`) and the tree limits it announces do not change while a client
|
|
328
|
+
* lives. A handshake that failed is not kept, so the next call asks again.
|
|
329
|
+
*/
|
|
330
|
+
versionInfo;
|
|
331
|
+
/** The client's request ceiling (`PipelexApiClientOptions.requestTimeoutMs`). */
|
|
332
|
+
requestTimeoutMs;
|
|
327
333
|
/**
|
|
328
334
|
* The `User-Agent` sent on every API request, computed once at construction;
|
|
329
335
|
* `undefined` in a browser, where no header may be set.
|
|
@@ -351,6 +357,13 @@ export class PipelexApiClient {
|
|
|
351
357
|
}
|
|
352
358
|
this.baseUrl = normalizedBaseUrl;
|
|
353
359
|
this.originUrl = new URL("/", this.baseUrl).origin;
|
|
360
|
+
const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
361
|
+
// A longer delay overflows the timer, which then fires at once as a false timeout.
|
|
362
|
+
if (!isTimerDelay(requestTimeoutMs)) {
|
|
363
|
+
throw new RangeError(`"requestTimeoutMs" must be a positive number no larger than ${MAX_TIMER_DELAY_MS}, got ` +
|
|
364
|
+
`${String(requestTimeoutMs)}.`);
|
|
365
|
+
}
|
|
366
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
354
367
|
}
|
|
355
368
|
// ── URL resolution ───────────────────────────────────────────────────
|
|
356
369
|
/** Build an API URL: `<base>/v1/<endpoint>`. */
|
|
@@ -395,7 +408,7 @@ export class PipelexApiClient {
|
|
|
395
408
|
// argument rather than caught below and reported as a network failure.
|
|
396
409
|
const bodyText = hasBody ? requestBodyText(method, url, options.body) : undefined;
|
|
397
410
|
const headers = this.requestHeaders(hasBody, options.idempotencyKey);
|
|
398
|
-
const timeoutMs = options.timeoutMs ??
|
|
411
|
+
const timeoutMs = options.timeoutMs ?? this.requestTimeoutMs;
|
|
399
412
|
// A longer delay overflows the timer, which then fires at once as a false timeout.
|
|
400
413
|
if (!isTimerDelay(timeoutMs)) {
|
|
401
414
|
throw new RangeError(`"timeoutMs" must be a positive number no larger than ${MAX_TIMER_DELAY_MS}, got ` +
|
|
@@ -486,28 +499,27 @@ export class PipelexApiClient {
|
|
|
486
499
|
return this.readObjectAnswer(method, endpoint, res);
|
|
487
500
|
}
|
|
488
501
|
/**
|
|
489
|
-
* A methods-catalog route answering one stored method (`
|
|
490
|
-
* `
|
|
502
|
+
* A methods-catalog route answering one stored method (`createMethod`, `writeDraft`,
|
|
503
|
+
* `renameMethod`), read as `readObjectAnswer` reads it.
|
|
491
504
|
*/
|
|
492
|
-
async requestMethodData(method, endpoint, body) {
|
|
493
|
-
const res = await this.requestProductAnswer(method, endpoint, body);
|
|
494
|
-
|
|
495
|
-
return this.readStoredSources(method, endpoint, res, wire, "a stored method");
|
|
505
|
+
async requestMethodData(method, endpoint, body, options = {}) {
|
|
506
|
+
const res = await this.requestProductAnswer(method, endpoint, body, options);
|
|
507
|
+
return this.readObjectAnswer(method, endpoint, res);
|
|
496
508
|
}
|
|
497
509
|
/**
|
|
498
|
-
*
|
|
499
|
-
*
|
|
500
|
-
*
|
|
501
|
-
*
|
|
502
|
-
*
|
|
503
|
-
* change to the call fixes it. `subject` names what carried them in the message.
|
|
510
|
+
* A route answering a saved tree's listing beside what names it (`getMethod`,
|
|
511
|
+
* `getMethodVersion`), read as `readObjectAnswer` reads it, then its `files` checked: a listing
|
|
512
|
+
* that is not an array of `{ path, sha256, size, url, expires_at }`, each a string but `size`, a
|
|
513
|
+
* non-negative integer, is an answer the SDK cannot read, since `writeMethodTree` would write
|
|
514
|
+
* from it. Throws the `ApiResponseError` `unreadableAnswer` builds, `subject` naming the answer.
|
|
504
515
|
*/
|
|
505
|
-
|
|
506
|
-
const
|
|
507
|
-
|
|
508
|
-
|
|
516
|
+
readSavedTree(method, endpoint, res, subject) {
|
|
517
|
+
const answer = this.readObjectAnswer(method, endpoint, res);
|
|
518
|
+
const fault = savedFilesFault(answer.files);
|
|
519
|
+
if (fault !== null) {
|
|
520
|
+
throw this.unreadableAnswer(method, `/${API_PREFIX}/${endpoint}`, res, `${subject} whose \`files\` ${fault}`);
|
|
509
521
|
}
|
|
510
|
-
return
|
|
522
|
+
return answer;
|
|
511
523
|
}
|
|
512
524
|
/**
|
|
513
525
|
* A verdict, the `200` a route answers discriminated on `outcome` with the object the verdict
|
|
@@ -592,11 +604,14 @@ export class PipelexApiClient {
|
|
|
592
604
|
}
|
|
593
605
|
return value;
|
|
594
606
|
}
|
|
595
|
-
/**
|
|
607
|
+
/**
|
|
608
|
+
* Issue a product request and return its 2xx answer, a non-2xx thrown as `ApiResponseError`.
|
|
609
|
+
* The request takes the poll timeout unless `options.timeoutMs` gives another.
|
|
610
|
+
*/
|
|
596
611
|
async requestProductAnswer(method, endpoint, body, options = {}) {
|
|
597
612
|
const res = await this.requestRaw(method, this.url(endpoint), {
|
|
598
613
|
body,
|
|
599
|
-
timeoutMs: POLL_REQUEST_TIMEOUT_MS,
|
|
614
|
+
timeoutMs: options.timeoutMs ?? POLL_REQUEST_TIMEOUT_MS,
|
|
600
615
|
signal: options.signal,
|
|
601
616
|
});
|
|
602
617
|
if (res.status < 200 || res.status >= 300) {
|
|
@@ -673,11 +688,13 @@ export class PipelexApiClient {
|
|
|
673
688
|
* `API <method> <path> answered <status> with <what>`: the answer's status, its raw text as
|
|
674
689
|
* `responseBody`, no hosted problem member but the `X-Request-ID` header, the failure that made it
|
|
675
690
|
* unreadable as `cause`, and the verdict the fallback gives a 2xx, `runtime` and not retryable.
|
|
676
|
-
*
|
|
691
|
+
* An answer that carries signed links (`RawResponse.carriesLinks`) is held with every link
|
|
692
|
+
* hidden, and with no `cause`, which could quote one. On a request that may create a run, the run may have started: the server accepted it.
|
|
677
693
|
*/
|
|
678
694
|
unreadableAnswer(method, path, res, what, cause, run = {}) {
|
|
679
|
-
|
|
680
|
-
|
|
695
|
+
const carriesLinks = res.carriesLinks === true;
|
|
696
|
+
return new ApiResponseError(`API ${method} ${path} answered ${res.status} with ${what}`, this.baseUrl, res.status, res.statusText, carriesLinks ? redactSignedLinks(res.body) : res.body, undefined, undefined, undefined, undefined, {
|
|
697
|
+
cause: carriesLinks ? undefined : cause,
|
|
681
698
|
details: { requestId: nonEmptyHeader(res.headers, REQUEST_ID_HEADER) },
|
|
682
699
|
headers: res.headers,
|
|
683
700
|
mayStartRun: run.mayStartRun === true,
|
|
@@ -910,7 +927,7 @@ export class PipelexApiClient {
|
|
|
910
927
|
nonEmptyString(options.method_ref) !== undefined;
|
|
911
928
|
const res = await this.requestRaw("POST", url, {
|
|
912
929
|
body: request,
|
|
913
|
-
timeoutMs: needsLongCeiling ?
|
|
930
|
+
timeoutMs: needsLongCeiling ? this.requestTimeoutMs : POLL_REQUEST_TIMEOUT_MS,
|
|
914
931
|
idempotencyKey,
|
|
915
932
|
...STARTS_RUN,
|
|
916
933
|
});
|
|
@@ -1297,7 +1314,7 @@ export class PipelexApiClient {
|
|
|
1297
1314
|
*/
|
|
1298
1315
|
async resolve(request) {
|
|
1299
1316
|
return this.requestExtension("resolve", await wireCrateRequest(request), {
|
|
1300
|
-
timeoutMs: crateRequestTimeoutMs(request),
|
|
1317
|
+
timeoutMs: crateRequestTimeoutMs(request, this.requestTimeoutMs),
|
|
1301
1318
|
});
|
|
1302
1319
|
}
|
|
1303
1320
|
/**
|
|
@@ -1317,7 +1334,7 @@ export class PipelexApiClient {
|
|
|
1317
1334
|
*/
|
|
1318
1335
|
async codegen(request) {
|
|
1319
1336
|
return this.requestExtension("codegen", await wireCrateRequest(request), {
|
|
1320
|
-
timeoutMs: crateRequestTimeoutMs(request),
|
|
1337
|
+
timeoutMs: crateRequestTimeoutMs(request, this.requestTimeoutMs),
|
|
1321
1338
|
});
|
|
1322
1339
|
}
|
|
1323
1340
|
/**
|
|
@@ -1359,7 +1376,7 @@ export class PipelexApiClient {
|
|
|
1359
1376
|
*/
|
|
1360
1377
|
async pipeIo(request) {
|
|
1361
1378
|
return this.requestExtension("pipe-io", await wireCrateRequest(request), {
|
|
1362
|
-
timeoutMs: crateRequestTimeoutMs(request),
|
|
1379
|
+
timeoutMs: crateRequestTimeoutMs(request, this.requestTimeoutMs),
|
|
1363
1380
|
});
|
|
1364
1381
|
}
|
|
1365
1382
|
// ── Hosted extension: durable run lifecycle (NOT part of the protocol) ──
|
|
@@ -1523,6 +1540,7 @@ export class PipelexApiClient {
|
|
|
1523
1540
|
if (this.lifecycleAvailable === undefined) {
|
|
1524
1541
|
try {
|
|
1525
1542
|
const info = await this.version();
|
|
1543
|
+
this.versionInfo ??= info;
|
|
1526
1544
|
const impl = info.implementation;
|
|
1527
1545
|
this.lifecycleAvailable = !(typeof impl === "string" && impl === BARE_RUNNER_IMPLEMENTATION);
|
|
1528
1546
|
}
|
|
@@ -1671,10 +1689,13 @@ export class PipelexApiClient {
|
|
|
1671
1689
|
yield* followCursor("listMethods", (cursor) => this.listMethods({ ...query, cursor }));
|
|
1672
1690
|
}
|
|
1673
1691
|
/**
|
|
1674
|
-
* Read one method — `GET /v1/methods/{id}`: its identity, its draft
|
|
1675
|
-
* `input_data`) with the draft's token (`updated_at`) and digest (`draft_digest`),
|
|
1676
|
-
* latest published version's summary (`latest_version`, `latest_published`), whatever
|
|
1677
|
-
* publish state. A method never published reads with both `null`, never as an error.
|
|
1692
|
+
* Read one method with its draft's files — `GET /v1/methods/{id}`: its identity, its draft
|
|
1693
|
+
* (`files`, `input_data`) with the draft's token (`updated_at`) and digest (`draft_digest`),
|
|
1694
|
+
* and its latest published version's summary (`latest_version`, `latest_published`), whatever
|
|
1695
|
+
* the publish state. A method never published reads with both `null`, never as an error.
|
|
1696
|
+
* `files` lists the draft's tree in tree path order, each file with its SHA-256, its size and a
|
|
1697
|
+
* link to its bytes valid for about fifteen minutes, and is empty for an empty draft;
|
|
1698
|
+
* `writeMethodTree` brings the tree to disk.
|
|
1678
1699
|
*
|
|
1679
1700
|
* Takes a bare catalog id: the method routes address the method itself, never a version of
|
|
1680
1701
|
* it. A caller holding `mt_…@3` strips the suffix with `parseMethodSelector` and reads that
|
|
@@ -1682,54 +1703,56 @@ export class PipelexApiClient {
|
|
|
1682
1703
|
* suffixed id, before any request, rather than read back the `404` the platform would answer,
|
|
1683
1704
|
* and for an id that is empty, `.` or `..`, which the URL would resolve into another route's
|
|
1684
1705
|
* path.
|
|
1706
|
+
*
|
|
1707
|
+
* Throws `ApiResponseError`: `404` `not_found` for an unknown or foreign-org method, `409`
|
|
1708
|
+
* `method_being_deleted` while its erasure runs, and a `409` whose verdict is `runtime` for a
|
|
1709
|
+
* method whose draft names a tree the platform holds no record of. An answer whose `files` is
|
|
1710
|
+
* not a listing is an answer the SDK cannot read, thrown as an `ApiResponseError` too.
|
|
1685
1711
|
*/
|
|
1686
1712
|
async getMethod(methodId) {
|
|
1687
|
-
|
|
1713
|
+
const endpoint = methodPath(methodId, "getMethod");
|
|
1714
|
+
const res = await this.requestProductAnswer("GET", endpoint, undefined);
|
|
1715
|
+
return this.readSavedTree("GET", endpoint, withLinks(res), "a stored method");
|
|
1688
1716
|
}
|
|
1689
1717
|
/**
|
|
1690
|
-
*
|
|
1691
|
-
*
|
|
1692
|
-
*
|
|
1693
|
-
* `mthds` source with `methodSourceToContents`, and labels each resulting file
|
|
1694
|
-
* with the `method_id` as its `source` provenance.
|
|
1718
|
+
* Create a method — `POST /v1/methods`. The new method holds its tree as its draft and has no
|
|
1719
|
+
* published version, so its bare id answers `409 method_not_published` on the run and tooling
|
|
1720
|
+
* routes until its first `publishMethod`; its draft runs as `mt_…@draft` at once.
|
|
1695
1721
|
*
|
|
1696
|
-
*
|
|
1697
|
-
*
|
|
1698
|
-
*
|
|
1699
|
-
*
|
|
1722
|
+
* The tree is `files`, sent as given, or `tree` or `contents`, which the client sends inline
|
|
1723
|
+
* when its inline map weighs 256 KiB or less, or when the platform keeps no blob store, and
|
|
1724
|
+
* otherwise every file by hash, after asking `POST /v1/blobs/missing` which files the
|
|
1725
|
+
* organization lacks and uploading those: a file uploaded once is never uploaded again. With
|
|
1726
|
+
* none of the three, the draft is empty (see `docs/method-trees.md`).
|
|
1700
1727
|
*
|
|
1701
|
-
*
|
|
1702
|
-
*
|
|
1703
|
-
*
|
|
1704
|
-
*
|
|
1705
|
-
*
|
|
1706
|
-
*
|
|
1707
|
-
*
|
|
1708
|
-
*
|
|
1709
|
-
* catalog is org-scoped to the key's org, so an unknown OR foreign-org id is a
|
|
1710
|
-
* `getMethod` `404` (`ApiResponseError` `not_found`), which propagates unchanged. A
|
|
1711
|
-
* real, in-org method whose draft parses to nothing throws `EmptyMethodSourceError`
|
|
1712
|
-
* (distinct from the 404) — the row exists but has no runnable source yet.
|
|
1728
|
+
* Throws `RequestArgumentError` for two forms of the tree, or `contents` holding no file, before
|
|
1729
|
+
* any request; `MethodFileTransferError` when a file the organization lacks could not be
|
|
1730
|
+
* uploaded, and nothing is saved. Throws `ApiResponseError`: `422` `invalid_method_files` for a
|
|
1731
|
+
* map the standard refuses, with its `tree_refusal`; `422` `method_tree_too_large` for a tree
|
|
1732
|
+
* past one of the deployment's limits, named in its `limit`; `422` `blob_mismatch` for a hash
|
|
1733
|
+
* stated at another size than the one held; `422` `missing_blobs` for a hash entry whose bytes
|
|
1734
|
+
* the organization does not hold; `413` `payload_too_large` for a method that would leave no
|
|
1735
|
+
* room for a publish.
|
|
1713
1736
|
*/
|
|
1714
|
-
async
|
|
1715
|
-
const
|
|
1716
|
-
const
|
|
1717
|
-
if (
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1737
|
+
async createMethod(input) {
|
|
1738
|
+
const files = await this.methodTreeFiles("createMethod", input);
|
|
1739
|
+
const body = { name: input.name };
|
|
1740
|
+
if (files != null)
|
|
1741
|
+
body.files = files;
|
|
1742
|
+
if (input.input_data !== undefined)
|
|
1743
|
+
body.input_data = input.input_data;
|
|
1744
|
+
return this.requestMethodData("POST", "methods", body, this.saveTimeout(files));
|
|
1721
1745
|
}
|
|
1722
1746
|
/**
|
|
1723
|
-
*
|
|
1724
|
-
*
|
|
1725
|
-
*
|
|
1726
|
-
*
|
|
1727
|
-
* Throws `ApiResponseError`: `413` `payload_too_large` for a method that would leave no room
|
|
1728
|
-
* for a publish, measured as a draft write is, and `422` `validation_failed` for Python files
|
|
1729
|
-
* no run could import or for text holding a lone surrogate.
|
|
1747
|
+
* The time limit of a save carrying `files`: the client's request ceiling, as for a `start`
|
|
1748
|
+
* carrying its method, since a tree sent inline to a deployment with no blob store has no size
|
|
1749
|
+
* bound and its whole upload is charged against the limit; the poll timeout for a save sending
|
|
1750
|
+
* no tree.
|
|
1730
1751
|
*/
|
|
1731
|
-
|
|
1732
|
-
return
|
|
1752
|
+
saveTimeout(files) {
|
|
1753
|
+
return files != null && Object.keys(files).length > 0
|
|
1754
|
+
? { timeoutMs: this.requestTimeoutMs }
|
|
1755
|
+
: {};
|
|
1733
1756
|
}
|
|
1734
1757
|
/**
|
|
1735
1758
|
* Replace a method's draft — `PUT /v1/methods/{id}/draft` — and return the method with the
|
|
@@ -1737,21 +1760,189 @@ export class PipelexApiClient {
|
|
|
1737
1760
|
*
|
|
1738
1761
|
* The draft is never validated on write, and writing it changes nothing for the callers of
|
|
1739
1762
|
* the method's bare id, who run the latest published version until the next `publishMethod`.
|
|
1740
|
-
*
|
|
1763
|
+
* The write replaces the draft's whole tree, given in exactly one form: `files`, sent as given,
|
|
1764
|
+
* `files: null` emptying the draft, or `tree` or `contents`, sent inline or by hash as
|
|
1765
|
+
* `createMethod` sends them. `input_data` is three-way (see `MethodDraftInput`). With
|
|
1741
1766
|
* `expected_updated_at`, the write is a compare-and-swap on the draft token: a draft that
|
|
1742
1767
|
* moved since is refused with a `409` whose `code` is `method_update_conflict`, and nothing is
|
|
1743
1768
|
* written. Without it, last writer wins.
|
|
1744
1769
|
*
|
|
1745
|
-
* Takes a bare catalog id, as `getMethod` does. Throws `
|
|
1746
|
-
*
|
|
1747
|
-
* `
|
|
1748
|
-
*
|
|
1749
|
-
*
|
|
1750
|
-
*
|
|
1751
|
-
*
|
|
1770
|
+
* Takes a bare catalog id, as `getMethod` does. Throws `RequestArgumentError` for a draft that
|
|
1771
|
+
* gives its tree in no form or in two, or `contents` holding no file, before any request;
|
|
1772
|
+
* `MethodFileTransferError` when a file the organization lacks could not be uploaded. Throws
|
|
1773
|
+
* `ApiResponseError`: `404` `not_found` for an unknown or foreign-org method, `409`
|
|
1774
|
+
* `method_being_deleted` while its erasure runs, the `422` refusals of a tree `createMethod`
|
|
1775
|
+
* lists, `413` `payload_too_large` for a draft that would grow the method past the room a
|
|
1776
|
+
* publish needs, `403` for a read-only key, and a `503` that wrote nothing, safe to retry,
|
|
1777
|
+
* when the method kept changing under the write.
|
|
1752
1778
|
*/
|
|
1753
1779
|
async writeDraft(methodId, input) {
|
|
1754
|
-
|
|
1780
|
+
const endpoint = `${methodPath(methodId, "writeDraft")}/draft`;
|
|
1781
|
+
if (input.files === undefined && input.tree == null && input.contents == null) {
|
|
1782
|
+
throw new RequestArgumentError("writeDraft() replaces the draft's whole tree: give it as files (files: null empties the " +
|
|
1783
|
+
"draft), tree or contents.");
|
|
1784
|
+
}
|
|
1785
|
+
const files = await this.methodTreeFiles("writeDraft", input);
|
|
1786
|
+
const body = { files: files ?? null };
|
|
1787
|
+
if (input.input_data !== undefined)
|
|
1788
|
+
body.input_data = input.input_data;
|
|
1789
|
+
if (input.expected_updated_at !== undefined) {
|
|
1790
|
+
body.expected_updated_at = input.expected_updated_at;
|
|
1791
|
+
}
|
|
1792
|
+
return this.requestMethodData("PUT", endpoint, body, this.saveTimeout(files));
|
|
1793
|
+
}
|
|
1794
|
+
/**
|
|
1795
|
+
* The limits the deployment holds a method's tree to — `GET /v1/version`, read from
|
|
1796
|
+
* `limits.method_tree` of the public handshake — in the shape the `mthds` package's
|
|
1797
|
+
* `listMethodTree` takes as its `limits` option, so a tree the platform would refuse is refused
|
|
1798
|
+
* while it is listed, before any file is hashed:
|
|
1799
|
+
*
|
|
1800
|
+
* ```ts
|
|
1801
|
+
* import { listMethodTree } from "mthds";
|
|
1802
|
+
* const tree = await listMethodTree("methods/receipts", { limits: await client.methodTreeLimits() });
|
|
1803
|
+
* await client.createMethod({ name: "Receipts", tree });
|
|
1804
|
+
* ```
|
|
1805
|
+
*
|
|
1806
|
+
* A limit the server does not announce is absent, so a bare runner, which announces none,
|
|
1807
|
+
* yields no limit, and none is ever assumed. A value that is not a positive integer is left out
|
|
1808
|
+
* rather than trusted.
|
|
1809
|
+
*/
|
|
1810
|
+
async methodTreeLimits() {
|
|
1811
|
+
return announcedTreeLimits(await this.cachedVersion());
|
|
1812
|
+
}
|
|
1813
|
+
/**
|
|
1814
|
+
* The version handshake, asked once per client: the first answer is kept (see `versionInfo`),
|
|
1815
|
+
* and a failure is thrown and not kept, so the next call asks again.
|
|
1816
|
+
*/
|
|
1817
|
+
async cachedVersion() {
|
|
1818
|
+
this.versionInfo ??= await this.version();
|
|
1819
|
+
return this.versionInfo;
|
|
1820
|
+
}
|
|
1821
|
+
/**
|
|
1822
|
+
* Write a saved method's tree to disk: each file of a listing `getMethod` or `getMethodVersion`
|
|
1823
|
+
* served, at its tree path under `options.destination`. **Node only.**
|
|
1824
|
+
*
|
|
1825
|
+
* Each file is downloaded through its link, at most eight at a time, with no credential of this
|
|
1826
|
+
* client, and checked against the listing: its size and its SHA-256 must be the ones listed, so
|
|
1827
|
+
* what lands on disk is the tree the method's digest names. The files are written into a fresh
|
|
1828
|
+
* hidden directory beside the destination, which takes its name only once every file is
|
|
1829
|
+
* written: an interrupted or failed write never leaves a partial tree under that name. The
|
|
1830
|
+
* destination must not exist, or be an empty directory; one holding anything is refused rather
|
|
1831
|
+
* than written into or replaced. An empty listing writes an empty directory.
|
|
1832
|
+
*
|
|
1833
|
+
* The links expire about fifteen minutes after the read that served them: write the tree soon
|
|
1834
|
+
* after reading the method, and read it again for fresh links when a download is refused.
|
|
1835
|
+
*
|
|
1836
|
+
* Throws `RequestArgumentError` before any download when the destination exists and is not an
|
|
1837
|
+
* empty directory, or the listing holds a path that is not a tree path, the same path twice, or
|
|
1838
|
+
* paths that collide once case-folded or where one names a directory of another;
|
|
1839
|
+
* `MethodFileTransferError` when a file could not be downloaded, or its bytes are not the file
|
|
1840
|
+
* listed, and nothing is left under the destination.
|
|
1841
|
+
*/
|
|
1842
|
+
async writeMethodTree(files, options) {
|
|
1843
|
+
await writeMethodTreeTo(files, options.destination, { timeoutMs: this.requestTimeoutMs });
|
|
1844
|
+
}
|
|
1845
|
+
/**
|
|
1846
|
+
* The `files` a save sends: `files` as given, or `tree` or `contents` prepared, uploading what
|
|
1847
|
+
* the organization lacks; `undefined` when the input gives none. Two forms are refused with
|
|
1848
|
+
* `RequestArgumentError` before any request, naming `call`.
|
|
1849
|
+
*/
|
|
1850
|
+
async methodTreeFiles(call, input) {
|
|
1851
|
+
const given = [
|
|
1852
|
+
["files", input.files !== undefined],
|
|
1853
|
+
["tree", input.tree != null],
|
|
1854
|
+
["contents", input.contents != null],
|
|
1855
|
+
]
|
|
1856
|
+
.filter(([, present]) => present)
|
|
1857
|
+
.map(([name]) => name);
|
|
1858
|
+
if (given.length > 1) {
|
|
1859
|
+
throw new RequestArgumentError(`${call}() takes the method's tree in one form, files, tree or contents; got ` +
|
|
1860
|
+
`${given.join(" and ")}.`);
|
|
1861
|
+
}
|
|
1862
|
+
if (input.tree != null)
|
|
1863
|
+
return this.prepareTreeFiles(input.tree, call);
|
|
1864
|
+
if (input.contents != null) {
|
|
1865
|
+
return this.prepareTreeFiles(await contentsTreeSource(input.contents, call), call);
|
|
1866
|
+
}
|
|
1867
|
+
return input.files;
|
|
1868
|
+
}
|
|
1869
|
+
/**
|
|
1870
|
+
* The `files` map that sends `tree`: inline, or by hash once the organization holds every file.
|
|
1871
|
+
*
|
|
1872
|
+
* A tree whose inline map weighs `INLINE_TREE_MAX_BYTES` or less travels inline, with no round
|
|
1873
|
+
* trip, measured as the request spends it, its JSON with no whitespace in UTF-8. A larger tree
|
|
1874
|
+
* travels inline too when the version handshake declares no blob store; otherwise
|
|
1875
|
+
* `POST /v1/blobs/missing` is asked which files the organization lacks, naming the tree's
|
|
1876
|
+
* digest, each one it lacks is uploaded once, and every file travels by hash. A file read to be
|
|
1877
|
+
* measured is uploaded from memory rather than read a second time. A tree holding no file is
|
|
1878
|
+
* refused with `RequestArgumentError` before any request, `call` naming the caller's method.
|
|
1879
|
+
*/
|
|
1880
|
+
async prepareTreeFiles(tree, call) {
|
|
1881
|
+
if (tree.files.length === 0) {
|
|
1882
|
+
throw new RequestArgumentError(`${call}() got a tree holding no file, and a method tree holds at least one; give ` +
|
|
1883
|
+
"files: null for an empty draft.");
|
|
1884
|
+
}
|
|
1885
|
+
let read;
|
|
1886
|
+
// An inline map is never smaller than the bytes it carries, so a tree whose bytes alone pass
|
|
1887
|
+
// the threshold is not read in full just to be measured.
|
|
1888
|
+
if (treeTotalSize(tree) <= INLINE_TREE_MAX_BYTES) {
|
|
1889
|
+
read = await readTreeBytes(tree);
|
|
1890
|
+
const inline = inlineFilesOf(tree, read);
|
|
1891
|
+
if (inlineMapBytes(inline) <= INLINE_TREE_MAX_BYTES)
|
|
1892
|
+
return inline;
|
|
1893
|
+
}
|
|
1894
|
+
if (!declaresBlobStore(await this.cachedVersion())) {
|
|
1895
|
+
return inlineFilesOf(tree, read ?? (await readTreeBytes(tree)));
|
|
1896
|
+
}
|
|
1897
|
+
const asked = await this.findMissingBlobs({ blobs: uniqueBlobs(tree), digest: tree.digest });
|
|
1898
|
+
await this.uploadMissing(tree, asked, read);
|
|
1899
|
+
return hashMethodFiles(tree);
|
|
1900
|
+
}
|
|
1901
|
+
/**
|
|
1902
|
+
* Which of these blobs the organization lacks, and where to upload each —
|
|
1903
|
+
* `POST /v1/blobs/missing` — with the answer it was read from, which carries signed links. A
|
|
1904
|
+
* blob held is left out, and an empty `missing` means nothing to upload. An answer whose
|
|
1905
|
+
* `missing` is not a list of blobs, each with its upload target, is an answer the SDK cannot
|
|
1906
|
+
* read, thrown as an `ApiResponseError` holding the answer with its links hidden.
|
|
1907
|
+
*/
|
|
1908
|
+
async findMissingBlobs(request) {
|
|
1909
|
+
const endpoint = "blobs/missing";
|
|
1910
|
+
const res = withLinks(await this.requestProductAnswer("POST", endpoint, request));
|
|
1911
|
+
const answer = this.readObjectAnswer("POST", endpoint, res);
|
|
1912
|
+
const missing = answer.missing;
|
|
1913
|
+
if (!Array.isArray(missing) || !missing.every(isMissingBlob)) {
|
|
1914
|
+
throw this.unreadableAnswer("POST", `/${API_PREFIX}/${endpoint}`, res, "a body whose `missing` is not a list of blobs, each with its upload target");
|
|
1915
|
+
}
|
|
1916
|
+
return { result: answer, res };
|
|
1917
|
+
}
|
|
1918
|
+
/**
|
|
1919
|
+
* Upload each missing blob from the tree's bytes, a few at a time, stopping at the first
|
|
1920
|
+
* failure. A blob the platform asks for that the tree does not hold, or holds at another size,
|
|
1921
|
+
* is the platform's fault, thrown before anything is sent as the `ApiResponseError` of an answer
|
|
1922
|
+
* the SDK cannot read. The bytes the tree reads for a blob are checked against it before they
|
|
1923
|
+
* are sent, since storage keeps a blob under the hash it is named by: bytes of another hash or
|
|
1924
|
+
* size are refused with `RequestArgumentError`, and nothing more is uploaded.
|
|
1925
|
+
*/
|
|
1926
|
+
async uploadMissing(tree, asked, read) {
|
|
1927
|
+
const missing = asked.result.missing;
|
|
1928
|
+
const byBlob = fileByBlob(tree);
|
|
1929
|
+
const strangers = missing
|
|
1930
|
+
.filter((blob) => byBlob.get(blob.sha256)?.size !== blob.size)
|
|
1931
|
+
.map((blob) => blob.sha256);
|
|
1932
|
+
if (strangers.length > 0) {
|
|
1933
|
+
throw this.unreadableAnswer("POST", `/${API_PREFIX}/blobs/missing`, asked.res, `a body asking for blobs this tree does not hold: ${[...new Set(strangers)].sort().join(", ")}`);
|
|
1934
|
+
}
|
|
1935
|
+
await forEachConcurrently(missing, MAX_CONCURRENT_TRANSFERS, async (blob, signal) => {
|
|
1936
|
+
const file = byBlob.get(blob.sha256);
|
|
1937
|
+
const bytes = read?.get(file.path) ?? (await tree.readFile(file.path));
|
|
1938
|
+
const bytesSha256 = await sha256Hex(bytes);
|
|
1939
|
+
if (bytes.length !== blob.size || bytesSha256 !== blob.sha256) {
|
|
1940
|
+
throw new RequestArgumentError(`The tree read ${bytes.length} bytes hashing to ${bytesSha256} for '${file.path}', which ` +
|
|
1941
|
+
`its listing names as ${blob.size} bytes hashing to ${blob.sha256}, so they were not ` +
|
|
1942
|
+
"uploaded and the method was not saved.");
|
|
1943
|
+
}
|
|
1944
|
+
await uploadBlob(blob, bytes, { timeoutMs: this.requestTimeoutMs, signal });
|
|
1945
|
+
});
|
|
1755
1946
|
}
|
|
1756
1947
|
/**
|
|
1757
1948
|
* Rename a method — `PATCH /v1/methods/{id}` — and return it. The name belongs to the method,
|
|
@@ -1817,11 +2008,10 @@ export class PipelexApiClient {
|
|
|
1817
2008
|
outcomes: ["published", "unchanged", "refused"],
|
|
1818
2009
|
member: "method",
|
|
1819
2010
|
});
|
|
1820
|
-
|
|
1821
|
-
return { ...answer, outcome, method };
|
|
2011
|
+
return { ...answer, outcome, method: subject };
|
|
1822
2012
|
}
|
|
1823
2013
|
/**
|
|
1824
|
-
* One page of a method's published versions, newest first, without their
|
|
2014
|
+
* One page of a method's published versions, newest first, without their files —
|
|
1825
2015
|
* `GET /v1/methods/{id}/versions`.
|
|
1826
2016
|
*
|
|
1827
2017
|
* Pass the page's `nextCursor` back as `cursor` to continue; `null` is the last page. A page
|
|
@@ -1839,14 +2029,16 @@ export class PipelexApiClient {
|
|
|
1839
2029
|
return this.requestPage(`${endpoint}${search}`);
|
|
1840
2030
|
}
|
|
1841
2031
|
/**
|
|
1842
|
-
* One published version of a method, with its
|
|
1843
|
-
* `
|
|
1844
|
-
*
|
|
2032
|
+
* One published version of a method, with its files — `GET /v1/methods/{id}/versions/{n}`.
|
|
2033
|
+
* `files` lists the version's tree as `getMethod` lists the draft's, each file with a link to
|
|
2034
|
+
* its bytes valid for about fifteen minutes, which `writeMethodTree` writes to disk.
|
|
1845
2035
|
*
|
|
1846
2036
|
* Takes a bare catalog id, as `getMethod` does, and the version number. Throws
|
|
1847
2037
|
* `RequestArgumentError` for a `version` that is not a positive integer, before any request.
|
|
1848
2038
|
* Throws `ApiResponseError`: `404` `method_version_not_found` for a version the method never
|
|
1849
|
-
* published, `404` `not_found` for an unknown method, `409` `method_being_deleted
|
|
2039
|
+
* published, `404` `not_found` for an unknown method, `409` `method_being_deleted`, and `409`
|
|
2040
|
+
* `conflict`, whose verdict is `runtime`, for a version stored before methods were trees and
|
|
2041
|
+
* never rewritten as one, which names no tree to serve.
|
|
1850
2042
|
*/
|
|
1851
2043
|
async getMethodVersion(methodId, version) {
|
|
1852
2044
|
if (!Number.isSafeInteger(version) || version < 1) {
|
|
@@ -1854,8 +2046,7 @@ export class PipelexApiClient {
|
|
|
1854
2046
|
}
|
|
1855
2047
|
const endpoint = `${methodPath(methodId, "getMethodVersion")}/versions/${version}`;
|
|
1856
2048
|
const res = await this.requestProductAnswer("GET", endpoint, undefined);
|
|
1857
|
-
|
|
1858
|
-
return this.readStoredSources("GET", endpoint, res, wire, "a published version");
|
|
2049
|
+
return this.readSavedTree("GET", endpoint, withLinks(res), "a published version");
|
|
1859
2050
|
}
|
|
1860
2051
|
/**
|
|
1861
2052
|
* Erase a method and everything it produced — `DELETE /v1/methods/{id}`.
|
|
@@ -2548,15 +2739,16 @@ const METHOD_REF_FETCH_TIMEOUT_MS = 180_000; // 3 min — covers the server's cl
|
|
|
2548
2739
|
*
|
|
2549
2740
|
* - **`resources`** make the request *body* as large as the runner's limits allow,
|
|
2550
2741
|
* about 13 MiB of base64 by default, and the whole upload is charged against
|
|
2551
|
-
* this budget. They get the
|
|
2552
|
-
* them
|
|
2553
|
-
* files never time out on one route yet go
|
|
2742
|
+
* this budget. They get the client's request ceiling (`requestTimeoutMs`),
|
|
2743
|
+
* which `start` gives a request carrying them and `validate` carries them under
|
|
2744
|
+
* by default, so the same bundle and files never time out on one route yet go
|
|
2745
|
+
* through on another.
|
|
2554
2746
|
* - **A `method_ref`** may make the server clone a repository first, which gets
|
|
2555
2747
|
* the fetch-sized budget.
|
|
2556
2748
|
*/
|
|
2557
|
-
function crateRequestTimeoutMs(request) {
|
|
2749
|
+
function crateRequestTimeoutMs(request, executeCeilingMs) {
|
|
2558
2750
|
if (hasBundleResources(request.resources))
|
|
2559
|
-
return
|
|
2751
|
+
return executeCeilingMs;
|
|
2560
2752
|
return nonEmptyString(request.method_ref) !== undefined ? METHOD_REF_FETCH_TIMEOUT_MS : undefined;
|
|
2561
2753
|
}
|
|
2562
2754
|
/**
|
|
@@ -2813,6 +3005,54 @@ const UNDECODABLE_BODY_CODE = /^(Z_|ERR__ERROR_|ZSTD_error_)/;
|
|
|
2813
3005
|
function isUndecodableBody(error) {
|
|
2814
3006
|
return error.code !== undefined && UNDECODABLE_BODY_CODE.test(error.code);
|
|
2815
3007
|
}
|
|
3008
|
+
/** A SHA-256 as the platform names a blob: 64 lowercase hexadecimal digits. */
|
|
3009
|
+
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
|
3010
|
+
/** `res`, marked as an answer whose body carries signed links (`RawResponse.carriesLinks`). */
|
|
3011
|
+
function withLinks(res) {
|
|
3012
|
+
return { ...res, carriesLinks: true };
|
|
3013
|
+
}
|
|
3014
|
+
/** Whether one entry of a `POST /v1/blobs/missing` answer is a blob with its upload target. */
|
|
3015
|
+
function isMissingBlob(entry) {
|
|
3016
|
+
if (!isPlainObject(entry))
|
|
3017
|
+
return false;
|
|
3018
|
+
const { sha256, size, upload } = entry;
|
|
3019
|
+
if (typeof sha256 !== "string" || !SHA256_HEX.test(sha256))
|
|
3020
|
+
return false;
|
|
3021
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0)
|
|
3022
|
+
return false;
|
|
3023
|
+
if (!isPlainObject(upload))
|
|
3024
|
+
return false;
|
|
3025
|
+
if (typeof upload.method !== "string" || typeof upload.url !== "string")
|
|
3026
|
+
return false;
|
|
3027
|
+
const headers = upload.headers;
|
|
3028
|
+
return (isPlainObject(headers) && Object.values(headers).every((value) => typeof value === "string"));
|
|
3029
|
+
}
|
|
3030
|
+
/**
|
|
3031
|
+
* Why a saved tree's listing is not one the SDK reads, or `null` when it is: an array of objects,
|
|
3032
|
+
* each with a string `path`, `sha256`, `url` and `expires_at` and a non-negative integer `size`.
|
|
3033
|
+
*/
|
|
3034
|
+
function savedFilesFault(files) {
|
|
3035
|
+
if (!Array.isArray(files))
|
|
3036
|
+
return "is not an array";
|
|
3037
|
+
for (let index = 0; index < files.length; index += 1) {
|
|
3038
|
+
const file = files[index];
|
|
3039
|
+
if (!isPlainObject(file))
|
|
3040
|
+
return `holds an entry that is not an object, at [${index}]`;
|
|
3041
|
+
for (const member of ["path", "sha256", "url", "expires_at"]) {
|
|
3042
|
+
if (typeof file[member] !== "string") {
|
|
3043
|
+
return `holds an entry whose \`${member}\` is not a string, at [${index}]`;
|
|
3044
|
+
}
|
|
3045
|
+
}
|
|
3046
|
+
if (!SHA256_HEX.test(file.sha256)) {
|
|
3047
|
+
return `holds an entry whose \`sha256\` is not 64 lowercase hex digits, at [${index}]`;
|
|
3048
|
+
}
|
|
3049
|
+
const size = file.size;
|
|
3050
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0) {
|
|
3051
|
+
return `holds an entry whose \`size\` is not a non-negative integer, at [${index}]`;
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
return null;
|
|
3055
|
+
}
|
|
2816
3056
|
function extractNetworkErrorCode(err) {
|
|
2817
3057
|
if (err instanceof DOMException && err.name === "TimeoutError") {
|
|
2818
3058
|
return "ABORT_TIMEOUT";
|