@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/LICENSE +21 -0
- package/README.md +173 -2
- package/dist/cjs/ArtifactBackupLease.d.ts +34 -0
- package/dist/cjs/ArtifactBackupLease.d.ts.map +1 -0
- package/dist/cjs/ArtifactBackupLease.js +175 -0
- package/dist/cjs/ArtifactBackupLease.js.map +7 -0
- package/dist/cjs/ArtifactStore.d.ts +129 -0
- package/dist/cjs/ArtifactStore.d.ts.map +1 -0
- package/dist/cjs/ArtifactStore.js +73 -0
- package/dist/cjs/ArtifactStore.js.map +7 -0
- package/dist/cjs/ArtifactStoreError.d.ts +82 -0
- package/dist/cjs/ArtifactStoreError.d.ts.map +1 -0
- package/dist/cjs/ArtifactStoreError.js +72 -0
- package/dist/cjs/ArtifactStoreError.js.map +7 -0
- package/dist/cjs/ArtifactStoreMetrics.d.ts +70 -0
- package/dist/cjs/ArtifactStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/ArtifactStoreMetrics.js +51 -0
- package/dist/cjs/ArtifactStoreMetrics.js.map +7 -0
- package/dist/cjs/ArtifactSweep.d.ts +185 -0
- package/dist/cjs/ArtifactSweep.d.ts.map +1 -0
- package/dist/cjs/ArtifactSweep.js +163 -0
- package/dist/cjs/ArtifactSweep.js.map +7 -0
- package/dist/cjs/CombinedArtifacts.d.ts +103 -0
- package/dist/cjs/CombinedArtifacts.d.ts.map +1 -0
- package/dist/cjs/CombinedArtifacts.js +189 -0
- package/dist/cjs/CombinedArtifacts.js.map +7 -0
- package/dist/cjs/FileLease.d.ts +113 -0
- package/dist/cjs/FileLease.d.ts.map +1 -0
- package/dist/cjs/FileLease.js +201 -0
- package/dist/cjs/FileLease.js.map +7 -0
- package/dist/cjs/FileSystemArtifactStore.d.ts +93 -0
- package/dist/cjs/FileSystemArtifactStore.d.ts.map +1 -0
- package/dist/cjs/FileSystemArtifactStore.js +276 -0
- package/dist/cjs/FileSystemArtifactStore.js.map +7 -0
- package/dist/cjs/MemoryArtifactStore.d.ts +25 -0
- package/dist/cjs/MemoryArtifactStore.d.ts.map +1 -0
- package/dist/cjs/MemoryArtifactStore.js +84 -0
- package/dist/cjs/MemoryArtifactStore.js.map +7 -0
- package/dist/cjs/NoopArtifactStore.d.ts +17 -0
- package/dist/cjs/NoopArtifactStore.d.ts.map +1 -0
- package/dist/cjs/NoopArtifactStore.js +46 -0
- package/dist/cjs/NoopArtifactStore.js.map +7 -0
- package/dist/cjs/RemoteArtifacts.d.ts +237 -0
- package/dist/cjs/RemoteArtifacts.d.ts.map +1 -0
- package/dist/cjs/RemoteArtifacts.js +405 -0
- package/dist/cjs/RemoteArtifacts.js.map +7 -0
- package/dist/cjs/index.d.ts +65 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +47 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/ArtifactLocks.d.ts +30 -0
- package/dist/cjs/internal/ArtifactLocks.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactLocks.js +93 -0
- package/dist/cjs/internal/ArtifactLocks.js.map +7 -0
- package/dist/cjs/internal/ArtifactPath.d.ts +19 -0
- package/dist/cjs/internal/ArtifactPath.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactPath.js +74 -0
- package/dist/cjs/internal/ArtifactPath.js.map +7 -0
- package/dist/cjs/measureBytes.d.ts +18 -0
- package/dist/cjs/measureBytes.d.ts.map +1 -0
- package/dist/cjs/measureBytes.js +43 -0
- package/dist/cjs/measureBytes.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/snapshotBytes.d.ts +19 -0
- package/dist/cjs/snapshotBytes.d.ts.map +1 -0
- package/dist/cjs/snapshotBytes.js +41 -0
- package/dist/cjs/snapshotBytes.js.map +7 -0
- package/dist/cjs/validateDigest.d.ts +22 -0
- package/dist/cjs/validateDigest.d.ts.map +1 -0
- package/dist/cjs/validateDigest.js +42 -0
- package/dist/cjs/validateDigest.js.map +7 -0
- package/dist/esm/ArtifactBackupLease.d.ts +34 -0
- package/dist/esm/ArtifactBackupLease.d.ts.map +1 -0
- package/dist/esm/ArtifactBackupLease.js +117 -0
- package/dist/esm/ArtifactBackupLease.js.map +1 -0
- package/dist/esm/ArtifactStore.d.ts +129 -0
- package/dist/esm/ArtifactStore.d.ts.map +1 -0
- package/dist/esm/ArtifactStore.js +83 -0
- package/dist/esm/ArtifactStore.js.map +1 -0
- package/dist/esm/ArtifactStoreError.d.ts +82 -0
- package/dist/esm/ArtifactStoreError.d.ts.map +1 -0
- package/dist/esm/ArtifactStoreError.js +77 -0
- package/dist/esm/ArtifactStoreError.js.map +1 -0
- package/dist/esm/ArtifactStoreMetrics.d.ts +70 -0
- package/dist/esm/ArtifactStoreMetrics.d.ts.map +1 -0
- package/dist/esm/ArtifactStoreMetrics.js +77 -0
- package/dist/esm/ArtifactStoreMetrics.js.map +1 -0
- package/dist/esm/ArtifactSweep.d.ts +185 -0
- package/dist/esm/ArtifactSweep.d.ts.map +1 -0
- package/dist/esm/ArtifactSweep.js +202 -0
- package/dist/esm/ArtifactSweep.js.map +1 -0
- package/dist/esm/CombinedArtifacts.d.ts +103 -0
- package/dist/esm/CombinedArtifacts.d.ts.map +1 -0
- package/dist/esm/CombinedArtifacts.js +214 -0
- package/dist/esm/CombinedArtifacts.js.map +1 -0
- package/dist/esm/FileLease.d.ts +113 -0
- package/dist/esm/FileLease.d.ts.map +1 -0
- package/dist/esm/FileLease.js +239 -0
- package/dist/esm/FileLease.js.map +1 -0
- package/dist/esm/FileSystemArtifactStore.d.ts +93 -0
- package/dist/esm/FileSystemArtifactStore.d.ts.map +1 -0
- package/dist/esm/FileSystemArtifactStore.js +336 -0
- package/dist/esm/FileSystemArtifactStore.js.map +1 -0
- package/dist/esm/MemoryArtifactStore.d.ts +25 -0
- package/dist/esm/MemoryArtifactStore.d.ts.map +1 -0
- package/dist/esm/MemoryArtifactStore.js +67 -0
- package/dist/esm/MemoryArtifactStore.js.map +1 -0
- package/dist/esm/NoopArtifactStore.d.ts +17 -0
- package/dist/esm/NoopArtifactStore.d.ts.map +1 -0
- package/dist/esm/NoopArtifactStore.js +27 -0
- package/dist/esm/NoopArtifactStore.js.map +1 -0
- package/dist/esm/RemoteArtifacts.d.ts +237 -0
- package/dist/esm/RemoteArtifacts.d.ts.map +1 -0
- package/dist/esm/RemoteArtifacts.js +554 -0
- package/dist/esm/RemoteArtifacts.js.map +1 -0
- package/dist/esm/index.d.ts +65 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +65 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/ArtifactLocks.d.ts +30 -0
- package/dist/esm/internal/ArtifactLocks.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactLocks.js +86 -0
- package/dist/esm/internal/ArtifactLocks.js.map +1 -0
- package/dist/esm/internal/ArtifactPath.d.ts +19 -0
- package/dist/esm/internal/ArtifactPath.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactPath.js +59 -0
- package/dist/esm/internal/ArtifactPath.js.map +1 -0
- package/dist/esm/measureBytes.d.ts +18 -0
- package/dist/esm/measureBytes.d.ts.map +1 -0
- package/dist/esm/measureBytes.js +22 -0
- package/dist/esm/measureBytes.js.map +1 -0
- package/dist/esm/snapshotBytes.d.ts +19 -0
- package/dist/esm/snapshotBytes.d.ts.map +1 -0
- package/dist/esm/snapshotBytes.js +30 -0
- package/dist/esm/snapshotBytes.js.map +1 -0
- package/dist/esm/validateDigest.d.ts +22 -0
- package/dist/esm/validateDigest.d.ts.map +1 -0
- package/dist/esm/validateDigest.js +26 -0
- package/dist/esm/validateDigest.js.map +1 -0
- package/docs/README.md +200 -0
- package/docs/api.md +474 -0
- package/docs/concepts/content-addressing.md +174 -0
- package/docs/concepts/coordination.md +141 -0
- package/docs/concepts/tiers.md +147 -0
- package/docs/guides/fence-a-backup.md +123 -0
- package/docs/guides/reclaim-disk-space.md +143 -0
- package/docs/guides/serve-the-artifact-protocol.md +140 -0
- package/docs/guides/share-artifacts-across-machines.md +180 -0
- package/docs/guides/test-against-an-artifact-store.md +158 -0
- package/docs/installation.md +98 -0
- package/docs/quickstart.md +182 -0
- package/docs/troubleshooting.md +232 -0
- package/package.json +179 -3
- package/src/ArtifactBackupLease.ts +213 -0
- package/src/ArtifactStore.ts +152 -0
- package/src/ArtifactStoreError.ts +96 -0
- package/src/ArtifactStoreMetrics.ts +83 -0
- package/src/ArtifactSweep.ts +336 -0
- package/src/CombinedArtifacts.ts +369 -0
- package/src/FileLease.ts +380 -0
- package/src/FileSystemArtifactStore.ts +459 -0
- package/src/MemoryArtifactStore.ts +83 -0
- package/src/NoopArtifactStore.ts +30 -0
- package/src/RemoteArtifacts.ts +809 -0
- package/src/index.ts +71 -0
- package/src/internal/ArtifactLocks.ts +110 -0
- package/src/internal/ArtifactPath.ts +72 -0
- package/src/measureBytes.ts +28 -0
- package/src/snapshotBytes.ts +34 -0
- package/src/validateDigest.ts +31 -0
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shared artifact tier: an {@link ArtifactStore.Service} spoken over HTTP.
|
|
3
|
+
*
|
|
4
|
+
* The wire protocol mirrors Bazel's dumb-HTTP remote cache, class
|
|
5
|
+
* `HttpCacheClient` in `com.google.devtools.build.lib.remote.http`, which
|
|
6
|
+
* documents it as: "CAS (Content Addressable Storage) blobs are stored under
|
|
7
|
+
* the path `/cas/base16-key`", uploaded with `PUT`, downloaded with `GET`. We add
|
|
8
|
+
* `HEAD /cas/{digest}` for a single existence probe and
|
|
9
|
+
* `POST /cas/findMissing` for the batched one, because Bazel's HTTP client has
|
|
10
|
+
* no `findMissingDigests` at all — it answers "everything is missing" and
|
|
11
|
+
* re-uploads — and a batched probe is the whole reason
|
|
12
|
+
* `MissingDigestsFinder` exists in the gRPC client.
|
|
13
|
+
*
|
|
14
|
+
* Transport is Effect's own `HttpClient` tag. There is no lower transport in
|
|
15
|
+
* this repository, and there does not need to be one: the kernel already
|
|
16
|
+
* decorates that tag with `net:get`/`net:post` capability checks, so a remote
|
|
17
|
+
* artifact fetch is permission-checked like every other egress.
|
|
18
|
+
*
|
|
19
|
+
* ## Chunked uploads
|
|
20
|
+
*
|
|
21
|
+
* With `chunkBytes` set, a blob past that size travels as a sequence of
|
|
22
|
+
* `PUT /cas/{digest}` requests carrying `Content-Range: bytes {a}-{b}/{total}`,
|
|
23
|
+
* preceded by a `HEAD /cas/{digest}` existence probe and a
|
|
24
|
+
* `Content-Range: bytes *\/{total}` probe with an empty body. This is the
|
|
25
|
+
* resumable-upload shape HTTP already has: the range probe asks what prefix
|
|
26
|
+
* the tier holds and `308` means "keep going", with a `Range: bytes=0-{last}`
|
|
27
|
+
* header on the probe or on a chunk answer moving the offset.
|
|
28
|
+
*
|
|
29
|
+
* Only a `308` continues the sequence, and the completing chunk's `2xx` is
|
|
30
|
+
* confirmed with `HEAD` before `put` reports the digest as published. A `2xx`
|
|
31
|
+
* anywhere else, to the empty probe or to a chunk the tier is still owed bytes
|
|
32
|
+
* after, reads as a tier that ignored `Content-Range` and stored the body it
|
|
33
|
+
* was handed, which is what plain WebDAV `PUT` does. That tier, the one that
|
|
34
|
+
* answers `400`, `411`, or `416`, and the one whose `HEAD` will not confirm
|
|
35
|
+
* the stored length all get the same treatment: one whole-blob `PUT`, which
|
|
36
|
+
* overwrites the partial body the sequence left behind. So the blob always
|
|
37
|
+
* lands whole, and the cost of turning the dial on against a server that
|
|
38
|
+
* never learned about it is round trips, never a digest published over bytes
|
|
39
|
+
* the server does not hold.
|
|
40
|
+
*
|
|
41
|
+
* This repository's own hosted and self-hosted cache services are such
|
|
42
|
+
* servers: neither implements the resumable sequence, and both refuse a
|
|
43
|
+
* ranged `PUT` with `400`, RFC 9110 section 14.5's answer from a resource
|
|
44
|
+
* that does not support partial `PUT`. Against them `chunkBytes` costs the
|
|
45
|
+
* probe round trip and the blob travels whole, and both cap one request body
|
|
46
|
+
* at 16 MiB, so a blob past that cap is refused with `413` in either mode.
|
|
47
|
+
*
|
|
48
|
+
* The endpoint and its credentials arrive as **layer construction options**.
|
|
49
|
+
* They are capabilities, never step inputs: they are not hashed into a step
|
|
50
|
+
* key, journaled, or returned in a recorded result. Invalid endpoint errors are
|
|
51
|
+
* sanitized before they cross the boundary.
|
|
52
|
+
*
|
|
53
|
+
* @since 1.0.0-rc.0
|
|
54
|
+
*/
|
|
55
|
+
import * as Duration from "effect/Duration";
|
|
56
|
+
import * as Effect from "effect/Effect";
|
|
57
|
+
import * as Layer from "effect/Layer";
|
|
58
|
+
import * as Schema from "effect/Schema";
|
|
59
|
+
import * as HttpClient from "effect/unstable/http/HttpClient";
|
|
60
|
+
import * as ArtifactStore from "./ArtifactStore.ts";
|
|
61
|
+
/**
|
|
62
|
+
* How eagerly a composition materializes shared blobs into its local store.
|
|
63
|
+
*
|
|
64
|
+
* @category models
|
|
65
|
+
* @since 1.0.0-rc.0
|
|
66
|
+
*/
|
|
67
|
+
export declare const DownloadPolicy: Schema.Literals<readonly ["all", "toplevel", "minimal"]>;
|
|
68
|
+
/**
|
|
69
|
+
* How eagerly a composition materializes remote blobs.
|
|
70
|
+
*
|
|
71
|
+
* @category models
|
|
72
|
+
* @since 1.0.0-rc.0
|
|
73
|
+
*/
|
|
74
|
+
export type DownloadPolicy = typeof DownloadPolicy.Type;
|
|
75
|
+
/**
|
|
76
|
+
* Every download policy, in materialization order.
|
|
77
|
+
*
|
|
78
|
+
* @category models
|
|
79
|
+
* @since 1.0.0-rc.0
|
|
80
|
+
*/
|
|
81
|
+
export declare const downloadPolicies: ReadonlyArray<DownloadPolicy>;
|
|
82
|
+
/**
|
|
83
|
+
* A remote artifact tier, which is an ordinary store that also states how
|
|
84
|
+
* eagerly a composition reading through it should materialize blobs locally.
|
|
85
|
+
*
|
|
86
|
+
* @category models
|
|
87
|
+
* @since 1.0.0-rc.0
|
|
88
|
+
*/
|
|
89
|
+
export interface Service extends ArtifactStore.Service {
|
|
90
|
+
readonly downloadPolicy: DownloadPolicy;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Reads the download policy a store declares, or `undefined` for a store that
|
|
94
|
+
* declares none — every local store, and any foreign implementation.
|
|
95
|
+
*
|
|
96
|
+
* @category getters
|
|
97
|
+
* @since 1.0.0-rc.0
|
|
98
|
+
*/
|
|
99
|
+
export declare const downloadPolicyOf: (store: ArtifactStore.Service) => DownloadPolicy | undefined;
|
|
100
|
+
/**
|
|
101
|
+
* How to reach the shared artifact tier.
|
|
102
|
+
*
|
|
103
|
+
* @category models
|
|
104
|
+
* @since 1.0.0-rc.0
|
|
105
|
+
* @slop
|
|
106
|
+
*/
|
|
107
|
+
export interface Options {
|
|
108
|
+
/**
|
|
109
|
+
* The cache root, e.g. `https://cache.example.com`. `/cas/{digest}` and
|
|
110
|
+
* `/cas/findMissing` are resolved beneath it. A trailing slash is ignored.
|
|
111
|
+
*
|
|
112
|
+
* Four shapes are refused at construction as `invalid_configuration`, before
|
|
113
|
+
* any request leaves and therefore before `headers` can reach a wire: a value
|
|
114
|
+
* that is not a string, one no `URL` parser accepts, a scheme other than
|
|
115
|
+
* `https:` (plain `http:` is accepted only for the literal loopback hosts
|
|
116
|
+
* `localhost`, `127.0.0.1`, and `[::1]`), and an endpoint carrying
|
|
117
|
+
* userinfo, a query, or a fragment. The
|
|
118
|
+
* last rule keeps a credential out of a place nothing here would redact: an
|
|
119
|
+
* endpoint is interpolated into every request path and into span attributes.
|
|
120
|
+
*
|
|
121
|
+
* The refusal message names only the violated rule and never echoes the
|
|
122
|
+
* endpoint, so a rejected `https://user:secret@host` cannot leak its
|
|
123
|
+
* credential into a log line or a durable error.
|
|
124
|
+
*/
|
|
125
|
+
readonly endpoint: string;
|
|
126
|
+
/**
|
|
127
|
+
* Headers sent with every request — an `Authorization` bearer token, a
|
|
128
|
+
* tenant header, whatever the deployment's gateway wants. This is the
|
|
129
|
+
* credential seam, and it is deliberately construction-time: a credential
|
|
130
|
+
* that arrived as a step input would be hashed into a step key and persisted
|
|
131
|
+
* everywhere the journal goes.
|
|
132
|
+
*
|
|
133
|
+
* The protocol wins every collision. These headers are applied to a request
|
|
134
|
+
* first. Configured `content-type`, `content-length`, and `content-range`
|
|
135
|
+
* are removed, then computed only where the protocol needs them. Whole-blob
|
|
136
|
+
* uploads, including fallbacks, never carry `content-range`.
|
|
137
|
+
* All configured header names are redacted in HTTP spans, in addition to
|
|
138
|
+
* the host's existing redaction rules.
|
|
139
|
+
*
|
|
140
|
+
* A name that is not an HTTP token, or a value longer than 16 KiB or
|
|
141
|
+
* carrying a control character (CR and LF included), is refused at
|
|
142
|
+
* construction as `invalid_configuration`.
|
|
143
|
+
*/
|
|
144
|
+
readonly headers?: Record<string, string> | undefined;
|
|
145
|
+
/**
|
|
146
|
+
* The deadline on a single download, from the request leaving to the last
|
|
147
|
+
* body byte arriving. A shared tier that stops answering must fail the read
|
|
148
|
+
* rather than park it forever. `CombinedArtifacts.get` propagates a remote
|
|
149
|
+
* `ArtifactStoreError`, including a download timeout. Only opportunistic
|
|
150
|
+
* uploads from `CombinedArtifacts.put` ignore remote failures;
|
|
151
|
+
* `ArtifactSync.hydrate` separately turns a failure into replay rejection.
|
|
152
|
+
* Defaults to 60 seconds, Bazel's `--remote_timeout`
|
|
153
|
+
* default for the same REST protocol (its `RemoteOptions`: "For the REST
|
|
154
|
+
* cache, this is both the connect and the read timeout").
|
|
155
|
+
*/
|
|
156
|
+
readonly downloadTimeout?: Duration.Input | undefined;
|
|
157
|
+
/**
|
|
158
|
+
* Deadline for an upload, including every resume probe and chunk. Defaults
|
|
159
|
+
* to 60 seconds.
|
|
160
|
+
*/
|
|
161
|
+
readonly uploadTimeout?: Duration.Input | undefined;
|
|
162
|
+
/**
|
|
163
|
+
* Deadline for an existence probe or one `findMissing` batch, including its
|
|
164
|
+
* response body. Defaults to 60 seconds.
|
|
165
|
+
*/
|
|
166
|
+
readonly requestTimeout?: Duration.Input | undefined;
|
|
167
|
+
/**
|
|
168
|
+
* The largest download this client will buffer, in bytes. The body is read
|
|
169
|
+
* incrementally and abandoned the moment it exceeds the bound — and refused
|
|
170
|
+
* outright when `Content-Length` already declares the excess — so a
|
|
171
|
+
* mis-serving or hostile cache cannot make this process buffer and hash an
|
|
172
|
+
* arbitrarily large body before the digest check refuses it. Defaults to
|
|
173
|
+
* 256 MiB.
|
|
174
|
+
*/
|
|
175
|
+
readonly maxDownloadBytes?: number | undefined;
|
|
176
|
+
/**
|
|
177
|
+
* Largest `findMissing` response body accepted. Defaults to the protocol's
|
|
178
|
+
* 256 KiB request/response bound and may only lower that bound.
|
|
179
|
+
*/
|
|
180
|
+
readonly maxFindMissingResponseBytes?: number | undefined;
|
|
181
|
+
/**
|
|
182
|
+
* Above this many bytes an upload travels as a sequence of `Content-Range`
|
|
183
|
+
* `PUT`s instead of one whole-blob body, and a transfer that died partway
|
|
184
|
+
* resumes from the prefix the tier kept rather than starting over. See
|
|
185
|
+
* {@link module:RemoteArtifacts | the chunked upload protocol} below.
|
|
186
|
+
*
|
|
187
|
+
* Absent by default: every upload is one whole-blob `PUT`, which is what
|
|
188
|
+
* Bazel's dumb-HTTP client does and what every deployment already serving
|
|
189
|
+
* this protocol expects. Set it when a proxy caps request bodies, or when
|
|
190
|
+
* artifacts are large enough that losing a transfer is expensive.
|
|
191
|
+
*
|
|
192
|
+
* The repository's own hosted and self-hosted cache services answer the
|
|
193
|
+
* ranged probe with `400`, so against them this dial degrades to one
|
|
194
|
+
* whole-blob `PUT`, and their 16 MiB body cap refuses a larger blob with
|
|
195
|
+
* `413` in either mode.
|
|
196
|
+
*/
|
|
197
|
+
readonly chunkBytes?: number | undefined;
|
|
198
|
+
/**
|
|
199
|
+
* How eagerly a composition reading through this tier materializes blobs
|
|
200
|
+
* into its local store. This is Bazel's `--remote_download_{all,toplevel,
|
|
201
|
+
* minimal}` dial (`RemoteOutputChecker`), and it lives here because the
|
|
202
|
+
* shared tier is the thing being conserved.
|
|
203
|
+
*
|
|
204
|
+
* - `all` (the default) prefetches every referenced blob when a replay is
|
|
205
|
+
* admitted, so the replay reads local bytes.
|
|
206
|
+
* - `toplevel` prefetches nothing and materializes a blob into the local
|
|
207
|
+
* store on the first read that actually needs it.
|
|
208
|
+
* - `minimal` prefetches nothing and materializes nothing: a read is served
|
|
209
|
+
* straight from the shared tier and the local store never grows.
|
|
210
|
+
*
|
|
211
|
+
* `CombinedArtifacts.get` honors the last two; `@smthrs/engine-store`'s
|
|
212
|
+
* `ArtifactSync.hydrate` honors the first.
|
|
213
|
+
*/
|
|
214
|
+
readonly downloadPolicy?: DownloadPolicy | undefined;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Builds a remote artifact store over Effect's `HttpClient`.
|
|
218
|
+
*
|
|
219
|
+
* @category constructors
|
|
220
|
+
* @since 1.0.0-rc.0
|
|
221
|
+
* @slop
|
|
222
|
+
*/
|
|
223
|
+
export declare const make: (options: Options) => Effect.Effect<Service, ArtifactStore.ArtifactStoreError, HttpClient.HttpClient>;
|
|
224
|
+
/**
|
|
225
|
+
* Provides a remote artifact store as the `ArtifactStore` tag.
|
|
226
|
+
*
|
|
227
|
+
* Composing this *alone* makes every artifact read and write a network round
|
|
228
|
+
* trip. The intended production shape is
|
|
229
|
+
* {@link CombinedArtifacts.layer | CombinedArtifacts}, with this as its remote
|
|
230
|
+
* tier.
|
|
231
|
+
*
|
|
232
|
+
* @category layers
|
|
233
|
+
* @since 1.0.0-rc.0
|
|
234
|
+
* @slop
|
|
235
|
+
*/
|
|
236
|
+
export declare const layer: (options: Options) => Layer.Layer<ArtifactStore.ArtifactStore, ArtifactStore.ArtifactStoreError, HttpClient.HttpClient>;
|
|
237
|
+
//# sourceMappingURL=RemoteArtifacts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RemoteArtifacts.d.ts","sourceRoot":"","sources":["../../src/RemoteArtifacts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AAEH,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAErC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAIvC,OAAO,KAAK,UAAU,MAAM,iCAAiC,CAAA;AAG7D,OAAO,KAAK,aAAa,MAAM,oBAAoB,CAAA;AAEnD;;;;;GAKG;AACH,eAAO,MAAM,cAAc,0DAA2D,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC,IAAI,CAAA;AAEvD;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,aAAa,CAAC,cAAc,CAAiD,CAAA;AAE5G;;;;;;GAMG;AACH,MAAM,WAAW,OAAQ,SAAQ,aAAa,CAAC,OAAO;IACpD,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAA;CACxC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO,aAAa,CAAC,OAAO,KAAG,cAAc,GAAG,SAGhF,CAAA;AAED;;;;;;GAMG;AACH,MAAM,WAAW,OAAO;IACtB;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAA;IACrD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IACrD;;;OAGG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IACnD;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IACpD;;;;;;;OAOG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9C;;;OAGG;IACH,QAAQ,CAAC,2BAA2B,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACzD;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,SAAS,CAAA;CACrD;AA4FD;;;;;;GAMG;AACH,eAAO,MAAM,IAAI,GACf,SAAS,OAAO,KACf,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,EAAE,UAAU,CAAC,UAAU,CA4c7E,CAAA;AAEJ;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,KAAK,GAChB,SAAS,OAAO,KACf,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,EAAE,aAAa,CAAC,kBAAkB,EAAE,UAAU,CAAC,UAAU,CACzC,CAAA"}
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
var RemoteArtifacts_exports = {};
|
|
30
|
+
__export(RemoteArtifacts_exports, {
|
|
31
|
+
DownloadPolicy: () => DownloadPolicy,
|
|
32
|
+
downloadPolicies: () => downloadPolicies,
|
|
33
|
+
downloadPolicyOf: () => downloadPolicyOf,
|
|
34
|
+
layer: () => layer,
|
|
35
|
+
make: () => make
|
|
36
|
+
});
|
|
37
|
+
module.exports = __toCommonJS(RemoteArtifacts_exports);
|
|
38
|
+
var Duration = __toESM(require("effect/Duration"), 1);
|
|
39
|
+
var Effect = __toESM(require("effect/Effect"), 1);
|
|
40
|
+
var Layer = __toESM(require("effect/Layer"), 1);
|
|
41
|
+
var Option = __toESM(require("effect/Option"), 1);
|
|
42
|
+
var Schema = __toESM(require("effect/Schema"), 1);
|
|
43
|
+
var Stream = __toESM(require("effect/Stream"), 1);
|
|
44
|
+
var FetchHttpClient = __toESM(require("effect/unstable/http/FetchHttpClient"), 1);
|
|
45
|
+
var Headers = __toESM(require("effect/unstable/http/Headers"), 1);
|
|
46
|
+
var HttpClient = __toESM(require("effect/unstable/http/HttpClient"), 1);
|
|
47
|
+
var HttpClientRequest = __toESM(require("effect/unstable/http/HttpClientRequest"), 1);
|
|
48
|
+
var ArtifactStore = __toESM(require("./ArtifactStore.js"), 1);
|
|
49
|
+
const DownloadPolicy = Schema.Literals(["all", "toplevel", "minimal"]);
|
|
50
|
+
const downloadPolicies = Object.freeze(["all", "toplevel", "minimal"]);
|
|
51
|
+
const downloadPolicyOf = (store) => {
|
|
52
|
+
const declared = store.downloadPolicy;
|
|
53
|
+
return Schema.is(DownloadPolicy)(declared) ? declared : void 0;
|
|
54
|
+
};
|
|
55
|
+
const defaultDownloadTimeout = Duration.seconds(60);
|
|
56
|
+
const defaultUploadTimeout = Duration.seconds(60);
|
|
57
|
+
const defaultRequestTimeout = Duration.seconds(60);
|
|
58
|
+
const defaultMaxDownloadBytes = 256 * 1024 * 1024;
|
|
59
|
+
const maxFindMissingBatchDigests = 1e3;
|
|
60
|
+
const maxFindMissingBodyBytes = 256 * 1024;
|
|
61
|
+
const encoder = new TextEncoder();
|
|
62
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
63
|
+
const transportReason = (cause) => {
|
|
64
|
+
const reason = cause?.reason;
|
|
65
|
+
const tag = typeof reason?._tag === "string" ? reason._tag : void 0;
|
|
66
|
+
let code;
|
|
67
|
+
let current = reason ?? cause;
|
|
68
|
+
for (let depth = 0; depth < 8 && typeof current === "object" && current !== null; depth++) {
|
|
69
|
+
const candidate = current.code;
|
|
70
|
+
if (typeof candidate === "string" && /^[A-Z][A-Z0-9_]{1,63}$/.test(candidate)) {
|
|
71
|
+
code = candidate;
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
current = current.cause;
|
|
75
|
+
}
|
|
76
|
+
const parts = [tag, code].filter((part) => part !== void 0);
|
|
77
|
+
return parts.length === 0 ? void 0 : parts.join(" ");
|
|
78
|
+
};
|
|
79
|
+
const transportFailure = (operation, cause) => {
|
|
80
|
+
const reason = transportReason(cause);
|
|
81
|
+
return new ArtifactStore.ArtifactStoreError({
|
|
82
|
+
code: "transport_failed",
|
|
83
|
+
message: `the remote artifact tier refused ${operation}${reason === void 0 ? "" : ` (${reason})`}`
|
|
84
|
+
});
|
|
85
|
+
};
|
|
86
|
+
const unexpectedStatus = (operation, status) => new ArtifactStore.ArtifactStoreError({
|
|
87
|
+
code: "transport_failed",
|
|
88
|
+
message: `the remote artifact tier answered ${operation} with HTTP ${status}`
|
|
89
|
+
});
|
|
90
|
+
const invalidOption = (name) => new ArtifactStore.ArtifactStoreError({
|
|
91
|
+
code: "invalid_configuration",
|
|
92
|
+
message: `invalid remote artifact option: ${name}`
|
|
93
|
+
});
|
|
94
|
+
const FindMissingResponse = Schema.Struct({ missing: Schema.Array(Schema.String) });
|
|
95
|
+
const headerName = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;
|
|
96
|
+
const hasControlText = (value) => {
|
|
97
|
+
for (let index = 0; index < value.length; index++) {
|
|
98
|
+
const unit = value.charCodeAt(index);
|
|
99
|
+
if (unit <= 31 || unit === 127) return true;
|
|
100
|
+
}
|
|
101
|
+
return false;
|
|
102
|
+
};
|
|
103
|
+
const configurationFailure = (message) => new ArtifactStore.ArtifactStoreError({ code: "invalid_configuration", message });
|
|
104
|
+
const isOk = (response) => response.status >= 200 && response.status < 300;
|
|
105
|
+
const make = (options) => Effect.gen(function* () {
|
|
106
|
+
const configured = yield* Effect.try({
|
|
107
|
+
try: () => {
|
|
108
|
+
const headers2 = options.headers === void 0 ? void 0 : Object.freeze(Object.fromEntries(
|
|
109
|
+
Object.entries(options.headers).map(([name, value]) => {
|
|
110
|
+
if (typeof value !== "string" || value.length > 16 * 1024 || hasControlText(value)) {
|
|
111
|
+
throw new TypeError("header value");
|
|
112
|
+
}
|
|
113
|
+
if (!headerName.test(name)) throw new TypeError("header name");
|
|
114
|
+
return [name, value];
|
|
115
|
+
})
|
|
116
|
+
));
|
|
117
|
+
return {
|
|
118
|
+
endpoint: options.endpoint,
|
|
119
|
+
headers: headers2,
|
|
120
|
+
downloadTimeout: options.downloadTimeout,
|
|
121
|
+
uploadTimeout: options.uploadTimeout,
|
|
122
|
+
requestTimeout: options.requestTimeout,
|
|
123
|
+
maxDownloadBytes: options.maxDownloadBytes,
|
|
124
|
+
maxFindMissingResponseBytes: options.maxFindMissingResponseBytes,
|
|
125
|
+
chunkBytes: options.chunkBytes,
|
|
126
|
+
downloadPolicy: options.downloadPolicy
|
|
127
|
+
};
|
|
128
|
+
},
|
|
129
|
+
catch: () => invalidOption("options")
|
|
130
|
+
});
|
|
131
|
+
if (typeof configured.endpoint !== "string") return yield* Effect.fail(invalidOption("endpoint"));
|
|
132
|
+
const endpoint = yield* Effect.try({
|
|
133
|
+
try: () => new URL(configured.endpoint),
|
|
134
|
+
catch: () => configurationFailure("invalid remote artifact endpoint")
|
|
135
|
+
});
|
|
136
|
+
const loopback = endpoint.hostname === "localhost" || endpoint.hostname === "127.0.0.1" || endpoint.hostname === "[::1]";
|
|
137
|
+
if (endpoint.protocol !== "https:" && !(endpoint.protocol === "http:" && loopback)) {
|
|
138
|
+
return yield* Effect.fail(configurationFailure("remote artifact endpoint must use HTTPS"));
|
|
139
|
+
}
|
|
140
|
+
if (endpoint.username !== "" || endpoint.password !== "" || endpoint.search !== "" || endpoint.hash !== "") {
|
|
141
|
+
return yield* Effect.fail(
|
|
142
|
+
configurationFailure("remote artifact endpoint must not contain credentials, a query, or a fragment")
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
endpoint.pathname = endpoint.pathname.replace(/\/+$/, "");
|
|
146
|
+
const base = endpoint.toString().replace(/\/+$/, "");
|
|
147
|
+
const headers = configured.headers;
|
|
148
|
+
const client = yield* HttpClient.HttpClient;
|
|
149
|
+
const credentialNames = Object.keys(headers ?? {}).map((name) => name.toLowerCase());
|
|
150
|
+
const authorized = (request) => (headers === void 0 ? request : HttpClientRequest.setHeaders(request, headers)).pipe(
|
|
151
|
+
HttpClientRequest.removeHeader("content-type"),
|
|
152
|
+
HttpClientRequest.removeHeader("content-length"),
|
|
153
|
+
HttpClientRequest.removeHeader("content-range")
|
|
154
|
+
);
|
|
155
|
+
const casUrl = (digest) => `${base}/cas/${digest}`;
|
|
156
|
+
const scopedClient = HttpClient.withScope(client);
|
|
157
|
+
const send = (operation, request, use) => Effect.gen(function* () {
|
|
158
|
+
const redactedNames = yield* Headers.CurrentRedactedNames;
|
|
159
|
+
const requestInit = yield* Effect.serviceOption(FetchHttpClient.RequestInit);
|
|
160
|
+
return yield* scopedClient.execute(request).pipe(
|
|
161
|
+
Effect.provideService(FetchHttpClient.RequestInit, {
|
|
162
|
+
...Option.getOrUndefined(requestInit),
|
|
163
|
+
redirect: "manual"
|
|
164
|
+
}),
|
|
165
|
+
Effect.mapError((cause) => transportFailure(operation, cause)),
|
|
166
|
+
Effect.flatMap(use),
|
|
167
|
+
Effect.provideService(Headers.CurrentRedactedNames, [...redactedNames, ...credentialNames]),
|
|
168
|
+
Effect.scoped
|
|
169
|
+
);
|
|
170
|
+
});
|
|
171
|
+
const parseDeadline = (name, input, fallback) => Effect.gen(function* () {
|
|
172
|
+
const parsed = Duration.fromInput(input ?? fallback);
|
|
173
|
+
if (Option.isNone(parsed) || !Number.isFinite(Duration.toMillis(parsed.value)) || Duration.toMillis(parsed.value) <= 0) {
|
|
174
|
+
return yield* Effect.fail(invalidOption(name));
|
|
175
|
+
}
|
|
176
|
+
return parsed.value;
|
|
177
|
+
});
|
|
178
|
+
const downloadDeadline = yield* parseDeadline(
|
|
179
|
+
"downloadTimeout",
|
|
180
|
+
configured.downloadTimeout,
|
|
181
|
+
defaultDownloadTimeout
|
|
182
|
+
);
|
|
183
|
+
const uploadDeadline = yield* parseDeadline("uploadTimeout", configured.uploadTimeout, defaultUploadTimeout);
|
|
184
|
+
const requestDeadline = yield* parseDeadline("requestTimeout", configured.requestTimeout, defaultRequestTimeout);
|
|
185
|
+
const maxDownloadBytes = configured.maxDownloadBytes ?? defaultMaxDownloadBytes;
|
|
186
|
+
if (!Number.isSafeInteger(maxDownloadBytes) || maxDownloadBytes <= 0) {
|
|
187
|
+
return yield* Effect.fail(invalidOption("maxDownloadBytes"));
|
|
188
|
+
}
|
|
189
|
+
const maxFindMissingResponseBytes = configured.maxFindMissingResponseBytes ?? maxFindMissingBodyBytes;
|
|
190
|
+
if (!Number.isSafeInteger(maxFindMissingResponseBytes) || maxFindMissingResponseBytes <= 0 || maxFindMissingResponseBytes > maxFindMissingBodyBytes) {
|
|
191
|
+
return yield* Effect.fail(invalidOption("maxFindMissingResponseBytes"));
|
|
192
|
+
}
|
|
193
|
+
const chunkBytes = configured.chunkBytes;
|
|
194
|
+
if (chunkBytes !== void 0 && (!Number.isSafeInteger(chunkBytes) || chunkBytes <= 0)) {
|
|
195
|
+
return yield* Effect.fail(invalidOption("chunkBytes"));
|
|
196
|
+
}
|
|
197
|
+
const downloadPolicy = configured.downloadPolicy ?? "all";
|
|
198
|
+
if (!Schema.is(DownloadPolicy)(downloadPolicy)) {
|
|
199
|
+
return yield* Effect.fail(invalidOption("downloadPolicy"));
|
|
200
|
+
}
|
|
201
|
+
const bodyTooLarge = (operation, received, bound) => new ArtifactStore.ArtifactStoreError({
|
|
202
|
+
code: "transport_failed",
|
|
203
|
+
message: `the remote artifact tier answered ${operation} with ${received} bytes, past the ${bound}-byte bound`
|
|
204
|
+
});
|
|
205
|
+
const timedOut = (operation) => new ArtifactStore.ArtifactStoreError({
|
|
206
|
+
code: "transport_failed",
|
|
207
|
+
message: `the remote artifact tier did not finish ${operation} within its configured deadline`
|
|
208
|
+
});
|
|
209
|
+
const within = (operation, deadline, effect) => effect.pipe(
|
|
210
|
+
Effect.timeout(deadline),
|
|
211
|
+
Effect.catchTag("TimeoutError", () => Effect.fail(timedOut(operation)))
|
|
212
|
+
);
|
|
213
|
+
const readBounded = (response, operation, bound) => Effect.gen(function* () {
|
|
214
|
+
const declared = response.headers["content-length"];
|
|
215
|
+
if (declared !== void 0) {
|
|
216
|
+
const declaredBytes = Number(declared);
|
|
217
|
+
if (!/^\d+$/.test(declared) || !Number.isSafeInteger(declaredBytes)) {
|
|
218
|
+
return yield* Effect.fail(transportFailure(`${operation} body`));
|
|
219
|
+
}
|
|
220
|
+
if (declaredBytes > bound) return yield* Effect.fail(bodyTooLarge(operation, declaredBytes, bound));
|
|
221
|
+
}
|
|
222
|
+
const chunks = [];
|
|
223
|
+
let received = 0;
|
|
224
|
+
yield* Stream.runForEach(response.stream, (chunk) => Effect.suspend(() => {
|
|
225
|
+
received += chunk.byteLength;
|
|
226
|
+
if (received > bound) return Effect.fail(bodyTooLarge(operation, received, bound));
|
|
227
|
+
chunks.push(new Uint8Array(chunk));
|
|
228
|
+
return Effect.void;
|
|
229
|
+
})).pipe(
|
|
230
|
+
Effect.catchTag("HttpClientError", (cause) => (
|
|
231
|
+
// An absent body is zero bytes, not a transport refusal: the
|
|
232
|
+
// digest check in `get` is the arbiter of whether empty content
|
|
233
|
+
// is the requested artifact.
|
|
234
|
+
cause.reason._tag === "EmptyBodyError" ? Effect.void : Effect.fail(transportFailure(`${operation} body`))
|
|
235
|
+
))
|
|
236
|
+
);
|
|
237
|
+
const bytes = new Uint8Array(received);
|
|
238
|
+
let offset = 0;
|
|
239
|
+
for (const chunk of chunks) {
|
|
240
|
+
bytes.set(chunk, offset);
|
|
241
|
+
offset += chunk.byteLength;
|
|
242
|
+
}
|
|
243
|
+
return bytes;
|
|
244
|
+
});
|
|
245
|
+
const download = (digest) => send("a download", authorized(HttpClientRequest.get(casUrl(digest))), (response) => Effect.gen(function* () {
|
|
246
|
+
if (response.status === 404) {
|
|
247
|
+
return yield* Effect.fail(new ArtifactStore.ArtifactMissing({ code: "artifact_missing", digest }));
|
|
248
|
+
}
|
|
249
|
+
if (!isOk(response)) return yield* Effect.fail(unexpectedStatus("a download", response.status));
|
|
250
|
+
return yield* readBounded(response, "a download", maxDownloadBytes);
|
|
251
|
+
}));
|
|
252
|
+
const uploadWhole = (digest, bytes) => send(
|
|
253
|
+
"an upload",
|
|
254
|
+
authorized(HttpClientRequest.put(casUrl(digest))).pipe(
|
|
255
|
+
HttpClientRequest.bodyUint8Array(bytes, "application/octet-stream")
|
|
256
|
+
),
|
|
257
|
+
(response) => isOk(response) ? Effect.void : Effect.fail(unexpectedStatus("an upload", response.status))
|
|
258
|
+
);
|
|
259
|
+
const resumeOffset = (response) => {
|
|
260
|
+
const header = response.headers["range"];
|
|
261
|
+
if (header === void 0) return void 0;
|
|
262
|
+
const match = /^bytes=0-(\d+)$/.exec(header.trim());
|
|
263
|
+
if (match === null) return void 0;
|
|
264
|
+
const last = Number(match[1]);
|
|
265
|
+
if (!Number.isSafeInteger(last)) return void 0;
|
|
266
|
+
return last + 1;
|
|
267
|
+
};
|
|
268
|
+
const rejectsRanges = (status) => status === 400 || status === 411 || status === 416;
|
|
269
|
+
const holdsWhole = (digest, total) => send("an existence probe", authorized(HttpClientRequest.head(casUrl(digest))), (response) => Effect.sync(() => {
|
|
270
|
+
if (!isOk(response)) return false;
|
|
271
|
+
const declared = response.headers["content-length"];
|
|
272
|
+
return declared !== void 0 && Number(declared) === total;
|
|
273
|
+
}));
|
|
274
|
+
const uploadChunked = (digest, bytes, chunk) => Effect.gen(function* () {
|
|
275
|
+
const total = bytes.byteLength;
|
|
276
|
+
const ranged = (range, body) => send(
|
|
277
|
+
"an upload",
|
|
278
|
+
authorized(HttpClientRequest.put(casUrl(digest))).pipe(
|
|
279
|
+
HttpClientRequest.bodyUint8Array(body, "application/octet-stream"),
|
|
280
|
+
HttpClientRequest.setHeader("content-range", range)
|
|
281
|
+
),
|
|
282
|
+
// Only status and headers are used after the response is released.
|
|
283
|
+
Effect.succeed
|
|
284
|
+
);
|
|
285
|
+
if (yield* holdsWhole(digest, total)) return true;
|
|
286
|
+
const probe = yield* ranged(`bytes */${total}`, new Uint8Array(0));
|
|
287
|
+
if (isOk(probe) || rejectsRanges(probe.status)) return false;
|
|
288
|
+
if (probe.status !== 308 && probe.status !== 404) {
|
|
289
|
+
return yield* Effect.fail(unexpectedStatus("an upload probe", probe.status));
|
|
290
|
+
}
|
|
291
|
+
let offset = probe.status === 308 ? resumeOffset(probe) ?? 0 : 0;
|
|
292
|
+
while (offset < total) {
|
|
293
|
+
const end = Math.min(offset + chunk, total);
|
|
294
|
+
const answer = yield* ranged(`bytes ${offset}-${end - 1}/${total}`, bytes.subarray(offset, end));
|
|
295
|
+
if (answer.status !== 308) {
|
|
296
|
+
if (end === total && isOk(answer)) break;
|
|
297
|
+
if (isOk(answer) || rejectsRanges(answer.status)) return false;
|
|
298
|
+
return yield* Effect.fail(unexpectedStatus("an upload", answer.status));
|
|
299
|
+
}
|
|
300
|
+
offset = Math.max(end, resumeOffset(answer) ?? 0);
|
|
301
|
+
}
|
|
302
|
+
return yield* holdsWhole(digest, total);
|
|
303
|
+
});
|
|
304
|
+
const put = Effect.fn("RemoteArtifacts.put")(
|
|
305
|
+
(bytes) => Effect.flatMap(ArtifactStore.snapshotBytes(bytes), (snapshot) => Effect.gen(function* () {
|
|
306
|
+
const digest = yield* ArtifactStore.measureBytes(snapshot);
|
|
307
|
+
yield* Effect.annotateCurrentSpan({ digest });
|
|
308
|
+
yield* within(
|
|
309
|
+
"an upload",
|
|
310
|
+
uploadDeadline,
|
|
311
|
+
Effect.gen(function* () {
|
|
312
|
+
if (chunkBytes === void 0 || snapshot.byteLength <= chunkBytes) {
|
|
313
|
+
yield* uploadWhole(digest, snapshot);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
const transferred = yield* uploadChunked(digest, snapshot, chunkBytes);
|
|
317
|
+
if (!transferred) yield* uploadWhole(digest, snapshot);
|
|
318
|
+
})
|
|
319
|
+
);
|
|
320
|
+
return digest;
|
|
321
|
+
}))
|
|
322
|
+
);
|
|
323
|
+
const get = Effect.fn("RemoteArtifacts.get")(
|
|
324
|
+
(digest) => Effect.gen(function* () {
|
|
325
|
+
const validated = yield* ArtifactStore.validateDigest(digest);
|
|
326
|
+
yield* Effect.annotateCurrentSpan({ digest: validated });
|
|
327
|
+
const bytes = yield* within("a download", downloadDeadline, download(validated));
|
|
328
|
+
const measured = yield* ArtifactStore.measureBytes(bytes);
|
|
329
|
+
if (measured !== validated) {
|
|
330
|
+
return yield* Effect.fail(
|
|
331
|
+
new ArtifactStore.ArtifactCorruption({
|
|
332
|
+
code: "artifact_corruption",
|
|
333
|
+
recordedDigest: validated,
|
|
334
|
+
measuredDigest: measured
|
|
335
|
+
})
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
return bytes;
|
|
339
|
+
})
|
|
340
|
+
);
|
|
341
|
+
const has = Effect.fn("RemoteArtifacts.has")(
|
|
342
|
+
(digest) => Effect.gen(function* () {
|
|
343
|
+
const validated = yield* ArtifactStore.validateDigest(digest);
|
|
344
|
+
yield* Effect.annotateCurrentSpan({ digest: validated });
|
|
345
|
+
const response = yield* within(
|
|
346
|
+
"an existence probe",
|
|
347
|
+
requestDeadline,
|
|
348
|
+
send("an existence probe", authorized(HttpClientRequest.head(casUrl(validated))), Effect.succeed)
|
|
349
|
+
);
|
|
350
|
+
if (response.status === 404) return false;
|
|
351
|
+
if (!isOk(response)) return yield* Effect.fail(unexpectedStatus("an existence probe", response.status));
|
|
352
|
+
return true;
|
|
353
|
+
})
|
|
354
|
+
);
|
|
355
|
+
const findMissing = Effect.fn("RemoteArtifacts.findMissing")(
|
|
356
|
+
(digests) => Effect.gen(function* () {
|
|
357
|
+
const requested = [...new Set(digests)];
|
|
358
|
+
if (requested.length === 0) return [];
|
|
359
|
+
const validated = [];
|
|
360
|
+
for (const digest of requested) validated.push(yield* ArtifactStore.validateDigest(digest));
|
|
361
|
+
yield* Effect.annotateCurrentSpan({ count: validated.length });
|
|
362
|
+
const missing = /* @__PURE__ */ new Set();
|
|
363
|
+
for (let offset = 0; offset < validated.length; offset += maxFindMissingBatchDigests) {
|
|
364
|
+
const batch = validated.slice(offset, offset + maxFindMissingBatchDigests);
|
|
365
|
+
const requestBody = encoder.encode(JSON.stringify({ digests: batch }));
|
|
366
|
+
if (requestBody.byteLength > maxFindMissingBodyBytes) {
|
|
367
|
+
return yield* Effect.fail(configurationFailure("remote artifact findMissing request exceeds 256 KiB"));
|
|
368
|
+
}
|
|
369
|
+
const decoded = yield* within(
|
|
370
|
+
"a batched existence probe",
|
|
371
|
+
requestDeadline,
|
|
372
|
+
send(
|
|
373
|
+
"a batched existence probe",
|
|
374
|
+
authorized(HttpClientRequest.post(`${base}/cas/findMissing`)).pipe(
|
|
375
|
+
HttpClientRequest.bodyUint8Array(requestBody, "application/json")
|
|
376
|
+
),
|
|
377
|
+
(response) => Effect.gen(function* () {
|
|
378
|
+
if (!isOk(response)) {
|
|
379
|
+
return yield* Effect.fail(unexpectedStatus("a batched existence probe", response.status));
|
|
380
|
+
}
|
|
381
|
+
const bodyBytes = yield* readBounded(
|
|
382
|
+
response,
|
|
383
|
+
"a batched existence probe",
|
|
384
|
+
maxFindMissingResponseBytes
|
|
385
|
+
);
|
|
386
|
+
const body = yield* Effect.try({
|
|
387
|
+
try: () => JSON.parse(decoder.decode(bodyBytes)),
|
|
388
|
+
catch: () => transportFailure("a batched existence probe body")
|
|
389
|
+
});
|
|
390
|
+
return yield* Schema.decodeUnknownEffect(FindMissingResponse)(body).pipe(
|
|
391
|
+
Effect.mapError(() => transportFailure("a batched existence probe body"))
|
|
392
|
+
);
|
|
393
|
+
})
|
|
394
|
+
)
|
|
395
|
+
);
|
|
396
|
+
const asked = new Set(batch);
|
|
397
|
+
for (const digest of decoded.missing) if (asked.has(digest)) missing.add(digest);
|
|
398
|
+
}
|
|
399
|
+
return validated.filter((digest) => missing.has(digest));
|
|
400
|
+
})
|
|
401
|
+
);
|
|
402
|
+
return { put, get, has, findMissing, downloadPolicy };
|
|
403
|
+
});
|
|
404
|
+
const layer = (options) => Layer.effect(ArtifactStore.ArtifactStore)(make(options));
|
|
405
|
+
//# sourceMappingURL=RemoteArtifacts.js.map
|