@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/LICENSE +21 -0
- package/README.md +173 -2
- package/dist/cjs/ArtifactBackupLease.d.ts +34 -0
- package/dist/cjs/ArtifactBackupLease.d.ts.map +1 -0
- package/dist/cjs/ArtifactBackupLease.js +175 -0
- package/dist/cjs/ArtifactBackupLease.js.map +7 -0
- package/dist/cjs/ArtifactStore.d.ts +129 -0
- package/dist/cjs/ArtifactStore.d.ts.map +1 -0
- package/dist/cjs/ArtifactStore.js +73 -0
- package/dist/cjs/ArtifactStore.js.map +7 -0
- package/dist/cjs/ArtifactStoreError.d.ts +82 -0
- package/dist/cjs/ArtifactStoreError.d.ts.map +1 -0
- package/dist/cjs/ArtifactStoreError.js +72 -0
- package/dist/cjs/ArtifactStoreError.js.map +7 -0
- package/dist/cjs/ArtifactStoreMetrics.d.ts +70 -0
- package/dist/cjs/ArtifactStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/ArtifactStoreMetrics.js +51 -0
- package/dist/cjs/ArtifactStoreMetrics.js.map +7 -0
- package/dist/cjs/ArtifactSweep.d.ts +185 -0
- package/dist/cjs/ArtifactSweep.d.ts.map +1 -0
- package/dist/cjs/ArtifactSweep.js +163 -0
- package/dist/cjs/ArtifactSweep.js.map +7 -0
- package/dist/cjs/CombinedArtifacts.d.ts +103 -0
- package/dist/cjs/CombinedArtifacts.d.ts.map +1 -0
- package/dist/cjs/CombinedArtifacts.js +189 -0
- package/dist/cjs/CombinedArtifacts.js.map +7 -0
- package/dist/cjs/FileLease.d.ts +113 -0
- package/dist/cjs/FileLease.d.ts.map +1 -0
- package/dist/cjs/FileLease.js +201 -0
- package/dist/cjs/FileLease.js.map +7 -0
- package/dist/cjs/FileSystemArtifactStore.d.ts +93 -0
- package/dist/cjs/FileSystemArtifactStore.d.ts.map +1 -0
- package/dist/cjs/FileSystemArtifactStore.js +276 -0
- package/dist/cjs/FileSystemArtifactStore.js.map +7 -0
- package/dist/cjs/MemoryArtifactStore.d.ts +25 -0
- package/dist/cjs/MemoryArtifactStore.d.ts.map +1 -0
- package/dist/cjs/MemoryArtifactStore.js +84 -0
- package/dist/cjs/MemoryArtifactStore.js.map +7 -0
- package/dist/cjs/NoopArtifactStore.d.ts +17 -0
- package/dist/cjs/NoopArtifactStore.d.ts.map +1 -0
- package/dist/cjs/NoopArtifactStore.js +46 -0
- package/dist/cjs/NoopArtifactStore.js.map +7 -0
- package/dist/cjs/RemoteArtifacts.d.ts +237 -0
- package/dist/cjs/RemoteArtifacts.d.ts.map +1 -0
- package/dist/cjs/RemoteArtifacts.js +405 -0
- package/dist/cjs/RemoteArtifacts.js.map +7 -0
- package/dist/cjs/index.d.ts +65 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +47 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/ArtifactLocks.d.ts +30 -0
- package/dist/cjs/internal/ArtifactLocks.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactLocks.js +93 -0
- package/dist/cjs/internal/ArtifactLocks.js.map +7 -0
- package/dist/cjs/internal/ArtifactPath.d.ts +19 -0
- package/dist/cjs/internal/ArtifactPath.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactPath.js +74 -0
- package/dist/cjs/internal/ArtifactPath.js.map +7 -0
- package/dist/cjs/measureBytes.d.ts +18 -0
- package/dist/cjs/measureBytes.d.ts.map +1 -0
- package/dist/cjs/measureBytes.js +43 -0
- package/dist/cjs/measureBytes.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/snapshotBytes.d.ts +19 -0
- package/dist/cjs/snapshotBytes.d.ts.map +1 -0
- package/dist/cjs/snapshotBytes.js +41 -0
- package/dist/cjs/snapshotBytes.js.map +7 -0
- package/dist/cjs/validateDigest.d.ts +22 -0
- package/dist/cjs/validateDigest.d.ts.map +1 -0
- package/dist/cjs/validateDigest.js +42 -0
- package/dist/cjs/validateDigest.js.map +7 -0
- package/dist/esm/ArtifactBackupLease.d.ts +34 -0
- package/dist/esm/ArtifactBackupLease.d.ts.map +1 -0
- package/dist/esm/ArtifactBackupLease.js +117 -0
- package/dist/esm/ArtifactBackupLease.js.map +1 -0
- package/dist/esm/ArtifactStore.d.ts +129 -0
- package/dist/esm/ArtifactStore.d.ts.map +1 -0
- package/dist/esm/ArtifactStore.js +83 -0
- package/dist/esm/ArtifactStore.js.map +1 -0
- package/dist/esm/ArtifactStoreError.d.ts +82 -0
- package/dist/esm/ArtifactStoreError.d.ts.map +1 -0
- package/dist/esm/ArtifactStoreError.js +77 -0
- package/dist/esm/ArtifactStoreError.js.map +1 -0
- package/dist/esm/ArtifactStoreMetrics.d.ts +70 -0
- package/dist/esm/ArtifactStoreMetrics.d.ts.map +1 -0
- package/dist/esm/ArtifactStoreMetrics.js +77 -0
- package/dist/esm/ArtifactStoreMetrics.js.map +1 -0
- package/dist/esm/ArtifactSweep.d.ts +185 -0
- package/dist/esm/ArtifactSweep.d.ts.map +1 -0
- package/dist/esm/ArtifactSweep.js +202 -0
- package/dist/esm/ArtifactSweep.js.map +1 -0
- package/dist/esm/CombinedArtifacts.d.ts +103 -0
- package/dist/esm/CombinedArtifacts.d.ts.map +1 -0
- package/dist/esm/CombinedArtifacts.js +214 -0
- package/dist/esm/CombinedArtifacts.js.map +1 -0
- package/dist/esm/FileLease.d.ts +113 -0
- package/dist/esm/FileLease.d.ts.map +1 -0
- package/dist/esm/FileLease.js +239 -0
- package/dist/esm/FileLease.js.map +1 -0
- package/dist/esm/FileSystemArtifactStore.d.ts +93 -0
- package/dist/esm/FileSystemArtifactStore.d.ts.map +1 -0
- package/dist/esm/FileSystemArtifactStore.js +336 -0
- package/dist/esm/FileSystemArtifactStore.js.map +1 -0
- package/dist/esm/MemoryArtifactStore.d.ts +25 -0
- package/dist/esm/MemoryArtifactStore.d.ts.map +1 -0
- package/dist/esm/MemoryArtifactStore.js +67 -0
- package/dist/esm/MemoryArtifactStore.js.map +1 -0
- package/dist/esm/NoopArtifactStore.d.ts +17 -0
- package/dist/esm/NoopArtifactStore.d.ts.map +1 -0
- package/dist/esm/NoopArtifactStore.js +27 -0
- package/dist/esm/NoopArtifactStore.js.map +1 -0
- package/dist/esm/RemoteArtifacts.d.ts +237 -0
- package/dist/esm/RemoteArtifacts.d.ts.map +1 -0
- package/dist/esm/RemoteArtifacts.js +554 -0
- package/dist/esm/RemoteArtifacts.js.map +1 -0
- package/dist/esm/index.d.ts +65 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +65 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/ArtifactLocks.d.ts +30 -0
- package/dist/esm/internal/ArtifactLocks.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactLocks.js +86 -0
- package/dist/esm/internal/ArtifactLocks.js.map +1 -0
- package/dist/esm/internal/ArtifactPath.d.ts +19 -0
- package/dist/esm/internal/ArtifactPath.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactPath.js +59 -0
- package/dist/esm/internal/ArtifactPath.js.map +1 -0
- package/dist/esm/measureBytes.d.ts +18 -0
- package/dist/esm/measureBytes.d.ts.map +1 -0
- package/dist/esm/measureBytes.js +22 -0
- package/dist/esm/measureBytes.js.map +1 -0
- package/dist/esm/snapshotBytes.d.ts +19 -0
- package/dist/esm/snapshotBytes.d.ts.map +1 -0
- package/dist/esm/snapshotBytes.js +30 -0
- package/dist/esm/snapshotBytes.js.map +1 -0
- package/dist/esm/validateDigest.d.ts +22 -0
- package/dist/esm/validateDigest.d.ts.map +1 -0
- package/dist/esm/validateDigest.js +26 -0
- package/dist/esm/validateDigest.js.map +1 -0
- package/docs/README.md +200 -0
- package/docs/api.md +474 -0
- package/docs/concepts/content-addressing.md +174 -0
- package/docs/concepts/coordination.md +141 -0
- package/docs/concepts/tiers.md +147 -0
- package/docs/guides/fence-a-backup.md +123 -0
- package/docs/guides/reclaim-disk-space.md +143 -0
- package/docs/guides/serve-the-artifact-protocol.md +140 -0
- package/docs/guides/share-artifacts-across-machines.md +180 -0
- package/docs/guides/test-against-an-artifact-store.md +158 -0
- package/docs/installation.md +98 -0
- package/docs/quickstart.md +182 -0
- package/docs/troubleshooting.md +232 -0
- package/package.json +179 -3
- package/src/ArtifactBackupLease.ts +213 -0
- package/src/ArtifactStore.ts +152 -0
- package/src/ArtifactStoreError.ts +96 -0
- package/src/ArtifactStoreMetrics.ts +83 -0
- package/src/ArtifactSweep.ts +336 -0
- package/src/CombinedArtifacts.ts +369 -0
- package/src/FileLease.ts +380 -0
- package/src/FileSystemArtifactStore.ts +459 -0
- package/src/MemoryArtifactStore.ts +83 -0
- package/src/NoopArtifactStore.ts +30 -0
- package/src/RemoteArtifacts.ts +809 -0
- package/src/index.ts +71 -0
- package/src/internal/ArtifactLocks.ts +110 -0
- package/src/internal/ArtifactPath.ts +72 -0
- package/src/measureBytes.ts +28 -0
- package/src/snapshotBytes.ts +34 -0
- package/src/validateDigest.ts +31 -0
|
@@ -0,0 +1,459 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The filesystem-backed artifact store: blobs in a workspace-relative objects
|
|
3
|
+
* directory under a two-hex-prefix fanout, published atomically, coordinated
|
|
4
|
+
* across processes, and digest-verified on every read.
|
|
5
|
+
*
|
|
6
|
+
* @since 1.0.0-rc.0
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import * as Clock from "effect/Clock"
|
|
10
|
+
import * as Effect from "effect/Effect"
|
|
11
|
+
import type * as FileSystem from "effect/FileSystem"
|
|
12
|
+
import * as Metric from "effect/Metric"
|
|
13
|
+
import * as Option from "effect/Option"
|
|
14
|
+
import type * as Path from "effect/Path"
|
|
15
|
+
import * as PlatformError from "effect/PlatformError"
|
|
16
|
+
import * as Random from "effect/Random"
|
|
17
|
+
import type { Service } from "./ArtifactStore.ts"
|
|
18
|
+
import { ArtifactCorruption, ArtifactMissing, ArtifactStoreError } from "./ArtifactStoreError.ts"
|
|
19
|
+
import * as ArtifactStoreMetrics from "./ArtifactStoreMetrics.ts"
|
|
20
|
+
import * as ArtifactLocks from "./internal/ArtifactLocks.ts"
|
|
21
|
+
import * as ArtifactPath from "./internal/ArtifactPath.ts"
|
|
22
|
+
import { measureBytes } from "./measureBytes.ts"
|
|
23
|
+
import { snapshotBytes } from "./snapshotBytes.ts"
|
|
24
|
+
import { validateDigest } from "./validateDigest.ts"
|
|
25
|
+
|
|
26
|
+
const hostFailure = (cause: unknown): ArtifactStoreError =>
|
|
27
|
+
cause instanceof ArtifactStoreError ? cause : new ArtifactStoreError({
|
|
28
|
+
code: "unavailable",
|
|
29
|
+
message: `the host filesystem refused an artifact operation: ${String(cause)}`,
|
|
30
|
+
cause
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Where the filesystem-backed store keeps its blobs.
|
|
35
|
+
*
|
|
36
|
+
* The directory is workspace-relative rather than absolute so a workspace can
|
|
37
|
+
* be moved or copied whole and still resolve its own artifacts.
|
|
38
|
+
*
|
|
39
|
+
* @category models
|
|
40
|
+
* @since 1.0.0-rc.0
|
|
41
|
+
* @slop
|
|
42
|
+
*/
|
|
43
|
+
export interface FileSystemOptions {
|
|
44
|
+
/**
|
|
45
|
+
* Where blobs are stored, content-addressed by digest. Workspace-relative;
|
|
46
|
+
* defaults to {@link defaultDirectory}. An `ArtifactSweep` over the same
|
|
47
|
+
* store must be built with this same directory, or it enumerates somewhere
|
|
48
|
+
* the store never publishes.
|
|
49
|
+
*/
|
|
50
|
+
readonly directory?: string | undefined
|
|
51
|
+
/** New payload mode, default `0600`, restricted by the umask. Existing blobs are unchanged. */
|
|
52
|
+
readonly fileMode?: number | undefined
|
|
53
|
+
/** New objects and fanout directory mode, default `0700`. Existing directories are unchanged. */
|
|
54
|
+
readonly directoryMode?: number | undefined
|
|
55
|
+
/**
|
|
56
|
+
* `required` reports success only after syncing the blob, its fanout,
|
|
57
|
+
* the objects directory, and every ancestor. `best-effort` is the explicit weaker
|
|
58
|
+
* capability for hosts that cannot sync file or directory handles.
|
|
59
|
+
* Both modes require exclusive writable handles and symlink inspection.
|
|
60
|
+
*/
|
|
61
|
+
readonly durability?: "required" | "best-effort" | undefined
|
|
62
|
+
/**
|
|
63
|
+
* `required` uses an atomic lock file to coordinate writers and sweepers
|
|
64
|
+
* across processes. `process` is the explicit weaker browser/test mode.
|
|
65
|
+
*
|
|
66
|
+
* The fence bounds the race rather than eliminating it: the lock is
|
|
67
|
+
* heartbeated every 10 seconds, another process reclaims it once it is 60
|
|
68
|
+
* seconds stale, and acquisition itself gives up after 2 minutes. A holder
|
|
69
|
+
* whose host stalls past the stale bound can therefore be reaped while it is
|
|
70
|
+
* still running.
|
|
71
|
+
*
|
|
72
|
+
* Contenders that measure the same lock as stale race for a claim file named
|
|
73
|
+
* after its owner token, so each lock generation is moved away at most once
|
|
74
|
+
* and a fresh replacement is never displaced by a late stale verdict.
|
|
75
|
+
* Neither the mtime check nor the backup lease independently protects
|
|
76
|
+
* against a reaped stalled holder: the check precedes a separate delete, and
|
|
77
|
+
* the gate uses this same protocol.
|
|
78
|
+
*
|
|
79
|
+
* It also only fences parties that agree. An `ArtifactSweep` over the same
|
|
80
|
+
* directory must be built with the same `coordination`: a store on `process`
|
|
81
|
+
* paired with a sweep on `required` takes lock files no writer observes, so
|
|
82
|
+
* the fence reads as armed and protects nothing.
|
|
83
|
+
*/
|
|
84
|
+
readonly coordination?: "required" | "process" | undefined
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The default objects directory. Workspace-relative, so a workspace carries
|
|
89
|
+
* its own artifacts and a sandbox that mounts the workspace inherits them.
|
|
90
|
+
*
|
|
91
|
+
* Exported because the store and its sweep must name the same directory, and a
|
|
92
|
+
* second private copy of the literal is exactly how they would drift apart.
|
|
93
|
+
*
|
|
94
|
+
* @category constants
|
|
95
|
+
* @since 1.0.0-rc.0
|
|
96
|
+
*/
|
|
97
|
+
export const defaultDirectory = ".flows/objects"
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* How old a scratch file must be before the sweep treats it as a crash orphan
|
|
101
|
+
* rather than a live writer's in-flight file. A publication writes and renames
|
|
102
|
+
* within one `put`, so an hour is far beyond any live writer's window, and it
|
|
103
|
+
* is sixty times the 60-second bound after which a lock file's own contention
|
|
104
|
+
* path reclaims it, so a lock this old belongs to no living holder either.
|
|
105
|
+
*/
|
|
106
|
+
const staleScratchMs = 60 * 60 * 1000
|
|
107
|
+
|
|
108
|
+
/** Whether an entry is scratch the sweep may reclaim once it is stale. */
|
|
109
|
+
const isScratch = (entry: string): boolean =>
|
|
110
|
+
entry.includes(".tmp-") || entry.startsWith(`${ArtifactLocks.directoryName}/`)
|
|
111
|
+
|
|
112
|
+
const isNotFound = (cause: unknown): boolean =>
|
|
113
|
+
cause instanceof PlatformError.PlatformError && cause.reason._tag === "NotFound"
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Bazel's `DiskCacheClient.toPath` layout: a two-hex-prefix subdirectory
|
|
117
|
+
* "to bypass possible folder file count limits"
|
|
118
|
+
* (class `DiskCacheClient` in `com.google.devtools.build.lib.remote.disk`). The
|
|
119
|
+
* store moved out of `StepBoundary` with a flat `${dir}/${digest}` layout, which puts every
|
|
120
|
+
* artifact a workspace ever spilled into one directory. The rc.0 contract has
|
|
121
|
+
* no compatibility shim for the provisional flat layout; old addresses are
|
|
122
|
+
* cache misses that re-publish.
|
|
123
|
+
*/
|
|
124
|
+
const fanout = (directory: string, digest: string): { readonly parent: string; readonly path: string } => {
|
|
125
|
+
const parent = `${directory}/${digest.slice(0, 2)}`
|
|
126
|
+
return { parent, path: `${parent}/${digest}` }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Builds the filesystem-backed artifact store.
|
|
131
|
+
*
|
|
132
|
+
* Host access arrives through Effect's `FileSystem` tag. Persistence needs a
|
|
133
|
+
* trusted host with exclusive writable handles, symlink inspection and, for
|
|
134
|
+
* required durability, file and directory syncing. Build this service before
|
|
135
|
+
* decorating the workflow filesystem with the capability kernel, as native
|
|
136
|
+
* `layerHost` does. The guarded native filesystem deliberately refuses open
|
|
137
|
+
* handles and cannot back this store; best-effort durability does not lift
|
|
138
|
+
* that requirement. A browser host can instead use the memory store.
|
|
139
|
+
* `platformPath` is the host's `Path` service; the directory fsync walks ancestors with it,
|
|
140
|
+
* so a Windows drive directory syncs its real parents.
|
|
141
|
+
*
|
|
142
|
+
* @category constructors
|
|
143
|
+
* @since 1.0.0-rc.0
|
|
144
|
+
* @slop
|
|
145
|
+
*/
|
|
146
|
+
export const makeFileSystem = (
|
|
147
|
+
fs: FileSystem.FileSystem,
|
|
148
|
+
platformPath: Path.Path,
|
|
149
|
+
options: FileSystemOptions = {}
|
|
150
|
+
): Service => {
|
|
151
|
+
const directory = (options.directory ?? defaultDirectory).replace(/([^/])\/+$/, "$1")
|
|
152
|
+
const durability = options.durability ?? "required"
|
|
153
|
+
const coordination = options.coordination ?? "required"
|
|
154
|
+
const fileMode = options.fileMode ?? 0o600
|
|
155
|
+
const directoryMode = options.directoryMode ?? 0o700
|
|
156
|
+
// Every attempt draws a new token; exclusivity, not unpredictability, protects
|
|
157
|
+
// an existing entry. Bound retries so a hostile directory cannot hang put.
|
|
158
|
+
const freshTempToken = Effect.map(
|
|
159
|
+
Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true }),
|
|
160
|
+
(drawn) => drawn.toString(36)
|
|
161
|
+
)
|
|
162
|
+
/**
|
|
163
|
+
* Best-effort reclamation of scratch files orphaned by a crash: `.tmp-*`
|
|
164
|
+
* payloads left between the temp write and the rename, and `.locks/*` files
|
|
165
|
+
* left by a holder that was hard-killed. Nothing else ever observes either —
|
|
166
|
+
* reads resolve only canonical paths, and a lock is reclaimed on contention
|
|
167
|
+
* alone, so a digest nobody publishes again keeps its lock file forever —
|
|
168
|
+
* which is how the objects directory would accumulate dead files unboundedly.
|
|
169
|
+
* The sweep runs once per store, on the first publication, and is
|
|
170
|
+
* conservative: a scratch file younger than the stale bound may belong to a
|
|
171
|
+
* live writer or lock holder in another process, and one whose age cannot be
|
|
172
|
+
* measured says nothing about its owner, so both survive. Every step is
|
|
173
|
+
* best-effort — a missing directory or failing host never fails the
|
|
174
|
+
* publication.
|
|
175
|
+
*
|
|
176
|
+
* It measures and then removes, which is not atomic, so a lock file replaced
|
|
177
|
+
* between those two steps is removed on the strength of its predecessor's
|
|
178
|
+
* age. The hour-long threshold is what keeps that narrow: a live holder
|
|
179
|
+
* heartbeats its lock every 10 seconds, so only a path abandoned for an hour
|
|
180
|
+
* is ever a candidate, and only a reclaim landing inside that one gap loses
|
|
181
|
+
* its lock. It is the same non-atomic reclamation `FileSystemOptions`'
|
|
182
|
+
* `coordination` documents, applied to the files that reclamation leaves
|
|
183
|
+
* behind.
|
|
184
|
+
*
|
|
185
|
+
* This is a sweep of scratch files, not garbage collection. Reclaiming
|
|
186
|
+
* *published* artifacts is `ArtifactSweep` driven by an explicit
|
|
187
|
+
* `ArtifactGc.gc()` call in `@smthrs/engine-store`, never folded in here.
|
|
188
|
+
*/
|
|
189
|
+
let sweepDone = false
|
|
190
|
+
const sweepOrphanedTemps = Effect.gen(function*() {
|
|
191
|
+
if (sweepDone) return
|
|
192
|
+
sweepDone = true
|
|
193
|
+
const checkRoot = yield* ArtifactPath.guard(fs, directory)
|
|
194
|
+
const parents = yield* fs.readDirectory(directory)
|
|
195
|
+
const now = yield* Clock.currentTimeMillis
|
|
196
|
+
for (const parent of parents) {
|
|
197
|
+
if (!/^[0-9a-f]{2}$/.test(parent) && parent !== ArtifactLocks.directoryName) continue
|
|
198
|
+
const parentPath = `${directory}/${parent}`
|
|
199
|
+
yield* checkRoot
|
|
200
|
+
const checkParent = yield* ArtifactPath.guard(fs, parentPath).pipe(Effect.option)
|
|
201
|
+
if (Option.isNone(checkParent)) continue
|
|
202
|
+
const entries = yield* fs.readDirectory(parentPath).pipe(Effect.orElseSucceed(() => [] as Array<string>))
|
|
203
|
+
for (const entry of entries) {
|
|
204
|
+
if (entry.includes("/") || entry.includes("\\") || !isScratch(`${parent}/${entry}`)) continue
|
|
205
|
+
const orphanPath = `${parentPath}/${entry}`
|
|
206
|
+
yield* Effect.gen(function*() {
|
|
207
|
+
const checkFile = yield* ArtifactPath.guard(fs, orphanPath, "File")
|
|
208
|
+
const info = yield* fs.stat(orphanPath)
|
|
209
|
+
const mtime = Option.getOrUndefined(info.mtime)
|
|
210
|
+
if (mtime === undefined || now - mtime.getTime() < staleScratchMs) return
|
|
211
|
+
yield* checkRoot
|
|
212
|
+
yield* checkParent.value
|
|
213
|
+
yield* checkFile
|
|
214
|
+
yield* fs.remove(orphanPath)
|
|
215
|
+
}).pipe(Effect.ignore)
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}).pipe(Effect.ignore)
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Flushes a freshly written temp file before it is renamed into place.
|
|
222
|
+
*
|
|
223
|
+
* Bazel does exactly this in `DiskCacheClient.saveFile`: "fsync temp before
|
|
224
|
+
* we rename it to avoid data loss in the case of machine crashes (the OS may
|
|
225
|
+
* reorder the writes and the rename)". Scratch files sync on their retained
|
|
226
|
+
* handle; this helper syncs published blobs and directories. Best-effort
|
|
227
|
+
* durability tolerates sync refusals, but still requires exclusive creation.
|
|
228
|
+
*/
|
|
229
|
+
const syncPath = (target: string, flag: "r" | "r+"): Effect.Effect<void, ArtifactStoreError> => {
|
|
230
|
+
const sync = Effect.scoped(Effect.flatMap(fs.open(target, { flag }), (file) => file.sync)).pipe(
|
|
231
|
+
Effect.mapError(hostFailure)
|
|
232
|
+
)
|
|
233
|
+
return durability === "best-effort" ? Effect.ignore(sync) : sync
|
|
234
|
+
}
|
|
235
|
+
// A directory sync persists its children, not its own name in its parent.
|
|
236
|
+
// Repeat the whole ancestry even on dedupe: an existing directory may have
|
|
237
|
+
// been created by an interrupted publication in another store or process.
|
|
238
|
+
// Syncing objects also persists the lock directory created by withDigest.
|
|
239
|
+
const syncDirectoryAncestry = Effect.gen(function*() {
|
|
240
|
+
let current = directory
|
|
241
|
+
while (true) {
|
|
242
|
+
yield* syncPath(current, "r")
|
|
243
|
+
const parent = platformPath.dirname(current)
|
|
244
|
+
if (parent === current) return
|
|
245
|
+
current = parent
|
|
246
|
+
}
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
const put: Service["put"] = Effect.fn("ArtifactStore.put")((bytes: Uint8Array) =>
|
|
250
|
+
Effect.flatMap(
|
|
251
|
+
snapshotBytes(bytes),
|
|
252
|
+
(snapshot) =>
|
|
253
|
+
Effect.flatMap(measureBytes(snapshot), (digest) =>
|
|
254
|
+
Effect.gen(function*() {
|
|
255
|
+
yield* ArtifactPath.guard(fs, directory).pipe(Effect.mapError(hostFailure))
|
|
256
|
+
yield* fs.makeDirectory(directory, { recursive: true, mode: directoryMode }).pipe(
|
|
257
|
+
Effect.mapError(hostFailure)
|
|
258
|
+
)
|
|
259
|
+
const checkRoot = yield* ArtifactPath.guard(fs, directory).pipe(Effect.mapError(hostFailure))
|
|
260
|
+
return yield* ArtifactLocks.withDigest(
|
|
261
|
+
fs,
|
|
262
|
+
directory,
|
|
263
|
+
digest,
|
|
264
|
+
Effect.gen(function*() {
|
|
265
|
+
yield* Effect.annotateCurrentSpan({ digest })
|
|
266
|
+
const blob = fanout(directory, digest)
|
|
267
|
+
yield* checkRoot.pipe(Effect.mapError(hostFailure))
|
|
268
|
+
yield* ArtifactPath.guard(fs, blob.parent).pipe(Effect.mapError(hostFailure))
|
|
269
|
+
yield* fs.makeDirectory(blob.parent, { mode: directoryMode, recursive: true }).pipe(
|
|
270
|
+
Effect.mapError(hostFailure)
|
|
271
|
+
)
|
|
272
|
+
const checkParent = yield* ArtifactPath.guard(fs, blob.parent).pipe(Effect.mapError(hostFailure))
|
|
273
|
+
const checkBlob = yield* ArtifactPath.guard(fs, blob.path, "File").pipe(Effect.mapError(hostFailure))
|
|
274
|
+
const stored = yield* fs.exists(blob.path).pipe(Effect.mapError(hostFailure))
|
|
275
|
+
// Existence alone is not validity: a truncated blob left by a crashing
|
|
276
|
+
// writer or by disk corruption would otherwise be trusted forever at
|
|
277
|
+
// write time while `get` digest-verifies and refuses — a permanent
|
|
278
|
+
// failure with no repair path even though this process holds the correct
|
|
279
|
+
// bytes. The existing blob is digest-verified on EVERY put (an
|
|
280
|
+
// unreadable blob counts as corrupt), and only a verified match skips
|
|
281
|
+
// the write; a mismatch falls through to the atomic rewrite below,
|
|
282
|
+
// healing the address. Verification is deliberately not memoized: the
|
|
283
|
+
// objects directory is workspace-shared, so a blob can change behind
|
|
284
|
+
// this store's back, and a remembered proof let a later `put` report
|
|
285
|
+
// success over corrupt bytes without repairing them — `get` would then
|
|
286
|
+
// refuse the digest forever even though every `put` held the cure.
|
|
287
|
+
// Re-verifying costs a constant factor, never a new asymptote: a `put`
|
|
288
|
+
// already pays one O(blob size) hash to measure its own input.
|
|
289
|
+
let verified = stored &&
|
|
290
|
+
(yield* fs.readFile(blob.path).pipe(
|
|
291
|
+
Effect.flatMap((existing) => Effect.map(measureBytes(existing), (measured) => measured === digest)),
|
|
292
|
+
Effect.catch(() => Effect.succeed(false))
|
|
293
|
+
))
|
|
294
|
+
if (verified) {
|
|
295
|
+
// Freshen the blob's mtime on a dedupe hit — git's loose-object
|
|
296
|
+
// freshening, and the touch Bazel's `DiskCacheClient` performs on a
|
|
297
|
+
// cache hit. The mtime is the age evidence a mark/sweep collector
|
|
298
|
+
// fences its deletions on (`ArtifactSweep`), so a re-publication of
|
|
299
|
+
// old bytes must read as a recent reference or the grace period
|
|
300
|
+
// cannot protect the entry recorded moments later. Best-effort on
|
|
301
|
+
// hosts without `utimes` (the browser filesystem): a failed freshen
|
|
302
|
+
// over a blob that still exists keeps the dedupe skip and accepts
|
|
303
|
+
// git's freshen-versus-prune race; a failed freshen over a blob that
|
|
304
|
+
// VANISHED — a sweep won it — falls through to the atomic rewrite
|
|
305
|
+
// below, healing the address.
|
|
306
|
+
const now = yield* Clock.currentTimeMillis
|
|
307
|
+
const timestamp = new Date(now)
|
|
308
|
+
yield* checkRoot.pipe(Effect.mapError(hostFailure))
|
|
309
|
+
yield* checkParent.pipe(Effect.mapError(hostFailure))
|
|
310
|
+
yield* checkBlob.pipe(Effect.mapError(hostFailure))
|
|
311
|
+
const alive = yield* fs.utimes(blob.path, timestamp, timestamp).pipe(
|
|
312
|
+
Effect.as(true),
|
|
313
|
+
Effect.catch(() => fs.exists(blob.path).pipe(Effect.catch(() => Effect.succeed(true))))
|
|
314
|
+
)
|
|
315
|
+
if (!alive) {
|
|
316
|
+
verified = false
|
|
317
|
+
}
|
|
318
|
+
if (verified) {
|
|
319
|
+
yield* syncPath(blob.path, "r+")
|
|
320
|
+
yield* syncPath(blob.parent, "r")
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
if (!verified) {
|
|
324
|
+
// Atomic publication: a plain write to the canonical address could be
|
|
325
|
+
// observed — or survive a crash — as a partial file that every later
|
|
326
|
+
// read of this digest would trust. The payload lands at a temp path in
|
|
327
|
+
// the same fanout directory (so the rename never crosses a filesystem)
|
|
328
|
+
// and is renamed into place; an existing blob is rewritten only when
|
|
329
|
+
// its bytes no longer match its address.
|
|
330
|
+
yield* sweepOrphanedTemps
|
|
331
|
+
yield* Effect.scoped(Effect.gen(function*() {
|
|
332
|
+
for (let attempt = 0; attempt < 16; attempt++) {
|
|
333
|
+
yield* checkRoot
|
|
334
|
+
yield* checkParent
|
|
335
|
+
const tempPath = `${blob.path}.tmp-${yield* freshTempToken}-${attempt}`
|
|
336
|
+
const file = yield* fs.open(tempPath, { flag: "wx", mode: fileMode }).pipe(
|
|
337
|
+
Effect.map(Option.some),
|
|
338
|
+
Effect.catch((cause) =>
|
|
339
|
+
cause.reason._tag === "AlreadyExists"
|
|
340
|
+
? Effect.succeed(Option.none())
|
|
341
|
+
: Effect.fail(cause)
|
|
342
|
+
)
|
|
343
|
+
)
|
|
344
|
+
if (Option.isNone(file)) continue
|
|
345
|
+
// Install cleanup only after exclusive acquisition. Never unlink
|
|
346
|
+
// a colliding entry, which may belong to another writer.
|
|
347
|
+
const checkTemp = yield* ArtifactPath.guard(fs, tempPath, "File", yield* file.value.stat)
|
|
348
|
+
yield* Effect.gen(function*() {
|
|
349
|
+
if (snapshot.byteLength > 0) yield* file.value.writeAll(snapshot)
|
|
350
|
+
yield* durability === "best-effort" ? Effect.ignore(file.value.sync) : file.value.sync
|
|
351
|
+
yield* checkRoot
|
|
352
|
+
yield* checkParent
|
|
353
|
+
yield* checkTemp
|
|
354
|
+
yield* fs.rename(tempPath, blob.path)
|
|
355
|
+
yield* syncPath(blob.parent, "r")
|
|
356
|
+
}).pipe(Effect.onError(() =>
|
|
357
|
+
Effect.gen(function*() {
|
|
358
|
+
yield* checkRoot
|
|
359
|
+
yield* checkParent
|
|
360
|
+
yield* checkTemp
|
|
361
|
+
yield* fs.remove(tempPath)
|
|
362
|
+
}).pipe(Effect.ignore)
|
|
363
|
+
))
|
|
364
|
+
return
|
|
365
|
+
}
|
|
366
|
+
return yield* Effect.fail(
|
|
367
|
+
new ArtifactStoreError({
|
|
368
|
+
code: "unavailable",
|
|
369
|
+
message: "artifact scratch creation exhausted collision retries"
|
|
370
|
+
})
|
|
371
|
+
)
|
|
372
|
+
})).pipe(Effect.mapError(hostFailure))
|
|
373
|
+
}
|
|
374
|
+
yield* syncDirectoryAncestry
|
|
375
|
+
yield* Metric.update(ArtifactStoreMetrics.puts, 1)
|
|
376
|
+
return digest
|
|
377
|
+
}),
|
|
378
|
+
hostFailure,
|
|
379
|
+
coordination
|
|
380
|
+
)
|
|
381
|
+
}))
|
|
382
|
+
)
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Refuses a symlinked or wrong-typed objects directory, fanout, or blob
|
|
387
|
+
* before a read or probe follows it. The digest check alone guards `get`'s
|
|
388
|
+
* bytes, but `has` would otherwise report a planted foreign link as a stored
|
|
389
|
+
* blob and let a publication skip its upload. The returned check re-inspects
|
|
390
|
+
* all three and refuses an observed replacement.
|
|
391
|
+
*/
|
|
392
|
+
const guardBlob = (blob: { readonly parent: string; readonly path: string }) =>
|
|
393
|
+
Effect.gen(function*() {
|
|
394
|
+
const checkRoot = yield* ArtifactPath.guard(fs, directory)
|
|
395
|
+
const checkParent = yield* ArtifactPath.guard(fs, blob.parent)
|
|
396
|
+
const checkFile = yield* ArtifactPath.guard(fs, blob.path, "File")
|
|
397
|
+
return Effect.gen(function*() {
|
|
398
|
+
yield* checkRoot
|
|
399
|
+
yield* checkParent
|
|
400
|
+
yield* checkFile
|
|
401
|
+
}).pipe(Effect.mapError(hostFailure))
|
|
402
|
+
}).pipe(Effect.mapError(hostFailure))
|
|
403
|
+
|
|
404
|
+
const get: Service["get"] = Effect.fn("ArtifactStore.get")((digest: string) =>
|
|
405
|
+
Effect.gen(function*() {
|
|
406
|
+
const validated = yield* validateDigest(digest)
|
|
407
|
+
yield* Effect.annotateCurrentSpan({ digest: validated })
|
|
408
|
+
const blob = fanout(directory, validated)
|
|
409
|
+
const checkBlob = yield* guardBlob(blob)
|
|
410
|
+
const bytes = yield* fs.readFile(blob.path).pipe(
|
|
411
|
+
Effect.catch((cause): Effect.Effect<Uint8Array, ArtifactMissing | ArtifactStoreError> => {
|
|
412
|
+
const missing = new ArtifactMissing({ code: "artifact_missing", digest: validated })
|
|
413
|
+
if (isNotFound(cause)) return Effect.fail(missing)
|
|
414
|
+
return fs.exists(blob.path).pipe(
|
|
415
|
+
Effect.mapError((probeCause) => hostFailure({ read: cause, existenceProbe: probeCause })),
|
|
416
|
+
Effect.flatMap((present): Effect.Effect<Uint8Array, ArtifactMissing | ArtifactStoreError> =>
|
|
417
|
+
Effect.fail(present ? hostFailure(cause) : missing)
|
|
418
|
+
)
|
|
419
|
+
)
|
|
420
|
+
})
|
|
421
|
+
)
|
|
422
|
+
yield* checkBlob
|
|
423
|
+
const measured = yield* measureBytes(bytes)
|
|
424
|
+
if (measured !== validated) {
|
|
425
|
+
return yield* Effect.fail(
|
|
426
|
+
new ArtifactCorruption({
|
|
427
|
+
code: "artifact_corruption",
|
|
428
|
+
recordedDigest: validated,
|
|
429
|
+
measuredDigest: measured
|
|
430
|
+
})
|
|
431
|
+
)
|
|
432
|
+
}
|
|
433
|
+
yield* Metric.update(ArtifactStoreMetrics.gets, 1)
|
|
434
|
+
return bytes
|
|
435
|
+
})
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
const has: Service["has"] = Effect.fn("ArtifactStore.has")((digest: string) =>
|
|
439
|
+
Effect.gen(function*() {
|
|
440
|
+
const validated = yield* validateDigest(digest)
|
|
441
|
+
yield* Effect.annotateCurrentSpan({ digest: validated })
|
|
442
|
+
const blob = fanout(directory, validated)
|
|
443
|
+
yield* guardBlob(blob)
|
|
444
|
+
return yield* fs.exists(blob.path).pipe(Effect.mapError(hostFailure))
|
|
445
|
+
})
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
const findMissing: Service["findMissing"] = Effect.fn("ArtifactStore.findMissing")((digests: Iterable<string>) =>
|
|
449
|
+
Effect.gen(function*() {
|
|
450
|
+
const requested = [...new Set(digests)]
|
|
451
|
+
yield* Effect.annotateCurrentSpan({ count: requested.length })
|
|
452
|
+
yield* Effect.forEach(requested, validateDigest, { discard: true })
|
|
453
|
+
const present = yield* Effect.forEach(requested, has, { concurrency: 16 })
|
|
454
|
+
return requested.filter((_, index) => !present[index])
|
|
455
|
+
})
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
return { put, get, has, findMissing }
|
|
459
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The in-memory artifact store: a private map from digest to a defensive copy
|
|
3
|
+
* of the bytes, for tests and hosts with no durable filesystem.
|
|
4
|
+
*
|
|
5
|
+
* @since 1.0.0-rc.0
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import * as Effect from "effect/Effect"
|
|
9
|
+
import * as Metric from "effect/Metric"
|
|
10
|
+
import type { Service } from "./ArtifactStore.ts"
|
|
11
|
+
import { ArtifactMissing } from "./ArtifactStoreError.ts"
|
|
12
|
+
import * as ArtifactStoreMetrics from "./ArtifactStoreMetrics.ts"
|
|
13
|
+
import { measureBytes } from "./measureBytes.ts"
|
|
14
|
+
import { snapshotBytes } from "./snapshotBytes.ts"
|
|
15
|
+
import { validateDigest } from "./validateDigest.ts"
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Builds an in-memory artifact store, for tests and for a browser host with no
|
|
19
|
+
* durable filesystem yet.
|
|
20
|
+
*
|
|
21
|
+
* Reads are not digest-verified here, and that is not an oversight: the map is
|
|
22
|
+
* keyed by the digest this store measured when it accepted the bytes, and both
|
|
23
|
+
* boundaries copy — `put` stores a copy of the caller's array and `get` hands
|
|
24
|
+
* out a copy of the stored one — so no reference a caller can still mutate
|
|
25
|
+
* aliases the stored content, and there is no window in which the address and
|
|
26
|
+
* the content can disagree. The filesystem and remote implementations verify
|
|
27
|
+
* because their address spaces are genuinely shared.
|
|
28
|
+
*
|
|
29
|
+
* @category constructors
|
|
30
|
+
* @since 1.0.0-rc.0
|
|
31
|
+
* @slop
|
|
32
|
+
*/
|
|
33
|
+
export const makeMemory = (): Service => {
|
|
34
|
+
const blobs = new Map<string, Uint8Array>()
|
|
35
|
+
const has: Service["has"] = Effect.fn("ArtifactStore.has")((digest: string) =>
|
|
36
|
+
Effect.flatMap(validateDigest(digest), (validated) =>
|
|
37
|
+
Effect.annotateCurrentSpan({ digest: validated }).pipe(
|
|
38
|
+
Effect.as(blobs.has(validated))
|
|
39
|
+
))
|
|
40
|
+
)
|
|
41
|
+
return {
|
|
42
|
+
put: Effect.fn("ArtifactStore.put")((bytes: Uint8Array) =>
|
|
43
|
+
Effect.flatMap(snapshotBytes(bytes), (snapshot) =>
|
|
44
|
+
Effect.map(measureBytes(snapshot), (digest) => {
|
|
45
|
+
// A defensive copy, never the caller's reference: the caller is free
|
|
46
|
+
// to reuse its buffer after `put` returns, and an aliased array would
|
|
47
|
+
// let that mutation corrupt the stored content for its digest.
|
|
48
|
+
blobs.set(digest, snapshot)
|
|
49
|
+
return digest
|
|
50
|
+
})).pipe(
|
|
51
|
+
Effect.tap((digest) => Effect.annotateCurrentSpan({ digest })),
|
|
52
|
+
Effect.tap(() => Metric.update(ArtifactStoreMetrics.puts, 1))
|
|
53
|
+
)
|
|
54
|
+
),
|
|
55
|
+
get: Effect.fn("ArtifactStore.get")((digest: string) =>
|
|
56
|
+
Effect.gen(function*() {
|
|
57
|
+
const validated = yield* validateDigest(digest)
|
|
58
|
+
yield* Effect.annotateCurrentSpan({ digest: validated })
|
|
59
|
+
const bytes = blobs.get(validated)
|
|
60
|
+
if (bytes === undefined) {
|
|
61
|
+
return yield* Effect.fail(new ArtifactMissing({ code: "artifact_missing", digest: validated }))
|
|
62
|
+
}
|
|
63
|
+
yield* Metric.update(ArtifactStoreMetrics.gets, 1)
|
|
64
|
+
// A copy for the same reason `put` stores one: handing out the stored
|
|
65
|
+
// array would let one reader's mutation corrupt every later read of
|
|
66
|
+
// the digest.
|
|
67
|
+
return bytes.slice()
|
|
68
|
+
})
|
|
69
|
+
),
|
|
70
|
+
has,
|
|
71
|
+
findMissing: Effect.fn("ArtifactStore.findMissing")((digests: Iterable<string>) =>
|
|
72
|
+
Effect.gen(function*() {
|
|
73
|
+
const requested = [...new Set(digests)]
|
|
74
|
+
yield* Effect.annotateCurrentSpan({ count: requested.length })
|
|
75
|
+
const missing: Array<string> = []
|
|
76
|
+
for (const digest of requested) {
|
|
77
|
+
if (!(yield* has(digest))) missing.push(digest)
|
|
78
|
+
}
|
|
79
|
+
return missing
|
|
80
|
+
})
|
|
81
|
+
)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The no-op artifact store: the honest refusal a composition gets when it has
|
|
3
|
+
* no artifact store at all, with per-method overrides.
|
|
4
|
+
*
|
|
5
|
+
* @since 1.0.0-rc.0
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import * as Effect from "effect/Effect"
|
|
9
|
+
import type { Service } from "./ArtifactStore.ts"
|
|
10
|
+
import { ArtifactStoreError } from "./ArtifactStoreError.ts"
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Builds an artifact store whose every operation fails as unavailable, with
|
|
14
|
+
* per-method overrides.
|
|
15
|
+
*
|
|
16
|
+
* @category constructors
|
|
17
|
+
* @since 1.0.0-rc.0
|
|
18
|
+
* @slop
|
|
19
|
+
*/
|
|
20
|
+
export const makeNoop = (overrides: Partial<Service> = {}): Service => {
|
|
21
|
+
const unavailable = (method: string) =>
|
|
22
|
+
Effect.fail(new ArtifactStoreError({ code: "unavailable", message: `${method} is unavailable` }))
|
|
23
|
+
return {
|
|
24
|
+
put: Effect.fn("ArtifactStore.put")(() => unavailable("put")),
|
|
25
|
+
get: Effect.fn("ArtifactStore.get")(() => unavailable("get")),
|
|
26
|
+
has: Effect.fn("ArtifactStore.has")(() => unavailable("has")),
|
|
27
|
+
findMissing: Effect.fn("ArtifactStore.findMissing")(() => unavailable("findMissing")),
|
|
28
|
+
...overrides
|
|
29
|
+
}
|
|
30
|
+
}
|