@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/CHANGELOG.md +77 -0
  2. package/LICENSE +21 -0
  3. package/README.md +173 -2
  4. package/dist/cjs/ArtifactBackupLease.d.ts +34 -0
  5. package/dist/cjs/ArtifactBackupLease.d.ts.map +1 -0
  6. package/dist/cjs/ArtifactBackupLease.js +175 -0
  7. package/dist/cjs/ArtifactBackupLease.js.map +7 -0
  8. package/dist/cjs/ArtifactStore.d.ts +129 -0
  9. package/dist/cjs/ArtifactStore.d.ts.map +1 -0
  10. package/dist/cjs/ArtifactStore.js +73 -0
  11. package/dist/cjs/ArtifactStore.js.map +7 -0
  12. package/dist/cjs/ArtifactStoreError.d.ts +82 -0
  13. package/dist/cjs/ArtifactStoreError.d.ts.map +1 -0
  14. package/dist/cjs/ArtifactStoreError.js +72 -0
  15. package/dist/cjs/ArtifactStoreError.js.map +7 -0
  16. package/dist/cjs/ArtifactStoreMetrics.d.ts +70 -0
  17. package/dist/cjs/ArtifactStoreMetrics.d.ts.map +1 -0
  18. package/dist/cjs/ArtifactStoreMetrics.js +51 -0
  19. package/dist/cjs/ArtifactStoreMetrics.js.map +7 -0
  20. package/dist/cjs/ArtifactSweep.d.ts +185 -0
  21. package/dist/cjs/ArtifactSweep.d.ts.map +1 -0
  22. package/dist/cjs/ArtifactSweep.js +163 -0
  23. package/dist/cjs/ArtifactSweep.js.map +7 -0
  24. package/dist/cjs/CombinedArtifacts.d.ts +103 -0
  25. package/dist/cjs/CombinedArtifacts.d.ts.map +1 -0
  26. package/dist/cjs/CombinedArtifacts.js +189 -0
  27. package/dist/cjs/CombinedArtifacts.js.map +7 -0
  28. package/dist/cjs/FileLease.d.ts +113 -0
  29. package/dist/cjs/FileLease.d.ts.map +1 -0
  30. package/dist/cjs/FileLease.js +201 -0
  31. package/dist/cjs/FileLease.js.map +7 -0
  32. package/dist/cjs/FileSystemArtifactStore.d.ts +93 -0
  33. package/dist/cjs/FileSystemArtifactStore.d.ts.map +1 -0
  34. package/dist/cjs/FileSystemArtifactStore.js +276 -0
  35. package/dist/cjs/FileSystemArtifactStore.js.map +7 -0
  36. package/dist/cjs/MemoryArtifactStore.d.ts +25 -0
  37. package/dist/cjs/MemoryArtifactStore.d.ts.map +1 -0
  38. package/dist/cjs/MemoryArtifactStore.js +84 -0
  39. package/dist/cjs/MemoryArtifactStore.js.map +7 -0
  40. package/dist/cjs/NoopArtifactStore.d.ts +17 -0
  41. package/dist/cjs/NoopArtifactStore.d.ts.map +1 -0
  42. package/dist/cjs/NoopArtifactStore.js +46 -0
  43. package/dist/cjs/NoopArtifactStore.js.map +7 -0
  44. package/dist/cjs/RemoteArtifacts.d.ts +237 -0
  45. package/dist/cjs/RemoteArtifacts.d.ts.map +1 -0
  46. package/dist/cjs/RemoteArtifacts.js +405 -0
  47. package/dist/cjs/RemoteArtifacts.js.map +7 -0
  48. package/dist/cjs/index.d.ts +65 -0
  49. package/dist/cjs/index.d.ts.map +1 -0
  50. package/dist/cjs/index.js +47 -0
  51. package/dist/cjs/index.js.map +7 -0
  52. package/dist/cjs/internal/ArtifactLocks.d.ts +30 -0
  53. package/dist/cjs/internal/ArtifactLocks.d.ts.map +1 -0
  54. package/dist/cjs/internal/ArtifactLocks.js +93 -0
  55. package/dist/cjs/internal/ArtifactLocks.js.map +7 -0
  56. package/dist/cjs/internal/ArtifactPath.d.ts +19 -0
  57. package/dist/cjs/internal/ArtifactPath.d.ts.map +1 -0
  58. package/dist/cjs/internal/ArtifactPath.js +74 -0
  59. package/dist/cjs/internal/ArtifactPath.js.map +7 -0
  60. package/dist/cjs/measureBytes.d.ts +18 -0
  61. package/dist/cjs/measureBytes.d.ts.map +1 -0
  62. package/dist/cjs/measureBytes.js +43 -0
  63. package/dist/cjs/measureBytes.js.map +7 -0
  64. package/dist/cjs/package.json +1 -0
  65. package/dist/cjs/snapshotBytes.d.ts +19 -0
  66. package/dist/cjs/snapshotBytes.d.ts.map +1 -0
  67. package/dist/cjs/snapshotBytes.js +41 -0
  68. package/dist/cjs/snapshotBytes.js.map +7 -0
  69. package/dist/cjs/validateDigest.d.ts +22 -0
  70. package/dist/cjs/validateDigest.d.ts.map +1 -0
  71. package/dist/cjs/validateDigest.js +42 -0
  72. package/dist/cjs/validateDigest.js.map +7 -0
  73. package/dist/esm/ArtifactBackupLease.d.ts +34 -0
  74. package/dist/esm/ArtifactBackupLease.d.ts.map +1 -0
  75. package/dist/esm/ArtifactBackupLease.js +117 -0
  76. package/dist/esm/ArtifactBackupLease.js.map +1 -0
  77. package/dist/esm/ArtifactStore.d.ts +129 -0
  78. package/dist/esm/ArtifactStore.d.ts.map +1 -0
  79. package/dist/esm/ArtifactStore.js +83 -0
  80. package/dist/esm/ArtifactStore.js.map +1 -0
  81. package/dist/esm/ArtifactStoreError.d.ts +82 -0
  82. package/dist/esm/ArtifactStoreError.d.ts.map +1 -0
  83. package/dist/esm/ArtifactStoreError.js +77 -0
  84. package/dist/esm/ArtifactStoreError.js.map +1 -0
  85. package/dist/esm/ArtifactStoreMetrics.d.ts +70 -0
  86. package/dist/esm/ArtifactStoreMetrics.d.ts.map +1 -0
  87. package/dist/esm/ArtifactStoreMetrics.js +77 -0
  88. package/dist/esm/ArtifactStoreMetrics.js.map +1 -0
  89. package/dist/esm/ArtifactSweep.d.ts +185 -0
  90. package/dist/esm/ArtifactSweep.d.ts.map +1 -0
  91. package/dist/esm/ArtifactSweep.js +202 -0
  92. package/dist/esm/ArtifactSweep.js.map +1 -0
  93. package/dist/esm/CombinedArtifacts.d.ts +103 -0
  94. package/dist/esm/CombinedArtifacts.d.ts.map +1 -0
  95. package/dist/esm/CombinedArtifacts.js +214 -0
  96. package/dist/esm/CombinedArtifacts.js.map +1 -0
  97. package/dist/esm/FileLease.d.ts +113 -0
  98. package/dist/esm/FileLease.d.ts.map +1 -0
  99. package/dist/esm/FileLease.js +239 -0
  100. package/dist/esm/FileLease.js.map +1 -0
  101. package/dist/esm/FileSystemArtifactStore.d.ts +93 -0
  102. package/dist/esm/FileSystemArtifactStore.d.ts.map +1 -0
  103. package/dist/esm/FileSystemArtifactStore.js +336 -0
  104. package/dist/esm/FileSystemArtifactStore.js.map +1 -0
  105. package/dist/esm/MemoryArtifactStore.d.ts +25 -0
  106. package/dist/esm/MemoryArtifactStore.d.ts.map +1 -0
  107. package/dist/esm/MemoryArtifactStore.js +67 -0
  108. package/dist/esm/MemoryArtifactStore.js.map +1 -0
  109. package/dist/esm/NoopArtifactStore.d.ts +17 -0
  110. package/dist/esm/NoopArtifactStore.d.ts.map +1 -0
  111. package/dist/esm/NoopArtifactStore.js +27 -0
  112. package/dist/esm/NoopArtifactStore.js.map +1 -0
  113. package/dist/esm/RemoteArtifacts.d.ts +237 -0
  114. package/dist/esm/RemoteArtifacts.d.ts.map +1 -0
  115. package/dist/esm/RemoteArtifacts.js +554 -0
  116. package/dist/esm/RemoteArtifacts.js.map +1 -0
  117. package/dist/esm/index.d.ts +65 -0
  118. package/dist/esm/index.d.ts.map +1 -0
  119. package/dist/esm/index.js +65 -0
  120. package/dist/esm/index.js.map +1 -0
  121. package/dist/esm/internal/ArtifactLocks.d.ts +30 -0
  122. package/dist/esm/internal/ArtifactLocks.d.ts.map +1 -0
  123. package/dist/esm/internal/ArtifactLocks.js +86 -0
  124. package/dist/esm/internal/ArtifactLocks.js.map +1 -0
  125. package/dist/esm/internal/ArtifactPath.d.ts +19 -0
  126. package/dist/esm/internal/ArtifactPath.d.ts.map +1 -0
  127. package/dist/esm/internal/ArtifactPath.js +59 -0
  128. package/dist/esm/internal/ArtifactPath.js.map +1 -0
  129. package/dist/esm/measureBytes.d.ts +18 -0
  130. package/dist/esm/measureBytes.d.ts.map +1 -0
  131. package/dist/esm/measureBytes.js +22 -0
  132. package/dist/esm/measureBytes.js.map +1 -0
  133. package/dist/esm/snapshotBytes.d.ts +19 -0
  134. package/dist/esm/snapshotBytes.d.ts.map +1 -0
  135. package/dist/esm/snapshotBytes.js +30 -0
  136. package/dist/esm/snapshotBytes.js.map +1 -0
  137. package/dist/esm/validateDigest.d.ts +22 -0
  138. package/dist/esm/validateDigest.d.ts.map +1 -0
  139. package/dist/esm/validateDigest.js +26 -0
  140. package/dist/esm/validateDigest.js.map +1 -0
  141. package/docs/README.md +200 -0
  142. package/docs/api.md +474 -0
  143. package/docs/concepts/content-addressing.md +174 -0
  144. package/docs/concepts/coordination.md +141 -0
  145. package/docs/concepts/tiers.md +147 -0
  146. package/docs/guides/fence-a-backup.md +123 -0
  147. package/docs/guides/reclaim-disk-space.md +143 -0
  148. package/docs/guides/serve-the-artifact-protocol.md +140 -0
  149. package/docs/guides/share-artifacts-across-machines.md +180 -0
  150. package/docs/guides/test-against-an-artifact-store.md +158 -0
  151. package/docs/installation.md +98 -0
  152. package/docs/quickstart.md +182 -0
  153. package/docs/troubleshooting.md +232 -0
  154. package/package.json +179 -3
  155. package/src/ArtifactBackupLease.ts +213 -0
  156. package/src/ArtifactStore.ts +152 -0
  157. package/src/ArtifactStoreError.ts +96 -0
  158. package/src/ArtifactStoreMetrics.ts +83 -0
  159. package/src/ArtifactSweep.ts +336 -0
  160. package/src/CombinedArtifacts.ts +369 -0
  161. package/src/FileLease.ts +380 -0
  162. package/src/FileSystemArtifactStore.ts +459 -0
  163. package/src/MemoryArtifactStore.ts +83 -0
  164. package/src/NoopArtifactStore.ts +30 -0
  165. package/src/RemoteArtifacts.ts +809 -0
  166. package/src/index.ts +71 -0
  167. package/src/internal/ArtifactLocks.ts +110 -0
  168. package/src/internal/ArtifactPath.ts +72 -0
  169. package/src/measureBytes.ts +28 -0
  170. package/src/snapshotBytes.ts +34 -0
  171. package/src/validateDigest.ts +31 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,77 @@
1
+ # @smthrs/artifacts
2
+
3
+ ## [Unreleased]
4
+
5
+ ### Fixed
6
+
7
+ - The filesystem store syncs the real ancestors of a Windows drive directory.
8
+ Breaking: `makeFileSystem(fs, path, options)` takes the host `Path`, and
9
+ `layerFileSystem` requires `Path.Path` (`NodePath.layer` on Node) (#2302).
10
+ - The filesystem store's `get`, `has`, and `findMissing` refuse a symlinked
11
+ objects root, fanout, or blob as `unavailable` instead of following it.
12
+ - Artifact locks refuse a symlinked `.locks` directory before and after
13
+ creating it and recheck its identity before every lock-file mutation;
14
+ `FileLease.hold` takes a `guard` option for that check.
15
+ - `RemoteArtifacts` never follows redirects (fetch runs with
16
+ `redirect: "manual"`), and allows plain `http:` only for the literal
17
+ loopback hosts `localhost`, `127.0.0.1`, and `[::1]`. Breaking: an
18
+ `http://*.localhost` endpoint is refused.
19
+
20
+ ## [1.0.0-rc.0] - 2026-08-31
21
+
22
+ ### Added
23
+
24
+ - New package. The content-addressed artifact store extracted out of
25
+ `@smthrs/engine-store`'s `StepBoundary`: `ArtifactStore` (contract,
26
+ filesystem, memory, and no-op implementations), `RemoteArtifacts` (the
27
+ dumb-HTTP CAS client), and `CombinedArtifacts` (local-first, remote-second
28
+ read-through with local write-back).
29
+ - Two improvements taken while moving, both from Bazel's `DiskCacheClient`: a
30
+ two-hex-prefix fanout directory layout, and an fsync of the temp file before
31
+ the rename that publishes it. There is no compatibility shim for the old flat
32
+ layout.
33
+
34
+ ### Changed
35
+
36
+ - Snapshotted all caller-owned byte arrays at Effect start and copied every
37
+ read result.
38
+ - Added strict SHA-256 address validation, typed configuration and crypto
39
+ failures, finite remote deadlines and body limits, and deterministic
40
+ `findMissing` deduplication.
41
+ - Added workspace-wide writer/sweeper coordination with crash recovery, and
42
+ reclaimed crash-orphaned lock files in the same conservative scratch sweep
43
+ that reclaims `.tmp-*` payloads.
44
+ - Made download policy immutable and consistent across constructors and layers.
45
+ - Made `CombinedArtifacts.get`'s local write-back opportunistic: a local tier
46
+ that cannot store bytes the shared tier already served costs the next read a
47
+ round trip instead of failing this one. `minimal` now repairs a corrupt local
48
+ address, because an address the local tier already claims must be one it can
49
+ serve.
50
+ - Applied caller-configured headers before the protocol's own, so a configured
51
+ `content-range`, `content-type`, or `content-length` can no longer overwrite
52
+ a chunked upload's computed headers.
53
+ - Reported a refused byte snapshot as `unavailable`, the host-refusal code,
54
+ rather than as a crypto `digest_failed`.
55
+ - Ignored a resume prefix past the safe integer range instead of trusting a
56
+ rounded offset.
57
+ - Gave `ArtifactSweep` its own `SweepOptions` rather than the store's options,
58
+ which carried a `durability` field it ignored, and exported
59
+ `ArtifactStore.defaultDirectory` so the store and its sweep name one
60
+ directory.
61
+ - Classified a backup-lease marker another process already reaped as a clean
62
+ release rather than logging a release failure.
63
+ - Stopped a lock heartbeat from freshening a lock file it no longer owns. A
64
+ holder reaped as stale and replaced kept touching that path every 10 seconds,
65
+ which held the replacement fresh past every stale bound, so a replacement that
66
+ was then hard-killed blocked the digest for as long as the reaped holder ran.
67
+ A lock that is gone or now names another owner ends the heartbeat; a read the
68
+ host refused for any other reason skips one beat and retries, so a transient
69
+ refusal cannot retire a lock still held.
70
+ - Made per-digest lock bookkeeping per execution rather than per construction.
71
+ A lock effect built once and run repeatedly, which is how the backup lease
72
+ heartbeats, retired its entry underneath a live holder, and the next caller
73
+ minted a second in-process semaphore for the same digest.
74
+ - Documented that reclaiming a stale lock measures and then removes in separate
75
+ steps, so simultaneous reclaims of one stale lock can both proceed, and that
76
+ `ifUnmodifiedSinceMs` and the backup lease are the fences that do not depend
77
+ on it.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 William Cory and the Smithers Flows contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,174 @@
1
- # Temporary Holding Version
1
+ # `@smthrs/artifacts`
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Release candidate scope, host requirements and compatibility review are defined in the [library support policy](https://github.com/smithersai/smithers/blob/main/RELEASE_SUPPORT.md).
4
+
5
+ This package declares `effect` as an exact
6
+ `4.0.0-rc.115` peer dependency. Keep the application on that version so
7
+ all Smithers packages share one Effect runtime.
8
+
9
+ **Documentation:** https://artifacts.smithers.sh
10
+
11
+ The content-addressed artifact store: bytes addressed by their own SHA-256
12
+ digest. You hand it a `Uint8Array`, it hands back a 64 character address, and
13
+ any store holding those bytes returns them for that address after checking that
14
+ they still hash to it.
15
+
16
+ Reach for it when you cache or ship large byte payloads, a compiled bundle, a
17
+ build log, a model transcript, a tarball, and you care that a read either
18
+ returns the exact bytes that were published or fails. It opens no file and no
19
+ socket by itself: the filesystem and the network arrive as Effect's
20
+ `FileSystem` and `HttpClient` services, which is what lets the same store code
21
+ run in Node.js, in Bun, in a browser tab, and inside a sandbox.
22
+
23
+ ## Install
24
+
25
+ Not on npm yet; see [Installation](https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/flow/docs/installation.md#use-the-libraries).
26
+
27
+ `@effect/platform-node` supplies the Node.js implementations of the services
28
+ the store asks for. The filesystem store needs exclusive writable handles,
29
+ symlink inspection, and (by default) file and directory syncing. Compose it
30
+ with the trusted host filesystem before applying the capability kernel to
31
+ workflow services; `NodeRuntime.layerHost` and `BunRuntime.layerHost` do this.
32
+ The guarded native filesystem refuses open handles and cannot back this store,
33
+ even with `durability: "best-effort"`. Use `layerMemory` on hosts without those
34
+ filesystem capabilities.
35
+
36
+ ## Publish bytes and read them back
37
+
38
+ ```ts
39
+ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"
40
+ import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"
41
+ import * as NodePath from "@effect/platform-node/NodePath"
42
+ import * as ArtifactStore from "@smthrs/artifacts/ArtifactStore"
43
+ import * as Effect from "effect/Effect"
44
+ import * as Layer from "effect/Layer"
45
+
46
+ const layer = ArtifactStore.layerFileSystem({ directory: ".flows/objects" }).pipe(
47
+ Layer.provideMerge(Layer.mergeAll(NodeFileSystem.layer, NodePath.layer, NodeCrypto.layer))
48
+ )
49
+
50
+ const program = Effect.gen(function*() {
51
+ const store = yield* ArtifactStore.ArtifactStore
52
+ const bytes = new TextEncoder().encode("the bytes a build step produced")
53
+
54
+ const address = yield* store.put(bytes)
55
+ const again = yield* store.put(bytes)
56
+
57
+ console.log(address)
58
+ console.log(again === address)
59
+ console.log(new TextDecoder().decode(yield* store.get(address)))
60
+ })
61
+
62
+ await Effect.runPromise(program.pipe(Effect.provide(layer)))
63
+ ```
64
+
65
+ ```text
66
+ 6bb29e0869012afcfc246886c647422236e0b7d3419d3dc4ded8da758a4dfeb3
67
+ true
68
+ the bytes a build step produced
69
+ ```
70
+
71
+ One blob landed on disk, under the first two characters of its own address. The
72
+ second `put` measured the bytes, took the digest's lock, verified the blob
73
+ already there, freshened its modification time, and synced it, then returned the
74
+ same address without rewriting the payload.
75
+
76
+ ## The stores you can compose
77
+
78
+ Every implementation is the same four operations, `put`, `get`, `has`, and
79
+ `findMissing`, so a composition swaps one for another without touching a
80
+ caller:
81
+
82
+ | Constructor | What backs it | Use it for |
83
+ | ------------------------------ | ------------------------------------------- | ------------------------------------------ |
84
+ | `ArtifactStore.makeFileSystem` | A directory, reached through `FileSystem` | The durable local tier |
85
+ | `ArtifactStore.makeMemory` | A private `Map` | Tests, and hosts with no durable disk |
86
+ | `ArtifactStore.makeNoop` | Nothing, with per-method overrides | Declaring a tier honestly unavailable |
87
+ | `RemoteArtifacts.make` | An HTTP cache, reached through `HttpClient` | The tier several machines share |
88
+ | `CombinedArtifacts.make` | A local store in front of a shared one | Read local first, fall through, write back |
89
+
90
+ `ArtifactSweep` enumerates an objects directory and deletes one blob behind a
91
+ modification time fence, and `ArtifactBackupLease` keeps a sweep from deleting
92
+ a blob a running backup is still copying.
93
+
94
+ Every export, option, and error code is in the
95
+ [API reference](https://artifacts.smithers.sh/reference/api/).
96
+
97
+ ## What it guarantees
98
+
99
+ - **Every read is digest verified.** A truncated blob left by a crashing
100
+ writer, a corrupted disk, or a mis-serving shared tier is refused with
101
+ `ArtifactCorruption`, never handed back as the recorded artifact. The memory
102
+ store is the one deliberate exception: its address space is a private `Map`
103
+ keyed by the digest it measured, so the address and the content cannot
104
+ disagree.
105
+ - **Publication is atomic.** Bytes land at a temp path in the destination
106
+ directory, are fsynced, and are renamed into place. Temp names fold a random
107
+ per-instance token, so two processes publishing the same digest into one
108
+ workspace never share a scratch path.
109
+ - **An existing blob is verified on every `put`.** The objects directory is
110
+ shared, so a remembered proof could outlive the bytes it proved. A mismatch
111
+ or failing read falls through to the atomic rewrite and heals the address.
112
+ - **The endpoint and its credentials are a capability, never an input.** They
113
+ arrive as layer construction options, so they are never hashed into a cache
114
+ key and never recorded. `RemoteArtifacts` refuses a non-HTTPS endpoint, and
115
+ any endpoint carrying credentials, a query, or a fragment, at construction.
116
+ The sanitized message names only the violated rule, so the failure text never
117
+ echoes the endpoint.
118
+
119
+ Every `put` snapshots its bytes when the Effect begins. Stores never retain a
120
+ caller-owned buffer, and every successful `get` returns a new byte array.
121
+
122
+ ## Defaults
123
+
124
+ | Boundary | Default | Guarantee |
125
+ | ----------------------------------------- | ----------------------- | ---------------------------------------------------------------------------- |
126
+ | Filesystem directory | `.flows/objects` | Two-hex fanout, atomic publication, digest verification, required fsync |
127
+ | Cross-process coordination | `required` | Writers and sweepers share per-digest lock files; stale owners are recovered |
128
+ | Remote request, upload, download deadline | 60 seconds each | No remote exchange can wait forever |
129
+ | Maximum downloaded blob | 256 MiB | The body is rejected before or while buffering past the limit |
130
+ | `findMissing` batch and response | 1,000 digests / 256 KiB | Inputs are validated and deduplicated in first-seen order |
131
+
132
+ `invalid_configuration` and `invalid_digest` are permanent caller failures.
133
+ `ArtifactMissing` is an ordinary miss, and `ArtifactCorruption` is an integrity
134
+ failure. `digest_failed`, `unavailable`, and `transport_failed` describe host
135
+ or transport failures whose retryability depends on the host and operation.
136
+ [Troubleshooting](https://artifacts.smithers.sh/troubleshooting/) lists each
137
+ one with what to change.
138
+
139
+ ## Prior art
140
+
141
+ The contract's ergonomics follow Effect's own `KeyValueStore`: one small set of
142
+ total operations over one address space, so memory, filesystem, and network
143
+ implementations are the same shape. Everything else follows Bazel's
144
+ remote-cache classes: `findMissing` is
145
+ [`MissingDigestsFinder`](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/remote/common/MissingDigestsFinder.java),
146
+ one batched probe whose result is a subset of its input; the two-hex fanout and
147
+ the fsync before the rename are
148
+ [`DiskCacheClient`](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/remote/disk/DiskCacheClient.java);
149
+ and the wire protocol, CAS blobs under `/cas/base16-key`, is
150
+ [`HttpCacheClient`](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java).
151
+ Where this package deviates, and why, is in
152
+ [content addressing](https://artifacts.smithers.sh/concepts/content-addressing/).
153
+
154
+ ## Where it fits
155
+
156
+ This package is one piece of the Smithers durable flow engine, whose whole
157
+ surface is re-exported by [`@smthrs/flows`](https://flows.smithers.sh). Inside
158
+ that engine the artifact store is the byte half of the result cache:
159
+ [`@smthrs/step-cache`](https://step-cache.smithers.sh) records what a step
160
+ returned, and any part of that result too large to sit inline is spilled here
161
+ and referenced by digest.
162
+
163
+ Install `@smthrs/artifacts` on its own when a content-addressed byte store is
164
+ all you want. Nothing in it knows what a flow, a step, or a run is.
165
+
166
+ Reclaiming published artifacts is an explicit operation, never a side effect of
167
+ a store call. The `.tmp-*` sweep in `layerFileSystem` reclaims crash orphans
168
+ only. `ArtifactSweep` is the deletion surface, and the mark phase that decides
169
+ what is live belongs to
170
+ [`@smthrs/engine-store`](https://engine-store.smithers.sh).
171
+
172
+ ## License
173
+
174
+ MIT
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Cross-process exclusion between a filesystem artifact backup and garbage
3
+ * collection.
4
+ *
5
+ * A backup holds one heartbeat-backed marker for the duration of its database
6
+ * snapshot and blob copy. Sweep deletion checks that marker while holding the
7
+ * same short-lived global gate. Publication remains unconstrained: new blobs
8
+ * may appear during a backup, but a blob already referenced by the frozen
9
+ * database cannot disappear underneath it.
10
+ *
11
+ * @since 1.0.0-rc.0
12
+ */
13
+ import * as Effect from "effect/Effect";
14
+ import type * as FileSystem from "effect/FileSystem";
15
+ import * as Option from "effect/Option";
16
+ /**
17
+ * Runs `effect` while sweep deletion is fenced across every process using
18
+ * this objects directory. A crashed holder is recovered after its heartbeat
19
+ * becomes stale.
20
+ *
21
+ * @category combinators
22
+ * @since 1.0.0-rc.0
23
+ */
24
+ export declare const withLease: <A, E, R, E2>(fs: FileSystem.FileSystem, directory: string, effect: Effect.Effect<A, E, R>, failure: (cause: unknown) => E2) => Effect.Effect<A, E | E2, R>;
25
+ /**
26
+ * Runs one sweep deletion only when no live backup lease exists. The marker
27
+ * check and `effect` share the global gate, closing the check/delete race.
28
+ * `None` means a backup deliberately fenced the operation.
29
+ *
30
+ * @category combinators
31
+ * @since 1.0.0-rc.0
32
+ */
33
+ export declare const unlessActive: <A, E, R, E2>(fs: FileSystem.FileSystem, directory: string, effect: Effect.Effect<A, E, R>, failure: (cause: unknown) => E2) => Effect.Effect<Option.Option<A>, E | E2, R>;
34
+ //# sourceMappingURL=ArtifactBackupLease.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ArtifactBackupLease.d.ts","sourceRoot":"","sources":["../../src/ArtifactBackupLease.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,UAAU,MAAM,mBAAmB,CAAA;AACpD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAqDvC;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EACnC,IAAI,UAAU,CAAC,UAAU,EACzB,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,SAAS,CAAC,KAAK,EAAE,OAAO,KAAK,EAAE,KAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CA0GzB,CAAA;AAEJ;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EACtC,IAAI,UAAU,CAAC,UAAU,EACzB,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,SAAS,CAAC,KAAK,EAAE,OAAO,KAAK,EAAE,KAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CASzC,CAAA"}
@@ -0,0 +1,175 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+ var ArtifactBackupLease_exports = {};
30
+ __export(ArtifactBackupLease_exports, {
31
+ unlessActive: () => unlessActive,
32
+ withLease: () => withLease
33
+ });
34
+ module.exports = __toCommonJS(ArtifactBackupLease_exports);
35
+ var Clock = __toESM(require("effect/Clock"), 1);
36
+ var Effect = __toESM(require("effect/Effect"), 1);
37
+ var Option = __toESM(require("effect/Option"), 1);
38
+ var PlatformError = __toESM(require("effect/PlatformError"), 1);
39
+ var Random = __toESM(require("effect/Random"), 1);
40
+ var ArtifactLocks = __toESM(require("./internal/ArtifactLocks.js"), 1);
41
+ const markerName = ".backup-lease";
42
+ const gateName = ".backup-lease-gate";
43
+ const heartbeatEvery = "10 seconds";
44
+ const staleAfterMs = 6e4;
45
+ const acquireWithin = "2 minutes";
46
+ const retryEvery = "25 millis";
47
+ const isReason = (cause, tag) => cause instanceof PlatformError.PlatformError && cause.reason._tag === tag;
48
+ const token = Effect.gen(function* () {
49
+ const first = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true });
50
+ const second = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true });
51
+ return `${first.toString(36)}-${second.toString(36)}-${yield* Clock.currentTimeMillis}`;
52
+ });
53
+ const gate = (fs, directory, effect, failure) => ArtifactLocks.withDigest(fs, directory, gateName, effect, failure);
54
+ const activeMarker = (fs, directory, failure) => Effect.gen(function* () {
55
+ const path = `${directory}/${markerName}`;
56
+ const info = yield* fs.stat(path).pipe(
57
+ Effect.map(Option.some),
58
+ Effect.catch(
59
+ (cause) => isReason(cause, "NotFound") ? Effect.succeed(Option.none()) : Effect.fail(failure(cause))
60
+ )
61
+ );
62
+ if (Option.isNone(info)) return false;
63
+ const modified = Option.getOrUndefined(info.value.mtime);
64
+ const now = yield* Clock.currentTimeMillis;
65
+ if (modified === void 0 || now - modified.getTime() <= staleAfterMs) return true;
66
+ yield* fs.remove(path).pipe(
67
+ Effect.catch(
68
+ (cause) => isReason(cause, "NotFound") ? Effect.void : Effect.fail(failure(cause))
69
+ )
70
+ );
71
+ return false;
72
+ });
73
+ const withLease = (fs, directory, effect, failure) => Effect.gen(function* () {
74
+ const owner = yield* token;
75
+ const marker = `${directory}/${markerName}`;
76
+ yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure));
77
+ let ownsMarker = false;
78
+ const acquire = Effect.gen(function* () {
79
+ while (true) {
80
+ const acquired = yield* gate(
81
+ fs,
82
+ directory,
83
+ Effect.gen(function* () {
84
+ if (yield* activeMarker(fs, directory, failure)) return false;
85
+ return yield* Effect.uninterruptible(
86
+ fs.writeFileString(marker, owner, { flag: "wx", mode: 384 }).pipe(
87
+ Effect.andThen(Effect.sync(() => {
88
+ ownsMarker = true;
89
+ return true;
90
+ })),
91
+ Effect.catch(
92
+ (cause) => isReason(cause, "AlreadyExists") ? Effect.succeed(false) : Effect.fail(failure(cause))
93
+ )
94
+ )
95
+ );
96
+ }),
97
+ failure
98
+ );
99
+ if (acquired) return;
100
+ yield* Effect.sleep(retryEvery);
101
+ }
102
+ }).pipe(
103
+ Effect.timeout(acquireWithin),
104
+ Effect.catchTag("TimeoutError", (cause) => Effect.fail(failure(cause)))
105
+ );
106
+ return yield* Effect.acquireUseRelease(
107
+ Effect.void,
108
+ () => acquire.pipe(Effect.andThen(
109
+ Effect.scoped(Effect.gen(function* () {
110
+ yield* Effect.forkScoped(
111
+ Effect.forever(
112
+ Effect.sleep(heartbeatEvery).pipe(
113
+ Effect.andThen(gate(
114
+ fs,
115
+ directory,
116
+ fs.readFileString(marker).pipe(
117
+ Effect.map(Option.some),
118
+ Effect.catch(
119
+ (cause) => isReason(cause, "NotFound") ? Effect.succeed(Option.none()) : Effect.fail(cause)
120
+ ),
121
+ Effect.flatMap((found) => {
122
+ if (Option.getOrUndefined(found) !== owner) {
123
+ return Effect.logWarning(
124
+ "Artifact backup lease was reclaimed while its backup was still running"
125
+ );
126
+ }
127
+ return Effect.flatMap(Clock.currentTimeMillis, (now) => {
128
+ const timestamp = new Date(now);
129
+ return fs.utimes(marker, timestamp, timestamp);
130
+ });
131
+ }),
132
+ Effect.mapError(failure)
133
+ ),
134
+ failure
135
+ )),
136
+ // A failed heartbeat is retried on the next tick, but a
137
+ // minute of them lets the marker go stale, so each one is
138
+ // reported rather than dropped.
139
+ Effect.catch((cause) => Effect.logWarning("Artifact backup lease heartbeat failed", cause))
140
+ )
141
+ )
142
+ );
143
+ return yield* effect;
144
+ }))
145
+ )),
146
+ () => ownsMarker ? gate(
147
+ fs,
148
+ directory,
149
+ fs.readFileString(marker).pipe(
150
+ Effect.flatMap((found) => found === owner ? fs.remove(marker) : Effect.void),
151
+ // A backup whose heartbeat lapsed has its marker reaped by whoever
152
+ // noticed, so `NotFound` is the ordinary end of a slow lease, not a
153
+ // fault: there is nothing left to release and nothing left to leak.
154
+ // The sibling release in `internal/ArtifactLocks.ts` classifies the
155
+ // same cause the same way.
156
+ Effect.catch(
157
+ (cause) => isReason(cause, "NotFound") ? Effect.void : Effect.fail(cause)
158
+ )
159
+ ),
160
+ failure
161
+ ).pipe(
162
+ Effect.catch((cause) => Effect.logWarning("Artifact backup lease release failed", cause))
163
+ ) : Effect.void
164
+ );
165
+ });
166
+ const unlessActive = (fs, directory, effect, failure) => gate(
167
+ fs,
168
+ directory,
169
+ Effect.gen(function* () {
170
+ if (yield* activeMarker(fs, directory, failure)) return Option.none();
171
+ return Option.some(yield* effect);
172
+ }),
173
+ failure
174
+ );
175
+ //# sourceMappingURL=ArtifactBackupLease.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/ArtifactBackupLease.ts"],
4
+ "sourcesContent": ["/**\n * Cross-process exclusion between a filesystem artifact backup and garbage\n * collection.\n *\n * A backup holds one heartbeat-backed marker for the duration of its database\n * snapshot and blob copy. Sweep deletion checks that marker while holding the\n * same short-lived global gate. Publication remains unconstrained: new blobs\n * may appear during a backup, but a blob already referenced by the frozen\n * database cannot disappear underneath it.\n *\n * @since 1.0.0-rc.0\n */\n\nimport * as Clock from \"effect/Clock\"\nimport * as Effect from \"effect/Effect\"\nimport type * as FileSystem from \"effect/FileSystem\"\nimport * as Option from \"effect/Option\"\nimport * as PlatformError from \"effect/PlatformError\"\nimport * as Random from \"effect/Random\"\nimport * as ArtifactLocks from \"./internal/ArtifactLocks.ts\"\n\nconst markerName = \".backup-lease\"\nconst gateName = \".backup-lease-gate\"\nconst heartbeatEvery = \"10 seconds\"\nconst staleAfterMs = 60_000\nconst acquireWithin = \"2 minutes\"\nconst retryEvery = \"25 millis\"\n\nconst isReason = (cause: unknown, tag: PlatformError.SystemErrorTag): boolean =>\n cause instanceof PlatformError.PlatformError && cause.reason._tag === tag\n\nconst token = Effect.gen(function*() {\n const first = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })\n const second = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })\n return `${first.toString(36)}-${second.toString(36)}-${yield* Clock.currentTimeMillis}`\n})\n\nconst gate = <A, E, R, E2>(\n fs: FileSystem.FileSystem,\n directory: string,\n effect: Effect.Effect<A, E, R>,\n failure: (cause: unknown) => E2\n): Effect.Effect<A, E | E2, R> => ArtifactLocks.withDigest(fs, directory, gateName, effect, failure)\n\nconst activeMarker = <E>(\n fs: FileSystem.FileSystem,\n directory: string,\n failure: (cause: unknown) => E\n): Effect.Effect<boolean, E> =>\n Effect.gen(function*() {\n const path = `${directory}/${markerName}`\n const info = yield* fs.stat(path).pipe(\n Effect.map(Option.some),\n Effect.catch((cause): Effect.Effect<Option.Option<FileSystem.File.Info>, E> =>\n isReason(cause, \"NotFound\") ? Effect.succeed(Option.none()) : Effect.fail(failure(cause))\n )\n )\n if (Option.isNone(info)) return false\n const modified = Option.getOrUndefined(info.value.mtime)\n const now = yield* Clock.currentTimeMillis\n if (modified === undefined || now - modified.getTime() <= staleAfterMs) return true\n yield* fs.remove(path).pipe(\n Effect.catch((cause): Effect.Effect<void, E> =>\n isReason(cause, \"NotFound\") ? Effect.void : Effect.fail(failure(cause))\n )\n )\n return false\n })\n\n/**\n * Runs `effect` while sweep deletion is fenced across every process using\n * this objects directory. A crashed holder is recovered after its heartbeat\n * becomes stale.\n *\n * @category combinators\n * @since 1.0.0-rc.0\n */\nexport const withLease = <A, E, R, E2>(\n fs: FileSystem.FileSystem,\n directory: string,\n effect: Effect.Effect<A, E, R>,\n failure: (cause: unknown) => E2\n): Effect.Effect<A, E | E2, R> =>\n Effect.gen(function*() {\n const owner = yield* token\n const marker = `${directory}/${markerName}`\n yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure))\n\n let ownsMarker = false\n const acquire = Effect.gen(function*() {\n while (true) {\n const acquired = yield* gate(\n fs,\n directory,\n Effect.gen(function*() {\n if (yield* activeMarker(fs, directory, failure)) return false\n return yield* Effect.uninterruptible(\n fs.writeFileString(marker, owner, { flag: \"wx\", mode: 0o600 }).pipe(\n Effect.andThen(Effect.sync(() => {\n ownsMarker = true\n return true\n })),\n Effect.catch((cause): Effect.Effect<boolean, E2> =>\n isReason(cause, \"AlreadyExists\") ? Effect.succeed(false) : Effect.fail(failure(cause))\n )\n )\n )\n }),\n failure\n )\n if (acquired) return\n yield* Effect.sleep(retryEvery)\n }\n }).pipe(\n Effect.timeout(acquireWithin),\n Effect.catchTag(\"TimeoutError\", (cause) => Effect.fail(failure(cause)))\n )\n\n // Install marker cleanup before acquisition, including gate release. The\n // marker write and ownership bookkeeping complete in one interruption mask.\n return yield* Effect.acquireUseRelease(\n Effect.void,\n () =>\n acquire.pipe(Effect.andThen(\n Effect.scoped(Effect.gen(function*() {\n yield* Effect.forkScoped(\n Effect.forever(\n Effect.sleep(heartbeatEvery).pipe(\n Effect.andThen(gate(\n fs,\n directory,\n fs.readFileString(marker).pipe(\n Effect.map(Option.some),\n Effect.catch((cause) =>\n isReason(cause, \"NotFound\") ? Effect.succeed(Option.none<string>()) : Effect.fail(cause)\n ),\n Effect.flatMap((found) => {\n // A missing or foreign marker means a sweep judged this\n // lease stale and reaped it: deletion is no longer\n // fenced while the backup still copies. The sibling\n // `internal/ArtifactLocks.ts` heartbeat reports the same\n // loss the same way.\n if (Option.getOrUndefined(found) !== owner) {\n return Effect.logWarning(\n \"Artifact backup lease was reclaimed while its backup was still running\"\n )\n }\n return Effect.flatMap(Clock.currentTimeMillis, (now) => {\n const timestamp = new Date(now)\n return fs.utimes(marker, timestamp, timestamp)\n })\n }),\n Effect.mapError(failure)\n ),\n failure\n )),\n // A failed heartbeat is retried on the next tick, but a\n // minute of them lets the marker go stale, so each one is\n // reported rather than dropped.\n Effect.catch((cause) => Effect.logWarning(\"Artifact backup lease heartbeat failed\", cause))\n )\n )\n )\n return yield* effect\n }))\n )),\n () =>\n ownsMarker\n ? gate(\n fs,\n directory,\n fs.readFileString(marker).pipe(\n Effect.flatMap((found) => found === owner ? fs.remove(marker) : Effect.void),\n // A backup whose heartbeat lapsed has its marker reaped by whoever\n // noticed, so `NotFound` is the ordinary end of a slow lease, not a\n // fault: there is nothing left to release and nothing left to leak.\n // The sibling release in `internal/ArtifactLocks.ts` classifies the\n // same cause the same way.\n Effect.catch((cause): Effect.Effect<void, PlatformError.PlatformError> =>\n isReason(cause, \"NotFound\") ? Effect.void : Effect.fail(cause)\n )\n ),\n failure\n ).pipe(\n Effect.catch((cause) => Effect.logWarning(\"Artifact backup lease release failed\", cause))\n )\n : Effect.void\n )\n })\n\n/**\n * Runs one sweep deletion only when no live backup lease exists. The marker\n * check and `effect` share the global gate, closing the check/delete race.\n * `None` means a backup deliberately fenced the operation.\n *\n * @category combinators\n * @since 1.0.0-rc.0\n */\nexport const unlessActive = <A, E, R, E2>(\n fs: FileSystem.FileSystem,\n directory: string,\n effect: Effect.Effect<A, E, R>,\n failure: (cause: unknown) => E2\n): Effect.Effect<Option.Option<A>, E | E2, R> =>\n gate(\n fs,\n directory,\n Effect.gen(function*() {\n if (yield* activeMarker(fs, directory, failure)) return Option.none<A>()\n return Option.some(yield* effect)\n }),\n failure\n )\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAaA,YAAuB;AACvB,aAAwB;AAExB,aAAwB;AACxB,oBAA+B;AAC/B,aAAwB;AACxB,oBAA+B;AAE/B,MAAM,aAAa;AACnB,MAAM,WAAW;AACjB,MAAM,iBAAiB;AACvB,MAAM,eAAe;AACrB,MAAM,gBAAgB;AACtB,MAAM,aAAa;AAEnB,MAAM,WAAW,CAAC,OAAgB,QAChC,iBAAiB,cAAc,iBAAiB,MAAM,OAAO,SAAS;AAExE,MAAM,QAAQ,OAAO,IAAI,aAAY;AACnC,QAAM,QAAQ,OAAO,OAAO,eAAe,GAAG,OAAO,kBAAkB,EAAE,UAAU,KAAK,CAAC;AACzF,QAAM,SAAS,OAAO,OAAO,eAAe,GAAG,OAAO,kBAAkB,EAAE,UAAU,KAAK,CAAC;AAC1F,SAAO,GAAG,MAAM,SAAS,EAAE,CAAC,IAAI,OAAO,SAAS,EAAE,CAAC,IAAI,OAAO,MAAM,iBAAiB;AACvF,CAAC;AAED,MAAM,OAAO,CACX,IACA,WACA,QACA,YACgC,cAAc,WAAW,IAAI,WAAW,UAAU,QAAQ,OAAO;AAEnG,MAAM,eAAe,CACnB,IACA,WACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,OAAO,GAAG,SAAS,IAAI,UAAU;AACvC,QAAM,OAAO,OAAO,GAAG,KAAK,IAAI,EAAE;AAAA,IAChC,OAAO,IAAI,OAAO,IAAI;AAAA,IACtB,OAAO;AAAA,MAAM,CAAC,UACZ,SAAS,OAAO,UAAU,IAAI,OAAO,QAAQ,OAAO,KAAK,CAAC,IAAI,OAAO,KAAK,QAAQ,KAAK,CAAC;AAAA,IAC1F;AAAA,EACF;AACA,MAAI,OAAO,OAAO,IAAI,EAAG,QAAO;AAChC,QAAM,WAAW,OAAO,eAAe,KAAK,MAAM,KAAK;AACvD,QAAM,MAAM,OAAO,MAAM;AACzB,MAAI,aAAa,UAAa,MAAM,SAAS,QAAQ,KAAK,aAAc,QAAO;AAC/E,SAAO,GAAG,OAAO,IAAI,EAAE;AAAA,IACrB,OAAO;AAAA,MAAM,CAAC,UACZ,SAAS,OAAO,UAAU,IAAI,OAAO,OAAO,OAAO,KAAK,QAAQ,KAAK,CAAC;AAAA,IACxE;AAAA,EACF;AACA,SAAO;AACT,CAAC;AAUI,MAAM,YAAY,CACvB,IACA,WACA,QACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,QAAQ,OAAO;AACrB,QAAM,SAAS,GAAG,SAAS,IAAI,UAAU;AACzC,SAAO,GAAG,cAAc,WAAW,EAAE,WAAW,KAAK,CAAC,EAAE,KAAK,OAAO,SAAS,OAAO,CAAC;AAErF,MAAI,aAAa;AACjB,QAAM,UAAU,OAAO,IAAI,aAAY;AACrC,WAAO,MAAM;AACX,YAAM,WAAW,OAAO;AAAA,QACtB;AAAA,QACA;AAAA,QACA,OAAO,IAAI,aAAY;AACrB,cAAI,OAAO,aAAa,IAAI,WAAW,OAAO,EAAG,QAAO;AACxD,iBAAO,OAAO,OAAO;AAAA,YACnB,GAAG,gBAAgB,QAAQ,OAAO,EAAE,MAAM,MAAM,MAAM,IAAM,CAAC,EAAE;AAAA,cAC7D,OAAO,QAAQ,OAAO,KAAK,MAAM;AAC/B,6BAAa;AACb,uBAAO;AAAA,cACT,CAAC,CAAC;AAAA,cACF,OAAO;AAAA,gBAAM,CAAC,UACZ,SAAS,OAAO,eAAe,IAAI,OAAO,QAAQ,KAAK,IAAI,OAAO,KAAK,QAAQ,KAAK,CAAC;AAAA,cACvF;AAAA,YACF;AAAA,UACF;AAAA,QACF,CAAC;AAAA,QACD;AAAA,MACF;AACA,UAAI,SAAU;AACd,aAAO,OAAO,MAAM,UAAU;AAAA,IAChC;AAAA,EACF,CAAC,EAAE;AAAA,IACD,OAAO,QAAQ,aAAa;AAAA,IAC5B,OAAO,SAAS,gBAAgB,CAAC,UAAU,OAAO,KAAK,QAAQ,KAAK,CAAC,CAAC;AAAA,EACxE;AAIA,SAAO,OAAO,OAAO;AAAA,IACnB,OAAO;AAAA,IACP,MACE,QAAQ,KAAK,OAAO;AAAA,MAClB,OAAO,OAAO,OAAO,IAAI,aAAY;AACnC,eAAO,OAAO;AAAA,UACZ,OAAO;AAAA,YACL,OAAO,MAAM,cAAc,EAAE;AAAA,cAC3B,OAAO,QAAQ;AAAA,gBACb;AAAA,gBACA;AAAA,gBACA,GAAG,eAAe,MAAM,EAAE;AAAA,kBACxB,OAAO,IAAI,OAAO,IAAI;AAAA,kBACtB,OAAO;AAAA,oBAAM,CAAC,UACZ,SAAS,OAAO,UAAU,IAAI,OAAO,QAAQ,OAAO,KAAa,CAAC,IAAI,OAAO,KAAK,KAAK;AAAA,kBACzF;AAAA,kBACA,OAAO,QAAQ,CAAC,UAAU;AAMxB,wBAAI,OAAO,eAAe,KAAK,MAAM,OAAO;AAC1C,6BAAO,OAAO;AAAA,wBACZ;AAAA,sBACF;AAAA,oBACF;AACA,2BAAO,OAAO,QAAQ,MAAM,mBAAmB,CAAC,QAAQ;AACtD,4BAAM,YAAY,IAAI,KAAK,GAAG;AAC9B,6BAAO,GAAG,OAAO,QAAQ,WAAW,SAAS;AAAA,oBAC/C,CAAC;AAAA,kBACH,CAAC;AAAA,kBACD,OAAO,SAAS,OAAO;AAAA,gBACzB;AAAA,gBACA;AAAA,cACF,CAAC;AAAA;AAAA;AAAA;AAAA,cAID,OAAO,MAAM,CAAC,UAAU,OAAO,WAAW,0CAA0C,KAAK,CAAC;AAAA,YAC5F;AAAA,UACF;AAAA,QACF;AACA,eAAO,OAAO;AAAA,MAChB,CAAC,CAAC;AAAA,IACJ,CAAC;AAAA,IACH,MACE,aACI;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAG,eAAe,MAAM,EAAE;AAAA,QACxB,OAAO,QAAQ,CAAC,UAAU,UAAU,QAAQ,GAAG,OAAO,MAAM,IAAI,OAAO,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAM3E,OAAO;AAAA,UAAM,CAAC,UACZ,SAAS,OAAO,UAAU,IAAI,OAAO,OAAO,OAAO,KAAK,KAAK;AAAA,QAC/D;AAAA,MACF;AAAA,MACA;AAAA,IACF,EAAE;AAAA,MACA,OAAO,MAAM,CAAC,UAAU,OAAO,WAAW,wCAAwC,KAAK,CAAC;AAAA,IAC1F,IACE,OAAO;AAAA,EACf;AACF,CAAC;AAUI,MAAM,eAAe,CAC1B,IACA,WACA,QACA,YAEA;AAAA,EACE;AAAA,EACA;AAAA,EACA,OAAO,IAAI,aAAY;AACrB,QAAI,OAAO,aAAa,IAAI,WAAW,OAAO,EAAG,QAAO,OAAO,KAAQ;AACvE,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC,CAAC;AAAA,EACD;AACF;",
6
+ "names": []
7
+ }
@@ -0,0 +1,129 @@
1
+ /**
2
+ * The content-addressed artifact store: bytes addressed by their own SHA-256
3
+ * digest.
4
+ *
5
+ * It is deliberately *not* the step cache: the step cache maps a step key to a
6
+ * recorded result, while large result bytes live here under their digest. The
7
+ * two tiers remain separate because artifacts must be published before a cache
8
+ * record may reference them. See the package README and
9
+ * {@link https://smithers.sh/docs/concepts/content-addressing | step-key documentation}.
10
+ *
11
+ * This module defines the contract: the address schema, the service interface
12
+ * and tag, and one layer per local implementation. The errors, byte helpers,
13
+ * and implementations are defined in the modules named for them and re-exported
14
+ * here unchanged, so this subpath and the root `ArtifactStore` namespace keep
15
+ * their whole surface.
16
+ *
17
+ * @since 1.0.0-rc.0
18
+ */
19
+ import { Sha256 } from "@smthrs/crypto";
20
+ import * as Context from "effect/Context";
21
+ import type * as Crypto from "effect/Crypto";
22
+ import * as Effect from "effect/Effect";
23
+ import * as FileSystem from "effect/FileSystem";
24
+ import * as Layer from "effect/Layer";
25
+ import * as Path from "effect/Path";
26
+ import type { ArtifactCorruption, ArtifactMissing, ArtifactStoreError } from "./ArtifactStoreError.ts";
27
+ import { type FileSystemOptions } from "./FileSystemArtifactStore.ts";
28
+ export { ArtifactCorruption, ArtifactMissing, ArtifactStoreError, ArtifactStoreErrorCode } from "./ArtifactStoreError.ts";
29
+ export { defaultDirectory, type FileSystemOptions, makeFileSystem } from "./FileSystemArtifactStore.ts";
30
+ export { measureBytes } from "./measureBytes.ts";
31
+ export { makeMemory } from "./MemoryArtifactStore.ts";
32
+ export { makeNoop } from "./NoopArtifactStore.ts";
33
+ export { snapshotBytes } from "./snapshotBytes.ts";
34
+ export { validateDigest } from "./validateDigest.ts";
35
+ /**
36
+ * Schema for a content address: exactly 64 lowercase hexadecimal SHA-256
37
+ * characters, branded by `@smthrs/crypto`. Re-exported so a consumer never has
38
+ * to reach past this package for the address type it stores under.
39
+ *
40
+ * @category schemas
41
+ * @since 1.0.0-rc.0
42
+ * @slop
43
+ */
44
+ export declare const Digest: import("effect/Schema").brand<import("effect/Schema").String, "@smthrs/crypto/Sha256/Digest">;
45
+ /**
46
+ * A content address produced by this store.
47
+ *
48
+ * Read operations accept a plain `string` rather than this brand on purpose: a
49
+ * digest read back out of a durable row is untrusted input, so the store
50
+ * validates it (see {@link ArtifactStoreError}'s `invalid_digest` code) instead
51
+ * of asking every caller to re-brand a persisted column. `put` returns the
52
+ * brand, because it measured the bytes itself.
53
+ *
54
+ * @category models
55
+ * @since 1.0.0-rc.0
56
+ * @slop
57
+ */
58
+ export type Digest = typeof Sha256.Digest.Type;
59
+ /**
60
+ * Content-addressed blob storage.
61
+ *
62
+ * The contract's ergonomics follow Effect's own `KeyValueStore`
63
+ * (`effect/unstable/persistence/KeyValueStore`): a small set of total
64
+ * operations over one address space, with a single typed error family, so a
65
+ * memory, filesystem, or network implementation is the same shape.
66
+ * `findMissing` is Bazel's `MissingDigestsFinder` — one batched round trip
67
+ * whose result is guaranteed to be a subset of its input — because a
68
+ * per-digest existence probe over a network tier is the wrong shape entirely.
69
+ *
70
+ * @category models
71
+ * @since 1.0.0-rc.0
72
+ * @slop
73
+ */
74
+ export interface Service {
75
+ /**
76
+ * Stores `bytes` under their own SHA-256 digest and returns that address.
77
+ * Storing the same bytes twice is idempotent.
78
+ */
79
+ readonly put: (bytes: Uint8Array) => Effect.Effect<Digest, ArtifactStoreError, Crypto.Crypto>;
80
+ /**
81
+ * Reads the bytes stored at `digest`, verifying that they still hash to it.
82
+ */
83
+ readonly get: (digest: string) => Effect.Effect<Uint8Array, ArtifactMissing | ArtifactCorruption | ArtifactStoreError, Crypto.Crypto>;
84
+ /** Whether this tier holds an artifact at `digest`. */
85
+ readonly has: (digest: string) => Effect.Effect<boolean, ArtifactStoreError>;
86
+ /**
87
+ * Which of `digests` this tier does not hold. The returned array is
88
+ * guaranteed to be a subset of the input and free of duplicates.
89
+ */
90
+ readonly findMissing: (digests: Iterable<string>) => Effect.Effect<Array<string>, ArtifactStoreError>;
91
+ }
92
+ declare const ArtifactStore_base: Context.ServiceClass<ArtifactStore, "@smthrs/artifacts/ArtifactStore", Service>;
93
+ /**
94
+ * Service tag for the content-addressed artifact store.
95
+ *
96
+ * The identity string equals this module's package path, per the house rule
97
+ * that an identity is the defining module path.
98
+ *
99
+ * @category services
100
+ * @since 1.0.0-rc.0
101
+ * @slop
102
+ */
103
+ export declare class ArtifactStore extends ArtifactStore_base {
104
+ }
105
+ /**
106
+ * Provides the filesystem-backed artifact store.
107
+ *
108
+ * @category layers
109
+ * @since 1.0.0-rc.0
110
+ * @slop
111
+ */
112
+ export declare const layerFileSystem: (options?: FileSystemOptions) => Layer.Layer<ArtifactStore, never, FileSystem.FileSystem | Path.Path>;
113
+ /**
114
+ * Provides an in-memory artifact store.
115
+ *
116
+ * @category layers
117
+ * @since 1.0.0-rc.0
118
+ * @slop
119
+ */
120
+ export declare const layerMemory: Layer.Layer<ArtifactStore>;
121
+ /**
122
+ * Provides a no-op artifact store.
123
+ *
124
+ * @category layers
125
+ * @since 1.0.0-rc.0
126
+ * @slop
127
+ */
128
+ export declare const layerNoop: (overrides?: Partial<Service>) => Layer.Layer<ArtifactStore>;
129
+ //# sourceMappingURL=ArtifactStore.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ArtifactStore.d.ts","sourceRoot":"","sources":["../../src/ArtifactStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,KAAK,MAAM,MAAM,eAAe,CAAA;AAC5C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AACtG,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,8BAA8B,CAAA;AAIrF,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AACvG,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAEpD;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM,+FAAgB,CAAA;AAEnC;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAA;AAE9C;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,OAAO;IACtB;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IAC7F;;OAEG;IACH,QAAQ,CAAC,GAAG,EAAE,CACZ,MAAM,EAAE,MAAM,KACX,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,GAAG,kBAAkB,GAAG,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IACxG,uDAAuD;IACvD,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;IAC5E;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,CACpB,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,KACtB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,kBAAkB,CAAC,CAAA;CACtD;;AAED;;;;;;;;;GASG;AACH,qBAAa,aAAc,SAAQ,kBAA4E;CAAG;AAElH;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAC1B,UAAS,iBAAsB,KAC9B,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAGjE,CAAA;AAEL;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,aAAa,CAAwD,CAAA;AAE3G;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,GAAI,YAAW,OAAO,CAAC,OAAO,CAAM,KAAG,KAAK,CAAC,KAAK,CAAC,aAAa,CACnC,CAAA"}