@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/LICENSE +21 -0
- package/README.md +175 -2
- package/dist/cjs/ArtifactBackupLease.d.ts +34 -0
- package/dist/cjs/ArtifactBackupLease.d.ts.map +1 -0
- package/dist/cjs/ArtifactBackupLease.js +183 -0
- package/dist/cjs/ArtifactBackupLease.js.map +7 -0
- package/dist/cjs/ArtifactStore.d.ts +129 -0
- package/dist/cjs/ArtifactStore.d.ts.map +1 -0
- package/dist/cjs/ArtifactStore.js +73 -0
- package/dist/cjs/ArtifactStore.js.map +7 -0
- package/dist/cjs/ArtifactStoreError.d.ts +82 -0
- package/dist/cjs/ArtifactStoreError.d.ts.map +1 -0
- package/dist/cjs/ArtifactStoreError.js +72 -0
- package/dist/cjs/ArtifactStoreError.js.map +7 -0
- package/dist/cjs/ArtifactStoreMetrics.d.ts +70 -0
- package/dist/cjs/ArtifactStoreMetrics.d.ts.map +1 -0
- package/dist/cjs/ArtifactStoreMetrics.js +51 -0
- package/dist/cjs/ArtifactStoreMetrics.js.map +7 -0
- package/dist/cjs/ArtifactSweep.d.ts +185 -0
- package/dist/cjs/ArtifactSweep.d.ts.map +1 -0
- package/dist/cjs/ArtifactSweep.js +171 -0
- package/dist/cjs/ArtifactSweep.js.map +7 -0
- package/dist/cjs/CombinedArtifacts.d.ts +103 -0
- package/dist/cjs/CombinedArtifacts.d.ts.map +1 -0
- package/dist/cjs/CombinedArtifacts.js +189 -0
- package/dist/cjs/CombinedArtifacts.js.map +7 -0
- package/dist/cjs/FileLease.d.ts +117 -0
- package/dist/cjs/FileLease.d.ts.map +1 -0
- package/dist/cjs/FileLease.js +208 -0
- package/dist/cjs/FileLease.js.map +7 -0
- package/dist/cjs/FileSystemArtifactStore.d.ts +93 -0
- package/dist/cjs/FileSystemArtifactStore.d.ts.map +1 -0
- package/dist/cjs/FileSystemArtifactStore.js +276 -0
- package/dist/cjs/FileSystemArtifactStore.js.map +7 -0
- package/dist/cjs/MemoryArtifactStore.d.ts +25 -0
- package/dist/cjs/MemoryArtifactStore.d.ts.map +1 -0
- package/dist/cjs/MemoryArtifactStore.js +84 -0
- package/dist/cjs/MemoryArtifactStore.js.map +7 -0
- package/dist/cjs/NoopArtifactStore.d.ts +17 -0
- package/dist/cjs/NoopArtifactStore.d.ts.map +1 -0
- package/dist/cjs/NoopArtifactStore.js +46 -0
- package/dist/cjs/NoopArtifactStore.js.map +7 -0
- package/dist/cjs/RemoteArtifacts.d.ts +237 -0
- package/dist/cjs/RemoteArtifacts.d.ts.map +1 -0
- package/dist/cjs/RemoteArtifacts.js +405 -0
- package/dist/cjs/RemoteArtifacts.js.map +7 -0
- package/dist/cjs/index.d.ts +65 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +47 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/ArtifactLocks.d.ts +30 -0
- package/dist/cjs/internal/ArtifactLocks.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactLocks.js +93 -0
- package/dist/cjs/internal/ArtifactLocks.js.map +7 -0
- package/dist/cjs/internal/ArtifactPath.d.ts +19 -0
- package/dist/cjs/internal/ArtifactPath.d.ts.map +1 -0
- package/dist/cjs/internal/ArtifactPath.js +74 -0
- package/dist/cjs/internal/ArtifactPath.js.map +7 -0
- package/dist/cjs/measureBytes.d.ts +18 -0
- package/dist/cjs/measureBytes.d.ts.map +1 -0
- package/dist/cjs/measureBytes.js +43 -0
- package/dist/cjs/measureBytes.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/snapshotBytes.d.ts +19 -0
- package/dist/cjs/snapshotBytes.d.ts.map +1 -0
- package/dist/cjs/snapshotBytes.js +41 -0
- package/dist/cjs/snapshotBytes.js.map +7 -0
- package/dist/cjs/validateDigest.d.ts +22 -0
- package/dist/cjs/validateDigest.d.ts.map +1 -0
- package/dist/cjs/validateDigest.js +42 -0
- package/dist/cjs/validateDigest.js.map +7 -0
- package/dist/esm/ArtifactBackupLease.d.ts +34 -0
- package/dist/esm/ArtifactBackupLease.d.ts.map +1 -0
- package/dist/esm/ArtifactBackupLease.js +126 -0
- package/dist/esm/ArtifactBackupLease.js.map +1 -0
- package/dist/esm/ArtifactStore.d.ts +129 -0
- package/dist/esm/ArtifactStore.d.ts.map +1 -0
- package/dist/esm/ArtifactStore.js +83 -0
- package/dist/esm/ArtifactStore.js.map +1 -0
- package/dist/esm/ArtifactStoreError.d.ts +82 -0
- package/dist/esm/ArtifactStoreError.d.ts.map +1 -0
- package/dist/esm/ArtifactStoreError.js +77 -0
- package/dist/esm/ArtifactStoreError.js.map +1 -0
- package/dist/esm/ArtifactStoreMetrics.d.ts +70 -0
- package/dist/esm/ArtifactStoreMetrics.d.ts.map +1 -0
- package/dist/esm/ArtifactStoreMetrics.js +77 -0
- package/dist/esm/ArtifactStoreMetrics.js.map +1 -0
- package/dist/esm/ArtifactSweep.d.ts +185 -0
- package/dist/esm/ArtifactSweep.d.ts.map +1 -0
- package/dist/esm/ArtifactSweep.js +210 -0
- package/dist/esm/ArtifactSweep.js.map +1 -0
- package/dist/esm/CombinedArtifacts.d.ts +103 -0
- package/dist/esm/CombinedArtifacts.d.ts.map +1 -0
- package/dist/esm/CombinedArtifacts.js +214 -0
- package/dist/esm/CombinedArtifacts.js.map +1 -0
- package/dist/esm/FileLease.d.ts +117 -0
- package/dist/esm/FileLease.d.ts.map +1 -0
- package/dist/esm/FileLease.js +249 -0
- package/dist/esm/FileLease.js.map +1 -0
- package/dist/esm/FileSystemArtifactStore.d.ts +93 -0
- package/dist/esm/FileSystemArtifactStore.d.ts.map +1 -0
- package/dist/esm/FileSystemArtifactStore.js +336 -0
- package/dist/esm/FileSystemArtifactStore.js.map +1 -0
- package/dist/esm/MemoryArtifactStore.d.ts +25 -0
- package/dist/esm/MemoryArtifactStore.d.ts.map +1 -0
- package/dist/esm/MemoryArtifactStore.js +67 -0
- package/dist/esm/MemoryArtifactStore.js.map +1 -0
- package/dist/esm/NoopArtifactStore.d.ts +17 -0
- package/dist/esm/NoopArtifactStore.d.ts.map +1 -0
- package/dist/esm/NoopArtifactStore.js +27 -0
- package/dist/esm/NoopArtifactStore.js.map +1 -0
- package/dist/esm/RemoteArtifacts.d.ts +237 -0
- package/dist/esm/RemoteArtifacts.d.ts.map +1 -0
- package/dist/esm/RemoteArtifacts.js +557 -0
- package/dist/esm/RemoteArtifacts.js.map +1 -0
- package/dist/esm/index.d.ts +65 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +65 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/ArtifactLocks.d.ts +30 -0
- package/dist/esm/internal/ArtifactLocks.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactLocks.js +86 -0
- package/dist/esm/internal/ArtifactLocks.js.map +1 -0
- package/dist/esm/internal/ArtifactPath.d.ts +19 -0
- package/dist/esm/internal/ArtifactPath.d.ts.map +1 -0
- package/dist/esm/internal/ArtifactPath.js +59 -0
- package/dist/esm/internal/ArtifactPath.js.map +1 -0
- package/dist/esm/measureBytes.d.ts +18 -0
- package/dist/esm/measureBytes.d.ts.map +1 -0
- package/dist/esm/measureBytes.js +22 -0
- package/dist/esm/measureBytes.js.map +1 -0
- package/dist/esm/snapshotBytes.d.ts +19 -0
- package/dist/esm/snapshotBytes.d.ts.map +1 -0
- package/dist/esm/snapshotBytes.js +30 -0
- package/dist/esm/snapshotBytes.js.map +1 -0
- package/dist/esm/validateDigest.d.ts +22 -0
- package/dist/esm/validateDigest.d.ts.map +1 -0
- package/dist/esm/validateDigest.js +26 -0
- package/dist/esm/validateDigest.js.map +1 -0
- package/docs/README.md +202 -0
- package/docs/api.md +484 -0
- package/docs/concepts/content-addressing.md +174 -0
- package/docs/concepts/coordination.md +141 -0
- package/docs/concepts/tiers.md +147 -0
- package/docs/guides/fence-a-backup.md +123 -0
- package/docs/guides/reclaim-disk-space.md +143 -0
- package/docs/guides/serve-the-artifact-protocol.md +140 -0
- package/docs/guides/share-artifacts-across-machines.md +180 -0
- package/docs/guides/test-against-an-artifact-store.md +158 -0
- package/docs/installation.md +102 -0
- package/docs/quickstart.md +183 -0
- package/docs/troubleshooting.md +232 -0
- package/package.json +179 -3
- package/src/ArtifactBackupLease.ts +223 -0
- package/src/ArtifactStore.ts +152 -0
- package/src/ArtifactStoreError.ts +96 -0
- package/src/ArtifactStoreMetrics.ts +83 -0
- package/src/ArtifactSweep.ts +346 -0
- package/src/CombinedArtifacts.ts +369 -0
- package/src/FileLease.ts +397 -0
- package/src/FileSystemArtifactStore.ts +459 -0
- package/src/MemoryArtifactStore.ts +83 -0
- package/src/NoopArtifactStore.ts +30 -0
- package/src/RemoteArtifacts.ts +814 -0
- package/src/index.ts +71 -0
- package/src/internal/ArtifactLocks.ts +110 -0
- package/src/internal/ArtifactPath.ts +72 -0
- package/src/measureBytes.ts +28 -0
- package/src/snapshotBytes.ts +34 -0
- package/src/validateDigest.ts +31 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
var snapshotBytes_exports = {};
|
|
30
|
+
__export(snapshotBytes_exports, {
|
|
31
|
+
snapshotBytes: () => snapshotBytes
|
|
32
|
+
});
|
|
33
|
+
module.exports = __toCommonJS(snapshotBytes_exports);
|
|
34
|
+
var Effect = __toESM(require("effect/Effect"), 1);
|
|
35
|
+
var import_ArtifactStoreError = require("./ArtifactStoreError.js");
|
|
36
|
+
const snapshotFailure = (cause) => new import_ArtifactStoreError.ArtifactStoreError({ code: "unavailable", message: "the host could not copy the artifact bytes", cause });
|
|
37
|
+
const snapshotBytes = (bytes) => Effect.try({
|
|
38
|
+
try: () => new Uint8Array(bytes),
|
|
39
|
+
catch: (cause) => snapshotFailure(cause)
|
|
40
|
+
});
|
|
41
|
+
//# sourceMappingURL=snapshotBytes.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/snapshotBytes.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Snapshotting caller-owned artifact bytes, so a caller that reuses its buffer\n * cannot change what is measured or stored.\n *\n * @since 1.0.0-rc.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport { ArtifactStoreError } from \"./ArtifactStoreError.ts\"\n\n/**\n * A refused byte copy is the host declining an allocation, not a crypto\n * failure: the Crypto service has not been consulted when this fires, so\n * reporting `digest_failed` would point an operator diagnosing memory pressure\n * at the wrong subsystem. The message is constant, and the buffer is never\n * attached.\n */\nconst snapshotFailure = (cause: unknown): ArtifactStoreError =>\n new ArtifactStoreError({ code: \"unavailable\", message: \"the host could not copy the artifact bytes\", cause })\n\n/**\n * Copies a caller-owned buffer when the returned Effect begins.\n *\n * A host that refuses the copy \u2014 a detached buffer, an allocation past what the\n * runtime will give \u2014 fails as `unavailable`, the code for a host refusal.\n *\n * @category utilities\n * @since 1.0.0-rc.0\n */\nexport const snapshotBytes = (bytes: Uint8Array): Effect.Effect<Uint8Array, ArtifactStoreError> =>\n Effect.try({\n try: () => new Uint8Array(bytes),\n catch: (cause) => snapshotFailure(cause)\n })\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,aAAwB;AACxB,gCAAmC;AASnC,MAAM,kBAAkB,CAAC,UACvB,IAAI,6CAAmB,EAAE,MAAM,eAAe,SAAS,8CAA8C,MAAM,CAAC;AAWvG,MAAM,gBAAgB,CAAC,UAC5B,OAAO,IAAI;AAAA,EACT,KAAK,MAAM,IAAI,WAAW,KAAK;AAAA,EAC/B,OAAO,CAAC,UAAU,gBAAgB,KAAK;AACzC,CAAC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Validating a content address before any implementation logs it or
|
|
3
|
+
* interpolates it into a path or URL.
|
|
4
|
+
*
|
|
5
|
+
* @since 1.0.0-rc.0
|
|
6
|
+
*/
|
|
7
|
+
import * as Effect from "effect/Effect";
|
|
8
|
+
import type { Digest } from "./ArtifactStore.ts";
|
|
9
|
+
import { ArtifactStoreError } from "./ArtifactStoreError.ts";
|
|
10
|
+
/**
|
|
11
|
+
* Refuses anything other than the canonical SHA-256 address representation.
|
|
12
|
+
*
|
|
13
|
+
* Every implementation validates before logging or interpolating an untrusted
|
|
14
|
+
* value into a path or URL. The failure text is constant and bounded, so even a
|
|
15
|
+
* hostile multi-megabyte value cannot be copied into logs or durable errors.
|
|
16
|
+
*
|
|
17
|
+
* @category predicates
|
|
18
|
+
* @since 1.0.0-rc.0
|
|
19
|
+
* @slop
|
|
20
|
+
*/
|
|
21
|
+
export declare const validateDigest: (digest: string) => Effect.Effect<Digest, ArtifactStoreError>;
|
|
22
|
+
//# sourceMappingURL=validateDigest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validateDigest.d.ts","sourceRoot":"","sources":["../../src/validateDigest.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAE5D;;;;;;;;;;GAUG;AACH,eAAO,MAAM,cAAc,GAAI,QAAQ,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAQnF,CAAA"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
var validateDigest_exports = {};
|
|
30
|
+
__export(validateDigest_exports, {
|
|
31
|
+
validateDigest: () => validateDigest
|
|
32
|
+
});
|
|
33
|
+
module.exports = __toCommonJS(validateDigest_exports);
|
|
34
|
+
var Effect = __toESM(require("effect/Effect"), 1);
|
|
35
|
+
var import_ArtifactStoreError = require("./ArtifactStoreError.js");
|
|
36
|
+
const validateDigest = (digest) => typeof digest === "string" && /^[0-9a-f]{64}$/.test(digest) ? Effect.succeed(digest) : Effect.fail(
|
|
37
|
+
new import_ArtifactStoreError.ArtifactStoreError({
|
|
38
|
+
code: "invalid_digest",
|
|
39
|
+
message: "artifact digest must be exactly 64 lowercase hexadecimal characters"
|
|
40
|
+
})
|
|
41
|
+
);
|
|
42
|
+
//# sourceMappingURL=validateDigest.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/validateDigest.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Validating a content address before any implementation logs it or\n * interpolates it into a path or URL.\n *\n * @since 1.0.0-rc.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Digest } from \"./ArtifactStore.ts\"\nimport { ArtifactStoreError } from \"./ArtifactStoreError.ts\"\n\n/**\n * Refuses anything other than the canonical SHA-256 address representation.\n *\n * Every implementation validates before logging or interpolating an untrusted\n * value into a path or URL. The failure text is constant and bounded, so even a\n * hostile multi-megabyte value cannot be copied into logs or durable errors.\n *\n * @category predicates\n * @since 1.0.0-rc.0\n * @slop\n */\nexport const validateDigest = (digest: string): Effect.Effect<Digest, ArtifactStoreError> =>\n typeof digest === \"string\" && /^[0-9a-f]{64}$/.test(digest)\n ? Effect.succeed(digest as Digest)\n : Effect.fail(\n new ArtifactStoreError({\n code: \"invalid_digest\",\n message: \"artifact digest must be exactly 64 lowercase hexadecimal characters\"\n })\n )\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,aAAwB;AAExB,gCAAmC;AAa5B,MAAM,iBAAiB,CAAC,WAC7B,OAAO,WAAW,YAAY,iBAAiB,KAAK,MAAM,IACtD,OAAO,QAAQ,MAAgB,IAC/B,OAAO;AAAA,EACP,IAAI,6CAAmB;AAAA,IACrB,MAAM;AAAA,IACN,SAAS;AAAA,EACX,CAAC;AACH;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-process exclusion between a filesystem artifact backup and garbage
|
|
3
|
+
* collection.
|
|
4
|
+
*
|
|
5
|
+
* A backup holds one heartbeat-backed marker for the duration of its database
|
|
6
|
+
* snapshot and blob copy. Sweep deletion checks that marker while holding the
|
|
7
|
+
* same short-lived global gate. Publication remains unconstrained: new blobs
|
|
8
|
+
* may appear during a backup, but a blob already referenced by the frozen
|
|
9
|
+
* database cannot disappear underneath it.
|
|
10
|
+
*
|
|
11
|
+
* @since 1.0.0-rc.0
|
|
12
|
+
*/
|
|
13
|
+
import * as Effect from "effect/Effect";
|
|
14
|
+
import type * as FileSystem from "effect/FileSystem";
|
|
15
|
+
import * as Option from "effect/Option";
|
|
16
|
+
/**
|
|
17
|
+
* Runs `effect` while sweep deletion is fenced across every process using
|
|
18
|
+
* this objects directory. A crashed holder is recovered after its heartbeat
|
|
19
|
+
* becomes stale.
|
|
20
|
+
*
|
|
21
|
+
* @category combinators
|
|
22
|
+
* @since 1.0.0-rc.0
|
|
23
|
+
*/
|
|
24
|
+
export declare const withLease: <A, E, R, E2>(fs: FileSystem.FileSystem, directory: string, effect: Effect.Effect<A, E, R>, failure: (cause: unknown) => E2) => Effect.Effect<A, E | E2, R>;
|
|
25
|
+
/**
|
|
26
|
+
* Runs one sweep deletion only when no live backup lease exists. The marker
|
|
27
|
+
* check and `effect` share the global gate, closing the check/delete race.
|
|
28
|
+
* `None` means a backup deliberately fenced the operation.
|
|
29
|
+
*
|
|
30
|
+
* @category combinators
|
|
31
|
+
* @since 1.0.0-rc.0
|
|
32
|
+
*/
|
|
33
|
+
export declare const unlessActive: <A, E, R, E2>(fs: FileSystem.FileSystem, directory: string, effect: Effect.Effect<A, E, R>, failure: (cause: unknown) => E2) => Effect.Effect<Option.Option<A>, E | E2, R>;
|
|
34
|
+
//# sourceMappingURL=ArtifactBackupLease.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ArtifactBackupLease.d.ts","sourceRoot":"","sources":["../../src/ArtifactBackupLease.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,UAAU,MAAM,mBAAmB,CAAA;AACpD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAqDvC;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EACnC,IAAI,UAAU,CAAC,UAAU,EACzB,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,SAAS,CAAC,KAAK,EAAE,OAAO,KAAK,EAAE,KAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAoHzB,CAAA;AAEJ;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EACtC,IAAI,UAAU,CAAC,UAAU,EACzB,WAAW,MAAM,EACjB,QAAQ,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAC9B,SAAS,CAAC,KAAK,EAAE,OAAO,KAAK,EAAE,KAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CASzC,CAAA"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-process exclusion between a filesystem artifact backup and garbage
|
|
3
|
+
* collection.
|
|
4
|
+
*
|
|
5
|
+
* A backup holds one heartbeat-backed marker for the duration of its database
|
|
6
|
+
* snapshot and blob copy. Sweep deletion checks that marker while holding the
|
|
7
|
+
* same short-lived global gate. Publication remains unconstrained: new blobs
|
|
8
|
+
* may appear during a backup, but a blob already referenced by the frozen
|
|
9
|
+
* database cannot disappear underneath it.
|
|
10
|
+
*
|
|
11
|
+
* @since 1.0.0-rc.0
|
|
12
|
+
*/
|
|
13
|
+
import * as Clock from "effect/Clock";
|
|
14
|
+
import * as Effect from "effect/Effect";
|
|
15
|
+
import * as Option from "effect/Option";
|
|
16
|
+
import * as PlatformError from "effect/PlatformError";
|
|
17
|
+
import * as Random from "effect/Random";
|
|
18
|
+
import * as ArtifactLocks from "./internal/ArtifactLocks.js";
|
|
19
|
+
const markerName = ".backup-lease";
|
|
20
|
+
const gateName = ".backup-lease-gate";
|
|
21
|
+
const heartbeatEvery = "10 seconds";
|
|
22
|
+
const staleAfterMs = 60_000;
|
|
23
|
+
const acquireWithin = "2 minutes";
|
|
24
|
+
const retryEvery = "25 millis";
|
|
25
|
+
const isReason = (cause, tag) => cause instanceof PlatformError.PlatformError && cause.reason._tag === tag;
|
|
26
|
+
const token = Effect.gen(function* () {
|
|
27
|
+
const first = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true });
|
|
28
|
+
const second = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true });
|
|
29
|
+
return `${first.toString(36)}-${second.toString(36)}-${yield* Clock.currentTimeMillis}`;
|
|
30
|
+
});
|
|
31
|
+
const gate = (fs, directory, effect, failure) => ArtifactLocks.withDigest(fs, directory, gateName, effect, failure);
|
|
32
|
+
const activeMarker = (fs, directory, failure) => Effect.gen(function* () {
|
|
33
|
+
const path = `${directory}/${markerName}`;
|
|
34
|
+
const info = yield* fs.stat(path).pipe(Effect.map(Option.some), Effect.catch((cause) => isReason(cause, "NotFound") ? Effect.succeed(Option.none()) : Effect.fail(failure(cause))));
|
|
35
|
+
if (Option.isNone(info))
|
|
36
|
+
return false;
|
|
37
|
+
const modified = Option.getOrUndefined(info.value.mtime);
|
|
38
|
+
const now = yield* Clock.currentTimeMillis;
|
|
39
|
+
if (modified === undefined || now - modified.getTime() <= staleAfterMs)
|
|
40
|
+
return true;
|
|
41
|
+
yield* fs.remove(path).pipe(Effect.catch((cause) => isReason(cause, "NotFound") ? Effect.void : Effect.fail(failure(cause))));
|
|
42
|
+
return false;
|
|
43
|
+
});
|
|
44
|
+
/**
|
|
45
|
+
* Runs `effect` while sweep deletion is fenced across every process using
|
|
46
|
+
* this objects directory. A crashed holder is recovered after its heartbeat
|
|
47
|
+
* becomes stale.
|
|
48
|
+
*
|
|
49
|
+
* @category combinators
|
|
50
|
+
* @since 1.0.0-rc.0
|
|
51
|
+
*/
|
|
52
|
+
export const withLease = (fs, directory, effect, failure) => Effect.gen(function* () {
|
|
53
|
+
const owner = yield* token;
|
|
54
|
+
const marker = `${directory}/${markerName}`;
|
|
55
|
+
yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure));
|
|
56
|
+
let ownsMarker = false;
|
|
57
|
+
const acquire = Effect.gen(function* () {
|
|
58
|
+
while (true) {
|
|
59
|
+
const acquired = yield* gate(fs, directory, Effect.gen(function* () {
|
|
60
|
+
if (yield* activeMarker(fs, directory, failure))
|
|
61
|
+
return false;
|
|
62
|
+
return yield* Effect.uninterruptible(fs.writeFileString(marker, owner, { flag: "wx", mode: 0o600 }).pipe(Effect.andThen(Effect.sync(() => {
|
|
63
|
+
ownsMarker = true;
|
|
64
|
+
return true;
|
|
65
|
+
})), Effect.catch((cause) => isReason(cause, "AlreadyExists") ? Effect.succeed(false) : Effect.fail(failure(cause)))));
|
|
66
|
+
}), failure);
|
|
67
|
+
if (acquired)
|
|
68
|
+
return;
|
|
69
|
+
yield* Effect.sleep(retryEvery);
|
|
70
|
+
}
|
|
71
|
+
}).pipe(Effect.timeout(acquireWithin), Effect.catchTag("TimeoutError", (cause) => Effect.fail(failure(cause))));
|
|
72
|
+
// Install marker cleanup before acquisition, including gate release. The
|
|
73
|
+
// marker write and ownership bookkeeping complete in one interruption mask.
|
|
74
|
+
return yield* Effect.acquireUseRelease(Effect.void, () => acquire.pipe(Effect.andThen(Effect.scoped(Effect.gen(function* () {
|
|
75
|
+
yield* Effect.forkScoped(Effect.gen(function* () {
|
|
76
|
+
while (true) {
|
|
77
|
+
yield* Effect.sleep(heartbeatEvery);
|
|
78
|
+
const reclaimed = yield* gate(fs, directory, fs.readFileString(marker).pipe(Effect.map(Option.some), Effect.catch((cause) => isReason(cause, "NotFound") ? Effect.succeed(Option.none()) : Effect.fail(cause)), Effect.flatMap((found) => {
|
|
79
|
+
// A missing or foreign marker means a sweep judged this
|
|
80
|
+
// lease stale and reaped it: deletion is no longer
|
|
81
|
+
// fenced while the backup still copies. The sibling
|
|
82
|
+
// `internal/ArtifactLocks.ts` heartbeat reports the same
|
|
83
|
+
// loss the same way. A reaped marker is never handed
|
|
84
|
+
// back, so the loss is reported once and the heartbeat
|
|
85
|
+
// ends instead of repeating it every tick.
|
|
86
|
+
if (Option.getOrUndefined(found) !== owner) {
|
|
87
|
+
return Effect.as(Effect.logWarning("Artifact backup lease was reclaimed while its backup was still running"), true);
|
|
88
|
+
}
|
|
89
|
+
return Effect.flatMap(Clock.currentTimeMillis, (now) => {
|
|
90
|
+
const timestamp = new Date(now);
|
|
91
|
+
return Effect.as(fs.utimes(marker, timestamp, timestamp), false);
|
|
92
|
+
});
|
|
93
|
+
}), Effect.mapError(failure)), failure).pipe(
|
|
94
|
+
// A failed heartbeat is retried on the next tick, but a
|
|
95
|
+
// minute of them lets the marker go stale, so each one is
|
|
96
|
+
// reported rather than dropped.
|
|
97
|
+
Effect.catch((cause) => Effect.as(Effect.logWarning("Artifact backup lease heartbeat failed", cause), false)));
|
|
98
|
+
if (reclaimed)
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
}));
|
|
102
|
+
return yield* effect;
|
|
103
|
+
})))), () => ownsMarker
|
|
104
|
+
? gate(fs, directory, fs.readFileString(marker).pipe(Effect.flatMap((found) => found === owner ? fs.remove(marker) : Effect.void),
|
|
105
|
+
// A backup whose heartbeat lapsed has its marker reaped by whoever
|
|
106
|
+
// noticed, so `NotFound` is the ordinary end of a slow lease, not a
|
|
107
|
+
// fault: there is nothing left to release and nothing left to leak.
|
|
108
|
+
// The sibling release in `internal/ArtifactLocks.ts` classifies the
|
|
109
|
+
// same cause the same way.
|
|
110
|
+
Effect.catch((cause) => isReason(cause, "NotFound") ? Effect.void : Effect.fail(cause))), failure).pipe(Effect.catch((cause) => Effect.logWarning("Artifact backup lease release failed", cause)))
|
|
111
|
+
: Effect.void);
|
|
112
|
+
});
|
|
113
|
+
/**
|
|
114
|
+
* Runs one sweep deletion only when no live backup lease exists. The marker
|
|
115
|
+
* check and `effect` share the global gate, closing the check/delete race.
|
|
116
|
+
* `None` means a backup deliberately fenced the operation.
|
|
117
|
+
*
|
|
118
|
+
* @category combinators
|
|
119
|
+
* @since 1.0.0-rc.0
|
|
120
|
+
*/
|
|
121
|
+
export const unlessActive = (fs, directory, effect, failure) => gate(fs, directory, Effect.gen(function* () {
|
|
122
|
+
if (yield* activeMarker(fs, directory, failure))
|
|
123
|
+
return Option.none();
|
|
124
|
+
return Option.some(yield* effect);
|
|
125
|
+
}), failure);
|
|
126
|
+
//# sourceMappingURL=ArtifactBackupLease.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ArtifactBackupLease.js","sourceRoot":"","sources":["../../src/ArtifactBackupLease.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,aAAa,MAAM,sBAAsB,CAAA;AACrD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,aAAa,MAAM,6BAA6B,CAAA;AAE5D,MAAM,UAAU,GAAG,eAAe,CAAA;AAClC,MAAM,QAAQ,GAAG,oBAAoB,CAAA;AACrC,MAAM,cAAc,GAAG,YAAY,CAAA;AACnC,MAAM,YAAY,GAAG,MAAM,CAAA;AAC3B,MAAM,aAAa,GAAG,WAAW,CAAA;AACjC,MAAM,UAAU,GAAG,WAAW,CAAA;AAE9B,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAE,GAAiC,EAAW,EAAE,CAC9E,KAAK,YAAY,aAAa,CAAC,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,GAAG,CAAA;AAE3E,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAC1F,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3F,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAA;AACzF,CAAC,CAAC,CAAA;AAEF,MAAM,IAAI,GAAG,CACX,EAAyB,EACzB,SAAiB,EACjB,MAA8B,EAC9B,OAA+B,EACF,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;AAEpG,MAAM,YAAY,GAAG,CACnB,EAAyB,EACzB,SAAiB,EACjB,OAA8B,EACH,EAAE,CAC7B,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,IAAI,GAAG,GAAG,SAAS,IAAI,UAAU,EAAE,CAAA;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CACpC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EACvB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAyD,EAAE,CAC5E,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAC1F,CACF,CAAA;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACxD,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAA;IAC1C,IAAI,QAAQ,KAAK,SAAS,IAAI,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,IAAI,YAAY;QAAE,OAAO,IAAI,CAAA;IACnF,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CACzB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAA0B,EAAE,CAC7C,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CACxE,CACF,CAAA;IACD,OAAO,KAAK,CAAA;AACd,CAAC,CAAC,CAAA;AAEJ;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,EAAyB,EACzB,SAAiB,EACjB,MAA8B,EAC9B,OAA+B,EACF,EAAE,CAC/B,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAA;IAC1B,MAAM,MAAM,GAAG,GAAG,SAAS,IAAI,UAAU,EAAE,CAAA;IAC3C,KAAK,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEtF,IAAI,UAAU,GAAG,KAAK,CAAA;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,IAAI,CAC1B,EAAE,EACF,SAAS,EACT,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAClB,IAAI,KAAK,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC;oBAAE,OAAO,KAAK,CAAA;gBAC7D,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,eAAe,CAClC,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CACjE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;oBAC9B,UAAU,GAAG,IAAI,CAAA;oBACjB,OAAO,IAAI,CAAA;gBACb,CAAC,CAAC,CAAC,EACH,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAA8B,EAAE,CACjD,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CACvF,CACF,CACF,CAAA;YACH,CAAC,CAAC,EACF,OAAO,CACR,CAAA;YACD,IAAI,QAAQ;gBAAE,OAAM;YACpB,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;QACjC,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAC7B,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CACxE,CAAA;IAED,yEAAyE;IACzE,4EAA4E;IAC5E,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,iBAAiB,CACpC,MAAM,CAAC,IAAI,EACX,GAAG,EAAE,CACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CACzB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CACtB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAClB,OAAO,IAAI,EAAE,CAAC;gBACZ,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;gBACnC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,IAAI,CAC3B,EAAE,EACF,SAAS,EACT,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAC5B,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EACvB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACrB,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CACzF,EACD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;oBACvB,wDAAwD;oBACxD,mDAAmD;oBACnD,oDAAoD;oBACpD,yDAAyD;oBACzD,qDAAqD;oBACrD,uDAAuD;oBACvD,2CAA2C;oBAC3C,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,KAAK,EAAE,CAAC;wBAC3C,OAAO,MAAM,CAAC,EAAE,CACd,MAAM,CAAC,UAAU,CACf,wEAAwE,CACzE,EACD,IAAI,CACL,CAAA;oBACH,CAAC;oBACD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,GAAG,EAAE,EAAE;wBACrD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAA;wBAC/B,OAAO,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,CAAA;oBAClE,CAAC,CAAC,CAAA;gBACJ,CAAC,CAAC,EACF,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CACzB,EACD,OAAO,CACR,CAAC,IAAI;gBACJ,wDAAwD;gBACxD,0DAA0D;gBAC1D,gCAAgC;gBAChC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACrB,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,wCAAwC,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CACrF,CACF,CAAA;gBACD,IAAI,SAAS;oBAAE,OAAM;YACvB,CAAC;QACH,CAAC,CAAC,CACH,CAAA;QACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAA;IACtB,CAAC,CAAC,CAAC,CACJ,CAAC,EACJ,GAAG,EAAE,CACH,UAAU;QACR,CAAC,CAAC,IAAI,CACJ,EAAE,EACF,SAAS,EACT,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QAC5E,mEAAmE;QACnE,oEAAoE;QACpE,oEAAoE;QACpE,oEAAoE;QACpE,2BAA2B;QAC3B,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAoD,EAAE,CACvE,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAC/D,CACF,EACD,OAAO,CACR,CAAC,IAAI,CACJ,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,sCAAsC,EAAE,KAAK,CAAC,CAAC,CAC1F;QACD,CAAC,CAAC,MAAM,CAAC,IAAI,CAClB,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAyB,EACzB,SAAiB,EACjB,MAA8B,EAC9B,OAA+B,EACa,EAAE,CAC9C,IAAI,CACF,EAAE,EACF,SAAS,EACT,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,EAAK,CAAA;IACxE,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAA;AACnC,CAAC,CAAC,EACF,OAAO,CACR,CAAA","sourcesContent":["/**\n * Cross-process exclusion between a filesystem artifact backup and garbage\n * collection.\n *\n * A backup holds one heartbeat-backed marker for the duration of its database\n * snapshot and blob copy. Sweep deletion checks that marker while holding the\n * same short-lived global gate. Publication remains unconstrained: new blobs\n * may appear during a backup, but a blob already referenced by the frozen\n * database cannot disappear underneath it.\n *\n * @since 1.0.0-rc.0\n */\n\nimport * as Clock from \"effect/Clock\"\nimport * as Effect from \"effect/Effect\"\nimport type * as FileSystem from \"effect/FileSystem\"\nimport * as Option from \"effect/Option\"\nimport * as PlatformError from \"effect/PlatformError\"\nimport * as Random from \"effect/Random\"\nimport * as ArtifactLocks from \"./internal/ArtifactLocks.ts\"\n\nconst markerName = \".backup-lease\"\nconst gateName = \".backup-lease-gate\"\nconst heartbeatEvery = \"10 seconds\"\nconst staleAfterMs = 60_000\nconst acquireWithin = \"2 minutes\"\nconst retryEvery = \"25 millis\"\n\nconst isReason = (cause: unknown, tag: PlatformError.SystemErrorTag): boolean =>\n cause instanceof PlatformError.PlatformError && cause.reason._tag === tag\n\nconst token = Effect.gen(function*() {\n const first = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })\n const second = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })\n return `${first.toString(36)}-${second.toString(36)}-${yield* Clock.currentTimeMillis}`\n})\n\nconst gate = <A, E, R, E2>(\n fs: FileSystem.FileSystem,\n directory: string,\n effect: Effect.Effect<A, E, R>,\n failure: (cause: unknown) => E2\n): Effect.Effect<A, E | E2, R> => ArtifactLocks.withDigest(fs, directory, gateName, effect, failure)\n\nconst activeMarker = <E>(\n fs: FileSystem.FileSystem,\n directory: string,\n failure: (cause: unknown) => E\n): Effect.Effect<boolean, E> =>\n Effect.gen(function*() {\n const path = `${directory}/${markerName}`\n const info = yield* fs.stat(path).pipe(\n Effect.map(Option.some),\n Effect.catch((cause): Effect.Effect<Option.Option<FileSystem.File.Info>, E> =>\n isReason(cause, \"NotFound\") ? Effect.succeed(Option.none()) : Effect.fail(failure(cause))\n )\n )\n if (Option.isNone(info)) return false\n const modified = Option.getOrUndefined(info.value.mtime)\n const now = yield* Clock.currentTimeMillis\n if (modified === undefined || now - modified.getTime() <= staleAfterMs) return true\n yield* fs.remove(path).pipe(\n Effect.catch((cause): Effect.Effect<void, E> =>\n isReason(cause, \"NotFound\") ? Effect.void : Effect.fail(failure(cause))\n )\n )\n return false\n })\n\n/**\n * Runs `effect` while sweep deletion is fenced across every process using\n * this objects directory. A crashed holder is recovered after its heartbeat\n * becomes stale.\n *\n * @category combinators\n * @since 1.0.0-rc.0\n */\nexport const withLease = <A, E, R, E2>(\n fs: FileSystem.FileSystem,\n directory: string,\n effect: Effect.Effect<A, E, R>,\n failure: (cause: unknown) => E2\n): Effect.Effect<A, E | E2, R> =>\n Effect.gen(function*() {\n const owner = yield* token\n const marker = `${directory}/${markerName}`\n yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure))\n\n let ownsMarker = false\n const acquire = Effect.gen(function*() {\n while (true) {\n const acquired = yield* gate(\n fs,\n directory,\n Effect.gen(function*() {\n if (yield* activeMarker(fs, directory, failure)) return false\n return yield* Effect.uninterruptible(\n fs.writeFileString(marker, owner, { flag: \"wx\", mode: 0o600 }).pipe(\n Effect.andThen(Effect.sync(() => {\n ownsMarker = true\n return true\n })),\n Effect.catch((cause): Effect.Effect<boolean, E2> =>\n isReason(cause, \"AlreadyExists\") ? Effect.succeed(false) : Effect.fail(failure(cause))\n )\n )\n )\n }),\n failure\n )\n if (acquired) return\n yield* Effect.sleep(retryEvery)\n }\n }).pipe(\n Effect.timeout(acquireWithin),\n Effect.catchTag(\"TimeoutError\", (cause) => Effect.fail(failure(cause)))\n )\n\n // Install marker cleanup before acquisition, including gate release. The\n // marker write and ownership bookkeeping complete in one interruption mask.\n return yield* Effect.acquireUseRelease(\n Effect.void,\n () =>\n acquire.pipe(Effect.andThen(\n Effect.scoped(Effect.gen(function*() {\n yield* Effect.forkScoped(\n Effect.gen(function*() {\n while (true) {\n yield* Effect.sleep(heartbeatEvery)\n const reclaimed = yield* gate(\n fs,\n directory,\n fs.readFileString(marker).pipe(\n Effect.map(Option.some),\n Effect.catch((cause) =>\n isReason(cause, \"NotFound\") ? Effect.succeed(Option.none<string>()) : Effect.fail(cause)\n ),\n Effect.flatMap((found) => {\n // A missing or foreign marker means a sweep judged this\n // lease stale and reaped it: deletion is no longer\n // fenced while the backup still copies. The sibling\n // `internal/ArtifactLocks.ts` heartbeat reports the same\n // loss the same way. A reaped marker is never handed\n // back, so the loss is reported once and the heartbeat\n // ends instead of repeating it every tick.\n if (Option.getOrUndefined(found) !== owner) {\n return Effect.as(\n Effect.logWarning(\n \"Artifact backup lease was reclaimed while its backup was still running\"\n ),\n true\n )\n }\n return Effect.flatMap(Clock.currentTimeMillis, (now) => {\n const timestamp = new Date(now)\n return Effect.as(fs.utimes(marker, timestamp, timestamp), false)\n })\n }),\n Effect.mapError(failure)\n ),\n failure\n ).pipe(\n // A failed heartbeat is retried on the next tick, but a\n // minute of them lets the marker go stale, so each one is\n // reported rather than dropped.\n Effect.catch((cause) =>\n Effect.as(Effect.logWarning(\"Artifact backup lease heartbeat failed\", cause), false)\n )\n )\n if (reclaimed) return\n }\n })\n )\n return yield* effect\n }))\n )),\n () =>\n ownsMarker\n ? gate(\n fs,\n directory,\n fs.readFileString(marker).pipe(\n Effect.flatMap((found) => found === owner ? fs.remove(marker) : Effect.void),\n // A backup whose heartbeat lapsed has its marker reaped by whoever\n // noticed, so `NotFound` is the ordinary end of a slow lease, not a\n // fault: there is nothing left to release and nothing left to leak.\n // The sibling release in `internal/ArtifactLocks.ts` classifies the\n // same cause the same way.\n Effect.catch((cause): Effect.Effect<void, PlatformError.PlatformError> =>\n isReason(cause, \"NotFound\") ? Effect.void : Effect.fail(cause)\n )\n ),\n failure\n ).pipe(\n Effect.catch((cause) => Effect.logWarning(\"Artifact backup lease release failed\", cause))\n )\n : Effect.void\n )\n })\n\n/**\n * Runs one sweep deletion only when no live backup lease exists. The marker\n * check and `effect` share the global gate, closing the check/delete race.\n * `None` means a backup deliberately fenced the operation.\n *\n * @category combinators\n * @since 1.0.0-rc.0\n */\nexport const unlessActive = <A, E, R, E2>(\n fs: FileSystem.FileSystem,\n directory: string,\n effect: Effect.Effect<A, E, R>,\n failure: (cause: unknown) => E2\n): Effect.Effect<Option.Option<A>, E | E2, R> =>\n gate(\n fs,\n directory,\n Effect.gen(function*() {\n if (yield* activeMarker(fs, directory, failure)) return Option.none<A>()\n return Option.some(yield* effect)\n }),\n failure\n )\n"]}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The content-addressed artifact store: bytes addressed by their own SHA-256
|
|
3
|
+
* digest.
|
|
4
|
+
*
|
|
5
|
+
* It is deliberately *not* the step cache: the step cache maps a step key to a
|
|
6
|
+
* recorded result, while large result bytes live here under their digest. The
|
|
7
|
+
* two tiers remain separate because artifacts must be published before a cache
|
|
8
|
+
* record may reference them. See the package README and
|
|
9
|
+
* {@link https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/plan/docs/concepts/step-keys.md | step-key documentation}.
|
|
10
|
+
*
|
|
11
|
+
* This module defines the contract: the address schema, the service interface
|
|
12
|
+
* and tag, and one layer per local implementation. The errors, byte helpers,
|
|
13
|
+
* and implementations are defined in the modules named for them and re-exported
|
|
14
|
+
* here unchanged, so this subpath and the root `ArtifactStore` namespace keep
|
|
15
|
+
* their whole surface.
|
|
16
|
+
*
|
|
17
|
+
* @since 1.0.0-rc.0
|
|
18
|
+
*/
|
|
19
|
+
import { Sha256 } from "@smthrs/crypto";
|
|
20
|
+
import * as Context from "effect/Context";
|
|
21
|
+
import type * as Crypto from "effect/Crypto";
|
|
22
|
+
import * as Effect from "effect/Effect";
|
|
23
|
+
import * as FileSystem from "effect/FileSystem";
|
|
24
|
+
import * as Layer from "effect/Layer";
|
|
25
|
+
import * as Path from "effect/Path";
|
|
26
|
+
import type { ArtifactCorruption, ArtifactMissing, ArtifactStoreError } from "./ArtifactStoreError.ts";
|
|
27
|
+
import { type FileSystemOptions } from "./FileSystemArtifactStore.ts";
|
|
28
|
+
export { ArtifactCorruption, ArtifactMissing, ArtifactStoreError, ArtifactStoreErrorCode } from "./ArtifactStoreError.ts";
|
|
29
|
+
export { defaultDirectory, type FileSystemOptions, makeFileSystem } from "./FileSystemArtifactStore.ts";
|
|
30
|
+
export { measureBytes } from "./measureBytes.ts";
|
|
31
|
+
export { makeMemory } from "./MemoryArtifactStore.ts";
|
|
32
|
+
export { makeNoop } from "./NoopArtifactStore.ts";
|
|
33
|
+
export { snapshotBytes } from "./snapshotBytes.ts";
|
|
34
|
+
export { validateDigest } from "./validateDigest.ts";
|
|
35
|
+
/**
|
|
36
|
+
* Schema for a content address: exactly 64 lowercase hexadecimal SHA-256
|
|
37
|
+
* characters, branded by `@smthrs/crypto`. Re-exported so a consumer never has
|
|
38
|
+
* to reach past this package for the address type it stores under.
|
|
39
|
+
*
|
|
40
|
+
* @category schemas
|
|
41
|
+
* @since 1.0.0-rc.0
|
|
42
|
+
* @slop
|
|
43
|
+
*/
|
|
44
|
+
export declare const Digest: import("effect/Schema").brand<import("effect/Schema").String, "@smthrs/crypto/Sha256/Digest">;
|
|
45
|
+
/**
|
|
46
|
+
* A content address produced by this store.
|
|
47
|
+
*
|
|
48
|
+
* Read operations accept a plain `string` rather than this brand on purpose: a
|
|
49
|
+
* digest read back out of a durable row is untrusted input, so the store
|
|
50
|
+
* validates it (see {@link ArtifactStoreError}'s `invalid_digest` code) instead
|
|
51
|
+
* of asking every caller to re-brand a persisted column. `put` returns the
|
|
52
|
+
* brand, because it measured the bytes itself.
|
|
53
|
+
*
|
|
54
|
+
* @category models
|
|
55
|
+
* @since 1.0.0-rc.0
|
|
56
|
+
* @slop
|
|
57
|
+
*/
|
|
58
|
+
export type Digest = typeof Sha256.Digest.Type;
|
|
59
|
+
/**
|
|
60
|
+
* Content-addressed blob storage.
|
|
61
|
+
*
|
|
62
|
+
* The contract's ergonomics follow Effect's own `KeyValueStore`
|
|
63
|
+
* (`effect/persistence/KeyValueStore`): a small set of total
|
|
64
|
+
* operations over one address space, with a single typed error family, so a
|
|
65
|
+
* memory, filesystem, or network implementation is the same shape.
|
|
66
|
+
* `findMissing` is Bazel's `MissingDigestsFinder` — one batched round trip
|
|
67
|
+
* whose result is guaranteed to be a subset of its input — because a
|
|
68
|
+
* per-digest existence probe over a network tier is the wrong shape entirely.
|
|
69
|
+
*
|
|
70
|
+
* @category models
|
|
71
|
+
* @since 1.0.0-rc.0
|
|
72
|
+
* @slop
|
|
73
|
+
*/
|
|
74
|
+
export interface Service {
|
|
75
|
+
/**
|
|
76
|
+
* Stores `bytes` under their own SHA-256 digest and returns that address.
|
|
77
|
+
* Storing the same bytes twice is idempotent.
|
|
78
|
+
*/
|
|
79
|
+
readonly put: (bytes: Uint8Array) => Effect.Effect<Digest, ArtifactStoreError, Crypto.Crypto>;
|
|
80
|
+
/**
|
|
81
|
+
* Reads the bytes stored at `digest`, verifying that they still hash to it.
|
|
82
|
+
*/
|
|
83
|
+
readonly get: (digest: string) => Effect.Effect<Uint8Array, ArtifactMissing | ArtifactCorruption | ArtifactStoreError, Crypto.Crypto>;
|
|
84
|
+
/** Whether this tier holds an artifact at `digest`. */
|
|
85
|
+
readonly has: (digest: string) => Effect.Effect<boolean, ArtifactStoreError>;
|
|
86
|
+
/**
|
|
87
|
+
* Which of `digests` this tier does not hold. The returned array is
|
|
88
|
+
* guaranteed to be a subset of the input and free of duplicates.
|
|
89
|
+
*/
|
|
90
|
+
readonly findMissing: (digests: Iterable<string>) => Effect.Effect<Array<string>, ArtifactStoreError>;
|
|
91
|
+
}
|
|
92
|
+
declare const ArtifactStore_base: Context.ServiceClass<ArtifactStore, "@smthrs/artifacts/ArtifactStore", Service>;
|
|
93
|
+
/**
|
|
94
|
+
* Service tag for the content-addressed artifact store.
|
|
95
|
+
*
|
|
96
|
+
* The identity string equals this module's package path, per the house rule
|
|
97
|
+
* that an identity is the defining module path.
|
|
98
|
+
*
|
|
99
|
+
* @category services
|
|
100
|
+
* @since 1.0.0-rc.0
|
|
101
|
+
* @slop
|
|
102
|
+
*/
|
|
103
|
+
export declare class ArtifactStore extends ArtifactStore_base {
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Provides the filesystem-backed artifact store.
|
|
107
|
+
*
|
|
108
|
+
* @category layers
|
|
109
|
+
* @since 1.0.0-rc.0
|
|
110
|
+
* @slop
|
|
111
|
+
*/
|
|
112
|
+
export declare const layerFileSystem: (options?: FileSystemOptions) => Layer.Layer<ArtifactStore, never, FileSystem.FileSystem | Path.Path>;
|
|
113
|
+
/**
|
|
114
|
+
* Provides an in-memory artifact store.
|
|
115
|
+
*
|
|
116
|
+
* @category layers
|
|
117
|
+
* @since 1.0.0-rc.0
|
|
118
|
+
* @slop
|
|
119
|
+
*/
|
|
120
|
+
export declare const layerMemory: Layer.Layer<ArtifactStore>;
|
|
121
|
+
/**
|
|
122
|
+
* Provides a no-op artifact store.
|
|
123
|
+
*
|
|
124
|
+
* @category layers
|
|
125
|
+
* @since 1.0.0-rc.0
|
|
126
|
+
* @slop
|
|
127
|
+
*/
|
|
128
|
+
export declare const layerNoop: (overrides?: Partial<Service>) => Layer.Layer<ArtifactStore>;
|
|
129
|
+
//# sourceMappingURL=ArtifactStore.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ArtifactStore.d.ts","sourceRoot":"","sources":["../../src/ArtifactStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,KAAK,MAAM,MAAM,eAAe,CAAA;AAC5C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AACtG,OAAO,EAAE,KAAK,iBAAiB,EAAkB,MAAM,8BAA8B,CAAA;AAIrF,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AACvG,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAEpD;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM,+FAAgB,CAAA;AAEnC;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAA;AAE9C;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,OAAO;IACtB;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IAC7F;;OAEG;IACH,QAAQ,CAAC,GAAG,EAAE,CACZ,MAAM,EAAE,MAAM,KACX,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,GAAG,kBAAkB,GAAG,kBAAkB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IACxG,uDAAuD;IACvD,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAA;IAC5E;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,CACpB,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,KACtB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,kBAAkB,CAAC,CAAA;CACtD;;AAED;;;;;;;;;GASG;AACH,qBAAa,aAAc,SAAQ,kBAA4E;CAAG;AAElH;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAC1B,UAAS,iBAAsB,KAC9B,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAGjE,CAAA;AAEL;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,aAAa,CAAwD,CAAA;AAE3G;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,GAAI,YAAW,OAAO,CAAC,OAAO,CAAM,KAAG,KAAK,CAAC,KAAK,CAAC,aAAa,CACnC,CAAA"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The content-addressed artifact store: bytes addressed by their own SHA-256
|
|
3
|
+
* digest.
|
|
4
|
+
*
|
|
5
|
+
* It is deliberately *not* the step cache: the step cache maps a step key to a
|
|
6
|
+
* recorded result, while large result bytes live here under their digest. The
|
|
7
|
+
* two tiers remain separate because artifacts must be published before a cache
|
|
8
|
+
* record may reference them. See the package README and
|
|
9
|
+
* {@link https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/plan/docs/concepts/step-keys.md | step-key documentation}.
|
|
10
|
+
*
|
|
11
|
+
* This module defines the contract: the address schema, the service interface
|
|
12
|
+
* and tag, and one layer per local implementation. The errors, byte helpers,
|
|
13
|
+
* and implementations are defined in the modules named for them and re-exported
|
|
14
|
+
* here unchanged, so this subpath and the root `ArtifactStore` namespace keep
|
|
15
|
+
* their whole surface.
|
|
16
|
+
*
|
|
17
|
+
* @since 1.0.0-rc.0
|
|
18
|
+
*/
|
|
19
|
+
import { Sha256 } from "@smthrs/crypto";
|
|
20
|
+
import * as Context from "effect/Context";
|
|
21
|
+
import * as Effect from "effect/Effect";
|
|
22
|
+
import * as FileSystem from "effect/FileSystem";
|
|
23
|
+
import * as Layer from "effect/Layer";
|
|
24
|
+
import * as Path from "effect/Path";
|
|
25
|
+
import { makeFileSystem } from "./FileSystemArtifactStore.js";
|
|
26
|
+
import { makeMemory } from "./MemoryArtifactStore.js";
|
|
27
|
+
import { makeNoop } from "./NoopArtifactStore.js";
|
|
28
|
+
export { ArtifactCorruption, ArtifactMissing, ArtifactStoreError, ArtifactStoreErrorCode } from "./ArtifactStoreError.js";
|
|
29
|
+
export { defaultDirectory, makeFileSystem } from "./FileSystemArtifactStore.js";
|
|
30
|
+
export { measureBytes } from "./measureBytes.js";
|
|
31
|
+
export { makeMemory } from "./MemoryArtifactStore.js";
|
|
32
|
+
export { makeNoop } from "./NoopArtifactStore.js";
|
|
33
|
+
export { snapshotBytes } from "./snapshotBytes.js";
|
|
34
|
+
export { validateDigest } from "./validateDigest.js";
|
|
35
|
+
/**
|
|
36
|
+
* Schema for a content address: exactly 64 lowercase hexadecimal SHA-256
|
|
37
|
+
* characters, branded by `@smthrs/crypto`. Re-exported so a consumer never has
|
|
38
|
+
* to reach past this package for the address type it stores under.
|
|
39
|
+
*
|
|
40
|
+
* @category schemas
|
|
41
|
+
* @since 1.0.0-rc.0
|
|
42
|
+
* @slop
|
|
43
|
+
*/
|
|
44
|
+
export const Digest = Sha256.Digest;
|
|
45
|
+
/**
|
|
46
|
+
* Service tag for the content-addressed artifact store.
|
|
47
|
+
*
|
|
48
|
+
* The identity string equals this module's package path, per the house rule
|
|
49
|
+
* that an identity is the defining module path.
|
|
50
|
+
*
|
|
51
|
+
* @category services
|
|
52
|
+
* @since 1.0.0-rc.0
|
|
53
|
+
* @slop
|
|
54
|
+
*/
|
|
55
|
+
export class ArtifactStore extends Context.Service()("@smthrs/artifacts/ArtifactStore") {
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Provides the filesystem-backed artifact store.
|
|
59
|
+
*
|
|
60
|
+
* @category layers
|
|
61
|
+
* @since 1.0.0-rc.0
|
|
62
|
+
* @slop
|
|
63
|
+
*/
|
|
64
|
+
export const layerFileSystem = (options = {}) => Layer.effect(ArtifactStore)(Effect.gen(function* () {
|
|
65
|
+
return makeFileSystem(yield* FileSystem.FileSystem, yield* Path.Path, options);
|
|
66
|
+
}));
|
|
67
|
+
/**
|
|
68
|
+
* Provides an in-memory artifact store.
|
|
69
|
+
*
|
|
70
|
+
* @category layers
|
|
71
|
+
* @since 1.0.0-rc.0
|
|
72
|
+
* @slop
|
|
73
|
+
*/
|
|
74
|
+
export const layerMemory = Layer.effect(ArtifactStore)(Effect.sync(makeMemory));
|
|
75
|
+
/**
|
|
76
|
+
* Provides a no-op artifact store.
|
|
77
|
+
*
|
|
78
|
+
* @category layers
|
|
79
|
+
* @since 1.0.0-rc.0
|
|
80
|
+
* @slop
|
|
81
|
+
*/
|
|
82
|
+
export const layerNoop = (overrides = {}) => Layer.succeed(ArtifactStore)(makeNoop(overrides));
|
|
83
|
+
//# sourceMappingURL=ArtifactStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ArtifactStore.js","sourceRoot":"","sources":["../../src/ArtifactStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AACvC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AAEnC,OAAO,EAA0B,cAAc,EAAE,MAAM,8BAA8B,CAAA;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAEjD,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,sBAAsB,EACvB,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAA0B,cAAc,EAAE,MAAM,8BAA8B,CAAA;AACvG,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAEpD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;AAuDnC;;;;;;;;;GASG;AACH,MAAM,OAAO,aAAc,SAAQ,OAAO,CAAC,OAAO,EAA0B,CAAC,iCAAiC,CAAC;CAAG;AAElH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,UAA6B,EAAE,EACuC,EAAE,CACxE,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9C,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;AAChF,CAAC,CAAC,CAAC,CAAA;AAEL;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,WAAW,GAA+B,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAA;AAE3G;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,YAA8B,EAAE,EAA8B,EAAE,CACxF,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAA","sourcesContent":["/**\n * The content-addressed artifact store: bytes addressed by their own SHA-256\n * digest.\n *\n * It is deliberately *not* the step cache: the step cache maps a step key to a\n * recorded result, while large result bytes live here under their digest. The\n * two tiers remain separate because artifacts must be published before a cache\n * record may reference them. See the package README and\n * {@link https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/plan/docs/concepts/step-keys.md | step-key documentation}.\n *\n * This module defines the contract: the address schema, the service interface\n * and tag, and one layer per local implementation. The errors, byte helpers,\n * and implementations are defined in the modules named for them and re-exported\n * here unchanged, so this subpath and the root `ArtifactStore` namespace keep\n * their whole surface.\n *\n * @since 1.0.0-rc.0\n */\n\nimport { Sha256 } from \"@smthrs/crypto\"\nimport * as Context from \"effect/Context\"\nimport type * as Crypto from \"effect/Crypto\"\nimport * as Effect from \"effect/Effect\"\nimport * as FileSystem from \"effect/FileSystem\"\nimport * as Layer from \"effect/Layer\"\nimport * as Path from \"effect/Path\"\nimport type { ArtifactCorruption, ArtifactMissing, ArtifactStoreError } from \"./ArtifactStoreError.ts\"\nimport { type FileSystemOptions, makeFileSystem } from \"./FileSystemArtifactStore.ts\"\nimport { makeMemory } from \"./MemoryArtifactStore.ts\"\nimport { makeNoop } from \"./NoopArtifactStore.ts\"\n\nexport {\n ArtifactCorruption,\n ArtifactMissing,\n ArtifactStoreError,\n ArtifactStoreErrorCode\n} from \"./ArtifactStoreError.ts\"\nexport { defaultDirectory, type FileSystemOptions, makeFileSystem } from \"./FileSystemArtifactStore.ts\"\nexport { measureBytes } from \"./measureBytes.ts\"\nexport { makeMemory } from \"./MemoryArtifactStore.ts\"\nexport { makeNoop } from \"./NoopArtifactStore.ts\"\nexport { snapshotBytes } from \"./snapshotBytes.ts\"\nexport { validateDigest } from \"./validateDigest.ts\"\n\n/**\n * Schema for a content address: exactly 64 lowercase hexadecimal SHA-256\n * characters, branded by `@smthrs/crypto`. Re-exported so a consumer never has\n * to reach past this package for the address type it stores under.\n *\n * @category schemas\n * @since 1.0.0-rc.0\n * @slop\n */\nexport const Digest = Sha256.Digest\n\n/**\n * A content address produced by this store.\n *\n * Read operations accept a plain `string` rather than this brand on purpose: a\n * digest read back out of a durable row is untrusted input, so the store\n * validates it (see {@link ArtifactStoreError}'s `invalid_digest` code) instead\n * of asking every caller to re-brand a persisted column. `put` returns the\n * brand, because it measured the bytes itself.\n *\n * @category models\n * @since 1.0.0-rc.0\n * @slop\n */\nexport type Digest = typeof Sha256.Digest.Type\n\n/**\n * Content-addressed blob storage.\n *\n * The contract's ergonomics follow Effect's own `KeyValueStore`\n * (`effect/persistence/KeyValueStore`): a small set of total\n * operations over one address space, with a single typed error family, so a\n * memory, filesystem, or network implementation is the same shape.\n * `findMissing` is Bazel's `MissingDigestsFinder` — one batched round trip\n * whose result is guaranteed to be a subset of its input — because a\n * per-digest existence probe over a network tier is the wrong shape entirely.\n *\n * @category models\n * @since 1.0.0-rc.0\n * @slop\n */\nexport interface Service {\n /**\n * Stores `bytes` under their own SHA-256 digest and returns that address.\n * Storing the same bytes twice is idempotent.\n */\n readonly put: (bytes: Uint8Array) => Effect.Effect<Digest, ArtifactStoreError, Crypto.Crypto>\n /**\n * Reads the bytes stored at `digest`, verifying that they still hash to it.\n */\n readonly get: (\n digest: string\n ) => Effect.Effect<Uint8Array, ArtifactMissing | ArtifactCorruption | ArtifactStoreError, Crypto.Crypto>\n /** Whether this tier holds an artifact at `digest`. */\n readonly has: (digest: string) => Effect.Effect<boolean, ArtifactStoreError>\n /**\n * Which of `digests` this tier does not hold. The returned array is\n * guaranteed to be a subset of the input and free of duplicates.\n */\n readonly findMissing: (\n digests: Iterable<string>\n ) => Effect.Effect<Array<string>, ArtifactStoreError>\n}\n\n/**\n * Service tag for the content-addressed artifact store.\n *\n * The identity string equals this module's package path, per the house rule\n * that an identity is the defining module path.\n *\n * @category services\n * @since 1.0.0-rc.0\n * @slop\n */\nexport class ArtifactStore extends Context.Service<ArtifactStore, Service>()(\"@smthrs/artifacts/ArtifactStore\") {}\n\n/**\n * Provides the filesystem-backed artifact store.\n *\n * @category layers\n * @since 1.0.0-rc.0\n * @slop\n */\nexport const layerFileSystem = (\n options: FileSystemOptions = {}\n): Layer.Layer<ArtifactStore, never, FileSystem.FileSystem | Path.Path> =>\n Layer.effect(ArtifactStore)(Effect.gen(function*() {\n return makeFileSystem(yield* FileSystem.FileSystem, yield* Path.Path, options)\n }))\n\n/**\n * Provides an in-memory artifact store.\n *\n * @category layers\n * @since 1.0.0-rc.0\n * @slop\n */\nexport const layerMemory: Layer.Layer<ArtifactStore> = Layer.effect(ArtifactStore)(Effect.sync(makeMemory))\n\n/**\n * Provides a no-op artifact store.\n *\n * @category layers\n * @since 1.0.0-rc.0\n * @slop\n */\nexport const layerNoop = (overrides: Partial<Service> = {}): Layer.Layer<ArtifactStore> =>\n Layer.succeed(ArtifactStore)(makeNoop(overrides))\n"]}
|