@inflexa-ai/harness 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/reference-data/catalog.d.ts +20 -34
- package/dist/reference-data/catalog.js +259 -54
- package/dist/reference-data/receipt.d.ts +4 -12
- package/dist/reference-data/receipt.js +4 -5
- package/dist/sandbox/create-sandbox.d.ts +39 -1
- package/dist/sandbox/create-sandbox.js +39 -19
- package/dist/sandbox/docker-client.d.ts +15 -0
- package/dist/sandbox/docker-client.js +38 -14
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -103,7 +103,7 @@ export { queryStepsByRun } from "./state/step-executions.js";
|
|
|
103
103
|
export type { CortexRunRow, StepExecutionRow, RunStatus } from "./state/schema.js";
|
|
104
104
|
export type { DbError } from "./lib/db-result.js";
|
|
105
105
|
export { REFERENCE_DATA_CATALOG, REFERENCE_DATA_CATALOG_VERSION, ReferenceArtifactPathSchema, ReferenceArtifactSchema, ReferenceDataCatalogSchema, ReferenceDatasetSchema, ReferenceSha256Schema, UnknownReferenceDatasetError, referenceArtifactKey, resolveReferenceInstallPlan, } from "./reference-data/catalog.js";
|
|
106
|
-
export type { ReferenceArtifact, ReferenceDataCatalog, ReferenceDataset, ReferenceInstallPlan, ReferenceInstallPlanDataset,
|
|
106
|
+
export type { ReferenceArtifact, ReferenceDataCatalog, ReferenceDataset, ReferenceInstallPlan, ReferenceInstallPlanDataset, } from "./reference-data/catalog.js";
|
|
107
107
|
export { REFERENCE_INSTALL_RECEIPT_VERSION, ReferenceInstallReceiptSchema, parseReferenceInstallReceipt } from "./reference-data/receipt.js";
|
|
108
108
|
export type { ReferenceInstallReceipt, ReferenceReceiptArtifact } from "./reference-data/receipt.js";
|
|
109
109
|
export { queryActiveSandboxes } from "./state/active-sandboxes.js";
|
|
@@ -6,20 +6,20 @@ export declare const REFERENCE_DATA_CATALOG_VERSION: 1;
|
|
|
6
6
|
export declare const ReferenceArtifactPathSchema: z.ZodString;
|
|
7
7
|
/** Lowercase SHA-256 digest of one artifact's bytes. */
|
|
8
8
|
export declare const ReferenceSha256Schema: z.ZodString;
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
9
|
+
/**
|
|
10
|
+
* One final file distributed for a reference dataset: a stable install-relative
|
|
11
|
+
* path and the third-party https URL it is fetched from. The catalog carries no
|
|
12
|
+
* size or digest — every upstream is authenticated over TLS at download time,
|
|
13
|
+
* and the installer records the bytes it actually received in the receipt so
|
|
14
|
+
* `verify` can later prove the local copy has not changed since install. This is
|
|
15
|
+
* one uniform trust-on-first-use model; there is deliberately no per-artifact
|
|
16
|
+
* integrity class, so adding a source is only ever a URL, and no checked-in
|
|
17
|
+
* digest is ours to maintain or lets go stale when a `current` upstream rebuilds.
|
|
18
|
+
*/
|
|
19
|
+
export declare const ReferenceArtifactSchema: z.ZodObject<{
|
|
18
20
|
path: z.ZodString;
|
|
19
21
|
url: z.ZodURL;
|
|
20
|
-
}, z.core.$strict
|
|
21
|
-
/** Which integrity guarantee an artifact can actually offer. */
|
|
22
|
-
export type ReferenceIntegrity = z.infer<typeof ReferenceArtifactSchema>["integrity"];
|
|
22
|
+
}, z.core.$strict>;
|
|
23
23
|
/** Provenance and licensing information for one supported reference dataset. */
|
|
24
24
|
export declare const ReferenceDatasetSchema: z.ZodObject<{
|
|
25
25
|
id: z.ZodString;
|
|
@@ -35,17 +35,10 @@ export declare const ReferenceDatasetSchema: z.ZodObject<{
|
|
|
35
35
|
group: z.ZodString;
|
|
36
36
|
recommended: z.ZodBoolean;
|
|
37
37
|
}, z.core.$strip>;
|
|
38
|
-
artifacts: z.ZodArray<z.
|
|
39
|
-
integrity: z.ZodLiteral<"pinned">;
|
|
38
|
+
artifacts: z.ZodArray<z.ZodObject<{
|
|
40
39
|
path: z.ZodString;
|
|
41
40
|
url: z.ZodURL;
|
|
42
|
-
|
|
43
|
-
sha256: z.ZodString;
|
|
44
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
45
|
-
integrity: z.ZodLiteral<"unpinned">;
|
|
46
|
-
path: z.ZodString;
|
|
47
|
-
url: z.ZodURL;
|
|
48
|
-
}, z.core.$strict>], "integrity">>;
|
|
41
|
+
}, z.core.$strict>>;
|
|
49
42
|
}, z.core.$strip>;
|
|
50
43
|
/** Versioned canonical catalog consumed identically by every harness embedder. */
|
|
51
44
|
export declare const ReferenceDataCatalogSchema: z.ZodObject<{
|
|
@@ -64,17 +57,10 @@ export declare const ReferenceDataCatalogSchema: z.ZodObject<{
|
|
|
64
57
|
group: z.ZodString;
|
|
65
58
|
recommended: z.ZodBoolean;
|
|
66
59
|
}, z.core.$strip>;
|
|
67
|
-
artifacts: z.ZodArray<z.
|
|
68
|
-
integrity: z.ZodLiteral<"pinned">;
|
|
69
|
-
path: z.ZodString;
|
|
70
|
-
url: z.ZodURL;
|
|
71
|
-
bytes: z.ZodNumber;
|
|
72
|
-
sha256: z.ZodString;
|
|
73
|
-
}, z.core.$strict>, z.ZodObject<{
|
|
74
|
-
integrity: z.ZodLiteral<"unpinned">;
|
|
60
|
+
artifacts: z.ZodArray<z.ZodObject<{
|
|
75
61
|
path: z.ZodString;
|
|
76
62
|
url: z.ZodURL;
|
|
77
|
-
}, z.core.$strict
|
|
63
|
+
}, z.core.$strict>>;
|
|
78
64
|
}, z.core.$strip>>;
|
|
79
65
|
}, z.core.$strip>;
|
|
80
66
|
type DeepReadonly<T> = T extends (...args: never[]) => unknown ? T : T extends readonly (infer Item)[] ? readonly DeepReadonly<Item>[] : T extends object ? {
|
|
@@ -97,14 +83,14 @@ export declare function referenceArtifactKey(dataset: {
|
|
|
97
83
|
/**
|
|
98
84
|
* Canonical release catalog. Every artifact is fetched directly from the third
|
|
99
85
|
* party that publishes it; this project hosts, mirrors, and redistributes
|
|
100
|
-
* nothing. Entries are added
|
|
101
|
-
* licensing data —
|
|
102
|
-
*
|
|
86
|
+
* nothing. Entries are added with a real upstream https URL, provenance, and
|
|
87
|
+
* licensing data — and nothing else per file: no size, no digest to compute or
|
|
88
|
+
* keep in sync.
|
|
103
89
|
*
|
|
104
90
|
* `version` is the dataset's upstream release identifier. Datasets built on an
|
|
105
91
|
* upstream that has no immutable release — NCBI regenerates `gene_info` daily,
|
|
106
92
|
* Reactome overwrites `current` and deletes prior releases — are versioned
|
|
107
|
-
* `current
|
|
93
|
+
* `current`; those with an immutable release carry it verbatim.
|
|
108
94
|
*/
|
|
109
95
|
export declare const REFERENCE_DATA_CATALOG: ReferenceDataCatalog;
|
|
110
96
|
/** One selected dataset and its host-neutral installation location. */
|
|
@@ -22,34 +22,19 @@ export const ReferenceSha256Schema = z.string().regex(SHA256, "Expected a lowerc
|
|
|
22
22
|
*/
|
|
23
23
|
const ReferenceArtifactUrlSchema = z.url().refine((value) => value.startsWith("https://"), "Reference artifacts must be fetched over https");
|
|
24
24
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
25
|
+
* One final file distributed for a reference dataset: a stable install-relative
|
|
26
|
+
* path and the third-party https URL it is fetched from. The catalog carries no
|
|
27
|
+
* size or digest — every upstream is authenticated over TLS at download time,
|
|
28
|
+
* and the installer records the bytes it actually received in the receipt so
|
|
29
|
+
* `verify` can later prove the local copy has not changed since install. This is
|
|
30
|
+
* one uniform trust-on-first-use model; there is deliberately no per-artifact
|
|
31
|
+
* integrity class, so adding a source is only ever a URL, and no checked-in
|
|
32
|
+
* digest is ours to maintain or lets go stale when a `current` upstream rebuilds.
|
|
29
33
|
*/
|
|
30
|
-
const
|
|
31
|
-
integrity: z.literal("pinned"),
|
|
34
|
+
export const ReferenceArtifactSchema = z.strictObject({
|
|
32
35
|
path: ReferenceArtifactPathSchema,
|
|
33
36
|
url: ReferenceArtifactUrlSchema,
|
|
34
|
-
bytes: z.number().int().positive(),
|
|
35
|
-
sha256: ReferenceSha256Schema,
|
|
36
37
|
});
|
|
37
|
-
/**
|
|
38
|
-
* An artifact whose upstream regenerates the same URL in place — NCBI rebuilds
|
|
39
|
-
* `gene_info` continuously and Reactome's `current` release is overwritten and
|
|
40
|
-
* its predecessors deleted. No checked-in digest can survive that, and pinning
|
|
41
|
-
* one would only guarantee a broken download. Integrity is therefore
|
|
42
|
-
* trust-on-first-use: the installer records the bytes it actually received in
|
|
43
|
-
* the receipt, and `verify` proves the files have not changed *since install*.
|
|
44
|
-
* This is a weaker guarantee than `pinned` and is surfaced as such to the user.
|
|
45
|
-
*/
|
|
46
|
-
const UnpinnedReferenceArtifactSchema = z.strictObject({
|
|
47
|
-
integrity: z.literal("unpinned"),
|
|
48
|
-
path: ReferenceArtifactPathSchema,
|
|
49
|
-
url: ReferenceArtifactUrlSchema,
|
|
50
|
-
});
|
|
51
|
-
/** One immutable final file distributed for a reference dataset. */
|
|
52
|
-
export const ReferenceArtifactSchema = z.discriminatedUnion("integrity", [PinnedReferenceArtifactSchema, UnpinnedReferenceArtifactSchema]);
|
|
53
38
|
/** Provenance and licensing information for one supported reference dataset. */
|
|
54
39
|
export const ReferenceDatasetSchema = z.object({
|
|
55
40
|
id: z.string().regex(SAFE_ID, "Expected a stable lowercase dataset id"),
|
|
@@ -112,14 +97,14 @@ function deepFreeze(value) {
|
|
|
112
97
|
/**
|
|
113
98
|
* Canonical release catalog. Every artifact is fetched directly from the third
|
|
114
99
|
* party that publishes it; this project hosts, mirrors, and redistributes
|
|
115
|
-
* nothing. Entries are added
|
|
116
|
-
* licensing data —
|
|
117
|
-
*
|
|
100
|
+
* nothing. Entries are added with a real upstream https URL, provenance, and
|
|
101
|
+
* licensing data — and nothing else per file: no size, no digest to compute or
|
|
102
|
+
* keep in sync.
|
|
118
103
|
*
|
|
119
104
|
* `version` is the dataset's upstream release identifier. Datasets built on an
|
|
120
105
|
* upstream that has no immutable release — NCBI regenerates `gene_info` daily,
|
|
121
106
|
* Reactome overwrites `current` and deletes prior releases — are versioned
|
|
122
|
-
* `current
|
|
107
|
+
* `current`; those with an immutable release carry it verbatim.
|
|
123
108
|
*/
|
|
124
109
|
export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.parse({
|
|
125
110
|
version: REFERENCE_DATA_CATALOG_VERSION,
|
|
@@ -137,7 +122,6 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
137
122
|
recommendation: { group: "gene-identifiers", recommended: true },
|
|
138
123
|
artifacts: [
|
|
139
124
|
{
|
|
140
|
-
integrity: "unpinned",
|
|
141
125
|
path: "Homo_sapiens.gene_info.gz",
|
|
142
126
|
url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz",
|
|
143
127
|
},
|
|
@@ -156,7 +140,6 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
156
140
|
recommendation: { group: "gene-identifiers", recommended: true },
|
|
157
141
|
artifacts: [
|
|
158
142
|
{
|
|
159
|
-
integrity: "unpinned",
|
|
160
143
|
path: "Mus_musculus.gene_info.gz",
|
|
161
144
|
url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Mus_musculus.gene_info.gz",
|
|
162
145
|
},
|
|
@@ -175,12 +158,85 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
175
158
|
recommendation: { group: "gene-identifiers", recommended: true },
|
|
176
159
|
artifacts: [
|
|
177
160
|
{
|
|
178
|
-
integrity: "unpinned",
|
|
179
161
|
path: "Rattus_norvegicus.gene_info.gz",
|
|
180
162
|
url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Rattus_norvegicus.gene_info.gz",
|
|
181
163
|
},
|
|
182
164
|
],
|
|
183
165
|
},
|
|
166
|
+
{
|
|
167
|
+
// All-species monolithic table (~274 MB), not the per-organism split the gene_info entries use:
|
|
168
|
+
// NCBI publishes Entrez↔Ensembl only in this one combined file. Opt-in for that reason.
|
|
169
|
+
id: "ncbi-gene2ensembl",
|
|
170
|
+
version: "current",
|
|
171
|
+
title: "NCBI Entrez-to-Ensembl gene mapping (all species)",
|
|
172
|
+
description: "NCBI Gene's Entrez Gene ID ↔ Ensembl gene/transcript/protein identifier cross-reference for every species (gene2ensembl), tab-separated and gzipped — roughly 274 MB. NCBI rebuilds this file in place, so it is verified against what you downloaded rather than a checked-in digest.",
|
|
173
|
+
sourceUrl: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/",
|
|
174
|
+
license: {
|
|
175
|
+
identifier: "NCBI-Molecular-Data-Usage-Policy",
|
|
176
|
+
url: "https://www.ncbi.nlm.nih.gov/home/about/policies/",
|
|
177
|
+
},
|
|
178
|
+
recommendation: { group: "gene-identifiers", recommended: false },
|
|
179
|
+
artifacts: [{ path: "gene2ensembl.gz", url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2ensembl.gz" }],
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
// ~2.2 GB all-species monolithic table — the largest catalog entry. Opt-in like reactome-mappings.
|
|
183
|
+
id: "ncbi-gene2refseq",
|
|
184
|
+
version: "current",
|
|
185
|
+
title: "NCBI Entrez-to-RefSeq accession mapping (all species, large)",
|
|
186
|
+
description: "NCBI Gene's Entrez Gene ID ↔ RefSeq RNA/protein/genomic accession cross-reference for every species (gene2refseq), tab-separated and gzipped — roughly 2.2 GB. NCBI rebuilds this file in place, so it is verified against what you downloaded rather than a checked-in digest. Only needed to join RefSeq accessions onto genes.",
|
|
187
|
+
sourceUrl: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/",
|
|
188
|
+
license: {
|
|
189
|
+
identifier: "NCBI-Molecular-Data-Usage-Policy",
|
|
190
|
+
url: "https://www.ncbi.nlm.nih.gov/home/about/policies/",
|
|
191
|
+
},
|
|
192
|
+
recommendation: { group: "gene-identifiers", recommended: false },
|
|
193
|
+
artifacts: [{ path: "gene2refseq.gz", url: "https://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2refseq.gz" }],
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
id: "uniprot-idmapping-human",
|
|
197
|
+
version: "current",
|
|
198
|
+
title: "UniProt human ID mapping",
|
|
199
|
+
description: "UniProtKB accession ↔ Entrez, Ensembl, RefSeq and other identifiers for Homo sapiens (idmapping_selected, ~61 MB gzipped). UniProt overwrites `current_release` each cycle, so it is verified against what you downloaded rather than a checked-in digest.",
|
|
200
|
+
sourceUrl: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/",
|
|
201
|
+
license: { identifier: "CC-BY-4.0", url: "https://www.uniprot.org/help/license" },
|
|
202
|
+
recommendation: { group: "gene-identifiers", recommended: true },
|
|
203
|
+
artifacts: [
|
|
204
|
+
{
|
|
205
|
+
path: "HUMAN_9606_idmapping_selected.tab.gz",
|
|
206
|
+
url: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/HUMAN_9606_idmapping_selected.tab.gz",
|
|
207
|
+
},
|
|
208
|
+
],
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
id: "uniprot-idmapping-mouse",
|
|
212
|
+
version: "current",
|
|
213
|
+
title: "UniProt mouse ID mapping",
|
|
214
|
+
description: "UniProtKB accession ↔ Entrez, Ensembl, RefSeq and other identifiers for Mus musculus (idmapping_selected, ~18 MB gzipped). UniProt overwrites `current_release` each cycle, so it is verified against what you downloaded rather than a checked-in digest.",
|
|
215
|
+
sourceUrl: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/",
|
|
216
|
+
license: { identifier: "CC-BY-4.0", url: "https://www.uniprot.org/help/license" },
|
|
217
|
+
recommendation: { group: "gene-identifiers", recommended: true },
|
|
218
|
+
artifacts: [
|
|
219
|
+
{
|
|
220
|
+
path: "MOUSE_10090_idmapping_selected.tab.gz",
|
|
221
|
+
url: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/MOUSE_10090_idmapping_selected.tab.gz",
|
|
222
|
+
},
|
|
223
|
+
],
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: "uniprot-idmapping-rat",
|
|
227
|
+
version: "current",
|
|
228
|
+
title: "UniProt rat ID mapping",
|
|
229
|
+
description: "UniProtKB accession ↔ Entrez, Ensembl, RefSeq and other identifiers for Rattus norvegicus (idmapping_selected, ~6 MB gzipped). UniProt overwrites `current_release` each cycle, so it is verified against what you downloaded rather than a checked-in digest.",
|
|
230
|
+
sourceUrl: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/",
|
|
231
|
+
license: { identifier: "CC-BY-4.0", url: "https://www.uniprot.org/help/license" },
|
|
232
|
+
recommendation: { group: "gene-identifiers", recommended: true },
|
|
233
|
+
artifacts: [
|
|
234
|
+
{
|
|
235
|
+
path: "RAT_10116_idmapping_selected.tab.gz",
|
|
236
|
+
url: "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/RAT_10116_idmapping_selected.tab.gz",
|
|
237
|
+
},
|
|
238
|
+
],
|
|
239
|
+
},
|
|
184
240
|
{
|
|
185
241
|
id: "reactome-pathways",
|
|
186
242
|
version: "current",
|
|
@@ -190,8 +246,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
190
246
|
license: { identifier: "CC0-1.0", url: "https://reactome.org/license" },
|
|
191
247
|
recommendation: { group: "pathways", recommended: true },
|
|
192
248
|
artifacts: [
|
|
193
|
-
{
|
|
194
|
-
{
|
|
249
|
+
{ path: "ReactomePathways.gmt", url: "https://reactome.org/download/current/ReactomePathways.gmt" },
|
|
250
|
+
{ path: "ReactomePathways.txt", url: "https://reactome.org/download/current/ReactomePathways.txt" },
|
|
195
251
|
],
|
|
196
252
|
},
|
|
197
253
|
{
|
|
@@ -208,17 +264,14 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
208
264
|
recommendation: { group: "pathways", recommended: false },
|
|
209
265
|
artifacts: [
|
|
210
266
|
{
|
|
211
|
-
integrity: "unpinned",
|
|
212
267
|
path: "NCBI2Reactome_All_Levels.txt",
|
|
213
268
|
url: "https://reactome.org/download/current/NCBI2Reactome_All_Levels.txt",
|
|
214
269
|
},
|
|
215
270
|
{
|
|
216
|
-
integrity: "unpinned",
|
|
217
271
|
path: "Ensembl2Reactome_All_Levels.txt",
|
|
218
272
|
url: "https://reactome.org/download/current/Ensembl2Reactome_All_Levels.txt",
|
|
219
273
|
},
|
|
220
274
|
{
|
|
221
|
-
integrity: "unpinned",
|
|
222
275
|
path: "UniProt2Reactome_All_Levels.txt",
|
|
223
276
|
url: "https://reactome.org/download/current/UniProt2Reactome_All_Levels.txt",
|
|
224
277
|
},
|
|
@@ -234,11 +287,68 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
234
287
|
recommendation: { group: "pathways", recommended: true },
|
|
235
288
|
artifacts: [
|
|
236
289
|
{
|
|
237
|
-
integrity: "pinned",
|
|
238
290
|
path: "wikipathways_Homo_sapiens.gmt",
|
|
239
291
|
url: "https://data.wikipathways.org/20260710/gmt/wikipathways-20260710-gmt-Homo_sapiens.gmt",
|
|
240
|
-
|
|
241
|
-
|
|
292
|
+
},
|
|
293
|
+
],
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
id: "wikipathways-mouse",
|
|
297
|
+
version: "2026.07.10",
|
|
298
|
+
title: "WikiPathways mouse pathways",
|
|
299
|
+
description: "Community-curated Mus musculus pathway gene sets (GMT) from the immutable 2026-07-10 WikiPathways snapshot.",
|
|
300
|
+
sourceUrl: "https://data.wikipathways.org/20260710/gmt/",
|
|
301
|
+
license: { identifier: "CC0-1.0", url: "https://www.wikipathways.org/about/terms.html" },
|
|
302
|
+
recommendation: { group: "pathways", recommended: true },
|
|
303
|
+
artifacts: [
|
|
304
|
+
{
|
|
305
|
+
path: "wikipathways_Mus_musculus.gmt",
|
|
306
|
+
url: "https://data.wikipathways.org/20260710/gmt/wikipathways-20260710-gmt-Mus_musculus.gmt",
|
|
307
|
+
},
|
|
308
|
+
],
|
|
309
|
+
},
|
|
310
|
+
{
|
|
311
|
+
id: "wikipathways-rat",
|
|
312
|
+
version: "2026.07.10",
|
|
313
|
+
title: "WikiPathways rat pathways",
|
|
314
|
+
description: "Community-curated Rattus norvegicus pathway gene sets (GMT) from the immutable 2026-07-10 WikiPathways snapshot.",
|
|
315
|
+
sourceUrl: "https://data.wikipathways.org/20260710/gmt/",
|
|
316
|
+
license: { identifier: "CC0-1.0", url: "https://www.wikipathways.org/about/terms.html" },
|
|
317
|
+
recommendation: { group: "pathways", recommended: true },
|
|
318
|
+
artifacts: [
|
|
319
|
+
{
|
|
320
|
+
path: "wikipathways_Rattus_norvegicus.gmt",
|
|
321
|
+
url: "https://data.wikipathways.org/20260710/gmt/wikipathways-20260710-gmt-Rattus_norvegicus.gmt",
|
|
322
|
+
},
|
|
323
|
+
],
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
id: "msigdb-hallmark-human",
|
|
327
|
+
version: "2026.1",
|
|
328
|
+
title: "MSigDB human hallmark gene sets",
|
|
329
|
+
description: "The 50 MSigDB hallmark gene sets for Homo sapiens (symbols GMT) — coherent, well-defined biological states and processes for enrichment. From the immutable MSigDB 2026.1 human release.",
|
|
330
|
+
sourceUrl: "https://data.broadinstitute.org/gsea-msigdb/msigdb/",
|
|
331
|
+
license: { identifier: "MSigDB-License", url: "https://www.gsea-msigdb.org/gsea/msigdb/license.jsp" },
|
|
332
|
+
recommendation: { group: "pathways", recommended: true },
|
|
333
|
+
artifacts: [
|
|
334
|
+
{
|
|
335
|
+
path: "h.all.v2026.1.Hs.symbols.gmt",
|
|
336
|
+
url: "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2026.1.Hs/h.all.v2026.1.Hs.symbols.gmt",
|
|
337
|
+
},
|
|
338
|
+
],
|
|
339
|
+
},
|
|
340
|
+
{
|
|
341
|
+
id: "msigdb-hallmark-mouse",
|
|
342
|
+
version: "2026.1",
|
|
343
|
+
title: "MSigDB mouse hallmark gene sets",
|
|
344
|
+
description: "The MSigDB mouse hallmark gene sets for Mus musculus (symbols GMT) — the murine ortholog-mapped hallmark collection for enrichment. From the immutable MSigDB 2026.1 mouse release.",
|
|
345
|
+
sourceUrl: "https://data.broadinstitute.org/gsea-msigdb/msigdb/",
|
|
346
|
+
license: { identifier: "MSigDB-License", url: "https://www.gsea-msigdb.org/gsea/msigdb/license.jsp" },
|
|
347
|
+
recommendation: { group: "pathways", recommended: true },
|
|
348
|
+
artifacts: [
|
|
349
|
+
{
|
|
350
|
+
path: "mh.all.v2026.1.Mm.symbols.gmt",
|
|
351
|
+
url: "https://data.broadinstitute.org/gsea-msigdb/msigdb/release/2026.1.Mm/mh.all.v2026.1.Mm.symbols.gmt",
|
|
242
352
|
},
|
|
243
353
|
],
|
|
244
354
|
},
|
|
@@ -252,11 +362,8 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
252
362
|
recommendation: { group: "regulatory-networks", recommended: true },
|
|
253
363
|
artifacts: [
|
|
254
364
|
{
|
|
255
|
-
integrity: "pinned",
|
|
256
365
|
path: "CollecTRI_regulons.csv",
|
|
257
366
|
url: "https://zenodo.org/records/8192729/files/CollecTRI_regulons.csv",
|
|
258
|
-
bytes: 4_345_649,
|
|
259
|
-
sha256: "4473c9189dd53dacc80297709ad1452dda1086a1cc2185f9a56146c261668701",
|
|
260
367
|
},
|
|
261
368
|
],
|
|
262
369
|
},
|
|
@@ -270,21 +377,25 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
270
377
|
recommendation: { group: "normal-expression", recommended: true },
|
|
271
378
|
artifacts: [
|
|
272
379
|
{
|
|
273
|
-
integrity: "pinned",
|
|
274
380
|
path: "GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz",
|
|
275
381
|
url: "https://storage.googleapis.com/adult-gtex/bulk-gex/v8/rna-seq/GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz",
|
|
276
|
-
bytes: 6_952_331,
|
|
277
|
-
sha256: "ee7201ff2f280b0de5657d4b08e9a240362d9757efed7f7bd5dba35a5f8617b8",
|
|
278
382
|
},
|
|
279
383
|
{
|
|
280
|
-
integrity: "pinned",
|
|
281
384
|
path: "GTEx_Analysis_v8_Annotations_SampleAttributesDS.txt",
|
|
282
385
|
url: "https://storage.googleapis.com/adult-gtex/annotations/v8/metadata-files/GTEx_Analysis_v8_Annotations_SampleAttributesDS.txt",
|
|
283
|
-
bytes: 11_512_258,
|
|
284
|
-
sha256: "74f6ab4c34ed2648d708a0ae6e6dff324f6c86ea723ae7d1c37d76f5221148f0",
|
|
285
386
|
},
|
|
286
387
|
],
|
|
287
388
|
},
|
|
389
|
+
{
|
|
390
|
+
id: "hpa-proteinatlas",
|
|
391
|
+
version: "current",
|
|
392
|
+
title: "Human Protein Atlas",
|
|
393
|
+
description: "The full Human Protein Atlas table (proteinatlas.tsv.zip, ~7 MB): per-gene RNA/protein tissue expression, tissue specificity, subcellular location, secretome and blood-concentration annotations for Homo sapiens. HPA republishes this file per release, so it is verified against what you downloaded rather than a checked-in digest.",
|
|
394
|
+
sourceUrl: "https://www.proteinatlas.org/about/download",
|
|
395
|
+
license: { identifier: "CC-BY-SA-3.0", url: "https://www.proteinatlas.org/about/licence" },
|
|
396
|
+
recommendation: { group: "normal-expression", recommended: true },
|
|
397
|
+
artifacts: [{ path: "proteinatlas.tsv.zip", url: "https://www.proteinatlas.org/download/proteinatlas.tsv.zip" }],
|
|
398
|
+
},
|
|
288
399
|
{
|
|
289
400
|
id: "celltypist-immune",
|
|
290
401
|
version: "2",
|
|
@@ -295,18 +406,112 @@ export const REFERENCE_DATA_CATALOG = deepFreeze(ReferenceDataCatalogSchema.pars
|
|
|
295
406
|
recommendation: { group: "cell-typing", recommended: true },
|
|
296
407
|
artifacts: [
|
|
297
408
|
{
|
|
298
|
-
integrity: "pinned",
|
|
299
409
|
path: "Immune_All_Low.pkl",
|
|
300
410
|
url: "https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_Low.pkl",
|
|
301
|
-
bytes: 2_824_990,
|
|
302
|
-
sha256: "290874d35dac039d4c9218c343fde4aac1077709b72a331ce7266f6828c36502",
|
|
303
411
|
},
|
|
304
412
|
{
|
|
305
|
-
integrity: "pinned",
|
|
306
413
|
path: "Immune_All_High.pkl",
|
|
307
414
|
url: "https://celltypist.cog.sanger.ac.uk/models/Pan_Immune_CellTypist/v2/Immune_All_High.pkl",
|
|
308
|
-
|
|
309
|
-
|
|
415
|
+
},
|
|
416
|
+
],
|
|
417
|
+
},
|
|
418
|
+
{
|
|
419
|
+
id: "celltypist-pan-fetal",
|
|
420
|
+
version: "2",
|
|
421
|
+
title: "CellTypist pan-fetal human model",
|
|
422
|
+
description: "CellTypist Pan_Fetal_Human model — cell-type annotation across human fetal tissues. Load by absolute path; CellTypist's by-name lookup expects its own cache layout, which the reference store deliberately does not impersonate.",
|
|
423
|
+
sourceUrl: "https://www.celltypist.org/models",
|
|
424
|
+
license: { identifier: "NOASSERTION" },
|
|
425
|
+
recommendation: { group: "cell-typing", recommended: true },
|
|
426
|
+
artifacts: [
|
|
427
|
+
{
|
|
428
|
+
path: "Pan_Fetal_Human.pkl",
|
|
429
|
+
url: "https://celltypist.cog.sanger.ac.uk/models/Pan_Fetal_Suo/v2/Pan_Fetal_Human.pkl",
|
|
430
|
+
},
|
|
431
|
+
],
|
|
432
|
+
},
|
|
433
|
+
{
|
|
434
|
+
id: "celltypist-covid19",
|
|
435
|
+
version: "1",
|
|
436
|
+
title: "CellTypist COVID-19 immune model",
|
|
437
|
+
description: "CellTypist COVID19_Immune_Landscape model — immune cell-type annotation trained on the cross-study COVID-19 immune atlas. Load by absolute path; CellTypist's by-name lookup expects its own cache layout, which the reference store deliberately does not impersonate.",
|
|
438
|
+
sourceUrl: "https://www.celltypist.org/models",
|
|
439
|
+
license: { identifier: "NOASSERTION" },
|
|
440
|
+
recommendation: { group: "cell-typing", recommended: true },
|
|
441
|
+
artifacts: [
|
|
442
|
+
{
|
|
443
|
+
path: "COVID19_Immune_Landscape.pkl",
|
|
444
|
+
url: "https://celltypist.cog.sanger.ac.uk/models/COVID19_Immune_Ren/v1/COVID19_Immune_Landscape.pkl",
|
|
445
|
+
},
|
|
446
|
+
],
|
|
447
|
+
},
|
|
448
|
+
{
|
|
449
|
+
id: "panglaodb-markers",
|
|
450
|
+
version: "2020.03.27",
|
|
451
|
+
title: "PanglaoDB cell-type markers",
|
|
452
|
+
description: "PanglaoDB cell-type marker panel (tab-separated, gzipped): curated marker genes per cell type across human and mouse, for marker-based annotation. From the immutable 27 March 2020 PanglaoDB marker release.",
|
|
453
|
+
sourceUrl: "https://panglaodb.se/markers.html",
|
|
454
|
+
license: { identifier: "NOASSERTION" },
|
|
455
|
+
recommendation: { group: "cell-typing", recommended: true },
|
|
456
|
+
artifacts: [
|
|
457
|
+
{
|
|
458
|
+
path: "PanglaoDB_markers_27_Mar_2020.tsv.gz",
|
|
459
|
+
url: "https://panglaodb.se/markers/PanglaoDB_markers_27_Mar_2020.tsv.gz",
|
|
460
|
+
},
|
|
461
|
+
],
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
id: "azimuth-pbmc",
|
|
465
|
+
version: "1.0.0",
|
|
466
|
+
title: "Azimuth human PBMC reference",
|
|
467
|
+
description: "Azimuth annotated reference for human peripheral blood mononuclear cells: the Seurat reference object plus its Annoy nearest-neighbour index, for supervised mapping and cell-type label transfer. Immutable Zenodo release 4546839.",
|
|
468
|
+
sourceUrl: "https://zenodo.org/records/4546839",
|
|
469
|
+
license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/4546839" },
|
|
470
|
+
recommendation: { group: "cell-typing", recommended: true },
|
|
471
|
+
artifacts: [
|
|
472
|
+
{
|
|
473
|
+
path: "ref.Rds",
|
|
474
|
+
url: "https://zenodo.org/records/4546839/files/ref.Rds",
|
|
475
|
+
},
|
|
476
|
+
{
|
|
477
|
+
path: "idx.annoy",
|
|
478
|
+
url: "https://zenodo.org/records/4546839/files/idx.annoy",
|
|
479
|
+
},
|
|
480
|
+
],
|
|
481
|
+
},
|
|
482
|
+
{
|
|
483
|
+
// ~443 MB (ref + index), so it is opt-in rather than part of a default setup.
|
|
484
|
+
id: "azimuth-tonsil",
|
|
485
|
+
version: "1.0.0",
|
|
486
|
+
title: "Azimuth human tonsil reference",
|
|
487
|
+
description: "Azimuth annotated reference for human tonsil: the Seurat reference object plus its Annoy nearest-neighbour index, for supervised mapping and cell-type label transfer. Immutable Zenodo release 7032928, roughly 443 MB.",
|
|
488
|
+
sourceUrl: "https://zenodo.org/records/7032928",
|
|
489
|
+
license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/7032928" },
|
|
490
|
+
recommendation: { group: "cell-typing", recommended: false },
|
|
491
|
+
artifacts: [
|
|
492
|
+
{
|
|
493
|
+
path: "ref.Rds",
|
|
494
|
+
url: "https://zenodo.org/records/7032928/files/ref.Rds",
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
path: "idx.annoy",
|
|
498
|
+
url: "https://zenodo.org/records/7032928/files/idx.annoy",
|
|
499
|
+
},
|
|
500
|
+
],
|
|
501
|
+
},
|
|
502
|
+
{
|
|
503
|
+
// ~842 MB reference genome bundle, PGx-specific, so it is opt-in rather than a default download.
|
|
504
|
+
id: "pharmcat-grch38-fasta",
|
|
505
|
+
version: "GRCh38.p13",
|
|
506
|
+
title: "PharmCAT GRCh38 reference FASTA",
|
|
507
|
+
description: "The GRCh38.p13 reference genome FASTA that PharmCAT's VCF preprocessor expects (bgzip'd with a .fai/.gzi faidx), bundled as a tar — roughly 842 MB. Immutable Zenodo release 7288118. Needed only for pharmacogenomics (PharmCAT) workflows.",
|
|
508
|
+
sourceUrl: "https://zenodo.org/records/7288118",
|
|
509
|
+
license: { identifier: "CC-BY-4.0", url: "https://zenodo.org/records/7288118" },
|
|
510
|
+
recommendation: { group: "pharmacogenomics", recommended: false },
|
|
511
|
+
artifacts: [
|
|
512
|
+
{
|
|
513
|
+
path: "PharmCAT_GRCh38_reference_fasta.tar",
|
|
514
|
+
url: "https://zenodo.org/records/7288118/files/GRCh38_reference_fasta.tar?download=1",
|
|
310
515
|
},
|
|
311
516
|
],
|
|
312
517
|
},
|
|
@@ -3,19 +3,15 @@ import { z } from "zod";
|
|
|
3
3
|
export declare const REFERENCE_INSTALL_RECEIPT_VERSION: 1;
|
|
4
4
|
/**
|
|
5
5
|
* What was actually written to disk for one artifact. The size and digest are
|
|
6
|
-
* *observed at install
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
6
|
+
* *observed at install* — the catalog carries neither, so this receipt is the
|
|
7
|
+
* sole record of the bytes the upstream served, and what `verify` later checks
|
|
8
|
+
* the local files against. A receipt written by an older build may still carry
|
|
9
|
+
* an `integrity` field; it is simply ignored (unknown keys are stripped).
|
|
10
10
|
*/
|
|
11
11
|
declare const ReferenceReceiptArtifactSchema: z.ZodObject<{
|
|
12
12
|
path: z.ZodString;
|
|
13
13
|
bytes: z.ZodNumber;
|
|
14
14
|
sha256: z.ZodString;
|
|
15
|
-
integrity: z.ZodEnum<{
|
|
16
|
-
pinned: "pinned";
|
|
17
|
-
unpinned: "unpinned";
|
|
18
|
-
}>;
|
|
19
15
|
}, z.core.$strip>;
|
|
20
16
|
/** Optional metadata describing one activated reference dataset version. */
|
|
21
17
|
export declare const ReferenceInstallReceiptSchema: z.ZodObject<{
|
|
@@ -27,10 +23,6 @@ export declare const ReferenceInstallReceiptSchema: z.ZodObject<{
|
|
|
27
23
|
path: z.ZodString;
|
|
28
24
|
bytes: z.ZodNumber;
|
|
29
25
|
sha256: z.ZodString;
|
|
30
|
-
integrity: z.ZodEnum<{
|
|
31
|
-
pinned: "pinned";
|
|
32
|
-
unpinned: "unpinned";
|
|
33
|
-
}>;
|
|
34
26
|
}, z.core.$strip>>;
|
|
35
27
|
}, z.core.$strip>;
|
|
36
28
|
export type ReferenceInstallReceipt = z.infer<typeof ReferenceInstallReceiptSchema>;
|
|
@@ -4,16 +4,15 @@ import { ReferenceArtifactPathSchema, ReferenceDatasetSchema, ReferenceSha256Sch
|
|
|
4
4
|
export const REFERENCE_INSTALL_RECEIPT_VERSION = 1;
|
|
5
5
|
/**
|
|
6
6
|
* What was actually written to disk for one artifact. The size and digest are
|
|
7
|
-
* *observed at install
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* *observed at install* — the catalog carries neither, so this receipt is the
|
|
8
|
+
* sole record of the bytes the upstream served, and what `verify` later checks
|
|
9
|
+
* the local files against. A receipt written by an older build may still carry
|
|
10
|
+
* an `integrity` field; it is simply ignored (unknown keys are stripped).
|
|
11
11
|
*/
|
|
12
12
|
const ReferenceReceiptArtifactSchema = z.object({
|
|
13
13
|
path: ReferenceArtifactPathSchema,
|
|
14
14
|
bytes: z.number().int().nonnegative(),
|
|
15
15
|
sha256: ReferenceSha256Schema,
|
|
16
|
-
integrity: z.enum(["pinned", "unpinned"]),
|
|
17
16
|
});
|
|
18
17
|
/** Optional metadata describing one activated reference dataset version. */
|
|
19
18
|
export const ReferenceInstallReceiptSchema = z
|
|
@@ -20,7 +20,7 @@ import { clearSandboxRef } from "../state/index.js";
|
|
|
20
20
|
import { type AwaitExecOptions } from "./await-exec.js";
|
|
21
21
|
import type { SandboxClient } from "./client.js";
|
|
22
22
|
import { type SubmitExecDeps } from "./submit-exec.js";
|
|
23
|
-
import { type SandboxTransport } from "./types.js";
|
|
23
|
+
import { type CreateSandboxMeta, type SandboxTransport } from "./types.js";
|
|
24
24
|
/**
|
|
25
25
|
* Narrow config slice the sandbox factory reads off `Env` — the backend
|
|
26
26
|
* default and the K8s namespace fallback. Composition roots map their
|
|
@@ -62,6 +62,25 @@ export interface CreateSandboxClientConfig {
|
|
|
62
62
|
refStorePath?: string;
|
|
63
63
|
/** Docker: force the container platform (e.g. `linux/amd64`) so the sandbox matches the mounted lib store's arch. */
|
|
64
64
|
platform?: string;
|
|
65
|
+
/**
|
|
66
|
+
* Docker: unix socket of the Docker-API engine to dial — a Docker daemon
|
|
67
|
+
* socket or a podman Docker-compat socket. Unset preserves dockerode's
|
|
68
|
+
* default resolution (`DOCKER_HOST`, then the default Docker socket).
|
|
69
|
+
*/
|
|
70
|
+
engineSocketPath?: string;
|
|
71
|
+
/**
|
|
72
|
+
* Docker: the engine's bind mounts expose honest host file ownership to
|
|
73
|
+
* the container — podman machine's virtiofs and native Linux binds present
|
|
74
|
+
* the embedder user's real uid and modes, which the uid-1000 sandbox
|
|
75
|
+
* workload cannot write; Docker Desktop's file-sharing layer masks that
|
|
76
|
+
* mismatch, so it stays unset there. When set, the client loosens each
|
|
77
|
+
* step's pre-created write tree so workload writes land. The field names
|
|
78
|
+
* the engine fact rather than the remediation, so how the harness
|
|
79
|
+
* compensates can evolve without changing the embedder surface; a
|
|
80
|
+
* single-valued union (not a boolean) so future ownership classes extend
|
|
81
|
+
* it without a breaking change.
|
|
82
|
+
*/
|
|
83
|
+
engineBindOwnership?: "host-preserved";
|
|
65
84
|
/** K8s: PVC claim backing the session PVC the workspace roots live under. */
|
|
66
85
|
sessionPvc?: string;
|
|
67
86
|
/**
|
|
@@ -100,6 +119,25 @@ export interface CreateSandboxClientConfig {
|
|
|
100
119
|
* Exported for tests.
|
|
101
120
|
*/
|
|
102
121
|
export declare function composeAwaitOptions(base: AwaitExecOptions | undefined, transport: SandboxTransport, isAlive: NonNullable<AwaitExecOptions["isAlive"]>): AwaitExecOptions;
|
|
122
|
+
/**
|
|
123
|
+
* Backend-agnostic pre-creation of the writable step tree under the analysis's
|
|
124
|
+
* resolved workspace root. On Docker that root is the host dir the container
|
|
125
|
+
* binds; on K8s it lives under the session PVC the sandbox pod mounts via
|
|
126
|
+
* subPath. `mkdir(recursive)` is idempotent, so an existing tree (replay, retry)
|
|
127
|
+
* is not an error. A read-only sandbox has no writable step mount, so there is
|
|
128
|
+
* nothing to pre-create.
|
|
129
|
+
*
|
|
130
|
+
* `stepTreeAccess: "world-writable"` chmods the step dir and each subdir so the
|
|
131
|
+
* uid-1000 sandbox workload can write them through engines that honor host bind
|
|
132
|
+
* ownership. The chmod is explicit, not `mkdir`'s `mode` option: the process
|
|
133
|
+
* umask masks `mkdir`'s mode, and on replay the dirs already exist so `mkdir`
|
|
134
|
+
* would not touch them either way. Scoped to the step write tree — the
|
|
135
|
+
* read-only mount sources are never re-moded. Exported for tests.
|
|
136
|
+
*/
|
|
137
|
+
export declare function precreateStepTree(deps: {
|
|
138
|
+
resolveWorkspaceRoot: ResolveWorkspaceRoot;
|
|
139
|
+
stepTreeAccess?: "world-writable";
|
|
140
|
+
}, meta: CreateSandboxMeta): Promise<void>;
|
|
103
141
|
export declare function createSandboxClient(config: CreateSandboxClientConfig): SandboxClient;
|
|
104
142
|
/** Re-exports so consumers import once from this module. */
|
|
105
143
|
export { clearSandboxRef };
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* coupling (they just call the closures we hand them). `submitExec` and
|
|
12
12
|
* `awaitExec` are backend-agnostic — they only need the SandboxRef.
|
|
13
13
|
*/
|
|
14
|
-
import { mkdir } from "node:fs/promises";
|
|
14
|
+
import { chmod, mkdir } from "node:fs/promises";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { clampResources } from "../config/resource-limits.js";
|
|
17
17
|
import { stepWritePrefix } from "../workspace/paths.js";
|
|
@@ -34,6 +34,36 @@ import { toPersistedRef } from "./types.js";
|
|
|
34
34
|
export function composeAwaitOptions(base, transport, isAlive) {
|
|
35
35
|
return { isAlive, ...base, transport };
|
|
36
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Backend-agnostic pre-creation of the writable step tree under the analysis's
|
|
39
|
+
* resolved workspace root. On Docker that root is the host dir the container
|
|
40
|
+
* binds; on K8s it lives under the session PVC the sandbox pod mounts via
|
|
41
|
+
* subPath. `mkdir(recursive)` is idempotent, so an existing tree (replay, retry)
|
|
42
|
+
* is not an error. A read-only sandbox has no writable step mount, so there is
|
|
43
|
+
* nothing to pre-create.
|
|
44
|
+
*
|
|
45
|
+
* `stepTreeAccess: "world-writable"` chmods the step dir and each subdir so the
|
|
46
|
+
* uid-1000 sandbox workload can write them through engines that honor host bind
|
|
47
|
+
* ownership. The chmod is explicit, not `mkdir`'s `mode` option: the process
|
|
48
|
+
* umask masks `mkdir`'s mode, and on replay the dirs already exist so `mkdir`
|
|
49
|
+
* would not touch them either way. Scoped to the step write tree — the
|
|
50
|
+
* read-only mount sources are never re-moded. Exported for tests.
|
|
51
|
+
*/
|
|
52
|
+
export async function precreateStepTree(deps, meta) {
|
|
53
|
+
if (meta.readOnly)
|
|
54
|
+
return;
|
|
55
|
+
const stepDir = stepWritePrefix({
|
|
56
|
+
workspaceRoot: deps.resolveWorkspaceRoot(meta.analysisId),
|
|
57
|
+
runId: meta.runId,
|
|
58
|
+
stepId: meta.stepId,
|
|
59
|
+
});
|
|
60
|
+
await mkdir(stepDir, { recursive: true });
|
|
61
|
+
await Promise.all(STEP_SUBDIRS.map((sub) => mkdir(join(stepDir, sub), { recursive: true })));
|
|
62
|
+
if (deps.stepTreeAccess === "world-writable") {
|
|
63
|
+
await chmod(stepDir, 0o777);
|
|
64
|
+
await Promise.all(STEP_SUBDIRS.map((sub) => chmod(join(stepDir, sub), 0o777)));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
37
67
|
export function createSandboxClient(config) {
|
|
38
68
|
const backend = config.backend ?? config.env.backend;
|
|
39
69
|
const transport = config.transport ?? "poll";
|
|
@@ -64,26 +94,10 @@ export function createSandboxClient(config) {
|
|
|
64
94
|
libStorePath: config.libStorePath,
|
|
65
95
|
refStorePath: config.refStorePath,
|
|
66
96
|
platform: config.platform,
|
|
97
|
+
engineSocketPath: config.engineSocketPath,
|
|
67
98
|
logger: config.logger,
|
|
68
99
|
registerSandbox,
|
|
69
100
|
});
|
|
70
|
-
// Backend-agnostic pre-creation of the writable step tree under the
|
|
71
|
-
// analysis's resolved workspace root. On Docker that root is the host dir
|
|
72
|
-
// the container binds; on K8s it lives under the session PVC the sandbox
|
|
73
|
-
// pod mounts via subPath. mkdir(recursive) is idempotent, so an existing
|
|
74
|
-
// tree (replay, retry) is not an error.
|
|
75
|
-
const precreateStepTree = async (meta) => {
|
|
76
|
-
// A read-only sandbox has no writable step mount — nothing to pre-create.
|
|
77
|
-
if (meta.readOnly)
|
|
78
|
-
return;
|
|
79
|
-
const stepDir = stepWritePrefix({
|
|
80
|
-
workspaceRoot: config.resolveWorkspaceRoot(meta.analysisId),
|
|
81
|
-
runId: meta.runId,
|
|
82
|
-
stepId: meta.stepId,
|
|
83
|
-
});
|
|
84
|
-
await mkdir(stepDir, { recursive: true });
|
|
85
|
-
await Promise.all(STEP_SUBDIRS.map((sub) => mkdir(join(stepDir, sub), { recursive: true })));
|
|
86
|
-
};
|
|
87
101
|
// `teardown` needs the registry-clear closure but the SandboxRef alone
|
|
88
102
|
// doesn't carry runId/stepId. Wrap the backend impl so callers pass the
|
|
89
103
|
// ref + step coordinates and the wrapper does the clear after the
|
|
@@ -109,7 +123,13 @@ export function createSandboxClient(config) {
|
|
|
109
123
|
const isAlive = async (ref) => unwrapOrThrow(await ops.isAlive(ref));
|
|
110
124
|
return {
|
|
111
125
|
createSandbox: async (meta, identity) => {
|
|
112
|
-
|
|
126
|
+
// The config states an engine fact (binds preserve host ownership);
|
|
127
|
+
// which remediation that fact demands — today, a world-writable step
|
|
128
|
+
// write tree — is harness-owned and chosen here, not by the embedder.
|
|
129
|
+
await precreateStepTree({
|
|
130
|
+
resolveWorkspaceRoot: config.resolveWorkspaceRoot,
|
|
131
|
+
stepTreeAccess: config.engineBindOwnership === "host-preserved" ? "world-writable" : undefined,
|
|
132
|
+
}, meta);
|
|
113
133
|
// Every caller must declare resources — a sandbox with no cpu/memory
|
|
114
134
|
// request is a semantic error, not something to paper over with a
|
|
115
135
|
// default (a DBOS replay of a pre-resources workflow input lands here).
|
|
@@ -53,6 +53,13 @@ export interface DockerClientConfig {
|
|
|
53
53
|
* create rather than crashing inside the sandbox.
|
|
54
54
|
*/
|
|
55
55
|
platform?: string;
|
|
56
|
+
/**
|
|
57
|
+
* Unix socket of the Docker-API engine to dial — a Docker daemon socket or a
|
|
58
|
+
* podman Docker-compat socket (both serve the identical REST API). Unset
|
|
59
|
+
* preserves dockerode's default resolution (`DOCKER_HOST`, then the default
|
|
60
|
+
* Docker socket).
|
|
61
|
+
*/
|
|
62
|
+
engineSocketPath?: string;
|
|
56
63
|
/** Injected for tests. */
|
|
57
64
|
docker?: Docker;
|
|
58
65
|
/** Injected for tests so `/health` polling can be stubbed. */
|
|
@@ -66,6 +73,14 @@ export interface DockerClientConfig {
|
|
|
66
73
|
/** Hook called after the registry row is written. */
|
|
67
74
|
registerSandbox: (meta: CreateSandboxMeta, ref: SandboxRef) => Promise<void>;
|
|
68
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* dockerode connection options for a configured engine socket, or `undefined`
|
|
78
|
+
* to leave construction as a bare `new Docker()` — which preserves dockerode's
|
|
79
|
+
* default resolution (`DOCKER_HOST`, then the default Docker socket). A set
|
|
80
|
+
* `engineSocketPath` dials that unix socket instead (a Docker daemon or a
|
|
81
|
+
* podman Docker-compat socket).
|
|
82
|
+
*/
|
|
83
|
+
export declare function engineConnectionOptions(engineSocketPath: string | undefined): Docker.DockerOptions | undefined;
|
|
69
84
|
export declare function createDockerSandboxOps(config: DockerClientConfig): {
|
|
70
85
|
createSandbox(meta: CreateSandboxMeta, identity: SandboxIdentity): ResultAsync<SandboxRef, SandboxError>;
|
|
71
86
|
teardown(ref: SandboxRef): ResultAsync<void, SandboxError>;
|
|
@@ -97,36 +97,58 @@ async function pollHealth(fetchImpl, url, timeoutMs) {
|
|
|
97
97
|
}
|
|
98
98
|
throw new Error(`sandbox /health did not return 200 within ${timeoutMs}ms: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
99
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* dockerode connection options for a configured engine socket, or `undefined`
|
|
102
|
+
* to leave construction as a bare `new Docker()` — which preserves dockerode's
|
|
103
|
+
* default resolution (`DOCKER_HOST`, then the default Docker socket). A set
|
|
104
|
+
* `engineSocketPath` dials that unix socket instead (a Docker daemon or a
|
|
105
|
+
* podman Docker-compat socket).
|
|
106
|
+
*/
|
|
107
|
+
export function engineConnectionOptions(engineSocketPath) {
|
|
108
|
+
return engineSocketPath === undefined ? undefined : { socketPath: engineSocketPath };
|
|
109
|
+
}
|
|
100
110
|
/**
|
|
101
111
|
* Create the container, or adopt the one already standing under this
|
|
102
|
-
* checkpointed name
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
* name
|
|
112
|
+
* checkpointed name on a recovery re-run. Any create failure triggers an
|
|
113
|
+
* inspect of the name: engines disagree on the duplicate-name status (Docker
|
|
114
|
+
* answers 409, podman's compat API 500), so the reconcile keys on whether a
|
|
115
|
+
* container stands under the name, not on a status code. A standing container
|
|
116
|
+
* this workflow owns is adopted as-is if running, removed and recreated if
|
|
117
|
+
* stopped; one owned by a different workflow is a refused name collision — never
|
|
118
|
+
* adopt or remove someone else's container. When no container stands, the
|
|
119
|
+
* failure was not a name collision and the original create error is returned;
|
|
120
|
+
* the inspect's own failure is never surfaced, so a transient engine outage
|
|
121
|
+
* (both calls fail) still reports the create failure.
|
|
106
122
|
*/
|
|
107
|
-
async function createOrAdopt(docker, createOpts, name, ownerWorkflowId, createFailed) {
|
|
123
|
+
async function createOrAdopt(docker, createOpts, name, ownerWorkflowId, createFailed, logger) {
|
|
108
124
|
const created = await trySandbox(() => docker.createContainer(createOpts), createFailed);
|
|
109
125
|
if (created.isOk())
|
|
110
126
|
return ok({ container: created.value, alreadyRunning: false });
|
|
111
|
-
if (statusOf(created.error) !== 409)
|
|
112
|
-
return err(created.error);
|
|
113
127
|
const existing = docker.getContainer(name);
|
|
114
128
|
const inspected = await trySandbox(() => existing.inspect(), createFailed);
|
|
129
|
+
// No container stands under the name → the create failure was not a name
|
|
130
|
+
// collision, so surface the original create error rather than the inspect's.
|
|
115
131
|
if (inspected.isErr())
|
|
116
|
-
return err(
|
|
132
|
+
return err(created.error);
|
|
117
133
|
const info = inspected.value;
|
|
118
134
|
const owner = info.Config?.Labels?.[OWNER_WORKFLOW_LABEL];
|
|
119
135
|
if (owner !== ownerWorkflowId) {
|
|
120
136
|
return err({ type: "name_conflict", op: "docker.createSandbox", sandboxId: name, owner: owner ?? null });
|
|
121
137
|
}
|
|
122
|
-
|
|
138
|
+
// Both mutating reconcile outcomes are logged: adoption reuses a container the
|
|
139
|
+
// current attempt did not create, so without a record here nothing ties the
|
|
140
|
+
// step's sandbox back to the prior attempt that made it.
|
|
141
|
+
if (info.State.Running) {
|
|
142
|
+
logger.info("adopted standing container on recovery re-run", { sandboxId: name, ownerWorkflowId });
|
|
123
143
|
return ok({ container: existing, alreadyRunning: true });
|
|
144
|
+
}
|
|
124
145
|
const removed = await removeContainerIgnoreMissing(docker, name);
|
|
125
146
|
if (removed.isErr())
|
|
126
147
|
return err(removed.error);
|
|
127
148
|
const recreated = await trySandbox(() => docker.createContainer(createOpts), createFailed);
|
|
128
149
|
if (recreated.isErr())
|
|
129
150
|
return err(recreated.error);
|
|
151
|
+
logger.info("removed stopped prior attempt and recreated", { sandboxId: name, ownerWorkflowId });
|
|
130
152
|
return ok({ container: recreated.value, alreadyRunning: false });
|
|
131
153
|
}
|
|
132
154
|
function removeContainerIgnoreMissing(docker, sandboxId) {
|
|
@@ -145,7 +167,11 @@ function removeContainerIgnoreMissing(docker, sandboxId) {
|
|
|
145
167
|
statusOf(e) === 404 ? ok(undefined) : err(e));
|
|
146
168
|
}
|
|
147
169
|
export function createDockerSandboxOps(config) {
|
|
148
|
-
|
|
170
|
+
// The injected test instance wins; otherwise a configured socket dials that
|
|
171
|
+
// engine and an unset one stays a bare `new Docker()` (default resolution).
|
|
172
|
+
const connection = engineConnectionOptions(config.engineSocketPath);
|
|
173
|
+
const docker = config.docker ?? (connection ? new Docker(connection) : new Docker());
|
|
174
|
+
const logger = (config.logger ?? createNoopLogger()).named("docker-client");
|
|
149
175
|
const fetchImpl = config.fetch ?? fetch;
|
|
150
176
|
const transport = config.transport ?? "poll";
|
|
151
177
|
const pollMode = transport === "poll";
|
|
@@ -159,9 +185,7 @@ export function createDockerSandboxOps(config) {
|
|
|
159
185
|
// since an otherwise-silent libs-mount drop is invisible to operators.
|
|
160
186
|
const libsMounted = !!config.libStorePath && libStoreUsable(config.libStorePath);
|
|
161
187
|
if (config.libStorePath && !libsMounted) {
|
|
162
|
-
(config.
|
|
163
|
-
.named("docker-client")
|
|
164
|
-
.warn("lib store configured but `current` is missing or incomplete at sandbox creation — mounting no library store (sandbox degrades to available:false)", { libStorePath: config.libStorePath, sandboxId });
|
|
188
|
+
logger.warn("lib store configured but `current` is missing or incomplete at sandbox creation — mounting no library store (sandbox degrades to available:false)", { libStorePath: config.libStorePath, sandboxId });
|
|
165
189
|
}
|
|
166
190
|
const plan = buildMountPlan(meta, {
|
|
167
191
|
libs: libsMounted,
|
|
@@ -234,7 +258,7 @@ export function createDockerSandboxOps(config) {
|
|
|
234
258
|
AutoRemove: false,
|
|
235
259
|
},
|
|
236
260
|
};
|
|
237
|
-
const sandbox = await createOrAdopt(docker, createOpts, sandboxId, meta.childWorkflowId, createFailed);
|
|
261
|
+
const sandbox = await createOrAdopt(docker, createOpts, sandboxId, meta.childWorkflowId, createFailed, logger);
|
|
238
262
|
if (sandbox.isErr())
|
|
239
263
|
return err(sandbox.error);
|
|
240
264
|
if (!sandbox.value.alreadyRunning) {
|
package/package.json
CHANGED