@extantape/onephylogeny-engine 0.0.0-stage → 0.1.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +75 -0
- package/README.md +8 -2
- package/dist/index.d.ts +441 -0
- package/dist/index.js +1327 -0
- package/package.json +46 -3
- package/schemas/dataset.schema.json +77 -0
- package/schemas/examples.schema.json +165 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1327 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
import {
|
|
3
|
+
ALIGNER,
|
|
4
|
+
alignToReference as alignToReference2,
|
|
5
|
+
LOCATE_FLANK as LOCATE_FLANK2,
|
|
6
|
+
locateMarker as locateMarker2,
|
|
7
|
+
assessAlignment as assessAlignment2,
|
|
8
|
+
buildPanel,
|
|
9
|
+
canonicalJson as canonicalJson4,
|
|
10
|
+
checkNumts,
|
|
11
|
+
collapseHaplotypes,
|
|
12
|
+
compareRuns,
|
|
13
|
+
comparePanels,
|
|
14
|
+
DEFAULT_GATES as DEFAULT_GATES2,
|
|
15
|
+
extractLocation,
|
|
16
|
+
findGeneFeatures,
|
|
17
|
+
fullValidation,
|
|
18
|
+
gatesForRecord,
|
|
19
|
+
parseFasta as parseFasta2,
|
|
20
|
+
parseGenBank,
|
|
21
|
+
evaluateQuery,
|
|
22
|
+
sampleQc,
|
|
23
|
+
toRecordSheet,
|
|
24
|
+
toSequenceRecord,
|
|
25
|
+
validatePanel,
|
|
26
|
+
verifyBundle,
|
|
27
|
+
validatedScopeFromReport,
|
|
28
|
+
validateProfile,
|
|
29
|
+
validateResult,
|
|
30
|
+
writeFasta,
|
|
31
|
+
geneChecks,
|
|
32
|
+
geneCharsets,
|
|
33
|
+
geneWindows,
|
|
34
|
+
partitionNexus,
|
|
35
|
+
rotateToReference
|
|
36
|
+
} from "@extantape/onephylogeny-core";
|
|
37
|
+
import { classifyAccessions, genomesHoldingMarkerQuery, linkedAccessions, NcbiClient, selectionQuery, studiesFromRecords } from "@extantape/onephylogeny-acquire";
|
|
38
|
+
|
|
39
|
+
// src/artifacts.ts
|
|
40
|
+
import { canonicalJson, sha256Hex } from "@extantape/onephylogeny-core";
|
|
41
|
+
|
|
42
|
+
// src/errors.ts
|
|
43
|
+
var EngineError = class extends Error {
|
|
44
|
+
constructor(code, message, action) {
|
|
45
|
+
super(message);
|
|
46
|
+
this.code = code;
|
|
47
|
+
this.action = action;
|
|
48
|
+
this.name = "EngineError";
|
|
49
|
+
}
|
|
50
|
+
code;
|
|
51
|
+
action;
|
|
52
|
+
};
|
|
53
|
+
var isCancelled = (e) => e instanceof EngineError && e.code === "cancelled";
|
|
54
|
+
|
|
55
|
+
// src/artifacts.ts
|
|
56
|
+
var hashText = (text) => `sha256:${sha256Hex(text)}`;
|
|
57
|
+
var artifactDir = ({ type, key }) => `artifacts/${type}/${key.replace("sha256:", "")}`;
|
|
58
|
+
function stepKey(spec) {
|
|
59
|
+
return hashText(canonicalJson({ type: spec.type, step: spec.step, inputs: spec.inputs, parameters: spec.parameters, tools: spec.tools }));
|
|
60
|
+
}
|
|
61
|
+
async function readManifest(store, ref2) {
|
|
62
|
+
const path = `${artifactDir(ref2)}/manifest.json`;
|
|
63
|
+
if (!await store.exists(path)) return null;
|
|
64
|
+
let manifest;
|
|
65
|
+
try {
|
|
66
|
+
manifest = JSON.parse(await store.read(path));
|
|
67
|
+
} catch {
|
|
68
|
+
throw new EngineError("artifact-corrupt", `The record of ${ref2.type} ${ref2.key} is damaged`);
|
|
69
|
+
}
|
|
70
|
+
if (manifest.type !== ref2.type) throw new EngineError("artifact-wrong-type", `Expected ${ref2.type}, but this artifact is ${manifest.type}`);
|
|
71
|
+
return manifest;
|
|
72
|
+
}
|
|
73
|
+
async function verifyFiles(store, manifest) {
|
|
74
|
+
for (const file of manifest.files) {
|
|
75
|
+
const path = `${artifactDir(manifest)}/${file.name}`;
|
|
76
|
+
if (!await store.exists(path)) return false;
|
|
77
|
+
if (hashText(await store.read(path)) !== file.sha256) return false;
|
|
78
|
+
}
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
var open = (store, manifest) => ({
|
|
82
|
+
manifest,
|
|
83
|
+
read: async (name) => {
|
|
84
|
+
if (!manifest.files.some((f) => f.name === name)) throw new EngineError("artifact-missing", `${manifest.type} has no file ${name}`);
|
|
85
|
+
return store.read(`${artifactDir(manifest)}/${name}`);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
async function loadArtifact(store, ref2) {
|
|
89
|
+
const manifest = await readManifest(store, ref2);
|
|
90
|
+
if (!manifest) throw new EngineError("artifact-missing", `${ref2.type} ${ref2.key} doesn't exist`, "Run the earlier step again");
|
|
91
|
+
if (!await verifyFiles(store, manifest)) throw new EngineError("artifact-corrupt", `Files of ${ref2.type} ${ref2.key} have changed or gone missing`, "Run the step again to rebuild them");
|
|
92
|
+
return open(store, manifest);
|
|
93
|
+
}
|
|
94
|
+
var openVerified = open;
|
|
95
|
+
async function traceArtifact(store, ref2) {
|
|
96
|
+
const manifest = await readManifest(store, ref2);
|
|
97
|
+
if (!manifest) throw new EngineError("artifact-missing", `${ref2.type} ${ref2.key} doesn't exist`);
|
|
98
|
+
const inputs = [];
|
|
99
|
+
for (const input of manifest.inputs) inputs.push(await traceArtifact(store, input));
|
|
100
|
+
return { type: manifest.type, key: manifest.key, step: `${manifest.step.name}@${manifest.step.version}`, parameters: manifest.parameters, tools: manifest.tools, createdAt: manifest.createdAt, inputs };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// src/run.ts
|
|
104
|
+
import { AcquireError } from "@extantape/onephylogeny-acquire";
|
|
105
|
+
var ENGINE_VERSION = "0.1.0-alpha.8";
|
|
106
|
+
var ENGINE = { name: "onephylogeny-engine", version: ENGINE_VERSION };
|
|
107
|
+
var profilePaths = (profileId) => ({
|
|
108
|
+
profile: `profiles/${profileId}/profile.json`,
|
|
109
|
+
curation: `profiles/${profileId}/curation.json`,
|
|
110
|
+
latest: `profiles/${profileId}/latest-run.txt`,
|
|
111
|
+
run: (id) => `profiles/${profileId}/runs/${id}`,
|
|
112
|
+
panel: (id, version) => `profiles/${profileId}/panels/${id}-${version}`,
|
|
113
|
+
results: `profiles/${profileId}/results`
|
|
114
|
+
});
|
|
115
|
+
var json = (v) => `${JSON.stringify(v, null, 2)}
|
|
116
|
+
`;
|
|
117
|
+
var stamp = (ctx) => (ctx.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
118
|
+
var RunLog = class {
|
|
119
|
+
constructor(ctx, file) {
|
|
120
|
+
this.ctx = ctx;
|
|
121
|
+
this.file = file;
|
|
122
|
+
}
|
|
123
|
+
ctx;
|
|
124
|
+
file;
|
|
125
|
+
events = [];
|
|
126
|
+
pending = Promise.resolve();
|
|
127
|
+
add(level, step, message, data) {
|
|
128
|
+
const event = { at: stamp(this.ctx), level, step, message, ...data ? { data } : {} };
|
|
129
|
+
this.events.push(event);
|
|
130
|
+
this.ctx.onEvent?.(event);
|
|
131
|
+
const { store } = this.ctx;
|
|
132
|
+
const write = store.append ? () => store.append(this.file, `${JSON.stringify(event)}
|
|
133
|
+
`) : () => store.write(this.file, `${this.events.map((e) => JSON.stringify(e)).join("\n")}
|
|
134
|
+
`);
|
|
135
|
+
this.pending = this.pending.then(write).catch(() => {
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
flush() {
|
|
139
|
+
return this.pending;
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
async function runStep(scope, spec) {
|
|
143
|
+
const { store } = scope.ctx;
|
|
144
|
+
const tools = spec.tools ?? [];
|
|
145
|
+
const key = stepKey({ type: spec.type, step: spec.step, inputs: spec.inputs, parameters: spec.parameters, tools });
|
|
146
|
+
const ref2 = { type: spec.type, key };
|
|
147
|
+
scope.checkpoint();
|
|
148
|
+
const existing = await readManifest(store, ref2);
|
|
149
|
+
if (existing && await verifyFiles(store, existing)) {
|
|
150
|
+
scope.log("info", spec.step.name, "Reused earlier output: inputs and settings are unchanged", { status: "reused", key });
|
|
151
|
+
scope.run.artifacts.push({ ...ref2, reused: true });
|
|
152
|
+
return openVerified(store, existing);
|
|
153
|
+
}
|
|
154
|
+
const dir = artifactDir(ref2);
|
|
155
|
+
scope.log("info", spec.step.name, "Running", { status: "running", key });
|
|
156
|
+
const files = [];
|
|
157
|
+
const put = async (name, text) => {
|
|
158
|
+
await store.write(`${dir}/${name}`, text);
|
|
159
|
+
files.push({ name, sha256: hashText(text) });
|
|
160
|
+
};
|
|
161
|
+
const register = async (name) => {
|
|
162
|
+
files.push({ name, sha256: hashText(await store.read(`${dir}/${name}`)) });
|
|
163
|
+
};
|
|
164
|
+
await spec.run(put, dir, register);
|
|
165
|
+
scope.checkpoint();
|
|
166
|
+
if (spec.ensure) {
|
|
167
|
+
const problem = await spec.ensure((name) => store.read(`${dir}/${name}`));
|
|
168
|
+
if (problem) throw new EngineError("step-output-invalid", `${spec.step.name} produced invalid output: ${problem}`);
|
|
169
|
+
}
|
|
170
|
+
const manifest = { schemaVersion: 1, type: spec.type, key, step: spec.step, inputs: spec.inputs, parameters: spec.parameters, tools, files, createdAt: stamp(scope.ctx) };
|
|
171
|
+
await store.write(`${dir}/manifest.json`, json(manifest));
|
|
172
|
+
scope.log("info", spec.step.name, "Finished", { status: "finished", key });
|
|
173
|
+
scope.run.artifacts.push({ ...ref2, reused: false });
|
|
174
|
+
return openVerified(store, manifest);
|
|
175
|
+
}
|
|
176
|
+
var newRunId = (ctx, kind) => `${stamp(ctx).replace(/[:.]/g, "-")}-${kind}`;
|
|
177
|
+
var toEngineError = (e) => {
|
|
178
|
+
if (e instanceof AcquireError) return new EngineError("source-failed", e.message, "Check your connection and try again. Anything already downloaded is kept");
|
|
179
|
+
return e;
|
|
180
|
+
};
|
|
181
|
+
async function withRun(ctx, profile, kind, work) {
|
|
182
|
+
const P = profilePaths(profile.id);
|
|
183
|
+
const id = newRunId(ctx, kind);
|
|
184
|
+
const dir = P.run(id);
|
|
185
|
+
const run = { schemaVersion: 1, id, kind, profile, artifacts: [], tools: [ENGINE], status: "running", startedAt: stamp(ctx), events: [], outputs: [], diagnostics: [] };
|
|
186
|
+
await ctx.store.write(`${dir}/run.json`, json(run));
|
|
187
|
+
const log = new RunLog(ctx, `${dir}/events.jsonl`);
|
|
188
|
+
const checkpoint = () => {
|
|
189
|
+
if (ctx.signal?.aborted) throw new EngineError("cancelled", "Cancelled");
|
|
190
|
+
};
|
|
191
|
+
let lastBreath = Date.now();
|
|
192
|
+
const scope = {
|
|
193
|
+
ctx,
|
|
194
|
+
run,
|
|
195
|
+
dir,
|
|
196
|
+
log: (level, step, message, data) => log.add(level, step, message, data),
|
|
197
|
+
checkpoint,
|
|
198
|
+
breathe: async () => {
|
|
199
|
+
checkpoint();
|
|
200
|
+
if (Date.now() - lastBreath > 30) {
|
|
201
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
202
|
+
lastBreath = Date.now();
|
|
203
|
+
checkpoint();
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
step: (spec) => runStep(scope, spec)
|
|
207
|
+
};
|
|
208
|
+
const finish = async (status) => {
|
|
209
|
+
await log.flush();
|
|
210
|
+
run.status = status;
|
|
211
|
+
run.finishedAt = stamp(ctx);
|
|
212
|
+
run.events = log.events;
|
|
213
|
+
await ctx.store.write(`${dir}/run.json`, json(run));
|
|
214
|
+
};
|
|
215
|
+
try {
|
|
216
|
+
const value = await work(scope);
|
|
217
|
+
await finish("succeeded");
|
|
218
|
+
return { value, run };
|
|
219
|
+
} catch (raw) {
|
|
220
|
+
const e = toEngineError(raw);
|
|
221
|
+
const cancelled = e instanceof EngineError && e.code === "cancelled";
|
|
222
|
+
scope.log(cancelled ? "warning" : "error", "run", cancelled ? "Cancelled. Finished steps are kept and will be reused" : e instanceof Error ? e.message : String(e));
|
|
223
|
+
await finish(cancelled ? "cancelled" : "failed").catch(() => {
|
|
224
|
+
});
|
|
225
|
+
throw e;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// src/route.ts
|
|
230
|
+
import { alignToReference, assessAlignment, coveredColumns, DEFAULT_GATES, LOCATE_FLANK, locateMarker } from "@extantape/onephylogeny-core";
|
|
231
|
+
function markerInSample(sequence, bundle) {
|
|
232
|
+
const reference = bundle.referenceSequence;
|
|
233
|
+
if (!reference || bundle.panel.marker.kind !== "gene") return { sequence };
|
|
234
|
+
const factor = bundle.panel.gates.containerRecordFactor ?? DEFAULT_GATES.containerRecordFactor ?? Infinity;
|
|
235
|
+
if (sequence.replace(/[-.\s]/g, "").length < factor * reference.length) return { sequence };
|
|
236
|
+
const location = locateMarker(sequence, reference);
|
|
237
|
+
if (location.status === "located") return { sequence: location.sequence, located: location.place };
|
|
238
|
+
if (location.status === "multiple") return { sequence, manyPlaces: location.places };
|
|
239
|
+
return { sequence };
|
|
240
|
+
}
|
|
241
|
+
var placeText = (marker, p) => `${marker} found in bases ${p.start + 1} to ${p.end} of the sample${p.strand === "-" ? ", on the reverse strand" : ""} (with up to ${LOCATE_FLANK} flanking bases each side)`;
|
|
242
|
+
var markerName = (bundle) => bundle.panel.marker.kind === "mitogenome" ? "mitogenome" : bundle.panel.marker.gene ?? bundle.panel.marker.kind;
|
|
243
|
+
var CURRENT = ["active", "candidate"];
|
|
244
|
+
var newer = (a, b) => {
|
|
245
|
+
const x = a.split(".").map(Number), y = b.split(".").map(Number);
|
|
246
|
+
for (let i = 0; i < Math.max(x.length, y.length); i++) if ((x[i] ?? 0) !== (y[i] ?? 0)) return (x[i] ?? 0) - (y[i] ?? 0);
|
|
247
|
+
return a.localeCompare(b);
|
|
248
|
+
};
|
|
249
|
+
var same = (a, b) => a.source === b.source && String(a.id) === String(b.id);
|
|
250
|
+
function routeSample(sequence, bundles, options = {}) {
|
|
251
|
+
const out = bundles.map((bundle) => {
|
|
252
|
+
const { panel } = bundle;
|
|
253
|
+
const marker = markerName(bundle);
|
|
254
|
+
const base = { panel: { id: panel.id, version: panel.version, status: panel.status }, windows: [], taxonMatches: options.taxonClaim ? panel.taxa.some((t) => same(t, options.taxonClaim)) : "unknown", marker };
|
|
255
|
+
if (!CURRENT.includes(panel.status)) return { ...base, applies: false, reason: `This panel is ${panel.status}, so it is not used for new samples` };
|
|
256
|
+
if (!bundle.referenceSequence) return { ...base, applies: false, reason: "This panel has no reference sequence; build it again" };
|
|
257
|
+
const part = markerInSample(sequence, bundle);
|
|
258
|
+
if (part.manyPlaces) return { ...base, applies: false, reason: `${marker} matches ${part.manyPlaces.length} places in this sample, so which one to test is ambiguous` };
|
|
259
|
+
const located = part.located ? { located: { start: part.located.start, end: part.located.end, strand: part.located.strand } } : {};
|
|
260
|
+
const found = part.located ? `${placeText(marker, part.located)}. ` : "";
|
|
261
|
+
const a = alignToReference(part.sequence, bundle.referenceSequence);
|
|
262
|
+
const errors = assessAlignment(a, { ...panel.gates, frameErrors: false }).filter((d) => d.severity === "error");
|
|
263
|
+
const measured = a ? { identity: a.identity, alignedColumns: a.matches + a.mismatches } : {};
|
|
264
|
+
if (!a || errors.length) return { ...base, ...located, ...measured, applies: false, reason: found + (errors.map((e) => e.message).join("; ") || "Does not align to this panel's reference") };
|
|
265
|
+
const covered = panel.windows.filter((w) => [...a.aligned.slice(w.start, w.end)].filter((c) => c !== "-" && c !== "N").length >= panel.rules.minSharedColumns);
|
|
266
|
+
const windows = covered.map((w) => w.id);
|
|
267
|
+
if (!windows.length) return { ...base, ...located, ...measured, applies: false, reason: `${found}Aligns to the reference, but covers none of this panel's windows with at least ${panel.rules.minSharedColumns} columns` };
|
|
268
|
+
if (base.taxonMatches === false) return { ...base, ...located, ...measured, windows, applies: false, reason: `${found}Aligns, but the sample is recorded as a taxon this panel is not for` };
|
|
269
|
+
const genes = panel.windows.some((w) => w.gene);
|
|
270
|
+
const windowColumns = genes ? { windowColumns: coveredColumns(covered) } : {};
|
|
271
|
+
const span = genes ? `covering ${windows.length} of ${panel.windows.length} genes (${windowColumns.windowColumns} reference columns)` : `covering ${windows.length} of ${panel.windows.length} windows`;
|
|
272
|
+
const taxon = base.taxonMatches === true ? ", for the taxon the sample is recorded as" : "";
|
|
273
|
+
return { ...base, ...located, ...measured, ...windowColumns, windows, applies: true, reason: `${found}Aligns at ${(a.identity * 100).toFixed(1)}% identity over ${measured.alignedColumns} columns, ${span}${taxon}` };
|
|
274
|
+
});
|
|
275
|
+
const rank = (s) => [s.applies ? 0 : 1, s.panel.status === "active" ? 0 : 1, -s.windows.length, -(s.alignedColumns ?? 0)];
|
|
276
|
+
return out.sort((x, y) => {
|
|
277
|
+
const [a, b] = [rank(x), rank(y)];
|
|
278
|
+
for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return a[i] - b[i];
|
|
279
|
+
if (x.panel.id === y.panel.id) return newer(y.panel.version, x.panel.version);
|
|
280
|
+
return String(x.panel.id).localeCompare(String(y.panel.id));
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// src/dataset.ts
|
|
285
|
+
import { canonicalJson as canonicalJson2, LOCATION_EVIDENCE } from "@extantape/onephylogeny-core";
|
|
286
|
+
var DATASET_FILES = ["panel.json", "panel.fasta", "validation.json", "diagnostics.json", "examples.json", "README.md"];
|
|
287
|
+
function datasetReadme(bundle, examples) {
|
|
288
|
+
const p = bundle.panel;
|
|
289
|
+
const lines = [
|
|
290
|
+
`# ${p.id} ${p.version}`,
|
|
291
|
+
"",
|
|
292
|
+
`A OnePhylogeny reference panel for ${p.marker.kind === "gene" ? `the ${p.marker.gene} gene` : "one marker"}, status **${p.status}**. It says which sampled maternal-line population a sequence is closer to. That is never proof of where an animal was collected.`,
|
|
293
|
+
"",
|
|
294
|
+
"## Groups",
|
|
295
|
+
"",
|
|
296
|
+
"| Group | Region | Records |",
|
|
297
|
+
"| --- | --- | --- |",
|
|
298
|
+
...p.groups.map((g) => `| ${g.label} | ${g.region ?? "(none)"} | ${g.evidence.length} |`),
|
|
299
|
+
"",
|
|
300
|
+
"## Limits",
|
|
301
|
+
"",
|
|
302
|
+
...p.notCovered.length ? p.notCovered.map((n) => `- Not covered: ${n}`) : ["- The panel states nothing about what it does not cover."],
|
|
303
|
+
...p.validatedScope.length ? p.validatedScope.map((v) => `- Validated: ${v}`) : ["- No claim has passed validation yet, so every result is exploratory."],
|
|
304
|
+
"- Statuses follow stated rules and are not probabilities.",
|
|
305
|
+
"",
|
|
306
|
+
"## Checking a copy",
|
|
307
|
+
"",
|
|
308
|
+
`\`onephylogeny verify --dir .\` checks every file against \`dataset.json\`, checks the seal (checksum \`${p.checksum}\`), and reruns the ${examples.examples.length} examples in \`examples.json\`. A copy is good only if all of them still give the answers recorded there.`,
|
|
309
|
+
"",
|
|
310
|
+
"## Files",
|
|
311
|
+
"",
|
|
312
|
+
"- `panel.json`: the sealed panel. `panel.fasta`: its reference and aligned records.",
|
|
313
|
+
"- `validation.json`: the validation report. `diagnostics.json`: notes from the build.",
|
|
314
|
+
"- `examples.json`: sequences with the answers they gave. `dataset.json`: every file's SHA-256.",
|
|
315
|
+
"",
|
|
316
|
+
"## Cite",
|
|
317
|
+
"",
|
|
318
|
+
...p.sources.length ? p.sources.map((s) => `- ${s.citation}${s.url ? ` ${s.url}` : ""}`) : ["- No sources recorded."],
|
|
319
|
+
"",
|
|
320
|
+
`Built with ${examples.producedBy.map((t) => `${t.name} ${t.version}`).join(", ")}.`,
|
|
321
|
+
""
|
|
322
|
+
];
|
|
323
|
+
return lines.join("\n");
|
|
324
|
+
}
|
|
325
|
+
function expectationOf(outcome) {
|
|
326
|
+
if (!outcome.result) return { outcome: "refused", codes: [...new Set(outcome.diagnostics.filter((d) => d.severity === "error").map((d) => d.code))].sort() };
|
|
327
|
+
return {
|
|
328
|
+
outcome: "result",
|
|
329
|
+
statuses: Object.fromEntries(outcome.result.findings.map((f) => [f.group, f.status]).sort(([a], [b]) => String(a).localeCompare(String(b)))),
|
|
330
|
+
pairs: Object.fromEntries(outcome.result.pairs.map((p) => [`${p.a}|${p.b}`, p.state === "resolved" ? `resolved:${p.favors}` : p.state]).sort(([a], [b]) => a.localeCompare(b)))
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
var ungap = (aligned) => aligned.replace(/[-.]/g, "");
|
|
334
|
+
var FRAGMENT = 300;
|
|
335
|
+
function chooseExamples(bundle) {
|
|
336
|
+
const out = [];
|
|
337
|
+
for (const group of [...bundle.panel.groups].sort((a, b) => String(a.id).localeCompare(String(b.id)))) {
|
|
338
|
+
const best = group.evidence.filter((e) => LOCATION_EVIDENCE.includes(e.provenanceClass) && bundle.sequences[e.record]).map((e) => ({ record: e.record, length: ungap(bundle.sequences[e.record]).length })).sort((a, b) => b.length - a.length || String(a.record).localeCompare(String(b.record)))[0];
|
|
339
|
+
if (best) out.push({ id: `member-${group.id}`, kind: "member", note: `The longest located member of ${group.label ?? group.id} (${best.record})`, sequence: ungap(bundle.sequences[best.record]) });
|
|
340
|
+
}
|
|
341
|
+
const first = out[0];
|
|
342
|
+
if (first && first.sequence.length >= FRAGMENT + 100) {
|
|
343
|
+
const start = Math.floor((first.sequence.length - FRAGMENT) / 2);
|
|
344
|
+
out.push({ id: `fragment-${first.id.slice("member-".length)}`, kind: "fragment", note: `${FRAGMENT} bases from the middle of ${first.id}`, sequence: first.sequence.slice(start, start + FRAGMENT) });
|
|
345
|
+
}
|
|
346
|
+
out.push({ id: "control-not-marker", kind: "control", note: "A repeat that is not the marker; the panel must refuse it", sequence: "ACGTTGCA".repeat(60) });
|
|
347
|
+
return out;
|
|
348
|
+
}
|
|
349
|
+
function makeExamples(bundle, test) {
|
|
350
|
+
const { panel } = bundle;
|
|
351
|
+
if (!panel.checksum) throw new EngineError("panel-unsealed", "Examples can only be made for a sealed panel");
|
|
352
|
+
return {
|
|
353
|
+
schemaVersion: 1,
|
|
354
|
+
panel: { id: panel.id, version: panel.version, checksum: panel.checksum },
|
|
355
|
+
producedBy: [ENGINE],
|
|
356
|
+
examples: chooseExamples(bundle).map((e) => ({ ...e, expected: expectationOf(test(e.sequence, e.id, bundle)) }))
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
function checkExamples(examples, bundle, test) {
|
|
360
|
+
const out = [];
|
|
361
|
+
if (examples.panel.checksum !== bundle.panel.checksum) return [{ severity: "error", code: "examples-wrong-panel", message: `The examples were made for panel checksum ${examples.panel.checksum}, not this one` }];
|
|
362
|
+
if (!examples.examples.length) out.push({ severity: "error", code: "no-examples", message: "The dataset has no examples, so nothing shows it still gives the same answers" });
|
|
363
|
+
for (const e of examples.examples) {
|
|
364
|
+
const now = expectationOf(test(e.sequence, e.id, bundle));
|
|
365
|
+
if (canonicalJson2(now) !== canonicalJson2(e.expected))
|
|
366
|
+
out.push({ severity: "error", code: "example-changed", message: `Example ${e.id} (${e.kind}) gave ${describe(now)}, not ${describe(e.expected)} as when the panel was built`, path: `examples.${e.id}` });
|
|
367
|
+
}
|
|
368
|
+
return out;
|
|
369
|
+
}
|
|
370
|
+
function describe(x) {
|
|
371
|
+
if (x.outcome === "refused") return `a refusal (${x.codes.join(", ") || "no code"})`;
|
|
372
|
+
return Object.entries(x.statuses).map(([g, s]) => `${g} ${s}`).join(", ");
|
|
373
|
+
}
|
|
374
|
+
async function writeDataset(store, dir, bundle, test) {
|
|
375
|
+
const examples = makeExamples(bundle, test);
|
|
376
|
+
const text = `${JSON.stringify(examples, null, 2)}
|
|
377
|
+
`;
|
|
378
|
+
if (await store.exists(`${dir}/examples.json`)) {
|
|
379
|
+
const existing = JSON.parse(await store.read(`${dir}/examples.json`));
|
|
380
|
+
if (canonicalJson2(existing.examples) !== canonicalJson2(examples.examples) || existing.panel.checksum !== examples.panel.checksum)
|
|
381
|
+
throw new EngineError("examples-differ", `Panel ${bundle.panel.id} ${bundle.panel.version} already has different examples`, "Run verify to see which ones changed; a panel version keeps the examples it was published with");
|
|
382
|
+
} else await store.write(`${dir}/examples.json`, text);
|
|
383
|
+
const kept = JSON.parse(await store.read(`${dir}/examples.json`));
|
|
384
|
+
await store.write(`${dir}/README.md`, datasetReadme(bundle, kept));
|
|
385
|
+
const files = {};
|
|
386
|
+
for (const f of DATASET_FILES) if (await store.exists(`${dir}/${f}`)) files[f] = hashText(await store.read(`${dir}/${f}`));
|
|
387
|
+
const manifest = { schemaVersion: 1, panel: examples.panel, files, builtWith: kept.producedBy, citations: bundle.panel.sources.map((s) => s.url ? `${s.citation} ${s.url}` : s.citation) };
|
|
388
|
+
await store.write(`${dir}/dataset.json`, `${JSON.stringify(manifest, null, 2)}
|
|
389
|
+
`);
|
|
390
|
+
return manifest;
|
|
391
|
+
}
|
|
392
|
+
async function verifyDataset(store, dir, load, test) {
|
|
393
|
+
const diagnostics = [];
|
|
394
|
+
let files = 0;
|
|
395
|
+
if (!await store.exists(`${dir}/dataset.json`)) diagnostics.push({ severity: "error", code: "no-dataset-manifest", message: "This folder has no dataset.json, so its files can't be checked against their hashes" });
|
|
396
|
+
else {
|
|
397
|
+
const manifest = JSON.parse(await store.read(`${dir}/dataset.json`));
|
|
398
|
+
for (const [name, hash] of Object.entries(manifest.files)) {
|
|
399
|
+
files++;
|
|
400
|
+
if (!await store.exists(`${dir}/${name}`)) diagnostics.push({ severity: "error", code: "file-missing", message: `${name} is listed in dataset.json but missing`, path: name });
|
|
401
|
+
else if (hashText(await store.read(`${dir}/${name}`)) !== hash) diagnostics.push({ severity: "error", code: "file-changed", message: `${name} has changed since the dataset was written`, path: name });
|
|
402
|
+
}
|
|
403
|
+
for (const required of ["panel.json", "panel.fasta", "examples.json"])
|
|
404
|
+
if (!(required in manifest.files)) diagnostics.push({ severity: "error", code: "file-not-covered", message: `dataset.json doesn't cover ${required}`, path: required });
|
|
405
|
+
}
|
|
406
|
+
let bundle;
|
|
407
|
+
try {
|
|
408
|
+
bundle = await load(store, dir);
|
|
409
|
+
} catch (e) {
|
|
410
|
+
diagnostics.push({ severity: "error", code: e instanceof EngineError ? e.code : "panel-unreadable", message: e.message });
|
|
411
|
+
return { diagnostics, checked: { files, examples: 0 } };
|
|
412
|
+
}
|
|
413
|
+
if (!await store.exists(`${dir}/examples.json`)) {
|
|
414
|
+
diagnostics.push({ severity: "error", code: "no-examples", message: "This panel has no examples.json, so nothing shows it still gives the same answers" });
|
|
415
|
+
return { diagnostics, checked: { files, examples: 0 } };
|
|
416
|
+
}
|
|
417
|
+
const examples = JSON.parse(await store.read(`${dir}/examples.json`));
|
|
418
|
+
diagnostics.push(...checkExamples(examples, bundle, test));
|
|
419
|
+
return { diagnostics, checked: { files, examples: examples.examples.length } };
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// src/curation.ts
|
|
423
|
+
import { parseCuration } from "@extantape/onephylogeny-core";
|
|
424
|
+
var historyPath = (profileId) => `profiles/${profileId}/curation-history.jsonl`;
|
|
425
|
+
function curationDiff(before, after) {
|
|
426
|
+
const out = [];
|
|
427
|
+
for (const record of [.../* @__PURE__ */ new Set([...Object.keys(before.records), ...Object.keys(after.records)])].sort()) {
|
|
428
|
+
const a = before.records[record] ?? {};
|
|
429
|
+
const b = after.records[record] ?? {};
|
|
430
|
+
for (const field of [.../* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)])].sort()) {
|
|
431
|
+
if (JSON.stringify(a[field]) === JSON.stringify(b[field])) continue;
|
|
432
|
+
out.push({ record, field, ...a[field] !== void 0 ? { from: a[field] } : {}, ...b[field] !== void 0 ? { to: b[field] } : {} });
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return out;
|
|
436
|
+
}
|
|
437
|
+
async function saveCuration(store, profileId, next, meta = {}) {
|
|
438
|
+
const P = profilePaths(profileId);
|
|
439
|
+
const before = await store.exists(P.curation) ? parseCuration(await store.read(P.curation)) : { schemaVersion: 1, records: {} };
|
|
440
|
+
const changes = curationDiff(before, next);
|
|
441
|
+
if (!changes.length) return null;
|
|
442
|
+
const entry = { at: meta.at ?? (/* @__PURE__ */ new Date()).toISOString(), ...meta.by ? { by: meta.by } : {}, ...meta.reason ? { reason: meta.reason } : {}, changes };
|
|
443
|
+
const line = `${JSON.stringify(entry)}
|
|
444
|
+
`;
|
|
445
|
+
if (store.append) await store.append(historyPath(profileId), line);
|
|
446
|
+
else await store.write(historyPath(profileId), `${await store.exists(historyPath(profileId)) ? await store.read(historyPath(profileId)) : ""}${line}`);
|
|
447
|
+
await store.write(P.curation, `${JSON.stringify(next, null, 2)}
|
|
448
|
+
`);
|
|
449
|
+
return entry;
|
|
450
|
+
}
|
|
451
|
+
async function readCurationHistory(store, profileId, record) {
|
|
452
|
+
if (!await store.exists(historyPath(profileId))) return { entries: [], damagedLines: [] };
|
|
453
|
+
const entries = [];
|
|
454
|
+
const damagedLines = [];
|
|
455
|
+
(await store.read(historyPath(profileId))).split("\n").forEach((l, i) => {
|
|
456
|
+
if (!l.trim()) return;
|
|
457
|
+
try {
|
|
458
|
+
const e = JSON.parse(l);
|
|
459
|
+
const changes = record ? e.changes.filter((c) => c.record === record) : e.changes;
|
|
460
|
+
if (changes.length) entries.push({ ...e, changes });
|
|
461
|
+
} catch {
|
|
462
|
+
damagedLines.push(i + 1);
|
|
463
|
+
}
|
|
464
|
+
});
|
|
465
|
+
return { entries, damagedLines };
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// src/hypothesis.ts
|
|
469
|
+
import {
|
|
470
|
+
canonicalJson as canonicalJson3,
|
|
471
|
+
claimText,
|
|
472
|
+
parseFasta,
|
|
473
|
+
planTest,
|
|
474
|
+
runTest
|
|
475
|
+
} from "@extantape/onephylogeny-core";
|
|
476
|
+
var foldersOf = (store) => store.folders?.bind(store);
|
|
477
|
+
var safeName = (id) => id.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
478
|
+
function namedVersions(claim) {
|
|
479
|
+
if (claim.kind === "panel-changed-call") return [claim.from, claim.to];
|
|
480
|
+
return claim.panel ? [claim.panel] : [];
|
|
481
|
+
}
|
|
482
|
+
var claimSample = (claim) => claim.kind === "groups-distinct" ? void 0 : claim.sample;
|
|
483
|
+
async function readBundle(store, folder) {
|
|
484
|
+
const panel = JSON.parse(await store.read(`${folder}/panel.json`));
|
|
485
|
+
const fasta = parseFasta(await store.read(`${folder}/panel.fasta`));
|
|
486
|
+
const referenceSequence = fasta.find((f) => f.id === "__reference__")?.sequence;
|
|
487
|
+
return { panel, sequences: Object.fromEntries(fasta.filter((f) => f.id !== "__reference__").map((f) => [f.id, f.sequence])), ...referenceSequence ? { referenceSequence } : {} };
|
|
488
|
+
}
|
|
489
|
+
var readJson = async (store, path) => {
|
|
490
|
+
if (!await store.exists(path)) return void 0;
|
|
491
|
+
try {
|
|
492
|
+
return JSON.parse(await store.read(path));
|
|
493
|
+
} catch {
|
|
494
|
+
return void 0;
|
|
495
|
+
}
|
|
496
|
+
};
|
|
497
|
+
async function loadPanelSource(store, folder, label, artifactCheck) {
|
|
498
|
+
let bundle;
|
|
499
|
+
const problems = [];
|
|
500
|
+
try {
|
|
501
|
+
bundle = await loadPanel(store, folder);
|
|
502
|
+
} catch (e) {
|
|
503
|
+
if (!(e instanceof EngineError) || e.code !== "panel-changed") return { note: `${label} was skipped: ${e.message}` };
|
|
504
|
+
bundle = await readBundle(store, folder);
|
|
505
|
+
problems.push(e.message);
|
|
506
|
+
}
|
|
507
|
+
let checked = 0;
|
|
508
|
+
const artifact = await readJson(store, `${folder}/artifact.json`);
|
|
509
|
+
if (typeof artifactCheck === "object" && (artifact?.type !== artifactCheck.type || artifact.key !== artifactCheck.key)) problems.push(`it no longer names panel artifact ${artifactCheck.key}`);
|
|
510
|
+
if (artifact && artifactCheck !== "folder") {
|
|
511
|
+
checked++;
|
|
512
|
+
const manifest = await readManifest(store, artifact).catch(() => null);
|
|
513
|
+
if (!manifest) problems.push(`the panel artifact ${artifact.key} it names is missing or damaged`);
|
|
514
|
+
else if (!await verifyFiles(store, manifest)) problems.push(`the files of panel artifact ${artifact.key} don't match their recorded hashes`);
|
|
515
|
+
else for (const name of ["panel.json", "panel.fasta"]) {
|
|
516
|
+
const recorded = manifest.files.find((f) => f.name === name)?.sha256;
|
|
517
|
+
if (!recorded || hashText(await store.read(`${folder}/${name}`)) !== recorded) problems.push(`${name} isn't the file panel artifact ${artifact.key} recorded`);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
const dataset = await readJson(store, `${folder}/dataset.json`);
|
|
521
|
+
if (dataset) {
|
|
522
|
+
checked++;
|
|
523
|
+
for (const [name, hash] of Object.entries(dataset.files)) {
|
|
524
|
+
if (!await store.exists(`${folder}/${name}`)) problems.push(`${name} is listed in dataset.json but missing`);
|
|
525
|
+
else if (hashText(await store.read(`${folder}/${name}`)) !== hash) problems.push(`${name} has changed since dataset.json was written`);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
const pointer = artifact ? { type: artifact.type, key: artifact.key } : { type: "panel-dataset", key: bundle.panel.checksum ?? hashText(canonicalJson3(bundle.panel)) };
|
|
529
|
+
const filesVerified = problems.length ? false : checked ? true : void 0;
|
|
530
|
+
const notes = problems.length ? [`${label} failed its file checks: ${problems.join("; ")}`] : checked ? [] : [`${label} has no artifact.json or dataset.json, so its files couldn't be checked against recorded hashes`];
|
|
531
|
+
const item = { artifact: pointer, bundle, ...filesVerified !== void 0 ? { filesVerified } : {} };
|
|
532
|
+
return { item, source: { kind: "panel", pointer, folder, ...artifact && artifactCheck !== "folder" ? { artifact } : {} }, ...notes.length ? { notes } : {} };
|
|
533
|
+
}
|
|
534
|
+
async function loadResultSource(store, profileId, file, label) {
|
|
535
|
+
let text;
|
|
536
|
+
let saved;
|
|
537
|
+
try {
|
|
538
|
+
text = await store.read(file);
|
|
539
|
+
saved = JSON.parse(text);
|
|
540
|
+
} catch {
|
|
541
|
+
return { note: `${label} couldn't be read as a saved result, so it was skipped` };
|
|
542
|
+
}
|
|
543
|
+
if (!saved.result) {
|
|
544
|
+
const why = (saved.diagnostics ?? []).filter((d) => d.severity === "error").map((d) => d.message).join("; ");
|
|
545
|
+
return { note: `${label} has no result: the sample was refused${why ? ` (${why})` : ""}` };
|
|
546
|
+
}
|
|
547
|
+
const result = saved.result;
|
|
548
|
+
const P = profilePaths(profileId);
|
|
549
|
+
const problems = [];
|
|
550
|
+
let artifact;
|
|
551
|
+
const run = await readJson(store, `${P.run(result.run)}/run.json`);
|
|
552
|
+
if (!run) problems.push(`the run that made it, ${result.run}, has no record`);
|
|
553
|
+
else {
|
|
554
|
+
for (const a of run.artifacts.filter((x) => x.type === "sample-result")) {
|
|
555
|
+
const ref2 = { type: "sample-result", key: a.key };
|
|
556
|
+
const manifest = await readManifest(store, ref2).catch(() => null);
|
|
557
|
+
if (!manifest || manifest.parameters.queryId !== result.query) continue;
|
|
558
|
+
artifact = ref2;
|
|
559
|
+
if (!await verifyFiles(store, manifest)) problems.push(`the files of sample-result ${a.key} don't match their recorded hashes`);
|
|
560
|
+
else if (manifest.files.find((f) => f.name === "outcome.json")?.sha256 !== hashText(text)) problems.push(`this copy isn't the outcome sample-result ${a.key} recorded`);
|
|
561
|
+
break;
|
|
562
|
+
}
|
|
563
|
+
if (!artifact) problems.push(`run ${result.run} recorded no saved result for ${result.query}`);
|
|
564
|
+
}
|
|
565
|
+
const pointer = artifact ? { type: artifact.type, key: artifact.key } : { type: "sample-result-copy", key: file };
|
|
566
|
+
const item = { artifact: pointer, result, ...saved.aligned ? { aligned: saved.aligned } : {}, filesVerified: problems.length === 0 };
|
|
567
|
+
const source = { kind: "result", pointer, file, ...artifact ? { artifact } : {} };
|
|
568
|
+
return { item, source, ...problems.length ? { notes: [`${label} failed its file checks: ${problems.join("; ")}`] } : {} };
|
|
569
|
+
}
|
|
570
|
+
function collect(into, loaded, push) {
|
|
571
|
+
if ("note" in loaded) return void into.notes.push(loaded.note);
|
|
572
|
+
push(loaded.item);
|
|
573
|
+
into.sources.push(loaded.source);
|
|
574
|
+
into.notes.push(...loaded.notes ?? []);
|
|
575
|
+
}
|
|
576
|
+
async function gather(ctx, profile, claim) {
|
|
577
|
+
const { store } = ctx;
|
|
578
|
+
const P = profilePaths(profile.id);
|
|
579
|
+
const panelsRoot = `profiles/${profile.id}/panels`;
|
|
580
|
+
const list = foldersOf(store);
|
|
581
|
+
const named = namedVersions(claim);
|
|
582
|
+
let panelFolders;
|
|
583
|
+
let resultFolders;
|
|
584
|
+
if (list) {
|
|
585
|
+
panelFolders = (await list(panelsRoot)).map((f) => `${panelsRoot}/${f}`);
|
|
586
|
+
resultFolders = (await list(P.results)).map((f) => `${P.results}/${f}`);
|
|
587
|
+
} else {
|
|
588
|
+
if (!named.length) return { untestable: "this store can't list the panel versions it holds, so the claim has to name one", notes: [] };
|
|
589
|
+
panelFolders = named.map((v) => P.panel(v.id, v.version));
|
|
590
|
+
resultFolders = named.map((v) => `${P.results}/${v.id}-${v.version}`);
|
|
591
|
+
}
|
|
592
|
+
const out = { artifacts: { panels: [], results: [] }, sources: [], notes: [] };
|
|
593
|
+
for (const folder of panelFolders) {
|
|
594
|
+
if (!await store.exists(`${folder}/panel.json`)) continue;
|
|
595
|
+
collect(out, await loadPanelSource(store, folder, `The panel in ${folder}`, "workspace"), (p) => out.artifacts.panels.push(p));
|
|
596
|
+
}
|
|
597
|
+
const sample = claimSample(claim);
|
|
598
|
+
if (sample) for (const folder of resultFolders) {
|
|
599
|
+
const file = `${folder}/${safeName(sample)}.json`;
|
|
600
|
+
if (!await store.exists(file)) continue;
|
|
601
|
+
collect(out, await loadResultSource(store, profile.id, file, `The saved result ${file}`), (r) => out.artifacts.results.push(r));
|
|
602
|
+
}
|
|
603
|
+
return out;
|
|
604
|
+
}
|
|
605
|
+
var sourcesFor = (stated, sources) => stated.uses.flatMap((u) => sources.filter((s) => s.pointer.type === u.artifact.type && s.pointer.key === u.artifact.key).slice(0, 1));
|
|
606
|
+
var notTestable = (claim, reason) => ({
|
|
607
|
+
claim,
|
|
608
|
+
claimText: claimText(claim),
|
|
609
|
+
testable: false,
|
|
610
|
+
reason,
|
|
611
|
+
uses: [],
|
|
612
|
+
text: `This can't be tested with the artifacts given: ${reason}.`
|
|
613
|
+
});
|
|
614
|
+
async function testHypothesis(ctx, profile, claim) {
|
|
615
|
+
const g = await gather(ctx, profile, claim);
|
|
616
|
+
if ("untestable" in g) return { stated: notTestable(claim, g.untestable), profile: profile.id, sources: [], notes: g.notes };
|
|
617
|
+
const stated = planTest(claim, g.artifacts);
|
|
618
|
+
return { stated, profile: profile.id, sources: sourcesFor(stated, g.sources), notes: g.notes };
|
|
619
|
+
}
|
|
620
|
+
async function testHypothesisOnPanel(ctx, folder, claim) {
|
|
621
|
+
const out = { artifacts: { panels: [], results: [] }, sources: [], notes: [] };
|
|
622
|
+
collect(out, await loadPanelSource(ctx.store, folder, "The panel folder", "folder"), (p) => out.artifacts.panels.push(p));
|
|
623
|
+
if (claimSample(claim)) out.notes.push("A panel folder holds no saved results; give a workspace and profile to test claims about a sample");
|
|
624
|
+
const stated = planTest(claim, out.artifacts);
|
|
625
|
+
return { stated, sources: sourcesFor(stated, out.sources), notes: out.notes };
|
|
626
|
+
}
|
|
627
|
+
async function runHypothesis(ctx, plan) {
|
|
628
|
+
if (!plan.stated.testable) return { ...runTest(plan.stated, { panels: [], results: [] }), notes: [] };
|
|
629
|
+
const out = { artifacts: { panels: [], results: [] }, sources: [], notes: [] };
|
|
630
|
+
for (const s of plan.sources) {
|
|
631
|
+
if (s.kind === "panel") {
|
|
632
|
+
if (!await ctx.store.exists(`${s.folder}/panel.json`)) {
|
|
633
|
+
out.notes.push(`The panel in ${s.folder} is gone`);
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
collect(out, await loadPanelSource(ctx.store, s.folder, `The panel in ${s.folder}`, s.artifact ?? (plan.profile ? "workspace" : "folder")), (p) => out.artifacts.panels.push(p));
|
|
637
|
+
} else {
|
|
638
|
+
if (!await ctx.store.exists(s.file)) {
|
|
639
|
+
out.notes.push(`The saved result ${s.file} is gone`);
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
642
|
+
collect(out, await loadResultSource(ctx.store, plan.profile ?? "", s.file, `The saved result ${s.file}`), (r) => out.artifacts.results.push(r));
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
const verdict = runTest(plan.stated, out.artifacts);
|
|
646
|
+
return { ...verdict, notes: out.notes };
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// src/index.ts
|
|
650
|
+
var cachePath = (version) => `cache/genbank/${version.replace(/[^A-Za-z0-9._-]/g, "_")}.gb`;
|
|
651
|
+
var ref = (a) => ({ type: a.manifest.type, key: a.manifest.key });
|
|
652
|
+
var artifactOf = (run, type) => run.artifacts.find((a) => a.type === type);
|
|
653
|
+
var isMitogenome = (profile) => profile.selection.marker.kind === "mitogenome";
|
|
654
|
+
var COMPLETE_SHARE = 0.9;
|
|
655
|
+
function geneDiagnostics(checks) {
|
|
656
|
+
const out = [];
|
|
657
|
+
for (const c of checks) {
|
|
658
|
+
if (c.status === "frame") out.push({ severity: "warning", code: "gene-frame", message: `${c.label}: ${c.detail}`, path: `genes.${c.window}` });
|
|
659
|
+
else if (c.status === "stops") out.push({ severity: "warning", code: "gene-stops", message: `${c.label}: ${c.detail}`, path: `genes.${c.window}` });
|
|
660
|
+
else if (c.status === "not-checked") out.push({ severity: "info", code: "gene-not-checked", message: `${c.label}: ${c.detail}`, path: `genes.${c.window}` });
|
|
661
|
+
}
|
|
662
|
+
const partial = checks.filter((c) => c.status === "partial");
|
|
663
|
+
if (partial.length) out.push({ severity: "info", code: "gene-partial", message: `${partial.length} protein-coding gene${partial.length === 1 ? " is" : "s are"} covered only in part, so frame and stops weren't judged there: ${partial.map((c) => `${c.label} (${Math.round(c.coverage * 100)}%)`).join(", ")}` });
|
|
664
|
+
return out;
|
|
665
|
+
}
|
|
666
|
+
function checkGenes(aligned, reference, windows, code) {
|
|
667
|
+
if (!windows.some((w) => w.gene?.kind === "cds")) return { checks: [], diagnostics: [] };
|
|
668
|
+
if (!code) return { checks: [], diagnostics: [{ severity: "info", code: "gene-not-checked", message: "No genetic code is set for the reference, so protein-coding genes weren't checked for frame or stops" }] };
|
|
669
|
+
const checks = geneChecks(aligned, reference, windows, code);
|
|
670
|
+
return { checks, diagnostics: geneDiagnostics(checks) };
|
|
671
|
+
}
|
|
672
|
+
var leftOutLines = (gw) => [...gw.leftOut.map((g) => `${g.name} (reference ${g.location}): ${g.reason}`), ...gw.notes];
|
|
673
|
+
function geneSequence(rec, profile) {
|
|
674
|
+
const marker = profile.selection.marker;
|
|
675
|
+
const names = [marker.gene, ...marker.synonyms ?? [], ...marker.products ?? []].filter(Boolean);
|
|
676
|
+
if (marker.kind === "gene") {
|
|
677
|
+
const feature = findGeneFeatures(rec, names)[0];
|
|
678
|
+
if (feature) {
|
|
679
|
+
try {
|
|
680
|
+
return { sequence: extractLocation(rec.sequence, feature.location), from: "annotation" };
|
|
681
|
+
} catch {
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
return { sequence: rec.sequence, from: "whole-record" };
|
|
686
|
+
}
|
|
687
|
+
var where = (p) => `bases ${p.start + 1} to ${p.end} on the ${p.strand === "+" ? "given" : "reverse"} strand${p.wrapsOrigin ? ", across the origin" : ""}`;
|
|
688
|
+
function locatedSequence(rec, reference, profile, found) {
|
|
689
|
+
const factor = panelGates(profile).containerRecordFactor ?? Infinity;
|
|
690
|
+
if (found.from !== "whole-record" || profile.selection.marker.kind !== "gene" || found.sequence.length < factor * reference.length) return found;
|
|
691
|
+
const location = locateMarker2(found.sequence, reference, { ...rec.circular ? { circular: true } : {} });
|
|
692
|
+
if (location.status === "located") return { sequence: location.sequence, from: "located", located: location.place };
|
|
693
|
+
if (location.status === "multiple") return { ...found, manyPlaces: location.places };
|
|
694
|
+
return found;
|
|
695
|
+
}
|
|
696
|
+
async function searchStep(scope, profile, client, options) {
|
|
697
|
+
const include = profile.selection.include;
|
|
698
|
+
const term = include?.length ? `chosen accessions, ${include.length} records, ${hashText([...include].sort().join("\n"))}` : selectionQuery(profile.selection);
|
|
699
|
+
scope.log("info", "search", include?.length ? "Using the accessions chosen in a Study" : "Searching NCBI", { term });
|
|
700
|
+
const found = include?.length ? include : await client.searchAccessions(term);
|
|
701
|
+
if (!found.length && !options.allowEmpty) throw new EngineError("empty-search", "NCBI found no records for this taxon and marker", "Check the taxon, gene names and organelle in Setup");
|
|
702
|
+
const genomes = include?.length ? null : genomesHoldingMarkerQuery(profile.selection);
|
|
703
|
+
if (genomes) {
|
|
704
|
+
try {
|
|
705
|
+
const missed = (await client.searchAccessions(genomes)).filter((a) => !found.includes(a));
|
|
706
|
+
if (missed.length) scope.log("warning", "search", `${missed.length} mitochondrial genomes of these taxa aren't in this search, probably because they don't annotate the marker. They may hold it, and they aren't used unless chosen in a Study, which cuts the marker out by similarity: ${missed.join(", ")}`, { missed });
|
|
707
|
+
} catch (e) {
|
|
708
|
+
if (isCancelled(e)) throw e;
|
|
709
|
+
scope.log("warning", "search", `Couldn't check for mitochondrial genomes the search missed: ${e instanceof Error ? e.message : String(e)}`);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
const extra = [profile.reference.accession, ...profile.selection.outgroups.map((o) => o.accession).filter((a) => Boolean(a))];
|
|
713
|
+
const accessions = [.../* @__PURE__ */ new Set([...found, ...extra])].sort();
|
|
714
|
+
scope.log("info", "search", `Found ${found.length} records, plus ${extra.length} named in the profile`, { found: found.length });
|
|
715
|
+
const artifact = await scope.step({
|
|
716
|
+
type: "accession-list",
|
|
717
|
+
step: { name: "search", version: 1 },
|
|
718
|
+
inputs: [],
|
|
719
|
+
parameters: { term, accessions: hashText(accessions.join("\n")) },
|
|
720
|
+
run: async (put) => {
|
|
721
|
+
await put("accessions.json", json({ term, accessions }));
|
|
722
|
+
}
|
|
723
|
+
});
|
|
724
|
+
return { artifact, accessions, term };
|
|
725
|
+
}
|
|
726
|
+
function rawStep(scope, list, accessions, client, batchSize) {
|
|
727
|
+
const { store } = scope.ctx;
|
|
728
|
+
return scope.step({
|
|
729
|
+
type: "raw-records",
|
|
730
|
+
step: { name: "fetch", version: 1 },
|
|
731
|
+
inputs: [ref(list)],
|
|
732
|
+
parameters: {},
|
|
733
|
+
run: async (put) => {
|
|
734
|
+
const texts = /* @__PURE__ */ new Map();
|
|
735
|
+
const missing = [];
|
|
736
|
+
for (const version of accessions) {
|
|
737
|
+
const path = cachePath(version);
|
|
738
|
+
if (await store.exists(path)) texts.set(version, await store.read(path));
|
|
739
|
+
else missing.push(version);
|
|
740
|
+
}
|
|
741
|
+
scope.log("info", "fetch", `${texts.size} records already downloaded, ${missing.length} to fetch`, { done: texts.size, total: accessions.length });
|
|
742
|
+
for (let i = 0; i < missing.length; i += batchSize) {
|
|
743
|
+
scope.checkpoint();
|
|
744
|
+
const batch = missing.slice(i, i + batchSize);
|
|
745
|
+
const { text, records } = await client.fetchGenBankText(batch);
|
|
746
|
+
const blocks = text.split(/^\/\/\s*$/m).filter((b) => b.includes("LOCUS"));
|
|
747
|
+
for (const [n, rec] of records.entries()) {
|
|
748
|
+
const raw = `${blocks[n].trimStart()}//
|
|
749
|
+
`;
|
|
750
|
+
const key = rec.version || rec.accession;
|
|
751
|
+
await store.write(cachePath(key), raw);
|
|
752
|
+
texts.set(key, raw);
|
|
753
|
+
}
|
|
754
|
+
scope.log("info", "fetch", `Fetched ${Math.min(i + batchSize, missing.length)} of ${missing.length}`, { done: texts.size, total: accessions.length });
|
|
755
|
+
}
|
|
756
|
+
const index = Object.fromEntries([...texts].map(([k, t]) => [k, hashText(t)]).sort(([a], [b]) => String(a).localeCompare(String(b))));
|
|
757
|
+
await put("index.json", json(index));
|
|
758
|
+
},
|
|
759
|
+
ensure: async (read) => Object.keys(JSON.parse(await read("index.json"))).length ? null : "no records were fetched"
|
|
760
|
+
});
|
|
761
|
+
}
|
|
762
|
+
var markerParameters = (profile) => ({ marker: profile.selection.marker, reference: profile.reference?.accession });
|
|
763
|
+
function anchorStep(scope, profile, raw) {
|
|
764
|
+
const { store } = scope.ctx;
|
|
765
|
+
return scope.step({
|
|
766
|
+
// A mitogenome profile adds its rotation rule, so a change to it is a new artifact. Gene profiles keep their keys.
|
|
767
|
+
type: "anchored-records",
|
|
768
|
+
step: { name: "anchor", version: 2 },
|
|
769
|
+
inputs: [ref(raw)],
|
|
770
|
+
parameters: { ...markerParameters(profile), ...isMitogenome(profile) ? { mitogenome: { rotate: "exact k-mer anchor", completeShare: COMPLETE_SHARE } } : {} },
|
|
771
|
+
tools: [ALIGNER],
|
|
772
|
+
run: async (put) => {
|
|
773
|
+
const index = JSON.parse(await raw.read("index.json"));
|
|
774
|
+
const parsed = /* @__PURE__ */ new Map();
|
|
775
|
+
for (const [version, sha] of Object.entries(index)) {
|
|
776
|
+
const text = await store.read(cachePath(version));
|
|
777
|
+
if (hashText(text) !== sha) throw new EngineError("raw-file-changed", `The downloaded file for ${version} changed after it was fetched`, `Delete ${cachePath(version)} and fetch again`);
|
|
778
|
+
const rec = parseGenBank(text)[0];
|
|
779
|
+
if (!rec) throw new EngineError("raw-file-changed", `The downloaded file for ${version} has no record in it`, `Delete ${cachePath(version)} and fetch again`);
|
|
780
|
+
parsed.set(version, rec);
|
|
781
|
+
}
|
|
782
|
+
const accession = profile.reference.accession;
|
|
783
|
+
const refRec = [...parsed.values()].find((r) => r.version === accession || r.accession === accession);
|
|
784
|
+
if (!refRec) throw new EngineError("reference-missing", `The reference ${accession} wasn't retrieved`, "Check the reference accession in Setup");
|
|
785
|
+
const reference = geneSequence(refRec, profile).sequence;
|
|
786
|
+
scope.log("info", "anchor", `Reference ${refRec.version}: ${reference.length} columns`);
|
|
787
|
+
const mito = isMitogenome(profile);
|
|
788
|
+
if (mito) {
|
|
789
|
+
const gw = geneWindows(refRec);
|
|
790
|
+
scope.log(gw.windows.length ? "info" : "warning", "anchor", `Reference ${refRec.version}: ${gw.windows.length} gene windows, ${gw.leftOut.length} features left out`);
|
|
791
|
+
await put("gene-windows.json", json(gw));
|
|
792
|
+
}
|
|
793
|
+
const anchored = [];
|
|
794
|
+
const records = [];
|
|
795
|
+
let done = 0;
|
|
796
|
+
for (const [version, rec] of parsed) {
|
|
797
|
+
await scope.breathe();
|
|
798
|
+
const { sequence: found, from, located, manyPlaces } = locatedSequence(rec, reference, profile, geneSequence(rec, profile));
|
|
799
|
+
let sequence = found;
|
|
800
|
+
let rotation;
|
|
801
|
+
if (mito && rec.circular) {
|
|
802
|
+
if (sequence.length >= COMPLETE_SHARE * reference.length) {
|
|
803
|
+
const turned = rotateToReference(sequence, reference);
|
|
804
|
+
if (turned.status === "rotated") {
|
|
805
|
+
sequence = turned.sequence;
|
|
806
|
+
rotation = { strand: turned.strand, shift: turned.shift };
|
|
807
|
+
} else rotation = { refused: turned.reason };
|
|
808
|
+
} else rotation = { notTurned: `Circular, but ${sequence.length} bp is under ${Math.round(COMPLETE_SHARE * 100)}% of the reference's ${reference.length}, so it was aligned as it is, without turning` };
|
|
809
|
+
}
|
|
810
|
+
const refused = rotation && "refused" in rotation;
|
|
811
|
+
anchored.push({
|
|
812
|
+
...located ? { located } : {},
|
|
813
|
+
...manyPlaces ? { manyPlaces } : {},
|
|
814
|
+
record: rec.version || version,
|
|
815
|
+
...rec.organism ? { organism: rec.organism } : {},
|
|
816
|
+
alignment: refused ? null : alignToReference2(sequence, reference),
|
|
817
|
+
extractedFrom: from,
|
|
818
|
+
length: sequence.length,
|
|
819
|
+
ambiguous: (sequence.match(/[^ACGTacgt]/g) ?? []).length,
|
|
820
|
+
unverified: rec.definition.startsWith("UNVERIFIED"),
|
|
821
|
+
...rotation ? { rotation } : {}
|
|
822
|
+
});
|
|
823
|
+
records.push(toSequenceRecord(rec));
|
|
824
|
+
if (++done % 100 === 0) scope.log("debug", "anchor", `Anchored ${done} of ${parsed.size}`, { done, total: parsed.size });
|
|
825
|
+
}
|
|
826
|
+
await put("reference.fasta", writeFasta([{ id: refRec.version, sequence: reference }]));
|
|
827
|
+
await put("anchored.json", JSON.stringify(anchored));
|
|
828
|
+
await put("records.json", JSON.stringify(records));
|
|
829
|
+
},
|
|
830
|
+
ensure: async (read) => parseFasta2(await read("reference.fasta"))[0]?.sequence.length ? null : "the reference has no sequence"
|
|
831
|
+
});
|
|
832
|
+
}
|
|
833
|
+
var panelGates = (profile) => isMitogenome(profile) ? mitogenomeGates(profile) : { ...DEFAULT_GATES2, ...profile.gates, frameErrors: true, ...profile.reference?.codonStart !== void 0 ? { codonStart: profile.reference.codonStart } : {}, ...profile.reference?.geneticCode ? { geneticCode: profile.reference.geneticCode } : {} };
|
|
834
|
+
function mitogenomeGates(profile) {
|
|
835
|
+
const { codonStart: _start, ...rest } = { ...DEFAULT_GATES2, ...profile.gates };
|
|
836
|
+
return { ...rest, frameErrors: false, ...profile.reference?.geneticCode ? { geneticCode: profile.reference.geneticCode } : {} };
|
|
837
|
+
}
|
|
838
|
+
function gateStep(scope, profile, anchoredArtifact) {
|
|
839
|
+
const { filters, exclude } = profile.selection;
|
|
840
|
+
const gates = panelGates(profile);
|
|
841
|
+
return scope.step({
|
|
842
|
+
type: "gated-records",
|
|
843
|
+
step: { name: "gate", version: 2 },
|
|
844
|
+
inputs: [ref(anchoredArtifact)],
|
|
845
|
+
parameters: { filters, exclude, gates },
|
|
846
|
+
run: async (put) => {
|
|
847
|
+
const excluded = new Map(exclude.map((e) => [e.accession, e.reason]));
|
|
848
|
+
const anchored = JSON.parse(await anchoredArtifact.read("anchored.json"));
|
|
849
|
+
const records = JSON.parse(await anchoredArtifact.read("records.json"));
|
|
850
|
+
const outcomes = [];
|
|
851
|
+
const aligned = [];
|
|
852
|
+
const mito = isMitogenome(profile);
|
|
853
|
+
const genes = mito ? JSON.parse(await anchoredArtifact.read("gene-windows.json")).windows : [];
|
|
854
|
+
const referenceSequence = mito ? parseFasta2(await anchoredArtifact.read("reference.fasta"))[0].sequence : "";
|
|
855
|
+
for (const a of anchored) {
|
|
856
|
+
const refusal = a.rotation && "refused" in a.rotation ? a.rotation.refused : void 0;
|
|
857
|
+
const assessed = refusal !== void 0 ? [{ severity: "error", code: "rotation-refused", message: `A circular, complete record that couldn't be turned to start where the reference starts, so it wasn't aligned: ${refusal}` }] : assessAlignment2(a.alignment, gatesForRecord(gates, { wholeRecord: a.extractedFrom === "whole-record", length: a.length, referenceLength: a.alignment?.aligned.length ?? 0 }));
|
|
858
|
+
const onlyFlank = a.located && a.alignment && a.alignment.clipped.start <= LOCATE_FLANK2 && a.alignment.clipped.end <= LOCATE_FLANK2;
|
|
859
|
+
const diagnostics = onlyFlank ? assessed.filter((d) => d.code !== "clipped-ends") : mito ? assessed.filter((d) => d.code !== "frame-shifting-indel") : assessed;
|
|
860
|
+
if (a.rotation && "shift" in a.rotation) diagnostics.push({ severity: "info", code: "rotated-to-reference", message: a.rotation.shift === 0 && a.rotation.strand === "+" ? "Circular and complete, and already starts where the reference starts" : `Circular and complete, so it was turned to start where the reference starts: moved by ${a.rotation.shift} bases${a.rotation.strand === "-" ? " and read on the other strand" : ""}` });
|
|
861
|
+
if (a.rotation && "notTurned" in a.rotation) diagnostics.push({ severity: "info", code: "not-rotated", message: a.rotation.notTurned });
|
|
862
|
+
if (mito && a.alignment) diagnostics.push(...checkGenes(a.alignment.aligned, referenceSequence, genes, gates.geneticCode).diagnostics);
|
|
863
|
+
const fail = (code, message) => diagnostics.push({ severity: "error", code, message });
|
|
864
|
+
const reason = excluded.get(a.record) ?? excluded.get(a.record.split(".")[0]);
|
|
865
|
+
if (reason) fail("excluded-by-profile", reason);
|
|
866
|
+
if (filters.minLength !== void 0 && a.length < filters.minLength) fail("too-short", `${a.length} bp is under the minimum of ${filters.minLength}`);
|
|
867
|
+
if (filters.maxLength !== void 0 && a.length > filters.maxLength) fail("too-long", `${a.length} bp is over the maximum of ${filters.maxLength}`);
|
|
868
|
+
if (filters.maxAmbiguous !== void 0 && a.ambiguous > filters.maxAmbiguous) fail("too-ambiguous", `${a.ambiguous} ambiguous bases is over the maximum of ${filters.maxAmbiguous}`);
|
|
869
|
+
if (a.unverified) fail("unverified", "NCBI marks this record UNVERIFIED");
|
|
870
|
+
if (a.located) diagnostics.push({ severity: "info", code: "located-by-similarity", message: `The record doesn't annotate this marker, so it was cut out by similarity: ${where(a.located)}. ${a.located.seedHits} shared seeds` });
|
|
871
|
+
if (a.manyPlaces) fail("marker-in-several-places", `The marker looks present in ${a.manyPlaces.length} places, so none was chosen: ${a.manyPlaces.map(where).join("; ")}`);
|
|
872
|
+
const passed = !diagnostics.some((d) => d.severity === "error");
|
|
873
|
+
outcomes.push({ record: a.record, ...a.organism ? { organism: a.organism } : {}, status: passed ? "passed" : "excluded", diagnostics, columns: a.alignment ? a.alignment.matches + a.alignment.mismatches : 0, identity: a.alignment?.identity ?? 0, extractedFrom: a.extractedFrom });
|
|
874
|
+
if (a.alignment && passed) aligned.push({ id: a.record, sequence: a.alignment.aligned });
|
|
875
|
+
}
|
|
876
|
+
const passing = outcomes.filter((o) => o.status === "passed").length;
|
|
877
|
+
scope.log("info", "gate", `${passing} records passed, ${outcomes.length - passing} excluded`);
|
|
878
|
+
const counts = /* @__PURE__ */ new Map();
|
|
879
|
+
for (const o of outcomes) for (const d of o.diagnostics) counts.set(d.code, (counts.get(d.code) ?? 0) + 1);
|
|
880
|
+
for (const [code, n] of counts) scope.log("info", "gate", `${code}: ${n}`);
|
|
881
|
+
await put("outcomes.json", json(outcomes));
|
|
882
|
+
await put("aligned.fasta", writeFasta(aligned));
|
|
883
|
+
await put("reference.fasta", await anchoredArtifact.read("reference.fasta"));
|
|
884
|
+
await put("records.csv", toRecordSheet(records, { checks: outcomes.map((o) => ({ record: o.record, status: o.status, reasons: o.diagnostics.map((d) => d.message) })) }));
|
|
885
|
+
}
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
async function fetchAndAnchor(ctx, profile, options = {}) {
|
|
889
|
+
const errors = validateProfile(profile).filter((d) => d.severity === "error");
|
|
890
|
+
if (errors.length) throw new EngineError("invalid-profile", `The profile can't run: ${errors.map((e) => e.message).join("; ")}`, "Fix it in Setup");
|
|
891
|
+
if (!profile.reference?.accession) throw new EngineError("invalid-profile", "The profile needs a reference accession to anchor records to", "Add one in Setup");
|
|
892
|
+
const P = profilePaths(profile.id);
|
|
893
|
+
const { value, run } = await withRun(ctx, profile, "fetch", async (scope) => {
|
|
894
|
+
scope.run.tools.push(ALIGNER);
|
|
895
|
+
const client = new NcbiClient({ ...ctx.ncbi, log: (level, message, data) => scope.log(level, "ncbi", message, data) });
|
|
896
|
+
const { artifact: list, accessions, term } = await searchStep(scope, profile, client, options);
|
|
897
|
+
const raw = await rawStep(scope, list, accessions, client, options.batchSize ?? 100);
|
|
898
|
+
const anchored = await anchorStep(scope, profile, raw);
|
|
899
|
+
const gated = await gateStep(scope, profile, anchored);
|
|
900
|
+
const records = JSON.parse(await anchored.read("records.json"));
|
|
901
|
+
const outcomes = JSON.parse(await gated.read("outcomes.json"));
|
|
902
|
+
const index = JSON.parse(await raw.read("index.json"));
|
|
903
|
+
const snapshot = {
|
|
904
|
+
schemaVersion: 1,
|
|
905
|
+
id: scope.run.id,
|
|
906
|
+
createdAt: scope.run.startedAt,
|
|
907
|
+
sources: [{ name: "NCBI nuccore", query: term, retrievedAt: scope.run.startedAt }],
|
|
908
|
+
records,
|
|
909
|
+
rawFiles: records.map((r) => ({ path: cachePath(r.version || r.accession), sha256: index[r.version || r.accession] ?? "" }))
|
|
910
|
+
};
|
|
911
|
+
snapshot.checksum = hashText(canonicalJson4(snapshot));
|
|
912
|
+
scope.run.snapshot = { id: scope.run.id, checksum: snapshot.checksum };
|
|
913
|
+
const publish = async (name, text) => {
|
|
914
|
+
await ctx.store.write(`${scope.dir}/${name}`, text);
|
|
915
|
+
scope.run.outputs.push({ path: name, sha256: hashText(text) });
|
|
916
|
+
};
|
|
917
|
+
await publish("snapshot.json", json(snapshot));
|
|
918
|
+
for (const name of ["records.csv", "outcomes.json", "aligned.fasta", "reference.fasta"]) await publish(name, await gated.read(name));
|
|
919
|
+
return { records, outcomes };
|
|
920
|
+
});
|
|
921
|
+
await ctx.store.write(P.latest, run.id);
|
|
922
|
+
return { run, records: value.records, outcomes: value.outcomes };
|
|
923
|
+
}
|
|
924
|
+
function spanOf(alignment) {
|
|
925
|
+
if (!alignment) return void 0;
|
|
926
|
+
const start = alignment.aligned.search(/[^-]/);
|
|
927
|
+
if (start < 0) return void 0;
|
|
928
|
+
return [start, alignment.aligned.length - (alignment.aligned.match(/-*$/)?.[0].length ?? 0)];
|
|
929
|
+
}
|
|
930
|
+
async function studyRecords(ctx, client, accessions, onProgress) {
|
|
931
|
+
const stop = () => {
|
|
932
|
+
if (ctx.signal?.aborted) throw new EngineError("cancelled", "Cancelled");
|
|
933
|
+
};
|
|
934
|
+
const out = [];
|
|
935
|
+
const missing = [];
|
|
936
|
+
for (const a of accessions) {
|
|
937
|
+
stop();
|
|
938
|
+
if (await ctx.store.exists(cachePath(a))) {
|
|
939
|
+
const r = parseGenBank(await ctx.store.read(cachePath(a)))[0];
|
|
940
|
+
if (r) {
|
|
941
|
+
out.push(r);
|
|
942
|
+
continue;
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
missing.push(a);
|
|
946
|
+
}
|
|
947
|
+
for (let i = 0; i < missing.length; i += 100) {
|
|
948
|
+
stop();
|
|
949
|
+
out.push(...(await client.fetchGenBankText(missing.slice(i, i + 100))).records);
|
|
950
|
+
onProgress?.(out.length, accessions.length);
|
|
951
|
+
}
|
|
952
|
+
return out;
|
|
953
|
+
}
|
|
954
|
+
async function studyReference(ctx, client, profile) {
|
|
955
|
+
const accession = profile.reference?.accession;
|
|
956
|
+
if (!accession) throw new EngineError("invalid-profile", "A Study places records on the reference, and this profile has none", "Add a reference accession in Setup");
|
|
957
|
+
const [rec] = await studyRecords(ctx, client, [accession]);
|
|
958
|
+
if (!rec) throw new EngineError("reference-missing", `The reference ${accession} wasn't retrieved`, "Check the reference accession in Setup");
|
|
959
|
+
return geneSequence(rec, profile).sequence;
|
|
960
|
+
}
|
|
961
|
+
var countryOf = (rec) => toSequenceRecord(rec).geoLocName?.split(":")[0]?.trim() || void 0;
|
|
962
|
+
async function placeRecords(ctx, profile, records, reference, classified) {
|
|
963
|
+
const placed = /* @__PURE__ */ new Map();
|
|
964
|
+
for (const rec of records) {
|
|
965
|
+
if (ctx.signal?.aborted) throw new EngineError("cancelled", "Cancelled");
|
|
966
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
967
|
+
const hasMarker = classified.hasMarker.has(rec.version);
|
|
968
|
+
const span = hasMarker ? spanOf(alignToReference2(geneSequence(rec, profile).sequence, reference)) : void 0;
|
|
969
|
+
const country = countryOf(rec);
|
|
970
|
+
placed.set(rec.version, { accession: rec.version, organism: rec.organism, length: rec.sequence.length, title: rec.definition, inTarget: classified.inTarget.has(rec.version), hasMarker, ...country ? { country } : {}, ...span ? { span } : {} });
|
|
971
|
+
}
|
|
972
|
+
return placed;
|
|
973
|
+
}
|
|
974
|
+
async function findRecordStudies(ctx, profile, onProgress) {
|
|
975
|
+
const client = new NcbiClient(ctx.ncbi);
|
|
976
|
+
const query = selectionQuery(profile.selection);
|
|
977
|
+
const genomes = genomesHoldingMarkerQuery(profile.selection);
|
|
978
|
+
const accessions = [.../* @__PURE__ */ new Set([...await client.searchAccessions(query), ...genomes ? await client.searchAccessions(genomes) : []])];
|
|
979
|
+
const reference = await studyReference(ctx, client, profile);
|
|
980
|
+
const records = await studyRecords(ctx, client, accessions, onProgress);
|
|
981
|
+
const all = new Set(records.map((r) => r.version));
|
|
982
|
+
const placed = await placeRecords(ctx, profile, records, reference, { inTarget: all, hasMarker: all });
|
|
983
|
+
return { papers: studiesFromRecords(records, (r) => placed.get(r.version)), columns: reference.length, query };
|
|
984
|
+
}
|
|
985
|
+
async function findPaperRecords(ctx, profile, pmid) {
|
|
986
|
+
const client = new NcbiClient(ctx.ncbi);
|
|
987
|
+
const accessions = await linkedAccessions(client, pmid);
|
|
988
|
+
const reference = await studyReference(ctx, client, profile);
|
|
989
|
+
if (!accessions.length) return { linked: [], columns: reference.length };
|
|
990
|
+
const classified = await classifyAccessions(client, accessions, profile.selection);
|
|
991
|
+
const records = await studyRecords(ctx, client, accessions);
|
|
992
|
+
return { linked: [...(await placeRecords(ctx, profile, records, reference, classified)).values()], columns: reference.length };
|
|
993
|
+
}
|
|
994
|
+
var defaultValidate = async (input) => fullValidation(input);
|
|
995
|
+
async function loadRunRecords(store, profileId, runId) {
|
|
996
|
+
const P = profilePaths(profileId);
|
|
997
|
+
const id = runId ?? (await store.exists(P.latest) ? (await store.read(P.latest)).trim() : "");
|
|
998
|
+
if (!id) throw new EngineError("run-missing", "No finished fetch run exists yet", "Fetch records first");
|
|
999
|
+
const run = JSON.parse(await store.read(`${P.run(id)}/run.json`));
|
|
1000
|
+
const gatedRef = artifactOf(run, "gated-records");
|
|
1001
|
+
if (!gatedRef) throw new EngineError("run-missing", `Run ${id} didn't finish fetching`, "Fetch records again");
|
|
1002
|
+
const gated = await loadArtifact(store, { type: "gated-records", key: gatedRef.key });
|
|
1003
|
+
return JSON.parse(await (await loadArtifact(store, gated.manifest.inputs[0])).read("records.json"));
|
|
1004
|
+
}
|
|
1005
|
+
async function buildPanelFromRun(ctx, profile, curation, id) {
|
|
1006
|
+
const settings = profile.panel;
|
|
1007
|
+
if (!settings) throw new EngineError("invalid-profile", "The profile has no panel settings", "Add groups in the Panel tab");
|
|
1008
|
+
if (!settings.rules) throw new EngineError("invalid-profile", "This profile predates evaluation rules, so a panel built from it would use values nobody chose", "Set the rules in the Panel tab and give the panel a new version");
|
|
1009
|
+
const P = profilePaths(profile.id);
|
|
1010
|
+
const sourceId = id ?? (await ctx.store.exists(P.latest) ? (await ctx.store.read(P.latest)).trim() : "");
|
|
1011
|
+
if (!sourceId) throw new EngineError("run-missing", "No finished fetch run exists yet", "Fetch records first");
|
|
1012
|
+
const source = JSON.parse(await ctx.store.read(`${P.run(sourceId)}/run.json`));
|
|
1013
|
+
const gatedRef = artifactOf(source, "gated-records");
|
|
1014
|
+
if (!gatedRef) throw new EngineError("run-missing", `Run ${sourceId} didn't finish fetching`, "Fetch records again");
|
|
1015
|
+
const { value, run } = await withRun(ctx, profile, "panel", async (scope) => {
|
|
1016
|
+
scope.run.tools.push(ALIGNER);
|
|
1017
|
+
const gated = await loadArtifact(ctx.store, { type: "gated-records", key: gatedRef.key });
|
|
1018
|
+
const aligned = parseFasta2(await gated.read("aligned.fasta"));
|
|
1019
|
+
const reference = parseFasta2(await gated.read("reference.fasta"))[0];
|
|
1020
|
+
const groupIds = new Set(settings.groups.map((g) => g.id));
|
|
1021
|
+
const anchoredArtifact = await loadArtifact(ctx.store, gated.manifest.inputs[0]);
|
|
1022
|
+
const records = JSON.parse(await anchoredArtifact.read("records.json"));
|
|
1023
|
+
const panelSettings = isMitogenome(profile) && !settings.windows?.length ? await geneWindowSettings(settings, anchoredArtifact, reference.id) : settings;
|
|
1024
|
+
const studyOf = new Map(records.map((r) => [r.version, r.publications.find((p) => p.title && p.title !== "Direct Submission")?.title]));
|
|
1025
|
+
const members = [];
|
|
1026
|
+
const diagnostics = [];
|
|
1027
|
+
let unassigned = 0;
|
|
1028
|
+
for (const e of aligned) {
|
|
1029
|
+
const decision = curation.records[e.id];
|
|
1030
|
+
if (decision?.exclude) continue;
|
|
1031
|
+
if (!decision?.group) {
|
|
1032
|
+
unassigned++;
|
|
1033
|
+
continue;
|
|
1034
|
+
}
|
|
1035
|
+
if (!groupIds.has(decision.group)) {
|
|
1036
|
+
diagnostics.push({ severity: "error", code: "unknown-group", message: `${e.id} names group ${decision.group}, which isn't in the panel settings`, ids: [e.id] });
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
1039
|
+
const study = decision.study ?? studyOf.get(e.id);
|
|
1040
|
+
members.push({ record: e.id, group: decision.group, aligned: e.sequence, provenanceClass: decision.provenanceClass ?? "unknown", ...decision.use ? { use: decision.use } : {}, ...decision.site ? { block: decision.site } : {}, ...decision.originSource ? { originSource: decision.originSource } : {}, ...study ? { study } : {} });
|
|
1041
|
+
}
|
|
1042
|
+
if (!members.length && !diagnostics.some((d) => d.severity === "error")) throw new EngineError("empty-panel", "No records are assigned to a group, so there is nothing to build", "Assign records to groups in the Records tab");
|
|
1043
|
+
const memberIds = new Set(members.map((m) => m.record));
|
|
1044
|
+
const citations = /* @__PURE__ */ new Map();
|
|
1045
|
+
for (const r of records) if (memberIds.has(r.version)) {
|
|
1046
|
+
for (const p of r.publications) if (p.title) citations.set(p.title, { citation: [p.authors, p.title, p.journal].filter(Boolean).join(". ") });
|
|
1047
|
+
}
|
|
1048
|
+
const numtPath = `profiles/${profile.id}/numt-screen/numt-library.fasta`;
|
|
1049
|
+
const numtCopies = await ctx.store.exists(numtPath) ? parseFasta2(await ctx.store.read(numtPath)).map((e) => ({ id: e.id, aligned: e.sequence })) : [];
|
|
1050
|
+
const input = {
|
|
1051
|
+
settings: panelSettings,
|
|
1052
|
+
taxa: profile.selection.taxa,
|
|
1053
|
+
marker: profile.selection.marker,
|
|
1054
|
+
reference: { accession: reference.id, length: reference.sequence.length, ...profile.reference?.codonStart !== void 0 ? { codonStart: profile.reference.codonStart } : {}, ...profile.reference?.geneticCode ? { geneticCode: profile.reference.geneticCode } : {} },
|
|
1055
|
+
sources: [...citations.values()],
|
|
1056
|
+
builtBy: gated.manifest.key,
|
|
1057
|
+
referenceSequence: reference.sequence,
|
|
1058
|
+
builtWith: [ENGINE, ALIGNER],
|
|
1059
|
+
// The checks that actually ran on these records, even if the profile has changed since.
|
|
1060
|
+
gates: gated.manifest.parameters.gates,
|
|
1061
|
+
...numtCopies.length ? { numtCopies } : {},
|
|
1062
|
+
// From the fetch these records came from, not today's profile, which may have moved to another Study.
|
|
1063
|
+
...source.profile.selection.fromStudy ? { study: source.profile.selection.fromStudy } : {}
|
|
1064
|
+
};
|
|
1065
|
+
if (settings.validatedScope.length) diagnostics.push({ severity: "warning", code: "hand-entered-scope-ignored", message: "The scope written in the panel settings is ignored. A panel's scope now comes only from claims that pass validation", path: "panel.validatedScope" });
|
|
1066
|
+
const draft = buildPanel({ ...input, members });
|
|
1067
|
+
diagnostics.push(...validatePanel(draft.panel).filter((d) => d.code !== "not-validated"));
|
|
1068
|
+
const empty = { scheme: "leave-one-block-out", tested: 0, skipped: 0, groups: [], untestedGroups: [], diagnostics: [] };
|
|
1069
|
+
const hasErrors = diagnostics.some((d) => d.severity === "error");
|
|
1070
|
+
if (hasErrors) return { bundle: draft, diagnostics, validation: { blocks: empty, groups: { ...empty, scheme: "leave-one-group-out" }, ladder: [], claims: [], controls: { shuffles: 0, simulations: 0, seed: 0 } }, unassigned, published: false };
|
|
1071
|
+
const fingerprint = hashText(canonicalJson4({ settings: { ...panelSettings, version: "", id: "" }, members: members.map(({ aligned: aligned2, ...m }) => ({ ...m, aligned: hashText(aligned2) })) }));
|
|
1072
|
+
const validationArtifact = await scope.step({
|
|
1073
|
+
type: "validation",
|
|
1074
|
+
step: { name: "validate", version: 3 },
|
|
1075
|
+
inputs: [ref(gated)],
|
|
1076
|
+
parameters: { fingerprint },
|
|
1077
|
+
run: async (put) => {
|
|
1078
|
+
const reports = await (ctx.compute?.validate ?? defaultValidate)({ ...input, members }, ctx.signal);
|
|
1079
|
+
await put("validation.json", json(reports));
|
|
1080
|
+
}
|
|
1081
|
+
});
|
|
1082
|
+
const validation = JSON.parse(await validationArtifact.read("validation.json"));
|
|
1083
|
+
const bundle = buildPanel({ ...input, members, claims: validatedScopeFromReport(validation.claims) });
|
|
1084
|
+
if (!bundle.panel.claims.length) diagnostics.push({ severity: "warning", code: "not-validated", message: "No claim passed validation, so this panel has no validated scope", path: "validatedScope" });
|
|
1085
|
+
const out = P.panel(bundle.panel.id, bundle.panel.version);
|
|
1086
|
+
if (await ctx.store.exists(`${out}/panel.json`)) {
|
|
1087
|
+
const existing = JSON.parse(await ctx.store.read(`${out}/panel.json`));
|
|
1088
|
+
if (existing.checksum !== bundle.panel.checksum) throw new EngineError("panel-version-exists", `Panel ${bundle.panel.id} version ${bundle.panel.version} already exists with different content`, "Give this build a new version, so earlier results stay true");
|
|
1089
|
+
}
|
|
1090
|
+
const panelArtifact = await scope.step({
|
|
1091
|
+
type: "panel",
|
|
1092
|
+
step: { name: "panel", version: 2 },
|
|
1093
|
+
inputs: [ref(gated), ref(validationArtifact)],
|
|
1094
|
+
parameters: { checksum: bundle.panel.checksum },
|
|
1095
|
+
tools: [ENGINE, ALIGNER],
|
|
1096
|
+
run: async (put) => {
|
|
1097
|
+
await put("panel.json", json(bundle.panel));
|
|
1098
|
+
await put("panel.fasta", writeFasta([{ id: "__reference__", sequence: reference.sequence }, ...Object.entries(bundle.sequences).map(([k, sequence]) => ({ id: k, sequence }))]));
|
|
1099
|
+
}
|
|
1100
|
+
});
|
|
1101
|
+
await ctx.store.write(`${out}/panel.json`, await panelArtifact.read("panel.json"));
|
|
1102
|
+
await ctx.store.write(`${out}/panel.fasta`, await panelArtifact.read("panel.fasta"));
|
|
1103
|
+
await ctx.store.write(`${out}/validation.json`, json(validation));
|
|
1104
|
+
await ctx.store.write(`${out}/diagnostics.json`, json({ diagnostics, unassigned }));
|
|
1105
|
+
await ctx.store.write(`${out}/artifact.json`, json(ref(panelArtifact)));
|
|
1106
|
+
await writeDataset(ctx.store, out, bundle, testSample);
|
|
1107
|
+
return { bundle, diagnostics, validation, unassigned, published: true };
|
|
1108
|
+
});
|
|
1109
|
+
return { ...value, run };
|
|
1110
|
+
}
|
|
1111
|
+
async function geneWindowSettings(settings, anchored, accession) {
|
|
1112
|
+
let gw;
|
|
1113
|
+
try {
|
|
1114
|
+
gw = JSON.parse(await anchored.read("gene-windows.json"));
|
|
1115
|
+
} catch {
|
|
1116
|
+
throw new EngineError("reference-unannotated", `The gene annotation of the reference ${accession} couldn't be read from this fetch run, so the panel has no gene windows`, "Fetch records again with this mitogenome profile, then build the panel");
|
|
1117
|
+
}
|
|
1118
|
+
if (!gw.windows.length) throw new EngineError("reference-unannotated", `The reference ${accession} has no usable gene annotation, so a mitogenome panel would have no windows${gw.leftOut.length ? `. Left out: ${gw.leftOut.map((g) => `${g.name} (${g.reason})`).join("; ")}` : ""}`, "Choose an annotated, complete RefSeq mitogenome as the reference in Setup");
|
|
1119
|
+
return { ...settings, windows: gw.windows, notCovered: [.../* @__PURE__ */ new Set([...settings.notCovered, ...leftOutLines(gw)])] };
|
|
1120
|
+
}
|
|
1121
|
+
async function loadPanel(store, dir) {
|
|
1122
|
+
if (!await store.exists(`${dir}/panel.json`)) throw new EngineError("panel-missing", "This panel hasn't been built yet", "Build it in the Panel tab");
|
|
1123
|
+
const panel = JSON.parse(await store.read(`${dir}/panel.json`));
|
|
1124
|
+
if (panel.schemaVersion !== 2) throw new EngineError("panel-outdated", `Panel ${panel.id} version ${panel.version} was built with an earlier format, so it can't be read by the current evaluator`, "Build it again under a new version");
|
|
1125
|
+
const fasta = parseFasta2(await store.read(`${dir}/panel.fasta`));
|
|
1126
|
+
const referenceSequence = fasta.find((f) => f.id === "__reference__")?.sequence;
|
|
1127
|
+
const bundle = { panel, sequences: Object.fromEntries(fasta.filter((f) => f.id !== "__reference__").map((f) => [f.id, f.sequence])), ...referenceSequence ? { referenceSequence } : {} };
|
|
1128
|
+
const problems = verifyBundle(bundle);
|
|
1129
|
+
if (problems.length) throw new EngineError("panel-changed", `Panel ${panel.id} version ${panel.version} doesn't match its seal: ${problems.map((p) => p.message).join("; ")}`, "Restore it from its source, or build it again under a new version");
|
|
1130
|
+
return bundle;
|
|
1131
|
+
}
|
|
1132
|
+
function panelVersions(profile, folders) {
|
|
1133
|
+
const prefix = `${profile.panel?.id ?? ""}-`;
|
|
1134
|
+
const parts = (v) => v.split(".").map(Number);
|
|
1135
|
+
const order = (a, b) => {
|
|
1136
|
+
const x = parts(a), y = parts(b);
|
|
1137
|
+
for (let i = 0; i < Math.max(x.length, y.length); i++) {
|
|
1138
|
+
const d = (x[i] ?? 0) - (y[i] ?? 0);
|
|
1139
|
+
if (d) return d;
|
|
1140
|
+
}
|
|
1141
|
+
return a.localeCompare(b);
|
|
1142
|
+
};
|
|
1143
|
+
return folders.filter((f) => f.startsWith(prefix)).map((f) => f.slice(prefix.length)).sort(order);
|
|
1144
|
+
}
|
|
1145
|
+
async function loadVersions(store, profile, a, b) {
|
|
1146
|
+
if (!profile.panel) throw new EngineError("invalid-profile", "The profile has no panel settings", "Add groups in the Panel tab");
|
|
1147
|
+
const P = profilePaths(profile.id);
|
|
1148
|
+
const load = async (version) => {
|
|
1149
|
+
const dir = P.panel(profile.panel.id, version);
|
|
1150
|
+
const { panel } = await loadPanel(store, dir);
|
|
1151
|
+
const validation = await store.exists(`${dir}/validation.json`) ? JSON.parse(await store.read(`${dir}/validation.json`)) : void 0;
|
|
1152
|
+
return { panel, validation };
|
|
1153
|
+
};
|
|
1154
|
+
return [await load(a), await load(b)];
|
|
1155
|
+
}
|
|
1156
|
+
async function comparePanelVersions(store, profile, a, b) {
|
|
1157
|
+
const [x, y] = await loadVersions(store, profile, a, b);
|
|
1158
|
+
return compareRuns(x.panel, y.panel, x.validation?.claims, y.validation?.claims);
|
|
1159
|
+
}
|
|
1160
|
+
async function panelVersionComparison(store, profile, a, b) {
|
|
1161
|
+
const [x, y] = await loadVersions(store, profile, a, b);
|
|
1162
|
+
return comparePanels(x.panel, y.panel, x.validation, y.validation);
|
|
1163
|
+
}
|
|
1164
|
+
var geneQcDiagnostics = (diagnostics) => [
|
|
1165
|
+
...diagnostics.filter((d) => d.code !== "internal-stop"),
|
|
1166
|
+
...diagnostics.some((d) => d.code === "gene-frame") ? [{ severity: "warning", code: "frame-shifting-indel", message: "A protein-coding gene is out of frame" }] : [],
|
|
1167
|
+
...diagnostics.some((d) => d.code === "gene-stops") ? [{ severity: "warning", code: "internal-stop", message: "A protein-coding gene has an internal stop" }] : []
|
|
1168
|
+
];
|
|
1169
|
+
function testSample(sequence, queryId, bundle, options = {}) {
|
|
1170
|
+
if (!bundle.referenceSequence) throw new EngineError("panel-outdated", "This panel has no reference sequence", "Build it again");
|
|
1171
|
+
const part = markerInSample(sequence, bundle);
|
|
1172
|
+
const marker = bundle.panel.marker.gene ?? "The marker";
|
|
1173
|
+
if (part.manyPlaces) return { result: null, diagnostics: [{ severity: "error", code: "marker-in-many-places", message: `${marker} matches ${part.manyPlaces.length} places in this sample, so which one to test is ambiguous` }] };
|
|
1174
|
+
const located = part.located ? [{ severity: "info", code: "marker-located", message: `${placeText(marker, part.located)}; only that part was tested` }] : [];
|
|
1175
|
+
const a = alignToReference2(part.sequence, bundle.referenceSequence);
|
|
1176
|
+
const mito = bundle.panel.marker.kind === "mitogenome";
|
|
1177
|
+
const assessed = assessAlignment2(a, { ...bundle.panel.gates, frameErrors: false });
|
|
1178
|
+
const diagnostics = mito ? assessed.filter((d) => d.code !== "frame-shifting-indel") : assessed;
|
|
1179
|
+
if (!a || diagnostics.some((d) => d.severity === "error")) return { result: null, diagnostics: [...located, ...diagnostics] };
|
|
1180
|
+
const genes = mito ? checkGenes(a.aligned, bundle.referenceSequence, bundle.panel.windows, bundle.panel.reference.geneticCode) : void 0;
|
|
1181
|
+
if (genes) diagnostics.push(...genes.diagnostics);
|
|
1182
|
+
const numtCaveats = [];
|
|
1183
|
+
if (bundle.panel.numtCopies?.length) {
|
|
1184
|
+
const numt = checkNumts(a.aligned, bundle.panel.numtCopies, bundle.referenceSequence);
|
|
1185
|
+
for (const f of numt.flagged) diagnostics.push({ severity: "warning", code: "numt-flagged", message: `This sequence is closer to nuclear copy ${f.copy} than to the reference, over ${f.shared} columns (${(f.toCopy * 100).toFixed(1)}% against ${(f.toReference * 100).toFixed(1)}%). Review it. This doesn't prove it's a nuclear copy` });
|
|
1186
|
+
numtCaveats.push(numt.flagged.length ? "Flagged for review as possibly a nuclear copy of the marker" : `Compared with ${numt.tested} of ${bundle.panel.numtCopies.length} known nuclear copies with no flag. That doesn't clear it, since copies absent from the screened genome can't be seen`);
|
|
1187
|
+
}
|
|
1188
|
+
const result = evaluateQuery(a.aligned, bundle, {
|
|
1189
|
+
queryId,
|
|
1190
|
+
runId: options.runId ?? "sample",
|
|
1191
|
+
querySha256: hashText(sequence.toUpperCase()),
|
|
1192
|
+
producedBy: [ENGINE, ALIGNER],
|
|
1193
|
+
caveats: ["Rules are configured values and these statuses are not probabilities", ...located.map((d) => d.message), ...diagnostics.filter((d) => d.severity === "warning").map((d) => d.message), ...numtCaveats]
|
|
1194
|
+
});
|
|
1195
|
+
result.qc = sampleQc(a.aligned, genes ? geneQcDiagnostics(diagnostics) : diagnostics, bundle.sequences, bundle.panel.qc);
|
|
1196
|
+
if (genes?.checks.length) result.qc.genes = genes.checks;
|
|
1197
|
+
diagnostics.push(...validateResult(result, bundle.panel));
|
|
1198
|
+
return { result, diagnostics: [...located, ...diagnostics], aligned: a.aligned, insertions: a.insertions.reduce((s, x) => s + x.length, 0), deletions: a.deletions.reduce((s, x) => s + x.length, 0), identity: a.identity, clipped: a.clipped };
|
|
1199
|
+
}
|
|
1200
|
+
async function runSample(ctx, profile, sequence, queryId) {
|
|
1201
|
+
if (!profile.panel) throw new EngineError("invalid-profile", "The profile has no panel settings", "Add groups in the Panel tab");
|
|
1202
|
+
const P = profilePaths(profile.id);
|
|
1203
|
+
const dir = P.panel(profile.panel.id, profile.panel.version);
|
|
1204
|
+
const bundle = await loadPanel(ctx.store, dir);
|
|
1205
|
+
const panelRef = JSON.parse(await ctx.store.read(`${dir}/artifact.json`));
|
|
1206
|
+
const { value, run } = await withRun(ctx, profile, "sample", async (scope) => {
|
|
1207
|
+
scope.run.tools.push(ALIGNER);
|
|
1208
|
+
const panelArtifact = await loadArtifact(ctx.store, panelRef);
|
|
1209
|
+
if (JSON.parse(await panelArtifact.read("panel.json")).checksum !== bundle.panel.checksum) throw new EngineError("panel-changed", `The saved copy of panel ${bundle.panel.id} isn't the one its record traces back to`, "Build it again under a new version");
|
|
1210
|
+
const artifact = await scope.step({
|
|
1211
|
+
type: "sample-result",
|
|
1212
|
+
step: { name: "sample", version: 1 },
|
|
1213
|
+
inputs: [ref(panelArtifact)],
|
|
1214
|
+
parameters: { queryId, sequence: hashText(sequence.toUpperCase()) },
|
|
1215
|
+
tools: [ENGINE, ALIGNER],
|
|
1216
|
+
run: async (put) => {
|
|
1217
|
+
await put("outcome.json", json(testSample(sequence, queryId, bundle, { runId: scope.run.id })));
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1220
|
+
const text = await artifact.read("outcome.json");
|
|
1221
|
+
await ctx.store.write(`${P.results}/${bundle.panel.id}-${bundle.panel.version}/${queryId.replace(/[^A-Za-z0-9._-]/g, "_")}.json`, text);
|
|
1222
|
+
return { outcome: JSON.parse(text), artifact: ref(artifact) };
|
|
1223
|
+
});
|
|
1224
|
+
return { ...value.outcome, run, artifact: value.artifact };
|
|
1225
|
+
}
|
|
1226
|
+
async function buildTree(ctx, profile, options) {
|
|
1227
|
+
if (!ctx.tools) throw new EngineError("tool-missing", "Building a tree needs a tool runner");
|
|
1228
|
+
const settings = profile.tree;
|
|
1229
|
+
if (!settings) throw new EngineError("invalid-profile", "The profile has no tree settings");
|
|
1230
|
+
const tool = options.tool ?? "iqtree3";
|
|
1231
|
+
const version = await ctx.tools.version(tool);
|
|
1232
|
+
if (!version) throw new EngineError("tool-missing", `${tool} isn't installed or can't be found`, `Install it with: brew install ${tool}`);
|
|
1233
|
+
const tools = ctx.tools;
|
|
1234
|
+
const P = profilePaths(profile.id);
|
|
1235
|
+
const sourceId = options.runId ?? (await ctx.store.exists(P.latest) ? (await ctx.store.read(P.latest)).trim() : "");
|
|
1236
|
+
if (!sourceId) throw new EngineError("run-missing", "No finished fetch run exists yet", "Fetch records first");
|
|
1237
|
+
const source = JSON.parse(await ctx.store.read(`${P.run(sourceId)}/run.json`));
|
|
1238
|
+
const gatedRef = artifactOf(source, "gated-records");
|
|
1239
|
+
if (!gatedRef) throw new EngineError("run-missing", `Run ${sourceId} didn't finish fetching`, "Fetch records again");
|
|
1240
|
+
const { value, run } = await withRun(ctx, profile, "tree", async (scope) => {
|
|
1241
|
+
scope.run.tools.push({ name: tool, version });
|
|
1242
|
+
const gated = await loadArtifact(ctx.store, { type: "gated-records", key: gatedRef.key });
|
|
1243
|
+
const aligned = parseFasta2(await gated.read("aligned.fasta"));
|
|
1244
|
+
const w = options.window ?? { start: 0, end: aligned[0]?.sequence.length ?? 0 };
|
|
1245
|
+
const min = options.minCoverage ?? 0.85;
|
|
1246
|
+
const usable = aligned.map((e) => ({ id: e.id, aligned: e.sequence.slice(w.start, w.end) })).filter((e) => (e.aligned.match(/[ACGT]/gi) ?? []).length >= min * (w.end - w.start));
|
|
1247
|
+
const haps = collapseHaplotypes(usable);
|
|
1248
|
+
if (haps.length < 4) throw new EngineError("step-output-invalid", `Only ${haps.length} haplotypes cover this window, and a tree needs at least 4`, "Choose another window, or lower the coverage needed");
|
|
1249
|
+
const outgroups = profile.selection.outgroups.map((o) => o.accession).filter((a) => Boolean(a));
|
|
1250
|
+
const names = new Map(haps.map((h, i) => [h, `H${String(i + 1).padStart(4, "0")}`]));
|
|
1251
|
+
const og = haps.filter((h) => h.members.some((m) => outgroups.includes(m))).map((h) => names.get(h));
|
|
1252
|
+
const partitionMode = isMitogenome(profile) ? settings.partitions ?? "codons" : "none";
|
|
1253
|
+
let geneWindowsForTree = null;
|
|
1254
|
+
if (partitionMode !== "none") {
|
|
1255
|
+
try {
|
|
1256
|
+
const anchoredArtifact = await loadArtifact(ctx.store, gated.manifest.inputs[0]);
|
|
1257
|
+
geneWindowsForTree = JSON.parse(await anchoredArtifact.read("gene-windows.json")).windows;
|
|
1258
|
+
} catch {
|
|
1259
|
+
scope.log("warning", "tree", "This run's gene windows couldn't be read, so the tree isn't partitioned by gene");
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
const artifact = await scope.step({
|
|
1263
|
+
type: "tree",
|
|
1264
|
+
step: { name: "tree", version: 1 },
|
|
1265
|
+
inputs: [ref(gated)],
|
|
1266
|
+
parameters: { window: w, minCoverage: min, tree: settings, outgroups: og, ...partitionMode !== "none" ? { partitions: partitionMode } : {} },
|
|
1267
|
+
tools: [{ name: tool, version }],
|
|
1268
|
+
run: async (put, dir, register) => {
|
|
1269
|
+
await put("input.fasta", writeFasta(haps.map((h) => ({ id: names.get(h), sequence: h.representative }))));
|
|
1270
|
+
await put("tips.json", json(haps.map((h) => ({ id: names.get(h), members: h.members }))));
|
|
1271
|
+
let partitioned = false;
|
|
1272
|
+
if (partitionMode !== "none" && geneWindowsForTree) {
|
|
1273
|
+
await put("partitions.nex", partitionNexus(geneCharsets(geneWindowsForTree, w, { codonPositions: partitionMode === "codons" })));
|
|
1274
|
+
partitioned = true;
|
|
1275
|
+
}
|
|
1276
|
+
const args = ["-s", "input.fasta", ...partitioned ? ["-p", "partitions.nex"] : [], "-m", settings.model, "-B", String(settings.bootstrap), "--alrt", String(settings.shAlrt), "--seed", String(settings.seed), "-T", String(settings.threads), "-pre", "tree", "-redo", ...og.length ? ["-o", og.join(",")] : []];
|
|
1277
|
+
const res = await tools.run(tool, args, { cwd: `${options.absoluteDir}/${dir}` });
|
|
1278
|
+
await ctx.store.write(`${dir}/tool.log`, `${res.stdout}
|
|
1279
|
+
${res.stderr}`);
|
|
1280
|
+
if (res.code !== 0) throw new EngineError("tool-failed", `${tool} failed: ${(res.stderr || res.stdout).slice(0, 600)}`, `See ${dir}/tool.log`);
|
|
1281
|
+
await register("tree.treefile");
|
|
1282
|
+
await register("tree.iqtree");
|
|
1283
|
+
},
|
|
1284
|
+
ensure: async (read) => (await read("tree.treefile")).includes("(") ? null : "the tree file has no tree in it"
|
|
1285
|
+
});
|
|
1286
|
+
return { newick: await artifact.read("tree.treefile"), tips: JSON.parse(await artifact.read("tips.json")), artifact: ref(artifact) };
|
|
1287
|
+
});
|
|
1288
|
+
return { tool, version, ...value, run };
|
|
1289
|
+
}
|
|
1290
|
+
export {
|
|
1291
|
+
DATASET_FILES,
|
|
1292
|
+
ENGINE_VERSION,
|
|
1293
|
+
EngineError,
|
|
1294
|
+
buildPanelFromRun,
|
|
1295
|
+
buildTree,
|
|
1296
|
+
checkExamples,
|
|
1297
|
+
chooseExamples,
|
|
1298
|
+
comparePanelVersions,
|
|
1299
|
+
curationDiff,
|
|
1300
|
+
datasetReadme,
|
|
1301
|
+
expectationOf,
|
|
1302
|
+
fetchAndAnchor,
|
|
1303
|
+
findPaperRecords,
|
|
1304
|
+
findRecordStudies,
|
|
1305
|
+
historyPath,
|
|
1306
|
+
isCancelled,
|
|
1307
|
+
loadArtifact,
|
|
1308
|
+
loadPanel,
|
|
1309
|
+
loadRunRecords,
|
|
1310
|
+
makeExamples,
|
|
1311
|
+
markerInSample,
|
|
1312
|
+
panelVersionComparison,
|
|
1313
|
+
panelVersions,
|
|
1314
|
+
placeText,
|
|
1315
|
+
profilePaths,
|
|
1316
|
+
readCurationHistory,
|
|
1317
|
+
routeSample,
|
|
1318
|
+
runHypothesis,
|
|
1319
|
+
runSample,
|
|
1320
|
+
saveCuration,
|
|
1321
|
+
testHypothesis,
|
|
1322
|
+
testHypothesisOnPanel,
|
|
1323
|
+
testSample,
|
|
1324
|
+
traceArtifact,
|
|
1325
|
+
verifyDataset,
|
|
1326
|
+
writeDataset
|
|
1327
|
+
};
|