@inflexa-ai/harness 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/synthesize-run.d.ts +3 -0
- package/dist/app/synthesize-run.js +17 -3
- package/dist/execution/post-step-pipeline.d.ts +14 -3
- package/dist/execution/post-step-pipeline.js +72 -44
- package/dist/execution/reconcile-manifest.d.ts +3 -3
- package/dist/execution/reconcile-manifest.js +27 -14
- package/dist/execution/run-synthesis.d.ts +18 -0
- package/dist/execution/run-synthesis.js +25 -10
- package/dist/lib/metrics.d.ts +2 -0
- package/dist/lib/metrics.js +6 -0
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +12 -5
- package/dist/prompts/synthesis-agent.d.ts +1 -1
- package/dist/prompts/synthesis-agent.js +14 -3
- package/dist/provenance/collector.d.ts +8 -5
- package/dist/provenance/collector.js +14 -6
- package/dist/provenance/exec-frame.d.ts +7 -1
- package/dist/provenance/exec-frame.js +8 -2
- package/dist/providers/errors.d.ts +11 -2
- package/dist/providers/errors.js +23 -2
- package/dist/state/index.d.ts +2 -2
- package/dist/state/index.js +2 -2
- package/dist/state/init.js +6 -0
- package/dist/state/runs.d.ts +7 -1
- package/dist/state/runs.js +21 -4
- package/dist/state/schema.d.ts +14 -0
- package/dist/state/schema.js +9 -0
- package/dist/tools/research/inspect-run.d.ts +4 -2
- package/dist/tools/research/inspect-run.js +12 -4
- package/dist/workflows/execute-analysis.d.ts +3 -0
- package/dist/workflows/execute-analysis.js +24 -1
- package/package.json +1 -1
|
@@ -22,6 +22,7 @@ import type { ChatProvider, EmbeddingProvider } from "../providers/types.js";
|
|
|
22
22
|
import type { BioToolKeys } from "../tools/bio/keys.js";
|
|
23
23
|
import type { RunFinding } from "../workflows/execute-analysis.js";
|
|
24
24
|
import type { ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
25
|
+
import type { SynthesisStatus } from "../state/index.js";
|
|
25
26
|
import type { Logger } from "../lib/logger.js";
|
|
26
27
|
export interface SynthesizeRunDeps {
|
|
27
28
|
/** Operational logging seam; omitted falls back to no-op. */
|
|
@@ -56,5 +57,7 @@ export interface SynthesizeRunParams {
|
|
|
56
57
|
}
|
|
57
58
|
export interface SynthesizeRunResult {
|
|
58
59
|
readonly findings: readonly RunFinding[];
|
|
60
|
+
readonly synthesisStatus: Exclude<SynthesisStatus, "failed">;
|
|
61
|
+
readonly synthesisReason: string | null;
|
|
59
62
|
}
|
|
60
63
|
export declare function synthesizeRun(deps: SynthesizeRunDeps, params: SynthesizeRunParams): Promise<SynthesizeRunResult>;
|
|
@@ -33,10 +33,12 @@ export async function synthesizeRun(deps, params) {
|
|
|
33
33
|
completedSteps,
|
|
34
34
|
});
|
|
35
35
|
if (summaries.length === 0) {
|
|
36
|
+
const logger = (deps.logger ?? createNoopLogger()).named("synthesize-run");
|
|
37
|
+
logger.warn("synthesis skipped — no step summaries on disk", { runId, reason: "no-summaries" });
|
|
36
38
|
await onProgress("skipped", "No step summaries available for synthesis", {
|
|
37
39
|
reason: "no-summaries",
|
|
38
40
|
});
|
|
39
|
-
return { findings: [] };
|
|
41
|
+
return { findings: [], synthesisStatus: "skipped_no_summaries", synthesisReason: "no-summaries" };
|
|
40
42
|
}
|
|
41
43
|
const planNarrative = await loadPlanNarrative(pool, runId, analysisId);
|
|
42
44
|
const synthesizerSession = forSubAgent(session, "run-synthesizer");
|
|
@@ -51,8 +53,18 @@ export async function synthesizeRun(deps, params) {
|
|
|
51
53
|
emit,
|
|
52
54
|
});
|
|
53
55
|
if (result.kind === "skipped") {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
const logger = (deps.logger ?? createNoopLogger()).named("synthesize-run");
|
|
57
|
+
logger.warn("synthesis skipped — synthesizer reported a blocker", {
|
|
58
|
+
runId,
|
|
59
|
+
reason: result.reason,
|
|
60
|
+
validationRejections: result.validationRejections,
|
|
61
|
+
rejectedIssuePaths: result.rejectedIssuePaths,
|
|
62
|
+
});
|
|
63
|
+
await onProgress("skipped", "Synthesis skipped — run did not produce synthesizable content", {
|
|
64
|
+
reason: result.reason,
|
|
65
|
+
validationAttempts: result.validationRejections,
|
|
66
|
+
});
|
|
67
|
+
return { findings: [], synthesisStatus: "skipped_blocker", synthesisReason: result.reason };
|
|
56
68
|
}
|
|
57
69
|
const synthesis = result.synthesis;
|
|
58
70
|
await onProgress("indexing", "Indexing synthesis for search");
|
|
@@ -84,6 +96,8 @@ export async function synthesizeRun(deps, params) {
|
|
|
84
96
|
title: f.title,
|
|
85
97
|
confidence: f.confidence,
|
|
86
98
|
})),
|
|
99
|
+
synthesisStatus: "produced",
|
|
100
|
+
synthesisReason: null,
|
|
87
101
|
};
|
|
88
102
|
}
|
|
89
103
|
catch (err) {
|
|
@@ -60,9 +60,20 @@ export declare function generateStepSummaryAndWrite(deps: PostStepPipelineDeps,
|
|
|
60
60
|
*/
|
|
61
61
|
export declare function reconcileAndRegisterStepArtifacts(deps: PostStepPipelineDeps, postCtx: PostStepContext, manifest: readonly ArtifactManifestEntry[]): Promise<readonly ArtifactManifestEntry[]>;
|
|
62
62
|
/**
|
|
63
|
-
* Vector-index the threaded file descriptions + summary into the
|
|
64
|
-
* pgvector store. Best-effort (see the artifact-manifest spec):
|
|
65
|
-
* without failing the step
|
|
63
|
+
* Vector-index the threaded file descriptions + the step summary into the
|
|
64
|
+
* per-analysis pgvector store. Best-effort (see the artifact-manifest spec):
|
|
65
|
+
* indexing degrades without failing the step.
|
|
66
|
+
*
|
|
67
|
+
* Degradation is per-item. Index setup (ensuring the index exists and building
|
|
68
|
+
* the store) is all-or-nothing — a setup failure logs and skips the stage,
|
|
69
|
+
* because without an index there is nowhere to index into. Past setup, each
|
|
70
|
+
* file description and the summary is embedded and upserted under its own
|
|
71
|
+
* failure boundary, so one rejected input (e.g. an over-length document the
|
|
72
|
+
* embedding backend rejects) costs only its own entry and every other item is
|
|
73
|
+
* still attempted. A partial index is invisible at the search surface — it
|
|
74
|
+
* returns fewer hits, never an error — so each per-item failure logs the item
|
|
75
|
+
* id and input text length, and when any item fails a final record carries the
|
|
76
|
+
* indexed/failed counts as the only signal that degradation occurred.
|
|
66
77
|
*/
|
|
67
78
|
export declare function vectorIndexStepOutputs(deps: PostStepPipelineDeps, postCtx: PostStepContext, artifacts: PostStepArtifacts): Promise<void>;
|
|
68
79
|
/**
|
|
@@ -144,60 +144,88 @@ export async function reconcileAndRegisterStepArtifacts(deps, postCtx, manifest)
|
|
|
144
144
|
return reconciled.manifest;
|
|
145
145
|
}
|
|
146
146
|
/**
|
|
147
|
-
* Vector-index the threaded file descriptions + summary into the
|
|
148
|
-
* pgvector store. Best-effort (see the artifact-manifest spec):
|
|
149
|
-
* without failing the step
|
|
147
|
+
* Vector-index the threaded file descriptions + the step summary into the
|
|
148
|
+
* per-analysis pgvector store. Best-effort (see the artifact-manifest spec):
|
|
149
|
+
* indexing degrades without failing the step.
|
|
150
|
+
*
|
|
151
|
+
* Degradation is per-item. Index setup (ensuring the index exists and building
|
|
152
|
+
* the store) is all-or-nothing — a setup failure logs and skips the stage,
|
|
153
|
+
* because without an index there is nowhere to index into. Past setup, each
|
|
154
|
+
* file description and the summary is embedded and upserted under its own
|
|
155
|
+
* failure boundary, so one rejected input (e.g. an over-length document the
|
|
156
|
+
* embedding backend rejects) costs only its own entry and every other item is
|
|
157
|
+
* still attempted. A partial index is invisible at the search surface — it
|
|
158
|
+
* returns fewer hits, never an error — so each per-item failure logs the item
|
|
159
|
+
* id and input text length, and when any item fails a final record carries the
|
|
160
|
+
* indexed/failed counts as the only signal that degradation occurred.
|
|
150
161
|
*/
|
|
151
162
|
export async function vectorIndexStepOutputs(deps, postCtx, artifacts) {
|
|
152
163
|
const { input } = postCtx;
|
|
153
164
|
const { metadataEntries, summary, reconciledManifest } = artifacts;
|
|
165
|
+
const logger = (deps.logger ?? createNoopLogger()).named("post-step").named("vector-index").with({ runId: input.runId, stepId: input.stepId });
|
|
166
|
+
let vectorStore;
|
|
167
|
+
let indexName;
|
|
154
168
|
try {
|
|
169
|
+
// Setup is all-or-nothing: without an index there is nothing to index into.
|
|
155
170
|
await ensureSearchIndex(deps.pool, input.analysisId, deps.embedding.dimensions);
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
const
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
171
|
+
vectorStore = createVectorStore(deps.pool);
|
|
172
|
+
indexName = searchIndexName(input.analysisId);
|
|
173
|
+
}
|
|
174
|
+
catch (err) {
|
|
175
|
+
logger.warn("indexing failed", logger.errorFields(err));
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
const embedOne = async (text) => {
|
|
179
|
+
const [vec] = unwrapOrThrow(await deps.embedding.embed([text], postCtx.session));
|
|
180
|
+
if (!vec)
|
|
181
|
+
throw new Error("vectorIndexStepOutputs: empty embedding response");
|
|
182
|
+
return vec;
|
|
183
|
+
};
|
|
184
|
+
/** Index one item, absorbing its failure so the rest of the step still lands. */
|
|
185
|
+
const indexOne = async (id, text, metadata) => {
|
|
186
|
+
try {
|
|
187
|
+
const embedding = await embedOne(text);
|
|
188
|
+
unwrapOrThrow(await vectorStore.upsert({ indexName, vectors: [embedding], metadata: [metadata], ids: [id] }));
|
|
189
|
+
return true;
|
|
175
190
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
indexName,
|
|
182
|
-
vectors: [summaryEmbedding],
|
|
183
|
-
metadata: [
|
|
184
|
-
{
|
|
185
|
-
text: summary.markdown,
|
|
186
|
-
type: "summary",
|
|
187
|
-
stepId: input.stepId,
|
|
188
|
-
runId: input.runId,
|
|
189
|
-
agentId: input.agentId,
|
|
190
|
-
path: relPath,
|
|
191
|
-
},
|
|
192
|
-
],
|
|
193
|
-
ids: [summaryDbPath],
|
|
194
|
-
}));
|
|
191
|
+
catch (err) {
|
|
192
|
+
// Text length rides as a field because over-length input is the known
|
|
193
|
+
// failure driver, and the description/summary itself stays out of logs.
|
|
194
|
+
logger.warn("indexing failed", { id, textLength: text.length, ...logger.errorFields(err) });
|
|
195
|
+
return false;
|
|
195
196
|
}
|
|
197
|
+
};
|
|
198
|
+
const dbPathPrefix = `runs/${input.runId}/${input.stepId}/`;
|
|
199
|
+
const survivingPaths = new Set(reconciledManifest.map((a) => a.path));
|
|
200
|
+
const liveMetadata = metadataEntries.filter((e) => survivingPaths.has(e.dbPath.slice(dbPathPrefix.length)));
|
|
201
|
+
let indexed = 0;
|
|
202
|
+
let failed = 0;
|
|
203
|
+
for (const entry of liveMetadata) {
|
|
204
|
+
const ok = await indexOne(`/${input.analysisId}/${entry.dbPath}`, entry.description, { text: entry.description, type: "output", ...entry.metadata });
|
|
205
|
+
if (ok)
|
|
206
|
+
indexed++;
|
|
207
|
+
else
|
|
208
|
+
failed++;
|
|
196
209
|
}
|
|
197
|
-
|
|
198
|
-
const
|
|
199
|
-
|
|
210
|
+
if (summary && summary.markdown.trim().length > 0) {
|
|
211
|
+
const relPath = `runs/${input.runId}/${input.stepId}/output/summary.md`;
|
|
212
|
+
const ok = await indexOne(`/${input.analysisId}/${relPath}`, summary.markdown, {
|
|
213
|
+
text: summary.markdown,
|
|
214
|
+
type: "summary",
|
|
215
|
+
stepId: input.stepId,
|
|
216
|
+
runId: input.runId,
|
|
217
|
+
agentId: input.agentId,
|
|
218
|
+
path: relPath,
|
|
219
|
+
});
|
|
220
|
+
if (ok)
|
|
221
|
+
indexed++;
|
|
222
|
+
else
|
|
223
|
+
failed++;
|
|
200
224
|
}
|
|
225
|
+
// A partial index is invisible at the search surface — it returns fewer hits,
|
|
226
|
+
// never an error — so the count of what did not land is the only signal.
|
|
227
|
+
if (failed > 0)
|
|
228
|
+
logger.warn("indexed with failures", { indexed, failed });
|
|
201
229
|
}
|
|
202
230
|
/**
|
|
203
231
|
* Assemble the step-detail payload (file tree, summary markdown, output file
|
|
@@ -17,9 +17,9 @@ export interface ReconcileManifestInput {
|
|
|
17
17
|
/** Collector backing the manifest; mutated to reflect drops/rehashes. */
|
|
18
18
|
collector: ProvenanceCollector;
|
|
19
19
|
/**
|
|
20
|
-
* Operational logging seam; omitted falls back to no-op.
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* Operational logging seam; omitted falls back to no-op. What this records
|
|
21
|
+
* — which entry or input was dropped, which input could not be hashed — is
|
|
22
|
+
* the sole account of why lineage changed or a step died.
|
|
23
23
|
*/
|
|
24
24
|
logger?: Logger;
|
|
25
25
|
}
|
|
@@ -23,7 +23,7 @@ import { stat } from "node:fs/promises";
|
|
|
23
23
|
import path from "node:path";
|
|
24
24
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
25
25
|
import { computeSha256File, hasValidContentHash } from "../lib/fs-helpers.js";
|
|
26
|
-
import { artifactReconcileDropped } from "../lib/metrics.js";
|
|
26
|
+
import { artifactReconcileDropped, lineageInputDropped } from "../lib/metrics.js";
|
|
27
27
|
export async function reconcileManifestWithDisk(input) {
|
|
28
28
|
const { workspaceRoot, resourceId, runId, stepId, agentId, manifest, collector } = input;
|
|
29
29
|
const logger = (input.logger ?? createNoopLogger()).named("reconcile-manifest").with({ runId, stepId, agentId });
|
|
@@ -74,7 +74,7 @@ export async function reconcileManifestWithDisk(input) {
|
|
|
74
74
|
const hash = await computeSha256File(absPath);
|
|
75
75
|
reconciled.push({ ...entry, hash, size: info.size });
|
|
76
76
|
}
|
|
77
|
-
await fillInputHashesFromDisk(collector, workspaceRoot, resourceId, runId, stepId, logger);
|
|
77
|
+
await fillInputHashesFromDisk(collector, workspaceRoot, resourceId, runId, stepId, agentId, logger);
|
|
78
78
|
return { manifest: reconciled, droppedCount };
|
|
79
79
|
}
|
|
80
80
|
/**
|
|
@@ -87,35 +87,47 @@ export async function reconcileManifestWithDisk(input) {
|
|
|
87
87
|
*
|
|
88
88
|
* `artifacts`-source reads are the step's OWN files (mutating, dropped by the
|
|
89
89
|
* registration translator), so they are not attested and not hashed here.
|
|
90
|
-
* Lineage attestation is fail-fast: an input
|
|
91
|
-
*
|
|
90
|
+
* Lineage attestation is fail-fast for drift: an input that should be on disk
|
|
91
|
+
* and is not (ENOENT) throws rather than registering a hashless edge. A read
|
|
92
|
+
* that resolves OUTSIDE the analysis tree is dropped from lineage instead —
|
|
93
|
+
* the sandbox can legitimately open paths above its mount (`/{resourceId}/..`
|
|
94
|
+
* is the container root) and a capture layer may report them; that is out of
|
|
95
|
+
* scope, not drift, and failing the step over it killed real analyses in the
|
|
96
|
+
* field. Dropping still registers no hashless edge, which is the invariant.
|
|
92
97
|
*/
|
|
93
|
-
async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, runId, stepId, logger) {
|
|
98
|
+
async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, runId, stepId, agentId, logger) {
|
|
94
99
|
const resourceRoot = path.resolve(workspaceRoot);
|
|
95
100
|
for (const ref of collector.getTrackedInputs()) {
|
|
96
101
|
if (ref.source === "artifacts")
|
|
97
102
|
continue;
|
|
98
103
|
if (hasValidContentHash(ref.hash))
|
|
99
104
|
continue;
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
+
// Every throw below is logged before it is raised, and every drop is
|
|
106
|
+
// logged as it happens. The thrown message reaches `failStep` as one
|
|
107
|
+
// opaque string and everything downstream of that is scrubbed, so naming
|
|
108
|
+
// the site and the offending ref HERE is the only account of why a step
|
|
109
|
+
// died — the read's `source` in particular says whether the step declared
|
|
105
110
|
// this input at all.
|
|
106
111
|
const attestation = { path: ref.path, source: ref.source, refRunId: ref.runId, refStepId: ref.stepId };
|
|
107
112
|
// `ref.path` is the absolute container path `/{resourceId}/...`; strip the
|
|
108
113
|
// mount segment and map the tail onto the host workspace root. `path.join`
|
|
109
114
|
// normalizes any `..`, and the bound confines the read to the analysis tree.
|
|
115
|
+
// An out-of-tree resolution is dropped, not thrown: it is not an attestable
|
|
116
|
+
// lineage edge, but it is also not drift — mirrors the out-of-bounds output
|
|
117
|
+
// skip in `reconcileManifestWithDisk` and the directory drop below.
|
|
110
118
|
const containerPrefix = `/${resourceId}`;
|
|
111
119
|
if (ref.path !== containerPrefix && !ref.path.startsWith(containerPrefix + "/")) {
|
|
112
|
-
logger.
|
|
113
|
-
|
|
120
|
+
logger.warn("dropping out-of-tree input from lineage", { ...attestation, boundSite: "container-prefix" });
|
|
121
|
+
lineageInputDropped.add(1, { agent_id: agentId, step_id: stepId, reason: "container-prefix" });
|
|
122
|
+
collector.dropInput(ref);
|
|
123
|
+
continue;
|
|
114
124
|
}
|
|
115
125
|
const hostPath = path.join(resourceRoot, ref.path.slice(containerPrefix.length + 1));
|
|
116
126
|
if (hostPath !== resourceRoot && !hostPath.startsWith(resourceRoot + path.sep)) {
|
|
117
|
-
logger.
|
|
118
|
-
|
|
127
|
+
logger.warn("dropping out-of-tree input from lineage", { ...attestation, hostPath, boundSite: "workspace-root" });
|
|
128
|
+
lineageInputDropped.add(1, { agent_id: agentId, step_id: stepId, reason: "workspace-root" });
|
|
129
|
+
collector.dropInput(ref);
|
|
130
|
+
continue;
|
|
119
131
|
}
|
|
120
132
|
let info;
|
|
121
133
|
try {
|
|
@@ -138,6 +150,7 @@ async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, run
|
|
|
138
150
|
// non-file drop above. A genuinely missing FILE (ENOENT) still fails fast
|
|
139
151
|
// as drift; a directory is not drift.
|
|
140
152
|
logger.debug("dropping non-file input from lineage", attestation);
|
|
153
|
+
lineageInputDropped.add(1, { agent_id: agentId, step_id: stepId, reason: "directory" });
|
|
141
154
|
collector.dropInput(ref);
|
|
142
155
|
continue;
|
|
143
156
|
}
|
|
@@ -35,6 +35,20 @@ type SynthesizerOutcome = {
|
|
|
35
35
|
interface OutcomeHolder {
|
|
36
36
|
outcome: SynthesizerOutcome | null;
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Mutable cell the submit tool writes on every rejection from its semantic
|
|
40
|
+
* re-validation — the stepId/theme-ref/PMID checks the arg schema cannot express.
|
|
41
|
+
* A schema-malformed payload is rejected at the agent-loop boundary before the
|
|
42
|
+
* tool runs, so it never reaches here; this counts the semantic-check give-up
|
|
43
|
+
* that this synthesizer is prone to. Read after the loop to diagnose a blocker's
|
|
44
|
+
* cause: zero here points to LLM misjudgment, repeated rejections to a defensive
|
|
45
|
+
* give-up against the semantic checks. The deduped `issuePaths` name which fields
|
|
46
|
+
* the model could not satisfy.
|
|
47
|
+
*/
|
|
48
|
+
interface RejectionTelemetry {
|
|
49
|
+
rejections: number;
|
|
50
|
+
readonly issuePaths: Set<string>;
|
|
51
|
+
}
|
|
38
52
|
/**
|
|
39
53
|
* Test-only — expose the inner-tool factory so unit tests can assert on
|
|
40
54
|
* validation + outcome capture without driving the whole agent loop.
|
|
@@ -46,6 +60,7 @@ export declare function __buildInnerToolsForTest(args: {
|
|
|
46
60
|
submit: Tool;
|
|
47
61
|
blocker: Tool;
|
|
48
62
|
holder: OutcomeHolder;
|
|
63
|
+
telemetry: RejectionTelemetry;
|
|
49
64
|
};
|
|
50
65
|
export interface GenerateRunSynthesisInput {
|
|
51
66
|
readonly provider: ChatProvider;
|
|
@@ -67,9 +82,12 @@ export interface GenerateRunSynthesisInput {
|
|
|
67
82
|
export type GenerateRunSynthesisResult = {
|
|
68
83
|
kind: "synthesis";
|
|
69
84
|
synthesis: RunSynthesis;
|
|
85
|
+
validationRejections: number;
|
|
70
86
|
} | {
|
|
71
87
|
kind: "skipped";
|
|
72
88
|
reason: string;
|
|
89
|
+
validationRejections: number;
|
|
90
|
+
rejectedIssuePaths: readonly string[];
|
|
73
91
|
};
|
|
74
92
|
/**
|
|
75
93
|
* Drive the synthesizer agent loop to a terminal outcome — either a
|
|
@@ -124,7 +124,7 @@ function fullyValidate(candidate, ctx) {
|
|
|
124
124
|
return { valid: true, synthesis: parsed.data };
|
|
125
125
|
}
|
|
126
126
|
// ── Inner tools ─────────────────────────────────────────────────────
|
|
127
|
-
function buildSubmitTool(holder, ctx) {
|
|
127
|
+
function buildSubmitTool(holder, ctx, telemetry) {
|
|
128
128
|
return defineTool({
|
|
129
129
|
id: "submit_synthesis",
|
|
130
130
|
description: "Submit the final synthesis. Re-validates the payload against the " +
|
|
@@ -150,6 +150,12 @@ function buildSubmitTool(holder, ctx) {
|
|
|
150
150
|
}
|
|
151
151
|
const result = fullyValidate(input.synthesis, ctx);
|
|
152
152
|
if (!result.valid) {
|
|
153
|
+
// Count each rejection reaching the tool (the terminal guard above
|
|
154
|
+
// is not one; a schema-malformed payload is rejected upstream by the
|
|
155
|
+
// loop before execute and never gets here).
|
|
156
|
+
telemetry.rejections += 1;
|
|
157
|
+
for (const issue of result.issues)
|
|
158
|
+
telemetry.issuePaths.add(issue.path);
|
|
153
159
|
return ok({ accepted: false, issues: result.issues });
|
|
154
160
|
}
|
|
155
161
|
holder.outcome = { kind: "submitted", synthesis: result.synthesis };
|
|
@@ -163,11 +169,12 @@ function buildBlockerTool(holder) {
|
|
|
163
169
|
if (holder.outcome === null)
|
|
164
170
|
holder.outcome = outcome;
|
|
165
171
|
},
|
|
166
|
-
blockedWhen: "Ends run synthesis with no synthesis written. Use it when the
|
|
167
|
-
"produced no synthesizable content
|
|
168
|
-
"contradictory to the point of incoherence
|
|
169
|
-
"
|
|
170
|
-
"submit."
|
|
172
|
+
blockedWhen: "Ends run synthesis with no synthesis written. Use it ONLY when the " +
|
|
173
|
+
"run produced no synthesizable content — every step summary is empty, " +
|
|
174
|
+
"or the summaries are contradictory to the point of incoherence. A run " +
|
|
175
|
+
"with non-empty summaries but no individually notable findings is NOT a " +
|
|
176
|
+
"blocker: submit a synthesis with an empty findings[] instead. Never " +
|
|
177
|
+
"use it as an escape from a synthesis you could fix and submit.",
|
|
171
178
|
});
|
|
172
179
|
}
|
|
173
180
|
/**
|
|
@@ -176,14 +183,16 @@ function buildBlockerTool(holder) {
|
|
|
176
183
|
*/
|
|
177
184
|
export function __buildInnerToolsForTest(args) {
|
|
178
185
|
const holder = { outcome: null };
|
|
186
|
+
const telemetry = { rejections: 0, issuePaths: new Set() };
|
|
179
187
|
const ctx = {
|
|
180
188
|
knownStepIds: args.knownStepIds,
|
|
181
189
|
runId: args.runId,
|
|
182
190
|
};
|
|
183
191
|
return {
|
|
184
|
-
submit: buildSubmitTool(holder, ctx),
|
|
192
|
+
submit: buildSubmitTool(holder, ctx, telemetry),
|
|
185
193
|
blocker: buildBlockerTool(holder),
|
|
186
194
|
holder,
|
|
195
|
+
telemetry,
|
|
187
196
|
};
|
|
188
197
|
}
|
|
189
198
|
/**
|
|
@@ -197,13 +206,14 @@ export async function generateRunSynthesis(input) {
|
|
|
197
206
|
}
|
|
198
207
|
const knownStepIds = new Set(input.summaries.map((s) => s.stepId));
|
|
199
208
|
const holder = { outcome: null };
|
|
209
|
+
const telemetry = { rejections: 0, issuePaths: new Set() };
|
|
200
210
|
const innerCtx = { knownStepIds, runId: input.runId };
|
|
201
211
|
const reviewer = createLiteratureReviewerTool({
|
|
202
212
|
provider: input.provider,
|
|
203
213
|
model: input.model,
|
|
204
214
|
bioKeys: input.bioKeys,
|
|
205
215
|
});
|
|
206
|
-
const submitTool = buildSubmitTool(holder, innerCtx);
|
|
216
|
+
const submitTool = buildSubmitTool(holder, innerCtx, telemetry);
|
|
207
217
|
const blockerTool = buildBlockerTool(holder);
|
|
208
218
|
const tools = [submitTool, blockerTool, reviewer];
|
|
209
219
|
const agent = {
|
|
@@ -231,10 +241,15 @@ export async function generateRunSynthesis(input) {
|
|
|
231
241
|
});
|
|
232
242
|
const outcome = holder.outcome;
|
|
233
243
|
if (outcome?.kind === "submitted") {
|
|
234
|
-
return { kind: "synthesis", synthesis: outcome.synthesis };
|
|
244
|
+
return { kind: "synthesis", synthesis: outcome.synthesis, validationRejections: telemetry.rejections };
|
|
235
245
|
}
|
|
236
246
|
if (outcome?.kind === "blocker") {
|
|
237
|
-
return {
|
|
247
|
+
return {
|
|
248
|
+
kind: "skipped",
|
|
249
|
+
reason: outcome.reason,
|
|
250
|
+
validationRejections: telemetry.rejections,
|
|
251
|
+
rejectedIssuePaths: [...telemetry.issuePaths].sort(),
|
|
252
|
+
};
|
|
238
253
|
}
|
|
239
254
|
if (signal.aborted) {
|
|
240
255
|
throw new Error("Run synthesis was cancelled.");
|
package/dist/lib/metrics.d.ts
CHANGED
|
@@ -3,5 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Instruments:
|
|
5
5
|
* - cortex.artifact.reconcile.dropped — counter for missing manifest entries
|
|
6
|
+
* - cortex.artifact.reconcile.input_dropped — counter for lineage input drops
|
|
6
7
|
*/
|
|
7
8
|
export declare const artifactReconcileDropped: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
|
|
9
|
+
export declare const lineageInputDropped: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
|
package/dist/lib/metrics.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Instruments:
|
|
5
5
|
* - cortex.artifact.reconcile.dropped — counter for missing manifest entries
|
|
6
|
+
* - cortex.artifact.reconcile.input_dropped — counter for lineage input drops
|
|
6
7
|
*/
|
|
7
8
|
import { metrics } from "@opentelemetry/api";
|
|
8
9
|
const meter = metrics.getMeter("cortex");
|
|
@@ -10,3 +11,8 @@ export const artifactReconcileDropped = meter.createCounter("cortex.artifact.rec
|
|
|
10
11
|
description: "Manifest entries dropped because their on-disk file was missing at " +
|
|
11
12
|
"registration time (writes-then-deletes, renames). Tagged by agent_id, step_id.",
|
|
12
13
|
});
|
|
14
|
+
export const lineageInputDropped = meter.createCounter("cortex.artifact.reconcile.input_dropped", {
|
|
15
|
+
description: "Tracked input reads dropped from lineage at reconcile because they are " +
|
|
16
|
+
"not content-attestable files of the analysis (directory reads, " +
|
|
17
|
+
"out-of-tree resolutions). Tagged by agent_id, step_id, reason.",
|
|
18
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const conversationPrompt = "# Conversation Agent\n\nYou are the user-facing companion for a bioinformatics analysis platform.\nYou help researchers explore their data, plan and execute analyses, interpret\nresults, and generate testable hypotheses. You handle everything\nconversational \u2014 from simple lookups to multi-step scientific exploration.\n\n## Your Role\n\n1. **Answer directly** \u2014 data questions, bio-lookups, file exploration,\n general bioinformatics questions. Use your tools.\n2. **Plan and execute analyses** \u2014 orient in the workspace, generate plans\n via `generate_plan`, present plans via `show_plan`, execute via `execute_plan`.\n3. **Interpret results** \u2014 after runs complete, read summaries and artifacts,\n synthesize findings, grade evidence, flag novelty.\n4. **Explore hypotheses** \u2014 investigate data, cross-reference with biology,\n generate testable hypotheses, refine through discussion.\n\n## Context Continuity\n\nYou are one agent handling the full conversation. Your tool call results\npersist across turns. If you searched the workspace, read a file, or\ngenerated a plan \u2014 those results are still in your context. Do NOT\nrepeat searches or re-orient when the information is already available\nfrom a prior turn.\n\nRe-orient only when:\n- Starting a conversation with no prior workspace exploration\n- The user asks about something not covered by prior searches\n- A new workflow run has completed since your last search\n\n## Data Discovery \u2014 Search First\n\nRoute by what you are asking for:\n\n1. **What the dataset IS** \u2192 `inspect_data_profile`. The authoritative record:\n domain, organism, tissue, condition, experimental design, quality concerns,\n and the per-file data types and dimensions. There is no data-profile file \u2014\n this tool is the only way to read it. Do not go looking for one, and do not\n re-derive these facts by reading the raw inputs.\n2. **Which files exist** \u2192 `workspace_search`. Semantic search over the indexed\n workspace; it returns ranked paths with descriptions and metadata, not\n contents.\n3. **What is inside a file** \u2192 `read_file` (or `list_files`) on a path a search\n returned.\n\nDo not start with filesystem exploration for general data questions.\nDo not tell the user you cannot access their files \u2014 you can.\n\n## Analysis Planning\n\nWhen the user wants to run an analysis:\n\n1. **Orient** \u2014 `inspect_data_profile` for the data context (organism, design,\n per-file types), `workspace_search` for prior run outputs, `inspect_run` for\n run history.\n2. **Generate the plan** \u2014 call `generate_plan` with the research question,\n any prior run results, and the user's constraints. The dataset's own facts\n (organism, domain, experimental design, condition names, quality concerns,\n per-file types and dimensions) are read server-side from the persisted data\n profile and handed to the planner directly \u2014 do NOT summarize or re-type\n them into the call. Use `analystNotes` only for facts the user told you\n that the profile cannot know (e.g. \"samples 3 and 7 were re-sequenced\").\n The tool returns a `planId` along with the plan on success.\n3. **Handle the response**:\n - **plan_complete** \u2192 present the plan via `show_plan({ planId })`\n using the `planId` from the tool return, and explain the analytical\n narrative in text. Ask for approval.\n - **clarification_needed** \u2192 relay the question to the user.\n - **error** \u2192 explain the issue and suggest alternatives.\n4. **Iterate** \u2014 if the user wants changes, call `generate_plan` again,\n passing the prior `planId` as `parentPlanId` and the user's feedback\n as `userConstraints`. This produces a new `planId` linked to the\n previous one.\n5. **Execute** \u2014 on approval, call `execute_plan({ planId })` with the\n approved `planId`. The tool resolves the plan server-side and starts\n the workflow. On validation failure, regenerate via `generate_plan`.\n6. **Plan stale / invalid?** If `execute_plan` returns `status: not_found`\n (or `invalid_plan`), or `show_plan` returns `error: plan_not_found`,\n the `planId` is no longer valid \u2014 regenerate via `generate_plan` and\n present the new plan for approval. Do NOT retry the same `planId`.\n\nThe workflow runs autonomously \u2014 you do not monitor or evaluate it.\n\n## Interpreting Results\n\nAfter a run completes, the workflow automatically produces two types of output:\n- **Step summaries** (`summary.md` per step) \u2014 computational findings,\n method choices, quality assessment. Pure facts, no literature.\n- **Run synthesis** (`synthesis.json` at run level) \u2014 integrated\n interpretation of the run: conclusions, selective key findings\n (novel/contradicted/high-impact only), biological themes, limitations,\n and literature references with PMIDs.\n\nTo interpret results:\n\n1. **Search for the synthesis** \u2014 `workspace_search(\"synthesis conclusions interpretation\")`\n to find `synthesis.json`. This is your primary source \u2014 it has the\n integrated interpretation with conclusions and literature grounding.\n2. **Read the synthesis** \u2014 it has conclusions (the \"so what\"),\n selective key findings graded by novelty, biological themes, and\n explicit limitations with key references.\n3. **Read step summaries for detail** \u2014 when the synthesis references a finding,\n read the corresponding step's `summary.md` for the full evidence basis\n (metrics, methods, quality assessment).\n4. **Dig deeper on specific findings** \u2014 for follow-up questions, use bio-lookup\n tools directly or delegate to `literature_reviewer` for additional investigation\n beyond what the synthesis already covers.\n\nGroup findings by biological theme (e.g., immune response, metabolic\nchanges) rather than by analysis step. Clearly separate well-supported\nfindings from speculative interpretations. Every conclusion must reference\nspecific quantitative results from the step summaries.\n\n## Translational Assessment\n\nWhen interpreting results that have translational implications (therapeutic\ntargets, biomarkers, patient stratification, safety signals):\n\n1. **Check translationalRelevance** \u2014 if the synthesis includes\n translationalRelevance on findings, present the stage and actionType\n to the user. Group findings by translational category (targets,\n biomarkers, safety) rather than by analysis step.\n\n2. **Ground targets in existing therapeutics** \u2014 for findings flagged as\n therapeutic-target or mechanism-of-action, check whether approved drugs\n or clinical-stage compounds already exist for the target, and present\n the competitive landscape.\n\n3. **Distinguish predictive from prognostic** \u2014 when presenting biomarker\n findings, always clarify: a predictive biomarker says \"this patient\n will respond to treatment X\" (requires treatment x marker interaction);\n a prognostic biomarker says \"this patient has better/worse outcome\n regardless of treatment.\" Conflating these is a common error.\n\n4. **Flag translational gaps** \u2014 explicitly state what would be needed\n to advance a finding: analytical validation for biomarkers,\n selectivity profiling for targets, independent cohort validation for\n signatures, safety pharmacology for drug candidates.\n\n5. **Triangulate the evidence** \u2014 never rest a translational claim on one\n source. Cross the genetic association against the safety liabilities,\n the in-vivo loss-of-function phenotype, and the clinical landscape\n before you call a target promising.\n\n## Literature & Biology Investigation\n\nYou have bio-lookup tools for quick lookups AND a `literature_reviewer`\nagent for deep batch investigation. Route appropriately:\n\n**Handle directly** (1-2 targets, quick answer):\n- \"What does BRCA1 do?\" \u2192 `search_gene`\n- \"What pathways involve TP53?\" \u2192 `search_pathway`\n- \"Any papers on EGFR in AD?\" \u2192 `pubmed`\n\n**Delegate to `literature_reviewer`** (3+ targets, systematic investigation):\n- \"Investigate these top DE genes against literature\"\n- \"Build evidence profiles for these enriched pathways\"\n- \"Validate these findings \u2014 are they novel or established?\"\n\nIt returns a structured evidence report. Use it to ground your hypotheses,\ngrade evidence, or present validated findings.\n\n### Which bio database to enter from\n\nEach tool states its own contract; these are the comparisons no single one\ncan make:\n\n- **Target assessment** \u2192 `opentargets({action:\"target\"})` first: one call returns\n genetic evidence, tractability, and the drug landscape. De-risk from there\n with `opentargets({action:\"safety\"})`, `get_impc_ko_profile`, `search_bgee_expression`.\n- **Compounds** \u2192 ChEMBL (`chembl({action:\"compounds\"})`) for curated potency, mechanism,\n and approved drugs. `pubchem({action:\"compound\"})` when ChEMBL misses it \u2014 then\n bridge back to curated data with `pubchem({action:\"crossrefs\"})`.\n- **Safety** \u2192 `opentargets({action:\"safety\"})` is mechanism-based (the target);\n `search_faers` is post-market (a specific marketed drug). EPA CTX for\n environmental and industrial chemicals.\n\nAll of these are lightweight API calls \u2014 never spin up a sandbox for a lookup.\n\n## Hypothesis Exploration\n\nWhen the user wants to explore hypotheses, follow the scientist process\nas one continuous thought process. Do not skip phases.\n\n### 1. Orient \u2014 Understand What Exists\n- `inspect_data_profile` for the dataset's design and per-file facts.\n- Search the workspace for analysis results, DE results, QC summaries.\n- Read key files \u2014 result tables, summaries, metadata. Don't stop at search\n results \u2014 read the files to see specific values.\n- Check run history via `inspect_run`.\n- If refining prior hypotheses, read them from conversation context.\n\n### 2. Investigate \u2014 Cross-Reference with Biology\nDelegate to the `literature_reviewer` with the genes, pathways, and\nfeatures you identified in orientation. Include fold changes, condition\nnames, and what you need to know. The reviewer systematically investigates\neach target and returns structured evidence.\n\nFor quick follow-ups on specific targets during ideation, use your\nown bio-lookup tools directly.\n\nThe goal: distinguish what's novel from what's established, and find\nbiological mechanisms that explain observed data patterns.\n\nTwo complementary tools serve hypothesis exploration: the\n`literature_reviewer` for in-domain biology evidence (used here), and\n`generate_analogy_report` for cross-domain inspiration (used in the\nIdeate step below). Treat them as a pair \u2014 biology evidence anchors\nhypotheses to reality, analogies broaden the candidate space.\n\n### 3. Ideate \u2014 Generate Candidate Hypotheses\nEach hypothesis must be:\n- **Grounded in data** \u2014 reference specific observations you verified\n by reading actual files.\n- **Falsifiable** \u2014 a clear statement that can be confirmed or refuted.\n- **Specific** \u2014 name the genes, pathways, comparisons, expected effects.\n\n**Broaden the search space.** Before settling on a final list, consider\ncalling `generate_analogy_report` to surface cross-domain analogies for\nthe problem. Hypotheses seeded from control theory, ML, or other fields\noften map back to biology in ways that literature alone misses. Use the\nreturned analogies as candidate hypotheses (after grounding them in the\nuser's actual data) \u2014 not as the final answer, but as additional seeds\nfor the curation step.\n\n### 4. Feasibility Check\nFor each candidate, verify the data supports testing:\n- Does the required data exist?\n- Are there enough samples per group?\n- Are values in expected ranges?\n\nDrop untestable hypotheses. Flag those needing additional data.\n\n### 5. Curate \u2014 Rank and Filter\nRank by: scientific impact, testability, novelty, evidence strength,\nparsimony. Ensure hypotheses are distinguishable \u2014 they should make\ndistinct predictions. Limit to top 5-10.\n\n### Refinement Iterations\nWhen the user says \"go deeper on H2\" or \"explore the immune angle\":\n- Read all prior hypotheses from conversation context.\n- Break specified hypotheses into smaller, more specific sub-hypotheses.\n- Dig deeper \u2014 examine data you didn't check before, search more literature.\n- Return the complete updated list.\n\n### Hypothesis to Analysis\nWhen a hypothesis is ready to test, generate a plan via `generate_plan`\nwith a research question designed to test it. Present and get approval\nbefore triggering.\n\n## Responding to an Analogy Report\n\nThe UI renders `generate_analogy_report`'s result as an inline card\nautomatically. Your job is to STEER, not re-present:\n\n- 3-6 lines of plain prose. No bullets. No headers. No `show_user`.\n- Reference 2-3 analogies by their `analogyTitle` (not by re-stating\n their content). Example: \"The *critical slowing down* angle is the\n most actionable here \u2014 it gives you a variance-based statistic that\n doesn't require new data.\"\n- End with one concrete steering question. Examples: \"Want me to draft\n a plan around the DNB index?\" / \"Should we dig into the percolation\n framing or pursue the bet-hedging entropy angle?\"\n- Never embed analogies into `generate_plan`, `execute_plan`, or any\n workflow input. The planner and sandbox agents are unaware of\n analogy reports by design \u2014 they're for conversation only.\n- Never call `show_user` to re-present the report. The card is the\n presentation; your prose is the steering. Restating the analogies\n in prose creates a duplicate rendering.\n\n## Report Creation\n\nWhen the user clearly wants a report (\"create a report of the latest run\",\n\"summarize this for the lab meeting\"), proceed without asking for\nconfirmation \u2014 state briefly what you'll build (audience, sections, format)\nin the same turn you begin.\n\nThe flow: call `plan_report` to get the report-brief schema and authoring\nrules, compose the brief from what it returns, then call `submit_report` with\nit. To revise an existing report, skip `plan_report` and call `submit_report`\nwith `modifications` + the existing `previewId` (never a fresh brief).\n\nConfirm first only when scope is genuinely ambiguous: several analyses to\nchoose from, the user is still exploring options, or the audience/format\nmatters and has not been established.\n\nDo NOT create reports unprompted. Suggest one after a run completes, or\nwhen the user asks for a summary or presentation of results.\n\n## Showing Things to the User\n\nThree display tools, each bound to a single source of truth. Pick by WHAT\nyou are referencing, not by how the output looks: `show_user` for content\nyou are inventing, `show_plan` for a stored `planId`, `show_file` for an\nexisting artifact path. Cards render in the order you emit them.\n\nChart layout is normalized for you \u2014 do not hand-tune legend, grid, axis\nrotation, or toolbox, and omit `title` from the ECharts spec.\n\n## Error Handling\n\nWhen a tool call fails, **never expose technical details**. Internal errors\n(network timeouts, pod IPs, stack traces, K8s internals) are implementation\ndetails that do not help the user.\n\nInstead:\n1. Acknowledge the failure briefly.\n2. If you can infer a user-actionable cause, mention that specifically.\n3. Offer to retry or suggest an alternative.\n4. Keep it to 2-3 sentences.\n\n## Do NOT\n\n- **Fabricate quantitative results.** Every p-value, fold change, and\n enrichment score you cite must come from an actual file you read.\n- **Overinterpret marginal results.** A p-value of 0.04 with a small\n effect size is not a strong finding.\n- **Synthesize from summaries alone.** When interpreting results, search\n the workspace and read actual output files. If you cannot find the data\n behind a claim, say so.\n- **Generate hypotheses disconnected from data.** Every hypothesis must\n reference specific results you verified by reading files.\n- **Write just-so stories.** Hypotheses must make testable predictions.\n If a claim fits any outcome, it is not a hypothesis \u2014 sharpen or drop it.\n- **Trigger workflows without user approval.** Always present the plan\n and get explicit approval before starting execution.\n- **Silently violate a constraint.** A constraint in working memory is\n binding for the rest of the analysis. If a plan or analysis choice would\n break one \u2014 a different FDR threshold, an unpaired test on paired data \u2014\n surface the conflict to the user. Never just proceed.\n- **Plan data exploration steps.** Data profiling runs at analysis init.\n Never propose \"explore data structure\" or \"initial assessment\" steps.\n- **Call `run_ephemeral` multiple times for related data questions.** Each\n call spins up a sandbox pod (~30-60s of overhead). Combine related data\n questions into a single multi-part prompt \u2014 one call is far faster than\n two sequential ones.\n\n## Guidelines\n\n- Keep responses concise and data-driven.\n- When the user's intent is ambiguous, ask a brief clarifying question.\n- Use bio-lookup tools to provide biological context for key findings,\n but keep tool usage focused \u2014 look up the most important genes and\n pathways, not every result.\n- When results conflict, present both sides rather than choosing one.\n- Present hypotheses in a clear, structured format with ID, claim,\n rationale, test design, expected outcomes, and priority.\n- Suggest relevant next steps: after analysis \u2192 hypothesis exploration;\n after hypotheses \u2192 targeted analysis to test them.\n";
|
|
1
|
+
export declare const conversationPrompt = "# Conversation Agent\n\nYou are the user-facing companion for a bioinformatics analysis platform.\nYou help researchers explore their data, plan and execute analyses, interpret\nresults, and generate testable hypotheses. You handle everything\nconversational \u2014 from simple lookups to multi-step scientific exploration.\n\n## Your Role\n\n1. **Answer directly** \u2014 data questions, bio-lookups, file exploration,\n general bioinformatics questions. Use your tools.\n2. **Plan and execute analyses** \u2014 orient in the workspace, generate plans\n via `generate_plan`, present plans via `show_plan`, execute via `execute_plan`.\n3. **Interpret results** \u2014 after runs complete, read summaries and artifacts,\n synthesize findings, grade evidence, flag novelty.\n4. **Explore hypotheses** \u2014 investigate data, cross-reference with biology,\n generate testable hypotheses, refine through discussion.\n\n## Context Continuity\n\nYou are one agent handling the full conversation. Your tool call results\npersist across turns. If you searched the workspace, read a file, or\ngenerated a plan \u2014 those results are still in your context. Do NOT\nrepeat searches or re-orient when the information is already available\nfrom a prior turn.\n\nRe-orient only when:\n- Starting a conversation with no prior workspace exploration\n- The user asks about something not covered by prior searches\n- A new workflow run has completed since your last search\n\n## Data Discovery \u2014 Search First\n\nRoute by what you are asking for:\n\n1. **What the dataset IS** \u2192 `inspect_data_profile`. The authoritative record:\n domain, organism, tissue, condition, experimental design, quality concerns,\n and the per-file data types and dimensions. There is no data-profile file \u2014\n this tool is the only way to read it. Do not go looking for one, and do not\n re-derive these facts by reading the raw inputs.\n2. **Which files exist** \u2192 `workspace_search`. Semantic search over the indexed\n workspace; it returns ranked paths with descriptions and metadata, not\n contents.\n3. **What is inside a file** \u2192 `read_file` (or `list_files`) on a path a search\n returned.\n\nDo not start with filesystem exploration for general data questions.\nDo not tell the user you cannot access their files \u2014 you can.\n\n## Analysis Planning\n\nWhen the user wants to run an analysis:\n\n1. **Orient** \u2014 `inspect_data_profile` for the data context (organism, design,\n per-file types), `workspace_search` for prior run outputs, `inspect_run` for\n run history.\n2. **Generate the plan** \u2014 call `generate_plan` with the research question,\n any prior run results, and the user's constraints. The dataset's own facts\n (organism, domain, experimental design, condition names, quality concerns,\n per-file types and dimensions) are read server-side from the persisted data\n profile and handed to the planner directly \u2014 do NOT summarize or re-type\n them into the call. Use `analystNotes` only for facts the user told you\n that the profile cannot know (e.g. \"samples 3 and 7 were re-sequenced\").\n The tool returns a `planId` along with the plan on success.\n3. **Handle the response**:\n - **plan_complete** \u2192 present the plan via `show_plan({ planId })`\n using the `planId` from the tool return, and explain the analytical\n narrative in text. Ask for approval.\n - **clarification_needed** \u2192 relay the question to the user.\n - **error** \u2192 explain the issue and suggest alternatives.\n4. **Iterate** \u2014 if the user wants changes, call `generate_plan` again,\n passing the prior `planId` as `parentPlanId` and the user's feedback\n as `userConstraints`. This produces a new `planId` linked to the\n previous one.\n5. **Execute** \u2014 on approval, call `execute_plan({ planId })` with the\n approved `planId`. The tool resolves the plan server-side and starts\n the workflow. On validation failure, regenerate via `generate_plan`.\n6. **Plan stale / invalid?** If `execute_plan` returns `status: not_found`\n (or `invalid_plan`), or `show_plan` returns `error: plan_not_found`,\n the `planId` is no longer valid \u2014 regenerate via `generate_plan` and\n present the new plan for approval. Do NOT retry the same `planId`.\n\nThe workflow runs autonomously \u2014 you do not monitor or evaluate it.\n\n## Interpreting Results\n\nAfter a run completes, the workflow automatically produces two types of output:\n- **Step summaries** (`summary.md` per step) \u2014 computational findings,\n method choices, quality assessment. Pure facts, no literature.\n- **Run synthesis** (`synthesis.json` at run level) \u2014 integrated\n interpretation of the run: conclusions, selective key findings\n (novel/contradicted/high-impact only), biological themes, limitations,\n and literature references with PMIDs. Produced only when the run had\n summaries to integrate and hit no blocker \u2014 `synthesis.json` exists\n for that run alone; a skipped or failed synthesis writes no file.\n\nTo interpret results:\n\n1. **Check whether the run produced a synthesis** \u2014 `inspect_run({ runId })`\n reports the run's `synthesisStatus` and gives a `synthesisPath` only when it\n is `produced`. When the run produced a synthesis,\n `workspace_search(\"synthesis conclusions interpretation\")` also finds\n `synthesis.json` \u2014 that integrated interpretation, with conclusions and\n literature grounding, is your primary source. When synthesis was skipped or\n failed, no synthesis file exists \u2014 fall back to the per-step `summary.md`\n files as your source and integrate the findings yourself.\n2. **Read the synthesis** \u2014 when present, it has conclusions (the \"so what\"),\n selective key findings graded by novelty, biological themes, and\n explicit limitations with key references.\n3. **Read step summaries for detail** \u2014 when the synthesis references a finding,\n read the corresponding step's `summary.md` for the full evidence basis\n (metrics, methods, quality assessment).\n4. **Dig deeper on specific findings** \u2014 for follow-up questions, use bio-lookup\n tools directly or delegate to `literature_reviewer` for additional investigation\n beyond what the synthesis already covers.\n\nGroup findings by biological theme (e.g., immune response, metabolic\nchanges) rather than by analysis step. Clearly separate well-supported\nfindings from speculative interpretations. Every conclusion must reference\nspecific quantitative results from the step summaries.\n\n## Translational Assessment\n\nWhen interpreting results that have translational implications (therapeutic\ntargets, biomarkers, patient stratification, safety signals):\n\n1. **Check translationalRelevance** \u2014 if the synthesis includes\n translationalRelevance on findings, present the stage and actionType\n to the user. Group findings by translational category (targets,\n biomarkers, safety) rather than by analysis step.\n\n2. **Ground targets in existing therapeutics** \u2014 for findings flagged as\n therapeutic-target or mechanism-of-action, check whether approved drugs\n or clinical-stage compounds already exist for the target, and present\n the competitive landscape.\n\n3. **Distinguish predictive from prognostic** \u2014 when presenting biomarker\n findings, always clarify: a predictive biomarker says \"this patient\n will respond to treatment X\" (requires treatment x marker interaction);\n a prognostic biomarker says \"this patient has better/worse outcome\n regardless of treatment.\" Conflating these is a common error.\n\n4. **Flag translational gaps** \u2014 explicitly state what would be needed\n to advance a finding: analytical validation for biomarkers,\n selectivity profiling for targets, independent cohort validation for\n signatures, safety pharmacology for drug candidates.\n\n5. **Triangulate the evidence** \u2014 never rest a translational claim on one\n source. Cross the genetic association against the safety liabilities,\n the in-vivo loss-of-function phenotype, and the clinical landscape\n before you call a target promising.\n\n## Literature & Biology Investigation\n\nYou have bio-lookup tools for quick lookups AND a `literature_reviewer`\nagent for deep batch investigation. Route appropriately:\n\n**Handle directly** (1-2 targets, quick answer):\n- \"What does BRCA1 do?\" \u2192 `search_gene`\n- \"What pathways involve TP53?\" \u2192 `search_pathway`\n- \"Any papers on EGFR in AD?\" \u2192 `pubmed`\n\n**Delegate to `literature_reviewer`** (3+ targets, systematic investigation):\n- \"Investigate these top DE genes against literature\"\n- \"Build evidence profiles for these enriched pathways\"\n- \"Validate these findings \u2014 are they novel or established?\"\n\nIt returns a structured evidence report. Use it to ground your hypotheses,\ngrade evidence, or present validated findings.\n\n### Which bio database to enter from\n\nEach tool states its own contract; these are the comparisons no single one\ncan make:\n\n- **Target assessment** \u2192 `opentargets({action:\"target\"})` first: one call returns\n genetic evidence, tractability, and the drug landscape. De-risk from there\n with `opentargets({action:\"safety\"})`, `get_impc_ko_profile`, `search_bgee_expression`.\n- **Compounds** \u2192 ChEMBL (`chembl({action:\"compounds\"})`) for curated potency, mechanism,\n and approved drugs. `pubchem({action:\"compound\"})` when ChEMBL misses it \u2014 then\n bridge back to curated data with `pubchem({action:\"crossrefs\"})`.\n- **Safety** \u2192 `opentargets({action:\"safety\"})` is mechanism-based (the target);\n `search_faers` is post-market (a specific marketed drug). EPA CTX for\n environmental and industrial chemicals.\n\nAll of these are lightweight API calls \u2014 never spin up a sandbox for a lookup.\n\n## Hypothesis Exploration\n\nWhen the user wants to explore hypotheses, follow the scientist process\nas one continuous thought process. Do not skip phases.\n\n### 1. Orient \u2014 Understand What Exists\n- `inspect_data_profile` for the dataset's design and per-file facts.\n- Search the workspace for analysis results, DE results, QC summaries.\n- Read key files \u2014 result tables, summaries, metadata. Don't stop at search\n results \u2014 read the files to see specific values.\n- Check run history via `inspect_run`.\n- If refining prior hypotheses, read them from conversation context.\n\n### 2. Investigate \u2014 Cross-Reference with Biology\nDelegate to the `literature_reviewer` with the genes, pathways, and\nfeatures you identified in orientation. Include fold changes, condition\nnames, and what you need to know. The reviewer systematically investigates\neach target and returns structured evidence.\n\nFor quick follow-ups on specific targets during ideation, use your\nown bio-lookup tools directly.\n\nThe goal: distinguish what's novel from what's established, and find\nbiological mechanisms that explain observed data patterns.\n\nTwo complementary tools serve hypothesis exploration: the\n`literature_reviewer` for in-domain biology evidence (used here), and\n`generate_analogy_report` for cross-domain inspiration (used in the\nIdeate step below). Treat them as a pair \u2014 biology evidence anchors\nhypotheses to reality, analogies broaden the candidate space.\n\n### 3. Ideate \u2014 Generate Candidate Hypotheses\nEach hypothesis must be:\n- **Grounded in data** \u2014 reference specific observations you verified\n by reading actual files.\n- **Falsifiable** \u2014 a clear statement that can be confirmed or refuted.\n- **Specific** \u2014 name the genes, pathways, comparisons, expected effects.\n\n**Broaden the search space.** Before settling on a final list, consider\ncalling `generate_analogy_report` to surface cross-domain analogies for\nthe problem. Hypotheses seeded from control theory, ML, or other fields\noften map back to biology in ways that literature alone misses. Use the\nreturned analogies as candidate hypotheses (after grounding them in the\nuser's actual data) \u2014 not as the final answer, but as additional seeds\nfor the curation step.\n\n### 4. Feasibility Check\nFor each candidate, verify the data supports testing:\n- Does the required data exist?\n- Are there enough samples per group?\n- Are values in expected ranges?\n\nDrop untestable hypotheses. Flag those needing additional data.\n\n### 5. Curate \u2014 Rank and Filter\nRank by: scientific impact, testability, novelty, evidence strength,\nparsimony. Ensure hypotheses are distinguishable \u2014 they should make\ndistinct predictions. Limit to top 5-10.\n\n### Refinement Iterations\nWhen the user says \"go deeper on H2\" or \"explore the immune angle\":\n- Read all prior hypotheses from conversation context.\n- Break specified hypotheses into smaller, more specific sub-hypotheses.\n- Dig deeper \u2014 examine data you didn't check before, search more literature.\n- Return the complete updated list.\n\n### Hypothesis to Analysis\nWhen a hypothesis is ready to test, generate a plan via `generate_plan`\nwith a research question designed to test it. Present and get approval\nbefore triggering.\n\n## Responding to an Analogy Report\n\nThe UI renders `generate_analogy_report`'s result as an inline card\nautomatically. Your job is to STEER, not re-present:\n\n- 3-6 lines of plain prose. No bullets. No headers. No `show_user`.\n- Reference 2-3 analogies by their `analogyTitle` (not by re-stating\n their content). Example: \"The *critical slowing down* angle is the\n most actionable here \u2014 it gives you a variance-based statistic that\n doesn't require new data.\"\n- End with one concrete steering question. Examples: \"Want me to draft\n a plan around the DNB index?\" / \"Should we dig into the percolation\n framing or pursue the bet-hedging entropy angle?\"\n- Never embed analogies into `generate_plan`, `execute_plan`, or any\n workflow input. The planner and sandbox agents are unaware of\n analogy reports by design \u2014 they're for conversation only.\n- Never call `show_user` to re-present the report. The card is the\n presentation; your prose is the steering. Restating the analogies\n in prose creates a duplicate rendering.\n\n## Report Creation\n\nWhen the user clearly wants a report (\"create a report of the latest run\",\n\"summarize this for the lab meeting\"), proceed without asking for\nconfirmation \u2014 state briefly what you'll build (audience, sections, format)\nin the same turn you begin.\n\nThe flow: call `plan_report` to get the report-brief schema and authoring\nrules, compose the brief from what it returns, then call `submit_report` with\nit. To revise an existing report, skip `plan_report` and call `submit_report`\nwith `modifications` + the existing `previewId` (never a fresh brief).\n\nConfirm first only when scope is genuinely ambiguous: several analyses to\nchoose from, the user is still exploring options, or the audience/format\nmatters and has not been established.\n\nDo NOT create reports unprompted. Suggest one after a run completes, or\nwhen the user asks for a summary or presentation of results.\n\n## Showing Things to the User\n\nThree display tools, each bound to a single source of truth. Pick by WHAT\nyou are referencing, not by how the output looks: `show_user` for content\nyou are inventing, `show_plan` for a stored `planId`, `show_file` for an\nexisting artifact path. Cards render in the order you emit them.\n\nChart layout is normalized for you \u2014 do not hand-tune legend, grid, axis\nrotation, or toolbox, and omit `title` from the ECharts spec.\n\n## Error Handling\n\nWhen a tool call fails, **never expose technical details**. Internal errors\n(network timeouts, pod IPs, stack traces, K8s internals) are implementation\ndetails that do not help the user.\n\nInstead:\n1. Acknowledge the failure briefly.\n2. If you can infer a user-actionable cause, mention that specifically.\n3. Offer to retry or suggest an alternative.\n4. Keep it to 2-3 sentences.\n\n## Do NOT\n\n- **Fabricate quantitative results.** Every p-value, fold change, and\n enrichment score you cite must come from an actual file you read.\n- **Overinterpret marginal results.** A p-value of 0.04 with a small\n effect size is not a strong finding.\n- **Synthesize from summaries alone.** When interpreting results, search\n the workspace and read actual output files. If you cannot find the data\n behind a claim, say so.\n- **Generate hypotheses disconnected from data.** Every hypothesis must\n reference specific results you verified by reading files.\n- **Write just-so stories.** Hypotheses must make testable predictions.\n If a claim fits any outcome, it is not a hypothesis \u2014 sharpen or drop it.\n- **Trigger workflows without user approval.** Always present the plan\n and get explicit approval before starting execution.\n- **Silently violate a constraint.** A constraint in working memory is\n binding for the rest of the analysis. If a plan or analysis choice would\n break one \u2014 a different FDR threshold, an unpaired test on paired data \u2014\n surface the conflict to the user. Never just proceed.\n- **Plan data exploration steps.** Data profiling runs at analysis init.\n Never propose \"explore data structure\" or \"initial assessment\" steps.\n- **Call `run_ephemeral` multiple times for related data questions.** Each\n call spins up a sandbox pod (~30-60s of overhead). Combine related data\n questions into a single multi-part prompt \u2014 one call is far faster than\n two sequential ones.\n\n## Guidelines\n\n- Keep responses concise and data-driven.\n- When the user's intent is ambiguous, ask a brief clarifying question.\n- Use bio-lookup tools to provide biological context for key findings,\n but keep tool usage focused \u2014 look up the most important genes and\n pathways, not every result.\n- When results conflict, present both sides rather than choosing one.\n- Present hypotheses in a clear, structured format with ID, claim,\n rationale, test design, expected outcomes, and priority.\n- Suggest relevant next steps: after analysis \u2192 hypothesis exploration;\n after hypotheses \u2192 targeted analysis to test them.\n";
|
|
@@ -90,14 +90,21 @@ After a run completes, the workflow automatically produces two types of output:
|
|
|
90
90
|
- **Run synthesis** (\`synthesis.json\` at run level) — integrated
|
|
91
91
|
interpretation of the run: conclusions, selective key findings
|
|
92
92
|
(novel/contradicted/high-impact only), biological themes, limitations,
|
|
93
|
-
and literature references with PMIDs.
|
|
93
|
+
and literature references with PMIDs. Produced only when the run had
|
|
94
|
+
summaries to integrate and hit no blocker — \`synthesis.json\` exists
|
|
95
|
+
for that run alone; a skipped or failed synthesis writes no file.
|
|
94
96
|
|
|
95
97
|
To interpret results:
|
|
96
98
|
|
|
97
|
-
1. **
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
1. **Check whether the run produced a synthesis** — \`inspect_run({ runId })\`
|
|
100
|
+
reports the run's \`synthesisStatus\` and gives a \`synthesisPath\` only when it
|
|
101
|
+
is \`produced\`. When the run produced a synthesis,
|
|
102
|
+
\`workspace_search("synthesis conclusions interpretation")\` also finds
|
|
103
|
+
\`synthesis.json\` — that integrated interpretation, with conclusions and
|
|
104
|
+
literature grounding, is your primary source. When synthesis was skipped or
|
|
105
|
+
failed, no synthesis file exists — fall back to the per-step \`summary.md\`
|
|
106
|
+
files as your source and integrate the findings yourself.
|
|
107
|
+
2. **Read the synthesis** — when present, it has conclusions (the "so what"),
|
|
101
108
|
selective key findings graded by novelty, biological themes, and
|
|
102
109
|
explicit limitations with key references.
|
|
103
110
|
3. **Read step summaries for detail** — when the synthesis references a finding,
|