@inflexa-ai/harness 0.31.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/agents/conversation-agent.d.ts +7 -0
  2. package/dist/agents/conversation-agent.js +1 -0
  3. package/dist/agents/report-session-agent.d.ts +9 -0
  4. package/dist/agents/report-session-agent.js +10 -2
  5. package/dist/app/chat-turn.d.ts +11 -0
  6. package/dist/app/chat-turn.js +17 -1
  7. package/dist/app/spawn-report-session.d.ts +12 -0
  8. package/dist/app/spawn-report-session.js +21 -1
  9. package/dist/contracts/report-blocks.d.ts +8 -8
  10. package/dist/contracts/schemas/chat-parts.d.ts +6 -6
  11. package/dist/index.d.ts +4 -1
  12. package/dist/index.js +8 -0
  13. package/dist/prompts/briefing.d.ts +12 -1
  14. package/dist/prompts/briefing.js +27 -1
  15. package/dist/prompts/sandbox-standards.d.ts +1 -1
  16. package/dist/prompts/sandbox-standards.js +27 -0
  17. package/dist/provenance/seam.d.ts +203 -0
  18. package/dist/provenance/seam.js +71 -0
  19. package/dist/report-model/authoring-grammar.d.ts +2 -2
  20. package/dist/report-render/assets.d.ts +10 -0
  21. package/dist/report-render/assets.js +14 -0
  22. package/dist/report-render/design.js +256 -0
  23. package/dist/report-render/page.d.ts +75 -0
  24. package/dist/report-render/page.js +793 -1
  25. package/dist/report-render/provenance-data.d.ts +42 -0
  26. package/dist/report-render/provenance-data.js +62 -0
  27. package/dist/report-render/render.d.ts +20 -1
  28. package/dist/report-render/render.js +23 -10
  29. package/dist/report-render/types.d.ts +3 -0
  30. package/dist/report-render/views/chart-view.d.ts +5 -1
  31. package/dist/report-render/views/chart-view.js +6 -2
  32. package/dist/report-render/views/lineage.d.ts +93 -0
  33. package/dist/report-render/views/lineage.js +86 -0
  34. package/dist/report-render/views/page-view.d.ts +22 -1
  35. package/dist/report-render/views/page-view.js +12 -3
  36. package/dist/report-render/views/prose.d.ts +7 -1
  37. package/dist/report-render/views/prose.js +10 -6
  38. package/dist/report-render/views/references-view.d.ts +17 -1
  39. package/dist/report-render/views/references-view.js +12 -4
  40. package/dist/report-render/views/values.d.ts +17 -4
  41. package/dist/report-render/views/values.js +21 -8
  42. package/dist/runtime/assemble.d.ts +22 -0
  43. package/dist/runtime/assemble.js +4 -0
  44. package/dist/sandbox/k8s-client.d.ts +0 -5
  45. package/dist/sandbox/k8s-client.js +9 -115
  46. package/dist/sandbox/thread-env.d.ts +18 -12
  47. package/dist/sandbox/thread-env.js +36 -25
  48. package/dist/schemas/artifact-manifest.d.ts +4 -4
  49. package/dist/tools/display/show-user.d.ts +1 -1
  50. package/dist/tools/report-authoring/authoring-tools.d.ts +16 -1
  51. package/dist/tools/report-authoring/authoring-tools.js +58 -16
  52. package/dist/tools/report-session/derive-table.d.ts +3 -0
  53. package/dist/tools/report-session/derive-table.js +15 -0
  54. package/dist/tools/report-session/preview-report.d.ts +13 -4
  55. package/dist/tools/report-session/preview-report.js +23 -6
  56. package/dist/tools/report-session/record-version.d.ts +3 -0
  57. package/dist/tools/report-session/record-version.js +7 -3
  58. package/dist/tools/start-report-session.d.ts +7 -0
  59. package/dist/tools/start-report-session.js +1 -0
  60. package/dist/workflows/execute-analysis.d.ts +15 -69
  61. package/dist/workflows/execute-analysis.js +5 -4
  62. package/package.json +2 -1
@@ -16,10 +16,26 @@ import { type DerivationChains, type ReferenceLedger } from "../references.js";
16
16
  *
17
17
  * A bracket reads as a reference inside prose, and it stays a span beside the words. One notation serves
18
18
  * every kind, thus one list answers every marker.
19
+ *
20
+ * Every grounded kind renders its marker here. Thus the lineage control has one emission point, and a
21
+ * marker that names no place carries no control. The page shows the control only where a stamp gives the
22
+ * script a key to walk.
19
23
  */
20
- export declare function Marker({ n }: {
24
+ export declare function Marker({ n, lineage }: {
21
25
  n: number;
26
+ lineage?: number;
22
27
  }): import("hono/jsx/jsx-dev-runtime").JSX.Element;
28
+ /**
29
+ * The count of hash characters that a chain line shows.
30
+ *
31
+ * A content hash is 64 hex characters, and a line that carries three whole ones reads as noise. The head
32
+ * identifies the bytes, and it matches the count that a staged file name carries. Thus a reader compares the
33
+ * two by sight.
34
+ *
35
+ * The appendix line and the hop of the lineage popover both cut a hash to this count. Thus one hash reads
36
+ * the same in the two places.
37
+ */
38
+ export declare const CHAIN_HASH_CHARS = 12;
23
39
  /**
24
40
  * The one appendix list: one entry for each distinct reference of the page, in the order of the ladder.
25
41
  *
@@ -1,14 +1,19 @@
1
- import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "hono/jsx/jsx-runtime";
1
+ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "hono/jsx/jsx-runtime";
2
2
  import { citationRecordOf } from "../../report-model/reference-resolver.js";
3
3
  import { citationKeyOf } from "../references.js";
4
+ import { LineageControl } from "./lineage.js";
4
5
  /**
5
6
  * One evidence marker as a bracket number that links to its appendix entry.
6
7
  *
7
8
  * A bracket reads as a reference inside prose, and it stays a span beside the words. One notation serves
8
9
  * every kind, thus one list answers every marker.
10
+ *
11
+ * Every grounded kind renders its marker here. Thus the lineage control has one emission point, and a
12
+ * marker that names no place carries no control. The page shows the control only where a stamp gives the
13
+ * script a key to walk.
9
14
  */
10
- export function Marker({ n }) {
11
- return (_jsx("span", { class: "report-marker", children: _jsx("a", { href: `#ref-${n}`, children: `[${n}]` }) }));
15
+ export function Marker({ n, lineage }) {
16
+ return (_jsxs("span", { class: "report-marker", children: [_jsx("a", { href: `#ref-${n}`, children: `[${n}]` }), lineage !== undefined ? _jsx(LineageControl, { place: lineage }) : null] }));
12
17
  }
13
18
  /** The kind label of one reference entry. */
14
19
  function RefKind({ children }) {
@@ -32,8 +37,11 @@ function HashCode({ children }) {
32
37
  * A content hash is 64 hex characters, and a line that carries three whole ones reads as noise. The head
33
38
  * identifies the bytes, and it matches the count that a staged file name carries. Thus a reader compares the
34
39
  * two by sight.
40
+ *
41
+ * The appendix line and the hop of the lineage popover both cut a hash to this count. Thus one hash reads
42
+ * the same in the two places.
35
43
  */
36
- const CHAIN_HASH_CHARS = 12;
44
+ export const CHAIN_HASH_CHARS = 12;
37
45
  /** The head of one content hash: the hex after the algorithm name, cut to the shown length. */
38
46
  function hashHead(hash) {
39
47
  return hash.slice(hash.lastIndexOf(":") + 1).slice(0, CHAIN_HASH_CHARS);
@@ -26,6 +26,7 @@
26
26
  import type { CitationBlock, FigureBlock, MetricBlock, TableBlock } from "../../contracts/report-blocks.js";
27
27
  import { type ArtifactTableReference } from "../../contracts/report-reference.js";
28
28
  import type { CitationRecord } from "../../report-model/reference-resolver.js";
29
+ import { type ViewOptions } from "./lineage.js";
29
30
  import { type ReferenceLedger } from "../references.js";
30
31
  import type { ColumnDisplay } from "../table-data.js";
31
32
  import type { RenderValue } from "../types.js";
@@ -76,8 +77,11 @@ type FigureValue = Extract<RenderValue, {
76
77
  * The value binding joins the reference ladder, thus the label line carries the marker and the appendix
77
78
  * names the cell. The marker sits on the label and never on the value, because the value line is the one
78
79
  * figure that a reader takes from the card.
80
+ *
81
+ * `view.lineage` states that the page carries a provenance document. The card then stamps its keys, and the
82
+ * marker carries the control that opens the chain.
79
83
  */
80
- export declare function renderMetric(block: MetricBlock, ledger: ReferenceLedger, value: ScalarValue): string;
84
+ export declare function renderMetric(block: MetricBlock, ledger: ReferenceLedger, value: ScalarValue, view?: ViewOptions): string;
81
85
  /**
82
86
  * Wrap a run of stat cards in one responsive grid. The caller renders each card first, and it passes the
83
87
  * markup as one already-escaped string, thus `raw()` inserts it byte for byte. The grid carries its own
@@ -141,8 +145,11 @@ export declare function tableStatusText(shown: number, total: number): string;
141
145
  * The whole-table binding joins the reference ladder, thus the title line carries the marker and the
142
146
  * appendix names the artifact. Every evidentiary block ledgers this way, and a card with no title still
143
147
  * shows its marker on the same line.
148
+ *
149
+ * `view.lineage` states that the page carries a provenance document. The card then stamps its keys, and the
150
+ * marker carries the control that opens the chain.
144
151
  */
145
- export declare function renderTable(block: TableBlock, ledger: ReferenceLedger, rowCount: number, total?: number): string;
152
+ export declare function renderTable(block: TableBlock, ledger: ReferenceLedger, rowCount: number, total?: number, view?: ViewOptions): string;
146
153
  /**
147
154
  * Render a figure block as a corner-accent card. The source rides the `src` attribute, and the caption
148
155
  * renders below the image. The caption also fills the `alt` attribute, thus the escape keeps a hostile
@@ -151,8 +158,11 @@ export declare function renderTable(block: TableBlock, ledger: ReferenceLedger,
151
158
  * The whole-file binding joins the reference ladder, thus the caption line carries the marker and the
152
159
  * appendix names the image. A figure with no caption still shows its marker on that line. The marker stays
153
160
  * out of the `alt` text, because a screen reader reads the alt as the picture and not as a footnote.
161
+ *
162
+ * `view.lineage` states that the page carries a provenance document. The card then stamps its keys, and the
163
+ * marker carries the control that opens the chain.
154
164
  */
155
- export declare function renderFigure(block: FigureBlock, ledger: ReferenceLedger, value: FigureValue): string;
165
+ export declare function renderFigure(block: FigureBlock, ledger: ReferenceLedger, value: FigureValue, view?: ViewOptions): string;
156
166
  /**
157
167
  * Render a citation block as a bibliography card. The binding joins the one reference ladder like a claim
158
168
  * binding, thus one shared source keeps one bracket number. The card shows the marker, the short citation
@@ -161,6 +171,9 @@ export declare function renderFigure(block: FigureBlock, ledger: ReferenceLedger
161
171
  * The short citation of a `pmid:` record carries the PubMed link of its identifier. A key of another
162
172
  * identifier space shows the citation as text. A key with no pinned record shows the key and the note
163
173
  * alone, because absence is a normal condition and no text stands in for the paper.
174
+ *
175
+ * `view.lineage` states that the page carries a provenance document. The card then stamps the identity of
176
+ * the external record in place of a pin, because a paper is no artifact and no data chain reaches it.
164
177
  */
165
- export declare function renderCitation(block: CitationBlock, ledger: ReferenceLedger, record?: CitationRecord): string;
178
+ export declare function renderCitation(block: CitationBlock, ledger: ReferenceLedger, record?: CitationRecord, view?: ViewOptions): string;
166
179
  export {};
@@ -27,6 +27,7 @@ import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "hono/jsx/jsx-
27
27
  import { raw } from "hono/html";
28
28
  import { declaredForColumn } from "../../contracts/report-reference.js";
29
29
  import { stagedSource, tableSidecarName } from "../assets.js";
30
+ import { DEFAULT_VIEW_OPTIONS, lineagePlace, lineageStamp } from "./lineage.js";
30
31
  import { Marker } from "./references-view.js";
31
32
  import { formatNumberCell, formatTableCell, holdsANumber, selectColumnKind, selectNumberKind, smallestPositiveValue } from "../number-format.js";
32
33
  import { citationKeyOf } from "../references.js";
@@ -67,11 +68,14 @@ const DOWNLOAD_LABEL = "Download";
67
68
  * The value binding joins the reference ladder, thus the label line carries the marker and the appendix
68
69
  * names the cell. The marker sits on the label and never on the value, because the value line is the one
69
70
  * figure that a reader takes from the card.
71
+ *
72
+ * `view.lineage` states that the page carries a provenance document. The card then stamps its keys, and the
73
+ * marker carries the control that opens the chain.
70
74
  */
71
- export function renderMetric(block, ledger, value) {
75
+ export function renderMetric(block, ledger, value, view = DEFAULT_VIEW_OPTIONS) {
72
76
  const n = ledger.mark(block.value);
73
77
  const shown = formatNumberCell(value.value, selectNumberKind(block.label, value.value));
74
- return String(_jsxs("div", { class: "stat-card corner-accents", children: [_jsx("div", { class: "stat-card-value", title: shown.full, children: shown.text }), _jsxs("div", { class: "stat-card-label", children: [block.label, _jsx(Marker, { n: n })] })] }));
78
+ return String(_jsxs("div", { class: "stat-card corner-accents", ...lineageStamp(view.lineage, block.id, [block.value]), children: [_jsx("div", { class: "stat-card-value", title: shown.full, children: shown.text }), _jsxs("div", { class: "stat-card-label", children: [block.label, _jsx(Marker, { n: n, lineage: lineagePlace(view.lineage, block.value) })] })] }));
75
79
  }
76
80
  /**
77
81
  * Wrap a run of stat cards in one responsive grid. The caller renders each card first, and it passes the
@@ -199,13 +203,16 @@ export function tableStatusText(shown, total) {
199
203
  * The whole-table binding joins the reference ladder, thus the title line carries the marker and the
200
204
  * appendix names the artifact. Every evidentiary block ledgers this way, and a card with no title still
201
205
  * shows its marker on the same line.
206
+ *
207
+ * `view.lineage` states that the page carries a provenance document. The card then stamps its keys, and the
208
+ * marker carries the control that opens the chain.
202
209
  */
203
- export function renderTable(block, ledger, rowCount, total) {
210
+ export function renderTable(block, ledger, rowCount, total, view = DEFAULT_VIEW_OPTIONS) {
204
211
  const binding = block.binding;
205
212
  const n = ledger.mark(binding);
206
213
  const download = tableSidecarName(binding.hash, binding.path);
207
214
  const bound = boundText(binding);
208
- return String(_jsxs("div", { class: "report-table", children: [_jsxs("div", { class: "report-table-title", children: [block.title, _jsx(Marker, { n: n })] }), _jsxs("div", { class: "corner-accents", children: [_jsx("div", { class: "report-grid", [GRID_MOUNT_ATTRIBUTE]: block.id }), _jsxs("div", { class: "report-table-footer", children: [_jsxs("div", { class: "report-table-footer-row", children: [_jsxs("span", { class: "report-table-status", children: [_jsx("span", { class: GRID_COUNT_CLASS, children: tableStatusText(rowCount, total ?? rowCount) }), bound !== undefined ? _jsx("span", { class: "report-table-bound", children: bound }) : null] }), _jsx("a", { class: "report-table-download", href: stagedSource(download), download: download, children: downloadLabel(binding.path) })] }), _jsx("div", { class: GRID_NOTE_CLASS })] })] }), block.caption !== undefined ? _jsx("p", { class: "report-caption", children: block.caption }) : null] }));
215
+ return String(_jsxs("div", { class: "report-table", ...lineageStamp(view.lineage, block.id, [binding]), children: [_jsxs("div", { class: "report-table-title", children: [block.title, _jsx(Marker, { n: n, lineage: lineagePlace(view.lineage, binding) })] }), _jsxs("div", { class: "corner-accents", children: [_jsx("div", { class: "report-grid", [GRID_MOUNT_ATTRIBUTE]: block.id }), _jsxs("div", { class: "report-table-footer", children: [_jsxs("div", { class: "report-table-footer-row", children: [_jsxs("span", { class: "report-table-status", children: [_jsx("span", { class: GRID_COUNT_CLASS, children: tableStatusText(rowCount, total ?? rowCount) }), bound !== undefined ? _jsx("span", { class: "report-table-bound", children: bound }) : null] }), _jsx("a", { class: "report-table-download", href: stagedSource(download), download: download, children: downloadLabel(binding.path) })] }), _jsx("div", { class: GRID_NOTE_CLASS })] })] }), block.caption !== undefined ? _jsx("p", { class: "report-caption", children: block.caption }) : null] }));
209
216
  }
210
217
  /**
211
218
  * Render a figure block as a corner-accent card. The source rides the `src` attribute, and the caption
@@ -215,11 +222,14 @@ export function renderTable(block, ledger, rowCount, total) {
215
222
  * The whole-file binding joins the reference ladder, thus the caption line carries the marker and the
216
223
  * appendix names the image. A figure with no caption still shows its marker on that line. The marker stays
217
224
  * out of the `alt` text, because a screen reader reads the alt as the picture and not as a footnote.
225
+ *
226
+ * `view.lineage` states that the page carries a provenance document. The card then stamps its keys, and the
227
+ * marker carries the control that opens the chain.
218
228
  */
219
- export function renderFigure(block, ledger, value) {
229
+ export function renderFigure(block, ledger, value, view = DEFAULT_VIEW_OPTIONS) {
220
230
  const n = ledger.mark(block.binding);
221
231
  const caption = block.caption;
222
- return String(_jsxs("figure", { class: "report-figure corner-accents", children: [_jsx("img", { src: value.src, alt: caption !== undefined ? caption : "", class: "report-figure-image" }), _jsxs("figcaption", { class: "report-caption", children: [caption, _jsx(Marker, { n: n })] })] }));
232
+ return String(_jsxs("figure", { class: "report-figure corner-accents", ...lineageStamp(view.lineage, block.id, [block.binding]), children: [_jsx("img", { src: value.src, alt: caption !== undefined ? caption : "", class: "report-figure-image" }), _jsxs("figcaption", { class: "report-caption", children: [caption, _jsx(Marker, { n: n, lineage: lineagePlace(view.lineage, block.binding) })] })] }));
223
233
  }
224
234
  /**
225
235
  * The PubMed address of one identifier. The address is deterministic, and an anchor loads nothing, thus a
@@ -234,10 +244,13 @@ const PUBMED_BASE = "https://pubmed.ncbi.nlm.nih.gov/";
234
244
  * The short citation of a `pmid:` record carries the PubMed link of its identifier. A key of another
235
245
  * identifier space shows the citation as text. A key with no pinned record shows the key and the note
236
246
  * alone, because absence is a normal condition and no text stands in for the paper.
247
+ *
248
+ * `view.lineage` states that the page carries a provenance document. The card then stamps the identity of
249
+ * the external record in place of a pin, because a paper is no artifact and no data chain reaches it.
237
250
  */
238
- export function renderCitation(block, ledger, record) {
251
+ export function renderCitation(block, ledger, record, view = DEFAULT_VIEW_OPTIONS) {
239
252
  const n = ledger.mark(block.binding);
240
253
  const binding = block.binding;
241
254
  const key = citationKeyOf(binding);
242
- return String(_jsxs("div", { class: "report-citation corner-accents", children: [_jsx(Marker, { n: n }), record !== undefined ? (_jsxs(_Fragment, { children: [" ", binding.idKind === "pmid" ? (_jsx("a", { class: "report-citation-source", href: `${PUBMED_BASE}${encodeURIComponent(binding.id)}/`, children: record.citation })) : (_jsx("span", { class: "report-citation-source", children: record.citation }))] })) : null, block.note !== undefined ? (_jsxs(_Fragment, { children: [" ", _jsx("span", { class: "report-citation-note", children: block.note })] })) : null, " ", _jsx("span", { class: "report-citation-key", children: key })] }));
255
+ return String(_jsxs("div", { class: "report-citation corner-accents", ...lineageStamp(view.lineage, block.id, [binding]), children: [_jsx(Marker, { n: n, lineage: lineagePlace(view.lineage, binding) }), record !== undefined ? (_jsxs(_Fragment, { children: [" ", binding.idKind === "pmid" ? (_jsx("a", { class: "report-citation-source", href: `${PUBMED_BASE}${encodeURIComponent(binding.id)}/`, children: record.citation })) : (_jsx("span", { class: "report-citation-source", children: record.citation }))] })) : null, block.note !== undefined ? (_jsxs(_Fragment, { children: [" ", _jsx("span", { class: "report-citation-note", children: block.note })] })) : null, " ", _jsx("span", { class: "report-citation-key", children: key })] }));
243
256
  }
@@ -34,6 +34,7 @@ import { type CitationResolverConfig } from "../citations/resolve.js";
34
34
  import type { CitationResolver } from "../citations/types.js";
35
35
  import type { AuthContext } from "../auth/types.js";
36
36
  import type { ReferenceResolver } from "../report-model/reference-resolver.js";
37
+ import type { ProvenanceSeam } from "../provenance/seam.js";
37
38
  import type { MakeSessionPagePublisher, ResolvePageAsset, ResolvePageUrl } from "../tools/report-session/index.js";
38
39
  /** Registered child sandbox-step callable the parent's child dispatch closes over. */
39
40
  export type SandboxStepCallable = (input: SandboxStepInput) => Promise<SandboxStepResult>;
@@ -160,6 +161,27 @@ export interface CoreRuntimeDeps {
160
161
  * the eyes tool of a report session then reports that condition.
161
162
  */
162
163
  readonly eyes?: AcquireEyes;
164
+ /**
165
+ * The provenance seam. Each act of a session — the creation of the session, the block operations, the
166
+ * title, the derivation, the preview, and the record — rides the session emit as one typed event, and
167
+ * the embedder lands the events in the record of the analysis that it owns. The emit is
168
+ * fire-and-forget: it gives no result, a throw of the realization reaches the log alone, and an unbound
169
+ * member emits nothing. Thus a composition that binds none behaves the same as one that never had the
170
+ * seam.
171
+ *
172
+ * The document read gives the signed document of the analysis back, as opaque text. The preview of a
173
+ * report session carries a frozen copy into the page, and the page then shows the chain of a grounded
174
+ * block with no host and no network. The harness parses nothing. Absence is a normal result: an unbound
175
+ * member, and a bound member that holds no document for the analysis, both give a page with no
176
+ * provenance and no other change.
177
+ *
178
+ * Each member is optional alone, thus a composition binds the members that it records.
179
+ *
180
+ * The assembly reaches two consumers with the seam: the report agent, and the conversation agent, whose
181
+ * start tool passes it to the spawn. The creation of a conversation thread rides `prepareChatTurn`,
182
+ * which an embedder calls on its own. Thus that composition binds the seam there as well.
183
+ */
184
+ readonly provenance?: ProvenanceSeam;
163
185
  }
164
186
  /**
165
187
  * The refusal a thread type carries no agent yet. `report` is a valid
@@ -124,6 +124,9 @@ export function assembleCoreRuntime(deps) {
124
124
  usageRecorder,
125
125
  citationResolver,
126
126
  ...(eyes ? { eyes } : {}),
127
+ // The start tool passes it to the spawn, which emits the creation of a
128
+ // report session. The same seam reaches the report agent below.
129
+ ...(deps.provenance ? { provenance: deps.provenance } : {}),
127
130
  });
128
131
  // The report agent is a singleton over the conversation deps, the same way the
129
132
  // session runtime is. Its preview tool writes the page into the analysis tree
@@ -173,6 +176,7 @@ export function assembleCoreRuntime(deps) {
173
176
  ...(deps.resolveReportPageAsset ? { resolvePageAsset: deps.resolveReportPageAsset } : {}),
174
177
  ...(deps.makeSessionPagePublisher ? { makeSessionPages: deps.makeSessionPagePublisher } : {}),
175
178
  ...(deps.resolveReportPageUrl ? { resolvePageUrl: deps.resolveReportPageUrl } : {}),
179
+ ...(deps.provenance ? { provenance: deps.provenance } : {}),
176
180
  ...(conversation.logger ? { logger: conversation.logger } : {}),
177
181
  });
178
182
  // The single registration point for every typed agent. A future thread type's
@@ -60,11 +60,6 @@ export interface K8sClientConfig {
60
60
  tolerations?: V1Toleration[];
61
61
  /** RuntimeClass for runtime isolation (e.g. `gvisor`). Omit for the default runtime. */
62
62
  runtimeClassName?: string;
63
- /**
64
- * The `/proc/cpuinfo` text that the cpu ConfigMap of each Job is cut from.
65
- * Defaults to the file of the host of this process, read once. Injected for tests.
66
- */
67
- readHostCpuinfo?: () => Promise<string | undefined>;
68
63
  /** Injected for tests. */
69
64
  batchApi?: BatchV1Api;
70
65
  coreApi?: CoreV1Api;
@@ -17,7 +17,6 @@ import { ResultAsync, err, ok } from "neverthrow";
17
17
  import { trySandbox } from "./sandbox-error.js";
18
18
  import { buildMountPlan, buildSessionSubPaths } from "./mount-plan.js";
19
19
  import { threadLimitEnv } from "./thread-env.js";
20
- import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
21
20
  import { createNoopLogger } from "../lib/console-logger.js";
22
21
  /** Read the originating HTTP status off any `SandboxError` variant that carries one. */
23
22
  function statusOf(e) {
@@ -26,8 +25,6 @@ function statusOf(e) {
26
25
  const SESSION_VOLUME_NAME = "session";
27
26
  const LIBS_VOLUME_NAME = "libs";
28
27
  const REFS_VOLUME_NAME = "refs";
29
- /** The ConfigMap volume that holds the two cpu files of the sandbox (`cpu-files.ts`). */
30
- const CPU_VOLUME_NAME = "cpu";
31
28
  const SANDBOX_SERVER_PORT = 8765;
32
29
  const POD_READY_TIMEOUT_MS = 5 * 60_000;
33
30
  const POD_POLL_INTERVAL_MS = 1_000;
@@ -63,19 +60,6 @@ export function sanitizeLabelValue(value) {
63
60
  function readOwnerWorkflowId(metadata) {
64
61
  return metadata?.annotations?.[OWNER_WORKFLOW_ANNOTATION] ?? null;
65
62
  }
66
- function deleteConfigMapIgnoreMissing(coreApi, namespace, name) {
67
- return trySandbox(async () => {
68
- await coreApi.deleteNamespacedConfigMap({ namespace, name });
69
- }, (status, cause) => ({
70
- type: "teardown_failed",
71
- op: "k8s.teardown",
72
- sandboxId: name,
73
- status,
74
- cause,
75
- })).orElse((e) =>
76
- // A 404 means the owner reference removed it with the Job already.
77
- statusOf(e) === 404 ? ok(undefined) : err(e));
78
- }
79
63
  function deleteJobIgnoreMissing(batchApi, namespace, name) {
80
64
  return trySandbox(async () => {
81
65
  await batchApi.deleteNamespacedJob({
@@ -124,7 +108,7 @@ function workspaceSubPathFor(config, analysisId) {
124
108
  }
125
109
  return posix;
126
110
  }
127
- function buildJobSpec(meta, config, identity, cpu) {
111
+ function buildJobSpec(meta, config, identity) {
128
112
  const { sandboxId } = identity;
129
113
  const plan = buildMountPlan(meta, {
130
114
  libs: !!config.libStorePvc,
@@ -205,16 +189,6 @@ function buildJobSpec(meta, config, identity, cpu) {
205
189
  readOnly: true,
206
190
  });
207
191
  }
208
- // The cpu files ride in a ConfigMap named after the Job (`ensureCpuConfigMap`).
209
- // A `subPath` mount of one key over one kernel path is accepted by the API
210
- // server, the kubelet, and gVisor. Under gVisor the sandbox sees no cgroup
211
- // at all, thus these two files are the only source of the quota for
212
- // `parallel::detectCores()` and `os.cpu_count()`.
213
- volumes.push({ name: CPU_VOLUME_NAME, configMap: { name: sandboxId } });
214
- volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPU_ONLINE_PATH, subPath: "online", readOnly: true });
215
- if (cpu.hasCpuinfo) {
216
- volumeMounts.push({ name: CPU_VOLUME_NAME, mountPath: CPUINFO_PATH, subPath: "cpuinfo", readOnly: true });
217
- }
218
192
  const podSpec = {
219
193
  automountServiceAccountToken: false,
220
194
  restartPolicy: "Never",
@@ -385,9 +359,6 @@ function waitForJobGone(batchApi, namespace, name) {
385
359
  * means an (astronomically rare) name collision with a *different* step —
386
360
  * adopting its pod would HMAC-fail every callback, and deleting its Job would
387
361
  * kill a live sibling. Refuse loudly instead.
388
- *
389
- * The result is the uid of the Job that the pod belongs to, for the owner
390
- * reference of the cpu ConfigMap.
391
362
  */
392
363
  function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflowId, job) {
393
364
  const createFailed = (status, cause) => ({
@@ -400,7 +371,7 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
400
371
  return new ResultAsync((async () => {
401
372
  const created = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
402
373
  if (created.isOk())
403
- return ok(created.value?.metadata?.uid);
374
+ return ok(undefined);
404
375
  if (statusOf(created.error) !== 409)
405
376
  return err(created.error);
406
377
  const existingRead = await trySandbox(() => batchApi.readNamespacedJob({ namespace, name: sandboxId }), createFailed);
@@ -428,92 +399,21 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
428
399
  const recreated = await trySandbox(() => batchApi.createNamespacedJob({ namespace, body: job }), createFailed);
429
400
  if (recreated.isErr())
430
401
  return err(recreated.error);
431
- return ok(recreated.value?.metadata?.uid);
432
- }
433
- return ok(existingRead.value.metadata?.uid);
434
- })());
435
- }
436
- /**
437
- * Make the cpu ConfigMap of a Job, or bring an existing one up to date.
438
- *
439
- * The ConfigMap comes after the Job, because the Job is its owner. Thus the
440
- * TTL of the Job and a Foreground delete both remove the ConfigMap with it.
441
- * The pod waits at its mount until the ConfigMap exists, and that wait is
442
- * shorter than the schedule of the pod.
443
- *
444
- * A `409 AlreadyExists` is the recovery re-run of the spawn step, or a
445
- * ConfigMap whose prior owner Job is not yet collected. Both cases get the
446
- * same data and the current owner through a replace.
447
- */
448
- function ensureCpuConfigMap(coreApi, namespace, sandboxId, jobUid, files) {
449
- const createFailed = (status, cause) => ({
450
- type: "container_create_failed",
451
- op: "k8s.createNamespacedConfigMap",
452
- sandboxId,
453
- status,
454
- cause,
455
- });
456
- const body = {
457
- apiVersion: "v1",
458
- kind: "ConfigMap",
459
- metadata: {
460
- name: sandboxId,
461
- namespace,
462
- labels: {
463
- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
464
- role: "sandbox",
465
- "cortex/sandbox-id": sandboxId,
466
- },
467
- ...(jobUid !== undefined && {
468
- ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: sandboxId, uid: jobUid }],
469
- }),
470
- },
471
- data: {
472
- online: files.online,
473
- ...(files.cpuinfo !== undefined && { cpuinfo: files.cpuinfo }),
474
- },
475
- };
476
- return new ResultAsync((async () => {
477
- const created = await trySandbox(() => coreApi.createNamespacedConfigMap({ namespace, body }), createFailed);
478
- if (created.isOk())
479
402
  return ok(undefined);
480
- if (statusOf(created.error) !== 409)
481
- return err(created.error);
482
- const existing = await trySandbox(() => coreApi.readNamespacedConfigMap({ namespace, name: sandboxId }), createFailed);
483
- if (existing.isErr())
484
- return err(existing.error);
485
- const replaced = await trySandbox(() => coreApi.replaceNamespacedConfigMap({
486
- namespace,
487
- name: sandboxId,
488
- body: { ...body, metadata: { ...body.metadata, resourceVersion: existing.value.metadata?.resourceVersion } },
489
- }), createFailed);
490
- if (replaced.isErr())
491
- return err(replaced.error);
403
+ }
492
404
  return ok(undefined);
493
405
  })());
494
406
  }
495
407
  export function createK8sSandboxOps(config) {
496
408
  const { batchApi, coreApi } = config.batchApi && config.coreApi ? { batchApi: config.batchApi, coreApi: config.coreApi } : buildKubeApi();
497
- const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
498
409
  return {
499
410
  createSandbox(meta, identity) {
500
411
  return new ResultAsync((async () => {
501
412
  const { sandboxId } = identity;
502
- // The same floor as `threadLimitEnv`: a fraction of a core still
503
- // keeps one thread busy, and a count of zero is not representable.
504
- const threads = Math.max(1, Math.floor(meta.resources.cpu));
505
- const files = cpuFiles(threads, await readHostCpuinfo());
506
- const job = buildJobSpec(meta, config, identity, { hasCpuinfo: files.cpuinfo !== undefined });
413
+ const job = buildJobSpec(meta, config, identity);
507
414
  const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, meta.childWorkflowId, job);
508
415
  if (adopted.isErr())
509
416
  return err(adopted.error);
510
- const cpuMap = await ensureCpuConfigMap(coreApi, config.namespace, sandboxId, adopted.value, files);
511
- if (cpuMap.isErr()) {
512
- // A pod whose ConfigMap never comes stays at ContainerCreating
513
- // until the ready wait expires. Fail now, with the real cause.
514
- await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
515
- return err(cpuMap.error);
516
- }
517
417
  // A Job whose pod never schedules (quota rejection, no nodes, spot
518
418
  // reclaim) is reaped by neither backoffLimit (an admission rejection
519
419
  // creates no pod to count) nor ttlSecondsAfterFinished (the Job never
@@ -537,22 +437,20 @@ export function createK8sSandboxOps(config) {
537
437
  cause,
538
438
  }));
539
439
  if (registered.isErr()) {
540
- await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
440
+ await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
541
441
  return err(registered.error);
542
442
  }
543
443
  return ok(ref);
544
444
  }
545
- await cleanupFailedJob(batchApi, coreApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
445
+ await cleanupFailedJob(batchApi, config.namespace, sandboxId, config.logger ?? createNoopLogger());
546
446
  return err(ready.error);
547
447
  })());
548
448
  },
549
- // The owner reference removes the cpu ConfigMap with the Job. The explicit
550
- // delete covers a ConfigMap that got no owner, and it is a no-op on 404.
551
449
  teardown(ref) {
552
- return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, ref.sandboxId));
450
+ return deleteJobIgnoreMissing(batchApi, config.namespace, ref.sandboxId);
553
451
  },
554
452
  teardownById(sandboxId) {
555
- return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId).andThen(() => deleteConfigMapIgnoreMissing(coreApi, config.namespace, sandboxId));
453
+ return deleteJobIgnoreMissing(batchApi, config.namespace, sandboxId);
556
454
  },
557
455
  listManagedSandboxes() {
558
456
  return trySandbox(() => batchApi.listNamespacedJob({
@@ -613,13 +511,9 @@ export function createK8sSandboxOps(config) {
613
511
  * matters; a delete error here is logged and swallowed so it never masks the
614
512
  * create error returned to the caller.
615
513
  */
616
- async function cleanupFailedJob(batchApi, coreApi, namespace, sandboxId, logger) {
514
+ async function cleanupFailedJob(batchApi, namespace, sandboxId, logger) {
617
515
  const deleted = await deleteJobIgnoreMissing(batchApi, namespace, sandboxId);
618
516
  if (deleted.isErr()) {
619
517
  logger.named("k8s-client").error("failed to delete Job after startup failure", { sandboxId, err: deleted.error });
620
518
  }
621
- const deletedMap = await deleteConfigMapIgnoreMissing(coreApi, namespace, sandboxId);
622
- if (deletedMap.isErr()) {
623
- logger.named("k8s-client").error("failed to delete cpu ConfigMap after startup failure", { sandboxId, err: deletedMap.error });
624
- }
625
519
  }
@@ -2,26 +2,32 @@
2
2
  * Thread-count env for a sandbox container.
3
3
  *
4
4
  * Each backend turns the `ResourceSpec` of a step into a hard cgroup limit:
5
- * `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s. A
6
- * cgroup quota is invisible to most runtimes inside it. `parallel::detectCores()`
7
- * greps `/proc/cpuinfo`, and `os.cpu_count()` reads
8
- * `/sys/devices/system/cpu/online`. Both files describe the host. Thus a step
9
- * with 2 CPUs on a host of 12 cores forks 12 workers into a cgroup of 2 CPUs,
10
- * and each fork copies its working set until the memory limit stops it.
5
+ * `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s.
6
+ * The mounted cpu files show the quota to `parallel::detectCores()` and to
7
+ * `os.cpu_count()`. Thus the fork count of a step follows the quota by default.
11
8
  *
12
- * The env below publishes the quota to each library that reads a thread count
13
- * from the environment. `parallelly::availableCores()` and `joblib.cpu_count()`
14
- * read the cgroup quota on their own, thus they need no value here.
9
+ * The thread pools get a different default. A fork copies the pool size of its
10
+ * parent, and no R runtime divides it. Thus a pool at the quota turns into
11
+ * `workers * quota` threads inside one cgroup. The env below pins each pool to
12
+ * one thread, which is the convention of Snakemake and of the HPC sites. A
13
+ * thread-parallel step raises the value per command, through the `env`
14
+ * parameter of `execute_command`. The rule for the agent is one line: workers
15
+ * times threads must not go above the quota.
16
+ *
17
+ * The two worker-count names stay at the quota. They cap process workers, not
18
+ * threads, and a worker with one thread is safe at each count up to the quota.
19
+ * joblib divides its own budget by its worker count, and a value of one in
20
+ * `OMP_NUM_THREADS` gives each of its workers one thread as well.
15
21
  *
16
22
  * `MC_CORES` is absent on purpose. R defaults `mclapply` to 2 workers, not to
17
23
  * the core count. A value there raises the fork count instead of a decrease.
18
24
  */
19
25
  import type { ResourceSpec } from "../config/resource-limits.js";
20
26
  /**
21
- * Env that binds each parallel library in the container to `spec.cpu`.
27
+ * Env that makes each parallel library in the container safe under `spec.cpu`.
22
28
  *
23
- * A fractional CPU request floors to one thread. A pool of zero is not
24
- * representable, and two threads on half a core only add context switches.
29
+ * A fractional CPU request floors to one worker. A pool of zero is not
30
+ * representable, and two workers on half a core only add context switches.
25
31
  *
26
32
  * The caller puts these before `plan.env` and before the `extraEnv` of the
27
33
  * embedder, thus a host that knows more about a step keeps the last word.
@@ -2,48 +2,59 @@
2
2
  * Thread-count env for a sandbox container.
3
3
  *
4
4
  * Each backend turns the `ResourceSpec` of a step into a hard cgroup limit:
5
- * `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s. A
6
- * cgroup quota is invisible to most runtimes inside it. `parallel::detectCores()`
7
- * greps `/proc/cpuinfo`, and `os.cpu_count()` reads
8
- * `/sys/devices/system/cpu/online`. Both files describe the host. Thus a step
9
- * with 2 CPUs on a host of 12 cores forks 12 workers into a cgroup of 2 CPUs,
10
- * and each fork copies its working set until the memory limit stops it.
5
+ * `NanoCpus` and `Memory` on Docker, `limits.cpu` and `limits.memory` on K8s.
6
+ * The mounted cpu files show the quota to `parallel::detectCores()` and to
7
+ * `os.cpu_count()`. Thus the fork count of a step follows the quota by default.
11
8
  *
12
- * The env below publishes the quota to each library that reads a thread count
13
- * from the environment. `parallelly::availableCores()` and `joblib.cpu_count()`
14
- * read the cgroup quota on their own, thus they need no value here.
9
+ * The thread pools get a different default. A fork copies the pool size of its
10
+ * parent, and no R runtime divides it. Thus a pool at the quota turns into
11
+ * `workers * quota` threads inside one cgroup. The env below pins each pool to
12
+ * one thread, which is the convention of Snakemake and of the HPC sites. A
13
+ * thread-parallel step raises the value per command, through the `env`
14
+ * parameter of `execute_command`. The rule for the agent is one line: workers
15
+ * times threads must not go above the quota.
16
+ *
17
+ * The two worker-count names stay at the quota. They cap process workers, not
18
+ * threads, and a worker with one thread is safe at each count up to the quota.
19
+ * joblib divides its own budget by its worker count, and a value of one in
20
+ * `OMP_NUM_THREADS` gives each of its workers one thread as well.
15
21
  *
16
22
  * `MC_CORES` is absent on purpose. R defaults `mclapply` to 2 workers, not to
17
23
  * the core count. A value there raises the fork count instead of a decrease.
18
24
  */
19
25
  /**
20
- * Env that binds each parallel library in the container to `spec.cpu`.
26
+ * Env that makes each parallel library in the container safe under `spec.cpu`.
21
27
  *
22
- * A fractional CPU request floors to one thread. A pool of zero is not
23
- * representable, and two threads on half a core only add context switches.
28
+ * A fractional CPU request floors to one worker. A pool of zero is not
29
+ * representable, and two workers on half a core only add context switches.
24
30
  *
25
31
  * The caller puts these before `plan.env` and before the `extraEnv` of the
26
32
  * embedder, thus a host that knows more about a step keeps the last word.
27
33
  */
28
34
  export function threadLimitEnv(spec) {
29
- const threads = String(Math.max(1, Math.floor(spec.cpu)));
35
+ const workers = String(Math.max(1, Math.floor(spec.cpu)));
30
36
  return {
37
+ // Thread pools: one thread each. The agent raises a value per command.
31
38
  // OpenMP: the compiled code of R and Python packages.
32
- OMP_NUM_THREADS: threads,
39
+ OMP_NUM_THREADS: "1",
33
40
  // OpenBLAS keeps a private pool. A pthread build reads this name first.
34
- OPENBLAS_NUM_THREADS: threads,
41
+ OPENBLAS_NUM_THREADS: "1",
35
42
  // MKL reads this name before OMP_NUM_THREADS.
36
- MKL_NUM_THREADS: threads,
43
+ MKL_NUM_THREADS: "1",
44
+ // data.table defaults to half of the visible cores.
45
+ R_DATATABLE_NUM_THREADS: "1",
46
+ // The Python numerical stack.
47
+ NUMBA_NUM_THREADS: "1",
48
+ NUMEXPR_MAX_THREADS: "1",
49
+ POLARS_MAX_THREADS: "1",
50
+ RAYON_NUM_THREADS: "1",
51
+ // Worker counts: the quota.
37
52
  // BiocParallel sizes `bpparam()` from this name. DESeq2 `parallel=TRUE`
38
53
  // and the other Bioconductor packages size their workers from `bpparam()`.
39
- BIOCPARALLEL_WORKER_NUMBER: threads,
40
- // data.table defaults to half of the host cores.
41
- R_DATATABLE_NUM_THREADS: threads,
42
- // The Python numerical stack.
43
- NUMBA_NUM_THREADS: threads,
44
- NUMEXPR_MAX_THREADS: threads,
45
- LOKY_MAX_CPU_COUNT: threads,
46
- POLARS_MAX_THREADS: threads,
47
- RAYON_NUM_THREADS: threads,
54
+ // `parallelly::availableCores()` reads it too, and under gVisor, where no
55
+ // cgroup is visible, it is the only bound that library sees.
56
+ BIOCPARALLEL_WORKER_NUMBER: workers,
57
+ // loky caps its worker default at this name.
58
+ LOKY_MAX_CPU_COUNT: workers,
48
59
  };
49
60
  }