@inflexa-ai/harness 0.30.2 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/dist/contracts/target-dossier.d.ts +61 -61
  2. package/dist/execution/run-synthesis.js +7 -2
  3. package/dist/lib/zod-issues.d.ts +8 -0
  4. package/dist/lib/zod-issues.js +13 -0
  5. package/dist/literature/sources/semantic-scholar.d.ts +34 -0
  6. package/dist/literature/sources/semantic-scholar.js +15 -4
  7. package/dist/sandbox/cpu-files.d.ts +37 -0
  8. package/dist/sandbox/cpu-files.js +55 -0
  9. package/dist/sandbox/docker-client.d.ts +15 -0
  10. package/dist/sandbox/docker-client.js +100 -11
  11. package/dist/sandbox/k8s-client.d.ts +5 -0
  12. package/dist/sandbox/k8s-client.js +128 -19
  13. package/dist/sandbox/thread-env.d.ts +29 -0
  14. package/dist/sandbox/thread-env.js +49 -0
  15. package/dist/state/data-profile.d.ts +35 -0
  16. package/dist/state/data-profile.js +56 -0
  17. package/dist/state/index.d.ts +1 -1
  18. package/dist/state/index.js +1 -1
  19. package/dist/tasks/data-profile.d.ts +16 -1
  20. package/dist/tasks/data-profile.js +31 -10
  21. package/dist/tools/bio/chembl.js +4 -3
  22. package/dist/tools/bio/drug-gene-interactions.d.ts +2 -14
  23. package/dist/tools/bio/drug-gene-interactions.js +10 -18
  24. package/dist/tools/bio/gene-disease-evidence.d.ts +3 -1
  25. package/dist/tools/bio/gene-disease-evidence.js +12 -9
  26. package/dist/tools/bio/keys.js +1 -0
  27. package/dist/tools/bio/lookup-annotation.d.ts +43 -0
  28. package/dist/tools/bio/lookup-annotation.js +19 -4
  29. package/dist/tools/bio/pubchem.js +5 -3
  30. package/dist/tools/bio/search-clinical-trials.js +5 -4
  31. package/dist/tools/bio/search-geo-datasets.d.ts +39 -9
  32. package/dist/tools/bio/search-geo-datasets.js +38 -20
  33. package/dist/tools/lib/api-utils.d.ts +27 -2
  34. package/dist/tools/lib/api-utils.js +43 -3
  35. package/dist/tools/lib/bgee-client.d.ts +16 -0
  36. package/dist/tools/lib/bgee-client.js +16 -5
  37. package/dist/tools/lib/cbioportal-client.d.ts +20 -0
  38. package/dist/tools/lib/cbioportal-client.js +16 -8
  39. package/dist/tools/lib/chembl-client.d.ts +105 -3
  40. package/dist/tools/lib/chembl-client.js +196 -84
  41. package/dist/tools/lib/clinical-benchmarks-client.d.ts +5 -0
  42. package/dist/tools/lib/clinical-benchmarks-client.js +5 -0
  43. package/dist/tools/lib/clinical-trials-client.d.ts +131 -2
  44. package/dist/tools/lib/clinical-trials-client.js +32 -6
  45. package/dist/tools/lib/clinvar-client.d.ts +35 -0
  46. package/dist/tools/lib/clinvar-client.js +34 -28
  47. package/dist/tools/lib/ctx-ops.d.ts +106 -35
  48. package/dist/tools/lib/ctx-ops.js +135 -69
  49. package/dist/tools/lib/dgidb-client.d.ts +73 -0
  50. package/dist/tools/lib/dgidb-client.js +25 -13
  51. package/dist/tools/lib/disgenet-client.d.ts +117 -24
  52. package/dist/tools/lib/disgenet-client.js +127 -34
  53. package/dist/tools/lib/disgenet-config.d.ts +1 -1
  54. package/dist/tools/lib/disgenet-config.js +5 -3
  55. package/dist/tools/lib/drugbank-client.d.ts +57 -58
  56. package/dist/tools/lib/drugbank-client.js +69 -64
  57. package/dist/tools/lib/drugbank-config.d.ts +6 -1
  58. package/dist/tools/lib/drugbank-config.js +6 -1
  59. package/dist/tools/lib/ema-client.d.ts +53 -0
  60. package/dist/tools/lib/ema-client.js +19 -7
  61. package/dist/tools/lib/ensembl-client.d.ts +4 -0
  62. package/dist/tools/lib/ensembl-client.js +4 -0
  63. package/dist/tools/lib/gwas-catalog-client.d.ts +64 -1
  64. package/dist/tools/lib/gwas-catalog-client.js +21 -6
  65. package/dist/tools/lib/identifier-resolver.d.ts +17 -1
  66. package/dist/tools/lib/identifier-resolver.js +13 -4
  67. package/dist/tools/lib/impc-client.d.ts +4 -0
  68. package/dist/tools/lib/impc-client.js +4 -0
  69. package/dist/tools/lib/iuphar-client.d.ts +10 -5
  70. package/dist/tools/lib/iuphar-client.js +49 -13
  71. package/dist/tools/lib/monarch-client.d.ts +4 -0
  72. package/dist/tools/lib/monarch-client.js +4 -0
  73. package/dist/tools/lib/openfda-client.d.ts +34 -0
  74. package/dist/tools/lib/openfda-client.js +22 -8
  75. package/dist/tools/lib/opentargets-client.d.ts +99 -4
  76. package/dist/tools/lib/opentargets-client.js +97 -37
  77. package/dist/tools/lib/pathway-client.d.ts +45 -0
  78. package/dist/tools/lib/pathway-client.js +88 -8
  79. package/dist/tools/lib/pharmgkb-client.d.ts +35 -1
  80. package/dist/tools/lib/pharmgkb-client.js +22 -6
  81. package/dist/tools/lib/pharmgkb-config.d.ts +8 -1
  82. package/dist/tools/lib/pharmgkb-config.js +8 -1
  83. package/dist/tools/lib/pubchem-client.d.ts +22 -0
  84. package/dist/tools/lib/pubchem-client.js +36 -33
  85. package/dist/tools/lib/pubchem-ops.d.ts +85 -5
  86. package/dist/tools/lib/pubchem-ops.js +96 -68
  87. package/dist/tools/lib/pubmed-client.d.ts +4 -0
  88. package/dist/tools/lib/pubmed-client.js +4 -0
  89. package/dist/tools/lib/string-client.d.ts +52 -0
  90. package/dist/tools/lib/string-client.js +12 -7
  91. package/dist/tools/lib/uniprot-client.d.ts +4 -0
  92. package/dist/tools/lib/uniprot-client.js +4 -0
  93. package/dist/tools/research/context7-docs.d.ts +23 -0
  94. package/dist/tools/research/context7-docs.js +49 -9
  95. package/dist/tools/research/search-github-repos.d.ts +4 -0
  96. package/dist/tools/research/search-github-repos.js +4 -0
  97. package/dist/workflows/execute-target-assessment.js +4 -4
  98. package/dist/workflows/target-assessment/assemblers/orchestrator.d.ts +2 -1
  99. package/dist/workflows/target-assessment/assemblers/orchestrator.js +3 -3
  100. package/dist/workflows/target-assessment/collectors/index.d.ts +8 -2
  101. package/dist/workflows/target-assessment/collectors/index.js +37 -10
  102. package/dist/workflows/target-assessment/fanout/index.d.ts +2 -1
  103. package/dist/workflows/target-assessment/fanout/index.js +2 -2
  104. package/dist/workflows/target-assessment/lib/safety-corroboration.js +2 -2
  105. package/dist/workflows/target-assessment/lib/target-identity-filter.d.ts +3 -2
  106. package/dist/workflows/target-assessment/lib/target-identity-filter.js +16 -5
  107. package/dist/workflows/target-assessment/phase4-assemble.d.ts +61 -60
  108. package/dist/workflows/target-assessment/phase4-assemble.js +2 -2
  109. package/dist/workflows/target-assessment/schemas.d.ts +2 -8
  110. package/dist/workflows/target-assessment/schemas.js +8 -2
  111. package/dist/workflows/target-assessment/steps/phase2-aggregate.d.ts +1 -4
  112. package/dist/workflows/target-assessment/steps/phase3-aggregate.d.ts +1 -4
  113. package/package.json +1 -1
  114. package/dist/agents/sandbox/__fixtures__/deps.d.ts +0 -23
  115. package/dist/agents/sandbox/__fixtures__/deps.js +0 -102
  116. package/dist/citations/__fixtures__/resolver.d.ts +0 -3
  117. package/dist/citations/__fixtures__/resolver.js +0 -9
  118. package/dist/loop/__fixtures__/scripted-provider.d.ts +0 -31
  119. package/dist/loop/__fixtures__/scripted-provider.js +0 -81
  120. package/dist/providers/__fixtures__/session.d.ts +0 -17
  121. package/dist/providers/__fixtures__/session.js +0 -20
  122. package/dist/tools/__fixtures__/tool-context.d.ts +0 -13
  123. package/dist/tools/__fixtures__/tool-context.js +0 -22
@@ -140,8 +140,8 @@ export declare const EntitySchema: z.ZodObject<{
140
140
  displayName: z.ZodString;
141
141
  entityType: z.ZodEnum<{
142
142
  gene: "gene";
143
- rna: "rna";
144
143
  protein: "protein";
144
+ rna: "rna";
145
145
  complex: "complex";
146
146
  }>;
147
147
  ontology: z.ZodString;
@@ -5985,8 +5985,8 @@ export declare const DossierSchema: z.ZodObject<{
5985
5985
  displayName: z.ZodString;
5986
5986
  entityType: z.ZodEnum<{
5987
5987
  gene: "gene";
5988
- rna: "rna";
5989
5988
  protein: "protein";
5989
+ rna: "rna";
5990
5990
  complex: "complex";
5991
5991
  }>;
5992
5992
  ontology: z.ZodString;
@@ -9243,64 +9243,6 @@ export type Dossier = z.infer<typeof DossierSchema>;
9243
9243
  * and validates the complete dossier against `DossierSchema`.
9244
9244
  */
9245
9245
  export declare const DossierBodySchema: z.ZodObject<{
9246
- drug_interactions: z.ZodDiscriminatedUnion<[z.ZodObject<{
9247
- coverage: z.ZodLiteral<"available">;
9248
- data: z.ZodObject<{
9249
- rows: z.ZodArray<z.ZodObject<{
9250
- drug_id: z.ZodNullable<z.ZodString>;
9251
- drug_name: z.ZodString;
9252
- best_score: z.ZodNumber;
9253
- predicates: z.ZodArray<z.ZodString>;
9254
- sources: z.ZodArray<z.ZodString>;
9255
- paper_count: z.ZodNumber;
9256
- dominant_direction: z.ZodEnum<{
9257
- unknown: "unknown";
9258
- positive: "positive";
9259
- negative: "negative";
9260
- mixed: "mixed";
9261
- }>;
9262
- has_human_evidence: z.ZodBoolean;
9263
- has_clinical_evidence: z.ZodBoolean;
9264
- evidence: z.ZodArray<z.ZodObject<{
9265
- pmid: z.ZodOptional<z.ZodString>;
9266
- doi: z.ZodOptional<z.ZodString>;
9267
- accession: z.ZodOptional<z.ZodString>;
9268
- source: z.ZodString;
9269
- predicate: z.ZodOptional<z.ZodString>;
9270
- score: z.ZodOptional<z.ZodNumber>;
9271
- strength: z.ZodOptional<z.ZodString>;
9272
- is_human: z.ZodOptional<z.ZodBoolean>;
9273
- is_clinical: z.ZodOptional<z.ZodBoolean>;
9274
- excerpt: z.ZodOptional<z.ZodString>;
9275
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
9276
- regulatory_reference: z.ZodOptional<z.ZodObject<{
9277
- document: z.ZodString;
9278
- section: z.ZodOptional<z.ZodString>;
9279
- doc_id: z.ZodOptional<z.ZodString>;
9280
- doc_url: z.ZodOptional<z.ZodString>;
9281
- }, z.core.$strip>>;
9282
- }, z.core.$strip>>;
9283
- }, z.core.$strip>>;
9284
- }, z.core.$strip>;
9285
- dropped_count: z.ZodOptional<z.ZodNumber>;
9286
- inference_path: z.ZodOptional<z.ZodString>;
9287
- truncated: z.ZodOptional<z.ZodBoolean>;
9288
- }, z.core.$strip>, z.ZodObject<{
9289
- coverage: z.ZodLiteral<"queried_no_data">;
9290
- error: z.ZodOptional<z.ZodObject<{
9291
- message: z.ZodString;
9292
- source: z.ZodOptional<z.ZodString>;
9293
- }, z.core.$strip>>;
9294
- inference_path: z.ZodOptional<z.ZodString>;
9295
- }, z.core.$strip>, z.ZodObject<{
9296
- coverage: z.ZodLiteral<"filtered">;
9297
- filter: z.ZodString;
9298
- dropped_count: z.ZodNumber;
9299
- inference_path: z.ZodOptional<z.ZodString>;
9300
- }, z.core.$strip>, z.ZodObject<{
9301
- coverage: z.ZodLiteral<"not_loaded">;
9302
- reason: z.ZodOptional<z.ZodString>;
9303
- }, z.core.$strip>], "coverage">;
9304
9246
  tractability: z.ZodDiscriminatedUnion<[z.ZodObject<{
9305
9247
  coverage: z.ZodLiteral<"available">;
9306
9248
  data: z.ZodObject<{
@@ -9348,8 +9290,8 @@ export declare const DossierBodySchema: z.ZodObject<{
9348
9290
  displayName: z.ZodString;
9349
9291
  entityType: z.ZodEnum<{
9350
9292
  gene: "gene";
9351
- rna: "rna";
9352
9293
  protein: "protein";
9294
+ rna: "rna";
9353
9295
  complex: "complex";
9354
9296
  }>;
9355
9297
  ontology: z.ZodString;
@@ -9647,6 +9589,64 @@ export declare const DossierBodySchema: z.ZodObject<{
9647
9589
  coverage: z.ZodLiteral<"not_loaded">;
9648
9590
  reason: z.ZodOptional<z.ZodString>;
9649
9591
  }, z.core.$strip>], "coverage">;
9592
+ drug_interactions: z.ZodDiscriminatedUnion<[z.ZodObject<{
9593
+ coverage: z.ZodLiteral<"available">;
9594
+ data: z.ZodObject<{
9595
+ rows: z.ZodArray<z.ZodObject<{
9596
+ drug_id: z.ZodNullable<z.ZodString>;
9597
+ drug_name: z.ZodString;
9598
+ best_score: z.ZodNumber;
9599
+ predicates: z.ZodArray<z.ZodString>;
9600
+ sources: z.ZodArray<z.ZodString>;
9601
+ paper_count: z.ZodNumber;
9602
+ dominant_direction: z.ZodEnum<{
9603
+ unknown: "unknown";
9604
+ positive: "positive";
9605
+ negative: "negative";
9606
+ mixed: "mixed";
9607
+ }>;
9608
+ has_human_evidence: z.ZodBoolean;
9609
+ has_clinical_evidence: z.ZodBoolean;
9610
+ evidence: z.ZodArray<z.ZodObject<{
9611
+ pmid: z.ZodOptional<z.ZodString>;
9612
+ doi: z.ZodOptional<z.ZodString>;
9613
+ accession: z.ZodOptional<z.ZodString>;
9614
+ source: z.ZodString;
9615
+ predicate: z.ZodOptional<z.ZodString>;
9616
+ score: z.ZodOptional<z.ZodNumber>;
9617
+ strength: z.ZodOptional<z.ZodString>;
9618
+ is_human: z.ZodOptional<z.ZodBoolean>;
9619
+ is_clinical: z.ZodOptional<z.ZodBoolean>;
9620
+ excerpt: z.ZodOptional<z.ZodString>;
9621
+ metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
9622
+ regulatory_reference: z.ZodOptional<z.ZodObject<{
9623
+ document: z.ZodString;
9624
+ section: z.ZodOptional<z.ZodString>;
9625
+ doc_id: z.ZodOptional<z.ZodString>;
9626
+ doc_url: z.ZodOptional<z.ZodString>;
9627
+ }, z.core.$strip>>;
9628
+ }, z.core.$strip>>;
9629
+ }, z.core.$strip>>;
9630
+ }, z.core.$strip>;
9631
+ dropped_count: z.ZodOptional<z.ZodNumber>;
9632
+ inference_path: z.ZodOptional<z.ZodString>;
9633
+ truncated: z.ZodOptional<z.ZodBoolean>;
9634
+ }, z.core.$strip>, z.ZodObject<{
9635
+ coverage: z.ZodLiteral<"queried_no_data">;
9636
+ error: z.ZodOptional<z.ZodObject<{
9637
+ message: z.ZodString;
9638
+ source: z.ZodOptional<z.ZodString>;
9639
+ }, z.core.$strip>>;
9640
+ inference_path: z.ZodOptional<z.ZodString>;
9641
+ }, z.core.$strip>, z.ZodObject<{
9642
+ coverage: z.ZodLiteral<"filtered">;
9643
+ filter: z.ZodString;
9644
+ dropped_count: z.ZodNumber;
9645
+ inference_path: z.ZodOptional<z.ZodString>;
9646
+ }, z.core.$strip>, z.ZodObject<{
9647
+ coverage: z.ZodLiteral<"not_loaded">;
9648
+ reason: z.ZodOptional<z.ZodString>;
9649
+ }, z.core.$strip>], "coverage">;
9650
9650
  clinical_development: z.ZodObject<{
9651
9651
  trials: z.ZodDiscriminatedUnion<[z.ZodObject<{
9652
9652
  coverage: z.ZodLiteral<"available">;
@@ -33,6 +33,7 @@ import { BiologicalThemeSchema, KeyReferenceSchema, RunSynthesisSchema, Synthesi
33
33
  import { synthesisAgentPrompt } from "../prompts/synthesis-agent.js";
34
34
  import { composeSystemPrompt } from "../agents/system-prompt.js";
35
35
  import { runDir } from "../workspace/paths.js";
36
+ import { decodeObjectString } from "../lib/zod-issues.js";
36
37
  import { runToTerminal } from "../loop/run-to-terminal.js";
37
38
  import { passthroughStep } from "../loop/run-step.js";
38
39
  import { defineTool } from "../tools/define-tool.js";
@@ -262,17 +263,21 @@ function buildValidateTool(ctx) {
262
263
  "field-by-field synthesis schema is the arg schema of " +
263
264
  "submit_synthesis — this tool deliberately does not restate it. " +
264
265
  "Non-terminal: call as often as needed to iterate toward a clean " +
265
- "synthesis, then submit_synthesis.",
266
+ "synthesis, then submit_synthesis. A candidate that arrives as a " +
267
+ "JSON-encoded string is decoded before the checks run, so the " +
268
+ "result describes the synthesis inside the string.",
266
269
  // Deliberately permissive: a structurally-invalid candidate must reach
267
270
  // `execute` so the model gets a structured {valid:false, issues} result —
268
271
  // including semantic issues — instead of a bare Zod rejection at the loop's
269
272
  // input boundary. `execute` re-parses against RunSynthesisSchema itself.
273
+ // The permissive schema also means the loop's string repair never runs
274
+ // here, so `execute` decodes a JSON-encoded string itself.
270
275
  inputSchema: z.object({
271
276
  synthesis: z.unknown().describe("The candidate synthesis, in any shape. Field-by-field schema: see submit_synthesis."),
272
277
  }),
273
278
  describeCall: "none",
274
279
  execute: async (input) => {
275
- const result = fullyValidate(input.synthesis, ctx);
280
+ const result = fullyValidate(decodeObjectString(input.synthesis), ctx);
276
281
  return ok(result.valid ? { valid: true, issues: [] } : { valid: false, issues: result.issues });
277
282
  },
278
283
  });
@@ -82,3 +82,11 @@ export declare function hintForZodIssue(issue: z.core.$ZodIssue, input: unknown)
82
82
  * is not re-examined — one pass, no recursion — and no input causes a throw.
83
83
  */
84
84
  export declare function repairToolInput(input: unknown, error: z.ZodError): unknown | undefined;
85
+ /**
86
+ * Decode a value that a model sent as a JSON-encoded object string, with or
87
+ * without a wrapper artifact. The decoded object is returned when the string
88
+ * parses to one. Each other value comes back as it is, thus a caller can apply
89
+ * this to an argument of any type. This is the repair that `repairToolInput`
90
+ * applies at the loop boundary, for a tool whose schema accepts the string.
91
+ */
92
+ export declare function decodeObjectString(value: unknown): unknown;
@@ -131,6 +131,19 @@ export function repairToolInput(input, error) {
131
131
  return undefined;
132
132
  }
133
133
  }
134
+ /**
135
+ * Decode a value that a model sent as a JSON-encoded object string, with or
136
+ * without a wrapper artifact. The decoded object is returned when the string
137
+ * parses to one. Each other value comes back as it is, thus a caller can apply
138
+ * this to an argument of any type. This is the repair that `repairToolInput`
139
+ * applies at the loop boundary, for a tool whose schema accepts the string.
140
+ */
141
+ export function decodeObjectString(value) {
142
+ if (typeof value !== "string" || value.length > MAX_REPAIRABLE_STRING_LENGTH)
143
+ return value;
144
+ const parsed = parseAs(value, "object") ?? parseAs(stripWrappers(value).text, "object");
145
+ return parsed === undefined ? value : parsed.value;
146
+ }
134
147
  /**
135
148
  * The measurement of a string that broke its maximum length, or `undefined` for
136
149
  * any other issue.
@@ -1,4 +1,38 @@
1
+ /**
2
+ * Absence policy: Semantic Scholar gives each requested key, and it sends an
3
+ * explicit `null` for an absent value such as `abstract` or `year`. An absent
4
+ * `venue` is the empty string.
5
+ */
6
+ import { z } from "zod";
1
7
  import { type SourceHttpOptions, type SourceHttpResult } from "./http.js";
8
+ export declare const ExactPaperSchema: z.ZodObject<{
9
+ title: z.ZodOptional<z.ZodNullable<z.ZodString>>;
10
+ abstract: z.ZodOptional<z.ZodNullable<z.ZodString>>;
11
+ year: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
12
+ venue: z.ZodOptional<z.ZodNullable<z.ZodString>>;
13
+ citationCount: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
14
+ url: z.ZodOptional<z.ZodNullable<z.ZodString>>;
15
+ authors: z.ZodOptional<z.ZodArray<z.ZodObject<{
16
+ name: z.ZodOptional<z.ZodNullable<z.ZodString>>;
17
+ }, z.core.$strip>>>;
18
+ externalIds: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>>>;
19
+ paperId: z.ZodString;
20
+ }, z.core.$loose>;
21
+ export declare const SearchResponseSchema: z.ZodObject<{
22
+ data: z.ZodOptional<z.ZodArray<z.ZodObject<{
23
+ paperId: z.ZodOptional<z.ZodString>;
24
+ title: z.ZodOptional<z.ZodNullable<z.ZodString>>;
25
+ abstract: z.ZodOptional<z.ZodNullable<z.ZodString>>;
26
+ year: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
27
+ venue: z.ZodOptional<z.ZodNullable<z.ZodString>>;
28
+ citationCount: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
29
+ url: z.ZodOptional<z.ZodNullable<z.ZodString>>;
30
+ authors: z.ZodOptional<z.ZodArray<z.ZodObject<{
31
+ name: z.ZodOptional<z.ZodNullable<z.ZodString>>;
32
+ }, z.core.$strip>>>;
33
+ externalIds: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber]>>>>;
34
+ }, z.core.$loose>>>;
35
+ }, z.core.$strip>;
2
36
  export interface SemanticScholarPaper {
3
37
  id: string;
4
38
  title?: string;
@@ -1,3 +1,8 @@
1
+ /**
2
+ * Absence policy: Semantic Scholar gives each requested key, and it sends an
3
+ * explicit `null` for an absent value such as `abstract` or `year`. An absent
4
+ * `venue` is the empty string.
5
+ */
1
6
  import { z } from "zod";
2
7
  import { requestJson } from "./http.js";
3
8
  const BASE_URL = "https://api.semanticscholar.org/graph/v1";
@@ -12,15 +17,21 @@ const WirePaperSchema = z
12
17
  citationCount: z.number().nullable().optional(),
13
18
  url: z.string().nullable().optional(),
14
19
  authors: z.array(z.object({ name: z.string().nullable().optional() })).optional(),
15
- externalIds: z.record(z.string(), z.unknown()).nullable().optional(),
20
+ // An external id is a string, but `CorpusId` is a JSON integer on every
21
+ // observed paper. A string-only value type drops it at the map site below.
22
+ externalIds: z
23
+ .record(z.string(), z.union([z.string(), z.number()]))
24
+ .nullable()
25
+ .optional(),
16
26
  })
17
27
  .passthrough();
18
- const ExactPaperSchema = WirePaperSchema.extend({ paperId: z.string() });
19
- const SearchResponseSchema = z.object({ data: z.array(WirePaperSchema).optional() });
28
+ export const ExactPaperSchema = WirePaperSchema.extend({ paperId: z.string() });
29
+ export const SearchResponseSchema = z.object({ data: z.array(WirePaperSchema).optional() });
30
+ /** Give one text form to each external id, thus a numeric `CorpusId` reaches a caller that keys on a string. */
20
31
  function stringExternalIds(value) {
21
32
  if (value === null || value === undefined)
22
33
  return undefined;
23
- const entries = Object.entries(value).filter((entry) => typeof entry[1] === "string");
34
+ const entries = Object.entries(value).map(([key, raw]) => [key, typeof raw === "string" ? raw : String(raw)]);
24
35
  return entries.length === 0 ? undefined : Object.fromEntries(entries);
25
36
  }
26
37
  function mapPaper(raw) {
@@ -0,0 +1,37 @@
1
+ /**
2
+ * The two cpu files that a sandbox reads for its core count.
3
+ *
4
+ * The thread env (`thread-env.ts`) reaches each library that reads a thread
5
+ * count from the environment. Two base calls read a kernel file instead:
6
+ * `parallel::detectCores()` greps `/proc/cpuinfo`, and `os.cpu_count()` of
7
+ * Python 3.12 reads `/sys/devices/system/cpu/online` through glibc. Both files
8
+ * describe the host, not the cgroup. A read-only bind of a small file over each
9
+ * path makes both calls report the quota.
10
+ *
11
+ * This module makes the content of the two files, and it reads the
12
+ * `/proc/cpuinfo` of the harness host that the content is cut from. The Docker
13
+ * backend writes the content to the host and binds it. The K8s backend puts
14
+ * the content in a ConfigMap and mounts each key with `subPath`.
15
+ */
16
+ /** The two container paths that the cpu files cover. */
17
+ export declare const CPU_ONLINE_PATH = "/sys/devices/system/cpu/online";
18
+ export declare const CPUINFO_PATH = "/proc/cpuinfo";
19
+ export declare function readHostCpuinfoOnce(): Promise<string | undefined>;
20
+ export interface CpuFiles {
21
+ /** Content of `/sys/devices/system/cpu/online`: `0` for one thread, else the range `0-<n-1>`. */
22
+ readonly online: string;
23
+ /**
24
+ * Content of `/proc/cpuinfo`: the first n `processor` blocks of the real file
25
+ * of the host. `undefined` when the host gives no `/proc/cpuinfo`.
26
+ */
27
+ readonly cpuinfo: string | undefined;
28
+ }
29
+ /**
30
+ * The content of the two cpu files for `threads` cpus.
31
+ *
32
+ * `threads` is a positive integer. The caller applies the same floor as
33
+ * `threadLimitEnv`: `Math.max(1, Math.floor(spec.cpu))`. When the host has fewer
34
+ * processor blocks than `threads`, all blocks stay. Each block keeps its own
35
+ * text, thus the `processor` numbers stay `0` to `n-1` as in the real file.
36
+ */
37
+ export declare function cpuFiles(threads: number, hostCpuinfo: string | undefined): CpuFiles;
@@ -0,0 +1,55 @@
1
+ /**
2
+ * The two cpu files that a sandbox reads for its core count.
3
+ *
4
+ * The thread env (`thread-env.ts`) reaches each library that reads a thread
5
+ * count from the environment. Two base calls read a kernel file instead:
6
+ * `parallel::detectCores()` greps `/proc/cpuinfo`, and `os.cpu_count()` of
7
+ * Python 3.12 reads `/sys/devices/system/cpu/online` through glibc. Both files
8
+ * describe the host, not the cgroup. A read-only bind of a small file over each
9
+ * path makes both calls report the quota.
10
+ *
11
+ * This module makes the content of the two files, and it reads the
12
+ * `/proc/cpuinfo` of the harness host that the content is cut from. The Docker
13
+ * backend writes the content to the host and binds it. The K8s backend puts
14
+ * the content in a ConfigMap and mounts each key with `subPath`.
15
+ */
16
+ import { readFile } from "node:fs/promises";
17
+ /** The two container paths that the cpu files cover. */
18
+ export const CPU_ONLINE_PATH = "/sys/devices/system/cpu/online";
19
+ export const CPUINFO_PATH = "/proc/cpuinfo";
20
+ /**
21
+ * The `/proc/cpuinfo` of the harness host, read once per process. The content
22
+ * does not change while the process runs. A host with no such file, for
23
+ * example macOS, gives `undefined`, and the caller covers only `online`.
24
+ */
25
+ let hostCpuinfo;
26
+ export function readHostCpuinfoOnce() {
27
+ hostCpuinfo ??= readFile(CPUINFO_PATH, "utf8").catch(() => undefined);
28
+ return hostCpuinfo;
29
+ }
30
+ /**
31
+ * Split the text of `/proc/cpuinfo` into its `processor` blocks. A blank line
32
+ * ends a block. A block with no `processor` line, for example the `Hardware`
33
+ * trailer of an ARM kernel, is not a processor and is dropped.
34
+ */
35
+ function processorBlocks(cpuinfo) {
36
+ return cpuinfo
37
+ .split(/\n\s*\n/)
38
+ .map((block) => block.trim())
39
+ .filter((block) => /^processor\b/m.test(block));
40
+ }
41
+ /**
42
+ * The content of the two cpu files for `threads` cpus.
43
+ *
44
+ * `threads` is a positive integer. The caller applies the same floor as
45
+ * `threadLimitEnv`: `Math.max(1, Math.floor(spec.cpu))`. When the host has fewer
46
+ * processor blocks than `threads`, all blocks stay. Each block keeps its own
47
+ * text, thus the `processor` numbers stay `0` to `n-1` as in the real file.
48
+ */
49
+ export function cpuFiles(threads, hostCpuinfo) {
50
+ const online = `${threads === 1 ? "0" : `0-${threads - 1}`}\n`;
51
+ if (hostCpuinfo === undefined)
52
+ return { online, cpuinfo: undefined };
53
+ const blocks = processorBlocks(hostCpuinfo).slice(0, threads);
54
+ return { online, cpuinfo: blocks.map((block) => `${block}\n\n`).join("") };
55
+ }
@@ -7,6 +7,15 @@
7
7
  * `/mnt/libs` / `/mnt/refs` when their host paths are configured. Container
8
8
  * paths and lib-store env come from the shared mount plan (`mount-plan.ts`).
9
9
  *
10
+ * ## The cpu files
11
+ *
12
+ * Two more binds put a small file over `/sys/devices/system/cpu/online` and
13
+ * over `/proc/cpuinfo` (`cpu-files.ts`). Each file describes `spec.cpu` cpus,
14
+ * not the host. The host side of each bind is a file under
15
+ * `<parent of the workspace root>/.cpu/<sandboxId>/`, thus outside the tree
16
+ * that the sandbox sees. `teardown` removes the directory when this process
17
+ * wrote it. A directory from an earlier process stays.
18
+ *
10
19
  * ## Transport and confinement
11
20
  *
12
21
  * The container joins the default bridge and publishes sandbox-server's port to
@@ -69,6 +78,12 @@ export interface DockerClientConfig {
69
78
  docker?: Docker;
70
79
  /** Injected for tests so `/health` polling can be stubbed. */
71
80
  fetch?: typeof fetch;
81
+ /**
82
+ * Injected for tests so the `/proc/cpuinfo` of the host can be stubbed. The
83
+ * default reads the file once per process. `undefined` means that the host
84
+ * has no such file, for example macOS.
85
+ */
86
+ readHostCpuinfo?: () => Promise<string | undefined>;
72
87
  /**
73
88
  * Optional logger so a lib-store degradation (a configured store whose `current`
74
89
  * vanished or went incomplete by sandbox-create time) is observable instead of a
@@ -7,6 +7,15 @@
7
7
  * `/mnt/libs` / `/mnt/refs` when their host paths are configured. Container
8
8
  * paths and lib-store env come from the shared mount plan (`mount-plan.ts`).
9
9
  *
10
+ * ## The cpu files
11
+ *
12
+ * Two more binds put a small file over `/sys/devices/system/cpu/online` and
13
+ * over `/proc/cpuinfo` (`cpu-files.ts`). Each file describes `spec.cpu` cpus,
14
+ * not the host. The host side of each bind is a file under
15
+ * `<parent of the workspace root>/.cpu/<sandboxId>/`, thus outside the tree
16
+ * that the sandbox sees. `teardown` removes the directory when this process
17
+ * wrote it. A directory from an earlier process stays.
18
+ *
10
19
  * ## Transport and confinement
11
20
  *
12
21
  * The container joins the default bridge and publishes sandbox-server's port to
@@ -29,13 +38,16 @@
29
38
  * effective capabilities.
30
39
  */
31
40
  import { existsSync, lstatSync, statSync } from "node:fs";
32
- import { join } from "node:path";
41
+ import { mkdir, rm, writeFile } from "node:fs/promises";
42
+ import { dirname, join } from "node:path";
33
43
  import Docker from "dockerode";
34
44
  import { ResultAsync, err, ok } from "neverthrow";
35
45
  import { createNoopLogger } from "../lib/console-logger.js";
36
46
  import { tailWritePrefix } from "../workspace/paths.js";
37
47
  import { trySandbox } from "./sandbox-error.js";
38
48
  import { buildMountPlan, sandboxWriteTail } from "./mount-plan.js";
49
+ import { CPU_ONLINE_PATH, CPUINFO_PATH, cpuFiles, readHostCpuinfoOnce } from "./cpu-files.js";
50
+ import { threadLimitEnv } from "./thread-env.js";
39
51
  /** Read the originating HTTP status off any `SandboxError` variant that carries one. */
40
52
  function statusOf(e) {
41
53
  return "status" in e ? e.status : undefined;
@@ -60,6 +72,8 @@ const MANAGED_BY_VALUE = "cortex";
60
72
  const OWNER_WORKFLOW_LABEL = "cortex/owner-workflow-id";
61
73
  const RUN_ID_LABEL = "cortex/run-id";
62
74
  const STEP_ID_LABEL = "cortex/step-id";
75
+ /** The directory next to the workspace root that holds the cpu files of each sandbox. */
76
+ const CPU_FILES_DIR = ".cpu";
63
77
  /**
64
78
  * Whether the lib store's `current` resolves to a COMPLETE, usable version, not merely a
65
79
  * present symlink. By sandbox-create time `current` may be gone (concurrent prune/`rm`), a
@@ -104,6 +118,39 @@ function refStoreUsable(refStorePath) {
104
118
  return false;
105
119
  }
106
120
  }
121
+ /**
122
+ * Write the cpu files of one sandbox into `dir` and return their binds.
123
+ *
124
+ * The directory `.cpu` is made with no `recursive` option on purpose. The
125
+ * parent of the workspace root must exist already. Thus a root that is not on
126
+ * disk gives no directory at an unexpected place, and the sandbox starts with
127
+ * no cpu files. The failure is a degradation, not an error: the thread env
128
+ * still limits each library, but `detectCores()` and `os.cpu_count()` report
129
+ * the host.
130
+ */
131
+ async function writeCpuFiles(dir, files, logger, sandboxId) {
132
+ const ignoreExists = (cause) => {
133
+ if (cause.code !== "EEXIST")
134
+ throw cause;
135
+ };
136
+ try {
137
+ await mkdir(dirname(dir)).catch(ignoreExists);
138
+ await mkdir(dir).catch(ignoreExists);
139
+ const online = join(dir, "online");
140
+ await writeFile(online, files.online);
141
+ const binds = [`${online}:${CPU_ONLINE_PATH}:ro`];
142
+ if (files.cpuinfo !== undefined) {
143
+ const cpuinfo = join(dir, "cpuinfo");
144
+ await writeFile(cpuinfo, files.cpuinfo);
145
+ binds.push(`${cpuinfo}:${CPUINFO_PATH}:ro`);
146
+ }
147
+ return binds;
148
+ }
149
+ catch (cause) {
150
+ logger.warn("cpu files not written — the sandbox reports the cores of the host", { sandboxId, dir, cause });
151
+ return [];
152
+ }
153
+ }
107
154
  async function pollHealth(fetchImpl, url, timeoutMs) {
108
155
  const deadline = Date.now() + timeoutMs;
109
156
  let lastErr;
@@ -199,6 +246,9 @@ export function createDockerSandboxOps(config) {
199
246
  const fetchImpl = config.fetch ?? fetch;
200
247
  const transport = config.transport ?? "poll";
201
248
  const pollMode = transport === "poll";
249
+ const readHostCpuinfo = config.readHostCpuinfo ?? readHostCpuinfoOnce;
250
+ /** The cpu directory of each sandbox that this process wrote, for `teardown`. */
251
+ const cpuDirs = new Map();
202
252
  return {
203
253
  createSandbox(meta, identity) {
204
254
  return new ResultAsync((async () => {
@@ -228,6 +278,18 @@ export function createDockerSandboxOps(config) {
228
278
  // a crafted tail cannot escape the resolved root into the container. The
229
279
  // tail is the step directory, or the one the caller declared.
230
280
  const write = sandboxWriteTail(meta);
281
+ // The same floor as `threadLimitEnv`: a fractional cpu request is one cpu.
282
+ const threads = Math.max(1, Math.floor(meta.resources.cpu));
283
+ const cpuinfo = await readHostCpuinfo();
284
+ if (cpuinfo === undefined) {
285
+ logger.debug("no host /proc/cpuinfo — the sandbox gets the online file only", { sandboxId });
286
+ }
287
+ // Next to the workspace root, not under it: the root is bound read-only
288
+ // at `/{analysisId}`, and a file under it is visible in the sandbox.
289
+ const cpuDir = join(dirname(hostTreePath), CPU_FILES_DIR, sandboxId);
290
+ const cpuBinds = await writeCpuFiles(cpuDir, cpuFiles(threads, cpuinfo), logger, sandboxId);
291
+ if (cpuBinds.length > 0)
292
+ cpuDirs.set(sandboxId, cpuDir);
231
293
  const binds = [
232
294
  `${hostTreePath}:${plan.readonlyTreePath}:ro`,
233
295
  ...(write && plan.writableStepPath
@@ -235,6 +297,7 @@ export function createDockerSandboxOps(config) {
235
297
  : []),
236
298
  ...(libsMounted && config.libStorePath ? [`${config.libStorePath}:${plan.libsPath}:ro`] : []),
237
299
  ...(refsMounted && config.refStorePath ? [`${config.refStorePath}:${plan.refsPath}:ro`] : []),
300
+ ...cpuBinds,
238
301
  ];
239
302
  const createFailed = (status, cause) => ({
240
303
  type: "container_create_failed",
@@ -244,17 +307,22 @@ export function createDockerSandboxOps(config) {
244
307
  cause,
245
308
  });
246
309
  const image = meta.image ?? config.image;
310
+ const spec = meta.resources;
247
311
  // Poll mode never dials out and carries no CORTEX_BASE_URL; it sets the
248
312
  // firewall flag so the root entrypoint installs the egress block before
249
313
  // dropping to uid 1000. Callback mode is the inverse.
250
- const env = [
251
- `SANDBOX_TRANSPORT=${transport}`,
252
- `SANDBOX_CALLBACK_SECRET=${callbackSecret}`,
253
- ...(pollMode ? ["SANDBOX_EGRESS_FIREWALL=1"] : [`CORTEX_BASE_URL=${config.cortexBaseUrl}`]),
254
- ...Object.entries(plan.env).map(([k, v]) => `${k}=${v}`),
255
- ...Object.entries(meta.extraEnv ?? {}).map(([k, v]) => `${k}=${v}`),
256
- ];
257
- const spec = meta.resources;
314
+ //
315
+ // Composed as one record, not as a list of `K=V`. A duplicate key in
316
+ // the Env array resolves at the libc that scans `environ`, thus the
317
+ // later spread must win here, not there.
318
+ const env = Object.entries({
319
+ SANDBOX_TRANSPORT: transport,
320
+ SANDBOX_CALLBACK_SECRET: callbackSecret,
321
+ ...(pollMode ? { SANDBOX_EGRESS_FIREWALL: "1" } : { CORTEX_BASE_URL: config.cortexBaseUrl }),
322
+ ...threadLimitEnv(spec),
323
+ ...plan.env,
324
+ ...(meta.extraEnv ?? {}),
325
+ }).map(([k, v]) => `${k}=${v}`);
258
326
  const createOpts = {
259
327
  name: sandboxId,
260
328
  ...(config.platform !== undefined && { platform: config.platform }),
@@ -289,6 +357,11 @@ export function createDockerSandboxOps(config) {
289
357
  SecurityOpt: ["no-new-privileges"],
290
358
  NanoCpus: Math.round(spec.cpu * 1e9),
291
359
  Memory: spec.memoryGb * 1024 ** 3,
360
+ // `Memory` alone lets the container swap as much again. A fork
361
+ // storm then thrashes for minutes, and sandbox-server stops to
362
+ // answer. With no swap the OOM killer removes the largest fork in
363
+ // seconds, and sandbox-server survives. K8s runs with no swap.
364
+ MemorySwap: spec.memoryGb * 1024 ** 3,
292
365
  AutoRemove: false,
293
366
  },
294
367
  };
@@ -329,10 +402,10 @@ export function createDockerSandboxOps(config) {
329
402
  })());
330
403
  },
331
404
  teardown(ref) {
332
- return removeContainerIgnoreMissing(docker, ref.sandboxId);
405
+ return teardownSandbox(ref.sandboxId);
333
406
  },
334
407
  teardownById(sandboxId) {
335
- return removeContainerIgnoreMissing(docker, sandboxId);
408
+ return teardownSandbox(sandboxId);
336
409
  },
337
410
  listManagedSandboxes() {
338
411
  return trySandbox(() => docker.listContainers({
@@ -361,6 +434,22 @@ export function createDockerSandboxOps(config) {
361
434
  },
362
435
  isAliveById,
363
436
  };
437
+ /**
438
+ * Remove the container, then the cpu files that this process wrote for it.
439
+ * A failure of the file removal does not fail the teardown: the container is
440
+ * gone, and a stale small directory is not a risk to a later sandbox.
441
+ */
442
+ function teardownSandbox(sandboxId) {
443
+ return removeContainerIgnoreMissing(docker, sandboxId).map(async () => {
444
+ const cpuDir = cpuDirs.get(sandboxId);
445
+ if (cpuDir === undefined)
446
+ return;
447
+ cpuDirs.delete(sandboxId);
448
+ await rm(cpuDir, { recursive: true, force: true }).catch((cause) => {
449
+ logger.debug("cpu files not removed", { sandboxId, cpuDir, cause });
450
+ });
451
+ });
452
+ }
364
453
  function isAliveById(sandboxId) {
365
454
  return trySandbox(async () => {
366
455
  const info = await docker.getContainer(sandboxId).inspect();
@@ -60,6 +60,11 @@ export interface K8sClientConfig {
60
60
  tolerations?: V1Toleration[];
61
61
  /** RuntimeClass for runtime isolation (e.g. `gvisor`). Omit for the default runtime. */
62
62
  runtimeClassName?: string;
63
+ /**
64
+ * The `/proc/cpuinfo` text that the cpu ConfigMap of each Job is cut from.
65
+ * Defaults to the file of the host of this process, read once. Injected for tests.
66
+ */
67
+ readHostCpuinfo?: () => Promise<string | undefined>;
63
68
  /** Injected for tests. */
64
69
  batchApi?: BatchV1Api;
65
70
  coreApi?: CoreV1Api;