@inflexa-ai/harness 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agents/conversation-agent.d.ts +3 -8
- package/dist/agents/conversation-agent.js +24 -4
- package/dist/agents/report-session-agent.d.ts +11 -14
- package/dist/agents/report-session-agent.js +30 -17
- package/dist/agents/sandbox/shared.d.ts +18 -13
- package/dist/agents/sandbox/shared.js +42 -27
- package/dist/agents/sandbox/types.d.ts +1 -1
- package/dist/agents/sandbox-catalog.d.ts +2 -0
- package/dist/agents/sandbox-catalog.js +30 -1
- package/dist/app/chat-turn-workflow.js +5 -6
- package/dist/app/synthesize-run.d.ts +10 -6
- package/dist/app/synthesize-run.js +4 -2
- package/dist/contracts/analogy-report.d.ts +46 -2
- package/dist/contracts/analogy-report.js +31 -25
- package/dist/contracts/chat-parts.d.ts +3 -14
- package/dist/contracts/report-reference.d.ts +2 -1
- package/dist/contracts/report-reference.js +2 -1
- package/dist/contracts/schemas/chat-parts.d.ts +0 -27
- package/dist/contracts/schemas/chat-parts.js +1 -3
- package/dist/execution/run-canceler.d.ts +2 -0
- package/dist/execution/run-canceler.js +9 -0
- package/dist/execution/run-synthesis.d.ts +36 -8
- package/dist/execution/run-synthesis.js +114 -32
- package/dist/index.d.ts +2 -5
- package/dist/index.js +3 -6
- package/dist/input-scan/enrich.d.ts +5 -10
- package/dist/input-scan/enrich.js +12 -8
- package/dist/lib/hooks.d.ts +9 -0
- package/dist/lib/hooks.js +6 -1
- package/dist/lib/markdown-pdf.d.ts +27 -0
- package/dist/lib/markdown-pdf.js +236 -0
- package/dist/lib/metrics.d.ts +31 -0
- package/dist/lib/metrics.js +64 -0
- package/dist/lib/pdf-text.d.ts +23 -0
- package/dist/lib/pdf-text.js +34 -0
- package/dist/lib/regulatory-corpus/fetch-fda-guidance.js +3 -5
- package/dist/lib/regulatory-corpus/fetch-ich.js +3 -5
- package/dist/loop/run-loop.d.ts +5 -0
- package/dist/loop/run-loop.js +30 -2
- package/dist/loop/sub-agent.js +5 -6
- package/dist/loop/tool-name-repair.d.ts +11 -0
- package/dist/loop/tool-name-repair.js +23 -0
- package/dist/loop/turn-credentials.d.ts +5 -0
- package/dist/loop/turn-credentials.js +17 -0
- package/dist/loop/types.d.ts +14 -0
- package/dist/memory/card-builders.d.ts +1 -1
- package/dist/prompts/SOUL.d.ts +6 -6
- package/dist/prompts/SOUL.js +22 -121
- package/dist/prompts/analogical-reasoner.d.ts +1 -1
- package/dist/prompts/analogical-reasoner.js +35 -101
- package/dist/prompts/briefing.d.ts +20 -11
- package/dist/prompts/briefing.js +36 -19
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +103 -383
- package/dist/prompts/planner.d.ts +1 -5
- package/dist/prompts/planner.js +45 -128
- package/dist/prompts/report-session.d.ts +1 -1
- package/dist/prompts/report-session.js +26 -57
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.js +3 -3
- package/dist/prompts/sandbox/cheminformatics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/cheminformatics-agent.js +12 -47
- package/dist/prompts/sandbox/chromatin-agent.d.ts +1 -1
- package/dist/prompts/sandbox/chromatin-agent.js +4 -5
- package/dist/prompts/sandbox/data-profiler.js +11 -34
- package/dist/prompts/sandbox/dna-methylation-agent.d.ts +1 -1
- package/dist/prompts/sandbox/dna-methylation-agent.js +3 -1
- package/dist/prompts/sandbox/drug-repurposing-agent.d.ts +1 -1
- package/dist/prompts/sandbox/drug-repurposing-agent.js +23 -73
- package/dist/prompts/sandbox/enrichment-agent.d.ts +1 -1
- package/dist/prompts/sandbox/enrichment-agent.js +3 -1
- package/dist/prompts/sandbox/genomic-variant-agent.d.ts +1 -1
- package/dist/prompts/sandbox/genomic-variant-agent.js +3 -4
- package/dist/prompts/sandbox/immune-profiling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/immune-profiling-agent.js +5 -4
- package/dist/prompts/sandbox/metabolomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/metabolomics-agent.js +3 -1
- package/dist/prompts/sandbox/microbiome-agent.d.ts +1 -1
- package/dist/prompts/sandbox/microbiome-agent.js +3 -1
- package/dist/prompts/sandbox/multi-omics-integration-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multi-omics-integration-agent.js +3 -1
- package/dist/prompts/sandbox/multimodal-sc-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multimodal-sc-agent.js +3 -1
- package/dist/prompts/sandbox/network-agent.d.ts +1 -1
- package/dist/prompts/sandbox/network-agent.js +3 -1
- package/dist/prompts/sandbox/pkpd-clinical-response-agent.d.ts +1 -1
- package/dist/prompts/sandbox/pkpd-clinical-response-agent.js +4 -7
- package/dist/prompts/sandbox/proteomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/proteomics-agent.js +3 -1
- package/dist/prompts/sandbox/scientific-executor.d.ts +1 -1
- package/dist/prompts/sandbox/scientific-executor.js +0 -12
- package/dist/prompts/sandbox/single-cell-agent.d.ts +1 -1
- package/dist/prompts/sandbox/single-cell-agent.js +3 -1
- package/dist/prompts/sandbox/spatial-omics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/spatial-omics-agent.js +3 -1
- package/dist/prompts/sandbox/statistical-modeling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/statistical-modeling-agent.js +4 -4
- package/dist/prompts/sandbox/translational-safety-agent.d.ts +1 -1
- package/dist/prompts/sandbox/translational-safety-agent.js +10 -29
- package/dist/prompts/sandbox-standards.d.ts +15 -18
- package/dist/prompts/sandbox-standards.js +96 -305
- package/dist/prompts/synthesis-agent.d.ts +1 -1
- package/dist/prompts/synthesis-agent.js +24 -90
- package/dist/provenance/collector.d.ts +2 -5
- package/dist/provenance/collector.js +2 -5
- package/dist/provenance/seam.d.ts +36 -1
- package/dist/provenance/workspace-reads.d.ts +36 -0
- package/dist/provenance/workspace-reads.js +33 -0
- package/dist/providers/request-headers.js +12 -3
- package/dist/report-model/production-resolver.d.ts +22 -52
- package/dist/report-model/production-resolver.js +43 -112
- package/dist/report-render/chart-export.d.ts +9 -0
- package/dist/report-render/chart-export.js +1 -1
- package/dist/report-render/views/chart-view.d.ts +1 -0
- package/dist/report-render/views/chart-view.js +1 -0
- package/dist/report-render/views/lineage.d.ts +1 -0
- package/dist/report-render/views/page-view.d.ts +1 -0
- package/dist/report-render/views/page-view.js +1 -0
- package/dist/report-render/views/prose.d.ts +1 -0
- package/dist/report-render/views/prose.js +1 -0
- package/dist/report-render/views/references-view.d.ts +1 -0
- package/dist/report-render/views/values.d.ts +1 -0
- package/dist/report-render/views/values.js +1 -0
- package/dist/runtime/assemble.d.ts +20 -9
- package/dist/runtime/assemble.js +47 -38
- package/dist/sandbox/client.d.ts +5 -3
- package/dist/sandbox/create-sandbox.d.ts +7 -12
- package/dist/sandbox/create-sandbox.js +15 -37
- package/dist/sandbox/docker-client.d.ts +1 -3
- package/dist/sandbox/docker-client.js +3 -5
- package/dist/sandbox/exec-outcome.d.ts +11 -11
- package/dist/sandbox/exec-outcome.js +22 -22
- package/dist/sandbox/exec.d.ts +5 -0
- package/dist/sandbox/exec.js +9 -1
- package/dist/sandbox/identity.d.ts +4 -16
- package/dist/sandbox/identity.js +5 -21
- package/dist/sandbox/k8s-client.d.ts +0 -1
- package/dist/sandbox/k8s-client.js +8 -13
- package/dist/sandbox/lease-names.d.ts +6 -0
- package/dist/sandbox/lease-names.js +12 -0
- package/dist/sandbox/leases.d.ts +105 -0
- package/dist/sandbox/leases.js +519 -0
- package/dist/sandbox/liveness.d.ts +5 -2
- package/dist/sandbox/mount-plan.d.ts +10 -1
- package/dist/sandbox/mount-plan.js +7 -1
- package/dist/sandbox/reconciler.d.ts +58 -0
- package/dist/sandbox/reconciler.js +153 -0
- package/dist/sandbox/sandbox-step-translate.d.ts +1 -1
- package/dist/sandbox/types.d.ts +25 -18
- package/dist/sandbox/types.js +10 -4
- package/dist/schemas/plan-schemas.d.ts +0 -21
- package/dist/schemas/plan-schemas.js +2 -3
- package/dist/schemas/run-synthesis.d.ts +2 -2
- package/dist/schemas/run-synthesis.js +3 -3
- package/dist/schemas/validate-plan.d.ts +1 -14
- package/dist/schemas/validate-plan.js +5 -24
- package/dist/schemas/workflow-state.d.ts +3 -32
- package/dist/schemas/workflow-state.js +4 -13
- package/dist/state/index.d.ts +0 -1
- package/dist/state/index.js +0 -1
- package/dist/state/migrations/20261030120000_lease_the_sandboxes.d.ts +7 -0
- package/dist/state/migrations/20261030120000_lease_the_sandboxes.js +34 -0
- package/dist/state/migrations/20261030120100_drop_the_report_derivations.d.ts +5 -0
- package/dist/state/migrations/20261030120100_drop_the_report_derivations.js +9 -0
- package/dist/state/migrations/20261030120200_keep_the_report_derivations.d.ts +6 -0
- package/dist/state/migrations/20261030120200_keep_the_report_derivations.js +10 -0
- package/dist/state/migrations/index.js +6 -0
- package/dist/state/purge-analysis.d.ts +4 -2
- package/dist/state/purge-analysis.js +22 -11
- package/dist/state/sandbox-leases.d.ts +81 -0
- package/dist/state/sandbox-leases.js +247 -0
- package/dist/state/schema.d.ts +1 -13
- package/dist/state/schema.js +1 -6
- package/dist/state/step-executions.d.ts +0 -2
- package/dist/state/step-executions.js +1 -4
- package/dist/tasks/data-profile.d.ts +40 -2
- package/dist/tasks/data-profile.js +89 -87
- package/dist/testing.d.ts +2 -0
- package/dist/testing.js +6 -1
- package/dist/tools/ad-hoc-router.d.ts +1 -6
- package/dist/tools/ad-hoc-router.js +5 -48
- package/dist/tools/bio/comptox.js +1 -1
- package/dist/tools/bio/drug-gene-interactions.js +1 -1
- package/dist/tools/bio/gene-disease-evidence.js +1 -1
- package/dist/tools/bio/keys.d.ts +1 -1
- package/dist/tools/bio/pubmed.js +31 -0
- package/dist/tools/bio/search-gene.js +17 -0
- package/dist/tools/bio/search-geo-datasets.js +13 -0
- package/dist/tools/bio/search-regulatory-approvals.js +1 -1
- package/dist/tools/define-tool.d.ts +25 -2
- package/dist/tools/define-tool.js +11 -1
- package/dist/tools/display/show-user.js +1 -1
- package/dist/tools/execute-analysis.d.ts +1 -3
- package/dist/tools/execute-analysis.js +5 -33
- package/dist/tools/report-session/index.d.ts +0 -1
- package/dist/tools/report-session/index.js +0 -1
- package/dist/tools/report-session/script-tools.d.ts +30 -0
- package/dist/tools/report-session/script-tools.js +177 -0
- package/dist/tools/research/generate-analogy-report.d.ts +16 -9
- package/dist/tools/research/generate-analogy-report.js +111 -49
- package/dist/tools/research/generate-plan.d.ts +102 -7
- package/dist/tools/research/generate-plan.js +74 -43
- package/dist/tools/research/index.d.ts +4 -3
- package/dist/tools/research/index.js +4 -3
- package/dist/tools/research/inspect-run.d.ts +0 -1
- package/dist/tools/research/inspect-run.js +5 -10
- package/dist/tools/research/search-arxiv.js +15 -0
- package/dist/tools/research/search-github-repos.js +13 -0
- package/dist/tools/research/search-semantic-scholar.js +16 -0
- package/dist/tools/research/synthesis-lookups.d.ts +21 -0
- package/dist/tools/research/synthesis-lookups.js +29 -0
- package/dist/tools/research/web-fetch.d.ts +1 -1
- package/dist/tools/research/web-fetch.js +14 -12
- package/dist/tools/sandbox/lease-tools.d.ts +32 -0
- package/dist/tools/sandbox/lease-tools.js +60 -0
- package/dist/tools/script/codemode-worker.d.ts +5 -0
- package/dist/tools/script/codemode-worker.js +5 -0
- package/dist/tools/script/file-tools.d.ts +100 -0
- package/dist/tools/script/file-tools.js +377 -0
- package/dist/tools/script/metrics.d.ts +21 -0
- package/dist/tools/script/metrics.js +38 -0
- package/dist/tools/script/quickjs.wasm +0 -0
- package/dist/tools/script/run-script.d.ts +76 -0
- package/dist/tools/script/run-script.js +326 -0
- package/dist/tools/script/scratch.d.ts +81 -0
- package/dist/tools/script/scratch.js +142 -0
- package/dist/tools/start-report-session.js +1 -1
- package/dist/tools/workspace/edit-file.js +2 -1
- package/dist/tools/workspace/execute-command.d.ts +9 -13
- package/dist/tools/workspace/execute-command.js +23 -19
- package/dist/tools/workspace/file-stat.js +1 -1
- package/dist/tools/workspace/grep.d.ts +5 -2
- package/dist/tools/workspace/grep.js +19 -10
- package/dist/tools/workspace/mutator.d.ts +48 -4
- package/dist/tools/workspace/mutator.js +43 -12
- package/dist/tools/workspace/read-file.d.ts +5 -2
- package/dist/tools/workspace/read-file.js +15 -11
- package/dist/tools/workspace/read-surface.d.ts +17 -0
- package/dist/tools/workspace/read-surface.js +17 -0
- package/dist/tools/workspace/result-bounds.d.ts +1 -0
- package/dist/tools/workspace/run-exec.d.ts +12 -13
- package/dist/tools/workspace/run-exec.js +13 -13
- package/dist/tools/workspace/scan-inputs.d.ts +2 -5
- package/dist/tools/workspace/scan-inputs.js +2 -3
- package/dist/tools/workspace/show-file.js +1 -1
- package/dist/tools/workspace/show-plan.js +4 -3
- package/dist/tools/workspace/write-file.js +2 -1
- package/dist/workflows/execute-analysis-scheduler.d.ts +0 -26
- package/dist/workflows/execute-analysis-scheduler.js +12 -32
- package/dist/workflows/execute-analysis.d.ts +17 -35
- package/dist/workflows/execute-analysis.js +13 -107
- package/dist/workflows/pause.js +29 -22
- package/dist/workflows/sandbox-step.d.ts +16 -31
- package/dist/workflows/sandbox-step.js +36 -71
- package/dist/workflows/test/sql-writes.d.ts +0 -1
- package/dist/workflows/test/sql-writes.js +0 -2
- package/dist/workspace/filesystem.d.ts +2 -2
- package/dist/workspace/filesystem.js +10 -5
- package/dist/workspace/paths.d.ts +19 -1
- package/dist/workspace/paths.js +22 -1
- package/package.json +6 -3
- package/dist/prompts/literature-reviewer.d.ts +0 -1
- package/dist/prompts/literature-reviewer.js +0 -114
- package/dist/sandbox/reaper.d.ts +0 -81
- package/dist/sandbox/reaper.js +0 -161
- package/dist/sandbox/step-sandbox.d.ts +0 -47
- package/dist/sandbox/step-sandbox.js +0 -53
- package/dist/state/active-sandboxes.d.ts +0 -33
- package/dist/state/active-sandboxes.js +0 -65
- package/dist/tasks/extract-values-script.d.ts +0 -64
- package/dist/tasks/extract-values-script.js +0 -180
- package/dist/tasks/extract-values.d.ts +0 -77
- package/dist/tasks/extract-values.js +0 -146
- package/dist/tools/report-session/derive-table.d.ts +0 -164
- package/dist/tools/report-session/derive-table.js +0 -419
- package/dist/tools/research/literature-reviewer.d.ts +0 -42
- package/dist/tools/research/literature-reviewer.js +0 -107
- package/dist/workflows/metrics.d.ts +0 -23
- package/dist/workflows/metrics.js +0 -38
|
@@ -84,18 +84,6 @@ export const AnalysisStepSchema = z.object({
|
|
|
84
84
|
.array(z.string())
|
|
85
85
|
.describe("IDs of steps in this plan that must complete before this one runs. Empty when the step has no prerequisites. The graph must be acyclic."),
|
|
86
86
|
status: z.enum(["pending", "running", "completed", "failed", "skipped"]).default("pending"),
|
|
87
|
-
// Optional on the persistence schema so historical plans (pre-resources)
|
|
88
|
-
// still parse on the read side. `PlanStepSchema` re-requires it for planner
|
|
89
|
-
// output, and the execution path rejects any step that reaches it without
|
|
90
|
-
// resources (no silent default) and clamps to cluster limits before use.
|
|
91
|
-
resources: z
|
|
92
|
-
.object({
|
|
93
|
-
cpu: z.number().describe("CPU cores the step needs."),
|
|
94
|
-
memoryGb: z.number().describe("Memory in GB the step needs."),
|
|
95
|
-
gpu: z.object({ count: z.number() }).optional().describe("GPUs the step needs. Omit unless the step genuinely requires one."),
|
|
96
|
-
})
|
|
97
|
-
.optional()
|
|
98
|
-
.describe("Resources the step needs. Ground the estimate in the actual data size — see the resource-estimation rules."),
|
|
99
87
|
grounding: GroundingSchema.optional().describe("The knowledge grounding of a method step: fill it from the answer of `knowledge_recommend` when that tool is attached. Omit it when no knowledge tool is attached."),
|
|
100
88
|
// Execution fields
|
|
101
89
|
agent: z.string().optional().describe("Assigned sandbox agent name from registry"),
|
|
@@ -108,7 +96,6 @@ export const AnalysisStepSchema = z.object({
|
|
|
108
96
|
.describe("The one capability phrase of the assigned agent that this step uses, copied EXACTLY from the agent catalog. " +
|
|
109
97
|
"One phrase, verbatim — never a paraphrase, never a list, never your own wording. " +
|
|
110
98
|
"The launch resolves the reference data that the capability declares before the run starts, so a reworded phrase loses that check and the step discovers an absent source itself."),
|
|
111
|
-
timeout: z.number().optional().describe("Execution timeout in seconds (falls back to image default)"),
|
|
112
99
|
maxSteps: z.number().describe("Max LLM agentic steps for sandbox agent"),
|
|
113
100
|
description: z.string().optional().describe("Human-readable description of what the step produces."),
|
|
114
101
|
// Step result fields (populated during execution)
|
|
@@ -128,6 +115,10 @@ export const AnalysisPlanSchema = z.object({
|
|
|
128
115
|
created_at: z.string().describe("ISO-8601 timestamp for when the plan was created."),
|
|
129
116
|
omicsType: z.string().optional().describe("Omics data type detected from the data context (e.g. transcriptomics, proteomics, metabolomics)."),
|
|
130
117
|
omicsSubtype: z.string().optional().describe("Subtype detected from the data context (e.g. bulk-rna-seq, single-cell, microarray)."),
|
|
118
|
+
request: z
|
|
119
|
+
.string()
|
|
120
|
+
.optional()
|
|
121
|
+
.describe("The request that the plan answers, as `generate_plan` received it. The briefing of each step renders it. Absent on an ad hoc plan and on a historical plan."),
|
|
131
122
|
});
|
|
132
123
|
// ── Suspension Payload ──────────────────────────────────────────────
|
|
133
124
|
/**
|
package/dist/state/index.d.ts
CHANGED
|
@@ -16,7 +16,6 @@ setRunSynthesisOutcome, queryRun, queryActiveRun, queryNonTerminalRunsByAnalysis
|
|
|
16
16
|
export type { InsertRunInput, RunPage, RunReservation } from "./runs.js";
|
|
17
17
|
export { insertStepExecution, seedStepExecutions, sweepPendingStepExecutions, updateStepExecution, queryStepsByRun } from "./step-executions.js";
|
|
18
18
|
export type { InsertStepExecutionInput, SeedStepExecutionRow, UpdateStepExecutionInput } from "./step-executions.js";
|
|
19
|
-
export { setSandboxRef, clearSandboxRef, reconcileReapedSandbox } from "./active-sandboxes.js";
|
|
20
19
|
export { insertPlan, upsertPlan, loadPlan, loadPlanRecord } from "./plans.js";
|
|
21
20
|
export type { InsertPlanInput, UpsertPlanInput, PlanRecord, PlanRowQuerier } from "./plans.js";
|
|
22
21
|
export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, completeEmptyDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
|
package/dist/state/index.js
CHANGED
|
@@ -12,7 +12,6 @@ export { upsertArtifact, upsertArtifacts, queryInputArtifacts, queryStepArtifact
|
|
|
12
12
|
export { insertRun, reserveRunById, RunDedupCollisionError, RunIdentityCollisionError, updateRunStatus, reconcileOrphanedRuns, ORPHANED_RUN_GRACE_MS, markRunCanceledIfActive, promoteFailedToPartial, setRunMandate, // oss-core-managed-ok: run-mandate ledger (nullable; OSS leaves null)
|
|
13
13
|
setRunSynthesisOutcome, queryRun, queryActiveRun, queryNonTerminalRunsByAnalysis, queryRunsForInspection, queryRunsByAnalysis, countRunsByAnalysis, queryRunsByThread, queryRunIdsByPlan, queryNewestPlanRun, } from "./runs.js";
|
|
14
14
|
export { insertStepExecution, seedStepExecutions, sweepPendingStepExecutions, updateStepExecution, queryStepsByRun } from "./step-executions.js";
|
|
15
|
-
export { setSandboxRef, clearSandboxRef, reconcileReapedSandbox } from "./active-sandboxes.js";
|
|
16
15
|
export { insertPlan, upsertPlan, loadPlan, loadPlanRecord } from "./plans.js";
|
|
17
16
|
export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, completeEmptyDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
|
|
18
17
|
export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses, queryLatestRunsByAnalyses, queryRunProgressByRuns, queryRunsEndedSince, } from "./analyses-metrics.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record each sandbox lease. A row is one generation of a lease and its lease
|
|
3
|
+
* workflow: the request, the grant, the current machine, and the release. The
|
|
4
|
+
* lease table replaces the sandbox handle on the step rows.
|
|
5
|
+
*/
|
|
6
|
+
import type { Migration } from "kysely/migration";
|
|
7
|
+
export declare const leaseTheSandboxes: Migration;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record each sandbox lease. A row is one generation of a lease and its lease
|
|
3
|
+
* workflow: the request, the grant, the current machine, and the release. The
|
|
4
|
+
* lease table replaces the sandbox handle on the step rows.
|
|
5
|
+
*/
|
|
6
|
+
import { sql } from "kysely";
|
|
7
|
+
export const leaseTheSandboxes = {
|
|
8
|
+
async up(db) {
|
|
9
|
+
await sql `CREATE TABLE IF NOT EXISTS cortex_sandbox_leases (
|
|
10
|
+
lease_workflow_id TEXT PRIMARY KEY,
|
|
11
|
+
lease_key TEXT NOT NULL,
|
|
12
|
+
generation INTEGER NOT NULL,
|
|
13
|
+
analysis_id TEXT NOT NULL,
|
|
14
|
+
owner_workflow_id TEXT NOT NULL,
|
|
15
|
+
work_kind TEXT NOT NULL,
|
|
16
|
+
work_id TEXT NOT NULL,
|
|
17
|
+
requested JSONB NOT NULL,
|
|
18
|
+
granted JSONB,
|
|
19
|
+
state TEXT NOT NULL,
|
|
20
|
+
release_reason TEXT,
|
|
21
|
+
machine JSONB,
|
|
22
|
+
machine_generation INTEGER NOT NULL DEFAULT 0,
|
|
23
|
+
exec_salt TEXT NOT NULL DEFAULT gen_random_uuid()::text,
|
|
24
|
+
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
25
|
+
granted_at TIMESTAMPTZ,
|
|
26
|
+
last_used_at TIMESTAMPTZ,
|
|
27
|
+
released_at TIMESTAMPTZ
|
|
28
|
+
)`.execute(db);
|
|
29
|
+
await sql `CREATE UNIQUE INDEX IF NOT EXISTS idx_cortex_sandbox_leases_active_key ON cortex_sandbox_leases(lease_key) WHERE state <> 'released'`.execute(db);
|
|
30
|
+
await sql `CREATE INDEX IF NOT EXISTS idx_cortex_sandbox_leases_waiting ON cortex_sandbox_leases(requested_at) WHERE state = 'requested'`.execute(db);
|
|
31
|
+
await sql `CREATE INDEX IF NOT EXISTS idx_cortex_sandbox_leases_analysis ON cortex_sandbox_leases(analysis_id)`.execute(db);
|
|
32
|
+
await sql `ALTER TABLE cortex_step_executions DROP COLUMN IF EXISTS sandbox_ref`.execute(db);
|
|
33
|
+
},
|
|
34
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A report session derives no table, thus the session state keeps no list of derivations.
|
|
3
|
+
*/
|
|
4
|
+
import { sql } from "kysely";
|
|
5
|
+
export const dropTheReportDerivations = {
|
|
6
|
+
async up(db) {
|
|
7
|
+
await sql `ALTER TABLE cortex_report_session_state DROP COLUMN IF EXISTS derivations`.execute(db);
|
|
8
|
+
},
|
|
9
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A report session derives a table in a script, thus the session state keeps its list of derivations again. A
|
|
3
|
+
* database that ran the drop gets the column back with no rows in it. Each other database keeps its column.
|
|
4
|
+
*/
|
|
5
|
+
import type { Migration } from "kysely/migration";
|
|
6
|
+
export declare const keepTheReportDerivations: Migration;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A report session derives a table in a script, thus the session state keeps its list of derivations again. A
|
|
3
|
+
* database that ran the drop gets the column back with no rows in it. Each other database keeps its column.
|
|
4
|
+
*/
|
|
5
|
+
import { sql } from "kysely";
|
|
6
|
+
export const keepTheReportDerivations = {
|
|
7
|
+
async up(db) {
|
|
8
|
+
await sql `ALTER TABLE cortex_report_session_state ADD COLUMN IF NOT EXISTS derivations JSONB`.execute(db);
|
|
9
|
+
},
|
|
10
|
+
};
|
|
@@ -21,6 +21,9 @@ import { recordThePausedWork } from "./20261028120000_record_the_paused_work.js"
|
|
|
21
21
|
import { endTheSuspendedRuns } from "./20261028120100_end_the_suspended_runs.js";
|
|
22
22
|
import { referenceTheRunMandate } from "./20261029120000_reference_the_run_mandate.js";
|
|
23
23
|
import { recordTheRunSessionOfAPause } from "./20261029120100_record_the_run_session_of_a_pause.js";
|
|
24
|
+
import { leaseTheSandboxes } from "./20261030120000_lease_the_sandboxes.js";
|
|
25
|
+
import { dropTheReportDerivations } from "./20261030120100_drop_the_report_derivations.js";
|
|
26
|
+
import { keepTheReportDerivations } from "./20261030120200_keep_the_report_derivations.js";
|
|
24
27
|
export function cortexMigrations(logger) {
|
|
25
28
|
return {
|
|
26
29
|
"20260927120000_baseline": baseline(logger),
|
|
@@ -39,5 +42,8 @@ export function cortexMigrations(logger) {
|
|
|
39
42
|
"20261028120100_end_the_suspended_runs": endTheSuspendedRuns,
|
|
40
43
|
"20261029120000_reference_the_run_mandate": referenceTheRunMandate,
|
|
41
44
|
"20261029120100_record_the_run_session_of_a_pause": recordTheRunSessionOfAPause,
|
|
45
|
+
"20261030120000_lease_the_sandboxes": leaseTheSandboxes,
|
|
46
|
+
"20261030120100_drop_the_report_derivations": dropTheReportDerivations,
|
|
47
|
+
"20261030120200_keep_the_report_derivations": keepTheReportDerivations,
|
|
42
48
|
};
|
|
43
49
|
}
|
|
@@ -29,8 +29,10 @@
|
|
|
29
29
|
* What it does not reach, so that absent coverage is never read as delivered:
|
|
30
30
|
* scheduled operational workflows (they belong to no analysis and accumulate
|
|
31
31
|
* independently of any purge), the shared regulatory corpus, `messages` rows whose
|
|
32
|
-
* thread row is already gone (nothing attributes them to an analysis),
|
|
33
|
-
*
|
|
32
|
+
* thread row is already gone (nothing attributes them to an analysis), the sandbox
|
|
33
|
+
* machines (the purge cancels their lease workflows, and the sandbox reconciler then
|
|
34
|
+
* removes each machine whose lease workflow ended), and workspace files on disk (the
|
|
35
|
+
* embedder owns their disposal).
|
|
34
36
|
*/
|
|
35
37
|
import { ResultAsync } from "neverthrow";
|
|
36
38
|
import type { Pool } from "pg";
|
|
@@ -29,8 +29,10 @@
|
|
|
29
29
|
* What it does not reach, so that absent coverage is never read as delivered:
|
|
30
30
|
* scheduled operational workflows (they belong to no analysis and accumulate
|
|
31
31
|
* independently of any purge), the shared regulatory corpus, `messages` rows whose
|
|
32
|
-
* thread row is already gone (nothing attributes them to an analysis),
|
|
33
|
-
*
|
|
32
|
+
* thread row is already gone (nothing attributes them to an analysis), the sandbox
|
|
33
|
+
* machines (the purge cancels their lease workflows, and the sandbox reconciler then
|
|
34
|
+
* removes each machine whose lease workflow ended), and workspace files on disk (the
|
|
35
|
+
* embedder owns their disposal).
|
|
34
36
|
*/
|
|
35
37
|
import { errAsync, okAsync } from "neverthrow";
|
|
36
38
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
@@ -45,8 +47,9 @@ import { isSqlIdentifier } from "./vector-store.js";
|
|
|
45
47
|
* id and interpolated into `DROP TABLE`, because an identifier cannot be a bind
|
|
46
48
|
* parameter. Confining the id to `[a-z0-9_-]` is what makes that derived name an
|
|
47
49
|
* identifier at all.
|
|
48
|
-
* - An unambiguous `dataprofile:` namespace. Profile workflows
|
|
49
|
-
*
|
|
50
|
+
* - An unambiguous `dataprofile:` and `sandboxlease:` namespace. Profile workflows
|
|
51
|
+
* and lease workflows are found by the id prefixes `dataprofile:{analysisId}:` and
|
|
52
|
+
* `sandboxlease:{analysisId}:`, matched with `starts_with` and no
|
|
50
53
|
* delimiter check. An id holding a `:` makes analysis `a`'s prefix equally a
|
|
51
54
|
* prefix of every workflow id of an analysis named `a:x` — and each id that
|
|
52
55
|
* prefix returns is cancelled, then deleted with its descendants, taking another
|
|
@@ -78,6 +81,7 @@ function isPurgeableAnalysisIdShape(analysisId) {
|
|
|
78
81
|
const ANALYSIS_KEYED_DELETES = [
|
|
79
82
|
{ op: "purgeAnalysis.artifacts", sql: "DELETE FROM cortex_artifacts WHERE analysis_id = $1" },
|
|
80
83
|
{ op: "purgeAnalysis.stepExecutions", sql: "DELETE FROM cortex_step_executions WHERE analysis_id = $1" },
|
|
84
|
+
{ op: "purgeAnalysis.sandboxLeases", sql: "DELETE FROM cortex_sandbox_leases WHERE analysis_id = $1" },
|
|
81
85
|
{ op: "purgeAnalysis.runs", sql: "DELETE FROM cortex_runs WHERE analysis_id = $1" },
|
|
82
86
|
{ op: "purgeAnalysis.workingMemory", sql: "DELETE FROM cortex_working_memory WHERE analysis_id = $1" },
|
|
83
87
|
{ op: "purgeAnalysis.asks", sql: "DELETE FROM cortex_asks WHERE analysis_id = $1" },
|
|
@@ -113,8 +117,10 @@ export function createAnalysisPurge({ pool, workflows, logger: injected }) {
|
|
|
113
117
|
/**
|
|
114
118
|
* Every workflow id the analysis can still be traced to, read while the rows
|
|
115
119
|
* that record them are all still on disk. `cortex_runs.run_id` IS the parent
|
|
116
|
-
* workflow id, and those rows together with the `dataprofile:{analysisId}:`
|
|
117
|
-
*
|
|
120
|
+
* workflow id, and those rows together with the `dataprofile:{analysisId}:` and the
|
|
121
|
+
* `sandboxlease:{analysisId}:` id namespaces are the ONLY mapping from an analysis to its
|
|
122
|
+
* workflows. A lease workflow is no descendant of its consumer, thus only its namespace
|
|
123
|
+
* reaches it. A delete
|
|
118
124
|
* of `cortex_runs` that lands before this read completes strands the largest
|
|
119
125
|
* part of the footprint permanently out of reach.
|
|
120
126
|
*/
|
|
@@ -124,12 +130,17 @@ export function createAnalysisPurge({ pool, workflows, logger: injected }) {
|
|
|
124
130
|
values: [analysisId],
|
|
125
131
|
});
|
|
126
132
|
return rows.map((row) => row.run_id);
|
|
127
|
-
}).andThen((runIds) => workflows.findByIdPrefix(`dataprofile:${analysisId}:`).
|
|
128
|
-
logger.debug("collected workflow identity", {
|
|
129
|
-
|
|
133
|
+
}).andThen((runIds) => workflows.findByIdPrefix(`dataprofile:${analysisId}:`).andThen((profileIds) => workflows.findByIdPrefix(`sandboxlease:${analysisId}:`).map((leaseIds) => {
|
|
134
|
+
logger.debug("collected workflow identity", {
|
|
135
|
+
analysisId,
|
|
136
|
+
runs: runIds.length,
|
|
137
|
+
dataProfiles: profileIds.length,
|
|
138
|
+
sandboxLeases: leaseIds.length,
|
|
139
|
+
});
|
|
140
|
+
// The id sources cannot overlap, but collapsing duplicates keeps a
|
|
130
141
|
// repeated id out of the reclaimed count regardless.
|
|
131
|
-
return [...new Set([...runIds, ...profileIds])];
|
|
132
|
-
}));
|
|
142
|
+
return [...new Set([...runIds, ...profileIds, ...leaseIds])];
|
|
143
|
+
})));
|
|
133
144
|
/**
|
|
134
145
|
* One transaction for the whole `cortex_*` stage, so a mid-stage failure leaves
|
|
135
146
|
* no half-deleted analysis behind.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rows of `cortex_sandbox_leases`: one row for each generation of a sandbox lease and its lease workflow. The
|
|
3
|
+
* sandbox module writes them, and the reconciler and the purge read them.
|
|
4
|
+
*
|
|
5
|
+
* The grant is the one place that compares a request against the budget. It runs under one transaction-scoped
|
|
6
|
+
* advisory lock, thus two grants never read the same free capacity.
|
|
7
|
+
*/
|
|
8
|
+
import type { ResultAsync } from "neverthrow";
|
|
9
|
+
import { type MachineBudget, type ResourceSpec } from "../config/resource-limits.js";
|
|
10
|
+
import type { WorkKind } from "../contracts/chat-parts.js";
|
|
11
|
+
import { type Connectable, type DbError } from "../lib/db-result.js";
|
|
12
|
+
import type { Querier } from "./db.js";
|
|
13
|
+
import { type PersistedSandboxRef } from "./schema.js";
|
|
14
|
+
export type LeaseState = "requested" | "granted" | "released";
|
|
15
|
+
/** One generation of a lease, as its row records it. */
|
|
16
|
+
export type LeaseRecord = {
|
|
17
|
+
readonly leaseWorkflowId: string;
|
|
18
|
+
readonly leaseKey: string;
|
|
19
|
+
readonly generation: number;
|
|
20
|
+
readonly analysisId: string;
|
|
21
|
+
readonly ownerWorkflowId: string;
|
|
22
|
+
readonly workKind: WorkKind;
|
|
23
|
+
readonly workId: string;
|
|
24
|
+
readonly requested: ResourceSpec;
|
|
25
|
+
readonly granted?: ResourceSpec;
|
|
26
|
+
readonly state: LeaseState;
|
|
27
|
+
readonly releaseReason?: string;
|
|
28
|
+
readonly machine?: PersistedSandboxRef;
|
|
29
|
+
readonly machineGeneration: number;
|
|
30
|
+
/** The random value of the exec ids of the lease. No machine receives it, thus a command cannot derive a later exec id. */
|
|
31
|
+
readonly execSalt: string;
|
|
32
|
+
};
|
|
33
|
+
/** What a request writes. The row starts in `requested`, and the time of the insert orders the queue. */
|
|
34
|
+
export type LeaseRequest = Omit<LeaseRecord, "granted" | "state" | "releaseReason" | "machine" | "machineGeneration" | "execSalt">;
|
|
35
|
+
/**
|
|
36
|
+
* Write the request of a new generation. A key that holds a lease that is not released keeps it: the insert does
|
|
37
|
+
* nothing, and the caller reads the active lease again.
|
|
38
|
+
*/
|
|
39
|
+
export declare function requestLease(pool: Querier, request: LeaseRequest): ResultAsync<void, DbError>;
|
|
40
|
+
/** The lease of the key that is not released, or `undefined`. */
|
|
41
|
+
export declare function activeLease(pool: Querier, leaseKey: string): ResultAsync<LeaseRecord | undefined, DbError>;
|
|
42
|
+
export declare function readLease(pool: Querier, leaseWorkflowId: string): ResultAsync<LeaseRecord | undefined, DbError>;
|
|
43
|
+
/** The highest generation of the key, or 0 for a key with no lease. */
|
|
44
|
+
export declare function latestGeneration(pool: Querier, leaseKey: string): ResultAsync<number, DbError>;
|
|
45
|
+
export type GrantOutcome = {
|
|
46
|
+
readonly kind: "granted";
|
|
47
|
+
readonly size: ResourceSpec;
|
|
48
|
+
} | {
|
|
49
|
+
readonly kind: "waiting";
|
|
50
|
+
} | {
|
|
51
|
+
readonly kind: "released";
|
|
52
|
+
};
|
|
53
|
+
/** Grant the lease when it is the oldest waiting request and it fits the budget. With no budget, grant at once. */
|
|
54
|
+
export declare function tryGrant(pool: Connectable, leaseWorkflowId: string, budget: MachineBudget | undefined): ResultAsync<GrantOutcome, DbError>;
|
|
55
|
+
/** The lease workflow of the oldest waiting request, the next one that a free capacity can wake. */
|
|
56
|
+
export declare function oldestWaiting(pool: Querier): ResultAsync<string | undefined, DbError>;
|
|
57
|
+
/** Record the machine of a machine generation. */
|
|
58
|
+
export declare function setMachine(pool: Querier, leaseWorkflowId: string, machine: PersistedSandboxRef, machineGeneration: number): ResultAsync<void, DbError>;
|
|
59
|
+
/**
|
|
60
|
+
* Clear the machine of one machine generation, and give the machine that the row held. A row whose generation
|
|
61
|
+
* moved on keeps its machine, thus a late report of an old death clears nothing.
|
|
62
|
+
*/
|
|
63
|
+
export declare function clearMachine(pool: Querier, leaseWorkflowId: string, machineGeneration: number): ResultAsync<PersistedSandboxRef | undefined, DbError>;
|
|
64
|
+
/**
|
|
65
|
+
* Clear the machine of an idle lease: no exec used the lease for `idleMs`. An exec writes `last_used_at` before it
|
|
66
|
+
* submits, under the same row lock, thus an exec that starts can never lose its machine to this clear.
|
|
67
|
+
*/
|
|
68
|
+
export declare function clearIdleMachine(pool: Querier, leaseWorkflowId: string, machineGeneration: number, idleMs: number): ResultAsync<PersistedSandboxRef | undefined, DbError>;
|
|
69
|
+
/** Mark the lease used now, and give its machine. A lease that is not granted gives `undefined`. */
|
|
70
|
+
export declare function touchLease(pool: Querier, leaseWorkflowId: string): ResultAsync<{
|
|
71
|
+
machine?: PersistedSandboxRef;
|
|
72
|
+
machineGeneration: number;
|
|
73
|
+
} | undefined, DbError>;
|
|
74
|
+
/** Mark the lease released, and give the machine that it still held. A second release gives `undefined`. */
|
|
75
|
+
export declare function releaseLease(pool: Querier, leaseWorkflowId: string, reason: string): ResultAsync<{
|
|
76
|
+
machine?: PersistedSandboxRef;
|
|
77
|
+
} | undefined, DbError>;
|
|
78
|
+
/** Each lease of the given owner workflows that is not released. */
|
|
79
|
+
export declare function leasesOwnedBy(pool: Querier, ownerWorkflowIds: readonly string[]): ResultAsync<LeaseRecord[], DbError>;
|
|
80
|
+
/** Each lease that is not released, for the sweep of the reconciler. */
|
|
81
|
+
export declare function openLeases(pool: Querier): ResultAsync<LeaseRecord[], DbError>;
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rows of `cortex_sandbox_leases`: one row for each generation of a sandbox lease and its lease workflow. The
|
|
3
|
+
* sandbox module writes them, and the reconciler and the purge read them.
|
|
4
|
+
*
|
|
5
|
+
* The grant is the one place that compares a request against the budget. It runs under one transaction-scoped
|
|
6
|
+
* advisory lock, thus two grants never read the same free capacity.
|
|
7
|
+
*/
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
import { ResourceSpecSchema } from "../config/resource-limits.js";
|
|
10
|
+
import { tryMutation, tryQuery, withTransaction } from "../lib/db-result.js";
|
|
11
|
+
import { PersistedSandboxRefSchema } from "./schema.js";
|
|
12
|
+
const LeaseRowSchema = z.object({
|
|
13
|
+
lease_workflow_id: z.string(),
|
|
14
|
+
lease_key: z.string(),
|
|
15
|
+
generation: z.number(),
|
|
16
|
+
analysis_id: z.string(),
|
|
17
|
+
owner_workflow_id: z.string(),
|
|
18
|
+
work_kind: z.enum(["turn", "run", "profile"]),
|
|
19
|
+
work_id: z.string(),
|
|
20
|
+
requested: ResourceSpecSchema,
|
|
21
|
+
granted: ResourceSpecSchema.nullable(),
|
|
22
|
+
state: z.enum(["requested", "granted", "released"]),
|
|
23
|
+
release_reason: z.string().nullable(),
|
|
24
|
+
machine: PersistedSandboxRefSchema.nullable(),
|
|
25
|
+
machine_generation: z.number(),
|
|
26
|
+
exec_salt: z.string(),
|
|
27
|
+
});
|
|
28
|
+
const COLUMNS = "lease_workflow_id, lease_key, generation, analysis_id, owner_workflow_id, work_kind, work_id, requested, granted, state, release_reason, machine, machine_generation, exec_salt";
|
|
29
|
+
/** The one key of the advisory lock of the grant. Any constant works; it only has to be the same for each grant. */
|
|
30
|
+
const GRANT_LOCK_KEY = 74_210_031;
|
|
31
|
+
function recordOf(row) {
|
|
32
|
+
const parsed = LeaseRowSchema.parse(row);
|
|
33
|
+
return {
|
|
34
|
+
leaseWorkflowId: parsed.lease_workflow_id,
|
|
35
|
+
leaseKey: parsed.lease_key,
|
|
36
|
+
generation: parsed.generation,
|
|
37
|
+
analysisId: parsed.analysis_id,
|
|
38
|
+
ownerWorkflowId: parsed.owner_workflow_id,
|
|
39
|
+
workKind: parsed.work_kind,
|
|
40
|
+
workId: parsed.work_id,
|
|
41
|
+
requested: parsed.requested,
|
|
42
|
+
granted: parsed.granted ?? undefined,
|
|
43
|
+
state: parsed.state,
|
|
44
|
+
releaseReason: parsed.release_reason ?? undefined,
|
|
45
|
+
machine: parsed.machine ?? undefined,
|
|
46
|
+
machineGeneration: parsed.machine_generation,
|
|
47
|
+
execSalt: parsed.exec_salt,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Write the request of a new generation. A key that holds a lease that is not released keeps it: the insert does
|
|
52
|
+
* nothing, and the caller reads the active lease again.
|
|
53
|
+
*/
|
|
54
|
+
export function requestLease(pool, request) {
|
|
55
|
+
return tryMutation("sandboxLeases.request", async () => {
|
|
56
|
+
await pool.query({
|
|
57
|
+
text: `INSERT INTO cortex_sandbox_leases
|
|
58
|
+
(lease_workflow_id, lease_key, generation, analysis_id, owner_workflow_id, work_kind, work_id, requested, state)
|
|
59
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, 'requested')
|
|
60
|
+
ON CONFLICT DO NOTHING`,
|
|
61
|
+
values: [
|
|
62
|
+
request.leaseWorkflowId,
|
|
63
|
+
request.leaseKey,
|
|
64
|
+
request.generation,
|
|
65
|
+
request.analysisId,
|
|
66
|
+
request.ownerWorkflowId,
|
|
67
|
+
request.workKind,
|
|
68
|
+
request.workId,
|
|
69
|
+
JSON.stringify(request.requested),
|
|
70
|
+
],
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
/** The lease of the key that is not released, or `undefined`. */
|
|
75
|
+
export function activeLease(pool, leaseKey) {
|
|
76
|
+
return tryQuery("sandboxLeases.active", async () => {
|
|
77
|
+
const { rows } = await pool.query({
|
|
78
|
+
text: `SELECT ${COLUMNS} FROM cortex_sandbox_leases WHERE lease_key = $1 AND state <> 'released'`,
|
|
79
|
+
values: [leaseKey],
|
|
80
|
+
});
|
|
81
|
+
return rows.length === 0 ? undefined : recordOf(rows[0]);
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
export function readLease(pool, leaseWorkflowId) {
|
|
85
|
+
return tryQuery("sandboxLeases.read", async () => {
|
|
86
|
+
const { rows } = await pool.query({
|
|
87
|
+
text: `SELECT ${COLUMNS} FROM cortex_sandbox_leases WHERE lease_workflow_id = $1`,
|
|
88
|
+
values: [leaseWorkflowId],
|
|
89
|
+
});
|
|
90
|
+
return rows.length === 0 ? undefined : recordOf(rows[0]);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
/** The highest generation of the key, or 0 for a key with no lease. */
|
|
94
|
+
export function latestGeneration(pool, leaseKey) {
|
|
95
|
+
return tryQuery("sandboxLeases.latestGeneration", async () => {
|
|
96
|
+
const { rows } = await pool.query({
|
|
97
|
+
text: `SELECT max(generation) AS generation FROM cortex_sandbox_leases WHERE lease_key = $1`,
|
|
98
|
+
values: [leaseKey],
|
|
99
|
+
});
|
|
100
|
+
return rows[0]?.generation ?? 0;
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
function fits(budget, held, request) {
|
|
104
|
+
return held.cpu + request.cpu <= budget.cpu && held.memoryGb + request.memoryGb <= budget.memoryGb;
|
|
105
|
+
}
|
|
106
|
+
function grantIn(client, leaseWorkflowId, budget) {
|
|
107
|
+
return tryMutation("sandboxLeases.grant", async () => {
|
|
108
|
+
await client.query({ text: "SELECT pg_advisory_xact_lock($1)", values: [GRANT_LOCK_KEY] });
|
|
109
|
+
const own = await client.query({ text: `SELECT ${COLUMNS} FROM cortex_sandbox_leases WHERE lease_workflow_id = $1`, values: [leaseWorkflowId] });
|
|
110
|
+
const lease = own.rows.length === 0 ? undefined : recordOf(own.rows[0]);
|
|
111
|
+
if (lease === undefined || lease.state === "released")
|
|
112
|
+
return { kind: "released" };
|
|
113
|
+
if (lease.state === "granted")
|
|
114
|
+
return { kind: "granted", size: lease.granted ?? lease.requested };
|
|
115
|
+
if (budget !== undefined) {
|
|
116
|
+
// The strict order of the requests: only the oldest waiting request can take a grant.
|
|
117
|
+
const head = await client.query(`SELECT lease_workflow_id FROM cortex_sandbox_leases WHERE state = 'requested' ORDER BY requested_at, lease_workflow_id LIMIT 1`);
|
|
118
|
+
if (head.rows[0]?.lease_workflow_id !== leaseWorkflowId)
|
|
119
|
+
return { kind: "waiting" };
|
|
120
|
+
const sums = await client.query(`SELECT sum((granted->>'cpu')::numeric) AS cpu, sum((granted->>'memoryGb')::numeric) AS memory_gb
|
|
121
|
+
FROM cortex_sandbox_leases WHERE state = 'granted'`);
|
|
122
|
+
const held = { cpu: Number(sums.rows[0]?.cpu ?? 0), memoryGb: Number(sums.rows[0]?.memory_gb ?? 0) };
|
|
123
|
+
if (!fits(budget, held, lease.requested))
|
|
124
|
+
return { kind: "waiting" };
|
|
125
|
+
}
|
|
126
|
+
await client.query({
|
|
127
|
+
text: `UPDATE cortex_sandbox_leases SET state = 'granted', granted = requested, granted_at = now(), last_used_at = now()
|
|
128
|
+
WHERE lease_workflow_id = $1`,
|
|
129
|
+
values: [leaseWorkflowId],
|
|
130
|
+
});
|
|
131
|
+
return { kind: "granted", size: lease.requested };
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
/** Grant the lease when it is the oldest waiting request and it fits the budget. With no budget, grant at once. */
|
|
135
|
+
export function tryGrant(pool, leaseWorkflowId, budget) {
|
|
136
|
+
return withTransaction(pool, "sandboxLeases.tryGrant", (client) => grantIn(client, leaseWorkflowId, budget));
|
|
137
|
+
}
|
|
138
|
+
/** The lease workflow of the oldest waiting request, the next one that a free capacity can wake. */
|
|
139
|
+
export function oldestWaiting(pool) {
|
|
140
|
+
return tryQuery("sandboxLeases.oldestWaiting", async () => {
|
|
141
|
+
const { rows } = await pool.query(`SELECT lease_workflow_id FROM cortex_sandbox_leases WHERE state = 'requested' ORDER BY requested_at, lease_workflow_id LIMIT 1`);
|
|
142
|
+
return rows[0]?.lease_workflow_id;
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
/** Record the machine of a machine generation. */
|
|
146
|
+
export function setMachine(pool, leaseWorkflowId, machine, machineGeneration) {
|
|
147
|
+
return tryMutation("sandboxLeases.setMachine", async () => {
|
|
148
|
+
await pool.query({
|
|
149
|
+
text: `UPDATE cortex_sandbox_leases SET machine = $2::jsonb, machine_generation = $3, last_used_at = now()
|
|
150
|
+
WHERE lease_workflow_id = $1 AND state = 'granted'`,
|
|
151
|
+
values: [leaseWorkflowId, JSON.stringify(machine), machineGeneration],
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Clear the machine of one machine generation, and give the machine that the row held. A row whose generation
|
|
157
|
+
* moved on keeps its machine, thus a late report of an old death clears nothing.
|
|
158
|
+
*/
|
|
159
|
+
export function clearMachine(pool, leaseWorkflowId, machineGeneration) {
|
|
160
|
+
return tryMutation("sandboxLeases.clearMachine", async () => {
|
|
161
|
+
const { rows } = await pool.query({
|
|
162
|
+
text: `WITH old AS (
|
|
163
|
+
SELECT lease_workflow_id, machine FROM cortex_sandbox_leases
|
|
164
|
+
WHERE lease_workflow_id = $1 AND machine_generation = $2 AND machine IS NOT NULL
|
|
165
|
+
FOR UPDATE
|
|
166
|
+
)
|
|
167
|
+
UPDATE cortex_sandbox_leases AS l SET machine = NULL FROM old
|
|
168
|
+
WHERE l.lease_workflow_id = old.lease_workflow_id
|
|
169
|
+
RETURNING old.machine`,
|
|
170
|
+
values: [leaseWorkflowId, machineGeneration],
|
|
171
|
+
});
|
|
172
|
+
return rows.length === 0 ? undefined : PersistedSandboxRefSchema.parse(rows[0]?.machine);
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Clear the machine of an idle lease: no exec used the lease for `idleMs`. An exec writes `last_used_at` before it
|
|
177
|
+
* submits, under the same row lock, thus an exec that starts can never lose its machine to this clear.
|
|
178
|
+
*/
|
|
179
|
+
export function clearIdleMachine(pool, leaseWorkflowId, machineGeneration, idleMs) {
|
|
180
|
+
return tryMutation("sandboxLeases.clearIdleMachine", async () => {
|
|
181
|
+
const { rows } = await pool.query({
|
|
182
|
+
text: `WITH old AS (
|
|
183
|
+
SELECT lease_workflow_id, machine FROM cortex_sandbox_leases
|
|
184
|
+
WHERE lease_workflow_id = $1 AND machine_generation = $2 AND machine IS NOT NULL
|
|
185
|
+
AND last_used_at < now() - make_interval(secs => $3::double precision / 1000)
|
|
186
|
+
FOR UPDATE
|
|
187
|
+
)
|
|
188
|
+
UPDATE cortex_sandbox_leases AS l SET machine = NULL FROM old
|
|
189
|
+
WHERE l.lease_workflow_id = old.lease_workflow_id
|
|
190
|
+
RETURNING old.machine`,
|
|
191
|
+
values: [leaseWorkflowId, machineGeneration, idleMs],
|
|
192
|
+
});
|
|
193
|
+
return rows.length === 0 ? undefined : PersistedSandboxRefSchema.parse(rows[0]?.machine);
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
/** Mark the lease used now, and give its machine. A lease that is not granted gives `undefined`. */
|
|
197
|
+
export function touchLease(pool, leaseWorkflowId) {
|
|
198
|
+
return tryMutation("sandboxLeases.touch", async () => {
|
|
199
|
+
const { rows } = await pool.query({
|
|
200
|
+
text: `UPDATE cortex_sandbox_leases SET last_used_at = now()
|
|
201
|
+
WHERE lease_workflow_id = $1 AND state = 'granted'
|
|
202
|
+
RETURNING machine, machine_generation`,
|
|
203
|
+
values: [leaseWorkflowId],
|
|
204
|
+
});
|
|
205
|
+
if (rows.length === 0)
|
|
206
|
+
return undefined;
|
|
207
|
+
const row = z.object({ machine: PersistedSandboxRefSchema.nullable(), machine_generation: z.number() }).parse(rows[0]);
|
|
208
|
+
return { machine: row.machine ?? undefined, machineGeneration: row.machine_generation };
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
/** Mark the lease released, and give the machine that it still held. A second release gives `undefined`. */
|
|
212
|
+
export function releaseLease(pool, leaseWorkflowId, reason) {
|
|
213
|
+
return tryMutation("sandboxLeases.release", async () => {
|
|
214
|
+
const { rows } = await pool.query({
|
|
215
|
+
text: `WITH old AS (
|
|
216
|
+
SELECT lease_workflow_id, machine FROM cortex_sandbox_leases
|
|
217
|
+
WHERE lease_workflow_id = $1 AND state <> 'released'
|
|
218
|
+
FOR UPDATE
|
|
219
|
+
)
|
|
220
|
+
UPDATE cortex_sandbox_leases AS l SET state = 'released', release_reason = $2, released_at = now(), machine = NULL
|
|
221
|
+
FROM old WHERE l.lease_workflow_id = old.lease_workflow_id
|
|
222
|
+
RETURNING old.machine`,
|
|
223
|
+
values: [leaseWorkflowId, reason],
|
|
224
|
+
});
|
|
225
|
+
if (rows.length === 0)
|
|
226
|
+
return undefined;
|
|
227
|
+
const machine = PersistedSandboxRefSchema.nullable().parse(rows[0]?.machine);
|
|
228
|
+
return { machine: machine ?? undefined };
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
/** Each lease of the given owner workflows that is not released. */
|
|
232
|
+
export function leasesOwnedBy(pool, ownerWorkflowIds) {
|
|
233
|
+
return tryQuery("sandboxLeases.ownedBy", async () => {
|
|
234
|
+
const { rows } = await pool.query({
|
|
235
|
+
text: `SELECT ${COLUMNS} FROM cortex_sandbox_leases WHERE owner_workflow_id = ANY($1::text[]) AND state <> 'released'`,
|
|
236
|
+
values: [[...ownerWorkflowIds]],
|
|
237
|
+
});
|
|
238
|
+
return rows.map(recordOf);
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
/** Each lease that is not released, for the sweep of the reconciler. */
|
|
242
|
+
export function openLeases(pool) {
|
|
243
|
+
return tryQuery("sandboxLeases.open", async () => {
|
|
244
|
+
const { rows } = await pool.query(`SELECT ${COLUMNS} FROM cortex_sandbox_leases WHERE state <> 'released'`);
|
|
245
|
+
return rows.map(recordOf);
|
|
246
|
+
});
|
|
247
|
+
}
|
package/dist/state/schema.d.ts
CHANGED
|
@@ -99,10 +99,7 @@ export declare const CortexRunRowSchema: z.ZodObject<{
|
|
|
99
99
|
planId: z.ZodNullable<z.ZodString>;
|
|
100
100
|
}, z.core.$strip>;
|
|
101
101
|
export type CortexRunRow = z.infer<typeof CortexRunRowSchema>;
|
|
102
|
-
/**
|
|
103
|
-
* Persistable sandbox handle — what the active-sandbox registry stores on
|
|
104
|
-
* `cortex_step_executions.sandbox_ref`.
|
|
105
|
-
*/
|
|
102
|
+
/** Persistable sandbox handle — the machine that a lease row of `cortex_sandbox_leases` holds. */
|
|
106
103
|
export declare const PersistedSandboxRefSchema: z.ZodObject<{
|
|
107
104
|
sandboxId: z.ZodString;
|
|
108
105
|
host: z.ZodString;
|
|
@@ -137,15 +134,6 @@ export declare const StepExecutionRowSchema: z.ZodObject<{
|
|
|
137
134
|
finishReason: z.ZodNullable<z.ZodString>;
|
|
138
135
|
hitMaxSteps: z.ZodDefault<z.ZodBoolean>;
|
|
139
136
|
blockedReason: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
140
|
-
sandboxRef: z.ZodDefault<z.ZodNullable<z.ZodObject<{
|
|
141
|
-
sandboxId: z.ZodString;
|
|
142
|
-
host: z.ZodString;
|
|
143
|
-
port: z.ZodNumber;
|
|
144
|
-
backend: z.ZodEnum<{
|
|
145
|
-
docker: "docker";
|
|
146
|
-
k8s: "k8s";
|
|
147
|
-
}>;
|
|
148
|
-
}, z.core.$strip>>>;
|
|
149
137
|
childWorkflowId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
150
138
|
}, z.core.$strip>;
|
|
151
139
|
export type StepExecutionRow = z.infer<typeof StepExecutionRowSchema>;
|
package/dist/state/schema.js
CHANGED
|
@@ -57,10 +57,7 @@ export const CortexRunRowSchema = z.object({
|
|
|
57
57
|
mandateId: z.string().nullable(), // oss-core-managed-ok: run-mandate ledger (nullable; OSS leaves null)
|
|
58
58
|
planId: z.string().nullable(),
|
|
59
59
|
});
|
|
60
|
-
/**
|
|
61
|
-
* Persistable sandbox handle — what the active-sandbox registry stores on
|
|
62
|
-
* `cortex_step_executions.sandbox_ref`.
|
|
63
|
-
*/
|
|
60
|
+
/** Persistable sandbox handle — the machine that a lease row of `cortex_sandbox_leases` holds. */
|
|
64
61
|
export const PersistedSandboxRefSchema = z.object({
|
|
65
62
|
sandboxId: z.string(),
|
|
66
63
|
host: z.string(),
|
|
@@ -88,8 +85,6 @@ export const StepExecutionRowSchema = z.object({
|
|
|
88
85
|
hitMaxSteps: z.boolean().default(false),
|
|
89
86
|
/** Agent-declared blocker reason (see the harness-sandbox-agents spec); null unless `status === "blocked"`. */
|
|
90
87
|
blockedReason: z.string().nullable().default(null),
|
|
91
|
-
/** Active-sandbox registry handle; null when no sandbox is live for this step. */
|
|
92
|
-
sandboxRef: PersistedSandboxRefSchema.nullable().default(null),
|
|
93
88
|
/** DBOS child workflow id (`"${parentWorkflowId}-${N}"`); null on pre-DBOS rows. */
|
|
94
89
|
childWorkflowId: z.string().nullable().default(null),
|
|
95
90
|
});
|