@opengeni/worker-bundle 0.12.10 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2729 -1208
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +873 -131
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import type { SessionEventType } from "@opengeni/contracts";
|
|
2
|
+
import type { EventLogger } from "@opengeni/events";
|
|
3
|
+
import type { Observability } from "@opengeni/observability";
|
|
4
|
+
import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
|
|
5
|
+
export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
|
|
6
|
+
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
7
|
+
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
8
|
+
export type CreditBalanceGauge = {
|
|
9
|
+
accountId: string;
|
|
10
|
+
balanceMicros: number;
|
|
11
|
+
};
|
|
12
|
+
export declare function observabilityEventLogger(observability: Observability): EventLogger;
|
|
13
|
+
export declare function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks;
|
|
14
|
+
/**
|
|
15
|
+
* Adapt the runtime's transport-agnostic `SelfhostedOpObserver` to the metrics
|
|
16
|
+
* hooks: map a completed-op observation onto `onSandboxOp`, converting the wire
|
|
17
|
+
* `ErrorCode` to its stable enum-name string (a bounded metric label) and the
|
|
18
|
+
* duration to seconds. Wired into the selfhosted session build so every Connected
|
|
19
|
+
* Machine control op meters.
|
|
20
|
+
*/
|
|
21
|
+
export declare function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): SelfhostedOpObserver;
|
|
22
|
+
/** A session-scoped `machine.op.*` event mapped from a completed-op observation. */
|
|
23
|
+
export type MachineOpSessionEvent = {
|
|
24
|
+
type: "machine.op.failed" | "machine.op.recovered";
|
|
25
|
+
payload: {
|
|
26
|
+
op: string;
|
|
27
|
+
faultClass: string;
|
|
28
|
+
attempts: number;
|
|
29
|
+
machineId?: string;
|
|
30
|
+
};
|
|
31
|
+
};
|
|
32
|
+
/** Map an observation to a `machine.op.*` session event, or null if it is not
|
|
33
|
+
* eventable. `machine.op.failed` fires ONLY for infrastructure fault classes (a
|
|
34
|
+
* semantic miss the model asked about is an outcome, not an infra fault);
|
|
35
|
+
* `machine.op.recovered` fires for a healed op (success after ≥1 retry). */
|
|
36
|
+
export declare function machineOpSessionEventFor(o: SelfhostedOpObservation): MachineOpSessionEvent | null;
|
|
37
|
+
/**
|
|
38
|
+
* The Connected Machine op observer wired into a turn: it meters EVERY op (the
|
|
39
|
+
* metrics sink) and BUFFERS the eventable ops (infra failures + healed recoveries)
|
|
40
|
+
* as `machine.op.*` session events. The observer is SYNC (fire-and-forget), so the
|
|
41
|
+
* turn drains the buffer to durable session events at a known checkpoint (turn end),
|
|
42
|
+
* awaited — never an unawaited DB write inside the Temporal activity.
|
|
43
|
+
*/
|
|
44
|
+
export declare function makeMachineOpObserver(hooks: RuntimeMetricsHooks): {
|
|
45
|
+
observer: SelfhostedOpObserver;
|
|
46
|
+
drainEvents(): MachineOpSessionEvent[];
|
|
47
|
+
};
|
|
48
|
+
export declare function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics;
|
|
49
|
+
export declare class TurnLifecycleMetrics {
|
|
50
|
+
private readonly observability;
|
|
51
|
+
private readonly options;
|
|
52
|
+
private readonly startedTurns;
|
|
53
|
+
private timer;
|
|
54
|
+
constructor(observability: Observability, options?: {
|
|
55
|
+
now?: () => number;
|
|
56
|
+
refreshIntervalMs?: number;
|
|
57
|
+
});
|
|
58
|
+
start(turnId: string): void;
|
|
59
|
+
finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void;
|
|
60
|
+
refreshGauges(): void;
|
|
61
|
+
stop(): void;
|
|
62
|
+
private oldestInflightAgeSeconds;
|
|
63
|
+
private ensureTimer;
|
|
64
|
+
private stopTimer;
|
|
65
|
+
private now;
|
|
66
|
+
}
|
|
67
|
+
export declare function recordTurnsQueuedGauge(observability: Observability, value: number): void;
|
|
68
|
+
export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
|
|
69
|
+
export declare function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void;
|
|
70
|
+
export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
|
|
71
|
+
export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
|
|
72
|
+
export declare function recordSandboxCheckpointArtifactGauges(observability: Observability, counts: Record<SandboxCheckpointArtifactMetricState, number>): void;
|
|
73
|
+
export type SandboxCheckpointArtifactOutcome = "legacy_adopted" | "claimed" | "deleted" | "delete_failed" | "tombstone_pruned";
|
|
74
|
+
export declare function recordSandboxCheckpointArtifactOutcome(observability: Observability, outcome: SandboxCheckpointArtifactOutcome, count?: number): void;
|
|
75
|
+
export declare function recordSandboxDeadlineRotationsRequested(observability: Observability, count: number): void;
|
|
76
|
+
export declare function recordSandboxRotationBacklogGauges(observability: Observability, backlog: {
|
|
77
|
+
requested: number;
|
|
78
|
+
overdue: number;
|
|
79
|
+
turnBlocked: number;
|
|
80
|
+
directBlocked: number;
|
|
81
|
+
processBlocked: number;
|
|
82
|
+
}): void;
|
|
83
|
+
export declare function recordRetainedProcessInventoryGauges(observability: Observability, counts: Array<{
|
|
84
|
+
ownerState: string;
|
|
85
|
+
activeCount: number;
|
|
86
|
+
terminalOwnerCount: number;
|
|
87
|
+
}>): void;
|
|
88
|
+
export declare function recordExpiredDrainingSandboxLeaseGauges(observability: Observability, counts: Array<{
|
|
89
|
+
backend: string;
|
|
90
|
+
ageBucket: string;
|
|
91
|
+
count: number;
|
|
92
|
+
}>): void;
|
|
93
|
+
export declare const RETAINED_PROCESS_RECONCILIATION_OUTCOMES: readonly ["claim_failed", "proof_exited", "proof_lost", "proof_checkpoint_failed", "settled_exited", "settled_lost", "settlement_failed", "identity_mismatch", "resume_state_missing", "backend_unsupported", "provider_running", "provider_unknown", "provider_timeout", "provider_error", "provider_binding_missing", "provider_binding_mismatch", "provider_binding_adopted", "defer_failed"];
|
|
94
|
+
export type RetainedProcessReconciliationOutcome = (typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];
|
|
95
|
+
export declare function recordRetainedProcessReconciliation(observability: Observability, outcome: RetainedProcessReconciliationOutcome): void;
|
|
96
|
+
export declare function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void;
|
|
97
|
+
export type StreamDeltaClass = "message" | "reasoning";
|
|
98
|
+
/**
|
|
99
|
+
* Per-turn stream-timing tracker fed every normalized runtime event in push order.
|
|
100
|
+
* It emits two model-responsiveness SLIs from the worker's seat on the stream:
|
|
101
|
+
*
|
|
102
|
+
* - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its
|
|
103
|
+
* first streamed content delta. The anchor starts at construction (≈ runStream
|
|
104
|
+
* start, so the first observation is "how long until text appears") and re-arms
|
|
105
|
+
* on every non-content event (a tool call, a completed message, a usage frame),
|
|
106
|
+
* so a post-tool response measures the model's restart latency, NOT our own
|
|
107
|
+
* tool-execution time.
|
|
108
|
+
* - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
|
|
109
|
+
* consecutive content deltas of the SAME class. The run resets on any
|
|
110
|
+
* non-content event so a gap never spans a tool call or a model boundary — it
|
|
111
|
+
* measures only the choppiness of a live token stream.
|
|
112
|
+
*
|
|
113
|
+
* Purely observational and clock-injectable; it never touches the events it sees.
|
|
114
|
+
*/
|
|
115
|
+
export declare class StreamTimingMetrics {
|
|
116
|
+
private readonly observability;
|
|
117
|
+
private readonly options;
|
|
118
|
+
private readonly now;
|
|
119
|
+
private ttftAnchor;
|
|
120
|
+
private ttftArmed;
|
|
121
|
+
private readonly lastDeltaAt;
|
|
122
|
+
constructor(observability: Observability, options: {
|
|
123
|
+
provider: string;
|
|
124
|
+
now?: () => number;
|
|
125
|
+
});
|
|
126
|
+
onEvent(type: SessionEventType): void;
|
|
127
|
+
}
|
|
128
|
+
/** Flush shape of the streaming batcher: how many events coalesced into one flush
|
|
129
|
+
* (the coalescing win) and how long that flush took (append + publish). */
|
|
130
|
+
export declare function recordBatchFlush(observability: Observability, input: {
|
|
131
|
+
events: number;
|
|
132
|
+
durationSeconds: number;
|
|
133
|
+
}): void;
|
|
134
|
+
/** Latency of the durable `appendSessionEvents` DB write — the write path. A p99
|
|
135
|
+
* climb here is our Postgres, not the model or NATS. */
|
|
136
|
+
export declare function recordSessionEventAppendLatency(observability: Observability, input: {
|
|
137
|
+
durationSeconds: number;
|
|
138
|
+
}): void;
|
|
139
|
+
/** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
|
|
140
|
+
* here (with append healthy) is delivery, not the write path. */
|
|
141
|
+
export declare function recordSessionEventPublishLatency(observability: Observability, input: {
|
|
142
|
+
durationSeconds: number;
|
|
143
|
+
}): void;
|
|
144
|
+
/** Observed input (context) tokens per model response — the context-pressure
|
|
145
|
+
* signal. Paired with `opengeni_context_compactions_total`, it makes the
|
|
146
|
+
* "compaction never firing while contexts run hot" failure mode expressible. */
|
|
147
|
+
export declare function recordModelInputTokens(observability: Observability, provider: string, inputTokens: number): void;
|
|
148
|
+
/** A context compaction actually fired, by trigger (operator | overflow | proactive
|
|
149
|
+
* | auto). The rate of this — against the input-tokens histogram above — is how an
|
|
150
|
+
* operator sees compaction working (or silently not). */
|
|
151
|
+
export declare function recordContextCompaction(observability: Observability, trigger: string): void;
|
|
152
|
+
/**
|
|
153
|
+
* Prompt-cache efficiency of one model response, by provider. Reads from the SAME
|
|
154
|
+
* usage frame that feeds input-token accounting, so the two are always consistent:
|
|
155
|
+
* - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the
|
|
156
|
+
* provider served from cache. Advanced only when >0, so a provider that never
|
|
157
|
+
* reports cached tokens contributes nothing rather than phantom zero-increments.
|
|
158
|
+
* - `opengeni_model_cache_write_tokens_total{provider}` — cumulative tokens a
|
|
159
|
+
* reporting provider wrote into its prompt cache. Absent telemetry contributes
|
|
160
|
+
* nothing; in particular, Codex subscription usage does not currently expose it.
|
|
161
|
+
* - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,
|
|
162
|
+
* observed only when prompt tokens are known (>0) AND cached tokens are reported.
|
|
163
|
+
* A reported cached-token zero records a real 0%; absent/null telemetry remains
|
|
164
|
+
* unknown and must not fabricate a low-cache signal.
|
|
165
|
+
* - `opengeni_model_cache_read_telemetry_total{provider,status}` — one bounded
|
|
166
|
+
* availability observation per authoritative call (`reported` or `missing`).
|
|
167
|
+
* Invalid, fractional, unsafe, or over-contract token telemetry is rejected. A call
|
|
168
|
+
* with no prompt tokens has no ratio (skipped).
|
|
169
|
+
*/
|
|
170
|
+
export declare function recordModelCacheTokens(observability: Observability, provider: string, input: {
|
|
171
|
+
cachedTokens: number | null | undefined;
|
|
172
|
+
cacheWriteTokens?: number | null | undefined;
|
|
173
|
+
promptTokens: number | null | undefined;
|
|
174
|
+
}): void;
|
|
175
|
+
/**
|
|
176
|
+
* An opaque, stable, non-reversible tag for a codex credential (account) — the
|
|
177
|
+
* per-call log dimension the account-switch hypothesis correlates against. It is
|
|
178
|
+
* a hash of the NON-SECRET credential ROW id (never a token/bearer), truncated so
|
|
179
|
+
* it is short in logs while still distinguishing a handful of accounts without
|
|
180
|
+
* collision. A null/absent credential (non-codex turn, no active account) tags as
|
|
181
|
+
* "none" so the field is always present and never leaks an id verbatim.
|
|
182
|
+
*/
|
|
183
|
+
export declare function stableAccountHash(credentialId: string | null | undefined): string;
|
|
184
|
+
/**
|
|
185
|
+
* The per-call account dimensions for the usage log: the opaque serving-account
|
|
186
|
+
* tag and whether that account CHANGED versus the session's previous call. Within
|
|
187
|
+
* one turn the serving credential is fixed, so a switch can only surface on the
|
|
188
|
+
* turn's FIRST call (compared against the session's durably-recorded prior
|
|
189
|
+
* credential); later calls in the same turn report `false`. A switch is reported
|
|
190
|
+
* only when there was a KNOWN prior account that differs — a session's very first
|
|
191
|
+
* call (no prior) is a cold start, not a switch.
|
|
192
|
+
*/
|
|
193
|
+
export declare function modelCallAccountContext(input: {
|
|
194
|
+
servingCredentialId: string | null;
|
|
195
|
+
priorSessionCredentialId: string | null;
|
|
196
|
+
isFirstCallOfTurn: boolean;
|
|
197
|
+
}): {
|
|
198
|
+
servingAccountHash: string;
|
|
199
|
+
accountChangedFromPrevCall: boolean;
|
|
200
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Context } from "@temporalio/activity";
|
|
2
|
+
import type { OpStreamJournal } from "@opengeni/runtime";
|
|
3
|
+
/** The activity's shared heartbeat-details object: every heartbeat call site
|
|
4
|
+
* spreads THIS one object (plus its own phase), so fields like the op-ack
|
|
5
|
+
* roster survive across sites instead of being lost to last-write-wins. */
|
|
6
|
+
export interface TurnHeartbeatDetails extends Record<string, unknown> {
|
|
7
|
+
/** Settled-op roster: durable op id → its exit seq (the final-ack frontier). */
|
|
8
|
+
opAcks: Record<string, string>;
|
|
9
|
+
}
|
|
10
|
+
export declare function makeTurnOpJournal(context: Context | null, details: TurnHeartbeatDetails): OpStreamJournal;
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { type Settings } from "@opengeni/config";
|
|
2
|
+
import { SandboxLeaseSupersededError, type Database, type LeaseHolderKind } from "@opengeni/db";
|
|
3
|
+
import { establishSandboxSessionFromEnvelope, type EstablishedSandboxSession, type RuntimeMetricsHooks } from "@opengeni/runtime";
|
|
4
|
+
export type ResumeHolderKind = LeaseHolderKind;
|
|
5
|
+
/**
|
|
6
|
+
* The durable identity of a turn holding a shared sandbox lease.
|
|
7
|
+
*
|
|
8
|
+
* Temporal activity ids cannot satisfy this type: they are workflow-local
|
|
9
|
+
* sequence numbers and collide across sessions that share a sandbox group.
|
|
10
|
+
*/
|
|
11
|
+
export type TurnSandboxLeaseHolderId = `turn-attempt:${string}`;
|
|
12
|
+
export declare function sandboxLeaseHolderIdForAttempt(attemptId: string): TurnSandboxLeaseHolderId;
|
|
13
|
+
export declare function isRetryableDegradedRestore(restore: {
|
|
14
|
+
status: string;
|
|
15
|
+
retryable?: boolean;
|
|
16
|
+
}): boolean;
|
|
17
|
+
/** The minimal services surface resumeBoxForTurn needs. A subset of
|
|
18
|
+
* ActivityServices so a test (and the API later) can pass a lean bag. */
|
|
19
|
+
export type SandboxResumeServices = {
|
|
20
|
+
db: Database;
|
|
21
|
+
settings: Settings;
|
|
22
|
+
sandboxMetrics?: RuntimeMetricsHooks;
|
|
23
|
+
/**
|
|
24
|
+
* The logical turn-attempt lifetime, not merely the provider request lifetime.
|
|
25
|
+
* Some provider SDK calls cannot be interrupted. Aborting this signal still
|
|
26
|
+
* releases the durable holder immediately and fences any late provider result.
|
|
27
|
+
*/
|
|
28
|
+
cancellationSignal?: AbortSignal;
|
|
29
|
+
/** Test seam for the attached/resumed path. Production uses the one runtime
|
|
30
|
+
* resume primitive; callers must not use this to create a replacement box. */
|
|
31
|
+
establishAttachedSandbox?: typeof establishSandboxSessionFromEnvelope;
|
|
32
|
+
/** Test seam for the bounded command-readiness proof performed before an
|
|
33
|
+
* attached provider box is handed to the agent. */
|
|
34
|
+
verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
|
|
35
|
+
/** Called only by the observer that wins the exact warm->cold loss CAS. */
|
|
36
|
+
onSandboxLost?: (input: {
|
|
37
|
+
sandboxGroupId: string;
|
|
38
|
+
instanceId: string;
|
|
39
|
+
leaseEpoch: number;
|
|
40
|
+
}) => Promise<void>;
|
|
41
|
+
};
|
|
42
|
+
export type ResumeBoxIds = {
|
|
43
|
+
accountId: string;
|
|
44
|
+
workspaceId: string;
|
|
45
|
+
sandboxGroupId: string;
|
|
46
|
+
/** The attributing session within the group (holders carry session_id for
|
|
47
|
+
* disclosure/attribution). For a singleton group this == sandboxGroupId. */
|
|
48
|
+
sessionId: string;
|
|
49
|
+
/** The backend the box runs on (sessions.sandbox_backend). */
|
|
50
|
+
backend: string;
|
|
51
|
+
/** The OS axis (sessions.sandbox_os); default 'linux'. */
|
|
52
|
+
os?: string;
|
|
53
|
+
/**
|
|
54
|
+
* The FULL environment the agent will declare for this run (the SAME object
|
|
55
|
+
* passed to runtime.buildAgent's `sandboxEnvironment`). The box's manifest is
|
|
56
|
+
* created with this environment so that when the SDK applies the agent's
|
|
57
|
+
* manifest to this NON-OWNED provided session, the environments match exactly
|
|
58
|
+
* and `validateNoEnvironmentDelta` finds an empty delta (otherwise it throws
|
|
59
|
+
* "Live sandbox sessions cannot change manifest environment variables" and the
|
|
60
|
+
* turn dies). Omitted → the leaf falls back to collectSandboxEnvironment(settings)
|
|
61
|
+
* (the legacy default; only the resume/spawn-without-an-agent callers rely on it).
|
|
62
|
+
*/
|
|
63
|
+
environment?: Record<string, string>;
|
|
64
|
+
/**
|
|
65
|
+
* IMAGE IS SHARED STATE (B3): the container image this run resolves (Modal image ref
|
|
66
|
+
* / docker image). Threaded to acquireLease, which stamps it on the cold-create and
|
|
67
|
+
* conflicts on a live box already running a DIFFERENT image (a solo holder requests
|
|
68
|
+
* capture-and-drain rotation; N-holders throw SandboxImageConflictError). Omitted ->
|
|
69
|
+
* image is not enforced (the selfhosted path never passes it; a legacy/null-image box
|
|
70
|
+
* never conflicts).
|
|
71
|
+
*/
|
|
72
|
+
image?: string;
|
|
73
|
+
/**
|
|
74
|
+
* RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
|
|
75
|
+
* acquireLease, which stamps it on the cold-create and conflicts on a live box
|
|
76
|
+
* set up under a DIFFERENT rig version (a solo holder requests capture-and-drain
|
|
77
|
+
* rotation; N-holders throw SandboxRigConflictError). Omitted for a rig-less
|
|
78
|
+
* session -> rig is never stamped or enforced (shares exactly as today).
|
|
79
|
+
*/
|
|
80
|
+
rigVersionId?: string;
|
|
81
|
+
};
|
|
82
|
+
/** What resumeBoxForTurn returns: the live NON-OWNED session to inject, the
|
|
83
|
+
* fence token (lease_epoch) it was established under, and a release function
|
|
84
|
+
* the caller invokes in `finally` (idempotent delete-my-holder-row). */
|
|
85
|
+
export type ResumedTurnSandbox = {
|
|
86
|
+
/** The live, externally-owned session — inject {client, session, sessionState}
|
|
87
|
+
* NON-OWNED into runStream's `ownedSandbox`; the SDK never reaps it. */
|
|
88
|
+
established: EstablishedSandboxSession;
|
|
89
|
+
/** The lease_epoch this turn holds; the heartbeat/fence token. */
|
|
90
|
+
leaseEpoch: number;
|
|
91
|
+
/** Idempotent release: deletes this holder row and (if refcount hits 0 with no
|
|
92
|
+
* turn holders) CASes warm->draining. NEVER stops the box. Safe to call once. */
|
|
93
|
+
release: () => Promise<void>;
|
|
94
|
+
};
|
|
95
|
+
export declare class SandboxWarmingTimeoutError extends Error {
|
|
96
|
+
readonly backend: string;
|
|
97
|
+
readonly timeoutMs: number;
|
|
98
|
+
readonly code = "sandbox_warming_timeout";
|
|
99
|
+
constructor(backend: string, timeoutMs: number);
|
|
100
|
+
}
|
|
101
|
+
/** The exact attached caller that won warm->cold after proving the provider
|
|
102
|
+
* instance gone. It is a lease supersession (the same logical turn recovers),
|
|
103
|
+
* plus the lost id needed for one durable observability event. */
|
|
104
|
+
export declare class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
|
|
105
|
+
readonly lostInstanceId: string;
|
|
106
|
+
constructor(sandboxGroupId: string, leaseEpoch: number, lostInstanceId: string);
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Modal may return a sandbox handle before its command router accepts the first
|
|
110
|
+
* exec. The upstream session's yieldTimeMs starts only after sandbox.exec()
|
|
111
|
+
* returns, so it cannot bound that initial RPC. Probe it before publishing the
|
|
112
|
+
* lease as warm and enforce our own wall-clock deadline.
|
|
113
|
+
*/
|
|
114
|
+
export declare function waitForSandboxExecReadiness(established: EstablishedSandboxSession, timeoutMs?: number): Promise<void>;
|
|
115
|
+
export declare function waitForWarmSnapshot(snapshot: Promise<unknown>, timeoutMs: number, signal?: AbortSignal): Promise<boolean>;
|
|
116
|
+
/**
|
|
117
|
+
* MID-SESSION /workspace snapshot (sandbox-file-persistence). The reaper's
|
|
118
|
+
* drain-persist only protects boxes the reaper itself kills; anything else —
|
|
119
|
+
* Modal's hard creation-time timeout catching a session busy past it, provider
|
|
120
|
+
* OOM/infra death — loses everything since the last clean drain (staging
|
|
121
|
+
* session e644e8a8, 2026-07-06: mid-turn box termination cost an unpushed
|
|
122
|
+
* branch + 2 commits). While a turn HOLDS the live box, this folds a fresh
|
|
123
|
+
* snapshot onto the lease through persistWarmSnapshot (the warm sibling of the
|
|
124
|
+
* drain seam: epoch-fenced CAS + atomic throttle re-check) and GCs the
|
|
125
|
+
* superseded snapshot image, bounding worst-case loss of ANY unclean box death
|
|
126
|
+
* to sandboxSnapshotIntervalMs.
|
|
127
|
+
*
|
|
128
|
+
* Never throws and never blocks turn progress semantics: every failure path
|
|
129
|
+
* returns false (the snapshot is protection, not a turn dependency). No-ops
|
|
130
|
+
* when the interval is 0, the backend has no persistWorkspace (selfhosted =
|
|
131
|
+
* the user's machine IS the persistence), or a snapshot newer than the
|
|
132
|
+
* interval already rides the lease.
|
|
133
|
+
*/
|
|
134
|
+
export declare function maybePersistWarmWorkspaceSnapshot(services: SandboxResumeServices, ids: {
|
|
135
|
+
accountId: string;
|
|
136
|
+
workspaceId: string;
|
|
137
|
+
sessionId: string;
|
|
138
|
+
turnId: string;
|
|
139
|
+
attemptId: string;
|
|
140
|
+
sandboxGroupId: string;
|
|
141
|
+
}, session: unknown, leaseEpoch: number, signal?: AbortSignal, force?: boolean): Promise<boolean>;
|
|
142
|
+
/**
|
|
143
|
+
* Resume the one box for a single turn (or any worker-side resume op). Returns
|
|
144
|
+
* the live non-owned session + the fence epoch + a release fn. The CALLER owns
|
|
145
|
+
* the lifecycle: inject non-owned, run, then `await release()` and drop the
|
|
146
|
+
* handle in `finally`.
|
|
147
|
+
*
|
|
148
|
+
* holderId is the globally unique durable turn-attempt id. It must not be a
|
|
149
|
+
* Temporal activity id, because activity ids are only workflow-local and
|
|
150
|
+
* collide when sibling sessions share one sandbox group.
|
|
151
|
+
*/
|
|
152
|
+
export declare function resumeBoxForTurn(services: SandboxResumeServices, ids: ResumeBoxIds, kind: "turn", holderId: TurnSandboxLeaseHolderId): Promise<ResumedTurnSandbox>;
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { type Settings } from "@opengeni/config";
|
|
2
|
+
import { type Database } from "@opengeni/db";
|
|
3
|
+
import type { EventBus } from "@opengeni/events";
|
|
4
|
+
import { type EstablishedSandboxSession, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
|
|
5
|
+
export type RoutingWiringServices = {
|
|
6
|
+
db: Database;
|
|
7
|
+
settings: Settings;
|
|
8
|
+
/** The events bus, for the selfhosted control-plane request/reply connection.
|
|
9
|
+
* Optional: when absent (or NATS unconfigured) a selfhosted swap target
|
|
10
|
+
* surfaces agent_offline on its first op rather than failing to build. */
|
|
11
|
+
bus?: EventBus;
|
|
12
|
+
/** The per-op observer wired into every selfhosted session this turn builds
|
|
13
|
+
* (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
|
|
14
|
+
onOp?: SelfhostedOpObserver;
|
|
15
|
+
/** The op-stream durable-resume journal (the Temporal adaptation from
|
|
16
|
+
* op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
|
|
17
|
+
* the runtime defaults (generation "1", no persistence) — tests / non-turn
|
|
18
|
+
* callers. Only consulted when op-stream is actually enabled for the turn. */
|
|
19
|
+
opJournal?: OpStreamJournal;
|
|
20
|
+
/** Durable lifecycle notification emitted only by the observer that wins the
|
|
21
|
+
* exact warm-instance loss CAS. */
|
|
22
|
+
onHomeSandboxLost?: (input: {
|
|
23
|
+
sandboxGroupId: string;
|
|
24
|
+
instanceId: string;
|
|
25
|
+
leaseEpoch: number;
|
|
26
|
+
}) => Promise<void>;
|
|
27
|
+
/** Called when a route repair replaces the worker's original home handle. */
|
|
28
|
+
onHomeSandboxRebound?: (input: {
|
|
29
|
+
established: EstablishedSandboxSession;
|
|
30
|
+
leaseEpoch: number;
|
|
31
|
+
}) => void;
|
|
32
|
+
};
|
|
33
|
+
export type RoutingWiringIds = {
|
|
34
|
+
workspaceId: string;
|
|
35
|
+
sessionId: string;
|
|
36
|
+
/** Canonical turn-attempt fence used to admit persistable-home mutations.
|
|
37
|
+
* Omit outside a running worker attempt; mutation hooks then remain disabled. */
|
|
38
|
+
workspaceMutationFence?: {
|
|
39
|
+
accountId: string;
|
|
40
|
+
turnId: string;
|
|
41
|
+
executionGeneration: number;
|
|
42
|
+
attemptId: string;
|
|
43
|
+
};
|
|
44
|
+
homeLease?: {
|
|
45
|
+
accountId: string;
|
|
46
|
+
sandboxGroupId: string;
|
|
47
|
+
leaseEpoch: number;
|
|
48
|
+
instanceId: string;
|
|
49
|
+
backend: string;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* The run's declared sandbox environment — the SAME object the turn passes to
|
|
53
|
+
* `runtime.buildAgent`'s `sandboxEnvironment` and to `resumeBoxForTurn` (so the
|
|
54
|
+
* group box's manifest carries it too). Threaded into a selfhosted swap target's
|
|
55
|
+
* manifest so its `environment` EQUALS the turn's, making the SDK's per-turn
|
|
56
|
+
* provided-session manifest-env delta empty (validateNoEnvironmentDelta).
|
|
57
|
+
* WITHOUT this a pin-to-vm turn throws "Live sandbox sessions cannot change
|
|
58
|
+
* manifest environment variables." Optional → the resolver defaults to `{}`.
|
|
59
|
+
*/
|
|
60
|
+
environment?: Record<string, string>;
|
|
61
|
+
/**
|
|
62
|
+
* Stage D machine-primary: PIN the already-established turn SelfhostedSession
|
|
63
|
+
* (the `established` arg's session) for THIS machine pointer `(sandboxId, epoch)`
|
|
64
|
+
* so the per-op resolver returns that SAME instance instead of building a fresh
|
|
65
|
+
* one — the turn-start manifest write + per-op reads then hit ONE
|
|
66
|
+
* SelfhostedSession/manifest. Set ONLY by the machine-primary establish branch
|
|
67
|
+
* (where `established.session` is the SelfhostedSession bound to this pointer);
|
|
68
|
+
* the group-box/swap path omits it (the default is the modal group box, and a
|
|
69
|
+
* swap target is built fresh).
|
|
70
|
+
*/
|
|
71
|
+
pinnedSelfhosted?: {
|
|
72
|
+
sandboxId: string;
|
|
73
|
+
epoch: number;
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Whether the turn's `defaultBackend` IS the session's home (so the null pointer may
|
|
77
|
+
* resolve to it). Defaults to TRUE (omitted). Set explicitly FALSE on a machine-primary
|
|
78
|
+
* turn of a Modal-HOME session (pinned to a machine, no Modal group box established this
|
|
79
|
+
* turn): the routing resolver's null branch then throws a typed `home_unavailable_this_turn`
|
|
80
|
+
* error on a mid-turn clear-to-null instead of silently serving the pinned machine — the
|
|
81
|
+
* detach's pointer commit stands and takes effect next turn. A genuine machine-HOME turn
|
|
82
|
+
* (home IS the machine) passes true.
|
|
83
|
+
*/
|
|
84
|
+
defaultIsHome?: boolean;
|
|
85
|
+
};
|
|
86
|
+
/** Map the deployment relay URL to the leaf's `SelfhostedRelayConfig` shape
|
|
87
|
+
* (host/port/tls). M8 wires the real relay; until then a configured/placeholder
|
|
88
|
+
* host yields a well-formed stream-URL shape behind `resolveExposedPort`. */
|
|
89
|
+
export declare function relayConfigFromSettings(settings: Settings): SelfhostedRelayConfig;
|
|
90
|
+
/** The selfhosted CONTROL vs EXEC op deadlines for a turn, from settings. Control
|
|
91
|
+
* ops (ping/fs/desktop/pty) stay on the short timeout so machine liveness is never
|
|
92
|
+
* masked by a slow op; exec gets its own much larger budget so a real command is not
|
|
93
|
+
* killed at the control wall. Threaded into every turn-path session build + resolver. */
|
|
94
|
+
export declare function selfhostedTimeoutsFromSettings(settings: Settings): {
|
|
95
|
+
timeoutMs: number;
|
|
96
|
+
execTimeoutMs: number;
|
|
97
|
+
};
|
|
98
|
+
export declare function providerIdentityFromResumeState(resumeState: Record<string, unknown>): string | null;
|
|
99
|
+
/**
|
|
100
|
+
* Wrap an established group-box session in a `RoutingSandboxSession` so a mid-turn
|
|
101
|
+
* swap routes the NEXT tool call to the new active sandbox. Returns the SAME
|
|
102
|
+
* established handle with its `session` replaced by the stable proxy; the
|
|
103
|
+
* client/sessionState/instanceId/backendId are preserved (the lease still owns
|
|
104
|
+
* the group box's lifecycle — the proxy is a routing veneer, not an owner).
|
|
105
|
+
*
|
|
106
|
+
* The DEFAULT pointer (active_sandbox_id == null) routes to the established group
|
|
107
|
+
* session unchanged (backward-compat). A swap to a selfhosted machine routes to a
|
|
108
|
+
* SelfhostedSession bound to the target's enrollment agentId, fenced under the
|
|
109
|
+
* swap's active_epoch.
|
|
110
|
+
*/
|
|
111
|
+
export declare function wrapTurnBoxWithRouting(services: RoutingWiringServices, ids: RoutingWiringIds, established: EstablishedSandboxSession): EstablishedSandboxSession;
|
|
112
|
+
export declare function wrapLazyTurnBoxWithRouting(services: RoutingWiringServices, ids: RoutingWiringIds, args: {
|
|
113
|
+
client: EstablishedSandboxSession["client"];
|
|
114
|
+
backendId: string;
|
|
115
|
+
agentDefaultManifest: unknown;
|
|
116
|
+
provisioner: {
|
|
117
|
+
get(): Promise<{
|
|
118
|
+
established: EstablishedSandboxSession;
|
|
119
|
+
leaseEpoch?: number;
|
|
120
|
+
}>;
|
|
121
|
+
};
|
|
122
|
+
homeLeaseIdentity?: {
|
|
123
|
+
accountId: string;
|
|
124
|
+
sandboxGroupId: string;
|
|
125
|
+
backend: string;
|
|
126
|
+
};
|
|
127
|
+
}): EstablishedSandboxSession;
|
|
128
|
+
export type SelfhostedTurnSessionArgs = {
|
|
129
|
+
workspaceId: string;
|
|
130
|
+
/** The target machine's enrollment id == the agent subject id. */
|
|
131
|
+
agentId: string;
|
|
132
|
+
/** Whether the target machine advertised Capabilities.op_stream in its latest
|
|
133
|
+
* Hello. The runtime-side transport gate must still require the server flag. */
|
|
134
|
+
opStream: boolean;
|
|
135
|
+
/** The active pointer's epoch — the control-op fence echoed to the agent. */
|
|
136
|
+
epoch: number;
|
|
137
|
+
/** The run's declared sandbox environment (the SAME object fed to buildAgent +
|
|
138
|
+
* the manifest), threaded so the SDK's per-turn provided-session env delta is
|
|
139
|
+
* empty. */
|
|
140
|
+
environment: Record<string, string>;
|
|
141
|
+
/** The session working directory (per-session pointer). Null ⇒ workspace_root. */
|
|
142
|
+
workingDir: string | null;
|
|
143
|
+
};
|
|
144
|
+
type LegacySelfhostedTurnSessionArgs = Omit<SelfhostedTurnSessionArgs, "opStream">;
|
|
145
|
+
export declare function establishSelfhostedTurnSession(services: RoutingWiringServices, args: SelfhostedTurnSessionArgs | LegacySelfhostedTurnSessionArgs): Promise<EstablishedSandboxSession>;
|
|
146
|
+
/** Whether the routing proxy should wrap the turn box: the hot-swap feature is
|
|
147
|
+
* gated by the selfhosted flag (the active pointer + swap tools are only
|
|
148
|
+
* meaningful when selfhosted is enabled). With the flag off the established
|
|
149
|
+
* group box is injected unchanged — byte-for-byte today. */
|
|
150
|
+
export declare function routingEnabled(settings: Settings): boolean;
|
|
151
|
+
/** Whether the turn should defer sandbox provisioning to the first dispatched op
|
|
152
|
+
* (the in-process single-flight provisioner behind the routing proxy's
|
|
153
|
+
* resolveActiveBackend). Lazy is a property of the OWNED path only — the SDK never
|
|
154
|
+
* creates/resumes an injected session, so we own establish timing — hence gated on
|
|
155
|
+
* BOTH flags. With either off the turn provisions eagerly at turn start, exactly as
|
|
156
|
+
* today. NB: under lazy the box is ALWAYS wrapped in the routing proxy (the proxy's
|
|
157
|
+
* resolver IS the establish seam), independent of `routingEnabled`. */
|
|
158
|
+
export declare function lazyProvisionEnabled(settings: Settings): boolean;
|
|
159
|
+
export {};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
import type { WorkerLifecycleState } from "./http";
|
|
3
|
+
export declare function constructWithOwnedConnection<Connection, Result>(connect: () => Promise<Connection>, construct: (connection: Connection) => Promise<Result>, close: (connection: Connection) => Promise<void>): Promise<Result>;
|
|
4
|
+
export type WorkerRunTarget = {
|
|
5
|
+
run(): Promise<void>;
|
|
6
|
+
shutdown(): void;
|
|
7
|
+
};
|
|
8
|
+
export type WorkerServiceLifecycle = {
|
|
9
|
+
state(): WorkerLifecycleState;
|
|
10
|
+
run(): Promise<void>;
|
|
11
|
+
drain(reason?: string): void;
|
|
12
|
+
close(): Promise<void>;
|
|
13
|
+
};
|
|
14
|
+
export declare function createWorkerServiceLifecycle(input: {
|
|
15
|
+
role: "control" | "turn";
|
|
16
|
+
worker: WorkerRunTarget;
|
|
17
|
+
observability: Observability;
|
|
18
|
+
closeOwnedResources: () => Promise<void>;
|
|
19
|
+
onReady?: () => void;
|
|
20
|
+
}): WorkerServiceLifecycle;
|