@opengeni/worker-bundle 0.12.7 → 0.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +778 -0
- package/dist/activities/capabilities.d.ts +18 -0
- package/dist/activities/codex-auth.d.ts +1 -0
- package/dist/activities/codex-capacity.d.ts +31 -0
- package/dist/activities/codex-fleet-shadow.d.ts +74 -0
- package/dist/activities/codex-rotation.d.ts +261 -0
- package/dist/activities/common.d.ts +17 -0
- package/dist/activities/context-compaction.d.ts +81 -0
- package/dist/activities/documents.d.ts +39 -0
- package/dist/activities/environment.d.ts +97 -0
- package/dist/activities/expiring-material-renewal.d.ts +38 -0
- package/dist/activities/file-upload-reaper.d.ts +25 -0
- package/dist/activities/git-credential-renewal.d.ts +41 -0
- package/dist/activities/goals.d.ts +16 -0
- package/dist/activities/mcp-credentials.d.ts +14 -0
- package/dist/activities/packs.d.ts +77 -0
- package/dist/activities/parent-wake.d.ts +54 -0
- package/dist/activities/recording.d.ts +90 -0
- package/dist/activities/redaction.d.ts +1 -0
- package/dist/activities/rig-verification.d.ts +131 -0
- package/dist/activities/run-credential-renewal.d.ts +28 -0
- package/dist/activities/run-credentials.d.ts +36 -0
- package/dist/activities/run-input.d.ts +106 -0
- package/dist/activities/sandbox-lease.d.ts +128 -0
- package/dist/activities/scheduled-tasks.d.ts +4 -0
- package/dist/activities/session-state.d.ts +39 -0
- package/dist/activities/streaming.d.ts +24 -0
- package/dist/activities/toolspace-token-renewal.d.ts +23 -0
- package/dist/activities/turn-attempt-fenced.d.ts +11 -0
- package/dist/activities/types.d.ts +240 -0
- package/dist/activities/workflow-wake.d.ts +15 -0
- package/dist/activities/workspace-capture.d.ts +105 -0
- package/dist/activities.d.ts +312 -0
- package/dist/concurrency.d.ts +19 -0
- package/dist/host-export-pump.d.ts +66 -0
- package/dist/host-export.d.ts +6 -67
- package/dist/http.d.ts +32 -0
- package/dist/index.d.ts +27 -388
- package/dist/index.js +2730 -1211
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +200 -0
- package/dist/op-journal.d.ts +10 -0
- package/dist/sandbox-resume.d.ts +152 -0
- package/dist/sandbox-routing.d.ts +159 -0
- package/dist/worker-service-lifecycle.d.ts +20 -0
- package/dist/workflow-bundle.js +88 -20
- package/dist/workflows/activities.d.ts +27 -0
- package/dist/workflows/document-index.d.ts +41 -0
- package/dist/workflows/file-upload-reaper.d.ts +2 -0
- package/dist/workflows/rig-verification.d.ts +3 -0
- package/dist/workflows/sandbox-reaper.d.ts +1 -0
- package/dist/workflows/scheduled-tasks.d.ts +17 -0
- package/dist/workflows/session.d.ts +58 -0
- package/dist/workflows/workflow-wake.d.ts +3 -0
- package/dist/workflows.d.ts +7 -0
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +874 -134
- package/src/activities/context-compaction.ts +265 -76
- package/src/activities/environment.ts +1 -0
- package/src/activities/goals.ts +14 -0
- package/src/activities/rig-verification.ts +2 -0
- package/src/activities/run-input.ts +18 -4
- package/src/activities/sandbox-lease.ts +787 -133
- package/src/activities/session-state.ts +56 -0
- package/src/activities/types.ts +21 -0
- package/src/activities/workspace-capture.ts +95 -4
- package/src/activities.ts +3 -0
- package/src/observability-metrics.ts +94 -0
- package/src/sandbox-resume.ts +383 -90
- package/src/sandbox-routing.ts +32 -10
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +70 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { FileAsset, ResourceRef } from "@opengeni/contracts";
|
|
2
|
+
import { getSessionEvent, type Database } from "@opengeni/db";
|
|
3
|
+
import { type OpenGeniRuntime } from "@opengeni/runtime";
|
|
4
|
+
/**
|
|
5
|
+
* The codex account THIS turn runs on, threaded into every history read path so a
|
|
6
|
+
* cross-account turn never replays another account's encrypted reasoning. The
|
|
7
|
+
* single rule across all paths: DROP any reasoning item whose producing codex
|
|
8
|
+
* account differs from `currentCodexCredentialId`.
|
|
9
|
+
*
|
|
10
|
+
* `currentCodexCredentialId` is the resolved codex credential id on a codex turn,
|
|
11
|
+
* or NULL on a non-codex turn (the "account" of the built-in Azure/OpenAI path).
|
|
12
|
+
* NULL is a real value in the comparison, not a "skip" sentinel: a non-codex turn
|
|
13
|
+
* (current = null) still drops codex-produced reasoning (producer != null) so a
|
|
14
|
+
* foreign encrypted blob never reaches the Azure/built-in Responses call. A
|
|
15
|
+
* session with no codex history (every producer == null == current) is a no-op.
|
|
16
|
+
*/
|
|
17
|
+
export type TurnCodexAccount = {
|
|
18
|
+
currentCodexCredentialId: string | null;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Apply the cross-account reasoning strip to a set of stored history rows. Pure +
|
|
22
|
+
* non-mutating. The single rule: a row whose producing codex account EQUALS the
|
|
23
|
+
* turn's current account replays verbatim (by reference); a row produced by a
|
|
24
|
+
* DIFFERENT account is treated by item type —
|
|
25
|
+
*
|
|
26
|
+
* - `reasoning` → DROPPED WHOLE (id + blob filtered out of the history). The
|
|
27
|
+
* foreign `rs_…` id is validated by the Responses backend, which rejects a
|
|
28
|
+
* reasoning item that has a foreign id and no encrypted_content (store:false),
|
|
29
|
+
* so blanking only the blob is not enough — the whole item must go.
|
|
30
|
+
* - `compaction` → kept, with only its account-bound `encrypted_content` blob
|
|
31
|
+
* stripped (its summary is real conversation content that must survive).
|
|
32
|
+
* - everything else (messages, tool calls, tool outputs) → kept verbatim by
|
|
33
|
+
* reference; message and tool content are never account-bound, never touched.
|
|
34
|
+
*
|
|
35
|
+
* Mismatch covers a foreign codex account, the non-codex/Azure producer (null on
|
|
36
|
+
* a codex turn), and legacy untagged rows (null): all are stripped, which is
|
|
37
|
+
* defensive and harmless (at most one turn of lost chain-of-thought continuity,
|
|
38
|
+
* never any content). No-op (rows by reference) when every producer equals the
|
|
39
|
+
* current account — a single-account workspace, an unchanged-account turn, or a
|
|
40
|
+
* non-codex turn over a history with no codex-produced reasoning.
|
|
41
|
+
*/
|
|
42
|
+
export declare function applyCodexHistoryStrip(rows: ReadonlyArray<{
|
|
43
|
+
item: Record<string, unknown>;
|
|
44
|
+
producerCodexCredentialId: string | null;
|
|
45
|
+
providerArtifactInvalidatedAt?: Date | null;
|
|
46
|
+
}>, current: TurnCodexAccount): Array<Record<string, unknown>>;
|
|
47
|
+
/**
|
|
48
|
+
* Resolve the serialized RunState used only for an approval resume, applying
|
|
49
|
+
* the same cross-account rule as the canonical history-items path. The blob
|
|
50
|
+
* carries no per-item producer tag, so we
|
|
51
|
+
* compare the codex account that FROZE the state to the resuming turn's account:
|
|
52
|
+
* when they differ, neutralize every reasoning item's account-bound identity
|
|
53
|
+
* (encrypted_content + provider id) in the blob; when they match (including
|
|
54
|
+
* null == null for non-codex / single-account) the blob replays byte-for-byte
|
|
55
|
+
* (same string reference). This closes the gap where a frozen A-minted RunState
|
|
56
|
+
* was replayed verbatim into a turn that switched to account B (or to a non-codex
|
|
57
|
+
* turn), 400ing the resume.
|
|
58
|
+
*/
|
|
59
|
+
export declare function resumeRunStateForCodexAccount(state: {
|
|
60
|
+
serializedRunState: string;
|
|
61
|
+
frozenCodexCredentialId: string | null;
|
|
62
|
+
providerArtifactInvalidatedAt?: Date | null;
|
|
63
|
+
}, current: TurnCodexAccount): string;
|
|
64
|
+
/**
|
|
65
|
+
* A prepared turn input plus the watermark-seed discriminator the reconcile pass
|
|
66
|
+
* needs (HOLE E). `modelHistoryFromItems` is TRUE iff `state.history` was seeded
|
|
67
|
+
* from the cross-account-STRIPPED active history items (the items read path) — so
|
|
68
|
+
* the turn-end reconcile must seed `persistedHistoryCount` from the SAME strip
|
|
69
|
+
* (HOLE D). It is FALSE only when `state.history` was seeded from the approval
|
|
70
|
+
* RunState: there
|
|
71
|
+
* foreign reasoning is NEUTRALIZED-IN-PLACE by {@link resumeRunStateForCodexAccount}
|
|
72
|
+
* (the item is KEPT, only its id/encrypted_content go), so the blob's history
|
|
73
|
+
* length still COUNTS those items. Seeding the watermark with the strip on that
|
|
74
|
+
* path under-counts by K and the reconcile re-appends K already-persisted items at
|
|
75
|
+
* fresh positions — that is HOLE E. The watermark must therefore NOT strip on the
|
|
76
|
+
* blob path (count the raw sanitized active length, matching the blob).
|
|
77
|
+
*/
|
|
78
|
+
export type PreparedTurnInput = {
|
|
79
|
+
input: Awaited<ReturnType<OpenGeniRuntime["prepareInput"]>>;
|
|
80
|
+
modelHistoryFromItems: boolean;
|
|
81
|
+
};
|
|
82
|
+
export type TurnInputOptions = {
|
|
83
|
+
turnId: string;
|
|
84
|
+
recovering?: boolean;
|
|
85
|
+
unavailableSandboxFilesNote?: string;
|
|
86
|
+
runCredentialsNote?: string;
|
|
87
|
+
readFileBytesForModel?: (file: FileAsset) => Promise<Uint8Array>;
|
|
88
|
+
};
|
|
89
|
+
export declare const MAX_INLINE_MODEL_ATTACHMENT_BYTES: number;
|
|
90
|
+
export type ModelAttachmentContent = {
|
|
91
|
+
kind: "image" | "file";
|
|
92
|
+
fileId: string;
|
|
93
|
+
filename: string;
|
|
94
|
+
contentType: string;
|
|
95
|
+
dataUrl: string;
|
|
96
|
+
};
|
|
97
|
+
export declare function modelAttachmentContentForFiles(files: FileAsset[], readFileBytes: (file: FileAsset) => Promise<Uint8Array>): Promise<ModelAttachmentContent[]>;
|
|
98
|
+
/**
|
|
99
|
+
* Enrich the current turn's durable user boundary for this model attempt only.
|
|
100
|
+
* The item count stays unchanged, so the turn reconciler still treats the
|
|
101
|
+
* enriched row as the already-persisted prefix and never writes inline bytes to
|
|
102
|
+
* session_history_items. Recovery rebuilds the same projection from the trigger.
|
|
103
|
+
*/
|
|
104
|
+
export declare function withCurrentUserAttachmentContent(historyItems: Array<Record<string, unknown>>, attachments: ModelAttachmentContent[]): Array<Record<string, unknown>>;
|
|
105
|
+
export declare function turnInput(db: Database, runtime: OpenGeniRuntime, agent: any, trigger: Awaited<ReturnType<typeof getSessionEvent>>, current: TurnCodexAccount | undefined, options: TurnInputOptions): Promise<PreparedTurnInput>;
|
|
106
|
+
export declare function userMessageTextWithAttachments(db: Database, workspaceId: string, text: string, resources: ResourceRef[]): Promise<string>;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { listLiveModalSandboxLeaseAttributions, readLease, type RetainedProcessProviderProof, type SandboxRetainedProcess, type LeaseSnapshot } from "@opengeni/db";
|
|
2
|
+
import { createSandboxClientForBackend, inspectModalSandboxLifecycle, resolveModalCheckpointProviderBindingForSession, type ModalOrphanSweepTermination, type ModalCheckpointProviderBinding, type WorkspaceArchiveDescriptor } from "@opengeni/runtime";
|
|
3
|
+
import type { ActivityServices } from "./types";
|
|
4
|
+
export type ReapSandboxLeasesResult = {
|
|
5
|
+
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
6
|
+
* the DB pass; here we report the terminate outcomes. */
|
|
7
|
+
examined: number;
|
|
8
|
+
/** Boxes whose provider terminate fired AND whose lease went cold. */
|
|
9
|
+
terminated: number;
|
|
10
|
+
/** Drainable rows skipped because the CAS no longer held (re-armed / newer
|
|
11
|
+
* epoch / already drained by a concurrent sweep) — provider stop() did NOT
|
|
12
|
+
* fire. */
|
|
13
|
+
skipped: number;
|
|
14
|
+
/** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
|
|
15
|
+
metered: number;
|
|
16
|
+
/** Viewer-only boxes force-drained because their workspace is over a limit
|
|
17
|
+
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
18
|
+
forceDrained: number;
|
|
19
|
+
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
20
|
+
modalOrphansTerminated: number;
|
|
21
|
+
};
|
|
22
|
+
type CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;
|
|
23
|
+
/**
|
|
24
|
+
* The CAS-write the terminate seam calls to PERSIST the captured /workspace
|
|
25
|
+
* archive onto the lease under the epoch fence, BEFORE the box is terminated
|
|
26
|
+
* (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over
|
|
27
|
+
* db/accountId/row and delegates to persistDrainSnapshot. Returns:
|
|
28
|
+
* `wrote:false` means the CAS missed (re-armed / newer epoch / vanished). The
|
|
29
|
+
* box is wanted again, so the seam MUST NOT terminate it. Provider-native
|
|
30
|
+
* deletion is owned independently by artifact-ledger GC.
|
|
31
|
+
*
|
|
32
|
+
* Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for
|
|
33
|
+
* backends with no persistWorkspace — ensures a re-arm during the snapshot window
|
|
34
|
+
* aborts the terminate before client.delete()).
|
|
35
|
+
*/
|
|
36
|
+
export type PersistArchiveFn = (archiveBase64: string | null, archiveMetadata?: WorkspaceArchiveDescriptor, providerSession?: unknown, providerBinding?: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>> | null) => Promise<{
|
|
37
|
+
wrote: boolean;
|
|
38
|
+
archiveRevision?: string | null;
|
|
39
|
+
}>;
|
|
40
|
+
export type ProviderTerminationOutcome = {
|
|
41
|
+
/** false means an epoch/refcount CAS proved the box was re-armed, so it was
|
|
42
|
+
* deliberately left running and the cold commit must not execute. */
|
|
43
|
+
terminated: boolean;
|
|
44
|
+
/** True only when a definitive provider NotFound proved the cloud box had
|
|
45
|
+
* already vanished before this drain could capture its workspace. */
|
|
46
|
+
providerMissingBeforeCapture: boolean;
|
|
47
|
+
};
|
|
48
|
+
/** The provider-terminate seam. Production wires the real resume-by-id ->
|
|
49
|
+
* persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->
|
|
50
|
+
* provider stop() (`terminateProviderBox`); a unit test injects a spy so the
|
|
51
|
+
* drain/CAS logic is exercised against a real DB without a live provider box.
|
|
52
|
+
* Test seams may return the legacy boolean shorthand. Production returns the
|
|
53
|
+
* typed outcome so the cold commit can distinguish a clean stop from definitive
|
|
54
|
+
* provider disappearance before capture. */
|
|
55
|
+
export type TerminateBoxFn = (settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn) => Promise<boolean | ProviderTerminationOutcome>;
|
|
56
|
+
export type SweepModalOrphansFn = (settings: ActivityServices["settings"], db: ActivityServices["db"], observability: ActivityServices["observability"]) => Promise<number>;
|
|
57
|
+
export type SandboxLeaseActivityOptions = {
|
|
58
|
+
/** Override the provider terminate (tests spy this; defaults to the real
|
|
59
|
+
* resume-by-id + provider stop()). */
|
|
60
|
+
terminateBox?: TerminateBoxFn;
|
|
61
|
+
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
62
|
+
* Modal list+tag comparison when Modal is configured). */
|
|
63
|
+
sweepModalOrphans?: SweepModalOrphansFn;
|
|
64
|
+
/** Override only the read-only provider process probe. The canonical DB
|
|
65
|
+
* settlement remains real in tests and production. */
|
|
66
|
+
probeRetainedProcess?: RetainedProcessProbeFn;
|
|
67
|
+
/** Observe a historical Modal box after its lease row has advanced to a
|
|
68
|
+
* successor identity. Lifecycle-only; a live box remains ambiguous. */
|
|
69
|
+
inspectHistoricalModalSandbox?: HistoricalModalSandboxLifecycleProbeFn;
|
|
70
|
+
/** Override the read-only exact-instance readiness probe used before the
|
|
71
|
+
* reaper settles blockers for a provider that has definitively vanished. */
|
|
72
|
+
probeDrainableProvider?: DrainableProviderProbeFn;
|
|
73
|
+
};
|
|
74
|
+
export type RetainedProcessProbeResult = {
|
|
75
|
+
status: "proved";
|
|
76
|
+
proof: RetainedProcessProviderProof;
|
|
77
|
+
} | {
|
|
78
|
+
status: "binding";
|
|
79
|
+
providerBindingKey: string;
|
|
80
|
+
providerBinding: ModalCheckpointProviderBinding;
|
|
81
|
+
} | {
|
|
82
|
+
status: "deferred";
|
|
83
|
+
reason: "identity_mismatch" | "resume_state_missing" | "backend_unsupported" | "provider_running" | "provider_unknown" | "provider_timeout" | "provider_error" | "provider_binding_missing" | "provider_binding_mismatch";
|
|
84
|
+
};
|
|
85
|
+
export type RetainedProcessProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess) => Promise<RetainedProcessProbeResult>;
|
|
86
|
+
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
87
|
+
export type DrainableProviderProbeFn = (settings: ActivityServices["settings"], lease: LeaseSnapshot) => Promise<"ready" | "missing">;
|
|
88
|
+
export declare const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
89
|
+
export declare const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS: number;
|
|
90
|
+
export declare const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5000;
|
|
91
|
+
export declare function createSandboxLeaseActivities(services: () => Promise<ActivityServices>, options?: SandboxLeaseActivityOptions): {
|
|
92
|
+
reapSandboxLeases: () => Promise<ReapSandboxLeasesResult>;
|
|
93
|
+
};
|
|
94
|
+
export declare function probeRetainedProcessAtProvider(settings: ActivityServices["settings"], lease: LeaseSnapshot, process: SandboxRetainedProcess): Promise<RetainedProcessProbeResult>;
|
|
95
|
+
/** Classify only exact SDK control banners. Arbitrary output, malformed text,
|
|
96
|
+
* and a running marker are observations to retry; none is physical exit proof. */
|
|
97
|
+
export declare function classifyRetainedProcessPollResult(result: unknown, providerSessionId: number): RetainedProcessProbeResult;
|
|
98
|
+
export declare function modalOrphanTerminationStillEligible(latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>, candidate: ModalOrphanSweepTermination): boolean;
|
|
99
|
+
/**
|
|
100
|
+
* Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the
|
|
101
|
+
* lease under the epoch fence, hand the superseded snapshot to durable GC, THEN
|
|
102
|
+
* terminate. Resumes DIRECTLY (no cold-restore fallback) so a gone box is a clean
|
|
103
|
+
* no-op, never a wasteful create-then-kill:
|
|
104
|
+
* - build the client for the lease's backend;
|
|
105
|
+
* - deserialize + resume the envelope (warm reattach by id, R4-safe);
|
|
106
|
+
* - session.persistWorkspace() -> capture the /workspace archive;
|
|
107
|
+
* - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark
|
|
108
|
+
* the displaced artifact delete-pending;
|
|
109
|
+
* - the provider-bound global GC deletes that object asynchronously;
|
|
110
|
+
* - terminate the live handle (client.delete / session kill|terminate|close).
|
|
111
|
+
*
|
|
112
|
+
* Returns true when the box was terminated (or was already gone), false when the
|
|
113
|
+
* persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.
|
|
114
|
+
*
|
|
115
|
+
* Failure discipline (NEVER lose files, NEVER leak a box):
|
|
116
|
+
* - resume NotFound -> the box is already down; SUCCESS (return true), nothing
|
|
117
|
+
* to persist, the caller colds the lease.
|
|
118
|
+
* - a transient/auth/network resume failure on a box that may be ALIVE -> throw
|
|
119
|
+
* (the caller skips; the lease stays draining for the next sweep).
|
|
120
|
+
* - a persistWorkspace failure (snapshot timeout / provider error) -> throw
|
|
121
|
+
* BEFORE any terminate, so the box is NOT torn down with un-captured files;
|
|
122
|
+
* the next sweep retries, and the provider idle-timeout is the ultimate
|
|
123
|
+
* backstop. We never terminate a box whose snapshot we could not capture.
|
|
124
|
+
* For a draining lease with no envelope (a warming-death row that committed no
|
|
125
|
+
* box, or a 'none'-backed group) there is no live box — a no-op (return true).
|
|
126
|
+
*/
|
|
127
|
+
export declare function terminateProviderBox(settings: ActivityServices["settings"], lease: NonNullable<Awaited<ReturnType<typeof readLease>>>, observability: ActivityServices["observability"], persistArchive: PersistArchiveFn, createClientForBackend?: CreateSandboxClientForBackendFn): Promise<ProviderTerminationOutcome>;
|
|
128
|
+
export {};
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ActivityServices, DispatchScheduledTaskRunInput, DispatchScheduledTaskRunResult } from "./types";
|
|
2
|
+
export declare function createScheduledTaskActivities(services: () => Promise<ActivityServices>): {
|
|
3
|
+
dispatchScheduledTaskRun: (input: DispatchScheduledTaskRunInput) => Promise<DispatchScheduledTaskRunResult>;
|
|
4
|
+
};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { settleSessionAttemptInterruptions, applySessionTurnSettlement, requestSessionTurnRecovery, recoverSessionDispatch, reconcileSessionAttemptQuiescence, peekSessionWork as peekSessionWorkDb, countQueuedTurns, getSessionAttemptActivityRef, getSessionEvent, getSessionTurnForAttempt, expireSessionHumanInputRequest, markSessionAttemptQuiesced, requireSession, settleSessionIdleWithParentOutbox } from "@opengeni/db";
|
|
2
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
3
|
+
import { deliverFailedChildTurnToParent, notifyParentOfChildIdle } from "./parent-wake";
|
|
4
|
+
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
5
|
+
import type { ActivityServices, ExpireSessionHumanInputInput, ExpireSessionHumanInputResult, PeekSessionWorkInput, FailSessionAttemptInput, SettleSessionInterruptionsInput, MarkSessionIdleInput, PersistSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceInput, ReconcileSessionAttemptQuiescenceResult, RecoverDispatchInput, RecoverDispatchResult, RecoverEscapedMcpTimeoutInput, RecoverEscapedMcpTimeoutResult } from "./types";
|
|
6
|
+
export type SessionStateActivityOverrides = Partial<{
|
|
7
|
+
settleSessionAttemptInterruptions: typeof settleSessionAttemptInterruptions;
|
|
8
|
+
applySessionTurnSettlement: typeof applySessionTurnSettlement;
|
|
9
|
+
requestSessionTurnRecovery: typeof requestSessionTurnRecovery;
|
|
10
|
+
recoverSessionDispatch: typeof recoverSessionDispatch;
|
|
11
|
+
reconcileSessionAttemptQuiescence: typeof reconcileSessionAttemptQuiescence;
|
|
12
|
+
peekSessionWork: typeof peekSessionWorkDb;
|
|
13
|
+
countQueuedTurns: typeof countQueuedTurns;
|
|
14
|
+
getSessionAttemptActivityRef: typeof getSessionAttemptActivityRef;
|
|
15
|
+
getSessionEvent: typeof getSessionEvent;
|
|
16
|
+
getSessionTurnForAttempt: typeof getSessionTurnForAttempt;
|
|
17
|
+
expireSessionHumanInputRequest: typeof expireSessionHumanInputRequest;
|
|
18
|
+
requireSession: typeof requireSession;
|
|
19
|
+
settleSessionIdleWithParentOutbox: typeof settleSessionIdleWithParentOutbox;
|
|
20
|
+
markSessionAttemptQuiesced: typeof markSessionAttemptQuiesced;
|
|
21
|
+
publishDurableSessionEvents: typeof publishDurableSessionEvents;
|
|
22
|
+
deliverFailedChildTurnToParent: typeof deliverFailedChildTurnToParent;
|
|
23
|
+
notifyParentOfChildIdle: typeof notifyParentOfChildIdle;
|
|
24
|
+
recordTurnsQueuedGauge: typeof recordTurnsQueuedGauge;
|
|
25
|
+
}>;
|
|
26
|
+
export declare const WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
27
|
+
export declare function createSessionStateActivities(services: () => Promise<ActivityServices>, overrides?: SessionStateActivityOverrides): {
|
|
28
|
+
failSessionAttempt: (input: FailSessionAttemptInput) => Promise<void>;
|
|
29
|
+
settleSessionInterruptions: (input: SettleSessionInterruptionsInput) => Promise<{
|
|
30
|
+
action: "paused" | "continue" | "stale";
|
|
31
|
+
}>;
|
|
32
|
+
persistSessionAttemptQuiescence: (input: PersistSessionAttemptQuiescenceInput) => Promise<void>;
|
|
33
|
+
reconcileSessionAttemptQuiescence: (input: ReconcileSessionAttemptQuiescenceInput) => Promise<ReconcileSessionAttemptQuiescenceResult>;
|
|
34
|
+
recoverDispatch: (input: RecoverDispatchInput) => Promise<RecoverDispatchResult>;
|
|
35
|
+
recoverEscapedMcpTimeout: (input: RecoverEscapedMcpTimeoutInput) => Promise<RecoverEscapedMcpTimeoutResult>;
|
|
36
|
+
peekSessionWork: (input: PeekSessionWorkInput) => Promise<import("@opengeni/db").SessionWorkPeek>;
|
|
37
|
+
expireSessionHumanInput: (input: ExpireSessionHumanInputInput) => Promise<ExpireSessionHumanInputResult>;
|
|
38
|
+
markSessionIdle: (input: MarkSessionIdleInput) => Promise<void>;
|
|
39
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { SessionEventType } from "@opengeni/contracts";
|
|
2
|
+
import type { AppendEventInput } from "@opengeni/db";
|
|
3
|
+
import { Context } from "@temporalio/activity";
|
|
4
|
+
/** Optional metrics seam for the batcher. `onFlush` fires once per flush (on both
|
|
5
|
+
* success and failure) with the coalesced event count and the flush duration, so
|
|
6
|
+
* the worker can meter batch shape without the batcher knowing about Observability.
|
|
7
|
+
* `now` is injectable for deterministic tests. Both are optional; absent ⇒ no-op. */
|
|
8
|
+
export type RuntimeBatcherOptions = {
|
|
9
|
+
onFlush?: (info: {
|
|
10
|
+
events: number;
|
|
11
|
+
durationSeconds: number;
|
|
12
|
+
}) => void;
|
|
13
|
+
now?: () => number;
|
|
14
|
+
};
|
|
15
|
+
export declare function createRuntimeBatcher(flushEvents: (events: AppendEventInput[]) => Promise<void>, options?: RuntimeBatcherOptions): {
|
|
16
|
+
push: (event: {
|
|
17
|
+
type: SessionEventType;
|
|
18
|
+
payload: unknown;
|
|
19
|
+
}) => Promise<void>;
|
|
20
|
+
flush: () => Promise<void>;
|
|
21
|
+
};
|
|
22
|
+
export declare function currentActivityContext(): Context | null;
|
|
23
|
+
export declare function startActivityHeartbeat(context: Context | null, details: Record<string, unknown>): ReturnType<typeof setInterval> | null;
|
|
24
|
+
export declare function nextStreamEvent<T>(iterator: AsyncIterator<T>, context: Context | null): Promise<IteratorResult<T>>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type ExpiringMaterialRenewalController } from "./expiring-material-renewal";
|
|
2
|
+
import type { MintedSandboxToolspaceToken } from "./environment";
|
|
3
|
+
export declare const TOOLSPACE_TOKEN_DEFAULT_REFRESH_MS: number;
|
|
4
|
+
export declare const TOOLSPACE_TOKEN_EXPIRY_LEAD_MS: number;
|
|
5
|
+
export declare const TOOLSPACE_TOKEN_MIN_REFRESH_MS = 5000;
|
|
6
|
+
export declare const TOOLSPACE_TOKEN_MAX_RETRY_MS: number;
|
|
7
|
+
export type ToolspaceTokenRenewalController = ExpiringMaterialRenewalController;
|
|
8
|
+
export type ToolspaceTokenRenewalOptions = {
|
|
9
|
+
initialExpiresAt: Date;
|
|
10
|
+
mint: () => Promise<MintedSandboxToolspaceToken | undefined>;
|
|
11
|
+
write: (material: MintedSandboxToolspaceToken) => Promise<void>;
|
|
12
|
+
now?: () => number;
|
|
13
|
+
schedule?: (callback: () => void, delayMs: number) => unknown;
|
|
14
|
+
clearSchedule?: (timer: unknown) => void;
|
|
15
|
+
onSuccess?: (result: {
|
|
16
|
+
nextDelayMs: number;
|
|
17
|
+
}) => void;
|
|
18
|
+
onFailure?: (failure: {
|
|
19
|
+
retryDelayMs: number;
|
|
20
|
+
errorClass: string;
|
|
21
|
+
}) => void;
|
|
22
|
+
};
|
|
23
|
+
export declare function startToolspaceTokenRenewalLoop(options: ToolspaceTokenRenewalOptions): ToolspaceTokenRenewalController;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The activity lost ownership of its exact turn generation/attempt.
|
|
3
|
+
*
|
|
4
|
+
* This is an ordinary stale-writer outcome, not a Temporal cancellation. The
|
|
5
|
+
* current owner or control transaction already holds authoritative state, so
|
|
6
|
+
* the losing activity must stop locally without asking the workflow to settle
|
|
7
|
+
* or fail the turn again.
|
|
8
|
+
*/
|
|
9
|
+
export declare class TurnAttemptFencedError extends Error {
|
|
10
|
+
constructor(message: string);
|
|
11
|
+
}
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import type { ConnectionCredentialsPort, EntitlementsPort, ScheduledTaskTriggerType, TurnInitiator } from "@opengeni/contracts";
|
|
3
|
+
import type { Database } from "@opengeni/db";
|
|
4
|
+
import type { DocumentServices } from "@opengeni/documents";
|
|
5
|
+
import type { EventBus } from "@opengeni/events";
|
|
6
|
+
import type { Observability } from "@opengeni/observability";
|
|
7
|
+
import type { OpenGeniRuntime } from "@opengeni/runtime";
|
|
8
|
+
import type { ObjectStorage } from "@opengeni/storage";
|
|
9
|
+
export type WakeSessionWorkflowSignal = (input: {
|
|
10
|
+
accountId: string;
|
|
11
|
+
workspaceId: string;
|
|
12
|
+
sessionId: string;
|
|
13
|
+
workflowId: string;
|
|
14
|
+
wakeRevision: number;
|
|
15
|
+
interruptionRequested?: boolean;
|
|
16
|
+
}) => Promise<void>;
|
|
17
|
+
export type SignalCodexCapacityWorkflow = (input: {
|
|
18
|
+
accountId: string;
|
|
19
|
+
workspaceId: string;
|
|
20
|
+
sessionId: string;
|
|
21
|
+
workflowId: string;
|
|
22
|
+
wakeRevision: number;
|
|
23
|
+
}) => Promise<void>;
|
|
24
|
+
/** Exact activity-owned proof that the hard sandbox/tool fence physically
|
|
25
|
+
* drained. This is delivery evidence only: the workflow still validates the
|
|
26
|
+
* persisted attempt dispatch and commits the authoritative Postgres receipt. */
|
|
27
|
+
export type SessionAttemptQuiescenceProof = {
|
|
28
|
+
accountId: string;
|
|
29
|
+
workspaceId: string;
|
|
30
|
+
sessionId: string;
|
|
31
|
+
attemptId: string;
|
|
32
|
+
workflowId: string;
|
|
33
|
+
workflowRunId: string;
|
|
34
|
+
activityId: string;
|
|
35
|
+
};
|
|
36
|
+
export type SignalSessionAttemptQuiesced = (input: SessionAttemptQuiescenceProof) => Promise<void>;
|
|
37
|
+
export type InspectSessionAttemptActivity = (input: {
|
|
38
|
+
workflowId: string;
|
|
39
|
+
workflowRunId: string;
|
|
40
|
+
activityId: string;
|
|
41
|
+
}) => Promise<"pending" | "settled">;
|
|
42
|
+
export type ActivityServices = {
|
|
43
|
+
settings: Settings;
|
|
44
|
+
db: Database;
|
|
45
|
+
bus: EventBus;
|
|
46
|
+
runtime: OpenGeniRuntime;
|
|
47
|
+
objectStorage: ObjectStorage | null;
|
|
48
|
+
documentServices: DocumentServices;
|
|
49
|
+
observability: Observability;
|
|
50
|
+
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
51
|
+
/** Durable signalWithStart fallback used only after the activity's direct
|
|
52
|
+
* physical-quiescence receipt write exhausts its bounded DB retries. */
|
|
53
|
+
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced | null;
|
|
54
|
+
/** Server-authoritative Temporal activity lease inspection used only to
|
|
55
|
+
* recover a missing quiescence receipt after the original activity vanished. */
|
|
56
|
+
inspectSessionAttemptActivity: InspectSessionAttemptActivity | null;
|
|
57
|
+
/** Revision-carrying capacity nudge; generic outbox repair is also sufficient. */
|
|
58
|
+
signalCodexCapacityWorkflow?: SignalCodexCapacityWorkflow | null;
|
|
59
|
+
entitlements?: EntitlementsPort | null;
|
|
60
|
+
connectionCredentials?: ConnectionCredentialsPort | null;
|
|
61
|
+
};
|
|
62
|
+
export type CodexCapacityWaitRef = {
|
|
63
|
+
waiterId: string;
|
|
64
|
+
generation: number;
|
|
65
|
+
nextCheckAt: string;
|
|
66
|
+
wakeRevision: number;
|
|
67
|
+
};
|
|
68
|
+
export type GetCodexCapacityWaitInput = {
|
|
69
|
+
workspaceId: string;
|
|
70
|
+
sessionId: string;
|
|
71
|
+
};
|
|
72
|
+
export type ReconcileCodexCapacityWaitInput = {
|
|
73
|
+
accountId: string;
|
|
74
|
+
workspaceId: string;
|
|
75
|
+
sessionId: string;
|
|
76
|
+
waiterId: string;
|
|
77
|
+
generation: number;
|
|
78
|
+
cause: "timer" | "signal" | "queue" | "recovery";
|
|
79
|
+
};
|
|
80
|
+
export type ReconcileCodexCapacityWaitResult = ({
|
|
81
|
+
action: "waiting";
|
|
82
|
+
} & CodexCapacityWaitRef) | {
|
|
83
|
+
action: "resumed" | "paused" | "superseded" | "stale";
|
|
84
|
+
};
|
|
85
|
+
export type ActivityDependencies = Partial<ActivityServices>;
|
|
86
|
+
export type RunAgentTurnInput = {
|
|
87
|
+
accountId: string;
|
|
88
|
+
workspaceId: string;
|
|
89
|
+
sessionId: string;
|
|
90
|
+
workflowId: string;
|
|
91
|
+
workflowRunId: string;
|
|
92
|
+
attemptId: string;
|
|
93
|
+
trigger: {
|
|
94
|
+
kind: "next";
|
|
95
|
+
} | {
|
|
96
|
+
kind: "approval";
|
|
97
|
+
triggerEventId: string;
|
|
98
|
+
};
|
|
99
|
+
};
|
|
100
|
+
export type SettleSessionInterruptionsInput = {
|
|
101
|
+
accountId: string;
|
|
102
|
+
workspaceId: string;
|
|
103
|
+
sessionId: string;
|
|
104
|
+
attemptId: string;
|
|
105
|
+
workflowId: string;
|
|
106
|
+
/**
|
|
107
|
+
* Replay-only compatibility for session workflow histories created before
|
|
108
|
+
* the receipt-gated cancellation v2 patch. New histories never send this
|
|
109
|
+
* phase; the exact activity writes the authoritative receipt itself.
|
|
110
|
+
*/
|
|
111
|
+
phase?: "logical" | "attempt_quiesced";
|
|
112
|
+
};
|
|
113
|
+
export type PersistSessionAttemptQuiescenceInput = SessionAttemptQuiescenceProof;
|
|
114
|
+
export type ReconcileSessionAttemptQuiescenceInput = {
|
|
115
|
+
accountId: string;
|
|
116
|
+
workspaceId: string;
|
|
117
|
+
sessionId: string;
|
|
118
|
+
attemptId: string;
|
|
119
|
+
workflowId: string;
|
|
120
|
+
};
|
|
121
|
+
export type ReconcileSessionAttemptQuiescenceResult = {
|
|
122
|
+
action: "quiesced" | "pending" | "stale";
|
|
123
|
+
};
|
|
124
|
+
export type FailSessionAttemptInput = {
|
|
125
|
+
accountId: string;
|
|
126
|
+
workspaceId: string;
|
|
127
|
+
sessionId: string;
|
|
128
|
+
attemptId: string;
|
|
129
|
+
error?: string;
|
|
130
|
+
};
|
|
131
|
+
export type RecoverDispatchInput = {
|
|
132
|
+
accountId: string;
|
|
133
|
+
workspaceId: string;
|
|
134
|
+
sessionId: string;
|
|
135
|
+
attemptId: string;
|
|
136
|
+
timeoutType: "HEARTBEAT" | "SCHEDULE_TO_START";
|
|
137
|
+
};
|
|
138
|
+
export type RecoverDispatchResult = {
|
|
139
|
+
action: "unclaimed";
|
|
140
|
+
} | {
|
|
141
|
+
action: "recovering";
|
|
142
|
+
turnId: string;
|
|
143
|
+
redispatches: number;
|
|
144
|
+
} | {
|
|
145
|
+
action: "stale";
|
|
146
|
+
} | {
|
|
147
|
+
action: "exceeded";
|
|
148
|
+
turnId: string;
|
|
149
|
+
redispatches: number;
|
|
150
|
+
};
|
|
151
|
+
export declare const ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
|
|
152
|
+
export declare const ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
|
|
153
|
+
export type EscapedMcpTimeoutRecoveryDetail = {
|
|
154
|
+
turnId: string;
|
|
155
|
+
triggerEventId: string;
|
|
156
|
+
executionGeneration: number;
|
|
157
|
+
};
|
|
158
|
+
export type RecoverEscapedMcpTimeoutInput = EscapedMcpTimeoutRecoveryDetail & {
|
|
159
|
+
accountId: string;
|
|
160
|
+
workspaceId: string;
|
|
161
|
+
sessionId: string;
|
|
162
|
+
attemptId: string;
|
|
163
|
+
};
|
|
164
|
+
export type RecoverEscapedMcpTimeoutResult = {
|
|
165
|
+
action: "recovering" | "stale" | "ineligible";
|
|
166
|
+
};
|
|
167
|
+
export type PeekSessionWorkInput = {
|
|
168
|
+
workspaceId: string;
|
|
169
|
+
sessionId: string;
|
|
170
|
+
};
|
|
171
|
+
export type ExpireSessionHumanInputInput = {
|
|
172
|
+
accountId: string;
|
|
173
|
+
workspaceId: string;
|
|
174
|
+
sessionId: string;
|
|
175
|
+
requestId: string;
|
|
176
|
+
};
|
|
177
|
+
export type ExpireSessionHumanInputResult = {
|
|
178
|
+
action: "expired" | "stale" | "not_found";
|
|
179
|
+
};
|
|
180
|
+
export type MarkSessionIdleInput = {
|
|
181
|
+
workspaceId: string;
|
|
182
|
+
sessionId: string;
|
|
183
|
+
};
|
|
184
|
+
export type MaybeContinueGoalInput = {
|
|
185
|
+
accountId: string;
|
|
186
|
+
workspaceId: string;
|
|
187
|
+
sessionId: string;
|
|
188
|
+
workflowId: string;
|
|
189
|
+
};
|
|
190
|
+
export type MaybeContinueGoalResult = {
|
|
191
|
+
action: "none" | "queue" | "continue" | "paused";
|
|
192
|
+
};
|
|
193
|
+
export type DispatchScheduledTaskRunInput = {
|
|
194
|
+
workspaceId: string;
|
|
195
|
+
taskId: string;
|
|
196
|
+
/** Stable Temporal workflow identity; retries must reuse the same source row. */
|
|
197
|
+
producerKey?: string;
|
|
198
|
+
} & ({
|
|
199
|
+
triggerType: Extract<ScheduledTaskTriggerType, "scheduled">;
|
|
200
|
+
agentRunUsageIdempotencyKey?: never;
|
|
201
|
+
initiator?: never;
|
|
202
|
+
} | {
|
|
203
|
+
triggerType: Extract<ScheduledTaskTriggerType, "manual">;
|
|
204
|
+
agentRunUsageIdempotencyKey: string;
|
|
205
|
+
/** Exact identity used by the API-side charge for this same trigger. */
|
|
206
|
+
initiator: TurnInitiator;
|
|
207
|
+
});
|
|
208
|
+
export type DispatchScheduledTaskRunResult = {
|
|
209
|
+
action: "deleted";
|
|
210
|
+
} | {
|
|
211
|
+
action: "blocked";
|
|
212
|
+
reason: "insufficient_credits" | "monthly_model_cost_limit" | "monthly_agent_run_limit" | "malformed_manual_trigger";
|
|
213
|
+
} | {
|
|
214
|
+
action: "start" | "signal";
|
|
215
|
+
accountId: string;
|
|
216
|
+
workspaceId: string;
|
|
217
|
+
sessionId: string;
|
|
218
|
+
triggerEventId: string;
|
|
219
|
+
workflowId: string;
|
|
220
|
+
workflowWakeRevision: number | null;
|
|
221
|
+
};
|
|
222
|
+
export type IndexDocumentInput = {
|
|
223
|
+
accountId: string;
|
|
224
|
+
workspaceId: string;
|
|
225
|
+
documentId: string;
|
|
226
|
+
};
|
|
227
|
+
type ClaimedRunAgentTurnResult = {
|
|
228
|
+
status: "idle" | "requires_action" | "failed" | "cancelled" | "recovering" | "waiting_capacity";
|
|
229
|
+
turnId: string;
|
|
230
|
+
attemptId: string;
|
|
231
|
+
continueDelayMs?: number;
|
|
232
|
+
idleUntilReset?: boolean;
|
|
233
|
+
capacityWait?: CodexCapacityWaitRef;
|
|
234
|
+
deferredUntilWake?: boolean;
|
|
235
|
+
};
|
|
236
|
+
export type RunAgentTurnResult = ClaimedRunAgentTurnResult | {
|
|
237
|
+
status: "unclaimed";
|
|
238
|
+
reason: "gate-closed" | "no-work" | "stale-approval" | "control-pending";
|
|
239
|
+
};
|
|
240
|
+
export {};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ActivityServices } from "./types";
|
|
2
|
+
export type DispatchSessionWorkflowWakesResult = {
|
|
3
|
+
claimed: number;
|
|
4
|
+
delivered: number;
|
|
5
|
+
failed: number;
|
|
6
|
+
exhaustedBatchLimit: boolean;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Drain committed session-workflow wake revisions through the same signal and
|
|
10
|
+
* acknowledgement path used by immediate delivery. This is a repair path, not
|
|
11
|
+
* an eligibility scan: producers have already decided what must be delivered.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createWorkflowWakeActivities(services: () => Promise<ActivityServices>): {
|
|
14
|
+
dispatchSessionWorkflowWakes(): Promise<DispatchSessionWorkflowWakesResult>;
|
|
15
|
+
};
|