@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { normalizeProtocolJsonValue, sanitizeHistoryItemsForModel } from "@opengeni/runtime";
|
|
2
|
+
import { opaqueProviderArtifactFingerprint } from "@opengeni/codex";
|
|
3
|
+
|
|
4
|
+
export function historyRowsToAppend(
|
|
5
|
+
rawHistory: Array<Record<string, unknown>>,
|
|
6
|
+
// How many items of the CURRENT in-memory history are already persisted (the
|
|
7
|
+
// slice index into `sanitized`). This is the in-memory history length, NOT the
|
|
8
|
+
// total persisted-row count: after a compaction the in-memory history is the
|
|
9
|
+
// short [summary, ...tail, ...new] list, far shorter than the total rows in
|
|
10
|
+
// the table (which still hold the superseded prefix).
|
|
11
|
+
persistedHistoryCount: number,
|
|
12
|
+
// Next free WHOLE-NUMBER absolute position to write at. Decoupled from the
|
|
13
|
+
// slice index because compaction inserts a fractional summary position, so the
|
|
14
|
+
// total-row count no longer equals max(position)+1. Defaults to
|
|
15
|
+
// persistedHistoryCount to preserve the pre-compaction behaviour (contiguous
|
|
16
|
+
// positions from 0) when callers do not pass an explicit next position.
|
|
17
|
+
nextPosition: number = persistedHistoryCount,
|
|
18
|
+
toolOutputTruncationTokens?: number,
|
|
19
|
+
): {
|
|
20
|
+
rows: Array<{ position: number; item: Record<string, unknown> }>;
|
|
21
|
+
nextWatermark: number;
|
|
22
|
+
nextPosition: number;
|
|
23
|
+
} {
|
|
24
|
+
const modelReady = sanitizeHistoryItemsForModel(rawHistory, toolOutputTruncationTokens);
|
|
25
|
+
if (modelReady.length <= persistedHistoryCount) {
|
|
26
|
+
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
27
|
+
}
|
|
28
|
+
// Canonical model-facing inputs (including machine-input batches) are
|
|
29
|
+
// persisted before inference and therefore live inside the prefix represented
|
|
30
|
+
// by persistedHistoryCount. System messages synthesized only for this
|
|
31
|
+
// attempt—recovery diagnostics, credential notices, attachment materialization
|
|
32
|
+
// notes—must not accidentally become conversation memory during reconciliation.
|
|
33
|
+
// Advance the in-memory watermark past them, but only allocate durable
|
|
34
|
+
// positions to actual model/tool output.
|
|
35
|
+
const rows: Array<{ position: number; item: Record<string, unknown> }> = [];
|
|
36
|
+
for (const [offset, item] of modelReady.slice(persistedHistoryCount).entries()) {
|
|
37
|
+
if (item.type === "message" && item.role === "system") {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
rows.push({
|
|
41
|
+
position: nextPosition + rows.length,
|
|
42
|
+
item: normalizeProtocolJsonValue(
|
|
43
|
+
item as Record<string, unknown>,
|
|
44
|
+
`$[${persistedHistoryCount + offset}]`,
|
|
45
|
+
),
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
rows,
|
|
50
|
+
nextWatermark: modelReady.length,
|
|
51
|
+
nextPosition: nextPosition + rows.length,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Select only opaque rows that participated in the provider's rejected wire request. */
|
|
56
|
+
export function selectRejectedProviderArtifactHistoryIds(
|
|
57
|
+
activeHistory: ReadonlyArray<{
|
|
58
|
+
id: string;
|
|
59
|
+
item: Record<string, unknown>;
|
|
60
|
+
providerArtifactInvalidatedAt: Date | null;
|
|
61
|
+
}>,
|
|
62
|
+
candidates: {
|
|
63
|
+
knownHistoryItemIds: readonly string[];
|
|
64
|
+
historyItemIds: readonly string[];
|
|
65
|
+
},
|
|
66
|
+
wireFingerprints: readonly string[],
|
|
67
|
+
): string[] {
|
|
68
|
+
const remainingWireArtifacts = new Map<string, number>();
|
|
69
|
+
for (const fingerprint of wireFingerprints) {
|
|
70
|
+
remainingWireArtifacts.set(fingerprint, (remainingWireArtifacts.get(fingerprint) ?? 0) + 1);
|
|
71
|
+
}
|
|
72
|
+
const knownHistoryItemIds = new Set(candidates.knownHistoryItemIds);
|
|
73
|
+
const initialCandidateIds = new Set(candidates.historyItemIds);
|
|
74
|
+
const selected: string[] = [];
|
|
75
|
+
for (const row of activeHistory) {
|
|
76
|
+
if (row.providerArtifactInvalidatedAt !== null) continue;
|
|
77
|
+
// A row present before this attempt is eligible only when the exact
|
|
78
|
+
// model-facing projection retained it. Rows appended during this run are
|
|
79
|
+
// eligible after durable reconciliation.
|
|
80
|
+
if (knownHistoryItemIds.has(row.id) && !initialCandidateIds.has(row.id)) continue;
|
|
81
|
+
const fingerprint = opaqueProviderArtifactFingerprint(row.item);
|
|
82
|
+
if (!fingerprint) continue;
|
|
83
|
+
const remaining = remainingWireArtifacts.get(fingerprint) ?? 0;
|
|
84
|
+
if (remaining === 0) continue;
|
|
85
|
+
selected.push(row.id);
|
|
86
|
+
remainingWireArtifacts.set(fingerprint, remaining - 1);
|
|
87
|
+
}
|
|
88
|
+
return selected;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Literal final per-call boundary for replayed and current-turn model input. */
|
|
92
|
+
export function isModelOrToolProgressHistoryItem(item: Record<string, unknown>): boolean {
|
|
93
|
+
if (item.type === "message") {
|
|
94
|
+
return item.role === "assistant";
|
|
95
|
+
}
|
|
96
|
+
if (item.type === "reasoning" || item.type === "compaction") {
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
if (typeof item.type === "string") {
|
|
100
|
+
return item.type !== "message";
|
|
101
|
+
}
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Resolve the EFFECTIVE/active compute backend a turn should gate
|
|
107
|
+
* filesystem-touching agent lifecycle hooks on (today: the repository clone).
|
|
108
|
+
*
|
|
109
|
+
* WHY (Case B — clone-onto-real-disk hazard): a session keeps its CLOUD HOME
|
|
110
|
+
* backend (`settings.sandboxBackend`, e.g. "modal") but its ACTIVE sandbox may
|
|
111
|
+
* have been swapped to a connected machine (`active_sandbox_id` → a selfhosted
|
|
112
|
+
* lease). `runtime.buildAgent`'s repository-clone hook keys off the backend it is
|
|
113
|
+
* told; if the worker passes nothing it defaults to the HOME backend and the hook
|
|
114
|
+
* would `git clone` a private GitHub-App repo onto the user's REAL disk — a
|
|
115
|
+
* bring-your-own machine owns its own filesystem and must NEVER be cloned onto. So
|
|
116
|
+
* we look at where the agent ACTUALLY runs, not where the session was created.
|
|
117
|
+
*
|
|
118
|
+
* Returns "selfhosted" ONLY when the selfhosted feature is on AND the session has
|
|
119
|
+
* a non-null active pointer whose sandbox `kind` is "selfhosted". Otherwise
|
|
120
|
+
* returns undefined so buildAgent falls back to the home backend — byte-for-byte
|
|
121
|
+
* unchanged cloud behavior.
|
|
122
|
+
*
|
|
123
|
+
* Total + best-effort by contract: it NEVER throws (a lookup failure is logged and
|
|
124
|
+
* falls back to the home default), so wiring it at turn start can't fail the turn.
|
|
125
|
+
* The DB I/O is injected (the real call site passes readActiveSandbox + getSandbox,
|
|
126
|
+
* the same helpers wrapTurnBoxWithRouting reuses) so the gate/decision/safety
|
|
127
|
+
* contract is unit-testable without a live database.
|
|
128
|
+
*/
|
|
129
|
+
export function pendingToolCallFromSdkEvent(event: unknown): {
|
|
130
|
+
callId: string;
|
|
131
|
+
callType: string;
|
|
132
|
+
callName: string | null;
|
|
133
|
+
callItem: Record<string, unknown>;
|
|
134
|
+
} | null {
|
|
135
|
+
if (!event || typeof event !== "object") return null;
|
|
136
|
+
if ((event as { type?: unknown }).type !== "run_item_stream_event") return null;
|
|
137
|
+
const item = (event as { item?: { type?: unknown; rawItem?: unknown } }).item;
|
|
138
|
+
if (
|
|
139
|
+
!item ||
|
|
140
|
+
(item.type !== "tool_call_item" && item.type !== "tool_search_call_item") ||
|
|
141
|
+
!item.rawItem ||
|
|
142
|
+
typeof item.rawItem !== "object" ||
|
|
143
|
+
Array.isArray(item.rawItem)
|
|
144
|
+
) {
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
const raw = item.rawItem as Record<string, unknown>;
|
|
148
|
+
// The hosted image call is a complete provider fact carried by one item; it
|
|
149
|
+
// never receives a separate function result and therefore must not enter the
|
|
150
|
+
// pending function-call ledger.
|
|
151
|
+
if (raw.type === "hosted_tool_call" && raw.name === "image_generation_call") return null;
|
|
152
|
+
const callId = raw.callId ?? raw.call_id ?? raw.id;
|
|
153
|
+
const callType = raw.type;
|
|
154
|
+
if (typeof callId !== "string" || callId.length === 0 || typeof callType !== "string") {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
const callItem = normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]');
|
|
158
|
+
return {
|
|
159
|
+
callId,
|
|
160
|
+
callType,
|
|
161
|
+
callName: typeof raw.name === "string" ? raw.name : null,
|
|
162
|
+
callItem,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Tools whose intentional visual result must survive inline-media sanitization. */
|
|
167
|
+
export function toolCallProducesRetainableSessionImage(name: string | null): boolean {
|
|
168
|
+
return name === "computer_screenshot" || name === "view_image";
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function completedToolCallFromSdkEvent(event: unknown): {
|
|
172
|
+
callId: string;
|
|
173
|
+
resultItem: Record<string, unknown>;
|
|
174
|
+
} | null {
|
|
175
|
+
if (!event || typeof event !== "object") return null;
|
|
176
|
+
if ((event as { type?: unknown }).type !== "run_item_stream_event") return null;
|
|
177
|
+
const item = (event as { item?: { type?: unknown; rawItem?: unknown; id?: unknown } }).item;
|
|
178
|
+
if (!item || (item.type !== "tool_call_output_item" && item.type !== "tool_search_output_item")) {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
const raw =
|
|
182
|
+
item.rawItem && typeof item.rawItem === "object" && !Array.isArray(item.rawItem)
|
|
183
|
+
? (item.rawItem as Record<string, unknown>)
|
|
184
|
+
: {};
|
|
185
|
+
const callId = raw.callId ?? raw.call_id ?? item.id;
|
|
186
|
+
if (typeof callId !== "string" || callId.length === 0) return null;
|
|
187
|
+
return {
|
|
188
|
+
callId,
|
|
189
|
+
resultItem: normalizeProtocolJsonValue(raw, '$["item"]["rawItem"]'),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Budget/limit exhaustion detected between model calls. This is account
|
|
195
|
+
* state, not an agent failure: the segment ends gracefully (session idles,
|
|
196
|
+
* run state preserved) so a top-up or limit reset lets the same session
|
|
197
|
+
* continue — a failed session would reject the user's next message. An
|
|
198
|
+
* active goal pauses visibly (reason "limits") at the next continuation
|
|
199
|
+
* evaluation without consuming continuation budget.
|
|
200
|
+
*/
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getMaterializedSandboxFileResources,
|
|
3
|
+
markSandboxFileResourcesMaterialized,
|
|
4
|
+
requireFile,
|
|
5
|
+
} from "@opengeni/db";
|
|
6
|
+
import {
|
|
7
|
+
materializeSandboxFileDownloads,
|
|
8
|
+
sandboxRunAs,
|
|
9
|
+
type CodemodeTokenWriterSession,
|
|
10
|
+
type RetainableSessionImageOutputHook,
|
|
11
|
+
type SandboxFileDownload,
|
|
12
|
+
type TurnToolCancellationFence,
|
|
13
|
+
} from "@opengeni/runtime";
|
|
14
|
+
import type { Settings } from "@opengeni/config";
|
|
15
|
+
import type { RetainedArtifactMetadata } from "@opengeni/contracts";
|
|
16
|
+
import type { ResumedTurnSandbox } from "../../sandbox-resume";
|
|
17
|
+
import type { SharedActivityServices } from "../types";
|
|
18
|
+
import {
|
|
19
|
+
assertGeneratedImageHistoryRetained,
|
|
20
|
+
collectGeneratedImageReceipts,
|
|
21
|
+
compactGeneratedImageRunState,
|
|
22
|
+
generatedImagesFromHistory,
|
|
23
|
+
retainGeneratedImage,
|
|
24
|
+
type GeneratedImageOutput,
|
|
25
|
+
type GeneratedImageReceipt,
|
|
26
|
+
} from "../generated-images";
|
|
27
|
+
import {
|
|
28
|
+
collectRetainedScreenshotReceipts,
|
|
29
|
+
compactRetainedScreenshotRunState,
|
|
30
|
+
materializeRetainedScreenshotHistory,
|
|
31
|
+
retainComputerScreenshot,
|
|
32
|
+
toolOutputContainsInlineImage,
|
|
33
|
+
typedScreenshotFromToolOutput,
|
|
34
|
+
unavailableRetainedSessionImage,
|
|
35
|
+
} from "../retained-screenshots";
|
|
36
|
+
import { objectStorageForSandboxDownloads } from "./file-resources";
|
|
37
|
+
import type { TurnEventPublisher } from "./model-usage";
|
|
38
|
+
|
|
39
|
+
export type NativeImageGenerationRetention = {
|
|
40
|
+
providerId: string;
|
|
41
|
+
providerBindingHash: string;
|
|
42
|
+
sessionId: string;
|
|
43
|
+
turnId: string;
|
|
44
|
+
attemptId: string;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export type TurnMediaArtifactDeps = {
|
|
48
|
+
db: SharedActivityServices["db"];
|
|
49
|
+
objectStorage: SharedActivityServices["objectStorage"];
|
|
50
|
+
observability: SharedActivityServices["observability"];
|
|
51
|
+
accountId: string;
|
|
52
|
+
workspaceId: string;
|
|
53
|
+
sessionId: string;
|
|
54
|
+
attemptId: string;
|
|
55
|
+
getTurnId: () => string | undefined;
|
|
56
|
+
getModelRunSettings: () => Settings;
|
|
57
|
+
getPublish: () => TurnEventPublisher | null;
|
|
58
|
+
toolCancellationFenceRef: { current: TurnToolCancellationFence | null };
|
|
59
|
+
getResolvedSandbox: () => ResumedTurnSandbox | null;
|
|
60
|
+
getSetupBoxSession: () => unknown;
|
|
61
|
+
getSandboxGroupId: () => string | null;
|
|
62
|
+
runWorkspaceMutation: <T>(
|
|
63
|
+
sandbox: ResumedTurnSandbox,
|
|
64
|
+
operation: string,
|
|
65
|
+
mutation: () => Promise<T>,
|
|
66
|
+
) => Promise<T>;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
export class TurnMediaArtifacts {
|
|
70
|
+
readonly retainedScreenshotReceiptsByCallId = new Map<string, RetainedArtifactMetadata>();
|
|
71
|
+
modelCanReceiveRetainedSessionImages = true;
|
|
72
|
+
readonly generatedImageReceiptsByProviderItemId = new Map<string, GeneratedImageReceipt>();
|
|
73
|
+
readonly generatedImageReceiptsCreatedThisTurn = new Map<string, GeneratedImageReceipt>();
|
|
74
|
+
private generatedImageMaterializationCache: {
|
|
75
|
+
instanceId: string;
|
|
76
|
+
fileIds: Set<string>;
|
|
77
|
+
} | null = null;
|
|
78
|
+
sdkOwnedSandboxSession: CodemodeTokenWriterSession | null = null;
|
|
79
|
+
sandboxFileDownloadBackend: Settings["sandboxBackend"];
|
|
80
|
+
nativeImageGenerationRetention: NativeImageGenerationRetention | null = null;
|
|
81
|
+
readonly retainedSessionImageCallIds = new Set<string>();
|
|
82
|
+
|
|
83
|
+
constructor(private readonly deps: TurnMediaArtifactDeps) {
|
|
84
|
+
this.sandboxFileDownloadBackend = deps.getModelRunSettings().sandboxBackend;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
rememberGeneratedImageCreatedThisTurn = (receipt: GeneratedImageReceipt): void => {
|
|
88
|
+
this.generatedImageReceiptsCreatedThisTurn.set(receipt.artifact.artifactId, receipt);
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
private prepareGeneratedImageDownload = async (receipt: GeneratedImageReceipt) => {
|
|
92
|
+
const { db, objectStorage } = this.deps;
|
|
93
|
+
if (!objectStorage) {
|
|
94
|
+
throw new Error("Generated image sandbox materialization requires object storage");
|
|
95
|
+
}
|
|
96
|
+
const file = await requireFile(db, this.deps.workspaceId, receipt.artifact.artifactId);
|
|
97
|
+
const downloadStorage = objectStorageForSandboxDownloads(
|
|
98
|
+
this.deps.getModelRunSettings(),
|
|
99
|
+
objectStorage,
|
|
100
|
+
this.sandboxFileDownloadBackend,
|
|
101
|
+
);
|
|
102
|
+
const signed = await downloadStorage.createGetUrl({
|
|
103
|
+
key: file.objectKey,
|
|
104
|
+
});
|
|
105
|
+
return {
|
|
106
|
+
file,
|
|
107
|
+
download: {
|
|
108
|
+
fileId: file.id,
|
|
109
|
+
mountPath: "generated-images",
|
|
110
|
+
filename: file.safeFilename,
|
|
111
|
+
url: signed.url,
|
|
112
|
+
expiresAt: signed.expiresAt,
|
|
113
|
+
sizeBytes: file.sizeBytes,
|
|
114
|
+
...(file.sha256 ? { sha256: file.sha256 } : {}),
|
|
115
|
+
} satisfies SandboxFileDownload,
|
|
116
|
+
};
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
private writeGeneratedImageDownload = async (
|
|
120
|
+
sessionForDownload: CodemodeTokenWriterSession,
|
|
121
|
+
download: SandboxFileDownload,
|
|
122
|
+
): Promise<void> => {
|
|
123
|
+
const runAs = sandboxRunAs(this.deps.getModelRunSettings());
|
|
124
|
+
const publish = this.deps.getPublish();
|
|
125
|
+
const fence = this.deps.toolCancellationFenceRef.current;
|
|
126
|
+
const materialized = await materializeSandboxFileDownloads(sessionForDownload, [download], {
|
|
127
|
+
onRuntimeEvent: async (event) => {
|
|
128
|
+
await publish?.([{ type: event.type, payload: event.payload }], true);
|
|
129
|
+
},
|
|
130
|
+
...(runAs ? { runAs } : {}),
|
|
131
|
+
...(fence ? { commandRunner: fence.runSandboxCommand.bind(fence) } : {}),
|
|
132
|
+
});
|
|
133
|
+
if (materialized.failures.length > 0) {
|
|
134
|
+
throw new Error(materialized.failures[0]!.reason);
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
private warnGeneratedImageMaterializationDeferred = (error: unknown): void => {
|
|
139
|
+
this.deps.observability.warn("Generated image sandbox materialization deferred", {
|
|
140
|
+
errorClass: error instanceof Error ? error.name : "UnknownError",
|
|
141
|
+
errorCode: "generated_image_materialization_deferred",
|
|
142
|
+
origin: "worker",
|
|
143
|
+
});
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
materializeGeneratedImageInSandbox = async (
|
|
147
|
+
receipt: GeneratedImageReceipt,
|
|
148
|
+
sandbox: ResumedTurnSandbox,
|
|
149
|
+
sessionForDownload: unknown,
|
|
150
|
+
): Promise<boolean> => {
|
|
151
|
+
try {
|
|
152
|
+
const { file, download } = await this.prepareGeneratedImageDownload(receipt);
|
|
153
|
+
if (this.generatedImageMaterializationCache?.instanceId !== sandbox.established.instanceId) {
|
|
154
|
+
const fileIds = this.deps.getSandboxGroupId()
|
|
155
|
+
? await getMaterializedSandboxFileResources(this.deps.db, {
|
|
156
|
+
accountId: this.deps.accountId,
|
|
157
|
+
workspaceId: this.deps.workspaceId,
|
|
158
|
+
sandboxGroupId: this.deps.getSandboxGroupId()!,
|
|
159
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
160
|
+
instanceId: sandbox.established.instanceId,
|
|
161
|
+
})
|
|
162
|
+
: new Set<string>();
|
|
163
|
+
this.generatedImageMaterializationCache = {
|
|
164
|
+
instanceId: sandbox.established.instanceId,
|
|
165
|
+
fileIds,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
if (this.generatedImageMaterializationCache.fileIds.has(file.id)) return true;
|
|
169
|
+
await this.deps.runWorkspaceMutation(
|
|
170
|
+
sandbox,
|
|
171
|
+
"generatedImageMaterialization",
|
|
172
|
+
async () =>
|
|
173
|
+
await this.writeGeneratedImageDownload(
|
|
174
|
+
sessionForDownload as CodemodeTokenWriterSession,
|
|
175
|
+
download,
|
|
176
|
+
),
|
|
177
|
+
);
|
|
178
|
+
this.generatedImageMaterializationCache.fileIds.add(file.id);
|
|
179
|
+
const sandboxGroupId = this.deps.getSandboxGroupId();
|
|
180
|
+
if (sandboxGroupId) {
|
|
181
|
+
await markSandboxFileResourcesMaterialized(this.deps.db, {
|
|
182
|
+
accountId: this.deps.accountId,
|
|
183
|
+
workspaceId: this.deps.workspaceId,
|
|
184
|
+
sandboxGroupId,
|
|
185
|
+
expectedEpoch: sandbox.leaseEpoch,
|
|
186
|
+
instanceId: sandbox.established.instanceId,
|
|
187
|
+
fileIds: [file.id],
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
return true;
|
|
191
|
+
} catch (error) {
|
|
192
|
+
this.warnGeneratedImageMaterializationDeferred(error);
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
materializeGeneratedImageInOwnedSdkSession = async (
|
|
198
|
+
receipt: GeneratedImageReceipt,
|
|
199
|
+
sessionForDownload: unknown,
|
|
200
|
+
): Promise<boolean> => {
|
|
201
|
+
try {
|
|
202
|
+
const { download } = await this.prepareGeneratedImageDownload(receipt);
|
|
203
|
+
await this.writeGeneratedImageDownload(
|
|
204
|
+
sessionForDownload as CodemodeTokenWriterSession,
|
|
205
|
+
download,
|
|
206
|
+
);
|
|
207
|
+
return true;
|
|
208
|
+
} catch (error) {
|
|
209
|
+
this.warnGeneratedImageMaterializationDeferred(error);
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
materializeGeneratedImage = async (receipt: GeneratedImageReceipt): Promise<boolean> => {
|
|
215
|
+
if (this.sandboxFileDownloadBackend === "none") return false;
|
|
216
|
+
const resolvedSandbox = this.deps.getResolvedSandbox();
|
|
217
|
+
const setupBoxSession = this.deps.getSetupBoxSession();
|
|
218
|
+
if (resolvedSandbox && setupBoxSession) {
|
|
219
|
+
return await this.materializeGeneratedImageInSandbox(
|
|
220
|
+
receipt,
|
|
221
|
+
resolvedSandbox,
|
|
222
|
+
setupBoxSession,
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
if (this.sdkOwnedSandboxSession) {
|
|
226
|
+
return await this.materializeGeneratedImageInOwnedSdkSession(
|
|
227
|
+
receipt,
|
|
228
|
+
this.sdkOwnedSandboxSession,
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
return false;
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
retainNativeGeneratedImage = async (
|
|
235
|
+
output: GeneratedImageOutput,
|
|
236
|
+
): Promise<GeneratedImageReceipt> => {
|
|
237
|
+
if (output.providerItemId) {
|
|
238
|
+
const existing = this.generatedImageReceiptsByProviderItemId.get(output.providerItemId);
|
|
239
|
+
if (existing) return existing;
|
|
240
|
+
}
|
|
241
|
+
const context = this.nativeImageGenerationRetention;
|
|
242
|
+
if (!context || !output.providerItemId) {
|
|
243
|
+
throw new Error("Native generated image arrived without provider retention identity");
|
|
244
|
+
}
|
|
245
|
+
const retained = await retainGeneratedImage({
|
|
246
|
+
db: this.deps.db,
|
|
247
|
+
objectStorage: this.deps.objectStorage,
|
|
248
|
+
accountId: this.deps.accountId,
|
|
249
|
+
workspaceId: this.deps.workspaceId,
|
|
250
|
+
sessionId: context.sessionId,
|
|
251
|
+
turnId: context.turnId,
|
|
252
|
+
attemptId: context.attemptId,
|
|
253
|
+
sourceStrategy: "native_hosted",
|
|
254
|
+
providerId: context.providerId,
|
|
255
|
+
providerBindingHash: context.providerBindingHash,
|
|
256
|
+
output,
|
|
257
|
+
});
|
|
258
|
+
this.generatedImageReceiptsByProviderItemId.set(output.providerItemId, retained.receipt);
|
|
259
|
+
this.rememberGeneratedImageCreatedThisTurn(retained.receipt);
|
|
260
|
+
await this.materializeGeneratedImage(retained.receipt);
|
|
261
|
+
return retained.receipt;
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
retainNativeGeneratedImagesFromHistory = async (
|
|
265
|
+
history: Array<Record<string, unknown>>,
|
|
266
|
+
): Promise<void> => {
|
|
267
|
+
collectGeneratedImageReceipts(history, this.generatedImageReceiptsByProviderItemId);
|
|
268
|
+
for (const output of generatedImagesFromHistory(
|
|
269
|
+
history,
|
|
270
|
+
this.generatedImageReceiptsByProviderItemId,
|
|
271
|
+
)) {
|
|
272
|
+
if (
|
|
273
|
+
!output.providerItemId ||
|
|
274
|
+
this.generatedImageReceiptsByProviderItemId.has(output.providerItemId)
|
|
275
|
+
) {
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
await this.retainNativeGeneratedImage(output);
|
|
279
|
+
}
|
|
280
|
+
assertGeneratedImageHistoryRetained(history, this.generatedImageReceiptsByProviderItemId);
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
compactMediaRunState = (serialized: string): string =>
|
|
284
|
+
compactGeneratedImageRunState(
|
|
285
|
+
compactRetainedScreenshotRunState(serialized, this.retainedScreenshotReceiptsByCallId),
|
|
286
|
+
this.generatedImageReceiptsByProviderItemId,
|
|
287
|
+
);
|
|
288
|
+
|
|
289
|
+
retainSessionImageAtToolBoundary: RetainableSessionImageOutputHook = async ({
|
|
290
|
+
toolCallId,
|
|
291
|
+
output,
|
|
292
|
+
}) => {
|
|
293
|
+
const activeTurnId = this.deps.getTurnId();
|
|
294
|
+
if (!activeTurnId) {
|
|
295
|
+
throw new Error("Session image tool completed before turn initialization");
|
|
296
|
+
}
|
|
297
|
+
const typedScreenshot = typedScreenshotFromToolOutput({
|
|
298
|
+
callId: toolCallId,
|
|
299
|
+
output,
|
|
300
|
+
});
|
|
301
|
+
this.retainedSessionImageCallIds.add(toolCallId);
|
|
302
|
+
if (!typedScreenshot) {
|
|
303
|
+
if (toolOutputContainsInlineImage(output)) {
|
|
304
|
+
this.retainedScreenshotReceiptsByCallId.set(
|
|
305
|
+
toolCallId,
|
|
306
|
+
unavailableRetainedSessionImage({
|
|
307
|
+
sessionId: this.deps.sessionId,
|
|
308
|
+
turnId: activeTurnId,
|
|
309
|
+
attemptId: this.deps.attemptId,
|
|
310
|
+
toolCallId,
|
|
311
|
+
toolOutputId: toolCallId,
|
|
312
|
+
reason: "unsupported",
|
|
313
|
+
}),
|
|
314
|
+
);
|
|
315
|
+
}
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
this.retainedScreenshotReceiptsByCallId.set(
|
|
319
|
+
toolCallId,
|
|
320
|
+
unavailableRetainedSessionImage({
|
|
321
|
+
sessionId: this.deps.sessionId,
|
|
322
|
+
turnId: activeTurnId,
|
|
323
|
+
attemptId: this.deps.attemptId,
|
|
324
|
+
toolCallId,
|
|
325
|
+
toolOutputId: toolCallId,
|
|
326
|
+
reason: "pending",
|
|
327
|
+
}),
|
|
328
|
+
);
|
|
329
|
+
this.retainedScreenshotReceiptsByCallId.set(
|
|
330
|
+
toolCallId,
|
|
331
|
+
await retainComputerScreenshot({
|
|
332
|
+
db: this.deps.db,
|
|
333
|
+
objectStorage: this.deps.objectStorage,
|
|
334
|
+
accountId: this.deps.accountId,
|
|
335
|
+
workspaceId: this.deps.workspaceId,
|
|
336
|
+
sessionId: this.deps.sessionId,
|
|
337
|
+
turnId: activeTurnId,
|
|
338
|
+
attemptId: this.deps.attemptId,
|
|
339
|
+
output: typedScreenshot,
|
|
340
|
+
}),
|
|
341
|
+
);
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
materializeScreenshotHistory = async (history: Array<Record<string, unknown>>) => {
|
|
345
|
+
collectRetainedScreenshotReceipts(history, this.retainedScreenshotReceiptsByCallId);
|
|
346
|
+
if (!this.modelCanReceiveRetainedSessionImages) return history;
|
|
347
|
+
return await materializeRetainedScreenshotHistory({
|
|
348
|
+
db: this.deps.db,
|
|
349
|
+
objectStorage: this.deps.objectStorage,
|
|
350
|
+
workspaceId: this.deps.workspaceId,
|
|
351
|
+
sessionId: this.deps.sessionId,
|
|
352
|
+
history,
|
|
353
|
+
});
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
export function createTurnMediaArtifacts(deps: TurnMediaArtifactDeps): TurnMediaArtifacts {
|
|
358
|
+
return new TurnMediaArtifacts(deps);
|
|
359
|
+
}
|