@opengeni/worker-bundle 0.16.17 → 0.16.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +23 -19
- package/dist/activities/file-upload-reaper.d.ts +6 -0
- package/dist/index.js +148 -63
- package/dist/index.js.map +1 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +77 -74
- package/src/activities/context-compaction.ts +0 -2
- package/src/activities/file-upload-reaper.ts +108 -7
- package/src/activities/sandbox-lease.ts +5 -2
- package/src/sandbox-routing.ts +5 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type AppendEventInput, type ActiveSandboxPointer, type SandboxRecord } from "@opengeni/db";
|
|
2
|
-
import { modelResponseServiceTierFromSdkEvent,
|
|
2
|
+
import { modelResponseServiceTierFromSdkEvent, modelTerminalResponseFromSdkEvent, type SandboxFileDownload, type ComputerToolMode, type ModelResponseUsage, type ModelCallUsageInput, type ModelCallUsageNormalization, type RunAgentStreamOptions, type TurnToolCancellationFence, type BackendUnresolvableCode } from "@opengeni/runtime";
|
|
3
3
|
import { resolveTurnExecutionPolicyV1, type ModelUsageInput, type ModelProviderApi, type RegistryProviderKind, type Settings } from "@opengeni/config";
|
|
4
4
|
import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
|
|
5
5
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
@@ -271,19 +271,21 @@ type TurnEventPublisher = (events: Array<Omit<AppendEventInput, "producerId" | "
|
|
|
271
271
|
events: SessionEvent[];
|
|
272
272
|
accepted: boolean;
|
|
273
273
|
}>;
|
|
274
|
-
export type
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
274
|
+
export type ModelResponseEventState = {
|
|
275
|
+
responseCount: number;
|
|
276
|
+
contextSignal: {
|
|
277
|
+
revision: number;
|
|
278
|
+
totalTokens: number;
|
|
279
|
+
} | null;
|
|
278
280
|
claimedSourceKeys: Set<string>;
|
|
279
281
|
};
|
|
280
282
|
export type CompactionModelUsageEventState = {
|
|
281
283
|
usageCount: number;
|
|
282
284
|
claimedSourceKeys: Set<string>;
|
|
283
285
|
};
|
|
284
|
-
export declare function
|
|
286
|
+
export declare function createModelResponseEventState(claimedSourceKeys?: Set<string>): ModelResponseEventState;
|
|
285
287
|
export declare function createCompactionModelUsageEventState(claimedSourceKeys?: Set<string>): CompactionModelUsageEventState;
|
|
286
|
-
export declare function
|
|
288
|
+
export declare function modelResponseContextSignal(state: ModelResponseEventState): {
|
|
287
289
|
revision: number;
|
|
288
290
|
totalTokens: number;
|
|
289
291
|
} | null;
|
|
@@ -295,19 +297,20 @@ export declare function assertModelResponseLatencyMode(input: {
|
|
|
295
297
|
providerId?: string;
|
|
296
298
|
}): void;
|
|
297
299
|
/**
|
|
298
|
-
* Process one SDK
|
|
300
|
+
* Process one SDK terminal-response event through the production authority path.
|
|
299
301
|
*
|
|
300
302
|
* The pinned Responses SDK mirrors one provider terminal response as both a
|
|
301
303
|
* normalized `response_done` and a raw `model/response.completed` event. Claim
|
|
302
|
-
* the stable response/source key before lease renewal or any
|
|
303
|
-
*
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
304
|
+
* the stable response/source key before lease renewal or any side effect, and
|
|
305
|
+
* use that one positional ordinal for both response identity and same-run
|
|
306
|
+
* context binding. A response without usage still clears attempt-owned token
|
|
307
|
+
* state. When usage exists, the durable `agent.model.usage` source-key fence
|
|
308
|
+
* remains the cross-restart authority: a replay may retry the idempotent billing
|
|
309
|
+
* write, but it cannot advance metrics, context, or attempt-owned signals.
|
|
310
|
+
*/
|
|
311
|
+
export declare function processModelResponseTerminalEvent(input: {
|
|
312
|
+
event: Parameters<typeof modelTerminalResponseFromSdkEvent>[0];
|
|
313
|
+
state: ModelResponseEventState;
|
|
311
314
|
dispatchId: string | null;
|
|
312
315
|
settings: Settings;
|
|
313
316
|
db: ActivityServices["db"];
|
|
@@ -330,9 +333,9 @@ export declare function processModelResponseUsageEvent(input: {
|
|
|
330
333
|
renewLease: () => Promise<void>;
|
|
331
334
|
leaseLost: () => boolean;
|
|
332
335
|
leaseLostMessage: string;
|
|
333
|
-
setLastInputTokens: (tokens: number) => Promise<void>;
|
|
336
|
+
setLastInputTokens: (tokens: number | null) => Promise<void>;
|
|
334
337
|
}): Promise<{
|
|
335
|
-
status: "
|
|
338
|
+
status: "not_response";
|
|
336
339
|
} | {
|
|
337
340
|
status: "duplicate";
|
|
338
341
|
sourceKey: string;
|
|
@@ -340,6 +343,7 @@ export declare function processModelResponseUsageEvent(input: {
|
|
|
340
343
|
status: "processed";
|
|
341
344
|
sourceKey: string;
|
|
342
345
|
authoritative: boolean;
|
|
346
|
+
usageReported: boolean;
|
|
343
347
|
}>;
|
|
344
348
|
/**
|
|
345
349
|
* Apply the same source-key authority ordering to the compaction summarizer's
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { claimDueTranscriptionRecordingObjectCleanup, claimExpiredFileUploadCleanup, completeDueTranscriptionRecordingObjectCleanup, completeExpiredFileUploadCleanup, purgeExpiredTranscriptionRecordings } from "@opengeni/db";
|
|
1
2
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
2
3
|
import type { ActivityServices } from "./types";
|
|
3
4
|
export declare const FILE_UPLOAD_CLEANUP_GRACE_MS: number;
|
|
@@ -14,6 +15,11 @@ export type FileUploadReaperActivityOptions = {
|
|
|
14
15
|
batchSize?: number;
|
|
15
16
|
/** Failure-injection seam; production uses the configured provider delete. */
|
|
16
17
|
deleteObject?: (storage: ObjectStorage, key: string) => Promise<void>;
|
|
18
|
+
claimFileUploads?: typeof claimExpiredFileUploadCleanup;
|
|
19
|
+
completeFileUpload?: typeof completeExpiredFileUploadCleanup;
|
|
20
|
+
claimTranscriptionObjects?: typeof claimDueTranscriptionRecordingObjectCleanup;
|
|
21
|
+
completeTranscriptionObject?: typeof completeDueTranscriptionRecordingObjectCleanup;
|
|
22
|
+
purgeTranscriptionRecordings?: typeof purgeExpiredTranscriptionRecordings;
|
|
17
23
|
};
|
|
18
24
|
/**
|
|
19
25
|
* Build the provider-neutral expired direct-upload reaper. Claims are durable
|
package/dist/index.js
CHANGED
|
@@ -127,7 +127,7 @@ import {
|
|
|
127
127
|
sandboxStateEntryFromRunState,
|
|
128
128
|
maxTurnsExceededRunState,
|
|
129
129
|
modelResponseServiceTierFromSdkEvent,
|
|
130
|
-
|
|
130
|
+
modelTerminalResponseFromSdkEvent,
|
|
131
131
|
normalizeModelCallUsage,
|
|
132
132
|
normalizeSdkEvent,
|
|
133
133
|
projectHistoryForProvider as projectHistoryForProvider2,
|
|
@@ -164,6 +164,7 @@ import {
|
|
|
164
164
|
runOwnedSandboxSetup,
|
|
165
165
|
RoutingMutationOutcomeUnknownError,
|
|
166
166
|
WorkspaceArchiveIntegrityError as WorkspaceArchiveIntegrityError2,
|
|
167
|
+
sdkBackendIdForSandboxBackend,
|
|
167
168
|
swapTargetEstablishability,
|
|
168
169
|
deleteRecordingArtifacts as deleteRecordingArtifacts2,
|
|
169
170
|
stopRecording as stopRecordingOnBox2
|
|
@@ -1615,7 +1616,6 @@ async function compactContextRemoteV2(db, settings, scope, canonicalItems, items
|
|
|
1615
1616
|
expectedAttemptId: scope.attemptId,
|
|
1616
1617
|
replacementItems: replacementHistory.slice(0, -1),
|
|
1617
1618
|
summaryItem: tailItem,
|
|
1618
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
1619
1619
|
...options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {},
|
|
1620
1620
|
eventPayload: {
|
|
1621
1621
|
trigger: options.trigger ?? "auto",
|
|
@@ -1668,7 +1668,6 @@ async function compactContextPortable(db, settings, scope, canonicalItems, items
|
|
|
1668
1668
|
expectedAttemptId: scope.attemptId,
|
|
1669
1669
|
replacementItems: replacementHistory.slice(0, -1),
|
|
1670
1670
|
summaryItem,
|
|
1671
|
-
replacementInputTokens: estimatedTokensAfter,
|
|
1672
1671
|
...options.clearRequestedCompaction ? { clearRequestedCompaction: true } : {},
|
|
1673
1672
|
eventPayload: {
|
|
1674
1673
|
trigger: options.trigger ?? "auto",
|
|
@@ -4215,6 +4214,7 @@ import {
|
|
|
4215
4214
|
RoutingBackendRecoveryRequiredError,
|
|
4216
4215
|
RoutingSandboxSession,
|
|
4217
4216
|
resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
|
|
4217
|
+
sandboxBackendForSdkBackendId,
|
|
4218
4218
|
verifySandboxExecReadiness as verifySandboxExecReadiness2
|
|
4219
4219
|
} from "@opengeni/runtime";
|
|
4220
4220
|
function relayConfigFromSettings(settings) {
|
|
@@ -4300,7 +4300,8 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
4300
4300
|
};
|
|
4301
4301
|
}
|
|
4302
4302
|
const resumeState = lease.resumeState;
|
|
4303
|
-
const
|
|
4303
|
+
const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
4304
|
+
const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
|
|
4304
4305
|
if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
|
|
4305
4306
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
4306
4307
|
}
|
|
@@ -4341,7 +4342,8 @@ async function resolveCurrentHomeBackend(services, ids, established) {
|
|
|
4341
4342
|
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
|
|
4342
4343
|
);
|
|
4343
4344
|
}
|
|
4344
|
-
|
|
4345
|
+
const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
|
|
4346
|
+
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
4345
4347
|
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
4346
4348
|
}
|
|
4347
4349
|
services.onHomeSandboxRebound?.({
|
|
@@ -6791,22 +6793,18 @@ async function recordCompletedModelCallBeforeOwnershipFences(input) {
|
|
|
6791
6793
|
}
|
|
6792
6794
|
await input.recordAttemptSignals?.();
|
|
6793
6795
|
}
|
|
6794
|
-
function
|
|
6796
|
+
function createModelResponseEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
|
|
6795
6797
|
return {
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
lastProviderContextTokensObserved: null,
|
|
6798
|
+
responseCount: 0,
|
|
6799
|
+
contextSignal: null,
|
|
6799
6800
|
claimedSourceKeys
|
|
6800
6801
|
};
|
|
6801
6802
|
}
|
|
6802
6803
|
function createCompactionModelUsageEventState(claimedSourceKeys = /* @__PURE__ */ new Set()) {
|
|
6803
6804
|
return { usageCount: 0, claimedSourceKeys };
|
|
6804
6805
|
}
|
|
6805
|
-
function
|
|
6806
|
-
return state.
|
|
6807
|
-
revision: state.providerContextRevision,
|
|
6808
|
-
totalTokens: state.lastProviderContextTokensObserved
|
|
6809
|
-
};
|
|
6806
|
+
function modelResponseContextSignal(state) {
|
|
6807
|
+
return state.contextSignal;
|
|
6810
6808
|
}
|
|
6811
6809
|
function assertModelResponseLatencyMode(input) {
|
|
6812
6810
|
if (input.requested === "standard") {
|
|
@@ -6825,14 +6823,14 @@ function assertModelResponseLatencyMode(input) {
|
|
|
6825
6823
|
);
|
|
6826
6824
|
}
|
|
6827
6825
|
}
|
|
6828
|
-
async function
|
|
6829
|
-
const
|
|
6830
|
-
if (!
|
|
6831
|
-
return { status: "
|
|
6826
|
+
async function processModelResponseTerminalEvent(input) {
|
|
6827
|
+
const terminal = modelTerminalResponseFromSdkEvent(input.event);
|
|
6828
|
+
if (!terminal) {
|
|
6829
|
+
return { status: "not_response" };
|
|
6832
6830
|
}
|
|
6833
|
-
const responseOrdinal = input.state.
|
|
6831
|
+
const responseOrdinal = input.state.responseCount + 1;
|
|
6834
6832
|
const sourceKey = modelUsageSourceKey({
|
|
6835
|
-
responseId:
|
|
6833
|
+
responseId: terminal.responseId,
|
|
6836
6834
|
dispatchId: input.dispatchId,
|
|
6837
6835
|
positionalKey: `response-${responseOrdinal}`
|
|
6838
6836
|
});
|
|
@@ -6840,19 +6838,21 @@ async function processModelResponseUsageEvent(input) {
|
|
|
6840
6838
|
return { status: "duplicate", sourceKey };
|
|
6841
6839
|
}
|
|
6842
6840
|
input.state.claimedSourceKeys.add(sourceKey);
|
|
6843
|
-
input.state.
|
|
6844
|
-
const
|
|
6841
|
+
input.state.responseCount = responseOrdinal;
|
|
6842
|
+
const responseUsage = terminal.usage;
|
|
6843
|
+
const normalizedUsage = normalizeModelCallUsage(responseUsage?.usage);
|
|
6845
6844
|
const accountContext = modelCallAccountContext({
|
|
6846
6845
|
servingCredentialId: input.servingCredentialId,
|
|
6847
6846
|
priorSessionCredentialId: input.priorSessionCredentialId,
|
|
6848
6847
|
isFirstCallOfTurn: responseOrdinal === 1
|
|
6849
6848
|
});
|
|
6850
|
-
let authoritative =
|
|
6849
|
+
let authoritative = responseUsage === null;
|
|
6851
6850
|
await recordCompletedModelCallBeforeOwnershipFences({
|
|
6852
6851
|
renewLease: input.renewLease,
|
|
6853
6852
|
leaseLost: input.leaseLost,
|
|
6854
6853
|
leaseLostMessage: input.leaseLostMessage,
|
|
6855
6854
|
recordUsage: async () => {
|
|
6855
|
+
if (!responseUsage) return;
|
|
6856
6856
|
const billing = await recordModelUsageAndDebitCredits(input.settings, input.db, {
|
|
6857
6857
|
accountId: input.accountId,
|
|
6858
6858
|
workspaceId: input.workspaceId,
|
|
@@ -6910,17 +6910,19 @@ async function processModelResponseUsageEvent(input) {
|
|
|
6910
6910
|
recordAttemptSignals: async () => {
|
|
6911
6911
|
if (!authoritative) return;
|
|
6912
6912
|
const observedTotal = normalizedUsage.totalTokens;
|
|
6913
|
-
|
|
6914
|
-
input.state.lastProviderContextTokensObserved = observedTotal;
|
|
6915
|
-
input.state.providerContextRevision += 1;
|
|
6916
|
-
}
|
|
6913
|
+
input.state.contextSignal = observedTotal !== null && observedTotal > 0 ? { revision: responseOrdinal, totalTokens: observedTotal } : null;
|
|
6917
6914
|
const observedInput = normalizedUsage.telemetry.inputTokens;
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6915
|
+
await input.setLastInputTokens(
|
|
6916
|
+
observedInput !== null && observedInput > 0 ? observedInput : null
|
|
6917
|
+
);
|
|
6921
6918
|
}
|
|
6922
6919
|
});
|
|
6923
|
-
return {
|
|
6920
|
+
return {
|
|
6921
|
+
status: "processed",
|
|
6922
|
+
sourceKey,
|
|
6923
|
+
authoritative,
|
|
6924
|
+
usageReported: responseUsage !== null
|
|
6925
|
+
};
|
|
6924
6926
|
}
|
|
6925
6927
|
async function processCompactionModelUsageEvent(input) {
|
|
6926
6928
|
const usageOrdinal = input.state.usageCount + 1;
|
|
@@ -7432,9 +7434,6 @@ function createTurnSandboxProvisioner(establish, options = {}) {
|
|
|
7432
7434
|
}
|
|
7433
7435
|
};
|
|
7434
7436
|
}
|
|
7435
|
-
function sdkBackendIdForSandboxBackend(backend) {
|
|
7436
|
-
return backend === "local" ? "unix_local" : backend;
|
|
7437
|
-
}
|
|
7438
7437
|
function acceptsPromptCacheKeyForTurn(resolvedModel) {
|
|
7439
7438
|
if (!resolvedModel) {
|
|
7440
7439
|
return true;
|
|
@@ -10666,10 +10665,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
10666
10665
|
attemptId: input.attemptId
|
|
10667
10666
|
},
|
|
10668
10667
|
// Never reuse the persisted prior-turn signal for recovery. Proactive
|
|
10669
|
-
// guards provide their exact current signal; provider
|
|
10670
|
-
//
|
|
10671
|
-
// recovery proves progress separately by comparing the
|
|
10672
|
-
// with the current active-history estimate
|
|
10668
|
+
// guards provide their exact current provider-based signal; provider
|
|
10669
|
+
// overflows do not, so their diagnostic signal stays unknown/zero.
|
|
10670
|
+
// Forced recovery proves progress separately by comparing the
|
|
10671
|
+
// replacement with the current active-history estimate.
|
|
10673
10672
|
recoverySignalTokens,
|
|
10674
10673
|
compactSummarizer,
|
|
10675
10674
|
{
|
|
@@ -10691,9 +10690,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10691
10690
|
}
|
|
10692
10691
|
return outcome;
|
|
10693
10692
|
};
|
|
10694
|
-
const
|
|
10695
|
-
claimedModelUsageSourceKeys
|
|
10696
|
-
);
|
|
10693
|
+
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
10697
10694
|
const runStreamAttempt = async () => {
|
|
10698
10695
|
if (!runInput) {
|
|
10699
10696
|
throw new Error("Run input was not prepared");
|
|
@@ -10702,6 +10699,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10702
10699
|
batcher = null;
|
|
10703
10700
|
let currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
10704
10701
|
let currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
10702
|
+
let streamSawPerResponseUsage = false;
|
|
10705
10703
|
throwIfWorkerShuttingDown();
|
|
10706
10704
|
throwIfTurnCancelled();
|
|
10707
10705
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
@@ -10785,7 +10783,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10785
10783
|
}
|
|
10786
10784
|
} : {}
|
|
10787
10785
|
} : {},
|
|
10788
|
-
contextCompactionSignal: () =>
|
|
10786
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
10789
10787
|
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
10790
10788
|
callModelInputFilter: secretRedactionModelInputFilter((value) => redact(value)),
|
|
10791
10789
|
...toolCancellationFenceRef.current ? { turnToolCancellationFence: toolCancellationFenceRef.current } : {}
|
|
@@ -10818,9 +10816,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
10818
10816
|
}
|
|
10819
10817
|
let stableToolCallIdsToClear = null;
|
|
10820
10818
|
let completedCurrentToolBatch = false;
|
|
10821
|
-
const
|
|
10819
|
+
const responseResult = await processModelResponseTerminalEvent({
|
|
10822
10820
|
event: next.value,
|
|
10823
|
-
state:
|
|
10821
|
+
state: modelResponseState,
|
|
10824
10822
|
dispatchId: modelUsageDispatchId,
|
|
10825
10823
|
settings,
|
|
10826
10824
|
db,
|
|
@@ -10851,7 +10849,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
10851
10849
|
model: turn.model,
|
|
10852
10850
|
...resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}
|
|
10853
10851
|
});
|
|
10854
|
-
if (
|
|
10852
|
+
if (responseResult.status === "processed") {
|
|
10853
|
+
streamSawPerResponseUsage ||= responseResult.usageReported;
|
|
10855
10854
|
currentToolBatchCallIds = /* @__PURE__ */ new Set();
|
|
10856
10855
|
currentToolBatchCompletedCallIds = /* @__PURE__ */ new Set();
|
|
10857
10856
|
await reconcileConversationTruth();
|
|
@@ -10964,7 +10963,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10964
10963
|
stream.completed.catch(() => void 0),
|
|
10965
10964
|
cancellationSignal
|
|
10966
10965
|
);
|
|
10967
|
-
if (
|
|
10966
|
+
if (!streamSawPerResponseUsage) {
|
|
10968
10967
|
const aggregateUsage = stream.state.usage;
|
|
10969
10968
|
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
10970
10969
|
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
@@ -11041,13 +11040,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
11041
11040
|
},
|
|
11042
11041
|
recordAttemptSignals: async () => {
|
|
11043
11042
|
if (!aggregateAuthoritative) return;
|
|
11044
|
-
|
|
11045
|
-
|
|
11046
|
-
modelResponseUsageState.providerContextRevision += 1;
|
|
11047
|
-
}
|
|
11048
|
-
if (aggregateInput !== null && aggregateInput > 0) {
|
|
11049
|
-
await setLastInputTokensFenced(aggregateInput);
|
|
11050
|
-
}
|
|
11043
|
+
modelResponseState.contextSignal = null;
|
|
11044
|
+
await setLastInputTokensFenced(null);
|
|
11051
11045
|
}
|
|
11052
11046
|
});
|
|
11053
11047
|
}
|
|
@@ -13108,21 +13102,69 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
|
|
|
13108
13102
|
}
|
|
13109
13103
|
|
|
13110
13104
|
// src/activities/file-upload-reaper.ts
|
|
13111
|
-
import {
|
|
13105
|
+
import {
|
|
13106
|
+
claimDueTranscriptionRecordingObjectCleanup,
|
|
13107
|
+
claimExpiredFileUploadCleanup,
|
|
13108
|
+
completeDueTranscriptionRecordingObjectCleanup,
|
|
13109
|
+
completeExpiredFileUploadCleanup,
|
|
13110
|
+
purgeExpiredTranscriptionRecordings
|
|
13111
|
+
} from "@opengeni/db";
|
|
13112
13112
|
var FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1e3;
|
|
13113
13113
|
var FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
13114
13114
|
var FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
|
|
13115
|
+
var SAFE_CLEANUP_ERROR_CATEGORIES = /* @__PURE__ */ new Set([
|
|
13116
|
+
"aborted",
|
|
13117
|
+
"conflict",
|
|
13118
|
+
"invalid_response",
|
|
13119
|
+
"network",
|
|
13120
|
+
"not_found",
|
|
13121
|
+
"permission_denied",
|
|
13122
|
+
"provider",
|
|
13123
|
+
"timeout",
|
|
13124
|
+
"unavailable"
|
|
13125
|
+
]);
|
|
13126
|
+
function cleanupErrorAttributes(error) {
|
|
13127
|
+
let errorCategory = "provider_error";
|
|
13128
|
+
let errorStatus;
|
|
13129
|
+
if (error && typeof error === "object") {
|
|
13130
|
+
const rawCode = error.code;
|
|
13131
|
+
if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
|
|
13132
|
+
errorCategory = rawCode;
|
|
13133
|
+
} else if (rawCode !== void 0) {
|
|
13134
|
+
errorCategory = "unknown";
|
|
13135
|
+
}
|
|
13136
|
+
const rawStatus = Number(
|
|
13137
|
+
error.status ?? error.statusCode
|
|
13138
|
+
);
|
|
13139
|
+
if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
|
|
13140
|
+
errorStatus = rawStatus;
|
|
13141
|
+
}
|
|
13142
|
+
}
|
|
13143
|
+
return { errorCategory, ...errorStatus === void 0 ? {} : { errorStatus } };
|
|
13144
|
+
}
|
|
13115
13145
|
function createFileUploadReaperActivities(services, options = {}) {
|
|
13116
13146
|
const graceMs = options.graceMs ?? FILE_UPLOAD_CLEANUP_GRACE_MS;
|
|
13117
13147
|
const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
|
|
13118
13148
|
const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
|
|
13119
13149
|
const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
|
|
13150
|
+
const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
|
|
13151
|
+
const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
|
|
13152
|
+
const claimTranscriptionObjects = options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
|
|
13153
|
+
const completeTranscriptionObject = options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
|
|
13154
|
+
const purgeTranscriptionRecordings = options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
|
|
13120
13155
|
async function reapExpiredFileUploads2() {
|
|
13121
13156
|
const { db, objectStorage, observability } = await services();
|
|
13122
13157
|
if (!objectStorage) {
|
|
13158
|
+
try {
|
|
13159
|
+
await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
|
|
13160
|
+
} catch (error) {
|
|
13161
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
13162
|
+
...cleanupErrorAttributes(error)
|
|
13163
|
+
});
|
|
13164
|
+
}
|
|
13123
13165
|
return { claimed: 0, deleted: 0, failed: 0 };
|
|
13124
13166
|
}
|
|
13125
|
-
const claims = await
|
|
13167
|
+
const claims = await claimFileUploads(db, {
|
|
13126
13168
|
graceMs,
|
|
13127
13169
|
claimTimeoutMs,
|
|
13128
13170
|
limit: batchSize
|
|
@@ -13132,7 +13174,7 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
13132
13174
|
for (const claim of claims) {
|
|
13133
13175
|
try {
|
|
13134
13176
|
await deleteObject(objectStorage, claim.objectKey);
|
|
13135
|
-
const settled = await
|
|
13177
|
+
const settled = await completeFileUpload(db, claim);
|
|
13136
13178
|
if (!settled) {
|
|
13137
13179
|
throw new Error("cleanup claim no longer owns a reclaimable upload");
|
|
13138
13180
|
}
|
|
@@ -13143,18 +13185,58 @@ function createFileUploadReaperActivities(services, options = {}) {
|
|
|
13143
13185
|
workspaceId: claim.workspaceId,
|
|
13144
13186
|
uploadId: claim.uploadId,
|
|
13145
13187
|
fileId: claim.fileId,
|
|
13146
|
-
|
|
13188
|
+
...cleanupErrorAttributes(error)
|
|
13147
13189
|
});
|
|
13148
13190
|
}
|
|
13149
13191
|
}
|
|
13150
|
-
|
|
13192
|
+
const recordingClaims = await claimTranscriptionObjects(db, {
|
|
13193
|
+
graceMs,
|
|
13194
|
+
claimTimeoutMs,
|
|
13195
|
+
limit: batchSize
|
|
13196
|
+
});
|
|
13197
|
+
for (const claim of recordingClaims) {
|
|
13198
|
+
try {
|
|
13199
|
+
await deleteObject(objectStorage, claim.objectKey);
|
|
13200
|
+
const settled = await completeTranscriptionObject(db, claim);
|
|
13201
|
+
if (!settled) {
|
|
13202
|
+
throw new Error("cleanup claim no longer owns a transcription recording object");
|
|
13203
|
+
}
|
|
13204
|
+
deleted += 1;
|
|
13205
|
+
} catch (error) {
|
|
13206
|
+
failed += 1;
|
|
13207
|
+
observability.warn(
|
|
13208
|
+
"expired transcription recording object cleanup failed; claim remains reclaimable",
|
|
13209
|
+
{
|
|
13210
|
+
workspaceId: claim.workspaceId,
|
|
13211
|
+
recordingId: claim.recordingId,
|
|
13212
|
+
...cleanupErrorAttributes(error)
|
|
13213
|
+
}
|
|
13214
|
+
);
|
|
13215
|
+
}
|
|
13216
|
+
}
|
|
13217
|
+
let transcriptionRecordingsPurged = 0;
|
|
13218
|
+
try {
|
|
13219
|
+
transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
|
|
13220
|
+
graceMs,
|
|
13221
|
+
limit: batchSize
|
|
13222
|
+
});
|
|
13223
|
+
} catch (error) {
|
|
13224
|
+
observability.warn("expired transcription recording metadata purge failed", {
|
|
13225
|
+
...cleanupErrorAttributes(error)
|
|
13226
|
+
});
|
|
13227
|
+
}
|
|
13228
|
+
const claimed = claims.length + recordingClaims.length;
|
|
13229
|
+
if (claimed > 0 || transcriptionRecordingsPurged > 0) {
|
|
13151
13230
|
observability.info("expired file upload cleanup swept", {
|
|
13152
|
-
claimed
|
|
13231
|
+
claimed,
|
|
13153
13232
|
deleted,
|
|
13154
|
-
failed
|
|
13233
|
+
failed,
|
|
13234
|
+
fileUploads: claims.length,
|
|
13235
|
+
transcriptionRecordingObjects: recordingClaims.length,
|
|
13236
|
+
transcriptionRecordingsPurged
|
|
13155
13237
|
});
|
|
13156
13238
|
}
|
|
13157
|
-
return { claimed
|
|
13239
|
+
return { claimed, deleted, failed };
|
|
13158
13240
|
}
|
|
13159
13241
|
return { reapExpiredFileUploads: reapExpiredFileUploads2 };
|
|
13160
13242
|
}
|
|
@@ -13215,6 +13297,7 @@ import {
|
|
|
13215
13297
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
13216
13298
|
resolveModalCheckpointProviderBinding,
|
|
13217
13299
|
resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession3,
|
|
13300
|
+
sandboxBackendForSdkBackendId as sandboxBackendForSdkBackendId2,
|
|
13218
13301
|
sandboxCommandExitCode,
|
|
13219
13302
|
sandboxCommandStillRunning,
|
|
13220
13303
|
sweepModalOrphanSandboxes,
|
|
@@ -14132,7 +14215,8 @@ async function probeDrainableProviderReadiness(settings, lease) {
|
|
|
14132
14215
|
if (!lease.instanceId || !lease.resumeState) {
|
|
14133
14216
|
throw new Error("Expired workspace capture has no resumable provider identity");
|
|
14134
14217
|
}
|
|
14135
|
-
const
|
|
14218
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
14219
|
+
const backend = sandboxBackendForSdkBackendId2(durableBackendId) ?? durableBackendId;
|
|
14136
14220
|
let established;
|
|
14137
14221
|
try {
|
|
14138
14222
|
established = await establishSandboxSessionFromEnvelope3(settings, lease.resumeState, {
|
|
@@ -14362,7 +14446,8 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
|
|
|
14362
14446
|
}
|
|
14363
14447
|
}
|
|
14364
14448
|
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend) {
|
|
14365
|
-
const
|
|
14449
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
14450
|
+
const backend = sandboxBackendForSdkBackendId2(durableBackendId) ?? durableBackendId;
|
|
14366
14451
|
if (!backend || backend === "none") {
|
|
14367
14452
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
14368
14453
|
}
|