@opengeni/worker-bundle 0.16.29 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +85 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +662 -17374
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +2 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1043 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1470 -246
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +6 -6
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +45 -42
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +77 -4
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +48 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +199 -4
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +19 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +3 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
package/src/sandbox-routing.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// target lookup), and the selfhosted ControlRpc built over the events bus's NATS
|
|
15
15
|
// request/reply connection.
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
|
|
18
18
|
import {
|
|
19
19
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
20
20
|
advanceWorkspaceGeneration,
|
|
@@ -45,7 +45,9 @@ import {
|
|
|
45
45
|
RoutingBackendRecoveryRequiredError,
|
|
46
46
|
RoutingSandboxSession,
|
|
47
47
|
resolveModalCheckpointProviderBindingForSession,
|
|
48
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
48
49
|
sandboxBackendForSdkBackendId,
|
|
50
|
+
selectBackend,
|
|
49
51
|
verifySandboxExecReadiness,
|
|
50
52
|
type ControlRpc,
|
|
51
53
|
type EstablishedSandboxSession,
|
|
@@ -100,6 +102,8 @@ export type RoutingWiringServices = {
|
|
|
100
102
|
established: EstablishedSandboxSession;
|
|
101
103
|
leaseEpoch: number;
|
|
102
104
|
}) => void;
|
|
105
|
+
/** Cancel bounded capture waits with the owning turn activity. */
|
|
106
|
+
waitSignal?: AbortSignal;
|
|
103
107
|
};
|
|
104
108
|
|
|
105
109
|
/** Worker-turn routing may final-ack streamed command output, so it must always
|
|
@@ -262,27 +266,7 @@ function homeRouteRecoveryError(
|
|
|
262
266
|
export function providerIdentityFromResumeState(
|
|
263
267
|
resumeState: Record<string, unknown>,
|
|
264
268
|
): string | null {
|
|
265
|
-
|
|
266
|
-
resumeState.sessionState && typeof resumeState.sessionState === "object"
|
|
267
|
-
? (resumeState.sessionState as Record<string, unknown>)
|
|
268
|
-
: null;
|
|
269
|
-
const providerState =
|
|
270
|
-
sessionState?.providerState && typeof sessionState.providerState === "object"
|
|
271
|
-
? (sessionState.providerState as Record<string, unknown>)
|
|
272
|
-
: null;
|
|
273
|
-
for (const field of [
|
|
274
|
-
"sandboxId",
|
|
275
|
-
"instanceId",
|
|
276
|
-
"id",
|
|
277
|
-
"hostId",
|
|
278
|
-
"containerId",
|
|
279
|
-
"workspaceRootPath",
|
|
280
|
-
"agentId",
|
|
281
|
-
]) {
|
|
282
|
-
const value = providerState?.[field];
|
|
283
|
-
if (typeof value === "string" && value.length > 0) return value;
|
|
284
|
-
}
|
|
285
|
-
return null;
|
|
269
|
+
return sandboxProviderInstanceIdFromEnvelope(resumeState);
|
|
286
270
|
}
|
|
287
271
|
|
|
288
272
|
/**
|
|
@@ -448,7 +432,8 @@ function beforePersistableHomeMutation(
|
|
|
448
432
|
expectedEpoch: backend.leaseEpoch,
|
|
449
433
|
expectedInstanceId: backend.providerInstanceId,
|
|
450
434
|
operation: op,
|
|
451
|
-
captureWaitMs:
|
|
435
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
|
|
436
|
+
...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
|
|
452
437
|
});
|
|
453
438
|
return { admission, providerBinding };
|
|
454
439
|
};
|
|
@@ -583,7 +568,8 @@ function beforeRetainedProcessMutation(
|
|
|
583
568
|
sessionId: ids.sessionId,
|
|
584
569
|
processId: process.id,
|
|
585
570
|
operation: op,
|
|
586
|
-
captureWaitMs:
|
|
571
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
|
|
572
|
+
...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
|
|
587
573
|
});
|
|
588
574
|
}
|
|
589
575
|
|
|
@@ -884,8 +870,17 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
884
870
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
885
871
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
886
872
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
873
|
+
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
874
|
+
const defaultSupportsPty = defaultBackend
|
|
875
|
+
? selectBackend(defaultBackend).capabilities.Terminal.pty
|
|
876
|
+
: false;
|
|
887
877
|
const syntheticSession: RoutableBackendSession = {
|
|
888
878
|
state: { manifest: args.agentDefaultManifest },
|
|
879
|
+
// The SDK binds shell tools synchronously before lazy provisioning. Preserve
|
|
880
|
+
// the authoritative descriptor's PTY shape so a Modal home exposes
|
|
881
|
+
// write_stdin on its first turn operation; the async cancellation transport
|
|
882
|
+
// below still resolves the exact live route before executing a command.
|
|
883
|
+
supportsPty: () => defaultSupportsPty,
|
|
889
884
|
};
|
|
890
885
|
const routedResolver = makeActiveBackendResolver({
|
|
891
886
|
workspaceId: ids.workspaceId,
|
|
@@ -20,10 +20,58 @@ export type WorkerRunTarget = {
|
|
|
20
20
|
shutdown(): void;
|
|
21
21
|
};
|
|
22
22
|
|
|
23
|
+
/** One process may poll several disjoint Temporal queues, but it still has one
|
|
24
|
+
* host lifecycle. Start and stop every poller as a unit; a failed poller drains
|
|
25
|
+
* its siblings and preserves the original failure. */
|
|
26
|
+
export function combineWorkerRunTargets(targets: readonly WorkerRunTarget[]): WorkerRunTarget {
|
|
27
|
+
if (targets.length === 0) {
|
|
28
|
+
throw new Error("a worker service requires at least one Temporal poller");
|
|
29
|
+
}
|
|
30
|
+
let runPromise: Promise<void> | undefined;
|
|
31
|
+
const shutdown = (): void => {
|
|
32
|
+
let firstError: unknown;
|
|
33
|
+
for (const target of targets) {
|
|
34
|
+
try {
|
|
35
|
+
target.shutdown();
|
|
36
|
+
} catch (error) {
|
|
37
|
+
firstError ??= error;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (firstError !== undefined) throw firstError;
|
|
41
|
+
};
|
|
42
|
+
return {
|
|
43
|
+
run: () => {
|
|
44
|
+
runPromise ??= (async () => {
|
|
45
|
+
const runs = targets.map((target) => {
|
|
46
|
+
try {
|
|
47
|
+
return target.run();
|
|
48
|
+
} catch (error) {
|
|
49
|
+
return Promise.reject(error);
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
try {
|
|
53
|
+
await Promise.all(runs);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
try {
|
|
56
|
+
shutdown();
|
|
57
|
+
} catch {
|
|
58
|
+
// Preserve the poller failure. Every sibling still received its
|
|
59
|
+
// shutdown request even when one request itself failed.
|
|
60
|
+
}
|
|
61
|
+
await Promise.allSettled(runs);
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
})();
|
|
65
|
+
return runPromise;
|
|
66
|
+
},
|
|
67
|
+
shutdown,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
23
71
|
export type WorkerServiceLifecycle = {
|
|
24
72
|
state(): WorkerLifecycleState;
|
|
25
73
|
run(): Promise<void>;
|
|
26
|
-
drain(reason?: string):
|
|
74
|
+
drain(reason?: string): boolean;
|
|
27
75
|
close(): Promise<void>;
|
|
28
76
|
};
|
|
29
77
|
|
|
@@ -74,27 +122,37 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
74
122
|
},
|
|
75
123
|
drain: (_reason = "host request") => {
|
|
76
124
|
if (state === "draining" || state === "stopped" || state === "failed") {
|
|
77
|
-
return;
|
|
125
|
+
return true;
|
|
78
126
|
}
|
|
127
|
+
const previousState = state;
|
|
128
|
+
safeLifecycleLog(() =>
|
|
129
|
+
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
130
|
+
role: input.role,
|
|
131
|
+
errorClass: "WorkerLifecycleOperation",
|
|
132
|
+
errorCode: "worker_draining",
|
|
133
|
+
origin: "worker-lifecycle",
|
|
134
|
+
}),
|
|
135
|
+
);
|
|
79
136
|
state = "draining";
|
|
80
|
-
input.observability.info("OpenGeni worker draining (graceful shutdown)", {
|
|
81
|
-
role: input.role,
|
|
82
|
-
errorClass: "WorkerLifecycleOperation",
|
|
83
|
-
errorCode: "worker_draining",
|
|
84
|
-
origin: "worker-lifecycle",
|
|
85
|
-
});
|
|
86
137
|
try {
|
|
87
138
|
input.worker.shutdown();
|
|
139
|
+
return true;
|
|
88
140
|
} catch {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
141
|
+
state = previousState;
|
|
142
|
+
safeLifecycleLog(() =>
|
|
143
|
+
input.observability.warn("worker shutdown request failed", {
|
|
144
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
145
|
+
errorCode: "worker_shutdown_request_failed",
|
|
146
|
+
origin: "worker-lifecycle",
|
|
147
|
+
}),
|
|
148
|
+
);
|
|
149
|
+
return false;
|
|
94
150
|
}
|
|
95
151
|
},
|
|
96
152
|
close: async () => {
|
|
97
|
-
lifecycle.drain("service close")
|
|
153
|
+
if (!lifecycle.drain("service close")) {
|
|
154
|
+
throw new Error("worker shutdown request failed");
|
|
155
|
+
}
|
|
98
156
|
if (runPromise) {
|
|
99
157
|
await runPromise.catch(() => undefined);
|
|
100
158
|
} else {
|
|
@@ -106,3 +164,11 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
106
164
|
|
|
107
165
|
return lifecycle;
|
|
108
166
|
}
|
|
167
|
+
|
|
168
|
+
function safeLifecycleLog(log: () => void): void {
|
|
169
|
+
try {
|
|
170
|
+
log();
|
|
171
|
+
} catch {
|
|
172
|
+
// Telemetry cannot change worker lifecycle state or shutdown delivery.
|
|
173
|
+
}
|
|
174
|
+
}
|
|
@@ -91,11 +91,41 @@ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCanc
|
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
export function videoGenerationActivityForTaskQueue(baseTaskQueue: string) {
|
|
95
|
+
return proxyActivities<Pick<typeof activities, "reconcileVideoGenerationOperation">>({
|
|
96
|
+
taskQueue: turnTaskQueue(baseTaskQueue),
|
|
97
|
+
startToCloseTimeout: "20 minutes",
|
|
98
|
+
retry: {
|
|
99
|
+
initialInterval: "1 second",
|
|
100
|
+
backoffCoefficient: 2,
|
|
101
|
+
maximumInterval: "30 seconds",
|
|
102
|
+
maximumAttempts: 3,
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
94
107
|
export const documentActivity = proxyActivities<Pick<typeof activities, "indexDocument">>({
|
|
95
108
|
startToCloseTimeout: "30 minutes",
|
|
96
109
|
retry: { maximumAttempts: 1 },
|
|
97
110
|
});
|
|
98
111
|
|
|
112
|
+
/** Connector inventory/download/index batches are provider-I/O activities. The
|
|
113
|
+
* implementation heartbeats between bounded items and owns its resumable
|
|
114
|
+
* checkpoint; Temporal retries therefore repeat only idempotent source/object
|
|
115
|
+
* operations and never invoke the agent runtime. */
|
|
116
|
+
export const knowledgeSourceSyncActivity = proxyActivities<
|
|
117
|
+
Pick<typeof activities, "runKnowledgeSourceSyncBatch">
|
|
118
|
+
>({
|
|
119
|
+
startToCloseTimeout: "1 hour",
|
|
120
|
+
heartbeatTimeout: "2 minutes",
|
|
121
|
+
retry: {
|
|
122
|
+
initialInterval: "2 seconds",
|
|
123
|
+
backoffCoefficient: 2,
|
|
124
|
+
maximumInterval: "1 minute",
|
|
125
|
+
maximumAttempts: 5,
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
|
|
99
129
|
export function workflowFailureMessage(error: unknown): string {
|
|
100
130
|
if (error instanceof Error) {
|
|
101
131
|
return error.message;
|
|
@@ -2,7 +2,8 @@ import { proxyActivities } from "@temporalio/workflow";
|
|
|
2
2
|
import type * as activities from "../activities";
|
|
3
3
|
|
|
4
4
|
const reaperActivity = proxyActivities<
|
|
5
|
-
Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots">
|
|
5
|
+
Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots"> &
|
|
6
|
+
Pick<typeof activities, "recoverVideoGenerationWorkflows">
|
|
6
7
|
>({
|
|
7
8
|
startToCloseTimeout: "5 minutes",
|
|
8
9
|
retry: { maximumAttempts: 1 },
|
|
@@ -12,4 +13,5 @@ const reaperActivity = proxyActivities<
|
|
|
12
13
|
export async function fileUploadReaperWorkflow(): Promise<void> {
|
|
13
14
|
await reaperActivity.reapExpiredFileUploads();
|
|
14
15
|
await reaperActivity.maintainRetainedScreenshots();
|
|
16
|
+
await reaperActivity.recoverVideoGenerationWorkflows();
|
|
15
17
|
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { continueAsNew, defineSignal, setHandler, workflowInfo } from "@temporalio/workflow";
|
|
2
|
+
import type { RunKnowledgeSourceSyncBatchInput } from "../activities";
|
|
3
|
+
import { knowledgeSourceSyncActivity } from "./activities";
|
|
4
|
+
|
|
5
|
+
export type KnowledgeSourceSyncWorkflowInput = RunKnowledgeSourceSyncBatchInput;
|
|
6
|
+
|
|
7
|
+
export const knowledgeSourceSyncWake =
|
|
8
|
+
defineSignal<[KnowledgeSourceSyncWorkflowInput]>("knowledgeSourceSyncWake");
|
|
9
|
+
|
|
10
|
+
type KnowledgeSourceSyncWorkflowState = {
|
|
11
|
+
pending: KnowledgeSourceSyncWorkflowInput | null;
|
|
12
|
+
batchesThisRun: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
const MAX_BATCHES_PER_WORKFLOW_RUN = 50;
|
|
16
|
+
|
|
17
|
+
/** One stable workflow ID serializes all fires for a source. Signals retain at
|
|
18
|
+
* most one newest pending wake; Postgres lease/checkpoint rows remain the
|
|
19
|
+
* authoritative overlap and resumability fence across workflow generations. */
|
|
20
|
+
export async function knowledgeSourceSyncWorkflow(
|
|
21
|
+
input: KnowledgeSourceSyncWorkflowInput,
|
|
22
|
+
carried?: KnowledgeSourceSyncWorkflowState,
|
|
23
|
+
): Promise<void> {
|
|
24
|
+
let current = input;
|
|
25
|
+
let pending = carried?.pending ?? null;
|
|
26
|
+
let batchesThisRun = carried?.batchesThisRun ?? 0;
|
|
27
|
+
|
|
28
|
+
setHandler(knowledgeSourceSyncWake, (next) => {
|
|
29
|
+
if (
|
|
30
|
+
next.accountId !== input.accountId ||
|
|
31
|
+
next.workspaceId !== input.workspaceId ||
|
|
32
|
+
next.sourceId !== input.sourceId
|
|
33
|
+
) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
pending = next;
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
while (true) {
|
|
40
|
+
const result = await knowledgeSourceSyncActivity.runKnowledgeSourceSyncBatch(current);
|
|
41
|
+
batchesThisRun += 1;
|
|
42
|
+
|
|
43
|
+
if (result.action === "continue") {
|
|
44
|
+
if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
|
|
45
|
+
await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
|
|
46
|
+
pending,
|
|
47
|
+
batchesThisRun: 0,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (
|
|
54
|
+
(result.action === "complete" || result.action === "failed") &&
|
|
55
|
+
result.bufferedScheduledTaskRunId
|
|
56
|
+
) {
|
|
57
|
+
current = {
|
|
58
|
+
...current,
|
|
59
|
+
scheduledTaskRunId: result.bufferedScheduledTaskRunId,
|
|
60
|
+
};
|
|
61
|
+
pending = null;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (pending) {
|
|
66
|
+
current = pending;
|
|
67
|
+
pending = null;
|
|
68
|
+
if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
|
|
69
|
+
await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
|
|
70
|
+
pending: null,
|
|
71
|
+
batchesThisRun: 0,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function knowledgeSourceSyncWorkflowId(sourceId: string): string {
|
|
82
|
+
return `knowledge-source-sync-${sourceId}`;
|
|
83
|
+
}
|
|
@@ -1,28 +1,190 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
import { proxyActivities } from "@temporalio/workflow";
|
|
1
|
+
// The global schedule performs only bounded inventory/maintenance work and
|
|
2
|
+
// starts one durable child per exact (workspace, group, epoch). Provider I/O is
|
|
3
|
+
// therefore parallel at the Temporal worker's activity-concurrency limit; one
|
|
4
|
+
// slow snapshot can delay only its own box, never the global reaper or a sibling.
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
ParentClosePolicy,
|
|
8
|
+
WorkflowIdReusePolicy,
|
|
9
|
+
log,
|
|
10
|
+
proxyActivities,
|
|
11
|
+
startChild,
|
|
12
|
+
uuid4,
|
|
13
|
+
workflowInfo,
|
|
14
|
+
} from "@temporalio/workflow";
|
|
16
15
|
import type * as activities from "../activities";
|
|
16
|
+
import {
|
|
17
|
+
SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
|
|
18
|
+
SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS,
|
|
19
|
+
SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
20
|
+
SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
|
|
21
|
+
SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
22
|
+
sandboxDrainWorkflowId,
|
|
23
|
+
sandboxLifecycleTaskQueue,
|
|
24
|
+
type SandboxDrainActivityInput,
|
|
25
|
+
type SandboxDrainWorkflowInput,
|
|
26
|
+
type SandboxLeaseSweepMaintenanceInput,
|
|
27
|
+
} from "../sandbox-reaper-contract";
|
|
28
|
+
|
|
29
|
+
function lifecycleTaskQueue(): string {
|
|
30
|
+
return sandboxLifecycleTaskQueue(workflowInfo().taskQueue);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function sweepActivities() {
|
|
34
|
+
return proxyActivities<Pick<typeof activities, "prepareSandboxLeaseSweep">>({
|
|
35
|
+
taskQueue: lifecycleTaskQueue(),
|
|
36
|
+
startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
37
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
38
|
+
// Inventory is the admission edge for every per-box child. A worker crash
|
|
39
|
+
// must not silently consume the only scheduled sweep.
|
|
40
|
+
retry: {
|
|
41
|
+
initialInterval: "1 second",
|
|
42
|
+
backoffCoefficient: 2,
|
|
43
|
+
maximumInterval: "30 seconds",
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function maintenanceActivity() {
|
|
49
|
+
return proxyActivities<Pick<typeof activities, "maintainSandboxLeaseSweep">>({
|
|
50
|
+
taskQueue: lifecycleTaskQueue(),
|
|
51
|
+
startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
52
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
53
|
+
// Maintenance is ancillary and internally exact/idempotent. Never let one
|
|
54
|
+
// pathological pass keep the fixed-id V2 workflow open forever and thereby
|
|
55
|
+
// suppress all later drain inventories. The next Schedule tick retries it.
|
|
56
|
+
retry: { maximumAttempts: 1 },
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function drainActivity(startToCloseTimeout: number) {
|
|
61
|
+
return proxyActivities<Pick<typeof activities, "drainSandboxLease">>({
|
|
62
|
+
taskQueue: lifecycleTaskQueue(),
|
|
63
|
+
startToCloseTimeout,
|
|
64
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
65
|
+
// Each retry is a new exact DB capture attempt under one logical operation.
|
|
66
|
+
// Heartbeat timeout moves a crashed worker to a healthy poller promptly;
|
|
67
|
+
// exact-attempt CAS fencing makes late callbacks harmless.
|
|
68
|
+
retry: {
|
|
69
|
+
initialInterval: "1 second",
|
|
70
|
+
backoffCoefficient: 2,
|
|
71
|
+
maximumInterval: "30 seconds",
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
}
|
|
17
75
|
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
|
|
22
|
-
|
|
76
|
+
// This proxy and workflow deliberately preserve the exact pre-refactor
|
|
77
|
+
// ScheduleActivity command. A patch marker would let a new binary replay old
|
|
78
|
+
// histories, but an old binary could not replay a history first executed by a
|
|
79
|
+
// new worker during a rolling deployment. Routing happens inside the activity,
|
|
80
|
+
// where changing implementation is safe and mixed old/new workers remain valid.
|
|
81
|
+
const legacyReaperActivity = proxyActivities<Pick<typeof activities, "reapSandboxLeases">>({
|
|
82
|
+
startToCloseTimeout: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
|
|
23
83
|
retry: { maximumAttempts: 1 },
|
|
24
84
|
});
|
|
25
85
|
|
|
86
|
+
export async function sandboxDrainWorkflow(input: SandboxDrainWorkflowInput): Promise<void> {
|
|
87
|
+
const activityInput: SandboxDrainActivityInput = {
|
|
88
|
+
...input,
|
|
89
|
+
operationId: uuid4(),
|
|
90
|
+
};
|
|
91
|
+
if (input.timeoutClass === "fast") {
|
|
92
|
+
await drainActivity(SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS).drainSandboxLease(activityInput);
|
|
93
|
+
} else {
|
|
94
|
+
await drainActivity(SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS).drainSandboxLease(
|
|
95
|
+
activityInput,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Ancillary billing/reconciliation/GC is durable but never allowed to hold the
|
|
101
|
+
* fixed-ID inventory workflow open behind a provider backlog. One singleton
|
|
102
|
+
* maintenance child may queue on the lifecycle workers while every schedule
|
|
103
|
+
* tick remains free to inventory and dispatch newly-drainable boxes. */
|
|
104
|
+
export async function sandboxReaperMaintenanceWorkflow(
|
|
105
|
+
input: SandboxLeaseSweepMaintenanceInput,
|
|
106
|
+
): Promise<void> {
|
|
107
|
+
try {
|
|
108
|
+
await maintenanceActivity().maintainSandboxLeaseSweep(input);
|
|
109
|
+
} catch (error) {
|
|
110
|
+
// The next Schedule tick starts another singleton after this run closes.
|
|
111
|
+
// Drain children were already made durable before this workflow was started.
|
|
112
|
+
log.warn("sandbox reaper maintenance deferred to a later sweep", {
|
|
113
|
+
error: error instanceof Error ? error.message : String(error),
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function alreadyRunningChild(error: unknown): boolean {
|
|
119
|
+
let current: unknown = error;
|
|
120
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
121
|
+
const record = current as { name?: unknown; cause?: unknown };
|
|
122
|
+
if (record.name === "WorkflowExecutionAlreadyStartedError") return true;
|
|
123
|
+
current = record.cause;
|
|
124
|
+
}
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
|
|
26
128
|
export async function sandboxReaperWorkflow(): Promise<void> {
|
|
27
|
-
await
|
|
129
|
+
await legacyReaperActivity.reapSandboxLeases();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export async function sandboxReaperWorkflowV2(): Promise<void> {
|
|
133
|
+
const sweep = sweepActivities();
|
|
134
|
+
const plan = await sweep.prepareSandboxLeaseSweep();
|
|
135
|
+
const starts = await Promise.all(
|
|
136
|
+
plan.drainable.map(async (target) => {
|
|
137
|
+
try {
|
|
138
|
+
await startChild(sandboxDrainWorkflow, {
|
|
139
|
+
workflowId: sandboxDrainWorkflowId(target),
|
|
140
|
+
taskQueue: lifecycleTaskQueue(),
|
|
141
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
142
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
143
|
+
args: [
|
|
144
|
+
{
|
|
145
|
+
target,
|
|
146
|
+
timeoutClass: plan.timeoutClass,
|
|
147
|
+
snapshotTimeoutMs: plan.snapshotTimeoutMs,
|
|
148
|
+
captureTimeoutMs: plan.captureTimeoutMs,
|
|
149
|
+
},
|
|
150
|
+
],
|
|
151
|
+
});
|
|
152
|
+
return "started" as const;
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (alreadyRunningChild(error)) return "already_running" as const;
|
|
155
|
+
log.error("sandbox reaper child start failed", {
|
|
156
|
+
workspaceId: target.workspaceId,
|
|
157
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
158
|
+
leaseEpoch: target.leaseEpoch,
|
|
159
|
+
instanceId: target.instanceId,
|
|
160
|
+
error: error instanceof Error ? error.message : String(error),
|
|
161
|
+
});
|
|
162
|
+
return "failed" as const;
|
|
163
|
+
}
|
|
164
|
+
}),
|
|
165
|
+
);
|
|
166
|
+
try {
|
|
167
|
+
await startChild(sandboxReaperMaintenanceWorkflow, {
|
|
168
|
+
workflowId: SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
|
|
169
|
+
taskQueue: lifecycleTaskQueue(),
|
|
170
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
171
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
172
|
+
args: [
|
|
173
|
+
{
|
|
174
|
+
examined: plan.drainable.length,
|
|
175
|
+
started: starts.filter((outcome) => outcome === "started").length,
|
|
176
|
+
alreadyRunning: starts.filter((outcome) => outcome === "already_running").length,
|
|
177
|
+
startFailed: starts.filter((outcome) => outcome === "failed").length,
|
|
178
|
+
rotationsRequested: plan.rotationsRequested,
|
|
179
|
+
},
|
|
180
|
+
],
|
|
181
|
+
});
|
|
182
|
+
} catch (error) {
|
|
183
|
+
if (alreadyRunningChild(error)) return;
|
|
184
|
+
// Every drain child is already durable and this inventory workflow still
|
|
185
|
+
// completes, so the next Schedule tick can retry maintenance independently.
|
|
186
|
+
log.warn("sandbox reaper maintenance child start deferred to next sweep", {
|
|
187
|
+
error: error instanceof Error ? error.message : String(error),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
28
190
|
}
|
|
@@ -1,6 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
getExternalWorkflowHandle,
|
|
3
|
+
ParentClosePolicy,
|
|
4
|
+
patched,
|
|
5
|
+
startChild,
|
|
6
|
+
workflowInfo,
|
|
7
|
+
WorkflowIdReusePolicy,
|
|
8
|
+
} from "@temporalio/workflow";
|
|
2
9
|
import type { TurnInitiator } from "@opengeni/contracts";
|
|
3
10
|
import { scheduledTaskActivity } from "./activities";
|
|
11
|
+
import {
|
|
12
|
+
knowledgeSourceSyncWake,
|
|
13
|
+
knowledgeSourceSyncWorkflow,
|
|
14
|
+
knowledgeSourceSyncWorkflowId,
|
|
15
|
+
} from "./knowledge-source-sync";
|
|
4
16
|
|
|
5
17
|
type ScheduledTaskFireWorkflowBase = {
|
|
6
18
|
accountId: string;
|
|
@@ -16,7 +28,7 @@ export type ScheduledTaskFireWorkflowInput = ScheduledTaskFireWorkflowBase &
|
|
|
16
28
|
initiator?: never;
|
|
17
29
|
}
|
|
18
30
|
| {
|
|
19
|
-
triggerType: "manual";
|
|
31
|
+
triggerType: "manual" | "initial" | "provider_event" | "retry" | "repair";
|
|
20
32
|
agentRunUsageIdempotencyKey: string;
|
|
21
33
|
initiator: TurnInitiator;
|
|
22
34
|
}
|
|
@@ -30,7 +42,7 @@ export async function scheduledTaskFireWorkflow(
|
|
|
30
42
|
// the exact charging identity.
|
|
31
43
|
if (
|
|
32
44
|
patched("scheduled-task-manual-initiator-v1") &&
|
|
33
|
-
input.triggerType
|
|
45
|
+
input.triggerType !== "scheduled" &&
|
|
34
46
|
(!input.agentRunUsageIdempotencyKey || !input.initiator)
|
|
35
47
|
) {
|
|
36
48
|
return;
|
|
@@ -40,14 +52,38 @@ export async function scheduledTaskFireWorkflow(
|
|
|
40
52
|
taskId: input.taskId,
|
|
41
53
|
producerKey: workflowInfo().workflowId,
|
|
42
54
|
};
|
|
43
|
-
await scheduledTaskActivity.dispatchScheduledTaskRun(
|
|
44
|
-
input.triggerType
|
|
55
|
+
const result = await scheduledTaskActivity.dispatchScheduledTaskRun(
|
|
56
|
+
input.triggerType !== "scheduled"
|
|
45
57
|
? {
|
|
46
58
|
...base,
|
|
47
|
-
triggerType:
|
|
59
|
+
triggerType: input.triggerType,
|
|
48
60
|
agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey,
|
|
49
61
|
initiator: input.initiator,
|
|
50
62
|
}
|
|
51
63
|
: { ...base, triggerType: "scheduled" },
|
|
52
64
|
);
|
|
65
|
+
if (result.action !== "knowledge_source_sync") return;
|
|
66
|
+
|
|
67
|
+
const workflowId = knowledgeSourceSyncWorkflowId(result.sourceId);
|
|
68
|
+
try {
|
|
69
|
+
await startChild(knowledgeSourceSyncWorkflow, {
|
|
70
|
+
workflowId,
|
|
71
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
72
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
73
|
+
args: [result],
|
|
74
|
+
});
|
|
75
|
+
} catch (error) {
|
|
76
|
+
if (!alreadyRunningWorkflow(error)) throw error;
|
|
77
|
+
await getExternalWorkflowHandle(workflowId).signal(knowledgeSourceSyncWake, result);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function alreadyRunningWorkflow(error: unknown): boolean {
|
|
82
|
+
let current: unknown = error;
|
|
83
|
+
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
|
|
84
|
+
const record = current as { name?: unknown; cause?: unknown };
|
|
85
|
+
if (record.name === "WorkflowExecutionAlreadyStartedError") return true;
|
|
86
|
+
current = record.cause;
|
|
87
|
+
}
|
|
88
|
+
return false;
|
|
53
89
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { continueAsNew, sleep } from "@temporalio/workflow";
|
|
2
|
+
import { videoGenerationActivityForTaskQueue } from "./activities";
|
|
3
|
+
|
|
4
|
+
export type VideoGenerationWorkflowInput = {
|
|
5
|
+
accountId: string;
|
|
6
|
+
workspaceId: string;
|
|
7
|
+
operationId: string;
|
|
8
|
+
baseTaskQueue: string;
|
|
9
|
+
iterations?: number;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const CONTINUE_AS_NEW_AFTER = 100;
|
|
13
|
+
|
|
14
|
+
export function videoGenerationWorkflowId(operationId: string): string {
|
|
15
|
+
return `video-generation:${operationId}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Small deterministic orchestration; all private/provider state remains in Postgres. */
|
|
19
|
+
export async function videoGenerationWorkflow(input: VideoGenerationWorkflowInput): Promise<void> {
|
|
20
|
+
const activity = videoGenerationActivityForTaskQueue(input.baseTaskQueue);
|
|
21
|
+
let iterations = input.iterations ?? 0;
|
|
22
|
+
while (iterations < CONTINUE_AS_NEW_AFTER) {
|
|
23
|
+
const result = await activity.reconcileVideoGenerationOperation({
|
|
24
|
+
accountId: input.accountId,
|
|
25
|
+
workspaceId: input.workspaceId,
|
|
26
|
+
operationId: input.operationId,
|
|
27
|
+
});
|
|
28
|
+
if (result.action === "terminal") return;
|
|
29
|
+
await sleep(result.delayMs);
|
|
30
|
+
iterations += 1;
|
|
31
|
+
}
|
|
32
|
+
await continueAsNew<typeof videoGenerationWorkflow>({
|
|
33
|
+
...input,
|
|
34
|
+
iterations: 0,
|
|
35
|
+
});
|
|
36
|
+
}
|