@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import type { ConnectionCredentialsPort, EntitlementsPort, ScheduledTaskTriggerType } from "@opengeni/contracts";
|
|
3
|
+
import type { Database } from "@opengeni/db";
|
|
4
|
+
import type { DocumentServices } from "@opengeni/documents";
|
|
5
|
+
import type { EventBus } from "@opengeni/events";
|
|
6
|
+
import type { Observability } from "@opengeni/observability";
|
|
7
|
+
import type { OpenGeniRuntime } from "@opengeni/runtime";
|
|
8
|
+
import type { ObjectStorage } from "@opengeni/storage";
|
|
9
|
+
|
|
10
|
+
// Signal (start-if-needed) a session's Temporal workflow so a queued turn it
|
|
11
|
+
// cannot otherwise observe gets claimed. Used to wake a PARENT session's
|
|
12
|
+
// workflow when a spawned worker completes: the parent may have idled and let
|
|
13
|
+
// its workflow run complete, so a plain signal would not start one — this must
|
|
14
|
+
// signalWithStart. Injected (not built from the worker's NativeConnection)
|
|
15
|
+
// because the worker package owns only the worker runtime, not a client; an
|
|
16
|
+
// undefined signaler degrades to "DB wake recorded, no workflow nudge" (the
|
|
17
|
+
// turn is still claimed on the parent's next natural wake).
|
|
18
|
+
export type WakeSessionWorkflowSignal = (input: { accountId: string; workspaceId: string; sessionId: string; workflowId: string }) => Promise<void>;
|
|
19
|
+
|
|
20
|
+
export type ActivityServices = {
|
|
21
|
+
settings: Settings;
|
|
22
|
+
db: Database;
|
|
23
|
+
bus: EventBus;
|
|
24
|
+
runtime: OpenGeniRuntime;
|
|
25
|
+
objectStorage: ObjectStorage | null;
|
|
26
|
+
documentServices: DocumentServices;
|
|
27
|
+
observability: Observability;
|
|
28
|
+
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
29
|
+
// §7.5 P3 — host-entitlements port, the WORKER half of the same seam the API
|
|
30
|
+
// edge exposes on `AppDependencies`. When set, `ensureRunAllowed` (turn-entry
|
|
31
|
+
// AND the mid-stream budget valve) delegates the funding decision to
|
|
32
|
+
// `admitRun` instead of reading `getBillingBalance` locally. null/undefined
|
|
33
|
+
// (standalone default) → today's local-ledger read runs unchanged.
|
|
34
|
+
//
|
|
35
|
+
// IDEMPOTENCY: the worker calls `admitRun` ONLY as an admission READ ("may
|
|
36
|
+
// this run/continue?"), never to RECORD consumption. Usage is recorded
|
|
37
|
+
// exactly once, by `recordUsageEvent` keyed on a deterministic idempotency
|
|
38
|
+
// key the API already wrote at create-time — so a host PULL meter that also
|
|
39
|
+
// observes that same recorded event is consulted without double-charging:
|
|
40
|
+
// admission and metering are separate operations, and only metering carries
|
|
41
|
+
// the idempotency key.
|
|
42
|
+
entitlements?: EntitlementsPort | null;
|
|
43
|
+
// §7.6 P4a — host connection-credential provider, the WORKER half of the
|
|
44
|
+
// federated-connection boundary. When set, the run's per-run credential mint
|
|
45
|
+
// delegates to the host instead of self-minting from `settings`:
|
|
46
|
+
// - `gitCredentials` REPLACES `createGitHubAppInstallationToken(settings,…)`
|
|
47
|
+
// in `sandboxEnvironmentForRun` (the GH_TOKEN / git-extraheader source).
|
|
48
|
+
// - `sandboxSecrets` REPLACES the `environmentsEncryptionKeyBytes(settings)`
|
|
49
|
+
// decrypt in `loadWorkspaceEnvironmentForRun`.
|
|
50
|
+
// Each leg is independently optional; an unset leg falls through to today's
|
|
51
|
+
// self-mint for THAT leg. null/undefined (standalone default) → both legs
|
|
52
|
+
// self-mint byte-for-byte as today.
|
|
53
|
+
//
|
|
54
|
+
// FORK-7 CROSS-CHECK: a provider echoes the `workspaceId` it scoped the
|
|
55
|
+
// credential to; the consuming activity ASSERTS agreement with the run's
|
|
56
|
+
// workspace BEFORE injecting `GH_TOKEN` (or applying decrypted values). A host
|
|
57
|
+
// mapping bug returning tenant B's creds for a tenant-A run is caught here.
|
|
58
|
+
connectionCredentials?: ConnectionCredentialsPort | null;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
export type ActivityDependencies = Partial<ActivityServices>;
|
|
62
|
+
|
|
63
|
+
export type RunAgentTurnInput = {
|
|
64
|
+
accountId: string;
|
|
65
|
+
workspaceId: string;
|
|
66
|
+
sessionId: string;
|
|
67
|
+
triggerEventId: string;
|
|
68
|
+
workflowId: string;
|
|
69
|
+
turnId?: string;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
export type RequeueTurnAfterWorkerDeathInput = {
|
|
73
|
+
accountId: string;
|
|
74
|
+
workspaceId: string;
|
|
75
|
+
sessionId: string;
|
|
76
|
+
// The trigger the dead attempt was actually running (for an approval rerun
|
|
77
|
+
// this is the approval-decision event, not the turn row's original trigger).
|
|
78
|
+
triggerEventId: string;
|
|
79
|
+
workflowId: string;
|
|
80
|
+
turnId: string;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
export type RequeueTurnAfterWorkerDeathResult =
|
|
84
|
+
// The turn is back on the queue; the session workflow's next claim
|
|
85
|
+
// re-dispatches it on a healthy worker. `redispatches` is the total number
|
|
86
|
+
// of worker-death re-dispatches this turn has now consumed.
|
|
87
|
+
| { action: "requeued"; redispatches: number }
|
|
88
|
+
// The turn is no longer running/requires_action: the timed-out attempt was
|
|
89
|
+
// a zombie that actually settled the turn after the server gave up on its
|
|
90
|
+
// heartbeats. Nothing to redo; the workflow just continues its loop.
|
|
91
|
+
| { action: "stale" }
|
|
92
|
+
// The per-turn crash-loop guard tripped; the workflow must fail the
|
|
93
|
+
// session for real. `redispatches` is the count already consumed (== the
|
|
94
|
+
// ceiling), so the failed attempt was worker death number redispatches + 1.
|
|
95
|
+
| { action: "exceeded"; redispatches: number };
|
|
96
|
+
|
|
97
|
+
export type ClaimNextQueuedTurnInput = {
|
|
98
|
+
workspaceId: string;
|
|
99
|
+
sessionId: string;
|
|
100
|
+
workflowId: string;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
export type MarkSessionIdleInput = {
|
|
104
|
+
workspaceId: string;
|
|
105
|
+
sessionId: string;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
export type MaybeContinueGoalInput = {
|
|
109
|
+
accountId: string;
|
|
110
|
+
workspaceId: string;
|
|
111
|
+
sessionId: string;
|
|
112
|
+
workflowId: string;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
export type MaybeContinueGoalResult = {
|
|
116
|
+
action: "none" | "queue" | "continue" | "paused";
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
export type PauseGoalForInterruptInput = {
|
|
120
|
+
workspaceId: string;
|
|
121
|
+
sessionId: string;
|
|
122
|
+
// The `user.interrupt` event that triggered the pause, when there is one.
|
|
123
|
+
// A steer-tagged interrupt (reason "steer") must NOT pause the goal:
|
|
124
|
+
// steering redirects the work, it does not stop it.
|
|
125
|
+
triggerEventId?: string;
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
export type DispatchScheduledTaskRunInput = {
|
|
129
|
+
workspaceId: string;
|
|
130
|
+
taskId: string;
|
|
131
|
+
triggerType: ScheduledTaskTriggerType;
|
|
132
|
+
agentRunUsageIdempotencyKey?: string;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
export type DispatchScheduledTaskRunResult = {
|
|
136
|
+
action: "start" | "signal";
|
|
137
|
+
accountId: string;
|
|
138
|
+
workspaceId: string;
|
|
139
|
+
sessionId: string;
|
|
140
|
+
triggerEventId: string;
|
|
141
|
+
workflowId: string;
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
export type IndexDocumentInput = {
|
|
145
|
+
accountId: string;
|
|
146
|
+
workspaceId: string;
|
|
147
|
+
documentId: string;
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
export type RunAgentTurnResult = {
|
|
151
|
+
// "preempted": the worker hosting this turn shut down gracefully mid-turn;
|
|
152
|
+
// the activity checkpointed conversation truth, re-queued the turn, and the
|
|
153
|
+
// session workflow re-dispatches it on a healthy worker.
|
|
154
|
+
status: "idle" | "requires_action" | "failed" | "cancelled" | "preempted";
|
|
155
|
+
// Provider backpressure pacing: when set on an idle result, the session
|
|
156
|
+
// workflow holds the loop this long before admitting the next turn (an
|
|
157
|
+
// active goal's continuation would otherwise immediately re-hit the limit).
|
|
158
|
+
continueDelayMs?: number;
|
|
159
|
+
// Multi-account rotation all-capped idle: every connected Codex subscription is
|
|
160
|
+
// rate-limited/cooling. This is a MANDATORY hold — session.ts must wait
|
|
161
|
+
// continueDelayMs (floored to a minimum) and must NOT treat a 0/elapsed delay as
|
|
162
|
+
// "continue now" (invariant 4: NO THRASH). Distinct from a normal continueDelayMs:0
|
|
163
|
+
// which legitimately means "a rotation candidate is ready, re-dispatch immediately".
|
|
164
|
+
idleUntilReset?: boolean;
|
|
165
|
+
};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
2
|
+
import { createDb } from "@opengeni/db";
|
|
3
|
+
import { createDocumentServices } from "@opengeni/documents";
|
|
4
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
5
|
+
import { createObservability } from "@opengeni/observability";
|
|
6
|
+
import { createProductionAgentRuntime } from "@opengeni/runtime";
|
|
7
|
+
import { createObjectStorage } from "@opengeni/storage";
|
|
8
|
+
import { createRunAgentTurnActivity } from "./activities/agent-turn";
|
|
9
|
+
import { createDocumentActivities } from "./activities/documents";
|
|
10
|
+
import { createGoalActivities } from "./activities/goals";
|
|
11
|
+
import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
|
|
12
|
+
import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
|
|
13
|
+
import { createSessionStateActivities } from "./activities/session-state";
|
|
14
|
+
import type { ActivityDependencies, ActivityServices } from "./activities/types";
|
|
15
|
+
|
|
16
|
+
export type {
|
|
17
|
+
ActivityDependencies,
|
|
18
|
+
DispatchScheduledTaskRunInput,
|
|
19
|
+
DispatchScheduledTaskRunResult,
|
|
20
|
+
IndexDocumentInput,
|
|
21
|
+
MaybeContinueGoalInput,
|
|
22
|
+
MaybeContinueGoalResult,
|
|
23
|
+
PauseGoalForInterruptInput,
|
|
24
|
+
RequeueTurnAfterWorkerDeathInput,
|
|
25
|
+
RequeueTurnAfterWorkerDeathResult,
|
|
26
|
+
RunAgentTurnInput,
|
|
27
|
+
RunAgentTurnResult,
|
|
28
|
+
} from "./activities/types";
|
|
29
|
+
|
|
30
|
+
export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
31
|
+
let servicesPromise: Promise<ActivityServices> | null = null;
|
|
32
|
+
|
|
33
|
+
async function services(): Promise<ActivityServices> {
|
|
34
|
+
servicesPromise ??= (async () => {
|
|
35
|
+
const settings = dependencies.settings ?? getSettings();
|
|
36
|
+
// Step I: when not injected, build the standalone handle — searchPath
|
|
37
|
+
// undefined for standalone (public), scoped to the dedicated schema +
|
|
38
|
+
// host RLS strategy when embedded config is set. An embedded host injects
|
|
39
|
+
// `dependencies.db` directly and this branch is skipped.
|
|
40
|
+
const searchPath = dbSearchPath(settings);
|
|
41
|
+
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
42
|
+
...(searchPath ? { searchPath } : {}),
|
|
43
|
+
rlsStrategy: settings.rlsStrategy,
|
|
44
|
+
});
|
|
45
|
+
// The PRIVILEGED control-plane NATS login (M-AUTH): the worker resolves the
|
|
46
|
+
// SAME static account user the API uses to request `agent.*.rpc`. Null in
|
|
47
|
+
// local dev → anonymous connect (the bus default).
|
|
48
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
49
|
+
return {
|
|
50
|
+
settings,
|
|
51
|
+
db: dependencies.db ?? dbClient!.db,
|
|
52
|
+
// §7 Step G — EventBus binding contract. `bus` is the INJECTED
|
|
53
|
+
// live-fanout port; the ONE production impl is `createNatsEventBus`,
|
|
54
|
+
// the default on BOTH processes (this worker edge + the API edge in
|
|
55
|
+
// `apps/api/src/index.ts`). A host that embeds OpenGeni injects ONE
|
|
56
|
+
// broker binding (the same `createNatsEventBus(natsUrl)`) here AND on
|
|
57
|
+
// the mounted API, so the two SEPARATE processes share one broker and
|
|
58
|
+
// derive the IDENTICAL `sessionSubject` — the only way live fanout
|
|
59
|
+
// (worker emit → API SSE) works cross-process (SPIKE-1 F5/F6, proven).
|
|
60
|
+
// NEVER default to an in-memory bus: it fans out intra-process only and
|
|
61
|
+
// would silently break live SSE. unset → today's NATS default,
|
|
62
|
+
// byte-for-byte. The bus is live-fanout ONLY — the durable Postgres
|
|
63
|
+
// `session_events` log is source-of-truth (the API backfills missed
|
|
64
|
+
// events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
|
|
65
|
+
bus:
|
|
66
|
+
dependencies.bus ??
|
|
67
|
+
(await createNatsEventBus(
|
|
68
|
+
settings.natsUrl,
|
|
69
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
70
|
+
)),
|
|
71
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime(),
|
|
72
|
+
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
73
|
+
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
74
|
+
observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
|
|
75
|
+
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
76
|
+
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
77
|
+
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
|
78
|
+
// local ledger exactly as today (mirrors `wakeSessionWorkflow`'s
|
|
79
|
+
// null-degrades-gracefully shape, not a `createX(settings)` default).
|
|
80
|
+
entitlements: dependencies.entitlements ?? null,
|
|
81
|
+
// §7.6 P4a — host connection-credential provider. No constructed
|
|
82
|
+
// default: standalone owns its own GitHub App + encryption key, so unset
|
|
83
|
+
// → null → the per-run credential mint self-mints from `settings`
|
|
84
|
+
// (createGitHubAppInstallationToken + environmentsEncryptionKeyBytes)
|
|
85
|
+
// exactly as today. Same null-degrades shape as `entitlements`.
|
|
86
|
+
connectionCredentials: dependencies.connectionCredentials ?? null,
|
|
87
|
+
};
|
|
88
|
+
})();
|
|
89
|
+
return servicesPromise;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const runAgentTurn = createRunAgentTurnActivity(services);
|
|
93
|
+
return {
|
|
94
|
+
runAgentTurn,
|
|
95
|
+
// Legacy Temporal activity name. In-flight session workflows (which can
|
|
96
|
+
// legitimately live for days) recorded ScheduleActivityTask events as
|
|
97
|
+
// "runAgentSegment"; the name must keep resolving until those histories
|
|
98
|
+
// drain. New workflow code must use runAgentTurn; migrate the session
|
|
99
|
+
// workflow call site via patched() before removing this alias.
|
|
100
|
+
runAgentSegment: runAgentTurn,
|
|
101
|
+
...createDocumentActivities(services),
|
|
102
|
+
...createSessionStateActivities(services),
|
|
103
|
+
...createScheduledTaskActivities(services),
|
|
104
|
+
...createGoalActivities(services),
|
|
105
|
+
// P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
|
|
106
|
+
// *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
|
|
107
|
+
...createSandboxLeaseActivities(services),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const defaultActivities = createActivities();
|
|
112
|
+
|
|
113
|
+
export const runAgentTurn = defaultActivities.runAgentTurn;
|
|
114
|
+
export const runAgentSegment = defaultActivities.runAgentSegment;
|
|
115
|
+
export const indexDocument = defaultActivities.indexDocument;
|
|
116
|
+
export const failSession = defaultActivities.failSession;
|
|
117
|
+
export const interruptActiveTurn = defaultActivities.interruptActiveTurn;
|
|
118
|
+
export const requeueTurnAfterWorkerDeath = defaultActivities.requeueTurnAfterWorkerDeath;
|
|
119
|
+
export const claimNextQueuedTurn = defaultActivities.claimNextQueuedTurn;
|
|
120
|
+
export const markSessionIdle = defaultActivities.markSessionIdle;
|
|
121
|
+
export const dispatchScheduledTaskRun = defaultActivities.dispatchScheduledTaskRun;
|
|
122
|
+
export const maybeContinueGoal = defaultActivities.maybeContinueGoal;
|
|
123
|
+
export const pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
|
|
124
|
+
export const reapSandboxLeases = defaultActivities.reapSandboxLeases;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
2
|
+
import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
|
|
3
|
+
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
4
|
+
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
5
|
+
import { createActivities, type ActivityDependencies } from "./activities";
|
|
6
|
+
import type { WakeSessionWorkflowSignal } from "./activities/types";
|
|
7
|
+
|
|
8
|
+
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
9
|
+
// create() is idempotent across every worker in the pool: the first worker to
|
|
10
|
+
// boot creates it, all others collide on this id (ScheduleAlreadyRunning) and
|
|
11
|
+
// no-op — so the Schedule is registered EXACTLY ONCE per deployment regardless
|
|
12
|
+
// of replica count.
|
|
13
|
+
const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
14
|
+
|
|
15
|
+
export type WorkerOptions = {
|
|
16
|
+
settings?: Settings;
|
|
17
|
+
activities?: ReturnType<typeof createActivities>;
|
|
18
|
+
activityDependencies?: ActivityDependencies;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise<{
|
|
22
|
+
worker: Worker;
|
|
23
|
+
connection: NativeConnection;
|
|
24
|
+
}> {
|
|
25
|
+
const settings = options.settings ?? getSettings();
|
|
26
|
+
const observability = options.activityDependencies?.observability ?? createObservability(settings, { component: "worker" });
|
|
27
|
+
const connection = await retryStartupDependency(
|
|
28
|
+
"Temporal",
|
|
29
|
+
() => NativeConnection.connect({ address: settings.temporalHost }),
|
|
30
|
+
{
|
|
31
|
+
...startupRetryOptions(settings),
|
|
32
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
33
|
+
},
|
|
34
|
+
);
|
|
35
|
+
const activities = options.activities ?? createActivities({
|
|
36
|
+
...options.activityDependencies,
|
|
37
|
+
settings,
|
|
38
|
+
observability,
|
|
39
|
+
});
|
|
40
|
+
const worker = await Worker.create({
|
|
41
|
+
connection,
|
|
42
|
+
namespace: settings.temporalNamespace,
|
|
43
|
+
taskQueue: settings.temporalTaskQueue,
|
|
44
|
+
workflowsPath: new URL("../src/workflows.ts", import.meta.url).pathname,
|
|
45
|
+
activities,
|
|
46
|
+
});
|
|
47
|
+
return { worker, connection };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// A signalWithStart capability so a worker activity can wake a PARENT
|
|
51
|
+
// session's workflow when a spawned worker completes (the parent may have
|
|
52
|
+
// idled and let its run finish, so a plain signal would not start one).
|
|
53
|
+
// Separate from the worker's NativeConnection: the @temporalio/client
|
|
54
|
+
// Connection is what exposes workflow.signalWithStart.
|
|
55
|
+
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{ wakeSessionWorkflow: WakeSessionWorkflowSignal; close: () => Promise<void> }> {
|
|
56
|
+
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
57
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
58
|
+
return {
|
|
59
|
+
wakeSessionWorkflow: async ({ accountId, workspaceId, sessionId, workflowId }) => {
|
|
60
|
+
await temporal.workflow.signalWithStart("sessionWorkflow", {
|
|
61
|
+
taskQueue: settings.temporalTaskQueue,
|
|
62
|
+
workflowId,
|
|
63
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
64
|
+
args: [{ accountId, workspaceId, sessionId }],
|
|
65
|
+
signal: "queueChanged",
|
|
66
|
+
});
|
|
67
|
+
},
|
|
68
|
+
close: async () => {
|
|
69
|
+
await connection.close();
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
76
|
+
* driver — P1.3 / OD-3). Gated on sandboxOwnershipEnabled: with the flag off the
|
|
77
|
+
* Schedule is never created, so the lease feature is fully dark (no sweeps, no
|
|
78
|
+
* terminates).
|
|
79
|
+
*
|
|
80
|
+
* The Schedule fires sandboxReaperWorkflow on the worker's global task queue
|
|
81
|
+
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
82
|
+
* `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
|
|
83
|
+
* validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
|
|
84
|
+
* sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
|
|
85
|
+
* worker pool collides on ScheduleAlreadyRunning and no-ops, so the Schedule is
|
|
86
|
+
* registered exactly once per deployment.
|
|
87
|
+
*
|
|
88
|
+
* Returns a `close()` for the dedicated client connection (separate from the
|
|
89
|
+
* worker's NativeConnection — the Schedule client is a @temporalio/client).
|
|
90
|
+
*/
|
|
91
|
+
export async function registerSandboxReaperSchedule(
|
|
92
|
+
settings: Settings,
|
|
93
|
+
observability: Observability,
|
|
94
|
+
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
95
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
96
|
+
return { registered: false, close: async () => {} };
|
|
97
|
+
}
|
|
98
|
+
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
99
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
100
|
+
try {
|
|
101
|
+
await temporal.schedule.create({
|
|
102
|
+
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
103
|
+
spec: {
|
|
104
|
+
// @every-style interval: fire once per reaper period. The boot invariant
|
|
105
|
+
// (config) guarantees reaperPeriod < viewerHolderTTL and
|
|
106
|
+
// reaperPeriod + idleGrace < providerLifetime.
|
|
107
|
+
intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }],
|
|
108
|
+
},
|
|
109
|
+
action: {
|
|
110
|
+
type: "startWorkflow",
|
|
111
|
+
workflowType: "sandboxReaperWorkflow",
|
|
112
|
+
taskQueue: settings.temporalTaskQueue,
|
|
113
|
+
args: [],
|
|
114
|
+
},
|
|
115
|
+
policies: {
|
|
116
|
+
// A slow sweep must never overlap itself; the next fire is skipped.
|
|
117
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
118
|
+
catchupWindow: "1m",
|
|
119
|
+
pauseOnFailure: false,
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
observability.info("Registered the global sandbox-lease reaper Schedule", {
|
|
123
|
+
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
124
|
+
reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs,
|
|
125
|
+
});
|
|
126
|
+
return { registered: true, close: async () => { await connection.close(); } };
|
|
127
|
+
} catch (error) {
|
|
128
|
+
if (error instanceof ScheduleAlreadyRunning) {
|
|
129
|
+
// Another worker in the pool already created it. The Schedule exists
|
|
130
|
+
// exactly once — this is the expected no-op on every replica after the
|
|
131
|
+
// first. (We do NOT update the spec here: a redeploy with a changed cadence
|
|
132
|
+
// is an operational concern handled by deleting+recreating the Schedule.)
|
|
133
|
+
observability.info("Global sandbox-lease reaper Schedule already registered", {
|
|
134
|
+
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
135
|
+
});
|
|
136
|
+
return { registered: false, close: async () => { await connection.close(); } };
|
|
137
|
+
}
|
|
138
|
+
await connection.close().catch(() => undefined);
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export async function startWorker() {
|
|
144
|
+
const settings = getSettings();
|
|
145
|
+
const observability = createObservability(settings, { component: "worker" });
|
|
146
|
+
const signaler = await retryStartupDependency(
|
|
147
|
+
"Temporal client",
|
|
148
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
149
|
+
{
|
|
150
|
+
...startupRetryOptions(settings),
|
|
151
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
152
|
+
},
|
|
153
|
+
);
|
|
154
|
+
const { worker, connection } = await createOpenGeniWorker({
|
|
155
|
+
settings,
|
|
156
|
+
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
|
|
157
|
+
});
|
|
158
|
+
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
159
|
+
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
160
|
+
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
161
|
+
// per-session worker factory.
|
|
162
|
+
const reaperSchedule = await retryStartupDependency(
|
|
163
|
+
"Temporal schedule (sandbox reaper)",
|
|
164
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
165
|
+
{
|
|
166
|
+
...startupRetryOptions(settings),
|
|
167
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
168
|
+
},
|
|
169
|
+
);
|
|
170
|
+
observability.info("OpenGeni worker listening", {
|
|
171
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
172
|
+
});
|
|
173
|
+
try {
|
|
174
|
+
await worker.run();
|
|
175
|
+
} finally {
|
|
176
|
+
await connection.close();
|
|
177
|
+
await signaler.close();
|
|
178
|
+
await reaperSchedule.close();
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (import.meta.main) {
|
|
183
|
+
await startWorker();
|
|
184
|
+
}
|