@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,787 @@
|
|
|
1
|
+
import { dbSearchPath, getSettings, type Settings } from "@opengeni/config";
|
|
2
|
+
import {
|
|
3
|
+
createDb,
|
|
4
|
+
dbSql,
|
|
5
|
+
PostgresEditableArtifactMaterializationRepository,
|
|
6
|
+
type Database,
|
|
7
|
+
} from "@opengeni/db";
|
|
8
|
+
import { createObservability, type Observability } from "@opengeni/observability";
|
|
9
|
+
import { createObjectStorage, type ObjectStorage } from "@opengeni/storage";
|
|
10
|
+
import { dirname } from "node:path";
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE,
|
|
14
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES,
|
|
15
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES,
|
|
16
|
+
EditableArtifactMaterializer,
|
|
17
|
+
createConfiguredEditableArtifactMaterializer,
|
|
18
|
+
systemEditableArtifactMaterializerClock,
|
|
19
|
+
systemEditableArtifactMaterializerScheduler,
|
|
20
|
+
type EditableArtifactMaterializerPassSummary,
|
|
21
|
+
} from "./editable-artifact-materializer";
|
|
22
|
+
import {
|
|
23
|
+
createEditableArtifactObjectStorageReader,
|
|
24
|
+
createEditableArtifactObjectStorageVerifier,
|
|
25
|
+
createEditableArtifactObjectStorageWriter,
|
|
26
|
+
} from "./editable-artifact-materializer-storage";
|
|
27
|
+
import {
|
|
28
|
+
createNativeEditableArtifactSubprocessPort,
|
|
29
|
+
type NativeEditableArtifactSubprocessOptions,
|
|
30
|
+
} from "./editable-artifact-materializer-subprocess";
|
|
31
|
+
import {
|
|
32
|
+
createDevelopmentEditableArtifactProcessLauncher,
|
|
33
|
+
createLinuxEditableArtifactProcessLauncher,
|
|
34
|
+
} from "./editable-artifact-materializer-launcher";
|
|
35
|
+
|
|
36
|
+
export type EditableArtifactRuntimeAuthority =
|
|
37
|
+
| Readonly<{
|
|
38
|
+
mode: "production";
|
|
39
|
+
environmentVariable: "OPENGENI_ARTIFACT_RUNTIME_MANIFEST";
|
|
40
|
+
manifestPath: string;
|
|
41
|
+
}>
|
|
42
|
+
| Readonly<{
|
|
43
|
+
mode: "development-current-host";
|
|
44
|
+
environmentVariable: "OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST";
|
|
45
|
+
manifestPath: string;
|
|
46
|
+
}>;
|
|
47
|
+
|
|
48
|
+
export type EditableArtifactMaterializerServiceState =
|
|
49
|
+
| "starting"
|
|
50
|
+
| "ready"
|
|
51
|
+
| "draining"
|
|
52
|
+
| "stopped"
|
|
53
|
+
| "failed";
|
|
54
|
+
|
|
55
|
+
export type EditableArtifactMaterializerService = Readonly<{
|
|
56
|
+
state(): EditableArtifactMaterializerServiceState;
|
|
57
|
+
run(): Promise<void>;
|
|
58
|
+
drain(reason?: string): boolean;
|
|
59
|
+
close(): Promise<void>;
|
|
60
|
+
dispatchOnce(signal?: AbortSignal): Promise<EditableArtifactMaterializerPassSummary>;
|
|
61
|
+
fetch(request: Request): Promise<Response>;
|
|
62
|
+
}>;
|
|
63
|
+
|
|
64
|
+
export type CreateEditableArtifactMaterializerServiceInput = Readonly<{
|
|
65
|
+
materializer: EditableArtifactMaterializer;
|
|
66
|
+
db: Database;
|
|
67
|
+
closeDatabase: () => Promise<void>;
|
|
68
|
+
observability: Observability;
|
|
69
|
+
serviceName: string;
|
|
70
|
+
environment: string;
|
|
71
|
+
deploymentRevision?: string;
|
|
72
|
+
readinessTimeoutMs?: number;
|
|
73
|
+
runtimeCapability?: Readonly<{
|
|
74
|
+
mode: EditableArtifactRuntimeAuthority["mode"];
|
|
75
|
+
sandboxEnforced: boolean;
|
|
76
|
+
}>;
|
|
77
|
+
}>;
|
|
78
|
+
|
|
79
|
+
export type ProductionEditableArtifactMaterializerServiceOptions = Readonly<{
|
|
80
|
+
settings: Settings;
|
|
81
|
+
/** Dedicated DSN; never defaults to settings.databaseUrl. */
|
|
82
|
+
databaseUrl: string;
|
|
83
|
+
declaredDatabaseRole: string;
|
|
84
|
+
executable: string;
|
|
85
|
+
bubblewrapExecutable: string | null;
|
|
86
|
+
prlimitExecutable: string | null;
|
|
87
|
+
runtimeAuthority: EditableArtifactRuntimeAuthority;
|
|
88
|
+
/** Explicit exception selected only by the validated local-dev sidecar config. */
|
|
89
|
+
allowUnsandboxedLocalDevelopment?: boolean;
|
|
90
|
+
skillFacadeEntrypoint: string;
|
|
91
|
+
owner: string;
|
|
92
|
+
objectStorage?: ObjectStorage;
|
|
93
|
+
batchSize?: number;
|
|
94
|
+
concurrency?: number;
|
|
95
|
+
leaseDurationMs?: number;
|
|
96
|
+
leaseRenewIntervalMs?: number;
|
|
97
|
+
pollIntervalMs?: number;
|
|
98
|
+
maxAttempts?: number;
|
|
99
|
+
memoryLimitBytes: number;
|
|
100
|
+
cpuTimeLimitMs: number;
|
|
101
|
+
fileDescriptorLimit: number;
|
|
102
|
+
processLimit: number;
|
|
103
|
+
fileSizeLimitBytes: number;
|
|
104
|
+
wallTimeoutMs: number;
|
|
105
|
+
nodeEnvironment?: string;
|
|
106
|
+
maxSourceBytes?: number;
|
|
107
|
+
maxOutputBytes?: number;
|
|
108
|
+
}>;
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Compose the distinct production sidecar. It creates its own connection pool
|
|
112
|
+
* from the dedicated DSN and never reuses a control/turn worker handle.
|
|
113
|
+
*/
|
|
114
|
+
export async function createProductionEditableArtifactMaterializerService(
|
|
115
|
+
options: ProductionEditableArtifactMaterializerServiceOptions,
|
|
116
|
+
): Promise<EditableArtifactMaterializerService> {
|
|
117
|
+
if (options.declaredDatabaseRole !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE) {
|
|
118
|
+
throw new Error("Artifact materializer database role declaration is invalid");
|
|
119
|
+
}
|
|
120
|
+
if (typeof options.databaseUrl !== "string" || options.databaseUrl.trim().length === 0) {
|
|
121
|
+
throw new Error("Artifact materializer requires its dedicated database DSN");
|
|
122
|
+
}
|
|
123
|
+
const maxSourceBytes = options.maxSourceBytes ?? EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES;
|
|
124
|
+
const maxOutputBytes = options.maxOutputBytes ?? EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES;
|
|
125
|
+
const concurrency = options.concurrency ?? 2;
|
|
126
|
+
const development = options.runtimeAuthority.mode === "development-current-host";
|
|
127
|
+
if (development !== (options.allowUnsandboxedLocalDevelopment === true)) {
|
|
128
|
+
throw new Error("Artifact materializer runtime authority and local-development gate differ");
|
|
129
|
+
}
|
|
130
|
+
const launcher = development
|
|
131
|
+
? await createDevelopmentEditableArtifactProcessLauncher({
|
|
132
|
+
materializerExecutable: options.executable,
|
|
133
|
+
explicitlyEnabled: options.allowUnsandboxedLocalDevelopment === true,
|
|
134
|
+
nodeEnvironment: options.nodeEnvironment,
|
|
135
|
+
})
|
|
136
|
+
: await createLinuxEditableArtifactProcessLauncher({
|
|
137
|
+
materializerExecutable: options.executable,
|
|
138
|
+
bubblewrapExecutable: requiredOption(options.bubblewrapExecutable, "bubblewrap"),
|
|
139
|
+
prlimitExecutable: requiredOption(options.prlimitExecutable, "prlimit"),
|
|
140
|
+
runtimeRoot: dirname(options.runtimeAuthority.manifestPath),
|
|
141
|
+
memoryLimitBytes: options.memoryLimitBytes,
|
|
142
|
+
cpuTimeLimitMs: options.cpuTimeLimitMs,
|
|
143
|
+
fileDescriptorLimit: options.fileDescriptorLimit,
|
|
144
|
+
processLimit: options.processLimit,
|
|
145
|
+
fileSizeLimitBytes: options.fileSizeLimitBytes,
|
|
146
|
+
});
|
|
147
|
+
const searchPath = dbSearchPath(options.settings);
|
|
148
|
+
const dbClient = createDb(options.databaseUrl, {
|
|
149
|
+
...(searchPath ? { searchPath } : {}),
|
|
150
|
+
...(options.settings.rlsStrategy ? { rlsStrategy: options.settings.rlsStrategy } : {}),
|
|
151
|
+
max: Math.max(2, concurrency + 1),
|
|
152
|
+
});
|
|
153
|
+
let created = false;
|
|
154
|
+
try {
|
|
155
|
+
await assertDedicatedMaterializerDatabaseRole(dbClient.db);
|
|
156
|
+
const objectStorage = options.objectStorage ?? createObjectStorage(options.settings);
|
|
157
|
+
if (!objectStorage) throw new Error("Artifact materializer object storage is unavailable");
|
|
158
|
+
const subprocessOptions: NativeEditableArtifactSubprocessOptions = {
|
|
159
|
+
executable: options.executable,
|
|
160
|
+
launcher,
|
|
161
|
+
childEnvironment:
|
|
162
|
+
options.runtimeAuthority.mode === "production"
|
|
163
|
+
? Object.freeze({
|
|
164
|
+
OPENGENI_ARTIFACT_RUNTIME_MANIFEST: options.runtimeAuthority.manifestPath,
|
|
165
|
+
OPENGENI_ARTIFACT_TOOL_ENTRY: options.skillFacadeEntrypoint,
|
|
166
|
+
})
|
|
167
|
+
: Object.freeze({
|
|
168
|
+
NODE_ENV: "development",
|
|
169
|
+
OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST: options.runtimeAuthority.manifestPath,
|
|
170
|
+
OPENGENI_ARTIFACT_TOOL_ENTRY: options.skillFacadeEntrypoint,
|
|
171
|
+
}),
|
|
172
|
+
...(development ? { allowUnsandboxedDevelopment: true } : {}),
|
|
173
|
+
wallTimeoutMs: options.wallTimeoutMs,
|
|
174
|
+
maxSourceBytes,
|
|
175
|
+
maxOutputBytes,
|
|
176
|
+
};
|
|
177
|
+
const kernel = await createNativeEditableArtifactSubprocessPort(subprocessOptions);
|
|
178
|
+
const repository = new PostgresEditableArtifactMaterializationRepository(dbClient.db, {
|
|
179
|
+
...(options.settings.dbSchema.trim() ? { dataSchema: options.settings.dbSchema.trim() } : {}),
|
|
180
|
+
});
|
|
181
|
+
const observability = createObservability(options.settings, {
|
|
182
|
+
component: "artifact-materializer",
|
|
183
|
+
});
|
|
184
|
+
if (development) {
|
|
185
|
+
observability.warn("Artifact materializer is running without an OS sandbox", {
|
|
186
|
+
errorClass: "DevelopmentSecurityPosture",
|
|
187
|
+
errorCode: "artifact_materializer_unsandboxed_development",
|
|
188
|
+
origin: "worker",
|
|
189
|
+
outcome: "degraded",
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
const materializer = createConfiguredEditableArtifactMaterializer({
|
|
193
|
+
enabled: true,
|
|
194
|
+
databaseRole: options.declaredDatabaseRole,
|
|
195
|
+
objectStorageConfigured: true,
|
|
196
|
+
...(development ? { allowUnsandboxedDevelopment: true } : {}),
|
|
197
|
+
dependencies: {
|
|
198
|
+
store: repository,
|
|
199
|
+
sourceReader: createEditableArtifactObjectStorageReader(objectStorage),
|
|
200
|
+
outputWriter: createEditableArtifactObjectStorageWriter(objectStorage),
|
|
201
|
+
outputVerifier: createEditableArtifactObjectStorageVerifier(objectStorage, {
|
|
202
|
+
verify: kernel.verifyMaterialization,
|
|
203
|
+
}),
|
|
204
|
+
kernel,
|
|
205
|
+
scheduler: systemEditableArtifactMaterializerScheduler,
|
|
206
|
+
clock: systemEditableArtifactMaterializerClock,
|
|
207
|
+
logger: {
|
|
208
|
+
warn(message, attributes) {
|
|
209
|
+
observability.warn(message, {
|
|
210
|
+
errorClass: "WorkerOperationError",
|
|
211
|
+
errorCode: materializerFailureErrorCode(attributes) ?? "worker_operation_failed",
|
|
212
|
+
origin: "worker",
|
|
213
|
+
outcome: typeof attributes.code === "string" ? attributes.code : "unknown",
|
|
214
|
+
});
|
|
215
|
+
},
|
|
216
|
+
},
|
|
217
|
+
metrics: {
|
|
218
|
+
increment(outcome, count = 1) {
|
|
219
|
+
observability.incrementCounter({
|
|
220
|
+
name: "opengeni_artifact_materializations_total",
|
|
221
|
+
help: "Editable artifact materialization attempts by closed outcome.",
|
|
222
|
+
labels: { outcome },
|
|
223
|
+
amount: count,
|
|
224
|
+
});
|
|
225
|
+
},
|
|
226
|
+
},
|
|
227
|
+
},
|
|
228
|
+
options: {
|
|
229
|
+
owner: options.owner,
|
|
230
|
+
...(options.batchSize !== undefined ? { batchSize: options.batchSize } : {}),
|
|
231
|
+
concurrency,
|
|
232
|
+
...(options.leaseDurationMs !== undefined
|
|
233
|
+
? { leaseDurationMs: options.leaseDurationMs }
|
|
234
|
+
: {}),
|
|
235
|
+
...(options.leaseRenewIntervalMs !== undefined
|
|
236
|
+
? { leaseRenewIntervalMs: options.leaseRenewIntervalMs }
|
|
237
|
+
: {}),
|
|
238
|
+
...(options.pollIntervalMs !== undefined ? { pollIntervalMs: options.pollIntervalMs } : {}),
|
|
239
|
+
...(options.maxAttempts !== undefined ? { maxAttempts: options.maxAttempts } : {}),
|
|
240
|
+
maxSourceBytes,
|
|
241
|
+
maxOutputBytes,
|
|
242
|
+
},
|
|
243
|
+
});
|
|
244
|
+
if (!materializer) throw new Error("Artifact materializer is unexpectedly disabled");
|
|
245
|
+
created = true;
|
|
246
|
+
return createEditableArtifactMaterializerService({
|
|
247
|
+
materializer,
|
|
248
|
+
db: dbClient.db,
|
|
249
|
+
closeDatabase: dbClient.close,
|
|
250
|
+
observability,
|
|
251
|
+
serviceName: options.settings.serviceName,
|
|
252
|
+
environment: options.settings.environment,
|
|
253
|
+
deploymentRevision: options.settings.deploymentRevision,
|
|
254
|
+
runtimeCapability: Object.freeze({
|
|
255
|
+
mode: options.runtimeAuthority.mode,
|
|
256
|
+
sandboxEnforced: launcher.identity.sandboxEnforced,
|
|
257
|
+
}),
|
|
258
|
+
});
|
|
259
|
+
} finally {
|
|
260
|
+
if (!created) await dbClient.close();
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function materializerFailureErrorCode(
|
|
265
|
+
attributes: Readonly<Record<string, unknown>>,
|
|
266
|
+
):
|
|
267
|
+
| "artifact_materializer_source_open_failed"
|
|
268
|
+
| "artifact_materializer_source_content_type_mismatch"
|
|
269
|
+
| "artifact_materializer_source_stream_identity_mismatch"
|
|
270
|
+
| "artifact_materializer_source_revalidation_failed"
|
|
271
|
+
| "artifact_materializer_native_input_framing_failed"
|
|
272
|
+
| "artifact_materializer_native_snapshot_open_failed"
|
|
273
|
+
| "artifact_materializer_native_state_mismatch"
|
|
274
|
+
| undefined {
|
|
275
|
+
const key = `${String(attributes.failureStage)}:${String(attributes.failureSubcode)}`;
|
|
276
|
+
switch (key) {
|
|
277
|
+
case "source_reader:open":
|
|
278
|
+
return "artifact_materializer_source_open_failed";
|
|
279
|
+
case "source_reader:content_type":
|
|
280
|
+
return "artifact_materializer_source_content_type_mismatch";
|
|
281
|
+
case "source_reader:stream_identity":
|
|
282
|
+
return "artifact_materializer_source_stream_identity_mismatch";
|
|
283
|
+
case "source_reader:revalidate":
|
|
284
|
+
return "artifact_materializer_source_revalidation_failed";
|
|
285
|
+
case "native:input_framing":
|
|
286
|
+
return "artifact_materializer_native_input_framing_failed";
|
|
287
|
+
case "native:snapshot_open":
|
|
288
|
+
return "artifact_materializer_native_snapshot_open_failed";
|
|
289
|
+
case "native:state_mismatch":
|
|
290
|
+
return "artifact_materializer_native_state_mismatch";
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
export function createEditableArtifactMaterializerService(
|
|
295
|
+
input: CreateEditableArtifactMaterializerServiceInput,
|
|
296
|
+
): EditableArtifactMaterializerService {
|
|
297
|
+
let state: EditableArtifactMaterializerServiceState = "starting";
|
|
298
|
+
let runPromise: Promise<void> | null = null;
|
|
299
|
+
let closePromise: Promise<void> | null = null;
|
|
300
|
+
let databaseClosed = false;
|
|
301
|
+
const abort = new AbortController();
|
|
302
|
+
const readinessTimeoutMs = input.readinessTimeoutMs ?? 2_000;
|
|
303
|
+
|
|
304
|
+
const ready = async (): Promise<void> => {
|
|
305
|
+
await withTimeout(assertDedicatedMaterializerDatabaseRole(input.db), readinessTimeoutMs);
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
const run = (): Promise<void> => {
|
|
309
|
+
if (runPromise) return runPromise;
|
|
310
|
+
if (state !== "starting") {
|
|
311
|
+
return Promise.reject(new Error("Artifact materializer service cannot be restarted"));
|
|
312
|
+
}
|
|
313
|
+
runPromise = (async () => {
|
|
314
|
+
try {
|
|
315
|
+
await ready();
|
|
316
|
+
if (abort.signal.aborted) {
|
|
317
|
+
state = "stopped";
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
state = "ready";
|
|
321
|
+
await input.materializer.run(abort.signal);
|
|
322
|
+
const exitWasDraining = (state as EditableArtifactMaterializerServiceState) === "draining";
|
|
323
|
+
state = exitWasDraining ? "stopped" : "failed";
|
|
324
|
+
if (!exitWasDraining) {
|
|
325
|
+
throw new Error("Artifact materializer loop exited without a drain request");
|
|
326
|
+
}
|
|
327
|
+
} catch (error) {
|
|
328
|
+
const caughtState = state as EditableArtifactMaterializerServiceState;
|
|
329
|
+
if (caughtState !== "draining" && caughtState !== "stopped") state = "failed";
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
332
|
+
})();
|
|
333
|
+
return runPromise;
|
|
334
|
+
};
|
|
335
|
+
|
|
336
|
+
const drain = (_reason = "shutdown"): boolean => {
|
|
337
|
+
if (state === "draining" || state === "stopped" || state === "failed") return false;
|
|
338
|
+
state = "draining";
|
|
339
|
+
abort.abort();
|
|
340
|
+
return true;
|
|
341
|
+
};
|
|
342
|
+
|
|
343
|
+
const close = (): Promise<void> => {
|
|
344
|
+
if (closePromise) return closePromise;
|
|
345
|
+
closePromise = (async () => {
|
|
346
|
+
drain("close");
|
|
347
|
+
if (runPromise) await runPromise.catch(() => undefined);
|
|
348
|
+
await input.materializer.drain();
|
|
349
|
+
if (!databaseClosed) {
|
|
350
|
+
databaseClosed = true;
|
|
351
|
+
await input.closeDatabase();
|
|
352
|
+
}
|
|
353
|
+
if (state !== "failed") state = "stopped";
|
|
354
|
+
})();
|
|
355
|
+
return closePromise;
|
|
356
|
+
};
|
|
357
|
+
|
|
358
|
+
const fetch = async (request: Request): Promise<Response> => {
|
|
359
|
+
const url = new URL(request.url);
|
|
360
|
+
if (request.method !== "GET") {
|
|
361
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
362
|
+
}
|
|
363
|
+
if (url.pathname === "/healthz") {
|
|
364
|
+
const ok = state !== "failed" && state !== "stopped";
|
|
365
|
+
return Response.json(
|
|
366
|
+
{
|
|
367
|
+
service: input.serviceName,
|
|
368
|
+
component: "artifact-materializer",
|
|
369
|
+
environment: input.environment,
|
|
370
|
+
...(input.deploymentRevision ? { deploymentRevision: input.deploymentRevision } : {}),
|
|
371
|
+
runtime: input.runtimeCapability ?? {
|
|
372
|
+
mode: "production",
|
|
373
|
+
sandboxEnforced: true,
|
|
374
|
+
},
|
|
375
|
+
state,
|
|
376
|
+
ok,
|
|
377
|
+
},
|
|
378
|
+
{ status: ok ? 200 : 503 },
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
if (url.pathname === "/readyz") {
|
|
382
|
+
if (state !== "ready") {
|
|
383
|
+
return Response.json(
|
|
384
|
+
{ ok: false, state, checks: { databaseRole: { ok: false } } },
|
|
385
|
+
{ status: 503 },
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
try {
|
|
389
|
+
await ready();
|
|
390
|
+
return Response.json({
|
|
391
|
+
ok: true,
|
|
392
|
+
state,
|
|
393
|
+
runtime: input.runtimeCapability ?? {
|
|
394
|
+
mode: "production",
|
|
395
|
+
sandboxEnforced: true,
|
|
396
|
+
},
|
|
397
|
+
checks: { databaseRole: { ok: true } },
|
|
398
|
+
});
|
|
399
|
+
} catch {
|
|
400
|
+
return Response.json(
|
|
401
|
+
{ ok: false, state, checks: { databaseRole: { ok: false } } },
|
|
402
|
+
{ status: 503 },
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (url.pathname === "/metrics") {
|
|
407
|
+
return new Response(await input.observability.prometheusMetrics(), {
|
|
408
|
+
status: 200,
|
|
409
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
return Object.freeze({
|
|
416
|
+
state: () => state,
|
|
417
|
+
run,
|
|
418
|
+
drain,
|
|
419
|
+
close,
|
|
420
|
+
dispatchOnce: async (signal?: AbortSignal) => await input.materializer.dispatchOnce(signal),
|
|
421
|
+
fetch,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
export async function assertDedicatedMaterializerDatabaseRole(db: Database): Promise<void> {
|
|
426
|
+
const raw = await db.execute<{
|
|
427
|
+
current_role: string;
|
|
428
|
+
session_role: string;
|
|
429
|
+
can_select_jobs: boolean;
|
|
430
|
+
can_insert_jobs: boolean;
|
|
431
|
+
can_update_jobs: boolean;
|
|
432
|
+
can_delete_jobs: boolean;
|
|
433
|
+
}>(dbSql`
|
|
434
|
+
select
|
|
435
|
+
current_user::text as current_role,
|
|
436
|
+
session_user::text as session_role,
|
|
437
|
+
has_table_privilege(
|
|
438
|
+
current_user,
|
|
439
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
440
|
+
'SELECT'
|
|
441
|
+
) as can_select_jobs,
|
|
442
|
+
has_table_privilege(
|
|
443
|
+
current_user,
|
|
444
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
445
|
+
'INSERT'
|
|
446
|
+
) as can_insert_jobs,
|
|
447
|
+
has_table_privilege(
|
|
448
|
+
current_user,
|
|
449
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
450
|
+
'UPDATE'
|
|
451
|
+
) as can_update_jobs,
|
|
452
|
+
has_table_privilege(
|
|
453
|
+
current_user,
|
|
454
|
+
format('%I.editable_artifact_materialization_jobs', current_schema()),
|
|
455
|
+
'DELETE'
|
|
456
|
+
) as can_delete_jobs
|
|
457
|
+
`);
|
|
458
|
+
const rows = Array.isArray(raw) ? raw : ((raw as unknown as { rows?: unknown[] }).rows ?? []);
|
|
459
|
+
const row = rows[0] as
|
|
460
|
+
| {
|
|
461
|
+
current_role?: unknown;
|
|
462
|
+
session_role?: unknown;
|
|
463
|
+
can_select_jobs?: unknown;
|
|
464
|
+
can_insert_jobs?: unknown;
|
|
465
|
+
can_update_jobs?: unknown;
|
|
466
|
+
can_delete_jobs?: unknown;
|
|
467
|
+
}
|
|
468
|
+
| undefined;
|
|
469
|
+
if (
|
|
470
|
+
row?.current_role !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE ||
|
|
471
|
+
row.session_role !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE ||
|
|
472
|
+
row.can_select_jobs !== false ||
|
|
473
|
+
row.can_insert_jobs !== false ||
|
|
474
|
+
row.can_update_jobs !== false ||
|
|
475
|
+
row.can_delete_jobs !== false
|
|
476
|
+
) {
|
|
477
|
+
throw new Error("Artifact materializer database connection has an unsafe role posture");
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
export type EditableArtifactMaterializerSidecarEnvironment = Readonly<{
|
|
482
|
+
databaseUrl: string;
|
|
483
|
+
declaredDatabaseRole: string;
|
|
484
|
+
executable: string;
|
|
485
|
+
bubblewrapExecutable: string | null;
|
|
486
|
+
prlimitExecutable: string | null;
|
|
487
|
+
runtimeAuthority: EditableArtifactRuntimeAuthority;
|
|
488
|
+
skillFacadeEntrypoint: string;
|
|
489
|
+
owner: string;
|
|
490
|
+
httpHostname: string;
|
|
491
|
+
httpPort: number;
|
|
492
|
+
batchSize: number;
|
|
493
|
+
concurrency: number;
|
|
494
|
+
leaseDurationMs: number;
|
|
495
|
+
leaseRenewIntervalMs: number;
|
|
496
|
+
pollIntervalMs: number;
|
|
497
|
+
maxAttempts: number;
|
|
498
|
+
memoryLimitBytes: number;
|
|
499
|
+
cpuTimeLimitMs: number;
|
|
500
|
+
fileDescriptorLimit: number;
|
|
501
|
+
processLimit: number;
|
|
502
|
+
fileSizeLimitBytes: number;
|
|
503
|
+
wallTimeoutMs: number;
|
|
504
|
+
maxSourceBytes: number;
|
|
505
|
+
maxOutputBytes: number;
|
|
506
|
+
allowUnsandboxedLocalDevelopment: boolean;
|
|
507
|
+
nodeEnvironment?: string;
|
|
508
|
+
}>;
|
|
509
|
+
|
|
510
|
+
export function readEditableArtifactMaterializerSidecarEnvironment(
|
|
511
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
512
|
+
): EditableArtifactMaterializerSidecarEnvironment | null {
|
|
513
|
+
if (environment.OPENGENI_ARTIFACT_MATERIALIZER_ENABLED !== "true") return null;
|
|
514
|
+
const databaseUrl = required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_DATABASE_URL");
|
|
515
|
+
const declaredDatabaseRole = required(
|
|
516
|
+
environment,
|
|
517
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_DATABASE_ROLE",
|
|
518
|
+
);
|
|
519
|
+
if (declaredDatabaseRole !== EDITABLE_ARTIFACT_MATERIALIZER_DATABASE_ROLE) {
|
|
520
|
+
throw new Error("OPENGENI_ARTIFACT_MATERIALIZER_DATABASE_ROLE is invalid");
|
|
521
|
+
}
|
|
522
|
+
const productionManifest = optional(environment, "OPENGENI_ARTIFACT_RUNTIME_MANIFEST");
|
|
523
|
+
const developmentManifest = optional(
|
|
524
|
+
environment,
|
|
525
|
+
"OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST",
|
|
526
|
+
);
|
|
527
|
+
if (productionManifest && developmentManifest) {
|
|
528
|
+
throw new Error(
|
|
529
|
+
"Production and development artifact runtime manifests cannot both be configured",
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
if (!productionManifest && !developmentManifest) {
|
|
533
|
+
throw new Error(
|
|
534
|
+
"An artifact runtime manifest is required when artifact materialization is enabled",
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
const development = developmentManifest !== null;
|
|
538
|
+
if (development) {
|
|
539
|
+
if (environment.NODE_ENV === "production") {
|
|
540
|
+
throw new Error("Development artifact runtime is forbidden when NODE_ENV=production");
|
|
541
|
+
}
|
|
542
|
+
if (
|
|
543
|
+
environment.OPENGENI_ARTIFACT_LOCAL_DEVELOPMENT !== "true" ||
|
|
544
|
+
environment.OPENGENI_ARTIFACT_MATERIALIZER_UNSANDBOXED_DEVELOPMENT !== "true"
|
|
545
|
+
) {
|
|
546
|
+
throw new Error(
|
|
547
|
+
"Unsandboxed artifact materialization requires explicit local-development gates",
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
assertLoopbackDatabaseUrl(databaseUrl);
|
|
551
|
+
assertLoopbackObjectStorageUrl(required(environment, "OPENGENI_OBJECT_STORAGE_ENDPOINT"));
|
|
552
|
+
}
|
|
553
|
+
const httpHostname =
|
|
554
|
+
optional(environment, "OPENGENI_ARTIFACT_MATERIALIZER_HTTP_HOST") ?? "0.0.0.0";
|
|
555
|
+
if (development && !isLoopbackHostname(httpHostname)) {
|
|
556
|
+
throw new Error("Unsandboxed artifact materialization requires a loopback HTTP listener");
|
|
557
|
+
}
|
|
558
|
+
const runtimeAuthority: EditableArtifactRuntimeAuthority = developmentManifest
|
|
559
|
+
? Object.freeze({
|
|
560
|
+
mode: "development-current-host",
|
|
561
|
+
environmentVariable: "OPENGENI_ARTIFACT_DEVELOPMENT_RUNTIME_MANIFEST",
|
|
562
|
+
manifestPath: developmentManifest,
|
|
563
|
+
})
|
|
564
|
+
: Object.freeze({
|
|
565
|
+
mode: "production",
|
|
566
|
+
environmentVariable: "OPENGENI_ARTIFACT_RUNTIME_MANIFEST",
|
|
567
|
+
manifestPath: productionManifest!,
|
|
568
|
+
});
|
|
569
|
+
return Object.freeze({
|
|
570
|
+
databaseUrl,
|
|
571
|
+
declaredDatabaseRole,
|
|
572
|
+
executable: required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_EXECUTABLE"),
|
|
573
|
+
bubblewrapExecutable: development
|
|
574
|
+
? null
|
|
575
|
+
: required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_BWRAP"),
|
|
576
|
+
prlimitExecutable: development
|
|
577
|
+
? null
|
|
578
|
+
: required(environment, "OPENGENI_ARTIFACT_MATERIALIZER_PRLIMIT"),
|
|
579
|
+
runtimeAuthority,
|
|
580
|
+
allowUnsandboxedLocalDevelopment: development,
|
|
581
|
+
skillFacadeEntrypoint: required(environment, "OPENGENI_ARTIFACT_TOOL_ENTRY"),
|
|
582
|
+
owner:
|
|
583
|
+
environment.OPENGENI_ARTIFACT_MATERIALIZER_OWNER?.trim() ||
|
|
584
|
+
`materializer-${process.pid}-${crypto.randomUUID()}`,
|
|
585
|
+
httpHostname,
|
|
586
|
+
httpPort: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_HTTP_PORT", 9465, 1, 65_535),
|
|
587
|
+
batchSize: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_BATCH_SIZE", 8, 1, 64),
|
|
588
|
+
concurrency: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_CONCURRENCY", 2, 1, 64),
|
|
589
|
+
leaseDurationMs: integer(
|
|
590
|
+
environment,
|
|
591
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_LEASE_MS",
|
|
592
|
+
120_000,
|
|
593
|
+
10_000,
|
|
594
|
+
86_400_000,
|
|
595
|
+
),
|
|
596
|
+
leaseRenewIntervalMs: integer(
|
|
597
|
+
environment,
|
|
598
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_RENEW_MS",
|
|
599
|
+
30_000,
|
|
600
|
+
1_000,
|
|
601
|
+
43_199_999,
|
|
602
|
+
),
|
|
603
|
+
pollIntervalMs: integer(
|
|
604
|
+
environment,
|
|
605
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_POLL_MS",
|
|
606
|
+
1_000,
|
|
607
|
+
10,
|
|
608
|
+
60_000,
|
|
609
|
+
),
|
|
610
|
+
maxAttempts: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_MAX_ATTEMPTS", 20, 1, 1_000),
|
|
611
|
+
memoryLimitBytes: integer(
|
|
612
|
+
environment,
|
|
613
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_MEMORY_BYTES",
|
|
614
|
+
512 * 1024 * 1024,
|
|
615
|
+
64 * 1024 * 1024,
|
|
616
|
+
8 * 1024 * 1024 * 1024,
|
|
617
|
+
),
|
|
618
|
+
cpuTimeLimitMs: integer(
|
|
619
|
+
environment,
|
|
620
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_CPU_MS",
|
|
621
|
+
60_000,
|
|
622
|
+
1_000,
|
|
623
|
+
3_600_000,
|
|
624
|
+
),
|
|
625
|
+
fileDescriptorLimit: integer(
|
|
626
|
+
environment,
|
|
627
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_OPEN_FILES",
|
|
628
|
+
64,
|
|
629
|
+
16,
|
|
630
|
+
4_096,
|
|
631
|
+
),
|
|
632
|
+
processLimit: integer(environment, "OPENGENI_ARTIFACT_MATERIALIZER_PROCESSES", 64, 1, 1_024),
|
|
633
|
+
fileSizeLimitBytes: integer(
|
|
634
|
+
environment,
|
|
635
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_FILE_BYTES",
|
|
636
|
+
512 * 1024 * 1024,
|
|
637
|
+
1024 * 1024,
|
|
638
|
+
8 * 1024 * 1024 * 1024,
|
|
639
|
+
),
|
|
640
|
+
wallTimeoutMs: integer(
|
|
641
|
+
environment,
|
|
642
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_WALL_MS",
|
|
643
|
+
120_000,
|
|
644
|
+
1_000,
|
|
645
|
+
3_600_000,
|
|
646
|
+
),
|
|
647
|
+
maxSourceBytes: integer(
|
|
648
|
+
environment,
|
|
649
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES",
|
|
650
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES,
|
|
651
|
+
1,
|
|
652
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_SOURCE_BYTES,
|
|
653
|
+
),
|
|
654
|
+
maxOutputBytes: integer(
|
|
655
|
+
environment,
|
|
656
|
+
"OPENGENI_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES",
|
|
657
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES,
|
|
658
|
+
1,
|
|
659
|
+
EDITABLE_ARTIFACT_MATERIALIZER_MAX_OUTPUT_BYTES,
|
|
660
|
+
),
|
|
661
|
+
...(environment.NODE_ENV === undefined ? {} : { nodeEnvironment: environment.NODE_ENV }),
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
export async function createMaterializerSidecarFromEnvironment(
|
|
666
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
667
|
+
): Promise<Readonly<{
|
|
668
|
+
service: EditableArtifactMaterializerService;
|
|
669
|
+
httpHostname: string;
|
|
670
|
+
httpPort: number;
|
|
671
|
+
}> | null> {
|
|
672
|
+
const sidecar = readEditableArtifactMaterializerSidecarEnvironment(environment);
|
|
673
|
+
if (!sidecar) return null;
|
|
674
|
+
const settings = getSettings();
|
|
675
|
+
const service = await createProductionEditableArtifactMaterializerService({
|
|
676
|
+
settings,
|
|
677
|
+
databaseUrl: sidecar.databaseUrl,
|
|
678
|
+
declaredDatabaseRole: sidecar.declaredDatabaseRole,
|
|
679
|
+
executable: sidecar.executable,
|
|
680
|
+
bubblewrapExecutable: sidecar.bubblewrapExecutable,
|
|
681
|
+
prlimitExecutable: sidecar.prlimitExecutable,
|
|
682
|
+
runtimeAuthority: sidecar.runtimeAuthority,
|
|
683
|
+
...(sidecar.allowUnsandboxedLocalDevelopment ? { allowUnsandboxedLocalDevelopment: true } : {}),
|
|
684
|
+
skillFacadeEntrypoint: sidecar.skillFacadeEntrypoint,
|
|
685
|
+
owner: sidecar.owner,
|
|
686
|
+
batchSize: sidecar.batchSize,
|
|
687
|
+
concurrency: sidecar.concurrency,
|
|
688
|
+
leaseDurationMs: sidecar.leaseDurationMs,
|
|
689
|
+
leaseRenewIntervalMs: sidecar.leaseRenewIntervalMs,
|
|
690
|
+
pollIntervalMs: sidecar.pollIntervalMs,
|
|
691
|
+
maxAttempts: sidecar.maxAttempts,
|
|
692
|
+
memoryLimitBytes: sidecar.memoryLimitBytes,
|
|
693
|
+
cpuTimeLimitMs: sidecar.cpuTimeLimitMs,
|
|
694
|
+
fileDescriptorLimit: sidecar.fileDescriptorLimit,
|
|
695
|
+
processLimit: sidecar.processLimit,
|
|
696
|
+
fileSizeLimitBytes: sidecar.fileSizeLimitBytes,
|
|
697
|
+
wallTimeoutMs: sidecar.wallTimeoutMs,
|
|
698
|
+
maxSourceBytes: sidecar.maxSourceBytes,
|
|
699
|
+
maxOutputBytes: sidecar.maxOutputBytes,
|
|
700
|
+
...(sidecar.nodeEnvironment === undefined ? {} : { nodeEnvironment: sidecar.nodeEnvironment }),
|
|
701
|
+
});
|
|
702
|
+
return Object.freeze({
|
|
703
|
+
service,
|
|
704
|
+
httpHostname: sidecar.httpHostname,
|
|
705
|
+
httpPort: sidecar.httpPort,
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
async function withTimeout<T>(promise: Promise<T>, milliseconds: number): Promise<T> {
|
|
710
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
711
|
+
try {
|
|
712
|
+
return await Promise.race([
|
|
713
|
+
promise,
|
|
714
|
+
new Promise<never>((_resolve, reject) => {
|
|
715
|
+
timer = setTimeout(() => reject(new Error("readiness check timed out")), milliseconds);
|
|
716
|
+
}),
|
|
717
|
+
]);
|
|
718
|
+
} finally {
|
|
719
|
+
if (timer) clearTimeout(timer);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
function required(environment: NodeJS.ProcessEnv, name: string): string {
|
|
724
|
+
const value = environment[name]?.trim();
|
|
725
|
+
if (!value) throw new Error(`${name} is required when artifact materialization is enabled`);
|
|
726
|
+
return value;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function optional(environment: NodeJS.ProcessEnv, name: string): string | null {
|
|
730
|
+
const value = environment[name]?.trim();
|
|
731
|
+
return value ? value : null;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
function requiredOption(value: string | null, name: string): string {
|
|
735
|
+
if (!value) throw new Error(`Artifact materializer ${name} is required in production`);
|
|
736
|
+
return value;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
function assertLoopbackDatabaseUrl(databaseUrl: string): void {
|
|
740
|
+
let parsed: URL;
|
|
741
|
+
try {
|
|
742
|
+
parsed = new URL(databaseUrl);
|
|
743
|
+
} catch {
|
|
744
|
+
throw new Error("Local-development artifact materializer database DSN is invalid");
|
|
745
|
+
}
|
|
746
|
+
if (parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") {
|
|
747
|
+
throw new Error("Local-development artifact materializer database DSN is invalid");
|
|
748
|
+
}
|
|
749
|
+
if (!isLoopbackHostname(parsed.hostname)) {
|
|
750
|
+
throw new Error("Unsandboxed artifact materialization requires a loopback database DSN");
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
function assertLoopbackObjectStorageUrl(endpoint: string): void {
|
|
755
|
+
let parsed: URL;
|
|
756
|
+
try {
|
|
757
|
+
parsed = new URL(endpoint);
|
|
758
|
+
} catch {
|
|
759
|
+
throw new Error("Local-development artifact object-storage endpoint is invalid");
|
|
760
|
+
}
|
|
761
|
+
if (
|
|
762
|
+
(parsed.protocol !== "http:" && parsed.protocol !== "https:") ||
|
|
763
|
+
!isLoopbackHostname(parsed.hostname)
|
|
764
|
+
) {
|
|
765
|
+
throw new Error("Unsandboxed artifact materialization requires loopback object storage");
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
function isLoopbackHostname(hostname: string): boolean {
|
|
770
|
+
const normalized = hostname.toLowerCase();
|
|
771
|
+
return normalized === "localhost" || normalized === "127.0.0.1" || normalized === "[::1]";
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
function integer(
|
|
775
|
+
environment: NodeJS.ProcessEnv,
|
|
776
|
+
name: string,
|
|
777
|
+
fallback: number,
|
|
778
|
+
minimum: number,
|
|
779
|
+
maximum: number,
|
|
780
|
+
): number {
|
|
781
|
+
const raw = environment[name];
|
|
782
|
+
const value = raw === undefined ? fallback : Number(raw);
|
|
783
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
784
|
+
throw new Error(`${name} is outside its allowed range`);
|
|
785
|
+
}
|
|
786
|
+
return value;
|
|
787
|
+
}
|