@package-broker/runtime-node 0.0.0-stage → 2.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/assets.d.ts +4 -0
- package/dist/assets.d.ts.map +1 -0
- package/dist/assets.js +68 -0
- package/dist/assets.js.map +1 -0
- package/dist/blob/archive-staging.d.ts +15 -0
- package/dist/blob/archive-staging.d.ts.map +1 -0
- package/dist/blob/archive-staging.js +154 -0
- package/dist/blob/archive-staging.js.map +1 -0
- package/dist/blob/filesystem.d.ts +18 -0
- package/dist/blob/filesystem.d.ts.map +1 -0
- package/dist/blob/filesystem.js +631 -0
- package/dist/blob/filesystem.js.map +1 -0
- package/dist/blob/hasher.d.ts +3 -0
- package/dist/blob/hasher.d.ts.map +1 -0
- package/dist/blob/hasher.js +26 -0
- package/dist/blob/hasher.js.map +1 -0
- package/dist/blob/s3.d.ts +40 -0
- package/dist/blob/s3.d.ts.map +1 -0
- package/dist/blob/s3.js +494 -0
- package/dist/blob/s3.js.map +1 -0
- package/dist/cache/lru.d.ts +8 -0
- package/dist/cache/lru.d.ts.map +1 -0
- package/dist/cache/lru.js +103 -0
- package/dist/cache/lru.js.map +1 -0
- package/dist/client-ip.d.ts +2 -0
- package/dist/client-ip.d.ts.map +1 -0
- package/dist/client-ip.js +51 -0
- package/dist/client-ip.js.map +1 -0
- package/dist/codec/node-codec.d.ts +3 -0
- package/dist/codec/node-codec.d.ts.map +1 -0
- package/dist/codec/node-codec.js +163 -0
- package/dist/codec/node-codec.js.map +1 -0
- package/dist/db/better-sqlite3-executor.d.ts +10 -0
- package/dist/db/better-sqlite3-executor.d.ts.map +1 -0
- package/dist/db/better-sqlite3-executor.js +129 -0
- package/dist/db/better-sqlite3-executor.js.map +1 -0
- package/dist/entry.d.ts +32 -0
- package/dist/entry.d.ts.map +1 -0
- package/dist/entry.js +158 -0
- package/dist/entry.js.map +1 -0
- package/dist/import-jobs.d.ts +18 -0
- package/dist/import-jobs.d.ts.map +1 -0
- package/dist/import-jobs.js +56 -0
- package/dist/import-jobs.js.map +1 -0
- package/dist/index.d.ts +112 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +981 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/seams-key.d.ts +2 -0
- package/dist/internal/seams-key.d.ts.map +1 -0
- package/dist/internal/seams-key.js +2 -0
- package/dist/internal/seams-key.js.map +1 -0
- package/dist/jobs/loop.d.ts +38 -0
- package/dist/jobs/loop.d.ts.map +1 -0
- package/dist/jobs/loop.js +318 -0
- package/dist/jobs/loop.js.map +1 -0
- package/dist/observability/stdio.d.ts +11 -0
- package/dist/observability/stdio.d.ts.map +1 -0
- package/dist/observability/stdio.js +47 -0
- package/dist/observability/stdio.js.map +1 -0
- package/dist/organization-administration.d.ts +5 -0
- package/dist/organization-administration.d.ts.map +1 -0
- package/dist/organization-administration.js +16 -0
- package/dist/organization-administration.js.map +1 -0
- package/dist/sources/dns-guard.d.ts +33 -0
- package/dist/sources/dns-guard.d.ts.map +1 -0
- package/dist/sources/dns-guard.js +202 -0
- package/dist/sources/dns-guard.js.map +1 -0
- package/dist/sources/git-provider.d.ts +30 -0
- package/dist/sources/git-provider.d.ts.map +1 -0
- package/dist/sources/git-provider.js +213 -0
- package/dist/sources/git-provider.js.map +1 -0
- package/dist/sources/guarded-fetch.d.ts +15 -0
- package/dist/sources/guarded-fetch.d.ts.map +1 -0
- package/dist/sources/guarded-fetch.js +217 -0
- package/dist/sources/guarded-fetch.js.map +1 -0
- package/dist/sources/ssh-archive-store.d.ts +19 -0
- package/dist/sources/ssh-archive-store.d.ts.map +1 -0
- package/dist/sources/ssh-archive-store.js +284 -0
- package/dist/sources/ssh-archive-store.js.map +1 -0
- package/dist/sources/ssh-git.d.ts +46 -0
- package/dist/sources/ssh-git.d.ts.map +1 -0
- package/dist/sources/ssh-git.js +430 -0
- package/dist/sources/ssh-git.js.map +1 -0
- package/dist/sources/ssh-provider.d.ts +55 -0
- package/dist/sources/ssh-provider.d.ts.map +1 -0
- package/dist/sources/ssh-provider.js +369 -0
- package/dist/sources/ssh-provider.js.map +1 -0
- package/dist/startup.d.ts +49 -0
- package/dist/startup.d.ts.map +1 -0
- package/dist/startup.js +687 -0
- package/dist/startup.js.map +1 -0
- package/dist/token-jobs.d.ts +18 -0
- package/dist/token-jobs.d.ts.map +1 -0
- package/dist/token-jobs.js +45 -0
- package/dist/token-jobs.js.map +1 -0
- package/package.json +45 -3
- package/README.md +0 -3
package/dist/startup.js
ADDED
|
@@ -0,0 +1,687 @@
|
|
|
1
|
+
import { assertMcpRateLimit } from '@package-broker/core/mcp/limit';
|
|
2
|
+
import { INGEST_EVENTS, INGEST_REASON_CODES } from '@package-broker/core/ingest/finalize-guard';
|
|
3
|
+
import { mkdir, readFile, stat } from 'node:fs/promises';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { AUTH_REASON_CODES, FINISH_FAILURE_DETAILS, isAuthReasonCode, } from '@package-broker/core/auth/reason-codes';
|
|
6
|
+
import { selfTestPasswordHashing } from '@package-broker/core/auth/password';
|
|
7
|
+
import { systemClock, ulid } from '@package-broker/core/ids';
|
|
8
|
+
import { parsePublicOrigin, PublicOriginError } from '@package-broker/core/kernel/public-origin';
|
|
9
|
+
import { AUDIT_EVENTS } from '@package-broker/core/kernel/obs-codes';
|
|
10
|
+
import { BlobValidationError, createBlobStore, createBlobUploadAdmission, } from '@package-broker/core/platform/blob';
|
|
11
|
+
import { completionSpan, upstreamSpan } from '@package-broker/core/platform/completion-span';
|
|
12
|
+
import { createObservability, safeErrorName, filterLogLevel, } from '@package-broker/core/platform/observability';
|
|
13
|
+
import { createOtlpExporter } from '@package-broker/core/platform/otlp';
|
|
14
|
+
import { createOtlpSink } from '@package-broker/core/platform/otlp-sink';
|
|
15
|
+
import { validatePlatformPorts, } from '@package-broker/core/platform/port-config';
|
|
16
|
+
import { createKeyring } from '@package-broker/core/secrets/keyring';
|
|
17
|
+
import { KeyringStoreUnavailableError, KeyringMismatchError, KeyringVerifierMissingError, KeyringVersionInUseError, RESEED_VARIABLE, verifyKeyring, } from '@package-broker/core/secrets/keyring-verifier';
|
|
18
|
+
import { readMasterKeys, SecretsConfigError } from '@package-broker/core/secrets/master-key';
|
|
19
|
+
import { migrate } from '@package-broker/db-sqlite/migrations/runner';
|
|
20
|
+
import { createBlobRepository } from '@package-broker/db-sqlite/repositories/blob';
|
|
21
|
+
import { createJobRepository } from '@package-broker/db-sqlite/repositories/job';
|
|
22
|
+
import { createKeyringVerifierStore } from '@package-broker/db-sqlite/repositories/keyring-verifier';
|
|
23
|
+
import { createOrganizationBootstrap } from '@package-broker/db-sqlite/repositories/organization';
|
|
24
|
+
import { FilesystemRootError, openFilesystemByteStore } from './blob/filesystem.js';
|
|
25
|
+
import { createS3ByteStore, S3ConfigError } from './blob/s3.js';
|
|
26
|
+
import { createLruCache } from './cache/lru.js';
|
|
27
|
+
import { openBetterSqliteExecutor } from './db/better-sqlite3-executor.js';
|
|
28
|
+
import { createNodeServer } from './index.js';
|
|
29
|
+
import { mcpEnabled, brokerObservabilityProblems, } from '@package-broker/core/config';
|
|
30
|
+
import { RegistryConfigError, resolveRegistryConfig, } from '@package-broker/core/registry/registry-config';
|
|
31
|
+
import { resolveTokenLifetimePolicy, TokenConfigError, } from '@package-broker/core/authz/token-lifetime';
|
|
32
|
+
import { createJobLoop, JOB_EVENTS } from './jobs/loop.js';
|
|
33
|
+
import { createStdioSink } from './observability/stdio.js';
|
|
34
|
+
export class AlreadyBootstrappedError extends Error {
|
|
35
|
+
slug;
|
|
36
|
+
constructor(slug) {
|
|
37
|
+
super(`the database is already bootstrapped with the slug ${slug}`);
|
|
38
|
+
this.name = 'AlreadyBootstrappedError';
|
|
39
|
+
this.slug = slug;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class StartupConfigError extends Error {
|
|
43
|
+
variable;
|
|
44
|
+
constructor(variable, problem) {
|
|
45
|
+
super(`${variable} ${problem}`);
|
|
46
|
+
this.name = 'StartupConfigError';
|
|
47
|
+
this.variable = variable;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export const BLOB_PURGE_LEASE_SECONDS = 900;
|
|
51
|
+
export const BLOB_DELETE_TIMEOUT_MS = 30_000;
|
|
52
|
+
export function blobPurgeTiming(deleteTimeoutMs, purgeLeaseSeconds) {
|
|
53
|
+
const leaseMs = purgeLeaseSeconds * 1000;
|
|
54
|
+
if (!Number.isInteger(leaseMs) ||
|
|
55
|
+
leaseMs <= 0 ||
|
|
56
|
+
!Number.isInteger(deleteTimeoutMs) ||
|
|
57
|
+
deleteTimeoutMs < 1000 ||
|
|
58
|
+
deleteTimeoutMs > leaseMs / 10) {
|
|
59
|
+
throw new StartupConfigError('blob.deleteTimeoutMs', 'must be at least 1000 ms and at most a tenth of the purge lease');
|
|
60
|
+
}
|
|
61
|
+
return { deleteTimeoutMs, leaseMs };
|
|
62
|
+
}
|
|
63
|
+
const DEFAULT_DATABASE_PATH = './data/broker.sqlite';
|
|
64
|
+
const IDENTITY_ENV_PREFIX = 'PB_IDP_';
|
|
65
|
+
function migrateLog(log) {
|
|
66
|
+
return (event) => {
|
|
67
|
+
const entry = { level: event.level, message: event.message };
|
|
68
|
+
if (event.id !== undefined)
|
|
69
|
+
entry['id'] = event.id;
|
|
70
|
+
if (event.holder !== undefined)
|
|
71
|
+
entry['holder'] = event.holder;
|
|
72
|
+
log(entry);
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
function present(value) {
|
|
76
|
+
return value !== undefined && value !== '';
|
|
77
|
+
}
|
|
78
|
+
const MAX_KEY_FILE_BYTES = 4096;
|
|
79
|
+
const ERRNO_CODE = /^E[A-Z0-9]{2,15}$/;
|
|
80
|
+
function errnoSuffix(error) {
|
|
81
|
+
const code = error instanceof Error ? error.code : undefined;
|
|
82
|
+
return typeof code === 'string' && ERRNO_CODE.test(code) ? ` (${code})` : '';
|
|
83
|
+
}
|
|
84
|
+
async function readKeyFile(variable, file) {
|
|
85
|
+
let bytes;
|
|
86
|
+
try {
|
|
87
|
+
const stats = await stat(file);
|
|
88
|
+
if (!stats.isFile())
|
|
89
|
+
throw new StartupConfigError(variable, 'is not a regular file');
|
|
90
|
+
if (stats.size > MAX_KEY_FILE_BYTES) {
|
|
91
|
+
throw new StartupConfigError(variable, `is larger than ${MAX_KEY_FILE_BYTES} bytes`);
|
|
92
|
+
}
|
|
93
|
+
bytes = await readFile(file);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
if (error instanceof StartupConfigError)
|
|
97
|
+
throw error;
|
|
98
|
+
throw new StartupConfigError(variable, `is unreadable${errnoSuffix(error)}`);
|
|
99
|
+
}
|
|
100
|
+
if (bytes.length > MAX_KEY_FILE_BYTES) {
|
|
101
|
+
throw new StartupConfigError(variable, `is larger than ${MAX_KEY_FILE_BYTES} bytes`);
|
|
102
|
+
}
|
|
103
|
+
const content = bytes.toString('utf8');
|
|
104
|
+
return content.endsWith('\n') ? content.slice(0, -1) : content;
|
|
105
|
+
}
|
|
106
|
+
async function secretValue(env, variable) {
|
|
107
|
+
const fileVariable = `${variable}_FILE`;
|
|
108
|
+
const direct = env[variable];
|
|
109
|
+
const file = env[fileVariable];
|
|
110
|
+
if (direct !== undefined && file !== undefined) {
|
|
111
|
+
throw new StartupConfigError(fileVariable, `and ${variable} are both set`);
|
|
112
|
+
}
|
|
113
|
+
if (direct !== undefined) {
|
|
114
|
+
if (direct === '')
|
|
115
|
+
throw new StartupConfigError(variable, 'is empty');
|
|
116
|
+
return { value: direct, fromFile: false };
|
|
117
|
+
}
|
|
118
|
+
if (file === undefined)
|
|
119
|
+
return { value: undefined, fromFile: false };
|
|
120
|
+
if (file === '')
|
|
121
|
+
throw new StartupConfigError(fileVariable, 'is empty');
|
|
122
|
+
const value = await readKeyFile(fileVariable, file);
|
|
123
|
+
if (value === '')
|
|
124
|
+
throw new StartupConfigError(fileVariable, 'has empty content');
|
|
125
|
+
return { value, fromFile: true };
|
|
126
|
+
}
|
|
127
|
+
async function readKeyring(env) {
|
|
128
|
+
const current = await secretValue(env, 'PB_MASTER_KEY');
|
|
129
|
+
const previous = await secretValue(env, 'PB_MASTER_KEY_PREVIOUS');
|
|
130
|
+
try {
|
|
131
|
+
return createKeyring(readMasterKeys({ current: current.value, previous: previous.value }));
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
if (!(error instanceof SecretsConfigError))
|
|
135
|
+
throw error;
|
|
136
|
+
const fromFile = error.variable === 'PB_MASTER_KEY'
|
|
137
|
+
? current.fromFile
|
|
138
|
+
: error.variable === 'PB_MASTER_KEY_PREVIOUS' && previous.fromFile;
|
|
139
|
+
if (!fromFile)
|
|
140
|
+
throw error;
|
|
141
|
+
throw new SecretsConfigError(error.problem, error.variable === 'PB_MASTER_KEY' ? 'PB_MASTER_KEY_FILE' : 'PB_MASTER_KEY_PREVIOUS_FILE');
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function publicOrigin(raw) {
|
|
145
|
+
try {
|
|
146
|
+
return parsePublicOrigin(raw, { http: 'loopback' });
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (!(error instanceof PublicOriginError))
|
|
150
|
+
throw error;
|
|
151
|
+
const problems = {
|
|
152
|
+
missing: 'is missing',
|
|
153
|
+
'absolute-url': 'is not an absolute URL',
|
|
154
|
+
scheme: 'must be https, or http on a loopback host',
|
|
155
|
+
origin: 'must be an origin without path, query or credentials',
|
|
156
|
+
};
|
|
157
|
+
throw new StartupConfigError('PB_PUBLIC_URL', problems[error.problem]);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
function registryConfig(input, origin) {
|
|
161
|
+
try {
|
|
162
|
+
return resolveRegistryConfig(input, 'node', origin);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
if (!(error instanceof RegistryConfigError))
|
|
166
|
+
throw error;
|
|
167
|
+
throw new StartupConfigError('registry', error.problems.join('; '));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
function checkTokens(input) {
|
|
171
|
+
try {
|
|
172
|
+
resolveTokenLifetimePolicy(input);
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
if (!(error instanceof TokenConfigError))
|
|
176
|
+
throw error;
|
|
177
|
+
throw new StartupConfigError('config', error.problems.join('; '));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
function identityEnv(env) {
|
|
181
|
+
return Object.freeze(Object.fromEntries(Object.entries(env).filter(([name]) => name.startsWith(IDENTITY_ENV_PREFIX))));
|
|
182
|
+
}
|
|
183
|
+
async function bootstrapOrganization(executor, slug, name) {
|
|
184
|
+
const result = await createOrganizationBootstrap(executor).bootstrap({
|
|
185
|
+
id: ulid(),
|
|
186
|
+
slug,
|
|
187
|
+
name,
|
|
188
|
+
now: systemClock.now().toISOString(),
|
|
189
|
+
});
|
|
190
|
+
if (!result.created && result.slug !== slug)
|
|
191
|
+
throw new AlreadyBootstrappedError(result.slug);
|
|
192
|
+
}
|
|
193
|
+
export const AUTH_EVENTS = Object.freeze([
|
|
194
|
+
'auth.hook.denied',
|
|
195
|
+
'auth.hook.error',
|
|
196
|
+
'auth.local.locked',
|
|
197
|
+
'auth.local.setup.completed',
|
|
198
|
+
'auth.local.setup.refused',
|
|
199
|
+
'auth.login.refused',
|
|
200
|
+
'auth.login.succeeded',
|
|
201
|
+
'auth.logout.refused',
|
|
202
|
+
'auth.reauth.refused',
|
|
203
|
+
'auth.reauth.succeeded',
|
|
204
|
+
'token_touch_failed',
|
|
205
|
+
]);
|
|
206
|
+
export const NODE_RUNTIME_EVENTS = new Set([
|
|
207
|
+
...AUDIT_EVENTS,
|
|
208
|
+
...INGEST_EVENTS,
|
|
209
|
+
...JOB_EVENTS,
|
|
210
|
+
...AUTH_EVENTS,
|
|
211
|
+
'http.error',
|
|
212
|
+
'shutdown.jobs_abandoned',
|
|
213
|
+
'startup.failed',
|
|
214
|
+
'process.uncaught_exception',
|
|
215
|
+
'process.unhandled_rejection',
|
|
216
|
+
]);
|
|
217
|
+
const FLUSH_INTERVAL_MS = 1000;
|
|
218
|
+
const JOBS_ABANDONED_COUNTER = 'pb.jobs.abandoned';
|
|
219
|
+
const LOGIN_OUTCOMES = {
|
|
220
|
+
'auth.login.succeeded': 'succeeded',
|
|
221
|
+
'auth.login.refused': 'refused',
|
|
222
|
+
};
|
|
223
|
+
export function registeredAuthReason(value) {
|
|
224
|
+
return isAuthReasonCode(value) ? value : undefined;
|
|
225
|
+
}
|
|
226
|
+
export function sinkLog(observe) {
|
|
227
|
+
const unscoped = observe();
|
|
228
|
+
return (entry) => {
|
|
229
|
+
const requestId = entry['request_id'] ?? entry['requestId'];
|
|
230
|
+
const plugin = entry['pluginId'];
|
|
231
|
+
const scope = {
|
|
232
|
+
...(typeof requestId === 'string' ? { requestId } : {}),
|
|
233
|
+
...(typeof plugin === 'string' ? { plugin } : {}),
|
|
234
|
+
};
|
|
235
|
+
let observability = unscoped;
|
|
236
|
+
if (scope.requestId !== undefined || scope.plugin !== undefined) {
|
|
237
|
+
try {
|
|
238
|
+
observability = observe(scope);
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
observability = unscoped;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
const error = entry['error_name'] ?? entry['error'];
|
|
245
|
+
const detail = registeredAuthReason(entry['detail']);
|
|
246
|
+
const reason = detail ?? registeredAuthReason(entry['reason']) ?? registeredAuthReason(entry['code']);
|
|
247
|
+
const event = entry['event'];
|
|
248
|
+
observability.log({
|
|
249
|
+
level: entry['level'],
|
|
250
|
+
event,
|
|
251
|
+
...(error === undefined ? {} : { error_name: error }),
|
|
252
|
+
...(reason === undefined ? {} : { reason_code: reason }),
|
|
253
|
+
});
|
|
254
|
+
const outcome = LOGIN_OUTCOMES[event];
|
|
255
|
+
const provider = entry['provider'];
|
|
256
|
+
if (outcome !== undefined && typeof provider === 'string') {
|
|
257
|
+
const finishDetail = outcome === 'refused' &&
|
|
258
|
+
entry['reason'] === 'provider_error' &&
|
|
259
|
+
detail !== undefined &&
|
|
260
|
+
FINISH_FAILURE_DETAILS.has(detail);
|
|
261
|
+
observability.counter('pb.auth.login', {
|
|
262
|
+
provider,
|
|
263
|
+
outcome,
|
|
264
|
+
...(finishDetail ? { detail } : {}),
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
function requiredEnv(env, variable) {
|
|
270
|
+
const value = env[variable];
|
|
271
|
+
if (!present(value))
|
|
272
|
+
throw new StartupConfigError(variable, 'is missing');
|
|
273
|
+
return value;
|
|
274
|
+
}
|
|
275
|
+
function otlpHeaders(env, variable) {
|
|
276
|
+
if (variable === undefined)
|
|
277
|
+
return {};
|
|
278
|
+
const raw = requiredEnv(env, variable);
|
|
279
|
+
let parsed;
|
|
280
|
+
try {
|
|
281
|
+
parsed = JSON.parse(raw);
|
|
282
|
+
}
|
|
283
|
+
catch {
|
|
284
|
+
throw new StartupConfigError(variable, 'is not a JSON object of header values');
|
|
285
|
+
}
|
|
286
|
+
const valid = typeof parsed === 'object' &&
|
|
287
|
+
parsed !== null &&
|
|
288
|
+
!Array.isArray(parsed) &&
|
|
289
|
+
Object.values(parsed).every((value) => typeof value === 'string');
|
|
290
|
+
if (!valid)
|
|
291
|
+
throw new StartupConfigError(variable, 'is not a JSON object of header values');
|
|
292
|
+
return { ...parsed };
|
|
293
|
+
}
|
|
294
|
+
function createTelemetry(ports, options) {
|
|
295
|
+
const { env } = options;
|
|
296
|
+
const deploymentId = requiredEnv(env, ports.observability.deploymentIdEnv);
|
|
297
|
+
let stdio;
|
|
298
|
+
try {
|
|
299
|
+
stdio = createStdioSink({
|
|
300
|
+
deploymentId,
|
|
301
|
+
...(options.stdout === undefined ? {} : { stdout: options.stdout }),
|
|
302
|
+
...(options.stderr === undefined ? {} : { stderr: options.stderr }),
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
catch {
|
|
306
|
+
throw new StartupConfigError(ports.observability.deploymentIdEnv, 'is not a valid deployment id');
|
|
307
|
+
}
|
|
308
|
+
const routes = new Set();
|
|
309
|
+
let exporter;
|
|
310
|
+
let otlp;
|
|
311
|
+
const { endpointEnv, headersEnv } = ports.observability;
|
|
312
|
+
if (endpointEnv !== undefined) {
|
|
313
|
+
const endpoint = requiredEnv(env, endpointEnv);
|
|
314
|
+
const headers = otlpHeaders(env, headersEnv);
|
|
315
|
+
try {
|
|
316
|
+
exporter = createOtlpExporter({
|
|
317
|
+
endpoint,
|
|
318
|
+
headers,
|
|
319
|
+
routeTemplates: routes,
|
|
320
|
+
resource: { service: 'package-broker', deploymentId, runtime: 'node' },
|
|
321
|
+
metrics: { counter: (name, labels, value) => otlp?.internal.counter(name, labels, value) },
|
|
322
|
+
...(options.fetch === undefined ? {} : { fetchImpl: options.fetch }),
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
throw new StartupConfigError(endpointEnv, 'is not a valid OTLP endpoint or header set');
|
|
327
|
+
}
|
|
328
|
+
otlp = createOtlpSink(exporter);
|
|
329
|
+
}
|
|
330
|
+
const bridge = (record) => {
|
|
331
|
+
if (otlp === undefined || exporter === undefined)
|
|
332
|
+
return;
|
|
333
|
+
otlp.emit(record);
|
|
334
|
+
};
|
|
335
|
+
const outputs = filterLogLevel({
|
|
336
|
+
emit(record) {
|
|
337
|
+
stdio.emit(record);
|
|
338
|
+
try {
|
|
339
|
+
bridge(record);
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
},
|
|
345
|
+
}, options.observability?.level);
|
|
346
|
+
const sink = {
|
|
347
|
+
emit(record) {
|
|
348
|
+
outputs.emit(record);
|
|
349
|
+
if (record.kind !== 'log' || exporter === undefined)
|
|
350
|
+
return;
|
|
351
|
+
try {
|
|
352
|
+
const span = completionSpan(record.record, routes) ?? upstreamSpan(record.record);
|
|
353
|
+
if (span !== null)
|
|
354
|
+
exporter.export('traces', span);
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
},
|
|
360
|
+
};
|
|
361
|
+
return {
|
|
362
|
+
deploymentId,
|
|
363
|
+
routes,
|
|
364
|
+
sink,
|
|
365
|
+
observe: (scope) => createObservability(sink, {
|
|
366
|
+
deploymentId,
|
|
367
|
+
runtime: 'node',
|
|
368
|
+
requestId: scope?.requestId ?? null,
|
|
369
|
+
...(scope?.plugin === undefined ? {} : { plugin: scope.plugin }),
|
|
370
|
+
}, {
|
|
371
|
+
events: NODE_RUNTIME_EVENTS,
|
|
372
|
+
reasonCodes: new Set([...AUTH_REASON_CODES, ...INGEST_REASON_CODES]),
|
|
373
|
+
routeTemplates: routes,
|
|
374
|
+
}),
|
|
375
|
+
flush: () => otlp?.flush() ?? Promise.resolve(),
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
async function openByteStore(config, env, fetchImpl) {
|
|
379
|
+
if (config.adapter === 'filesystem') {
|
|
380
|
+
if (config.rootEnv === undefined)
|
|
381
|
+
throw new StartupConfigError('blob.rootEnv', 'is required');
|
|
382
|
+
const variable = config.rootEnv;
|
|
383
|
+
const root = requiredEnv(env, variable);
|
|
384
|
+
try {
|
|
385
|
+
return await openFilesystemByteStore({ root });
|
|
386
|
+
}
|
|
387
|
+
catch (error) {
|
|
388
|
+
const reason = error instanceof FilesystemRootError
|
|
389
|
+
? error.message
|
|
390
|
+
: error instanceof Error
|
|
391
|
+
? error.name
|
|
392
|
+
: 'failed';
|
|
393
|
+
throw new StartupConfigError(variable, `is not a usable blob root: ${reason}`);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
const value = (variable, field) => {
|
|
397
|
+
if (variable === undefined)
|
|
398
|
+
throw new StartupConfigError(`blob.${field}`, 'is required');
|
|
399
|
+
return requiredEnv(env, variable);
|
|
400
|
+
};
|
|
401
|
+
const endpoint = value(config.endpointEnv, 'endpointEnv');
|
|
402
|
+
const region = value(config.regionEnv, 'regionEnv');
|
|
403
|
+
const bucket = value(config.bucketEnv, 'bucketEnv');
|
|
404
|
+
const accessKeyId = value(config.accessKeyIdEnv, 'accessKeyIdEnv');
|
|
405
|
+
const secretAccessKey = value(config.secretAccessKeyEnv, 'secretAccessKeyEnv');
|
|
406
|
+
const sessionToken = config.sessionTokenEnv === undefined ? undefined : requiredEnv(env, config.sessionTokenEnv);
|
|
407
|
+
try {
|
|
408
|
+
return createS3ByteStore({
|
|
409
|
+
endpoint,
|
|
410
|
+
region,
|
|
411
|
+
bucket,
|
|
412
|
+
accessKeyId,
|
|
413
|
+
secretAccessKey,
|
|
414
|
+
...(sessionToken === undefined ? {} : { sessionToken }),
|
|
415
|
+
...(config.pathStyle === undefined ? {} : { pathStyle: config.pathStyle }),
|
|
416
|
+
...(fetchImpl === undefined ? {} : { fetch: fetchImpl }),
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
catch (error) {
|
|
420
|
+
if (!(error instanceof S3ConfigError))
|
|
421
|
+
throw error;
|
|
422
|
+
throw new StartupConfigError('blob', `S3 setting ${error.field} is invalid`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
async function buildServices(ports, options, executor) {
|
|
426
|
+
blobPurgeTiming(BLOB_DELETE_TIMEOUT_MS, BLOB_PURGE_LEASE_SECONDS);
|
|
427
|
+
const byteStore = await openByteStore(ports.blob, options.env, options.fetch);
|
|
428
|
+
let blob;
|
|
429
|
+
try {
|
|
430
|
+
blob = createBlobStore(byteStore, createBlobRepository(executor, { purgeLeaseSeconds: BLOB_PURGE_LEASE_SECONDS }), {
|
|
431
|
+
admission: createBlobUploadAdmission(),
|
|
432
|
+
maxBytes: ports.blob.maxBytes,
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
catch (error) {
|
|
436
|
+
if (!(error instanceof BlobValidationError))
|
|
437
|
+
throw error;
|
|
438
|
+
throw new StartupConfigError('blob.maxBytes', 'is not usable with the upload budget');
|
|
439
|
+
}
|
|
440
|
+
const cache = createLruCache(ports.cache.maxBytes === undefined ? {} : { maxBytes: ports.cache.maxBytes });
|
|
441
|
+
const services = {
|
|
442
|
+
blob,
|
|
443
|
+
cache,
|
|
444
|
+
jobStore: createJobRepository(executor),
|
|
445
|
+
retention: {
|
|
446
|
+
store: byteStore,
|
|
447
|
+
deleteTimeoutMs: BLOB_DELETE_TIMEOUT_MS,
|
|
448
|
+
leaseMs: BLOB_PURGE_LEASE_SECONDS * 1000,
|
|
449
|
+
},
|
|
450
|
+
};
|
|
451
|
+
return services;
|
|
452
|
+
}
|
|
453
|
+
function startJobs(kernel, ports, executor, telemetry) {
|
|
454
|
+
const registry = {
|
|
455
|
+
events: NODE_RUNTIME_EVENTS,
|
|
456
|
+
reasonCodes: INGEST_REASON_CODES,
|
|
457
|
+
routeTemplates: telemetry.routes,
|
|
458
|
+
};
|
|
459
|
+
const scope = {
|
|
460
|
+
deploymentId: telemetry.deploymentId,
|
|
461
|
+
runtime: 'node',
|
|
462
|
+
requestId: null,
|
|
463
|
+
};
|
|
464
|
+
const loop = createJobLoop({
|
|
465
|
+
executor,
|
|
466
|
+
jobs: kernel.registration.jobs,
|
|
467
|
+
scope: kernel.jobScope,
|
|
468
|
+
maxRunning: ports.jobs.maxRunning,
|
|
469
|
+
observability: createObservability(telemetry.sink, scope, registry),
|
|
470
|
+
runnerObservability: ({ jobId, attempt }) => createObservability(telemetry.sink, { ...scope, jobId, attempt }, registry),
|
|
471
|
+
});
|
|
472
|
+
loop.start();
|
|
473
|
+
return loop;
|
|
474
|
+
}
|
|
475
|
+
async function keyringGate(keyring, executor, options) {
|
|
476
|
+
const store = createKeyringVerifierStore(executor);
|
|
477
|
+
const reseed = options.env[RESEED_VARIABLE] === '1';
|
|
478
|
+
const attempt = async () => {
|
|
479
|
+
const result = await verifyKeyring(keyring, store, {
|
|
480
|
+
reseed,
|
|
481
|
+
onReseedIgnored: (keyVersion) => options.log?.({ level: 'warn', event: 'keyring.reseed_ignored', key_version: keyVersion }),
|
|
482
|
+
});
|
|
483
|
+
if (result.reseeded) {
|
|
484
|
+
options.log?.({ level: 'warn', event: 'keyring.reseeded', key_version: result.keyVersion });
|
|
485
|
+
}
|
|
486
|
+
};
|
|
487
|
+
try {
|
|
488
|
+
await attempt();
|
|
489
|
+
return undefined;
|
|
490
|
+
}
|
|
491
|
+
catch (error) {
|
|
492
|
+
const deferred = error instanceof KeyringStoreUnavailableError && options.env['PB_MIGRATE'] === 'off';
|
|
493
|
+
if (!deferred)
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
496
|
+
let verified = false;
|
|
497
|
+
let inFlight;
|
|
498
|
+
let nextAttempt = Date.now() + 1000;
|
|
499
|
+
const retry = async () => {
|
|
500
|
+
try {
|
|
501
|
+
await attempt();
|
|
502
|
+
verified = true;
|
|
503
|
+
}
|
|
504
|
+
catch (error) {
|
|
505
|
+
options.log?.({
|
|
506
|
+
level: 'error',
|
|
507
|
+
event: 'keyring.unverified',
|
|
508
|
+
error_name: error instanceof Error ? error.name : 'Error',
|
|
509
|
+
...(error instanceof KeyringMismatchError ||
|
|
510
|
+
error instanceof KeyringVerifierMissingError ||
|
|
511
|
+
error instanceof KeyringVersionInUseError
|
|
512
|
+
? { key_version: error.keyVersion }
|
|
513
|
+
: {}),
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
return verified;
|
|
517
|
+
};
|
|
518
|
+
return async () => {
|
|
519
|
+
if (verified)
|
|
520
|
+
return true;
|
|
521
|
+
if (inFlight !== undefined)
|
|
522
|
+
return inFlight;
|
|
523
|
+
if (Date.now() < nextAttempt)
|
|
524
|
+
return false;
|
|
525
|
+
nextAttempt = Date.now() + 1000;
|
|
526
|
+
inFlight = retry().finally(() => {
|
|
527
|
+
inFlight = undefined;
|
|
528
|
+
});
|
|
529
|
+
return inFlight;
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
export async function startNode(options) {
|
|
533
|
+
const rawTrustedProxy = options.env['PB_TRUSTED_PROXY'];
|
|
534
|
+
if (rawTrustedProxy !== undefined && rawTrustedProxy !== '0' && rawTrustedProxy !== '1') {
|
|
535
|
+
throw new StartupConfigError('PB_TRUSTED_PROXY', 'must be 0 or 1');
|
|
536
|
+
}
|
|
537
|
+
const trustedProxy = rawTrustedProxy === '1';
|
|
538
|
+
const log = options.log;
|
|
539
|
+
const path = options.env['PB_SQLITE_PATH'] ?? DEFAULT_DATABASE_PATH;
|
|
540
|
+
const ports = options.platformPorts ?? null;
|
|
541
|
+
let opened;
|
|
542
|
+
let broker;
|
|
543
|
+
let loop;
|
|
544
|
+
let flushTimer;
|
|
545
|
+
let telemetry;
|
|
546
|
+
const closeDatabase = async () => {
|
|
547
|
+
if (flushTimer !== undefined)
|
|
548
|
+
clearInterval(flushTimer);
|
|
549
|
+
flushTimer = undefined;
|
|
550
|
+
try {
|
|
551
|
+
const stopped = await loop?.stop();
|
|
552
|
+
if (telemetry !== undefined && stopped !== undefined && stopped.abandoned > 0) {
|
|
553
|
+
const observability = telemetry.observe();
|
|
554
|
+
observability.log({ level: 'warn', event: 'shutdown.jobs_abandoned' });
|
|
555
|
+
observability.counter(JOBS_ABANDONED_COUNTER, {}, stopped.abandoned);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
finally {
|
|
559
|
+
try {
|
|
560
|
+
await telemetry?.flush();
|
|
561
|
+
}
|
|
562
|
+
catch {
|
|
563
|
+
// A failed final flush must not keep the database open.
|
|
564
|
+
}
|
|
565
|
+
finally {
|
|
566
|
+
await opened?.close();
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
};
|
|
570
|
+
try {
|
|
571
|
+
assertMcpRateLimit({
|
|
572
|
+
runtime: 'node',
|
|
573
|
+
mcpEnabled: mcpEnabled({
|
|
574
|
+
platform: 'node',
|
|
575
|
+
plugins: options.plugins,
|
|
576
|
+
...(options.mcp === undefined ? {} : { mcp: options.mcp }),
|
|
577
|
+
}),
|
|
578
|
+
});
|
|
579
|
+
await mkdir(dirname(path), { recursive: true });
|
|
580
|
+
opened = openBetterSqliteExecutor(path);
|
|
581
|
+
const observabilityProblems = brokerObservabilityProblems({
|
|
582
|
+
platform: 'node',
|
|
583
|
+
plugins: options.plugins,
|
|
584
|
+
...(options.observability === undefined ? {} : { observability: options.observability }),
|
|
585
|
+
});
|
|
586
|
+
if (observabilityProblems.length > 0) {
|
|
587
|
+
throw new StartupConfigError('observability', observabilityProblems.join('; '));
|
|
588
|
+
}
|
|
589
|
+
let services;
|
|
590
|
+
if (ports !== null) {
|
|
591
|
+
const problems = validatePlatformPorts({ node: ports }, 'node');
|
|
592
|
+
if (problems.length > 0)
|
|
593
|
+
throw new StartupConfigError('platformPorts', problems.join('; '));
|
|
594
|
+
}
|
|
595
|
+
const keyring = await readKeyring(options.env);
|
|
596
|
+
const origin = publicOrigin(options.env['PB_PUBLIC_URL']);
|
|
597
|
+
if (options.registry !== undefined && ports === null) {
|
|
598
|
+
throw new StartupConfigError('registry', 'requires platformPorts');
|
|
599
|
+
}
|
|
600
|
+
const registry = registryConfig(options.registry, origin);
|
|
601
|
+
checkTokens(options.tokens);
|
|
602
|
+
await selfTestPasswordHashing(keyring);
|
|
603
|
+
if (ports !== null) {
|
|
604
|
+
telemetry = createTelemetry(ports, options);
|
|
605
|
+
services = await buildServices(ports, options, opened.executor);
|
|
606
|
+
}
|
|
607
|
+
if (options.env['PB_MIGRATE'] !== 'off') {
|
|
608
|
+
await migrate(opened.executor, options.migrations, {
|
|
609
|
+
holder: ulid(),
|
|
610
|
+
...(log === undefined ? {} : { log: migrateLog(log) }),
|
|
611
|
+
});
|
|
612
|
+
}
|
|
613
|
+
const servingGate = await keyringGate(keyring, opened.executor, options);
|
|
614
|
+
const slug = options.env['PB_ORG_SLUG'];
|
|
615
|
+
const name = options.env['PB_ORG_NAME'];
|
|
616
|
+
if (slug !== undefined && name !== undefined) {
|
|
617
|
+
if (servingGate !== undefined) {
|
|
618
|
+
throw new StartupConfigError('PB_ORG_SLUG', 'organization bootstrap requires applied migrations and a verified keyring');
|
|
619
|
+
}
|
|
620
|
+
await bootstrapOrganization(opened.executor, slug, name);
|
|
621
|
+
}
|
|
622
|
+
else if (slug !== undefined || name !== undefined) {
|
|
623
|
+
throw new Error('PB_ORG_SLUG and PB_ORG_NAME must be set together');
|
|
624
|
+
}
|
|
625
|
+
broker = createNodeServer(options.plugins, {
|
|
626
|
+
trustedProxy,
|
|
627
|
+
executor: opened.executor,
|
|
628
|
+
migrations: options.migrations,
|
|
629
|
+
closeDatabase,
|
|
630
|
+
...(options.tokens === undefined ? {} : { tokens: options.tokens }),
|
|
631
|
+
...(options.vulnerability === undefined ? {} : { vulnerability: options.vulnerability }),
|
|
632
|
+
...(options.mcp === undefined ? {} : { mcp: options.mcp }),
|
|
633
|
+
auth: { keyring, publicOrigin: origin, env: identityEnv(options.env) },
|
|
634
|
+
...(servingGate === undefined ? {} : { servingGate }),
|
|
635
|
+
...(telemetry !== undefined
|
|
636
|
+
? { log: sinkLog(telemetry.observe) }
|
|
637
|
+
: log === undefined
|
|
638
|
+
? {}
|
|
639
|
+
: { log }),
|
|
640
|
+
...(telemetry === undefined
|
|
641
|
+
? {}
|
|
642
|
+
: {
|
|
643
|
+
observability: {
|
|
644
|
+
sink: telemetry.sink,
|
|
645
|
+
level: options.observability?.level ?? 'info',
|
|
646
|
+
deploymentId: telemetry.deploymentId,
|
|
647
|
+
trustInboundTraceparent: options.observability?.trustInboundTraceparent === true,
|
|
648
|
+
},
|
|
649
|
+
}),
|
|
650
|
+
...(services === undefined ? {} : { platformServices: services }),
|
|
651
|
+
...(services === undefined
|
|
652
|
+
? {}
|
|
653
|
+
: {
|
|
654
|
+
registry: {
|
|
655
|
+
config: registry,
|
|
656
|
+
sshArchiveRoot: options.env['PB_SSH_ARCHIVE_ROOT'] ?? join(dirname(path), 'ssh-archives'),
|
|
657
|
+
gitArchiveRoot: options.env['PB_GIT_ARCHIVE_ROOT'] ?? join(dirname(path), 'git-archives'),
|
|
658
|
+
},
|
|
659
|
+
}),
|
|
660
|
+
});
|
|
661
|
+
const kernel = await broker.ready();
|
|
662
|
+
if (telemetry !== undefined) {
|
|
663
|
+
for (const template of kernel.routeTemplates)
|
|
664
|
+
telemetry.routes.add(template);
|
|
665
|
+
}
|
|
666
|
+
if (ports !== null && telemetry !== undefined) {
|
|
667
|
+
loop = startJobs(kernel, ports, opened.executor, telemetry);
|
|
668
|
+
const active = telemetry;
|
|
669
|
+
flushTimer = setInterval(() => {
|
|
670
|
+
active.flush().catch(() => undefined);
|
|
671
|
+
}, FLUSH_INTERVAL_MS);
|
|
672
|
+
flushTimer.unref();
|
|
673
|
+
}
|
|
674
|
+
return broker;
|
|
675
|
+
}
|
|
676
|
+
catch (error) {
|
|
677
|
+
telemetry?.observe().log({
|
|
678
|
+
level: 'error',
|
|
679
|
+
event: 'startup.failed',
|
|
680
|
+
error_name: safeErrorName(error),
|
|
681
|
+
});
|
|
682
|
+
const closing = broker === undefined ? closeDatabase() : broker.close();
|
|
683
|
+
await closing.catch(() => undefined);
|
|
684
|
+
throw error;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
//# sourceMappingURL=startup.js.map
|