@kici-dev/orchestrator 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/__test-helpers__/mock-db.d.ts +2 -0
- package/dist/agent/dispatcher.d.ts +110 -6
- package/dist/agent/registry.d.ts +14 -0
- package/dist/app.d.ts +11 -0
- package/dist/cache/agent-job-failed-error.d.ts +13 -0
- package/dist/cache/dispatch-cache-ref-tracker.d.ts +45 -0
- package/dist/cache/index.d.ts +2 -0
- package/dist/cache/user-cache.d.ts +116 -0
- package/dist/cancel/cancel-run.d.ts +56 -0
- package/dist/cli/commands/environment.d.ts +1 -0
- package/dist/cli.js +523 -111
- package/dist/cluster/peer-registry.d.ts +6 -0
- package/dist/config/schema.d.ts +4 -0
- package/dist/config.d.ts +13 -0
- package/dist/dashboard/handler.d.ts +92 -1
- package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts +11 -0
- package/dist/db/migrations/027_workflow_timeout.d.ts +20 -0
- package/dist/db/migrations/028_org_settings_user_cache.d.ts +4 -0
- package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +16 -0
- package/dist/db/migrations/030_held_runs_env_set_null.d.ts +13 -0
- package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +19 -0
- package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts +14 -0
- package/dist/db/types.d.ts +37 -2
- package/dist/environments/environment-store.d.ts +14 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +362 -40
- package/dist/lockfile-cache.d.ts +1 -1
- package/dist/metrics/prometheus.d.ts +8 -0
- package/dist/orchestrator-core.d.ts +4 -1
- package/dist/pipeline/dispatch-matched-workflow.d.ts +9 -0
- package/dist/pipeline/inline-eval.d.ts +17 -2
- package/dist/pipeline/process-webhook.d.ts +19 -0
- package/dist/pipeline/processor.d.ts +6 -1
- package/dist/pipeline/test-pipeline.d.ts +10 -0
- package/dist/providers/github/lock-file.d.ts +1 -1
- package/dist/providers/internal/lock-file-fetcher.d.ts +3 -2
- package/dist/queue/job-queue.d.ts +53 -1
- package/dist/reporting/execution-tracker.d.ts +3 -1
- package/dist/routes/admin-environments.d.ts +1 -0
- package/dist/scaler/bare-metal-backend.d.ts +1 -0
- package/dist/scaler/container-backend.d.ts +3 -2
- package/dist/scaler/firecracker-backend.d.ts +17 -0
- package/dist/scaler/manager.d.ts +2 -0
- package/dist/scaler/nftables.d.ts +25 -3
- package/dist/scaler/types.d.ts +26 -1
- package/dist/server.js +3981 -1421
- package/dist/stale-detector/workflow-deadline-detector.d.ts +49 -0
- package/dist/standalone.js +16819 -14768
- package/dist/storage/filesystem.d.ts +12 -3
- package/dist/storage/s3.d.ts +17 -4
- package/dist/storage/types.d.ts +25 -5
- package/dist/worker/in-memory-job-queue.d.ts +40 -7
- package/dist/ws/agent-handler.d.ts +13 -0
- package/dist/ws/dashboard-env-handler.d.ts +1 -0
- package/dist/ws/platform-client.d.ts +10 -1
- package/package.json +13 -10
- package/sbom.spdx.json +47 -47
package/dist/index.js
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
|
|
2
|
-
import { dirname as __cjs_dirname } from "node:path";
|
|
3
|
-
__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
|
|
4
1
|
import "node:module";
|
|
5
|
-
import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
2
|
+
import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
6
3
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
7
4
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
8
5
|
import { createLogger, createS3Client, sha256 } from "@kici-dev/shared";
|
|
9
6
|
import { promises } from "node:fs";
|
|
10
7
|
import { dirname, join } from "node:path";
|
|
11
|
-
import { createHmac, randomBytes } from "node:crypto";
|
|
8
|
+
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
12
9
|
import "node:fs/promises";
|
|
13
10
|
import { Kysely, PostgresDialect, sql } from "kysely";
|
|
14
11
|
import pg from "pg";
|
|
@@ -36,6 +33,8 @@ var S3CacheStorage = class {
|
|
|
36
33
|
client;
|
|
37
34
|
/** Separate client for pre-signed URL generation (uses externalEndpoint if configured). */
|
|
38
35
|
presignClient;
|
|
36
|
+
/** Separate client for the host CLI's pre-signed upload URL (uses uploadEndpoint if configured). */
|
|
37
|
+
uploadPresignClient;
|
|
39
38
|
bucket;
|
|
40
39
|
prefix;
|
|
41
40
|
ttlMs;
|
|
@@ -49,15 +48,21 @@ var S3CacheStorage = class {
|
|
|
49
48
|
endpoint: options.externalEndpoint
|
|
50
49
|
});
|
|
51
50
|
else this.presignClient = this.client;
|
|
51
|
+
if (options.uploadEndpoint) this.uploadPresignClient = createS3Client({
|
|
52
|
+
...options,
|
|
53
|
+
endpoint: options.uploadEndpoint
|
|
54
|
+
});
|
|
55
|
+
else this.uploadPresignClient = this.client;
|
|
52
56
|
}
|
|
53
57
|
/** Build the full S3 object key from a cache key. */
|
|
54
58
|
objectKey(key) {
|
|
55
59
|
return `${this.prefix}${key}`;
|
|
56
60
|
}
|
|
57
|
-
/** Check if metadata indicates an expired item. */
|
|
58
|
-
isExpired(meta) {
|
|
61
|
+
/** Check if metadata indicates an expired item, honoring a per-op TTL override. */
|
|
62
|
+
isExpired(meta, ttlMsOverride) {
|
|
63
|
+
const ttl = ttlMsOverride ?? this.ttlMs;
|
|
59
64
|
const lastAccessed = new Date(meta.lastAccessedAt).getTime();
|
|
60
|
-
return Date.now() - lastAccessed >
|
|
65
|
+
return Date.now() - lastAccessed > ttl;
|
|
61
66
|
}
|
|
62
67
|
/**
|
|
63
68
|
* Read metadata from an S3 object's custom headers.
|
|
@@ -121,10 +126,10 @@ var S3CacheStorage = class {
|
|
|
121
126
|
}
|
|
122
127
|
}).done();
|
|
123
128
|
}
|
|
124
|
-
async get(key) {
|
|
129
|
+
async get(key, ttlMsOverride) {
|
|
125
130
|
const meta = await this.readMeta(key);
|
|
126
131
|
if (!meta) return null;
|
|
127
|
-
if (this.isExpired(meta)) {
|
|
132
|
+
if (this.isExpired(meta, ttlMsOverride)) {
|
|
128
133
|
await this.deleteObject(key);
|
|
129
134
|
return null;
|
|
130
135
|
}
|
|
@@ -146,10 +151,10 @@ var S3CacheStorage = class {
|
|
|
146
151
|
} catch {}
|
|
147
152
|
return data;
|
|
148
153
|
}
|
|
149
|
-
async has(key) {
|
|
154
|
+
async has(key, ttlMsOverride) {
|
|
150
155
|
const meta = await this.readMeta(key);
|
|
151
156
|
if (!meta) return false;
|
|
152
|
-
if (this.isExpired(meta)) {
|
|
157
|
+
if (this.isExpired(meta, ttlMsOverride)) {
|
|
153
158
|
await this.deleteObject(key);
|
|
154
159
|
return false;
|
|
155
160
|
}
|
|
@@ -166,10 +171,10 @@ var S3CacheStorage = class {
|
|
|
166
171
|
meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
167
172
|
await this.updateMeta(key, meta);
|
|
168
173
|
}
|
|
169
|
-
async getUrl(key) {
|
|
174
|
+
async getUrl(key, ttlMsOverride) {
|
|
170
175
|
const meta = await this.readMeta(key);
|
|
171
176
|
if (!meta) return null;
|
|
172
|
-
if (this.isExpired(meta)) {
|
|
177
|
+
if (this.isExpired(meta, ttlMsOverride)) {
|
|
173
178
|
await this.deleteObject(key);
|
|
174
179
|
return null;
|
|
175
180
|
}
|
|
@@ -187,7 +192,7 @@ var S3CacheStorage = class {
|
|
|
187
192
|
}
|
|
188
193
|
async getInternalUploadUrl(key) {
|
|
189
194
|
const objectKey = this.objectKey(key);
|
|
190
|
-
return getSignedUrl(this.
|
|
195
|
+
return getSignedUrl(this.uploadPresignClient, new PutObjectCommand({
|
|
191
196
|
Bucket: this.bucket,
|
|
192
197
|
Key: objectKey
|
|
193
198
|
}), { expiresIn: UPLOAD_URL_EXPIRY_SECONDS });
|
|
@@ -206,6 +211,43 @@ var S3CacheStorage = class {
|
|
|
206
211
|
}
|
|
207
212
|
}));
|
|
208
213
|
}
|
|
214
|
+
async list(subPrefix) {
|
|
215
|
+
const fullPrefix = this.objectKey(subPrefix);
|
|
216
|
+
const items = [];
|
|
217
|
+
let token;
|
|
218
|
+
do {
|
|
219
|
+
const resp = await this.client.send(new ListObjectsV2Command({
|
|
220
|
+
Bucket: this.bucket,
|
|
221
|
+
Prefix: fullPrefix,
|
|
222
|
+
ContinuationToken: token
|
|
223
|
+
}));
|
|
224
|
+
for (const obj of resp.Contents ?? []) {
|
|
225
|
+
if (!obj.Key) continue;
|
|
226
|
+
items.push({
|
|
227
|
+
key: obj.Key.slice(this.prefix.length),
|
|
228
|
+
created: obj.LastModified ? obj.LastModified.getTime() : 0
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
token = resp.IsTruncated ? resp.NextContinuationToken : void 0;
|
|
232
|
+
} while (token);
|
|
233
|
+
items.sort((a, b) => b.created - a.created);
|
|
234
|
+
return items.map((i) => i.key);
|
|
235
|
+
}
|
|
236
|
+
async copy(srcKey, destKey) {
|
|
237
|
+
const srcObj = this.objectKey(srcKey);
|
|
238
|
+
const destObj = this.objectKey(destKey);
|
|
239
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
240
|
+
await this.client.send(new CopyObjectCommand({
|
|
241
|
+
Bucket: this.bucket,
|
|
242
|
+
Key: destObj,
|
|
243
|
+
CopySource: `${this.bucket}/${srcObj}`,
|
|
244
|
+
MetadataDirective: "REPLACE",
|
|
245
|
+
Metadata: {
|
|
246
|
+
"created-at": now,
|
|
247
|
+
"last-accessed-at": now
|
|
248
|
+
}
|
|
249
|
+
}));
|
|
250
|
+
}
|
|
209
251
|
/** Delete an S3 object. Idempotent (doesn't error if missing). */
|
|
210
252
|
async deleteObject(key) {
|
|
211
253
|
try {
|
|
@@ -298,9 +340,10 @@ var FilesystemCacheStorage = class {
|
|
|
298
340
|
metaPath(key) {
|
|
299
341
|
return `${this.resolvePath(key)}.meta.json`;
|
|
300
342
|
}
|
|
301
|
-
isExpired(meta) {
|
|
343
|
+
isExpired(meta, ttlMsOverride) {
|
|
344
|
+
const ttl = ttlMsOverride ?? this.ttlMs;
|
|
302
345
|
const lastAccessed = new Date(meta.lastAccessedAt).getTime();
|
|
303
|
-
return Date.now() - lastAccessed >
|
|
346
|
+
return Date.now() - lastAccessed > ttl;
|
|
304
347
|
}
|
|
305
348
|
async readMeta(key) {
|
|
306
349
|
try {
|
|
@@ -328,10 +371,10 @@ var FilesystemCacheStorage = class {
|
|
|
328
371
|
lastAccessedAt: now
|
|
329
372
|
});
|
|
330
373
|
}
|
|
331
|
-
async get(key) {
|
|
374
|
+
async get(key, ttlMsOverride) {
|
|
332
375
|
const meta = await this.readMeta(key);
|
|
333
376
|
if (!meta) return null;
|
|
334
|
-
if (this.isExpired(meta)) {
|
|
377
|
+
if (this.isExpired(meta, ttlMsOverride)) {
|
|
335
378
|
await this.deleteFiles(key);
|
|
336
379
|
return null;
|
|
337
380
|
}
|
|
@@ -348,10 +391,10 @@ var FilesystemCacheStorage = class {
|
|
|
348
391
|
} catch {}
|
|
349
392
|
return data;
|
|
350
393
|
}
|
|
351
|
-
async has(key) {
|
|
394
|
+
async has(key, ttlMsOverride) {
|
|
352
395
|
const meta = await this.readMeta(key);
|
|
353
396
|
if (!meta) return false;
|
|
354
|
-
if (this.isExpired(meta)) {
|
|
397
|
+
if (this.isExpired(meta, ttlMsOverride)) {
|
|
355
398
|
await this.deleteFiles(key);
|
|
356
399
|
return false;
|
|
357
400
|
}
|
|
@@ -368,10 +411,10 @@ var FilesystemCacheStorage = class {
|
|
|
368
411
|
meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
369
412
|
await this.writeMeta(key, meta);
|
|
370
413
|
}
|
|
371
|
-
async getUrl(key) {
|
|
414
|
+
async getUrl(key, ttlMsOverride) {
|
|
372
415
|
const meta = await this.readMeta(key);
|
|
373
416
|
if (!meta) return null;
|
|
374
|
-
if (this.isExpired(meta)) {
|
|
417
|
+
if (this.isExpired(meta, ttlMsOverride)) {
|
|
375
418
|
await this.deleteFiles(key);
|
|
376
419
|
return null;
|
|
377
420
|
}
|
|
@@ -390,6 +433,54 @@ var FilesystemCacheStorage = class {
|
|
|
390
433
|
lastAccessedAt: now
|
|
391
434
|
});
|
|
392
435
|
}
|
|
436
|
+
async list(subPrefix) {
|
|
437
|
+
const root = this.resolvePath(subPrefix.replace(/\/$/, ""));
|
|
438
|
+
let entries;
|
|
439
|
+
try {
|
|
440
|
+
entries = await this.walkDataFiles(root, subPrefix.replace(/\/$/, ""));
|
|
441
|
+
} catch (err) {
|
|
442
|
+
if (isNotFoundFsError(err)) return [];
|
|
443
|
+
throw err;
|
|
444
|
+
}
|
|
445
|
+
entries.sort((a, b) => b.mtime - a.mtime);
|
|
446
|
+
return entries.map((e) => e.key);
|
|
447
|
+
}
|
|
448
|
+
async copy(srcKey, destKey) {
|
|
449
|
+
const srcPath = this.resolvePath(srcKey);
|
|
450
|
+
const destPath = this.resolvePath(destKey);
|
|
451
|
+
await promises.mkdir(dirname(destPath), { recursive: true });
|
|
452
|
+
await promises.copyFile(srcPath, destPath);
|
|
453
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
454
|
+
await this.writeMeta(destKey, {
|
|
455
|
+
createdAt: now,
|
|
456
|
+
lastAccessedAt: now
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Recursively collect data files under `dir`, returning each as a cache key
|
|
461
|
+
* relative to `basePath` paired with its mtime. Metadata sidecars
|
|
462
|
+
* (`*.meta.json`) and the atomic-write temp files (`*.tmp-*`) are skipped so
|
|
463
|
+
* `list()` only surfaces real cache objects.
|
|
464
|
+
*/
|
|
465
|
+
async walkDataFiles(dir, keyPrefix) {
|
|
466
|
+
const out = [];
|
|
467
|
+
const dirents = await promises.readdir(dir, { withFileTypes: true });
|
|
468
|
+
for (const dirent of dirents) {
|
|
469
|
+
const childPath = join(dir, dirent.name);
|
|
470
|
+
const childKey = keyPrefix ? `${keyPrefix}/${dirent.name}` : dirent.name;
|
|
471
|
+
if (dirent.isDirectory()) {
|
|
472
|
+
out.push(...await this.walkDataFiles(childPath, childKey));
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
475
|
+
if (dirent.name.endsWith(".meta.json") || /\.tmp-[0-9a-f]+$/.test(dirent.name)) continue;
|
|
476
|
+
const stat = await promises.stat(childPath);
|
|
477
|
+
out.push({
|
|
478
|
+
key: childKey,
|
|
479
|
+
mtime: stat.mtimeMs
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
return out;
|
|
483
|
+
}
|
|
393
484
|
signedUrl(method, key) {
|
|
394
485
|
const { token } = signToken(this.signingSecret, method, key);
|
|
395
486
|
const encodedKey = key.split("/").map((seg) => encodeURIComponent(seg)).join("/");
|
|
@@ -432,6 +523,7 @@ function createCacheStorage(config) {
|
|
|
432
523
|
region: config.region,
|
|
433
524
|
endpoint: config.endpoint,
|
|
434
525
|
externalEndpoint: config.externalEndpoint,
|
|
526
|
+
uploadEndpoint: config.uploadEndpoint,
|
|
435
527
|
forcePathStyle: config.forcePathStyle
|
|
436
528
|
});
|
|
437
529
|
if (config.type === "filesystem") return new FilesystemCacheStorage({
|
|
@@ -460,7 +552,7 @@ function createCacheStorage(config) {
|
|
|
460
552
|
* downloads the source tarball, extracts it, and imports the workflow entry
|
|
461
553
|
* via the shared oxc-transform ESM loader hook.
|
|
462
554
|
*/
|
|
463
|
-
const logger$
|
|
555
|
+
const logger$2 = createLogger({ prefix: "source-cache" });
|
|
464
556
|
/** Cache key format: source/{contentHash}.tar.gz */
|
|
465
557
|
function sourceKey(contentHash) {
|
|
466
558
|
return `source/${contentHash}.tar.gz`;
|
|
@@ -473,7 +565,7 @@ var SourceCache = class {
|
|
|
473
565
|
async has(contentHash) {
|
|
474
566
|
const key = sourceKey(contentHash);
|
|
475
567
|
const exists = await this.storage.has(key);
|
|
476
|
-
logger$
|
|
568
|
+
logger$2.debug(`has(${contentHash}): ${exists}`);
|
|
477
569
|
return exists;
|
|
478
570
|
}
|
|
479
571
|
async get(contentHash) {
|
|
@@ -481,8 +573,8 @@ var SourceCache = class {
|
|
|
481
573
|
const data = await this.storage.get(key);
|
|
482
574
|
if (data) {
|
|
483
575
|
await this.storage.touch(key);
|
|
484
|
-
logger$
|
|
485
|
-
} else logger$
|
|
576
|
+
logger$2.debug(`get(${contentHash}): hit (${data.length} bytes)`);
|
|
577
|
+
} else logger$2.debug(`get(${contentHash}): miss`);
|
|
486
578
|
return data;
|
|
487
579
|
}
|
|
488
580
|
async getUrl(contentHash) {
|
|
@@ -490,8 +582,8 @@ var SourceCache = class {
|
|
|
490
582
|
const url = await this.storage.getUrl(key);
|
|
491
583
|
if (url) {
|
|
492
584
|
await this.storage.touch(key);
|
|
493
|
-
logger$
|
|
494
|
-
} else logger$
|
|
585
|
+
logger$2.debug(`getUrl(${contentHash}): hit`);
|
|
586
|
+
} else logger$2.debug(`getUrl(${contentHash}): miss`);
|
|
495
587
|
return url;
|
|
496
588
|
}
|
|
497
589
|
async getUploadUrl(contentHash) {
|
|
@@ -502,12 +594,12 @@ var SourceCache = class {
|
|
|
502
594
|
const key = sourceKey(contentHash);
|
|
503
595
|
await this.storage.put(key, tarball);
|
|
504
596
|
const size = typeof tarball === "string" ? Buffer.byteLength(tarball) : tarball.length;
|
|
505
|
-
logger$
|
|
597
|
+
logger$2.info(`store(${contentHash}): stored (${size} bytes)`);
|
|
506
598
|
}
|
|
507
599
|
async remove(contentHash) {
|
|
508
600
|
const key = sourceKey(contentHash);
|
|
509
601
|
const removed = await this.storage.delete(key);
|
|
510
|
-
logger$
|
|
602
|
+
logger$2.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`);
|
|
511
603
|
return removed;
|
|
512
604
|
}
|
|
513
605
|
};
|
|
@@ -522,7 +614,7 @@ var SourceCache = class {
|
|
|
522
614
|
*
|
|
523
615
|
* Cache key format: deps/{platform}-{arch}/{lockfileHash}.tar.gz
|
|
524
616
|
*/
|
|
525
|
-
const logger = createLogger({ prefix: "dep-cache" });
|
|
617
|
+
const logger$1 = createLogger({ prefix: "dep-cache" });
|
|
526
618
|
/** Default max tarball size: 500MB */
|
|
527
619
|
const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
|
|
528
620
|
/** Build cache key for dependency tarball: deps/{platform}-{arch}/{lockfileHash}.tar.gz */
|
|
@@ -540,7 +632,7 @@ var DepCache = class {
|
|
|
540
632
|
async has(lockfileHash, platform, arch) {
|
|
541
633
|
const key = depKey(lockfileHash, platform, arch);
|
|
542
634
|
const exists = await this.storage.has(key);
|
|
543
|
-
logger.debug(`has(${lockfileHash}): ${exists}`, {
|
|
635
|
+
logger$1.debug(`has(${lockfileHash}): ${exists}`, {
|
|
544
636
|
platform,
|
|
545
637
|
arch
|
|
546
638
|
});
|
|
@@ -555,11 +647,11 @@ var DepCache = class {
|
|
|
555
647
|
const url = await this.storage.getUrl(key);
|
|
556
648
|
if (url) {
|
|
557
649
|
await this.storage.touch(key);
|
|
558
|
-
logger.debug(`getUrl(${lockfileHash}): hit`, {
|
|
650
|
+
logger$1.debug(`getUrl(${lockfileHash}): hit`, {
|
|
559
651
|
platform,
|
|
560
652
|
arch
|
|
561
653
|
});
|
|
562
|
-
} else logger.debug(`getUrl(${lockfileHash}): miss`, {
|
|
654
|
+
} else logger$1.debug(`getUrl(${lockfileHash}): miss`, {
|
|
563
655
|
platform,
|
|
564
656
|
arch
|
|
565
657
|
});
|
|
@@ -574,7 +666,7 @@ var DepCache = class {
|
|
|
574
666
|
const key = depKey(lockfileHash, platform, arch);
|
|
575
667
|
const url = await this.storage.getUrl(key);
|
|
576
668
|
if (!url) {
|
|
577
|
-
logger.debug(`getUrlAndHash(${lockfileHash}): miss`, {
|
|
669
|
+
logger$1.debug(`getUrlAndHash(${lockfileHash}): miss`, {
|
|
578
670
|
platform,
|
|
579
671
|
arch
|
|
580
672
|
});
|
|
@@ -583,7 +675,7 @@ var DepCache = class {
|
|
|
583
675
|
await this.storage.touch(key);
|
|
584
676
|
const hashKey = `deps/${platform}-${arch}/${lockfileHash}.hash`;
|
|
585
677
|
const hash = (await this.storage.get(hashKey))?.toString("utf-8") || void 0;
|
|
586
|
-
logger.debug(`getUrlAndHash(${lockfileHash}): hit`, {
|
|
678
|
+
logger$1.debug(`getUrlAndHash(${lockfileHash}): hit`, {
|
|
587
679
|
platform,
|
|
588
680
|
arch,
|
|
589
681
|
hasHash: !!hash
|
|
@@ -608,7 +700,7 @@ var DepCache = class {
|
|
|
608
700
|
if (tarballData.length > this.maxTarballBytes) throw new Error(`Dep tarball exceeds max size: ${tarballData.length} bytes > ${this.maxTarballBytes} bytes limit`);
|
|
609
701
|
const key = depKey(lockfileHash, platform, arch);
|
|
610
702
|
await this.storage.put(key, tarballData);
|
|
611
|
-
logger.info(`store: ${tarballData.length} bytes`, {
|
|
703
|
+
logger$1.info(`store: ${tarballData.length} bytes`, {
|
|
612
704
|
lockfileHash,
|
|
613
705
|
platform,
|
|
614
706
|
arch
|
|
@@ -625,7 +717,7 @@ var DepCache = class {
|
|
|
625
717
|
async remove(lockfileHash, platform, arch) {
|
|
626
718
|
const key = depKey(lockfileHash, platform, arch);
|
|
627
719
|
const removed = await this.storage.delete(key);
|
|
628
|
-
logger.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
|
|
720
|
+
logger$1.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
|
|
629
721
|
platform,
|
|
630
722
|
arch
|
|
631
723
|
});
|
|
@@ -633,6 +725,236 @@ var DepCache = class {
|
|
|
633
725
|
}
|
|
634
726
|
};
|
|
635
727
|
//#endregion
|
|
728
|
+
//#region src/cache/user-cache.ts
|
|
729
|
+
/**
|
|
730
|
+
* User-facing cache layer wrapping CacheStorage.
|
|
731
|
+
*
|
|
732
|
+
* Namespacing: `cache/<orgId>/<repoId>/<refScope>/<key>.tar.gz`, where
|
|
733
|
+
* `refScope` is `shared` for trusted refs (org-shared, default-branch cache)
|
|
734
|
+
* or `iso/<runId>` for untrusted refs (per-run isolated scope). Restores from
|
|
735
|
+
* an untrusted ref read the shared scope as a fallback but writes can NEVER
|
|
736
|
+
* land in the shared scope — the GitHub Actions cache-isolation model. Keyed
|
|
737
|
+
* strictly per org so no tenant can read another tenant's cache.
|
|
738
|
+
*
|
|
739
|
+
* Saves are immutable (first save under an exact key wins) and atomic
|
|
740
|
+
* (upload to a `.tmp-<uuid>` key, then server-side copy to the final key and
|
|
741
|
+
* delete the temp), so a crashed save never leaves a corrupt final entry.
|
|
742
|
+
*
|
|
743
|
+
* Eviction: per-org byte quota plus the TTL the backing CacheStorage already
|
|
744
|
+
* enforces lazily on access. On a save that pushes the org over quota, oldest
|
|
745
|
+
* entries (by createdAt) are evicted until under quota; each eviction is
|
|
746
|
+
* logged. The companion `.hash` / `.size` sidecar objects carry the integrity
|
|
747
|
+
* hash and size accounting outside the tarball's own (presigned, metadata-less)
|
|
748
|
+
* upload.
|
|
749
|
+
*/
|
|
750
|
+
const logger = createLogger({ prefix: "user-cache" });
|
|
751
|
+
/**
|
|
752
|
+
* Cluster-wide default quota: 5 GiB. Serves as the fallback when an org has no
|
|
753
|
+
* per-org override in `org_settings.user_cache_quota_bytes`. The cluster-wide
|
|
754
|
+
* value is itself operator-configurable via KICI_USER_CACHE_QUOTA_BYTES.
|
|
755
|
+
*/
|
|
756
|
+
const DEFAULT_USER_CACHE_QUOTA_BYTES = 5 * 1024 * 1024 * 1024;
|
|
757
|
+
/**
|
|
758
|
+
* Cluster-wide default entry TTL: 7 days. Fallback when an org has no per-org
|
|
759
|
+
* override in `org_settings.user_cache_ttl_ms`. The cluster-wide value is
|
|
760
|
+
* operator-configurable via KICI_USER_CACHE_TTL_MS.
|
|
761
|
+
*/
|
|
762
|
+
const DEFAULT_USER_CACHE_TTL_MS = 10080 * 60 * 1e3;
|
|
763
|
+
/** Tarball suffix for committed cache entries. */
|
|
764
|
+
const TAR_SUFFIX = ".tar.gz";
|
|
765
|
+
var UserCache = class {
|
|
766
|
+
storage;
|
|
767
|
+
/** Cluster-wide default quota (the `KICI_USER_CACHE_QUOTA_BYTES` value). */
|
|
768
|
+
defaultQuotaBytes;
|
|
769
|
+
/** Cluster-wide default TTL (the `KICI_USER_CACHE_TTL_MS` value). */
|
|
770
|
+
defaultTtlMs;
|
|
771
|
+
/** Optional per-org override reader; absent = always use the cluster defaults. */
|
|
772
|
+
orgLimitsReader;
|
|
773
|
+
constructor(opts) {
|
|
774
|
+
this.storage = opts.storage;
|
|
775
|
+
this.defaultQuotaBytes = opts.quotaBytes ?? 5368709120;
|
|
776
|
+
this.defaultTtlMs = opts.ttlMs ?? 6048e5;
|
|
777
|
+
this.orgLimitsReader = opts.orgLimitsReader;
|
|
778
|
+
}
|
|
779
|
+
/**
|
|
780
|
+
* Resolve the effective quota + TTL for an org: the per-org override from
|
|
781
|
+
* `org_settings` when present, otherwise the cluster-wide default. A reader
|
|
782
|
+
* failure falls back to the defaults (logged) — the cache must never fail a
|
|
783
|
+
* restore/save because the settings lookup hiccupped.
|
|
784
|
+
*/
|
|
785
|
+
async resolveLimits(org) {
|
|
786
|
+
if (!this.orgLimitsReader) return {
|
|
787
|
+
quotaBytes: this.defaultQuotaBytes,
|
|
788
|
+
ttlMs: this.defaultTtlMs
|
|
789
|
+
};
|
|
790
|
+
let limits = {};
|
|
791
|
+
try {
|
|
792
|
+
limits = await this.orgLimitsReader(org);
|
|
793
|
+
} catch (err) {
|
|
794
|
+
logger.warn("user-cache org-limits lookup failed — using cluster defaults", {
|
|
795
|
+
org,
|
|
796
|
+
error: err instanceof Error ? err.message : String(err)
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
return {
|
|
800
|
+
quotaBytes: limits.quotaBytes ?? this.defaultQuotaBytes,
|
|
801
|
+
ttlMs: limits.ttlMs ?? this.defaultTtlMs
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* Sanitize a path segment so a key can never escape its org/repo/scope
|
|
806
|
+
* namespace. Beyond stripping disallowed characters, a segment consisting
|
|
807
|
+
* only of dots (`.`, `..`, …) is replaced wholesale: such a segment is a
|
|
808
|
+
* dot-segment that HTTP/S3 path canonicalization collapses (`a/./b` → `a/b`,
|
|
809
|
+
* `a/../b` → `b`), which both corrupts the namespace and breaks the SigV4
|
|
810
|
+
* signature on a pre-signed PUT/GET. Repo identifiers like `.` (the internal
|
|
811
|
+
* provider's repo id) hit exactly this case, so the all-dots guard keeps the
|
|
812
|
+
* object key canonical and the namespace boundary intact.
|
|
813
|
+
*/
|
|
814
|
+
seg(s) {
|
|
815
|
+
const cleaned = s.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
816
|
+
return /^\.+$/.test(cleaned) ? `_${cleaned}` : cleaned;
|
|
817
|
+
}
|
|
818
|
+
/** Org-level prefix: the per-tenant isolation boundary and quota scope. */
|
|
819
|
+
orgPrefix(ref) {
|
|
820
|
+
return `cache/${this.seg(ref.org)}/`;
|
|
821
|
+
}
|
|
822
|
+
/** Org + repo prefix shared by every scope of a repo. */
|
|
823
|
+
repoPrefix(ref) {
|
|
824
|
+
return `${this.orgPrefix(ref)}${this.seg(ref.repo)}`;
|
|
825
|
+
}
|
|
826
|
+
/** Namespace prefix for the WRITE scope of a ref (shared OR per-run isolated). */
|
|
827
|
+
writePrefix(ref) {
|
|
828
|
+
const base = this.repoPrefix(ref);
|
|
829
|
+
if (ref.scope === "isolated") {
|
|
830
|
+
if (!ref.runId) throw new Error("isolated cache scope requires a runId");
|
|
831
|
+
return `${base}/iso/${this.seg(ref.runId)}/`;
|
|
832
|
+
}
|
|
833
|
+
return `${base}/shared/`;
|
|
834
|
+
}
|
|
835
|
+
/** Namespace prefixes the ref may READ, in priority order. Isolated reads its own run scope, then shared. */
|
|
836
|
+
readPrefixes(ref) {
|
|
837
|
+
const base = this.repoPrefix(ref);
|
|
838
|
+
if (ref.scope === "isolated") {
|
|
839
|
+
if (!ref.runId) throw new Error("isolated cache scope requires a runId");
|
|
840
|
+
return [`${base}/iso/${this.seg(ref.runId)}/`, `${base}/shared/`];
|
|
841
|
+
}
|
|
842
|
+
return [`${base}/shared/`];
|
|
843
|
+
}
|
|
844
|
+
finalKey(prefix, key) {
|
|
845
|
+
return `${prefix}${this.seg(key)}${TAR_SUFFIX}`;
|
|
846
|
+
}
|
|
847
|
+
/** Restore: try the exact key across read prefixes, then restoreKeys prefix scan (newest wins). */
|
|
848
|
+
async restore(ref) {
|
|
849
|
+
const { ttlMs } = await this.resolveLimits(ref.org);
|
|
850
|
+
const prefixes = this.readPrefixes(ref);
|
|
851
|
+
const exact = await this.restoreExact(ref, prefixes, ttlMs);
|
|
852
|
+
if (exact) return exact;
|
|
853
|
+
return await this.restoreByPrefix(ref, prefixes, ttlMs) ?? { hit: false };
|
|
854
|
+
}
|
|
855
|
+
/** Try the exact key in read-prefix priority order. */
|
|
856
|
+
async restoreExact(ref, prefixes, ttlMs) {
|
|
857
|
+
for (const prefix of prefixes) {
|
|
858
|
+
const key = this.finalKey(prefix, ref.key);
|
|
859
|
+
const url = await this.storage.getUrl(key, ttlMs);
|
|
860
|
+
if (url) {
|
|
861
|
+
await this.storage.touch(key);
|
|
862
|
+
return {
|
|
863
|
+
hit: true,
|
|
864
|
+
matchedKey: ref.key,
|
|
865
|
+
downloadUrl: url,
|
|
866
|
+
tarHash: await this.readHash(key)
|
|
867
|
+
};
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
return null;
|
|
871
|
+
}
|
|
872
|
+
/** restoreKeys prefix fallback (ordered); within a prefix, list() returns newest-first. */
|
|
873
|
+
async restoreByPrefix(ref, prefixes, ttlMs) {
|
|
874
|
+
for (const rk of ref.restoreKeys ?? []) for (const prefix of prefixes) {
|
|
875
|
+
const matches = (await this.storage.list(`${prefix}${this.seg(rk)}`)).filter((k) => k.endsWith(TAR_SUFFIX));
|
|
876
|
+
if (matches.length === 0) continue;
|
|
877
|
+
const winner = matches[0];
|
|
878
|
+
const url = await this.storage.getUrl(winner, ttlMs);
|
|
879
|
+
if (!url) continue;
|
|
880
|
+
await this.storage.touch(winner);
|
|
881
|
+
return {
|
|
882
|
+
hit: true,
|
|
883
|
+
matchedKey: winner.slice(prefix.length, -7),
|
|
884
|
+
downloadUrl: url,
|
|
885
|
+
tarHash: await this.readHash(winner)
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
return null;
|
|
889
|
+
}
|
|
890
|
+
/** Begin a save: presigned PUT to a temp key, or skip=true when the immutable key exists. */
|
|
891
|
+
async beginSave(ref) {
|
|
892
|
+
const prefix = this.writePrefix(ref);
|
|
893
|
+
const final = this.finalKey(prefix, ref.key);
|
|
894
|
+
if (await this.storage.has(final)) {
|
|
895
|
+
logger.info("user-cache save skipped (immutable key exists)", { key: ref.key });
|
|
896
|
+
return { skip: true };
|
|
897
|
+
}
|
|
898
|
+
const tempKey = `${prefix}.tmp-${randomUUID()}${TAR_SUFFIX}`;
|
|
899
|
+
return {
|
|
900
|
+
skip: false,
|
|
901
|
+
uploadUrl: await this.storage.getUploadUrl(tempKey),
|
|
902
|
+
tempKey
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
/** Commit a save: copy temp -> final, init metadata, store companion hash/size, delete temp, enforce quota. */
|
|
906
|
+
async commitSave(ref) {
|
|
907
|
+
const prefix = this.writePrefix(ref);
|
|
908
|
+
const final = this.finalKey(prefix, ref.key);
|
|
909
|
+
if (await this.storage.has(final)) return;
|
|
910
|
+
if (ref.tempKey) {
|
|
911
|
+
await this.storage.copy(ref.tempKey, final);
|
|
912
|
+
await this.storage.delete(ref.tempKey);
|
|
913
|
+
}
|
|
914
|
+
await this.storage.initMeta(final);
|
|
915
|
+
await this.storage.put(`${final}.hash`, ref.tarHash);
|
|
916
|
+
await this.storage.put(`${final}.size`, String(ref.sizeBytes));
|
|
917
|
+
logger.info("user-cache entry committed", {
|
|
918
|
+
key: ref.key,
|
|
919
|
+
sizeBytes: ref.sizeBytes
|
|
920
|
+
});
|
|
921
|
+
await this.enforceQuota(ref);
|
|
922
|
+
}
|
|
923
|
+
async readHash(key) {
|
|
924
|
+
return (await this.storage.get(`${key}.hash`))?.toString("utf-8") || void 0;
|
|
925
|
+
}
|
|
926
|
+
/** Evict oldest entries for the org until total tarball size <= the per-org quota. */
|
|
927
|
+
async enforceQuota(ref) {
|
|
928
|
+
const { quotaBytes } = await this.resolveLimits(ref.org);
|
|
929
|
+
const orgPrefix = this.orgPrefix(ref);
|
|
930
|
+
const keys = (await this.storage.list(orgPrefix)).filter((k) => k.endsWith(TAR_SUFFIX));
|
|
931
|
+
const sized = [];
|
|
932
|
+
let total = 0;
|
|
933
|
+
for (const k of keys) {
|
|
934
|
+
const sizeData = await this.storage.get(`${k}.size`);
|
|
935
|
+
const size = sizeData ? Number(sizeData.toString("utf-8")) : 0;
|
|
936
|
+
total += size;
|
|
937
|
+
sized.push({
|
|
938
|
+
key: k,
|
|
939
|
+
size
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
if (total <= quotaBytes) return;
|
|
943
|
+
for (let i = sized.length - 1; i >= 0 && total > quotaBytes; i--) {
|
|
944
|
+
const { key, size } = sized[i];
|
|
945
|
+
await this.storage.delete(key);
|
|
946
|
+
await this.storage.delete(`${key}.hash`);
|
|
947
|
+
await this.storage.delete(`${key}.size`);
|
|
948
|
+
total -= size;
|
|
949
|
+
logger.info("user-cache eviction (org over quota)", {
|
|
950
|
+
org: ref.org,
|
|
951
|
+
key,
|
|
952
|
+
freedBytes: size
|
|
953
|
+
});
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
};
|
|
957
|
+
//#endregion
|
|
636
958
|
//#region src/cluster/peer-credentials.ts
|
|
637
959
|
/**
|
|
638
960
|
* Peer credential management for cluster authentication.
|
|
@@ -767,6 +1089,6 @@ async function createPeerCredentialStoreFromUrl(databaseUrl, opts) {
|
|
|
767
1089
|
};
|
|
768
1090
|
}
|
|
769
1091
|
//#endregion
|
|
770
|
-
export { DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, createCacheStorage, createPeerCredentialStoreFromUrl };
|
|
1092
|
+
export { DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, UserCache, createCacheStorage, createPeerCredentialStoreFromUrl };
|
|
771
1093
|
|
|
772
1094
|
//# sourceMappingURL=index.js.map
|
package/dist/lockfile-cache.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* Replaces the old GitHub-specific LockFileCache in github/lockfile.ts.
|
|
9
9
|
*/
|
|
10
|
-
import type
|
|
10
|
+
import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
|
|
11
11
|
export declare class LockFileCache {
|
|
12
12
|
private readonly cache;
|
|
13
13
|
private hits;
|
|
@@ -64,6 +64,14 @@ export declare const triggerMatchDurationSeconds: import("@opentelemetry/api").H
|
|
|
64
64
|
* Total deduplication cache hits (duplicate webhooks rejected).
|
|
65
65
|
*/
|
|
66
66
|
export declare const dedupHitsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
|
|
67
|
+
/**
|
|
68
|
+
* pg connection errors absorbed by the pool error handlers instead of
|
|
69
|
+
* crashing the process (e.g. a database failover terminating idle pooled
|
|
70
|
+
* backends).
|
|
71
|
+
* Labels:
|
|
72
|
+
* - source: idle-pool | client
|
|
73
|
+
*/
|
|
74
|
+
export declare const pgPoolClientErrorsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
|
|
67
75
|
/**
|
|
68
76
|
* Total source cache hits (`.kici/` source tarball found for content hash).
|
|
69
77
|
*/
|
|
@@ -26,7 +26,7 @@ import { type PeerHeartbeat } from '@kici-dev/engine';
|
|
|
26
26
|
import { ScalerManager } from './scaler/index.js';
|
|
27
27
|
import type { ScalerConfig } from './scaler/index.js';
|
|
28
28
|
import type { CacheStorage } from './storage/index.js';
|
|
29
|
-
import { SourceCache, BuildCoordinator, DepCache, PendingBuildTracker, PendingInitTracker, PendingDynamicTracker } from './cache/index.js';
|
|
29
|
+
import { SourceCache, BuildCoordinator, DepCache, UserCache, DispatchCacheRefTracker, PendingBuildTracker, PendingInitTracker, PendingDynamicTracker } from './cache/index.js';
|
|
30
30
|
import { CheckRunReporter } from './reporting/check-run-reporter.js';
|
|
31
31
|
import { StepLogBuffer } from './reporting/step-log-buffer.js';
|
|
32
32
|
import { type LogStorage } from './reporting/log-storage.js';
|
|
@@ -70,6 +70,9 @@ export interface OrchestratorSubsystems {
|
|
|
70
70
|
cacheStorage: CacheStorage | undefined;
|
|
71
71
|
sourceCache: SourceCache | undefined;
|
|
72
72
|
depCache: DepCache | undefined;
|
|
73
|
+
userCache: UserCache | undefined;
|
|
74
|
+
/** Server-side jobId -> user-cache-namespace store, written at dispatch, read by the agent-WS handler. */
|
|
75
|
+
dispatchCacheRefs: DispatchCacheRefTracker;
|
|
73
76
|
buildCoordinator: BuildCoordinator | undefined;
|
|
74
77
|
pendingBuilds: PendingBuildTracker | undefined;
|
|
75
78
|
pendingInits: PendingInitTracker;
|
|
@@ -13,11 +13,20 @@
|
|
|
13
13
|
* exported function is a narrative orchestrator that threads the typed
|
|
14
14
|
* results through the pipeline.
|
|
15
15
|
*/
|
|
16
|
+
import { CacheRefScope } from '@kici-dev/engine';
|
|
16
17
|
import type { LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
|
|
17
18
|
import type { WebhookInfo } from '../webhook/handler.js';
|
|
18
19
|
import type { ProviderBundle } from '../provider-registry.js';
|
|
19
20
|
import type { TrustResolution } from '../security/trust-resolver.js';
|
|
20
21
|
import { type ProcessingDeps } from './processor.js';
|
|
22
|
+
/**
|
|
23
|
+
* Trusted refs (write+ contributor, default-branch) get the org-shared cache
|
|
24
|
+
* write scope; everyone else (fork PR, unknown/known-but-not-trusted) is
|
|
25
|
+
* confined to a per-run isolated write scope. Absent trust resolution =>
|
|
26
|
+
* isolated (fail-closed), so an unresolved trust state can never poison the
|
|
27
|
+
* org-shared cache.
|
|
28
|
+
*/
|
|
29
|
+
export declare function deriveCacheRefScope(trust: TrustResolution | undefined): CacheRefScope;
|
|
21
30
|
/**
|
|
22
31
|
* Context for dispatching a single matched workflow.
|
|
23
32
|
*
|