@kici-dev/orchestrator 0.1.14 → 0.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +13 -1
  2. package/dist/__test-helpers__/mock-db.d.ts +2 -0
  3. package/dist/agent/dispatcher.d.ts +110 -6
  4. package/dist/agent/registry.d.ts +14 -0
  5. package/dist/app.d.ts +11 -0
  6. package/dist/cache/agent-job-failed-error.d.ts +13 -0
  7. package/dist/cache/dispatch-cache-ref-tracker.d.ts +45 -0
  8. package/dist/cache/index.d.ts +2 -0
  9. package/dist/cache/user-cache.d.ts +116 -0
  10. package/dist/cancel/cancel-run.d.ts +56 -0
  11. package/dist/cli/commands/environment.d.ts +1 -0
  12. package/dist/cli.js +523 -111
  13. package/dist/cluster/peer-registry.d.ts +6 -0
  14. package/dist/config/schema.d.ts +4 -0
  15. package/dist/config.d.ts +13 -0
  16. package/dist/dashboard/handler.d.ts +92 -1
  17. package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts +11 -0
  18. package/dist/db/migrations/027_workflow_timeout.d.ts +20 -0
  19. package/dist/db/migrations/028_org_settings_user_cache.d.ts +4 -0
  20. package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +16 -0
  21. package/dist/db/migrations/030_held_runs_env_set_null.d.ts +13 -0
  22. package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +19 -0
  23. package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts +14 -0
  24. package/dist/db/types.d.ts +37 -2
  25. package/dist/environments/environment-store.d.ts +14 -1
  26. package/dist/index.d.ts +1 -0
  27. package/dist/index.js +362 -40
  28. package/dist/lockfile-cache.d.ts +1 -1
  29. package/dist/metrics/prometheus.d.ts +8 -0
  30. package/dist/orchestrator-core.d.ts +4 -1
  31. package/dist/pipeline/dispatch-matched-workflow.d.ts +9 -0
  32. package/dist/pipeline/inline-eval.d.ts +17 -2
  33. package/dist/pipeline/process-webhook.d.ts +19 -0
  34. package/dist/pipeline/processor.d.ts +6 -1
  35. package/dist/pipeline/test-pipeline.d.ts +10 -0
  36. package/dist/providers/github/lock-file.d.ts +1 -1
  37. package/dist/providers/internal/lock-file-fetcher.d.ts +3 -2
  38. package/dist/queue/job-queue.d.ts +53 -1
  39. package/dist/reporting/execution-tracker.d.ts +3 -1
  40. package/dist/routes/admin-environments.d.ts +1 -0
  41. package/dist/scaler/bare-metal-backend.d.ts +1 -0
  42. package/dist/scaler/container-backend.d.ts +3 -2
  43. package/dist/scaler/firecracker-backend.d.ts +17 -0
  44. package/dist/scaler/manager.d.ts +2 -0
  45. package/dist/scaler/nftables.d.ts +25 -3
  46. package/dist/scaler/types.d.ts +26 -1
  47. package/dist/server.js +3981 -1421
  48. package/dist/stale-detector/workflow-deadline-detector.d.ts +49 -0
  49. package/dist/standalone.js +16819 -14768
  50. package/dist/storage/filesystem.d.ts +12 -3
  51. package/dist/storage/s3.d.ts +17 -4
  52. package/dist/storage/types.d.ts +25 -5
  53. package/dist/worker/in-memory-job-queue.d.ts +40 -7
  54. package/dist/ws/agent-handler.d.ts +13 -0
  55. package/dist/ws/dashboard-env-handler.d.ts +1 -0
  56. package/dist/ws/platform-client.d.ts +10 -1
  57. package/package.json +13 -10
  58. package/sbom.spdx.json +47 -47
package/dist/index.js CHANGED
@@ -1,14 +1,11 @@
1
- import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
2
- import { dirname as __cjs_dirname } from "node:path";
3
- __cjs_dirname(__cjs_fileURLToPath(import.meta.url));
4
1
  import "node:module";
5
- import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
2
+ import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
6
3
  import { Upload } from "@aws-sdk/lib-storage";
7
4
  import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
8
5
  import { createLogger, createS3Client, sha256 } from "@kici-dev/shared";
9
6
  import { promises } from "node:fs";
10
7
  import { dirname, join } from "node:path";
11
- import { createHmac, randomBytes } from "node:crypto";
8
+ import { createHmac, randomBytes, randomUUID } from "node:crypto";
12
9
  import "node:fs/promises";
13
10
  import { Kysely, PostgresDialect, sql } from "kysely";
14
11
  import pg from "pg";
@@ -36,6 +33,8 @@ var S3CacheStorage = class {
36
33
  client;
37
34
  /** Separate client for pre-signed URL generation (uses externalEndpoint if configured). */
38
35
  presignClient;
36
+ /** Separate client for the host CLI's pre-signed upload URL (uses uploadEndpoint if configured). */
37
+ uploadPresignClient;
39
38
  bucket;
40
39
  prefix;
41
40
  ttlMs;
@@ -49,15 +48,21 @@ var S3CacheStorage = class {
49
48
  endpoint: options.externalEndpoint
50
49
  });
51
50
  else this.presignClient = this.client;
51
+ if (options.uploadEndpoint) this.uploadPresignClient = createS3Client({
52
+ ...options,
53
+ endpoint: options.uploadEndpoint
54
+ });
55
+ else this.uploadPresignClient = this.client;
52
56
  }
53
57
  /** Build the full S3 object key from a cache key. */
54
58
  objectKey(key) {
55
59
  return `${this.prefix}${key}`;
56
60
  }
57
- /** Check if metadata indicates an expired item. */
58
- isExpired(meta) {
61
+ /** Check if metadata indicates an expired item, honoring a per-op TTL override. */
62
+ isExpired(meta, ttlMsOverride) {
63
+ const ttl = ttlMsOverride ?? this.ttlMs;
59
64
  const lastAccessed = new Date(meta.lastAccessedAt).getTime();
60
- return Date.now() - lastAccessed > this.ttlMs;
65
+ return Date.now() - lastAccessed > ttl;
61
66
  }
62
67
  /**
63
68
  * Read metadata from an S3 object's custom headers.
@@ -121,10 +126,10 @@ var S3CacheStorage = class {
121
126
  }
122
127
  }).done();
123
128
  }
124
- async get(key) {
129
+ async get(key, ttlMsOverride) {
125
130
  const meta = await this.readMeta(key);
126
131
  if (!meta) return null;
127
- if (this.isExpired(meta)) {
132
+ if (this.isExpired(meta, ttlMsOverride)) {
128
133
  await this.deleteObject(key);
129
134
  return null;
130
135
  }
@@ -146,10 +151,10 @@ var S3CacheStorage = class {
146
151
  } catch {}
147
152
  return data;
148
153
  }
149
- async has(key) {
154
+ async has(key, ttlMsOverride) {
150
155
  const meta = await this.readMeta(key);
151
156
  if (!meta) return false;
152
- if (this.isExpired(meta)) {
157
+ if (this.isExpired(meta, ttlMsOverride)) {
153
158
  await this.deleteObject(key);
154
159
  return false;
155
160
  }
@@ -166,10 +171,10 @@ var S3CacheStorage = class {
166
171
  meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
167
172
  await this.updateMeta(key, meta);
168
173
  }
169
- async getUrl(key) {
174
+ async getUrl(key, ttlMsOverride) {
170
175
  const meta = await this.readMeta(key);
171
176
  if (!meta) return null;
172
- if (this.isExpired(meta)) {
177
+ if (this.isExpired(meta, ttlMsOverride)) {
173
178
  await this.deleteObject(key);
174
179
  return null;
175
180
  }
@@ -187,7 +192,7 @@ var S3CacheStorage = class {
187
192
  }
188
193
  async getInternalUploadUrl(key) {
189
194
  const objectKey = this.objectKey(key);
190
- return getSignedUrl(this.client, new PutObjectCommand({
195
+ return getSignedUrl(this.uploadPresignClient, new PutObjectCommand({
191
196
  Bucket: this.bucket,
192
197
  Key: objectKey
193
198
  }), { expiresIn: UPLOAD_URL_EXPIRY_SECONDS });
@@ -206,6 +211,43 @@ var S3CacheStorage = class {
206
211
  }
207
212
  }));
208
213
  }
214
+ async list(subPrefix) {
215
+ const fullPrefix = this.objectKey(subPrefix);
216
+ const items = [];
217
+ let token;
218
+ do {
219
+ const resp = await this.client.send(new ListObjectsV2Command({
220
+ Bucket: this.bucket,
221
+ Prefix: fullPrefix,
222
+ ContinuationToken: token
223
+ }));
224
+ for (const obj of resp.Contents ?? []) {
225
+ if (!obj.Key) continue;
226
+ items.push({
227
+ key: obj.Key.slice(this.prefix.length),
228
+ created: obj.LastModified ? obj.LastModified.getTime() : 0
229
+ });
230
+ }
231
+ token = resp.IsTruncated ? resp.NextContinuationToken : void 0;
232
+ } while (token);
233
+ items.sort((a, b) => b.created - a.created);
234
+ return items.map((i) => i.key);
235
+ }
236
+ async copy(srcKey, destKey) {
237
+ const srcObj = this.objectKey(srcKey);
238
+ const destObj = this.objectKey(destKey);
239
+ const now = (/* @__PURE__ */ new Date()).toISOString();
240
+ await this.client.send(new CopyObjectCommand({
241
+ Bucket: this.bucket,
242
+ Key: destObj,
243
+ CopySource: `${this.bucket}/${srcObj}`,
244
+ MetadataDirective: "REPLACE",
245
+ Metadata: {
246
+ "created-at": now,
247
+ "last-accessed-at": now
248
+ }
249
+ }));
250
+ }
209
251
  /** Delete an S3 object. Idempotent (doesn't error if missing). */
210
252
  async deleteObject(key) {
211
253
  try {
@@ -298,9 +340,10 @@ var FilesystemCacheStorage = class {
298
340
  metaPath(key) {
299
341
  return `${this.resolvePath(key)}.meta.json`;
300
342
  }
301
- isExpired(meta) {
343
+ isExpired(meta, ttlMsOverride) {
344
+ const ttl = ttlMsOverride ?? this.ttlMs;
302
345
  const lastAccessed = new Date(meta.lastAccessedAt).getTime();
303
- return Date.now() - lastAccessed > this.ttlMs;
346
+ return Date.now() - lastAccessed > ttl;
304
347
  }
305
348
  async readMeta(key) {
306
349
  try {
@@ -328,10 +371,10 @@ var FilesystemCacheStorage = class {
328
371
  lastAccessedAt: now
329
372
  });
330
373
  }
331
- async get(key) {
374
+ async get(key, ttlMsOverride) {
332
375
  const meta = await this.readMeta(key);
333
376
  if (!meta) return null;
334
- if (this.isExpired(meta)) {
377
+ if (this.isExpired(meta, ttlMsOverride)) {
335
378
  await this.deleteFiles(key);
336
379
  return null;
337
380
  }
@@ -348,10 +391,10 @@ var FilesystemCacheStorage = class {
348
391
  } catch {}
349
392
  return data;
350
393
  }
351
- async has(key) {
394
+ async has(key, ttlMsOverride) {
352
395
  const meta = await this.readMeta(key);
353
396
  if (!meta) return false;
354
- if (this.isExpired(meta)) {
397
+ if (this.isExpired(meta, ttlMsOverride)) {
355
398
  await this.deleteFiles(key);
356
399
  return false;
357
400
  }
@@ -368,10 +411,10 @@ var FilesystemCacheStorage = class {
368
411
  meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
369
412
  await this.writeMeta(key, meta);
370
413
  }
371
- async getUrl(key) {
414
+ async getUrl(key, ttlMsOverride) {
372
415
  const meta = await this.readMeta(key);
373
416
  if (!meta) return null;
374
- if (this.isExpired(meta)) {
417
+ if (this.isExpired(meta, ttlMsOverride)) {
375
418
  await this.deleteFiles(key);
376
419
  return null;
377
420
  }
@@ -390,6 +433,54 @@ var FilesystemCacheStorage = class {
390
433
  lastAccessedAt: now
391
434
  });
392
435
  }
436
+ async list(subPrefix) {
437
+ const root = this.resolvePath(subPrefix.replace(/\/$/, ""));
438
+ let entries;
439
+ try {
440
+ entries = await this.walkDataFiles(root, subPrefix.replace(/\/$/, ""));
441
+ } catch (err) {
442
+ if (isNotFoundFsError(err)) return [];
443
+ throw err;
444
+ }
445
+ entries.sort((a, b) => b.mtime - a.mtime);
446
+ return entries.map((e) => e.key);
447
+ }
448
+ async copy(srcKey, destKey) {
449
+ const srcPath = this.resolvePath(srcKey);
450
+ const destPath = this.resolvePath(destKey);
451
+ await promises.mkdir(dirname(destPath), { recursive: true });
452
+ await promises.copyFile(srcPath, destPath);
453
+ const now = (/* @__PURE__ */ new Date()).toISOString();
454
+ await this.writeMeta(destKey, {
455
+ createdAt: now,
456
+ lastAccessedAt: now
457
+ });
458
+ }
459
+ /**
460
+ * Recursively collect data files under `dir`, returning each as a cache key
461
+ * relative to `basePath` paired with its mtime. Metadata sidecars
462
+ * (`*.meta.json`) and the atomic-write temp files (`*.tmp-*`) are skipped so
463
+ * `list()` only surfaces real cache objects.
464
+ */
465
+ async walkDataFiles(dir, keyPrefix) {
466
+ const out = [];
467
+ const dirents = await promises.readdir(dir, { withFileTypes: true });
468
+ for (const dirent of dirents) {
469
+ const childPath = join(dir, dirent.name);
470
+ const childKey = keyPrefix ? `${keyPrefix}/${dirent.name}` : dirent.name;
471
+ if (dirent.isDirectory()) {
472
+ out.push(...await this.walkDataFiles(childPath, childKey));
473
+ continue;
474
+ }
475
+ if (dirent.name.endsWith(".meta.json") || /\.tmp-[0-9a-f]+$/.test(dirent.name)) continue;
476
+ const stat = await promises.stat(childPath);
477
+ out.push({
478
+ key: childKey,
479
+ mtime: stat.mtimeMs
480
+ });
481
+ }
482
+ return out;
483
+ }
393
484
  signedUrl(method, key) {
394
485
  const { token } = signToken(this.signingSecret, method, key);
395
486
  const encodedKey = key.split("/").map((seg) => encodeURIComponent(seg)).join("/");
@@ -432,6 +523,7 @@ function createCacheStorage(config) {
432
523
  region: config.region,
433
524
  endpoint: config.endpoint,
434
525
  externalEndpoint: config.externalEndpoint,
526
+ uploadEndpoint: config.uploadEndpoint,
435
527
  forcePathStyle: config.forcePathStyle
436
528
  });
437
529
  if (config.type === "filesystem") return new FilesystemCacheStorage({
@@ -460,7 +552,7 @@ function createCacheStorage(config) {
460
552
  * downloads the source tarball, extracts it, and imports the workflow entry
461
553
  * via the shared oxc-transform ESM loader hook.
462
554
  */
463
- const logger$1 = createLogger({ prefix: "source-cache" });
555
+ const logger$2 = createLogger({ prefix: "source-cache" });
464
556
  /** Cache key format: source/{contentHash}.tar.gz */
465
557
  function sourceKey(contentHash) {
466
558
  return `source/${contentHash}.tar.gz`;
@@ -473,7 +565,7 @@ var SourceCache = class {
473
565
  async has(contentHash) {
474
566
  const key = sourceKey(contentHash);
475
567
  const exists = await this.storage.has(key);
476
- logger$1.debug(`has(${contentHash}): ${exists}`);
568
+ logger$2.debug(`has(${contentHash}): ${exists}`);
477
569
  return exists;
478
570
  }
479
571
  async get(contentHash) {
@@ -481,8 +573,8 @@ var SourceCache = class {
481
573
  const data = await this.storage.get(key);
482
574
  if (data) {
483
575
  await this.storage.touch(key);
484
- logger$1.debug(`get(${contentHash}): hit (${data.length} bytes)`);
485
- } else logger$1.debug(`get(${contentHash}): miss`);
576
+ logger$2.debug(`get(${contentHash}): hit (${data.length} bytes)`);
577
+ } else logger$2.debug(`get(${contentHash}): miss`);
486
578
  return data;
487
579
  }
488
580
  async getUrl(contentHash) {
@@ -490,8 +582,8 @@ var SourceCache = class {
490
582
  const url = await this.storage.getUrl(key);
491
583
  if (url) {
492
584
  await this.storage.touch(key);
493
- logger$1.debug(`getUrl(${contentHash}): hit`);
494
- } else logger$1.debug(`getUrl(${contentHash}): miss`);
585
+ logger$2.debug(`getUrl(${contentHash}): hit`);
586
+ } else logger$2.debug(`getUrl(${contentHash}): miss`);
495
587
  return url;
496
588
  }
497
589
  async getUploadUrl(contentHash) {
@@ -502,12 +594,12 @@ var SourceCache = class {
502
594
  const key = sourceKey(contentHash);
503
595
  await this.storage.put(key, tarball);
504
596
  const size = typeof tarball === "string" ? Buffer.byteLength(tarball) : tarball.length;
505
- logger$1.info(`store(${contentHash}): stored (${size} bytes)`);
597
+ logger$2.info(`store(${contentHash}): stored (${size} bytes)`);
506
598
  }
507
599
  async remove(contentHash) {
508
600
  const key = sourceKey(contentHash);
509
601
  const removed = await this.storage.delete(key);
510
- logger$1.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`);
602
+ logger$2.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`);
511
603
  return removed;
512
604
  }
513
605
  };
@@ -522,7 +614,7 @@ var SourceCache = class {
522
614
  *
523
615
  * Cache key format: deps/{platform}-{arch}/{lockfileHash}.tar.gz
524
616
  */
525
- const logger = createLogger({ prefix: "dep-cache" });
617
+ const logger$1 = createLogger({ prefix: "dep-cache" });
526
618
  /** Default max tarball size: 500MB */
527
619
  const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
528
620
  /** Build cache key for dependency tarball: deps/{platform}-{arch}/{lockfileHash}.tar.gz */
@@ -540,7 +632,7 @@ var DepCache = class {
540
632
  async has(lockfileHash, platform, arch) {
541
633
  const key = depKey(lockfileHash, platform, arch);
542
634
  const exists = await this.storage.has(key);
543
- logger.debug(`has(${lockfileHash}): ${exists}`, {
635
+ logger$1.debug(`has(${lockfileHash}): ${exists}`, {
544
636
  platform,
545
637
  arch
546
638
  });
@@ -555,11 +647,11 @@ var DepCache = class {
555
647
  const url = await this.storage.getUrl(key);
556
648
  if (url) {
557
649
  await this.storage.touch(key);
558
- logger.debug(`getUrl(${lockfileHash}): hit`, {
650
+ logger$1.debug(`getUrl(${lockfileHash}): hit`, {
559
651
  platform,
560
652
  arch
561
653
  });
562
- } else logger.debug(`getUrl(${lockfileHash}): miss`, {
654
+ } else logger$1.debug(`getUrl(${lockfileHash}): miss`, {
563
655
  platform,
564
656
  arch
565
657
  });
@@ -574,7 +666,7 @@ var DepCache = class {
574
666
  const key = depKey(lockfileHash, platform, arch);
575
667
  const url = await this.storage.getUrl(key);
576
668
  if (!url) {
577
- logger.debug(`getUrlAndHash(${lockfileHash}): miss`, {
669
+ logger$1.debug(`getUrlAndHash(${lockfileHash}): miss`, {
578
670
  platform,
579
671
  arch
580
672
  });
@@ -583,7 +675,7 @@ var DepCache = class {
583
675
  await this.storage.touch(key);
584
676
  const hashKey = `deps/${platform}-${arch}/${lockfileHash}.hash`;
585
677
  const hash = (await this.storage.get(hashKey))?.toString("utf-8") || void 0;
586
- logger.debug(`getUrlAndHash(${lockfileHash}): hit`, {
678
+ logger$1.debug(`getUrlAndHash(${lockfileHash}): hit`, {
587
679
  platform,
588
680
  arch,
589
681
  hasHash: !!hash
@@ -608,7 +700,7 @@ var DepCache = class {
608
700
  if (tarballData.length > this.maxTarballBytes) throw new Error(`Dep tarball exceeds max size: ${tarballData.length} bytes > ${this.maxTarballBytes} bytes limit`);
609
701
  const key = depKey(lockfileHash, platform, arch);
610
702
  await this.storage.put(key, tarballData);
611
- logger.info(`store: ${tarballData.length} bytes`, {
703
+ logger$1.info(`store: ${tarballData.length} bytes`, {
612
704
  lockfileHash,
613
705
  platform,
614
706
  arch
@@ -625,7 +717,7 @@ var DepCache = class {
625
717
  async remove(lockfileHash, platform, arch) {
626
718
  const key = depKey(lockfileHash, platform, arch);
627
719
  const removed = await this.storage.delete(key);
628
- logger.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
720
+ logger$1.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
629
721
  platform,
630
722
  arch
631
723
  });
@@ -633,6 +725,236 @@ var DepCache = class {
633
725
  }
634
726
  };
635
727
  //#endregion
728
+ //#region src/cache/user-cache.ts
729
+ /**
730
+ * User-facing cache layer wrapping CacheStorage.
731
+ *
732
+ * Namespacing: `cache/<orgId>/<repoId>/<refScope>/<key>.tar.gz`, where
733
+ * `refScope` is `shared` for trusted refs (org-shared, default-branch cache)
734
+ * or `iso/<runId>` for untrusted refs (per-run isolated scope). Restores from
735
+ * an untrusted ref read the shared scope as a fallback but writes can NEVER
736
+ * land in the shared scope — the GitHub Actions cache-isolation model. Keyed
737
+ * strictly per org so no tenant can read another tenant's cache.
738
+ *
739
+ * Saves are immutable (first save under an exact key wins) and atomic
740
+ * (upload to a `.tmp-<uuid>` key, then server-side copy to the final key and
741
+ * delete the temp), so a crashed save never leaves a corrupt final entry.
742
+ *
743
+ * Eviction: per-org byte quota plus the TTL the backing CacheStorage already
744
+ * enforces lazily on access. On a save that pushes the org over quota, oldest
745
+ * entries (by createdAt) are evicted until under quota; each eviction is
746
+ * logged. The companion `.hash` / `.size` sidecar objects carry the integrity
747
+ * hash and size accounting outside the tarball's own (presigned, metadata-less)
748
+ * upload.
749
+ */
750
+ const logger = createLogger({ prefix: "user-cache" });
751
+ /**
752
+ * Cluster-wide default quota: 5 GiB. Serves as the fallback when an org has no
753
+ * per-org override in `org_settings.user_cache_quota_bytes`. The cluster-wide
754
+ * value is itself operator-configurable via KICI_USER_CACHE_QUOTA_BYTES.
755
+ */
756
+ const DEFAULT_USER_CACHE_QUOTA_BYTES = 5 * 1024 * 1024 * 1024;
757
+ /**
758
+ * Cluster-wide default entry TTL: 7 days. Fallback when an org has no per-org
759
+ * override in `org_settings.user_cache_ttl_ms`. The cluster-wide value is
760
+ * operator-configurable via KICI_USER_CACHE_TTL_MS.
761
+ */
762
+ const DEFAULT_USER_CACHE_TTL_MS = 10080 * 60 * 1e3;
763
+ /** Tarball suffix for committed cache entries. */
764
+ const TAR_SUFFIX = ".tar.gz";
765
+ var UserCache = class {
766
+ storage;
767
+ /** Cluster-wide default quota (the `KICI_USER_CACHE_QUOTA_BYTES` value). */
768
+ defaultQuotaBytes;
769
+ /** Cluster-wide default TTL (the `KICI_USER_CACHE_TTL_MS` value). */
770
+ defaultTtlMs;
771
+ /** Optional per-org override reader; absent = always use the cluster defaults. */
772
+ orgLimitsReader;
773
+ constructor(opts) {
774
+ this.storage = opts.storage;
775
+ this.defaultQuotaBytes = opts.quotaBytes ?? 5368709120;
776
+ this.defaultTtlMs = opts.ttlMs ?? 6048e5;
777
+ this.orgLimitsReader = opts.orgLimitsReader;
778
+ }
779
+ /**
780
+ * Resolve the effective quota + TTL for an org: the per-org override from
781
+ * `org_settings` when present, otherwise the cluster-wide default. A reader
782
+ * failure falls back to the defaults (logged) — the cache must never fail a
783
+ * restore/save because the settings lookup hiccupped.
784
+ */
785
+ async resolveLimits(org) {
786
+ if (!this.orgLimitsReader) return {
787
+ quotaBytes: this.defaultQuotaBytes,
788
+ ttlMs: this.defaultTtlMs
789
+ };
790
+ let limits = {};
791
+ try {
792
+ limits = await this.orgLimitsReader(org);
793
+ } catch (err) {
794
+ logger.warn("user-cache org-limits lookup failed — using cluster defaults", {
795
+ org,
796
+ error: err instanceof Error ? err.message : String(err)
797
+ });
798
+ }
799
+ return {
800
+ quotaBytes: limits.quotaBytes ?? this.defaultQuotaBytes,
801
+ ttlMs: limits.ttlMs ?? this.defaultTtlMs
802
+ };
803
+ }
804
+ /**
805
+ * Sanitize a path segment so a key can never escape its org/repo/scope
806
+ * namespace. Beyond stripping disallowed characters, a segment consisting
807
+ * only of dots (`.`, `..`, …) is replaced wholesale: such a segment is a
808
+ * dot-segment that HTTP/S3 path canonicalization collapses (`a/./b` → `a/b`,
809
+ * `a/../b` → `b`), which both corrupts the namespace and breaks the SigV4
810
+ * signature on a pre-signed PUT/GET. Repo identifiers like `.` (the internal
811
+ * provider's repo id) hit exactly this case, so the all-dots guard keeps the
812
+ * object key canonical and the namespace boundary intact.
813
+ */
814
+ seg(s) {
815
+ const cleaned = s.replace(/[^A-Za-z0-9._-]/g, "_");
816
+ return /^\.+$/.test(cleaned) ? `_${cleaned}` : cleaned;
817
+ }
818
+ /** Org-level prefix: the per-tenant isolation boundary and quota scope. */
819
+ orgPrefix(ref) {
820
+ return `cache/${this.seg(ref.org)}/`;
821
+ }
822
+ /** Org + repo prefix shared by every scope of a repo. */
823
+ repoPrefix(ref) {
824
+ return `${this.orgPrefix(ref)}${this.seg(ref.repo)}`;
825
+ }
826
+ /** Namespace prefix for the WRITE scope of a ref (shared OR per-run isolated). */
827
+ writePrefix(ref) {
828
+ const base = this.repoPrefix(ref);
829
+ if (ref.scope === "isolated") {
830
+ if (!ref.runId) throw new Error("isolated cache scope requires a runId");
831
+ return `${base}/iso/${this.seg(ref.runId)}/`;
832
+ }
833
+ return `${base}/shared/`;
834
+ }
835
+ /** Namespace prefixes the ref may READ, in priority order. Isolated reads its own run scope, then shared. */
836
+ readPrefixes(ref) {
837
+ const base = this.repoPrefix(ref);
838
+ if (ref.scope === "isolated") {
839
+ if (!ref.runId) throw new Error("isolated cache scope requires a runId");
840
+ return [`${base}/iso/${this.seg(ref.runId)}/`, `${base}/shared/`];
841
+ }
842
+ return [`${base}/shared/`];
843
+ }
844
+ finalKey(prefix, key) {
845
+ return `${prefix}${this.seg(key)}${TAR_SUFFIX}`;
846
+ }
847
+ /** Restore: try the exact key across read prefixes, then restoreKeys prefix scan (newest wins). */
848
+ async restore(ref) {
849
+ const { ttlMs } = await this.resolveLimits(ref.org);
850
+ const prefixes = this.readPrefixes(ref);
851
+ const exact = await this.restoreExact(ref, prefixes, ttlMs);
852
+ if (exact) return exact;
853
+ return await this.restoreByPrefix(ref, prefixes, ttlMs) ?? { hit: false };
854
+ }
855
+ /** Try the exact key in read-prefix priority order. */
856
+ async restoreExact(ref, prefixes, ttlMs) {
857
+ for (const prefix of prefixes) {
858
+ const key = this.finalKey(prefix, ref.key);
859
+ const url = await this.storage.getUrl(key, ttlMs);
860
+ if (url) {
861
+ await this.storage.touch(key);
862
+ return {
863
+ hit: true,
864
+ matchedKey: ref.key,
865
+ downloadUrl: url,
866
+ tarHash: await this.readHash(key)
867
+ };
868
+ }
869
+ }
870
+ return null;
871
+ }
872
+ /** restoreKeys prefix fallback (ordered); within a prefix, list() returns newest-first. */
873
+ async restoreByPrefix(ref, prefixes, ttlMs) {
874
+ for (const rk of ref.restoreKeys ?? []) for (const prefix of prefixes) {
875
+ const matches = (await this.storage.list(`${prefix}${this.seg(rk)}`)).filter((k) => k.endsWith(TAR_SUFFIX));
876
+ if (matches.length === 0) continue;
877
+ const winner = matches[0];
878
+ const url = await this.storage.getUrl(winner, ttlMs);
879
+ if (!url) continue;
880
+ await this.storage.touch(winner);
881
+ return {
882
+ hit: true,
883
+ matchedKey: winner.slice(prefix.length, -7),
884
+ downloadUrl: url,
885
+ tarHash: await this.readHash(winner)
886
+ };
887
+ }
888
+ return null;
889
+ }
890
+ /** Begin a save: presigned PUT to a temp key, or skip=true when the immutable key exists. */
891
+ async beginSave(ref) {
892
+ const prefix = this.writePrefix(ref);
893
+ const final = this.finalKey(prefix, ref.key);
894
+ if (await this.storage.has(final)) {
895
+ logger.info("user-cache save skipped (immutable key exists)", { key: ref.key });
896
+ return { skip: true };
897
+ }
898
+ const tempKey = `${prefix}.tmp-${randomUUID()}${TAR_SUFFIX}`;
899
+ return {
900
+ skip: false,
901
+ uploadUrl: await this.storage.getUploadUrl(tempKey),
902
+ tempKey
903
+ };
904
+ }
905
+ /** Commit a save: copy temp -> final, init metadata, store companion hash/size, delete temp, enforce quota. */
906
+ async commitSave(ref) {
907
+ const prefix = this.writePrefix(ref);
908
+ const final = this.finalKey(prefix, ref.key);
909
+ if (await this.storage.has(final)) return;
910
+ if (ref.tempKey) {
911
+ await this.storage.copy(ref.tempKey, final);
912
+ await this.storage.delete(ref.tempKey);
913
+ }
914
+ await this.storage.initMeta(final);
915
+ await this.storage.put(`${final}.hash`, ref.tarHash);
916
+ await this.storage.put(`${final}.size`, String(ref.sizeBytes));
917
+ logger.info("user-cache entry committed", {
918
+ key: ref.key,
919
+ sizeBytes: ref.sizeBytes
920
+ });
921
+ await this.enforceQuota(ref);
922
+ }
923
+ async readHash(key) {
924
+ return (await this.storage.get(`${key}.hash`))?.toString("utf-8") || void 0;
925
+ }
926
+ /** Evict oldest entries for the org until total tarball size <= the per-org quota. */
927
+ async enforceQuota(ref) {
928
+ const { quotaBytes } = await this.resolveLimits(ref.org);
929
+ const orgPrefix = this.orgPrefix(ref);
930
+ const keys = (await this.storage.list(orgPrefix)).filter((k) => k.endsWith(TAR_SUFFIX));
931
+ const sized = [];
932
+ let total = 0;
933
+ for (const k of keys) {
934
+ const sizeData = await this.storage.get(`${k}.size`);
935
+ const size = sizeData ? Number(sizeData.toString("utf-8")) : 0;
936
+ total += size;
937
+ sized.push({
938
+ key: k,
939
+ size
940
+ });
941
+ }
942
+ if (total <= quotaBytes) return;
943
+ for (let i = sized.length - 1; i >= 0 && total > quotaBytes; i--) {
944
+ const { key, size } = sized[i];
945
+ await this.storage.delete(key);
946
+ await this.storage.delete(`${key}.hash`);
947
+ await this.storage.delete(`${key}.size`);
948
+ total -= size;
949
+ logger.info("user-cache eviction (org over quota)", {
950
+ org: ref.org,
951
+ key,
952
+ freedBytes: size
953
+ });
954
+ }
955
+ }
956
+ };
957
+ //#endregion
636
958
  //#region src/cluster/peer-credentials.ts
637
959
  /**
638
960
  * Peer credential management for cluster authentication.
@@ -767,6 +1089,6 @@ async function createPeerCredentialStoreFromUrl(databaseUrl, opts) {
767
1089
  };
768
1090
  }
769
1091
  //#endregion
770
- export { DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, createCacheStorage, createPeerCredentialStoreFromUrl };
1092
+ export { DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, UserCache, createCacheStorage, createPeerCredentialStoreFromUrl };
771
1093
 
772
1094
  //# sourceMappingURL=index.js.map
@@ -7,7 +7,7 @@
7
7
  *
8
8
  * Replaces the old GitHub-specific LockFileCache in github/lockfile.ts.
9
9
  */
10
- import type { LockFileFetcher, LockFile } from '@kici-dev/engine';
10
+ import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
11
11
  export declare class LockFileCache {
12
12
  private readonly cache;
13
13
  private hits;
@@ -64,6 +64,14 @@ export declare const triggerMatchDurationSeconds: import("@opentelemetry/api").H
64
64
  * Total deduplication cache hits (duplicate webhooks rejected).
65
65
  */
66
66
  export declare const dedupHitsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
67
+ /**
68
+ * pg connection errors absorbed by the pool error handlers instead of
69
+ * crashing the process (e.g. a database failover terminating idle pooled
70
+ * backends).
71
+ * Labels:
72
+ * - source: idle-pool | client
73
+ */
74
+ export declare const pgPoolClientErrorsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
67
75
  /**
68
76
  * Total source cache hits (`.kici/` source tarball found for content hash).
69
77
  */
@@ -26,7 +26,7 @@ import { type PeerHeartbeat } from '@kici-dev/engine';
26
26
  import { ScalerManager } from './scaler/index.js';
27
27
  import type { ScalerConfig } from './scaler/index.js';
28
28
  import type { CacheStorage } from './storage/index.js';
29
- import { SourceCache, BuildCoordinator, DepCache, PendingBuildTracker, PendingInitTracker, PendingDynamicTracker } from './cache/index.js';
29
+ import { SourceCache, BuildCoordinator, DepCache, UserCache, DispatchCacheRefTracker, PendingBuildTracker, PendingInitTracker, PendingDynamicTracker } from './cache/index.js';
30
30
  import { CheckRunReporter } from './reporting/check-run-reporter.js';
31
31
  import { StepLogBuffer } from './reporting/step-log-buffer.js';
32
32
  import { type LogStorage } from './reporting/log-storage.js';
@@ -70,6 +70,9 @@ export interface OrchestratorSubsystems {
70
70
  cacheStorage: CacheStorage | undefined;
71
71
  sourceCache: SourceCache | undefined;
72
72
  depCache: DepCache | undefined;
73
+ userCache: UserCache | undefined;
74
+ /** Server-side jobId -> user-cache-namespace store, written at dispatch, read by the agent-WS handler. */
75
+ dispatchCacheRefs: DispatchCacheRefTracker;
73
76
  buildCoordinator: BuildCoordinator | undefined;
74
77
  pendingBuilds: PendingBuildTracker | undefined;
75
78
  pendingInits: PendingInitTracker;
@@ -13,11 +13,20 @@
13
13
  * exported function is a narrative orchestrator that threads the typed
14
14
  * results through the pipeline.
15
15
  */
16
+ import { CacheRefScope } from '@kici-dev/engine';
16
17
  import type { LockWorkflow, SimulatedEvent, WorkflowDecision } from '@kici-dev/engine';
17
18
  import type { WebhookInfo } from '../webhook/handler.js';
18
19
  import type { ProviderBundle } from '../provider-registry.js';
19
20
  import type { TrustResolution } from '../security/trust-resolver.js';
20
21
  import { type ProcessingDeps } from './processor.js';
22
+ /**
23
+ * Trusted refs (write+ contributor, default-branch) get the org-shared cache
24
+ * write scope; everyone else (fork PR, unknown/known-but-not-trusted) is
25
+ * confined to a per-run isolated write scope. Absent trust resolution =>
26
+ * isolated (fail-closed), so an unresolved trust state can never poison the
27
+ * org-shared cache.
28
+ */
29
+ export declare function deriveCacheRefScope(trust: TrustResolution | undefined): CacheRefScope;
21
30
  /**
22
31
  * Context for dispatching a single matched workflow.
23
32
  *