@kici-dev/orchestrator 0.0.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +1 -6
- package/dist/__test-helpers__/mock-db.d.ts +103 -0
- package/dist/__test-helpers__/mock-ws.d.ts +13 -0
- package/dist/agent/dispatcher.d.ts +231 -0
- package/dist/agent/ownership-tracker.d.ts +46 -0
- package/dist/agent/registry.d.ts +255 -0
- package/dist/agent/token-store.d.ts +96 -0
- package/dist/app.d.ts +234 -0
- package/dist/audit/access-log.d.ts +92 -0
- package/dist/audit/sampling-rate-limiter.d.ts +41 -0
- package/dist/cache/build-coordinator.d.ts +42 -0
- package/dist/cache/dep-cache.d.ts +51 -0
- package/dist/cache/index.d.ts +16 -0
- package/dist/cache/pending-builds.d.ts +14 -0
- package/dist/cache/pending-dynamics.d.ts +15 -0
- package/dist/cache/pending-inits.d.ts +19 -0
- package/dist/cache/pending-tracker.d.ts +65 -0
- package/dist/cache/source-cache.d.ts +30 -0
- package/dist/cli/api-client.d.ts +411 -0
- package/dist/cli/commands/access-log.d.ts +30 -0
- package/dist/cli/commands/agent-service/index.d.ts +15 -0
- package/dist/cli/commands/agent-service/install.d.ts +9 -0
- package/dist/cli/commands/agent-service/logs.d.ts +9 -0
- package/dist/cli/commands/agent-service/restart.d.ts +8 -0
- package/dist/cli/commands/agent-service/start.d.ts +8 -0
- package/dist/cli/commands/agent-service/status.d.ts +10 -0
- package/dist/cli/commands/agent-service/stop.d.ts +8 -0
- package/dist/cli/commands/agent-service/uninstall.d.ts +9 -0
- package/dist/cli/commands/agent-service/upgrade.d.ts +11 -0
- package/dist/cli/commands/agent.d.ts +10 -0
- package/dist/cli/commands/api-key.d.ts +12 -0
- package/dist/cli/commands/audit.d.ts +9 -0
- package/dist/cli/commands/backend.d.ts +17 -0
- package/dist/cli/commands/cold-store-impl.d.ts +101 -0
- package/dist/cli/commands/cold-store.d.ts +30 -0
- package/dist/cli/commands/config.d.ts +16 -0
- package/dist/cli/commands/db.d.ts +19 -0
- package/dist/cli/commands/debug-bundle.d.ts +16 -0
- package/dist/cli/commands/diagnose.d.ts +15 -0
- package/dist/cli/commands/environment.d.ts +24 -0
- package/dist/cli/commands/event-dlq.d.ts +19 -0
- package/dist/cli/commands/event-log.d.ts +27 -0
- package/dist/cli/commands/event.d.ts +4 -0
- package/dist/cli/commands/execution.d.ts +17 -0
- package/dist/cli/commands/inspect-bundle.d.ts +16 -0
- package/dist/cli/commands/maintenance.d.ts +4 -0
- package/dist/cli/commands/orchestrator-service/index.d.ts +12 -0
- package/dist/cli/commands/orchestrator-service/install.d.ts +10 -0
- package/dist/cli/commands/orchestrator-service/logs.d.ts +10 -0
- package/dist/cli/commands/orchestrator-service/restart.d.ts +8 -0
- package/dist/cli/commands/orchestrator-service/start.d.ts +8 -0
- package/dist/cli/commands/orchestrator-service/status.d.ts +10 -0
- package/dist/cli/commands/orchestrator-service/stop.d.ts +8 -0
- package/dist/cli/commands/orchestrator-service/uninstall.d.ts +9 -0
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +12 -0
- package/dist/cli/commands/org-settings.d.ts +17 -0
- package/dist/cli/commands/peer.d.ts +13 -0
- package/dist/cli/commands/queue.d.ts +16 -0
- package/dist/cli/commands/registration.d.ts +18 -0
- package/dist/cli/commands/rotate.d.ts +9 -0
- package/dist/cli/commands/runs.d.ts +34 -0
- package/dist/cli/commands/secret.d.ts +12 -0
- package/dist/cli/commands/shared/db.d.ts +16 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +38 -0
- package/dist/cli/commands/source.d.ts +24 -0
- package/dist/cli/commands/token.d.ts +10 -0
- package/dist/cli/commands/workflow.d.ts +31 -0
- package/dist/cli/join.d.ts +14 -0
- package/dist/cli/kici-admin.d.ts +13 -0
- package/dist/cli/lazy-deps/cache.d.ts +17 -0
- package/dist/cli/lazy-deps/downloader.d.ts +33 -0
- package/dist/cli/lazy-deps/registry.d.ts +43 -0
- package/dist/cli/service/compose.d.ts +25 -0
- package/dist/cli/service/index.d.ts +24 -0
- package/dist/cli/service/launchd.d.ts +34 -0
- package/dist/cli/service/platform-detect.d.ts +59 -0
- package/dist/cli/service/systemd.d.ts +41 -0
- package/dist/cli/service/types.d.ts +91 -0
- package/dist/cli/service/windows.d.ts +20 -0
- package/dist/cli/wizard/agent-wizard.d.ts +24 -0
- package/dist/cli/wizard/orchestrator-wizard.d.ts +38 -0
- package/dist/cli/wizard/prompts.d.ts +23 -0
- package/dist/cli.js +11435 -0
- package/dist/cluster/cluster-identity.d.ts +59 -0
- package/dist/cluster/coordinator.d.ts +204 -0
- package/dist/cluster/health-api.d.ts +30 -0
- package/dist/cluster/index.d.ts +24 -0
- package/dist/cluster/join-client.d.ts +75 -0
- package/dist/cluster/join-handler.d.ts +58 -0
- package/dist/cluster/join-token.d.ts +94 -0
- package/dist/cluster/orphan-recovery.d.ts +67 -0
- package/dist/cluster/peer-client.d.ts +190 -0
- package/dist/cluster/peer-credentials.d.ts +108 -0
- package/dist/cluster/peer-crypto.d.ts +57 -0
- package/dist/cluster/peer-handler.d.ts +97 -0
- package/dist/cluster/peer-registry.d.ts +186 -0
- package/dist/cluster/raft-state.d.ts +36 -0
- package/dist/cluster/raft.d.ts +140 -0
- package/dist/cold-store/load-access-log-range.d.ts +73 -0
- package/dist/cold-store/load-event-log-range.d.ts +62 -0
- package/dist/cold-store/load-secret-audit-log-range.d.ts +38 -0
- package/dist/cold-store/orchestrator-cold-store.d.ts +58 -0
- package/dist/cold-store/tables/access-log.d.ts +80 -0
- package/dist/cold-store/tables/event-log.d.ts +70 -0
- package/dist/cold-store/tables/execution-jobs.d.ts +63 -0
- package/dist/cold-store/tables/execution-runs.d.ts +81 -0
- package/dist/cold-store/tables/execution-steps.d.ts +65 -0
- package/dist/cold-store/tables/secret-audit-log.d.ts +77 -0
- package/dist/concurrency/dispatch-next-queued.d.ts +63 -0
- package/dist/concurrency/group-tracker.d.ts +51 -0
- package/dist/concurrency/index.d.ts +3 -0
- package/dist/concurrency/queue-manager.d.ts +68 -0
- package/dist/concurrency/waiters.d.ts +58 -0
- package/dist/config/encryption.d.ts +82 -0
- package/dist/config/env-overlay.d.ts +45 -0
- package/dist/config/index.d.ts +16 -0
- package/dist/config/loader.d.ts +26 -0
- package/dist/config/reload.d.ts +113 -0
- package/dist/config/resolver.d.ts +47 -0
- package/dist/config/schema.d.ts +210 -0
- package/dist/config/shared-store.d.ts +142 -0
- package/dist/config/types.d.ts +218 -0
- package/dist/config.d.ts +249 -0
- package/dist/cron/cron-scheduler.d.ts +78 -0
- package/dist/cron/cron-store.d.ts +38 -0
- package/dist/dashboard/handler.d.ts +243 -0
- package/dist/db/client.d.ts +13 -0
- package/dist/db/migration-provider.d.ts +9 -0
- package/dist/db/migrations/001_initial.d.ts +8 -0
- package/dist/db/migrations/002_config_versions_key_version.d.ts +16 -0
- package/dist/db/migrations/003_access_log.d.ts +26 -0
- package/dist/db/migrations/004_rename_bundle_to_source.d.ts +19 -0
- package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +19 -0
- package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +38 -0
- package/dist/db/migrations/007_audit_logs_archived_at.d.ts +42 -0
- package/dist/db/migrations/008_event_log_archived_at.d.ts +40 -0
- package/dist/db/migrations/009_access_log_trigram.d.ts +15 -0
- package/dist/db/migrations/010_cold_store_chunks.d.ts +34 -0
- package/dist/db/migrations/011_drop_source_secrets_notify.d.ts +21 -0
- package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +28 -0
- package/dist/db/migrations/013_execution_log_bytes.d.ts +21 -0
- package/dist/db/migrations/014_kici_events_lease_retry.d.ts +36 -0
- package/dist/db/migrations/015_org_settings_customer_scoped.d.ts +30 -0
- package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +15 -0
- package/dist/db/migrations/017_org_id_widen.d.ts +4 -0
- package/dist/db/migrations/018_org_id_prefix_backfill.d.ts +4 -0
- package/dist/db/migrator.d.ts +32 -0
- package/dist/db/types.d.ts +1266 -0
- package/dist/diagnostics/bundle-reader.d.ts +48 -0
- package/dist/diagnostics/bundle-writer.d.ts +57 -0
- package/dist/diagnostics/checks/agents.d.ts +10 -0
- package/dist/diagnostics/checks/certs.d.ts +9 -0
- package/dist/diagnostics/checks/config.d.ts +10 -0
- package/dist/diagnostics/checks/db.d.ts +9 -0
- package/dist/diagnostics/checks/disk.d.ts +9 -0
- package/dist/diagnostics/checks/index.d.ts +17 -0
- package/dist/diagnostics/checks/ws.d.ts +9 -0
- package/dist/diagnostics/index.d.ts +13 -0
- package/dist/diagnostics/runner.d.ts +23 -0
- package/dist/diagnostics/types.d.ts +38 -0
- package/dist/entry-helpers.d.ts +93 -0
- package/dist/environments/binding-store.d.ts +27 -0
- package/dist/environments/environment-store.d.ts +74 -0
- package/dist/environments/held-runs.d.ts +64 -0
- package/dist/environments/index.d.ts +10 -0
- package/dist/environments/protection/branch-gate.d.ts +5 -0
- package/dist/environments/protection/concurrency-gate.d.ts +7 -0
- package/dist/environments/protection/pipeline.d.ts +18 -0
- package/dist/environments/protection/reviewer-gate.d.ts +7 -0
- package/dist/environments/protection/trust-gate.d.ts +7 -0
- package/dist/environments/protection/wait-timer-gate.d.ts +7 -0
- package/dist/environments/variable-store.d.ts +39 -0
- package/dist/events/circuit-breaker.d.ts +33 -0
- package/dist/events/event-emitter.d.ts +55 -0
- package/dist/events/event-retry-scanner.d.ts +52 -0
- package/dist/events/event-router.d.ts +183 -0
- package/dist/events/event-store.d.ts +157 -0
- package/dist/events/trust-store.d.ts +54 -0
- package/dist/events/types.d.ts +61 -0
- package/dist/helpers/ip-extraction.d.ts +29 -0
- package/dist/helpers/rate-limiter.d.ts +23 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +772 -0
- package/dist/lockfile-cache.d.ts +39 -0
- package/dist/metrics/agent-metrics-aggregator.d.ts +116 -0
- package/dist/metrics/metrics-reporter.d.ts +40 -0
- package/dist/metrics/prometheus.d.ts +261 -0
- package/dist/metrics/scheduled-jobs.d.ts +20 -0
- package/dist/orchestrator-core.d.ts +193 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +102 -0
- package/dist/pipeline/inline-eval.d.ts +28 -0
- package/dist/pipeline/install-secrets-resolver.d.ts +73 -0
- package/dist/pipeline/manual-schedule.d.ts +21 -0
- package/dist/pipeline/needs-scheduler.d.ts +85 -0
- package/dist/pipeline/process-webhook.d.ts +38 -0
- package/dist/pipeline/processor.d.ts +326 -0
- package/dist/pipeline/rerun.d.ts +79 -0
- package/dist/pipeline/test-pipeline.d.ts +110 -0
- package/dist/provider-registry.d.ts +122 -0
- package/dist/providers/generic/index.d.ts +23 -0
- package/dist/providers/generic/normalizer.d.ts +112 -0
- package/dist/providers/generic/verification.d.ts +47 -0
- package/dist/providers/github/auth.d.ts +60 -0
- package/dist/providers/github/changed-files.d.ts +51 -0
- package/dist/providers/github/check-status-poster.d.ts +35 -0
- package/dist/providers/github/contributor-resolver.d.ts +30 -0
- package/dist/providers/github/index.d.ts +20 -0
- package/dist/providers/github/lock-file.d.ts +32 -0
- package/dist/providers/github/normalizer.d.ts +103 -0
- package/dist/providers/github/repo-url.d.ts +35 -0
- package/dist/providers/internal/index.d.ts +32 -0
- package/dist/providers/internal/lock-file-fetcher.d.ts +34 -0
- package/dist/providers/internal/normalizer.d.ts +89 -0
- package/dist/providers/internal/repo-url-builder.d.ts +42 -0
- package/dist/providers/universal-git/changed-files.d.ts +42 -0
- package/dist/providers/universal-git/clone-token.d.ts +83 -0
- package/dist/providers/universal-git/config.d.ts +192 -0
- package/dist/providers/universal-git/index.d.ts +55 -0
- package/dist/providers/universal-git/lock-file.d.ts +71 -0
- package/dist/providers/universal-git/normalizer.d.ts +90 -0
- package/dist/providers/universal-git/repo-url.d.ts +50 -0
- package/dist/providers/universal-git/ssh-auth.d.ts +94 -0
- package/dist/queue/bootstrap.d.ts +50 -0
- package/dist/queue/cleanup.d.ts +41 -0
- package/dist/queue/depth-refresher.d.ts +65 -0
- package/dist/queue/job-queue.d.ts +343 -0
- package/dist/queue/scheduled-job.d.ts +91 -0
- package/dist/registration/extractor.d.ts +44 -0
- package/dist/registration/registration-index.d.ts +162 -0
- package/dist/registration/registration-store.d.ts +111 -0
- package/dist/reporting/check-run-summary.d.ts +102 -0
- package/dist/reporting/commit-status.d.ts +382 -0
- package/dist/reporting/execution-tracker.d.ts +538 -0
- package/dist/reporting/fs-log-storage.d.ts +41 -0
- package/dist/reporting/log-pull-handler.d.ts +35 -0
- package/dist/reporting/log-storage.d.ts +70 -0
- package/dist/reporting/log-writer.d.ts +45 -0
- package/dist/reporting/s3-log-storage.d.ts +42 -0
- package/dist/reporting/step-log-buffer.d.ts +57 -0
- package/dist/routes/admin-access-log.d.ts +34 -0
- package/dist/routes/admin-backends.d.ts +28 -0
- package/dist/routes/admin-config.d.ts +57 -0
- package/dist/routes/admin-db.d.ts +20 -0
- package/dist/routes/admin-environments.d.ts +42 -0
- package/dist/routes/admin-errors.d.ts +4 -0
- package/dist/routes/admin-event-dlq.d.ts +39 -0
- package/dist/routes/admin-event-log.d.ts +42 -0
- package/dist/routes/admin-events.d.ts +46 -0
- package/dist/routes/admin-maintenance.d.ts +22 -0
- package/dist/routes/admin-org-settings.d.ts +29 -0
- package/dist/routes/admin-queue-execution.d.ts +27 -0
- package/dist/routes/admin-registrations.d.ts +37 -0
- package/dist/routes/admin-runs.d.ts +57 -0
- package/dist/routes/admin-scheduled-jobs.d.ts +36 -0
- package/dist/routes/admin-sources.d.ts +18 -0
- package/dist/routes/admin.d.ts +103 -0
- package/dist/routes/capabilities.d.ts +24 -0
- package/dist/routes/diagnostics.d.ts +16 -0
- package/dist/routes/health.d.ts +20 -0
- package/dist/routes/test-trigger.d.ts +41 -0
- package/dist/routes/uploads.d.ts +38 -0
- package/dist/routes/webhooks.d.ts +36 -0
- package/dist/scaler/bare-metal-backend.d.ts +107 -0
- package/dist/scaler/config.d.ts +303 -0
- package/dist/scaler/container-backend.d.ts +125 -0
- package/dist/scaler/file-tail.d.ts +23 -0
- package/dist/scaler/firecracker-api.d.ts +100 -0
- package/dist/scaler/firecracker-backend.d.ts +263 -0
- package/dist/scaler/index.d.ts +24 -0
- package/dist/scaler/ip-allocator.d.ts +177 -0
- package/dist/scaler/label-matcher.d.ts +73 -0
- package/dist/scaler/log-forwarder.d.ts +54 -0
- package/dist/scaler/machine-ledger.d.ts +160 -0
- package/dist/scaler/manager.d.ts +338 -0
- package/dist/scaler/nftables.d.ts +85 -0
- package/dist/scaler/types.d.ts +388 -0
- package/dist/scaler/warm-pool.d.ts +81 -0
- package/dist/secrets/audit-logger.d.ts +65 -0
- package/dist/secrets/backend-health.d.ts +60 -0
- package/dist/secrets/backend-registry.d.ts +68 -0
- package/dist/secrets/backend-sync.d.ts +40 -0
- package/dist/secrets/cleanup.d.ts +40 -0
- package/dist/secrets/config.d.ts +51 -0
- package/dist/secrets/crypto.d.ts +49 -0
- package/dist/secrets/ephemeral-keys.d.ts +49 -0
- package/dist/secrets/index.d.ts +17 -0
- package/dist/secrets/pg-secret-store.d.ts +105 -0
- package/dist/secrets/rbac.d.ts +47 -0
- package/dist/secrets/secret-output-store.d.ts +46 -0
- package/dist/secrets/secret-resolver.d.ts +137 -0
- package/dist/secrets/source-credentials.d.ts +61 -0
- package/dist/secrets/token-manager.d.ts +63 -0
- package/dist/secrets/vault-secret-store.d.ts +85 -0
- package/dist/security/comment-handler.d.ts +67 -0
- package/dist/security/contributor-cache.d.ts +75 -0
- package/dist/security/global-workflow-policy.d.ts +72 -0
- package/dist/security/lock-source.d.ts +34 -0
- package/dist/security/trust-resolver.d.ts +108 -0
- package/dist/security/workflow-diff.d.ts +26 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.js +48055 -0
- package/dist/sources/index.d.ts +10 -0
- package/dist/sources/source-manager.d.ts +63 -0
- package/dist/sources/source-store.d.ts +79 -0
- package/dist/sources/source-validator.d.ts +16 -0
- package/dist/stale-detector/stale-run-detector.d.ts +109 -0
- package/dist/standalone.d.ts +20 -0
- package/dist/standalone.js +41546 -0
- package/dist/storage/blob-routes.d.ts +20 -0
- package/dist/storage/filesystem.d.ts +60 -0
- package/dist/storage/index.d.ts +20 -0
- package/dist/storage/s3.d.ts +68 -0
- package/dist/storage/sign-url.d.ts +45 -0
- package/dist/storage/types.d.ts +98 -0
- package/dist/webhook/dedup.d.ts +44 -0
- package/dist/webhook/event-log.d.ts +83 -0
- package/dist/webhook/generic-sources.d.ts +169 -0
- package/dist/webhook/handler.d.ts +23 -0
- package/dist/webhook/relay-buffer.d.ts +96 -0
- package/dist/webhook/verify-inbound.d.ts +78 -0
- package/dist/worker/in-memory-execution-tracker.d.ts +94 -0
- package/dist/worker/in-memory-job-queue.d.ts +74 -0
- package/dist/worker/static-agent-token-store.d.ts +58 -0
- package/dist/worker/worker-status.d.ts +36 -0
- package/dist/worker-core.d.ts +74 -0
- package/dist/ws/agent-api-registry.d.ts +49 -0
- package/dist/ws/agent-handler.d.ts +176 -0
- package/dist/ws/agent-heartbeat.d.ts +45 -0
- package/dist/ws/dashboard-backends-handler.d.ts +51 -0
- package/dist/ws/dashboard-diagnostics-handler.d.ts +56 -0
- package/dist/ws/dashboard-env-handler.d.ts +107 -0
- package/dist/ws/dashboard-global-workflows-handler.d.ts +72 -0
- package/dist/ws/dashboard-registrations-handler.d.ts +84 -0
- package/dist/ws/event-buffer.d.ts +16 -0
- package/dist/ws/observer-handler.d.ts +42 -0
- package/dist/ws/observer-registry.d.ts +88 -0
- package/dist/ws/platform-client.d.ts +270 -0
- package/dist/ws/server-options.d.ts +15 -0
- package/package.json +87 -6
- package/sbom.spdx.json +14370 -0
- package/index.js +0 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,772 @@
|
|
|
1
|
+
import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
|
|
2
|
+
import { dirname as __cjs_dirname } from "node:path";
|
|
3
|
+
__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
|
|
4
|
+
import "node:module";
|
|
5
|
+
import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
6
|
+
import { Upload } from "@aws-sdk/lib-storage";
|
|
7
|
+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
8
|
+
import { createLogger, createS3Client, sha256 } from "@kici-dev/shared";
|
|
9
|
+
import { promises } from "node:fs";
|
|
10
|
+
import { dirname, join } from "node:path";
|
|
11
|
+
import { createHmac, randomBytes } from "node:crypto";
|
|
12
|
+
import "node:fs/promises";
|
|
13
|
+
import { Kysely, PostgresDialect, sql } from "kysely";
|
|
14
|
+
import pg from "pg";
|
|
15
|
+
import.meta.url;
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/storage/s3.ts
|
|
18
|
+
/**
|
|
19
|
+
* S3-backed cache storage.
|
|
20
|
+
*
|
|
21
|
+
* Stores cached items in S3 with custom metadata headers for TTL tracking.
|
|
22
|
+
* Uses AWS SDK v3 modular imports -- only loaded when S3 storage is configured.
|
|
23
|
+
*
|
|
24
|
+
* Metadata storage:
|
|
25
|
+
* x-amz-meta-created-at -- ISO timestamp of creation
|
|
26
|
+
* x-amz-meta-last-accessed-at -- ISO timestamp of last access
|
|
27
|
+
*
|
|
28
|
+
* TTL is enforced lazily: expired items are deleted on access.
|
|
29
|
+
* Pre-signed URLs are generated for getUrl() with 15-minute expiry.
|
|
30
|
+
*/
|
|
31
|
+
/** Pre-signed download URL expiry in seconds (15 minutes) */
|
|
32
|
+
const PRESIGNED_URL_EXPIRY_SECONDS = 900;
|
|
33
|
+
/** Pre-signed upload URL expiry in seconds (30 minutes -- uploads take longer) */
|
|
34
|
+
const UPLOAD_URL_EXPIRY_SECONDS = 1800;
|
|
35
|
+
var S3CacheStorage = class {
|
|
36
|
+
client;
|
|
37
|
+
/** Separate client for pre-signed URL generation (uses externalEndpoint if configured). */
|
|
38
|
+
presignClient;
|
|
39
|
+
bucket;
|
|
40
|
+
prefix;
|
|
41
|
+
ttlMs;
|
|
42
|
+
constructor(options) {
|
|
43
|
+
this.bucket = options.bucket;
|
|
44
|
+
this.prefix = options.prefix;
|
|
45
|
+
this.ttlMs = options.ttlMs;
|
|
46
|
+
this.client = createS3Client(options);
|
|
47
|
+
if (options.externalEndpoint) this.presignClient = createS3Client({
|
|
48
|
+
...options,
|
|
49
|
+
endpoint: options.externalEndpoint
|
|
50
|
+
});
|
|
51
|
+
else this.presignClient = this.client;
|
|
52
|
+
}
|
|
53
|
+
/** Build the full S3 object key from a cache key. */
|
|
54
|
+
objectKey(key) {
|
|
55
|
+
return `${this.prefix}${key}`;
|
|
56
|
+
}
|
|
57
|
+
/** Check if metadata indicates an expired item. */
|
|
58
|
+
isExpired(meta) {
|
|
59
|
+
const lastAccessed = new Date(meta.lastAccessedAt).getTime();
|
|
60
|
+
return Date.now() - lastAccessed > this.ttlMs;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Read metadata from an S3 object's custom headers.
|
|
64
|
+
* Returns null if the object doesn't exist (NoSuchKey/NotFound).
|
|
65
|
+
*/
|
|
66
|
+
async readMeta(key) {
|
|
67
|
+
try {
|
|
68
|
+
const head = await this.client.send(new HeadObjectCommand({
|
|
69
|
+
Bucket: this.bucket,
|
|
70
|
+
Key: this.objectKey(key)
|
|
71
|
+
}));
|
|
72
|
+
const createdAt = head.Metadata?.["created-at"] ?? "";
|
|
73
|
+
const lastAccessedAt = head.Metadata?.["last-accessed-at"] ?? "";
|
|
74
|
+
if (!createdAt || !lastAccessedAt) return null;
|
|
75
|
+
return {
|
|
76
|
+
createdAt,
|
|
77
|
+
lastAccessedAt
|
|
78
|
+
};
|
|
79
|
+
} catch (err) {
|
|
80
|
+
if (this.isNotFoundError(err)) return null;
|
|
81
|
+
throw err;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Update metadata on an S3 object by copying it to itself.
|
|
86
|
+
* This is the standard S3 pattern for updating metadata without re-uploading data.
|
|
87
|
+
*/
|
|
88
|
+
async updateMeta(key, meta) {
|
|
89
|
+
const objectKey = this.objectKey(key);
|
|
90
|
+
await this.client.send(new CopyObjectCommand({
|
|
91
|
+
Bucket: this.bucket,
|
|
92
|
+
Key: objectKey,
|
|
93
|
+
CopySource: `${this.bucket}/${objectKey}`,
|
|
94
|
+
MetadataDirective: "REPLACE",
|
|
95
|
+
Metadata: {
|
|
96
|
+
"created-at": meta.createdAt,
|
|
97
|
+
"last-accessed-at": meta.lastAccessedAt
|
|
98
|
+
}
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
/** Check if an error is a "not found" error (NoSuchKey, NotFound, 404). */
|
|
102
|
+
isNotFoundError(err) {
|
|
103
|
+
if (typeof err !== "object" || err === null) return false;
|
|
104
|
+
const name = err.name;
|
|
105
|
+
if (name === "NoSuchKey" || name === "NotFound") return true;
|
|
106
|
+
return err.$metadata?.httpStatusCode === 404;
|
|
107
|
+
}
|
|
108
|
+
async put(key, data) {
|
|
109
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
110
|
+
const body = typeof data === "string" ? Buffer.from(data) : data;
|
|
111
|
+
await new Upload({
|
|
112
|
+
client: this.client,
|
|
113
|
+
params: {
|
|
114
|
+
Bucket: this.bucket,
|
|
115
|
+
Key: this.objectKey(key),
|
|
116
|
+
Body: body,
|
|
117
|
+
Metadata: {
|
|
118
|
+
"created-at": now,
|
|
119
|
+
"last-accessed-at": now
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}).done();
|
|
123
|
+
}
|
|
124
|
+
async get(key) {
|
|
125
|
+
const meta = await this.readMeta(key);
|
|
126
|
+
if (!meta) return null;
|
|
127
|
+
if (this.isExpired(meta)) {
|
|
128
|
+
await this.deleteObject(key);
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
let data;
|
|
132
|
+
try {
|
|
133
|
+
const response = await this.client.send(new GetObjectCommand({
|
|
134
|
+
Bucket: this.bucket,
|
|
135
|
+
Key: this.objectKey(key)
|
|
136
|
+
}));
|
|
137
|
+
if (!response.Body) return null;
|
|
138
|
+
data = Buffer.from(await response.Body.transformToByteArray());
|
|
139
|
+
} catch (err) {
|
|
140
|
+
if (this.isNotFoundError(err)) return null;
|
|
141
|
+
throw err;
|
|
142
|
+
}
|
|
143
|
+
try {
|
|
144
|
+
meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
145
|
+
await this.updateMeta(key, meta);
|
|
146
|
+
} catch {}
|
|
147
|
+
return data;
|
|
148
|
+
}
|
|
149
|
+
async has(key) {
|
|
150
|
+
const meta = await this.readMeta(key);
|
|
151
|
+
if (!meta) return false;
|
|
152
|
+
if (this.isExpired(meta)) {
|
|
153
|
+
await this.deleteObject(key);
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
async delete(key) {
|
|
159
|
+
const existed = await this.readMeta(key) !== null;
|
|
160
|
+
await this.deleteObject(key);
|
|
161
|
+
return existed;
|
|
162
|
+
}
|
|
163
|
+
async touch(key) {
|
|
164
|
+
const meta = await this.readMeta(key);
|
|
165
|
+
if (!meta) return;
|
|
166
|
+
meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
167
|
+
await this.updateMeta(key, meta);
|
|
168
|
+
}
|
|
169
|
+
async getUrl(key) {
|
|
170
|
+
const meta = await this.readMeta(key);
|
|
171
|
+
if (!meta) return null;
|
|
172
|
+
if (this.isExpired(meta)) {
|
|
173
|
+
await this.deleteObject(key);
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
return await getSignedUrl(this.presignClient, new GetObjectCommand({
|
|
177
|
+
Bucket: this.bucket,
|
|
178
|
+
Key: this.objectKey(key)
|
|
179
|
+
}), { expiresIn: PRESIGNED_URL_EXPIRY_SECONDS });
|
|
180
|
+
}
|
|
181
|
+
async getUploadUrl(key) {
|
|
182
|
+
const objectKey = this.objectKey(key);
|
|
183
|
+
return getSignedUrl(this.presignClient, new PutObjectCommand({
|
|
184
|
+
Bucket: this.bucket,
|
|
185
|
+
Key: objectKey
|
|
186
|
+
}), { expiresIn: UPLOAD_URL_EXPIRY_SECONDS });
|
|
187
|
+
}
|
|
188
|
+
async getInternalUploadUrl(key) {
|
|
189
|
+
const objectKey = this.objectKey(key);
|
|
190
|
+
return getSignedUrl(this.client, new PutObjectCommand({
|
|
191
|
+
Bucket: this.bucket,
|
|
192
|
+
Key: objectKey
|
|
193
|
+
}), { expiresIn: UPLOAD_URL_EXPIRY_SECONDS });
|
|
194
|
+
}
|
|
195
|
+
async initMeta(key) {
|
|
196
|
+
const objectKey = this.objectKey(key);
|
|
197
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
198
|
+
await this.client.send(new CopyObjectCommand({
|
|
199
|
+
Bucket: this.bucket,
|
|
200
|
+
Key: objectKey,
|
|
201
|
+
CopySource: `${this.bucket}/${objectKey}`,
|
|
202
|
+
MetadataDirective: "REPLACE",
|
|
203
|
+
Metadata: {
|
|
204
|
+
"created-at": now,
|
|
205
|
+
"last-accessed-at": now
|
|
206
|
+
}
|
|
207
|
+
}));
|
|
208
|
+
}
|
|
209
|
+
/** Delete an S3 object. Idempotent (doesn't error if missing). */
|
|
210
|
+
async deleteObject(key) {
|
|
211
|
+
try {
|
|
212
|
+
await this.client.send(new DeleteObjectCommand({
|
|
213
|
+
Bucket: this.bucket,
|
|
214
|
+
Key: this.objectKey(key)
|
|
215
|
+
}));
|
|
216
|
+
} catch (err) {
|
|
217
|
+
if (!this.isNotFoundError(err)) throw err;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
//#endregion
|
|
222
|
+
//#region src/storage/sign-url.ts
|
|
223
|
+
/**
|
|
224
|
+
* HMAC-signed URL tokens for the filesystem cache backend.
|
|
225
|
+
*
|
|
226
|
+
* Pre-signed S3 URLs are not available for filesystem storage, so the
|
|
227
|
+
* filesystem backend mints HMAC-SHA256 tokens that the orchestrator's HTTP
|
|
228
|
+
* blob route verifies before serving / writing a file. Token shape:
|
|
229
|
+
*
|
|
230
|
+
* `${expiresUnixMs}.${hexSig}`
|
|
231
|
+
*
|
|
232
|
+
* Signature input: `${method}:${key}:${expiresUnixMs}` with HMAC-SHA256 keyed
|
|
233
|
+
* by a per-orchestrator-process secret (random 32-byte hex at boot, persists
|
|
234
|
+
* for the lifetime of the process). Tokens become invalid on orchestrator
|
|
235
|
+
* restart — fine for the filesystem backend's E2E / single-host use case.
|
|
236
|
+
*/
|
|
237
|
+
/** Default URL lifetime: 1 hour (matches S3 backend's PUT-URL lifetime). */
|
|
238
|
+
const DEFAULT_SIGN_URL_TTL_MS = 3600 * 1e3;
|
|
239
|
+
function computeSig(secret, method, key, expiresMs) {
|
|
240
|
+
const hmac = createHmac("sha256", secret);
|
|
241
|
+
hmac.update(`${method}:${key}:${expiresMs}`);
|
|
242
|
+
return hmac.digest("hex");
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Mint a signed token for the given (method, key, expiry) triple.
|
|
246
|
+
*/
|
|
247
|
+
function signToken(secret, method, key, ttlMs = DEFAULT_SIGN_URL_TTL_MS) {
|
|
248
|
+
const expiresMs = Date.now() + ttlMs;
|
|
249
|
+
return {
|
|
250
|
+
token: `${expiresMs}.${computeSig(secret, method, key, expiresMs)}`,
|
|
251
|
+
expiresMs
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
//#endregion
|
|
255
|
+
//#region src/storage/filesystem.ts
|
|
256
|
+
/**
|
|
257
|
+
* Filesystem-backed cache storage.
|
|
258
|
+
*
|
|
259
|
+
* Mirrors the `CacheStorage` interface that `S3CacheStorage` provides, but
|
|
260
|
+
* stores blobs as files under a configured base directory. Pre-signed URLs
|
|
261
|
+
* are replaced by HMAC-signed `http://<orchUrl>/api/v1/cache/blob/...` URLs
|
|
262
|
+
* — see `sign-url.ts` for the token mechanics and `app.ts` for the route.
|
|
263
|
+
*
|
|
264
|
+
* Intended for single-host deployments and E2E sandboxes where standing up
|
|
265
|
+
* an S3-compatible service is overkill. Production should still use S3.
|
|
266
|
+
*
|
|
267
|
+
* Metadata is co-located as a sibling JSON file (`<key>.meta.json`) so the
|
|
268
|
+
* data file is byte-identical to what the agent receives over HTTP. TTL is
|
|
269
|
+
* enforced lazily on access — same model as the S3 backend.
|
|
270
|
+
*/
|
|
271
|
+
const DEFAULT_ROUTE_PREFIX = "/api/v1/cache/blob/";
|
|
272
|
+
var FilesystemCacheStorage = class {
|
|
273
|
+
basePath;
|
|
274
|
+
ttlMs;
|
|
275
|
+
baseUrl;
|
|
276
|
+
signingSecret;
|
|
277
|
+
routePrefix;
|
|
278
|
+
constructor(options) {
|
|
279
|
+
this.basePath = options.basePath;
|
|
280
|
+
this.ttlMs = options.ttlMs;
|
|
281
|
+
this.baseUrl = options.baseUrl.replace(/\/$/, "");
|
|
282
|
+
this.signingSecret = options.signingSecret;
|
|
283
|
+
this.routePrefix = (options.routePrefix ?? DEFAULT_ROUTE_PREFIX).replace(/\/?$/, "/");
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Map a cache key to its on-disk path. Keys may contain `/`, which becomes a
|
|
287
|
+
* directory separator. Reject keys containing `..` segments or NUL bytes to
|
|
288
|
+
* prevent path traversal — the same defence the route layer enforces too.
|
|
289
|
+
*/
|
|
290
|
+
resolvePath(key) {
|
|
291
|
+
if (key.includes("\0") || key.split("/").some((seg) => seg === ".." || seg === "")) throw new Error(`Invalid cache key: ${JSON.stringify(key)}`);
|
|
292
|
+
return join(this.basePath, key);
|
|
293
|
+
}
|
|
294
|
+
/** Public for the HTTP route. The route validates the key before calling. */
|
|
295
|
+
pathFor(key) {
|
|
296
|
+
return this.resolvePath(key);
|
|
297
|
+
}
|
|
298
|
+
metaPath(key) {
|
|
299
|
+
return `${this.resolvePath(key)}.meta.json`;
|
|
300
|
+
}
|
|
301
|
+
isExpired(meta) {
|
|
302
|
+
const lastAccessed = new Date(meta.lastAccessedAt).getTime();
|
|
303
|
+
return Date.now() - lastAccessed > this.ttlMs;
|
|
304
|
+
}
|
|
305
|
+
async readMeta(key) {
|
|
306
|
+
try {
|
|
307
|
+
const raw = await promises.readFile(this.metaPath(key), "utf-8");
|
|
308
|
+
const parsed = JSON.parse(raw);
|
|
309
|
+
if (typeof parsed.createdAt !== "string" || typeof parsed.lastAccessedAt !== "string") return null;
|
|
310
|
+
return parsed;
|
|
311
|
+
} catch (err) {
|
|
312
|
+
if (isNotFoundFsError(err)) return null;
|
|
313
|
+
throw err;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
async writeMeta(key, meta) {
|
|
317
|
+
const path = this.metaPath(key);
|
|
318
|
+
await promises.mkdir(dirname(path), { recursive: true });
|
|
319
|
+
await writeFileAtomic(path, JSON.stringify(meta));
|
|
320
|
+
}
|
|
321
|
+
async put(key, data) {
|
|
322
|
+
const path = this.resolvePath(key);
|
|
323
|
+
await promises.mkdir(dirname(path), { recursive: true });
|
|
324
|
+
await writeFileAtomic(path, typeof data === "string" ? Buffer.from(data) : data);
|
|
325
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
326
|
+
await this.writeMeta(key, {
|
|
327
|
+
createdAt: now,
|
|
328
|
+
lastAccessedAt: now
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
async get(key) {
|
|
332
|
+
const meta = await this.readMeta(key);
|
|
333
|
+
if (!meta) return null;
|
|
334
|
+
if (this.isExpired(meta)) {
|
|
335
|
+
await this.deleteFiles(key);
|
|
336
|
+
return null;
|
|
337
|
+
}
|
|
338
|
+
let data;
|
|
339
|
+
try {
|
|
340
|
+
data = await promises.readFile(this.resolvePath(key));
|
|
341
|
+
} catch (err) {
|
|
342
|
+
if (isNotFoundFsError(err)) return null;
|
|
343
|
+
throw err;
|
|
344
|
+
}
|
|
345
|
+
try {
|
|
346
|
+
meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
347
|
+
await this.writeMeta(key, meta);
|
|
348
|
+
} catch {}
|
|
349
|
+
return data;
|
|
350
|
+
}
|
|
351
|
+
async has(key) {
|
|
352
|
+
const meta = await this.readMeta(key);
|
|
353
|
+
if (!meta) return false;
|
|
354
|
+
if (this.isExpired(meta)) {
|
|
355
|
+
await this.deleteFiles(key);
|
|
356
|
+
return false;
|
|
357
|
+
}
|
|
358
|
+
return true;
|
|
359
|
+
}
|
|
360
|
+
async delete(key) {
|
|
361
|
+
const existed = await this.readMeta(key) !== null;
|
|
362
|
+
await this.deleteFiles(key);
|
|
363
|
+
return existed;
|
|
364
|
+
}
|
|
365
|
+
async touch(key) {
|
|
366
|
+
const meta = await this.readMeta(key);
|
|
367
|
+
if (!meta) return;
|
|
368
|
+
meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
369
|
+
await this.writeMeta(key, meta);
|
|
370
|
+
}
|
|
371
|
+
async getUrl(key) {
|
|
372
|
+
const meta = await this.readMeta(key);
|
|
373
|
+
if (!meta) return null;
|
|
374
|
+
if (this.isExpired(meta)) {
|
|
375
|
+
await this.deleteFiles(key);
|
|
376
|
+
return null;
|
|
377
|
+
}
|
|
378
|
+
return this.signedUrl("GET", key);
|
|
379
|
+
}
|
|
380
|
+
async getUploadUrl(key) {
|
|
381
|
+
return this.signedUrl("PUT", key);
|
|
382
|
+
}
|
|
383
|
+
async getInternalUploadUrl(key) {
|
|
384
|
+
return this.signedUrl("PUT", key);
|
|
385
|
+
}
|
|
386
|
+
async initMeta(key) {
|
|
387
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
388
|
+
await this.writeMeta(key, {
|
|
389
|
+
createdAt: now,
|
|
390
|
+
lastAccessedAt: now
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
signedUrl(method, key) {
|
|
394
|
+
const { token } = signToken(this.signingSecret, method, key);
|
|
395
|
+
const encodedKey = key.split("/").map((seg) => encodeURIComponent(seg)).join("/");
|
|
396
|
+
return `${this.baseUrl}${this.routePrefix}${encodedKey}?sig=${encodeURIComponent(token)}`;
|
|
397
|
+
}
|
|
398
|
+
async deleteFiles(key) {
|
|
399
|
+
await promises.rm(this.resolvePath(key), { force: true });
|
|
400
|
+
await promises.rm(this.metaPath(key), { force: true });
|
|
401
|
+
}
|
|
402
|
+
};
|
|
403
|
+
function isNotFoundFsError(err) {
|
|
404
|
+
return typeof err === "object" && err !== null && err.code === "ENOENT";
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Write a file atomically: write to a temp sibling, fsync, rename. Avoids
|
|
408
|
+
* partial reads if the orchestrator crashes mid-write. The temp suffix is
|
|
409
|
+
* randomised so concurrent writers don't collide on the same temp name.
|
|
410
|
+
*/
|
|
411
|
+
async function writeFileAtomic(path, data) {
|
|
412
|
+
const tmp = `${path}.tmp-${randomBytes(6).toString("hex")}`;
|
|
413
|
+
const handle = await promises.open(tmp, "w");
|
|
414
|
+
try {
|
|
415
|
+
await handle.writeFile(data);
|
|
416
|
+
await handle.sync();
|
|
417
|
+
} finally {
|
|
418
|
+
await handle.close();
|
|
419
|
+
}
|
|
420
|
+
await promises.rename(tmp, path);
|
|
421
|
+
}
|
|
422
|
+
//#endregion
|
|
423
|
+
//#region src/storage/index.ts
|
|
424
|
+
/**
|
|
425
|
+
* Create a cache storage backend from configuration.
|
|
426
|
+
*/
|
|
427
|
+
function createCacheStorage(config) {
|
|
428
|
+
if (config.type === "s3") return new S3CacheStorage({
|
|
429
|
+
bucket: config.bucket,
|
|
430
|
+
prefix: config.prefix,
|
|
431
|
+
ttlMs: config.ttlMs,
|
|
432
|
+
region: config.region,
|
|
433
|
+
endpoint: config.endpoint,
|
|
434
|
+
externalEndpoint: config.externalEndpoint,
|
|
435
|
+
forcePathStyle: config.forcePathStyle
|
|
436
|
+
});
|
|
437
|
+
if (config.type === "filesystem") return new FilesystemCacheStorage({
|
|
438
|
+
basePath: config.basePath,
|
|
439
|
+
ttlMs: config.ttlMs,
|
|
440
|
+
baseUrl: config.baseUrl,
|
|
441
|
+
signingSecret: config.signingSecret
|
|
442
|
+
});
|
|
443
|
+
throw new Error(`Unknown cache storage type: ${config.type}`);
|
|
444
|
+
}
|
|
445
|
+
//#endregion
|
|
446
|
+
//#region src/cache/source-cache.ts
|
|
447
|
+
/**
|
|
448
|
+
* Source-tarball cache layer wrapping CacheStorage.
|
|
449
|
+
*
|
|
450
|
+
* Provides content-hash-keyed operations for customer-workflow source tarballs
|
|
451
|
+
* (the `.kici/` directory minus `node_modules/`, packed by the agent's build
|
|
452
|
+
* job via `source-packer.ts`). Tarballs are platform-independent: raw TypeScript
|
|
453
|
+
* source files are identical across architectures. Refreshes TTL on reads
|
|
454
|
+
* (touch-on-read) so actively used sources stay in cache longer.
|
|
455
|
+
*
|
|
456
|
+
* Cache key format: source/{contentHash}.tar.gz
|
|
457
|
+
*
|
|
458
|
+
* Replaces the former `BundleCache` / `bundles/{contentHash}.js` layout that
|
|
459
|
+
* stored Rolldown-bundled `.compiled.mjs` artifacts — the new execution path
|
|
460
|
+
* downloads the source tarball, extracts it, and imports the workflow entry
|
|
461
|
+
* via the shared oxc-transform ESM loader hook.
|
|
462
|
+
*/
|
|
463
|
+
const logger$1 = createLogger({ prefix: "source-cache" });
|
|
464
|
+
/** Cache key format: source/{contentHash}.tar.gz */
|
|
465
|
+
function sourceKey(contentHash) {
|
|
466
|
+
return `source/${contentHash}.tar.gz`;
|
|
467
|
+
}
|
|
468
|
+
var SourceCache = class {
|
|
469
|
+
storage;
|
|
470
|
+
constructor(options) {
|
|
471
|
+
this.storage = options.storage;
|
|
472
|
+
}
|
|
473
|
+
async has(contentHash) {
|
|
474
|
+
const key = sourceKey(contentHash);
|
|
475
|
+
const exists = await this.storage.has(key);
|
|
476
|
+
logger$1.debug(`has(${contentHash}): ${exists}`);
|
|
477
|
+
return exists;
|
|
478
|
+
}
|
|
479
|
+
async get(contentHash) {
|
|
480
|
+
const key = sourceKey(contentHash);
|
|
481
|
+
const data = await this.storage.get(key);
|
|
482
|
+
if (data) {
|
|
483
|
+
await this.storage.touch(key);
|
|
484
|
+
logger$1.debug(`get(${contentHash}): hit (${data.length} bytes)`);
|
|
485
|
+
} else logger$1.debug(`get(${contentHash}): miss`);
|
|
486
|
+
return data;
|
|
487
|
+
}
|
|
488
|
+
async getUrl(contentHash) {
|
|
489
|
+
const key = sourceKey(contentHash);
|
|
490
|
+
const url = await this.storage.getUrl(key);
|
|
491
|
+
if (url) {
|
|
492
|
+
await this.storage.touch(key);
|
|
493
|
+
logger$1.debug(`getUrl(${contentHash}): hit`);
|
|
494
|
+
} else logger$1.debug(`getUrl(${contentHash}): miss`);
|
|
495
|
+
return url;
|
|
496
|
+
}
|
|
497
|
+
async getUploadUrl(contentHash) {
|
|
498
|
+
const key = sourceKey(contentHash);
|
|
499
|
+
return this.storage.getUploadUrl(key);
|
|
500
|
+
}
|
|
501
|
+
async store(contentHash, tarball) {
|
|
502
|
+
const key = sourceKey(contentHash);
|
|
503
|
+
await this.storage.put(key, tarball);
|
|
504
|
+
const size = typeof tarball === "string" ? Buffer.byteLength(tarball) : tarball.length;
|
|
505
|
+
logger$1.info(`store(${contentHash}): stored (${size} bytes)`);
|
|
506
|
+
}
|
|
507
|
+
async remove(contentHash) {
|
|
508
|
+
const key = sourceKey(contentHash);
|
|
509
|
+
const removed = await this.storage.delete(key);
|
|
510
|
+
logger$1.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`);
|
|
511
|
+
return removed;
|
|
512
|
+
}
|
|
513
|
+
};
|
|
514
|
+
//#endregion
|
|
515
|
+
//#region src/cache/dep-cache.ts
|
|
516
|
+
/**
|
|
517
|
+
* Dependency-specific cache layer wrapping CacheStorage.
|
|
518
|
+
*
|
|
519
|
+
* Stores and retrieves dependency tarballs keyed by lockfileHash + platform + arch.
|
|
520
|
+
* Shared CacheStorage backend with SourceCache (same S3 bucket).
|
|
521
|
+
* Refreshes TTL on reads (touch-on-read).
|
|
522
|
+
*
|
|
523
|
+
* Cache key format: deps/{platform}-{arch}/{lockfileHash}.tar.gz
|
|
524
|
+
*/
|
|
525
|
+
const logger = createLogger({ prefix: "dep-cache" });
|
|
526
|
+
/** Default max tarball size: 500MB */
|
|
527
|
+
const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
|
|
528
|
+
/** Build cache key for dependency tarball: deps/{platform}-{arch}/{lockfileHash}.tar.gz */
|
|
529
|
+
function depKey(lockfileHash, platform, arch) {
|
|
530
|
+
return `deps/${platform}-${arch}/${lockfileHash}.tar.gz`;
|
|
531
|
+
}
|
|
532
|
+
var DepCache = class {
|
|
533
|
+
storage;
|
|
534
|
+
maxTarballBytes;
|
|
535
|
+
constructor(options) {
|
|
536
|
+
this.storage = options.storage;
|
|
537
|
+
this.maxTarballBytes = options.maxTarballBytes ?? DEFAULT_MAX_TARBALL_BYTES;
|
|
538
|
+
}
|
|
539
|
+
/** Check if a dep tarball exists in cache. */
|
|
540
|
+
async has(lockfileHash, platform, arch) {
|
|
541
|
+
const key = depKey(lockfileHash, platform, arch);
|
|
542
|
+
const exists = await this.storage.has(key);
|
|
543
|
+
logger.debug(`has(${lockfileHash}): ${exists}`, {
|
|
544
|
+
platform,
|
|
545
|
+
arch
|
|
546
|
+
});
|
|
547
|
+
return exists;
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Get a pre-signed download URL for the dep tarball (for agent delivery).
|
|
551
|
+
* Refreshes TTL on hit.
|
|
552
|
+
*/
|
|
553
|
+
async getUrl(lockfileHash, platform, arch) {
|
|
554
|
+
const key = depKey(lockfileHash, platform, arch);
|
|
555
|
+
const url = await this.storage.getUrl(key);
|
|
556
|
+
if (url) {
|
|
557
|
+
await this.storage.touch(key);
|
|
558
|
+
logger.debug(`getUrl(${lockfileHash}): hit`, {
|
|
559
|
+
platform,
|
|
560
|
+
arch
|
|
561
|
+
});
|
|
562
|
+
} else logger.debug(`getUrl(${lockfileHash}): miss`, {
|
|
563
|
+
platform,
|
|
564
|
+
arch
|
|
565
|
+
});
|
|
566
|
+
return url;
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* Get a pre-signed download URL and the tarball content hash (if available).
|
|
570
|
+
* Returns null on cache miss. Hash may be undefined for old entries stored
|
|
571
|
+
* before integrity tracking was added.
|
|
572
|
+
*/
|
|
573
|
+
async getUrlAndHash(lockfileHash, platform, arch) {
|
|
574
|
+
const key = depKey(lockfileHash, platform, arch);
|
|
575
|
+
const url = await this.storage.getUrl(key);
|
|
576
|
+
if (!url) {
|
|
577
|
+
logger.debug(`getUrlAndHash(${lockfileHash}): miss`, {
|
|
578
|
+
platform,
|
|
579
|
+
arch
|
|
580
|
+
});
|
|
581
|
+
return null;
|
|
582
|
+
}
|
|
583
|
+
await this.storage.touch(key);
|
|
584
|
+
const hashKey = `deps/${platform}-${arch}/${lockfileHash}.hash`;
|
|
585
|
+
const hash = (await this.storage.get(hashKey))?.toString("utf-8") || void 0;
|
|
586
|
+
logger.debug(`getUrlAndHash(${lockfileHash}): hit`, {
|
|
587
|
+
platform,
|
|
588
|
+
arch,
|
|
589
|
+
hasHash: !!hash
|
|
590
|
+
});
|
|
591
|
+
return {
|
|
592
|
+
url,
|
|
593
|
+
hash
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Get a pre-signed upload URL for direct agent-to-S3 upload.
|
|
598
|
+
*/
|
|
599
|
+
async getUploadUrl(lockfileHash, platform, arch) {
|
|
600
|
+
const key = depKey(lockfileHash, platform, arch);
|
|
601
|
+
return this.storage.getUploadUrl(key);
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* Store a dep tarball in cache.
|
|
605
|
+
* Throws if tarball exceeds max size (per user decision).
|
|
606
|
+
*/
|
|
607
|
+
async store(lockfileHash, platform, arch, tarballData) {
|
|
608
|
+
if (tarballData.length > this.maxTarballBytes) throw new Error(`Dep tarball exceeds max size: ${tarballData.length} bytes > ${this.maxTarballBytes} bytes limit`);
|
|
609
|
+
const key = depKey(lockfileHash, platform, arch);
|
|
610
|
+
await this.storage.put(key, tarballData);
|
|
611
|
+
logger.info(`store: ${tarballData.length} bytes`, {
|
|
612
|
+
lockfileHash,
|
|
613
|
+
platform,
|
|
614
|
+
arch
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* Compute SHA-256 hash of a tarball buffer.
|
|
619
|
+
* Used by build agents to compute the hash for depsHash protocol field.
|
|
620
|
+
*/
|
|
621
|
+
static computeHash(data) {
|
|
622
|
+
return sha256(data);
|
|
623
|
+
}
|
|
624
|
+
/** Remove a dep tarball from cache. */
|
|
625
|
+
async remove(lockfileHash, platform, arch) {
|
|
626
|
+
const key = depKey(lockfileHash, platform, arch);
|
|
627
|
+
const removed = await this.storage.delete(key);
|
|
628
|
+
logger.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
|
|
629
|
+
platform,
|
|
630
|
+
arch
|
|
631
|
+
});
|
|
632
|
+
return removed;
|
|
633
|
+
}
|
|
634
|
+
};
|
|
635
|
+
//#endregion
|
|
636
|
+
//#region src/cluster/peer-credentials.ts
|
|
637
|
+
/**
|
|
638
|
+
* Peer credential management for cluster authentication.
|
|
639
|
+
*
|
|
640
|
+
* Provides two persistence layers:
|
|
641
|
+
* 1. DB CRUD (PeerCredentialStore) — coordinator stores credentials for all peers
|
|
642
|
+
* 2. File I/O (readCredentialFile/writeCredentialFile) — workers persist their
|
|
643
|
+
* credential locally for reconnection after restart
|
|
644
|
+
*
|
|
645
|
+
* Credential files are written with 0600 permissions to protect the secret.
|
|
646
|
+
*/
|
|
647
|
+
/** Default credential expiry: 90 days. */
|
|
648
|
+
const DEFAULT_EXPIRY_DAYS = 90;
|
|
649
|
+
/**
|
|
650
|
+
* Database CRUD operations for peer credentials.
|
|
651
|
+
* Used by the coordinator to manage credentials for all cluster members.
|
|
652
|
+
*/
|
|
653
|
+
var PeerCredentialStore = class {
|
|
654
|
+
constructor(db) {
|
|
655
|
+
this.db = db;
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
658
|
+
* Save a new peer credential.
|
|
659
|
+
*/
|
|
660
|
+
async save(opts) {
|
|
661
|
+
const expiryDays = opts.expiryDays ?? DEFAULT_EXPIRY_DAYS;
|
|
662
|
+
const expiresAt = new Date(Date.now() + expiryDays * 24 * 60 * 60 * 1e3);
|
|
663
|
+
const runOnce = async () => {
|
|
664
|
+
await this.db.transaction().execute(async (trx) => {
|
|
665
|
+
await trx.updateTable("peer_credentials").set({ revoked_at: /* @__PURE__ */ new Date() }).where("instance_id", "=", opts.instanceId).where("revoked_at", "is", null).execute();
|
|
666
|
+
await trx.insertInto("peer_credentials").values({
|
|
667
|
+
instance_id: opts.instanceId,
|
|
668
|
+
credential_hash: opts.credentialHash,
|
|
669
|
+
role: opts.role,
|
|
670
|
+
routing_keys: sql`${sql.val(opts.routingKeys)}`,
|
|
671
|
+
source_token_hash: opts.sourceTokenHash ?? null,
|
|
672
|
+
expires_at: expiresAt
|
|
673
|
+
}).execute();
|
|
674
|
+
});
|
|
675
|
+
};
|
|
676
|
+
try {
|
|
677
|
+
await runOnce();
|
|
678
|
+
} catch (err) {
|
|
679
|
+
if (err instanceof Error && "code" in err && err.code === "23505") {
|
|
680
|
+
await runOnce();
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
throw err;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Find a non-revoked, non-expired credential by its hash.
|
|
688
|
+
*/
|
|
689
|
+
async findByCredentialHash(hash) {
|
|
690
|
+
const row = await this.db.selectFrom("peer_credentials").selectAll().where("credential_hash", "=", hash).where("revoked_at", "is", null).where("expires_at", ">", /* @__PURE__ */ new Date()).executeTakeFirst();
|
|
691
|
+
return row ? mapRow(row) : null;
|
|
692
|
+
}
|
|
693
|
+
/**
|
|
694
|
+
* Find a non-revoked, non-expired credential by instance ID.
|
|
695
|
+
*/
|
|
696
|
+
async findByInstanceId(instanceId) {
|
|
697
|
+
const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is", null).where("expires_at", ">", /* @__PURE__ */ new Date()).orderBy("created_at", "desc").executeTakeFirst();
|
|
698
|
+
return row ? mapRow(row) : null;
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* Update the last_seen_at timestamp for a credential.
|
|
702
|
+
* Optionally records which coordinator validated it.
|
|
703
|
+
*/
|
|
704
|
+
async updateLastSeen(credentialHash, validatedBy) {
|
|
705
|
+
const updates = { last_seen_at: /* @__PURE__ */ new Date() };
|
|
706
|
+
if (validatedBy) updates.last_validated_by = validatedBy;
|
|
707
|
+
await this.db.updateTable("peer_credentials").set(updates).where("credential_hash", "=", credentialHash).execute();
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Revoke a peer's credential by instance ID.
|
|
711
|
+
*/
|
|
712
|
+
async revoke(instanceId) {
|
|
713
|
+
await this.db.updateTable("peer_credentials").set({ revoked_at: /* @__PURE__ */ new Date() }).where("instance_id", "=", instanceId).where("revoked_at", "is", null).execute();
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Revoke all active credentials.
|
|
717
|
+
* @returns The number of credentials revoked.
|
|
718
|
+
*/
|
|
719
|
+
async revokeAll() {
|
|
720
|
+
const result = await this.db.updateTable("peer_credentials").set({ revoked_at: /* @__PURE__ */ new Date() }).where("revoked_at", "is", null).executeTakeFirst();
|
|
721
|
+
return Number(result?.numUpdatedRows ?? 0);
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* List all active (non-revoked, non-expired) credentials.
|
|
725
|
+
*/
|
|
726
|
+
async listActive() {
|
|
727
|
+
return (await this.db.selectFrom("peer_credentials").selectAll().where("revoked_at", "is", null).where("expires_at", ">", /* @__PURE__ */ new Date()).orderBy("created_at", "asc").execute()).map(mapRow);
|
|
728
|
+
}
|
|
729
|
+
};
|
|
730
|
+
/**
|
|
731
|
+
* Map a database row to a PeerCredential interface.
|
|
732
|
+
*/
|
|
733
|
+
function mapRow(row) {
|
|
734
|
+
return {
|
|
735
|
+
id: row.id,
|
|
736
|
+
instanceId: row.instance_id,
|
|
737
|
+
credentialHash: row.credential_hash,
|
|
738
|
+
role: row.role,
|
|
739
|
+
routingKeys: typeof row.routing_keys === "string" ? JSON.parse(row.routing_keys) : row.routing_keys,
|
|
740
|
+
sourceTokenHash: row.source_token_hash ?? null,
|
|
741
|
+
createdAt: new Date(row.created_at),
|
|
742
|
+
lastSeenAt: row.last_seen_at ? new Date(row.last_seen_at) : null,
|
|
743
|
+
lastValidatedBy: row.last_validated_by ?? null,
|
|
744
|
+
expiresAt: new Date(row.expires_at),
|
|
745
|
+
revokedAt: row.revoked_at ? new Date(row.revoked_at) : null
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Construct a PeerCredentialStore bound to a Postgres URL, returning the
|
|
750
|
+
* store plus a disposer that closes the underlying pool. Lets tests
|
|
751
|
+
* exercise the real Kysely-backed store without importing pg/kysely.
|
|
752
|
+
*
|
|
753
|
+
* Intended for e2e coverage of the store's own CRUD methods — production
|
|
754
|
+
* code constructs the store with an already-allocated Kysely instance
|
|
755
|
+
* shared with the orchestrator.
|
|
756
|
+
*/
|
|
757
|
+
async function createPeerCredentialStoreFromUrl(databaseUrl, opts) {
|
|
758
|
+
const db = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({
|
|
759
|
+
connectionString: databaseUrl,
|
|
760
|
+
max: opts?.maxConnections ?? 3
|
|
761
|
+
}) }) });
|
|
762
|
+
return {
|
|
763
|
+
store: new PeerCredentialStore(db),
|
|
764
|
+
dispose: async () => {
|
|
765
|
+
await db.destroy();
|
|
766
|
+
}
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
//#endregion
|
|
770
|
+
export { DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, createCacheStorage, createPeerCredentialStoreFromUrl };
|
|
771
|
+
|
|
772
|
+
//# sourceMappingURL=index.js.map
|