@kici-dev/orchestrator 0.1.26 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__test-helpers__/mock-db.d.ts +2 -0
- package/dist/agent/dispatcher.d.ts +52 -2
- package/dist/agent/host-roster-reaper.d.ts +1 -2
- package/dist/agent/host-roster.d.ts +34 -0
- package/dist/agent/ownership-refusal.d.ts +12 -0
- package/dist/agent/ownership-tracker.d.ts +64 -16
- package/dist/agent/token-store.d.ts +13 -0
- package/dist/agent-packaging/availability.d.ts +27 -0
- package/dist/agent-packaging/build-package.d.ts +16 -0
- package/dist/agent-packaging/download.d.ts +30 -0
- package/dist/agent-packaging/index.d.ts +9 -0
- package/dist/agent-packaging/install-closure.d.ts +30 -0
- package/dist/agent-packaging/node-binary.d.ts +26 -0
- package/dist/agent-packaging/refresh.d.ts +50 -0
- package/dist/agent-packaging/store.d.ts +15 -0
- package/dist/agent-packaging/upload.d.ts +24 -0
- package/dist/app-on-error.d.ts +29 -0
- package/dist/app.d.ts +69 -8
- package/dist/approvals/apply-decision.d.ts +1 -1
- package/dist/approvals/step-approval-bridge.d.ts +1 -1
- package/dist/artifacts/artifact-store.d.ts +233 -0
- package/dist/artifacts/failure-messages.d.ts +46 -0
- package/dist/audit/access-log.d.ts +18 -1
- package/dist/cache/dep-cache.d.ts +15 -0
- package/dist/cache/pending-inits.d.ts +4 -4
- package/dist/cache/user-cache.d.ts +37 -0
- package/dist/cancel/cancel-run.d.ts +10 -3
- package/dist/cancel/dashboard-cancel-handler.d.ts +19 -0
- package/dist/cli/api-client.d.ts +51 -5
- package/dist/cli/commands/agent-package.d.ts +34 -0
- package/dist/cli/commands/check-run.d.ts +26 -0
- package/dist/cli/commands/cluster-settings.d.ts +55 -0
- package/dist/cli/commands/context.d.ts +25 -0
- package/dist/cli/commands/dashboard-encryption-key.d.ts +3 -0
- package/dist/cli/commands/db-backup.d.ts +71 -0
- package/dist/cli/commands/debug-bundle.d.ts +1 -1
- package/dist/cli/commands/orchestrator-service/drain.d.ts +31 -0
- package/dist/cli/commands/orchestrator-service/index.d.ts +7 -4
- package/dist/cli/commands/orchestrator-service/install-env.d.ts +58 -0
- package/dist/cli/commands/secret.d.ts +45 -1
- package/dist/cli/commands/shared/admin-cli-access-log.d.ts +46 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +65 -2
- package/dist/cli/commands/signing-key.d.ts +3 -0
- package/dist/cli/commands/source.d.ts +12 -0
- package/dist/cli/commands/token.d.ts +9 -0
- package/dist/cli/commands/trust-policy.d.ts +48 -0
- package/dist/cli/commands/variable.d.ts +6 -6
- package/dist/cli/join.d.ts +1 -1
- package/dist/cli/service/index.d.ts +2 -2
- package/dist/cli/service/instance/manifest.d.ts +17 -0
- package/dist/cli/service/instance/resolve.d.ts +11 -0
- package/dist/cli/service/windows.d.ts +6 -0
- package/dist/cli/wizard/orchestrator-wizard.d.ts +39 -11
- package/dist/cli.js +10263 -3667
- package/dist/cluster/cluster-settings-reader.d.ts +78 -0
- package/dist/cluster/coordinator.d.ts +81 -9
- package/dist/cluster/join-client.d.ts +1 -1
- package/dist/cluster/leader-gated-scheduler.d.ts +50 -0
- package/dist/cluster/orphan-recovery.d.ts +14 -0
- package/dist/cluster/peer-client.d.ts +12 -2
- package/dist/cluster/peer-handler.d.ts +15 -4
- package/dist/cluster/peer-registry.d.ts +12 -0
- package/dist/cluster/verified-issuer-poller.d.ts +11 -0
- package/dist/cluster/verified-issuer.d.ts +63 -0
- package/dist/cold-store/tables/access-log.d.ts +1 -1
- package/dist/cold-store/tables/event-log.d.ts +1 -1
- package/dist/cold-store/tables/execution-jobs.d.ts +1 -1
- package/dist/cold-store/tables/execution-runs.d.ts +1 -1
- package/dist/cold-store/tables/execution-steps.d.ts +1 -1
- package/dist/cold-store/tables/secret-audit-log.d.ts +1 -1
- package/dist/config/loader.d.ts +1 -1
- package/dist/config/resolver.d.ts +2 -1
- package/dist/config/schema.d.ts +8 -6
- package/dist/config/types.d.ts +3 -2
- package/dist/config.d.ts +372 -46
- package/dist/contexts/binding-store.d.ts +36 -0
- package/dist/contexts/context-store.d.ts +100 -0
- package/dist/contexts/glob-specificity.d.ts +30 -0
- package/dist/{environments → contexts}/held-runs.d.ts +64 -32
- package/dist/{environments → contexts}/index.d.ts +1 -1
- package/dist/contexts/protection/aggregate.d.ts +43 -0
- package/dist/contexts/protection/branch-gate.d.ts +5 -0
- package/dist/contexts/protection/concurrency-gate.d.ts +4 -0
- package/dist/{environments → contexts}/protection/pipeline.d.ts +3 -3
- package/dist/contexts/protection/reviewer-gate.d.ts +4 -0
- package/dist/{environments → contexts}/protection/satisfiability.d.ts +22 -22
- package/dist/contexts/protection/trust-gate.d.ts +4 -0
- package/dist/contexts/protection/wait-timer-gate.d.ts +4 -0
- package/dist/contexts/variable-store.d.ts +39 -0
- package/dist/cron/cron-scheduler.d.ts +24 -4
- package/dist/cron/cron-store.d.ts +21 -7
- package/dist/dashboard/attestation-filters.d.ts +94 -74
- package/dist/dashboard/handler.d.ts +63 -3
- package/dist/db/migration-test-harness.d.ts +41 -0
- package/dist/db/migrations/067_environments_to_contexts.d.ts +36 -0
- package/dist/db/migrations/068_request_idempotency.d.ts +17 -0
- package/dist/db/migrations/069_reroute_tunables.d.ts +21 -0
- package/dist/db/migrations/070_execution_runs_failure_class.d.ts +4 -0
- package/dist/db/migrations/071_batch_accumulation.d.ts +26 -0
- package/dist/db/migrations/072_dispatch_queue_run_id_index.d.ts +15 -0
- package/dist/db/migrations/073_org_settings_ingest_concurrency.d.ts +13 -0
- package/dist/db/migrations/074_normalize_zero_concurrency_limit.d.ts +13 -0
- package/dist/db/migrations/075_ingest_overflow_buffer.d.ts +16 -0
- package/dist/db/migrations/076_artifacts.d.ts +19 -0
- package/dist/db/migrations/077_backup_runs.d.ts +13 -0
- package/dist/db/migrations/078_org_settings_backup_staleness.d.ts +12 -0
- package/dist/db/migrations/079_org_settings_scaler_spawn_timeout.d.ts +14 -0
- package/dist/db/migrations/080_cluster_settings.d.ts +17 -0
- package/dist/db/migrations/081_org_settings_queue_timeout.d.ts +19 -0
- package/dist/db/migrations/082_host_s3_reachable.d.ts +16 -0
- package/dist/db/migrations/083_org_settings_artifact_caps.d.ts +18 -0
- package/dist/db/migrations/084_orchestrator_signing_keys.d.ts +20 -0
- package/dist/db/migrations/085_cluster_settings_reroute_flap_grace_ms.d.ts +12 -0
- package/dist/db/migrations/086_cluster_settings_max_fanout_hosts.d.ts +12 -0
- package/dist/db/migrations/087_cluster_settings_event_router_rate_limit.d.ts +12 -0
- package/dist/db/migrations/088_cluster_settings_cache_max_tarball_bytes.d.ts +12 -0
- package/dist/db/migrations/089_cluster_settings_cache_ttl_days.d.ts +12 -0
- package/dist/db/migrations/090_cluster_settings_concurrency_wait_timeout_ms.d.ts +12 -0
- package/dist/db/migrations/091_cluster_settings_agent_token_ttl_ms.d.ts +12 -0
- package/dist/db/migrations/092_cluster_settings_version.d.ts +16 -0
- package/dist/db/migrations/093_org_settings_sandbox_allowlist.d.ts +4 -0
- package/dist/db/migrations/094_dashboard_encryption_keys.d.ts +21 -0
- package/dist/db/migrations/095_dashboard_write_policy_tristate.d.ts +20 -0
- package/dist/db/migrations/096_multi_schedule_cron_last_fired.d.ts +4 -0
- package/dist/db/migrations/097_execution_runs_pr_number.d.ts +4 -0
- package/dist/db/migrations/098_execution_runs_customer_id.d.ts +4 -0
- package/dist/db/migrations/099_cluster_settings_dashboard_verified_issuer.d.ts +21 -0
- package/dist/db/migrations/100_held_runs_hold_type_vocabulary.d.ts +31 -0
- package/dist/db/migrations/101_contexts_hold_expiry_drop_default.d.ts +31 -0
- package/dist/db/migrations/102_dispatch_queue_agent_id.d.ts +20 -0
- package/dist/db/migrations/103_cluster_settings_ownership_db_check_timeout_ms.d.ts +20 -0
- package/dist/db/migrations/104_check_run_terminal_sent.d.ts +22 -0
- package/dist/db/migrations/105_org_trust_policy.d.ts +19 -0
- package/dist/db/migrations/106_cluster_settings_check_run_tracking_ttl_days.d.ts +18 -0
- package/dist/db/migrations/107_check_run_tracking_updated_at_index.d.ts +23 -0
- package/dist/db/repos/dashboard-encryption-keys-repo.d.ts +48 -0
- package/dist/db/repos/signing-keys-repo.d.ts +34 -0
- package/dist/db/types.d.ts +499 -66
- package/dist/deployment/deployment-identity.d.ts +87 -2
- package/dist/diagnostics/checks/backup.d.ts +18 -0
- package/dist/diagnostics/checks/index.d.ts +2 -1
- package/dist/diagnostics/fleet-collector.d.ts +1 -1
- package/dist/drain/drain-controller.d.ts +52 -0
- package/dist/events/batch-accumulator.d.ts +63 -0
- package/dist/events/circuit-breaker.d.ts +9 -3
- package/dist/events/event-emitter.d.ts +31 -0
- package/dist/events/event-retry-scanner.d.ts +12 -2
- package/dist/events/event-router.d.ts +29 -0
- package/dist/firecracker/host-network.d.ts +2 -0
- package/dist/firecracker/net-interfaces.d.ts +14 -0
- package/dist/github-app-name-refresher/github-app-name-refresher.d.ts +19 -9
- package/dist/index.d.ts +2 -0
- package/dist/index.js +625 -22
- package/dist/lockfile-cache.d.ts +4 -0
- package/dist/lockfile-validate.d.ts +16 -9
- package/dist/metrics/prometheus.d.ts +114 -15
- package/dist/oidc/aws-kms-signer.d.ts +37 -0
- package/dist/oidc/command-signer.d.ts +23 -0
- package/dist/oidc/db-signer.d.ts +48 -0
- package/dist/oidc/id-token-claims.d.ts +76 -0
- package/dist/oidc/jwt.d.ts +18 -0
- package/dist/oidc/local-dev-signer.d.ts +31 -0
- package/dist/oidc/local-mint.d.ts +77 -0
- package/dist/oidc/oidc-mint-registration.d.ts +65 -0
- package/dist/oidc/orchestrator-mint.d.ts +97 -0
- package/dist/oidc/orchestrator-signer-factory.d.ts +33 -0
- package/dist/oidc/reconcile-signing-key.d.ts +21 -0
- package/dist/oidc/signer.d.ts +25 -0
- package/dist/oidc/signing-key-status.d.ts +23 -0
- package/dist/orchestrator-core.d.ts +42 -2
- package/dist/pipeline/decorating-secret-resolver.d.ts +5 -4
- package/dist/pipeline/dispatch-matched-workflow.d.ts +44 -3
- package/dist/pipeline/inline-eval.d.ts +2 -2
- package/dist/pipeline/install-secrets-resolver.d.ts +25 -14
- package/dist/pipeline/{job-environments.d.ts → job-contexts.d.ts} +23 -23
- package/dist/pipeline/manual-schedule.d.ts +28 -14
- package/dist/pipeline/pending-workflow-context.d.ts +1 -1
- package/dist/pipeline/process-webhook.d.ts +57 -2
- package/dist/pipeline/processor.d.ts +49 -13
- package/dist/pipeline/request-idempotency.d.ts +28 -0
- package/dist/pipeline/rerun.d.ts +20 -13
- package/dist/pipeline/resolve-sandbox-grant.d.ts +28 -0
- package/dist/pipeline/resume-workflow.d.ts +1 -1
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +71 -0
- package/dist/pipeline/sandbox-allowlist-reader.d.ts +33 -0
- package/dist/pipeline/test-pipeline.d.ts +2 -2
- package/dist/policy/dashboard-write-policy.d.ts +9 -4
- package/dist/provenance/backfill-run.d.ts +1 -0
- package/dist/provenance/trust-root.d.ts +20 -0
- package/dist/provider-registry.d.ts +8 -0
- package/dist/providers/generic/normalizer.d.ts +1 -1
- package/dist/providers/github/auth.d.ts +1 -1
- package/dist/providers/github/changed-files.d.ts +14 -6
- package/dist/providers/github/check-status-poster.d.ts +19 -3
- package/dist/providers/github/contributor-resolver.d.ts +1 -1
- package/dist/providers/github/lock-file.d.ts +1 -1
- package/dist/providers/github/normalizer.d.ts +1 -1
- package/dist/providers/github/repo-url.d.ts +1 -1
- package/dist/providers/local/index.d.ts +1 -0
- package/dist/providers/local/local-source-config.d.ts +1 -0
- package/dist/providers/local/lock-file-fetcher.d.ts +1 -1
- package/dist/providers/local/normalizer.d.ts +1 -1
- package/dist/providers/local/repo-url-builder.d.ts +1 -1
- package/dist/providers/universal-git/changed-files.d.ts +8 -7
- package/dist/providers/universal-git/clone-token.d.ts +1 -1
- package/dist/providers/universal-git/config.d.ts +4 -4
- package/dist/providers/universal-git/index.d.ts +5 -1
- package/dist/providers/universal-git/lock-file.d.ts +6 -1
- package/dist/providers/universal-git/normalizer.d.ts +1 -1
- package/dist/providers/universal-git/repo-url.d.ts +1 -1
- package/dist/queue/cleanup.d.ts +69 -8
- package/dist/queue/job-queue.d.ts +94 -14
- package/dist/queue/scheduled-job.d.ts +2 -2
- package/dist/registration/registration-index.d.ts +1 -1
- package/dist/reporting/agent-failure-category.d.ts +4 -1
- package/dist/reporting/check-run-reporter.d.ts +34 -14
- package/dist/reporting/check-run-tracking-store.d.ts +70 -8
- package/dist/reporting/execution-tracker.d.ts +173 -17
- package/dist/reporting/fs-log-storage.d.ts +31 -2
- package/dist/reporting/job-check-run-completion.d.ts +50 -0
- package/dist/reporting/log-chunk-sink.d.ts +47 -0
- package/dist/reporting/log-retention.d.ts +16 -0
- package/dist/reporting/log-storage.d.ts +64 -4
- package/dist/reporting/log-writer.d.ts +22 -7
- package/dist/reporting/peer-log-normalize.d.ts +14 -0
- package/dist/reporting/run-aggregator.d.ts +36 -36
- package/dist/reporting/s3-log-storage.d.ts +102 -8
- package/dist/routes/admin-auth.d.ts +90 -0
- package/dist/routes/admin-cluster-settings.d.ts +31 -0
- package/dist/routes/admin-contexts.d.ts +47 -0
- package/dist/routes/admin-events.d.ts +3 -0
- package/dist/routes/admin-orchestrator-drain.d.ts +35 -0
- package/dist/routes/admin-registrations.d.ts +2 -2
- package/dist/routes/admin-runs.d.ts +9 -0
- package/dist/routes/admin-sources.d.ts +7 -0
- package/dist/routes/admin-trust-policy.d.ts +49 -0
- package/dist/routes/admin.d.ts +15 -1
- package/dist/routes/github-webhook.d.ts +9 -3
- package/dist/routes/provenance-oidc.d.ts +38 -0
- package/dist/routes/verify-attestation.d.ts +21 -0
- package/dist/scaler/bare-metal-backend.d.ts +9 -2
- package/dist/scaler/config.d.ts +25 -4
- package/dist/scaler/container-backend.d.ts +9 -1
- package/dist/scaler/firecracker-backend.d.ts +29 -2
- package/dist/scaler/index.d.ts +1 -1
- package/dist/scaler/manager.d.ts +166 -8
- package/dist/scaler/pending-scale-sweeper.d.ts +35 -0
- package/dist/scaler/types.d.ts +75 -6
- package/dist/secrets/backend-registry.d.ts +32 -1
- package/dist/secrets/context-secret-resolver.d.ts +30 -0
- package/dist/secrets/dashboard-encryption-key.d.ts +48 -0
- package/dist/secrets/ephemeral-keys.d.ts +16 -0
- package/dist/secrets/index.d.ts +1 -1
- package/dist/secrets/pg-secret-store.d.ts +13 -3
- package/dist/secrets/rbac.d.ts +5 -1
- package/dist/secrets/routing-key-scope.d.ts +6 -0
- package/dist/secrets/scope-routing.d.ts +126 -0
- package/dist/secrets/secret-resolver.d.ts +19 -32
- package/dist/secrets/token-manager.d.ts +5 -1
- package/dist/security/comment-handler.d.ts +1 -1
- package/dist/security/contributor-cache.d.ts +9 -1
- package/dist/security/trust-policy-gate.d.ts +117 -0
- package/dist/security/trust-policy-store.d.ts +78 -0
- package/dist/server.js +44012 -32721
- package/dist/sources/source-manager.d.ts +24 -0
- package/dist/stale-detector/stale-run-detector.d.ts +8 -1
- package/dist/standalone.js +44106 -33690
- package/dist/storage/blob-routes.d.ts +16 -0
- package/dist/storage/filesystem.d.ts +17 -0
- package/dist/storage/key-discriminator.d.ts +20 -0
- package/dist/storage/s3.d.ts +2 -0
- package/dist/storage/types.d.ts +16 -0
- package/dist/webhook/dedup.d.ts +9 -1
- package/dist/webhook/event-log.d.ts +13 -2
- package/dist/webhook/generic-sources-listener.d.ts +4 -0
- package/dist/webhook/ingest-admission.d.ts +140 -0
- package/dist/webhook/ingest-overflow-buffer.d.ts +27 -0
- package/dist/webhook/ingest-overflow-replayer.d.ts +49 -0
- package/dist/webhook/ingest-overflow-types.d.ts +73 -0
- package/dist/webhook/loop-lag-source.d.ts +40 -0
- package/dist/webhook/observed-relay-guard.d.ts +40 -0
- package/dist/webhook/org-ingest-cap-reader.d.ts +24 -0
- package/dist/webhook/register-source-bundle.d.ts +3 -0
- package/dist/webhook/relay-reinject.d.ts +43 -0
- package/dist/worker/in-memory-job-queue.d.ts +8 -3
- package/dist/worker-core.d.ts +10 -0
- package/dist/ws/agent-api-registry.d.ts +20 -0
- package/dist/ws/agent-handler.d.ts +63 -7
- package/dist/ws/bringup-api.d.ts +99 -0
- package/dist/ws/{dashboard-env-handler.d.ts → dashboard-context-handler.d.ts} +43 -25
- package/dist/ws/dashboard-run-state-handler.d.ts +31 -0
- package/dist/ws/failure-messages.d.ts +29 -0
- package/dist/ws/oidc-token-relay.d.ts +15 -0
- package/dist/ws/ownership-gate.d.ts +22 -0
- package/dist/ws/platform-client.d.ts +96 -1
- package/dist/ws/replay-chunker.d.ts +37 -0
- package/installer-image-digests.json +3 -3
- package/package.json +22 -21
- package/sbom.spdx.json +1245 -1793
- package/dist/cli/commands/environment.d.ts +0 -25
- package/dist/environments/binding-store.d.ts +0 -36
- package/dist/environments/environment-store.d.ts +0 -87
- package/dist/environments/protection/aggregate.d.ts +0 -43
- package/dist/environments/protection/branch-gate.d.ts +0 -5
- package/dist/environments/protection/concurrency-gate.d.ts +0 -7
- package/dist/environments/protection/reviewer-gate.d.ts +0 -7
- package/dist/environments/protection/trust-gate.d.ts +0 -7
- package/dist/environments/protection/wait-timer-gate.d.ts +0 -7
- package/dist/environments/variable-store.d.ts +0 -39
- package/dist/routes/admin-environments.d.ts +0 -47
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type CheckRunReporter } from './check-run-reporter.js';
|
|
2
|
+
import type { ExecutionContext } from './execution-tracker.js';
|
|
3
|
+
/**
|
|
4
|
+
* Job-terminal signal this module reacts to. Shaped after the payload the
|
|
5
|
+
* execution tracker hands its `onJobComplete` hook — its primary caller — plus
|
|
6
|
+
* the agent-supplied `data` blob used to build the enriched check-run summary.
|
|
7
|
+
*/
|
|
8
|
+
export interface JobCheckRunCompletionInput {
|
|
9
|
+
runId: string;
|
|
10
|
+
jobId: string;
|
|
11
|
+
jobName: string;
|
|
12
|
+
/** Raw job status string as recorded by the tracker. */
|
|
13
|
+
status: string;
|
|
14
|
+
/** Agent-supplied job payload (step results, error, duration). */
|
|
15
|
+
data?: Record<string, unknown>;
|
|
16
|
+
/**
|
|
17
|
+
* Explicit check-run description, overriding the conclusion mapper's default
|
|
18
|
+
* wording. A caller that already holds the specific reason a job ended the way
|
|
19
|
+
* it did passes it here — the queue-expiry sweep's `unroutable` message, for
|
|
20
|
+
* instance, is the only statement of WHICH `runsOn` selectors went unmatched,
|
|
21
|
+
* and the mapper's generic phrasing cannot name them.
|
|
22
|
+
*/
|
|
23
|
+
description?: string;
|
|
24
|
+
}
|
|
25
|
+
/** Collaborators needed to post a job's terminal check-run conclusion. */
|
|
26
|
+
export interface JobCheckRunCompletionDeps {
|
|
27
|
+
checkRunReporter: Pick<CheckRunReporter, 'updateJobStatus'>;
|
|
28
|
+
/** Resolves the run's provider/repo/sha context — `ExecutionTracker.getExecutionContext`. */
|
|
29
|
+
getExecutionContext: (runId: string) => ExecutionContext | undefined;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Post the terminal conclusion for a job's check run.
|
|
33
|
+
*
|
|
34
|
+
* Every job that reaches a terminal state must resolve the check run that
|
|
35
|
+
* `setPending()` created for it at trigger match. A check run left `queued` is
|
|
36
|
+
* not cosmetic: branch protection that requires it can never be satisfied, and
|
|
37
|
+
* a developer reads it as "still running" indefinitely with nothing to point at.
|
|
38
|
+
*
|
|
39
|
+
* A non-terminal status is a no-op — a check run is completed once, when the job
|
|
40
|
+
* stops.
|
|
41
|
+
*
|
|
42
|
+
* Never throws. Its callers are hot paths whose remaining work matters more than
|
|
43
|
+
* this report: the tracker's terminal-job hook runs inside `onJobStatus`, ahead
|
|
44
|
+
* of the job-complete event emission and the needs/wave scheduler hooks, and the
|
|
45
|
+
* expiry sweep runs it per job inside a loop. `buildJobFailureDescription` reads
|
|
46
|
+
* an agent-supplied payload, so a malformed `stepResults` entry is enough to
|
|
47
|
+
* raise — and a check-run report is not worth stalling a run's scheduling for.
|
|
48
|
+
*/
|
|
49
|
+
export declare function reportJobCheckRunCompletion(deps: JobCheckRunCompletionDeps, input: JobCheckRunCompletionInput): void;
|
|
50
|
+
//# sourceMappingURL=job-check-run-completion.d.ts.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The single place that decides what happens to a step-log chunk.
|
|
3
|
+
*
|
|
4
|
+
* Both ingresses feed this sink: the local agent WebSocket handler and the
|
|
5
|
+
* coordinator's peer handler, which receives chunks relayed by a worker
|
|
6
|
+
* orchestrator. Keeping the policy in one place is what makes the two paths
|
|
7
|
+
* behave identically — a worker-dispatched job's logs are persisted, counted
|
|
8
|
+
* and forwarded exactly like a locally-dispatched one's.
|
|
9
|
+
*
|
|
10
|
+
* The storage key is derived from `executionTracker.getJobName`, the same call
|
|
11
|
+
* `ExecutionTracker.onStepStatus` uses to fill `execution_steps.log_path`, so
|
|
12
|
+
* the reader and the writer cannot disagree about the naming rule.
|
|
13
|
+
*/
|
|
14
|
+
import type { LogStream } from '@kici-dev/engine';
|
|
15
|
+
import type { LogWriter } from './log-writer.js';
|
|
16
|
+
import type { StepLogBuffer } from './step-log-buffer.js';
|
|
17
|
+
/** A chunk whose lines all share one timestamp and one originating stream. */
|
|
18
|
+
export interface NormalizedLogChunk {
|
|
19
|
+
runId: string;
|
|
20
|
+
jobId: string;
|
|
21
|
+
stepIndex: number;
|
|
22
|
+
lines: string[];
|
|
23
|
+
timestamp: number;
|
|
24
|
+
stream?: LogStream;
|
|
25
|
+
}
|
|
26
|
+
/** Which ingress produced the chunk. Stamped onto the two log counters. */
|
|
27
|
+
export type LogChunkSource = 'local' | 'peer';
|
|
28
|
+
export interface LogChunkSinkDeps {
|
|
29
|
+
/** Ingress this sink instance serves; becomes the `source` metric attribute. */
|
|
30
|
+
source: LogChunkSource;
|
|
31
|
+
/** Short in-memory tail feeding the GitHub check-run summary. */
|
|
32
|
+
stepLogBuffer?: StepLogBuffer;
|
|
33
|
+
/** Durable step-log persistence. Absent when the orchestrator has no database. */
|
|
34
|
+
logWriter?: LogWriter;
|
|
35
|
+
/** Resolves the job name that names the storage path. */
|
|
36
|
+
executionTracker?: {
|
|
37
|
+
getJobName(runId: string, jobId: string): string | undefined;
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* Forward to the Platform for browser fan-out. Absent in independent mode.
|
|
41
|
+
* The caller wraps the chunk in the `log.chunk` envelope, which keeps this
|
|
42
|
+
* module free of Platform-protocol knowledge.
|
|
43
|
+
*/
|
|
44
|
+
forwardToPlatform?: (chunk: NormalizedLogChunk) => void;
|
|
45
|
+
}
|
|
46
|
+
export declare function createLogChunkSink(deps: LogChunkSinkDeps): (chunk: NormalizedLogChunk) => void;
|
|
47
|
+
//# sourceMappingURL=log-chunk-sink.d.ts.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { LogStorage } from './log-storage.js';
|
|
2
|
+
/**
|
|
3
|
+
* Delete step-log objects older than `ttlDays` in one bulk pass.
|
|
4
|
+
*
|
|
5
|
+
* Step-log objects are write-once (appended during the run, untouched after
|
|
6
|
+
* finalize), so last-modified time is the correct expiry basis — a recent or
|
|
7
|
+
* still-running run's objects were all written within the window and survive.
|
|
8
|
+
* `ttlDays <= 0` disables the sweep (returns 0). Expired objects are collected
|
|
9
|
+
* then bulk-deleted via `deleteMany` (S3 DeleteObjects), so a large sweep is a
|
|
10
|
+
* handful of API calls, not one-per-object. A delete failure is logged and the
|
|
11
|
+
* cleanup tick is not aborted.
|
|
12
|
+
*
|
|
13
|
+
* @returns Number of objects deleted.
|
|
14
|
+
*/
|
|
15
|
+
export declare function pruneExpiredLogs(storage: LogStorage, ttlDays: number, now?: Date): Promise<number>;
|
|
16
|
+
//# sourceMappingURL=log-retention.d.ts.map
|
|
@@ -2,14 +2,33 @@
|
|
|
2
2
|
* Log storage abstraction for the orchestrator.
|
|
3
3
|
*
|
|
4
4
|
* Provides a pluggable backend interface for persisting execution logs.
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* Retention on the S3 backend is bounded by the orchestrator cleanup sweep
|
|
6
|
+
* (`KICI_STEP_LOG_TTL_DAYS`, default 90; 0 disables it); the filesystem backend
|
|
7
|
+
* leaves lifecycle to the operator.
|
|
7
8
|
*
|
|
8
9
|
* Backends: FilesystemLogStorage, S3LogStorage
|
|
9
10
|
*
|
|
10
11
|
* Log file layout: executions/{runId}/job-{name}/step-{index}.log
|
|
11
12
|
* Format: JSONL (one JSON object per line with timestamp, level, message, metadata)
|
|
12
13
|
*/
|
|
14
|
+
/**
|
|
15
|
+
* Assert that a logical log path is safe to hand to a storage backend.
|
|
16
|
+
*
|
|
17
|
+
* The invariant every backend relies on: a log path is relative, contains no
|
|
18
|
+
* `..` segment, and carries no NUL byte. Both backends call this at their own
|
|
19
|
+
* boundary, so a malformed key is refused identically whichever backend the
|
|
20
|
+
* operator configured, and a new caller is guarded on arrival rather than by
|
|
21
|
+
* remembering to validate.
|
|
22
|
+
*
|
|
23
|
+
* The `..` test is per-segment, not a substring match: a job name such as
|
|
24
|
+
* `job-a..b` is legitimate and is accepted.
|
|
25
|
+
*
|
|
26
|
+
* `FilesystemLogStorage.fullPath()` re-states the same invariant physically,
|
|
27
|
+
* comparing the resolved absolute path against the resolved root. Both checks
|
|
28
|
+
* are lexical: neither resolves symlinks, so a symlink planted inside the log
|
|
29
|
+
* root still redirects a logically-clean path through it.
|
|
30
|
+
*/
|
|
31
|
+
export declare function assertSafeLogPath(path: string): void;
|
|
13
32
|
/**
|
|
14
33
|
* Read result from LogStorage.read().
|
|
15
34
|
* Supports cursor-based pagination for large log files.
|
|
@@ -34,17 +53,54 @@ export interface LogReadOptions {
|
|
|
34
53
|
/**
|
|
35
54
|
* Pluggable log storage interface.
|
|
36
55
|
*
|
|
37
|
-
* All operations are async.
|
|
56
|
+
* All operations are async. Retention is enforced by the cleanup sweep on the
|
|
57
|
+
* S3 backend (see `pruneExpiredLogs`); the filesystem backend is operator-managed.
|
|
38
58
|
*/
|
|
39
59
|
export interface LogStorage {
|
|
40
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Append lines to a log file, immediately durable. Creates the file (and
|
|
62
|
+
* directories) if it doesn't exist. Used by single-shot writers (webhook
|
|
63
|
+
* payloads, rerun payloads, orchestration logs) that write once and expect
|
|
64
|
+
* the content to be readable without any subsequent finalize call.
|
|
65
|
+
*/
|
|
41
66
|
append(path: string, data: string): Promise<void>;
|
|
67
|
+
/**
|
|
68
|
+
* High-frequency append-only log write. Durability is guaranteed by a
|
|
69
|
+
* subsequent finalize(path) (called at run completion). S3 buffers per key
|
|
70
|
+
* and seals immutable segments on a size/time threshold; the filesystem
|
|
71
|
+
* appends immediately (appendFile is already durable).
|
|
72
|
+
*/
|
|
73
|
+
appendStreaming(path: string, data: string): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Seal any buffered-but-unwritten content for a path. S3 flushes its current
|
|
76
|
+
* tail segment; the filesystem is a no-op. Safe on any path — a no-op when
|
|
77
|
+
* nothing is buffered.
|
|
78
|
+
*/
|
|
79
|
+
finalize(path: string): Promise<void>;
|
|
42
80
|
/** Read log file content. Returns { data, cursor, complete } for pagination. */
|
|
43
81
|
read(path: string, options?: LogReadOptions): Promise<LogReadResult>;
|
|
44
82
|
/** Check if a log file exists. */
|
|
45
83
|
exists(path: string): Promise<boolean>;
|
|
46
84
|
/** List files matching a prefix (directory listing). */
|
|
47
85
|
list(prefix: string): Promise<string[]>;
|
|
86
|
+
/**
|
|
87
|
+
* List every physical object under a prefix with its last-modified time,
|
|
88
|
+
* `path` relative to the storage prefix. Unlike {@link LogStorage.list} this
|
|
89
|
+
* does NOT collapse S3 segment objects (`.../seg-NNNNNN`) to their logical
|
|
90
|
+
* step path — the retention sweep must see and delete each physical object,
|
|
91
|
+
* and each segment carries its own last-modified time. Step-log objects are
|
|
92
|
+
* write-once, so last-modified is the correct expiry basis.
|
|
93
|
+
*/
|
|
94
|
+
listWithMetadata(prefix: string): Promise<Array<{
|
|
95
|
+
path: string;
|
|
96
|
+
lastModified: Date;
|
|
97
|
+
}>>;
|
|
98
|
+
/**
|
|
99
|
+
* Bulk-delete objects by their storage-relative path (as returned by
|
|
100
|
+
* {@link LogStorage.listWithMetadata}). S3 uses `DeleteObjects` (≤1000
|
|
101
|
+
* keys/call, chunked); missing keys are no-ops. Returns the count deleted.
|
|
102
|
+
*/
|
|
103
|
+
deleteMany(paths: string[]): Promise<number>;
|
|
48
104
|
}
|
|
49
105
|
/**
|
|
50
106
|
* Configuration for creating a log storage backend.
|
|
@@ -61,6 +117,10 @@ type LogStorageConfig = {
|
|
|
61
117
|
endpoint?: string;
|
|
62
118
|
/** Use path-style access instead of virtual-hosted-style (required for most S3-compatible services) */
|
|
63
119
|
forcePathStyle?: boolean;
|
|
120
|
+
/** Seal a step-log segment once the in-memory buffer reaches this many bytes. */
|
|
121
|
+
segmentFlushBytes: number;
|
|
122
|
+
/** Seal a step-log segment once the oldest buffered byte reaches this age (ms). */
|
|
123
|
+
segmentFlushMs: number;
|
|
64
124
|
};
|
|
65
125
|
/**
|
|
66
126
|
* Factory function to create the appropriate LogStorage backend.
|
|
@@ -5,10 +5,14 @@
|
|
|
5
5
|
* and metadata fields. Lines are written to the standard log path layout:
|
|
6
6
|
* executions/{runId}/job-{name}/step-{index}.log
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
8
|
+
* Step-log writes go through the high-frequency `appendStreaming` verb: the
|
|
9
|
+
* filesystem backend appends immediately (no buffering); the S3 backend buffers
|
|
10
|
+
* per step key and seals immutable append-only segments. `drain(runId)` awaits
|
|
11
|
+
* the in-flight appends and then finalizes each step path so the S3 tail
|
|
12
|
+
* segment is sealed before a reader declares the stream complete.
|
|
10
13
|
*/
|
|
11
14
|
import type { LogStorage } from './log-storage.js';
|
|
15
|
+
import { LogStream } from '@kici-dev/engine';
|
|
12
16
|
import type { ObserverRegistry } from '../ws/observer-registry.js';
|
|
13
17
|
interface LogWriterDeps {
|
|
14
18
|
logStorage: LogStorage;
|
|
@@ -32,13 +36,22 @@ export declare class LogWriter {
|
|
|
32
36
|
* disk, dropping it from a blocking `kici run remote` follow.
|
|
33
37
|
*/
|
|
34
38
|
private readonly pendingAppends;
|
|
39
|
+
/**
|
|
40
|
+
* Log paths written for each run, populated in `appendChunk`. `drain(runId)`
|
|
41
|
+
* finalizes each one so the S3 backend seals its buffered tail segment before
|
|
42
|
+
* a reader declares the stream complete. The filesystem backend's `finalize`
|
|
43
|
+
* is a no-op.
|
|
44
|
+
*/
|
|
45
|
+
private readonly runPaths;
|
|
35
46
|
constructor(deps: LogWriterDeps);
|
|
36
47
|
/**
|
|
37
48
|
* Await every in-flight log append for `runId` that was registered before
|
|
38
|
-
* this call
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
49
|
+
* this call, then seal each step log's buffered tail segment so the durable
|
|
50
|
+
* S3 record is complete. Settles even if an append rejected (errors are
|
|
51
|
+
* already logged by `appendChunk`); the point is ordering, not error
|
|
52
|
+
* propagation. New appends started after this call are not waited on — a
|
|
53
|
+
* terminal run emits no further chunks, so the snapshot taken at call time is
|
|
54
|
+
* complete.
|
|
42
55
|
*/
|
|
43
56
|
drain(runId: string): Promise<void>;
|
|
44
57
|
/**
|
|
@@ -57,8 +70,10 @@ export declare class LogWriter {
|
|
|
57
70
|
* @param timestamp - Timestamp from the agent message (epoch ms)
|
|
58
71
|
* @param jobId - Job ID for observer broadcasting (optional)
|
|
59
72
|
* @param stepName - Step name for observer broadcasting (optional)
|
|
73
|
+
* @param stream - Which subprocess stream the lines came from. Absent means
|
|
74
|
+
* the agent did not report one, which is recorded as `stdout`.
|
|
60
75
|
*/
|
|
61
|
-
appendChunk(runId: string, jobName: string, stepIndex: number, lines: string[], timestamp: number, jobId?: string, stepName?: string): Promise<void>;
|
|
76
|
+
appendChunk(runId: string, jobName: string, stepIndex: number, lines: string[], timestamp: number, jobId?: string, stepName?: string, stream?: LogStream): Promise<void>;
|
|
62
77
|
}
|
|
63
78
|
export {};
|
|
64
79
|
//# sourceMappingURL=log-writer.d.ts.map
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a worker-relayed `peer.log.chunk` into the chunk shape the log sink
|
|
3
|
+
* consumes.
|
|
4
|
+
*
|
|
5
|
+
* The peer frame carries a timestamp and an originating stream per line, while
|
|
6
|
+
* the sink (and `LogWriter.appendChunk` beneath it) takes one of each per
|
|
7
|
+
* chunk. Grouping consecutive lines that agree on both is lossless and keeps
|
|
8
|
+
* line order intact. A worker today gives every line in a chunk the same
|
|
9
|
+
* timestamp and stream, so this yields a single group in practice.
|
|
10
|
+
*/
|
|
11
|
+
import type { PeerLogChunk } from '@kici-dev/engine';
|
|
12
|
+
import type { NormalizedLogChunk } from './log-chunk-sink.js';
|
|
13
|
+
export declare function normalizePeerLogChunk(chunk: PeerLogChunk): NormalizedLogChunk[];
|
|
14
|
+
//# sourceMappingURL=peer-log-normalize.d.ts.map
|
|
@@ -34,12 +34,6 @@ export interface RunDetailStepRow {
|
|
|
34
34
|
}
|
|
35
35
|
/** Map a step row to the dashboard run-detail step shape (epoch-ms timestamps). */
|
|
36
36
|
export declare function mapRunDetailStep(step: RunDetailStepRow): {
|
|
37
|
-
groupId?: string | undefined;
|
|
38
|
-
concurrencyKind?: string | undefined;
|
|
39
|
-
driftSummary?: string | undefined;
|
|
40
|
-
checkOutcome?: string | undefined;
|
|
41
|
-
secretsAccessed: string | null;
|
|
42
|
-
stepType?: string | undefined;
|
|
43
37
|
stepIndex: number;
|
|
44
38
|
stepName: string;
|
|
45
39
|
status: string;
|
|
@@ -48,6 +42,12 @@ export declare function mapRunDetailStep(step: RunDetailStepRow): {
|
|
|
48
42
|
durationMs: number | null;
|
|
49
43
|
exitCode: number | null;
|
|
50
44
|
errorMessage: string | null;
|
|
45
|
+
stepType?: string | undefined;
|
|
46
|
+
secretsAccessed: string | null;
|
|
47
|
+
checkOutcome?: string | undefined;
|
|
48
|
+
driftSummary?: string | undefined;
|
|
49
|
+
concurrencyKind?: string | undefined;
|
|
50
|
+
groupId?: string | undefined;
|
|
51
51
|
};
|
|
52
52
|
/** A run-detail job row as queried from execution_jobs (warm + cold paths). */
|
|
53
53
|
export interface RunDetailJobRow {
|
|
@@ -64,8 +64,8 @@ export interface RunDetailJobRow {
|
|
|
64
64
|
agent_id: string | null;
|
|
65
65
|
error_message: string | null;
|
|
66
66
|
runs_on_labels: unknown;
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
contexts: unknown;
|
|
68
|
+
skipped_contexts: unknown;
|
|
69
69
|
env_warning: string | null;
|
|
70
70
|
outputs: unknown;
|
|
71
71
|
init_failure: unknown;
|
|
@@ -81,32 +81,6 @@ export interface RunDetailJobLookups {
|
|
|
81
81
|
}
|
|
82
82
|
/** Map queried job + step rows into the dashboard run-detail job DTO shape. */
|
|
83
83
|
export declare function buildRunDetailJobs(jobs: RunDetailJobRow[], lookups: RunDetailJobLookups): {
|
|
84
|
-
needs: {
|
|
85
|
-
upstreamName: string;
|
|
86
|
-
runOn: ExecutionJobStatus[];
|
|
87
|
-
}[] | null;
|
|
88
|
-
steps: {
|
|
89
|
-
groupId?: string | undefined;
|
|
90
|
-
concurrencyKind?: string | undefined;
|
|
91
|
-
driftSummary?: string | undefined;
|
|
92
|
-
checkOutcome?: string | undefined;
|
|
93
|
-
secretsAccessed: string | null;
|
|
94
|
-
stepType?: string | undefined;
|
|
95
|
-
stepIndex: number;
|
|
96
|
-
stepName: string;
|
|
97
|
-
status: string;
|
|
98
|
-
startedAt: number | null;
|
|
99
|
-
completedAt: number | null;
|
|
100
|
-
durationMs: number | null;
|
|
101
|
-
exitCode: number | null;
|
|
102
|
-
errorMessage: string | null;
|
|
103
|
-
}[];
|
|
104
|
-
initFailure?: {
|
|
105
|
-
scope: "job" | "run";
|
|
106
|
-
category: "secret_resolution" | "install_secrets" | "lock_resolution" | "build_coordination" | "environment_rules" | "dynamic_eval" | "no_agent" | "matrix_expansion";
|
|
107
|
-
message: string;
|
|
108
|
-
jobName?: string | undefined;
|
|
109
|
-
} | undefined;
|
|
110
84
|
jobId: string;
|
|
111
85
|
jobName: string;
|
|
112
86
|
status: string;
|
|
@@ -121,11 +95,37 @@ export declare function buildRunDetailJobs(jobs: RunDetailJobRow[], lookups: Run
|
|
|
121
95
|
orchestratorId: null;
|
|
122
96
|
errorMessage: string | null;
|
|
123
97
|
runsOnLabels: string[] | null;
|
|
124
|
-
|
|
125
|
-
|
|
98
|
+
contexts: string[] | null;
|
|
99
|
+
skippedContexts: string[] | null;
|
|
126
100
|
envWarning: string | null;
|
|
127
101
|
outputs: any;
|
|
128
102
|
secretOutputKeys: string[] | null;
|
|
103
|
+
initFailure?: {
|
|
104
|
+
scope: "job" | "run";
|
|
105
|
+
category: "approval_misconfig" | "build_coordination" | "context_rules" | "dynamic_eval" | "install_secrets" | "lock_resolution" | "matrix_expansion" | "no_agent" | "sandbox_denied" | "secret_resolution" | "trust_policy";
|
|
106
|
+
message: string;
|
|
107
|
+
jobName?: string | undefined;
|
|
108
|
+
} | undefined;
|
|
109
|
+
needs: {
|
|
110
|
+
upstreamName: string;
|
|
111
|
+
runOn: ExecutionJobStatus[];
|
|
112
|
+
}[] | null;
|
|
113
|
+
steps: {
|
|
114
|
+
stepIndex: number;
|
|
115
|
+
stepName: string;
|
|
116
|
+
status: string;
|
|
117
|
+
startedAt: number | null;
|
|
118
|
+
completedAt: number | null;
|
|
119
|
+
durationMs: number | null;
|
|
120
|
+
exitCode: number | null;
|
|
121
|
+
errorMessage: string | null;
|
|
122
|
+
stepType?: string | undefined;
|
|
123
|
+
secretsAccessed: string | null;
|
|
124
|
+
checkOutcome?: string | undefined;
|
|
125
|
+
driftSummary?: string | undefined;
|
|
126
|
+
concurrencyKind?: string | undefined;
|
|
127
|
+
groupId?: string | undefined;
|
|
128
|
+
}[];
|
|
129
129
|
}[];
|
|
130
130
|
/** A dashboard-shaped run-detail job (epoch-ms timestamps). */
|
|
131
131
|
export type CanonicalRunDetailJob = ReturnType<typeof buildRunDetailJobs>[number];
|
|
@@ -1,14 +1,37 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* S3-backed log storage.
|
|
3
3
|
*
|
|
4
|
-
* Stores execution logs in S3.
|
|
4
|
+
* Stores execution logs in S3. Retention is bounded by the orchestrator cleanup
|
|
5
|
+
* sweep, which bulk-deletes objects older than `KICI_STEP_LOG_TTL_DAYS` (default
|
|
6
|
+
* 90; 0 disables the sweep for operators running their own bucket lifecycle).
|
|
5
7
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Two write verbs with different durability contracts:
|
|
9
|
+
*
|
|
10
|
+
* - append() is an immediately-durable read-modify-write: GetObject the whole
|
|
11
|
+
* object, concatenate, PutObject it back. Correct for single-shot writers
|
|
12
|
+
* (webhook delivery payloads, rerun payloads, orchestration logs) that write
|
|
13
|
+
* once and read back without any finalize — N=1 means no amplification and no
|
|
14
|
+
* race. Its content is a plain single object at objectKey(path).
|
|
15
|
+
* - appendStreaming() is the high-frequency step-log verb. S3 has no native
|
|
16
|
+
* append primitive, so it buffers chunks per step key in memory and seals an
|
|
17
|
+
* immutable `{objectKey(path)}/seg-NNNNNN` object (one PutObject, unique key)
|
|
18
|
+
* once the buffer reaches segmentFlushBytes OR its oldest byte reaches
|
|
19
|
+
* segmentFlushMs. Each byte is written exactly once (no O(N^2) amplification)
|
|
20
|
+
* and the unique seal key means concurrent appends to the same step can never
|
|
21
|
+
* overwrite one another. Durability of the final tail is guaranteed by
|
|
22
|
+
* finalize(path) at run completion.
|
|
23
|
+
*
|
|
24
|
+
* - finalize() seals the remaining tail buffer for a step key.
|
|
25
|
+
* - read() lists sealed segments (plus any legacy/single object written by
|
|
26
|
+
* append()) and streams them via the byte-offset cursor.
|
|
27
|
+
* - exists()/list() are segment-aware: `exists` is true when a single object or
|
|
28
|
+
* any segment is present; `list` collapses segment keys to their logical step
|
|
29
|
+
* path.
|
|
30
|
+
*
|
|
31
|
+
* All appendStreaming/finalize operations for one step key run through a
|
|
32
|
+
* sequential execution chain so the in-memory buffer and seq counter are the
|
|
33
|
+
* authoritative single writer even when the caller fires appends without
|
|
34
|
+
* awaiting.
|
|
12
35
|
*
|
|
13
36
|
* Uses AWS SDK v3 modular imports -- only loaded when S3 storage is configured.
|
|
14
37
|
*/
|
|
@@ -21,22 +44,93 @@ interface S3LogStorageOptions {
|
|
|
21
44
|
endpoint?: string;
|
|
22
45
|
/** Use path-style access instead of virtual-hosted-style (required for most S3-compatible services) */
|
|
23
46
|
forcePathStyle?: boolean;
|
|
47
|
+
/** Seal a step-log segment once the in-memory buffer reaches this many bytes. */
|
|
48
|
+
segmentFlushBytes: number;
|
|
49
|
+
/** Seal a step-log segment once the oldest buffered byte reaches this age (ms). */
|
|
50
|
+
segmentFlushMs: number;
|
|
24
51
|
}
|
|
25
52
|
export declare class S3LogStorage implements LogStorage {
|
|
26
53
|
private readonly client;
|
|
27
54
|
private readonly bucket;
|
|
28
55
|
private readonly prefix;
|
|
56
|
+
private readonly segmentFlushBytes;
|
|
57
|
+
private readonly segmentFlushMs;
|
|
58
|
+
/** Per-key in-memory tail buffers awaiting seal (appendStreaming path). */
|
|
59
|
+
private readonly buffers;
|
|
60
|
+
/** Per-key sequential execution chain (serializes appendStreaming/finalize). */
|
|
61
|
+
private readonly locks;
|
|
29
62
|
constructor(options: S3LogStorageOptions);
|
|
30
|
-
/**
|
|
63
|
+
/**
|
|
64
|
+
* Build the full S3 object key from a log path.
|
|
65
|
+
*
|
|
66
|
+
* The assert is not traversal protection — S3 keys are opaque byte strings,
|
|
67
|
+
* so `..` is a literal segment and this concatenation cannot escape the
|
|
68
|
+
* prefix. It refuses to mint a malformed key at all: `list()` and
|
|
69
|
+
* `listWithMetadata()` rebuild storage-relative paths by slicing the prefix
|
|
70
|
+
* off, so a key carrying `..` or a leading `/` would be handed back to the
|
|
71
|
+
* caller as an odd relative path — and on a filesystem-backed peer that path
|
|
72
|
+
* reaches a resolve. Keep it; it is deliberate.
|
|
73
|
+
*
|
|
74
|
+
* Removals do NOT go through here: see {@link S3LogStorage.deleteKey}.
|
|
75
|
+
*/
|
|
31
76
|
private objectKey;
|
|
77
|
+
/**
|
|
78
|
+
* Object key for a removal, built without the minting assert.
|
|
79
|
+
*
|
|
80
|
+
* `deleteMany` consumes this store's own `listWithMetadata()` output, which
|
|
81
|
+
* reports whatever keys the bucket actually holds — including a legacy
|
|
82
|
+
* malformed one written before `objectKey()` began refusing them. Routing a
|
|
83
|
+
* removal through the minting assert would throw while building the batch,
|
|
84
|
+
* so one such object would abort the whole retention sweep and leave every
|
|
85
|
+
* expired object in place forever. There is no protection to lose: an S3 key
|
|
86
|
+
* is an opaque byte string, so a `..` segment is literal and this
|
|
87
|
+
* concatenation addresses an object inside the prefix or nothing at all.
|
|
88
|
+
*/
|
|
89
|
+
private deleteKey;
|
|
90
|
+
/** Segment object key for a given sequence number (zero-padded to 6 digits). */
|
|
91
|
+
private segKey;
|
|
32
92
|
/** Check if an error is a "not found" error (NoSuchKey, NotFound, 404). */
|
|
33
93
|
private isNotFoundError;
|
|
34
94
|
/** Read the full content of an S3 object as a string. Returns null if not found. */
|
|
35
95
|
private getContent;
|
|
96
|
+
/**
|
|
97
|
+
* Run `fn` after every previously-queued operation for `key` has settled, so
|
|
98
|
+
* all appendStreaming/finalize calls for one step key execute strictly in
|
|
99
|
+
* order. The chain swallows prior errors (they are surfaced to their own
|
|
100
|
+
* caller) so one failed op does not wedge the key.
|
|
101
|
+
*/
|
|
102
|
+
private runExclusive;
|
|
36
103
|
append(path: string, data: string): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Highest existing segment number under a key (for mid-run redeploy
|
|
106
|
+
* continuity), + 1. Returns 0 when no segments exist yet.
|
|
107
|
+
*/
|
|
108
|
+
private nextSeq;
|
|
109
|
+
/** Seal the buffer as the next immutable segment object. */
|
|
110
|
+
private seal;
|
|
111
|
+
appendStreaming(path: string, data: string): Promise<void>;
|
|
112
|
+
finalize(path: string): Promise<void>;
|
|
113
|
+
/**
|
|
114
|
+
* Single-object size (or null) + sealed segments in ascending seq order for a
|
|
115
|
+
* key. The single object is the content written by append() (single-shot
|
|
116
|
+
* writers) or any legacy step log written before the append-only change.
|
|
117
|
+
*/
|
|
118
|
+
private layout;
|
|
119
|
+
/** Fetch an inclusive byte range from one object. */
|
|
120
|
+
private getRange;
|
|
121
|
+
/**
|
|
122
|
+
* Map a global byte cursor + limit onto per-object range reads across an
|
|
123
|
+
* ordered entry list. Returns the reads to perform and the total byte size.
|
|
124
|
+
*/
|
|
125
|
+
private planReads;
|
|
37
126
|
read(path: string, options?: LogReadOptions): Promise<LogReadResult>;
|
|
38
127
|
exists(path: string): Promise<boolean>;
|
|
39
128
|
list(prefix: string): Promise<string[]>;
|
|
129
|
+
listWithMetadata(prefix: string): Promise<Array<{
|
|
130
|
+
path: string;
|
|
131
|
+
lastModified: Date;
|
|
132
|
+
}>>;
|
|
133
|
+
deleteMany(paths: string[]): Promise<number>;
|
|
40
134
|
}
|
|
41
135
|
export {};
|
|
42
136
|
//# sourceMappingURL=s3-log-storage.d.ts.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Bearer-token auth middleware for the orchestrator's admin routes.
|
|
3
|
+
*
|
|
4
|
+
* One factory serves every DB-backed admin router, which is what makes the
|
|
5
|
+
* error boundary universal: a router that forgets to wrap `validate()` cannot
|
|
6
|
+
* exist, because no router writes that call itself.
|
|
7
|
+
*
|
|
8
|
+
* Three outcomes, deliberately distinct:
|
|
9
|
+
* - header absent / not `Bearer <token>` -> 401 AUTH_ERROR.missing
|
|
10
|
+
* - token unknown / revoked / expired -> 401 AUTH_ERROR.invalid
|
|
11
|
+
* - validation could not be completed -> 503 AUTH_ERROR.unavailable
|
|
12
|
+
*
|
|
13
|
+
* The third is the reason this file exists. `validate()` returns `null` for a
|
|
14
|
+
* bad credential, so it only throws when the lookup itself fails (unreachable
|
|
15
|
+
* DB, exhausted pool, failover). That is not the caller's fault and is not a
|
|
16
|
+
* bug in this request: the server cannot authenticate anyone right now, which
|
|
17
|
+
* is what 503 means. It also keeps a DB blip out of the 5xx error-rate signal
|
|
18
|
+
* that alerting reads as an application outage.
|
|
19
|
+
*
|
|
20
|
+
* The boundary fails closed: an unresolvable auth decision is a rejection with
|
|
21
|
+
* a 5xx, never a fall-through to the protected handler.
|
|
22
|
+
*/
|
|
23
|
+
import type { Context, MiddlewareHandler } from 'hono';
|
|
24
|
+
import type { Role } from '../secrets/rbac.js';
|
|
25
|
+
/**
|
|
26
|
+
* The three admin-auth response bodies.
|
|
27
|
+
*
|
|
28
|
+
* A single map rather than string literals repeated across every admin router:
|
|
29
|
+
* the 401 wordings are observed by operators and by E2E assertions, so they are
|
|
30
|
+
* pinned in exactly one place.
|
|
31
|
+
*/
|
|
32
|
+
export declare const AUTH_ERROR: {
|
|
33
|
+
readonly missing: 'Missing authorization';
|
|
34
|
+
readonly invalid: 'Invalid or expired token';
|
|
35
|
+
readonly unavailable: 'Authentication unavailable';
|
|
36
|
+
};
|
|
37
|
+
/** What a successful admin-token validation yields. */
|
|
38
|
+
export interface AuthTokenInfo {
|
|
39
|
+
id: string;
|
|
40
|
+
role: Role;
|
|
41
|
+
routingKey: string | null;
|
|
42
|
+
label: string;
|
|
43
|
+
}
|
|
44
|
+
/** The slice of TokenManager this middleware needs (keeps tests trivial to fake). */
|
|
45
|
+
export interface AuthTokenValidator {
|
|
46
|
+
validate(token: string): Promise<AuthTokenInfo | null>;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The result of resolving one `Authorization` header.
|
|
50
|
+
*
|
|
51
|
+
* Failures carry the status and body rather than a `Response` so a caller that
|
|
52
|
+
* is not a middleware (the inline run-cancel route on the coordinator app) can
|
|
53
|
+
* answer with the identical shape.
|
|
54
|
+
*/
|
|
55
|
+
export type BearerAuthOutcome = {
|
|
56
|
+
ok: true;
|
|
57
|
+
tokenInfo: AuthTokenInfo;
|
|
58
|
+
} | {
|
|
59
|
+
ok: false;
|
|
60
|
+
status: 401 | 503;
|
|
61
|
+
error: string;
|
|
62
|
+
};
|
|
63
|
+
export interface BearerAuthDeps {
|
|
64
|
+
tokenManager: AuthTokenValidator;
|
|
65
|
+
/** Route-family label recorded on the 503 log line (e.g. 'admin-runs'). */
|
|
66
|
+
scope: string;
|
|
67
|
+
/** Overridable for tests; defaults to the module logger. */
|
|
68
|
+
logger?: {
|
|
69
|
+
error(msg: string, meta?: Record<string, unknown>): void;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Resolve one request's `Authorization` header into an authenticated token or
|
|
74
|
+
* a rejection. This is the boundary itself; the middleware below is a thin
|
|
75
|
+
* wrapper so a route that needs the token info inline can share it.
|
|
76
|
+
*
|
|
77
|
+
* @param c - The request context (only the header, method, and path are read).
|
|
78
|
+
* @param deps - Token validator, a scope label for logs, and an optional logger.
|
|
79
|
+
* @returns The token info, or the status + body to answer with.
|
|
80
|
+
*/
|
|
81
|
+
export declare function resolveBearerAuth(c: Context<any>, deps: BearerAuthDeps): Promise<BearerAuthOutcome>;
|
|
82
|
+
/**
|
|
83
|
+
* Build the bearer-auth middleware for one admin route family.
|
|
84
|
+
*
|
|
85
|
+
* @param deps - Token validator, a scope label for logs, and an optional logger.
|
|
86
|
+
* @returns A Hono middleware that authenticates or rejects; it never calls
|
|
87
|
+
* `next()` on an unresolved credential.
|
|
88
|
+
*/
|
|
89
|
+
export declare function createBearerAuthMiddleware(deps: BearerAuthDeps): MiddlewareHandler<any>;
|
|
90
|
+
//# sourceMappingURL=admin-auth.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Admin API routes for cluster-global (fleet-wide) orchestrator tunables.
|
|
3
|
+
*
|
|
4
|
+
* Exposes GET / PATCH `/api/v1/admin/cluster-settings` so `kici-admin
|
|
5
|
+
* cluster-settings` can read and write the single `cluster_settings` row
|
|
6
|
+
* (id='default') without going through the Platform dashboard proxy — the CLI
|
|
7
|
+
* stays operable even when Platform is unavailable.
|
|
8
|
+
*
|
|
9
|
+
* These knobs are fleet-wide, not per-tenant, so the route is deliberately NOT
|
|
10
|
+
* under `/orgs/:customerId/*`. Mutations require the `secret.write` RBAC
|
|
11
|
+
* capability (already granted to admin / owner roles), matching the gating
|
|
12
|
+
* posture used by `admin-org-settings.ts`.
|
|
13
|
+
*/
|
|
14
|
+
import { Hono } from 'hono';
|
|
15
|
+
import { type Kysely } from 'kysely';
|
|
16
|
+
import type { Database } from '../db/types.js';
|
|
17
|
+
import type { RbacEnforcer, Role } from '../secrets/rbac.js';
|
|
18
|
+
interface ClusterSettingsRouteDeps {
|
|
19
|
+
db: Kysely<Database>;
|
|
20
|
+
rbac: RbacEnforcer;
|
|
21
|
+
}
|
|
22
|
+
type AdminEnv = {
|
|
23
|
+
Variables: {
|
|
24
|
+
role: Role;
|
|
25
|
+
userId: string;
|
|
26
|
+
routingKey: string | null;
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
export declare function createClusterSettingsRoutes(deps: ClusterSettingsRouteDeps): Hono<AdminEnv>;
|
|
30
|
+
export {};
|
|
31
|
+
//# sourceMappingURL=admin-cluster-settings.d.ts.map
|