@kici-dev/orchestrator 0.0.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +1 -6
- package/dist/__test-helpers__/mock-db.d.ts +103 -0
- package/dist/__test-helpers__/mock-ws.d.ts +13 -0
- package/dist/agent/dispatcher.d.ts +231 -0
- package/dist/agent/ownership-tracker.d.ts +46 -0
- package/dist/agent/registry.d.ts +255 -0
- package/dist/agent/token-store.d.ts +96 -0
- package/dist/app.d.ts +234 -0
- package/dist/audit/access-log.d.ts +92 -0
- package/dist/audit/sampling-rate-limiter.d.ts +41 -0
- package/dist/cache/build-coordinator.d.ts +42 -0
- package/dist/cache/dep-cache.d.ts +51 -0
- package/dist/cache/index.d.ts +16 -0
- package/dist/cache/pending-builds.d.ts +14 -0
- package/dist/cache/pending-dynamics.d.ts +15 -0
- package/dist/cache/pending-inits.d.ts +19 -0
- package/dist/cache/pending-tracker.d.ts +65 -0
- package/dist/cache/source-cache.d.ts +30 -0
- package/dist/cli/api-client.d.ts +411 -0
- package/dist/cli/commands/access-log.d.ts +30 -0
- package/dist/cli/commands/agent-service/index.d.ts +15 -0
- package/dist/cli/commands/agent-service/install.d.ts +9 -0
- package/dist/cli/commands/agent-service/logs.d.ts +9 -0
- package/dist/cli/commands/agent-service/restart.d.ts +8 -0
- package/dist/cli/commands/agent-service/start.d.ts +8 -0
- package/dist/cli/commands/agent-service/status.d.ts +10 -0
- package/dist/cli/commands/agent-service/stop.d.ts +8 -0
- package/dist/cli/commands/agent-service/uninstall.d.ts +9 -0
- package/dist/cli/commands/agent-service/upgrade.d.ts +11 -0
- package/dist/cli/commands/agent.d.ts +10 -0
- package/dist/cli/commands/api-key.d.ts +12 -0
- package/dist/cli/commands/audit.d.ts +9 -0
- package/dist/cli/commands/backend.d.ts +17 -0
- package/dist/cli/commands/cold-store-impl.d.ts +101 -0
- package/dist/cli/commands/cold-store.d.ts +30 -0
- package/dist/cli/commands/config.d.ts +16 -0
- package/dist/cli/commands/db.d.ts +19 -0
- package/dist/cli/commands/debug-bundle.d.ts +16 -0
- package/dist/cli/commands/diagnose.d.ts +15 -0
- package/dist/cli/commands/environment.d.ts +24 -0
- package/dist/cli/commands/event-dlq.d.ts +19 -0
- package/dist/cli/commands/event-log.d.ts +27 -0
- package/dist/cli/commands/event.d.ts +4 -0
- package/dist/cli/commands/execution.d.ts +17 -0
- package/dist/cli/commands/inspect-bundle.d.ts +16 -0
- package/dist/cli/commands/maintenance.d.ts +4 -0
- package/dist/cli/commands/orchestrator-service/index.d.ts +12 -0
- package/dist/cli/commands/orchestrator-service/install.d.ts +10 -0
- package/dist/cli/commands/orchestrator-service/logs.d.ts +10 -0
- package/dist/cli/commands/orchestrator-service/restart.d.ts +8 -0
- package/dist/cli/commands/orchestrator-service/start.d.ts +8 -0
- package/dist/cli/commands/orchestrator-service/status.d.ts +10 -0
- package/dist/cli/commands/orchestrator-service/stop.d.ts +8 -0
- package/dist/cli/commands/orchestrator-service/uninstall.d.ts +9 -0
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +12 -0
- package/dist/cli/commands/org-settings.d.ts +17 -0
- package/dist/cli/commands/peer.d.ts +13 -0
- package/dist/cli/commands/queue.d.ts +16 -0
- package/dist/cli/commands/registration.d.ts +18 -0
- package/dist/cli/commands/rotate.d.ts +9 -0
- package/dist/cli/commands/runs.d.ts +34 -0
- package/dist/cli/commands/secret.d.ts +12 -0
- package/dist/cli/commands/shared/db.d.ts +16 -0
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +38 -0
- package/dist/cli/commands/source.d.ts +24 -0
- package/dist/cli/commands/token.d.ts +10 -0
- package/dist/cli/commands/workflow.d.ts +31 -0
- package/dist/cli/join.d.ts +14 -0
- package/dist/cli/kici-admin.d.ts +13 -0
- package/dist/cli/lazy-deps/cache.d.ts +17 -0
- package/dist/cli/lazy-deps/downloader.d.ts +33 -0
- package/dist/cli/lazy-deps/registry.d.ts +43 -0
- package/dist/cli/service/compose.d.ts +25 -0
- package/dist/cli/service/index.d.ts +24 -0
- package/dist/cli/service/launchd.d.ts +34 -0
- package/dist/cli/service/platform-detect.d.ts +59 -0
- package/dist/cli/service/systemd.d.ts +41 -0
- package/dist/cli/service/types.d.ts +91 -0
- package/dist/cli/service/windows.d.ts +20 -0
- package/dist/cli/wizard/agent-wizard.d.ts +24 -0
- package/dist/cli/wizard/orchestrator-wizard.d.ts +38 -0
- package/dist/cli/wizard/prompts.d.ts +23 -0
- package/dist/cli.js +11435 -0
- package/dist/cluster/cluster-identity.d.ts +59 -0
- package/dist/cluster/coordinator.d.ts +204 -0
- package/dist/cluster/health-api.d.ts +30 -0
- package/dist/cluster/index.d.ts +24 -0
- package/dist/cluster/join-client.d.ts +75 -0
- package/dist/cluster/join-handler.d.ts +58 -0
- package/dist/cluster/join-token.d.ts +94 -0
- package/dist/cluster/orphan-recovery.d.ts +67 -0
- package/dist/cluster/peer-client.d.ts +190 -0
- package/dist/cluster/peer-credentials.d.ts +108 -0
- package/dist/cluster/peer-crypto.d.ts +57 -0
- package/dist/cluster/peer-handler.d.ts +97 -0
- package/dist/cluster/peer-registry.d.ts +186 -0
- package/dist/cluster/raft-state.d.ts +36 -0
- package/dist/cluster/raft.d.ts +140 -0
- package/dist/cold-store/load-access-log-range.d.ts +73 -0
- package/dist/cold-store/load-event-log-range.d.ts +62 -0
- package/dist/cold-store/load-secret-audit-log-range.d.ts +38 -0
- package/dist/cold-store/orchestrator-cold-store.d.ts +58 -0
- package/dist/cold-store/tables/access-log.d.ts +80 -0
- package/dist/cold-store/tables/event-log.d.ts +70 -0
- package/dist/cold-store/tables/execution-jobs.d.ts +63 -0
- package/dist/cold-store/tables/execution-runs.d.ts +81 -0
- package/dist/cold-store/tables/execution-steps.d.ts +65 -0
- package/dist/cold-store/tables/secret-audit-log.d.ts +77 -0
- package/dist/concurrency/dispatch-next-queued.d.ts +63 -0
- package/dist/concurrency/group-tracker.d.ts +51 -0
- package/dist/concurrency/index.d.ts +3 -0
- package/dist/concurrency/queue-manager.d.ts +68 -0
- package/dist/concurrency/waiters.d.ts +58 -0
- package/dist/config/encryption.d.ts +82 -0
- package/dist/config/env-overlay.d.ts +45 -0
- package/dist/config/index.d.ts +16 -0
- package/dist/config/loader.d.ts +26 -0
- package/dist/config/reload.d.ts +113 -0
- package/dist/config/resolver.d.ts +47 -0
- package/dist/config/schema.d.ts +210 -0
- package/dist/config/shared-store.d.ts +142 -0
- package/dist/config/types.d.ts +218 -0
- package/dist/config.d.ts +249 -0
- package/dist/cron/cron-scheduler.d.ts +78 -0
- package/dist/cron/cron-store.d.ts +38 -0
- package/dist/dashboard/handler.d.ts +243 -0
- package/dist/db/client.d.ts +13 -0
- package/dist/db/migration-provider.d.ts +9 -0
- package/dist/db/migrations/001_initial.d.ts +8 -0
- package/dist/db/migrations/002_config_versions_key_version.d.ts +16 -0
- package/dist/db/migrations/003_access_log.d.ts +26 -0
- package/dist/db/migrations/004_rename_bundle_to_source.d.ts +19 -0
- package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +19 -0
- package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +38 -0
- package/dist/db/migrations/007_audit_logs_archived_at.d.ts +42 -0
- package/dist/db/migrations/008_event_log_archived_at.d.ts +40 -0
- package/dist/db/migrations/009_access_log_trigram.d.ts +15 -0
- package/dist/db/migrations/010_cold_store_chunks.d.ts +34 -0
- package/dist/db/migrations/011_drop_source_secrets_notify.d.ts +21 -0
- package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +28 -0
- package/dist/db/migrations/013_execution_log_bytes.d.ts +21 -0
- package/dist/db/migrations/014_kici_events_lease_retry.d.ts +36 -0
- package/dist/db/migrations/015_org_settings_customer_scoped.d.ts +30 -0
- package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +15 -0
- package/dist/db/migrations/017_org_id_widen.d.ts +4 -0
- package/dist/db/migrations/018_org_id_prefix_backfill.d.ts +4 -0
- package/dist/db/migrator.d.ts +32 -0
- package/dist/db/types.d.ts +1266 -0
- package/dist/diagnostics/bundle-reader.d.ts +48 -0
- package/dist/diagnostics/bundle-writer.d.ts +57 -0
- package/dist/diagnostics/checks/agents.d.ts +10 -0
- package/dist/diagnostics/checks/certs.d.ts +9 -0
- package/dist/diagnostics/checks/config.d.ts +10 -0
- package/dist/diagnostics/checks/db.d.ts +9 -0
- package/dist/diagnostics/checks/disk.d.ts +9 -0
- package/dist/diagnostics/checks/index.d.ts +17 -0
- package/dist/diagnostics/checks/ws.d.ts +9 -0
- package/dist/diagnostics/index.d.ts +13 -0
- package/dist/diagnostics/runner.d.ts +23 -0
- package/dist/diagnostics/types.d.ts +38 -0
- package/dist/entry-helpers.d.ts +93 -0
- package/dist/environments/binding-store.d.ts +27 -0
- package/dist/environments/environment-store.d.ts +74 -0
- package/dist/environments/held-runs.d.ts +64 -0
- package/dist/environments/index.d.ts +10 -0
- package/dist/environments/protection/branch-gate.d.ts +5 -0
- package/dist/environments/protection/concurrency-gate.d.ts +7 -0
- package/dist/environments/protection/pipeline.d.ts +18 -0
- package/dist/environments/protection/reviewer-gate.d.ts +7 -0
- package/dist/environments/protection/trust-gate.d.ts +7 -0
- package/dist/environments/protection/wait-timer-gate.d.ts +7 -0
- package/dist/environments/variable-store.d.ts +39 -0
- package/dist/events/circuit-breaker.d.ts +33 -0
- package/dist/events/event-emitter.d.ts +55 -0
- package/dist/events/event-retry-scanner.d.ts +52 -0
- package/dist/events/event-router.d.ts +183 -0
- package/dist/events/event-store.d.ts +157 -0
- package/dist/events/trust-store.d.ts +54 -0
- package/dist/events/types.d.ts +61 -0
- package/dist/helpers/ip-extraction.d.ts +29 -0
- package/dist/helpers/rate-limiter.d.ts +23 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +772 -0
- package/dist/lockfile-cache.d.ts +39 -0
- package/dist/metrics/agent-metrics-aggregator.d.ts +116 -0
- package/dist/metrics/metrics-reporter.d.ts +40 -0
- package/dist/metrics/prometheus.d.ts +261 -0
- package/dist/metrics/scheduled-jobs.d.ts +20 -0
- package/dist/orchestrator-core.d.ts +193 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +102 -0
- package/dist/pipeline/inline-eval.d.ts +28 -0
- package/dist/pipeline/install-secrets-resolver.d.ts +73 -0
- package/dist/pipeline/manual-schedule.d.ts +21 -0
- package/dist/pipeline/needs-scheduler.d.ts +85 -0
- package/dist/pipeline/process-webhook.d.ts +38 -0
- package/dist/pipeline/processor.d.ts +326 -0
- package/dist/pipeline/rerun.d.ts +79 -0
- package/dist/pipeline/test-pipeline.d.ts +110 -0
- package/dist/provider-registry.d.ts +122 -0
- package/dist/providers/generic/index.d.ts +23 -0
- package/dist/providers/generic/normalizer.d.ts +112 -0
- package/dist/providers/generic/verification.d.ts +47 -0
- package/dist/providers/github/auth.d.ts +60 -0
- package/dist/providers/github/changed-files.d.ts +51 -0
- package/dist/providers/github/check-status-poster.d.ts +35 -0
- package/dist/providers/github/contributor-resolver.d.ts +30 -0
- package/dist/providers/github/index.d.ts +20 -0
- package/dist/providers/github/lock-file.d.ts +32 -0
- package/dist/providers/github/normalizer.d.ts +103 -0
- package/dist/providers/github/repo-url.d.ts +35 -0
- package/dist/providers/internal/index.d.ts +32 -0
- package/dist/providers/internal/lock-file-fetcher.d.ts +34 -0
- package/dist/providers/internal/normalizer.d.ts +89 -0
- package/dist/providers/internal/repo-url-builder.d.ts +42 -0
- package/dist/providers/universal-git/changed-files.d.ts +42 -0
- package/dist/providers/universal-git/clone-token.d.ts +83 -0
- package/dist/providers/universal-git/config.d.ts +192 -0
- package/dist/providers/universal-git/index.d.ts +55 -0
- package/dist/providers/universal-git/lock-file.d.ts +71 -0
- package/dist/providers/universal-git/normalizer.d.ts +90 -0
- package/dist/providers/universal-git/repo-url.d.ts +50 -0
- package/dist/providers/universal-git/ssh-auth.d.ts +94 -0
- package/dist/queue/bootstrap.d.ts +50 -0
- package/dist/queue/cleanup.d.ts +41 -0
- package/dist/queue/depth-refresher.d.ts +65 -0
- package/dist/queue/job-queue.d.ts +343 -0
- package/dist/queue/scheduled-job.d.ts +91 -0
- package/dist/registration/extractor.d.ts +44 -0
- package/dist/registration/registration-index.d.ts +162 -0
- package/dist/registration/registration-store.d.ts +111 -0
- package/dist/reporting/check-run-summary.d.ts +102 -0
- package/dist/reporting/commit-status.d.ts +382 -0
- package/dist/reporting/execution-tracker.d.ts +538 -0
- package/dist/reporting/fs-log-storage.d.ts +41 -0
- package/dist/reporting/log-pull-handler.d.ts +35 -0
- package/dist/reporting/log-storage.d.ts +70 -0
- package/dist/reporting/log-writer.d.ts +45 -0
- package/dist/reporting/s3-log-storage.d.ts +42 -0
- package/dist/reporting/step-log-buffer.d.ts +57 -0
- package/dist/routes/admin-access-log.d.ts +34 -0
- package/dist/routes/admin-backends.d.ts +28 -0
- package/dist/routes/admin-config.d.ts +57 -0
- package/dist/routes/admin-db.d.ts +20 -0
- package/dist/routes/admin-environments.d.ts +42 -0
- package/dist/routes/admin-errors.d.ts +4 -0
- package/dist/routes/admin-event-dlq.d.ts +39 -0
- package/dist/routes/admin-event-log.d.ts +42 -0
- package/dist/routes/admin-events.d.ts +46 -0
- package/dist/routes/admin-maintenance.d.ts +22 -0
- package/dist/routes/admin-org-settings.d.ts +29 -0
- package/dist/routes/admin-queue-execution.d.ts +27 -0
- package/dist/routes/admin-registrations.d.ts +37 -0
- package/dist/routes/admin-runs.d.ts +57 -0
- package/dist/routes/admin-scheduled-jobs.d.ts +36 -0
- package/dist/routes/admin-sources.d.ts +18 -0
- package/dist/routes/admin.d.ts +103 -0
- package/dist/routes/capabilities.d.ts +24 -0
- package/dist/routes/diagnostics.d.ts +16 -0
- package/dist/routes/health.d.ts +20 -0
- package/dist/routes/test-trigger.d.ts +41 -0
- package/dist/routes/uploads.d.ts +38 -0
- package/dist/routes/webhooks.d.ts +36 -0
- package/dist/scaler/bare-metal-backend.d.ts +107 -0
- package/dist/scaler/config.d.ts +303 -0
- package/dist/scaler/container-backend.d.ts +125 -0
- package/dist/scaler/file-tail.d.ts +23 -0
- package/dist/scaler/firecracker-api.d.ts +100 -0
- package/dist/scaler/firecracker-backend.d.ts +263 -0
- package/dist/scaler/index.d.ts +24 -0
- package/dist/scaler/ip-allocator.d.ts +177 -0
- package/dist/scaler/label-matcher.d.ts +73 -0
- package/dist/scaler/log-forwarder.d.ts +54 -0
- package/dist/scaler/machine-ledger.d.ts +160 -0
- package/dist/scaler/manager.d.ts +338 -0
- package/dist/scaler/nftables.d.ts +85 -0
- package/dist/scaler/types.d.ts +388 -0
- package/dist/scaler/warm-pool.d.ts +81 -0
- package/dist/secrets/audit-logger.d.ts +65 -0
- package/dist/secrets/backend-health.d.ts +60 -0
- package/dist/secrets/backend-registry.d.ts +68 -0
- package/dist/secrets/backend-sync.d.ts +40 -0
- package/dist/secrets/cleanup.d.ts +40 -0
- package/dist/secrets/config.d.ts +51 -0
- package/dist/secrets/crypto.d.ts +49 -0
- package/dist/secrets/ephemeral-keys.d.ts +49 -0
- package/dist/secrets/index.d.ts +17 -0
- package/dist/secrets/pg-secret-store.d.ts +105 -0
- package/dist/secrets/rbac.d.ts +47 -0
- package/dist/secrets/secret-output-store.d.ts +46 -0
- package/dist/secrets/secret-resolver.d.ts +137 -0
- package/dist/secrets/source-credentials.d.ts +61 -0
- package/dist/secrets/token-manager.d.ts +63 -0
- package/dist/secrets/vault-secret-store.d.ts +85 -0
- package/dist/security/comment-handler.d.ts +67 -0
- package/dist/security/contributor-cache.d.ts +75 -0
- package/dist/security/global-workflow-policy.d.ts +72 -0
- package/dist/security/lock-source.d.ts +34 -0
- package/dist/security/trust-resolver.d.ts +108 -0
- package/dist/security/workflow-diff.d.ts +26 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.js +48055 -0
- package/dist/sources/index.d.ts +10 -0
- package/dist/sources/source-manager.d.ts +63 -0
- package/dist/sources/source-store.d.ts +79 -0
- package/dist/sources/source-validator.d.ts +16 -0
- package/dist/stale-detector/stale-run-detector.d.ts +109 -0
- package/dist/standalone.d.ts +20 -0
- package/dist/standalone.js +41546 -0
- package/dist/storage/blob-routes.d.ts +20 -0
- package/dist/storage/filesystem.d.ts +60 -0
- package/dist/storage/index.d.ts +20 -0
- package/dist/storage/s3.d.ts +68 -0
- package/dist/storage/sign-url.d.ts +45 -0
- package/dist/storage/types.d.ts +98 -0
- package/dist/webhook/dedup.d.ts +44 -0
- package/dist/webhook/event-log.d.ts +83 -0
- package/dist/webhook/generic-sources.d.ts +169 -0
- package/dist/webhook/handler.d.ts +23 -0
- package/dist/webhook/relay-buffer.d.ts +96 -0
- package/dist/webhook/verify-inbound.d.ts +78 -0
- package/dist/worker/in-memory-execution-tracker.d.ts +94 -0
- package/dist/worker/in-memory-job-queue.d.ts +74 -0
- package/dist/worker/static-agent-token-store.d.ts +58 -0
- package/dist/worker/worker-status.d.ts +36 -0
- package/dist/worker-core.d.ts +74 -0
- package/dist/ws/agent-api-registry.d.ts +49 -0
- package/dist/ws/agent-handler.d.ts +176 -0
- package/dist/ws/agent-heartbeat.d.ts +45 -0
- package/dist/ws/dashboard-backends-handler.d.ts +51 -0
- package/dist/ws/dashboard-diagnostics-handler.d.ts +56 -0
- package/dist/ws/dashboard-env-handler.d.ts +107 -0
- package/dist/ws/dashboard-global-workflows-handler.d.ts +72 -0
- package/dist/ws/dashboard-registrations-handler.d.ts +84 -0
- package/dist/ws/event-buffer.d.ts +16 -0
- package/dist/ws/observer-handler.d.ts +42 -0
- package/dist/ws/observer-registry.d.ts +88 -0
- package/dist/ws/platform-client.d.ts +270 -0
- package/dist/ws/server-options.d.ts +15 -0
- package/package.json +87 -6
- package/sbom.spdx.json +14370 -0
- package/index.js +0 -3
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal Raft leader election implementation.
|
|
3
|
+
*
|
|
4
|
+
* Implements only leader election (no log replication) for cluster coordination.
|
|
5
|
+
* State machine: follower -> candidate -> leader. State is persisted to Postgres
|
|
6
|
+
* via RaftStateStore for crash recovery. Election timeout is 5-10s with randomized
|
|
7
|
+
* jitter (WAN-appropriate per research). Leader sends heartbeats every 2s.
|
|
8
|
+
*
|
|
9
|
+
* Dormant mode: When peerRegistry has 0 connected peers, the node self-elects
|
|
10
|
+
* immediately (single-orchestrator deployment). If peers connect later, normal
|
|
11
|
+
* election resumes.
|
|
12
|
+
*
|
|
13
|
+
* Raft heartbeat (raft.append.entries every 2s) is separate from peer inventory
|
|
14
|
+
* heartbeat (peer.heartbeat every 30s). Raft heartbeat is purely for leader
|
|
15
|
+
* liveness detection.
|
|
16
|
+
*/
|
|
17
|
+
import type { RaftVoteRequest, RaftVoteResponse, RaftAppendEntries } from '@kici-dev/engine';
|
|
18
|
+
import type { PeerRegistry } from './peer-registry.js';
|
|
19
|
+
import type { RaftStateStore } from './raft-state.js';
|
|
20
|
+
export type RaftRole = 'follower' | 'candidate' | 'leader';
|
|
21
|
+
interface RaftNodeDeps {
|
|
22
|
+
instanceId: string;
|
|
23
|
+
stateStore: RaftStateStore;
|
|
24
|
+
peerRegistry: PeerRegistry;
|
|
25
|
+
broadcastToPeers: (msg: RaftVoteRequest | RaftAppendEntries) => void;
|
|
26
|
+
onBecomeLeader: () => void;
|
|
27
|
+
onLoseLeadership: () => void;
|
|
28
|
+
electionTimeoutMinMs?: number;
|
|
29
|
+
electionTimeoutMaxMs?: number;
|
|
30
|
+
leaderHeartbeatMs?: number;
|
|
31
|
+
/** Grace period in ms before dormant-mode self-election. Default: 0 (no grace period). */
|
|
32
|
+
gracePeriodMs?: number;
|
|
33
|
+
}
|
|
34
|
+
export declare class RaftNode {
|
|
35
|
+
private readonly instanceId;
|
|
36
|
+
private readonly stateStore;
|
|
37
|
+
private readonly peerRegistry;
|
|
38
|
+
private readonly broadcastToPeers;
|
|
39
|
+
private readonly onBecomeLeader;
|
|
40
|
+
private readonly onLoseLeadership;
|
|
41
|
+
private readonly electionTimeoutMinMs;
|
|
42
|
+
private readonly electionTimeoutMaxMs;
|
|
43
|
+
private readonly leaderHeartbeatMs;
|
|
44
|
+
private readonly gracePeriodMs;
|
|
45
|
+
private startedAt;
|
|
46
|
+
private currentTerm;
|
|
47
|
+
private votedFor;
|
|
48
|
+
private leaderId;
|
|
49
|
+
private role;
|
|
50
|
+
private electionTimer;
|
|
51
|
+
private leaderHeartbeatTimer;
|
|
52
|
+
/** Votes received during the current election as candidate. */
|
|
53
|
+
private votesReceived;
|
|
54
|
+
/**
|
|
55
|
+
* Consecutive elections where no peer responded with a vote.
|
|
56
|
+
* When this exceeds the threshold, the node force-self-elects as a safety
|
|
57
|
+
* valve against delayed WebSocket close detection (peer appears "connected"
|
|
58
|
+
* but is actually dead).
|
|
59
|
+
*/
|
|
60
|
+
private consecutiveUnansweredElections;
|
|
61
|
+
constructor(deps: RaftNodeDeps);
|
|
62
|
+
/**
|
|
63
|
+
* Start the Raft node. Loads persisted state from DB and starts
|
|
64
|
+
* as follower with election timer.
|
|
65
|
+
*/
|
|
66
|
+
start(): Promise<void>;
|
|
67
|
+
/**
|
|
68
|
+
* Stop the Raft node. Clears all timers and persists state.
|
|
69
|
+
*/
|
|
70
|
+
stop(): Promise<void>;
|
|
71
|
+
/** Whether this node is the current leader. */
|
|
72
|
+
isLeader(): boolean;
|
|
73
|
+
/** Get the current known leader's instance ID. */
|
|
74
|
+
getLeaderId(): string | null;
|
|
75
|
+
/** Get the current Raft term. */
|
|
76
|
+
getCurrentTerm(): number;
|
|
77
|
+
/** Get the current role (for testing/diagnostics). */
|
|
78
|
+
getRole(): RaftRole;
|
|
79
|
+
/**
|
|
80
|
+
* Handle a peer.leaving announcement from a gracefully shutting down peer.
|
|
81
|
+
* If the leaving peer is the current leader, immediately clear leaderId and
|
|
82
|
+
* start a new election. Otherwise, no election action (peer registry update
|
|
83
|
+
* is done by the caller before calling this method).
|
|
84
|
+
*/
|
|
85
|
+
handlePeerLeaving(instanceId: string): void;
|
|
86
|
+
/**
|
|
87
|
+
* Reset the election timer with a randomized timeout.
|
|
88
|
+
* On expiry, starts an election.
|
|
89
|
+
*/
|
|
90
|
+
resetElectionTimer(): void;
|
|
91
|
+
private clearElectionTimer;
|
|
92
|
+
/**
|
|
93
|
+
* Notify Raft that a peer disconnected. If this node is a follower or
|
|
94
|
+
* candidate, reset the election timer to a short value so Raft quickly
|
|
95
|
+
* re-evaluates whether it should self-elect (dormant mode).
|
|
96
|
+
* Without this, the node waits for the full election timeout (3-5s)
|
|
97
|
+
* before discovering all peers are gone.
|
|
98
|
+
*/
|
|
99
|
+
onPeerDisconnected(): void;
|
|
100
|
+
/**
|
|
101
|
+
* Start a new election. Increment term, vote for self, request votes
|
|
102
|
+
* from peers. If no peers are connected, self-elect immediately
|
|
103
|
+
* (dormant mode for single-orchestrator deployments).
|
|
104
|
+
*/
|
|
105
|
+
private startElection;
|
|
106
|
+
/**
|
|
107
|
+
* Handle an incoming vote request from another candidate.
|
|
108
|
+
*/
|
|
109
|
+
handleVoteRequest(msg: RaftVoteRequest): RaftVoteResponse;
|
|
110
|
+
/**
|
|
111
|
+
* Handle a vote response from a peer.
|
|
112
|
+
*/
|
|
113
|
+
handleVoteResponse(msg: RaftVoteResponse): void;
|
|
114
|
+
/**
|
|
115
|
+
* Check if we have a majority of votes to become leader.
|
|
116
|
+
* Cluster size = connected peers + self.
|
|
117
|
+
*/
|
|
118
|
+
private checkMajority;
|
|
119
|
+
/**
|
|
120
|
+
* Transition to leader role. Start heartbeat timer, persist state,
|
|
121
|
+
* call onBecomeLeader callback.
|
|
122
|
+
*/
|
|
123
|
+
private becomeLeader;
|
|
124
|
+
/**
|
|
125
|
+
* Step down from leader/candidate to follower.
|
|
126
|
+
*/
|
|
127
|
+
private stepDown;
|
|
128
|
+
/**
|
|
129
|
+
* Start sending periodic leader heartbeats (raft.append.entries).
|
|
130
|
+
*/
|
|
131
|
+
private startLeaderHeartbeat;
|
|
132
|
+
private sendHeartbeat;
|
|
133
|
+
private clearLeaderHeartbeat;
|
|
134
|
+
/**
|
|
135
|
+
* Handle an incoming append entries (heartbeat) from a leader.
|
|
136
|
+
*/
|
|
137
|
+
handleAppendEntries(msg: RaftAppendEntries): void;
|
|
138
|
+
}
|
|
139
|
+
export {};
|
|
140
|
+
//# sourceMappingURL=raft.d.ts.map
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-through helper for `access_log` (Orchestrator) — Phase D.
|
|
3
|
+
*
|
|
4
|
+
* Unlike `audit_log` and `secret_audit_log`, the `access_log` callers
|
|
5
|
+
* (dashboard "Data access" tab, admin HTTP route, `kici-admin
|
|
6
|
+
* access-log list` CLI) all use the same cursor-based pagination
|
|
7
|
+
* already implemented by `AccessLogWriter.query()`. Phase D's design
|
|
8
|
+
* §7 row 11 classifies access_log as "paginated/transparent" — there
|
|
9
|
+
* is NO `--include-archived` flag. The helper merges cold rows
|
|
10
|
+
* automatically when pagination crosses the warm cutoff or the caller
|
|
11
|
+
* supplies a `from` timestamp older than warm.
|
|
12
|
+
*
|
|
13
|
+
* Cursor extension: the existing cursor encodes `{createdAt, id}` for
|
|
14
|
+
* the hot tuple comparison `(created_at, id) < (cursor.createdAt, cursor.id)`.
|
|
15
|
+
* The cursor also carries a `source: 'hot' | 'cold'` discriminator so a
|
|
16
|
+
* cursor minted while the page boundary was inside cold can resume in
|
|
17
|
+
* the cold stream on the next request. Cursors without a `source` field
|
|
18
|
+
* default to `'hot'` — backwards-compatible with any tab the user had
|
|
19
|
+
* open across a deploy boundary.
|
|
20
|
+
*/
|
|
21
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
22
|
+
import { type ColdStore } from '@kici-dev/shared';
|
|
23
|
+
import { type AccessLogAction, type AccessLogItem, type AccessLogOutcome, type AccessLogSource, type AccessLogTargetType, type ActorType } from '@kici-dev/engine';
|
|
24
|
+
import type { AccessLogTable, Database } from '../db/types.js';
|
|
25
|
+
/**
|
|
26
|
+
* Synthetic tenant placeholder for orchestrator-level rows whose `org_id IS
|
|
27
|
+
* NULL` — collapses every NULL-tenant row under one cold-store prefix so the
|
|
28
|
+
* `(tenant_id, day)` partition scheme still yields a single scan key. Shared
|
|
29
|
+
* across every cold-store reader / adapter touching tables that may emit
|
|
30
|
+
* NULL-tenant rows (`access_log`, `secret_audit_log`); export once here so
|
|
31
|
+
* the four call sites in `cold-store/` and the fallback in
|
|
32
|
+
* `audit/access-log.ts` cannot drift apart.
|
|
33
|
+
*/
|
|
34
|
+
export declare const SYNTHETIC_ORCH_TENANT = "__orchestrator__";
|
|
35
|
+
export type AccessLogColdRow = Selectable<AccessLogTable>;
|
|
36
|
+
export interface LoadAccessLogRangeArgs {
|
|
37
|
+
db: Kysely<Database>;
|
|
38
|
+
coldStore: ColdStore | undefined;
|
|
39
|
+
filter: {
|
|
40
|
+
orgId?: string;
|
|
41
|
+
actorType?: ActorType;
|
|
42
|
+
actorId?: string;
|
|
43
|
+
action?: AccessLogAction;
|
|
44
|
+
source?: AccessLogSource;
|
|
45
|
+
outcome?: AccessLogOutcome;
|
|
46
|
+
targetType?: AccessLogTargetType;
|
|
47
|
+
targetId?: string;
|
|
48
|
+
fromTimestamp?: Date | string;
|
|
49
|
+
toTimestamp?: Date | string;
|
|
50
|
+
/**
|
|
51
|
+
* Full-text search over `error_message`. Trigram-indexed via migration
|
|
52
|
+
* `009_access_log_trigram.ts`. Min ~3 chars for the index to help.
|
|
53
|
+
*/
|
|
54
|
+
q?: string;
|
|
55
|
+
};
|
|
56
|
+
limit: number;
|
|
57
|
+
cursor?: string;
|
|
58
|
+
}
|
|
59
|
+
export interface LoadAccessLogRangeResult {
|
|
60
|
+
items: AccessLogItem[];
|
|
61
|
+
nextCursor: string | null;
|
|
62
|
+
}
|
|
63
|
+
export declare function loadAccessLogRange(args: LoadAccessLogRangeArgs): Promise<LoadAccessLogRangeResult>;
|
|
64
|
+
/**
|
|
65
|
+
* Canonical hot-or-cold `AccessLogRow → AccessLogItem` adapter. Exported
|
|
66
|
+
* because `AccessLogWriter.getById` (in `audit/access-log.ts`) needs to
|
|
67
|
+
* convert both hot (`AccessLogRow`) and cold (`AccessLogColdRow`) rows
|
|
68
|
+
* into the same `AccessLogItem` shape — the two row types are
|
|
69
|
+
* structurally identical (`Selectable<AccessLogTable>`), so a single
|
|
70
|
+
* converter avoids drift.
|
|
71
|
+
*/
|
|
72
|
+
export declare function toAccessLogItem(row: AccessLogColdRow): AccessLogItem;
|
|
73
|
+
//# sourceMappingURL=load-access-log-range.d.ts.map
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-through helper for `event_log` (Orchestrator) — Phase E.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the Platform-side `load-event-log-range.ts`. Tenant column is
|
|
5
|
+
* `routing_key` (NOT NULL on this side) so cold-store partitioning
|
|
6
|
+
* doesn't need a synthetic-tenant fallback. Partition column is
|
|
7
|
+
* `received_at`.
|
|
8
|
+
*
|
|
9
|
+
* The orchestrator's `event_log` row carries the `payload_key` reference
|
|
10
|
+
* (an S3 object that holds the gzipped webhook body). Cold-store
|
|
11
|
+
* archives the row metadata and preserves `payload_key` verbatim — the
|
|
12
|
+
* payload blob itself stays in object storage indefinitely so the
|
|
13
|
+
* dashboard's payload-detail view continues to resolve archived
|
|
14
|
+
* deliveries identically to hot ones.
|
|
15
|
+
*/
|
|
16
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
17
|
+
import { type ColdStore } from '@kici-dev/shared';
|
|
18
|
+
import type { Database, EventLogTable } from '../db/types.js';
|
|
19
|
+
export type EventLogColdStoreRow = Selectable<EventLogTable>;
|
|
20
|
+
export interface LoadOrchEventLogRangeArgs {
|
|
21
|
+
db: Kysely<Database>;
|
|
22
|
+
coldStore: ColdStore | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* Routing key (tenant column on this side). Required when
|
|
25
|
+
* `includeArchived` is true — cold-store fetch is keyed by tenant.
|
|
26
|
+
* The hot list endpoint also typically narrows by routing_key.
|
|
27
|
+
*/
|
|
28
|
+
routingKey: string;
|
|
29
|
+
/** Optional org filter (event_log carries org_id for cross-tier joins). */
|
|
30
|
+
orgId?: string;
|
|
31
|
+
event?: string;
|
|
32
|
+
action?: string;
|
|
33
|
+
status?: string;
|
|
34
|
+
deliveryId?: string;
|
|
35
|
+
fromTs?: Date;
|
|
36
|
+
toTs?: Date;
|
|
37
|
+
limit: number;
|
|
38
|
+
offset: number;
|
|
39
|
+
includeArchived: boolean;
|
|
40
|
+
}
|
|
41
|
+
export declare function loadEventLogRange(args: LoadOrchEventLogRangeArgs): Promise<EventLogColdStoreRow[]>;
|
|
42
|
+
/**
|
|
43
|
+
* Detail lookup by `(orgId, deliveryId)`. The dashboard handler joins
|
|
44
|
+
* cross-tier on this composite key. Hot first; on miss scans cold for
|
|
45
|
+
* the matching tenant. The caller must know `routingKey` to scope the
|
|
46
|
+
* cold scan — when only `orgId+deliveryId` are known, callers fall
|
|
47
|
+
* back to enumerating all routing keys for the org (rare; the dashboard
|
|
48
|
+
* always knows the routing_key from the Platform-side row).
|
|
49
|
+
*/
|
|
50
|
+
export declare function loadEventLogByDeliveryId(args: {
|
|
51
|
+
db: Kysely<Database>;
|
|
52
|
+
coldStore: ColdStore | undefined;
|
|
53
|
+
orgId: string;
|
|
54
|
+
deliveryId: string;
|
|
55
|
+
/**
|
|
56
|
+
* Optional routing-key hint. When omitted, the cold scan iterates
|
|
57
|
+
* every routing_key for the org — bounded but slower. The dashboard
|
|
58
|
+
* always provides this hint from the Platform-side join.
|
|
59
|
+
*/
|
|
60
|
+
routingKey?: string;
|
|
61
|
+
}): Promise<EventLogColdStoreRow | null>;
|
|
62
|
+
//# sourceMappingURL=load-event-log-range.d.ts.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-through helper for `secret_audit_log` (Orchestrator) — Phase D.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the Phase B/C reader pattern. Wraps `AuditLogger.query()`'s
|
|
5
|
+
* existing offset/limit shape; cold-store rows are fetched only when
|
|
6
|
+
* `includeArchived === true`.
|
|
7
|
+
*
|
|
8
|
+
* Design contract: the `kici-admin secret audit` CLI surfaces an
|
|
9
|
+
* explicit `--include-archived` flag; admin HTTP route mirrors via a
|
|
10
|
+
* query param. Default is hot-only so existing tooling doesn't pay an
|
|
11
|
+
* S3 round-trip on every invocation.
|
|
12
|
+
*
|
|
13
|
+
* Cold-side filter handling: any filter that the hot SELECT applies
|
|
14
|
+
* (contextName, action, run_id, ...) is re-applied in-process to cold
|
|
15
|
+
* rows after streaming. This is correct because cold chunks are
|
|
16
|
+
* partitioned by `(routing_key, day)`, not by the secondary filters.
|
|
17
|
+
*/
|
|
18
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
19
|
+
import { type ColdStore } from '@kici-dev/shared';
|
|
20
|
+
import type { Database, SecretAuditLogTable } from '../db/types.js';
|
|
21
|
+
export type SecretAuditLogRow = Selectable<SecretAuditLogTable>;
|
|
22
|
+
export interface LoadSecretAuditLogRangeArgs {
|
|
23
|
+
db: Kysely<Database>;
|
|
24
|
+
coldStore: ColdStore | undefined;
|
|
25
|
+
/** Optional filters mirroring AuditLogger.query(). */
|
|
26
|
+
contextName?: string;
|
|
27
|
+
routingKey?: string;
|
|
28
|
+
action?: string;
|
|
29
|
+
/** Inclusive lower bound on `timestamp`. */
|
|
30
|
+
fromTs?: Date;
|
|
31
|
+
/** Exclusive upper bound on `timestamp`. */
|
|
32
|
+
toTs?: Date;
|
|
33
|
+
limit: number;
|
|
34
|
+
offset: number;
|
|
35
|
+
includeArchived: boolean;
|
|
36
|
+
}
|
|
37
|
+
export declare function loadSecretAuditLogRange(args: LoadSecretAuditLogRangeArgs): Promise<SecretAuditLogRow[]>;
|
|
38
|
+
//# sourceMappingURL=load-secret-audit-log-range.d.ts.map
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestrator-side cold-store.
|
|
3
|
+
*
|
|
4
|
+
* Extends BaseColdStore with orchestrator-specific table adapters.
|
|
5
|
+
* Phase A registered no adapters; Phase C added execution_runs /
|
|
6
|
+
* execution_jobs / execution_steps. Phase D adds secret_audit_log
|
|
7
|
+
* and access_log.
|
|
8
|
+
*/
|
|
9
|
+
import { type Kysely } from 'kysely';
|
|
10
|
+
import { BaseColdStore, type BaseColdStoreDeps, type ColdStoreConfig, type PurgeableChunk } from '@kici-dev/shared';
|
|
11
|
+
import type { Database } from '../db/types.js';
|
|
12
|
+
export interface OrchestratorColdStoreDeps extends Omit<BaseColdStoreDeps, 'db'> {
|
|
13
|
+
/** Kysely handle used by every registered Orchestrator adapter. */
|
|
14
|
+
kdb: Kysely<Database>;
|
|
15
|
+
}
|
|
16
|
+
export declare class OrchestratorColdStore extends BaseColdStore {
|
|
17
|
+
private readonly kdb;
|
|
18
|
+
constructor(deps: OrchestratorColdStoreDeps);
|
|
19
|
+
/**
|
|
20
|
+
* Phase 2 — query the orchestrator's `cold_store_chunks` table for
|
|
21
|
+
* chunks past their per-row cold-retention horizon. See the matching
|
|
22
|
+
* Platform-side method for the full rationale.
|
|
23
|
+
*/
|
|
24
|
+
protected listPurgeableChunks(opts: {
|
|
25
|
+
tableFilter?: string;
|
|
26
|
+
bucketFilter?: string;
|
|
27
|
+
limit: number;
|
|
28
|
+
}): Promise<PurgeableChunk[]>;
|
|
29
|
+
/**
|
|
30
|
+
* Phase 2 — per-chunk advisory lock keyed on
|
|
31
|
+
* `hashtext('cold-store-purge|orchestrator|<table>|<chunkId>')`.
|
|
32
|
+
*/
|
|
33
|
+
protected withPurgeLock<T>(args: {
|
|
34
|
+
table: string;
|
|
35
|
+
chunkId: string;
|
|
36
|
+
}, fn: () => Promise<T>): Promise<T | null>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Build the orchestrator cold-store config from env vars. Reads
|
|
40
|
+
* `KICI_COLD_STORE_*` vars mirrored with the Platform side. YAML
|
|
41
|
+
* integration (per the design doc) lands in a follow-up commit that
|
|
42
|
+
* wires the `coldStore` section of the orchestrator config schema;
|
|
43
|
+
* Phase A ships the env-var path only.
|
|
44
|
+
*/
|
|
45
|
+
export declare function readOrchestratorColdStoreConfig(env?: NodeJS.ProcessEnv): ColdStoreConfig;
|
|
46
|
+
/**
|
|
47
|
+
* Build the cold-store-archive scheduled-job handler for the
|
|
48
|
+
* orchestrator. Re-reads env vars on every tick so SIGHUP-driven
|
|
49
|
+
* config reload picks up new bucket/prefix values without a restart.
|
|
50
|
+
*/
|
|
51
|
+
export declare function createColdStoreArchiveHandler(instanceId: string, kdb: Kysely<Database>): () => Promise<void>;
|
|
52
|
+
/**
|
|
53
|
+
* Phase 2 — build the cold-store-purge scheduled-job handler. Same
|
|
54
|
+
* env-var re-read pattern as the archive handler so SIGHUP picks up
|
|
55
|
+
* config changes mid-run.
|
|
56
|
+
*/
|
|
57
|
+
export declare function createColdStorePurgeHandler(instanceId: string, kdb: Kysely<Database>): () => Promise<void>;
|
|
58
|
+
//# sourceMappingURL=orchestrator-cold-store.d.ts.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `access_log` cold-store adapter (Orchestrator side).
|
|
3
|
+
*
|
|
4
|
+
*
|
|
5
|
+
* Contract:
|
|
6
|
+
* - tenant column: `org_id` (NULL → synthetic `__orchestrator__`)
|
|
7
|
+
* - partition column: `created_at`
|
|
8
|
+
* - warm TTL: 30 days (design §5 matrix row 11)
|
|
9
|
+
*
|
|
10
|
+
* Replaces the previous 90-day `expires_at`-based hard delete (removed
|
|
11
|
+
* by migration 007). Rows older than 30 days now live in S3
|
|
12
|
+
* indefinitely.
|
|
13
|
+
*
|
|
14
|
+
* Synthetic tenant: orchestrator-level access events (e.g. scheduler
|
|
15
|
+
* tick failures, admin-triggered scheduled jobs) carry `org_id IS
|
|
16
|
+
* NULL`; they collapse to the `__orchestrator__` synthetic prefix.
|
|
17
|
+
*
|
|
18
|
+
* Recursion: archiving access_log rows writes a new access_log row
|
|
19
|
+
* recording the archive (one per chunk). The new row is itself
|
|
20
|
+
* eligible for archive in a future cycle. Volume is bounded — one
|
|
21
|
+
* audit row per ~50K-row chunk.
|
|
22
|
+
*/
|
|
23
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
24
|
+
import { type ChunkCommitMetadata, type ColdRetention, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter } from '@kici-dev/shared';
|
|
25
|
+
import type { AccessLogTable, Database } from '../../db/types.js';
|
|
26
|
+
export type AccessLogColdStoreRow = Selectable<AccessLogTable>;
|
|
27
|
+
export interface AccessLogAdapterOptions {
|
|
28
|
+
overrides?: Partial<ColdStoreTableConfig>;
|
|
29
|
+
}
|
|
30
|
+
export declare class AccessLogAdapter implements TableAdapter<AccessLogColdStoreRow> {
|
|
31
|
+
private readonly kdb;
|
|
32
|
+
private readonly instanceId;
|
|
33
|
+
readonly db: "orchestrator";
|
|
34
|
+
readonly table = "access_log";
|
|
35
|
+
readonly tenantColumn = "org_id";
|
|
36
|
+
readonly partitionColumn = "created_at";
|
|
37
|
+
readonly config: ColdStoreTableConfig;
|
|
38
|
+
constructor(kdb: Kysely<Database>, instanceId: string, opts?: AccessLogAdapterOptions);
|
|
39
|
+
listEligiblePartitions(args: {
|
|
40
|
+
warmCutoff: Date;
|
|
41
|
+
}): AsyncIterable<EligiblePartition>;
|
|
42
|
+
countTenantWarmBytes(args: {
|
|
43
|
+
tenantId: string;
|
|
44
|
+
warmCutoff: Date;
|
|
45
|
+
}): Promise<number>;
|
|
46
|
+
withPartitionLock<T>(args: {
|
|
47
|
+
tenantId: string;
|
|
48
|
+
partitionDate: string;
|
|
49
|
+
}, fn: () => Promise<T>): Promise<T | null>;
|
|
50
|
+
selectEligible(args: {
|
|
51
|
+
tenantId: string;
|
|
52
|
+
partitionDate: string;
|
|
53
|
+
limit: number;
|
|
54
|
+
}): AsyncIterable<AccessLogColdStoreRow>;
|
|
55
|
+
encodeRow(row: AccessLogColdStoreRow): string;
|
|
56
|
+
decodeRow(line: string): AccessLogColdStoreRow;
|
|
57
|
+
rowId(row: AccessLogColdStoreRow): string | number;
|
|
58
|
+
rowTimestamp(row: AccessLogColdStoreRow): Date | string;
|
|
59
|
+
/**
|
|
60
|
+
* Phase 2 — per-row cold-retention TTL. Mirrors the warm-side override
|
|
61
|
+
* layering: outcome=denied/error → 730d (forensic), platform_operator →
|
|
62
|
+
* `'forever'` (compliance), otherwise per-action map. The framework
|
|
63
|
+
* groups rows by `coldDaysToBucket(coldTtlDays(row))` at archive time.
|
|
64
|
+
*/
|
|
65
|
+
coldTtlDays(row: AccessLogColdStoreRow): ColdRetention;
|
|
66
|
+
markArchivedAndDelete(args: {
|
|
67
|
+
rowIds: ReadonlyArray<string | number>;
|
|
68
|
+
chunkMeta: ChunkCommitMetadata;
|
|
69
|
+
}): Promise<void>;
|
|
70
|
+
purgeChunkRecord(args: {
|
|
71
|
+
tenantId: string;
|
|
72
|
+
chunkId: string;
|
|
73
|
+
gzipBytes: number;
|
|
74
|
+
rowCount: number;
|
|
75
|
+
bucket: string;
|
|
76
|
+
maxColdDays: number;
|
|
77
|
+
objectKey: string;
|
|
78
|
+
}): Promise<void>;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=access-log.d.ts.map
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `event_log` cold-store adapter (Orchestrator side) —.
|
|
3
|
+
*
|
|
4
|
+
* §2 row 6
|
|
5
|
+
* and §10.
|
|
6
|
+
*
|
|
7
|
+
* Contract:
|
|
8
|
+
* - tenant column: `routing_key` (NOT NULL on this side — no
|
|
9
|
+
* synthetic-tenant fallback like access_log / secret_audit_log)
|
|
10
|
+
* - partition column: `received_at`
|
|
11
|
+
* - warm TTL: 30 days (replaces the previous 30-day hard delete in
|
|
12
|
+
* `packages/orchestrator/src/webhook/event-log.ts:cleanup` +
|
|
13
|
+
* `packages/orchestrator/src/queue/cleanup.ts:runCleanup` step 4)
|
|
14
|
+
*
|
|
15
|
+
* Per-row payload retention: rows on this table carry a `payload_key`
|
|
16
|
+
* pointing at a gzipped webhook body in object storage (LogStorage).
|
|
17
|
+
* Cold-store packages the row metadata — including `payload_key` —
|
|
18
|
+
* but does NOT touch the body the key points to. The body remains in
|
|
19
|
+
* S3 indefinitely so the dashboard delivery-detail page resolves
|
|
20
|
+
* payload reads identically for hot and cold rows. The pre-Phase-E
|
|
21
|
+
* cleanup that deleted both the row and the payload blob in
|
|
22
|
+
* lock-step is retired.
|
|
23
|
+
*
|
|
24
|
+
* Recursive write: archiving event_log rows writes one access_log
|
|
25
|
+
* audit row per chunk (the orchestrator's audit surface). The new
|
|
26
|
+
* access_log rows themselves get archived in a later cycle by the
|
|
27
|
+
* AccessLogAdapter, bounded.
|
|
28
|
+
*/
|
|
29
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
30
|
+
import { type ChunkCommitMetadata, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter } from '@kici-dev/shared';
|
|
31
|
+
import type { Database, EventLogTable } from '../../db/types.js';
|
|
32
|
+
export type EventLogColdStoreRow = Selectable<EventLogTable>;
|
|
33
|
+
export interface EventLogAdapterOptions {
|
|
34
|
+
overrides?: Partial<ColdStoreTableConfig>;
|
|
35
|
+
}
|
|
36
|
+
export declare class EventLogAdapter implements TableAdapter<EventLogColdStoreRow> {
|
|
37
|
+
private readonly kdb;
|
|
38
|
+
private readonly instanceId;
|
|
39
|
+
readonly db: "orchestrator";
|
|
40
|
+
readonly table = "event_log";
|
|
41
|
+
readonly tenantColumn = "routing_key";
|
|
42
|
+
readonly partitionColumn = "received_at";
|
|
43
|
+
readonly config: ColdStoreTableConfig;
|
|
44
|
+
constructor(kdb: Kysely<Database>, instanceId: string, opts?: EventLogAdapterOptions);
|
|
45
|
+
listEligiblePartitions(args: {
|
|
46
|
+
warmCutoff: Date;
|
|
47
|
+
}): AsyncIterable<EligiblePartition>;
|
|
48
|
+
countTenantWarmBytes(args: {
|
|
49
|
+
tenantId: string;
|
|
50
|
+
warmCutoff: Date;
|
|
51
|
+
}): Promise<number>;
|
|
52
|
+
withPartitionLock<T>(args: {
|
|
53
|
+
tenantId: string;
|
|
54
|
+
partitionDate: string;
|
|
55
|
+
}, fn: () => Promise<T>): Promise<T | null>;
|
|
56
|
+
selectEligible(args: {
|
|
57
|
+
tenantId: string;
|
|
58
|
+
partitionDate: string;
|
|
59
|
+
limit: number;
|
|
60
|
+
}): AsyncIterable<EventLogColdStoreRow>;
|
|
61
|
+
encodeRow(row: EventLogColdStoreRow): string;
|
|
62
|
+
decodeRow(line: string): EventLogColdStoreRow;
|
|
63
|
+
rowId(row: EventLogColdStoreRow): string | number;
|
|
64
|
+
rowTimestamp(row: EventLogColdStoreRow): Date | string;
|
|
65
|
+
markArchivedAndDelete(args: {
|
|
66
|
+
rowIds: ReadonlyArray<string | number>;
|
|
67
|
+
chunkMeta: ChunkCommitMetadata;
|
|
68
|
+
}): Promise<void>;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=event-log.d.ts.map
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `execution_jobs` cold-store adapter (Orchestrator side).
|
|
3
|
+
*
|
|
4
|
+
*
|
|
5
|
+
* Contract:
|
|
6
|
+
* - tenant column: `routing_key` (denormalized in migration 006)
|
|
7
|
+
* - partition column: `created_at`
|
|
8
|
+
* - warm TTL: 30 days (design §5 matrix row 8)
|
|
9
|
+
* - eligibility: terminal job status AND no live steps remain for this
|
|
10
|
+
* (run, job)
|
|
11
|
+
*
|
|
12
|
+
* Order: registered AFTER `ExecutionStepsAdapter`, BEFORE
|
|
13
|
+
* `ExecutionRunsAdapter` (within one cycle: steps → jobs → runs). The
|
|
14
|
+
* "no live steps" predicate makes the ordering self-correcting across
|
|
15
|
+
* cycle interruptions.
|
|
16
|
+
*
|
|
17
|
+
* FK ordering on the orchestrator side: `execution_jobs.run_id` is a
|
|
18
|
+
* FK to `execution_runs(run_id)`. Deleting a job is safe from the FK's
|
|
19
|
+
* perspective (the FK only constrains job → run direction). The runs
|
|
20
|
+
* adapter's eligibility predicate handles the dependent-rows check.
|
|
21
|
+
*/
|
|
22
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
23
|
+
import { type ChunkCommitMetadata, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter } from '@kici-dev/shared';
|
|
24
|
+
import type { Database, ExecutionJobTable } from '../../db/types.js';
|
|
25
|
+
export type ExecutionJobRow = Selectable<ExecutionJobTable>;
|
|
26
|
+
export interface ExecutionJobsAdapterOptions {
|
|
27
|
+
overrides?: Partial<ColdStoreTableConfig>;
|
|
28
|
+
}
|
|
29
|
+
export declare class ExecutionJobsAdapter implements TableAdapter<ExecutionJobRow> {
|
|
30
|
+
private readonly kdb;
|
|
31
|
+
private readonly instanceId;
|
|
32
|
+
readonly db: "orchestrator";
|
|
33
|
+
readonly table = "execution_jobs";
|
|
34
|
+
readonly tenantColumn = "routing_key";
|
|
35
|
+
readonly partitionColumn = "created_at";
|
|
36
|
+
readonly config: ColdStoreTableConfig;
|
|
37
|
+
constructor(kdb: Kysely<Database>, instanceId: string, opts?: ExecutionJobsAdapterOptions);
|
|
38
|
+
listEligiblePartitions(args: {
|
|
39
|
+
warmCutoff: Date;
|
|
40
|
+
}): AsyncIterable<EligiblePartition>;
|
|
41
|
+
countTenantWarmBytes(args: {
|
|
42
|
+
tenantId: string;
|
|
43
|
+
warmCutoff: Date;
|
|
44
|
+
}): Promise<number>;
|
|
45
|
+
withPartitionLock<T>(args: {
|
|
46
|
+
tenantId: string;
|
|
47
|
+
partitionDate: string;
|
|
48
|
+
}, fn: () => Promise<T>): Promise<T | null>;
|
|
49
|
+
selectEligible(args: {
|
|
50
|
+
tenantId: string;
|
|
51
|
+
partitionDate: string;
|
|
52
|
+
limit: number;
|
|
53
|
+
}): AsyncIterable<ExecutionJobRow>;
|
|
54
|
+
encodeRow(row: ExecutionJobRow): string;
|
|
55
|
+
decodeRow(line: string): ExecutionJobRow;
|
|
56
|
+
rowId(row: ExecutionJobRow): string | number;
|
|
57
|
+
rowTimestamp(row: ExecutionJobRow): Date | string;
|
|
58
|
+
markArchivedAndDelete(args: {
|
|
59
|
+
rowIds: ReadonlyArray<string | number>;
|
|
60
|
+
chunkMeta: ChunkCommitMetadata;
|
|
61
|
+
}): Promise<void>;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=execution-jobs.d.ts.map
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `execution_runs` cold-store adapter (Orchestrator side).
|
|
3
|
+
*
|
|
4
|
+
*
|
|
5
|
+
* Contract:
|
|
6
|
+
* - tenant column: `routing_key`
|
|
7
|
+
* - partition column: `created_at`
|
|
8
|
+
* - warm TTL: 30 days (design §5 matrix row 7)
|
|
9
|
+
* - eligibility: terminal run status AND no live jobs reference this run
|
|
10
|
+
*
|
|
11
|
+
* Order: registered LAST in the orchestrator cold-store. Within one
|
|
12
|
+
* cycle the framework iterates adapters steps → jobs → runs; this
|
|
13
|
+
* adapter goes last so the FK `execution_jobs.run_id →
|
|
14
|
+
* execution_runs(run_id)` doesn't fire on DELETE. The "no live jobs"
|
|
15
|
+
* predicate enforces the same invariant across cycle interruptions.
|
|
16
|
+
*/
|
|
17
|
+
import { type Kysely, type Selectable } from 'kysely';
|
|
18
|
+
import { type ChunkCommitMetadata, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter } from '@kici-dev/shared';
|
|
19
|
+
import type { Database, ExecutionRunTable } from '../../db/types.js';
|
|
20
|
+
export type ExecutionRunRow = Selectable<ExecutionRunTable>;
|
|
21
|
+
export interface ExecutionRunsAdapterOptions {
|
|
22
|
+
overrides?: Partial<ColdStoreTableConfig>;
|
|
23
|
+
}
|
|
24
|
+
export declare class ExecutionRunsAdapter implements TableAdapter<ExecutionRunRow> {
|
|
25
|
+
private readonly kdb;
|
|
26
|
+
private readonly instanceId;
|
|
27
|
+
readonly db: "orchestrator";
|
|
28
|
+
readonly table = "execution_runs";
|
|
29
|
+
readonly tenantColumn = "routing_key";
|
|
30
|
+
readonly partitionColumn = "created_at";
|
|
31
|
+
readonly config: ColdStoreTableConfig;
|
|
32
|
+
constructor(kdb: Kysely<Database>, instanceId: string, opts?: ExecutionRunsAdapterOptions);
|
|
33
|
+
listEligiblePartitions(args: {
|
|
34
|
+
warmCutoff: Date;
|
|
35
|
+
}): AsyncIterable<EligiblePartition>;
|
|
36
|
+
countTenantWarmBytes(args: {
|
|
37
|
+
tenantId: string;
|
|
38
|
+
warmCutoff: Date;
|
|
39
|
+
}): Promise<number>;
|
|
40
|
+
withPartitionLock<T>(args: {
|
|
41
|
+
tenantId: string;
|
|
42
|
+
partitionDate: string;
|
|
43
|
+
}, fn: () => Promise<T>): Promise<T | null>;
|
|
44
|
+
selectEligible(args: {
|
|
45
|
+
tenantId: string;
|
|
46
|
+
partitionDate: string;
|
|
47
|
+
limit: number;
|
|
48
|
+
}): AsyncIterable<ExecutionRunRow>;
|
|
49
|
+
encodeRow(row: ExecutionRunRow): string;
|
|
50
|
+
decodeRow(line: string): ExecutionRunRow;
|
|
51
|
+
rowId(row: ExecutionRunRow): string | number;
|
|
52
|
+
rowTimestamp(row: ExecutionRunRow): Date | string;
|
|
53
|
+
/**
|
|
54
|
+
* Phase F — natural-key extractor used by `replayRow` so callers
|
|
55
|
+
* (the rerun pipeline) can locate the chunk holding a given UUID
|
|
56
|
+
* `run_id` without knowing the internal SERIAL `id` returned by
|
|
57
|
+
* `rowId()`.
|
|
58
|
+
*/
|
|
59
|
+
replayLookupKey(row: ExecutionRunRow): string;
|
|
60
|
+
/**
|
|
61
|
+
* Phase F — replay a chunk's `execution_runs` rows back into
|
|
62
|
+
* Orchestrator PG. Mirrors the Platform-side replayInsert; the
|
|
63
|
+
* audit row goes to `access_log` (not `audit_log`) because the
|
|
64
|
+
* orchestrator's audit surface is access_log per design §8.
|
|
65
|
+
*
|
|
66
|
+
* Idempotent on duplicate replays via `ON CONFLICT (run_id) DO
|
|
67
|
+
* NOTHING`. Decrements `cold_store_chunk_counts` (floored at 0).
|
|
68
|
+
*/
|
|
69
|
+
replayInsert(args: {
|
|
70
|
+
rows: ReadonlyArray<ExecutionRunRow>;
|
|
71
|
+
chunkMeta: ChunkCommitMetadata;
|
|
72
|
+
}): Promise<{
|
|
73
|
+
inserted: number;
|
|
74
|
+
skipped: number;
|
|
75
|
+
}>;
|
|
76
|
+
markArchivedAndDelete(args: {
|
|
77
|
+
rowIds: ReadonlyArray<string | number>;
|
|
78
|
+
chunkMeta: ChunkCommitMetadata;
|
|
79
|
+
}): Promise<void>;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=execution-runs.d.ts.map
|