@hivemagnus/magnus-server 0.0.0-stage → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +171 -0
- package/LICENSE +202 -0
- package/README.md +169 -2
- package/agent-version.json +1 -0
- package/dist/ai/HEARTBEAT.md +56 -0
- package/dist/ai/IDENTITY.md +54 -0
- package/dist/ai/MEMORY.md +40 -0
- package/dist/ai/TOOLS.md +538 -0
- package/dist/ai/USER.md +32 -0
- package/dist/cli/args.js +62 -0
- package/dist/cli/env-catalog.js +131 -0
- package/dist/cli/preflight.js +89 -0
- package/dist/cli.js +175 -0
- package/dist/config/cors.js +16 -0
- package/dist/config/env.js +392 -0
- package/dist/config/features.js +16 -0
- package/dist/config/index.js +3 -0
- package/dist/config/listener-security.js +38 -0
- package/dist/config/version.js +31 -0
- package/dist/db/client.js +44 -0
- package/dist/db/generated/prisma/browser.js +18 -0
- package/dist/db/generated/prisma/client.js +35 -0
- package/dist/db/generated/prisma/commonInputTypes.js +11 -0
- package/dist/db/generated/prisma/enums.js +42 -0
- package/dist/db/generated/prisma/internal/class.js +50 -0
- package/dist/db/generated/prisma/internal/prismaNamespace.js +1598 -0
- package/dist/db/generated/prisma/internal/prismaNamespaceBrowser.js +1569 -0
- package/dist/db/generated/prisma/models/A2ATask.js +2 -0
- package/dist/db/generated/prisma/models/AdmissionFence.js +2 -0
- package/dist/db/generated/prisma/models/Agent.js +2 -0
- package/dist/db/generated/prisma/models/AiConversation.js +2 -0
- package/dist/db/generated/prisma/models/AiDashboard.js +2 -0
- package/dist/db/generated/prisma/models/AiMemory.js +2 -0
- package/dist/db/generated/prisma/models/AiMessage.js +2 -0
- package/dist/db/generated/prisma/models/AiScheduledOp.js +2 -0
- package/dist/db/generated/prisma/models/AlertConditionState.js +2 -0
- package/dist/db/generated/prisma/models/AlertRule.js +2 -0
- package/dist/db/generated/prisma/models/AlertViolation.js +2 -0
- package/dist/db/generated/prisma/models/AnomalyEvent.js +2 -0
- package/dist/db/generated/prisma/models/ApplicationObservation.js +2 -0
- package/dist/db/generated/prisma/models/ArdentInstallation.js +2 -0
- package/dist/db/generated/prisma/models/ArdentProfile.js +2 -0
- package/dist/db/generated/prisma/models/ArdentScopeEnrollment.js +2 -0
- package/dist/db/generated/prisma/models/CommandResourceFence.js +2 -0
- package/dist/db/generated/prisma/models/CommandResourceLease.js +2 -0
- package/dist/db/generated/prisma/models/CommandResult.js +2 -0
- package/dist/db/generated/prisma/models/CronJob.js +2 -0
- package/dist/db/generated/prisma/models/CronJobExecution.js +2 -0
- package/dist/db/generated/prisma/models/DeployJob.js +2 -0
- package/dist/db/generated/prisma/models/DriftScan.js +2 -0
- package/dist/db/generated/prisma/models/EngineeringMetricSample.js +2 -0
- package/dist/db/generated/prisma/models/EngineeringRecoveryExercise.js +2 -0
- package/dist/db/generated/prisma/models/ExecutionRun.js +2 -0
- package/dist/db/generated/prisma/models/FilePatchTask.js +2 -0
- package/dist/db/generated/prisma/models/GpuSnapshot.js +2 -0
- package/dist/db/generated/prisma/models/HealthSnapshot.js +2 -0
- package/dist/db/generated/prisma/models/HermesCoreRollout.js +2 -0
- package/dist/db/generated/prisma/models/HermesCoreRolloutHost.js +2 -0
- package/dist/db/generated/prisma/models/HermesCredentialPool.js +2 -0
- package/dist/db/generated/prisma/models/HermesCredentialPoolAttachment.js +2 -0
- package/dist/db/generated/prisma/models/HermesCronJob.js +2 -0
- package/dist/db/generated/prisma/models/HermesHook.js +2 -0
- package/dist/db/generated/prisma/models/HermesInstallation.js +2 -0
- package/dist/db/generated/prisma/models/HermesMcpServer.js +2 -0
- package/dist/db/generated/prisma/models/HermesMcpServerAttachment.js +2 -0
- package/dist/db/generated/prisma/models/HermesMessagingBinding.js +2 -0
- package/dist/db/generated/prisma/models/HermesOperationJob.js +2 -0
- package/dist/db/generated/prisma/models/HermesPluginInstall.js +2 -0
- package/dist/db/generated/prisma/models/HermesProfile.js +2 -0
- package/dist/db/generated/prisma/models/HermesProviderConfig.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryClient.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryInventory.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryReceipt.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryRevision.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryTarget.js +2 -0
- package/dist/db/generated/prisma/models/HermesRole.js +2 -0
- package/dist/db/generated/prisma/models/HermesRoleApplication.js +2 -0
- package/dist/db/generated/prisma/models/HermesSkillAssignment.js +2 -0
- package/dist/db/generated/prisma/models/HermesToolsetPolicy.js +2 -0
- package/dist/db/generated/prisma/models/IncidentLesson.js +2 -0
- package/dist/db/generated/prisma/models/LibraryCategory.js +2 -0
- package/dist/db/generated/prisma/models/LibraryFile.js +2 -0
- package/dist/db/generated/prisma/models/LogEntry.js +2 -0
- package/dist/db/generated/prisma/models/MagnusReleaseCanary.js +2 -0
- package/dist/db/generated/prisma/models/MaintenanceWindow.js +2 -0
- package/dist/db/generated/prisma/models/ManagedCronDefinition.js +2 -0
- package/dist/db/generated/prisma/models/ManagedCronOccurrence.js +2 -0
- package/dist/db/generated/prisma/models/Node.js +2 -0
- package/dist/db/generated/prisma/models/Notification.js +2 -0
- package/dist/db/generated/prisma/models/NotificationPreference.js +2 -0
- package/dist/db/generated/prisma/models/RemediationExecution.js +2 -0
- package/dist/db/generated/prisma/models/RemediationRule.js +2 -0
- package/dist/db/generated/prisma/models/RemoteCommand.js +2 -0
- package/dist/db/generated/prisma/models/ResourceDrainParticipant.js +2 -0
- package/dist/db/generated/prisma/models/ResourceEnrollment.js +2 -0
- package/dist/db/generated/prisma/models/ResourceParticipantWork.js +2 -0
- package/dist/db/generated/prisma/models/Role.js +2 -0
- package/dist/db/generated/prisma/models/RoleApplication.js +2 -0
- package/dist/db/generated/prisma/models/RolloutTarget.js +2 -0
- package/dist/db/generated/prisma/models/SecurityExportOutbox.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionAction.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionActionApproval.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCase.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseActivity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseEvent.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseRelationship.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCollector.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCommandOutbox.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionDeletionReceipt.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionDelivery.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEntityAlias.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEvent.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEventEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEvidence.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionInvestigation.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionNonce.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionReceiptChainHead.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionRuleVersion.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionSourceState.js +2 -0
- package/dist/db/generated/prisma/models/SecurityServiceState.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceFence.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceLease.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceLeaseResource.js +2 -0
- package/dist/db/generated/prisma/models/SkillInstallation.js +2 -0
- package/dist/db/generated/prisma/models/SkillRegistryEntry.js +2 -0
- package/dist/db/generated/prisma/models/SkillTag.js +2 -0
- package/dist/db/generated/prisma/models/SkillVersion.js +2 -0
- package/dist/db/generated/prisma/models/TeamApprovalDecision.js +2 -0
- package/dist/db/generated/prisma/models/User.js +2 -0
- package/dist/db/generated/prisma/models/WebhookDelivery.js +2 -0
- package/dist/db/generated/prisma/models/WebhookEndpoint.js +2 -0
- package/dist/db/generated/prisma/models.js +2 -0
- package/dist/db/index.js +3 -0
- package/dist/db/read-retry.js +35 -0
- package/dist/db/seed.js +348 -0
- package/dist/index.js +162 -0
- package/dist/middleware/auth.js +361 -0
- package/dist/middleware/body-limit.js +14 -0
- package/dist/middleware/correlation-id.js +13 -0
- package/dist/middleware/error-handler.js +28 -0
- package/dist/middleware/iam-token.js +21 -0
- package/dist/middleware/rate-limit.js +78 -0
- package/dist/middleware/request-logger.js +85 -0
- package/dist/middleware/security-headers.js +23 -0
- package/dist/middleware/tracing.js +120 -0
- package/dist/middleware/user-status.js +33 -0
- package/dist/openapi/spec.js +2656 -0
- package/dist/queue/redis.js +20 -0
- package/dist/queue/requeue.js +27 -0
- package/dist/queue/workers.js +342 -0
- package/dist/routes/ai-schedule.js +73 -0
- package/dist/routes/alerts.js +79 -0
- package/dist/routes/anomalies.js +101 -0
- package/dist/routes/ardent.js +82 -0
- package/dist/routes/assistant.js +417 -0
- package/dist/routes/audit.js +138 -0
- package/dist/routes/codex.js +65 -0
- package/dist/routes/commands.js +226 -0
- package/dist/routes/credentials.js +58 -0
- package/dist/routes/cron.js +292 -0
- package/dist/routes/dependencies.js +49 -0
- package/dist/routes/deploy.js +18 -0
- package/dist/routes/discovery.js +168 -0
- package/dist/routes/docs.js +24 -0
- package/dist/routes/drift.js +126 -0
- package/dist/routes/events-stream.js +87 -0
- package/dist/routes/files.js +214 -0
- package/dist/routes/fleet-checks.js +21 -0
- package/dist/routes/health.js +133 -0
- package/dist/routes/heartbeat.js +545 -0
- package/dist/routes/hermes-registry.js +278 -0
- package/dist/routes/hermes.js +61 -0
- package/dist/routes/library.js +402 -0
- package/dist/routes/logs.js +172 -0
- package/dist/routes/managed-cron.js +23 -0
- package/dist/routes/mcp.js +39 -0
- package/dist/routes/metrics.js +187 -0
- package/dist/routes/nodes.js +951 -0
- package/dist/routes/notifications.js +130 -0
- package/dist/routes/ops.js +258 -0
- package/dist/routes/orchestra.js +430 -0
- package/dist/routes/predictions.js +23 -0
- package/dist/routes/profiling.js +68 -0
- package/dist/routes/quick-command.js +72 -0
- package/dist/routes/remediation.js +111 -0
- package/dist/routes/resource-leases.js +297 -0
- package/dist/routes/roles.js +370 -0
- package/dist/routes/rollout.js +124 -0
- package/dist/routes/scan.js +116 -0
- package/dist/routes/security-backup.js +58 -0
- package/dist/routes/security-fusion.js +605 -0
- package/dist/routes/security-integration.js +71 -0
- package/dist/routes/self-update.js +129 -0
- package/dist/routes/shared-resource-leases.js +78 -0
- package/dist/routes/skills.js +485 -0
- package/dist/routes/stats.js +96 -0
- package/dist/routes/team-approvals.js +23 -0
- package/dist/routes/terminal-ticket.js +15 -0
- package/dist/routes/terminal.js +17 -0
- package/dist/routes/topology.js +37 -0
- package/dist/routes/users.js +219 -0
- package/dist/routes/webhooks.js +137 -0
- package/dist/schemas/command-reconciliation.js +14 -0
- package/dist/schemas/commands.js +51 -0
- package/dist/schemas/cron.js +33 -0
- package/dist/schemas/deploy.js +20 -0
- package/dist/schemas/files.js +14 -0
- package/dist/schemas/hermes.js +69 -0
- package/dist/schemas/index.js +11 -0
- package/dist/schemas/library.js +48 -0
- package/dist/schemas/logs.js +35 -0
- package/dist/schemas/managed-cron.js +127 -0
- package/dist/schemas/node-responses.js +87 -0
- package/dist/schemas/node-session-receipt.js +84 -0
- package/dist/schemas/nodes.js +36 -0
- package/dist/schemas/orchestra.js +118 -0
- package/dist/schemas/resource-participant-reconciliation.js +99 -0
- package/dist/schemas/roles.js +54 -0
- package/dist/schemas/shared-resource-lease.js +132 -0
- package/dist/schemas/skills.js +44 -0
- package/dist/schemas/users.js +28 -0
- package/dist/scripts/run-disposable-recovery-exercise.js +39 -0
- package/dist/sdk/index.js +2 -0
- package/dist/sdk/node-api-client.js +474 -0
- package/dist/security/approval-assertion.js +66 -0
- package/dist/security/command-completion-registry.js +35 -0
- package/dist/security/command-request.js +12 -0
- package/dist/security/command-result-signature.js +34 -0
- package/dist/security/managed-http.js +45 -0
- package/dist/security/native-completion-registry.js +44 -0
- package/dist/server.js +178 -0
- package/dist/services/admission-fence.js +71 -0
- package/dist/services/ai-assistant-service.js +2523 -0
- package/dist/services/ai-correlation-service.js +225 -0
- package/dist/services/ai-export-service.js +141 -0
- package/dist/services/ai-insights-service.js +219 -0
- package/dist/services/ai-schedule-service.js +145 -0
- package/dist/services/ai-scratchpad-service.js +222 -0
- package/dist/services/ai-tools.js +1343 -0
- package/dist/services/alert-service.js +313 -0
- package/dist/services/anomaly-detector.js +419 -0
- package/dist/services/api-local-work.js +79 -0
- package/dist/services/application-probes.js +309 -0
- package/dist/services/ardent-scope-enrollment.js +123 -0
- package/dist/services/ardent-service.js +86 -0
- package/dist/services/audit-service.js +308 -0
- package/dist/services/codex-service.js +209 -0
- package/dist/services/command-lifecycle.js +60 -0
- package/dist/services/command-resource-leases.js +305 -0
- package/dist/services/control-host-node.js +242 -0
- package/dist/services/credential-rotation-service.js +294 -0
- package/dist/services/dependency-service.js +145 -0
- package/dist/services/disposable-recovery-runner.js +595 -0
- package/dist/services/docker-group-health.js +121 -0
- package/dist/services/engineering-assurance.js +492 -0
- package/dist/services/engineering-evidence-ledger.js +94 -0
- package/dist/services/engineering-metric-evidence.js +26 -0
- package/dist/services/event-bus.js +227 -0
- package/dist/services/fleet-check-log.js +102 -0
- package/dist/services/forecast-service.js +154 -0
- package/dist/services/health-retention.js +175 -0
- package/dist/services/heartbeat-stream.js +63 -0
- package/dist/services/hermes-catalog-import.js +62 -0
- package/dist/services/hermes-cli-parser.js +671 -0
- package/dist/services/hermes-core-rollout.js +90 -0
- package/dist/services/hermes-dashboard-endpoint.js +13 -0
- package/dist/services/hermes-plugin-source.js +149 -0
- package/dist/services/hermes-registry-service.js +258 -0
- package/dist/services/hermes-service.js +566 -0
- package/dist/services/hivemem-client.js +85 -0
- package/dist/services/host-local-writer.js +82 -0
- package/dist/services/incident-evidence.js +110 -0
- package/dist/services/incident-lessons.js +186 -0
- package/dist/services/llm-providers/anthropic.js +143 -0
- package/dist/services/llm-providers/index.js +7 -0
- package/dist/services/llm-providers/llama-cpp.js +237 -0
- package/dist/services/llm-providers/ollama.js +292 -0
- package/dist/services/llm-providers/openai-compatible.js +200 -0
- package/dist/services/llm-providers/provider-error.js +36 -0
- package/dist/services/llm-providers/registry.js +45 -0
- package/dist/services/llm-providers/types.js +2 -0
- package/dist/services/log-emitter.js +19 -0
- package/dist/services/log-retention.js +66 -0
- package/dist/services/maintenance-service.js +29 -0
- package/dist/services/managed-cron.js +493 -0
- package/dist/services/managed-ssh-admission.js +68 -0
- package/dist/services/monitor-pulse.js +10 -0
- package/dist/services/node-issue-monitor.js +71 -0
- package/dist/services/node-service.js +35 -0
- package/dist/services/notification-retention.js +51 -0
- package/dist/services/notification-service.js +94 -0
- package/dist/services/operational-observations.js +112 -0
- package/dist/services/operations-config.js +67 -0
- package/dist/services/ops-service.js +491 -0
- package/dist/services/orchestra-extras-service.js +370 -0
- package/dist/services/orchestra-resources-service.js +530 -0
- package/dist/services/os-scheduler-writer.js +73 -0
- package/dist/services/password-service.js +22 -0
- package/dist/services/prediction-service.js +284 -0
- package/dist/services/recovery-exercises.js +682 -0
- package/dist/services/remediation-service.js +325 -0
- package/dist/services/resource-enrollment.js +433 -0
- package/dist/services/resource-node-lock.js +18 -0
- package/dist/services/resource-participant-work.js +302 -0
- package/dist/services/role-service.js +388 -0
- package/dist/services/rollout-service.js +440 -0
- package/dist/services/security-export.js +185 -0
- package/dist/services/security-fusion-actions.js +22 -0
- package/dist/services/security-fusion-approvals.js +83 -0
- package/dist/services/security-fusion-evidence-integrity.js +144 -0
- package/dist/services/security-fusion-ingest.js +101 -0
- package/dist/services/security-fusion-outbox.js +185 -0
- package/dist/services/security-fusion-receipt-chain.js +134 -0
- package/dist/services/security-fusion-service.js +751 -0
- package/dist/services/security-fusion-verification.js +50 -0
- package/dist/services/self-update-service.js +200 -0
- package/dist/services/shared-resource-leases.js +413 -0
- package/dist/services/skill-publication-bundle.js +90 -0
- package/dist/services/skill-publication-settlement.js +36 -0
- package/dist/services/skill-publication-state.js +40 -0
- package/dist/services/skill-registry-service.js +327 -0
- package/dist/services/skill-service.js +148 -0
- package/dist/services/ssh-exec-service.js +139 -0
- package/dist/services/ssh-host-verification.js +27 -0
- package/dist/services/tailscale-node-key-service.js +75 -0
- package/dist/services/team-approvals.js +118 -0
- package/dist/services/topology-service.js +138 -0
- package/dist/services/transfer-authority-snapshot.js +151 -0
- package/dist/services/webhook-credential-migration.js +97 -0
- package/dist/services/webhook-security.js +242 -0
- package/dist/services/webhook-service.js +494 -0
- package/dist/types/auth.js +3 -0
- package/dist/types/index.js +3 -0
- package/dist/types/library.js +3 -0
- package/dist/types/notifications.js +3 -0
- package/dist/types/roles.js +3 -0
- package/dist/types/shared-contracts.js +3 -0
- package/dist/utils/env.js +141 -0
- package/dist/utils/heartbeat-peer.js +19 -0
- package/dist/utils/idempotency.js +45 -0
- package/dist/utils/ip.js +25 -0
- package/dist/utils/json-fields.js +94 -0
- package/dist/utils/local-tls.js +39 -0
- package/dist/utils/magnus-home.js +15 -0
- package/dist/utils/node-url.js +59 -0
- package/dist/utils/redact.js +26 -0
- package/dist/utils/safe-equal.js +13 -0
- package/dist/utils/sentinel-nodes.js +16 -0
- package/dist/utils/validate.js +41 -0
- package/package.json +119 -4
- package/prisma/migrations/20261031000000_baseline/migration.sql +2935 -0
- package/prisma/migrations/migration_lock.toml +3 -0
- package/prisma/schema.prisma +2185 -0
- package/prisma.config.ts +14 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import { assertEnrollmentComplete, lockResourceNode } from './resource-enrollment.js';
|
|
2
|
+
import { assertArdentScope } from './ardent-scope-enrollment.js';
|
|
3
|
+
import { assertNodeAdmissionOpen } from './admission-fence.js';
|
|
4
|
+
/** Finite managed-command ownership. This is coordination, never action approval.
|
|
5
|
+
* Default-off until dashboard terminal and SSH lifecycle coverage is complete.
|
|
6
|
+
* A node requires the explicit durable participant drain before its first lease; switching the flag off does
|
|
7
|
+
* not remove held/unknown ownership or permit a legacy bypass on that node.
|
|
8
|
+
*/
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
import { nanoid } from 'nanoid';
|
|
11
|
+
import { getDb } from '../db/index.js';
|
|
12
|
+
import { HttpError } from '../middleware/error-handler.js';
|
|
13
|
+
import { commandRequestDigest } from '../security/command-request.js';
|
|
14
|
+
import { commandFenceSchema } from '../schemas/node-responses.js';
|
|
15
|
+
import { verifyCommandResultSignature } from '../security/command-result-signature.js';
|
|
16
|
+
import { RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET } from '../utils/env.js';
|
|
17
|
+
import { getNodeBaseUrl } from '../utils/node-url.js';
|
|
18
|
+
import { COMMAND_COMPLETION_REGISTRY, COMMAND_COMPLETION_REGISTRY_SHA256, commandCompletionDescriptor } from '../security/command-completion-registry.js';
|
|
19
|
+
import { MANAGED_HTTP_COMMAND } from '../security/managed-http.js';
|
|
20
|
+
import { NATIVE_COMPLETION_REGISTRY, NATIVE_COMPLETION_REGISTRY_SHA256, nativeCompletionDescriptor } from '../security/native-completion-registry.js';
|
|
21
|
+
const HELD = ['reserved', 'running', 'unknown'];
|
|
22
|
+
export class ResourceLeaseConflict extends HttpError {
|
|
23
|
+
constructor() { super(409, 'RESOURCE_LEASE_CONFLICT', 'Node or service has an unsettled command owner; reconcile its evidence before retrying'); }
|
|
24
|
+
}
|
|
25
|
+
export function resourceLeasesRequested() { return process.env['MAGNUS_RESOURCE_LEASES_ENABLED'] === 'true'; }
|
|
26
|
+
function canonicalServiceUnit(value) {
|
|
27
|
+
// A narrow scope is granted only to the exact allowlisted immutable runbook.
|
|
28
|
+
// Arbitrary shell syntax, aliases, user units and wrapper commands stay node-wide.
|
|
29
|
+
if (typeof value !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9_.@-]{0,100}\.service$/.test(value))
|
|
30
|
+
throw new HttpError(409, 'RESOURCE_SCOPE_INVALID', 'Approved system service identity is invalid');
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
async function completionAuthority(tx, payload, command, nodeId) {
|
|
34
|
+
const occurrenceId = payload.commandId.startsWith('cron-') ? payload.commandId.slice('cron-'.length) : '';
|
|
35
|
+
if (/^[a-f0-9]{64}$/.test(occurrenceId) && command.nodeId === nodeId
|
|
36
|
+
&& command.idempotencyKey === `managed-cron:${occurrenceId}` && command.correlationId?.startsWith('managed-cron:')) {
|
|
37
|
+
const occurrence = await tx.managedCronOccurrence.findUnique({ where: { commandId: command.id } });
|
|
38
|
+
const definition = occurrence && await tx.managedCronDefinition.findUnique({ where: { cronJobId: occurrence.cronJobId } });
|
|
39
|
+
try {
|
|
40
|
+
const spec = definition ? JSON.parse(definition.specJson) : null;
|
|
41
|
+
const canonicalSpec = spec && createHash('sha256').update(JSON.stringify(spec)).digest('hex');
|
|
42
|
+
const canonicalOccurrence = occurrence && definition && createHash('sha256')
|
|
43
|
+
.update(JSON.stringify([definition.jobId, occurrence.revision, occurrence.scheduledAt.getTime()])).digest('hex');
|
|
44
|
+
if (occurrence && definition && definition.enabled && definition.acknowledgementProtocolVersion === 1
|
|
45
|
+
&& definition.stageCommandSha256 && definition.stageReceiptSha256 && definition.stageAcknowledgedAt
|
|
46
|
+
&& definition.activationCommandSha256 && definition.activationReceiptSha256 && definition.activationAcknowledgedAt
|
|
47
|
+
&& definition.acknowledgedJobId === definition.jobId && definition.acknowledgedRevision === definition.revision
|
|
48
|
+
&& definition.acknowledgedSpecSha256 === definition.specSha256 && definition.acknowledgedSpecJson === definition.specJson
|
|
49
|
+
&& occurrence.id === occurrenceId && canonicalOccurrence === occurrenceId
|
|
50
|
+
&& occurrence.nodeId === nodeId && occurrence.commandId === payload.commandId && occurrence.state === 'admitting'
|
|
51
|
+
&& occurrence.cronJobId === definition.cronJobId && command.correlationId === `managed-cron:${definition.cronJobId}`
|
|
52
|
+
&& occurrence.revision === definition.revision && occurrence.specSha256 === definition.specSha256
|
|
53
|
+
&& canonicalSpec === definition.specSha256 && occurrence.deadline.getTime() === payload.coordination?.expiresAt
|
|
54
|
+
&& spec?.['command'] === payload.command && JSON.stringify(spec?.['args']) === JSON.stringify(payload.args)
|
|
55
|
+
&& (spec?.['workingDirectory'] ?? undefined) === payload.workingDirectory
|
|
56
|
+
&& JSON.stringify(spec?.['environment'] ?? {}) === JSON.stringify(payload.environment ?? {})
|
|
57
|
+
&& spec?.['timeoutSeconds'] === payload.timeoutSeconds)
|
|
58
|
+
return { completionClass: 'managed-scheduler', completionOccurrenceId: occurrence.id,
|
|
59
|
+
completionSpecSha256: occurrence.specSha256 };
|
|
60
|
+
}
|
|
61
|
+
catch { /* Corrupt or changed scheduler identity remains unverified. */ }
|
|
62
|
+
}
|
|
63
|
+
const descriptor = commandCompletionDescriptor(payload);
|
|
64
|
+
if (descriptor)
|
|
65
|
+
return { completionClass: 'node-supervised-command', completionRegistryVersion: COMMAND_COMPLETION_REGISTRY.version,
|
|
66
|
+
completionRegistrySha256: COMMAND_COMPLETION_REGISTRY_SHA256, completionDescriptorId: descriptor.id };
|
|
67
|
+
if (payload.command === MANAGED_HTTP_COMMAND && payload.args.length === 3 && /^[a-f0-9]{64}$/.test(payload.args[2])) {
|
|
68
|
+
const native = nativeCompletionDescriptor(payload.args[0], payload.args[1]);
|
|
69
|
+
if (native)
|
|
70
|
+
return { completionClass: 'native-http', completionRegistryVersion: NATIVE_COMPLETION_REGISTRY.version,
|
|
71
|
+
completionRegistrySha256: NATIVE_COMPLETION_REGISTRY_SHA256, completionDescriptorId: native.id };
|
|
72
|
+
}
|
|
73
|
+
return { completionClass: 'unverified' };
|
|
74
|
+
}
|
|
75
|
+
export async function admitResourceCommand(context, raw) {
|
|
76
|
+
const db = getDb();
|
|
77
|
+
const payload = structuredClone(raw);
|
|
78
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(payload.commandId)
|
|
79
|
+
|| typeof payload.command !== 'string' || !payload.command || !Array.isArray(payload.args)
|
|
80
|
+
|| payload.args.some(arg => typeof arg !== 'string') || !Number.isInteger(payload.timeoutSeconds)
|
|
81
|
+
|| payload.timeoutSeconds < 1 || payload.timeoutSeconds > 3600)
|
|
82
|
+
throw new HttpError(400, 'INVALID_COMMAND', 'A finite well-formed command is required');
|
|
83
|
+
const scopedArdent = context.scope === 'ardent-managed-http'
|
|
84
|
+
&& payload.command === MANAGED_HTTP_COMMAND && payload.args.length === 3
|
|
85
|
+
&& payload.args[0] === 'POST' && payload.args[1] === '/ardent/action'
|
|
86
|
+
&& /^[a-f0-9]{64}$/.test(payload.args[2])
|
|
87
|
+
&& payload.timeoutSeconds <= 900 && !payload.workingDirectory
|
|
88
|
+
&& Object.keys(payload.environment ?? {}).length === 0 && !payload.coordination;
|
|
89
|
+
if (context.scope && !scopedArdent)
|
|
90
|
+
throw new HttpError(409, 'RESOURCE_SCOPE_INVALID', 'Ardent scope accepts only the reviewed native action');
|
|
91
|
+
return db.$transaction(async (tx) => {
|
|
92
|
+
// One canonical node lock orders node-wide and service-specific admissions.
|
|
93
|
+
await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`command-resource-node:${context.nodeId}`}, 0))`;
|
|
94
|
+
const node = await tx.node.findUnique({ where: { id: context.nodeId } });
|
|
95
|
+
if (!node || getNodeBaseUrl(node.ipAddress, node.port).replace(/\/$/, '') !== context.baseUrl.replace(/\/$/, ''))
|
|
96
|
+
throw new HttpError(409, 'RESOURCE_TARGET_INVALID', 'Canonical command target differs from inventory');
|
|
97
|
+
await assertNodeAdmissionOpen(tx, node.id);
|
|
98
|
+
const enrollment = await tx.resourceEnrollment.findUnique({ where: { nodeId: node.id } });
|
|
99
|
+
if (!scopedArdent && !resourceLeasesRequested() && (!enrollment || enrollment.state === 'legacy')
|
|
100
|
+
&& !await tx.commandResourceLease.findFirst({ where: { nodeId: node.id }, select: { id: true } }))
|
|
101
|
+
return null;
|
|
102
|
+
if (scopedArdent)
|
|
103
|
+
await assertArdentScope(tx, node.id, node.hostname);
|
|
104
|
+
else
|
|
105
|
+
await assertEnrollmentComplete(tx, node.id, node.hostname);
|
|
106
|
+
if (await tx.resourceParticipantWork.findFirst({ where: { nodeId: node.id,
|
|
107
|
+
...(scopedArdent ? { participant: { in: ['agent', 'api', 'dashboard'] } } : {}), state: { in: HELD } } }))
|
|
108
|
+
throw new ResourceLeaseConflict();
|
|
109
|
+
if (node.status !== 'ONLINE' || !node.lastHeartbeatAt || Date.now() - node.lastHeartbeatAt.getTime() > 120_000)
|
|
110
|
+
throw new HttpError(409, 'RESOURCE_TARGET_INVALID', 'Canonical command target lacks fresh online health');
|
|
111
|
+
if (await tx.commandResourceLease.findUnique({ where: { commandId: payload.commandId } }))
|
|
112
|
+
throw new HttpError(409, 'COMMAND_ALREADY_OWNED', 'Command already has a durable resource owner; inspect its receipt without replaying');
|
|
113
|
+
let command = await tx.remoteCommand.findUnique({ where: { id: payload.commandId } });
|
|
114
|
+
const directCommand = !command;
|
|
115
|
+
let serviceUnit = null;
|
|
116
|
+
if (command) {
|
|
117
|
+
if (scopedArdent)
|
|
118
|
+
throw new HttpError(409, 'RESOURCE_SCOPE_INVALID', 'Ardent scope requires a fresh direct native action');
|
|
119
|
+
const stored = { commandId: command.id, command: command.command, args: JSON.parse(command.args),
|
|
120
|
+
timeoutSeconds: command.timeoutSeconds, workingDirectory: command.workingDirectory ?? undefined,
|
|
121
|
+
environment: JSON.parse(command.environment),
|
|
122
|
+
...(command.coordinationJson ? { coordination: commandFenceSchema.parse(JSON.parse(command.coordinationJson)) } : {}) };
|
|
123
|
+
if (command.nodeId !== node.id || command.status !== 'RUNNING' || commandRequestDigest(stored) !== commandRequestDigest(payload))
|
|
124
|
+
throw new HttpError(409, 'COMMAND_OWNER_MISMATCH', 'Durable command differs from its claimed execution');
|
|
125
|
+
if (command.correlationId?.startsWith('remediation_')) {
|
|
126
|
+
const { validateRemediationCommandForDispatch } = await import('./remediation-service.js');
|
|
127
|
+
await validateRemediationCommandForDispatch(command);
|
|
128
|
+
const remediation = await tx.remediationExecution.findUnique({ where: { id: command.correlationId.slice('remediation_'.length) } });
|
|
129
|
+
if (remediation && JSON.parse(remediation.input).runbookVersion === 'restart-service.v1') {
|
|
130
|
+
if (payload.command !== '/usr/bin/systemctl' || payload.args.length !== 2 || payload.args[0] !== 'restart')
|
|
131
|
+
throw new HttpError(409, 'RESOURCE_SCOPE_INVALID', 'Restart plan is not a typed system service action');
|
|
132
|
+
serviceUnit = canonicalServiceUnit(payload.args[1]);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
const resourceKey = scopedArdent ? `node:${node.id}:ardent`
|
|
137
|
+
: serviceUnit ? `node:${node.id}:system-service:${serviceUnit}` : `node:${node.id}:all`;
|
|
138
|
+
const coordinationKey = scopedArdent ? resourceKey : `node:${node.id}`;
|
|
139
|
+
// The node ledger has one execution slot: service metadata never promises parallel fences.
|
|
140
|
+
const conflict = await tx.commandResourceLease.findFirst({ where: { nodeId: node.id, state: { in: HELD } } });
|
|
141
|
+
if (conflict)
|
|
142
|
+
throw new ResourceLeaseConflict();
|
|
143
|
+
const now = new Date();
|
|
144
|
+
if (payload.coordination) {
|
|
145
|
+
commandFenceSchema.parse(payload.coordination);
|
|
146
|
+
if (payload.coordination.resourceKey !== coordinationKey || !command?.executionDeadline
|
|
147
|
+
|| command.executionDeadline.getTime() !== payload.coordination.expiresAt || payload.coordination.expiresAt <= now.getTime())
|
|
148
|
+
throw new HttpError(409, 'RESOURCE_DEADLINE_INVALID', 'Existing approval deadline and node fence must remain exact');
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
if (command?.executionDeadline)
|
|
152
|
+
throw new HttpError(409, 'RESOURCE_DEADLINE_INVALID', 'Approved execution is missing its fence');
|
|
153
|
+
const fence = await tx.commandResourceFence.upsert({ where: { resourceKey: coordinationKey },
|
|
154
|
+
create: { resourceKey: coordinationKey, generation: 1n }, update: { generation: { increment: 1n } } });
|
|
155
|
+
payload.coordination = { version: 1, resourceKey: coordinationKey, fence: fence.generation.toString(),
|
|
156
|
+
expiresAt: now.getTime() + (payload.timeoutSeconds + 10) * 1000 };
|
|
157
|
+
}
|
|
158
|
+
const coordinationJson = JSON.stringify(payload.coordination);
|
|
159
|
+
if (!command) {
|
|
160
|
+
command = await tx.remoteCommand.create({ data: { id: payload.commandId, nodeId: node.id,
|
|
161
|
+
command: payload.command, args: JSON.stringify(payload.args), environment: JSON.stringify(payload.environment ?? {}),
|
|
162
|
+
workingDirectory: payload.workingDirectory ?? null, timeoutSeconds: payload.timeoutSeconds,
|
|
163
|
+
idempotencyKey: `resource-direct:${payload.commandId}`, status: 'RUNNING', startedAt: now,
|
|
164
|
+
createdBy: context.actorId, executionDeadline: new Date(payload.coordination.expiresAt), coordinationJson } });
|
|
165
|
+
}
|
|
166
|
+
else if (!command.coordinationJson) {
|
|
167
|
+
await tx.remoteCommand.update({ where: { id: command.id }, data: {
|
|
168
|
+
executionDeadline: new Date(payload.coordination.expiresAt), coordinationJson
|
|
169
|
+
} });
|
|
170
|
+
}
|
|
171
|
+
const completion = await completionAuthority(tx, payload, command, node.id);
|
|
172
|
+
const lease = await tx.commandResourceLease.create({ data: { id: nanoid(), nodeId: node.id, resourceKey,
|
|
173
|
+
serviceUnit, commandId: payload.commandId, ownerActor: command.createdBy, directCommand,
|
|
174
|
+
requestSha256: commandRequestDigest(payload), coordinationJson, deadline: new Date(payload.coordination.expiresAt), ...completion } });
|
|
175
|
+
return { leaseId: lease.id, commandId: payload.commandId, nodeId: node.id, nodeFqdn: node.hostname, payload };
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
/** Persist intent immediately before the sole POST; interruption from here is unknown. */
|
|
179
|
+
export async function markResourceDispatched(admission, verifyBeforeDispatch) {
|
|
180
|
+
await getDb().$transaction(async (tx) => {
|
|
181
|
+
await lockResourceNode(tx, admission.nodeId);
|
|
182
|
+
await assertNodeAdmissionOpen(tx, admission.nodeId);
|
|
183
|
+
const lease = await tx.commandResourceLease.findUnique({ where: { id: admission.leaseId } });
|
|
184
|
+
if (lease?.resourceKey === `node:${admission.nodeId}:ardent`
|
|
185
|
+
&& lease.commandId === admission.commandId && lease.completionDescriptorId === 'ardent-action')
|
|
186
|
+
await assertArdentScope(tx, admission.nodeId, admission.nodeFqdn);
|
|
187
|
+
else
|
|
188
|
+
await assertEnrollmentComplete(tx, admission.nodeId, admission.nodeFqdn);
|
|
189
|
+
await verifyBeforeDispatch?.(tx);
|
|
190
|
+
const changed = await tx.commandResourceLease.updateMany({ where: { id: admission.leaseId, commandId: admission.commandId,
|
|
191
|
+
state: 'reserved', deadline: { gt: new Date() } }, data: { state: 'running', dispatchedAt: new Date() } });
|
|
192
|
+
if (changed.count !== 1)
|
|
193
|
+
throw new HttpError(409, 'RESOURCE_DEADLINE_INVALID', 'Resource admission expired or changed before dispatch');
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
export async function failResourceCommand(admission) {
|
|
197
|
+
const db = getDb();
|
|
198
|
+
await db.$transaction(async (tx) => {
|
|
199
|
+
const row = await tx.commandResourceLease.findUnique({ where: { id: admission.leaseId } });
|
|
200
|
+
if (!row || row.state === 'released')
|
|
201
|
+
return;
|
|
202
|
+
const notSent = row.state === 'reserved';
|
|
203
|
+
const changed = await tx.commandResourceLease.updateMany({ where: { id: row.id, state: row.state }, data: notSent
|
|
204
|
+
? { state: 'released', settledAt: new Date(), settlement: 'not-dispatched',
|
|
205
|
+
evidenceSha256: createHash('sha256').update(`pre-post:${row.commandId}:${row.requestSha256}`).digest('hex') }
|
|
206
|
+
: { state: 'unknown' } });
|
|
207
|
+
if (!changed.count)
|
|
208
|
+
return;
|
|
209
|
+
if (row.directCommand) {
|
|
210
|
+
await tx.remoteCommand.updateMany({ where: { id: row.commandId, status: 'RUNNING' }, data: { status: 'FAILED', completedAt: new Date() } });
|
|
211
|
+
await tx.commandResult.upsert({ where: { commandId: row.commandId }, update: {}, create: {
|
|
212
|
+
commandId: row.commandId, exitCode: -1, stdout: '', durationMs: 0,
|
|
213
|
+
stderr: notSent ? 'Resource admission ended before dispatch.' : 'Execution outcome unknown; resource ownership retained until verified evidence reconciliation.'
|
|
214
|
+
} });
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
export async function retainUnknownResourceCommand(commandId) {
|
|
219
|
+
await getDb().$transaction(async (tx) => {
|
|
220
|
+
// Persisted 'reserved' proves the sole POST barrier was never crossed. This
|
|
221
|
+
// CAS also makes a delayed markResourceDispatched fail before any network I/O.
|
|
222
|
+
const row = await tx.commandResourceLease.findUnique({ where: { commandId } });
|
|
223
|
+
if (!row)
|
|
224
|
+
return;
|
|
225
|
+
const stopped = await tx.commandResourceLease.updateMany({ where: { commandId, state: 'reserved' }, data: {
|
|
226
|
+
state: 'released', settledAt: new Date(), settlement: 'not-dispatched',
|
|
227
|
+
evidenceSha256: createHash('sha256').update(`recovered-pre-post:${commandId}:${row.requestSha256}`).digest('hex')
|
|
228
|
+
} });
|
|
229
|
+
if (!stopped.count)
|
|
230
|
+
await tx.commandResourceLease.updateMany({ where: { commandId, state: 'running' }, data: { state: 'unknown' } });
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
/** A signed, digest/fence-bound finite execution result is settlement evidence.
|
|
234
|
+
* Exit != 0 is known completion, not proof of application recovery or rollback. */
|
|
235
|
+
export async function settleResourceResult(commandId, result) {
|
|
236
|
+
const db = getDb();
|
|
237
|
+
const row = await db.commandResourceLease.findUnique({ where: { commandId } });
|
|
238
|
+
if (!row)
|
|
239
|
+
return;
|
|
240
|
+
const registered = row.completionClass === 'node-supervised-command'
|
|
241
|
+
&& row.completionRegistryVersion === COMMAND_COMPLETION_REGISTRY.version
|
|
242
|
+
&& row.completionRegistrySha256 === COMMAND_COMPLETION_REGISTRY_SHA256
|
|
243
|
+
&& COMMAND_COMPLETION_REGISTRY.descriptors.some(descriptor => descriptor.id === row.completionDescriptorId);
|
|
244
|
+
const native = row.completionClass === 'native-http'
|
|
245
|
+
&& row.completionRegistryVersion === NATIVE_COMPLETION_REGISTRY.version
|
|
246
|
+
&& row.completionRegistrySha256 === NATIVE_COMPLETION_REGISTRY_SHA256
|
|
247
|
+
&& NATIVE_COMPLETION_REGISTRY.descriptors.some(descriptor => descriptor.id === row.completionDescriptorId);
|
|
248
|
+
const scheduler = row.completionClass === 'managed-scheduler'
|
|
249
|
+
&& /^[a-f0-9]{64}$/.test(row.completionOccurrenceId ?? '')
|
|
250
|
+
&& /^[a-f0-9]{64}$/.test(row.completionSpecSha256 ?? '')
|
|
251
|
+
&& row.completionRegistryVersion === null && row.completionRegistrySha256 === null && row.completionDescriptorId === null;
|
|
252
|
+
const authorityMatches = registered ? result.completion?.registryVersion === row.completionRegistryVersion
|
|
253
|
+
&& result.completion.registrySha256 === row.completionRegistrySha256
|
|
254
|
+
&& result.completion.descriptorId === row.completionDescriptorId
|
|
255
|
+
: native || scheduler ? result.completion === undefined : false;
|
|
256
|
+
if (!authorityMatches)
|
|
257
|
+
throw new HttpError(409, 'RESOURCE_EVIDENCE_INVALID', 'Command lacks its exact registered whole-effect evidence');
|
|
258
|
+
if (!verifyCommandResultSignature(result, RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET)
|
|
259
|
+
|| result.commandId !== commandId || result.coordination?.requestSha256 !== row.requestSha256
|
|
260
|
+
|| JSON.stringify(result.coordination && { version: result.coordination.version, resourceKey: result.coordination.resourceKey,
|
|
261
|
+
fence: result.coordination.fence, expiresAt: result.coordination.expiresAt }) !== row.coordinationJson)
|
|
262
|
+
throw new HttpError(409, 'RESOURCE_EVIDENCE_INVALID', 'Signed result does not settle the owned request/fence');
|
|
263
|
+
const evidence = createHash('sha256').update(JSON.stringify(result)).digest('hex');
|
|
264
|
+
await db.$transaction(async (tx) => {
|
|
265
|
+
await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`command-resource-node:${row.nodeId}`}, 0))`;
|
|
266
|
+
const changed = await tx.commandResourceLease.updateMany({ where: { id: row.id, state: { in: HELD } }, data: {
|
|
267
|
+
state: 'released', settledAt: new Date(), settlement: 'signed-command-result', evidenceSha256: evidence
|
|
268
|
+
} });
|
|
269
|
+
if (!changed.count)
|
|
270
|
+
return;
|
|
271
|
+
if (row.directCommand) {
|
|
272
|
+
await tx.remoteCommand.update({ where: { id: commandId }, data: { status: result.exitCode === 0 ? 'COMPLETED' : 'FAILED', completedAt: new Date() } });
|
|
273
|
+
await tx.commandResult.upsert({ where: { commandId }, create: { commandId, exitCode: result.exitCode,
|
|
274
|
+
stdout: result.stdout, stderr: result.stderr, durationMs: result.durationMs, truncated: result.truncated },
|
|
275
|
+
update: { exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr, durationMs: result.durationMs, truncated: result.truncated } });
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
/** Adopt the node's already verified explicit reconciliation receipt. A local
|
|
280
|
+
* timeout, command status, heartbeat or arbitrary evidence URL never releases. */
|
|
281
|
+
export async function settleResourceReconciliation(nodeId, assertion) {
|
|
282
|
+
const { createHmac, timingSafeEqual } = await import('node:crypto');
|
|
283
|
+
const { commandReconciliationSigningPayload, commandReconciliationSchema } = await import('../sdk/node-api-client.js');
|
|
284
|
+
const { getNodeManagerSecret } = await import('../utils/env.js');
|
|
285
|
+
const value = commandReconciliationSchema.parse(assertion);
|
|
286
|
+
const secret = process.env['COMMAND_RECONCILIATION_SECRET'] ?? '';
|
|
287
|
+
if (secret.length < 32 || secret === RESULT_SIGNING_SECRET || secret === getNodeManagerSecret()
|
|
288
|
+
|| !timingSafeEqual(createHmac('sha256', secret).update(commandReconciliationSigningPayload(value)).digest(), Buffer.from(value.signature, 'hex')))
|
|
289
|
+
throw new HttpError(403, 'RESOURCE_EVIDENCE_INVALID', 'Explicit reconciliation signature is invalid');
|
|
290
|
+
const db = getDb();
|
|
291
|
+
await db.$transaction(async (tx) => {
|
|
292
|
+
await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`command-resource-node:${nodeId}`}, 0))`;
|
|
293
|
+
const row = await tx.commandResourceLease.findUnique({ where: { commandId: value.commandId } });
|
|
294
|
+
const node = await tx.node.findUnique({ where: { id: nodeId } });
|
|
295
|
+
if (!row || !node || row.nodeId !== nodeId || value.nodeFqdn !== node.hostname
|
|
296
|
+
|| value.requestSha256 !== row.requestSha256 || JSON.stringify(value.coordination ?? null) !== row.coordinationJson)
|
|
297
|
+
throw new HttpError(409, 'RESOURCE_EVIDENCE_INVALID', 'Explicit reconciliation does not bind this node, owner, request and fence');
|
|
298
|
+
if (row.state === 'released')
|
|
299
|
+
return;
|
|
300
|
+
await tx.commandResourceLease.update({ where: { id: row.id }, data: { state: 'released', settledAt: new Date(),
|
|
301
|
+
settlement: `signed-reconciliation:${value.disposition}`, evidenceSha256: value.evidenceSha256 } });
|
|
302
|
+
// Reconciliation does not invent an exit code or promote execution success.
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
//# sourceMappingURL=command-resource-leases.js.map
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { request as httpRequest } from 'node:http';
|
|
2
|
+
import { request as httpsRequest } from 'node:https';
|
|
3
|
+
import { promises as dns } from 'node:dns';
|
|
4
|
+
import { arch, hostname as osHostname, networkInterfaces, platform } from 'node:os';
|
|
5
|
+
import { isIP } from 'node:net';
|
|
6
|
+
import { nanoid } from 'nanoid';
|
|
7
|
+
import { getDb } from '../db/index.js';
|
|
8
|
+
import { isUsableNodeIp, normalizeIpAddress } from '../utils/ip.js';
|
|
9
|
+
import { controlHostAgentTlsOptions, controlHostAgentTlsServername, } from '../utils/local-tls.js';
|
|
10
|
+
const LOCAL_AGENT_PORT = Number(process.env['CONTROL_HOST_AGENT_PORT'] ?? process.env['AGENT_PORT'] ?? 4100);
|
|
11
|
+
const CACHE_TTL_MS = 30_000;
|
|
12
|
+
let lastEnsureAt = 0;
|
|
13
|
+
let inFlight = null;
|
|
14
|
+
function controlHostNodeEnabled() {
|
|
15
|
+
return process.env['CONTROL_HOST_NODE_ENABLED'] !== 'false';
|
|
16
|
+
}
|
|
17
|
+
function getControlHostName() {
|
|
18
|
+
return (process.env['CONTROL_HOST_NODE_NAME'] ?? process.env['CONTROL_NODE_SELF_NAME'] ?? osHostname() ?? 'magnus').trim();
|
|
19
|
+
}
|
|
20
|
+
function shouldRegisterOfflineControlHost(name) {
|
|
21
|
+
if (process.env['CONTROL_HOST_NODE_REGISTER_OFFLINE'] === 'true')
|
|
22
|
+
return true;
|
|
23
|
+
if (process.env['CONTROL_HOST_NODE_REGISTER_OFFLINE'] === 'false')
|
|
24
|
+
return false;
|
|
25
|
+
return name === 'magnus';
|
|
26
|
+
}
|
|
27
|
+
function parseMetadata(metadata) {
|
|
28
|
+
if (!metadata)
|
|
29
|
+
return {};
|
|
30
|
+
try {
|
|
31
|
+
const parsed = JSON.parse(metadata);
|
|
32
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
33
|
+
? parsed
|
|
34
|
+
: {};
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return {};
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function getInterfaceIp() {
|
|
41
|
+
for (const addresses of Object.values(networkInterfaces())) {
|
|
42
|
+
for (const address of addresses ?? []) {
|
|
43
|
+
if (address.family === 'IPv4' && !address.internal && isUsableNodeIp(address.address)) {
|
|
44
|
+
return normalizeIpAddress(address.address);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
export async function resolveControlHostAddress(hostname) {
|
|
51
|
+
const configured = normalizeIpAddress(process.env['CONTROL_HOST_NODE_IP'] ?? '');
|
|
52
|
+
if (configured && isUsableNodeIp(configured))
|
|
53
|
+
return configured;
|
|
54
|
+
try {
|
|
55
|
+
const records = await dns.lookup(hostname, { all: true, verbatim: true });
|
|
56
|
+
const resolved = records
|
|
57
|
+
.map((record) => normalizeIpAddress(record.address))
|
|
58
|
+
.find((address) => isUsableNodeIp(address));
|
|
59
|
+
if (resolved)
|
|
60
|
+
return resolved;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// Fall through to local interface probing.
|
|
64
|
+
}
|
|
65
|
+
return getInterfaceIp() ?? '127.0.0.1';
|
|
66
|
+
}
|
|
67
|
+
function requestAgentPing(protocol, port) {
|
|
68
|
+
const request = protocol === 'https' ? httpsRequest : httpRequest;
|
|
69
|
+
return new Promise((resolve) => {
|
|
70
|
+
const req = request({
|
|
71
|
+
hostname: '127.0.0.1',
|
|
72
|
+
port,
|
|
73
|
+
path: '/api/v1/ping',
|
|
74
|
+
method: 'GET',
|
|
75
|
+
...(protocol === 'https' ? controlHostAgentTlsOptions() : {}),
|
|
76
|
+
timeout: 1_500,
|
|
77
|
+
}, (res) => {
|
|
78
|
+
const chunks = [];
|
|
79
|
+
res.on('data', (chunk) => chunks.push(chunk));
|
|
80
|
+
res.on('end', () => {
|
|
81
|
+
if ((res.statusCode ?? 500) >= 400) {
|
|
82
|
+
resolve(null);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
try {
|
|
86
|
+
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
resolve(null);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
req.on('timeout', () => {
|
|
94
|
+
req.destroy();
|
|
95
|
+
resolve(null);
|
|
96
|
+
});
|
|
97
|
+
req.on('error', () => resolve(null));
|
|
98
|
+
req.end();
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
export async function probeLocalAgent() {
|
|
102
|
+
for (const protocol of ['https', 'http']) {
|
|
103
|
+
const response = await requestAgentPing(protocol, LOCAL_AGENT_PORT);
|
|
104
|
+
if (response?.success === true &&
|
|
105
|
+
response.data?.ok === true &&
|
|
106
|
+
response.data.features?.nodeManagement !== false) {
|
|
107
|
+
const tlsServername = protocol === 'https'
|
|
108
|
+
? controlHostAgentTlsOptions().servername
|
|
109
|
+
: undefined;
|
|
110
|
+
return {
|
|
111
|
+
port: LOCAL_AGENT_PORT,
|
|
112
|
+
version: response.data.version ?? '0.0.0',
|
|
113
|
+
nodeStatus: 'ONLINE',
|
|
114
|
+
agentStatus: 'RUNNING',
|
|
115
|
+
...(typeof tlsServername === 'string' && tlsServername.trim()
|
|
116
|
+
? { verifiedTlsHostname: tlsServername.trim().toLowerCase() }
|
|
117
|
+
: {}),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
/** Prefer the DNS identity proven by the local HTTPS probe over a LAN address. */
|
|
124
|
+
export function selectControlHostNodeAddress(verifiedTlsHostname, fallbackAddress, existingAddress, expectedTlsHostname) {
|
|
125
|
+
const verified = verifiedTlsHostname?.trim().toLowerCase();
|
|
126
|
+
if (verified)
|
|
127
|
+
return verified;
|
|
128
|
+
const existing = existingAddress?.trim().toLowerCase();
|
|
129
|
+
const expected = expectedTlsHostname?.trim().toLowerCase();
|
|
130
|
+
if (existing && expected && existing === expected && isIP(expected) === 0)
|
|
131
|
+
return existing;
|
|
132
|
+
return fallbackAddress;
|
|
133
|
+
}
|
|
134
|
+
export async function ensureControlHostNodeNow(options) {
|
|
135
|
+
if (!controlHostNodeEnabled())
|
|
136
|
+
return;
|
|
137
|
+
const name = getControlHostName();
|
|
138
|
+
const probe = await (options.probeAgent ?? probeLocalAgent)();
|
|
139
|
+
if (!probe && !shouldRegisterOfflineControlHost(name))
|
|
140
|
+
return;
|
|
141
|
+
const agentState = probe ?? {
|
|
142
|
+
port: LOCAL_AGENT_PORT,
|
|
143
|
+
version: '0.0.0',
|
|
144
|
+
nodeStatus: 'OFFLINE',
|
|
145
|
+
agentStatus: 'STOPPED',
|
|
146
|
+
};
|
|
147
|
+
const db = options.db ?? getDb();
|
|
148
|
+
const fallbackHostname = (process.env['CONTROL_HOST_NODE_HOSTNAME']
|
|
149
|
+
?? osHostname()
|
|
150
|
+
?? name).trim().toLowerCase();
|
|
151
|
+
const now = new Date();
|
|
152
|
+
const existing = await db.node.findFirst({
|
|
153
|
+
where: {
|
|
154
|
+
OR: [
|
|
155
|
+
{ name },
|
|
156
|
+
{ hostname: fallbackHostname },
|
|
157
|
+
],
|
|
158
|
+
},
|
|
159
|
+
select: { id: true, metadata: true, hostname: true, ipAddress: true },
|
|
160
|
+
});
|
|
161
|
+
const fallbackAddress = await (options.resolveAddress ?? resolveControlHostAddress)(fallbackHostname);
|
|
162
|
+
const hostname = selectControlHostNodeAddress(agentState.verifiedTlsHostname, fallbackHostname, existing?.hostname, controlHostAgentTlsServername());
|
|
163
|
+
const ipAddress = selectControlHostNodeAddress(agentState.verifiedTlsHostname, fallbackAddress, existing?.ipAddress, controlHostAgentTlsServername());
|
|
164
|
+
const metadata = {
|
|
165
|
+
...parseMetadata(existing?.metadata),
|
|
166
|
+
managedControlHost: true,
|
|
167
|
+
localAgentProbeAt: now.toISOString(),
|
|
168
|
+
localAgentReachable: Boolean(probe),
|
|
169
|
+
};
|
|
170
|
+
if (existing) {
|
|
171
|
+
await db.node.update({
|
|
172
|
+
where: { id: existing.id },
|
|
173
|
+
data: {
|
|
174
|
+
name,
|
|
175
|
+
hostname,
|
|
176
|
+
ipAddress,
|
|
177
|
+
port: agentState.port,
|
|
178
|
+
status: agentState.nodeStatus,
|
|
179
|
+
os: platform(),
|
|
180
|
+
arch: arch(),
|
|
181
|
+
metadata: JSON.stringify(metadata),
|
|
182
|
+
...(probe ? { lastHeartbeatAt: now } : {}),
|
|
183
|
+
},
|
|
184
|
+
});
|
|
185
|
+
await db.agent.upsert({
|
|
186
|
+
where: { nodeId: existing.id },
|
|
187
|
+
update: {
|
|
188
|
+
version: agentState.version,
|
|
189
|
+
status: agentState.agentStatus,
|
|
190
|
+
lastCheckIn: now,
|
|
191
|
+
},
|
|
192
|
+
create: {
|
|
193
|
+
nodeId: existing.id,
|
|
194
|
+
version: agentState.version,
|
|
195
|
+
status: agentState.agentStatus,
|
|
196
|
+
capabilities: '[]',
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
const id = nanoid();
|
|
202
|
+
await db.node.create({
|
|
203
|
+
data: {
|
|
204
|
+
id,
|
|
205
|
+
name,
|
|
206
|
+
hostname,
|
|
207
|
+
ipAddress,
|
|
208
|
+
port: agentState.port,
|
|
209
|
+
status: agentState.nodeStatus,
|
|
210
|
+
os: platform(),
|
|
211
|
+
arch: arch(),
|
|
212
|
+
token: `control-host-${id}`,
|
|
213
|
+
metadata: JSON.stringify(metadata),
|
|
214
|
+
...(probe ? { lastHeartbeatAt: now } : {}),
|
|
215
|
+
},
|
|
216
|
+
});
|
|
217
|
+
await db.agent.create({
|
|
218
|
+
data: {
|
|
219
|
+
nodeId: id,
|
|
220
|
+
version: agentState.version,
|
|
221
|
+
status: agentState.agentStatus,
|
|
222
|
+
capabilities: '[]',
|
|
223
|
+
},
|
|
224
|
+
});
|
|
225
|
+
console.log(`[ControlHost] Registered local managed node: ${name} (${ipAddress}:${agentState.port})`);
|
|
226
|
+
}
|
|
227
|
+
export async function ensureControlHostNode(options = {}) {
|
|
228
|
+
if (!options.force && Date.now() - lastEnsureAt < CACHE_TTL_MS)
|
|
229
|
+
return;
|
|
230
|
+
if (inFlight)
|
|
231
|
+
return inFlight;
|
|
232
|
+
inFlight = ensureControlHostNodeNow(options)
|
|
233
|
+
.catch((err) => {
|
|
234
|
+
console.warn(`[ControlHost] Failed to ensure local managed node: ${err instanceof Error ? err.message : String(err)}`);
|
|
235
|
+
})
|
|
236
|
+
.finally(() => {
|
|
237
|
+
lastEnsureAt = Date.now();
|
|
238
|
+
inFlight = null;
|
|
239
|
+
});
|
|
240
|
+
return inFlight;
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=control-host-node.js.map
|