@hivemagnus/magnus-server 0.0.0-stage → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +171 -0
- package/LICENSE +202 -0
- package/README.md +169 -2
- package/agent-version.json +1 -0
- package/dist/ai/HEARTBEAT.md +56 -0
- package/dist/ai/IDENTITY.md +54 -0
- package/dist/ai/MEMORY.md +40 -0
- package/dist/ai/TOOLS.md +538 -0
- package/dist/ai/USER.md +32 -0
- package/dist/cli/args.js +62 -0
- package/dist/cli/env-catalog.js +131 -0
- package/dist/cli/preflight.js +89 -0
- package/dist/cli.js +175 -0
- package/dist/config/cors.js +16 -0
- package/dist/config/env.js +392 -0
- package/dist/config/features.js +16 -0
- package/dist/config/index.js +3 -0
- package/dist/config/listener-security.js +38 -0
- package/dist/config/version.js +31 -0
- package/dist/db/client.js +44 -0
- package/dist/db/generated/prisma/browser.js +18 -0
- package/dist/db/generated/prisma/client.js +35 -0
- package/dist/db/generated/prisma/commonInputTypes.js +11 -0
- package/dist/db/generated/prisma/enums.js +42 -0
- package/dist/db/generated/prisma/internal/class.js +50 -0
- package/dist/db/generated/prisma/internal/prismaNamespace.js +1598 -0
- package/dist/db/generated/prisma/internal/prismaNamespaceBrowser.js +1569 -0
- package/dist/db/generated/prisma/models/A2ATask.js +2 -0
- package/dist/db/generated/prisma/models/AdmissionFence.js +2 -0
- package/dist/db/generated/prisma/models/Agent.js +2 -0
- package/dist/db/generated/prisma/models/AiConversation.js +2 -0
- package/dist/db/generated/prisma/models/AiDashboard.js +2 -0
- package/dist/db/generated/prisma/models/AiMemory.js +2 -0
- package/dist/db/generated/prisma/models/AiMessage.js +2 -0
- package/dist/db/generated/prisma/models/AiScheduledOp.js +2 -0
- package/dist/db/generated/prisma/models/AlertConditionState.js +2 -0
- package/dist/db/generated/prisma/models/AlertRule.js +2 -0
- package/dist/db/generated/prisma/models/AlertViolation.js +2 -0
- package/dist/db/generated/prisma/models/AnomalyEvent.js +2 -0
- package/dist/db/generated/prisma/models/ApplicationObservation.js +2 -0
- package/dist/db/generated/prisma/models/ArdentInstallation.js +2 -0
- package/dist/db/generated/prisma/models/ArdentProfile.js +2 -0
- package/dist/db/generated/prisma/models/ArdentScopeEnrollment.js +2 -0
- package/dist/db/generated/prisma/models/CommandResourceFence.js +2 -0
- package/dist/db/generated/prisma/models/CommandResourceLease.js +2 -0
- package/dist/db/generated/prisma/models/CommandResult.js +2 -0
- package/dist/db/generated/prisma/models/CronJob.js +2 -0
- package/dist/db/generated/prisma/models/CronJobExecution.js +2 -0
- package/dist/db/generated/prisma/models/DeployJob.js +2 -0
- package/dist/db/generated/prisma/models/DriftScan.js +2 -0
- package/dist/db/generated/prisma/models/EngineeringMetricSample.js +2 -0
- package/dist/db/generated/prisma/models/EngineeringRecoveryExercise.js +2 -0
- package/dist/db/generated/prisma/models/ExecutionRun.js +2 -0
- package/dist/db/generated/prisma/models/FilePatchTask.js +2 -0
- package/dist/db/generated/prisma/models/GpuSnapshot.js +2 -0
- package/dist/db/generated/prisma/models/HealthSnapshot.js +2 -0
- package/dist/db/generated/prisma/models/HermesCoreRollout.js +2 -0
- package/dist/db/generated/prisma/models/HermesCoreRolloutHost.js +2 -0
- package/dist/db/generated/prisma/models/HermesCredentialPool.js +2 -0
- package/dist/db/generated/prisma/models/HermesCredentialPoolAttachment.js +2 -0
- package/dist/db/generated/prisma/models/HermesCronJob.js +2 -0
- package/dist/db/generated/prisma/models/HermesHook.js +2 -0
- package/dist/db/generated/prisma/models/HermesInstallation.js +2 -0
- package/dist/db/generated/prisma/models/HermesMcpServer.js +2 -0
- package/dist/db/generated/prisma/models/HermesMcpServerAttachment.js +2 -0
- package/dist/db/generated/prisma/models/HermesMessagingBinding.js +2 -0
- package/dist/db/generated/prisma/models/HermesOperationJob.js +2 -0
- package/dist/db/generated/prisma/models/HermesPluginInstall.js +2 -0
- package/dist/db/generated/prisma/models/HermesProfile.js +2 -0
- package/dist/db/generated/prisma/models/HermesProviderConfig.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryClient.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryInventory.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryReceipt.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryRevision.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryTarget.js +2 -0
- package/dist/db/generated/prisma/models/HermesRole.js +2 -0
- package/dist/db/generated/prisma/models/HermesRoleApplication.js +2 -0
- package/dist/db/generated/prisma/models/HermesSkillAssignment.js +2 -0
- package/dist/db/generated/prisma/models/HermesToolsetPolicy.js +2 -0
- package/dist/db/generated/prisma/models/IncidentLesson.js +2 -0
- package/dist/db/generated/prisma/models/LibraryCategory.js +2 -0
- package/dist/db/generated/prisma/models/LibraryFile.js +2 -0
- package/dist/db/generated/prisma/models/LogEntry.js +2 -0
- package/dist/db/generated/prisma/models/MagnusReleaseCanary.js +2 -0
- package/dist/db/generated/prisma/models/MaintenanceWindow.js +2 -0
- package/dist/db/generated/prisma/models/ManagedCronDefinition.js +2 -0
- package/dist/db/generated/prisma/models/ManagedCronOccurrence.js +2 -0
- package/dist/db/generated/prisma/models/Node.js +2 -0
- package/dist/db/generated/prisma/models/Notification.js +2 -0
- package/dist/db/generated/prisma/models/NotificationPreference.js +2 -0
- package/dist/db/generated/prisma/models/RemediationExecution.js +2 -0
- package/dist/db/generated/prisma/models/RemediationRule.js +2 -0
- package/dist/db/generated/prisma/models/RemoteCommand.js +2 -0
- package/dist/db/generated/prisma/models/ResourceDrainParticipant.js +2 -0
- package/dist/db/generated/prisma/models/ResourceEnrollment.js +2 -0
- package/dist/db/generated/prisma/models/ResourceParticipantWork.js +2 -0
- package/dist/db/generated/prisma/models/Role.js +2 -0
- package/dist/db/generated/prisma/models/RoleApplication.js +2 -0
- package/dist/db/generated/prisma/models/RolloutTarget.js +2 -0
- package/dist/db/generated/prisma/models/SecurityExportOutbox.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionAction.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionActionApproval.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCase.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseActivity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseEvent.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseRelationship.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCollector.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCommandOutbox.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionDeletionReceipt.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionDelivery.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEntityAlias.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEvent.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEventEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEvidence.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionInvestigation.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionNonce.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionReceiptChainHead.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionRuleVersion.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionSourceState.js +2 -0
- package/dist/db/generated/prisma/models/SecurityServiceState.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceFence.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceLease.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceLeaseResource.js +2 -0
- package/dist/db/generated/prisma/models/SkillInstallation.js +2 -0
- package/dist/db/generated/prisma/models/SkillRegistryEntry.js +2 -0
- package/dist/db/generated/prisma/models/SkillTag.js +2 -0
- package/dist/db/generated/prisma/models/SkillVersion.js +2 -0
- package/dist/db/generated/prisma/models/TeamApprovalDecision.js +2 -0
- package/dist/db/generated/prisma/models/User.js +2 -0
- package/dist/db/generated/prisma/models/WebhookDelivery.js +2 -0
- package/dist/db/generated/prisma/models/WebhookEndpoint.js +2 -0
- package/dist/db/generated/prisma/models.js +2 -0
- package/dist/db/index.js +3 -0
- package/dist/db/read-retry.js +35 -0
- package/dist/db/seed.js +348 -0
- package/dist/index.js +162 -0
- package/dist/middleware/auth.js +361 -0
- package/dist/middleware/body-limit.js +14 -0
- package/dist/middleware/correlation-id.js +13 -0
- package/dist/middleware/error-handler.js +28 -0
- package/dist/middleware/iam-token.js +21 -0
- package/dist/middleware/rate-limit.js +78 -0
- package/dist/middleware/request-logger.js +85 -0
- package/dist/middleware/security-headers.js +23 -0
- package/dist/middleware/tracing.js +120 -0
- package/dist/middleware/user-status.js +33 -0
- package/dist/openapi/spec.js +2656 -0
- package/dist/queue/redis.js +20 -0
- package/dist/queue/requeue.js +27 -0
- package/dist/queue/workers.js +342 -0
- package/dist/routes/ai-schedule.js +73 -0
- package/dist/routes/alerts.js +79 -0
- package/dist/routes/anomalies.js +101 -0
- package/dist/routes/ardent.js +82 -0
- package/dist/routes/assistant.js +417 -0
- package/dist/routes/audit.js +138 -0
- package/dist/routes/codex.js +65 -0
- package/dist/routes/commands.js +226 -0
- package/dist/routes/credentials.js +58 -0
- package/dist/routes/cron.js +292 -0
- package/dist/routes/dependencies.js +49 -0
- package/dist/routes/deploy.js +18 -0
- package/dist/routes/discovery.js +168 -0
- package/dist/routes/docs.js +24 -0
- package/dist/routes/drift.js +126 -0
- package/dist/routes/events-stream.js +87 -0
- package/dist/routes/files.js +214 -0
- package/dist/routes/fleet-checks.js +21 -0
- package/dist/routes/health.js +133 -0
- package/dist/routes/heartbeat.js +545 -0
- package/dist/routes/hermes-registry.js +278 -0
- package/dist/routes/hermes.js +61 -0
- package/dist/routes/library.js +402 -0
- package/dist/routes/logs.js +172 -0
- package/dist/routes/managed-cron.js +23 -0
- package/dist/routes/mcp.js +39 -0
- package/dist/routes/metrics.js +187 -0
- package/dist/routes/nodes.js +951 -0
- package/dist/routes/notifications.js +130 -0
- package/dist/routes/ops.js +258 -0
- package/dist/routes/orchestra.js +430 -0
- package/dist/routes/predictions.js +23 -0
- package/dist/routes/profiling.js +68 -0
- package/dist/routes/quick-command.js +72 -0
- package/dist/routes/remediation.js +111 -0
- package/dist/routes/resource-leases.js +297 -0
- package/dist/routes/roles.js +370 -0
- package/dist/routes/rollout.js +124 -0
- package/dist/routes/scan.js +116 -0
- package/dist/routes/security-backup.js +58 -0
- package/dist/routes/security-fusion.js +605 -0
- package/dist/routes/security-integration.js +71 -0
- package/dist/routes/self-update.js +129 -0
- package/dist/routes/shared-resource-leases.js +78 -0
- package/dist/routes/skills.js +485 -0
- package/dist/routes/stats.js +96 -0
- package/dist/routes/team-approvals.js +23 -0
- package/dist/routes/terminal-ticket.js +15 -0
- package/dist/routes/terminal.js +17 -0
- package/dist/routes/topology.js +37 -0
- package/dist/routes/users.js +219 -0
- package/dist/routes/webhooks.js +137 -0
- package/dist/schemas/command-reconciliation.js +14 -0
- package/dist/schemas/commands.js +51 -0
- package/dist/schemas/cron.js +33 -0
- package/dist/schemas/deploy.js +20 -0
- package/dist/schemas/files.js +14 -0
- package/dist/schemas/hermes.js +69 -0
- package/dist/schemas/index.js +11 -0
- package/dist/schemas/library.js +48 -0
- package/dist/schemas/logs.js +35 -0
- package/dist/schemas/managed-cron.js +127 -0
- package/dist/schemas/node-responses.js +87 -0
- package/dist/schemas/node-session-receipt.js +84 -0
- package/dist/schemas/nodes.js +36 -0
- package/dist/schemas/orchestra.js +118 -0
- package/dist/schemas/resource-participant-reconciliation.js +99 -0
- package/dist/schemas/roles.js +54 -0
- package/dist/schemas/shared-resource-lease.js +132 -0
- package/dist/schemas/skills.js +44 -0
- package/dist/schemas/users.js +28 -0
- package/dist/scripts/run-disposable-recovery-exercise.js +39 -0
- package/dist/sdk/index.js +2 -0
- package/dist/sdk/node-api-client.js +474 -0
- package/dist/security/approval-assertion.js +66 -0
- package/dist/security/command-completion-registry.js +35 -0
- package/dist/security/command-request.js +12 -0
- package/dist/security/command-result-signature.js +34 -0
- package/dist/security/managed-http.js +45 -0
- package/dist/security/native-completion-registry.js +44 -0
- package/dist/server.js +178 -0
- package/dist/services/admission-fence.js +71 -0
- package/dist/services/ai-assistant-service.js +2523 -0
- package/dist/services/ai-correlation-service.js +225 -0
- package/dist/services/ai-export-service.js +141 -0
- package/dist/services/ai-insights-service.js +219 -0
- package/dist/services/ai-schedule-service.js +145 -0
- package/dist/services/ai-scratchpad-service.js +222 -0
- package/dist/services/ai-tools.js +1343 -0
- package/dist/services/alert-service.js +313 -0
- package/dist/services/anomaly-detector.js +419 -0
- package/dist/services/api-local-work.js +79 -0
- package/dist/services/application-probes.js +309 -0
- package/dist/services/ardent-scope-enrollment.js +123 -0
- package/dist/services/ardent-service.js +86 -0
- package/dist/services/audit-service.js +308 -0
- package/dist/services/codex-service.js +209 -0
- package/dist/services/command-lifecycle.js +60 -0
- package/dist/services/command-resource-leases.js +305 -0
- package/dist/services/control-host-node.js +242 -0
- package/dist/services/credential-rotation-service.js +294 -0
- package/dist/services/dependency-service.js +145 -0
- package/dist/services/disposable-recovery-runner.js +595 -0
- package/dist/services/docker-group-health.js +121 -0
- package/dist/services/engineering-assurance.js +492 -0
- package/dist/services/engineering-evidence-ledger.js +94 -0
- package/dist/services/engineering-metric-evidence.js +26 -0
- package/dist/services/event-bus.js +227 -0
- package/dist/services/fleet-check-log.js +102 -0
- package/dist/services/forecast-service.js +154 -0
- package/dist/services/health-retention.js +175 -0
- package/dist/services/heartbeat-stream.js +63 -0
- package/dist/services/hermes-catalog-import.js +62 -0
- package/dist/services/hermes-cli-parser.js +671 -0
- package/dist/services/hermes-core-rollout.js +90 -0
- package/dist/services/hermes-dashboard-endpoint.js +13 -0
- package/dist/services/hermes-plugin-source.js +149 -0
- package/dist/services/hermes-registry-service.js +258 -0
- package/dist/services/hermes-service.js +566 -0
- package/dist/services/hivemem-client.js +85 -0
- package/dist/services/host-local-writer.js +82 -0
- package/dist/services/incident-evidence.js +110 -0
- package/dist/services/incident-lessons.js +186 -0
- package/dist/services/llm-providers/anthropic.js +143 -0
- package/dist/services/llm-providers/index.js +7 -0
- package/dist/services/llm-providers/llama-cpp.js +237 -0
- package/dist/services/llm-providers/ollama.js +292 -0
- package/dist/services/llm-providers/openai-compatible.js +200 -0
- package/dist/services/llm-providers/provider-error.js +36 -0
- package/dist/services/llm-providers/registry.js +45 -0
- package/dist/services/llm-providers/types.js +2 -0
- package/dist/services/log-emitter.js +19 -0
- package/dist/services/log-retention.js +66 -0
- package/dist/services/maintenance-service.js +29 -0
- package/dist/services/managed-cron.js +493 -0
- package/dist/services/managed-ssh-admission.js +68 -0
- package/dist/services/monitor-pulse.js +10 -0
- package/dist/services/node-issue-monitor.js +71 -0
- package/dist/services/node-service.js +35 -0
- package/dist/services/notification-retention.js +51 -0
- package/dist/services/notification-service.js +94 -0
- package/dist/services/operational-observations.js +112 -0
- package/dist/services/operations-config.js +67 -0
- package/dist/services/ops-service.js +491 -0
- package/dist/services/orchestra-extras-service.js +370 -0
- package/dist/services/orchestra-resources-service.js +530 -0
- package/dist/services/os-scheduler-writer.js +73 -0
- package/dist/services/password-service.js +22 -0
- package/dist/services/prediction-service.js +284 -0
- package/dist/services/recovery-exercises.js +682 -0
- package/dist/services/remediation-service.js +325 -0
- package/dist/services/resource-enrollment.js +433 -0
- package/dist/services/resource-node-lock.js +18 -0
- package/dist/services/resource-participant-work.js +302 -0
- package/dist/services/role-service.js +388 -0
- package/dist/services/rollout-service.js +440 -0
- package/dist/services/security-export.js +185 -0
- package/dist/services/security-fusion-actions.js +22 -0
- package/dist/services/security-fusion-approvals.js +83 -0
- package/dist/services/security-fusion-evidence-integrity.js +144 -0
- package/dist/services/security-fusion-ingest.js +101 -0
- package/dist/services/security-fusion-outbox.js +185 -0
- package/dist/services/security-fusion-receipt-chain.js +134 -0
- package/dist/services/security-fusion-service.js +751 -0
- package/dist/services/security-fusion-verification.js +50 -0
- package/dist/services/self-update-service.js +200 -0
- package/dist/services/shared-resource-leases.js +413 -0
- package/dist/services/skill-publication-bundle.js +90 -0
- package/dist/services/skill-publication-settlement.js +36 -0
- package/dist/services/skill-publication-state.js +40 -0
- package/dist/services/skill-registry-service.js +327 -0
- package/dist/services/skill-service.js +148 -0
- package/dist/services/ssh-exec-service.js +139 -0
- package/dist/services/ssh-host-verification.js +27 -0
- package/dist/services/tailscale-node-key-service.js +75 -0
- package/dist/services/team-approvals.js +118 -0
- package/dist/services/topology-service.js +138 -0
- package/dist/services/transfer-authority-snapshot.js +151 -0
- package/dist/services/webhook-credential-migration.js +97 -0
- package/dist/services/webhook-security.js +242 -0
- package/dist/services/webhook-service.js +494 -0
- package/dist/types/auth.js +3 -0
- package/dist/types/index.js +3 -0
- package/dist/types/library.js +3 -0
- package/dist/types/notifications.js +3 -0
- package/dist/types/roles.js +3 -0
- package/dist/types/shared-contracts.js +3 -0
- package/dist/utils/env.js +141 -0
- package/dist/utils/heartbeat-peer.js +19 -0
- package/dist/utils/idempotency.js +45 -0
- package/dist/utils/ip.js +25 -0
- package/dist/utils/json-fields.js +94 -0
- package/dist/utils/local-tls.js +39 -0
- package/dist/utils/magnus-home.js +15 -0
- package/dist/utils/node-url.js +59 -0
- package/dist/utils/redact.js +26 -0
- package/dist/utils/safe-equal.js +13 -0
- package/dist/utils/sentinel-nodes.js +16 -0
- package/dist/utils/validate.js +41 -0
- package/package.json +119 -4
- package/prisma/migrations/20261031000000_baseline/migration.sql +2935 -0
- package/prisma/migrations/migration_lock.toml +3 -0
- package/prisma/schema.prisma +2185 -0
- package/prisma.config.ts +14 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { recordMonitorPulse } from './monitor-pulse.js';
|
|
3
|
+
import { getDb } from '../db/index.js';
|
|
4
|
+
import { getApplicationConfigs } from './operations-config.js';
|
|
5
|
+
import { createNotificationInTransaction } from './notification-service.js';
|
|
6
|
+
import { matchingMaintenance } from './maintenance-service.js';
|
|
7
|
+
import { engineeringMetricEvidenceSha256 } from './engineering-metric-evidence.js';
|
|
8
|
+
export function applicationProbeContractDigest(config) {
|
|
9
|
+
return createHash('sha256').update(JSON.stringify({
|
|
10
|
+
version: 1, id: config.id, projectId: config.projectId, serviceId: config.serviceId,
|
|
11
|
+
nodeId: config.nodeId, baseUrl: config.baseUrl, healthPath: config.healthPath,
|
|
12
|
+
readinessPath: config.readinessPath ?? null, queuePath: config.queuePath ?? null,
|
|
13
|
+
maxQueueAgeMs: config.maxQueueAgeMs, tokenEnv: config.tokenEnv ?? null, kind: config.kind,
|
|
14
|
+
})).digest('hex');
|
|
15
|
+
}
|
|
16
|
+
export async function boundedJson(response, maxBytes = 65536) {
|
|
17
|
+
const reader = response.body?.getReader();
|
|
18
|
+
if (!reader)
|
|
19
|
+
throw new Error('Empty response');
|
|
20
|
+
const chunks = [];
|
|
21
|
+
let bytes = 0;
|
|
22
|
+
try {
|
|
23
|
+
for (;;) {
|
|
24
|
+
const part = await reader.read();
|
|
25
|
+
if (part.done)
|
|
26
|
+
break;
|
|
27
|
+
bytes += part.value.byteLength;
|
|
28
|
+
if (bytes > maxBytes)
|
|
29
|
+
throw new Error('Response limit exceeded');
|
|
30
|
+
chunks.push(part.value);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
await reader.cancel();
|
|
35
|
+
}
|
|
36
|
+
return JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
37
|
+
}
|
|
38
|
+
export function summarizeProbeBody(body, kind) {
|
|
39
|
+
if (!body || typeof body !== 'object' || Array.isArray(body))
|
|
40
|
+
return { status: 'unknown', releaseRevision: null };
|
|
41
|
+
const value = body;
|
|
42
|
+
const status = String(value['status'] ?? '').toLowerCase();
|
|
43
|
+
const failed = ['fail', 'failed', 'degraded', 'unhealthy', 'error', 'unavailable'].includes(status) ||
|
|
44
|
+
value['ok'] === false;
|
|
45
|
+
const good = ['ok', 'ready', 'healthy', 'pass', 'online'].includes(status) || value['ok'] === true;
|
|
46
|
+
const revision = value['revision'] ?? value['commit'];
|
|
47
|
+
const result = {
|
|
48
|
+
status: failed ? 'fail' : good ? 'pass' : 'unknown',
|
|
49
|
+
releaseRevision: typeof revision === 'string' && /^[a-zA-Z0-9._-]{1,100}$/.test(revision) ? revision : null,
|
|
50
|
+
};
|
|
51
|
+
const nested = value['data'] && typeof value['data'] === 'object'
|
|
52
|
+
? value['data']
|
|
53
|
+
: value;
|
|
54
|
+
// `workflow-queue`: a worker service that reports readiness and a workflow queue.
|
|
55
|
+
if (kind === 'workflow-queue' && typeof nested['ready'] === 'boolean')
|
|
56
|
+
result.status = nested['ready'] ? 'pass' : 'fail';
|
|
57
|
+
if (kind === 'workflow-queue' && value['workflows'] && typeof value['workflows'] === 'object') {
|
|
58
|
+
const queue = value['workflows'];
|
|
59
|
+
result.status =
|
|
60
|
+
typeof queue['workers'] !== 'number'
|
|
61
|
+
? 'unknown'
|
|
62
|
+
: queue['workers'] < 1 || queue['paused'] === true
|
|
63
|
+
? 'fail'
|
|
64
|
+
: 'pass';
|
|
65
|
+
}
|
|
66
|
+
// `backend-pool`: a gateway that reports a pool of backends and a draining flag.
|
|
67
|
+
if (kind === 'backend-pool' && Array.isArray(value['backends'])) {
|
|
68
|
+
result.status = value['backends'].some((b) => b && typeof b === 'object' && b.status === 'healthy')
|
|
69
|
+
? 'pass'
|
|
70
|
+
: 'fail';
|
|
71
|
+
}
|
|
72
|
+
if (kind === 'backend-pool' && value['draining'] === true)
|
|
73
|
+
result.status = 'fail';
|
|
74
|
+
return result;
|
|
75
|
+
}
|
|
76
|
+
export async function probeApplication(config) {
|
|
77
|
+
const result = {
|
|
78
|
+
status: 'unknown',
|
|
79
|
+
observedAt: new Date().toISOString(),
|
|
80
|
+
checks: [],
|
|
81
|
+
releaseRevision: null,
|
|
82
|
+
probeContractSha256: applicationProbeContractDigest(config),
|
|
83
|
+
};
|
|
84
|
+
for (const [name, path, authenticated] of [
|
|
85
|
+
['reachability', config.healthPath, false],
|
|
86
|
+
...(config.readinessPath ? [['readiness', config.readinessPath, true]] : []),
|
|
87
|
+
...(config.queuePath ? [['worker-queue', config.queuePath, true]] : []),
|
|
88
|
+
]) {
|
|
89
|
+
const token = config.tokenEnv ? process.env[config.tokenEnv] : undefined;
|
|
90
|
+
if (authenticated && config.tokenEnv && !token) {
|
|
91
|
+
result.checks.push({
|
|
92
|
+
name,
|
|
93
|
+
status: 'unconfigured',
|
|
94
|
+
detail: 'Scoped credential is not configured',
|
|
95
|
+
method: 'GET',
|
|
96
|
+
path, httpStatus: null,
|
|
97
|
+
});
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
const probeStarted = Date.now();
|
|
101
|
+
try {
|
|
102
|
+
// Targets are operator-defined HTTPS origins; never follow a redirect with credentials.
|
|
103
|
+
const response = await fetch(new URL(path, config.baseUrl), {
|
|
104
|
+
method: 'GET',
|
|
105
|
+
redirect: 'error',
|
|
106
|
+
signal: AbortSignal.timeout(8000),
|
|
107
|
+
headers: {
|
|
108
|
+
...(authenticated && token ? { Authorization: `Bearer ${token}` } : {}),
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
if (!response.ok) {
|
|
112
|
+
await response.body?.cancel();
|
|
113
|
+
result.checks.push({
|
|
114
|
+
name,
|
|
115
|
+
status: 'fail',
|
|
116
|
+
method: 'GET', path, httpStatus: response.status,
|
|
117
|
+
durationMs: Date.now() - probeStarted,
|
|
118
|
+
detail: response.status === 401 || response.status === 403
|
|
119
|
+
? 'Credential rejected'
|
|
120
|
+
: `HTTP ${response.status}`,
|
|
121
|
+
});
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const body = await boundedJson(response);
|
|
125
|
+
const summary = summarizeProbeBody(body, config.kind);
|
|
126
|
+
let gpuQueueDepth;
|
|
127
|
+
if (config.kind === 'hive-router' && name === 'readiness') {
|
|
128
|
+
summary.status =
|
|
129
|
+
body &&
|
|
130
|
+
typeof body === 'object' &&
|
|
131
|
+
Array.isArray(body['data'])
|
|
132
|
+
? 'pass'
|
|
133
|
+
: 'unknown';
|
|
134
|
+
}
|
|
135
|
+
if (config.kind === 'workflow-queue' && body && typeof body === 'object') {
|
|
136
|
+
const queue = body.workflows;
|
|
137
|
+
if (typeof queue?.latencyMsP50 === 'number' && queue.latencyMsP50 > config.maxQueueAgeMs)
|
|
138
|
+
summary.status = 'fail';
|
|
139
|
+
if (name === 'worker-queue' && queue?.counts && typeof queue.counts === 'object') {
|
|
140
|
+
const values = ['waiting', 'delayed', 'paused'].map(key => queue.counts?.[key]);
|
|
141
|
+
if (values.every(value => typeof value === 'number' && Number.isInteger(value) && value >= 0))
|
|
142
|
+
gpuQueueDepth = values.reduce((total, value) => total + value, 0);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
result.checks.push({
|
|
146
|
+
name,
|
|
147
|
+
status: summary.status,
|
|
148
|
+
method: 'GET', path, httpStatus: response.status,
|
|
149
|
+
durationMs: Date.now() - probeStarted,
|
|
150
|
+
...(gpuQueueDepth === undefined ? {} : { gpuQueueDepth }),
|
|
151
|
+
detail: summary.status === 'pass'
|
|
152
|
+
? 'Application contract passed'
|
|
153
|
+
: 'Application readiness not proven',
|
|
154
|
+
});
|
|
155
|
+
result.releaseRevision = summary.releaseRevision ?? result.releaseRevision;
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
result.checks.push({
|
|
159
|
+
name,
|
|
160
|
+
status: 'fail',
|
|
161
|
+
detail: 'Transport, timeout, or response-contract failure',
|
|
162
|
+
method: 'GET',
|
|
163
|
+
path, httpStatus: null, durationMs: Date.now() - probeStarted,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
result.status = result.checks.some((c) => c.status === 'fail')
|
|
168
|
+
? 'fail'
|
|
169
|
+
: result.checks.some((c) => c.status === 'unconfigured')
|
|
170
|
+
? 'unconfigured'
|
|
171
|
+
: result.checks.length && result.checks.every((c) => c.status === 'pass')
|
|
172
|
+
? 'pass'
|
|
173
|
+
: 'unknown';
|
|
174
|
+
return result;
|
|
175
|
+
}
|
|
176
|
+
const collector = { startedAt: null, completedAt: null, durationMs: 0, configured: 0, errors: 0 };
|
|
177
|
+
export function getApplicationProbeCollectorStatus() { return { ...collector, running }; }
|
|
178
|
+
export async function collectApplicationProbes() {
|
|
179
|
+
const configs = getApplicationConfigs();
|
|
180
|
+
const started = Date.now();
|
|
181
|
+
collector.startedAt = new Date(started).toISOString();
|
|
182
|
+
collector.configured = configs.length;
|
|
183
|
+
collector.errors = 0;
|
|
184
|
+
let next = 0;
|
|
185
|
+
// At most 25 requests at once. Even 100 applications with three eight-second
|
|
186
|
+
// timeouts fit in 96 seconds, leaving room inside the 180-second freshness TTL.
|
|
187
|
+
await Promise.all(Array.from({ length: Math.min(25, configs.length) }, async () => {
|
|
188
|
+
for (;;) {
|
|
189
|
+
const config = configs[next++];
|
|
190
|
+
if (!config)
|
|
191
|
+
return;
|
|
192
|
+
try {
|
|
193
|
+
await collectApplicationProbe(config);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
collector.errors++;
|
|
197
|
+
console.error('[applications] observation persistence failed');
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}));
|
|
201
|
+
collector.durationMs = Date.now() - started;
|
|
202
|
+
collector.completedAt = new Date().toISOString();
|
|
203
|
+
}
|
|
204
|
+
async function collectApplicationProbe(config) {
|
|
205
|
+
const result = await probeApplication(config);
|
|
206
|
+
await persistApplicationProbeResult(config, result);
|
|
207
|
+
}
|
|
208
|
+
export async function persistApplicationProbeResult(config, result, afterObservation) {
|
|
209
|
+
const db = getDb();
|
|
210
|
+
const retainedResult = { ...result, probeContractSha256: applicationProbeContractDigest(config) };
|
|
211
|
+
const maintenance = await matchingMaintenance(config.nodeId, config.serviceId);
|
|
212
|
+
await db.$transaction(async (tx) => {
|
|
213
|
+
await tx.$executeRaw `SELECT pg_advisory_xact_lock(hashtextextended(${`application:${config.id}`}, 0))`;
|
|
214
|
+
const previous = await tx.applicationObservation.findUnique({ where: { id: config.id } });
|
|
215
|
+
await tx.applicationObservation.upsert({
|
|
216
|
+
where: { id: config.id },
|
|
217
|
+
create: {
|
|
218
|
+
id: config.id,
|
|
219
|
+
projectId: config.projectId,
|
|
220
|
+
serviceId: config.serviceId,
|
|
221
|
+
nodeId: config.nodeId,
|
|
222
|
+
status: result.status,
|
|
223
|
+
resultJson: JSON.stringify(retainedResult),
|
|
224
|
+
observedAt: new Date(result.observedAt),
|
|
225
|
+
},
|
|
226
|
+
update: {
|
|
227
|
+
projectId: config.projectId,
|
|
228
|
+
serviceId: config.serviceId,
|
|
229
|
+
nodeId: config.nodeId,
|
|
230
|
+
status: result.status,
|
|
231
|
+
resultJson: JSON.stringify(retainedResult),
|
|
232
|
+
observedAt: new Date(result.observedAt),
|
|
233
|
+
},
|
|
234
|
+
});
|
|
235
|
+
const observedAt = new Date(result.observedAt);
|
|
236
|
+
const metricRows = result.checks.flatMap(check => {
|
|
237
|
+
const rows = [];
|
|
238
|
+
if (check.status === 'pass' && check.httpStatus !== null && check.httpStatus !== undefined
|
|
239
|
+
&& check.method && check.path && check.durationMs !== undefined && Number.isFinite(check.durationMs) && check.durationMs >= 0) {
|
|
240
|
+
const sample = { applicationId: config.id, nodeId: config.nodeId, metric: 'dependency_latency',
|
|
241
|
+
sourceKey: check.name, unit: 'ms', value: check.durationMs, observedAt,
|
|
242
|
+
probeContractSha256: applicationProbeContractDigest(config), method: check.method, path: check.path,
|
|
243
|
+
httpStatus: check.httpStatus };
|
|
244
|
+
rows.push({ ...sample, evidenceSha256: engineeringMetricEvidenceSha256(sample) });
|
|
245
|
+
}
|
|
246
|
+
if (check.status === 'pass' && check.gpuQueueDepth !== undefined
|
|
247
|
+
&& check.method && check.path && check.httpStatus !== null && check.httpStatus !== undefined
|
|
248
|
+
&& Number.isInteger(check.gpuQueueDepth) && check.gpuQueueDepth >= 0) {
|
|
249
|
+
const sample = { applicationId: config.id, nodeId: config.nodeId, metric: 'gpu_queue',
|
|
250
|
+
sourceKey: check.name, unit: 'jobs', value: check.gpuQueueDepth, observedAt,
|
|
251
|
+
probeContractSha256: applicationProbeContractDigest(config), method: check.method, path: check.path,
|
|
252
|
+
httpStatus: check.httpStatus };
|
|
253
|
+
rows.push({ ...sample, evidenceSha256: engineeringMetricEvidenceSha256(sample) });
|
|
254
|
+
}
|
|
255
|
+
return rows;
|
|
256
|
+
});
|
|
257
|
+
if (metricRows.length)
|
|
258
|
+
await tx.engineeringMetricSample.createMany({ data: metricRows, skipDuplicates: true });
|
|
259
|
+
await afterObservation?.();
|
|
260
|
+
if (result.status !== previous?.status &&
|
|
261
|
+
(result.status === 'fail' || (previous?.status === 'fail' && result.status === 'pass'))) {
|
|
262
|
+
await createNotificationInTransaction(tx, {
|
|
263
|
+
type: `application.${result.status === 'fail' ? 'failed' : 'recovered'}`,
|
|
264
|
+
severity: result.status === 'fail' ? 'warning' : 'info',
|
|
265
|
+
title: `${config.serviceId}: ${result.status === 'fail' ? 'readiness failed' : 'recovered'}`,
|
|
266
|
+
message: result.checks.map((c) => `${c.name}: ${c.detail}`).join('; '),
|
|
267
|
+
nodeId: config.nodeId,
|
|
268
|
+
sourceRef: config.id,
|
|
269
|
+
metadata: {
|
|
270
|
+
serviceId: config.serviceId,
|
|
271
|
+
applicationId: config.id,
|
|
272
|
+
observedAt: result.observedAt,
|
|
273
|
+
},
|
|
274
|
+
}, maintenance?.id ?? null);
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
let timer;
|
|
279
|
+
let running = false;
|
|
280
|
+
export function startApplicationProbes() {
|
|
281
|
+
if (timer)
|
|
282
|
+
return;
|
|
283
|
+
const tick = async () => {
|
|
284
|
+
if (running)
|
|
285
|
+
return;
|
|
286
|
+
running = true;
|
|
287
|
+
try {
|
|
288
|
+
await collectApplicationProbes();
|
|
289
|
+
if (!collector.errors)
|
|
290
|
+
recordMonitorPulse('applications');
|
|
291
|
+
await (await import('./incident-lessons.js')).reconcileLessonInvalidations();
|
|
292
|
+
}
|
|
293
|
+
catch {
|
|
294
|
+
console.error('[applications] collection failed');
|
|
295
|
+
}
|
|
296
|
+
finally {
|
|
297
|
+
running = false;
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
timer = setInterval(() => void tick(), 60000);
|
|
301
|
+
timer.unref();
|
|
302
|
+
void tick();
|
|
303
|
+
}
|
|
304
|
+
export function stopApplicationProbes() {
|
|
305
|
+
if (timer)
|
|
306
|
+
clearInterval(timer);
|
|
307
|
+
timer = undefined;
|
|
308
|
+
}
|
|
309
|
+
//# sourceMappingURL=application-probes.js.map
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/** One-way Ardent-only authority, independent of the node enrollment state. */
|
|
2
|
+
import { createHash, createHmac, timingSafeEqual } from 'node:crypto';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { getDb } from '../db/index.js';
|
|
5
|
+
import { HttpError } from '../middleware/error-handler.js';
|
|
6
|
+
import { apiDrainAttestationSchema, apiDrainAttestationSigningPayload, canonical, dashboardEnrollmentSecret, drainReceiptSchema, enrollmentSigningPayload } from './resource-enrollment.js';
|
|
7
|
+
import { lockResourceNode, resourceNodeIdentity } from './resource-node-lock.js';
|
|
8
|
+
import { reconciliationSecret } from './resource-participant-work.js';
|
|
9
|
+
import { RESULT_SIGNING_SECRET } from '../utils/env.js';
|
|
10
|
+
const PARTICIPANTS = ['agent', 'api', 'dashboard'];
|
|
11
|
+
const HELD = ['reserved', 'running', 'unknown'];
|
|
12
|
+
const sha256 = (value) => createHash('sha256').update(value).digest('hex');
|
|
13
|
+
const hex = z.string().regex(/^[a-f0-9]{64}$/);
|
|
14
|
+
export const ardentScopeActivationSchema = z.object({
|
|
15
|
+
version: z.literal(1), scope: z.literal('managed-ardent-actions-only'),
|
|
16
|
+
excludedParticipant: z.literal('scheduler'),
|
|
17
|
+
nodeId: z.string().min(1).max(128), nodeFqdn: z.string().min(1).max(253),
|
|
18
|
+
epoch: z.number().int().positive(), barrierId: z.string().guid(),
|
|
19
|
+
decisionId: z.string().guid(), approvalRef: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9:._/-]{0,199}$/),
|
|
20
|
+
reviewSha256: hex, schedulerSnapshotSha256: hex,
|
|
21
|
+
receiptSha256: z.object({ agent: hex, api: hex, dashboard: hex }).strict(),
|
|
22
|
+
expiresAt: z.number().int().positive(), signature: hex,
|
|
23
|
+
}).strict();
|
|
24
|
+
export function ardentScopeSigningPayload(value) {
|
|
25
|
+
return 'magnus.ardent-scope-activation.v1\n' + canonical(value);
|
|
26
|
+
}
|
|
27
|
+
function fail() {
|
|
28
|
+
throw new HttpError(409, 'ARDENT_SCOPE_EVIDENCE_INVALID', 'Current reviewed Ardent scope evidence is required');
|
|
29
|
+
}
|
|
30
|
+
export async function assertArdentScope(tx, nodeId, nodeFqdn) {
|
|
31
|
+
const [scope, whole] = await Promise.all([
|
|
32
|
+
tx.ardentScopeEnrollment.findUnique({ where: { nodeId } }),
|
|
33
|
+
tx.resourceEnrollment.findUnique({ where: { nodeId } }),
|
|
34
|
+
]);
|
|
35
|
+
if (!scope || !whole || scope.nodeFqdn !== nodeFqdn || whole.nodeFqdn !== nodeFqdn
|
|
36
|
+
|| whole.epoch !== scope.epoch || whole.barrierId !== scope.barrierId || !['draining', 'enrolled'].includes(whole.state))
|
|
37
|
+
throw new HttpError(409, 'ARDENT_SCOPE_REQUIRED', 'Reviewed Ardent-only claim is required');
|
|
38
|
+
}
|
|
39
|
+
async function receiptHashes(tx, nodeId, nodeFqdn, epoch, barrierId) {
|
|
40
|
+
const hashes = { agent: '', api: '', dashboard: '' };
|
|
41
|
+
for (const participant of PARTICIPANTS) {
|
|
42
|
+
const ack = await tx.resourceDrainParticipant.findUnique({ where: { nodeId_participant: { nodeId, participant } } });
|
|
43
|
+
if (!ack || ack.status !== 'verified' || ack.epoch !== epoch || ack.barrierId !== barrierId
|
|
44
|
+
|| ack.challengeExpiresAt.getTime() <= Date.now() || !ack.receiptJson)
|
|
45
|
+
fail();
|
|
46
|
+
let receipt;
|
|
47
|
+
try {
|
|
48
|
+
receipt = participant === 'api' ? apiDrainAttestationSchema.parse(JSON.parse(ack.receiptJson))
|
|
49
|
+
: drainReceiptSchema.parse(JSON.parse(ack.receiptJson));
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
fail();
|
|
53
|
+
}
|
|
54
|
+
const { signature, ...unsigned } = receipt;
|
|
55
|
+
const key = participant === 'api' ? reconciliationSecret()
|
|
56
|
+
: participant === 'dashboard' ? dashboardEnrollmentSecret() : RESULT_SIGNING_SECRET;
|
|
57
|
+
const payload = participant === 'api'
|
|
58
|
+
? apiDrainAttestationSigningPayload(unsigned)
|
|
59
|
+
: enrollmentSigningPayload(unsigned);
|
|
60
|
+
const expected = createHmac('sha256', key).update(payload).digest();
|
|
61
|
+
if (receipt.participant !== participant || receipt.nodeId !== nodeId || receipt.nodeFqdn !== nodeFqdn
|
|
62
|
+
|| receipt.epoch !== epoch || receipt.barrierId !== barrierId
|
|
63
|
+
|| receipt.challengeId !== ack.challengeId || receipt.expiresAt !== ack.challengeExpiresAt.getTime()
|
|
64
|
+
|| receipt.status !== 'drained' || receipt.activeCount !== 0 || receipt.unknownCount !== 0
|
|
65
|
+
|| receipt.evidenceSha256 !== ack.evidenceSha256 || !timingSafeEqual(expected, Buffer.from(signature, 'hex')))
|
|
66
|
+
fail();
|
|
67
|
+
hashes[participant] = sha256(ack.receiptJson);
|
|
68
|
+
}
|
|
69
|
+
return hashes;
|
|
70
|
+
}
|
|
71
|
+
async function schedulerSnapshot(tx, nodeId) {
|
|
72
|
+
const [ack, definitions, unsettled] = await Promise.all([
|
|
73
|
+
tx.resourceDrainParticipant.findUnique({ where: { nodeId_participant: { nodeId, participant: 'scheduler' } },
|
|
74
|
+
select: { status: true, epoch: true, barrierId: true, evidenceSha256: true } }),
|
|
75
|
+
tx.managedCronDefinition.count({ where: { nodeId, enabled: true } }),
|
|
76
|
+
tx.managedCronOccurrence.count({ where: { nodeId, state: { not: 'completed' } } }),
|
|
77
|
+
]);
|
|
78
|
+
return sha256(canonical({ ack, definitions, unsettled }));
|
|
79
|
+
}
|
|
80
|
+
export async function inspectArdentScope(nodeId) {
|
|
81
|
+
return getDb().$transaction(async (tx) => {
|
|
82
|
+
await lockResourceNode(tx, nodeId);
|
|
83
|
+
const { node, row } = await resourceNodeIdentity(tx, nodeId);
|
|
84
|
+
const scope = await tx.ardentScopeEnrollment.findUnique({ where: { nodeId } });
|
|
85
|
+
return { nodeId, nodeFqdn: node.hostname, wholeNodeState: row?.state ?? 'legacy',
|
|
86
|
+
epoch: row?.epoch ?? 0, barrierId: row?.barrierId ?? null, schedulerSnapshotSha256: await schedulerSnapshot(tx, nodeId),
|
|
87
|
+
scope: scope ? { state: 'active', decisionId: scope.decisionId, evidenceSha256: scope.evidenceSha256,
|
|
88
|
+
activatedAt: scope.activatedAt } : { state: 'pending' } };
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
export async function activateArdentScope(nodeId, raw) {
|
|
92
|
+
const assertion = ardentScopeActivationSchema.parse(raw);
|
|
93
|
+
const { signature, ...unsigned } = assertion;
|
|
94
|
+
const key = reconciliationSecret();
|
|
95
|
+
const expected = createHmac('sha256', key).update(ardentScopeSigningPayload(unsigned)).digest();
|
|
96
|
+
if (!timingSafeEqual(expected, Buffer.from(signature, 'hex'))
|
|
97
|
+
|| assertion.expiresAt <= Date.now() || assertion.expiresAt > Date.now() + 300_000)
|
|
98
|
+
fail();
|
|
99
|
+
return getDb().$transaction(async (tx) => {
|
|
100
|
+
await lockResourceNode(tx, nodeId);
|
|
101
|
+
const { node, row } = await resourceNodeIdentity(tx, nodeId);
|
|
102
|
+
const existing = await tx.ardentScopeEnrollment.findUnique({ where: { nodeId } });
|
|
103
|
+
if (existing) {
|
|
104
|
+
if (existing.assertionJson !== JSON.stringify(assertion))
|
|
105
|
+
fail();
|
|
106
|
+
return { nodeId, nodeFqdn: node.hostname, state: 'active', decisionId: existing.decisionId };
|
|
107
|
+
}
|
|
108
|
+
if (row?.state !== 'draining' || !row.barrierId || assertion.nodeId !== nodeId || assertion.nodeFqdn !== node.hostname
|
|
109
|
+
|| assertion.epoch !== row.epoch || assertion.barrierId !== row.barrierId)
|
|
110
|
+
fail();
|
|
111
|
+
const hashes = await receiptHashes(tx, nodeId, node.hostname, row.epoch, row.barrierId);
|
|
112
|
+
if (canonical(hashes) !== canonical(assertion.receiptSha256)
|
|
113
|
+
|| assertion.schedulerSnapshotSha256 !== await schedulerSnapshot(tx, nodeId)
|
|
114
|
+
|| await tx.commandResourceLease.count({ where: { nodeId, state: { in: HELD } } })
|
|
115
|
+
|| await tx.resourceParticipantWork.count({ where: { nodeId, participant: { in: [...PARTICIPANTS] }, state: { in: HELD } } }))
|
|
116
|
+
fail();
|
|
117
|
+
await tx.ardentScopeEnrollment.create({ data: { nodeId, nodeFqdn: node.hostname,
|
|
118
|
+
epoch: row.epoch, barrierId: row.barrierId, decisionId: assertion.decisionId,
|
|
119
|
+
assertionJson: JSON.stringify(assertion), evidenceSha256: sha256(ardentScopeSigningPayload(unsigned)) } });
|
|
120
|
+
return { nodeId, nodeFqdn: node.hostname, state: 'active', decisionId: assertion.decisionId };
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=ardent-scope-enrollment.js.map
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { ArdentNodeStateSchema } from '@hivemagnus/magnus-contracts';
|
|
2
|
+
import { getDb } from '../db/index.js';
|
|
3
|
+
import { NodeApiClient } from '../sdk/index.js';
|
|
4
|
+
import { getNodeBaseUrl } from '../utils/node-url.js';
|
|
5
|
+
import { getNodeManagerSecret } from '../utils/env.js';
|
|
6
|
+
import { SENTINEL_NODE_IDS } from '../utils/sentinel-nodes.js';
|
|
7
|
+
function clientFor(node) {
|
|
8
|
+
return new NodeApiClient({ baseUrl: getNodeBaseUrl(node.ipAddress, node.port), resourceNodeId: node.id,
|
|
9
|
+
resourceScope: 'ardent-managed-http', nodeToken: getNodeManagerSecret() });
|
|
10
|
+
}
|
|
11
|
+
async function nodeTarget(id) {
|
|
12
|
+
const node = await getDb().node.findFirst({ where: { id, status: { not: 'DECOMMISSIONED' } },
|
|
13
|
+
select: { id: true, name: true, hostname: true, ipAddress: true, port: true } });
|
|
14
|
+
if (!node || SENTINEL_NODE_IDS.includes(node.id))
|
|
15
|
+
throw new Error('Managed node not found');
|
|
16
|
+
return node;
|
|
17
|
+
}
|
|
18
|
+
async function persistObservation(nodeId, state) {
|
|
19
|
+
const observedAt = state.reachable ? new Date(state.checkedAt) : undefined;
|
|
20
|
+
await getDb().$transaction(async (tx) => {
|
|
21
|
+
const installation = await tx.ardentInstallation.upsert({ where: { nodeId },
|
|
22
|
+
create: { nodeId, owner: state.reachable ? state.owner : null, version: state.version,
|
|
23
|
+
installed: state.installed, reachable: state.reachable, error: state.error,
|
|
24
|
+
...(observedAt ? { observedAt } : {}) },
|
|
25
|
+
update: { reachable: state.reachable, error: state.error, checkedAt: new Date(),
|
|
26
|
+
...(state.reachable ? { owner: state.owner, version: state.version, installed: state.installed, observedAt } : {}) } });
|
|
27
|
+
if (!state.reachable)
|
|
28
|
+
return;
|
|
29
|
+
const names = state.profiles.map(profile => profile.id);
|
|
30
|
+
await tx.ardentProfile.deleteMany({ where: { installationId: installation.id, ...(names.length ? { name: { notIn: names } } : {}) } });
|
|
31
|
+
for (const profile of state.profiles) {
|
|
32
|
+
const channels = profile.channels;
|
|
33
|
+
const summary = { displayName: profile.name, backend: profile.backend, apiPort: profile.apiPort,
|
|
34
|
+
running: profile.running, a2aEnabled: profile.a2aEnabled,
|
|
35
|
+
telegramEnabled: channels['telegram'] === true,
|
|
36
|
+
error: profile.error, observedAt: observedAt ?? new Date() };
|
|
37
|
+
await tx.ardentProfile.upsert({ where: { installationId_name: { installationId: installation.id, name: profile.id } },
|
|
38
|
+
create: { installationId: installation.id, name: profile.id, ...summary }, update: summary });
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
async function inspect(node, timeoutMs = 15_000) {
|
|
43
|
+
let state;
|
|
44
|
+
try {
|
|
45
|
+
const response = await clientFor(node).getArdentState(timeoutMs);
|
|
46
|
+
if (!response.success)
|
|
47
|
+
throw new Error('Node did not return Ardent inventory');
|
|
48
|
+
const parsed = ArdentNodeStateSchema.parse(response.data);
|
|
49
|
+
state = { ...parsed, nodeId: node.id, nodeName: node.name, hostname: node.hostname, reachable: true, error: null };
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
state = { nodeId: node.id, nodeName: node.name, hostname: node.hostname, reachable: false,
|
|
53
|
+
installed: false, version: null, owner: '', profiles: [], checkedAt: new Date().toISOString(),
|
|
54
|
+
error: error instanceof Error ? error.message.slice(0, 300) : 'Node inventory unavailable' };
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
await persistObservation(node.id, state);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return { ...state, error: 'Ardent inventory observed, but its central reference could not be stored' };
|
|
61
|
+
}
|
|
62
|
+
return state;
|
|
63
|
+
}
|
|
64
|
+
export async function listArdentInstallations() {
|
|
65
|
+
const db = getDb();
|
|
66
|
+
const nodes = await db.node.findMany({ where: { id: { notIn: SENTINEL_NODE_IDS }, status: { not: 'DECOMMISSIONED' } },
|
|
67
|
+
orderBy: { name: 'asc' }, select: { id: true, name: true, hostname: true, ipAddress: true, port: true } });
|
|
68
|
+
const known = await db.ardentInstallation.findMany({ where: { nodeId: { in: nodes.map(node => node.id) }, installed: true }, select: { nodeId: true } });
|
|
69
|
+
const installed = new Set(known.map(row => row.nodeId));
|
|
70
|
+
// Probe the bounded inventory concurrently. Uninstalled/offline candidates
|
|
71
|
+
// must not hold the entire management page beyond the dashboard proxy limit.
|
|
72
|
+
return Promise.all(nodes.map(node => inspect(node, installed.has(node.id) ? 5_000 : 1_500)));
|
|
73
|
+
}
|
|
74
|
+
export async function getArdentInstallation(id) {
|
|
75
|
+
return inspect(await nodeTarget(id));
|
|
76
|
+
}
|
|
77
|
+
export async function getArdentProfileResource(nodeId, profile, resource, query) {
|
|
78
|
+
return clientFor(await nodeTarget(nodeId)).getArdentProfileResource(profile, resource, query);
|
|
79
|
+
}
|
|
80
|
+
export async function getArdentArtifact(nodeId, profile, artifactId) {
|
|
81
|
+
return clientFor(await nodeTarget(nodeId)).getArdentArtifact(profile, artifactId);
|
|
82
|
+
}
|
|
83
|
+
export async function performArdentAction(nodeId, profile, action, target, body) {
|
|
84
|
+
return clientFor(await nodeTarget(nodeId)).performArdentAction(profile, action, target, body);
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=ardent-service.js.map
|