@hivemagnus/magnus-server 0.0.0-stage → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +171 -0
- package/LICENSE +202 -0
- package/README.md +169 -2
- package/agent-version.json +1 -0
- package/dist/ai/HEARTBEAT.md +56 -0
- package/dist/ai/IDENTITY.md +54 -0
- package/dist/ai/MEMORY.md +40 -0
- package/dist/ai/TOOLS.md +538 -0
- package/dist/ai/USER.md +32 -0
- package/dist/cli/args.js +62 -0
- package/dist/cli/env-catalog.js +131 -0
- package/dist/cli/preflight.js +89 -0
- package/dist/cli.js +175 -0
- package/dist/config/cors.js +16 -0
- package/dist/config/env.js +392 -0
- package/dist/config/features.js +16 -0
- package/dist/config/index.js +3 -0
- package/dist/config/listener-security.js +38 -0
- package/dist/config/version.js +31 -0
- package/dist/db/client.js +44 -0
- package/dist/db/generated/prisma/browser.js +18 -0
- package/dist/db/generated/prisma/client.js +35 -0
- package/dist/db/generated/prisma/commonInputTypes.js +11 -0
- package/dist/db/generated/prisma/enums.js +42 -0
- package/dist/db/generated/prisma/internal/class.js +50 -0
- package/dist/db/generated/prisma/internal/prismaNamespace.js +1598 -0
- package/dist/db/generated/prisma/internal/prismaNamespaceBrowser.js +1569 -0
- package/dist/db/generated/prisma/models/A2ATask.js +2 -0
- package/dist/db/generated/prisma/models/AdmissionFence.js +2 -0
- package/dist/db/generated/prisma/models/Agent.js +2 -0
- package/dist/db/generated/prisma/models/AiConversation.js +2 -0
- package/dist/db/generated/prisma/models/AiDashboard.js +2 -0
- package/dist/db/generated/prisma/models/AiMemory.js +2 -0
- package/dist/db/generated/prisma/models/AiMessage.js +2 -0
- package/dist/db/generated/prisma/models/AiScheduledOp.js +2 -0
- package/dist/db/generated/prisma/models/AlertConditionState.js +2 -0
- package/dist/db/generated/prisma/models/AlertRule.js +2 -0
- package/dist/db/generated/prisma/models/AlertViolation.js +2 -0
- package/dist/db/generated/prisma/models/AnomalyEvent.js +2 -0
- package/dist/db/generated/prisma/models/ApplicationObservation.js +2 -0
- package/dist/db/generated/prisma/models/ArdentInstallation.js +2 -0
- package/dist/db/generated/prisma/models/ArdentProfile.js +2 -0
- package/dist/db/generated/prisma/models/ArdentScopeEnrollment.js +2 -0
- package/dist/db/generated/prisma/models/CommandResourceFence.js +2 -0
- package/dist/db/generated/prisma/models/CommandResourceLease.js +2 -0
- package/dist/db/generated/prisma/models/CommandResult.js +2 -0
- package/dist/db/generated/prisma/models/CronJob.js +2 -0
- package/dist/db/generated/prisma/models/CronJobExecution.js +2 -0
- package/dist/db/generated/prisma/models/DeployJob.js +2 -0
- package/dist/db/generated/prisma/models/DriftScan.js +2 -0
- package/dist/db/generated/prisma/models/EngineeringMetricSample.js +2 -0
- package/dist/db/generated/prisma/models/EngineeringRecoveryExercise.js +2 -0
- package/dist/db/generated/prisma/models/ExecutionRun.js +2 -0
- package/dist/db/generated/prisma/models/FilePatchTask.js +2 -0
- package/dist/db/generated/prisma/models/GpuSnapshot.js +2 -0
- package/dist/db/generated/prisma/models/HealthSnapshot.js +2 -0
- package/dist/db/generated/prisma/models/HermesCoreRollout.js +2 -0
- package/dist/db/generated/prisma/models/HermesCoreRolloutHost.js +2 -0
- package/dist/db/generated/prisma/models/HermesCredentialPool.js +2 -0
- package/dist/db/generated/prisma/models/HermesCredentialPoolAttachment.js +2 -0
- package/dist/db/generated/prisma/models/HermesCronJob.js +2 -0
- package/dist/db/generated/prisma/models/HermesHook.js +2 -0
- package/dist/db/generated/prisma/models/HermesInstallation.js +2 -0
- package/dist/db/generated/prisma/models/HermesMcpServer.js +2 -0
- package/dist/db/generated/prisma/models/HermesMcpServerAttachment.js +2 -0
- package/dist/db/generated/prisma/models/HermesMessagingBinding.js +2 -0
- package/dist/db/generated/prisma/models/HermesOperationJob.js +2 -0
- package/dist/db/generated/prisma/models/HermesPluginInstall.js +2 -0
- package/dist/db/generated/prisma/models/HermesProfile.js +2 -0
- package/dist/db/generated/prisma/models/HermesProviderConfig.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryClient.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryInventory.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryReceipt.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryRevision.js +2 -0
- package/dist/db/generated/prisma/models/HermesRegistryTarget.js +2 -0
- package/dist/db/generated/prisma/models/HermesRole.js +2 -0
- package/dist/db/generated/prisma/models/HermesRoleApplication.js +2 -0
- package/dist/db/generated/prisma/models/HermesSkillAssignment.js +2 -0
- package/dist/db/generated/prisma/models/HermesToolsetPolicy.js +2 -0
- package/dist/db/generated/prisma/models/IncidentLesson.js +2 -0
- package/dist/db/generated/prisma/models/LibraryCategory.js +2 -0
- package/dist/db/generated/prisma/models/LibraryFile.js +2 -0
- package/dist/db/generated/prisma/models/LogEntry.js +2 -0
- package/dist/db/generated/prisma/models/MagnusReleaseCanary.js +2 -0
- package/dist/db/generated/prisma/models/MaintenanceWindow.js +2 -0
- package/dist/db/generated/prisma/models/ManagedCronDefinition.js +2 -0
- package/dist/db/generated/prisma/models/ManagedCronOccurrence.js +2 -0
- package/dist/db/generated/prisma/models/Node.js +2 -0
- package/dist/db/generated/prisma/models/Notification.js +2 -0
- package/dist/db/generated/prisma/models/NotificationPreference.js +2 -0
- package/dist/db/generated/prisma/models/RemediationExecution.js +2 -0
- package/dist/db/generated/prisma/models/RemediationRule.js +2 -0
- package/dist/db/generated/prisma/models/RemoteCommand.js +2 -0
- package/dist/db/generated/prisma/models/ResourceDrainParticipant.js +2 -0
- package/dist/db/generated/prisma/models/ResourceEnrollment.js +2 -0
- package/dist/db/generated/prisma/models/ResourceParticipantWork.js +2 -0
- package/dist/db/generated/prisma/models/Role.js +2 -0
- package/dist/db/generated/prisma/models/RoleApplication.js +2 -0
- package/dist/db/generated/prisma/models/RolloutTarget.js +2 -0
- package/dist/db/generated/prisma/models/SecurityExportOutbox.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionAction.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionActionApproval.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCase.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseActivity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseEvent.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCaseRelationship.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCollector.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionCommandOutbox.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionDeletionReceipt.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionDelivery.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEntityAlias.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEvent.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEventEntity.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionEvidence.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionInvestigation.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionNonce.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionReceiptChainHead.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionRuleVersion.js +2 -0
- package/dist/db/generated/prisma/models/SecurityFusionSourceState.js +2 -0
- package/dist/db/generated/prisma/models/SecurityServiceState.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceFence.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceLease.js +2 -0
- package/dist/db/generated/prisma/models/SharedResourceLeaseResource.js +2 -0
- package/dist/db/generated/prisma/models/SkillInstallation.js +2 -0
- package/dist/db/generated/prisma/models/SkillRegistryEntry.js +2 -0
- package/dist/db/generated/prisma/models/SkillTag.js +2 -0
- package/dist/db/generated/prisma/models/SkillVersion.js +2 -0
- package/dist/db/generated/prisma/models/TeamApprovalDecision.js +2 -0
- package/dist/db/generated/prisma/models/User.js +2 -0
- package/dist/db/generated/prisma/models/WebhookDelivery.js +2 -0
- package/dist/db/generated/prisma/models/WebhookEndpoint.js +2 -0
- package/dist/db/generated/prisma/models.js +2 -0
- package/dist/db/index.js +3 -0
- package/dist/db/read-retry.js +35 -0
- package/dist/db/seed.js +348 -0
- package/dist/index.js +162 -0
- package/dist/middleware/auth.js +361 -0
- package/dist/middleware/body-limit.js +14 -0
- package/dist/middleware/correlation-id.js +13 -0
- package/dist/middleware/error-handler.js +28 -0
- package/dist/middleware/iam-token.js +21 -0
- package/dist/middleware/rate-limit.js +78 -0
- package/dist/middleware/request-logger.js +85 -0
- package/dist/middleware/security-headers.js +23 -0
- package/dist/middleware/tracing.js +120 -0
- package/dist/middleware/user-status.js +33 -0
- package/dist/openapi/spec.js +2656 -0
- package/dist/queue/redis.js +20 -0
- package/dist/queue/requeue.js +27 -0
- package/dist/queue/workers.js +342 -0
- package/dist/routes/ai-schedule.js +73 -0
- package/dist/routes/alerts.js +79 -0
- package/dist/routes/anomalies.js +101 -0
- package/dist/routes/ardent.js +82 -0
- package/dist/routes/assistant.js +417 -0
- package/dist/routes/audit.js +138 -0
- package/dist/routes/codex.js +65 -0
- package/dist/routes/commands.js +226 -0
- package/dist/routes/credentials.js +58 -0
- package/dist/routes/cron.js +292 -0
- package/dist/routes/dependencies.js +49 -0
- package/dist/routes/deploy.js +18 -0
- package/dist/routes/discovery.js +168 -0
- package/dist/routes/docs.js +24 -0
- package/dist/routes/drift.js +126 -0
- package/dist/routes/events-stream.js +87 -0
- package/dist/routes/files.js +214 -0
- package/dist/routes/fleet-checks.js +21 -0
- package/dist/routes/health.js +133 -0
- package/dist/routes/heartbeat.js +545 -0
- package/dist/routes/hermes-registry.js +278 -0
- package/dist/routes/hermes.js +61 -0
- package/dist/routes/library.js +402 -0
- package/dist/routes/logs.js +172 -0
- package/dist/routes/managed-cron.js +23 -0
- package/dist/routes/mcp.js +39 -0
- package/dist/routes/metrics.js +187 -0
- package/dist/routes/nodes.js +951 -0
- package/dist/routes/notifications.js +130 -0
- package/dist/routes/ops.js +258 -0
- package/dist/routes/orchestra.js +430 -0
- package/dist/routes/predictions.js +23 -0
- package/dist/routes/profiling.js +68 -0
- package/dist/routes/quick-command.js +72 -0
- package/dist/routes/remediation.js +111 -0
- package/dist/routes/resource-leases.js +297 -0
- package/dist/routes/roles.js +370 -0
- package/dist/routes/rollout.js +124 -0
- package/dist/routes/scan.js +116 -0
- package/dist/routes/security-backup.js +58 -0
- package/dist/routes/security-fusion.js +605 -0
- package/dist/routes/security-integration.js +71 -0
- package/dist/routes/self-update.js +129 -0
- package/dist/routes/shared-resource-leases.js +78 -0
- package/dist/routes/skills.js +485 -0
- package/dist/routes/stats.js +96 -0
- package/dist/routes/team-approvals.js +23 -0
- package/dist/routes/terminal-ticket.js +15 -0
- package/dist/routes/terminal.js +17 -0
- package/dist/routes/topology.js +37 -0
- package/dist/routes/users.js +219 -0
- package/dist/routes/webhooks.js +137 -0
- package/dist/schemas/command-reconciliation.js +14 -0
- package/dist/schemas/commands.js +51 -0
- package/dist/schemas/cron.js +33 -0
- package/dist/schemas/deploy.js +20 -0
- package/dist/schemas/files.js +14 -0
- package/dist/schemas/hermes.js +69 -0
- package/dist/schemas/index.js +11 -0
- package/dist/schemas/library.js +48 -0
- package/dist/schemas/logs.js +35 -0
- package/dist/schemas/managed-cron.js +127 -0
- package/dist/schemas/node-responses.js +87 -0
- package/dist/schemas/node-session-receipt.js +84 -0
- package/dist/schemas/nodes.js +36 -0
- package/dist/schemas/orchestra.js +118 -0
- package/dist/schemas/resource-participant-reconciliation.js +99 -0
- package/dist/schemas/roles.js +54 -0
- package/dist/schemas/shared-resource-lease.js +132 -0
- package/dist/schemas/skills.js +44 -0
- package/dist/schemas/users.js +28 -0
- package/dist/scripts/run-disposable-recovery-exercise.js +39 -0
- package/dist/sdk/index.js +2 -0
- package/dist/sdk/node-api-client.js +474 -0
- package/dist/security/approval-assertion.js +66 -0
- package/dist/security/command-completion-registry.js +35 -0
- package/dist/security/command-request.js +12 -0
- package/dist/security/command-result-signature.js +34 -0
- package/dist/security/managed-http.js +45 -0
- package/dist/security/native-completion-registry.js +44 -0
- package/dist/server.js +178 -0
- package/dist/services/admission-fence.js +71 -0
- package/dist/services/ai-assistant-service.js +2523 -0
- package/dist/services/ai-correlation-service.js +225 -0
- package/dist/services/ai-export-service.js +141 -0
- package/dist/services/ai-insights-service.js +219 -0
- package/dist/services/ai-schedule-service.js +145 -0
- package/dist/services/ai-scratchpad-service.js +222 -0
- package/dist/services/ai-tools.js +1343 -0
- package/dist/services/alert-service.js +313 -0
- package/dist/services/anomaly-detector.js +419 -0
- package/dist/services/api-local-work.js +79 -0
- package/dist/services/application-probes.js +309 -0
- package/dist/services/ardent-scope-enrollment.js +123 -0
- package/dist/services/ardent-service.js +86 -0
- package/dist/services/audit-service.js +308 -0
- package/dist/services/codex-service.js +209 -0
- package/dist/services/command-lifecycle.js +60 -0
- package/dist/services/command-resource-leases.js +305 -0
- package/dist/services/control-host-node.js +242 -0
- package/dist/services/credential-rotation-service.js +294 -0
- package/dist/services/dependency-service.js +145 -0
- package/dist/services/disposable-recovery-runner.js +595 -0
- package/dist/services/docker-group-health.js +121 -0
- package/dist/services/engineering-assurance.js +492 -0
- package/dist/services/engineering-evidence-ledger.js +94 -0
- package/dist/services/engineering-metric-evidence.js +26 -0
- package/dist/services/event-bus.js +227 -0
- package/dist/services/fleet-check-log.js +102 -0
- package/dist/services/forecast-service.js +154 -0
- package/dist/services/health-retention.js +175 -0
- package/dist/services/heartbeat-stream.js +63 -0
- package/dist/services/hermes-catalog-import.js +62 -0
- package/dist/services/hermes-cli-parser.js +671 -0
- package/dist/services/hermes-core-rollout.js +90 -0
- package/dist/services/hermes-dashboard-endpoint.js +13 -0
- package/dist/services/hermes-plugin-source.js +149 -0
- package/dist/services/hermes-registry-service.js +258 -0
- package/dist/services/hermes-service.js +566 -0
- package/dist/services/hivemem-client.js +85 -0
- package/dist/services/host-local-writer.js +82 -0
- package/dist/services/incident-evidence.js +110 -0
- package/dist/services/incident-lessons.js +186 -0
- package/dist/services/llm-providers/anthropic.js +143 -0
- package/dist/services/llm-providers/index.js +7 -0
- package/dist/services/llm-providers/llama-cpp.js +237 -0
- package/dist/services/llm-providers/ollama.js +292 -0
- package/dist/services/llm-providers/openai-compatible.js +200 -0
- package/dist/services/llm-providers/provider-error.js +36 -0
- package/dist/services/llm-providers/registry.js +45 -0
- package/dist/services/llm-providers/types.js +2 -0
- package/dist/services/log-emitter.js +19 -0
- package/dist/services/log-retention.js +66 -0
- package/dist/services/maintenance-service.js +29 -0
- package/dist/services/managed-cron.js +493 -0
- package/dist/services/managed-ssh-admission.js +68 -0
- package/dist/services/monitor-pulse.js +10 -0
- package/dist/services/node-issue-monitor.js +71 -0
- package/dist/services/node-service.js +35 -0
- package/dist/services/notification-retention.js +51 -0
- package/dist/services/notification-service.js +94 -0
- package/dist/services/operational-observations.js +112 -0
- package/dist/services/operations-config.js +67 -0
- package/dist/services/ops-service.js +491 -0
- package/dist/services/orchestra-extras-service.js +370 -0
- package/dist/services/orchestra-resources-service.js +530 -0
- package/dist/services/os-scheduler-writer.js +73 -0
- package/dist/services/password-service.js +22 -0
- package/dist/services/prediction-service.js +284 -0
- package/dist/services/recovery-exercises.js +682 -0
- package/dist/services/remediation-service.js +325 -0
- package/dist/services/resource-enrollment.js +433 -0
- package/dist/services/resource-node-lock.js +18 -0
- package/dist/services/resource-participant-work.js +302 -0
- package/dist/services/role-service.js +388 -0
- package/dist/services/rollout-service.js +440 -0
- package/dist/services/security-export.js +185 -0
- package/dist/services/security-fusion-actions.js +22 -0
- package/dist/services/security-fusion-approvals.js +83 -0
- package/dist/services/security-fusion-evidence-integrity.js +144 -0
- package/dist/services/security-fusion-ingest.js +101 -0
- package/dist/services/security-fusion-outbox.js +185 -0
- package/dist/services/security-fusion-receipt-chain.js +134 -0
- package/dist/services/security-fusion-service.js +751 -0
- package/dist/services/security-fusion-verification.js +50 -0
- package/dist/services/self-update-service.js +200 -0
- package/dist/services/shared-resource-leases.js +413 -0
- package/dist/services/skill-publication-bundle.js +90 -0
- package/dist/services/skill-publication-settlement.js +36 -0
- package/dist/services/skill-publication-state.js +40 -0
- package/dist/services/skill-registry-service.js +327 -0
- package/dist/services/skill-service.js +148 -0
- package/dist/services/ssh-exec-service.js +139 -0
- package/dist/services/ssh-host-verification.js +27 -0
- package/dist/services/tailscale-node-key-service.js +75 -0
- package/dist/services/team-approvals.js +118 -0
- package/dist/services/topology-service.js +138 -0
- package/dist/services/transfer-authority-snapshot.js +151 -0
- package/dist/services/webhook-credential-migration.js +97 -0
- package/dist/services/webhook-security.js +242 -0
- package/dist/services/webhook-service.js +494 -0
- package/dist/types/auth.js +3 -0
- package/dist/types/index.js +3 -0
- package/dist/types/library.js +3 -0
- package/dist/types/notifications.js +3 -0
- package/dist/types/roles.js +3 -0
- package/dist/types/shared-contracts.js +3 -0
- package/dist/utils/env.js +141 -0
- package/dist/utils/heartbeat-peer.js +19 -0
- package/dist/utils/idempotency.js +45 -0
- package/dist/utils/ip.js +25 -0
- package/dist/utils/json-fields.js +94 -0
- package/dist/utils/local-tls.js +39 -0
- package/dist/utils/magnus-home.js +15 -0
- package/dist/utils/node-url.js +59 -0
- package/dist/utils/redact.js +26 -0
- package/dist/utils/safe-equal.js +13 -0
- package/dist/utils/sentinel-nodes.js +16 -0
- package/dist/utils/validate.js +41 -0
- package/package.json +119 -4
- package/prisma/migrations/20261031000000_baseline/migration.sql +2935 -0
- package/prisma/migrations/migration_lock.toml +3 -0
- package/prisma/schema.prisma +2185 -0
- package/prisma.config.ts +14 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Redis } from 'ioredis';
|
|
2
|
+
let redis;
|
|
3
|
+
/** Singleton Redis connection (local Redis with prefix support). */
|
|
4
|
+
export function getRedis() {
|
|
5
|
+
if (!redis) {
|
|
6
|
+
redis = new Redis(process.env['REDIS_URL'] ?? 'redis://127.0.0.1:6379', {
|
|
7
|
+
maxRetriesPerRequest: null,
|
|
8
|
+
enableReadyCheck: false,
|
|
9
|
+
keyPrefix: process.env['REDIS_PREFIX'] ?? 'magnus:',
|
|
10
|
+
});
|
|
11
|
+
}
|
|
12
|
+
return redis;
|
|
13
|
+
}
|
|
14
|
+
export async function disconnectRedis() {
|
|
15
|
+
if (redis) {
|
|
16
|
+
await redis.quit();
|
|
17
|
+
redis = undefined;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=redis.js.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Add a job under a durable, deterministic job ID. BullMQ silently ignores
|
|
3
|
+
* `add()` while a job with the same ID is retained, including retained
|
|
4
|
+
* finished jobs, so a durable reconciler's requeue would never run again.
|
|
5
|
+
* A retained job in one of `replaceFinished` states is removed first; waiting,
|
|
6
|
+
* delayed, and active jobs are left untouched so the add stays a no-op for them.
|
|
7
|
+
* The database CAS claim in each worker remains the execution boundary.
|
|
8
|
+
*/
|
|
9
|
+
export async function addDurableJob(queue, name, data, opts, replaceFinished = ['completed', 'failed']) {
|
|
10
|
+
const existing = await queue.getJob(opts.jobId);
|
|
11
|
+
if (existing) {
|
|
12
|
+
const state = await existing.getState();
|
|
13
|
+
if (state !== 'unknown' && !replaceFinished.includes(state))
|
|
14
|
+
return;
|
|
15
|
+
if (state !== 'unknown')
|
|
16
|
+
try {
|
|
17
|
+
await existing.remove();
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
// A concurrent reconciler or worker changed the job; the add below is
|
|
21
|
+
// then ignored and the next reconciliation pass retries.
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// BullMQ's generic name/data typing is stricter than the runtime contract.
|
|
25
|
+
await queue.add(name, data, opts);
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=requeue.js.map
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import { ResourceLeaseConflict } from '../services/command-resource-leases.js';
|
|
2
|
+
import { securityFusionLocal } from '../services/security-export.js';
|
|
3
|
+
import { recordMonitorPulse } from '../services/monitor-pulse.js';
|
|
4
|
+
import { Queue, Worker } from 'bullmq';
|
|
5
|
+
import { addDurableJob } from './requeue.js';
|
|
6
|
+
import { validateRemediationCommandForDispatch } from '../services/remediation-service.js';
|
|
7
|
+
import { reconcileCommands } from '../services/command-lifecycle.js';
|
|
8
|
+
import { reconcileRestartVerification } from '../services/security-fusion-verification.js';
|
|
9
|
+
import { getDb } from '../db/index.js';
|
|
10
|
+
import { NodeApiClient } from '../sdk/index.js';
|
|
11
|
+
import { getNodeBaseUrl } from '../utils/node-url.js';
|
|
12
|
+
import { createNotification } from '../services/notification-service.js';
|
|
13
|
+
import { RESULT_SIGNING_PREVIOUS_SECRET, RESULT_SIGNING_SECRET, getNodeManagerSecret, } from '../utils/env.js';
|
|
14
|
+
import { safeParse } from '../utils/validate.js';
|
|
15
|
+
import { commandFenceSchema, commandResultSchema } from '../schemas/node-responses.js';
|
|
16
|
+
import { incMetric } from '../routes/metrics.js';
|
|
17
|
+
import { securityFusionActionSources } from '../services/security-fusion-actions.js';
|
|
18
|
+
import { verifyCommandResultSignature } from '../security/command-result-signature.js';
|
|
19
|
+
// ─── Command execution queue ───
|
|
20
|
+
const REDIS_PREFIX = process.env['REDIS_PREFIX'] ?? 'magnus-';
|
|
21
|
+
const COMMAND_QUEUE = `${REDIS_PREFIX}commands`;
|
|
22
|
+
const REDIS_URL = process.env['REDIS_URL'] ?? 'redis://127.0.0.1:6379';
|
|
23
|
+
let _commandQueue = null;
|
|
24
|
+
let _commandWorker = null;
|
|
25
|
+
let reconciliationTimer;
|
|
26
|
+
let reconciliationBusy = false;
|
|
27
|
+
const COMMAND_EXECUTION_LEASE_MS = 5 * 60 * 1_000;
|
|
28
|
+
const NODE_HEARTBEAT_FRESHNESS_MS = Number(process.env['STALE_NODE_THRESHOLD_MS'] ?? 120_000);
|
|
29
|
+
/** Get the singleton command queue instance. */
|
|
30
|
+
export function getCommandQueue() {
|
|
31
|
+
if (!_commandQueue) {
|
|
32
|
+
_commandQueue = new Queue(COMMAND_QUEUE, {
|
|
33
|
+
connection: { url: REDIS_URL },
|
|
34
|
+
defaultJobOptions: {
|
|
35
|
+
attempts: 1,
|
|
36
|
+
removeOnComplete: { age: 60 * 60, count: 1_000 },
|
|
37
|
+
removeOnFail: { age: 24 * 60 * 60, count: 1_000 },
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
return _commandQueue;
|
|
42
|
+
}
|
|
43
|
+
/** Close the singleton command queue. */
|
|
44
|
+
export async function closeCommandQueue() {
|
|
45
|
+
if (reconciliationTimer)
|
|
46
|
+
clearInterval(reconciliationTimer);
|
|
47
|
+
reconciliationTimer = undefined;
|
|
48
|
+
if (_commandWorker) {
|
|
49
|
+
await _commandWorker.close();
|
|
50
|
+
_commandWorker = null;
|
|
51
|
+
}
|
|
52
|
+
if (_commandQueue) {
|
|
53
|
+
await _commandQueue.close();
|
|
54
|
+
_commandQueue = null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
export function createCommandJobPayload(commandId) {
|
|
58
|
+
return { commandId };
|
|
59
|
+
}
|
|
60
|
+
/** Persist only the database identifier in Redis. All credentials and command
|
|
61
|
+
* fields are loaded immediately before the worker atomically claims execution. */
|
|
62
|
+
export async function enqueueCommand(commandId) {
|
|
63
|
+
// A retained finished job (claim skipped for a stale node, lease conflict
|
|
64
|
+
// reset to QUEUED, transient claim error) must not swallow the requeue.
|
|
65
|
+
await addDurableJob(getCommandQueue(), 'execute', createCommandJobPayload(commandId), {
|
|
66
|
+
jobId: `cmd-${commandId}`,
|
|
67
|
+
attempts: 1,
|
|
68
|
+
removeOnComplete: { age: 60 * 60, count: 1_000 },
|
|
69
|
+
removeOnFail: { age: 24 * 60 * 60, count: 1_000 },
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Verify agent HMAC-SHA256 signature over command result fields.
|
|
74
|
+
* Returns true if valid, false if tampered. Skips if secret is not configured.
|
|
75
|
+
*/
|
|
76
|
+
async function recordSecurityFusionCommandResult(commandId, succeeded, detail) {
|
|
77
|
+
const db = getDb();
|
|
78
|
+
const outbox = await db.securityFusionCommandOutbox.findUnique({
|
|
79
|
+
where: { remoteCommandId: commandId },
|
|
80
|
+
});
|
|
81
|
+
const action = outbox
|
|
82
|
+
? await db.securityFusionAction.findUnique({ where: { id: outbox.actionId } })
|
|
83
|
+
: null;
|
|
84
|
+
if (action && outbox) {
|
|
85
|
+
if (succeeded && action.kind === 'service_restart') {
|
|
86
|
+
const transitioned = await db.securityFusionAction.updateMany({
|
|
87
|
+
where: { id: action.id, status: { in: securityFusionActionSources('verifying') } },
|
|
88
|
+
data: { status: 'verifying', resultJson: JSON.stringify({ commandId, ...detail }) },
|
|
89
|
+
});
|
|
90
|
+
if (transitioned.count !== 1)
|
|
91
|
+
return;
|
|
92
|
+
await db.securityFusionCaseActivity.create({
|
|
93
|
+
data: {
|
|
94
|
+
caseId: action.caseId,
|
|
95
|
+
kind: 'containment.restart_verifying',
|
|
96
|
+
actorId: 'system:command-worker',
|
|
97
|
+
detailJson: JSON.stringify({ actionId: action.id, commandId }),
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const releasing = outbox.operation === 'release';
|
|
103
|
+
const status = succeeded ? (releasing ? 'released' : 'succeeded') : 'failed';
|
|
104
|
+
const transitioned = await db.securityFusionAction.updateMany({
|
|
105
|
+
where: {
|
|
106
|
+
id: action.id,
|
|
107
|
+
status: {
|
|
108
|
+
in: securityFusionActionSources(status),
|
|
109
|
+
},
|
|
110
|
+
},
|
|
111
|
+
data: { status, resultJson: JSON.stringify({ commandId, ...detail }) },
|
|
112
|
+
});
|
|
113
|
+
if (transitioned.count !== 1)
|
|
114
|
+
return;
|
|
115
|
+
await db.securityFusionCaseActivity.create({
|
|
116
|
+
data: {
|
|
117
|
+
caseId: action.caseId,
|
|
118
|
+
kind: `containment.${status}`,
|
|
119
|
+
actorId: 'system:command-worker',
|
|
120
|
+
detailJson: JSON.stringify({ actionId: action.id, commandId, ...detail }),
|
|
121
|
+
},
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
/** Start the command execution worker. */
|
|
126
|
+
export function startCommandWorker() {
|
|
127
|
+
if (_commandWorker)
|
|
128
|
+
return _commandWorker;
|
|
129
|
+
const worker = new Worker(COMMAND_QUEUE, async (job) => {
|
|
130
|
+
const db = getDb();
|
|
131
|
+
const { commandId } = job.data;
|
|
132
|
+
const now = new Date();
|
|
133
|
+
// This compare-and-set is the execution idempotency boundary. Duplicate,
|
|
134
|
+
// delayed, cancelled, recovered, stale-node, or already-finished jobs
|
|
135
|
+
// never reach the side-effecting agent call. The lease cutoff also closes
|
|
136
|
+
// the startup race where BullMQ can deliver a legacy job before stale
|
|
137
|
+
// recovery has completed.
|
|
138
|
+
const claimed = await db.remoteCommand.updateMany({
|
|
139
|
+
where: {
|
|
140
|
+
id: commandId,
|
|
141
|
+
status: 'QUEUED',
|
|
142
|
+
OR: [{ executionDeadline: null }, { executionDeadline: { gt: now } }],
|
|
143
|
+
createdAt: { gte: new Date(now.getTime() - COMMAND_EXECUTION_LEASE_MS) },
|
|
144
|
+
node: {
|
|
145
|
+
status: 'ONLINE',
|
|
146
|
+
lastHeartbeatAt: {
|
|
147
|
+
gte: new Date(now.getTime() - NODE_HEARTBEAT_FRESHNESS_MS),
|
|
148
|
+
},
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
data: { status: 'RUNNING', startedAt: now },
|
|
152
|
+
});
|
|
153
|
+
if (claimed.count !== 1)
|
|
154
|
+
return;
|
|
155
|
+
let command = '<unavailable>';
|
|
156
|
+
let args = [];
|
|
157
|
+
let nodeId = null;
|
|
158
|
+
try {
|
|
159
|
+
// Hydrate only after the atomic claim, so Redis never contains command
|
|
160
|
+
// arguments, environment, endpoint data, or credentials. Keep all
|
|
161
|
+
// parsing and client construction inside the failure boundary so a
|
|
162
|
+
// malformed stored record cannot remain RUNNING.
|
|
163
|
+
const commandRecord = await db.remoteCommand.findUnique({
|
|
164
|
+
where: { id: commandId },
|
|
165
|
+
include: { node: true },
|
|
166
|
+
});
|
|
167
|
+
if (!commandRecord)
|
|
168
|
+
throw new Error('Claimed command no longer exists');
|
|
169
|
+
command = commandRecord.command;
|
|
170
|
+
args = JSON.parse(commandRecord.args);
|
|
171
|
+
if (!Array.isArray(args) || args.some((arg) => typeof arg !== 'string')) {
|
|
172
|
+
throw new Error('Stored command args are malformed');
|
|
173
|
+
}
|
|
174
|
+
const environment = commandRecord.environment
|
|
175
|
+
? JSON.parse(commandRecord.environment)
|
|
176
|
+
: undefined;
|
|
177
|
+
if (environment
|
|
178
|
+
&& (Array.isArray(environment)
|
|
179
|
+
|| typeof environment !== 'object'
|
|
180
|
+
|| Object.entries(environment).some(([key, value]) => !key || typeof value !== 'string'))) {
|
|
181
|
+
throw new Error('Stored command environment is malformed');
|
|
182
|
+
}
|
|
183
|
+
nodeId = commandRecord.nodeId;
|
|
184
|
+
const heartbeat = commandRecord.node.lastHeartbeatAt;
|
|
185
|
+
if (commandRecord.node.status !== 'ONLINE'
|
|
186
|
+
|| !heartbeat
|
|
187
|
+
|| Date.now() - heartbeat.getTime() > NODE_HEARTBEAT_FRESHNESS_MS) {
|
|
188
|
+
throw new Error('Command target is no longer online with a fresh heartbeat');
|
|
189
|
+
}
|
|
190
|
+
const nodeBaseUrl = getNodeBaseUrl(commandRecord.node.ipAddress, commandRecord.node.port);
|
|
191
|
+
const client = new NodeApiClient({
|
|
192
|
+
baseUrl: nodeBaseUrl,
|
|
193
|
+
resourceNodeId: commandRecord.nodeId, resourceActorId: commandRecord.createdBy,
|
|
194
|
+
nodeToken: getNodeManagerSecret(),
|
|
195
|
+
});
|
|
196
|
+
const coordination = commandRecord.coordinationJson ? commandFenceSchema.parse(JSON.parse(commandRecord.coordinationJson)) : undefined;
|
|
197
|
+
if (commandRecord.executionDeadline && (!coordination
|
|
198
|
+
|| coordination.expiresAt !== commandRecord.executionDeadline.getTime()
|
|
199
|
+
|| coordination.resourceKey !== `node:${commandRecord.nodeId}`))
|
|
200
|
+
throw new Error('Approved command deadline/fence binding is missing or differs');
|
|
201
|
+
if (commandRecord.executionDeadline && commandRecord.executionDeadline.getTime() <= Date.now())
|
|
202
|
+
throw new Error('Approved command deadline expired before dispatch');
|
|
203
|
+
await validateRemediationCommandForDispatch(commandRecord);
|
|
204
|
+
const response = await client.executeCommand({
|
|
205
|
+
commandId,
|
|
206
|
+
command,
|
|
207
|
+
args,
|
|
208
|
+
workingDirectory: commandRecord.workingDirectory ?? undefined,
|
|
209
|
+
environment,
|
|
210
|
+
timeoutSeconds: commandRecord.timeoutSeconds,
|
|
211
|
+
...(coordination ? { coordination } : {}),
|
|
212
|
+
});
|
|
213
|
+
const rawResult = response.data;
|
|
214
|
+
// Validate response shape before persisting to DB
|
|
215
|
+
const parsed = safeParse(rawResult, commandResultSchema);
|
|
216
|
+
if (!parsed.success) {
|
|
217
|
+
throw new Error(`Invalid command result from node-manager: ${parsed.error}`);
|
|
218
|
+
}
|
|
219
|
+
const result = parsed.data;
|
|
220
|
+
if (result.commandId !== commandId) {
|
|
221
|
+
throw new Error('Command result commandId does not match the dispatched command');
|
|
222
|
+
}
|
|
223
|
+
// Verify HMAC signature to detect tampered results
|
|
224
|
+
if (!verifyCommandResultSignature(result, RESULT_SIGNING_SECRET, RESULT_SIGNING_PREVIOUS_SECRET)) {
|
|
225
|
+
console.warn(`[CommandWorker] HMAC verification failed for command ${commandId}`);
|
|
226
|
+
throw new Error('HMAC signature verification failed — result may be tampered');
|
|
227
|
+
}
|
|
228
|
+
incMetric('commandsCompleted');
|
|
229
|
+
// Persist the signed result and terminal state atomically. Late workers
|
|
230
|
+
// cannot overwrite a cancellation or an expired unknown outcome.
|
|
231
|
+
const recorded = await db.$transaction(async (tx) => {
|
|
232
|
+
const changed = await tx.remoteCommand.updateMany({
|
|
233
|
+
where: { id: commandId, status: 'RUNNING' },
|
|
234
|
+
data: { status: result.exitCode === 0 ? 'COMPLETED' : 'FAILED', completedAt: new Date() },
|
|
235
|
+
});
|
|
236
|
+
if (!changed.count)
|
|
237
|
+
return false;
|
|
238
|
+
await tx.commandResult.create({ data: { commandId, exitCode: result.exitCode,
|
|
239
|
+
stdout: result.stdout, stderr: result.stderr, durationMs: result.durationMs, truncated: result.truncated } });
|
|
240
|
+
return true;
|
|
241
|
+
});
|
|
242
|
+
if (!recorded)
|
|
243
|
+
return;
|
|
244
|
+
if (securityFusionLocal())
|
|
245
|
+
await recordSecurityFusionCommandResult(commandId, result.exitCode === 0, {
|
|
246
|
+
exitCode: result.exitCode,
|
|
247
|
+
durationMs: result.durationMs,
|
|
248
|
+
});
|
|
249
|
+
// Emit notification on command failure
|
|
250
|
+
if (result.exitCode !== 0) {
|
|
251
|
+
// Look up node name for the notification
|
|
252
|
+
const node = await db.node.findUnique({
|
|
253
|
+
where: { id: nodeId },
|
|
254
|
+
select: { name: true },
|
|
255
|
+
});
|
|
256
|
+
await createNotification({
|
|
257
|
+
type: 'command.failed',
|
|
258
|
+
severity: 'warning',
|
|
259
|
+
title: `Command failed on ${node?.name ?? 'unknown'}`,
|
|
260
|
+
message: `"${command} ${args.join(' ')}" exited with code ${result.exitCode}`,
|
|
261
|
+
nodeId,
|
|
262
|
+
nodeName: node?.name ?? null,
|
|
263
|
+
sourceRef: commandId,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
catch (err) {
|
|
268
|
+
if (err instanceof ResourceLeaseConflict) {
|
|
269
|
+
await db.remoteCommand.updateMany({ where: { id: commandId, status: 'RUNNING' },
|
|
270
|
+
data: { status: 'QUEUED', startedAt: null } });
|
|
271
|
+
return; // Durable reconciler retries only before the original approval deadline.
|
|
272
|
+
}
|
|
273
|
+
const errMsg = err instanceof Error ? err.message : 'unknown error';
|
|
274
|
+
incMetric('commandsFailed');
|
|
275
|
+
const failed = await db.$transaction(async (tx) => {
|
|
276
|
+
const changed = await tx.remoteCommand.updateMany({
|
|
277
|
+
where: { id: commandId, status: 'RUNNING' },
|
|
278
|
+
data: { status: 'FAILED', completedAt: new Date() },
|
|
279
|
+
});
|
|
280
|
+
if (!changed.count)
|
|
281
|
+
return false;
|
|
282
|
+
await tx.commandResult.upsert({
|
|
283
|
+
where: { commandId }, create: { commandId, exitCode: -1, stdout: '',
|
|
284
|
+
stderr: `Worker error; outcome may be unknown: ${errMsg}`, durationMs: 0, truncated: false }, update: {},
|
|
285
|
+
});
|
|
286
|
+
return true;
|
|
287
|
+
});
|
|
288
|
+
if (!failed)
|
|
289
|
+
throw err;
|
|
290
|
+
if (securityFusionLocal())
|
|
291
|
+
await recordSecurityFusionCommandResult(commandId, false, { error: errMsg });
|
|
292
|
+
// Emit notification on worker-level failure
|
|
293
|
+
const node = nodeId ? await db.node.findUnique({
|
|
294
|
+
where: { id: nodeId },
|
|
295
|
+
select: { name: true },
|
|
296
|
+
}).catch(() => null) : null;
|
|
297
|
+
await createNotification({
|
|
298
|
+
type: 'command.failed',
|
|
299
|
+
severity: 'warning',
|
|
300
|
+
title: `Command failed on ${node?.name ?? 'unknown'}`,
|
|
301
|
+
message: `"${command} ${args.join(' ')}" failed: ${errMsg}`,
|
|
302
|
+
nodeId: nodeId ?? undefined,
|
|
303
|
+
nodeName: node?.name ?? null,
|
|
304
|
+
sourceRef: commandId,
|
|
305
|
+
});
|
|
306
|
+
throw err;
|
|
307
|
+
}
|
|
308
|
+
}, {
|
|
309
|
+
connection: { url: REDIS_URL },
|
|
310
|
+
concurrency: 10,
|
|
311
|
+
});
|
|
312
|
+
worker.on('failed', (job, err) => {
|
|
313
|
+
console.error(`[CommandWorker] Job ${job?.id} failed:`, err.message);
|
|
314
|
+
});
|
|
315
|
+
worker.on('error', (err) => {
|
|
316
|
+
console.error('[CommandWorker] Worker error:', err.message);
|
|
317
|
+
});
|
|
318
|
+
const reconcile = async () => {
|
|
319
|
+
if (reconciliationBusy)
|
|
320
|
+
return;
|
|
321
|
+
reconciliationBusy = true;
|
|
322
|
+
try {
|
|
323
|
+
await reconcileCommands();
|
|
324
|
+
if (securityFusionLocal())
|
|
325
|
+
await reconcileRestartVerification();
|
|
326
|
+
recordMonitorPulse('commands');
|
|
327
|
+
}
|
|
328
|
+
catch {
|
|
329
|
+
console.error('[commands] reconciliation pending');
|
|
330
|
+
}
|
|
331
|
+
finally {
|
|
332
|
+
reconciliationBusy = false;
|
|
333
|
+
}
|
|
334
|
+
};
|
|
335
|
+
void reconcile();
|
|
336
|
+
reconciliationTimer = setInterval(() => void reconcile(), 5000);
|
|
337
|
+
reconciliationTimer.unref();
|
|
338
|
+
console.log('⚡ Command worker started');
|
|
339
|
+
_commandWorker = worker;
|
|
340
|
+
return worker;
|
|
341
|
+
}
|
|
342
|
+
//# sourceMappingURL=workers.js.map
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ai-schedule.ts — Routes for Scheduled AI Operations.
|
|
3
|
+
*
|
|
4
|
+
* CRUD + manual trigger for recurring AI tasks.
|
|
5
|
+
*/
|
|
6
|
+
import { Hono } from 'hono';
|
|
7
|
+
import { z } from 'zod';
|
|
8
|
+
import { createScheduledOp, updateScheduledOp, deleteScheduledOp, listScheduledOps, getScheduledOp, executeDueOps, } from '../services/ai-schedule-service.js';
|
|
9
|
+
import { authMiddleware, requireRole } from '../middleware/auth.js';
|
|
10
|
+
export const aiScheduleRouter = new Hono();
|
|
11
|
+
aiScheduleRouter.use('*', authMiddleware);
|
|
12
|
+
const CreateSchema = z.object({
|
|
13
|
+
name: z.string().min(1).max(100),
|
|
14
|
+
description: z.string().max(500).optional(),
|
|
15
|
+
cronExpr: z.string().min(5).max(100),
|
|
16
|
+
type: z.enum(['chat', 'analysis', 'runbook']).optional(),
|
|
17
|
+
prompt: z.string().min(1).max(5000),
|
|
18
|
+
nodeId: z.string().optional(),
|
|
19
|
+
});
|
|
20
|
+
const UpdateSchema = z.object({
|
|
21
|
+
name: z.string().min(1).max(100).optional(),
|
|
22
|
+
description: z.string().max(500).optional(),
|
|
23
|
+
cronExpr: z.string().min(5).max(100).optional(),
|
|
24
|
+
prompt: z.string().min(1).max(5000).optional(),
|
|
25
|
+
enabled: z.boolean().optional(),
|
|
26
|
+
nodeId: z.string().nullable().optional(),
|
|
27
|
+
});
|
|
28
|
+
// List scheduled ops
|
|
29
|
+
aiScheduleRouter.get('/', async (c) => {
|
|
30
|
+
const page = parseInt(c.req.query('page') ?? '1', 10) || 1;
|
|
31
|
+
const limit = Math.min(parseInt(c.req.query('limit') ?? '20', 10) || 20, 100);
|
|
32
|
+
const { data, total } = await listScheduledOps(page, limit);
|
|
33
|
+
return c.json({
|
|
34
|
+
data,
|
|
35
|
+
pagination: { page, limit, total, totalPages: Math.ceil(total / limit) },
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
// Get single scheduled op
|
|
39
|
+
aiScheduleRouter.get('/:id', async (c) => {
|
|
40
|
+
const op = await getScheduledOp(c.req.param('id'));
|
|
41
|
+
if (!op)
|
|
42
|
+
return c.json({ success: false, error: 'Not found' }, 404);
|
|
43
|
+
return c.json({ data: op });
|
|
44
|
+
});
|
|
45
|
+
// Create scheduled op
|
|
46
|
+
aiScheduleRouter.post('/', requireRole('ADMIN', 'OPERATOR'), async (c) => {
|
|
47
|
+
const raw = await c.req.json();
|
|
48
|
+
const body = CreateSchema.parse(raw);
|
|
49
|
+
const user = c.get('user');
|
|
50
|
+
const op = await createScheduledOp({
|
|
51
|
+
...body,
|
|
52
|
+
createdBy: user?.email ?? user?.userId ?? 'unknown',
|
|
53
|
+
});
|
|
54
|
+
return c.json({ data: op }, 201);
|
|
55
|
+
});
|
|
56
|
+
// Update scheduled op
|
|
57
|
+
aiScheduleRouter.patch('/:id', requireRole('ADMIN', 'OPERATOR'), async (c) => {
|
|
58
|
+
const raw = await c.req.json();
|
|
59
|
+
const body = UpdateSchema.parse(raw);
|
|
60
|
+
const op = await updateScheduledOp(c.req.param('id'), body);
|
|
61
|
+
return c.json({ data: op });
|
|
62
|
+
});
|
|
63
|
+
// Delete scheduled op
|
|
64
|
+
aiScheduleRouter.delete('/:id', requireRole('ADMIN', 'OPERATOR'), async (c) => {
|
|
65
|
+
await deleteScheduledOp(c.req.param('id'));
|
|
66
|
+
return c.json({ success: true });
|
|
67
|
+
});
|
|
68
|
+
// Manually trigger all due ops (admin/cron endpoint)
|
|
69
|
+
aiScheduleRouter.post('/execute-due', requireRole('ADMIN'), async (c) => {
|
|
70
|
+
const executed = await executeDueOps();
|
|
71
|
+
return c.json({ success: true, data: { executed } });
|
|
72
|
+
});
|
|
73
|
+
//# sourceMappingURL=ai-schedule.js.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { Hono } from 'hono';
|
|
2
|
+
import { authMiddleware, getUser, requireRole } from '../middleware/auth.js';
|
|
3
|
+
import { logAuditEvent } from '../services/audit-service.js';
|
|
4
|
+
import { HttpError } from '../middleware/error-handler.js';
|
|
5
|
+
import { listAlertRules, getAlertRule, createAlertRule, updateAlertRule, deleteAlertRule, listAlertViolations, } from '../services/alert-service.js';
|
|
6
|
+
export const alertsRouter = new Hono();
|
|
7
|
+
alertsRouter.use('*', authMiddleware);
|
|
8
|
+
// ── List all alert rules ──
|
|
9
|
+
alertsRouter.get('/rules', async (c) => {
|
|
10
|
+
const rules = await listAlertRules();
|
|
11
|
+
return c.json({ rules });
|
|
12
|
+
});
|
|
13
|
+
// ── Get a specific alert rule ──
|
|
14
|
+
alertsRouter.get('/rules/:id', async (c) => {
|
|
15
|
+
const rule = await getAlertRule(c.req.param('id'));
|
|
16
|
+
if (!rule)
|
|
17
|
+
throw new HttpError(404, 'NOT_FOUND', 'Alert rule not found');
|
|
18
|
+
return c.json({ rule });
|
|
19
|
+
});
|
|
20
|
+
// ── Create an alert rule (ADMIN only) ──
|
|
21
|
+
alertsRouter.post('/rules', requireRole('ADMIN'), async (c) => {
|
|
22
|
+
const body = await c.req.json();
|
|
23
|
+
if (!body.name || !body.metric || !body.operator || body.threshold === undefined) {
|
|
24
|
+
throw new HttpError(400, 'VALIDATION_ERROR', 'name, metric, operator, and threshold are required');
|
|
25
|
+
}
|
|
26
|
+
const validMetrics = ['cpu', 'memory', 'disk', 'swap', 'gpu_fan'];
|
|
27
|
+
if (!validMetrics.includes(body.metric)) {
|
|
28
|
+
throw new HttpError(400, 'VALIDATION_ERROR', `metric must be one of: ${validMetrics.join(', ')}`);
|
|
29
|
+
}
|
|
30
|
+
const validOperators = ['gt', 'gte', 'lt', 'lte', 'eq'];
|
|
31
|
+
if (!validOperators.includes(body.operator)) {
|
|
32
|
+
throw new HttpError(400, 'VALIDATION_ERROR', `operator must be one of: ${validOperators.join(', ')}`);
|
|
33
|
+
}
|
|
34
|
+
const user = c.get('user');
|
|
35
|
+
const rule = await createAlertRule({
|
|
36
|
+
...body,
|
|
37
|
+
createdBy: user?.userId ?? user?.email ?? 'system',
|
|
38
|
+
});
|
|
39
|
+
await logAuditEvent({ action: 'alert.rule.create', actorId: getUser(c).userId, targetType: 'alert-rule',
|
|
40
|
+
targetId: rule.id ?? 'unknown', details: { name: body.name, metric: body.metric, operator: body.operator,
|
|
41
|
+
threshold: body.threshold, severity: body.severity ?? null } });
|
|
42
|
+
return c.json({ rule }, 201);
|
|
43
|
+
});
|
|
44
|
+
// ── Update an alert rule (ADMIN only) ──
|
|
45
|
+
alertsRouter.patch('/rules/:id', requireRole('ADMIN'), async (c) => {
|
|
46
|
+
const id = c.req.param('id');
|
|
47
|
+
const existing = await getAlertRule(id);
|
|
48
|
+
if (!existing)
|
|
49
|
+
throw new HttpError(404, 'NOT_FOUND', 'Alert rule not found');
|
|
50
|
+
const body = await c.req.json();
|
|
51
|
+
const rule = await updateAlertRule(id, body);
|
|
52
|
+
await logAuditEvent({ action: 'alert.rule.update', actorId: getUser(c).userId, targetType: 'alert-rule',
|
|
53
|
+
targetId: id, details: { changedFields: Object.keys(body).sort(),
|
|
54
|
+
...(body.active !== undefined && { active: body.active }) } });
|
|
55
|
+
return c.json({ rule });
|
|
56
|
+
});
|
|
57
|
+
// ── Delete an alert rule (ADMIN only) ──
|
|
58
|
+
alertsRouter.delete('/rules/:id', requireRole('ADMIN'), async (c) => {
|
|
59
|
+
const id = c.req.param('id');
|
|
60
|
+
const existing = await getAlertRule(id);
|
|
61
|
+
if (!existing)
|
|
62
|
+
throw new HttpError(404, 'NOT_FOUND', 'Alert rule not found');
|
|
63
|
+
await deleteAlertRule(id);
|
|
64
|
+
await logAuditEvent({ action: 'alert.rule.delete', actorId: getUser(c).userId, targetType: 'alert-rule',
|
|
65
|
+
targetId: id, details: { name: existing.name ?? null } });
|
|
66
|
+
return c.json({ success: true });
|
|
67
|
+
});
|
|
68
|
+
// ── List violations (with optional filters) ──
|
|
69
|
+
alertsRouter.get('/violations', async (c) => {
|
|
70
|
+
const result = await listAlertViolations({
|
|
71
|
+
ruleId: c.req.query('ruleId'),
|
|
72
|
+
nodeId: c.req.query('nodeId'),
|
|
73
|
+
status: c.req.query('status'),
|
|
74
|
+
page: parseInt(c.req.query('page') ?? '1', 10),
|
|
75
|
+
pageSize: parseInt(c.req.query('pageSize') ?? '50', 10),
|
|
76
|
+
});
|
|
77
|
+
return c.json(result);
|
|
78
|
+
});
|
|
79
|
+
//# sourceMappingURL=alerts.js.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* anomalies.ts — Routes for anomaly detection events.
|
|
3
|
+
*
|
|
4
|
+
* GET /anomalies — Fleet-wide anomaly feed
|
|
5
|
+
* GET /anomalies/:nodeId — Anomalies for a specific node
|
|
6
|
+
* GET /anomalies/:nodeId/summary — Anomaly summary for a node
|
|
7
|
+
* POST /anomalies/:id/resolve — Mark an anomaly as resolved
|
|
8
|
+
* POST /anomalies/resolve-node — Bulk-resolve anomalies for a node
|
|
9
|
+
*/
|
|
10
|
+
import { Hono } from 'hono';
|
|
11
|
+
import { authMiddleware, requireRole } from '../middleware/auth.js';
|
|
12
|
+
import { HttpError } from '../middleware/error-handler.js';
|
|
13
|
+
import { listAnomalyEvents, getNodeAnomalySummary, resolveAnomaly, resolveNodeAnomalies, getFleetAnomalySummary, purgeResolvedAnomalies, } from '../services/anomaly-detector.js';
|
|
14
|
+
import { correlateAnomalies, getAnomalyCorrelationDetails, getAnomalyTimeline, } from '../services/ai-correlation-service.js';
|
|
15
|
+
export const anomaliesRouter = new Hono();
|
|
16
|
+
anomaliesRouter.use('*', authMiddleware);
|
|
17
|
+
// ── Fleet-wide anomaly feed ──
|
|
18
|
+
anomaliesRouter.get('/', async (c) => {
|
|
19
|
+
const result = await listAnomalyEvents({
|
|
20
|
+
nodeId: c.req.query('nodeId'),
|
|
21
|
+
metric: c.req.query('metric'),
|
|
22
|
+
severity: c.req.query('severity'),
|
|
23
|
+
resolved: c.req.query('resolved') === 'true' ? true : c.req.query('resolved') === 'false' ? false : undefined,
|
|
24
|
+
page: parseInt(c.req.query('page') ?? '1', 10),
|
|
25
|
+
pageSize: parseInt(c.req.query('pageSize') ?? '50', 10),
|
|
26
|
+
});
|
|
27
|
+
return c.json({ success: true, data: result });
|
|
28
|
+
});
|
|
29
|
+
// ── Fleet-wide anomaly summary (for AI / dashboard overview) ──
|
|
30
|
+
anomaliesRouter.get('/summary', async (c) => {
|
|
31
|
+
const summary = await getFleetAnomalySummary();
|
|
32
|
+
return c.json({ success: true, data: summary });
|
|
33
|
+
});
|
|
34
|
+
// ── Cross-fleet anomaly groups ──
|
|
35
|
+
anomaliesRouter.get('/correlations', async (c) => {
|
|
36
|
+
const windowMinutes = Math.min(Math.max(Number(c.req.query('windowMinutes') ?? '5'), 1), 120);
|
|
37
|
+
const includeResolved = c.req.query('includeResolved') === 'true';
|
|
38
|
+
const groups = await correlateAnomalies({ windowMinutes, includeResolved });
|
|
39
|
+
return c.json({ success: true, data: groups });
|
|
40
|
+
});
|
|
41
|
+
// ── Time-ordered anomaly and health context ──
|
|
42
|
+
anomaliesRouter.get('/timeline', async (c) => {
|
|
43
|
+
const hours = Math.min(Math.max(Number(c.req.query('hours') ?? '24'), 1), 24 * 30);
|
|
44
|
+
const nodeId = c.req.query('nodeId') ?? undefined;
|
|
45
|
+
return c.json({ success: true, data: await getAnomalyTimeline({ hours, nodeId }) });
|
|
46
|
+
});
|
|
47
|
+
// ── Per-anomaly AI correlation details ──
|
|
48
|
+
anomaliesRouter.get('/event/:id/correlation', async (c) => {
|
|
49
|
+
const details = await getAnomalyCorrelationDetails(c.req.param('id'));
|
|
50
|
+
if (!details) {
|
|
51
|
+
throw new HttpError(404, 'NOT_FOUND', 'Anomaly event not found');
|
|
52
|
+
}
|
|
53
|
+
return c.json({ success: true, data: details });
|
|
54
|
+
});
|
|
55
|
+
// ── Node-specific anomaly summary ──
|
|
56
|
+
anomaliesRouter.get('/:nodeId/summary', async (c) => {
|
|
57
|
+
const nodeId = c.req.param('nodeId');
|
|
58
|
+
const summary = await getNodeAnomalySummary(nodeId);
|
|
59
|
+
return c.json({ success: true, data: summary });
|
|
60
|
+
});
|
|
61
|
+
// ── Node anomaly list ──
|
|
62
|
+
anomaliesRouter.get('/:nodeId', async (c) => {
|
|
63
|
+
const nodeId = c.req.param('nodeId');
|
|
64
|
+
const result = await listAnomalyEvents({
|
|
65
|
+
nodeId,
|
|
66
|
+
metric: c.req.query('metric'),
|
|
67
|
+
severity: c.req.query('severity'),
|
|
68
|
+
resolved: c.req.query('resolved') === 'true' ? true : c.req.query('resolved') === 'false' ? false : undefined,
|
|
69
|
+
page: parseInt(c.req.query('page') ?? '1', 10),
|
|
70
|
+
pageSize: parseInt(c.req.query('pageSize') ?? '50', 10),
|
|
71
|
+
});
|
|
72
|
+
return c.json({ success: true, data: result });
|
|
73
|
+
});
|
|
74
|
+
// ── Resolve a single anomaly ──
|
|
75
|
+
anomaliesRouter.post('/:id/resolve', requireRole('ADMIN', 'OPERATOR'), async (c) => {
|
|
76
|
+
const id = c.req.param('id');
|
|
77
|
+
try {
|
|
78
|
+
await resolveAnomaly(id);
|
|
79
|
+
return c.json({ success: true });
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
throw new HttpError(404, 'NOT_FOUND', 'Anomaly event not found');
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
// ── Bulk-resolve anomalies for a node ──
|
|
86
|
+
anomaliesRouter.post('/resolve-node', requireRole('ADMIN', 'OPERATOR'), async (c) => {
|
|
87
|
+
const body = await c.req.json();
|
|
88
|
+
if (!body.nodeId) {
|
|
89
|
+
throw new HttpError(400, 'VALIDATION_ERROR', 'nodeId is required');
|
|
90
|
+
}
|
|
91
|
+
const count = await resolveNodeAnomalies(body.nodeId, body.metric);
|
|
92
|
+
return c.json({ success: true, data: { resolved: count } });
|
|
93
|
+
});
|
|
94
|
+
// ── Purge old resolved anomalies ──
|
|
95
|
+
anomaliesRouter.post('/purge', requireRole('ADMIN'), async (c) => {
|
|
96
|
+
const body = await c.req.json().catch(() => ({}));
|
|
97
|
+
const retentionDays = body.retentionDays ?? 30;
|
|
98
|
+
const count = await purgeResolvedAnomalies(retentionDays);
|
|
99
|
+
return c.json({ success: true, data: { deleted: count, retentionDays } });
|
|
100
|
+
});
|
|
101
|
+
//# sourceMappingURL=anomalies.js.map
|