@resolveio/server-lib 22.3.221 → 22.3.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/assistant-core-heuristics.d.ts +11 -0
- package/ai/assistant-core-heuristics.js +356 -0
- package/ai/assistant-core-heuristics.js.map +1 -0
- package/ai/resolveio-platform-intelligence-memory-corpus.d.ts +3 -0
- package/ai/resolveio-platform-intelligence-memory-corpus.js +214 -0
- package/ai/resolveio-platform-intelligence-memory-corpus.js.map +1 -0
- package/ai/resolveio-platform-intelligence-memory.d.ts +20 -0
- package/ai/resolveio-platform-intelligence-memory.js +341 -0
- package/ai/resolveio-platform-intelligence-memory.js.map +1 -0
- package/{src/ai/resolveio-platform-intelligence-types.ts → ai/resolveio-platform-intelligence-types.d.ts} +15 -20
- package/ai/resolveio-platform-intelligence-types.js +4 -0
- package/ai/resolveio-platform-intelligence-types.js.map +1 -0
- package/ai/resolveio-platform-intelligence.d.ts +6 -0
- package/ai/resolveio-platform-intelligence.js +463 -0
- package/ai/resolveio-platform-intelligence.js.map +1 -0
- package/client-server-app.d.ts +1 -0
- package/client-server-app.js +68 -0
- package/client-server-app.js.map +1 -0
- package/collections/ai-run.collection.d.ts +3 -0
- package/collections/ai-run.collection.js +170 -0
- package/collections/ai-run.collection.js.map +1 -0
- package/collections/ai-terminal-conversation.collection.d.ts +2 -0
- package/collections/ai-terminal-conversation.collection.js +140 -0
- package/collections/ai-terminal-conversation.collection.js.map +1 -0
- package/collections/ai-terminal-issue-report.collection.d.ts +2 -0
- package/collections/ai-terminal-issue-report.collection.js +148 -0
- package/collections/ai-terminal-issue-report.collection.js.map +1 -0
- package/collections/ai-terminal-message.collection.d.ts +2 -0
- package/collections/ai-terminal-message.collection.js +121 -0
- package/collections/ai-terminal-message.collection.js.map +1 -0
- package/collections/app-setting.collection.d.ts +3 -0
- package/collections/app-setting.collection.js +103 -0
- package/collections/app-setting.collection.js.map +1 -0
- package/collections/app-status.collection.d.ts +3 -0
- package/collections/app-status.collection.js +57 -0
- package/collections/app-status.collection.js.map +1 -0
- package/collections/communication-metric.collection.d.ts +2 -0
- package/collections/communication-metric.collection.js +133 -0
- package/collections/communication-metric.collection.js.map +1 -0
- package/collections/counter.collection.d.ts +3 -0
- package/collections/counter.collection.js +56 -0
- package/collections/counter.collection.js.map +1 -0
- package/collections/cron-job-history.collection.d.ts +3 -0
- package/collections/cron-job-history.collection.js +137 -0
- package/collections/cron-job-history.collection.js.map +1 -0
- package/collections/cron-job.collection.d.ts +3 -0
- package/collections/cron-job.collection.js +92 -0
- package/collections/cron-job.collection.js.map +1 -0
- package/collections/customer-notification.collection.d.ts +3 -0
- package/collections/customer-notification.collection.js +130 -0
- package/collections/customer-notification.collection.js.map +1 -0
- package/collections/customer-portal-password.collection.d.ts +3 -0
- package/collections/customer-portal-password.collection.js +75 -0
- package/collections/customer-portal-password.collection.js.map +1 -0
- package/collections/email-history.collection.d.ts +3 -0
- package/collections/email-history.collection.js +134 -0
- package/collections/email-history.collection.js.map +1 -0
- package/collections/email-verified.collection.d.ts +3 -0
- package/collections/email-verified.collection.js +62 -0
- package/collections/email-verified.collection.js.map +1 -0
- package/collections/file.collection.d.ts +3 -0
- package/collections/file.collection.js +74 -0
- package/collections/file.collection.js.map +1 -0
- package/collections/flag-update.collection.d.ts +3 -0
- package/collections/flag-update.collection.js +57 -0
- package/collections/flag-update.collection.js.map +1 -0
- package/collections/flag.collection.d.ts +3 -0
- package/collections/flag.collection.js +57 -0
- package/collections/flag.collection.js.map +1 -0
- package/collections/log-method-latency.collection.d.ts +3 -0
- package/collections/log-method-latency.collection.js +77 -0
- package/collections/log-method-latency.collection.js.map +1 -0
- package/collections/log-subscription.collection.d.ts +3 -0
- package/collections/log-subscription.collection.js +80 -0
- package/collections/log-subscription.collection.js.map +1 -0
- package/collections/log.collection.d.ts +3 -0
- package/collections/log.collection.js +93 -0
- package/collections/log.collection.js.map +1 -0
- package/collections/logged-in-users.collection.d.ts +3 -0
- package/collections/logged-in-users.collection.js +67 -0
- package/collections/logged-in-users.collection.js.map +1 -0
- package/collections/monitor-cpu.collection.d.ts +3 -0
- package/collections/monitor-cpu.collection.js +65 -0
- package/collections/monitor-cpu.collection.js.map +1 -0
- package/collections/monitor-function.collection.d.ts +3 -0
- package/collections/monitor-function.collection.js +74 -0
- package/collections/monitor-function.collection.js.map +1 -0
- package/collections/monitor-memory.collection.d.ts +3 -0
- package/collections/monitor-memory.collection.js +77 -0
- package/collections/monitor-memory.collection.js.map +1 -0
- package/collections/monitor-mongo.collection.d.ts +3 -0
- package/collections/monitor-mongo.collection.js +71 -0
- package/collections/monitor-mongo.collection.js.map +1 -0
- package/collections/notification.collection.d.ts +3 -0
- package/collections/notification.collection.js +57 -0
- package/collections/notification.collection.js.map +1 -0
- package/collections/openai-usage-ledger.collection.d.ts +2 -0
- package/collections/openai-usage-ledger.collection.js +188 -0
- package/collections/openai-usage-ledger.collection.js.map +1 -0
- package/collections/report-builder-dashboard-builder.collection.d.ts +3 -0
- package/collections/report-builder-dashboard-builder.collection.js +109 -0
- package/collections/report-builder-dashboard-builder.collection.js.map +1 -0
- package/collections/report-builder-library.collection.d.ts +3 -0
- package/collections/report-builder-library.collection.js +87 -0
- package/collections/report-builder-library.collection.js.map +1 -0
- package/collections/report-builder-report.collection.d.ts +4 -0
- package/collections/report-builder-report.collection.js +184 -0
- package/collections/report-builder-report.collection.js.map +1 -0
- package/collections/user-group.collection.d.ts +4 -0
- package/collections/user-group.collection.js +89 -0
- package/collections/user-group.collection.js.map +1 -0
- package/collections/user-guide.collection.d.ts +3 -0
- package/collections/user-guide.collection.js +57 -0
- package/collections/user-guide.collection.js.map +1 -0
- package/collections/user.collection.d.ts +4 -0
- package/collections/user.collection.js +180 -0
- package/collections/user.collection.js.map +1 -0
- package/cron/cron.d.ts +14 -0
- package/cron/cron.js +216 -0
- package/cron/cron.js.map +1 -0
- package/fixtures/cron-jobs.d.ts +1 -0
- package/fixtures/cron-jobs.js +150 -0
- package/fixtures/cron-jobs.js.map +1 -0
- package/fixtures/init.d.ts +1 -0
- package/fixtures/init.js +91 -0
- package/fixtures/init.js.map +1 -0
- package/http/auth.d.ts +2 -0
- package/http/auth.js +951 -0
- package/http/auth.js.map +1 -0
- package/http/health.d.ts +1 -0
- package/http/health.js +11 -0
- package/http/health.js.map +1 -0
- package/http/home.d.ts +1 -0
- package/http/home.js +134 -0
- package/http/home.js.map +1 -0
- package/http/slow-query-publication.d.ts +2 -0
- package/http/slow-query-publication.js +99 -0
- package/http/slow-query-publication.js.map +1 -0
- package/index.d.ts +1 -0
- package/index.js +19 -0
- package/index.js.map +1 -0
- package/managers/ai-assistant-codex-manager.manager.d.ts +67 -0
- package/managers/ai-assistant-codex-manager.manager.js +1113 -0
- package/managers/ai-assistant-codex-manager.manager.js.map +1 -0
- package/managers/ai-run-evidence.manager.d.ts +36 -0
- package/managers/ai-run-evidence.manager.js +377 -0
- package/managers/ai-run-evidence.manager.js.map +1 -0
- package/managers/communication-metric.manager.d.ts +16 -0
- package/managers/communication-metric.manager.js +134 -0
- package/managers/communication-metric.manager.js.map +1 -0
- package/managers/cron.manager.d.ts +20 -0
- package/managers/cron.manager.js +534 -0
- package/managers/cron.manager.js.map +1 -0
- package/managers/customer-notification-content.manager.d.ts +55 -0
- package/managers/customer-notification-content.manager.js +158 -0
- package/managers/customer-notification-content.manager.js.map +1 -0
- package/managers/diagnostic-manager-bootstrap.d.ts +9 -0
- package/managers/diagnostic-manager-bootstrap.js +260 -0
- package/managers/diagnostic-manager-bootstrap.js.map +1 -0
- package/managers/error-auto-fix.manager.d.ts +149 -0
- package/managers/error-auto-fix.manager.js +3064 -0
- package/managers/error-auto-fix.manager.js.map +1 -0
- package/managers/local-log.manager.d.ts +18 -0
- package/managers/local-log.manager.js +88 -0
- package/managers/local-log.manager.js.map +1 -0
- package/managers/method.manager.d.ts +84 -0
- package/managers/method.manager.js +1964 -0
- package/managers/method.manager.js.map +1 -0
- package/managers/mongo.manager.d.ts +224 -0
- package/managers/mongo.manager.js +5000 -0
- package/managers/mongo.manager.js.map +1 -0
- package/managers/monitor.manager.d.ts +70 -0
- package/managers/monitor.manager.js +550 -0
- package/managers/monitor.manager.js.map +1 -0
- package/managers/openai-usage-ledger.manager.d.ts +30 -0
- package/managers/openai-usage-ledger.manager.js +142 -0
- package/managers/openai-usage-ledger.manager.js.map +1 -0
- package/managers/slow-query-verifier.manager.d.ts +144 -0
- package/managers/slow-query-verifier.manager.js +3857 -0
- package/managers/slow-query-verifier.manager.js.map +1 -0
- package/managers/slow-query.manager.d.ts +28 -0
- package/managers/slow-query.manager.js +468 -0
- package/managers/slow-query.manager.js.map +1 -0
- package/managers/subscription.manager.d.ts +169 -0
- package/managers/subscription.manager.js +3434 -0
- package/managers/subscription.manager.js.map +1 -0
- package/managers/websocket.manager.d.ts +73 -0
- package/managers/websocket.manager.js +673 -0
- package/managers/websocket.manager.js.map +1 -0
- package/managers/worker-dispatcher.manager.d.ts +120 -0
- package/managers/worker-dispatcher.manager.js +1266 -0
- package/managers/worker-dispatcher.manager.js.map +1 -0
- package/managers/worker-server.manager.d.ts +35 -0
- package/managers/worker-server.manager.js +582 -0
- package/managers/worker-server.manager.js.map +1 -0
- package/methods/accounts.d.ts +2 -0
- package/methods/accounts.js +624 -0
- package/methods/accounts.js.map +1 -0
- package/methods/ai-terminal.d.ts +458 -0
- package/methods/ai-terminal.js +27991 -0
- package/methods/ai-terminal.js.map +1 -0
- package/methods/app-settings.d.ts +2 -0
- package/methods/app-settings.js +169 -0
- package/methods/app-settings.js.map +1 -0
- package/methods/aws.d.ts +2 -0
- package/methods/aws.js +877 -0
- package/methods/aws.js.map +1 -0
- package/methods/collections.d.ts +2 -0
- package/methods/collections.js +719 -0
- package/methods/collections.js.map +1 -0
- package/methods/counters.d.ts +2 -0
- package/methods/counters.js +113 -0
- package/methods/counters.js.map +1 -0
- package/methods/cron-jobs.d.ts +2 -0
- package/methods/cron-jobs.js +2475 -0
- package/methods/cron-jobs.js.map +1 -0
- package/methods/customer-notifications.d.ts +2 -0
- package/methods/customer-notifications.js +528 -0
- package/methods/customer-notifications.js.map +1 -0
- package/methods/diagnostics.d.ts +2 -0
- package/methods/diagnostics.js +703 -0
- package/methods/diagnostics.js.map +1 -0
- package/methods/flag-updates.d.ts +2 -0
- package/methods/flag-updates.js +8 -0
- package/methods/flag-updates.js.map +1 -0
- package/methods/flags.d.ts +2 -0
- package/methods/flags.js +8 -0
- package/methods/flags.js.map +1 -0
- package/methods/logs.d.ts +2 -0
- package/methods/logs.js +751 -0
- package/methods/logs.js.map +1 -0
- package/methods/mongo-explorer.d.ts +2 -0
- package/methods/mongo-explorer.js +1808 -0
- package/methods/mongo-explorer.js.map +1 -0
- package/methods/monitor.d.ts +2 -0
- package/methods/monitor.js +543 -0
- package/methods/monitor.js.map +1 -0
- package/methods/pdf.d.ts +2 -0
- package/methods/pdf.js +1216 -0
- package/methods/pdf.js.map +1 -0
- package/methods/publications.d.ts +1 -0
- package/methods/publications.js +183 -0
- package/methods/publications.js.map +1 -0
- package/methods/report-builder.d.ts +2 -0
- package/methods/report-builder.js +3094 -0
- package/methods/report-builder.js.map +1 -0
- package/methods/support.d.ts +2 -0
- package/methods/support.js +430 -0
- package/methods/support.js.map +1 -0
- package/models/ai-run.model.d.ts +19 -0
- package/models/ai-run.model.js +4 -0
- package/models/ai-run.model.js.map +1 -0
- package/models/ai-terminal-conversation.model.d.ts +17 -0
- package/models/ai-terminal-conversation.model.js +4 -0
- package/models/ai-terminal-conversation.model.js.map +1 -0
- package/models/ai-terminal-issue-report.model.d.ts +19 -0
- package/models/ai-terminal-issue-report.model.js +4 -0
- package/models/ai-terminal-issue-report.model.js.map +1 -0
- package/models/ai-terminal-message.model.d.ts +22 -0
- package/models/ai-terminal-message.model.js +4 -0
- package/models/ai-terminal-message.model.js.map +1 -0
- package/models/app-setting.model.d.ts +16 -0
- package/models/app-setting.model.js +4 -0
- package/models/app-setting.model.js.map +1 -0
- package/{src/models/app-status.model.ts → models/app-status.model.d.ts} +2 -3
- package/models/app-status.model.js +4 -0
- package/models/app-status.model.js.map +1 -0
- package/{src/models/billing-logged-in-users.model.ts → models/billing-logged-in-users.model.d.ts} +4 -5
- package/models/billing-logged-in-users.model.js +4 -0
- package/models/billing-logged-in-users.model.js.map +1 -0
- package/models/collection-document.model.d.ts +21 -0
- package/models/collection-document.model.js +4 -0
- package/models/collection-document.model.js.map +1 -0
- package/models/communication-metric.model.d.ts +20 -0
- package/models/communication-metric.model.js +4 -0
- package/models/communication-metric.model.js.map +1 -0
- package/{src/models/counter.model.ts → models/counter.model.d.ts} +3 -4
- package/models/counter.model.js +4 -0
- package/models/counter.model.js.map +1 -0
- package/models/cron-job-history.model.d.ts +15 -0
- package/models/cron-job-history.model.js +4 -0
- package/models/cron-job-history.model.js.map +1 -0
- package/models/cron-job.model.d.ts +14 -0
- package/models/cron-job.model.js +4 -0
- package/models/cron-job.model.js.map +1 -0
- package/models/customer-notification.model.d.ts +26 -0
- package/models/customer-notification.model.js +4 -0
- package/models/customer-notification.model.js.map +1 -0
- package/models/customer-portal-password.model.d.ts +11 -0
- package/models/customer-portal-password.model.js +4 -0
- package/models/customer-portal-password.model.js.map +1 -0
- package/models/dialog.model.d.ts +23 -0
- package/models/dialog.model.js +4 -0
- package/models/dialog.model.js.map +1 -0
- package/models/email-history.model.d.ts +32 -0
- package/{src/models/email-history.model.ts → models/email-history.model.js} +4 -36
- package/models/email-history.model.js.map +1 -0
- package/{src/models/email-verified.model.ts → models/email-verified.model.d.ts} +5 -6
- package/models/email-verified.model.js +4 -0
- package/models/email-verified.model.js.map +1 -0
- package/{src/models/file.model.ts → models/file.model.d.ts} +7 -8
- package/models/file.model.js +4 -0
- package/models/file.model.js.map +1 -0
- package/{src/models/flag-update.model.ts → models/flag-update.model.d.ts} +3 -4
- package/models/flag-update.model.js +4 -0
- package/models/flag-update.model.js.map +1 -0
- package/{src/models/flag.model.ts → models/flag.model.d.ts} +3 -4
- package/models/flag.model.js +4 -0
- package/models/flag.model.js.map +1 -0
- package/models/log-method-latency.model.d.ts +10 -0
- package/models/log-method-latency.model.js +4 -0
- package/models/log-method-latency.model.js.map +1 -0
- package/{src/models/log-subscription.model.ts → models/log-subscription.model.d.ts} +9 -11
- package/models/log-subscription.model.js +4 -0
- package/models/log-subscription.model.js.map +1 -0
- package/models/log.model.d.ts +17 -0
- package/models/log.model.js +4 -0
- package/models/log.model.js.map +1 -0
- package/{src/models/logged-in-users.model.ts → models/logged-in-users.model.d.ts} +5 -6
- package/models/logged-in-users.model.js +4 -0
- package/models/logged-in-users.model.js.map +1 -0
- package/{src/models/method-response.model.ts → models/method-response.model.d.ts} +6 -7
- package/models/method-response.model.js +4 -0
- package/models/method-response.model.js.map +1 -0
- package/models/method.model.d.ts +26 -0
- package/models/method.model.js +4 -0
- package/models/method.model.js.map +1 -0
- package/{src/models/monitor-cpu.model.ts → models/monitor-cpu.model.d.ts} +7 -9
- package/models/monitor-cpu.model.js +4 -0
- package/models/monitor-cpu.model.js.map +1 -0
- package/models/monitor-function.model.d.ts +14 -0
- package/models/monitor-function.model.js +4 -0
- package/models/monitor-function.model.js.map +1 -0
- package/models/monitor-memory.model.d.ts +15 -0
- package/models/monitor-memory.model.js +4 -0
- package/models/monitor-memory.model.js.map +1 -0
- package/models/monitor-mongo.model.d.ts +13 -0
- package/models/monitor-mongo.model.js +4 -0
- package/models/monitor-mongo.model.js.map +1 -0
- package/{src/models/notification.model.ts → models/notification.model.d.ts} +4 -6
- package/models/notification.model.js +4 -0
- package/models/notification.model.js.map +1 -0
- package/models/openai-usage-ledger.model.d.ts +30 -0
- package/models/openai-usage-ledger.model.js +4 -0
- package/models/openai-usage-ledger.model.js.map +1 -0
- package/models/pagination.model.d.ts +11 -0
- package/models/pagination.model.js +28 -0
- package/models/pagination.model.js.map +1 -0
- package/models/permission.model.d.ts +12 -0
- package/models/permission.model.js +4 -0
- package/models/permission.model.js.map +1 -0
- package/models/report-builder-dashboard-builder.model.d.ts +25 -0
- package/models/report-builder-dashboard-builder.model.js +4 -0
- package/models/report-builder-dashboard-builder.model.js.map +1 -0
- package/models/report-builder-library.model.d.ts +17 -0
- package/models/report-builder-library.model.js +4 -0
- package/models/report-builder-library.model.js.map +1 -0
- package/models/report-builder-report.model.d.ts +121 -0
- package/models/report-builder-report.model.js +4 -0
- package/models/report-builder-report.model.js.map +1 -0
- package/models/report-builder.model.d.ts +61 -0
- package/models/report-builder.model.js +4 -0
- package/models/report-builder.model.js.map +1 -0
- package/models/select-data-label.model.d.ts +9 -0
- package/models/select-data-label.model.js +4 -0
- package/models/select-data-label.model.js.map +1 -0
- package/models/server-message.model.d.ts +32 -0
- package/models/server-message.model.js +4 -0
- package/models/server-message.model.js.map +1 -0
- package/models/slow-query-report.model.d.ts +23 -0
- package/models/slow-query-report.model.js +4 -0
- package/models/slow-query-report.model.js.map +1 -0
- package/models/subscription.model.d.ts +31 -0
- package/models/subscription.model.js +4 -0
- package/models/subscription.model.js.map +1 -0
- package/models/support-ticket.model.d.ts +87 -0
- package/models/support-ticket.model.js +4 -0
- package/models/support-ticket.model.js.map +1 -0
- package/models/user-group.model.d.ts +20 -0
- package/models/user-group.model.js +4 -0
- package/models/user-group.model.js.map +1 -0
- package/{src/models/user-guide.model.ts → models/user-guide.model.d.ts} +4 -5
- package/models/user-guide.model.js +4 -0
- package/models/user-guide.model.js.map +1 -0
- package/models/user.model.d.ts +84 -0
- package/models/user.model.js +4 -0
- package/models/user.model.js.map +1 -0
- package/package.json +1 -1
- package/private/images/ResolveIO.png +0 -0
- package/public_api.js +127 -0
- package/public_api.js.map +1 -0
- package/publications/ai-terminal.d.ts +1 -0
- package/publications/ai-terminal.js +122 -0
- package/publications/ai-terminal.js.map +1 -0
- package/publications/app-settings.d.ts +2 -0
- package/publications/app-settings.js +28 -0
- package/publications/app-settings.js.map +1 -0
- package/publications/app-status.d.ts +2 -0
- package/publications/app-status.js +16 -0
- package/publications/app-status.js.map +1 -0
- package/publications/cron-jobs.d.ts +2 -0
- package/publications/cron-jobs.js +88 -0
- package/publications/cron-jobs.js.map +1 -0
- package/publications/customer-notifications.d.ts +2 -0
- package/publications/customer-notifications.js +161 -0
- package/publications/customer-notifications.js.map +1 -0
- package/publications/files.d.ts +2 -0
- package/publications/files.js +36 -0
- package/publications/files.js.map +1 -0
- package/publications/flags-update.d.ts +2 -0
- package/publications/flags-update.js +22 -0
- package/publications/flags-update.js.map +1 -0
- package/publications/flags.d.ts +2 -0
- package/publications/flags.js +22 -0
- package/publications/flags.js.map +1 -0
- package/publications/logs.d.ts +2 -0
- package/publications/logs.js +164 -0
- package/publications/logs.js.map +1 -0
- package/publications/notifications.d.ts +2 -0
- package/publications/notifications.js +16 -0
- package/publications/notifications.js.map +1 -0
- package/publications/report-builder-dashboard-builders.d.ts +2 -0
- package/publications/report-builder-dashboard-builders.js +42 -0
- package/publications/report-builder-dashboard-builders.js.map +1 -0
- package/publications/report-builder-libraries.d.ts +2 -0
- package/publications/report-builder-libraries.js +90 -0
- package/publications/report-builder-libraries.js.map +1 -0
- package/publications/report-builder-reports.d.ts +2 -0
- package/publications/report-builder-reports.js +50 -0
- package/publications/report-builder-reports.js.map +1 -0
- package/publications/super-admin.d.ts +2 -0
- package/publications/super-admin.js +16 -0
- package/publications/super-admin.js.map +1 -0
- package/publications/user-groups.d.ts +1 -0
- package/publications/user-groups.js +16 -0
- package/publications/user-groups.js.map +1 -0
- package/publications/user-guides.d.ts +1 -0
- package/publications/user-guides.js +16 -0
- package/publications/user-guides.js.map +1 -0
- package/resolveio-server-app.d.ts +70 -0
- package/resolveio-server-app.js +801 -0
- package/resolveio-server-app.js.map +1 -0
- package/server-app.d.ts +228 -0
- package/server-app.js +3566 -0
- package/server-app.js.map +1 -0
- package/services/codex-client.d.ts +128 -0
- package/services/codex-client.js +1629 -0
- package/services/codex-client.js.map +1 -0
- package/services/openai-client.d.ts +46 -0
- package/services/openai-client.js +318 -0
- package/services/openai-client.js.map +1 -0
- package/types/error-report.d.ts +25 -0
- package/types/error-report.js +4 -0
- package/types/error-report.js.map +1 -0
- package/types/slow-query-report.d.ts +27 -0
- package/types/slow-query-report.js +6 -0
- package/types/slow-query-report.js.map +1 -0
- package/util/ai-qa-policy.d.ts +124 -0
- package/util/ai-qa-policy.js +736 -0
- package/util/ai-qa-policy.js.map +1 -0
- package/util/ai-run-evidence-adapters.d.ts +109 -0
- package/util/ai-run-evidence-adapters.js +7234 -0
- package/util/ai-run-evidence-adapters.js.map +1 -0
- package/util/ai-run-evidence-dashboard.d.ts +88 -0
- package/util/ai-run-evidence-dashboard.js +343 -0
- package/util/ai-run-evidence-dashboard.js.map +1 -0
- package/util/ai-run-evidence-eval.d.ts +86 -0
- package/util/ai-run-evidence-eval.js +1018 -0
- package/util/ai-run-evidence-eval.js.map +1 -0
- package/util/ai-run-evidence.d.ts +244 -0
- package/util/ai-run-evidence.js +1096 -0
- package/util/ai-run-evidence.js.map +1 -0
- package/util/ai-runner-artifacts.d.ts +82 -0
- package/util/ai-runner-artifacts.js +713 -0
- package/util/ai-runner-artifacts.js.map +1 -0
- package/util/ai-runner-manager-autopilot.d.ts +210 -0
- package/util/ai-runner-manager-autopilot.js +642 -0
- package/util/ai-runner-manager-autopilot.js.map +1 -0
- package/util/ai-runner-manager-policy.d.ts +807 -0
- package/util/ai-runner-manager-policy.js +3501 -0
- package/util/ai-runner-manager-policy.js.map +1 -0
- package/util/ai-runner-qa-auth.d.ts +5 -0
- package/util/ai-runner-qa-auth.js +839 -0
- package/util/ai-runner-qa-auth.js.map +1 -0
- package/util/ai-runner-qa-tools.d.ts +26 -0
- package/util/ai-runner-qa-tools.js +3520 -0
- package/util/ai-runner-qa-tools.js.map +1 -0
- package/util/aicoder-runner-v6.d.ts +426 -0
- package/util/aicoder-runner-v6.js +2464 -0
- package/util/aicoder-runner-v6.js.map +1 -0
- package/util/common.d.ts +31 -0
- package/util/common.js +683 -0
- package/util/common.js.map +1 -0
- package/util/customer-portal-password.d.ts +13 -0
- package/util/customer-portal-password.js +209 -0
- package/util/customer-portal-password.js.map +1 -0
- package/util/error-reporter.d.ts +52 -0
- package/util/error-reporter.js +326 -0
- package/util/error-reporter.js.map +1 -0
- package/util/error-tracking.d.ts +13 -0
- package/util/error-tracking.js +120 -0
- package/util/error-tracking.js.map +1 -0
- package/util/openai-usage-cost.d.ts +6 -0
- package/util/openai-usage-cost.js +103 -0
- package/util/openai-usage-cost.js.map +1 -0
- package/util/report-builder-unwinds.d.ts +15 -0
- package/util/report-builder-unwinds.js +156 -0
- package/util/report-builder-unwinds.js.map +1 -0
- package/util/runner-process-janitor.d.ts +27 -0
- package/util/runner-process-janitor.js +208 -0
- package/util/runner-process-janitor.js.map +1 -0
- package/util/schema-report-builder.d.ts +6 -0
- package/util/schema-report-builder.js +481 -0
- package/util/schema-report-builder.js.map +1 -0
- package/util/slow-query-reporter.d.ts +28 -0
- package/util/slow-query-reporter.js +226 -0
- package/util/slow-query-reporter.js.map +1 -0
- package/util/subscription-dependency-context.d.ts +34 -0
- package/util/subscription-dependency-context.js +1283 -0
- package/util/subscription-dependency-context.js.map +1 -0
- package/util/support-runner-v5.d.ts +1426 -0
- package/util/support-runner-v5.js +7643 -0
- package/util/support-runner-v5.js.map +1 -0
- package/util/tokenizer.d.ts +5 -0
- package/util/tokenizer.js +41 -0
- package/util/tokenizer.js.map +1 -0
- package/workers/codex-runner.worker.d.ts +1 -0
- package/workers/codex-runner.worker.js +192 -0
- package/workers/codex-runner.worker.js.map +1 -0
- package/.nodemon.json +0 -5
- package/.vscode/settings.json +0 -21
- package/AGENTS.md +0 -195
- package/README.md +0 -22
- package/build_package.sh +0 -5
- package/compileDTS.pl +0 -64
- package/docs/ai-assistant-nightly-eval.md +0 -65
- package/docs/ai-assistant-preflight-checklist.md +0 -23
- package/docs/ai-assistant-report-builder-bridge-playbook.md +0 -115
- package/eslint-plugin-custom/index.js +0 -7
- package/eslint-plugin-custom/rules/no-filter-zero-index.js +0 -44
- package/eslint.config.js +0 -103
- package/gulpfile.js +0 -216
- package/methodAndPublicationListGenerator.py +0 -375
- package/mongodbensurers.js +0 -2
- package/mongostop.js +0 -3
- package/scripts/cleanup-bypassed-callmethod-logs.js +0 -616
- package/settings.development.json +0 -25
- package/settings.development.redacted.json +0 -25
- package/src/.env +0 -12
- package/src/ai/assistant-core-heuristics.ts +0 -379
- package/src/ai/resolveio-platform-intelligence-memory-corpus.ts +0 -185
- package/src/ai/resolveio-platform-intelligence-memory.ts +0 -325
- package/src/ai/resolveio-platform-intelligence.ts +0 -462
- package/src/client-server-app.ts +0 -12
- package/src/collections/ai-run.collection.ts +0 -117
- package/src/collections/ai-terminal-conversation.collection.ts +0 -91
- package/src/collections/ai-terminal-issue-report.collection.ts +0 -99
- package/src/collections/ai-terminal-message.collection.ts +0 -77
- package/src/collections/app-setting.collection.ts +0 -104
- package/src/collections/app-status.collection.ts +0 -58
- package/src/collections/communication-metric.collection.ts +0 -84
- package/src/collections/counter.collection.ts +0 -56
- package/src/collections/cron-job-history.collection.ts +0 -94
- package/src/collections/cron-job.collection.ts +0 -92
- package/src/collections/customer-notification.collection.ts +0 -131
- package/src/collections/customer-portal-password.collection.ts +0 -76
- package/src/collections/email-history.collection.ts +0 -134
- package/src/collections/email-verified.collection.ts +0 -62
- package/src/collections/file.collection.ts +0 -74
- package/src/collections/flag-update.collection.ts +0 -57
- package/src/collections/flag.collection.ts +0 -57
- package/src/collections/log-method-latency.collection.ts +0 -77
- package/src/collections/log-subscription.collection.ts +0 -80
- package/src/collections/log.collection.ts +0 -93
- package/src/collections/logged-in-users.collection.ts +0 -67
- package/src/collections/monitor-cpu.collection.ts +0 -65
- package/src/collections/monitor-function.collection.ts +0 -74
- package/src/collections/monitor-memory.collection.ts +0 -77
- package/src/collections/monitor-mongo.collection.ts +0 -71
- package/src/collections/notification.collection.ts +0 -57
- package/src/collections/openai-usage-ledger.collection.ts +0 -131
- package/src/collections/report-builder-dashboard-builder.collection.ts +0 -109
- package/src/collections/report-builder-library.collection.ts +0 -89
- package/src/collections/report-builder-report.collection.ts +0 -184
- package/src/collections/user-group.collection.ts +0 -89
- package/src/collections/user-guide.collection.ts +0 -57
- package/src/collections/user.collection.ts +0 -181
- package/src/cron/cron.ts +0 -117
- package/src/fixtures/cron-jobs.ts +0 -95
- package/src/fixtures/init.ts +0 -35
- package/src/http/auth.ts +0 -818
- package/src/http/health.ts +0 -7
- package/src/http/home.ts +0 -90
- package/src/http/slow-query-publication.ts +0 -49
- package/src/index.ts +0 -1
- package/src/managers/ai-assistant-codex-manager.manager.ts +0 -1131
- package/src/managers/ai-run-evidence.manager.ts +0 -264
- package/src/managers/communication-metric.manager.ts +0 -82
- package/src/managers/cron.manager.ts +0 -333
- package/src/managers/customer-notification-content.manager.ts +0 -236
- package/src/managers/diagnostic-manager-bootstrap.ts +0 -165
- package/src/managers/error-auto-fix.manager.ts +0 -2767
- package/src/managers/local-log.manager.ts +0 -113
- package/src/managers/method.manager.ts +0 -1857
- package/src/managers/mongo.manager.ts +0 -4575
- package/src/managers/monitor.manager.ts +0 -507
- package/src/managers/openai-usage-ledger.manager.ts +0 -112
- package/src/managers/slow-query-verifier.manager.ts +0 -3590
- package/src/managers/slow-query.manager.ts +0 -519
- package/src/managers/subscription.manager.ts +0 -3128
- package/src/managers/websocket.manager.ts +0 -746
- package/src/managers/worker-dispatcher.manager.ts +0 -1360
- package/src/managers/worker-server.manager.ts +0 -536
- package/src/methods/accounts.ts +0 -532
- package/src/methods/ai-terminal.ts +0 -29070
- package/src/methods/app-settings.ts +0 -114
- package/src/methods/aws.ts +0 -649
- package/src/methods/collections.ts +0 -641
- package/src/methods/counters.ts +0 -69
- package/src/methods/cron-jobs.ts +0 -2614
- package/src/methods/customer-notifications.ts +0 -458
- package/src/methods/diagnostics.ts +0 -616
- package/src/methods/flag-updates.ts +0 -7
- package/src/methods/flags.ts +0 -7
- package/src/methods/logs.ts +0 -657
- package/src/methods/mongo-explorer.ts +0 -1880
- package/src/methods/monitor.ts +0 -540
- package/src/methods/pdf.ts +0 -1236
- package/src/methods/publications.ts +0 -129
- package/src/methods/report-builder.ts +0 -3300
- package/src/methods/support.ts +0 -335
- package/src/models/ai-run.model.ts +0 -27
- package/src/models/ai-terminal-conversation.model.ts +0 -19
- package/src/models/ai-terminal-issue-report.model.ts +0 -21
- package/src/models/ai-terminal-message.model.ts +0 -24
- package/src/models/app-setting.model.ts +0 -17
- package/src/models/collection-document.model.ts +0 -24
- package/src/models/communication-metric.model.ts +0 -23
- package/src/models/cron-job-history.model.ts +0 -16
- package/src/models/cron-job.model.ts +0 -15
- package/src/models/customer-notification.model.ts +0 -28
- package/src/models/customer-portal-password.model.ts +0 -12
- package/src/models/dialog.model.ts +0 -25
- package/src/models/log-method-latency.model.ts +0 -11
- package/src/models/log.model.ts +0 -19
- package/src/models/method.model.ts +0 -25
- package/src/models/monitor-function.model.ts +0 -16
- package/src/models/monitor-memory.model.ts +0 -17
- package/src/models/monitor-mongo.model.ts +0 -15
- package/src/models/openai-usage-ledger.model.ts +0 -56
- package/src/models/pagination.model.ts +0 -35
- package/src/models/permission.model.ts +0 -14
- package/src/models/report-builder-dashboard-builder.model.ts +0 -29
- package/src/models/report-builder-library.model.ts +0 -20
- package/src/models/report-builder-report.model.ts +0 -136
- package/src/models/report-builder.model.ts +0 -68
- package/src/models/select-data-label.model.ts +0 -9
- package/src/models/server-message.model.ts +0 -31
- package/src/models/slow-query-report.model.ts +0 -23
- package/src/models/subscription.model.ts +0 -73
- package/src/models/support-ticket.model.ts +0 -104
- package/src/models/user-group.model.ts +0 -24
- package/src/models/user.model.ts +0 -96
- package/src/private/images/ResolveIO.png +0 -0
- package/src/publications/ai-terminal.ts +0 -73
- package/src/publications/app-settings.ts +0 -25
- package/src/publications/app-status.ts +0 -13
- package/src/publications/cron-jobs.ts +0 -40
- package/src/publications/customer-notifications.ts +0 -101
- package/src/publications/files.ts +0 -33
- package/src/publications/flags-update.ts +0 -19
- package/src/publications/flags.ts +0 -19
- package/src/publications/logs.ts +0 -163
- package/src/publications/notifications.ts +0 -13
- package/src/publications/report-builder-dashboard-builders.ts +0 -39
- package/src/publications/report-builder-libraries.ts +0 -41
- package/src/publications/report-builder-reports.ts +0 -47
- package/src/publications/super-admin.ts +0 -13
- package/src/publications/user-groups.ts +0 -12
- package/src/publications/user-guides.ts +0 -12
- package/src/resolveio-server-app.ts +0 -617
- package/src/server-app.ts +0 -3354
- package/src/services/codex-client.ts +0 -1231
- package/src/services/openai-client.ts +0 -265
- package/src/types/error-report.ts +0 -26
- package/src/types/js-tiktoken.d.ts +0 -11
- package/src/types/slow-query-report.ts +0 -28
- package/src/util/ai-qa-policy.ts +0 -925
- package/src/util/ai-run-evidence-adapters.ts +0 -8347
- package/src/util/ai-run-evidence-dashboard.ts +0 -323
- package/src/util/ai-run-evidence-eval.ts +0 -1057
- package/src/util/ai-run-evidence.ts +0 -1430
- package/src/util/ai-runner-artifacts.ts +0 -586
- package/src/util/ai-runner-manager-autopilot.ts +0 -961
- package/src/util/ai-runner-manager-policy.ts +0 -5011
- package/src/util/ai-runner-qa-auth.ts +0 -838
- package/src/util/ai-runner-qa-tools.ts +0 -3536
- package/src/util/aicoder-runner-v6.ts +0 -3121
- package/src/util/common.ts +0 -649
- package/src/util/customer-portal-password.ts +0 -183
- package/src/util/error-reporter.ts +0 -332
- package/src/util/error-tracking.ts +0 -79
- package/src/util/openai-usage-cost.ts +0 -114
- package/src/util/report-builder-unwinds.ts +0 -180
- package/src/util/runner-process-janitor.ts +0 -219
- package/src/util/schema-report-builder.ts +0 -448
- package/src/util/slow-query-reporter.ts +0 -216
- package/src/util/subscription-dependency-context.ts +0 -1096
- package/src/util/support-runner-v5.ts +0 -10040
- package/src/util/tokenizer.ts +0 -38
- package/src/workers/codex-runner.worker.ts +0 -142
- package/start_server.sh +0 -5
- package/tests/ai-assistant-corpus-build.ts +0 -484
- package/tests/ai-assistant-corpus-replay-e2e.ts +0 -774
- package/tests/ai-assistant-data-parity-e2e.ts +0 -1989
- package/tests/ai-assistant-eval-triage.ts +0 -831
- package/tests/ai-assistant-openai-e2e.ts +0 -1061
- package/tests/ai-assistant-openai-git-e2e.ts +0 -155
- package/tests/ai-assistant-preflight-matrix.ts +0 -215
- package/tests/ai-assistant-routing-eval.test.ts +0 -585
- package/tests/ai-assistant-snf-live-eval.ts +0 -975
- package/tests/ai-assistant-utils.test.ts +0 -4834
- package/tests/ai-manager-autopilot-snapshot.test.ts +0 -193
- package/tests/ai-manager-recovery-checkpoint.test.ts +0 -1383
- package/tests/ai-run-eval.test.ts +0 -132
- package/tests/ai-run-evidence.test.ts +0 -3773
- package/tests/ai-runner-contract.test.ts +0 -515
- package/tests/aicoder-runner-v6.test.ts +0 -822
- package/tests/error-reporter.test.ts +0 -145
- package/tests/method-publication-generator.test.ts +0 -46
- package/tests/report-builder-linking.test.ts +0 -79
- package/tests/resolveio-platform-intelligence.test.ts +0 -352
- package/tests/server-app-cron-owner.test.ts +0 -127
- package/tests/subscription-connect-race.test.ts +0 -158
- package/tests/subscription-dependency-context.test.ts +0 -324
- package/tests/subscription-manager-collection-tracking.test.ts +0 -86
- package/tests/subscription-manager-invalidation.test.ts +0 -86
- package/tests/support-runner-v5.test.ts +0 -3201
- package/tsconfig.json +0 -34
- /package/{src/private → private}/email-templates/enrollment.html +0 -0
- /package/{src/private → private}/email-templates/forgot-password.html +0 -0
- /package/{src/private → private}/email-templates/support-ticket-deleted.html +0 -0
- /package/{src/private → private}/email-templates/support-ticket-modified.html +0 -0
- /package/{src/private → private}/email-templates/support-ticket.html +0 -0
- /package/{src/public_api.ts → public_api.d.ts} +0 -0
package/src/server-app.ts
DELETED
|
@@ -1,3354 +0,0 @@
|
|
|
1
|
-
import * as express from 'express';
|
|
2
|
-
import * as xmlParser from 'express-xml-bodyparser';
|
|
3
|
-
import { createServer, Server } from 'http';
|
|
4
|
-
import * as crypto from 'crypto';
|
|
5
|
-
import * as fs from 'fs';
|
|
6
|
-
import * as jwt from 'jsonwebtoken';
|
|
7
|
-
import * as moment from 'moment-timezone';
|
|
8
|
-
import { unpack } from 'msgpackr';
|
|
9
|
-
import * as os from 'os';
|
|
10
|
-
import * as path from 'path';
|
|
11
|
-
import { monitorEventLoopDelay } from 'perf_hooks';
|
|
12
|
-
import * as inspector from 'inspector';
|
|
13
|
-
import { execFile } from 'child_process';
|
|
14
|
-
import { URL } from 'url';
|
|
15
|
-
import * as WebSocket from 'ws';
|
|
16
|
-
|
|
17
|
-
import { Logs } from './collections/log.collection';
|
|
18
|
-
import { Users } from './collections/user.collection';
|
|
19
|
-
import { CronManager } from './managers/cron.manager';
|
|
20
|
-
import { MethodManager } from './managers/method.manager';
|
|
21
|
-
import { MonitorManager, MonitorManagerFunction } from './managers/monitor.manager';
|
|
22
|
-
import { SubscriptionManager } from './managers/subscription.manager';
|
|
23
|
-
import { ServerResponseModel } from './models/server-message.model';
|
|
24
|
-
import { dateReviver, getBinarySize, isAllowedOrigin, objectIdHexString, round } from './util/common';
|
|
25
|
-
import { ErrorReporter } from './util/error-reporter';
|
|
26
|
-
import { ensureErrorWithCorrelation } from './util/error-tracking';
|
|
27
|
-
|
|
28
|
-
import { MongoNetworkTimeoutError } from 'mongodb';
|
|
29
|
-
import { setupAuthRoutes } from './http/auth';
|
|
30
|
-
import { setupHealthRoutes } from './http/health';
|
|
31
|
-
import { setupHomeRoutes } from './http/home';
|
|
32
|
-
import { setupSlowQueryPublicationRoutes } from './http/slow-query-publication';
|
|
33
|
-
|
|
34
|
-
import { WebSocketManager } from './managers/websocket.manager';
|
|
35
|
-
import { WorkerDispatcherManager } from './managers/worker-dispatcher.manager';
|
|
36
|
-
import { WorkerServerManager } from './managers/worker-server.manager';
|
|
37
|
-
import { AiAssistantCodexManager } from './managers/ai-assistant-codex-manager.manager';
|
|
38
|
-
import { ResolveIOServer } from './resolveio-server-app';
|
|
39
|
-
|
|
40
|
-
const CLIENT_REQUEST_LOG_SKIP_LIST = new Set([
|
|
41
|
-
'reportBuilderGetResults',
|
|
42
|
-
'reportBuilderGetDistinctValue',
|
|
43
|
-
'reportBuilderBuildTree',
|
|
44
|
-
'generatePDF',
|
|
45
|
-
'getWOOfflineData',
|
|
46
|
-
'countQuery',
|
|
47
|
-
'countWithQuery',
|
|
48
|
-
'countCollectionWithQuery',
|
|
49
|
-
'find',
|
|
50
|
-
'findOne',
|
|
51
|
-
'findWithOptions',
|
|
52
|
-
'getDrivers',
|
|
53
|
-
'processAirdropDistribution',
|
|
54
|
-
'qbHandleResponse',
|
|
55
|
-
'getSignedUrls',
|
|
56
|
-
'getSignedUrl',
|
|
57
|
-
'getSignedUrlWithId',
|
|
58
|
-
'getSignedUrlsAndFilesWithId',
|
|
59
|
-
'updateDocumentProps',
|
|
60
|
-
'insertDocument',
|
|
61
|
-
'updateDocument',
|
|
62
|
-
'uploadFileAndSave'
|
|
63
|
-
]);
|
|
64
|
-
|
|
65
|
-
function isCorsPreflightRequest(req: express.Request): boolean {
|
|
66
|
-
return req.method === 'OPTIONS' && !!req.headers.origin && !!req.headers['access-control-request-method'];
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function shouldWriteClientRequestLog(methodName: string): boolean {
|
|
70
|
-
return !CLIENT_REQUEST_LOG_SKIP_LIST.has(methodName);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
interface ProcessSnapshot {
|
|
74
|
-
pid: number;
|
|
75
|
-
ppid: number;
|
|
76
|
-
ageSeconds: number;
|
|
77
|
-
cpuPct: number;
|
|
78
|
-
rssKb: number;
|
|
79
|
-
command: string;
|
|
80
|
-
args: string;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export class ResolveIOMainServer {
|
|
84
|
-
private _app: express.Application;
|
|
85
|
-
private _serverHTTP: Server;
|
|
86
|
-
private _portHTTP: number;
|
|
87
|
-
private _serverWSS: WebSocket.Server;
|
|
88
|
-
private _offlineUpdates = [];
|
|
89
|
-
public sesMail = false;
|
|
90
|
-
private publicProgram = false;
|
|
91
|
-
private _rebootFlag = false;
|
|
92
|
-
|
|
93
|
-
private LOGGER = 'ERROR'; //ERROR / DEBUG
|
|
94
|
-
|
|
95
|
-
private _websocketManager: WebSocketManager;
|
|
96
|
-
private _monitorManager: MonitorManager;
|
|
97
|
-
private _monitorManagerFunction: MonitorManagerFunction;
|
|
98
|
-
private _subscriptionManager: SubscriptionManager;
|
|
99
|
-
private _methodManager: MethodManager;
|
|
100
|
-
private _cronManager: CronManager;
|
|
101
|
-
private _clientRoutes: string[] = [];
|
|
102
|
-
private _workerDispatcherManager: WorkerDispatcherManager;
|
|
103
|
-
private _workerServerManager: WorkerServerManager;
|
|
104
|
-
private _aiAssistantCodexManager: AiAssistantCodexManager | null = null;
|
|
105
|
-
private _httpServerClosePromise: Promise<void> | null = null;
|
|
106
|
-
private _websocketServerClosePromise: Promise<void> | null = null;
|
|
107
|
-
private _wsConnectDebug = false;
|
|
108
|
-
private _perfDebug = false;
|
|
109
|
-
private _perfDebugIntervalMs = 2000;
|
|
110
|
-
private _perfDebugTimer: NodeJS.Timeout | null = null;
|
|
111
|
-
private _perfDebugLastCpu: NodeJS.CpuUsage | null = null;
|
|
112
|
-
private _perfDebugLastTs = 0;
|
|
113
|
-
private _eventLoopHistogram: ReturnType<typeof monitorEventLoopDelay> | null = null;
|
|
114
|
-
private _cpuProfileOnStart = false;
|
|
115
|
-
private _cpuProfileAuto = false;
|
|
116
|
-
private _cpuProfileDurationMs = 15000;
|
|
117
|
-
private _cpuProfileThresholdPct = 90;
|
|
118
|
-
private _cpuProfileTriggerCount = 3;
|
|
119
|
-
private _cpuProfileHighCount = 0;
|
|
120
|
-
private _cpuProfileDir: string | null = null;
|
|
121
|
-
private _cpuProfileSession: inspector.Session | null = null;
|
|
122
|
-
private _timerDebug = false;
|
|
123
|
-
private _timerDebugThresholdMs = 50;
|
|
124
|
-
private _timerDebugMinDelayMs = 5;
|
|
125
|
-
private _timerDebugSampleRate = 1;
|
|
126
|
-
private _timerDebugLogLimit = 100;
|
|
127
|
-
private _timerDebugLogCount = 0;
|
|
128
|
-
private _aiWorkerDebug = false;
|
|
129
|
-
private _standaloneNodeReaperEnabled = false;
|
|
130
|
-
private _standaloneNodeReaperIntervalMs = 60000;
|
|
131
|
-
private _standaloneNodeReaperMinAgeSeconds = 300;
|
|
132
|
-
private _standaloneNodeReaperMaxKillsPerSignature = 1;
|
|
133
|
-
private _standaloneNodeReaperKillWindowMs = 60 * 60 * 1000;
|
|
134
|
-
private _standaloneNodeReaperIncludeSystemdServices = false;
|
|
135
|
-
private _standaloneNodeReaperHighCpuPct = 85;
|
|
136
|
-
private _standaloneNodeReaperHighRssMb = 4096;
|
|
137
|
-
private _standaloneNodeReaperHighMinAgeSeconds = 60;
|
|
138
|
-
private _standaloneNodeReaperHighConsecutiveScans = 2;
|
|
139
|
-
private _standaloneNodeReaperDryRun = false;
|
|
140
|
-
private _standaloneNodeReaperAlertWindowMs = 60 * 60 * 1000;
|
|
141
|
-
private _standaloneNodeReaperTimer: NodeJS.Timeout | null = null;
|
|
142
|
-
private _standaloneNodeReaperRunning = false;
|
|
143
|
-
private _standaloneNodeReaperKillCounts = new Map<string, { count: number; windowStartMs: number }>();
|
|
144
|
-
private _standaloneNodeReaperSuppressedSignatures = new Set<string>();
|
|
145
|
-
private _standaloneNodeReaperResourceHits = new Map<string, { count: number; lastSeenMs: number }>();
|
|
146
|
-
private _standaloneNodeReaperAlertTimes = new Map<string, number>();
|
|
147
|
-
|
|
148
|
-
private _serverStartTime: Date;
|
|
149
|
-
private _lastErrorMsg: Date = null;
|
|
150
|
-
|
|
151
|
-
private _debugMsgRecv = 0;
|
|
152
|
-
private _debugMsgQueue = 0;
|
|
153
|
-
|
|
154
|
-
private _isWorkersEnabled = false;
|
|
155
|
-
private _isWorkerInstance = false;
|
|
156
|
-
|
|
157
|
-
private _safeShutdown = false;
|
|
158
|
-
private _dynamicAppGatewayEnabled = false;
|
|
159
|
-
private _dynamicAppGatewayCache = new Map<string, { expiresAt: number; app: any }>();
|
|
160
|
-
private _socketTier = '';
|
|
161
|
-
private _maxClientSockets = 0;
|
|
162
|
-
private _singleIpPerUser = false;
|
|
163
|
-
private _socketPolicyUpgradeUrl = '';
|
|
164
|
-
|
|
165
|
-
private readonly _clientHeartbeatIntervalMs = 20000;
|
|
166
|
-
private readonly _clientHeartbeatInitialDelayMs = 5000;
|
|
167
|
-
private readonly _clientHeartbeatBackpressureBytes = 5 * 1024 * 1024;
|
|
168
|
-
private readonly _dynamicAppGatewayCacheMs = 30 * 1000;
|
|
169
|
-
|
|
170
|
-
constructor() {}
|
|
171
|
-
|
|
172
|
-
static async create() {
|
|
173
|
-
const resolveioMainServer = new ResolveIOMainServer();
|
|
174
|
-
await resolveioMainServer.initialize();
|
|
175
|
-
return resolveioMainServer;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
private async initialize() {
|
|
179
|
-
this._serverStartTime = new Date();
|
|
180
|
-
this._lastErrorMsg = null;
|
|
181
|
-
this._wsConnectDebug = this.resolveConnectDebug();
|
|
182
|
-
this._perfDebug = this.resolvePerfDebug();
|
|
183
|
-
this._cpuProfileOnStart = this.resolveCpuProfileOnStart();
|
|
184
|
-
this._cpuProfileAuto = this.resolveCpuProfileAuto();
|
|
185
|
-
this._cpuProfileDurationMs = this.resolveCpuProfileDurationMs();
|
|
186
|
-
this._cpuProfileThresholdPct = this.resolveCpuProfileThresholdPct();
|
|
187
|
-
this._cpuProfileTriggerCount = this.resolveCpuProfileTriggerCount();
|
|
188
|
-
this._cpuProfileDir = this.resolveCpuProfileDir();
|
|
189
|
-
this._timerDebug = this.resolveTimerDebug();
|
|
190
|
-
this._timerDebugThresholdMs = this.resolveTimerDebugThresholdMs();
|
|
191
|
-
this._timerDebugMinDelayMs = this.resolveTimerDebugMinDelayMs();
|
|
192
|
-
this._timerDebugSampleRate = this.resolveTimerDebugSampleRate();
|
|
193
|
-
this._timerDebugLogLimit = this.resolveTimerDebugLogLimit();
|
|
194
|
-
this._aiWorkerDebug = this.parseDebugFlag(process.env.AI_ASSISTANT_WORKER_DEBUG);
|
|
195
|
-
this._standaloneNodeReaperEnabled = this.resolveStandaloneNodeReaperEnabled();
|
|
196
|
-
this._standaloneNodeReaperIntervalMs = this.resolveStandaloneNodeReaperIntervalMs();
|
|
197
|
-
this._standaloneNodeReaperMinAgeSeconds = this.resolveStandaloneNodeReaperMinAgeSeconds();
|
|
198
|
-
this._standaloneNodeReaperMaxKillsPerSignature = this.resolveStandaloneNodeReaperMaxKillsPerSignature();
|
|
199
|
-
this._standaloneNodeReaperKillWindowMs = this.resolveStandaloneNodeReaperKillWindowMs();
|
|
200
|
-
this._standaloneNodeReaperIncludeSystemdServices = this.resolveStandaloneNodeReaperIncludeSystemdServices();
|
|
201
|
-
this._standaloneNodeReaperHighCpuPct = this.resolveStandaloneNodeReaperHighCpuPct();
|
|
202
|
-
this._standaloneNodeReaperHighRssMb = this.resolveStandaloneNodeReaperHighRssMb();
|
|
203
|
-
this._standaloneNodeReaperHighMinAgeSeconds = this.resolveStandaloneNodeReaperHighMinAgeSeconds();
|
|
204
|
-
this._standaloneNodeReaperHighConsecutiveScans = this.resolveStandaloneNodeReaperHighConsecutiveScans();
|
|
205
|
-
this._standaloneNodeReaperDryRun = this.resolveStandaloneNodeReaperDryRun();
|
|
206
|
-
this._standaloneNodeReaperAlertWindowMs = this.resolveStandaloneNodeReaperAlertWindowMs();
|
|
207
|
-
this._dynamicAppGatewayEnabled = this.resolveDynamicAppGatewayEnabled();
|
|
208
|
-
this._socketTier = this.resolveSocketTier();
|
|
209
|
-
this._maxClientSockets = this.resolveMaxClientSockets(this._socketTier);
|
|
210
|
-
this._singleIpPerUser = this.resolveSingleIpPerUserPolicy(this._socketTier);
|
|
211
|
-
this._socketPolicyUpgradeUrl = this.resolveSocketPolicyUpgradeUrl();
|
|
212
|
-
if (this._maxClientSockets > 0 || this._singleIpPerUser) {
|
|
213
|
-
console.info(new Date(), '[Socket Policy] configured', {
|
|
214
|
-
tier: this._socketTier || 'none',
|
|
215
|
-
maxClientSockets: this._maxClientSockets,
|
|
216
|
-
singleIpPerUser: this._singleIpPerUser
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
this._monitorManager = await MonitorManager.create();
|
|
220
|
-
this._monitorManagerFunction = new MonitorManagerFunction();
|
|
221
|
-
this.installTimerDebug();
|
|
222
|
-
this.startPerfDebug();
|
|
223
|
-
if (this._cpuProfileOnStart) {
|
|
224
|
-
this.startCpuProfile('on-start');
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Check for workers and decide what to start
|
|
228
|
-
this._isWorkersEnabled = process.env.IS_WORKERS_ENABLED === 'true';
|
|
229
|
-
this._isWorkerInstance = process.env.IS_WORKER_INSTANCE === 'true';
|
|
230
|
-
this.startStandaloneNodeReaper();
|
|
231
|
-
|
|
232
|
-
setInterval(() => {
|
|
233
|
-
if (this._methodManager && this._methodManager.getEnableDebug()) {
|
|
234
|
-
console.log(new Date(), 'Server App', 'Msg Recv Hits', this._debugMsgRecv);
|
|
235
|
-
console.log(new Date(), 'Server App', 'Msg Queue Hits', this._debugMsgQueue);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
this._debugMsgQueue = 0;
|
|
239
|
-
this._debugMsgRecv = 0;
|
|
240
|
-
}, 60000);
|
|
241
|
-
|
|
242
|
-
process.removeAllListeners('unhandledRejection');
|
|
243
|
-
|
|
244
|
-
process.on('unhandledRejection', async (error, rej) => {
|
|
245
|
-
const { error: normalizedError, correlationId } = ensureErrorWithCorrelation(error);
|
|
246
|
-
|
|
247
|
-
if (this._methodManager.getEnableDebug()) {
|
|
248
|
-
console.error(new Date(), 'ERROR DETECTED w/ Debug Flag Active: unhandledRejection', [normalizedError, rej, { correlationId }]);
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
// Condition to filter out the MongoError with specific codes
|
|
252
|
-
if (normalizedError && normalizedError['name'] === 'MongoError' && (normalizedError['code'] === 48 || normalizedError['code'] === 26 || normalizedError['code'] === 11000 || normalizedError['code'] === 251)) {
|
|
253
|
-
return; // Simply return without doing anything further
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
// if (normalizedError && normalizedError['name'] === 'MongoServerError' && (!initServerFlag || normalizedError['code'] === 26 || normalizedError['code'] === 11000 || normalizedError['code'] === 86 || normalizedError['code'] === 251)) {
|
|
257
|
-
// return; // Simply return without doing anything further
|
|
258
|
-
// }
|
|
259
|
-
|
|
260
|
-
if (normalizedError && normalizedError['name'] === 'MongoServerError') {
|
|
261
|
-
return; // Simply return without doing anything further
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
const errorDetails = {
|
|
265
|
-
id: correlationId,
|
|
266
|
-
name: normalizedError?.name,
|
|
267
|
-
message: normalizedError?.message,
|
|
268
|
-
stack: normalizedError?.stack,
|
|
269
|
-
code: normalizedError?.code,
|
|
270
|
-
codeName: normalizedError?.codeName
|
|
271
|
-
};
|
|
272
|
-
|
|
273
|
-
console.error(new Date(), 'Unhandled Rejection at Promise', [errorDetails]);
|
|
274
|
-
|
|
275
|
-
let diffTimeSec = moment().diff(this._serverStartTime, 'seconds');
|
|
276
|
-
|
|
277
|
-
// If this is a MongoNetworkTimeoutError, handle it specifically
|
|
278
|
-
if (normalizedError && (normalizedError['name'] === 'MongoNetworkTimeoutError' || normalizedError instanceof MongoNetworkTimeoutError)) {
|
|
279
|
-
if (diffTimeSec > 60 && !this._lastErrorMsg) {
|
|
280
|
-
this._lastErrorMsg = new Date();
|
|
281
|
-
setTimeout(() => {
|
|
282
|
-
this._lastErrorMsg = null;
|
|
283
|
-
}, 60000);
|
|
284
|
-
|
|
285
|
-
// Exiting the process
|
|
286
|
-
process.exit(1);
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
else if (normalizedError && normalizedError['name'] === 'MongoError' && normalizedError['message'] === 'not master') {
|
|
290
|
-
if (diffTimeSec > 60 && !this._lastErrorMsg) {
|
|
291
|
-
this._lastErrorMsg = new Date();
|
|
292
|
-
|
|
293
|
-
setTimeout(() => {
|
|
294
|
-
this._lastErrorMsg = null;
|
|
295
|
-
}, 60000);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
process.exit(1);
|
|
299
|
-
}
|
|
300
|
-
else if (normalizedError && normalizedError['name'] === 'MongoError' && normalizedError['message'] === 'not master and slaveOk=false') {
|
|
301
|
-
if (diffTimeSec > 60 && !this._lastErrorMsg) {
|
|
302
|
-
this._lastErrorMsg = new Date();
|
|
303
|
-
|
|
304
|
-
setTimeout(() => {
|
|
305
|
-
this._lastErrorMsg = null;
|
|
306
|
-
}, 60000);
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
process.exit(1);
|
|
310
|
-
}
|
|
311
|
-
else if (normalizedError && normalizedError['name'] !== 'StatusError' && normalizedError['message'] !== '') {
|
|
312
|
-
if (diffTimeSec > 60 && !this._lastErrorMsg) {
|
|
313
|
-
this._lastErrorMsg = new Date();
|
|
314
|
-
|
|
315
|
-
setTimeout(() => {
|
|
316
|
-
this._lastErrorMsg = null;
|
|
317
|
-
}, 60000);
|
|
318
|
-
|
|
319
|
-
await this.reportServerError(
|
|
320
|
-
'SERVER - Unhandled Rejection - ' + ResolveIOServer.getServerConfig()['CLIENT_NAME'],
|
|
321
|
-
correlationId,
|
|
322
|
-
errorDetails,
|
|
323
|
-
{
|
|
324
|
-
context: 'unhandledRejection',
|
|
325
|
-
scenario: 'General'
|
|
326
|
-
}
|
|
327
|
-
);
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
});
|
|
331
|
-
|
|
332
|
-
process.on('uncaughtException', async error => {
|
|
333
|
-
const { error: normalizedError, correlationId } = ensureErrorWithCorrelation(error);
|
|
334
|
-
console.error(normalizedError, 'Uncaught Exception thrown', { correlationId });
|
|
335
|
-
|
|
336
|
-
let diffTimeSec = moment().diff(this._serverStartTime, 'seconds');
|
|
337
|
-
|
|
338
|
-
if (diffTimeSec > 60 && !this._lastErrorMsg) {
|
|
339
|
-
this._lastErrorMsg = new Date();
|
|
340
|
-
|
|
341
|
-
setTimeout(() => {
|
|
342
|
-
this._lastErrorMsg = null;
|
|
343
|
-
}, 60000);
|
|
344
|
-
|
|
345
|
-
const errorDetails = {
|
|
346
|
-
id: correlationId,
|
|
347
|
-
name: normalizedError?.name,
|
|
348
|
-
message: normalizedError?.message,
|
|
349
|
-
stack: normalizedError?.stack,
|
|
350
|
-
code: normalizedError?.code,
|
|
351
|
-
codeName: normalizedError?.codeName
|
|
352
|
-
};
|
|
353
|
-
|
|
354
|
-
await this.reportServerError(
|
|
355
|
-
'SERVER - Unhandled Exception - ' + ResolveIOServer.getServerConfig()['CLIENT_NAME'],
|
|
356
|
-
correlationId,
|
|
357
|
-
errorDetails,
|
|
358
|
-
{
|
|
359
|
-
context: 'uncaughtException'
|
|
360
|
-
}
|
|
361
|
-
);
|
|
362
|
-
}
|
|
363
|
-
});
|
|
364
|
-
|
|
365
|
-
//PM2 wants to reboot/restart
|
|
366
|
-
process.on('SIGINT', async () => {
|
|
367
|
-
this._rebootFlag = true;
|
|
368
|
-
try {
|
|
369
|
-
await this.shutdownNetworkServers();
|
|
370
|
-
}
|
|
371
|
-
catch (error) {
|
|
372
|
-
console.error(new Date(), 'Error closing network servers (SIGINT)', error);
|
|
373
|
-
}
|
|
374
|
-
await this.safeShutdown();
|
|
375
|
-
});
|
|
376
|
-
|
|
377
|
-
process.on('SIGTERM', async () => {
|
|
378
|
-
this._rebootFlag = true;
|
|
379
|
-
try {
|
|
380
|
-
await this.shutdownNetworkServers();
|
|
381
|
-
}
|
|
382
|
-
catch (error) {
|
|
383
|
-
console.error(new Date(), 'Error closing network servers (SIGTERM)', error);
|
|
384
|
-
}
|
|
385
|
-
await this.safeShutdown();
|
|
386
|
-
});
|
|
387
|
-
|
|
388
|
-
process.on('SIGQUIT', async () => {
|
|
389
|
-
this._rebootFlag = true;
|
|
390
|
-
try {
|
|
391
|
-
await this.shutdownNetworkServers();
|
|
392
|
-
}
|
|
393
|
-
catch (error) {
|
|
394
|
-
console.error(new Date(), 'Error closing network servers (SIGQUIT)', error);
|
|
395
|
-
}
|
|
396
|
-
await this.safeShutdown();
|
|
397
|
-
});
|
|
398
|
-
|
|
399
|
-
if (this.LOGGER === 'DEBUG') {
|
|
400
|
-
console.log('Starting ResolveIO Server');
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
if (this._isWorkersEnabled) {
|
|
404
|
-
if (this._isWorkerInstance) {
|
|
405
|
-
const workerRole = this.resolveWorkerRole();
|
|
406
|
-
const workerIndex = this.normalizeWorkerSelectorValue(process.env.WORKER_INDEX) || 'UNKNOWN';
|
|
407
|
-
const workerInstance = this.normalizeWorkerSelectorValue(process.env.NODE_APP_INSTANCE) || 'UNKNOWN';
|
|
408
|
-
console.log(`Running as Worker: ${workerRole}`, workerIndex, workerInstance);
|
|
409
|
-
this._methodManager = MethodManager.create(null, this._monitorManagerFunction, this._isWorkersEnabled, this._isWorkerInstance);
|
|
410
|
-
this._subscriptionManager = SubscriptionManager.createPublicationRegistry(ResolveIOServer.getServerConfig());
|
|
411
|
-
const skipWorkerServerConnection = this.parseDebugFlag(process.env.DISABLE_WORKER_SERVER_CONNECTION)
|
|
412
|
-
|| this.parseDebugFlag(process.env.SUPPORT_CODEX_MANAGER_PROCESS_ONLY)
|
|
413
|
-
|| this.parseDebugFlag(process.env.SUPPORT_AUTO_MANAGER_PROCESS_ONLY)
|
|
414
|
-
|| this.parseDebugFlag(process.env.AI_ASSISTANT_CODEX_MANAGER_PROCESS_ONLY);
|
|
415
|
-
if (skipWorkerServerConnection) {
|
|
416
|
-
console.log(new Date(), 'Worker server connection disabled for process-only runtime', {
|
|
417
|
-
workerIndex: process.env.WORKER_INDEX || null,
|
|
418
|
-
workerInstance: process.env.NODE_APP_INSTANCE || null
|
|
419
|
-
});
|
|
420
|
-
this._workerServerManager = null;
|
|
421
|
-
}
|
|
422
|
-
else {
|
|
423
|
-
this._workerServerManager = WorkerServerManager.create(this._methodManager, this.getServerConfig());
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
const skipCronForProcessOnly = this.parseDebugFlag(process.env.SUPPORT_CODEX_MANAGER_PROCESS_ONLY)
|
|
427
|
-
|| this.parseDebugFlag(process.env.SUPPORT_AUTO_MANAGER_PROCESS_ONLY)
|
|
428
|
-
|| this.parseDebugFlag(process.env.AI_ASSISTANT_CODEX_MANAGER_PROCESS_ONLY);
|
|
429
|
-
if (this.shouldStartCronManagerForWorker() && !skipCronForProcessOnly) {
|
|
430
|
-
this._cronManager = CronManager.create();
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
else {
|
|
434
|
-
console.log('Running as a Server instance', process.env.NODE_APP_INSTANCE);
|
|
435
|
-
this._websocketManager = WebSocketManager.create(this);
|
|
436
|
-
this._methodManager = MethodManager.create(this._websocketManager, this._monitorManagerFunction, this._isWorkersEnabled, this._isWorkerInstance);
|
|
437
|
-
this._workerDispatcherManager = WorkerDispatcherManager.create(this._websocketManager, this._methodManager);
|
|
438
|
-
this._subscriptionManager = SubscriptionManager.create(this._serverWSS, ResolveIOServer.getServerConfig(), this._monitorManagerFunction);
|
|
439
|
-
this.startServerInstance();
|
|
440
|
-
this.listen();
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
else {
|
|
444
|
-
console.log('Running with Workers Disabled', process.env.NODE_APP_INSTANCE);
|
|
445
|
-
this._websocketManager = WebSocketManager.create(this);
|
|
446
|
-
this._methodManager = MethodManager.create(this._websocketManager, this._monitorManagerFunction, this._isWorkersEnabled, this._isWorkerInstance);
|
|
447
|
-
this._subscriptionManager = SubscriptionManager.create(this._serverWSS, ResolveIOServer.getServerConfig(), this._monitorManagerFunction);
|
|
448
|
-
this._cronManager = CronManager.create();
|
|
449
|
-
this.startServerInstance();
|
|
450
|
-
this.listen();
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
this.startAiAssistantCodexManagerIfEnabled();
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
private startAiAssistantCodexManagerIfEnabled(): void {
|
|
457
|
-
try {
|
|
458
|
-
this._aiAssistantCodexManager = AiAssistantCodexManager.createIfEnabled({
|
|
459
|
-
isWorkersEnabled: this._isWorkersEnabled,
|
|
460
|
-
isWorkerInstance: this._isWorkerInstance,
|
|
461
|
-
workerIndex: process.env.WORKER_INDEX || null,
|
|
462
|
-
workerInstance: process.env.NODE_APP_INSTANCE || null
|
|
463
|
-
});
|
|
464
|
-
ResolveIOServer['AiAssistantCodexManager'] = this._aiAssistantCodexManager;
|
|
465
|
-
}
|
|
466
|
-
catch (error) {
|
|
467
|
-
console.error(new Date(), 'Failed to start AI Assistant Codex Manager', error);
|
|
468
|
-
ResolveIOServer['AiAssistantCodexManager'] = null;
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
private startStandaloneNodeReaper(): void {
|
|
473
|
-
if (!this._standaloneNodeReaperEnabled || this._standaloneNodeReaperTimer) {
|
|
474
|
-
return;
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
if (process.platform !== 'linux' || !this.isPm2ManagedRuntime() || !this.isPrimaryPm2Instance()) {
|
|
478
|
-
return;
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
const initialDelayMs = Math.min(30000, this._standaloneNodeReaperIntervalMs);
|
|
482
|
-
const firstRunTimer = setTimeout(async () => {
|
|
483
|
-
try {
|
|
484
|
-
await this.reapStandaloneNodeProcesses();
|
|
485
|
-
}
|
|
486
|
-
catch (error) {
|
|
487
|
-
console.error(new Date(), '[Standalone Node Reaper] failed', error);
|
|
488
|
-
}
|
|
489
|
-
}, initialDelayMs);
|
|
490
|
-
firstRunTimer.unref?.();
|
|
491
|
-
|
|
492
|
-
this._standaloneNodeReaperTimer = setInterval(async () => {
|
|
493
|
-
try {
|
|
494
|
-
await this.reapStandaloneNodeProcesses();
|
|
495
|
-
}
|
|
496
|
-
catch (error) {
|
|
497
|
-
console.error(new Date(), '[Standalone Node Reaper] failed', error);
|
|
498
|
-
}
|
|
499
|
-
}, this._standaloneNodeReaperIntervalMs);
|
|
500
|
-
this._standaloneNodeReaperTimer.unref?.();
|
|
501
|
-
|
|
502
|
-
console.log(new Date(), '[Standalone Node Reaper] enabled', {
|
|
503
|
-
intervalMs: this._standaloneNodeReaperIntervalMs,
|
|
504
|
-
minAgeSeconds: this._standaloneNodeReaperMinAgeSeconds,
|
|
505
|
-
maxKillsPerSignature: this._standaloneNodeReaperMaxKillsPerSignature,
|
|
506
|
-
killWindowMs: this._standaloneNodeReaperKillWindowMs,
|
|
507
|
-
highCpuPct: this._standaloneNodeReaperHighCpuPct,
|
|
508
|
-
highRssMb: this._standaloneNodeReaperHighRssMb,
|
|
509
|
-
highMinAgeSeconds: this._standaloneNodeReaperHighMinAgeSeconds,
|
|
510
|
-
highConsecutiveScans: this._standaloneNodeReaperHighConsecutiveScans,
|
|
511
|
-
alertWindowMs: this._standaloneNodeReaperAlertWindowMs,
|
|
512
|
-
dryRun: this._standaloneNodeReaperDryRun
|
|
513
|
-
});
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
private async reapStandaloneNodeProcesses(): Promise<void> {
|
|
517
|
-
if (this._standaloneNodeReaperRunning) {
|
|
518
|
-
return;
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
this._standaloneNodeReaperRunning = true;
|
|
522
|
-
try {
|
|
523
|
-
const processes = await this.listSystemProcesses();
|
|
524
|
-
const protectedPids = this.resolvePm2ProtectedPids(processes);
|
|
525
|
-
for (const proc of processes) {
|
|
526
|
-
if (!this.shouldReapStandaloneNodeProcess(proc, protectedPids)) {
|
|
527
|
-
continue;
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
const signature = this.resolveStandaloneNodeProcessSignature(proc);
|
|
531
|
-
if (!this.canReapStandaloneNodeSignature(signature)) {
|
|
532
|
-
this.logStandaloneNodeReaperSuppressed(proc, signature);
|
|
533
|
-
continue;
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
const reason = this.resolveStandaloneNodeReaperReason(proc);
|
|
537
|
-
if (this._standaloneNodeReaperDryRun) {
|
|
538
|
-
this.recordStandaloneNodeReaperKill(signature);
|
|
539
|
-
const details = this.createStandaloneNodeReaperDetails(proc, reason, 'dry-run');
|
|
540
|
-
console.warn(new Date(), '[Standalone Node Reaper] dry-run would terminate stale node process', details);
|
|
541
|
-
await this.reportStandaloneNodeReaperAction(proc, reason, 'dry-run');
|
|
542
|
-
continue;
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
try {
|
|
546
|
-
process.kill(proc.pid, 'SIGTERM');
|
|
547
|
-
this.recordStandaloneNodeReaperKill(signature);
|
|
548
|
-
const details = this.createStandaloneNodeReaperDetails(proc, reason, 'terminated');
|
|
549
|
-
console.warn(new Date(), '[Standalone Node Reaper] terminated stale node process', details);
|
|
550
|
-
await this.reportStandaloneNodeReaperAction(proc, reason, 'terminated');
|
|
551
|
-
|
|
552
|
-
const killTimer = setTimeout(async () => {
|
|
553
|
-
try {
|
|
554
|
-
process.kill(proc.pid, 0);
|
|
555
|
-
process.kill(proc.pid, 'SIGKILL');
|
|
556
|
-
console.warn(new Date(), '[Standalone Node Reaper] force killed stale node process', {
|
|
557
|
-
pid: proc.pid,
|
|
558
|
-
reason,
|
|
559
|
-
command: proc.args
|
|
560
|
-
});
|
|
561
|
-
await this.reportStandaloneNodeReaperAction(proc, reason, 'force-killed');
|
|
562
|
-
}
|
|
563
|
-
catch {}
|
|
564
|
-
}, 10000);
|
|
565
|
-
killTimer.unref?.();
|
|
566
|
-
}
|
|
567
|
-
catch (error) {
|
|
568
|
-
if (error?.['code'] !== 'ESRCH') {
|
|
569
|
-
console.error(new Date(), '[Standalone Node Reaper] kill failed', {
|
|
570
|
-
pid: proc.pid,
|
|
571
|
-
command: proc.args,
|
|
572
|
-
error: error?.['message'] || error
|
|
573
|
-
});
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
finally {
|
|
579
|
-
this._standaloneNodeReaperRunning = false;
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
private createStandaloneNodeReaperDetails(proc: ProcessSnapshot, reason: string, action: string): Record<string, any> {
|
|
584
|
-
return {
|
|
585
|
-
action,
|
|
586
|
-
pid: proc.pid,
|
|
587
|
-
ppid: proc.ppid,
|
|
588
|
-
ageSeconds: proc.ageSeconds,
|
|
589
|
-
cpuPct: proc.cpuPct,
|
|
590
|
-
rssMb: round(proc.rssKb / 1024),
|
|
591
|
-
reason,
|
|
592
|
-
command: proc.args
|
|
593
|
-
};
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
private async reportStandaloneNodeReaperAction(proc: ProcessSnapshot, reason: string, action: string): Promise<void> {
|
|
597
|
-
const signature = this.resolveStandaloneNodeProcessSignature(proc);
|
|
598
|
-
const alertKey = `${action}|${signature}`;
|
|
599
|
-
const now = Date.now();
|
|
600
|
-
const lastReportedAt = this._standaloneNodeReaperAlertTimes.get(alertKey);
|
|
601
|
-
if (lastReportedAt && (now - lastReportedAt) < this._standaloneNodeReaperAlertWindowMs) {
|
|
602
|
-
return;
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
try {
|
|
606
|
-
await this.reportServerError(
|
|
607
|
-
`SERVER - Standalone Node Reaper ${action} process - ${ResolveIOServer.getServerConfig()?.['CLIENT_NAME'] || ResolveIOServer.getClientName()}`,
|
|
608
|
-
crypto.randomUUID(),
|
|
609
|
-
{
|
|
610
|
-
...this.createStandaloneNodeReaperDetails(proc, reason, action),
|
|
611
|
-
host: os.hostname(),
|
|
612
|
-
workerIndex: process.env.WORKER_INDEX || null,
|
|
613
|
-
workerInstance: process.env.NODE_APP_INSTANCE || null,
|
|
614
|
-
dryRun: this._standaloneNodeReaperDryRun,
|
|
615
|
-
alertWindowMs: this._standaloneNodeReaperAlertWindowMs
|
|
616
|
-
},
|
|
617
|
-
{
|
|
618
|
-
context: 'standaloneNodeReaper',
|
|
619
|
-
action,
|
|
620
|
-
reason,
|
|
621
|
-
signature
|
|
622
|
-
},
|
|
623
|
-
'warning'
|
|
624
|
-
);
|
|
625
|
-
this._standaloneNodeReaperAlertTimes.set(alertKey, now);
|
|
626
|
-
}
|
|
627
|
-
catch (error) {
|
|
628
|
-
console.error(new Date(), '[Standalone Node Reaper] alert failed', {
|
|
629
|
-
action,
|
|
630
|
-
pid: proc.pid,
|
|
631
|
-
error: error?.['message'] || error
|
|
632
|
-
});
|
|
633
|
-
}
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
private async listSystemProcesses(): Promise<ProcessSnapshot[]> {
|
|
637
|
-
const output = await this.execFileText('ps', ['-eo', 'pid=,ppid=,etimes=,pcpu=,rss=,comm=,args='], 5000);
|
|
638
|
-
return output
|
|
639
|
-
.split('\n')
|
|
640
|
-
.map(line => line.trim())
|
|
641
|
-
.filter(Boolean)
|
|
642
|
-
.map(line => this.parseProcessSnapshot(line))
|
|
643
|
-
.filter((proc): proc is ProcessSnapshot => proc !== null);
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
private parseProcessSnapshot(line: string): ProcessSnapshot | null {
|
|
647
|
-
const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s+(.*)$/);
|
|
648
|
-
if (!match) {
|
|
649
|
-
return null;
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
return {
|
|
653
|
-
pid: parseInt(match[1], 10),
|
|
654
|
-
ppid: parseInt(match[2], 10),
|
|
655
|
-
ageSeconds: parseInt(match[3], 10),
|
|
656
|
-
cpuPct: parseFloat(match[4]),
|
|
657
|
-
rssKb: parseInt(match[5], 10),
|
|
658
|
-
command: match[6],
|
|
659
|
-
args: match[7] || ''
|
|
660
|
-
};
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
private execFileText(command: string, args: string[], timeoutMs: number): Promise<string> {
|
|
664
|
-
// eslint-disable-next-line no-restricted-syntax
|
|
665
|
-
return new Promise((resolve, reject) => {
|
|
666
|
-
execFile(command, args, { timeout: timeoutMs, maxBuffer: 1024 * 1024 }, (error, stdout) => {
|
|
667
|
-
if (error) {
|
|
668
|
-
reject(error);
|
|
669
|
-
return;
|
|
670
|
-
}
|
|
671
|
-
resolve(stdout || '');
|
|
672
|
-
});
|
|
673
|
-
});
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
private resolvePm2ProtectedPids(processes: ProcessSnapshot[]): Set<number> {
|
|
677
|
-
const byPid = new Map<number, ProcessSnapshot>();
|
|
678
|
-
const children = new Map<number, number[]>();
|
|
679
|
-
for (const proc of processes) {
|
|
680
|
-
byPid.set(proc.pid, proc);
|
|
681
|
-
if (!children.has(proc.ppid)) {
|
|
682
|
-
children.set(proc.ppid, []);
|
|
683
|
-
}
|
|
684
|
-
children.get(proc.ppid).push(proc.pid);
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
const protectedPids = new Set<number>();
|
|
688
|
-
const addWithDescendants = (pid: number) => {
|
|
689
|
-
if (protectedPids.has(pid)) {
|
|
690
|
-
return;
|
|
691
|
-
}
|
|
692
|
-
protectedPids.add(pid);
|
|
693
|
-
for (const childPid of children.get(pid) || []) {
|
|
694
|
-
addWithDescendants(childPid);
|
|
695
|
-
}
|
|
696
|
-
};
|
|
697
|
-
|
|
698
|
-
for (const proc of processes) {
|
|
699
|
-
if (proc.args.includes('pm2-runtime') || proc.args.includes('/pm2/')) {
|
|
700
|
-
addWithDescendants(proc.pid);
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
let currentPid = process.pid;
|
|
705
|
-
while (currentPid && byPid.has(currentPid)) {
|
|
706
|
-
protectedPids.add(currentPid);
|
|
707
|
-
currentPid = byPid.get(currentPid).ppid;
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
return protectedPids;
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
private shouldReapStandaloneNodeProcess(proc: ProcessSnapshot, protectedPids: Set<number>): boolean {
|
|
714
|
-
if (proc.pid === process.pid || proc.ppid !== 1 || protectedPids.has(proc.pid)) {
|
|
715
|
-
return false;
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
if (!this.isNodeProcess(proc) || !this.isSameUidProcess(proc.pid)) {
|
|
719
|
-
return false;
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
if (!this._standaloneNodeReaperIncludeSystemdServices && this.isProtectedSystemdServiceProcess(proc)) {
|
|
723
|
-
return false;
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
if (!this.isResolveIOAppProcess(proc)) {
|
|
727
|
-
return false;
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
if (proc.ageSeconds >= this._standaloneNodeReaperMinAgeSeconds) {
|
|
731
|
-
return true;
|
|
732
|
-
}
|
|
733
|
-
|
|
734
|
-
if (proc.ageSeconds < this._standaloneNodeReaperHighMinAgeSeconds || !this.hasHighStandaloneNodeResourceUsage(proc)) {
|
|
735
|
-
this.clearStandaloneNodeResourceHit(proc);
|
|
736
|
-
return false;
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
return this.recordStandaloneNodeResourceHit(proc) >= this._standaloneNodeReaperHighConsecutiveScans;
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
private resolveStandaloneNodeReaperReason(proc: ProcessSnapshot): string {
|
|
743
|
-
if (proc.ageSeconds >= this._standaloneNodeReaperMinAgeSeconds) {
|
|
744
|
-
return `orphan_age_seconds_gte_${this._standaloneNodeReaperMinAgeSeconds}`;
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
const rssMb = proc.rssKb / 1024;
|
|
748
|
-
if (proc.cpuPct >= this._standaloneNodeReaperHighCpuPct && rssMb >= this._standaloneNodeReaperHighRssMb) {
|
|
749
|
-
return `orphan_high_cpu_and_rss_for_${this._standaloneNodeReaperHighConsecutiveScans}_scans`;
|
|
750
|
-
}
|
|
751
|
-
if (proc.cpuPct >= this._standaloneNodeReaperHighCpuPct) {
|
|
752
|
-
return `orphan_high_cpu_for_${this._standaloneNodeReaperHighConsecutiveScans}_scans`;
|
|
753
|
-
}
|
|
754
|
-
if (rssMb >= this._standaloneNodeReaperHighRssMb) {
|
|
755
|
-
return `orphan_high_rss_for_${this._standaloneNodeReaperHighConsecutiveScans}_scans`;
|
|
756
|
-
}
|
|
757
|
-
return 'orphan_stale_node_process';
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
private isNodeProcess(proc: ProcessSnapshot): boolean {
|
|
761
|
-
return proc.command === 'node' || proc.command.startsWith('node ') || proc.args === 'node' || proc.args.startsWith('node ');
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
private isResolveIOAppProcess(proc: ProcessSnapshot): boolean {
|
|
765
|
-
const normalizedArgs = proc.args || '';
|
|
766
|
-
const appDirs = this.resolveCurrentAppDirs();
|
|
767
|
-
for (const appDir of appDirs) {
|
|
768
|
-
if (normalizedArgs.includes(appDir)) {
|
|
769
|
-
return true;
|
|
770
|
-
}
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
if (normalizedArgs.includes('server-app.js') && normalizedArgs.includes('require(')) {
|
|
774
|
-
return true;
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
const cwd = this.resolveProcessCwd(proc.pid);
|
|
778
|
-
return cwd !== null && appDirs.has(cwd) && /\b(index|server-app)\.js\b/.test(normalizedArgs);
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
private hasHighStandaloneNodeResourceUsage(proc: ProcessSnapshot): boolean {
|
|
782
|
-
const rssMb = proc.rssKb / 1024;
|
|
783
|
-
return proc.cpuPct >= this._standaloneNodeReaperHighCpuPct || rssMb >= this._standaloneNodeReaperHighRssMb;
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
private recordStandaloneNodeResourceHit(proc: ProcessSnapshot): number {
|
|
787
|
-
const signature = this.resolveStandaloneNodeProcessSignature(proc);
|
|
788
|
-
const now = Date.now();
|
|
789
|
-
const current = this._standaloneNodeReaperResourceHits.get(signature);
|
|
790
|
-
if (!current || (now - current.lastSeenMs) > (this._standaloneNodeReaperIntervalMs * 3)) {
|
|
791
|
-
this._standaloneNodeReaperResourceHits.set(signature, { count: 1, lastSeenMs: now });
|
|
792
|
-
return 1;
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
current.count += 1;
|
|
796
|
-
current.lastSeenMs = now;
|
|
797
|
-
return current.count;
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
private clearStandaloneNodeResourceHit(proc: ProcessSnapshot): void {
|
|
801
|
-
this._standaloneNodeReaperResourceHits.delete(this.resolveStandaloneNodeProcessSignature(proc));
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
private resolveStandaloneNodeProcessSignature(proc: ProcessSnapshot): string {
|
|
805
|
-
const cwd = this.resolveProcessCwd(proc.pid) || '';
|
|
806
|
-
return `${cwd}|${proc.args}`;
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
private canReapStandaloneNodeSignature(signature: string): boolean {
|
|
810
|
-
const now = Date.now();
|
|
811
|
-
const current = this._standaloneNodeReaperKillCounts.get(signature);
|
|
812
|
-
if (!current || (now - current.windowStartMs) > this._standaloneNodeReaperKillWindowMs) {
|
|
813
|
-
this._standaloneNodeReaperKillCounts.set(signature, { count: 0, windowStartMs: now });
|
|
814
|
-
this._standaloneNodeReaperSuppressedSignatures.delete(signature);
|
|
815
|
-
return true;
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
return current.count < this._standaloneNodeReaperMaxKillsPerSignature;
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
private recordStandaloneNodeReaperKill(signature: string): void {
|
|
822
|
-
const now = Date.now();
|
|
823
|
-
const current = this._standaloneNodeReaperKillCounts.get(signature);
|
|
824
|
-
if (!current || (now - current.windowStartMs) > this._standaloneNodeReaperKillWindowMs) {
|
|
825
|
-
this._standaloneNodeReaperKillCounts.set(signature, { count: 1, windowStartMs: now });
|
|
826
|
-
this._standaloneNodeReaperSuppressedSignatures.delete(signature);
|
|
827
|
-
return;
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
current.count += 1;
|
|
831
|
-
}
|
|
832
|
-
|
|
833
|
-
private logStandaloneNodeReaperSuppressed(proc: ProcessSnapshot, signature: string): void {
|
|
834
|
-
if (this._standaloneNodeReaperSuppressedSignatures.has(signature)) {
|
|
835
|
-
return;
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
this._standaloneNodeReaperSuppressedSignatures.add(signature);
|
|
839
|
-
console.warn(new Date(), '[Standalone Node Reaper] suppressing recurring stale node process', {
|
|
840
|
-
pid: proc.pid,
|
|
841
|
-
ppid: proc.ppid,
|
|
842
|
-
ageSeconds: proc.ageSeconds,
|
|
843
|
-
cpuPct: proc.cpuPct,
|
|
844
|
-
rssMb: round(proc.rssKb / 1024),
|
|
845
|
-
maxKillsPerSignature: this._standaloneNodeReaperMaxKillsPerSignature,
|
|
846
|
-
killWindowMs: this._standaloneNodeReaperKillWindowMs,
|
|
847
|
-
command: proc.args
|
|
848
|
-
});
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
private resolveCurrentAppDirs(): Set<string> {
|
|
852
|
-
const dirs = new Set<string>();
|
|
853
|
-
for (const value of [process.cwd(), ResolveIOServer.getClientDir?.()]) {
|
|
854
|
-
if (!value) {
|
|
855
|
-
continue;
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
const resolved = this.safeRealpath(value);
|
|
859
|
-
if (resolved) {
|
|
860
|
-
dirs.add(resolved);
|
|
861
|
-
}
|
|
862
|
-
}
|
|
863
|
-
return dirs;
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
private resolveProcessCwd(pid: number): string | null {
|
|
867
|
-
return this.safeRealpath(`/proc/${pid}/cwd`);
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
private safeRealpath(value: string): string | null {
|
|
871
|
-
try {
|
|
872
|
-
return fs.realpathSync(value);
|
|
873
|
-
}
|
|
874
|
-
catch {
|
|
875
|
-
return null;
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
|
|
879
|
-
private isSameUidProcess(pid: number): boolean {
|
|
880
|
-
if (typeof process.getuid !== 'function') {
|
|
881
|
-
return true;
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
try {
|
|
885
|
-
const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8');
|
|
886
|
-
const uidLine = status.split('\n').find(line => line.startsWith('Uid:'));
|
|
887
|
-
if (!uidLine) {
|
|
888
|
-
return false;
|
|
889
|
-
}
|
|
890
|
-
const uid = parseInt(uidLine.split(/\s+/)[1], 10);
|
|
891
|
-
return uid === process.getuid();
|
|
892
|
-
}
|
|
893
|
-
catch {
|
|
894
|
-
return false;
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
private isProtectedSystemdServiceProcess(proc: ProcessSnapshot): boolean {
|
|
899
|
-
const serviceName = this.resolveSystemdServiceName(proc.pid);
|
|
900
|
-
if (!serviceName) {
|
|
901
|
-
return false;
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
return /\bindex\.js\b/.test(proc.args);
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
private resolveSystemdServiceName(pid: number): string | null {
|
|
908
|
-
try {
|
|
909
|
-
const cgroup = fs.readFileSync(`/proc/${pid}/cgroup`, 'utf8');
|
|
910
|
-
const serviceLine = cgroup.split('\n').find(line => line.includes('/system.slice/') && line.includes('.service'));
|
|
911
|
-
if (!serviceLine) {
|
|
912
|
-
return null;
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
const match = serviceLine.match(/\/system\.slice\/([^/]+\.service)/);
|
|
916
|
-
return match ? match[1] : null;
|
|
917
|
-
}
|
|
918
|
-
catch {
|
|
919
|
-
return null;
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
private async shutdownNetworkServers() {
|
|
924
|
-
await this.closeWebSocketServerGracefully();
|
|
925
|
-
await this.closeHttpServerGracefully();
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
private async closeHttpServerGracefully() {
|
|
929
|
-
if (!this._serverHTTP) {
|
|
930
|
-
return;
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
if (this._httpServerClosePromise !== null) {
|
|
934
|
-
await this._httpServerClosePromise;
|
|
935
|
-
return;
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
// eslint-disable-next-line no-restricted-syntax
|
|
939
|
-
this._httpServerClosePromise = new Promise(resolve => {
|
|
940
|
-
try {
|
|
941
|
-
this._serverHTTP.close(error => {
|
|
942
|
-
if (error && error['code'] !== 'ERR_SERVER_NOT_RUNNING') {
|
|
943
|
-
console.error(new Date(), 'Error closing HTTP server before shutdown', error);
|
|
944
|
-
}
|
|
945
|
-
resolve();
|
|
946
|
-
});
|
|
947
|
-
}
|
|
948
|
-
catch (error) {
|
|
949
|
-
if (error && error['code'] !== 'ERR_SERVER_NOT_RUNNING') {
|
|
950
|
-
console.error(new Date(), 'Error closing HTTP server before shutdown', error);
|
|
951
|
-
}
|
|
952
|
-
resolve();
|
|
953
|
-
}
|
|
954
|
-
});
|
|
955
|
-
|
|
956
|
-
await this._httpServerClosePromise;
|
|
957
|
-
}
|
|
958
|
-
|
|
959
|
-
private async closeWebSocketServerGracefully() {
|
|
960
|
-
if (!this._serverWSS) {
|
|
961
|
-
return;
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
if (this._websocketServerClosePromise !== null) {
|
|
965
|
-
await this._websocketServerClosePromise;
|
|
966
|
-
return;
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
this._serverWSS.clients.forEach(ws => {
|
|
970
|
-
try {
|
|
971
|
-
ws.close(1001, 'Server restarting');
|
|
972
|
-
}
|
|
973
|
-
catch (error) {
|
|
974
|
-
console.error(new Date(), 'Error closing WebSocket client before shutdown', error);
|
|
975
|
-
}
|
|
976
|
-
});
|
|
977
|
-
|
|
978
|
-
// eslint-disable-next-line no-restricted-syntax
|
|
979
|
-
this._websocketServerClosePromise = new Promise(resolve => {
|
|
980
|
-
try {
|
|
981
|
-
this._serverWSS.close(error => {
|
|
982
|
-
if (error) {
|
|
983
|
-
console.error(new Date(), 'Error closing WebSocket server before shutdown', error);
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
resolve();
|
|
987
|
-
});
|
|
988
|
-
}
|
|
989
|
-
catch (error) {
|
|
990
|
-
console.error(new Date(), 'Error closing WebSocket server before shutdown', error);
|
|
991
|
-
resolve();
|
|
992
|
-
}
|
|
993
|
-
});
|
|
994
|
-
|
|
995
|
-
await this._websocketServerClosePromise;
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
private startServerInstance() {
|
|
999
|
-
// Start express app
|
|
1000
|
-
this._app = express();
|
|
1001
|
-
|
|
1002
|
-
// Use built-in express JSON parser
|
|
1003
|
-
this._app.use(express.json({
|
|
1004
|
-
limit: '50mb',
|
|
1005
|
-
reviver: dateReviver // Note: 'reviver' is an option for JSON.parse, which can be passed here
|
|
1006
|
-
}));
|
|
1007
|
-
|
|
1008
|
-
// Use built-in express URL-encoded parser
|
|
1009
|
-
this._app.use(express.urlencoded({
|
|
1010
|
-
limit: '50mb',
|
|
1011
|
-
extended: true, // `extended` must be explicitly true or false
|
|
1012
|
-
parameterLimit: 1000000
|
|
1013
|
-
}));
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
this._app.use(xmlParser());
|
|
1017
|
-
|
|
1018
|
-
// Set port
|
|
1019
|
-
this._portHTTP = process.env.NODE_APP_INSTANCE ? parseInt('808' + process.env.NODE_APP_INSTANCE) : 8080;
|
|
1020
|
-
|
|
1021
|
-
if (this.LOGGER === 'DEBUG') {
|
|
1022
|
-
console.log('Setup ports');
|
|
1023
|
-
}
|
|
1024
|
-
|
|
1025
|
-
// Create http server and websock server
|
|
1026
|
-
this.createServer();
|
|
1027
|
-
|
|
1028
|
-
if (this.LOGGER === 'DEBUG') {
|
|
1029
|
-
console.log('Create server');
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
// Set CORS
|
|
1033
|
-
this._app.use((req, res, next) => {
|
|
1034
|
-
const requestOrigin = String(req.headers.origin || '').trim();
|
|
1035
|
-
if (requestOrigin && isAllowedOrigin(requestOrigin, ResolveIOServer.getServerConfig())) {
|
|
1036
|
-
res.setHeader('Access-Control-Allow-Origin', requestOrigin);
|
|
1037
|
-
res.setHeader('Vary', 'Origin');
|
|
1038
|
-
res.setHeader('Access-Control-Allow-Credentials', 'true');
|
|
1039
|
-
}
|
|
1040
|
-
else {
|
|
1041
|
-
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
1042
|
-
res.setHeader('Access-Control-Allow-Credentials', 'false');
|
|
1043
|
-
}
|
|
1044
|
-
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS');
|
|
1045
|
-
res.setHeader('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, Authorization, X-Rio-Token, X-API-Key, X-AI-Coder-App-Token, X-AI-Coder-Session');
|
|
1046
|
-
res.setHeader('Access-Control-Allow-Private-Network', 'true');
|
|
1047
|
-
if (isCorsPreflightRequest(req)) {
|
|
1048
|
-
res.status(204).end();
|
|
1049
|
-
return;
|
|
1050
|
-
}
|
|
1051
|
-
next();
|
|
1052
|
-
});
|
|
1053
|
-
|
|
1054
|
-
if (this.LOGGER === 'DEBUG') {
|
|
1055
|
-
console.log('Setup cors');
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
this.installDynamicAppGatewayMiddleware();
|
|
1059
|
-
|
|
1060
|
-
// Set up http login route
|
|
1061
|
-
setupAuthRoutes(this, this._app, ResolveIOServer.getServerConfig());
|
|
1062
|
-
setupHealthRoutes(this._app);
|
|
1063
|
-
setupSlowQueryPublicationRoutes(this._app);
|
|
1064
|
-
|
|
1065
|
-
if (ResolveIOServer.getServerConfig()['CLIENT_NAME'] === 'ResolveIO' || this.publicProgram) {
|
|
1066
|
-
setupHomeRoutes(this, this._app, ResolveIOServer.getServerConfig());
|
|
1067
|
-
}
|
|
1068
|
-
|
|
1069
|
-
if (this.LOGGER === 'DEBUG') {
|
|
1070
|
-
console.log('Setup express routes');
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
private installDynamicAppGatewayMiddleware(): void {
|
|
1075
|
-
if (!this._dynamicAppGatewayEnabled) {
|
|
1076
|
-
return;
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
this._app.use(async (req, res, next) => {
|
|
1080
|
-
const appId = this.extractDynamicAppGatewayId(req);
|
|
1081
|
-
if (!appId) {
|
|
1082
|
-
next();
|
|
1083
|
-
return;
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
try {
|
|
1087
|
-
const appDoc = await this.resolveCachedDynamicAppGatewayApp(appId);
|
|
1088
|
-
if (!appDoc) {
|
|
1089
|
-
res.status(404).send(JSON.stringify({
|
|
1090
|
-
error: true,
|
|
1091
|
-
result: 'App not found.'
|
|
1092
|
-
}));
|
|
1093
|
-
return;
|
|
1094
|
-
}
|
|
1095
|
-
|
|
1096
|
-
const allowedHosts = this.resolveDynamicAppAllowedHosts(appDoc);
|
|
1097
|
-
const controlHosts = this.resolveDynamicAppControlHosts();
|
|
1098
|
-
const requestHost = this.normalizeHostname(
|
|
1099
|
-
this.normalizeHeaderValue(req.headers?.['x-forwarded-host'])
|
|
1100
|
-
|| this.normalizeHeaderValue(req.headers?.host)
|
|
1101
|
-
);
|
|
1102
|
-
const originHeader = this.normalizeHeaderValue(req.headers?.origin);
|
|
1103
|
-
const originHost = this.resolveOriginHostname(originHeader);
|
|
1104
|
-
|
|
1105
|
-
if (requestHost && controlHosts.has(requestHost) && !this.isLocalHostname(requestHost)) {
|
|
1106
|
-
res.status(403).send(JSON.stringify({
|
|
1107
|
-
error: true,
|
|
1108
|
-
result: 'App API must be requested from an app domain.'
|
|
1109
|
-
}));
|
|
1110
|
-
return;
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
if (requestHost && !allowedHosts.has(requestHost) && !this.isLocalHostname(requestHost)) {
|
|
1114
|
-
res.status(403).send(JSON.stringify({
|
|
1115
|
-
error: true,
|
|
1116
|
-
result: 'Host not allowed for app API.'
|
|
1117
|
-
}));
|
|
1118
|
-
return;
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
if (originHost && controlHosts.has(originHost) && !this.isLocalHostname(originHost)) {
|
|
1122
|
-
res.status(403).send(JSON.stringify({
|
|
1123
|
-
error: true,
|
|
1124
|
-
result: 'App API origin must be an app domain.'
|
|
1125
|
-
}));
|
|
1126
|
-
return;
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
if (originHost && !allowedHosts.has(originHost)) {
|
|
1130
|
-
res.status(403).send(JSON.stringify({
|
|
1131
|
-
error: true,
|
|
1132
|
-
result: 'Origin not allowed for app API.'
|
|
1133
|
-
}));
|
|
1134
|
-
return;
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
if (originHeader) {
|
|
1138
|
-
this.applyDynamicAppGatewayCorsHeaders(res, originHeader);
|
|
1139
|
-
}
|
|
1140
|
-
|
|
1141
|
-
if (isCorsPreflightRequest(req)) {
|
|
1142
|
-
res.status(204).end();
|
|
1143
|
-
return;
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
const expectedToken = this.normalizeHeaderValue(appDoc?.rio_token);
|
|
1147
|
-
if (!expectedToken) {
|
|
1148
|
-
res.status(500).send(JSON.stringify({
|
|
1149
|
-
error: true,
|
|
1150
|
-
result: 'App token is not configured.'
|
|
1151
|
-
}));
|
|
1152
|
-
return;
|
|
1153
|
-
}
|
|
1154
|
-
|
|
1155
|
-
const providedToken = this.resolveDynamicAppGatewayToken(req);
|
|
1156
|
-
if (!providedToken || providedToken !== expectedToken) {
|
|
1157
|
-
res.status(401).send(JSON.stringify({
|
|
1158
|
-
error: true,
|
|
1159
|
-
result: 'Invalid or missing RIO token.'
|
|
1160
|
-
}));
|
|
1161
|
-
return;
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
|
-
next();
|
|
1165
|
-
}
|
|
1166
|
-
catch (error) {
|
|
1167
|
-
console.error(new Date(), '[DynamicAppGateway] middleware failure', error);
|
|
1168
|
-
res.status(500).send(JSON.stringify({
|
|
1169
|
-
error: true,
|
|
1170
|
-
result: (error as Error)?.message || 'Dynamic app gateway error.'
|
|
1171
|
-
}));
|
|
1172
|
-
}
|
|
1173
|
-
});
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
private resolveDynamicAppGatewayEnabled(): boolean {
|
|
1177
|
-
const config = ResolveIOServer.getServerConfig() || {};
|
|
1178
|
-
const raw = config['AI_CODER_DYNAMIC_APP_GATEWAY'] ?? process.env.AI_CODER_DYNAMIC_APP_GATEWAY;
|
|
1179
|
-
if (raw !== undefined && raw !== null && `${raw}`.trim() !== '') {
|
|
1180
|
-
return this.parseDebugFlag(raw);
|
|
1181
|
-
}
|
|
1182
|
-
|
|
1183
|
-
const urls = [
|
|
1184
|
-
config['ROOT_URL'],
|
|
1185
|
-
config['SEC_ROOT_URL'],
|
|
1186
|
-
config['SERVER_URL'],
|
|
1187
|
-
process.env.ROOT_URL,
|
|
1188
|
-
process.env.SEC_ROOT_URL,
|
|
1189
|
-
process.env.SERVER_URL
|
|
1190
|
-
]
|
|
1191
|
-
.map(value => `${value || ''}`.trim().toLowerCase())
|
|
1192
|
-
.filter(Boolean);
|
|
1193
|
-
|
|
1194
|
-
return urls.some(value => value.includes('aicoder'));
|
|
1195
|
-
}
|
|
1196
|
-
|
|
1197
|
-
private extractDynamicAppGatewayId(req: express.Request): string {
|
|
1198
|
-
const rawPath = `${req.originalUrl || req.url || req.path || ''}`;
|
|
1199
|
-
const normalizedPath = rawPath.split('?')[0];
|
|
1200
|
-
if (!normalizedPath.startsWith('/api/apps/')) {
|
|
1201
|
-
return '';
|
|
1202
|
-
}
|
|
1203
|
-
|
|
1204
|
-
const parts = normalizedPath.split('/').filter(Boolean);
|
|
1205
|
-
if (parts.length < 3) {
|
|
1206
|
-
return '';
|
|
1207
|
-
}
|
|
1208
|
-
|
|
1209
|
-
return `${parts[2] || ''}`.trim();
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
private resolveDynamicAppGatewayToken(req: express.Request): string {
|
|
1213
|
-
const authHeader = this.normalizeHeaderValue(req.headers?.authorization || req.headers?.Authorization);
|
|
1214
|
-
if (authHeader) {
|
|
1215
|
-
const bearerMatch = authHeader.match(/^bearer\s+(.+)$/i);
|
|
1216
|
-
if (bearerMatch && bearerMatch[1]) {
|
|
1217
|
-
return bearerMatch[1].trim();
|
|
1218
|
-
}
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
const headerCandidates = [
|
|
1222
|
-
req.headers?.['x-rio-token'],
|
|
1223
|
-
req.headers?.['x-app-token'],
|
|
1224
|
-
req.headers?.['x-api-key']
|
|
1225
|
-
];
|
|
1226
|
-
for (const candidate of headerCandidates) {
|
|
1227
|
-
const value = this.normalizeHeaderValue(candidate);
|
|
1228
|
-
if (value) {
|
|
1229
|
-
return value;
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
const body: any = req.body || {};
|
|
1234
|
-
const query: any = req.query || {};
|
|
1235
|
-
const bodyCandidates = [body.rioToken, body.rio_token, body.apiKey, body.token];
|
|
1236
|
-
for (const candidate of bodyCandidates) {
|
|
1237
|
-
const value = this.normalizeHeaderValue(candidate);
|
|
1238
|
-
if (value) {
|
|
1239
|
-
return value;
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
const queryCandidates = [query.rioToken, query.rio_token, query.apiKey, query.token];
|
|
1243
|
-
for (const candidate of queryCandidates) {
|
|
1244
|
-
const value = this.normalizeHeaderValue(candidate);
|
|
1245
|
-
if (value) {
|
|
1246
|
-
return value;
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
|
|
1250
|
-
return '';
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
private normalizeHeaderValue(value: any): string {
|
|
1254
|
-
if (Array.isArray(value)) {
|
|
1255
|
-
return value.map(entry => `${entry || ''}`.trim()).filter(Boolean).join(',');
|
|
1256
|
-
}
|
|
1257
|
-
return `${value || ''}`.trim();
|
|
1258
|
-
}
|
|
1259
|
-
|
|
1260
|
-
private normalizeHostname(value: string): string {
|
|
1261
|
-
const raw = `${value || ''}`.trim();
|
|
1262
|
-
if (!raw) {
|
|
1263
|
-
return '';
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
let candidate = raw.split(',')[0].trim();
|
|
1267
|
-
if (!candidate) {
|
|
1268
|
-
return '';
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
try {
|
|
1272
|
-
if (candidate.startsWith('http://') || candidate.startsWith('https://')) {
|
|
1273
|
-
return new URL(candidate).hostname.toLowerCase();
|
|
1274
|
-
}
|
|
1275
|
-
if (candidate.includes('/')) {
|
|
1276
|
-
return new URL(`http://${candidate}`).hostname.toLowerCase();
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
catch {}
|
|
1280
|
-
|
|
1281
|
-
candidate = candidate.replace(/^\[/, '').replace(/\]$/, '');
|
|
1282
|
-
const lastColon = candidate.lastIndexOf(':');
|
|
1283
|
-
if (lastColon > -1 && candidate.indexOf(':') === lastColon) {
|
|
1284
|
-
candidate = candidate.slice(0, lastColon);
|
|
1285
|
-
}
|
|
1286
|
-
return candidate.toLowerCase();
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
private resolveOriginHostname(origin: string): string {
|
|
1290
|
-
const raw = `${origin || ''}`.trim();
|
|
1291
|
-
if (!raw) {
|
|
1292
|
-
return '';
|
|
1293
|
-
}
|
|
1294
|
-
try {
|
|
1295
|
-
return new URL(raw).hostname.toLowerCase();
|
|
1296
|
-
}
|
|
1297
|
-
catch {
|
|
1298
|
-
return this.normalizeHostname(raw);
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
|
|
1302
|
-
private isLocalHostname(hostname: string): boolean {
|
|
1303
|
-
const normalized = this.normalizeHostname(hostname);
|
|
1304
|
-
return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '::1';
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
|
-
private resolveDynamicAppControlHosts(): Set<string> {
|
|
1308
|
-
const config = ResolveIOServer.getServerConfig() || {};
|
|
1309
|
-
const hosts = new Set<string>();
|
|
1310
|
-
const candidates = [
|
|
1311
|
-
config['ROOT_URL'],
|
|
1312
|
-
config['SEC_ROOT_URL'],
|
|
1313
|
-
config['SERVER_URL'],
|
|
1314
|
-
process.env.ROOT_URL,
|
|
1315
|
-
process.env.SEC_ROOT_URL,
|
|
1316
|
-
process.env.SERVER_URL,
|
|
1317
|
-
process.env.AI_CODER_ROOT_URL,
|
|
1318
|
-
process.env.AI_CODER_SEC_ROOT_URL,
|
|
1319
|
-
process.env.AI_CODER_SERVER_URL
|
|
1320
|
-
];
|
|
1321
|
-
for (const candidate of candidates) {
|
|
1322
|
-
const normalized = this.normalizeHostname(`${candidate || ''}`);
|
|
1323
|
-
if (normalized) {
|
|
1324
|
-
hosts.add(normalized);
|
|
1325
|
-
}
|
|
1326
|
-
}
|
|
1327
|
-
return hosts;
|
|
1328
|
-
}
|
|
1329
|
-
|
|
1330
|
-
private resolveDynamicAppAllowedHosts(appDoc: any): Set<string> {
|
|
1331
|
-
const hosts = new Set<string>();
|
|
1332
|
-
const candidates = [
|
|
1333
|
-
appDoc?.domain,
|
|
1334
|
-
appDoc?.backend_domain
|
|
1335
|
-
];
|
|
1336
|
-
if (appDoc?.subdomain && appDoc?.domain_base) {
|
|
1337
|
-
candidates.push(`${appDoc.subdomain}.${appDoc.domain_base}`);
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
for (const candidate of candidates) {
|
|
1341
|
-
const normalized = this.normalizeHostname(`${candidate || ''}`);
|
|
1342
|
-
if (normalized) {
|
|
1343
|
-
hosts.add(normalized);
|
|
1344
|
-
}
|
|
1345
|
-
}
|
|
1346
|
-
return hosts;
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
private applyDynamicAppGatewayCorsHeaders(res: express.Response, origin: string): void {
|
|
1350
|
-
res.setHeader('Access-Control-Allow-Origin', origin);
|
|
1351
|
-
res.setHeader('Vary', 'Origin');
|
|
1352
|
-
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS');
|
|
1353
|
-
res.setHeader('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept, Authorization, X-Rio-Token, X-API-Key, X-AI-Coder-App-Token, X-AI-Coder-Session');
|
|
1354
|
-
res.setHeader('Access-Control-Allow-Credentials', 'true');
|
|
1355
|
-
res.setHeader('Access-Control-Allow-Private-Network', 'true');
|
|
1356
|
-
res.setHeader('Access-Control-Max-Age', '600');
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
private async resolveCachedDynamicAppGatewayApp(appId: string): Promise<any | null> {
|
|
1360
|
-
const now = Date.now();
|
|
1361
|
-
const cached = this._dynamicAppGatewayCache.get(appId);
|
|
1362
|
-
if (cached && cached.expiresAt > now) {
|
|
1363
|
-
return cached.app;
|
|
1364
|
-
}
|
|
1365
|
-
|
|
1366
|
-
const db = ResolveIOServer.getMainDB();
|
|
1367
|
-
if (!db) {
|
|
1368
|
-
return null;
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
const appCollection: any = db.collection('ai-coder-apps');
|
|
1372
|
-
const appDoc = await appCollection.findOne(
|
|
1373
|
-
{ _id: appId },
|
|
1374
|
-
{
|
|
1375
|
-
projection: {
|
|
1376
|
-
_id: 1,
|
|
1377
|
-
domain: 1,
|
|
1378
|
-
backend_domain: 1,
|
|
1379
|
-
subdomain: 1,
|
|
1380
|
-
domain_base: 1,
|
|
1381
|
-
rio_token: 1
|
|
1382
|
-
}
|
|
1383
|
-
}
|
|
1384
|
-
);
|
|
1385
|
-
if (!appDoc) {
|
|
1386
|
-
this._dynamicAppGatewayCache.delete(appId);
|
|
1387
|
-
return null;
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
if (!this.normalizeHeaderValue(appDoc.rio_token)) {
|
|
1391
|
-
const generated = crypto.randomBytes(32).toString('hex');
|
|
1392
|
-
await appCollection.updateOne(
|
|
1393
|
-
{ _id: appId },
|
|
1394
|
-
{
|
|
1395
|
-
$set: {
|
|
1396
|
-
rio_token: generated,
|
|
1397
|
-
updatedAt: new Date()
|
|
1398
|
-
}
|
|
1399
|
-
}
|
|
1400
|
-
);
|
|
1401
|
-
appDoc.rio_token = generated;
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
this._dynamicAppGatewayCache.set(appId, {
|
|
1405
|
-
expiresAt: now + this._dynamicAppGatewayCacheMs,
|
|
1406
|
-
app: appDoc
|
|
1407
|
-
});
|
|
1408
|
-
return appDoc;
|
|
1409
|
-
}
|
|
1410
|
-
|
|
1411
|
-
private async safeShutdown() {
|
|
1412
|
-
if (!this._safeShutdown) {
|
|
1413
|
-
console.log(new Date(), 'Safe Shutdown Command Received');
|
|
1414
|
-
}
|
|
1415
|
-
|
|
1416
|
-
if (
|
|
1417
|
-
!this._monitorManagerFunction.getActiveMonitorFunctions().length
|
|
1418
|
-
&& !this._offlineUpdates.length && (!this._workerDispatcherManager || this._workerDispatcherManager.isSafeShutdown())
|
|
1419
|
-
) {
|
|
1420
|
-
if (ResolveIOServer.getMongoConnection()) {
|
|
1421
|
-
try {
|
|
1422
|
-
await ResolveIOServer.getMongoConnection().close(false);
|
|
1423
|
-
console.log(new Date(), 'Safe Exit Complete, Process Exit');
|
|
1424
|
-
process.exit(0);
|
|
1425
|
-
}
|
|
1426
|
-
catch {
|
|
1427
|
-
process.exit(1);
|
|
1428
|
-
};
|
|
1429
|
-
}
|
|
1430
|
-
else {
|
|
1431
|
-
process.exit(0);
|
|
1432
|
-
}
|
|
1433
|
-
}
|
|
1434
|
-
else {
|
|
1435
|
-
if (!this._safeShutdown) {
|
|
1436
|
-
this._safeShutdown = true;
|
|
1437
|
-
|
|
1438
|
-
setTimeout(() => {
|
|
1439
|
-
this._safeShutdown = false;
|
|
1440
|
-
}, 1000);
|
|
1441
|
-
|
|
1442
|
-
console.log(new Date(), 'Safe Exit In Progress',
|
|
1443
|
-
this._monitorManagerFunction.getActiveMonitorFunctions().length,
|
|
1444
|
-
this._offlineUpdates.length
|
|
1445
|
-
);
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
setImmediate(async () => {
|
|
1449
|
-
await this.safeShutdown();
|
|
1450
|
-
});
|
|
1451
|
-
}
|
|
1452
|
-
}
|
|
1453
|
-
|
|
1454
|
-
getIsWorkersEnabled() {
|
|
1455
|
-
return this._isWorkersEnabled;
|
|
1456
|
-
}
|
|
1457
|
-
|
|
1458
|
-
getIsWorkerInstance() {
|
|
1459
|
-
return this._isWorkerInstance;
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
public getWSList() {
|
|
1463
|
-
let res = [];
|
|
1464
|
-
this._serverWSS.clients.forEach((ws: WebSocket) => {
|
|
1465
|
-
res.push(ws['id_socket']);
|
|
1466
|
-
});
|
|
1467
|
-
return res;
|
|
1468
|
-
}
|
|
1469
|
-
|
|
1470
|
-
public getWSUserList() {
|
|
1471
|
-
let res = [];
|
|
1472
|
-
this._serverWSS.clients.forEach((ws: WebSocket) => {
|
|
1473
|
-
res.push(ws['id_user']);
|
|
1474
|
-
});
|
|
1475
|
-
return res;
|
|
1476
|
-
}
|
|
1477
|
-
|
|
1478
|
-
public getHTTPServer() {
|
|
1479
|
-
return this._serverHTTP;
|
|
1480
|
-
}
|
|
1481
|
-
|
|
1482
|
-
public getCronManager() {
|
|
1483
|
-
return this._cronManager;
|
|
1484
|
-
}
|
|
1485
|
-
|
|
1486
|
-
private async reportServerError(
|
|
1487
|
-
subject: string,
|
|
1488
|
-
correlationId: string,
|
|
1489
|
-
context: Record<string, any>,
|
|
1490
|
-
meta?: Record<string, any>,
|
|
1491
|
-
severity = 'error',
|
|
1492
|
-
stackOverride?: string
|
|
1493
|
-
) {
|
|
1494
|
-
const config = ResolveIOServer.getServerConfig();
|
|
1495
|
-
const metadata = Object.assign({}, meta || {});
|
|
1496
|
-
if (correlationId && !metadata.correlationId) {
|
|
1497
|
-
metadata.correlationId = correlationId;
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
await ErrorReporter.report({
|
|
1501
|
-
sourceApp: 'server-app',
|
|
1502
|
-
message: subject,
|
|
1503
|
-
environment: config?.ROOT_URL || process.env.NODE_ENV || 'unknown',
|
|
1504
|
-
clientSlug: ResolveIOServer.getClientName(),
|
|
1505
|
-
clientName: config?.CLIENT_NAME,
|
|
1506
|
-
severity,
|
|
1507
|
-
stack: stackOverride || (typeof context?.stack === 'string' ? context.stack : undefined),
|
|
1508
|
-
context,
|
|
1509
|
-
metadata,
|
|
1510
|
-
correlationId
|
|
1511
|
-
});
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
public getMethodManager() {
|
|
1515
|
-
return this._methodManager;
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
public getSubscriptionManager() {
|
|
1519
|
-
return this._subscriptionManager;
|
|
1520
|
-
}
|
|
1521
|
-
|
|
1522
|
-
public getMonitorManager() {
|
|
1523
|
-
return this._monitorManager;
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
public getRebootFlag() {
|
|
1527
|
-
return this._rebootFlag;
|
|
1528
|
-
}
|
|
1529
|
-
|
|
1530
|
-
public getWebSocketManager(): WebSocketManager {
|
|
1531
|
-
return this._websocketManager;
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
private createServer(): void {
|
|
1535
|
-
this._serverHTTP = createServer(this._app);
|
|
1536
|
-
this._serverHTTP.keepAliveTimeout = 65000;
|
|
1537
|
-
this._serverHTTP.headersTimeout = 66000;
|
|
1538
|
-
|
|
1539
|
-
this._serverWSS = new WebSocket.Server({
|
|
1540
|
-
server: this._serverHTTP,
|
|
1541
|
-
verifyClient: this.publicProgram ? null : (info, cb) => {
|
|
1542
|
-
if (this._rebootFlag) {
|
|
1543
|
-
cb(false, 409, 'Unable To Process');
|
|
1544
|
-
}
|
|
1545
|
-
else {
|
|
1546
|
-
if (this.LOGGER === 'DEBUG') {
|
|
1547
|
-
console.log('Verify Client', info, cb);
|
|
1548
|
-
}
|
|
1549
|
-
|
|
1550
|
-
// If it's a worker, we might skip token checks or do a simple check:
|
|
1551
|
-
if (info.req.url && info.req.url.includes('workerToken=')) {
|
|
1552
|
-
let requestUrl: URL;
|
|
1553
|
-
let rootUrl = ResolveIOServer.getServerConfig()['ROOT_URL'] || 'http://localhost';
|
|
1554
|
-
try {
|
|
1555
|
-
requestUrl = new URL(info.req.url, rootUrl);
|
|
1556
|
-
}
|
|
1557
|
-
catch {
|
|
1558
|
-
cb(false, 400, 'Bad Request');
|
|
1559
|
-
return;
|
|
1560
|
-
}
|
|
1561
|
-
|
|
1562
|
-
let workerToken = requestUrl.searchParams.get('workerToken') || '';
|
|
1563
|
-
let workerIndex = this.normalizeWorkerSelectorValue(requestUrl.searchParams.get('workerIndex'));
|
|
1564
|
-
let workerInstance = this.normalizeWorkerSelectorValue(requestUrl.searchParams.get('workerInstance'));
|
|
1565
|
-
let expectedWorkerToken = String(ResolveIOServer.getServerConfig()['WORKER_TOKEN'] || '');
|
|
1566
|
-
|
|
1567
|
-
if (!workerIndex || !workerInstance) {
|
|
1568
|
-
cb(false, 400, 'Missing worker identity');
|
|
1569
|
-
return;
|
|
1570
|
-
}
|
|
1571
|
-
|
|
1572
|
-
if (workerToken === expectedWorkerToken) {
|
|
1573
|
-
info.req['workerIndex'] = workerIndex;
|
|
1574
|
-
info.req['workerInstance'] = workerInstance;
|
|
1575
|
-
cb(true);
|
|
1576
|
-
}
|
|
1577
|
-
else {
|
|
1578
|
-
cb(false, 401, 'Unauthorized');
|
|
1579
|
-
}
|
|
1580
|
-
|
|
1581
|
-
return;
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
|
-
const protocolsHeader = info.req.headers['sec-websocket-protocol'];
|
|
1585
|
-
if (!protocolsHeader || typeof protocolsHeader !== 'string') {
|
|
1586
|
-
cb(false, 401, 'Unauthorized');
|
|
1587
|
-
return;
|
|
1588
|
-
}
|
|
1589
|
-
|
|
1590
|
-
let infoData = protocolsHeader.split(/,/);
|
|
1591
|
-
|
|
1592
|
-
if (!isAllowedOrigin(info.origin, ResolveIOServer.getServerConfig())) {
|
|
1593
|
-
cb(false, 401, 'Unauthorized');
|
|
1594
|
-
}
|
|
1595
|
-
else {
|
|
1596
|
-
let token = infoData[0];
|
|
1597
|
-
if (!token) {
|
|
1598
|
-
cb(false, 401, 'Unauthorized');
|
|
1599
|
-
}
|
|
1600
|
-
else {
|
|
1601
|
-
jwt.verify(token, ResolveIOServer.getServerConfig()['JWT_SECRET'], async (err, decoded) => {
|
|
1602
|
-
if (err) {
|
|
1603
|
-
cb(false, 401, 'Unauthorized');
|
|
1604
|
-
}
|
|
1605
|
-
else {
|
|
1606
|
-
info.req['id_user'] = decoded['id_user'];
|
|
1607
|
-
try {
|
|
1608
|
-
let user = await Users.findById(decoded['id_user']);
|
|
1609
|
-
if (user) {
|
|
1610
|
-
const socketAdmission = await this.evaluateClientSocketAdmission(decoded['id_user'], info.req);
|
|
1611
|
-
if (!socketAdmission.allowed) {
|
|
1612
|
-
cb(false, socketAdmission.statusCode, socketAdmission.message || 'Socket connection rejected.');
|
|
1613
|
-
return;
|
|
1614
|
-
}
|
|
1615
|
-
info.req['user'] = user.fullname;
|
|
1616
|
-
info.req['user_readonly'] = user.readonly || false;
|
|
1617
|
-
info.req['doc_user'] = user;
|
|
1618
|
-
info.req['client_ip'] = socketAdmission.clientIp;
|
|
1619
|
-
cb(true);
|
|
1620
|
-
}
|
|
1621
|
-
else {
|
|
1622
|
-
cb(false);
|
|
1623
|
-
}
|
|
1624
|
-
}
|
|
1625
|
-
catch {
|
|
1626
|
-
cb(false);
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
});
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
}
|
|
1633
|
-
}
|
|
1634
|
-
});
|
|
1635
|
-
}
|
|
1636
|
-
|
|
1637
|
-
/**
|
|
1638
|
-
* Listen for connections from clients or workers.
|
|
1639
|
-
*/
|
|
1640
|
-
private listen(): void {
|
|
1641
|
-
const host = process.env.RESOLVEIO_BIND_HOST || '127.0.0.1';
|
|
1642
|
-
this._serverHTTP.listen(this._portHTTP, host, () => {
|
|
1643
|
-
console.log('Running HTTP/WS server on port %s', this._portHTTP);
|
|
1644
|
-
});
|
|
1645
|
-
|
|
1646
|
-
this._serverWSS.on('connection', async (ws, req) => {
|
|
1647
|
-
if (req.url && req.url.includes('workerToken=')) {
|
|
1648
|
-
// It's a WORKER
|
|
1649
|
-
let workerId = objectIdHexString();
|
|
1650
|
-
ws['id_worker'] = workerId;
|
|
1651
|
-
let workerIndex = null;
|
|
1652
|
-
let workerInstance = null;
|
|
1653
|
-
ws['supportsBinary'] = true;
|
|
1654
|
-
|
|
1655
|
-
if (req.url) {
|
|
1656
|
-
let rootUrl = ResolveIOServer.getServerConfig()['ROOT_URL'] || 'http://localhost';
|
|
1657
|
-
try {
|
|
1658
|
-
let requestUrl = new URL(req.url, rootUrl);
|
|
1659
|
-
workerIndex = requestUrl.searchParams.get('workerIndex');
|
|
1660
|
-
workerInstance = requestUrl.searchParams.get('workerInstance');
|
|
1661
|
-
}
|
|
1662
|
-
catch {
|
|
1663
|
-
workerIndex = null;
|
|
1664
|
-
workerInstance = null;
|
|
1665
|
-
}
|
|
1666
|
-
}
|
|
1667
|
-
|
|
1668
|
-
if (!workerIndex && req['workerIndex']) {
|
|
1669
|
-
workerIndex = req['workerIndex'];
|
|
1670
|
-
}
|
|
1671
|
-
if (!workerInstance && req['workerInstance']) {
|
|
1672
|
-
workerInstance = req['workerInstance'];
|
|
1673
|
-
}
|
|
1674
|
-
|
|
1675
|
-
workerIndex = this.normalizeWorkerSelectorValue(workerIndex);
|
|
1676
|
-
workerInstance = this.normalizeWorkerSelectorValue(workerInstance);
|
|
1677
|
-
|
|
1678
|
-
if (!workerIndex || !workerInstance) {
|
|
1679
|
-
console.warn(new Date(), 'Rejected worker connection with missing identity', {
|
|
1680
|
-
workerId,
|
|
1681
|
-
workerIndex: workerIndex || null,
|
|
1682
|
-
workerInstance: workerInstance || null
|
|
1683
|
-
});
|
|
1684
|
-
ws.close(1008, 'Missing worker identity');
|
|
1685
|
-
return;
|
|
1686
|
-
}
|
|
1687
|
-
|
|
1688
|
-
ws['workerIndex'] = workerIndex;
|
|
1689
|
-
ws['workerInstance'] = workerInstance;
|
|
1690
|
-
|
|
1691
|
-
let workerIndexForLog = ws['workerIndex'] || 'UNKNOWN';
|
|
1692
|
-
let workerInstanceForLog = ws['workerInstance'] || 'UNKNOWN';
|
|
1693
|
-
console.log(new Date(), 'Worker Connected', workerIndexForLog, workerInstanceForLog);
|
|
1694
|
-
|
|
1695
|
-
this._workerDispatcherManager.addWorker(ws);
|
|
1696
|
-
|
|
1697
|
-
let interval = null;
|
|
1698
|
-
let lastComm = new Date();
|
|
1699
|
-
let missedPongs = 0;
|
|
1700
|
-
const heartbeatIntervalMs = 30000;
|
|
1701
|
-
const maxMissedPongs = 2;
|
|
1702
|
-
const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);
|
|
1703
|
-
|
|
1704
|
-
this._workerDispatcherManager.sendWorkerPayload(ws, 'ping');
|
|
1705
|
-
|
|
1706
|
-
interval = setInterval(() => {
|
|
1707
|
-
const now = Date.now();
|
|
1708
|
-
const last = lastComm ? lastComm.getTime() : 0;
|
|
1709
|
-
const silenceMs = last ? now - last : maxSilenceMs + 1;
|
|
1710
|
-
if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {
|
|
1711
|
-
this._workerDispatcherManager.disconnectWorker(ws['id_worker']);
|
|
1712
|
-
ws.close();
|
|
1713
|
-
return;
|
|
1714
|
-
}
|
|
1715
|
-
missedPongs += 1;
|
|
1716
|
-
this._workerDispatcherManager.sendWorkerPayload(ws, 'ping');
|
|
1717
|
-
}, heartbeatIntervalMs);
|
|
1718
|
-
|
|
1719
|
-
ws.on('message', (message: WebSocket.RawData) => {
|
|
1720
|
-
lastComm = new Date();
|
|
1721
|
-
if (typeof message === 'string') {
|
|
1722
|
-
if (message === 'ping') {
|
|
1723
|
-
this._workerDispatcherManager.sendWorkerPayload(ws, 'pong');
|
|
1724
|
-
}
|
|
1725
|
-
else if (message === 'pong') {
|
|
1726
|
-
missedPongs = 0;
|
|
1727
|
-
}
|
|
1728
|
-
else {
|
|
1729
|
-
this._workerDispatcherManager.handleWorkerMessage(ws['id_worker'], message);
|
|
1730
|
-
}
|
|
1731
|
-
|
|
1732
|
-
return;
|
|
1733
|
-
}
|
|
1734
|
-
|
|
1735
|
-
let buffer: Buffer;
|
|
1736
|
-
|
|
1737
|
-
if (Buffer.isBuffer(message)) {
|
|
1738
|
-
buffer = message;
|
|
1739
|
-
}
|
|
1740
|
-
else if (Array.isArray(message)) {
|
|
1741
|
-
const chunks = message as unknown as ReadonlyArray<Uint8Array>;
|
|
1742
|
-
buffer = Buffer.concat(chunks);
|
|
1743
|
-
}
|
|
1744
|
-
else if (message instanceof ArrayBuffer) {
|
|
1745
|
-
buffer = Buffer.from(message);
|
|
1746
|
-
}
|
|
1747
|
-
else if (ArrayBuffer.isView(message)) {
|
|
1748
|
-
const view = message as NodeJS.ArrayBufferView;
|
|
1749
|
-
buffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
|
|
1750
|
-
}
|
|
1751
|
-
else {
|
|
1752
|
-
buffer = Buffer.from(message as any);
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
|
-
if (buffer.length === 4) {
|
|
1756
|
-
let heartbeat = buffer.toString('utf8');
|
|
1757
|
-
|
|
1758
|
-
if (heartbeat === 'ping') {
|
|
1759
|
-
this._workerDispatcherManager.sendWorkerPayload(ws, 'pong');
|
|
1760
|
-
return;
|
|
1761
|
-
}
|
|
1762
|
-
else if (heartbeat === 'pong') {
|
|
1763
|
-
missedPongs = 0;
|
|
1764
|
-
return;
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
this._workerDispatcherManager.handleWorkerMessage(ws['id_worker'], buffer);
|
|
1769
|
-
});
|
|
1770
|
-
|
|
1771
|
-
ws.on('close', () => {
|
|
1772
|
-
this._workerDispatcherManager.disconnectWorker(ws['id_worker']);
|
|
1773
|
-
|
|
1774
|
-
console.log(new Date(), 'Worker disconnected:', workerId);
|
|
1775
|
-
|
|
1776
|
-
if (interval) {
|
|
1777
|
-
clearInterval(interval);
|
|
1778
|
-
}
|
|
1779
|
-
});
|
|
1780
|
-
|
|
1781
|
-
ws.on('error', (error) => {
|
|
1782
|
-
this._workerDispatcherManager.disconnectWorker(ws['id_worker']);
|
|
1783
|
-
|
|
1784
|
-
console.error('Error on WS Worker', error);
|
|
1785
|
-
ws.close();
|
|
1786
|
-
});
|
|
1787
|
-
}
|
|
1788
|
-
else {
|
|
1789
|
-
// Normal client
|
|
1790
|
-
ws['id_socket'] = objectIdHexString();
|
|
1791
|
-
ws['supportsBinary'] = true;
|
|
1792
|
-
ws['id_user'] = req['id_user'];
|
|
1793
|
-
ws['user'] = req['user'];
|
|
1794
|
-
ws['user_readonly'] = req['user_readonly'];
|
|
1795
|
-
ws['doc_user'] = req['doc_user'];
|
|
1796
|
-
ws['client_ip'] = this.resolveClientIp(req);
|
|
1797
|
-
|
|
1798
|
-
let socketAdmission: { allowed: boolean, statusCode: number, message: string, clientIp: string };
|
|
1799
|
-
try {
|
|
1800
|
-
socketAdmission = await this.evaluateClientSocketAdmission(ws['id_user'], req);
|
|
1801
|
-
}
|
|
1802
|
-
catch (socketPolicyError) {
|
|
1803
|
-
this.logConnectDebug('WS socket policy evaluation failed', {
|
|
1804
|
-
id_socket: ws['id_socket'],
|
|
1805
|
-
id_user: ws['id_user'],
|
|
1806
|
-
user: ws['user'],
|
|
1807
|
-
ip: ws['client_ip'],
|
|
1808
|
-
error: (socketPolicyError as Error)?.message || socketPolicyError
|
|
1809
|
-
});
|
|
1810
|
-
try {
|
|
1811
|
-
ws.close(1011, 'Socket policy error');
|
|
1812
|
-
}
|
|
1813
|
-
catch {}
|
|
1814
|
-
return;
|
|
1815
|
-
}
|
|
1816
|
-
|
|
1817
|
-
if (!socketAdmission.allowed) {
|
|
1818
|
-
this.logConnectDebug('WS client rejected', {
|
|
1819
|
-
id_socket: ws['id_socket'],
|
|
1820
|
-
id_user: ws['id_user'],
|
|
1821
|
-
user: ws['user'],
|
|
1822
|
-
ip: ws['client_ip'],
|
|
1823
|
-
reason: socketAdmission.message
|
|
1824
|
-
});
|
|
1825
|
-
try {
|
|
1826
|
-
ws.close(1008, this.buildSocketLimitCloseReason());
|
|
1827
|
-
}
|
|
1828
|
-
catch {}
|
|
1829
|
-
return;
|
|
1830
|
-
}
|
|
1831
|
-
ws['client_ip'] = socketAdmission.clientIp || ws['client_ip'];
|
|
1832
|
-
|
|
1833
|
-
this._websocketManager.addWebSocket(ws);
|
|
1834
|
-
|
|
1835
|
-
this.logConnectDebug('WS client connected', {
|
|
1836
|
-
id_socket: ws['id_socket'],
|
|
1837
|
-
id_user: ws['id_user'],
|
|
1838
|
-
user: ws['user'],
|
|
1839
|
-
url: req?.url,
|
|
1840
|
-
ip: ws['client_ip'],
|
|
1841
|
-
origin: req?.headers?.origin
|
|
1842
|
-
});
|
|
1843
|
-
|
|
1844
|
-
setTimeout(async () => {
|
|
1845
|
-
await this.triggerClientHeartbeat(ws);
|
|
1846
|
-
}, this._clientHeartbeatInitialDelayMs);
|
|
1847
|
-
|
|
1848
|
-
if (this.LOGGER === 'DEBUG') {
|
|
1849
|
-
console.log('Connection from user: ' + req['user']);
|
|
1850
|
-
}
|
|
1851
|
-
|
|
1852
|
-
ws['isAlive'] = true;
|
|
1853
|
-
ws['retryCnt'] = 0;
|
|
1854
|
-
ws.on('pong', () => {
|
|
1855
|
-
ws['isAlive'] = true;
|
|
1856
|
-
ws['pongTime'] = new Date();
|
|
1857
|
-
if (ws['pingTime']) {
|
|
1858
|
-
ws['latency'] = moment.duration(moment(ws['pongTime']).diff(ws['pingTime'])).asMilliseconds();
|
|
1859
|
-
this._subscriptionManager.loggedInLatency(ws);
|
|
1860
|
-
}
|
|
1861
|
-
});
|
|
1862
|
-
|
|
1863
|
-
ws.on('message', async (message: WebSocket.RawData) => {
|
|
1864
|
-
this._debugMsgRecv += 1;
|
|
1865
|
-
let socketData = [];
|
|
1866
|
-
let usedBinary = false;
|
|
1867
|
-
let bufferPayload: Buffer;
|
|
1868
|
-
|
|
1869
|
-
try {
|
|
1870
|
-
if (typeof message === 'string') {
|
|
1871
|
-
if (message === 'ping' || message === 'pong') {
|
|
1872
|
-
socketData = message;
|
|
1873
|
-
}
|
|
1874
|
-
else {
|
|
1875
|
-
socketData = JSON.parse(message, dateReviver);
|
|
1876
|
-
}
|
|
1877
|
-
}
|
|
1878
|
-
else if (Buffer.isBuffer(message)) {
|
|
1879
|
-
bufferPayload = message;
|
|
1880
|
-
let decodeResult = this.decodeBufferPayload(bufferPayload);
|
|
1881
|
-
socketData = decodeResult.data;
|
|
1882
|
-
usedBinary = decodeResult.usedBinary;
|
|
1883
|
-
}
|
|
1884
|
-
else if (Array.isArray(message)) {
|
|
1885
|
-
bufferPayload = Buffer.concat(message as unknown as ReadonlyArray<Uint8Array>);
|
|
1886
|
-
let decodeResult = this.decodeBufferPayload(bufferPayload);
|
|
1887
|
-
socketData = decodeResult.data;
|
|
1888
|
-
usedBinary = decodeResult.usedBinary;
|
|
1889
|
-
}
|
|
1890
|
-
else if (message instanceof ArrayBuffer) {
|
|
1891
|
-
bufferPayload = Buffer.from(message);
|
|
1892
|
-
let decodeResult = this.decodeBufferPayload(bufferPayload);
|
|
1893
|
-
socketData = decodeResult.data;
|
|
1894
|
-
usedBinary = decodeResult.usedBinary;
|
|
1895
|
-
}
|
|
1896
|
-
else if (ArrayBuffer.isView(message)) {
|
|
1897
|
-
const view = message as NodeJS.ArrayBufferView;
|
|
1898
|
-
bufferPayload = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
|
|
1899
|
-
let decodeResult = this.decodeBufferPayload(bufferPayload);
|
|
1900
|
-
socketData = decodeResult.data;
|
|
1901
|
-
usedBinary = decodeResult.usedBinary;
|
|
1902
|
-
}
|
|
1903
|
-
else {
|
|
1904
|
-
throw new Error('Unsupported WebSocket message type: ' + typeof message);
|
|
1905
|
-
}
|
|
1906
|
-
}
|
|
1907
|
-
catch (e) {
|
|
1908
|
-
console.log('Error - WS message parse', e);
|
|
1909
|
-
const correlationId = objectIdHexString();
|
|
1910
|
-
const context = {
|
|
1911
|
-
rawBinary: bufferPayload ? bufferPayload.toString('base64') : undefined,
|
|
1912
|
-
rawMessage: typeof message === 'string' ? message : undefined,
|
|
1913
|
-
error: e instanceof Error ? { name: e.name, message: e.message, stack: e.stack } : e
|
|
1914
|
-
};
|
|
1915
|
-
await this.reportServerError(
|
|
1916
|
-
'SERVER - JSON Parse Error - ' + ResolveIOServer.getServerConfig()['CLIENT_NAME'],
|
|
1917
|
-
correlationId,
|
|
1918
|
-
context,
|
|
1919
|
-
{ context: 'websocket-message-parse' },
|
|
1920
|
-
'error',
|
|
1921
|
-
e instanceof Error ? e.stack : undefined
|
|
1922
|
-
);
|
|
1923
|
-
return;
|
|
1924
|
-
}
|
|
1925
|
-
|
|
1926
|
-
if (usedBinary) {
|
|
1927
|
-
ws['supportsBinary'] = true;
|
|
1928
|
-
}
|
|
1929
|
-
|
|
1930
|
-
// call our existing processSocketMessage
|
|
1931
|
-
await this.processSocketMessage(ws, socketData);
|
|
1932
|
-
})
|
|
1933
|
-
.on('end', () => {
|
|
1934
|
-
ws.close();
|
|
1935
|
-
})
|
|
1936
|
-
.on('error', () => {
|
|
1937
|
-
ws.close()
|
|
1938
|
-
})
|
|
1939
|
-
.on('close', async () => {
|
|
1940
|
-
this.logConnectDebug('WS client closed', {
|
|
1941
|
-
id_socket: ws['id_socket'],
|
|
1942
|
-
id_user: ws['id_user'],
|
|
1943
|
-
user: ws['user']
|
|
1944
|
-
});
|
|
1945
|
-
await this.unsubscribeWS(ws);
|
|
1946
|
-
});
|
|
1947
|
-
|
|
1948
|
-
// Do not block message handler registration on DB write; this avoids losing
|
|
1949
|
-
// very-early subscription messages sent immediately after websocket open.
|
|
1950
|
-
setTimeout(async () => {
|
|
1951
|
-
try {
|
|
1952
|
-
await this._subscriptionManager.createLoggedInUser(ws['id_socket']);
|
|
1953
|
-
}
|
|
1954
|
-
catch (error) {
|
|
1955
|
-
console.error(new Date(), 'Error creating logged-in user', ws['id_socket'], error);
|
|
1956
|
-
this.logConnectDebug('Create logged-in user failed', {
|
|
1957
|
-
id_socket: ws['id_socket'],
|
|
1958
|
-
id_user: ws['id_user'],
|
|
1959
|
-
user: ws['user'],
|
|
1960
|
-
error: error?.message || error
|
|
1961
|
-
});
|
|
1962
|
-
}
|
|
1963
|
-
}, 0);
|
|
1964
|
-
}
|
|
1965
|
-
});
|
|
1966
|
-
|
|
1967
|
-
// Keep alive timer
|
|
1968
|
-
setInterval(async () => {
|
|
1969
|
-
for (let ws of this._serverWSS.clients) {
|
|
1970
|
-
if (ws['pingTime'] && Date.now() - ws['pingTime'].getTime() >= this._clientHeartbeatIntervalMs) {
|
|
1971
|
-
if (this.shouldDeferHeartbeat(ws)) {
|
|
1972
|
-
ws['isAlive'] = true;
|
|
1973
|
-
ws['retryCnt'] = 0;
|
|
1974
|
-
ws['pingTime'] = new Date();
|
|
1975
|
-
continue;
|
|
1976
|
-
}
|
|
1977
|
-
|
|
1978
|
-
if (ws['isAlive'] === false) {
|
|
1979
|
-
ws['retryCnt']++;
|
|
1980
|
-
if (ws['retryCnt'] >= 3) {
|
|
1981
|
-
await this.unsubscribeWS(ws);
|
|
1982
|
-
}
|
|
1983
|
-
else {
|
|
1984
|
-
await this.triggerClientHeartbeat(ws);
|
|
1985
|
-
}
|
|
1986
|
-
}
|
|
1987
|
-
else {
|
|
1988
|
-
ws['retryCnt'] = 0;
|
|
1989
|
-
ws['isAlive'] = false;
|
|
1990
|
-
await this.triggerClientHeartbeat(ws);
|
|
1991
|
-
}
|
|
1992
|
-
}
|
|
1993
|
-
}
|
|
1994
|
-
}, this._clientHeartbeatIntervalMs);
|
|
1995
|
-
}
|
|
1996
|
-
|
|
1997
|
-
private async processSocketMessage(ws: WebSocket, socketData: any) {
|
|
1998
|
-
if (typeof socketData === 'string' && socketData === 'ping') {
|
|
1999
|
-
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
2000
|
-
ws.send('pong');
|
|
2001
|
-
}
|
|
2002
|
-
return;
|
|
2003
|
-
}
|
|
2004
|
-
else if (typeof socketData === 'string' && socketData === 'pong') {
|
|
2005
|
-
ws['isAlive'] = true;
|
|
2006
|
-
ws['pongTime'] = new Date();
|
|
2007
|
-
ws['latency'] = moment.duration(moment(ws['pongTime']).diff(ws['pingTime'])).asMilliseconds();
|
|
2008
|
-
this._subscriptionManager.loggedInLatency(ws);
|
|
2009
|
-
return;
|
|
2010
|
-
}
|
|
2011
|
-
|
|
2012
|
-
// If the top level is not an array, let's skip
|
|
2013
|
-
if (!Array.isArray(socketData[0])) {
|
|
2014
|
-
console.log('Invalid message format (expected array of arrays)', socketData);
|
|
2015
|
-
this.logConnectDebug('Invalid message format', {
|
|
2016
|
-
id_socket: ws ? ws['id_socket'] : null,
|
|
2017
|
-
user: ws ? ws['user'] : null,
|
|
2018
|
-
preview: Array.isArray(socketData) ? socketData.slice(0, 3) : socketData
|
|
2019
|
-
});
|
|
2020
|
-
return;
|
|
2021
|
-
}
|
|
2022
|
-
|
|
2023
|
-
// Handle each sub-message
|
|
2024
|
-
for (let message of socketData) {
|
|
2025
|
-
await this.handleClientMessage(ws, message);
|
|
2026
|
-
}
|
|
2027
|
-
}
|
|
2028
|
-
|
|
2029
|
-
private decodeBufferPayload(buffer: Buffer): { data: any; usedBinary: boolean } {
|
|
2030
|
-
const textPayload = buffer.toString('utf8');
|
|
2031
|
-
|
|
2032
|
-
if (this.looksLikeTextPayload(textPayload)) {
|
|
2033
|
-
try {
|
|
2034
|
-
return { data: this.parseTextFallback(textPayload), usedBinary: false };
|
|
2035
|
-
}
|
|
2036
|
-
catch {
|
|
2037
|
-
// fall through to attempt MessagePack decode
|
|
2038
|
-
}
|
|
2039
|
-
}
|
|
2040
|
-
|
|
2041
|
-
try {
|
|
2042
|
-
return { data: unpack(buffer), usedBinary: true };
|
|
2043
|
-
}
|
|
2044
|
-
catch (binaryErr) {
|
|
2045
|
-
try {
|
|
2046
|
-
return { data: this.parseTextFallback(textPayload), usedBinary: false };
|
|
2047
|
-
}
|
|
2048
|
-
catch {
|
|
2049
|
-
throw binaryErr;
|
|
2050
|
-
}
|
|
2051
|
-
}
|
|
2052
|
-
}
|
|
2053
|
-
|
|
2054
|
-
private parseTextFallback(rawMessage: string): any {
|
|
2055
|
-
if (rawMessage === 'ping' || rawMessage === 'pong') {
|
|
2056
|
-
return rawMessage;
|
|
2057
|
-
}
|
|
2058
|
-
|
|
2059
|
-
try {
|
|
2060
|
-
return JSON.parse(rawMessage, dateReviver);
|
|
2061
|
-
}
|
|
2062
|
-
catch (err) {
|
|
2063
|
-
throw err;
|
|
2064
|
-
}
|
|
2065
|
-
}
|
|
2066
|
-
|
|
2067
|
-
private looksLikeTextPayload(text: string): boolean {
|
|
2068
|
-
if (!text) {
|
|
2069
|
-
return false;
|
|
2070
|
-
}
|
|
2071
|
-
|
|
2072
|
-
const trimmed = text.trim();
|
|
2073
|
-
if (!trimmed) {
|
|
2074
|
-
return false;
|
|
2075
|
-
}
|
|
2076
|
-
|
|
2077
|
-
const first = trimmed[0];
|
|
2078
|
-
return first === '[' || first === '{' || first === '"' || first === 'p' || first === 'P';
|
|
2079
|
-
}
|
|
2080
|
-
|
|
2081
|
-
private resolveConnectDebug(): boolean {
|
|
2082
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2083
|
-
const raw = process.env.WS_CONNECT_DEBUG
|
|
2084
|
-
?? process.env.CONNECT_DEBUG
|
|
2085
|
-
?? config?.['WS_CONNECT_DEBUG']
|
|
2086
|
-
?? config?.['CONNECT_DEBUG'];
|
|
2087
|
-
return this.parseDebugFlag(raw);
|
|
2088
|
-
}
|
|
2089
|
-
|
|
2090
|
-
private resolvePerfDebug(): boolean {
|
|
2091
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2092
|
-
const raw = process.env.PERF_DEBUG
|
|
2093
|
-
?? process.env.CPU_DEBUG
|
|
2094
|
-
?? config?.['PERF_DEBUG']
|
|
2095
|
-
?? config?.['CPU_DEBUG'];
|
|
2096
|
-
return this.parseDebugFlag(raw);
|
|
2097
|
-
}
|
|
2098
|
-
|
|
2099
|
-
private resolveCpuProfileOnStart(): boolean {
|
|
2100
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2101
|
-
const raw = process.env.CPU_PROFILE_ON_START
|
|
2102
|
-
?? process.env.CPU_PROFILE_START
|
|
2103
|
-
?? config?.['CPU_PROFILE_ON_START']
|
|
2104
|
-
?? config?.['CPU_PROFILE_START'];
|
|
2105
|
-
return this.parseDebugFlag(raw);
|
|
2106
|
-
}
|
|
2107
|
-
|
|
2108
|
-
private resolveCpuProfileAuto(): boolean {
|
|
2109
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2110
|
-
const raw = process.env.CPU_PROFILE_AUTO
|
|
2111
|
-
?? config?.['CPU_PROFILE_AUTO'];
|
|
2112
|
-
return this.parseDebugFlag(raw);
|
|
2113
|
-
}
|
|
2114
|
-
|
|
2115
|
-
private resolveCpuProfileDurationMs(): number {
|
|
2116
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2117
|
-
const raw = process.env.CPU_PROFILE_DURATION_MS
|
|
2118
|
-
?? process.env.PERF_PROFILE_DURATION_MS
|
|
2119
|
-
?? config?.['CPU_PROFILE_DURATION_MS']
|
|
2120
|
-
?? config?.['PERF_PROFILE_DURATION_MS'];
|
|
2121
|
-
return this.parsePositiveInt(raw, this._cpuProfileDurationMs);
|
|
2122
|
-
}
|
|
2123
|
-
|
|
2124
|
-
private resolveCpuProfileThresholdPct(): number {
|
|
2125
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2126
|
-
const raw = process.env.CPU_PROFILE_THRESHOLD_PCT
|
|
2127
|
-
?? process.env.PERF_PROFILE_THRESHOLD_PCT
|
|
2128
|
-
?? config?.['CPU_PROFILE_THRESHOLD_PCT']
|
|
2129
|
-
?? config?.['PERF_PROFILE_THRESHOLD_PCT'];
|
|
2130
|
-
return this.parsePositiveFloat(raw, this._cpuProfileThresholdPct);
|
|
2131
|
-
}
|
|
2132
|
-
|
|
2133
|
-
private resolveCpuProfileTriggerCount(): number {
|
|
2134
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2135
|
-
const raw = process.env.CPU_PROFILE_TRIGGER_COUNT
|
|
2136
|
-
?? config?.['CPU_PROFILE_TRIGGER_COUNT'];
|
|
2137
|
-
return this.parsePositiveInt(raw, this._cpuProfileTriggerCount);
|
|
2138
|
-
}
|
|
2139
|
-
|
|
2140
|
-
private resolveCpuProfileDir(): string | null {
|
|
2141
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2142
|
-
const raw = process.env.CPU_PROFILE_DIR
|
|
2143
|
-
?? config?.['CPU_PROFILE_DIR'];
|
|
2144
|
-
return typeof raw === 'string' && raw.trim() ? raw.trim() : null;
|
|
2145
|
-
}
|
|
2146
|
-
|
|
2147
|
-
private resolveTimerDebug(): boolean {
|
|
2148
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2149
|
-
const raw = process.env.TIMER_DEBUG
|
|
2150
|
-
?? config?.['TIMER_DEBUG'];
|
|
2151
|
-
return this.parseDebugFlag(raw);
|
|
2152
|
-
}
|
|
2153
|
-
|
|
2154
|
-
private resolveTimerDebugThresholdMs(): number {
|
|
2155
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2156
|
-
const raw = process.env.TIMER_DEBUG_THRESHOLD_MS
|
|
2157
|
-
?? config?.['TIMER_DEBUG_THRESHOLD_MS'];
|
|
2158
|
-
return this.parsePositiveFloat(raw, this._timerDebugThresholdMs);
|
|
2159
|
-
}
|
|
2160
|
-
|
|
2161
|
-
private resolveTimerDebugMinDelayMs(): number {
|
|
2162
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2163
|
-
const raw = process.env.TIMER_DEBUG_MIN_DELAY_MS
|
|
2164
|
-
?? config?.['TIMER_DEBUG_MIN_DELAY_MS'];
|
|
2165
|
-
return this.parsePositiveFloat(raw, this._timerDebugMinDelayMs);
|
|
2166
|
-
}
|
|
2167
|
-
|
|
2168
|
-
private resolveTimerDebugSampleRate(): number {
|
|
2169
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2170
|
-
const raw = process.env.TIMER_DEBUG_SAMPLE_RATE
|
|
2171
|
-
?? config?.['TIMER_DEBUG_SAMPLE_RATE'];
|
|
2172
|
-
const parsed = parseFloat(raw ?? '');
|
|
2173
|
-
if (Number.isFinite(parsed) && parsed > 0 && parsed <= 1) {
|
|
2174
|
-
return parsed;
|
|
2175
|
-
}
|
|
2176
|
-
return this._timerDebugSampleRate;
|
|
2177
|
-
}
|
|
2178
|
-
|
|
2179
|
-
private resolveTimerDebugLogLimit(): number {
|
|
2180
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2181
|
-
const raw = process.env.TIMER_DEBUG_LOG_LIMIT
|
|
2182
|
-
?? config?.['TIMER_DEBUG_LOG_LIMIT'];
|
|
2183
|
-
return this.parsePositiveInt(raw, this._timerDebugLogLimit);
|
|
2184
|
-
}
|
|
2185
|
-
|
|
2186
|
-
private resolveStandaloneNodeReaperEnabled(): boolean {
|
|
2187
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2188
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_ENABLED
|
|
2189
|
-
?? process.env.STANDALONE_NODE_REAPER_ENABLED
|
|
2190
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_ENABLED']
|
|
2191
|
-
?? config?.['STANDALONE_NODE_REAPER_ENABLED'];
|
|
2192
|
-
const configured = this.parseOptionalBoolean(raw);
|
|
2193
|
-
if (configured !== null) {
|
|
2194
|
-
return configured;
|
|
2195
|
-
}
|
|
2196
|
-
|
|
2197
|
-
return this.isPm2ManagedRuntime() && !this.isLocalRuntime();
|
|
2198
|
-
}
|
|
2199
|
-
|
|
2200
|
-
private resolveStandaloneNodeReaperIntervalMs(): number {
|
|
2201
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2202
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_INTERVAL_MS
|
|
2203
|
-
?? process.env.STANDALONE_NODE_REAPER_INTERVAL_MS
|
|
2204
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_INTERVAL_MS']
|
|
2205
|
-
?? config?.['STANDALONE_NODE_REAPER_INTERVAL_MS'];
|
|
2206
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperIntervalMs);
|
|
2207
|
-
}
|
|
2208
|
-
|
|
2209
|
-
private resolveStandaloneNodeReaperMinAgeSeconds(): number {
|
|
2210
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2211
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_MIN_AGE_SECONDS
|
|
2212
|
-
?? process.env.STANDALONE_NODE_REAPER_MIN_AGE_SECONDS
|
|
2213
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_MIN_AGE_SECONDS']
|
|
2214
|
-
?? config?.['STANDALONE_NODE_REAPER_MIN_AGE_SECONDS'];
|
|
2215
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperMinAgeSeconds);
|
|
2216
|
-
}
|
|
2217
|
-
|
|
2218
|
-
private resolveStandaloneNodeReaperMaxKillsPerSignature(): number {
|
|
2219
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2220
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_MAX_KILLS_PER_SIGNATURE
|
|
2221
|
-
?? process.env.STANDALONE_NODE_REAPER_MAX_KILLS_PER_SIGNATURE
|
|
2222
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_MAX_KILLS_PER_SIGNATURE']
|
|
2223
|
-
?? config?.['STANDALONE_NODE_REAPER_MAX_KILLS_PER_SIGNATURE'];
|
|
2224
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperMaxKillsPerSignature);
|
|
2225
|
-
}
|
|
2226
|
-
|
|
2227
|
-
private resolveStandaloneNodeReaperKillWindowMs(): number {
|
|
2228
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2229
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_KILL_WINDOW_MS
|
|
2230
|
-
?? process.env.STANDALONE_NODE_REAPER_KILL_WINDOW_MS
|
|
2231
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_KILL_WINDOW_MS']
|
|
2232
|
-
?? config?.['STANDALONE_NODE_REAPER_KILL_WINDOW_MS'];
|
|
2233
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperKillWindowMs);
|
|
2234
|
-
}
|
|
2235
|
-
|
|
2236
|
-
private resolveStandaloneNodeReaperIncludeSystemdServices(): boolean {
|
|
2237
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2238
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_INCLUDE_SYSTEMD_SERVICES
|
|
2239
|
-
?? process.env.STANDALONE_NODE_REAPER_INCLUDE_SYSTEMD_SERVICES
|
|
2240
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_INCLUDE_SYSTEMD_SERVICES']
|
|
2241
|
-
?? config?.['STANDALONE_NODE_REAPER_INCLUDE_SYSTEMD_SERVICES'];
|
|
2242
|
-
return this.parseDebugFlag(raw);
|
|
2243
|
-
}
|
|
2244
|
-
|
|
2245
|
-
private resolveStandaloneNodeReaperHighCpuPct(): number {
|
|
2246
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2247
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_CPU_PCT
|
|
2248
|
-
?? process.env.STANDALONE_NODE_REAPER_HIGH_CPU_PCT
|
|
2249
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_CPU_PCT']
|
|
2250
|
-
?? config?.['STANDALONE_NODE_REAPER_HIGH_CPU_PCT'];
|
|
2251
|
-
return this.parsePositiveFloat(raw, this._standaloneNodeReaperHighCpuPct);
|
|
2252
|
-
}
|
|
2253
|
-
|
|
2254
|
-
private resolveStandaloneNodeReaperHighRssMb(): number {
|
|
2255
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2256
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_RSS_MB
|
|
2257
|
-
?? process.env.STANDALONE_NODE_REAPER_HIGH_RSS_MB
|
|
2258
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_RSS_MB']
|
|
2259
|
-
?? config?.['STANDALONE_NODE_REAPER_HIGH_RSS_MB'];
|
|
2260
|
-
return this.parsePositiveFloat(raw, this._standaloneNodeReaperHighRssMb);
|
|
2261
|
-
}
|
|
2262
|
-
|
|
2263
|
-
private resolveStandaloneNodeReaperHighMinAgeSeconds(): number {
|
|
2264
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2265
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_MIN_AGE_SECONDS
|
|
2266
|
-
?? process.env.STANDALONE_NODE_REAPER_HIGH_MIN_AGE_SECONDS
|
|
2267
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_MIN_AGE_SECONDS']
|
|
2268
|
-
?? config?.['STANDALONE_NODE_REAPER_HIGH_MIN_AGE_SECONDS'];
|
|
2269
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperHighMinAgeSeconds);
|
|
2270
|
-
}
|
|
2271
|
-
|
|
2272
|
-
private resolveStandaloneNodeReaperHighConsecutiveScans(): number {
|
|
2273
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2274
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_CONSECUTIVE_SCANS
|
|
2275
|
-
?? process.env.STANDALONE_NODE_REAPER_HIGH_CONSECUTIVE_SCANS
|
|
2276
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_HIGH_CONSECUTIVE_SCANS']
|
|
2277
|
-
?? config?.['STANDALONE_NODE_REAPER_HIGH_CONSECUTIVE_SCANS'];
|
|
2278
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperHighConsecutiveScans);
|
|
2279
|
-
}
|
|
2280
|
-
|
|
2281
|
-
private resolveStandaloneNodeReaperDryRun(): boolean {
|
|
2282
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2283
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_DRY_RUN
|
|
2284
|
-
?? process.env.STANDALONE_NODE_REAPER_DRY_RUN
|
|
2285
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_DRY_RUN']
|
|
2286
|
-
?? config?.['STANDALONE_NODE_REAPER_DRY_RUN'];
|
|
2287
|
-
return this.parseDebugFlag(raw);
|
|
2288
|
-
}
|
|
2289
|
-
|
|
2290
|
-
private resolveStandaloneNodeReaperAlertWindowMs(): number {
|
|
2291
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2292
|
-
const raw = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_ALERT_WINDOW_MS
|
|
2293
|
-
?? process.env.STANDALONE_NODE_REAPER_ALERT_WINDOW_MS
|
|
2294
|
-
?? config?.['RESOLVEIO_STANDALONE_NODE_REAPER_ALERT_WINDOW_MS']
|
|
2295
|
-
?? config?.['STANDALONE_NODE_REAPER_ALERT_WINDOW_MS'];
|
|
2296
|
-
return this.parsePositiveInt(raw, this._standaloneNodeReaperAlertWindowMs);
|
|
2297
|
-
}
|
|
2298
|
-
|
|
2299
|
-
private isPm2ManagedRuntime(): boolean {
|
|
2300
|
-
return Boolean(
|
|
2301
|
-
process.env.pm_id
|
|
2302
|
-
|| process.env.PM2_HOME
|
|
2303
|
-
|| process.env.NODE_APP_INSTANCE
|
|
2304
|
-
|| `${process.env._ || ''}`.includes('pm2')
|
|
2305
|
-
);
|
|
2306
|
-
}
|
|
2307
|
-
|
|
2308
|
-
private isPrimaryPm2Instance(): boolean {
|
|
2309
|
-
const appInstance = this.normalizeWorkerSelectorValue(process.env.NODE_APP_INSTANCE);
|
|
2310
|
-
return !appInstance || appInstance === '0';
|
|
2311
|
-
}
|
|
2312
|
-
|
|
2313
|
-
private isLocalRuntime(): boolean {
|
|
2314
|
-
const config = ResolveIOServer.getServerConfig ? ResolveIOServer.getServerConfig() : null;
|
|
2315
|
-
const rootUrl = `${process.env.ROOT_URL || config?.['ROOT_URL'] || ''}`.trim().toLowerCase();
|
|
2316
|
-
const serverUrl = `${process.env.SERVER_URL || config?.['SERVER_URL'] || ''}`.trim().toLowerCase();
|
|
2317
|
-
return rootUrl.includes('localhost') || rootUrl.includes('127.0.0.1') || serverUrl.includes('localhost') || serverUrl.includes('127.0.0.1');
|
|
2318
|
-
}
|
|
2319
|
-
|
|
2320
|
-
private normalizeWorkerSelectorValue(value?: string | number | null): string | null {
|
|
2321
|
-
if (value === null || value === undefined) {
|
|
2322
|
-
return null;
|
|
2323
|
-
}
|
|
2324
|
-
|
|
2325
|
-
const normalized = String(value).trim();
|
|
2326
|
-
return normalized.length ? normalized : null;
|
|
2327
|
-
}
|
|
2328
|
-
|
|
2329
|
-
private shouldStartCronManagerForWorker(): boolean {
|
|
2330
|
-
const workerIndex = this.normalizeWorkerSelectorValue(process.env.WORKER_INDEX);
|
|
2331
|
-
const workerInstance = this.normalizeWorkerSelectorValue(process.env.NODE_APP_INSTANCE) || '0';
|
|
2332
|
-
const hasExplicitServerManagerOwner = process.env.SERVER_MANAGER_WORKER_INDEX !== undefined
|
|
2333
|
-
|| process.env.SERVER_MANAGER_WORKER_INSTANCE !== undefined
|
|
2334
|
-
|| process.env.SERVER_MANAGER_NODE_APP_INSTANCE !== undefined;
|
|
2335
|
-
|
|
2336
|
-
if (!hasExplicitServerManagerOwner) {
|
|
2337
|
-
return workerIndex === '0';
|
|
2338
|
-
}
|
|
2339
|
-
|
|
2340
|
-
const serverManagerWorkerIndex = this.normalizeWorkerSelectorValue(process.env.SERVER_MANAGER_WORKER_INDEX) || '0';
|
|
2341
|
-
const serverManagerWorkerInstance = this.normalizeWorkerSelectorValue(
|
|
2342
|
-
process.env.SERVER_MANAGER_WORKER_INSTANCE || process.env.SERVER_MANAGER_NODE_APP_INSTANCE
|
|
2343
|
-
) || '0';
|
|
2344
|
-
|
|
2345
|
-
return workerIndex === serverManagerWorkerIndex && workerInstance === serverManagerWorkerInstance;
|
|
2346
|
-
}
|
|
2347
|
-
|
|
2348
|
-
private parseWorkerSelector(value: any): Set<string> | null {
|
|
2349
|
-
if (value === null || value === undefined) {
|
|
2350
|
-
return null;
|
|
2351
|
-
}
|
|
2352
|
-
|
|
2353
|
-
const raw = Array.isArray(value) ? value.join(',') : String(value);
|
|
2354
|
-
const parts = raw.split(',').map(part => part.trim()).filter(Boolean);
|
|
2355
|
-
if (!parts.length) {
|
|
2356
|
-
return null;
|
|
2357
|
-
}
|
|
2358
|
-
|
|
2359
|
-
return new Set(parts);
|
|
2360
|
-
}
|
|
2361
|
-
|
|
2362
|
-
private workerMatchesSelector(
|
|
2363
|
-
workerIndex: string | null,
|
|
2364
|
-
workerInstance: string | null,
|
|
2365
|
-
indexes: Set<string> | null,
|
|
2366
|
-
instances: Set<string> | null
|
|
2367
|
-
): boolean {
|
|
2368
|
-
if (!indexes && !instances) {
|
|
2369
|
-
return false;
|
|
2370
|
-
}
|
|
2371
|
-
|
|
2372
|
-
const indexMatch = indexes ? (workerIndex ? indexes.has(workerIndex) : false) : true;
|
|
2373
|
-
const instanceMatch = instances ? (workerInstance ? instances.has(workerInstance) : false) : true;
|
|
2374
|
-
return indexMatch && instanceMatch;
|
|
2375
|
-
}
|
|
2376
|
-
|
|
2377
|
-
private resolveWorkerRole(): 'Codex' | 'Subscription' | 'Other' {
|
|
2378
|
-
const workerIndex = this.normalizeWorkerSelectorValue(process.env.WORKER_INDEX);
|
|
2379
|
-
const workerInstance = this.normalizeWorkerSelectorValue(process.env.NODE_APP_INSTANCE);
|
|
2380
|
-
const publicationIndexes = this.parseWorkerSelector(
|
|
2381
|
-
process.env.PUBLICATION_WORKER_INDEX
|
|
2382
|
-
|| process.env.SUBSCRIPTION_WORKER_INDEX
|
|
2383
|
-
|| process.env.WORKER_PUBLICATION_INDEX
|
|
2384
|
-
);
|
|
2385
|
-
const publicationInstances = this.parseWorkerSelector(
|
|
2386
|
-
process.env.PUBLICATION_WORKER_INSTANCE
|
|
2387
|
-
|| process.env.SUBSCRIPTION_WORKER_INSTANCE
|
|
2388
|
-
|| process.env.WORKER_PUBLICATION_INSTANCE
|
|
2389
|
-
);
|
|
2390
|
-
const codexIndexes = this.parseWorkerSelector(
|
|
2391
|
-
process.env.AI_ASSISTANT_CODEX_WORKER_INDEX
|
|
2392
|
-
|| process.env.CODEX_WORKER_INDEX
|
|
2393
|
-
|| process.env.WORKER_CODEX_INDEX
|
|
2394
|
-
);
|
|
2395
|
-
const codexInstances = this.parseWorkerSelector(
|
|
2396
|
-
process.env.AI_ASSISTANT_CODEX_WORKER_INSTANCE
|
|
2397
|
-
|| process.env.CODEX_WORKER_INSTANCE
|
|
2398
|
-
|| process.env.WORKER_CODEX_INSTANCE
|
|
2399
|
-
);
|
|
2400
|
-
|
|
2401
|
-
if (this.workerMatchesSelector(workerIndex, workerInstance, publicationIndexes, publicationInstances)) {
|
|
2402
|
-
return 'Subscription';
|
|
2403
|
-
}
|
|
2404
|
-
|
|
2405
|
-
if (this.workerMatchesSelector(workerIndex, workerInstance, codexIndexes, codexInstances)) {
|
|
2406
|
-
return 'Codex';
|
|
2407
|
-
}
|
|
2408
|
-
|
|
2409
|
-
return 'Other';
|
|
2410
|
-
}
|
|
2411
|
-
|
|
2412
|
-
private parsePositiveInt(value: any, fallback: number): number {
|
|
2413
|
-
const parsed = parseInt(value ?? '', 10);
|
|
2414
|
-
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
2415
|
-
return fallback;
|
|
2416
|
-
}
|
|
2417
|
-
return parsed;
|
|
2418
|
-
}
|
|
2419
|
-
|
|
2420
|
-
private parsePositiveFloat(value: any, fallback: number): number {
|
|
2421
|
-
const parsed = parseFloat(value ?? '');
|
|
2422
|
-
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
2423
|
-
return fallback;
|
|
2424
|
-
}
|
|
2425
|
-
return parsed;
|
|
2426
|
-
}
|
|
2427
|
-
|
|
2428
|
-
private startPerfDebug(): void {
|
|
2429
|
-
if (!this._perfDebug || this._perfDebugTimer) {
|
|
2430
|
-
return;
|
|
2431
|
-
}
|
|
2432
|
-
|
|
2433
|
-
this._perfDebugIntervalMs = this.parsePositiveInt(
|
|
2434
|
-
process.env.PERF_DEBUG_INTERVAL_MS ?? process.env.CPU_DEBUG_INTERVAL_MS,
|
|
2435
|
-
this._perfDebugIntervalMs
|
|
2436
|
-
);
|
|
2437
|
-
|
|
2438
|
-
this._perfDebugLastCpu = process.cpuUsage();
|
|
2439
|
-
this._perfDebugLastTs = Date.now();
|
|
2440
|
-
this._eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
|
|
2441
|
-
this._eventLoopHistogram.enable();
|
|
2442
|
-
|
|
2443
|
-
this._perfDebugTimer = setInterval(() => {
|
|
2444
|
-
const now = Date.now();
|
|
2445
|
-
const elapsedMs = now - this._perfDebugLastTs;
|
|
2446
|
-
const cpuDiff = this._perfDebugLastCpu ? process.cpuUsage(this._perfDebugLastCpu) : process.cpuUsage();
|
|
2447
|
-
const cpuMs = (cpuDiff.user + cpuDiff.system) / 1000;
|
|
2448
|
-
const cpuPct = elapsedMs > 0 ? (cpuMs / elapsedMs) * 100 : 0;
|
|
2449
|
-
|
|
2450
|
-
const mem = process.memoryUsage();
|
|
2451
|
-
const heapUsedMb = round((mem.heapUsed / 1024 / 1024) * 10) / 10;
|
|
2452
|
-
const rssMb = round((mem.rss / 1024 / 1024) * 10) / 10;
|
|
2453
|
-
|
|
2454
|
-
const handles = typeof (process as any)._getActiveHandles === 'function'
|
|
2455
|
-
? (process as any)._getActiveHandles().length
|
|
2456
|
-
: null;
|
|
2457
|
-
const requests = typeof (process as any)._getActiveRequests === 'function'
|
|
2458
|
-
? (process as any)._getActiveRequests().length
|
|
2459
|
-
: null;
|
|
2460
|
-
|
|
2461
|
-
const histogram = this._eventLoopHistogram;
|
|
2462
|
-
const eventLoop = histogram ? {
|
|
2463
|
-
meanMs: round((histogram.mean / 1e6) * 100) / 100,
|
|
2464
|
-
p50Ms: round((histogram.percentile(50) / 1e6) * 100) / 100,
|
|
2465
|
-
p95Ms: round((histogram.percentile(95) / 1e6) * 100) / 100,
|
|
2466
|
-
p99Ms: round((histogram.percentile(99) / 1e6) * 100) / 100,
|
|
2467
|
-
maxMs: round((histogram.max / 1e6) * 100) / 100
|
|
2468
|
-
} : null;
|
|
2469
|
-
histogram?.reset();
|
|
2470
|
-
|
|
2471
|
-
console.log(new Date(), '[Perf Debug]', JSON.stringify({
|
|
2472
|
-
cpuPct: round(cpuPct * 10) / 10,
|
|
2473
|
-
cpuMs: round(cpuMs),
|
|
2474
|
-
elapsedMs,
|
|
2475
|
-
eventLoop,
|
|
2476
|
-
heapUsedMb,
|
|
2477
|
-
rssMb,
|
|
2478
|
-
activeHandles: handles,
|
|
2479
|
-
activeRequests: requests,
|
|
2480
|
-
msgRecv: this._debugMsgRecv,
|
|
2481
|
-
msgQueue: this._debugMsgQueue
|
|
2482
|
-
}));
|
|
2483
|
-
|
|
2484
|
-
if (this._cpuProfileAuto) {
|
|
2485
|
-
if (cpuPct >= this._cpuProfileThresholdPct) {
|
|
2486
|
-
this._cpuProfileHighCount += 1;
|
|
2487
|
-
}
|
|
2488
|
-
else {
|
|
2489
|
-
this._cpuProfileHighCount = 0;
|
|
2490
|
-
}
|
|
2491
|
-
|
|
2492
|
-
if (this._cpuProfileHighCount >= this._cpuProfileTriggerCount) {
|
|
2493
|
-
this._cpuProfileHighCount = 0;
|
|
2494
|
-
this.startCpuProfile('auto-high-cpu');
|
|
2495
|
-
}
|
|
2496
|
-
}
|
|
2497
|
-
|
|
2498
|
-
this._perfDebugLastCpu = process.cpuUsage();
|
|
2499
|
-
this._perfDebugLastTs = now;
|
|
2500
|
-
}, this._perfDebugIntervalMs);
|
|
2501
|
-
}
|
|
2502
|
-
|
|
2503
|
-
private installTimerDebug(): void {
|
|
2504
|
-
if (!this._timerDebug) {
|
|
2505
|
-
return;
|
|
2506
|
-
}
|
|
2507
|
-
|
|
2508
|
-
const g = global as unknown as { __resolveioTimerDebugInstalled?: boolean };
|
|
2509
|
-
if (g.__resolveioTimerDebugInstalled) {
|
|
2510
|
-
return;
|
|
2511
|
-
}
|
|
2512
|
-
g.__resolveioTimerDebugInstalled = true;
|
|
2513
|
-
|
|
2514
|
-
const originalSetTimeout = global.setTimeout;
|
|
2515
|
-
const originalSetInterval = global.setInterval;
|
|
2516
|
-
const thresholdMs = this._timerDebugThresholdMs;
|
|
2517
|
-
const minDelayMs = this._timerDebugMinDelayMs;
|
|
2518
|
-
const sampleRate = this._timerDebugSampleRate;
|
|
2519
|
-
|
|
2520
|
-
const shouldSample = () => sampleRate >= 1 || Math.random() <= sampleRate;
|
|
2521
|
-
|
|
2522
|
-
const logTimer = (type: string, durationMs: number, delayMs: number | undefined, createdStack: string | undefined) => {
|
|
2523
|
-
if (this._timerDebugLogLimit > 0 && this._timerDebugLogCount >= this._timerDebugLogLimit) {
|
|
2524
|
-
return;
|
|
2525
|
-
}
|
|
2526
|
-
this._timerDebugLogCount += 1;
|
|
2527
|
-
const stackLines = createdStack
|
|
2528
|
-
? createdStack.split('\n').slice(1, 6).map(line => line.trim()).join(' | ')
|
|
2529
|
-
: undefined;
|
|
2530
|
-
console.log(new Date(), '[Timer Debug]', JSON.stringify({
|
|
2531
|
-
type,
|
|
2532
|
-
durationMs: round(durationMs * 10) / 10,
|
|
2533
|
-
delayMs: delayMs ?? null,
|
|
2534
|
-
createdAt: stackLines
|
|
2535
|
-
}));
|
|
2536
|
-
};
|
|
2537
|
-
|
|
2538
|
-
const wrapTimer = (handler: any, type: string, delayMs?: number) => {
|
|
2539
|
-
if (typeof handler !== 'function') {
|
|
2540
|
-
return handler;
|
|
2541
|
-
}
|
|
2542
|
-
|
|
2543
|
-
const createdStack = new Error().stack;
|
|
2544
|
-
const shouldFlagShortDelay = typeof delayMs === 'number' && delayMs <= minDelayMs;
|
|
2545
|
-
|
|
2546
|
-
return function wrappedTimer(this: any, ...args: any[]) {
|
|
2547
|
-
const start = process.hrtime.bigint();
|
|
2548
|
-
try {
|
|
2549
|
-
return handler.apply(this, args);
|
|
2550
|
-
}
|
|
2551
|
-
finally {
|
|
2552
|
-
const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6;
|
|
2553
|
-
if (shouldSample() && (elapsedMs >= thresholdMs || shouldFlagShortDelay)) {
|
|
2554
|
-
logTimer(type, elapsedMs, delayMs, createdStack);
|
|
2555
|
-
}
|
|
2556
|
-
}
|
|
2557
|
-
};
|
|
2558
|
-
};
|
|
2559
|
-
|
|
2560
|
-
global.setTimeout = ((handler: any, timeout?: number, ...args: any[]) => {
|
|
2561
|
-
return originalSetTimeout(wrapTimer(handler, 'setTimeout', timeout), timeout as any, ...args);
|
|
2562
|
-
}) as typeof setTimeout;
|
|
2563
|
-
|
|
2564
|
-
global.setInterval = ((handler: any, timeout?: number, ...args: any[]) => {
|
|
2565
|
-
return originalSetInterval(wrapTimer(handler, 'setInterval', timeout), timeout as any, ...args);
|
|
2566
|
-
}) as typeof setInterval;
|
|
2567
|
-
}
|
|
2568
|
-
|
|
2569
|
-
private startCpuProfile(trigger: string): void {
|
|
2570
|
-
if (this._cpuProfileSession) {
|
|
2571
|
-
return;
|
|
2572
|
-
}
|
|
2573
|
-
|
|
2574
|
-
try {
|
|
2575
|
-
const session = new inspector.Session();
|
|
2576
|
-
session.connect();
|
|
2577
|
-
session.post('Profiler.enable', () => {
|
|
2578
|
-
session.post('Profiler.start', () => {
|
|
2579
|
-
this._cpuProfileSession = session;
|
|
2580
|
-
console.log(new Date(), '[Perf Debug]', 'CPU profile started', trigger);
|
|
2581
|
-
setTimeout(() => {
|
|
2582
|
-
this.stopCpuProfile(trigger);
|
|
2583
|
-
}, this._cpuProfileDurationMs);
|
|
2584
|
-
});
|
|
2585
|
-
});
|
|
2586
|
-
}
|
|
2587
|
-
catch (error) {
|
|
2588
|
-
console.error(new Date(), '[Perf Debug]', 'CPU profile start failed', error);
|
|
2589
|
-
}
|
|
2590
|
-
}
|
|
2591
|
-
|
|
2592
|
-
private stopCpuProfile(trigger: string): void {
|
|
2593
|
-
const session = this._cpuProfileSession;
|
|
2594
|
-
if (!session) {
|
|
2595
|
-
return;
|
|
2596
|
-
}
|
|
2597
|
-
|
|
2598
|
-
session.post('Profiler.stop', (err, res: { profile?: unknown }) => {
|
|
2599
|
-
if (err) {
|
|
2600
|
-
console.error(new Date(), '[Perf Debug]', 'CPU profile stop failed', err);
|
|
2601
|
-
}
|
|
2602
|
-
else {
|
|
2603
|
-
try {
|
|
2604
|
-
const dir = this.resolveWritableProfileDir();
|
|
2605
|
-
const filename = `resolveio-cpuprofile-${process.pid}-${Date.now()}.cpuprofile`;
|
|
2606
|
-
const filePath = path.join(dir, filename);
|
|
2607
|
-
fs.writeFileSync(filePath, JSON.stringify(res?.profile ?? {}));
|
|
2608
|
-
console.log(new Date(), '[Perf Debug]', 'CPU profile saved', filePath, trigger);
|
|
2609
|
-
}
|
|
2610
|
-
catch (writeErr) {
|
|
2611
|
-
console.error(new Date(), '[Perf Debug]', 'CPU profile write failed', writeErr);
|
|
2612
|
-
}
|
|
2613
|
-
}
|
|
2614
|
-
|
|
2615
|
-
try {
|
|
2616
|
-
session.disconnect();
|
|
2617
|
-
}
|
|
2618
|
-
catch {}
|
|
2619
|
-
|
|
2620
|
-
this._cpuProfileSession = null;
|
|
2621
|
-
});
|
|
2622
|
-
}
|
|
2623
|
-
|
|
2624
|
-
private resolveWritableProfileDir(): string {
|
|
2625
|
-
const preferred = this._cpuProfileDir;
|
|
2626
|
-
if (preferred) {
|
|
2627
|
-
try {
|
|
2628
|
-
fs.mkdirSync(preferred, { recursive: true });
|
|
2629
|
-
return preferred;
|
|
2630
|
-
}
|
|
2631
|
-
catch {}
|
|
2632
|
-
}
|
|
2633
|
-
|
|
2634
|
-
return os.tmpdir();
|
|
2635
|
-
}
|
|
2636
|
-
|
|
2637
|
-
private parseDebugFlag(value: any): boolean {
|
|
2638
|
-
if (value === true) {
|
|
2639
|
-
return true;
|
|
2640
|
-
}
|
|
2641
|
-
|
|
2642
|
-
if (value === false || value === null || value === undefined) {
|
|
2643
|
-
return false;
|
|
2644
|
-
}
|
|
2645
|
-
|
|
2646
|
-
if (typeof value === 'number') {
|
|
2647
|
-
return value === 1;
|
|
2648
|
-
}
|
|
2649
|
-
|
|
2650
|
-
if (typeof value === 'string') {
|
|
2651
|
-
const normalized = value.trim().toLowerCase();
|
|
2652
|
-
return ['1', 'true', 'yes', 'y', 'on'].includes(normalized);
|
|
2653
|
-
}
|
|
2654
|
-
|
|
2655
|
-
return false;
|
|
2656
|
-
}
|
|
2657
|
-
|
|
2658
|
-
private parseOptionalBoolean(value: any): boolean | null {
|
|
2659
|
-
if (value === null || value === undefined) {
|
|
2660
|
-
return null;
|
|
2661
|
-
}
|
|
2662
|
-
const normalized = `${value}`.trim();
|
|
2663
|
-
if (!normalized) {
|
|
2664
|
-
return null;
|
|
2665
|
-
}
|
|
2666
|
-
return this.parseDebugFlag(normalized);
|
|
2667
|
-
}
|
|
2668
|
-
|
|
2669
|
-
private parseNonNegativeInt(value: any, fallback: number): number {
|
|
2670
|
-
const parsed = parseInt(`${value ?? ''}`, 10);
|
|
2671
|
-
if (Number.isNaN(parsed) || parsed < 0) {
|
|
2672
|
-
return fallback;
|
|
2673
|
-
}
|
|
2674
|
-
return parsed;
|
|
2675
|
-
}
|
|
2676
|
-
|
|
2677
|
-
private resolveSocketTier(): string {
|
|
2678
|
-
const config = ResolveIOServer.getServerConfig() || {};
|
|
2679
|
-
return `${process.env.AI_CODER_PLAN_TIER || config['AI_CODER_PLAN_TIER'] || ''}`.trim().toLowerCase();
|
|
2680
|
-
}
|
|
2681
|
-
|
|
2682
|
-
private resolveSocketTierKeySuffix(planTier: string): string {
|
|
2683
|
-
return `${planTier || ''}`
|
|
2684
|
-
.trim()
|
|
2685
|
-
.toUpperCase()
|
|
2686
|
-
.replace(/[^A-Z0-9]+/g, '_');
|
|
2687
|
-
}
|
|
2688
|
-
|
|
2689
|
-
private resolveMaxClientSockets(planTier: string): number {
|
|
2690
|
-
const config = ResolveIOServer.getServerConfig() || {};
|
|
2691
|
-
const explicitLimit = this.parseNonNegativeInt(
|
|
2692
|
-
config['AI_CODER_MAX_SOCKETS'] ?? process.env.AI_CODER_MAX_SOCKETS,
|
|
2693
|
-
-1
|
|
2694
|
-
);
|
|
2695
|
-
if (explicitLimit >= 0) {
|
|
2696
|
-
return explicitLimit;
|
|
2697
|
-
}
|
|
2698
|
-
|
|
2699
|
-
const normalizedTier = `${planTier || ''}`.trim().toLowerCase();
|
|
2700
|
-
const tierKeySuffix = this.resolveSocketTierKeySuffix(normalizedTier);
|
|
2701
|
-
if (tierKeySuffix) {
|
|
2702
|
-
const tierLimitKey = `AI_CODER_MAX_SOCKETS_${tierKeySuffix}`;
|
|
2703
|
-
const tierLimit = this.parseNonNegativeInt(
|
|
2704
|
-
config[tierLimitKey] ?? process.env[tierLimitKey],
|
|
2705
|
-
-1
|
|
2706
|
-
);
|
|
2707
|
-
if (tierLimit >= 0) {
|
|
2708
|
-
return tierLimit;
|
|
2709
|
-
}
|
|
2710
|
-
}
|
|
2711
|
-
|
|
2712
|
-
const maxUsers = this.parseNonNegativeInt(
|
|
2713
|
-
config['AI_CODER_MAX_USERS'] ?? process.env.AI_CODER_MAX_USERS,
|
|
2714
|
-
-1
|
|
2715
|
-
);
|
|
2716
|
-
if (maxUsers > 0) {
|
|
2717
|
-
return maxUsers === 1 ? 1 : maxUsers * 2;
|
|
2718
|
-
}
|
|
2719
|
-
|
|
2720
|
-
if (normalizedTier === 'tool') {
|
|
2721
|
-
return 1;
|
|
2722
|
-
}
|
|
2723
|
-
if (normalizedTier === 'small') {
|
|
2724
|
-
return 10;
|
|
2725
|
-
}
|
|
2726
|
-
if (normalizedTier === 'medium') {
|
|
2727
|
-
return 50;
|
|
2728
|
-
}
|
|
2729
|
-
if (normalizedTier === 'large') {
|
|
2730
|
-
return 200;
|
|
2731
|
-
}
|
|
2732
|
-
if (normalizedTier === 'enterprise') {
|
|
2733
|
-
return 0;
|
|
2734
|
-
}
|
|
2735
|
-
|
|
2736
|
-
return 0;
|
|
2737
|
-
}
|
|
2738
|
-
|
|
2739
|
-
private resolveSingleIpPerUserPolicy(planTier: string): boolean {
|
|
2740
|
-
const config = ResolveIOServer.getServerConfig() || {};
|
|
2741
|
-
const normalizedTier = `${planTier || ''}`.trim().toLowerCase();
|
|
2742
|
-
const tierKeySuffix = this.resolveSocketTierKeySuffix(normalizedTier);
|
|
2743
|
-
if (tierKeySuffix) {
|
|
2744
|
-
const tierPolicyKey = `AI_CODER_SINGLE_IP_PER_USER_${tierKeySuffix}`;
|
|
2745
|
-
const tierPolicy = this.parseOptionalBoolean(
|
|
2746
|
-
config[tierPolicyKey] ?? process.env[tierPolicyKey]
|
|
2747
|
-
);
|
|
2748
|
-
if (tierPolicy !== null) {
|
|
2749
|
-
return tierPolicy;
|
|
2750
|
-
}
|
|
2751
|
-
}
|
|
2752
|
-
|
|
2753
|
-
const policy = this.parseOptionalBoolean(
|
|
2754
|
-
config['AI_CODER_SINGLE_IP_PER_USER'] ?? process.env.AI_CODER_SINGLE_IP_PER_USER
|
|
2755
|
-
);
|
|
2756
|
-
if (policy !== null) {
|
|
2757
|
-
return policy;
|
|
2758
|
-
}
|
|
2759
|
-
|
|
2760
|
-
return !!normalizedTier;
|
|
2761
|
-
}
|
|
2762
|
-
|
|
2763
|
-
private resolveSocketPolicyUpgradeUrl(): string {
|
|
2764
|
-
const config = ResolveIOServer.getServerConfig() || {};
|
|
2765
|
-
const direct = `${config['AI_CODER_SOCKET_UPGRADE_URL'] || process.env.AI_CODER_SOCKET_UPGRADE_URL || ''}`.trim();
|
|
2766
|
-
if (direct) {
|
|
2767
|
-
return direct.replace(/\/$/, '');
|
|
2768
|
-
}
|
|
2769
|
-
|
|
2770
|
-
const dashboard = `${config['AI_CODER_CLIENT_DASHBOARD_URL'] || process.env.AI_CODER_CLIENT_DASHBOARD_URL || ''}`.trim();
|
|
2771
|
-
if (dashboard) {
|
|
2772
|
-
return dashboard.replace(/\/$/, '');
|
|
2773
|
-
}
|
|
2774
|
-
|
|
2775
|
-
const root = `${config['AI_CODER_ROOT_URL'] || process.env.AI_CODER_ROOT_URL || config['ROOT_URL'] || process.env.ROOT_URL || ''}`.trim();
|
|
2776
|
-
if (!root) {
|
|
2777
|
-
return '';
|
|
2778
|
-
}
|
|
2779
|
-
|
|
2780
|
-
return `${root.replace(/\/$/, '')}/dashboard/client`;
|
|
2781
|
-
}
|
|
2782
|
-
|
|
2783
|
-
private resolveSocketUpgradeUrlWithAppId(): string {
|
|
2784
|
-
const base = `${this._socketPolicyUpgradeUrl || ''}`.trim();
|
|
2785
|
-
if (!base) {
|
|
2786
|
-
return '';
|
|
2787
|
-
}
|
|
2788
|
-
const appId = `${process.env.AI_CODER_APP_ID || ''}`.trim();
|
|
2789
|
-
if (!appId) {
|
|
2790
|
-
return base;
|
|
2791
|
-
}
|
|
2792
|
-
const separator = base.includes('?') ? '&' : '?';
|
|
2793
|
-
return `${base}${separator}appId=${encodeURIComponent(appId)}`;
|
|
2794
|
-
}
|
|
2795
|
-
|
|
2796
|
-
private normalizeIpAddress(value: any): string {
|
|
2797
|
-
let ip = `${value || ''}`.trim();
|
|
2798
|
-
if (!ip) {
|
|
2799
|
-
return '';
|
|
2800
|
-
}
|
|
2801
|
-
if (ip.includes(',')) {
|
|
2802
|
-
ip = ip.split(',')[0].trim();
|
|
2803
|
-
}
|
|
2804
|
-
if (ip.startsWith('::ffff:')) {
|
|
2805
|
-
ip = ip.slice(7);
|
|
2806
|
-
}
|
|
2807
|
-
if (/^\d+\.\d+\.\d+\.\d+:\d+$/.test(ip)) {
|
|
2808
|
-
ip = ip.split(':')[0].trim();
|
|
2809
|
-
}
|
|
2810
|
-
if (ip === '::1') {
|
|
2811
|
-
return '127.0.0.1';
|
|
2812
|
-
}
|
|
2813
|
-
return ip.toLowerCase();
|
|
2814
|
-
}
|
|
2815
|
-
|
|
2816
|
-
private resolveClientIp(req: any): string {
|
|
2817
|
-
const forwardedFor = this.normalizeHeaderValue(req?.headers?.['x-forwarded-for']);
|
|
2818
|
-
if (forwardedFor) {
|
|
2819
|
-
return this.normalizeIpAddress(forwardedFor);
|
|
2820
|
-
}
|
|
2821
|
-
const realIp = this.normalizeHeaderValue(req?.headers?.['x-real-ip']);
|
|
2822
|
-
if (realIp) {
|
|
2823
|
-
return this.normalizeIpAddress(realIp);
|
|
2824
|
-
}
|
|
2825
|
-
const socketIp = req?.socket?.remoteAddress || req?.connection?.remoteAddress || req?.ip || '';
|
|
2826
|
-
return this.normalizeIpAddress(socketIp);
|
|
2827
|
-
}
|
|
2828
|
-
|
|
2829
|
-
private buildSocketLimitMessage(activeSockets: number): string {
|
|
2830
|
-
const tierLabel = this._socketTier ? this._socketTier[0].toUpperCase() + this._socketTier.slice(1) : 'Current';
|
|
2831
|
-
const upgradeUrl = this.resolveSocketUpgradeUrlWithAppId();
|
|
2832
|
-
const base = `Socket connection limit reached (${activeSockets}/${this._maxClientSockets}) for the ${tierLabel} tier.`;
|
|
2833
|
-
if (!upgradeUrl) {
|
|
2834
|
-
return `${base} Upgrade to increase capacity.`;
|
|
2835
|
-
}
|
|
2836
|
-
return `${base} Upgrade at ${upgradeUrl}.`;
|
|
2837
|
-
}
|
|
2838
|
-
|
|
2839
|
-
private buildSocketLimitCloseReason(): string {
|
|
2840
|
-
return `Socket limit reached (${this._maxClientSockets})`;
|
|
2841
|
-
}
|
|
2842
|
-
|
|
2843
|
-
private async disconnectUserSocketsFromDifferentIps(idUser: string, incomingIp: string): Promise<number> {
|
|
2844
|
-
if (!this._websocketManager) {
|
|
2845
|
-
return 0;
|
|
2846
|
-
}
|
|
2847
|
-
|
|
2848
|
-
const normalizedUser = `${idUser || ''}`.trim();
|
|
2849
|
-
const normalizedIncomingIp = this.normalizeIpAddress(incomingIp);
|
|
2850
|
-
if (!normalizedUser || !normalizedIncomingIp) {
|
|
2851
|
-
return 0;
|
|
2852
|
-
}
|
|
2853
|
-
|
|
2854
|
-
const userSockets = this._websocketManager.getUserWebSockets(normalizedUser);
|
|
2855
|
-
let disconnected = 0;
|
|
2856
|
-
for (const existingSocket of userSockets) {
|
|
2857
|
-
const existingIp = this.normalizeIpAddress(existingSocket?.['client_ip']);
|
|
2858
|
-
if (!existingIp || existingIp === normalizedIncomingIp) {
|
|
2859
|
-
continue;
|
|
2860
|
-
}
|
|
2861
|
-
|
|
2862
|
-
this.logConnectDebug('WS single-ip enforcement disconnect', {
|
|
2863
|
-
id_socket: existingSocket?.['id_socket'],
|
|
2864
|
-
id_user: normalizedUser,
|
|
2865
|
-
existingIp,
|
|
2866
|
-
incomingIp: normalizedIncomingIp
|
|
2867
|
-
});
|
|
2868
|
-
|
|
2869
|
-
await this.unsubscribeWS(existingSocket);
|
|
2870
|
-
if (existingSocket.readyState === WebSocket.OPEN || existingSocket.readyState === WebSocket.CONNECTING) {
|
|
2871
|
-
try {
|
|
2872
|
-
existingSocket.close(1008, 'Signed in from another IP');
|
|
2873
|
-
}
|
|
2874
|
-
catch {}
|
|
2875
|
-
}
|
|
2876
|
-
disconnected += 1;
|
|
2877
|
-
}
|
|
2878
|
-
|
|
2879
|
-
return disconnected;
|
|
2880
|
-
}
|
|
2881
|
-
|
|
2882
|
-
private async evaluateClientSocketAdmission(idUser: string, req: any): Promise<{ allowed: boolean, statusCode: number, message: string, clientIp: string }> {
|
|
2883
|
-
const normalizedUser = `${idUser || ''}`.trim();
|
|
2884
|
-
const clientIp = this.resolveClientIp(req);
|
|
2885
|
-
|
|
2886
|
-
if (this._singleIpPerUser) {
|
|
2887
|
-
await this.disconnectUserSocketsFromDifferentIps(normalizedUser, clientIp);
|
|
2888
|
-
}
|
|
2889
|
-
|
|
2890
|
-
if (this._maxClientSockets > 0 && this._websocketManager) {
|
|
2891
|
-
const activeSockets = this._websocketManager.getActiveWebSocketCount();
|
|
2892
|
-
if (activeSockets >= this._maxClientSockets) {
|
|
2893
|
-
const message = this.buildSocketLimitMessage(activeSockets);
|
|
2894
|
-
this.logConnectDebug('WS socket limit blocked', {
|
|
2895
|
-
id_user: normalizedUser,
|
|
2896
|
-
clientIp,
|
|
2897
|
-
activeSockets,
|
|
2898
|
-
maxClientSockets: this._maxClientSockets
|
|
2899
|
-
});
|
|
2900
|
-
return {
|
|
2901
|
-
allowed: false,
|
|
2902
|
-
statusCode: 429,
|
|
2903
|
-
message,
|
|
2904
|
-
clientIp
|
|
2905
|
-
};
|
|
2906
|
-
}
|
|
2907
|
-
}
|
|
2908
|
-
|
|
2909
|
-
return {
|
|
2910
|
-
allowed: true,
|
|
2911
|
-
statusCode: 200,
|
|
2912
|
-
message: '',
|
|
2913
|
-
clientIp
|
|
2914
|
-
};
|
|
2915
|
-
}
|
|
2916
|
-
|
|
2917
|
-
private logConnectDebug(message: string, details?: Record<string, unknown>): void {
|
|
2918
|
-
if (!this._wsConnectDebug) {
|
|
2919
|
-
return;
|
|
2920
|
-
}
|
|
2921
|
-
|
|
2922
|
-
if (details) {
|
|
2923
|
-
console.log(new Date(), '[Connect Debug]', message, JSON.stringify(details));
|
|
2924
|
-
}
|
|
2925
|
-
else {
|
|
2926
|
-
console.log(new Date(), '[Connect Debug]', message);
|
|
2927
|
-
}
|
|
2928
|
-
}
|
|
2929
|
-
|
|
2930
|
-
private async triggerClientHeartbeat(ws: WebSocket): Promise<void> {
|
|
2931
|
-
if (!ws || ws.readyState !== WebSocket.OPEN) {
|
|
2932
|
-
return;
|
|
2933
|
-
}
|
|
2934
|
-
|
|
2935
|
-
ws['pingTime'] = new Date();
|
|
2936
|
-
|
|
2937
|
-
try {
|
|
2938
|
-
if (typeof ws.ping === 'function') {
|
|
2939
|
-
ws.ping();
|
|
2940
|
-
}
|
|
2941
|
-
}
|
|
2942
|
-
catch (err) {
|
|
2943
|
-
if (this._methodManager?.getEnableDebug()) {
|
|
2944
|
-
console.log(new Date(), 'Server App', 'Error WS Ping Frame', err);
|
|
2945
|
-
}
|
|
2946
|
-
await this.unsubscribeWS(ws);
|
|
2947
|
-
return;
|
|
2948
|
-
}
|
|
2949
|
-
|
|
2950
|
-
ws.send('ping', async (error) => {
|
|
2951
|
-
if (error) {
|
|
2952
|
-
if (this._methodManager?.getEnableDebug()) {
|
|
2953
|
-
console.log(new Date(), 'Server App', 'Error WS Ping');
|
|
2954
|
-
}
|
|
2955
|
-
await this.unsubscribeWS(ws);
|
|
2956
|
-
}
|
|
2957
|
-
});
|
|
2958
|
-
}
|
|
2959
|
-
|
|
2960
|
-
private shouldDeferHeartbeat(ws: WebSocket): boolean {
|
|
2961
|
-
if (!ws || ws.readyState !== WebSocket.OPEN) {
|
|
2962
|
-
return false;
|
|
2963
|
-
}
|
|
2964
|
-
|
|
2965
|
-
const bufferedAmount = typeof ws.bufferedAmount === 'number' ? ws.bufferedAmount : 0;
|
|
2966
|
-
return bufferedAmount >= this._clientHeartbeatBackpressureBytes;
|
|
2967
|
-
}
|
|
2968
|
-
|
|
2969
|
-
private async handleClientMessage(ws: WebSocket, msg: any[]): Promise<void> {
|
|
2970
|
-
// This is basically your old logic from processSocketMessage,
|
|
2971
|
-
// but we'll insert our worker-queue logic for "method" calls.
|
|
2972
|
-
try {
|
|
2973
|
-
let messageRoute = msg[0];
|
|
2974
|
-
let messageDate = msg[1];
|
|
2975
|
-
let messageId = msg[2];
|
|
2976
|
-
let type = msg[3];
|
|
2977
|
-
|
|
2978
|
-
if (!this.publicProgram && this._clientRoutes.some(a => messageRoute.includes(a)) && !ws['doc_user'].roles.groups.some(a => a.views.some(b => messageRoute.includes(b) || b.includes(messageRoute))) && !ws['doc_user'].roles.super_admin) {
|
|
2979
|
-
return;
|
|
2980
|
-
}
|
|
2981
|
-
|
|
2982
|
-
if (type === 'subscription') {
|
|
2983
|
-
let subType = msg[4];
|
|
2984
|
-
let pub = msg[5];
|
|
2985
|
-
this.logConnectDebug('Subscription message', {
|
|
2986
|
-
subType,
|
|
2987
|
-
publication: pub,
|
|
2988
|
-
messageId,
|
|
2989
|
-
messageRoute,
|
|
2990
|
-
args: Math.max(0, msg.length - 6),
|
|
2991
|
-
id_socket: ws ? ws['id_socket'] : null,
|
|
2992
|
-
user: ws ? ws['user'] : null
|
|
2993
|
-
});
|
|
2994
|
-
|
|
2995
|
-
if (subType === 'sub') {
|
|
2996
|
-
await this._subscriptionManager.subscribe(messageRoute, messageDate, ws, messageId, pub, msg.slice(6));
|
|
2997
|
-
}
|
|
2998
|
-
else {
|
|
2999
|
-
this._subscriptionManager.unsubscribe(messageRoute, messageDate, ws, messageId, pub, msg.slice(6));
|
|
3000
|
-
}
|
|
3001
|
-
}
|
|
3002
|
-
else if (!this.publicProgram && type === 'offline') {
|
|
3003
|
-
let serverRes: ServerResponseModel = {
|
|
3004
|
-
messageId: messageId,
|
|
3005
|
-
hasError: false,
|
|
3006
|
-
data: 'ACK'
|
|
3007
|
-
};
|
|
3008
|
-
|
|
3009
|
-
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
3010
|
-
this._websocketManager.send(ws, serverRes);
|
|
3011
|
-
}
|
|
3012
|
-
|
|
3013
|
-
this._offlineUpdates.push(ws);
|
|
3014
|
-
let offlineUpdates = msg[4];
|
|
3015
|
-
|
|
3016
|
-
for (let i = 0; i < offlineUpdates.length; i++) {
|
|
3017
|
-
let update = offlineUpdates[i];
|
|
3018
|
-
|
|
3019
|
-
let data = update.data;
|
|
3020
|
-
|
|
3021
|
-
// eslint-disable-next-line no-unused-vars
|
|
3022
|
-
let updateRoute = data.shift();
|
|
3023
|
-
// eslint-disable-next-line no-unused-vars
|
|
3024
|
-
let updateDate = data.shift();
|
|
3025
|
-
let updateMessageId = data.shift();
|
|
3026
|
-
// eslint-disable-next-line no-unused-vars
|
|
3027
|
-
let updateType = data.shift();
|
|
3028
|
-
let method = data.shift();
|
|
3029
|
-
|
|
3030
|
-
let serverResMethod: ServerResponseModel = {
|
|
3031
|
-
messageId: updateMessageId,
|
|
3032
|
-
hasError: false,
|
|
3033
|
-
data: 'ACK'
|
|
3034
|
-
};
|
|
3035
|
-
|
|
3036
|
-
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
3037
|
-
this._websocketManager.send(ws, serverResMethod);
|
|
3038
|
-
}
|
|
3039
|
-
|
|
3040
|
-
if (method === 'insertDocument' && data[0] === 'driver-gps') {
|
|
3041
|
-
continue;
|
|
3042
|
-
}
|
|
3043
|
-
|
|
3044
|
-
if (shouldWriteClientRequestLog(method)) {
|
|
3045
|
-
if (
|
|
3046
|
-
ResolveIOServer.shouldWriteLogsOffline()
|
|
3047
|
-
) {
|
|
3048
|
-
ResolveIOServer.getLocalLogManager().writeLog({
|
|
3049
|
-
type: 'log',
|
|
3050
|
-
data: {
|
|
3051
|
-
_id: objectIdHexString(),
|
|
3052
|
-
createdAt: new Date(),
|
|
3053
|
-
type: 'client-request',
|
|
3054
|
-
collection: '',
|
|
3055
|
-
id_document: '',
|
|
3056
|
-
payload: getBinarySize(JSON.stringify([data])) < 1000000 ? JSON.stringify([data], null, 2) : 'Too Big',
|
|
3057
|
-
method: method,
|
|
3058
|
-
id_user: ws['id_user'] || '',
|
|
3059
|
-
user: ws['user'] || '',
|
|
3060
|
-
messageId: messageId,
|
|
3061
|
-
route: messageRoute,
|
|
3062
|
-
instance_index: process.env.NODE_APP_INSTANCE || '0'
|
|
3063
|
-
}
|
|
3064
|
-
});
|
|
3065
|
-
}
|
|
3066
|
-
else {
|
|
3067
|
-
await Logs.insertOne({
|
|
3068
|
-
_id: objectIdHexString(),
|
|
3069
|
-
type: 'client-request',
|
|
3070
|
-
collection: '',
|
|
3071
|
-
id_document: '',
|
|
3072
|
-
payload: getBinarySize(JSON.stringify([data])) < 1000000 ? JSON.stringify([data], null, 2) : 'Too Big',
|
|
3073
|
-
method: method,
|
|
3074
|
-
id_user: ws['id_user'] || '',
|
|
3075
|
-
user: ws['user'] || '',
|
|
3076
|
-
messageId: messageId,
|
|
3077
|
-
route: messageRoute,
|
|
3078
|
-
client: 'ResolveIO',
|
|
3079
|
-
instance: ResolveIOServer.getInstanceHost(),
|
|
3080
|
-
instance_index: process.env.NODE_APP_INSTANCE || '0'
|
|
3081
|
-
});
|
|
3082
|
-
}
|
|
3083
|
-
}
|
|
3084
|
-
|
|
3085
|
-
if (this._methodManager._methods[method]) {
|
|
3086
|
-
try {
|
|
3087
|
-
await this._methodManager.callMethod.call(Object.assign({}, this._methodManager, MethodManager.prototype, {id_user: ws['id_user'], user: ws['user'], id_ws: ws['id_socket']}), method, ...data);
|
|
3088
|
-
}
|
|
3089
|
-
catch (err) {
|
|
3090
|
-
console.log(new Date(), 'Offline Error', JSON.stringify(err, null, 2));
|
|
3091
|
-
}
|
|
3092
|
-
|
|
3093
|
-
if (method === 'updateDocumentOffline' || method === 'updateDocumentPropsOffline') {
|
|
3094
|
-
ResolveIOServer.getMongoManager().invalidateQueryCache(data[0]);
|
|
3095
|
-
}
|
|
3096
|
-
}
|
|
3097
|
-
else {
|
|
3098
|
-
console.log('Offline - Could not find method: ' + method);
|
|
3099
|
-
}
|
|
3100
|
-
}
|
|
3101
|
-
|
|
3102
|
-
this._offlineUpdates.splice(this._offlineUpdates.map(a => a['id_socket']).indexOf(ws['id_socket']), 1);
|
|
3103
|
-
}
|
|
3104
|
-
else {
|
|
3105
|
-
// It's presumably a 'method' or something else
|
|
3106
|
-
let dataCopy = [...msg];
|
|
3107
|
-
|
|
3108
|
-
dataCopy.shift();
|
|
3109
|
-
// eslint-disable-next-line no-unused-vars
|
|
3110
|
-
let date = dataCopy.shift();
|
|
3111
|
-
let msgId = dataCopy.shift();
|
|
3112
|
-
let msgType = dataCopy.shift();
|
|
3113
|
-
|
|
3114
|
-
if (msgType === 'method') {
|
|
3115
|
-
let methodName = dataCopy.shift();
|
|
3116
|
-
|
|
3117
|
-
if (ws['user_readonly']) {
|
|
3118
|
-
return;
|
|
3119
|
-
}
|
|
3120
|
-
|
|
3121
|
-
if (shouldWriteClientRequestLog(methodName)) {
|
|
3122
|
-
if (
|
|
3123
|
-
ResolveIOServer.shouldWriteLogsOffline()
|
|
3124
|
-
) {
|
|
3125
|
-
ResolveIOServer.getLocalLogManager().writeLog({
|
|
3126
|
-
type: 'log',
|
|
3127
|
-
data: {
|
|
3128
|
-
_id: objectIdHexString(),
|
|
3129
|
-
createdAt: new Date(),
|
|
3130
|
-
type: 'client-request',
|
|
3131
|
-
collection: '',
|
|
3132
|
-
id_document: '',
|
|
3133
|
-
payload: getBinarySize(JSON.stringify([dataCopy])) < 1000000 ? JSON.stringify([dataCopy], null, 2) : 'Too Big',
|
|
3134
|
-
method: methodName,
|
|
3135
|
-
id_user: ws['id_user'] || '',
|
|
3136
|
-
user: ws['user'] || '',
|
|
3137
|
-
messageId: messageId,
|
|
3138
|
-
route: messageRoute,
|
|
3139
|
-
instance_index: process.env.NODE_APP_INSTANCE || '0'
|
|
3140
|
-
}
|
|
3141
|
-
});
|
|
3142
|
-
}
|
|
3143
|
-
else {
|
|
3144
|
-
await Logs.insertOne({
|
|
3145
|
-
_id: objectIdHexString(),
|
|
3146
|
-
type: 'client-request',
|
|
3147
|
-
collection: '',
|
|
3148
|
-
id_document: '',
|
|
3149
|
-
payload: getBinarySize(JSON.stringify([dataCopy])) < 1000000 ? JSON.stringify([dataCopy], null, 2) : 'Too Big',
|
|
3150
|
-
method: methodName,
|
|
3151
|
-
id_user: ws['id_user'] || '',
|
|
3152
|
-
user: ws['user'] || '',
|
|
3153
|
-
messageId: messageId,
|
|
3154
|
-
route: messageRoute,
|
|
3155
|
-
client: 'ResolveIO',
|
|
3156
|
-
instance: ResolveIOServer.getInstanceHost(),
|
|
3157
|
-
instance_index: process.env.NODE_APP_INSTANCE || '0'
|
|
3158
|
-
});
|
|
3159
|
-
}
|
|
3160
|
-
}
|
|
3161
|
-
|
|
3162
|
-
// Immediately ACK
|
|
3163
|
-
let ack: ServerResponseModel = {
|
|
3164
|
-
messageId: msgId,
|
|
3165
|
-
hasError: false,
|
|
3166
|
-
data: 'ACK'
|
|
3167
|
-
};
|
|
3168
|
-
|
|
3169
|
-
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
3170
|
-
this._websocketManager.send(ws, ack);
|
|
3171
|
-
}
|
|
3172
|
-
|
|
3173
|
-
let method = this._methodManager.getMethod(methodName);
|
|
3174
|
-
const forceWorker = this._isWorkersEnabled && !!method?.forceWorker;
|
|
3175
|
-
const targetWorkerIndex = this._isWorkersEnabled && method ? method.targetWorkerIndex : null;
|
|
3176
|
-
const targetWorkerInstance = this._isWorkersEnabled && method ? method.targetWorkerInstance : null;
|
|
3177
|
-
const hasWorkerForMethod = this._workerDispatcherManager ? this._workerDispatcherManager.hasWorkersForMethod(methodName) : false;
|
|
3178
|
-
const isAiCodex = methodName === 'aiCoderTerminalRunCodex' || methodName === 'aiCoderAppRunCodex';
|
|
3179
|
-
const isExcludedFromWorker = (
|
|
3180
|
-
methodName === 'find' ||
|
|
3181
|
-
methodName === 'insertDocument' ||
|
|
3182
|
-
methodName === 'countWithQuery' ||
|
|
3183
|
-
methodName === 'findOne' ||
|
|
3184
|
-
methodName === 'updateDocumentProps' ||
|
|
3185
|
-
methodName === 'findWithOptions' ||
|
|
3186
|
-
methodName === 'updateDocument' ||
|
|
3187
|
-
methodName === 'insertErrorLog' ||
|
|
3188
|
-
methodName === 'removeDocument' ||
|
|
3189
|
-
methodName === 'supportCreateBillingUser' ||
|
|
3190
|
-
methodName === 'getSignedUrl' ||
|
|
3191
|
-
methodName === 'getSignedUrls' ||
|
|
3192
|
-
methodName === 'getSignedUrlWithId' ||
|
|
3193
|
-
methodName === 'incorrectUser' ||
|
|
3194
|
-
methodName === 'reloadWS' ||
|
|
3195
|
-
methodName === 'reconnectWS' ||
|
|
3196
|
-
methodName === 'disconnectWS'
|
|
3197
|
-
);
|
|
3198
|
-
|
|
3199
|
-
if ((targetWorkerIndex || targetWorkerInstance || forceWorker) && this._isWorkersEnabled && !hasWorkerForMethod && this._methodManager.getEnableDebug()) {
|
|
3200
|
-
console.warn(new Date(), '[WorkerDispatcher] Worker unavailable, running method locally', {
|
|
3201
|
-
method: methodName,
|
|
3202
|
-
targetWorkerIndex: targetWorkerIndex || null,
|
|
3203
|
-
targetWorkerInstance: targetWorkerInstance || null,
|
|
3204
|
-
forceWorker
|
|
3205
|
-
});
|
|
3206
|
-
}
|
|
3207
|
-
|
|
3208
|
-
const shouldDispatchToWorker = (
|
|
3209
|
-
method &&
|
|
3210
|
-
!method.skipWorker &&
|
|
3211
|
-
this._isWorkersEnabled &&
|
|
3212
|
-
this._workerDispatcherManager &&
|
|
3213
|
-
hasWorkerForMethod &&
|
|
3214
|
-
(forceWorker || !isExcludedFromWorker)
|
|
3215
|
-
);
|
|
3216
|
-
|
|
3217
|
-
if (isAiCodex && this._aiWorkerDebug) {
|
|
3218
|
-
const queueSnapshot = this._workerDispatcherManager ? this._workerDispatcherManager.getQueueSnapshot() : null;
|
|
3219
|
-
console.log(new Date(), '[AI Worker Debug] dispatch check', {
|
|
3220
|
-
isWorkersEnabled: this._isWorkersEnabled,
|
|
3221
|
-
isWorkerInstance: this._isWorkerInstance,
|
|
3222
|
-
forceWorker,
|
|
3223
|
-
hasWorkerForMethod,
|
|
3224
|
-
targetWorkerIndex: targetWorkerIndex || null,
|
|
3225
|
-
targetWorkerInstance: targetWorkerInstance || null,
|
|
3226
|
-
shouldDispatchToWorker,
|
|
3227
|
-
queueSnapshot
|
|
3228
|
-
});
|
|
3229
|
-
}
|
|
3230
|
-
|
|
3231
|
-
if (shouldDispatchToWorker) {
|
|
3232
|
-
this._workerDispatcherManager.sendClientTask(msgId, methodName, dataCopy, {
|
|
3233
|
-
id_user: ws['id_user'],
|
|
3234
|
-
user: ws['user'],
|
|
3235
|
-
id_ws: ws['id_socket']
|
|
3236
|
-
});
|
|
3237
|
-
}
|
|
3238
|
-
else {
|
|
3239
|
-
// No worker available: do method locally
|
|
3240
|
-
if (methodName === 'aiCoderTerminalRunCodex' || methodName === 'aiCoderAppRunCodex') {
|
|
3241
|
-
if (this._aiWorkerDebug) {
|
|
3242
|
-
console.warn(new Date(), '[AI Worker Debug] AI execution running locally', {
|
|
3243
|
-
isWorkersEnabled: this._isWorkersEnabled,
|
|
3244
|
-
isWorkerInstance: this._isWorkerInstance,
|
|
3245
|
-
targetWorkerIndex: targetWorkerIndex || null,
|
|
3246
|
-
targetWorkerInstance: targetWorkerInstance || null,
|
|
3247
|
-
hasWorkerForMethod
|
|
3248
|
-
});
|
|
3249
|
-
}
|
|
3250
|
-
setTimeout(async () => {
|
|
3251
|
-
try {
|
|
3252
|
-
await this.callMethodLocally(ws, msgId, methodName, dataCopy);
|
|
3253
|
-
}
|
|
3254
|
-
catch (error) {
|
|
3255
|
-
console.error(new Date(), 'AI execution run failed:', error);
|
|
3256
|
-
}
|
|
3257
|
-
}, 0);
|
|
3258
|
-
}
|
|
3259
|
-
else {
|
|
3260
|
-
await this.callMethodLocally(ws, msgId, methodName, dataCopy);
|
|
3261
|
-
}
|
|
3262
|
-
}
|
|
3263
|
-
}
|
|
3264
|
-
}
|
|
3265
|
-
}
|
|
3266
|
-
catch (err) {
|
|
3267
|
-
throw err;
|
|
3268
|
-
}
|
|
3269
|
-
}
|
|
3270
|
-
|
|
3271
|
-
/**
|
|
3272
|
-
* callMethodLocally is your old approach for invoking the method in-process.
|
|
3273
|
-
*/
|
|
3274
|
-
private async callMethodLocally(ws: WebSocket, messageId: number, method: string, params: any[]) {
|
|
3275
|
-
let serverRes: ServerResponseModel = {
|
|
3276
|
-
messageId: messageId,
|
|
3277
|
-
hasError: false,
|
|
3278
|
-
data: null
|
|
3279
|
-
};
|
|
3280
|
-
try {
|
|
3281
|
-
// You can keep your logging code (LogMethodLatencies, Logs.insertOne, etc.)
|
|
3282
|
-
let result = await this._methodManager.callMethod.call(
|
|
3283
|
-
Object.assign({}, this._methodManager, MethodManager.prototype, {
|
|
3284
|
-
id_user: ws['id_user'],
|
|
3285
|
-
user: ws['user'],
|
|
3286
|
-
id_ws: ws['id_socket']
|
|
3287
|
-
}),
|
|
3288
|
-
method,
|
|
3289
|
-
...params
|
|
3290
|
-
);
|
|
3291
|
-
|
|
3292
|
-
serverRes.data = result;
|
|
3293
|
-
if (this._aiWorkerDebug && typeof method === 'string' && method.startsWith('ai')) {
|
|
3294
|
-
let resultBytes: number | null = null;
|
|
3295
|
-
try {
|
|
3296
|
-
resultBytes = Buffer.byteLength(JSON.stringify(result));
|
|
3297
|
-
}
|
|
3298
|
-
catch {
|
|
3299
|
-
resultBytes = null;
|
|
3300
|
-
}
|
|
3301
|
-
console.log(new Date(), '[AI Worker Debug] local method result', {
|
|
3302
|
-
method,
|
|
3303
|
-
messageId,
|
|
3304
|
-
id_socket: ws ? ws['id_socket'] : null,
|
|
3305
|
-
id_user: ws ? ws['id_user'] : null,
|
|
3306
|
-
resultBytes
|
|
3307
|
-
});
|
|
3308
|
-
}
|
|
3309
|
-
}
|
|
3310
|
-
catch (err) {
|
|
3311
|
-
serverRes.hasError = true;
|
|
3312
|
-
serverRes.data = err || 'Unknown error';
|
|
3313
|
-
}
|
|
3314
|
-
|
|
3315
|
-
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
3316
|
-
this._websocketManager.send(ws, serverRes);
|
|
3317
|
-
}
|
|
3318
|
-
}
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
/**
|
|
3323
|
-
* Cleanly remove a client from the subscription manager, etc.
|
|
3324
|
-
*/
|
|
3325
|
-
public async unsubscribeWS(ws: WebSocket) {
|
|
3326
|
-
if (this._subscriptionManager && this._methodManager.getEnableDebug()) {
|
|
3327
|
-
console.log(new Date(), 'Server App', 'Unsub WS', ws['user'], ws['id_socket']);
|
|
3328
|
-
}
|
|
3329
|
-
this.logConnectDebug('WS unsubscribe', {
|
|
3330
|
-
id_socket: ws ? ws['id_socket'] : null,
|
|
3331
|
-
id_user: ws ? ws['id_user'] : null,
|
|
3332
|
-
user: ws ? ws['user'] : null
|
|
3333
|
-
});
|
|
3334
|
-
await this._subscriptionManager.unsubscribeAll(ws);
|
|
3335
|
-
ws.removeAllListeners();
|
|
3336
|
-
ws = null;
|
|
3337
|
-
}
|
|
3338
|
-
|
|
3339
|
-
public getApp() {
|
|
3340
|
-
return this._app;
|
|
3341
|
-
}
|
|
3342
|
-
|
|
3343
|
-
public getServerConfig() {
|
|
3344
|
-
return ResolveIOServer.getServerConfig();
|
|
3345
|
-
}
|
|
3346
|
-
|
|
3347
|
-
public getWorkerDispatcherManager() {
|
|
3348
|
-
return this._workerDispatcherManager;
|
|
3349
|
-
}
|
|
3350
|
-
|
|
3351
|
-
public getWorkerServerManager() {
|
|
3352
|
-
return this._workerServerManager;
|
|
3353
|
-
}
|
|
3354
|
-
}
|