evolcore 0.0.43-test.3 → 0.0.43-test.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/bin/ecmsg.js +492 -0
- package/bin/install-codex-managed-hooks.mjs +130 -8
- package/dist/ability/application-service.js +37 -0
- package/dist/ability/resources.js +28 -0
- package/dist/ability/runtime.js +6 -1
- package/dist/agents/claude-runner.js +7 -5
- package/dist/agents/codex/app-server-client.js +0 -12
- package/dist/agents/codex-runner.js +12 -54
- package/dist/agents/ecagent-runner.js +130 -35
- package/dist/aun/aid/agentmd.js +56 -12
- package/dist/aun/aid/control-aid.js +15 -0
- package/dist/aun/aid/identity.js +8 -0
- package/dist/aun/aid/ownership-operation.js +12 -3
- package/dist/aun/menu/client.js +8 -7
- package/dist/aun/menu/index.js +1 -1
- package/dist/aun/menu/transport.js +8 -5
- package/dist/aun/service-proxy.js +3 -0
- package/dist/channels/aun.js +56 -22
- package/dist/channels/daemon.js +12 -4
- package/dist/cli/agent-command.js +5 -0
- package/dist/cli/agent-create.js +32 -8
- package/dist/cli/agent.js +9 -3
- package/dist/cli/aid-bind-command.js +3 -0
- package/dist/cli/aun-commands.js +12 -5
- package/dist/cli/code-stats.js +451 -213
- package/dist/cli/ctl-command.js +1 -3
- package/dist/cli/daemon-commands.js +42 -21
- package/dist/cli/directory.js +7 -8
- package/dist/cli/ecmsg-gateway.js +204 -0
- package/dist/cli/getmeta-command.js +2 -1
- package/dist/cli/index.js +6 -0
- package/dist/cli/init.js +58 -18
- package/dist/cli/provider-command.js +82 -5
- package/dist/cli/restart-monitor.js +11 -19
- package/dist/cli/stats.js +8 -1
- package/dist/cli/task-context.js +3 -0
- package/dist/cli/trigger-command.js +20 -14
- package/dist/config/boot-log.js +23 -2
- package/dist/config/config-field-policy.js +2 -0
- package/dist/config/config-manager.js +13 -17
- package/dist/config/daemon-services.js +1 -0
- package/dist/config/peer-role-resolver.js +6 -6
- package/dist/config/provider-identity-migration.js +202 -0
- package/dist/config/provider-identity.js +37 -0
- package/dist/config/provider-ipc-service.js +40 -3
- package/dist/config/provider-local-sync.js +153 -0
- package/dist/config/providers-config.js +32 -4
- package/dist/config/providers-management.js +144 -68
- package/dist/config/providers.js +4 -1
- package/dist/config/role-schema.js +3 -3
- package/dist/config/schema-migration.js +188 -3
- package/dist/core/agent-application-service.js +11 -0
- package/dist/core/agent-ipc-registration.js +2 -0
- package/dist/core/agent-runtime.js +45 -11
- package/dist/core/auth/auth-gateway.js +8 -6
- package/dist/core/auth/delegation-diagnostics.js +6 -2
- package/dist/core/auth/operation-authorizer.js +5 -2
- package/dist/core/auth/operation-catalog.js +6 -4
- package/dist/core/baseagent-loader.js +11 -3
- package/dist/core/bootstrap-service.js +8 -0
- package/dist/core/channel-loader.js +61 -23
- package/dist/core/command/agent-control.js +4 -1
- package/dist/core/command/chatmode-config.js +46 -0
- package/dist/core/command/cli-intent-parser.js +2 -1
- package/dist/core/command/command-handler.js +16 -20
- package/dist/core/command/daemon-diagnostics.js +2 -1
- package/dist/core/command/menu-catalog.js +1 -1
- package/dist/core/command/menu-handler.js +16 -16
- package/dist/core/command/menu-protocol.js +4 -2
- package/dist/core/command/provider-menu.js +4 -2
- package/dist/core/command/role-menu.js +2 -2
- package/dist/core/command/slash-handler.js +56 -42
- package/dist/core/daemon-lifecycle.js +35 -17
- package/dist/core/event-catalog.js +19 -0
- package/dist/core/evolagent-registry.js +3 -0
- package/dist/core/handoff/runtime.js +9 -3
- package/dist/core/initialization/service.js +4 -3
- package/dist/core/ipc-registration.js +6 -0
- package/dist/core/message/aun-ipc-sender.js +81 -3
- package/dist/core/message/im-renderer.js +24 -19
- package/dist/core/message/message-admission.js +13 -2
- package/dist/core/message/response-engine.js +40 -41
- package/dist/core/message/send-options.js +40 -0
- package/dist/core/message/send-routing.js +310 -0
- package/dist/core/model/model-capabilities.js +37 -0
- package/dist/core/model/model-catalog.js +6 -2
- package/dist/core/ownership/owner-resolver.js +117 -14
- package/dist/core/permission/ec-command-parser.js +195 -27
- package/dist/core/permission/readonly-shell-query.js +66 -7
- package/dist/core/permission/tool-error-code.js +13 -1
- package/dist/core/permission/tool-policy.js +74 -20
- package/dist/core/plugin-lifecycle.js +6 -2
- package/dist/core/relation/pa-team.js +7 -2
- package/dist/core/relation/same-owner-proof.js +131 -0
- package/dist/core/session/session-idle-monitor.js +94 -16
- package/dist/core/session/session-manager.js +69 -16
- package/dist/core/thread/thread-route.js +5 -2
- package/dist/directory/owner-directory-cache.js +690 -99
- package/dist/directory/owner-directory-markdown.js +146 -36
- package/dist/directory/owner-directory-query-operation.js +7 -9
- package/dist/directory/owner-directory-scope.js +12 -0
- package/dist/directory/owner-directory-search.js +5 -1
- package/dist/directory/owner-directory-store.js +28 -0
- package/dist/ecweb/basic-settings-service.js +15 -1
- package/dist/ecweb/message-context-service.js +2 -2
- package/dist/ecweb/providers-service.js +3 -1
- package/dist/ecweb/runtime/lifecycle.js +38 -5
- package/dist/ecweb/runtime/supervisor.js +157 -49
- package/dist/index.js +202 -38
- package/dist/paths.js +2 -1
- package/dist/proxy/manager.js +2 -0
- package/dist/proxy/service-supervisor.js +379 -143
- package/dist/response-system/engines/v1/proactive-flow.js +4 -0
- package/dist/trigger/application-service.js +1 -0
- package/dist/trigger/audit.js +3 -2
- package/dist/trigger/authorization.js +2 -3
- package/dist/trigger/event-source.js +4 -0
- package/dist/trigger/feedback.js +26 -2
- package/dist/trigger/history.js +4 -3
- package/dist/trigger/ipc-service.js +24 -19
- package/dist/trigger/manager.js +3 -5
- package/dist/trigger/parser.js +4 -21
- package/dist/trigger/patch.js +3 -15
- package/dist/trigger/scheduler.js +48 -13
- package/dist/trigger/system-definitions.js +2 -2
- package/dist/trigger/validation.js +4 -13
- package/dist/utils/autostart.js +2 -2
- package/dist/utils/cross-platform.js +36 -8
- package/dist/utils/error-utils.js +8 -0
- package/dist/utils/macos-autostart.js +31 -0
- package/dist/utils/model-prices.jsonl +15 -8
- package/dist/utils/process-introspect.js +21 -1
- package/dist/utils/public-suffix.js +5811 -0
- package/dist/utils/startup-trace.js +7 -1
- package/dist/utils/windows-autostart.js +42 -7
- package/ecagent/dist/harness/compaction/branch-summarization.js +1 -1
- package/ecagent/dist/harness/compaction/compaction.js +4 -4
- package/ecagent/dist/harness/env/nodejs.js +46 -2
- package/kits/docs/aun/ABILITY_CALLING.md +3 -3
- package/kits/docs/channels/aun.md +1 -1
- package/kits/docs/channels/feishu.md +2 -2
- package/kits/docs/context-assembly.md +20 -4
- package/kits/docs/evolcore/INDEX.md +3 -2
- package/kits/docs/evolcore/agent.md +2 -2
- package/kits/docs/evolcore/aid.md +8 -2
- package/kits/docs/evolcore/ctl.md +13 -4
- package/kits/docs/evolcore/directory.md +48 -15
- package/kits/docs/evolcore/file-navigation.md +63 -49
- package/kits/docs/evolcore/group.md +6 -6
- package/kits/docs/evolcore/memory-consolidation.md +1 -1
- package/kits/docs/evolcore/menu-protocol.md +33 -3
- package/kits/docs/evolcore/model.md +5 -3
- package/kits/docs/evolcore/msg.md +28 -27
- package/kits/docs/evolcore/provider.md +39 -5
- package/kits/docs/evolcore/response.md +3 -1
- package/kits/docs/evolcore/thread-chain.md +12 -14
- package/kits/docs/evolcore/tool-authorization.md +21 -1
- package/kits/docs/evolcore/trigger.md +12 -9
- package/kits/docs/identity/ROLE_DETAIL.md +1 -1
- package/kits/docs/prompt-loading-architecture.md +30 -20
- package/kits/eck_manifest.auxiliary.json +5 -5
- package/kits/eck_manifest.json +28 -49
- package/kits/eck_manifest.memory.json +18 -18
- package/kits/rules/01-overview.md +43 -45
- package/kits/rules/02-navigation.md +44 -53
- package/kits/rules/03-identity.md +21 -25
- package/kits/rules/04-relation.md +23 -33
- package/kits/rules/05-venue.md +24 -25
- package/kits/rules/06-channel.md +3 -3
- package/kits/schemas/_meta.json +20 -27
- package/kits/schemas/agent-config.schema.19.json +10 -6
- package/kits/schemas/providers.schema.2.json +74 -0
- package/kits/schemas/relation-config.schema.12.json +2 -2
- package/kits/schemas/role-config.schema.2.json +5 -5
- package/kits/schemas/role-registry.schema.1.json +1 -1
- package/kits/skills/ec-agent-create/SKILL.md +1 -0
- package/kits/skills/ec-service-proxy/SKILL.md +3 -0
- package/kits/skills/ec-trigger/SKILL.md +4 -4
- package/kits/skills/ec-trigger/references/examples.md +1 -1
- package/kits/templates/system-fragments/channel.md +5 -5
- package/kits/templates/system-fragments/commands.md +3 -2
- package/kits/templates/system-fragments/directory-contacts.md +10 -3
- package/kits/templates/system-fragments/identity.md +0 -1
- package/kits/templates/system-fragments/personal-assistant.md +22 -3
- package/kits/templates/system-fragments/reply.md +47 -28
- package/kits/templates/system-fragments/session.md +2 -9
- package/kits/templates/system-fragments/thread.md +5 -5
- package/package.json +17 -13
- package/kits/schemas/agent-config.schema.20.json +0 -925
- package/kits/schemas/relation-config.schema.13.json +0 -289
- package/kits/schemas/role-config.schema.3.json +0 -1038
|
@@ -4,7 +4,7 @@ import path from 'node:path';
|
|
|
4
4
|
import { spawn } from 'node:child_process';
|
|
5
5
|
import crypto from 'node:crypto';
|
|
6
6
|
import { resolvePaths } from '../paths.js';
|
|
7
|
-
import { SERVICE_HEALTH_INTERVALS, SERVICE_RESTART_BACKOFF_MS, SERVICE_RESTART_MAX_ATTEMPTS, SERVICE_RESTART_WINDOW_MS, serviceRuntimeMode, } from '../config/daemon-services.js';
|
|
7
|
+
import { SERVICE_HEALTH_INTERVALS, SERVICE_RESTART_BACKOFF_MS, SERVICE_RESTART_MAX_ATTEMPTS, SERVICE_RESTART_WINDOW_MS, SERVICE_STARTUP_TIMEOUT_MS, serviceRuntimeMode, } from '../config/daemon-services.js';
|
|
8
8
|
import { getProcessStartTime, startTimeMatches } from '../utils/process-introspect.js';
|
|
9
9
|
import * as platform from '../utils/cross-platform.js';
|
|
10
10
|
import { logger as defaultLogger } from '../utils/logger.js';
|
|
@@ -44,6 +44,8 @@ function readInstanceRecords(name) {
|
|
|
44
44
|
if (record.owner !== 'evolcore-service-supervisor'
|
|
45
45
|
|| record.pid !== Number(match[1])
|
|
46
46
|
|| typeof record.startedAt !== 'number'
|
|
47
|
+
|| !Number.isFinite(record.startedAt)
|
|
48
|
+
|| record.startedAt <= 0
|
|
47
49
|
|| !Number.isInteger(record.port))
|
|
48
50
|
continue;
|
|
49
51
|
output.push({
|
|
@@ -103,7 +105,36 @@ export class ServiceSupervisor {
|
|
|
103
105
|
log;
|
|
104
106
|
active = new Map();
|
|
105
107
|
desired = new Map();
|
|
108
|
+
operations = new Map();
|
|
106
109
|
stopped = false;
|
|
110
|
+
logLifecycle(active, event, details = {}) {
|
|
111
|
+
const record = {
|
|
112
|
+
ts: new Date().toISOString(), event, service: active.config.name,
|
|
113
|
+
attemptId: active.attemptId, pid: active.pid || undefined,
|
|
114
|
+
generation: active.generation, ...details,
|
|
115
|
+
};
|
|
116
|
+
const content = JSON.stringify(record);
|
|
117
|
+
this.log.info('[ServiceSupervisor] ' + content);
|
|
118
|
+
try {
|
|
119
|
+
const directory = path.join(resolvePaths().logs, 'services');
|
|
120
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
121
|
+
fs.appendFileSync(path.join(directory, active.config.name + '.log'), '\n' + content + '\n');
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
this.log.warn('[ServiceSupervisor] lifecycle log failed service=' + active.config.name + ': ' + String(error));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
queueOperation(name, operation) {
|
|
128
|
+
const previous = this.operations.get(name) ?? Promise.resolve();
|
|
129
|
+
const result = previous.then(operation);
|
|
130
|
+
const settled = result.then(() => { }, () => { });
|
|
131
|
+
this.operations.set(name, settled);
|
|
132
|
+
void settled.then(() => {
|
|
133
|
+
if (this.operations.get(name) === settled)
|
|
134
|
+
this.operations.delete(name);
|
|
135
|
+
});
|
|
136
|
+
return result;
|
|
137
|
+
}
|
|
107
138
|
constructor(log = defaultLogger) {
|
|
108
139
|
this.log = log;
|
|
109
140
|
}
|
|
@@ -116,41 +147,46 @@ export class ServiceSupervisor {
|
|
|
116
147
|
return;
|
|
117
148
|
const next = new Map(services.map(service => [service.name, service]));
|
|
118
149
|
for (const [name, active] of this.active) {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
active.configHash = serviceConfigHash(desired);
|
|
129
|
-
active.generation += 1;
|
|
130
|
-
if (serviceRuntimeMode(desired) === 'managed')
|
|
131
|
-
await this.restartEntry(active, 'service configuration changed');
|
|
132
|
-
else {
|
|
133
|
-
if (active.healthTimer)
|
|
134
|
-
clearTimeout(active.healthTimer);
|
|
135
|
-
active.healthTimer = undefined;
|
|
136
|
-
active.state = desired.port ? 'starting' : 'unknown';
|
|
137
|
-
void this.monitorHealth(active).finally(() => this.scheduleHealthCheck(active));
|
|
150
|
+
await this.queueOperation(name, async () => {
|
|
151
|
+
if (this.active.get(name) !== active)
|
|
152
|
+
return;
|
|
153
|
+
const desired = next.get(name);
|
|
154
|
+
if (!desired || desired.enabled !== true
|
|
155
|
+
|| (desired.name === 'ecweb' && serviceRuntimeMode(desired) === 'managed')
|
|
156
|
+
|| serviceRuntimeMode(desired) !== serviceRuntimeMode(active.config)) {
|
|
157
|
+
if (await this.stopEntry(active, true))
|
|
158
|
+
this.active.delete(name);
|
|
138
159
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
if (
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
160
|
+
else if (serviceConfigHash(desired) !== active.configHash) {
|
|
161
|
+
active.config = desired;
|
|
162
|
+
active.configHash = serviceConfigHash(desired);
|
|
163
|
+
active.adoptionAllowed = false;
|
|
164
|
+
active.generation += 1;
|
|
165
|
+
if (serviceRuntimeMode(desired) === 'managed')
|
|
166
|
+
await this.restartEntry(active, 'service configuration changed');
|
|
167
|
+
else {
|
|
168
|
+
if (active.healthTimer)
|
|
169
|
+
clearTimeout(active.healthTimer);
|
|
170
|
+
active.healthTimer = undefined;
|
|
171
|
+
active.state = desired.port ? 'starting' : 'unknown';
|
|
172
|
+
void this.monitorHealth(active).finally(() => this.scheduleHealthCheck(active));
|
|
173
|
+
}
|
|
152
174
|
}
|
|
153
|
-
|
|
175
|
+
else {
|
|
176
|
+
const recoveryChanged = stableHash(active.config.runtime?.restart) !== stableHash(desired.runtime?.restart);
|
|
177
|
+
active.config = desired;
|
|
178
|
+
if (recoveryChanged && serviceRuntimeMode(desired) === 'managed') {
|
|
179
|
+
if (active.restartTimer)
|
|
180
|
+
clearTimeout(active.restartTimer);
|
|
181
|
+
active.restartTimer = undefined;
|
|
182
|
+
if (active.state === 'backoff')
|
|
183
|
+
active.state = 'failed';
|
|
184
|
+
if (active.state === 'failed')
|
|
185
|
+
this.scheduleRestart(active);
|
|
186
|
+
this.scheduleHealthCheck(active);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
});
|
|
154
190
|
}
|
|
155
191
|
this.desired.clear();
|
|
156
192
|
for (const service of services)
|
|
@@ -174,7 +210,7 @@ export class ServiceSupervisor {
|
|
|
174
210
|
const active = this.adoptOrCreate(service);
|
|
175
211
|
this.active.set(service.name, active);
|
|
176
212
|
if (active.state !== 'ready') {
|
|
177
|
-
void this.startEntry(active).catch(() => { });
|
|
213
|
+
void this.queueOperation(service.name, () => this.startEntry(active)).catch(() => { });
|
|
178
214
|
}
|
|
179
215
|
else if (!active.healthy) {
|
|
180
216
|
void this.verifyAdopted(active);
|
|
@@ -185,41 +221,54 @@ export class ServiceSupervisor {
|
|
|
185
221
|
return this.restartService(name);
|
|
186
222
|
}
|
|
187
223
|
async restartService(name) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
224
|
+
return this.queueOperation(name, async () => {
|
|
225
|
+
const active = this.active.get(name);
|
|
226
|
+
if (!active)
|
|
227
|
+
throw Object.assign(new Error('managed service is not running: ' + name), { code: 'SERVICE_NOT_RUNNING' });
|
|
228
|
+
if (active.config.runtime?.mode !== 'managed') {
|
|
229
|
+
throw Object.assign(new Error('external service lifecycle is not managed: ' + name), { code: 'EXTERNAL_SERVICE_UNMANAGED' });
|
|
230
|
+
}
|
|
231
|
+
await this.restartEntry(active, 'manual restart');
|
|
232
|
+
return this.snapshotOne(active);
|
|
233
|
+
});
|
|
196
234
|
}
|
|
197
235
|
async startService(name) {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
236
|
+
return this.queueOperation(name, async () => {
|
|
237
|
+
const service = this.desired.get(name);
|
|
238
|
+
if (!service || service.runtime?.mode !== 'managed') {
|
|
239
|
+
throw Object.assign(new Error('service is not configured as managed: ' + name), { code: 'EXTERNAL_SERVICE_UNMANAGED' });
|
|
240
|
+
}
|
|
241
|
+
service.enabled = true;
|
|
242
|
+
this.desired.set(name, service);
|
|
243
|
+
let active = this.active.get(name);
|
|
244
|
+
if (!active) {
|
|
245
|
+
active = this.adoptOrCreate(service);
|
|
246
|
+
this.active.set(name, active);
|
|
247
|
+
}
|
|
248
|
+
await this.startEntry(active);
|
|
249
|
+
return this.snapshotOne(active);
|
|
250
|
+
});
|
|
211
251
|
}
|
|
212
252
|
async stopService(name) {
|
|
213
|
-
const
|
|
214
|
-
if (
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
throw Object.assign(new Error('external service lifecycle is not managed: ' + name), { code: 'EXTERNAL_SERVICE_UNMANAGED' });
|
|
253
|
+
const starting = this.active.get(name);
|
|
254
|
+
if (starting?.startPromise) {
|
|
255
|
+
starting.intentionalStop = true;
|
|
256
|
+
starting.generation += 1;
|
|
218
257
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
258
|
+
return this.queueOperation(name, async () => {
|
|
259
|
+
const active = this.active.get(name);
|
|
260
|
+
if (!active)
|
|
261
|
+
return { name, state: 'disabled', failureCount: 0, restartCount: 0, generation: 0, managed: false };
|
|
262
|
+
if (active.config.runtime?.mode !== 'managed') {
|
|
263
|
+
throw Object.assign(new Error('external service lifecycle is not managed: ' + name), { code: 'EXTERNAL_SERVICE_UNMANAGED' });
|
|
264
|
+
}
|
|
265
|
+
const stopped = await this.stopEntry(active, true);
|
|
266
|
+
await active.startPromise?.catch(() => { });
|
|
267
|
+
if (!stopped)
|
|
268
|
+
throw Object.assign(new Error(active.lastError ?? 'managed process did not exit: ' + name), { code: active.lastErrorCode ?? 'PROCESS_STOP_FAILED' });
|
|
269
|
+
this.active.delete(name);
|
|
270
|
+
return this.snapshotOne(active);
|
|
271
|
+
});
|
|
223
272
|
}
|
|
224
273
|
snapshot(name) {
|
|
225
274
|
return [...this.active.values()]
|
|
@@ -232,25 +281,29 @@ export class ServiceSupervisor {
|
|
|
232
281
|
return;
|
|
233
282
|
this.stopped = true;
|
|
234
283
|
for (const active of [...this.active.values()]) {
|
|
235
|
-
await this.stopEntry(active, true);
|
|
284
|
+
const stopped = await this.stopEntry(active, true);
|
|
236
285
|
await active.startPromise?.catch(() => { });
|
|
286
|
+
if (stopped)
|
|
287
|
+
this.active.delete(active.config.name);
|
|
237
288
|
}
|
|
238
|
-
this.active.clear();
|
|
239
289
|
}
|
|
240
290
|
async clearFailure(name) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
291
|
+
return this.queueOperation(name, async () => {
|
|
292
|
+
const active = this.active.get(name);
|
|
293
|
+
if (!active)
|
|
294
|
+
throw Object.assign(new Error('managed service not found: ' + name), { code: 'NOT_FOUND' });
|
|
295
|
+
if (active.config.runtime?.mode !== 'managed') {
|
|
296
|
+
throw Object.assign(new Error('external service lifecycle is not managed: ' + name), { code: 'EXTERNAL_SERVICE_UNMANAGED' });
|
|
297
|
+
}
|
|
298
|
+
active.failures = [];
|
|
299
|
+
active.failureCount = 0;
|
|
300
|
+
active.retryExhausted = false;
|
|
301
|
+
active.lastError = undefined;
|
|
302
|
+
active.lastErrorCode = undefined;
|
|
303
|
+
if (active.config.enabled === true && active.state !== 'ready')
|
|
304
|
+
await this.restartEntry(active, 'failure reset');
|
|
305
|
+
return this.snapshotOne(active);
|
|
306
|
+
});
|
|
254
307
|
}
|
|
255
308
|
async verifyAdopted(active) {
|
|
256
309
|
if (active.state !== 'starting' || !active.pid || !platform.isProcessRunning(active.pid))
|
|
@@ -262,73 +315,156 @@ export class ServiceSupervisor {
|
|
|
262
315
|
active.checkedAt = Date.now();
|
|
263
316
|
active.healthy = healthy;
|
|
264
317
|
active.state = healthy ? 'ready' : 'failed';
|
|
265
|
-
if (
|
|
318
|
+
if (healthy) {
|
|
319
|
+
if (active.recordPath) {
|
|
320
|
+
atomicWriteJson(active.recordPath, {
|
|
321
|
+
owner: 'evolcore-service-supervisor',
|
|
322
|
+
pid: active.pid,
|
|
323
|
+
startedAt: active.startedAt,
|
|
324
|
+
port: active.config.port ?? 0,
|
|
325
|
+
configHash: active.configHash,
|
|
326
|
+
state: 'ready',
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
this.scheduleHealthCheck(active);
|
|
330
|
+
}
|
|
331
|
+
else {
|
|
266
332
|
active.lastError = 'adopted service failed health check';
|
|
267
333
|
this.scheduleRestart(active);
|
|
268
334
|
}
|
|
269
|
-
else
|
|
270
|
-
this.scheduleHealthCheck(active);
|
|
271
335
|
}
|
|
272
336
|
adoptOrCreate(service) {
|
|
273
337
|
const hash = serviceConfigHash(service);
|
|
274
338
|
const records = readInstanceRecords(service.name);
|
|
275
|
-
const
|
|
339
|
+
const liveRecords = records
|
|
276
340
|
.filter(record => platform.isProcessRunning(record.pid)
|
|
277
|
-
&& startTimeMatches(record.startedAt, getProcessStartTime(record.pid))
|
|
278
|
-
|
|
279
|
-
|
|
341
|
+
&& startTimeMatches(record.startedAt, getProcessStartTime(record.pid)))
|
|
342
|
+
.sort((a, b) => b.startedAt - a.startedAt);
|
|
343
|
+
const adopted = liveRecords.find(record => record.configHash === hash);
|
|
280
344
|
if (adopted) {
|
|
281
345
|
return {
|
|
282
346
|
config: service, process: null, pid: adopted.pid, startedAt: adopted.startedAt,
|
|
283
347
|
recordPath: adopted.filePath, configHash: hash, generation: 1, state: 'starting',
|
|
284
348
|
failureCount: 0, restartCount: 0, failures: [],
|
|
285
349
|
healthFailures: 0, healthCheckInFlight: false,
|
|
286
|
-
intentionalStop: false, stopping: false,
|
|
350
|
+
intentionalStop: false, stopping: false, adoptionAllowed: true,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
const staleConfig = liveRecords[0];
|
|
354
|
+
if (staleConfig) {
|
|
355
|
+
this.log.warn('[ServiceSupervisor] managed process config changed; replacing recorded process after exit confirmation: service=' + service.name + ' pid=' + staleConfig.pid);
|
|
356
|
+
return {
|
|
357
|
+
config: service, process: null, pid: staleConfig.pid, startedAt: staleConfig.startedAt,
|
|
358
|
+
recordPath: staleConfig.filePath, configHash: hash, generation: 1, state: 'starting',
|
|
359
|
+
failureCount: 0, restartCount: 0, failures: [],
|
|
360
|
+
healthFailures: 0, healthCheckInFlight: false,
|
|
361
|
+
intentionalStop: false, stopping: false, adoptionAllowed: false,
|
|
287
362
|
};
|
|
288
363
|
}
|
|
289
364
|
for (const record of records) {
|
|
290
|
-
if (platform.isProcessRunning(record.pid)
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
platform.killProcess(record.pid, true);
|
|
295
|
-
}
|
|
296
|
-
catch { }
|
|
297
|
-
try {
|
|
298
|
-
fs.unlinkSync(record.filePath);
|
|
299
|
-
}
|
|
300
|
-
catch { }
|
|
301
|
-
}
|
|
302
|
-
else {
|
|
303
|
-
try {
|
|
304
|
-
fs.unlinkSync(record.filePath);
|
|
305
|
-
}
|
|
306
|
-
catch { }
|
|
365
|
+
if (platform.isProcessRunning(record.pid) && getProcessStartTime(record.pid) === null)
|
|
366
|
+
continue;
|
|
367
|
+
try {
|
|
368
|
+
fs.unlinkSync(record.filePath);
|
|
307
369
|
}
|
|
370
|
+
catch { }
|
|
308
371
|
}
|
|
309
372
|
return {
|
|
310
373
|
config: service, process: null, pid: 0, startedAt: 0,
|
|
311
374
|
recordPath: '', configHash: hash, generation: 0, state: 'disabled',
|
|
312
375
|
failureCount: 0, restartCount: 0, failures: [],
|
|
313
376
|
healthFailures: 0, healthCheckInFlight: false,
|
|
314
|
-
intentionalStop: false, stopping: false,
|
|
377
|
+
intentionalStop: false, stopping: false, adoptionAllowed: true,
|
|
315
378
|
};
|
|
316
379
|
}
|
|
317
|
-
|
|
318
|
-
if (
|
|
380
|
+
startEntry(active) {
|
|
381
|
+
if (active.startPromise)
|
|
319
382
|
return active.startPromise;
|
|
383
|
+
if (this.stopped || active.stopping) {
|
|
384
|
+
return Promise.reject(Object.assign(new Error('service start cancelled: ' + active.config.name), { code: 'SERVICE_START_CANCELLED' }));
|
|
385
|
+
}
|
|
386
|
+
active.intentionalStop = false;
|
|
387
|
+
const promise = Promise.resolve().then(() => this.runStartEntry(active)).catch(async (error) => {
|
|
388
|
+
if (active.intentionalStop || this.stopped || error?.code === 'SERVICE_START_CANCELLED')
|
|
389
|
+
throw error;
|
|
390
|
+
const failureMessage = error instanceof Error ? error.message : String(error);
|
|
391
|
+
const failureCode = String(error?.code ?? 'UNKNOWN');
|
|
392
|
+
active.lastError = failureMessage;
|
|
393
|
+
active.lastErrorCode = failureCode;
|
|
394
|
+
if (!active.restartTimer)
|
|
395
|
+
active.state = 'failed';
|
|
396
|
+
this.log.warn('[ServiceSupervisor] start failed service=' + active.config.name + ' code=' + active.lastErrorCode + ': ' + active.lastError);
|
|
397
|
+
const stopped = !active.process?.pid || await this.stopEntry(active, false);
|
|
398
|
+
if (stopped && !active.intentionalStop && !this.stopped) {
|
|
399
|
+
active.lastError = failureMessage;
|
|
400
|
+
active.lastErrorCode = failureCode;
|
|
401
|
+
active.state = 'failed';
|
|
402
|
+
if (failureCode === 'PORT_IN_USE') {
|
|
403
|
+
if (active.restartTimer)
|
|
404
|
+
clearTimeout(active.restartTimer);
|
|
405
|
+
active.restartTimer = undefined;
|
|
406
|
+
active.state = 'failed';
|
|
407
|
+
this.scheduleHealthCheck(active);
|
|
408
|
+
}
|
|
409
|
+
else if (failureCode !== 'PROCESS_STOP_FAILED'
|
|
410
|
+
&& failureCode !== 'PROCESS_IDENTITY_UNVERIFIED') {
|
|
411
|
+
this.scheduleRestart(active);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
throw error;
|
|
415
|
+
}).finally(() => {
|
|
416
|
+
if (active.startPromise === promise)
|
|
417
|
+
active.startPromise = undefined;
|
|
418
|
+
});
|
|
419
|
+
active.startPromise = promise;
|
|
420
|
+
return promise;
|
|
421
|
+
}
|
|
422
|
+
async runStartEntry(active) {
|
|
423
|
+
if (this.stopped || active.stopping || active.intentionalStop)
|
|
424
|
+
return;
|
|
425
|
+
const generation = active.generation;
|
|
320
426
|
if (active.pid && platform.isProcessRunning(active.pid)) {
|
|
321
|
-
if (startTimeMatches(active.startedAt, getProcessStartTime(active.pid))) {
|
|
427
|
+
if (active.adoptionAllowed !== false && startTimeMatches(active.startedAt, getProcessStartTime(active.pid))) {
|
|
322
428
|
const healthy = await checkHealth(active.config, active.pid);
|
|
429
|
+
if (generation !== active.generation || this.stopped || active.intentionalStop) {
|
|
430
|
+
throw Object.assign(new Error('service start cancelled: ' + active.config.name), { code: 'SERVICE_START_CANCELLED' });
|
|
431
|
+
}
|
|
323
432
|
active.checkedAt = Date.now();
|
|
324
433
|
active.healthy = healthy;
|
|
325
434
|
if (healthy) {
|
|
326
435
|
active.state = 'ready';
|
|
436
|
+
active.lastError = undefined;
|
|
437
|
+
active.lastErrorCode = undefined;
|
|
438
|
+
if (active.recordPath) {
|
|
439
|
+
atomicWriteJson(active.recordPath, {
|
|
440
|
+
owner: 'evolcore-service-supervisor', pid: active.pid, startedAt: active.startedAt,
|
|
441
|
+
port: active.config.port ?? 0, configHash: active.configHash, state: 'ready',
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
this.log.info('[ServiceSupervisor] adopted service=' + active.config.name + ' pid=' + active.pid);
|
|
445
|
+
this.logLifecycle(active, 'service.adopted');
|
|
327
446
|
this.scheduleHealthCheck(active);
|
|
328
447
|
return;
|
|
329
448
|
}
|
|
330
449
|
}
|
|
331
|
-
await this.stopEntry(active, false)
|
|
450
|
+
if (!await this.stopEntry(active, false)) {
|
|
451
|
+
throw Object.assign(new Error('managed process did not exit; refusing to start a replacement: ' + active.config.name), { code: 'PROCESS_STOP_FAILED' });
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (active.config.port) {
|
|
455
|
+
const listeners = platform.findProcessByPort(active.config.port);
|
|
456
|
+
if (listeners.length > 0) {
|
|
457
|
+
active.portConflictPids = listeners;
|
|
458
|
+
const error = Object.assign(new Error('configured service port is owned by another process: ' + active.config.port + ' (pid=' + listeners.join(',') + ')'), { code: 'PORT_IN_USE' });
|
|
459
|
+
active.lastError = error.message;
|
|
460
|
+
active.lastErrorCode = error.code;
|
|
461
|
+
active.state = 'failed';
|
|
462
|
+
active.healthy = undefined;
|
|
463
|
+
active.checkedAt = undefined;
|
|
464
|
+
this.log.error('[ServiceSupervisor] start blocked service=' + active.config.name + ' code=PORT_IN_USE port=' + active.config.port + ' pids=' + listeners.join(','));
|
|
465
|
+
this.logLifecycle(active, 'service.start.blocked', { code: error.code, port: active.config.port, listenerPids: listeners });
|
|
466
|
+
throw error;
|
|
467
|
+
}
|
|
332
468
|
}
|
|
333
469
|
const command = active.config.runtime?.command;
|
|
334
470
|
if (!command)
|
|
@@ -341,7 +477,10 @@ export class ServiceSupervisor {
|
|
|
341
477
|
active.intentionalStop = false;
|
|
342
478
|
active.state = 'starting';
|
|
343
479
|
active.lastError = undefined;
|
|
480
|
+
active.lastErrorCode = undefined;
|
|
344
481
|
const run = async () => {
|
|
482
|
+
active.attemptId = crypto.randomUUID();
|
|
483
|
+
this.logLifecycle(active, 'service.starting');
|
|
345
484
|
const logDir = path.join(resolvePaths().logs, 'services');
|
|
346
485
|
fs.mkdirSync(logDir, { recursive: true });
|
|
347
486
|
const logPath = path.join(logDir, active.config.name + '.log');
|
|
@@ -379,8 +518,10 @@ export class ServiceSupervisor {
|
|
|
379
518
|
if (!pid)
|
|
380
519
|
throw Object.assign(new Error('failed to obtain PID for ' + active.config.name), { code: 'PROCESS_START_FAILED' });
|
|
381
520
|
active.pid = pid;
|
|
382
|
-
active.startedAt = getProcessStartTime(pid) ??
|
|
521
|
+
active.startedAt = getProcessStartTime(pid) ?? 0;
|
|
383
522
|
active.recordPath = recordPath(active.config.name, pid);
|
|
523
|
+
active.adoptionAllowed = true;
|
|
524
|
+
active.portConflictPids = undefined;
|
|
384
525
|
active.generation += 1;
|
|
385
526
|
child.once('exit', (code, signal) => {
|
|
386
527
|
if (active.process !== child)
|
|
@@ -401,10 +542,30 @@ export class ServiceSupervisor {
|
|
|
401
542
|
return;
|
|
402
543
|
}
|
|
403
544
|
active.lastError = 'process exited (code=' + (code ?? 'null') + ', signal=' + (signal ?? 'null') + ')';
|
|
545
|
+
active.lastErrorCode = 'PROCESS_EXITED';
|
|
546
|
+
this.logLifecycle(active, 'service.exited', { pid, exitCode: code, signal });
|
|
547
|
+
this.log.warn('[ServiceSupervisor] exited service=' + active.config.name + ' pid=' + pid + ' code=' + (code ?? 'null') + ' signal=' + (signal ?? 'null'));
|
|
404
548
|
active.state = 'failed';
|
|
549
|
+
// The process may exit while the initial readiness promise is still
|
|
550
|
+
// pending. scheduleRestart already deduplicates timers and must run
|
|
551
|
+
// for that case as well.
|
|
405
552
|
this.scheduleRestart(active);
|
|
406
553
|
});
|
|
554
|
+
fs.mkdirSync(resolvePaths().instanceDir, { recursive: true });
|
|
555
|
+
atomicWriteJson(active.recordPath, {
|
|
556
|
+
owner: 'evolcore-service-supervisor',
|
|
557
|
+
pid,
|
|
558
|
+
startedAt: active.startedAt,
|
|
559
|
+
port: active.config.port ?? 0,
|
|
560
|
+
configHash: active.configHash,
|
|
561
|
+
state: 'starting',
|
|
562
|
+
attemptId: active.attemptId,
|
|
563
|
+
});
|
|
564
|
+
this.logLifecycle(active, 'service.spawned', { startedAt: active.startedAt });
|
|
407
565
|
child.unref();
|
|
566
|
+
if (!active.startedAt) {
|
|
567
|
+
throw Object.assign(new Error('cannot verify newly spawned process identity: ' + active.config.name), { code: 'PROCESS_IDENTITY_UNVERIFIED' });
|
|
568
|
+
}
|
|
408
569
|
await this.waitForHealth(active);
|
|
409
570
|
if (active.stopping || this.stopped || !platform.isProcessRunning(pid))
|
|
410
571
|
return;
|
|
@@ -415,46 +576,41 @@ export class ServiceSupervisor {
|
|
|
415
576
|
startedAt: active.startedAt,
|
|
416
577
|
port: active.config.port ?? 0,
|
|
417
578
|
configHash: active.configHash,
|
|
579
|
+
state: 'ready',
|
|
580
|
+
attemptId: active.attemptId,
|
|
418
581
|
});
|
|
419
582
|
active.state = 'ready';
|
|
420
583
|
this.scheduleHealthCheck(active);
|
|
421
584
|
this.log.info('[ServiceSupervisor] ready service=' + active.config.name + ' pid=' + pid);
|
|
585
|
+
this.logLifecycle(active, 'service.ready');
|
|
422
586
|
};
|
|
423
|
-
|
|
424
|
-
if (active.intentionalStop || this.stopped || error?.code === 'SERVICE_START_CANCELLED')
|
|
425
|
-
throw error;
|
|
426
|
-
active.lastError = error instanceof Error ? error.message : String(error);
|
|
427
|
-
if (!active.restartTimer)
|
|
428
|
-
active.state = 'failed';
|
|
429
|
-
this.log.warn('[ServiceSupervisor] start failed service=' + active.config.name + ': ' + active.lastError);
|
|
430
|
-
if (active.process?.pid)
|
|
431
|
-
await this.stopEntry(active, false);
|
|
432
|
-
this.scheduleRestart(active);
|
|
433
|
-
throw error;
|
|
434
|
-
}).finally(() => { active.startPromise = undefined; });
|
|
435
|
-
return active.startPromise;
|
|
587
|
+
await run();
|
|
436
588
|
}
|
|
437
589
|
async waitForHealth(active) {
|
|
438
|
-
const deadline = Date.now() + (active.config.health?.startupTimeoutMs ??
|
|
590
|
+
const deadline = Date.now() + (active.config.health?.startupTimeoutMs ?? SERVICE_STARTUP_TIMEOUT_MS);
|
|
439
591
|
const interval = SERVICE_HEALTH_INTERVALS[active.config.health?.type ?? 'tcp'].startup;
|
|
440
592
|
const generation = active.generation;
|
|
441
593
|
do {
|
|
442
594
|
if (generation !== active.generation || active.intentionalStop || this.stopped) {
|
|
443
595
|
throw Object.assign(new Error('service start cancelled: ' + active.config.name), { code: 'SERVICE_START_CANCELLED' });
|
|
444
596
|
}
|
|
445
|
-
if (active.stopping || this.stopped || !platform.isProcessRunning(active.pid)) {
|
|
446
|
-
throw Object.assign(new Error('service exited before ready: ' + active.config.name), { code: 'READINESS_TIMEOUT' });
|
|
447
|
-
}
|
|
448
597
|
if (active.config.port) {
|
|
449
598
|
const listeners = platform.findProcessByPort(active.config.port);
|
|
450
599
|
if (listeners.length > 0 && !listeners.includes(active.pid)) {
|
|
451
|
-
|
|
600
|
+
active.portConflictPids = listeners;
|
|
601
|
+
throw Object.assign(new Error('configured service port is owned by another process: ' + active.config.port + ' (pid=' + listeners.join(',') + ')'), { code: 'PORT_IN_USE' });
|
|
452
602
|
}
|
|
453
603
|
}
|
|
604
|
+
if (active.stopping || this.stopped || !platform.isProcessRunning(active.pid)) {
|
|
605
|
+
throw Object.assign(new Error('service exited before ready: ' + active.config.name), { code: 'READINESS_TIMEOUT' });
|
|
606
|
+
}
|
|
454
607
|
const healthy = await checkHealth(active.config, active.pid);
|
|
455
608
|
if (generation !== active.generation || active.intentionalStop || this.stopped) {
|
|
456
609
|
throw Object.assign(new Error('service start cancelled: ' + active.config.name), { code: 'SERVICE_START_CANCELLED' });
|
|
457
610
|
}
|
|
611
|
+
if (!platform.isProcessRunning(active.pid)) {
|
|
612
|
+
throw Object.assign(new Error('service exited before ready: ' + active.config.name), { code: 'PROCESS_EXITED' });
|
|
613
|
+
}
|
|
458
614
|
if (healthy) {
|
|
459
615
|
active.healthy = true;
|
|
460
616
|
active.checkedAt = Date.now();
|
|
@@ -469,12 +625,26 @@ export class ServiceSupervisor {
|
|
|
469
625
|
active.generation += 1;
|
|
470
626
|
active.state = 'reloading';
|
|
471
627
|
this.log.info('[ServiceSupervisor] restart service=' + active.config.name + ' reason=' + reason);
|
|
472
|
-
await this.stopEntry(active, false)
|
|
628
|
+
if (!await this.stopEntry(active, false)) {
|
|
629
|
+
throw Object.assign(new Error('managed process did not exit; refusing to start a replacement: ' + active.config.name), { code: 'PROCESS_STOP_FAILED' });
|
|
630
|
+
}
|
|
473
631
|
await active.startPromise?.catch(() => { });
|
|
474
632
|
active.stopping = false;
|
|
475
633
|
await this.startEntry(active);
|
|
476
634
|
}
|
|
477
|
-
|
|
635
|
+
stopEntry(active, intentional) {
|
|
636
|
+
if (intentional)
|
|
637
|
+
active.intentionalStop = true;
|
|
638
|
+
if (active.stopPromise)
|
|
639
|
+
return active.stopPromise;
|
|
640
|
+
const promise = this.runStopEntry(active, intentional).finally(() => {
|
|
641
|
+
if (active.stopPromise === promise)
|
|
642
|
+
active.stopPromise = undefined;
|
|
643
|
+
});
|
|
644
|
+
active.stopPromise = promise;
|
|
645
|
+
return promise;
|
|
646
|
+
}
|
|
647
|
+
async runStopEntry(active, intentional) {
|
|
478
648
|
active.intentionalStop = intentional;
|
|
479
649
|
active.stopping = true;
|
|
480
650
|
active.generation += 1;
|
|
@@ -487,18 +657,42 @@ export class ServiceSupervisor {
|
|
|
487
657
|
active.restartTimer = undefined;
|
|
488
658
|
const pid = active.pid;
|
|
489
659
|
const expectedStartedAt = active.startedAt;
|
|
490
|
-
if (pid > 0 && platform.isProcessRunning(pid)
|
|
491
|
-
|
|
492
|
-
|
|
660
|
+
if (pid > 0 && platform.isProcessRunning(pid)) {
|
|
661
|
+
const actualStartedAt = getProcessStartTime(pid);
|
|
662
|
+
if (actualStartedAt === null) {
|
|
663
|
+
active.state = 'failed';
|
|
664
|
+
active.lastError = 'cannot verify managed process identity; refusing to stop or replace it (pid=' + pid + ')';
|
|
665
|
+
active.lastErrorCode = 'PROCESS_IDENTITY_UNVERIFIED';
|
|
666
|
+
active.stopping = false;
|
|
667
|
+
this.log.error('[ServiceSupervisor] stop blocked service=' + active.config.name + ' pid=' + pid + ' reason=identity-unverified');
|
|
668
|
+
return false;
|
|
493
669
|
}
|
|
494
|
-
|
|
495
|
-
const exited = await platform.waitForProcessExit(pid, active.config.runtime?.stopTimeoutMs ?? 10000);
|
|
496
|
-
if (!exited && startTimeMatches(expectedStartedAt, getProcessStartTime(pid))) {
|
|
670
|
+
if (startTimeMatches(expectedStartedAt, actualStartedAt)) {
|
|
497
671
|
try {
|
|
498
|
-
platform.killProcessTree(pid,
|
|
672
|
+
platform.killProcessTree(pid, false);
|
|
499
673
|
}
|
|
500
674
|
catch { }
|
|
501
|
-
await platform.waitForProcessExit(pid,
|
|
675
|
+
let exited = await platform.waitForProcessExit(pid, active.config.runtime?.stopTimeoutMs ?? 10000);
|
|
676
|
+
if (!exited && startTimeMatches(expectedStartedAt, getProcessStartTime(pid))) {
|
|
677
|
+
try {
|
|
678
|
+
platform.killProcessTree(pid, true);
|
|
679
|
+
}
|
|
680
|
+
catch { }
|
|
681
|
+
exited = await platform.waitForProcessExit(pid, 5000);
|
|
682
|
+
}
|
|
683
|
+
const remainingStartedAt = !exited && platform.isProcessRunning(pid) ? getProcessStartTime(pid) : undefined;
|
|
684
|
+
if (remainingStartedAt === null || (remainingStartedAt !== undefined && startTimeMatches(expectedStartedAt, remainingStartedAt))) {
|
|
685
|
+
active.state = 'failed';
|
|
686
|
+
active.lastError = 'process did not exit after graceful and forced termination (pid=' + pid + ')';
|
|
687
|
+
active.lastErrorCode = 'PROCESS_STOP_FAILED';
|
|
688
|
+
active.stopping = false;
|
|
689
|
+
this.log.error('[ServiceSupervisor] stop failed service=' + active.config.name + ' pid=' + pid);
|
|
690
|
+
this.logLifecycle(active, 'service.stop.failed', { code: active.lastErrorCode });
|
|
691
|
+
return false;
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
else {
|
|
695
|
+
this.log.warn('[ServiceSupervisor] recorded PID has been reused; refusing to signal unrelated process: service=' + active.config.name + ' pid=' + pid);
|
|
502
696
|
}
|
|
503
697
|
}
|
|
504
698
|
if (active.recordPath) {
|
|
@@ -512,12 +706,17 @@ export class ServiceSupervisor {
|
|
|
512
706
|
active.startedAt = 0;
|
|
513
707
|
active.healthy = false;
|
|
514
708
|
active.stopping = false;
|
|
515
|
-
active.state =
|
|
709
|
+
active.state = active.intentionalStop ? 'disabled' : 'starting';
|
|
710
|
+
this.logLifecycle(active, 'service.stopped', { pid });
|
|
711
|
+
return true;
|
|
516
712
|
}
|
|
517
713
|
scheduleRestart(active) {
|
|
518
714
|
if (this.stopped || active.intentionalStop || active.restartTimer || active.config.enabled !== true
|
|
519
715
|
|| active.config.runtime?.mode !== 'managed')
|
|
520
716
|
return;
|
|
717
|
+
if (active.lastErrorCode === 'PORT_IN_USE' || active.lastErrorCode === 'PROCESS_STOP_FAILED'
|
|
718
|
+
|| active.lastErrorCode === 'PROCESS_IDENTITY_UNVERIFIED')
|
|
719
|
+
return;
|
|
521
720
|
const restart = active.config.runtime?.restart;
|
|
522
721
|
if (restart?.enabled === false)
|
|
523
722
|
return;
|
|
@@ -528,8 +727,8 @@ export class ServiceSupervisor {
|
|
|
528
727
|
if (active.failures.length >= SERVICE_RESTART_MAX_ATTEMPTS) {
|
|
529
728
|
active.retryExhausted = true;
|
|
530
729
|
active.state = 'failed';
|
|
531
|
-
active.lastError = 'restart limit reached; use reset-failure before retrying';
|
|
532
730
|
this.log.error('[ServiceSupervisor] restart limit reached service=' + active.config.name);
|
|
731
|
+
this.logLifecycle(active, 'service.recovery.exhausted', { cause: active.lastError, code: active.lastErrorCode });
|
|
533
732
|
return;
|
|
534
733
|
}
|
|
535
734
|
active.failures.push(now);
|
|
@@ -540,12 +739,17 @@ export class ServiceSupervisor {
|
|
|
540
739
|
active.restartTimer = setTimeout(() => {
|
|
541
740
|
active.restartTimer = undefined;
|
|
542
741
|
active.restartCount += 1;
|
|
543
|
-
void this.
|
|
742
|
+
void this.queueOperation(active.config.name, async () => {
|
|
743
|
+
if (this.stopped || active.intentionalStop || this.active.get(active.config.name) !== active)
|
|
744
|
+
return;
|
|
745
|
+
await this.restartEntry(active, 'automatic recovery');
|
|
746
|
+
}).catch(error => this.log.warn('[ServiceSupervisor] automatic recovery failed service=' + active.config.name + ': ' + String(error)));
|
|
544
747
|
}, delay);
|
|
545
748
|
active.restartTimer.unref?.();
|
|
546
749
|
}
|
|
547
750
|
scheduleHealthCheck(active) {
|
|
548
|
-
if (this.stopped || active.intentionalStop
|
|
751
|
+
if (this.stopped || active.intentionalStop
|
|
752
|
+
|| (!active.pid && serviceRuntimeMode(active.config) !== 'external' && active.lastErrorCode !== 'PORT_IN_USE') || active.healthTimer
|
|
549
753
|
|| (active.state !== 'ready' && active.state !== 'failed'
|
|
550
754
|
&& !(active.state === 'starting' && serviceRuntimeMode(active.config) === 'external')))
|
|
551
755
|
return;
|
|
@@ -566,6 +770,28 @@ export class ServiceSupervisor {
|
|
|
566
770
|
active.healthCheckInFlight = true;
|
|
567
771
|
const generation = active.generation;
|
|
568
772
|
try {
|
|
773
|
+
if (serviceRuntimeMode(active.config) === 'managed' && !active.pid && active.lastErrorCode === 'PORT_IN_USE') {
|
|
774
|
+
const healthy = await checkHealth(active.config, 0);
|
|
775
|
+
if (generation !== active.generation || active.stopping || this.stopped)
|
|
776
|
+
return;
|
|
777
|
+
active.healthy = healthy;
|
|
778
|
+
active.checkedAt = Date.now();
|
|
779
|
+
const listeners = active.config.port ? platform.findProcessByPort(active.config.port) : [];
|
|
780
|
+
active.portConflictPids = listeners.length ? listeners : undefined;
|
|
781
|
+
if (listeners.length === 0 && active.config.runtime?.restart?.enabled !== false) {
|
|
782
|
+
active.failures = [];
|
|
783
|
+
active.failureCount = 0;
|
|
784
|
+
active.retryExhausted = false;
|
|
785
|
+
active.lastError = undefined;
|
|
786
|
+
active.lastErrorCode = undefined;
|
|
787
|
+
void this.queueOperation(active.config.name, async () => {
|
|
788
|
+
if (this.stopped || active.intentionalStop || this.active.get(active.config.name) !== active)
|
|
789
|
+
return;
|
|
790
|
+
await this.startEntry(active);
|
|
791
|
+
}).catch(error => this.log.warn('[ServiceSupervisor] conflict recovery failed service=' + active.config.name + ': ' + String(error)));
|
|
792
|
+
}
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
569
795
|
const healthy = (serviceRuntimeMode(active.config) === 'external'
|
|
570
796
|
|| startTimeMatches(active.startedAt, getProcessStartTime(active.pid)))
|
|
571
797
|
&& await checkHealth(active.config, active.pid);
|
|
@@ -600,11 +826,21 @@ export class ServiceSupervisor {
|
|
|
600
826
|
...(active.startedAt ? { startedAt: active.startedAt } : {}),
|
|
601
827
|
...(active.healthy === undefined ? {} : { healthy: active.healthy }),
|
|
602
828
|
...(active.checkedAt ? { checkedAt: active.checkedAt } : {}),
|
|
829
|
+
endpointHealth: active.healthy === undefined ? 'unknown' : active.healthy ? 'healthy' : 'unhealthy',
|
|
603
830
|
failureCount: active.failureCount,
|
|
604
831
|
restartCount: active.restartCount,
|
|
605
832
|
...(active.lastError ? { lastError: active.lastError } : {}),
|
|
833
|
+
...(active.lastErrorCode ? { lastErrorCode: active.lastErrorCode } : {}),
|
|
834
|
+
recoveryState: active.retryExhausted ? 'exhausted' : active.state === 'backoff' ? 'backoff'
|
|
835
|
+
: active.lastErrorCode === 'PORT_IN_USE' || active.lastErrorCode === 'PROCESS_STOP_FAILED'
|
|
836
|
+
|| active.lastErrorCode === 'PROCESS_IDENTITY_UNVERIFIED' ? 'blocked' : 'idle',
|
|
606
837
|
generation: active.generation,
|
|
607
838
|
managed: active.config.runtime?.mode === 'managed',
|
|
839
|
+
processOwnership: active.config.runtime?.mode === 'external'
|
|
840
|
+
? 'external'
|
|
841
|
+
: active.pid > 0 && active.startedAt > 0 && active.adoptionAllowed !== false ? 'managed'
|
|
842
|
+
: active.pid > 0 || active.portConflictPids?.length ? 'unknown' : 'none',
|
|
843
|
+
...(active.portConflictPids?.length ? { endpointOwnerPids: active.portConflictPids } : {}),
|
|
608
844
|
};
|
|
609
845
|
}
|
|
610
846
|
}
|