@upx-us/shield 0.7.13 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -220
- package/README.md +47 -30
- package/dist/index.d.ts +64 -1
- package/dist/index.js +495 -133
- package/dist/src/case-monitor.d.ts +10 -1
- package/dist/src/case-monitor.js +50 -10
- package/dist/src/config.js +1 -1
- package/dist/src/events/browser/validations.js +8 -4
- package/dist/src/sender.d.ts +4 -0
- package/dist/src/sender.js +17 -12
- package/dist/src/updater.d.ts +7 -0
- package/dist/src/updater.js +66 -10
- package/openclaw.plugin.json +3 -3
- package/package.json +2 -2
- package/skills/shield/SKILL.md +14 -6
package/dist/index.js
CHANGED
|
@@ -39,6 +39,7 @@ exports.maskPluginConfigForLogs = maskPluginConfigForLogs;
|
|
|
39
39
|
exports.createSingleflightRunner = createSingleflightRunner;
|
|
40
40
|
exports.createStartGuard = createStartGuard;
|
|
41
41
|
exports.getStatusWarnings = getStatusWarnings;
|
|
42
|
+
exports._resetForTesting = _resetForTesting;
|
|
42
43
|
const config_1 = require("./src/config");
|
|
43
44
|
const log_1 = require("./src/log");
|
|
44
45
|
const log = __importStar(require("./src/log"));
|
|
@@ -231,6 +232,82 @@ function flushAllTimeStats() {
|
|
|
231
232
|
}
|
|
232
233
|
let _stateDirty = true;
|
|
233
234
|
function markStateDirty() { _stateDirty = true; }
|
|
235
|
+
function createInitialState() {
|
|
236
|
+
return {
|
|
237
|
+
activated: false,
|
|
238
|
+
running: false,
|
|
239
|
+
startedAt: 0,
|
|
240
|
+
lastPollAt: 0,
|
|
241
|
+
lastSuccessfulPollAt: 0,
|
|
242
|
+
eventsProcessed: 0,
|
|
243
|
+
quarantineCount: 0,
|
|
244
|
+
consecutiveFailures: 0,
|
|
245
|
+
telemetryConsecutiveFailures: 0,
|
|
246
|
+
instanceId: '',
|
|
247
|
+
lastCaptureAt: 0,
|
|
248
|
+
captureSeenSinceLastSync: false,
|
|
249
|
+
lastSync: null,
|
|
250
|
+
lastTelemetryAt: 0,
|
|
251
|
+
lastSuccessfulTelemetryAt: 0,
|
|
252
|
+
lastTelemetryError: null,
|
|
253
|
+
lastPollError: null,
|
|
254
|
+
lastLifecyclePhase: 'idle',
|
|
255
|
+
lastStopReason: 'none',
|
|
256
|
+
lastError: null,
|
|
257
|
+
lastStartupCheckpoint: null,
|
|
258
|
+
sessionDirCount: 0,
|
|
259
|
+
eventsRetainedForRetry: 0,
|
|
260
|
+
lastDeliveryIssue: null,
|
|
261
|
+
lastDeliveryIssueAt: 0,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
function setLifecyclePhase(phase, checkpoint) {
|
|
265
|
+
state.lastLifecyclePhase = phase;
|
|
266
|
+
if (checkpoint !== undefined)
|
|
267
|
+
state.lastStartupCheckpoint = checkpoint;
|
|
268
|
+
markStateDirty();
|
|
269
|
+
}
|
|
270
|
+
function setLastError(message) {
|
|
271
|
+
state.lastError = message;
|
|
272
|
+
markStateDirty();
|
|
273
|
+
}
|
|
274
|
+
function setStopReason(reason) {
|
|
275
|
+
state.lastStopReason = reason;
|
|
276
|
+
markStateDirty();
|
|
277
|
+
}
|
|
278
|
+
function hasMeaningfulRuntimeState(snapshot) {
|
|
279
|
+
return Boolean(snapshot.running ||
|
|
280
|
+
snapshot.startedAt ||
|
|
281
|
+
snapshot.lastPollAt ||
|
|
282
|
+
snapshot.lastSuccessfulPollAt ||
|
|
283
|
+
snapshot.eventsProcessed ||
|
|
284
|
+
snapshot.quarantineCount ||
|
|
285
|
+
snapshot.lastTelemetryAt ||
|
|
286
|
+
snapshot.lastSuccessfulTelemetryAt ||
|
|
287
|
+
snapshot.lastStartupCheckpoint ||
|
|
288
|
+
snapshot.instanceId);
|
|
289
|
+
}
|
|
290
|
+
function hasHealthyRuntimeState(snapshot) {
|
|
291
|
+
return Boolean(snapshot.lastSuccessfulPollAt || snapshot.lastSync?.at);
|
|
292
|
+
}
|
|
293
|
+
function buildLastKnownGoodSnapshot(updatedAt) {
|
|
294
|
+
return {
|
|
295
|
+
pid: process.pid,
|
|
296
|
+
startedAt: state.startedAt,
|
|
297
|
+
lastPollAt: state.lastPollAt,
|
|
298
|
+
lastSuccessfulPollAt: state.lastSuccessfulPollAt,
|
|
299
|
+
lastCaptureAt: state.lastCaptureAt,
|
|
300
|
+
lastSync: state.lastSync,
|
|
301
|
+
instanceId: state.instanceId,
|
|
302
|
+
eventsProcessed: state.eventsProcessed,
|
|
303
|
+
quarantineCount: state.quarantineCount,
|
|
304
|
+
updatedAt,
|
|
305
|
+
version: version_1.VERSION,
|
|
306
|
+
lastLifecyclePhase: state.lastLifecyclePhase,
|
|
307
|
+
lastStopReason: state.lastStopReason,
|
|
308
|
+
lastStartupCheckpoint: state.lastStartupCheckpoint,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
234
311
|
function persistState(extra = {}) {
|
|
235
312
|
flushAllTimeStats();
|
|
236
313
|
if (!_stateDirty)
|
|
@@ -239,6 +316,7 @@ function persistState(extra = {}) {
|
|
|
239
316
|
const dir = (0, path_1.join)((0, os_1.homedir)(), '.openclaw', 'shield', 'data');
|
|
240
317
|
if (!(0, fs_1.existsSync)(dir))
|
|
241
318
|
(0, fs_1.mkdirSync)(dir, { recursive: true });
|
|
319
|
+
const previous = (0, safe_io_1.readJsonSafe)(STATUS_FILE, {}, 'status-merge');
|
|
242
320
|
let countersSnapshot = {};
|
|
243
321
|
try {
|
|
244
322
|
countersSnapshot = {
|
|
@@ -249,13 +327,22 @@ function persistState(extra = {}) {
|
|
|
249
327
|
};
|
|
250
328
|
}
|
|
251
329
|
catch { }
|
|
252
|
-
|
|
330
|
+
const updatedAt = Date.now();
|
|
331
|
+
const mergedCurrent = {
|
|
253
332
|
...state, ...extra,
|
|
254
333
|
version: version_1.VERSION,
|
|
255
|
-
updatedAt
|
|
334
|
+
updatedAt,
|
|
256
335
|
pid: process.pid,
|
|
257
336
|
counters: countersSnapshot,
|
|
258
337
|
allTime: readAllTimeStats(),
|
|
338
|
+
};
|
|
339
|
+
const previousLastKnownGood = previous.lastKnownGood ?? null;
|
|
340
|
+
const nextLastKnownGood = hasHealthyRuntimeState(state)
|
|
341
|
+
? buildLastKnownGoodSnapshot(updatedAt)
|
|
342
|
+
: previousLastKnownGood;
|
|
343
|
+
(0, safe_io_1.writeJsonSafe)(STATUS_FILE, {
|
|
344
|
+
...mergedCurrent,
|
|
345
|
+
lastKnownGood: nextLastKnownGood,
|
|
259
346
|
});
|
|
260
347
|
_stateDirty = false;
|
|
261
348
|
}
|
|
@@ -264,15 +351,27 @@ function persistState(extra = {}) {
|
|
|
264
351
|
function readPersistedState() {
|
|
265
352
|
try {
|
|
266
353
|
const d = (0, safe_io_1.readJsonSafe)(STATUS_FILE, null, 'status');
|
|
267
|
-
if (!d)
|
|
268
|
-
|
|
354
|
+
if (!d) {
|
|
355
|
+
if (hasMeaningfulRuntimeState(state)) {
|
|
356
|
+
return { data: state, source: 'runtime', message: null };
|
|
357
|
+
}
|
|
358
|
+
return { data: null, source: 'missing', message: 'No persisted Shield runtime state found yet.' };
|
|
359
|
+
}
|
|
269
360
|
const age = Date.now() - (Number(d.updatedAt) || 0);
|
|
270
|
-
if (age > 10 * 60 * 1000)
|
|
271
|
-
return
|
|
272
|
-
|
|
361
|
+
if (age > 10 * 60 * 1000) {
|
|
362
|
+
return {
|
|
363
|
+
data: d,
|
|
364
|
+
source: 'stale',
|
|
365
|
+
message: `Last persisted Shield state is stale (${Math.floor(age / 60_000)}m old).`,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
return { data: d, source: 'persisted', message: null };
|
|
273
369
|
}
|
|
274
370
|
catch {
|
|
275
|
-
|
|
371
|
+
if (hasMeaningfulRuntimeState(state)) {
|
|
372
|
+
return { data: state, source: 'runtime', message: null };
|
|
373
|
+
}
|
|
374
|
+
return { data: null, source: 'missing', message: 'Shield status state could not be read.' };
|
|
276
375
|
}
|
|
277
376
|
}
|
|
278
377
|
const STALE_POLL_WARN_MS = 2 * 60 * 1000;
|
|
@@ -304,19 +403,7 @@ function getStatusWarnings(input) {
|
|
|
304
403
|
}
|
|
305
404
|
return warnings;
|
|
306
405
|
}
|
|
307
|
-
const state =
|
|
308
|
-
activated: false,
|
|
309
|
-
running: false,
|
|
310
|
-
startedAt: 0,
|
|
311
|
-
lastPollAt: 0,
|
|
312
|
-
eventsProcessed: 0,
|
|
313
|
-
quarantineCount: 0,
|
|
314
|
-
consecutiveFailures: 0,
|
|
315
|
-
instanceId: '',
|
|
316
|
-
lastCaptureAt: 0,
|
|
317
|
-
captureSeenSinceLastSync: false,
|
|
318
|
-
lastSync: null,
|
|
319
|
-
};
|
|
406
|
+
const state = createInitialState();
|
|
320
407
|
let firstEventDelivered = false;
|
|
321
408
|
let teardownPreviousRuntime = null;
|
|
322
409
|
let pendingTeardown = null;
|
|
@@ -330,6 +417,26 @@ function getBackoffInterval(baseMs) {
|
|
|
330
417
|
const backoff = baseMs * Math.pow(2, Math.min(state.consecutiveFailures, 10));
|
|
331
418
|
return Math.min(backoff, MAX_BACKOFF_MS);
|
|
332
419
|
}
|
|
420
|
+
function describeSendFailure(kind, statusCode, body) {
|
|
421
|
+
switch (kind) {
|
|
422
|
+
case 'missing_credentials':
|
|
423
|
+
return 'Missing Shield credentials';
|
|
424
|
+
case 'pending_namespace':
|
|
425
|
+
return 'Namespace allocation pending';
|
|
426
|
+
case 'needs_registration':
|
|
427
|
+
return 'Instance registration invalidated by platform';
|
|
428
|
+
case 'quota_exceeded':
|
|
429
|
+
return 'Subscription inactive or quota exhausted';
|
|
430
|
+
case 'circuit_breaker':
|
|
431
|
+
return 'Circuit breaker opened after repeated batch failures';
|
|
432
|
+
case 'network_error':
|
|
433
|
+
return `Network error while sending events${body ? `: ${body}` : ''}`;
|
|
434
|
+
case 'http_error':
|
|
435
|
+
return `HTTP ${statusCode ?? 0} while sending events${body ? `: ${body}` : ''}`;
|
|
436
|
+
default:
|
|
437
|
+
return body || `HTTP ${statusCode ?? 0} while sending events`;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
333
440
|
function printNotActivatedStatus() {
|
|
334
441
|
console.log(`OpenClaw Shield — v${version_1.VERSION}`);
|
|
335
442
|
console.log('');
|
|
@@ -344,50 +451,131 @@ function printNotActivatedStatus() {
|
|
|
344
451
|
console.log(' Get your key at: https://uss.upx.com → APPS → OpenClaw Shield');
|
|
345
452
|
}
|
|
346
453
|
function printActivatedStatus() {
|
|
347
|
-
const
|
|
454
|
+
const snapshot = readPersistedState();
|
|
455
|
+
const s = snapshot.data;
|
|
456
|
+
const fmtTime = (ms) => ms < 60_000 ? `${Math.round(ms / 1000)}s ago`
|
|
457
|
+
: ms < 3_600_000 ? `${Math.floor(ms / 60_000)}m ago`
|
|
458
|
+
: `${(ms / 3_600_000).toFixed(1)}h ago`;
|
|
459
|
+
if (!s) {
|
|
460
|
+
console.log(`OpenClaw Shield — v${version_1.VERSION}`);
|
|
461
|
+
console.log('');
|
|
462
|
+
console.log('── Status ────────────────────────────────────');
|
|
463
|
+
console.log(' ⚠️ Unknown · Status unavailable');
|
|
464
|
+
console.log(` Version: ${version_1.VERSION}`);
|
|
465
|
+
console.log(` Note: ${snapshot.message ?? 'Shield has not persisted runtime state yet.'}`);
|
|
466
|
+
console.log(' - This usually means the gateway has not started the plugin yet, or state persistence is unavailable.');
|
|
467
|
+
console.log(' - If this persists, run: openclaw gateway restart');
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
348
470
|
const isRunning = Boolean(s.running);
|
|
349
471
|
const ageMs = s.updatedAt ? Date.now() - s.updatedAt : null;
|
|
350
|
-
const ageLabel = ageMs != null
|
|
351
|
-
? (ageMs < 60_000 ? `${Math.round(ageMs / 1000)}s ago`
|
|
352
|
-
: ageMs < 3_600_000 ? `${Math.floor(ageMs / 60_000)}m ago`
|
|
353
|
-
: `${(ageMs / 3_600_000).toFixed(1)}h ago`)
|
|
354
|
-
: '';
|
|
472
|
+
const ageLabel = ageMs != null ? fmtTime(ageMs) : '';
|
|
355
473
|
const lastPollMs = s.lastPollAt ? Date.now() - s.lastPollAt : null;
|
|
356
|
-
const lastPollLabel = s.lastPollAt
|
|
357
|
-
? (lastPollMs < 60_000 ? `${Math.round(lastPollMs / 1000)}s ago`
|
|
358
|
-
: lastPollMs < 3_600_000 ? `${Math.floor(lastPollMs / 60_000)}m ago`
|
|
359
|
-
: `${(lastPollMs / 3_600_000).toFixed(1)}h ago`)
|
|
360
|
-
: 'never';
|
|
474
|
+
const lastPollLabel = s.lastPollAt ? fmtTime(lastPollMs) : 'never';
|
|
361
475
|
const instanceId = s.instanceId;
|
|
362
476
|
const shortId = instanceId ? `${instanceId.slice(0, 8)}…` : '';
|
|
477
|
+
const lifecyclePhase = s.lastLifecyclePhase ?? 'unknown';
|
|
478
|
+
const stopReason = s.lastStopReason ?? 'unknown';
|
|
479
|
+
const lastKnownGood = (s.lastKnownGood ?? null);
|
|
363
480
|
console.log(`OpenClaw Shield — v${s.version ?? version_1.VERSION}${ageLabel ? ` (${ageLabel})` : ''}`);
|
|
481
|
+
const cachedUpdate = (0, updater_1.loadUpdateState)();
|
|
364
482
|
console.log('');
|
|
365
|
-
|
|
366
|
-
|
|
483
|
+
const lastCaptureMs = s.lastCaptureAt ? Date.now() - s.lastCaptureAt : null;
|
|
484
|
+
const lastCaptureLabel = s.lastCaptureAt ? fmtTime(lastCaptureMs) : null;
|
|
485
|
+
const updateState = cachedUpdate;
|
|
486
|
+
const allTime = (s.allTime ?? readAllTimeStats());
|
|
487
|
+
const phaseIcon = isRunning ? '✅' : (snapshot.source === 'stale' ? '⚠️' : '❌');
|
|
488
|
+
const phaseDisplay = lifecyclePhase.charAt(0).toUpperCase() + lifecyclePhase.slice(1);
|
|
489
|
+
const connectionDisplay = isRunning ? 'Connected' : snapshot.source === 'stale' ? 'Status stale' : 'Disconnected';
|
|
490
|
+
console.log('── Status ────────────────────────────────────');
|
|
491
|
+
console.log(` ${phaseIcon} ${phaseDisplay} · ${connectionDisplay}`);
|
|
367
492
|
console.log(` Version: ${s.version ?? version_1.VERSION}`);
|
|
368
493
|
if (shortId)
|
|
369
494
|
console.log(` Instance: ${shortId}`);
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
495
|
+
if (snapshot.message)
|
|
496
|
+
console.log(` Note: ${snapshot.message}`);
|
|
497
|
+
if (stopReason && stopReason !== 'none')
|
|
498
|
+
console.log(` Stop reason: ${stopReason}`);
|
|
499
|
+
if (lastKnownGood && !isRunning) {
|
|
500
|
+
const lastGoodPoll = lastKnownGood.lastPollAt ? fmtTime(Date.now() - lastKnownGood.lastPollAt) : 'never';
|
|
501
|
+
const lastGoodCapture = lastKnownGood.lastCaptureAt ? fmtTime(Date.now() - lastKnownGood.lastCaptureAt) : null;
|
|
502
|
+
const lastGoodParts = [`poll ${lastGoodPoll}`];
|
|
503
|
+
if (lastGoodCapture)
|
|
504
|
+
lastGoodParts.push(`capture ${lastGoodCapture}`);
|
|
505
|
+
console.log(` Last healthy: ${lastGoodParts.join(' · ')}`);
|
|
506
|
+
}
|
|
507
|
+
console.log('');
|
|
508
|
+
const telFailures = s.telemetryConsecutiveFailures ?? 0;
|
|
509
|
+
const pollFailures = s.consecutiveFailures ?? 0;
|
|
510
|
+
const sessionMs = s.startedAt ? Date.now() - s.startedAt : null;
|
|
511
|
+
const sessionLabel = sessionMs != null
|
|
512
|
+
? sessionMs < 60_000 ? `${Math.round(sessionMs / 1000)}s`
|
|
513
|
+
: sessionMs < 3_600_000 ? `${Math.floor(sessionMs / 60_000)}m`
|
|
514
|
+
: `${(sessionMs / 3_600_000).toFixed(1)}h`
|
|
376
515
|
: null;
|
|
516
|
+
const lastDataParts = [];
|
|
517
|
+
if (lastPollLabel !== 'never')
|
|
518
|
+
lastDataParts.push(`poll ${lastPollLabel}`);
|
|
377
519
|
if (lastCaptureLabel)
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
console.log(` Events
|
|
381
|
-
console.log(`
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
520
|
+
lastDataParts.push(`capture ${lastCaptureLabel}`);
|
|
521
|
+
console.log('── Monitoring ────────────────────────────────');
|
|
522
|
+
console.log(` Events: ${allTime.eventsProcessed.toLocaleString()} sent · ${allTime.quarantineCount.toLocaleString()} quarantined`);
|
|
523
|
+
console.log(` Failures: ${pollFailures} poll · ${telFailures} telemetry`);
|
|
524
|
+
const checkpointHuman = (() => {
|
|
525
|
+
const cp = s.lastStartupCheckpoint ?? '';
|
|
526
|
+
if (cp.includes('not_activated'))
|
|
527
|
+
return 'Not activated';
|
|
528
|
+
if (cp.includes('activation_failed'))
|
|
529
|
+
return 'Activation failed';
|
|
530
|
+
if (cp.includes('update_restart'))
|
|
531
|
+
return 'Restarting for update';
|
|
532
|
+
if (cp === 'checkpoint:11' || cp === 'checkpoint:9' || cp === 'checkpoint:9a')
|
|
533
|
+
return 'Monitoring active';
|
|
534
|
+
if (cp === 'checkpoint:8')
|
|
535
|
+
return 'Starting up';
|
|
536
|
+
if (cp.includes('checkpoint:') || cp === 'SIGTERM')
|
|
537
|
+
return 'Starting up';
|
|
538
|
+
if (cp.includes('registration_invalid') || cp.includes('poll_registration_invalid'))
|
|
539
|
+
return 'Registration invalid';
|
|
540
|
+
return null;
|
|
541
|
+
})();
|
|
542
|
+
if (sessionLabel)
|
|
543
|
+
console.log(` Session: ${sessionLabel} active${checkpointHuman ? ` (${checkpointHuman})` : ''}`);
|
|
544
|
+
if (lastDataParts.length > 0)
|
|
545
|
+
console.log(` Last data: ${lastDataParts.join(' · ')}`);
|
|
546
|
+
if (lastKnownGood && !isRunning) {
|
|
547
|
+
const lastGoodSync = lastKnownGood.lastSync?.at ? fmtTime(Date.now() - lastKnownGood.lastSync.at) : null;
|
|
548
|
+
const lastGoodSession = lastKnownGood.startedAt ? fmtTime(Date.now() - lastKnownGood.startedAt) : null;
|
|
549
|
+
const lastGoodLabel = [
|
|
550
|
+
lastGoodSession ? `session ${lastGoodSession}` : null,
|
|
551
|
+
lastGoodSync ? `sync ${lastGoodSync}` : null,
|
|
552
|
+
].filter(Boolean).join(' · ');
|
|
553
|
+
if (lastGoodLabel)
|
|
554
|
+
console.log(` Last good: ${lastGoodLabel}`);
|
|
555
|
+
}
|
|
556
|
+
if (s.eventsRetainedForRetry && Number(s.eventsRetainedForRetry) > 0) {
|
|
557
|
+
console.log(` Retry backlog:${String(s.eventsRetainedForRetry).padStart(3)} event(s) retained`);
|
|
558
|
+
}
|
|
559
|
+
if (updateState.pendingRestart || updateState.lastFailureStage || updateState.rollbackPending) {
|
|
560
|
+
console.log(` Update: ${updateState.pendingRestart ? 'pending restart' : updateState.rollbackPending ? 'rollback needed' : 'attention required'}`);
|
|
561
|
+
if (updateState.lastFailureStage)
|
|
562
|
+
console.log(` Update stage: ${updateState.lastFailureStage}`);
|
|
563
|
+
if (updateState.lastError)
|
|
564
|
+
console.log(` Update error: ${updateState.lastError}`);
|
|
565
|
+
}
|
|
566
|
+
if (s.lastError) {
|
|
567
|
+
console.log(` Last error: ${s.lastError}`);
|
|
568
|
+
}
|
|
569
|
+
if (s.lastDeliveryIssue) {
|
|
570
|
+
console.log(` Delivery: ${s.lastDeliveryIssue}`);
|
|
571
|
+
}
|
|
572
|
+
if (cachedUpdate.updateAvailable && cachedUpdate.latestVersion) {
|
|
573
|
+
console.log('');
|
|
574
|
+
console.log(`⚡ Update available: v${cachedUpdate.latestVersion}`);
|
|
575
|
+
console.log(` To update, run:`);
|
|
576
|
+
console.log(` openclaw plugins update shield`);
|
|
577
|
+
console.log(` openclaw gateway restart`);
|
|
578
|
+
console.log(` Or tell your agent: "Update the OpenClaw Shield plugin to the latest version"`);
|
|
391
579
|
}
|
|
392
580
|
const statusWarnings = getStatusWarnings({
|
|
393
581
|
running: isRunning,
|
|
@@ -397,22 +585,24 @@ function printActivatedStatus() {
|
|
|
397
585
|
captureSeenSinceLastSync: Boolean(s.captureSeenSinceLastSync ?? false),
|
|
398
586
|
consecutiveFailures: s.consecutiveFailures ?? 0,
|
|
399
587
|
});
|
|
400
|
-
|
|
588
|
+
const telemetryWarnings = [];
|
|
589
|
+
if (s.telemetryConsecutiveFailures && Number(s.telemetryConsecutiveFailures) >= CONSECUTIVE_FAILURES_WARN) {
|
|
590
|
+
telemetryWarnings.push(`Instance telemetry is failing (${s.telemetryConsecutiveFailures} consecutive attempts), but capture remains active.`);
|
|
591
|
+
}
|
|
592
|
+
if (s.lastTelemetryError) {
|
|
593
|
+
telemetryWarnings.push(`Last telemetry error: ${s.lastTelemetryError}`);
|
|
594
|
+
}
|
|
595
|
+
if (statusWarnings.length > 0 || telemetryWarnings.length > 0 || snapshot.source === 'stale') {
|
|
401
596
|
console.log(' Warning: ⚠️ Capture health degraded');
|
|
402
|
-
for (const warning of statusWarnings) {
|
|
597
|
+
for (const warning of [...statusWarnings, ...telemetryWarnings]) {
|
|
403
598
|
console.log(` - ${warning}`);
|
|
404
599
|
}
|
|
600
|
+
if (snapshot.source === 'stale') {
|
|
601
|
+
console.log(' - Persisted status is stale; the gateway may have restarted or status writes may be blocked.');
|
|
602
|
+
}
|
|
405
603
|
console.log(' - If this persists, run: openclaw gateway restart');
|
|
406
604
|
}
|
|
407
605
|
const startedAt = s.startedAt;
|
|
408
|
-
if (startedAt) {
|
|
409
|
-
const uptimeMs = Date.now() - startedAt;
|
|
410
|
-
const uptimeLabel = uptimeMs < 3_600_000
|
|
411
|
-
? `${Math.floor(uptimeMs / 60_000)}m`
|
|
412
|
-
: `${(uptimeMs / 3_600_000).toFixed(1)}h`;
|
|
413
|
-
console.log(` Session: ${uptimeLabel}`);
|
|
414
|
-
}
|
|
415
|
-
console.log('');
|
|
416
606
|
console.log('── Activity ──────────────────────────────────');
|
|
417
607
|
const BAR_CHARS = '████████████████████';
|
|
418
608
|
const BAR_MAX = 8;
|
|
@@ -422,9 +612,6 @@ function printActivatedStatus() {
|
|
|
422
612
|
const filled = Math.max(1, Math.round((count / max) * BAR_MAX));
|
|
423
613
|
return BAR_CHARS.slice(0, filled);
|
|
424
614
|
};
|
|
425
|
-
const fmtTime = (ms) => ms < 60_000 ? `${Math.round(ms / 1000)}s ago`
|
|
426
|
-
: ms < 3_600_000 ? `${Math.floor(ms / 60_000)}m ago`
|
|
427
|
-
: `${(ms / 3_600_000).toFixed(1)}h ago`;
|
|
428
615
|
const lastSync = s.lastSync;
|
|
429
616
|
console.log('');
|
|
430
617
|
if (lastSync && lastSync.at) {
|
|
@@ -437,7 +624,12 @@ function printActivatedStatus() {
|
|
|
437
624
|
}
|
|
438
625
|
else {
|
|
439
626
|
console.log('📡 Last sync');
|
|
440
|
-
|
|
627
|
+
if (lastKnownGood?.lastSync?.at && !isRunning) {
|
|
628
|
+
console.log(` Current runtime has no sync yet. Last healthy sync was ${fmtTime(Date.now() - lastKnownGood.lastSync.at)}.`);
|
|
629
|
+
}
|
|
630
|
+
else {
|
|
631
|
+
console.log(` ${snapshot.source === 'stale' ? 'Last known sync is unavailable because persisted state is stale.' : 'No sync yet. Bridge will send on the next poll cycle.'}`);
|
|
632
|
+
}
|
|
441
633
|
}
|
|
442
634
|
const counters = (s.counters ?? {});
|
|
443
635
|
const sessionEvents = counters.totalEvents ?? 0;
|
|
@@ -445,10 +637,10 @@ function printActivatedStatus() {
|
|
|
445
637
|
const sessionRows = Object.entries(sessionTypes).sort(([, a], [, b]) => b - a);
|
|
446
638
|
const sessionMax = sessionRows[0]?.[1] ?? 0;
|
|
447
639
|
console.log('');
|
|
448
|
-
const
|
|
640
|
+
const sessionActivityLabel = startedAt
|
|
449
641
|
? `since restart ${fmtTime(Date.now() - startedAt)}`
|
|
450
642
|
: 'this session';
|
|
451
|
-
console.log(`📊 This session (${
|
|
643
|
+
console.log(`📊 This session (${sessionActivityLabel} — ${sessionEvents} event${sessionEvents !== 1 ? 's' : ''})`);
|
|
452
644
|
if (sessionRows.length === 0) {
|
|
453
645
|
console.log(' No events recorded yet.');
|
|
454
646
|
}
|
|
@@ -472,6 +664,16 @@ function printActivatedStatus() {
|
|
|
472
664
|
console.log(' (original values never stored or transmitted)');
|
|
473
665
|
}
|
|
474
666
|
}
|
|
667
|
+
function _resetForTesting() {
|
|
668
|
+
Object.assign(state, createInitialState());
|
|
669
|
+
_allTimeStats = null;
|
|
670
|
+
_allTimeStatsDirty = false;
|
|
671
|
+
_stateDirty = true;
|
|
672
|
+
firstEventDelivered = false;
|
|
673
|
+
teardownPreviousRuntime = null;
|
|
674
|
+
pendingTeardown = null;
|
|
675
|
+
serviceStartFn = null;
|
|
676
|
+
}
|
|
475
677
|
exports.default = {
|
|
476
678
|
id: 'shield',
|
|
477
679
|
name: 'OpenClaw Shield',
|
|
@@ -561,6 +763,8 @@ exports.default = {
|
|
|
561
763
|
const markStopped = opts?.markStopped ?? true;
|
|
562
764
|
if (markStopped) {
|
|
563
765
|
state.running = false;
|
|
766
|
+
state.lastLifecyclePhase = opts?.finalPhase ?? 'stopped';
|
|
767
|
+
state.lastStopReason = opts?.stopReason ?? 'unknown';
|
|
564
768
|
markStateDirty();
|
|
565
769
|
persistState();
|
|
566
770
|
}
|
|
@@ -569,17 +773,32 @@ exports.default = {
|
|
|
569
773
|
}
|
|
570
774
|
if (opts?.flushRedactor) {
|
|
571
775
|
try {
|
|
572
|
-
const { flush: flushRedactor } =
|
|
776
|
+
const { flush: flushRedactor } = require('./src/redactor');
|
|
573
777
|
flushRedactor();
|
|
574
778
|
}
|
|
575
779
|
catch { }
|
|
576
780
|
}
|
|
577
781
|
};
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
782
|
+
const inheritedRuntimeTeardown = teardownPreviousRuntime;
|
|
783
|
+
let inheritedRuntimeHandedOff = false;
|
|
784
|
+
const currentRuntimeTeardown = () => cleanupRuntime({
|
|
785
|
+
markStopped: true,
|
|
786
|
+
resetGuard: true,
|
|
787
|
+
flushRedactor: false,
|
|
788
|
+
stopReason: 'runtime_replaced',
|
|
789
|
+
});
|
|
790
|
+
teardownPreviousRuntime = inheritedRuntimeTeardown;
|
|
791
|
+
const deactivateForRegistrationInvalidation = async (checkpoint, message) => {
|
|
792
|
+
setLifecyclePhase('deactivated', checkpoint);
|
|
793
|
+
setStopReason('registration_invalidated');
|
|
794
|
+
setLastError(message);
|
|
795
|
+
await cleanupRuntime({
|
|
796
|
+
markStopped: true,
|
|
797
|
+
stopReason: 'registration_invalidated',
|
|
798
|
+
finalPhase: 'deactivated',
|
|
799
|
+
flushRedactor: true,
|
|
800
|
+
});
|
|
801
|
+
};
|
|
583
802
|
const serviceDefinition = {
|
|
584
803
|
id: 'shield-monitor',
|
|
585
804
|
async start() {
|
|
@@ -594,6 +813,11 @@ exports.default = {
|
|
|
594
813
|
try {
|
|
595
814
|
await cleanupRuntime({ markStopped: false, resetGuard: false, flushRedactor: false });
|
|
596
815
|
const activeGeneration = ++runtimeGeneration;
|
|
816
|
+
Object.assign(state, createInitialState());
|
|
817
|
+
setLastError(null);
|
|
818
|
+
setStopReason('none');
|
|
819
|
+
setLifecyclePhase('starting', 'checkpoint:1');
|
|
820
|
+
persistState();
|
|
597
821
|
log.info('shield', `[checkpoint:1] Service start() entered — generation=${activeGeneration}`);
|
|
598
822
|
let credentials = (0, config_1.loadCredentials)();
|
|
599
823
|
let validCreds = hasValidCredentials(credentials);
|
|
@@ -603,6 +827,10 @@ exports.default = {
|
|
|
603
827
|
const autoCreds = await performAutoRegistration(installationKey);
|
|
604
828
|
if (!autoCreds) {
|
|
605
829
|
log.error('shield', 'Activation failed. Verify your Installation Key and try again.');
|
|
830
|
+
setLifecyclePhase('startup_failed', 'checkpoint:activation_failed');
|
|
831
|
+
setStopReason('activation_failed');
|
|
832
|
+
setLastError('Activation failed. Verify your Installation Key and try again.');
|
|
833
|
+
persistState();
|
|
606
834
|
startGuard.endFailure();
|
|
607
835
|
return;
|
|
608
836
|
}
|
|
@@ -617,11 +845,17 @@ exports.default = {
|
|
|
617
845
|
log.warn('shield', ' Activate via CLI: openclaw shield activate <YOUR_KEY>');
|
|
618
846
|
log.warn('shield', ' Or set plugins.entries.shield.config.installationKey in openclaw.json and restart.');
|
|
619
847
|
log.warn('shield', ' Get your key at: https://uss.upx.com → APPS → OpenClaw Shield');
|
|
848
|
+
setLifecyclePhase('stopped', 'checkpoint:not_activated');
|
|
849
|
+
setStopReason('not_activated');
|
|
850
|
+
setLastError('Shield is not activated.');
|
|
851
|
+
persistState();
|
|
620
852
|
startGuard.endFailure();
|
|
621
853
|
return;
|
|
622
854
|
}
|
|
623
855
|
state.activated = true;
|
|
624
856
|
state.startedAt = Date.now();
|
|
857
|
+
setLastError(null);
|
|
858
|
+
setLifecyclePhase('starting', 'checkpoint:3');
|
|
625
859
|
const config = (0, config_1.loadConfig)({
|
|
626
860
|
credentials,
|
|
627
861
|
dryRun: dryRunVal,
|
|
@@ -630,25 +864,33 @@ exports.default = {
|
|
|
630
864
|
collectHostMetrics: hostMetricsVal,
|
|
631
865
|
});
|
|
632
866
|
state.instanceId = config.credentials.instanceId ?? '';
|
|
867
|
+
state.sessionDirCount = config.sessionDirs.length;
|
|
868
|
+
if (config.sessionDirs.length === 0) {
|
|
869
|
+
state.lastDeliveryIssue = 'No OpenClaw session directories discovered yet — Shield is loaded but has no event sources.';
|
|
870
|
+
state.lastDeliveryIssueAt = Date.now();
|
|
871
|
+
}
|
|
633
872
|
const persistedStats = readAllTimeStats();
|
|
634
873
|
if (persistedStats.lastSync)
|
|
635
874
|
state.lastSync = persistedStats.lastSync;
|
|
636
875
|
log.info('shield', `[checkpoint:3] Config loaded — sessionDirs=${config.sessionDirs.length} poll=${config.pollIntervalMs}ms dryRun=${config.dryRun}`);
|
|
637
876
|
log.info('shield', `Starting Shield v${version_1.VERSION} (poll: ${config.pollIntervalMs}ms, dryRun: ${config.dryRun})`);
|
|
877
|
+
persistState();
|
|
638
878
|
(0, exclusions_1.initExclusions)((0, path_1.join)((0, os_1.homedir)(), '.openclaw', 'shield', 'data'));
|
|
639
879
|
log.info('shield', '[checkpoint:4] Exclusions initialized');
|
|
640
880
|
(0, case_monitor_1.initCaseMonitor)((0, path_1.join)((0, os_1.homedir)(), '.openclaw', 'shield', 'data'));
|
|
641
881
|
log.info('shield', '[checkpoint:5] Case monitor initialized');
|
|
882
|
+
state.lastStartupCheckpoint = 'checkpoint:5';
|
|
883
|
+
persistState();
|
|
642
884
|
if (config.localEventBuffer) {
|
|
643
885
|
(0, event_store_1.initEventStore)((0, path_1.join)((0, os_1.homedir)(), '.openclaw', 'shield', 'data'), { maxEvents: config.localEventLimit });
|
|
644
886
|
}
|
|
645
887
|
const runtime = api.runtime;
|
|
646
|
-
|
|
647
|
-
|
|
888
|
+
const openclawConfig = api.config;
|
|
889
|
+
if (typeof runtime?.system?.enqueueSystemEvent === 'function' && typeof runtime?.system?.requestHeartbeatNow === 'function') {
|
|
648
890
|
(0, case_monitor_1.setCaseNotificationDispatcher)({
|
|
649
891
|
enqueueSystemEvent: runtime.system.enqueueSystemEvent,
|
|
650
892
|
requestHeartbeatNow: runtime.system.requestHeartbeatNow,
|
|
651
|
-
agentId:
|
|
893
|
+
agentId: openclawConfig?.agents?.default ?? 'main',
|
|
652
894
|
});
|
|
653
895
|
}
|
|
654
896
|
else {
|
|
@@ -656,7 +898,6 @@ exports.default = {
|
|
|
656
898
|
}
|
|
657
899
|
try {
|
|
658
900
|
const channels = runtime?.channel;
|
|
659
|
-
const cfg = api.config;
|
|
660
901
|
const channelCandidates = [
|
|
661
902
|
{ name: 'telegram', sendFn: channels?.telegram?.sendMessageTelegram, configPath: 'telegram' },
|
|
662
903
|
{ name: 'discord', sendFn: channels?.discord?.sendMessageDiscord, configPath: 'discord' },
|
|
@@ -667,7 +908,7 @@ exports.default = {
|
|
|
667
908
|
for (const candidate of channelCandidates) {
|
|
668
909
|
if (typeof candidate.sendFn !== 'function')
|
|
669
910
|
continue;
|
|
670
|
-
const channelCfg =
|
|
911
|
+
const channelCfg = openclawConfig?.channels?.[candidate.configPath];
|
|
671
912
|
if (!channelCfg?.enabled && channelCfg?.enabled !== undefined)
|
|
672
913
|
continue;
|
|
673
914
|
let targetId = null;
|
|
@@ -694,6 +935,9 @@ exports.default = {
|
|
|
694
935
|
break;
|
|
695
936
|
}
|
|
696
937
|
}
|
|
938
|
+
if (!channels) {
|
|
939
|
+
log.debug('case-monitor', 'Direct send runtime channels unavailable — direct message notifications disabled');
|
|
940
|
+
}
|
|
697
941
|
}
|
|
698
942
|
catch (err) {
|
|
699
943
|
log.debug('case-monitor', `Direct send setup failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -702,7 +946,7 @@ exports.default = {
|
|
|
702
946
|
const updateState = (0, updater_1.loadUpdateState)();
|
|
703
947
|
const previousVersion = updateState.rollbackVersion ?? null;
|
|
704
948
|
const startReason = previousVersion ? 'update' : 'normal';
|
|
705
|
-
const { reportLifecycleEvent } =
|
|
949
|
+
const { reportLifecycleEvent } = require('./src/sender');
|
|
706
950
|
void reportLifecycleEvent('plugin_started', {
|
|
707
951
|
version: version_1.VERSION,
|
|
708
952
|
reason: startReason,
|
|
@@ -712,14 +956,21 @@ exports.default = {
|
|
|
712
956
|
catch { }
|
|
713
957
|
const autoUpdateMode = pluginConfig.autoUpdate ?? true;
|
|
714
958
|
const _bootState = (0, updater_1.loadUpdateState)();
|
|
959
|
+
state.lastStartupCheckpoint = 'checkpoint:6';
|
|
960
|
+
persistState();
|
|
715
961
|
log.info('shield', `[checkpoint:6] Pre-update-check — autoUpdate=${autoUpdateMode} pendingRestart=${_bootState.pendingRestart} updateAvailable=${_bootState.updateAvailable} latestVersion=${_bootState.latestVersion}`);
|
|
716
962
|
log.info('updater', `Startup update check (autoUpdate=${autoUpdateMode}, current=${version_1.VERSION}, pendingRestart=${_bootState.pendingRestart})`);
|
|
717
963
|
const startupUpdate = (0, updater_1.performAutoUpdate)(autoUpdateMode, _bootState.pendingRestart ? undefined : 0);
|
|
964
|
+
state.lastStartupCheckpoint = 'checkpoint:7';
|
|
965
|
+
persistState();
|
|
718
966
|
log.info('shield', `[checkpoint:7] Update check done — action=${startupUpdate.action}`);
|
|
719
967
|
if (startupUpdate.action === 'updated') {
|
|
720
968
|
log.info('updater', startupUpdate.message);
|
|
721
969
|
const restarted = (0, updater_1.requestGatewayRestart)();
|
|
722
970
|
if (restarted) {
|
|
971
|
+
setLifecyclePhase('stopped', 'checkpoint:update_restart');
|
|
972
|
+
setStopReason('update_restart');
|
|
973
|
+
persistState();
|
|
723
974
|
startGuard.endFailure();
|
|
724
975
|
return;
|
|
725
976
|
}
|
|
@@ -736,21 +987,37 @@ exports.default = {
|
|
|
736
987
|
}
|
|
737
988
|
try {
|
|
738
989
|
const inv = (0, inventory_1.collectInventory)();
|
|
739
|
-
const { setCachedInventory } =
|
|
990
|
+
const { setCachedInventory } = require('./src/inventory');
|
|
740
991
|
setCachedInventory(inv);
|
|
741
992
|
}
|
|
742
993
|
catch (err) {
|
|
743
994
|
log.warn('shield', `Inventory collection failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
744
995
|
}
|
|
745
|
-
const { fetchNewEntries, commitCursors } =
|
|
746
|
-
const { transformEntries, generateHostTelemetry, resolveOpenClawVersion, resolveAgentLabel } =
|
|
747
|
-
const { sendEvents, reportInstance } =
|
|
748
|
-
const { init: initRedactor, flush: flushRedactor, redactEvent } =
|
|
749
|
-
const { validate } =
|
|
996
|
+
const { fetchNewEntries, commitCursors } = require('./src/fetcher');
|
|
997
|
+
const { transformEntries, generateHostTelemetry, resolveOpenClawVersion, resolveAgentLabel } = require('./src/transformer');
|
|
998
|
+
const { sendEvents, reportInstance } = require('./src/sender');
|
|
999
|
+
const { init: initRedactor, flush: flushRedactor, redactEvent } = require('./src/redactor');
|
|
1000
|
+
const { validate } = require('./src/validator');
|
|
750
1001
|
log.info('shield', '[checkpoint:8] Dynamic imports loaded');
|
|
751
1002
|
if (config.redactionEnabled)
|
|
752
1003
|
initRedactor();
|
|
1004
|
+
if (inheritedRuntimeTeardown && !inheritedRuntimeHandedOff) {
|
|
1005
|
+
try {
|
|
1006
|
+
log.info('shield', '[checkpoint:8a] Taking over previous healthy runtime');
|
|
1007
|
+
pendingTeardown = inheritedRuntimeTeardown()
|
|
1008
|
+
.catch((err) => log.warn('shield', `Previous runtime teardown during takeover failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
1009
|
+
await pendingTeardown.catch(() => { });
|
|
1010
|
+
pendingTeardown = null;
|
|
1011
|
+
inheritedRuntimeHandedOff = true;
|
|
1012
|
+
}
|
|
1013
|
+
catch (err) {
|
|
1014
|
+
log.warn('shield', `Takeover handoff failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1015
|
+
throw err;
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
753
1018
|
state.running = true;
|
|
1019
|
+
setLifecyclePhase('running', 'checkpoint:9');
|
|
1020
|
+
teardownPreviousRuntime = currentRuntimeTeardown;
|
|
754
1021
|
persistState();
|
|
755
1022
|
log.info('shield', '[checkpoint:9] state.running=true — entering poll loop');
|
|
756
1023
|
const runTelemetry = async () => {
|
|
@@ -779,19 +1046,31 @@ exports.default = {
|
|
|
779
1046
|
const result = await reportInstance(instancePayload, config.credentials);
|
|
780
1047
|
if (activeGeneration !== runtimeGeneration)
|
|
781
1048
|
return;
|
|
1049
|
+
state.lastTelemetryAt = Date.now();
|
|
782
1050
|
log.info('shield', `Instance report → Platform: success=${result.ok}`);
|
|
783
1051
|
if (result.ok) {
|
|
784
|
-
state.
|
|
1052
|
+
state.telemetryConsecutiveFailures = 0;
|
|
1053
|
+
state.lastSuccessfulTelemetryAt = Date.now();
|
|
1054
|
+
state.lastTelemetryError = null;
|
|
1055
|
+
if (state.lastLifecyclePhase === 'degraded' && state.running) {
|
|
1056
|
+
state.lastLifecyclePhase = 'running';
|
|
1057
|
+
}
|
|
785
1058
|
}
|
|
786
1059
|
else {
|
|
787
|
-
state.
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
1060
|
+
state.telemetryConsecutiveFailures++;
|
|
1061
|
+
state.lastTelemetryError = result.error ? String(result.error).slice(0, 200) : `HTTP ${result.statusCode ?? 0}`;
|
|
1062
|
+
if (result.needsRegistration) {
|
|
1063
|
+
log.error('shield', 'Instance report says this Shield instance is no longer registered — deactivating monitoring.');
|
|
1064
|
+
await deactivateForRegistrationInvalidation('checkpoint:telemetry_registration_invalid', 'Instance registration invalidated by platform.');
|
|
1065
|
+
return;
|
|
1066
|
+
}
|
|
1067
|
+
if (state.telemetryConsecutiveFailures >= MAX_REGISTRATION_FAILURES) {
|
|
1068
|
+
log.error('shield', `Instance telemetry failed ${state.telemetryConsecutiveFailures} consecutive times — capture remains active but health is degraded.`);
|
|
1069
|
+
setLifecyclePhase('degraded');
|
|
793
1070
|
}
|
|
794
1071
|
}
|
|
1072
|
+
markStateDirty();
|
|
1073
|
+
persistState();
|
|
795
1074
|
};
|
|
796
1075
|
const runTelemetrySingleflight = createSingleflightRunner(runTelemetry);
|
|
797
1076
|
log.info('shield', '[checkpoint:9a] Firing initial telemetry');
|
|
@@ -828,7 +1107,10 @@ exports.default = {
|
|
|
828
1107
|
if (entries.length === 0) {
|
|
829
1108
|
commitCursors(config, []);
|
|
830
1109
|
state.consecutiveFailures = 0;
|
|
1110
|
+
state.lastPollError = null;
|
|
831
1111
|
state.lastPollAt = Date.now();
|
|
1112
|
+
state.lastSuccessfulPollAt = state.lastPollAt;
|
|
1113
|
+
state.eventsRetainedForRetry = 0;
|
|
832
1114
|
markStateDirty();
|
|
833
1115
|
persistState();
|
|
834
1116
|
const idlePlatformConfig = { apiUrl: config.credentials.apiUrl, instanceId: config.credentials.instanceId, hmacSecret: config.credentials.hmacSecret };
|
|
@@ -858,8 +1140,7 @@ exports.default = {
|
|
|
858
1140
|
const needsReg = results.some(r => r.needsRegistration);
|
|
859
1141
|
if (needsReg) {
|
|
860
1142
|
log.error('shield', 'Instance not registered on platform — Shield deactivated.');
|
|
861
|
-
|
|
862
|
-
markStateDirty();
|
|
1143
|
+
await deactivateForRegistrationInvalidation('checkpoint:poll_registration_invalid', 'Instance registration invalidated by platform.');
|
|
863
1144
|
return;
|
|
864
1145
|
}
|
|
865
1146
|
const pending = results.find(r => r.pendingNamespace);
|
|
@@ -867,6 +1148,10 @@ exports.default = {
|
|
|
867
1148
|
const waitMs = Math.min(pending.retryAfterMs ?? 300_000, MAX_BACKOFF_MS);
|
|
868
1149
|
log.warn('shield', `Namespace allocation in progress — holding events, backing off ${Math.round(waitMs / 1000)}s`);
|
|
869
1150
|
state.lastPollAt = Date.now();
|
|
1151
|
+
state.lastPollError = null;
|
|
1152
|
+
state.lastDeliveryIssue = describeSendFailure(pending.failureKind, pending.statusCode, pending.body);
|
|
1153
|
+
state.lastDeliveryIssueAt = Date.now();
|
|
1154
|
+
state.eventsRetainedForRetry = pending.eventCount;
|
|
870
1155
|
markStateDirty();
|
|
871
1156
|
persistState();
|
|
872
1157
|
await new Promise(r => setTimeout(r, waitMs));
|
|
@@ -875,7 +1160,9 @@ exports.default = {
|
|
|
875
1160
|
return;
|
|
876
1161
|
}
|
|
877
1162
|
const accepted = results.reduce((sum, r) => sum + (r.success ? r.eventCount : 0), 0);
|
|
878
|
-
|
|
1163
|
+
const allBatchesSucceeded = results.length > 0 && results.every(r => r.success);
|
|
1164
|
+
const firstFailure = results.find(r => !r.success);
|
|
1165
|
+
if (allBatchesSucceeded && accepted > 0) {
|
|
879
1166
|
(0, case_monitor_1.notifyCaseMonitorActivity)();
|
|
880
1167
|
if (!firstEventDelivered) {
|
|
881
1168
|
firstEventDelivered = true;
|
|
@@ -885,6 +1172,14 @@ exports.default = {
|
|
|
885
1172
|
flushRedactor();
|
|
886
1173
|
state.eventsProcessed += accepted;
|
|
887
1174
|
state.consecutiveFailures = 0;
|
|
1175
|
+
state.lastPollError = null;
|
|
1176
|
+
state.lastDeliveryIssue = null;
|
|
1177
|
+
state.lastDeliveryIssueAt = 0;
|
|
1178
|
+
state.eventsRetainedForRetry = 0;
|
|
1179
|
+
state.lastSuccessfulPollAt = Date.now();
|
|
1180
|
+
if (state.lastLifecyclePhase === 'degraded') {
|
|
1181
|
+
state.lastLifecyclePhase = 'running';
|
|
1182
|
+
}
|
|
888
1183
|
markStateDirty();
|
|
889
1184
|
const syncEventTypes = {};
|
|
890
1185
|
for (const env of envelopes) {
|
|
@@ -908,8 +1203,30 @@ exports.default = {
|
|
|
908
1203
|
(0, event_store_1.appendEvents)(summaries);
|
|
909
1204
|
}
|
|
910
1205
|
}
|
|
1206
|
+
else if (accepted > 0) {
|
|
1207
|
+
state.consecutiveFailures++;
|
|
1208
|
+
state.lastPollError = 'Partial delivery detected — retaining cursor position to retry all events safely.';
|
|
1209
|
+
state.lastDeliveryIssue = firstFailure
|
|
1210
|
+
? `${describeSendFailure(firstFailure.failureKind, firstFailure.statusCode, firstFailure.body)}. Retrying entire poll window without advancing cursors.`
|
|
1211
|
+
: 'Partial delivery detected — retrying without advancing cursors.';
|
|
1212
|
+
state.lastDeliveryIssueAt = Date.now();
|
|
1213
|
+
state.eventsRetainedForRetry = envelopes.length;
|
|
1214
|
+
setLifecyclePhase('degraded');
|
|
1215
|
+
log.warn('shield', 'Partial delivery detected — not advancing cursors so events can be retried safely. Some already-accepted events may be resent.');
|
|
1216
|
+
markStateDirty();
|
|
1217
|
+
}
|
|
911
1218
|
else {
|
|
912
1219
|
state.consecutiveFailures++;
|
|
1220
|
+
state.lastPollError = firstFailure
|
|
1221
|
+
? describeSendFailure(firstFailure.failureKind, firstFailure.statusCode, firstFailure.body)
|
|
1222
|
+
: 'Event delivery failed';
|
|
1223
|
+
state.lastDeliveryIssue = state.lastPollError;
|
|
1224
|
+
state.lastDeliveryIssueAt = Date.now();
|
|
1225
|
+
state.eventsRetainedForRetry = envelopes.length;
|
|
1226
|
+
state.lastLifecyclePhase = 'degraded';
|
|
1227
|
+
if (firstFailure?.failureKind === 'quota_exceeded') {
|
|
1228
|
+
setLifecyclePhase('degraded');
|
|
1229
|
+
}
|
|
913
1230
|
markStateDirty();
|
|
914
1231
|
}
|
|
915
1232
|
state.lastPollAt = Date.now();
|
|
@@ -924,8 +1241,13 @@ exports.default = {
|
|
|
924
1241
|
if (activeGeneration !== runtimeGeneration)
|
|
925
1242
|
return;
|
|
926
1243
|
state.consecutiveFailures++;
|
|
1244
|
+
state.lastPollError = err instanceof Error ? err.message : String(err);
|
|
1245
|
+
state.lastDeliveryIssue = state.lastPollError;
|
|
1246
|
+
state.lastDeliveryIssueAt = Date.now();
|
|
1247
|
+
state.lastLifecyclePhase = 'degraded';
|
|
927
1248
|
markStateDirty();
|
|
928
1249
|
log.error('shield', `Poll error: ${err instanceof Error ? err.message : String(err)}`);
|
|
1250
|
+
persistState();
|
|
929
1251
|
}
|
|
930
1252
|
};
|
|
931
1253
|
const runPollSingleflight = createSingleflightRunner(poll);
|
|
@@ -957,7 +1279,8 @@ exports.default = {
|
|
|
957
1279
|
if (!state.running)
|
|
958
1280
|
return;
|
|
959
1281
|
log.info('shield', '[checkpoint:SIGTERM] SIGTERM received — shutting down');
|
|
960
|
-
|
|
1282
|
+
setLifecyclePhase('stopping', 'checkpoint:SIGTERM');
|
|
1283
|
+
await cleanupRuntime({ markStopped: true, resetGuard: true, flushRedactor: true, stopReason: 'signal' });
|
|
961
1284
|
log.info('shield', 'Service stopped (signal)');
|
|
962
1285
|
};
|
|
963
1286
|
process.once('SIGTERM', onSignalHandler);
|
|
@@ -966,13 +1289,18 @@ exports.default = {
|
|
|
966
1289
|
startGuard.endSuccess();
|
|
967
1290
|
}
|
|
968
1291
|
catch (err) {
|
|
1292
|
+
setLifecyclePhase('startup_failed');
|
|
1293
|
+
setStopReason('startup_failed');
|
|
1294
|
+
setLastError(err instanceof Error ? err.message : String(err));
|
|
1295
|
+
persistState();
|
|
969
1296
|
startGuard.endFailure();
|
|
970
1297
|
throw err;
|
|
971
1298
|
}
|
|
972
1299
|
},
|
|
973
1300
|
async stop() {
|
|
974
1301
|
const wasRunning = state.running;
|
|
975
|
-
|
|
1302
|
+
setLifecyclePhase('stopping');
|
|
1303
|
+
await cleanupRuntime({ markStopped: true, resetGuard: true, flushRedactor: true, stopReason: 'service_stop' });
|
|
976
1304
|
if (wasRunning)
|
|
977
1305
|
log.info('shield', 'Service stopped');
|
|
978
1306
|
},
|
|
@@ -984,6 +1312,7 @@ exports.default = {
|
|
|
984
1312
|
const activated = state.activated || hasValidCredentials(creds);
|
|
985
1313
|
const caseStatus = (0, case_monitor_1.getCaseMonitorStatus)();
|
|
986
1314
|
const monitorHealth = (0, case_monitor_1.getMonitorHealth)();
|
|
1315
|
+
const updateState = (0, updater_1.loadUpdateState)();
|
|
987
1316
|
const caseMonitorDisplay = monitorHealth.status === 'degraded'
|
|
988
1317
|
? `⚠️ Case Monitor: DEGRADED — ${monitorHealth.consecutiveFailures} consecutive failures since ${monitorHealth.degradedSinceMs ? new Date(monitorHealth.degradedSinceMs).toISOString() : 'unknown'}. Last error: ${monitorHealth.lastErrorMessage}`
|
|
989
1318
|
: `✅ Case Monitor: ok`;
|
|
@@ -1005,18 +1334,47 @@ exports.default = {
|
|
|
1005
1334
|
}
|
|
1006
1335
|
}
|
|
1007
1336
|
const hasSecret = !!creds?.hmacSecret && !PLACEHOLDER_VALUES.has((creds.hmacSecret || '').trim().toLowerCase());
|
|
1008
|
-
const stateField = hasSecret
|
|
1337
|
+
const stateField = !hasSecret
|
|
1338
|
+
? 'unconfigured'
|
|
1339
|
+
: !activated
|
|
1340
|
+
? 'pending'
|
|
1341
|
+
: state.running
|
|
1342
|
+
? (state.lastLifecyclePhase === 'degraded' ? 'degraded' : 'connected')
|
|
1343
|
+
: state.lastLifecyclePhase;
|
|
1009
1344
|
respond(true, {
|
|
1010
1345
|
activated,
|
|
1011
1346
|
state: stateField,
|
|
1012
1347
|
running: state.running,
|
|
1013
1348
|
lastPollAt: state.lastPollAt,
|
|
1349
|
+
lastSuccessfulPollAt: state.lastSuccessfulPollAt,
|
|
1014
1350
|
lastCaptureAt: state.lastCaptureAt,
|
|
1015
1351
|
captureSeenSinceLastSync: state.captureSeenSinceLastSync,
|
|
1016
1352
|
eventsProcessed: state.eventsProcessed,
|
|
1017
1353
|
quarantineCount: state.quarantineCount,
|
|
1018
1354
|
consecutiveFailures: state.consecutiveFailures,
|
|
1355
|
+
telemetryConsecutiveFailures: state.telemetryConsecutiveFailures,
|
|
1356
|
+
lastTelemetryAt: state.lastTelemetryAt,
|
|
1357
|
+
lastSuccessfulTelemetryAt: state.lastSuccessfulTelemetryAt,
|
|
1358
|
+
lastTelemetryError: state.lastTelemetryError,
|
|
1359
|
+
lastPollError: state.lastPollError,
|
|
1360
|
+
lastLifecyclePhase: state.lastLifecyclePhase,
|
|
1361
|
+
lastStopReason: state.lastStopReason,
|
|
1362
|
+
lastError: state.lastError,
|
|
1363
|
+
lastStartupCheckpoint: state.lastStartupCheckpoint,
|
|
1364
|
+
sessionDirCount: state.sessionDirCount,
|
|
1365
|
+
eventsRetainedForRetry: state.eventsRetainedForRetry,
|
|
1366
|
+
lastDeliveryIssue: state.lastDeliveryIssue,
|
|
1367
|
+
lastDeliveryIssueAt: state.lastDeliveryIssueAt,
|
|
1019
1368
|
version: version_1.VERSION,
|
|
1369
|
+
update: {
|
|
1370
|
+
pendingRestart: updateState.pendingRestart,
|
|
1371
|
+
restartAttempts: updateState.restartAttempts,
|
|
1372
|
+
latestVersion: updateState.latestVersion,
|
|
1373
|
+
updateAvailable: updateState.updateAvailable,
|
|
1374
|
+
lastError: updateState.lastError,
|
|
1375
|
+
lastFailureStage: updateState.lastFailureStage,
|
|
1376
|
+
rollbackPending: updateState.rollbackPending,
|
|
1377
|
+
},
|
|
1020
1378
|
caseMonitor: {
|
|
1021
1379
|
intervalMs: caseStatus.intervalMs,
|
|
1022
1380
|
nextCheckInMs: caseStatus.nextCheckIn,
|
|
@@ -1071,34 +1429,39 @@ exports.default = {
|
|
|
1071
1429
|
hmacSecret: rpcCreds?.hmacSecret || '',
|
|
1072
1430
|
};
|
|
1073
1431
|
(0, rpc_1.registerAllRpcs)(api, platformApiConfig);
|
|
1074
|
-
api.registerCommand
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
const
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1432
|
+
if (typeof api.registerCommand === 'function') {
|
|
1433
|
+
api.registerCommand({
|
|
1434
|
+
name: 'shieldcases',
|
|
1435
|
+
description: 'Show pending Shield security cases',
|
|
1436
|
+
requireAuth: true,
|
|
1437
|
+
handler: () => {
|
|
1438
|
+
const { getPendingCases } = require('./src/case-monitor');
|
|
1439
|
+
const cases = getPendingCases();
|
|
1440
|
+
if (cases.length === 0) {
|
|
1441
|
+
return { text: '✅ No pending Shield security cases.' };
|
|
1442
|
+
}
|
|
1443
|
+
const severityEmoji = { CRITICAL: '🔴', HIGH: '🟠', MEDIUM: '🟡', LOW: '🔵' };
|
|
1444
|
+
const lines = cases.map((c) => {
|
|
1445
|
+
const emoji = severityEmoji[c.severity] || '⚠️';
|
|
1446
|
+
const age = Math.floor((Date.now() - new Date(c.created_at).getTime()) / 60000);
|
|
1447
|
+
const ageStr = age < 60 ? `${age}m` : `${Math.floor(age / 60)}h`;
|
|
1448
|
+
return `${emoji} **${c.rule_title}** (${c.severity || 'unknown'}) — ${c.event_count} events — ${ageStr} ago`;
|
|
1449
|
+
});
|
|
1450
|
+
return {
|
|
1451
|
+
text: [
|
|
1452
|
+
`🛡️ **Shield — ${cases.length} Pending Case${cases.length > 1 ? 's' : ''}**`,
|
|
1453
|
+
'',
|
|
1454
|
+
...lines,
|
|
1455
|
+
'',
|
|
1456
|
+
'Use `openclaw shield cases show <id>` for details.',
|
|
1457
|
+
].join('\n'),
|
|
1458
|
+
};
|
|
1459
|
+
},
|
|
1460
|
+
});
|
|
1461
|
+
}
|
|
1462
|
+
else {
|
|
1463
|
+
log.debug('shield', 'registerCommand not available — auto-reply command /shieldcases disabled on this OpenClaw runtime');
|
|
1464
|
+
}
|
|
1102
1465
|
api.registerCli(({ program }) => {
|
|
1103
1466
|
const shield = program.command('shield');
|
|
1104
1467
|
(0, cli_cases_1.registerCasesCli)(shield);
|
|
@@ -1237,10 +1600,9 @@ exports.default = {
|
|
|
1237
1600
|
step('Stale updateAvailable cleared');
|
|
1238
1601
|
}
|
|
1239
1602
|
step('Resetting session state...');
|
|
1240
|
-
state
|
|
1241
|
-
state.
|
|
1242
|
-
state.
|
|
1243
|
-
state.captureSeenSinceLastSync = false;
|
|
1603
|
+
Object.assign(state, createInitialState());
|
|
1604
|
+
state.lastLifecyclePhase = 'stopping';
|
|
1605
|
+
state.lastStopReason = 'manual_restart';
|
|
1244
1606
|
markStateDirty();
|
|
1245
1607
|
persistState();
|
|
1246
1608
|
step('State reset');
|
|
@@ -1332,7 +1694,7 @@ exports.default = {
|
|
|
1332
1694
|
console.log(` agent:${agentHash} workspace:${wsHash} identity=${identityLabel} bootstrapped=${bootstrapLabel}`);
|
|
1333
1695
|
}
|
|
1334
1696
|
try {
|
|
1335
|
-
const { initVault, getAllMappings } =
|
|
1697
|
+
const { initVault, getAllMappings } = require('./src/redactor/vault');
|
|
1336
1698
|
initVault();
|
|
1337
1699
|
const mappings = getAllMappings();
|
|
1338
1700
|
const tokens = Object.keys(mappings);
|