openclaw-hybrid-memory 2026.7.50 → 2026.7.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/commands/manage/register-maintenance-health.ts +68 -7
- package/cli/commands/manage/register-maintenance-orchestrator.ts +79 -27
- package/dist/cli/commands/manage/register-maintenance-health.js +34 -6
- package/dist/cli/commands/manage/register-maintenance-health.js.map +1 -1
- package/dist/cli/commands/manage/register-maintenance-orchestrator.js +20 -4
- package/dist/cli/commands/manage/register-maintenance-orchestrator.js.map +1 -1
- package/dist/lifecycle/stage-capture/run-capture.js +1 -1
- package/dist/services/cron-guard.js +91 -7
- package/dist/services/cron-guard.js.map +1 -1
- package/dist/services/maintenance-orchestrator.js +4 -3
- package/dist/services/maintenance-orchestrator.js.map +1 -1
- package/dist/services/passive-observer.js +12 -3
- package/dist/services/passive-observer.js.map +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/services/cron-guard.ts +109 -8
- package/services/maintenance-orchestrator.ts +191 -184
- package/services/passive-observer.ts +52 -34
|
@@ -1,19 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Maintenance inventory, status, and cron-health subcommands (Issue #281).
|
|
3
3
|
*/
|
|
4
|
+
import { existsSync } from "node:fs";
|
|
4
5
|
import { homedir } from "node:os";
|
|
5
6
|
import { join } from "node:path";
|
|
6
7
|
|
|
7
8
|
import type { HybridMemoryConfig } from "../../../config.js";
|
|
9
|
+
import { formatStepLockDetail, listActiveStepLocks } from "../../../services/cron-guard.js";
|
|
8
10
|
import {
|
|
9
11
|
collectMaintenanceInventory,
|
|
10
12
|
renderMaintenanceInventoryMarkdown,
|
|
11
13
|
renderMaintenanceInventoryText,
|
|
12
14
|
} from "../../../services/maintenance-inventory.js";
|
|
15
|
+
import {
|
|
16
|
+
collectMaintenanceSteps,
|
|
17
|
+
summarizeMaintenanceLogAnalysis,
|
|
18
|
+
} from "../../../services/maintenance-log-analyzer.js";
|
|
19
|
+
import { describeCronStoreLocation, readOpenClawCronStore } from "../../../services/openclaw-cron-store.js";
|
|
13
20
|
import { formatTimestampUtcFromMs } from "../../../utils/dates.js";
|
|
14
|
-
import { readOpenClawCronStore, describeCronStoreLocation } from "../../../services/openclaw-cron-store.js";
|
|
15
21
|
import { type Chainable, relativeTime, withExit } from "../../shared.js";
|
|
16
22
|
|
|
23
|
+
/** Lookback window for the log-health check folded into `maintenance status` (#2033). */
|
|
24
|
+
const STATUS_LOG_HEALTH_SINCE = "24h";
|
|
25
|
+
|
|
17
26
|
export function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: HybridMemoryConfig): void {
|
|
18
27
|
maintenance
|
|
19
28
|
.command("inventory")
|
|
@@ -48,8 +57,12 @@ export function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: H
|
|
|
48
57
|
.command("status")
|
|
49
58
|
.description("Show maintenance cron job health: nightly cycle, weekly backup, and any reliability issues.")
|
|
50
59
|
.option("--json", "Output as JSON")
|
|
60
|
+
.option(
|
|
61
|
+
"-v, --verbose",
|
|
62
|
+
"Also show log-health detail when there are no strict findings (active/stale locks are always shown when present)",
|
|
63
|
+
)
|
|
51
64
|
.action(
|
|
52
|
-
withExit(async (opts?: { json?: boolean }) => {
|
|
65
|
+
withExit(async (opts?: { json?: boolean; verbose?: boolean }) => {
|
|
53
66
|
const openclawDir = join(homedir(), ".openclaw");
|
|
54
67
|
const staleThresholdMs = (cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28) * 60 * 60 * 1000;
|
|
55
68
|
const nightlyCronExpr = cfg.maintenance?.cronReliability?.nightlyCron ?? "0 3 * * *";
|
|
@@ -168,7 +181,29 @@ export function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: H
|
|
|
168
181
|
|
|
169
182
|
const issues = results.filter((r) => r.issue);
|
|
170
183
|
|
|
171
|
-
|
|
184
|
+
// Log health (#2033): scheduler cadence looking healthy must not imply the maintenance
|
|
185
|
+
// logs are clean — analyze-maintenance-logs can report strict findings for the same window
|
|
186
|
+
// even when cron cadence looks fine, and an operator who stops at `status` would miss it.
|
|
187
|
+
const logRoot = join(openclawDir, "logs", "cron-hybrid-mem");
|
|
188
|
+
let logHealth: { summary: string; strictFailed: boolean; findingsCount: number } | null = null;
|
|
189
|
+
if (existsSync(logRoot)) {
|
|
190
|
+
try {
|
|
191
|
+
const steps = collectMaintenanceSteps(logRoot, STATUS_LOG_HEALTH_SINCE);
|
|
192
|
+
logHealth = summarizeMaintenanceLogAnalysis(steps);
|
|
193
|
+
} catch {
|
|
194
|
+
// Best-effort — a log-analysis failure must not break the cron-freshness check below.
|
|
195
|
+
logHealth = null;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Active/stale step locks (#2031): surfaces what `maintenance step <name> --force` would
|
|
200
|
+
// report as "locked by a concurrent maintenance run" instead of only discovering it there.
|
|
201
|
+
const locks = listActiveStepLocks(openclawDir);
|
|
202
|
+
|
|
203
|
+
const schedulerHealthy = issues.length === 0;
|
|
204
|
+
const logHealthy = !logHealth?.strictFailed;
|
|
205
|
+
|
|
206
|
+
if (issues.length > 0 || logHealth?.strictFailed) {
|
|
172
207
|
process.exitCode = 1;
|
|
173
208
|
}
|
|
174
209
|
|
|
@@ -176,9 +211,11 @@ export function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: H
|
|
|
176
211
|
console.log(
|
|
177
212
|
JSON.stringify(
|
|
178
213
|
{
|
|
179
|
-
ok:
|
|
214
|
+
ok: schedulerHealthy && logHealthy,
|
|
180
215
|
jobs: results,
|
|
181
216
|
issueCount: issues.length,
|
|
217
|
+
logHealth: logHealth ? { ...logHealth, since: STATUS_LOG_HEALTH_SINCE } : null,
|
|
218
|
+
locks,
|
|
182
219
|
},
|
|
183
220
|
null,
|
|
184
221
|
2,
|
|
@@ -209,14 +246,38 @@ export function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: H
|
|
|
209
246
|
}
|
|
210
247
|
|
|
211
248
|
console.log("");
|
|
212
|
-
if (
|
|
213
|
-
console.log("✅
|
|
249
|
+
if (schedulerHealthy) {
|
|
250
|
+
console.log("✅ Scheduler freshness: all maintenance jobs healthy (cron cadence only).");
|
|
214
251
|
} else {
|
|
215
|
-
console.log(`⚠️ ${issues.length} issue(s) detected. Run \`hybrid-mem verify --fix\` to repair.`);
|
|
252
|
+
console.log(`⚠️ ${issues.length} scheduler issue(s) detected. Run \`hybrid-mem verify --fix\` to repair.`);
|
|
216
253
|
if (issues.some((r) => r.isMissing)) {
|
|
217
254
|
console.log(" Missing jobs can be registered with: hybrid-mem install");
|
|
218
255
|
}
|
|
219
256
|
}
|
|
257
|
+
|
|
258
|
+
if (logHealth?.strictFailed) {
|
|
259
|
+
console.log(
|
|
260
|
+
`⚠️ Log health: analyze-maintenance-logs reports strict findings in the last ${STATUS_LOG_HEALTH_SINCE} ` +
|
|
261
|
+
`(${logHealth.summary}). Run \`hybrid-mem maintenance step analyze-maintenance-logs --force --verbose\` for detail.`,
|
|
262
|
+
);
|
|
263
|
+
} else if (opts?.verbose) {
|
|
264
|
+
console.log(
|
|
265
|
+
logHealth
|
|
266
|
+
? `✅ Log health: no strict findings in the last ${STATUS_LOG_HEALTH_SINCE} (${logHealth.summary}).`
|
|
267
|
+
: `ℹ️ Log health: unchecked (no logs found under ${logRoot}).`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (locks.length > 0) {
|
|
272
|
+
console.log("");
|
|
273
|
+
console.log(`🔒 Active/stale maintenance step locks (${locks.length}):`);
|
|
274
|
+
for (const lock of locks) {
|
|
275
|
+
console.log(` step--${lock.stepName} ${formatStepLockDetail(lock)}`);
|
|
276
|
+
}
|
|
277
|
+
console.log(
|
|
278
|
+
" A lock here will make `maintenance step <name> --force` report skipped_guard until it clears.",
|
|
279
|
+
);
|
|
280
|
+
}
|
|
220
281
|
}),
|
|
221
282
|
);
|
|
222
283
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { existsSync } from "node:fs";
|
|
6
6
|
import { homedir } from "node:os";
|
|
7
7
|
import { dirname, join } from "node:path";
|
|
8
|
-
import {
|
|
8
|
+
import { readStepGuardTimestampMs, stepGuardEligible } from "../../../services/cron-guard.js";
|
|
9
9
|
import {
|
|
10
10
|
effectiveCadenceLabel,
|
|
11
11
|
formatMaintenanceSummary,
|
|
@@ -14,7 +14,7 @@ import {
|
|
|
14
14
|
runMaintenanceOrchestrator,
|
|
15
15
|
toOrchestratorRunSummary,
|
|
16
16
|
} from "../../../services/maintenance-orchestrator.js";
|
|
17
|
-
import {
|
|
17
|
+
import { atomicWriteFile } from "../../../utils/atomic-write.js";
|
|
18
18
|
import { formatTimestampUtcFromMs } from "../../../utils/dates.js";
|
|
19
19
|
import { type CommanderOptsParent, readHybridMemVerbose } from "../../global-verbose.js";
|
|
20
20
|
import { type Chainable, withExit } from "../../shared.js";
|
|
@@ -44,6 +44,7 @@ export function registerMaintenanceOrchestratorCommands(maintenance: Chainable,
|
|
|
44
44
|
maxRuntimeMin?: string;
|
|
45
45
|
json?: boolean;
|
|
46
46
|
summaryOut?: string;
|
|
47
|
+
allowSkip?: boolean;
|
|
47
48
|
},
|
|
48
49
|
) => {
|
|
49
50
|
const verbose = !!opts?.verbose;
|
|
@@ -59,9 +60,15 @@ export function registerMaintenanceOrchestratorCommands(maintenance: Chainable,
|
|
|
59
60
|
backfillDecayMarker: backfillMarker,
|
|
60
61
|
scanOverrides: { force: opts?.force, full: opts?.full },
|
|
61
62
|
});
|
|
63
|
+
// `> 0` (not just finite) — "0" is a truthy, finite string that would otherwise produce a
|
|
64
|
+
// zero-minute budget, which the orchestrator's `elapsed >= maxRuntimeMs` check treats as
|
|
65
|
+
// already-exceeded before the first step runs, deferring every step including the one the
|
|
66
|
+
// operator asked for. Treat a non-positive value the same as "no budget", consistent with
|
|
67
|
+
// guardIntervalMs <= 0 meaning "always eligible" elsewhere in this file.
|
|
68
|
+
const parsedMaxRuntimeMin = opts?.maxRuntimeMin ? Number(opts.maxRuntimeMin) : undefined;
|
|
62
69
|
const maxRuntimeMs =
|
|
63
|
-
|
|
64
|
-
?
|
|
70
|
+
parsedMaxRuntimeMin !== undefined && Number.isFinite(parsedMaxRuntimeMin) && parsedMaxRuntimeMin > 0
|
|
71
|
+
? parsedMaxRuntimeMin * 60_000
|
|
65
72
|
: undefined;
|
|
66
73
|
|
|
67
74
|
const isJsonMode = opts?.json || !!opts?.summaryOut?.trim() || !!process.env.HM_SUMMARY?.trim();
|
|
@@ -109,6 +116,39 @@ export function registerMaintenanceOrchestratorCommands(maintenance: Chainable,
|
|
|
109
116
|
for (const line of lines) console.log(line);
|
|
110
117
|
}
|
|
111
118
|
if (result.exitCode !== 0) process.exitCode = result.exitCode;
|
|
119
|
+
|
|
120
|
+
// An explicit, forced --include run (whether via the dedicated `step <name>` command or
|
|
121
|
+
// `cycle`/`nightly`/`full --include x,y --force`) is a targeted "run this now and verify it"
|
|
122
|
+
// diagnostic — computeExitCode() treats "skipped" as benign because it's normal cadence-guard
|
|
123
|
+
// behavior in an ordinary full/nightly run, so it doesn't cover "the step(s) I explicitly asked
|
|
124
|
+
// for didn't execute" (#2031). Scoped to include+force so ordinary unforced --include automation
|
|
125
|
+
// (which can legitimately skip by cadence) keeps its existing exit-0-on-skip behavior.
|
|
126
|
+
const includeNames = parseStepList(opts?.include);
|
|
127
|
+
if ((opts?.force || opts?.full) && includeNames?.length && !opts?.allowSkip) {
|
|
128
|
+
// Look up each requested name individually rather than filtering result.steps down to the
|
|
129
|
+
// matches — a name that belongs to a tier not requested (e.g. `cycle --include <nightly-step>
|
|
130
|
+
// --force`) or that --exclude also removed never produces a step result at all, so it would
|
|
131
|
+
// otherwise vanish from `selected` entirely and slip past the "every(...)" check below with a
|
|
132
|
+
// false "didn't run" report never firing (an even quieter version of the bug #2031 closes).
|
|
133
|
+
const requested = includeNames.map((name) => ({ name, result: result.steps.find((s) => s.name === name) }));
|
|
134
|
+
const didNotRun = requested.every((r) => !r.result || r.result.status.startsWith("skipped"));
|
|
135
|
+
if (didNotRun) {
|
|
136
|
+
const detail = requested
|
|
137
|
+
.map((r) =>
|
|
138
|
+
r.result
|
|
139
|
+
? `${r.name}: ${r.result.status} — ${r.result.summary}`
|
|
140
|
+
: `${r.name}: no result (wrong tier, excluded, or unregistered runner)`,
|
|
141
|
+
)
|
|
142
|
+
.join("; ");
|
|
143
|
+
console.error(
|
|
144
|
+
`maintenance ${result.tierLabel}: selected step(s) did not run (${detail}). ` +
|
|
145
|
+
"Pass --allow-skip to treat this as a non-strict status check.",
|
|
146
|
+
);
|
|
147
|
+
if (!process.exitCode) process.exitCode = 3;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return result;
|
|
112
152
|
};
|
|
113
153
|
|
|
114
154
|
const commonOptions = (cmd: Chainable) =>
|
|
@@ -121,7 +161,11 @@ export function registerMaintenanceOrchestratorCommands(maintenance: Chainable,
|
|
|
121
161
|
.option("--exclude <steps>", "Comma-separated step names to skip")
|
|
122
162
|
.option("--max-runtime-min <n>", "Optional time budget in minutes")
|
|
123
163
|
.option("--json", "Emit orchestrator run summary as JSON")
|
|
124
|
-
.option("--summary-out <path>", "Write orchestrator run summary JSON to path")
|
|
164
|
+
.option("--summary-out <path>", "Write orchestrator run summary JSON to path")
|
|
165
|
+
.option(
|
|
166
|
+
"--allow-skip",
|
|
167
|
+
"With --include --force: exit 0 even if the selected step(s) didn't execute (locked/gated/guarded)",
|
|
168
|
+
);
|
|
125
169
|
|
|
126
170
|
commonOptions(
|
|
127
171
|
maintenance
|
|
@@ -158,34 +202,42 @@ export function registerMaintenanceOrchestratorCommands(maintenance: Chainable,
|
|
|
158
202
|
|
|
159
203
|
// `maintenance step <name>` runs exactly one named step in isolation (#2028). Named `step` (singular)
|
|
160
204
|
// to avoid colliding with the `maintenance run` JobRun-inspection group and the `steps` list command.
|
|
205
|
+
//
|
|
206
|
+
// Bounded by default (#2032): this command is meant for isolated diagnostic/verification runs, so
|
|
207
|
+
// unlike cycle/nightly/full it caps runtime unless the operator overrides --max-runtime-min. Long
|
|
208
|
+
// CPU/LLM-bound steps (e.g. passive-observer scanning a large first-run backlog) already consult the
|
|
209
|
+
// orchestrator-wide deadline between sessions/chunks, so this cap actually bounds wall-clock time.
|
|
210
|
+
const DEFAULT_STEP_MAX_RUNTIME_MIN = "15";
|
|
211
|
+
|
|
161
212
|
commonOptions(
|
|
162
213
|
maintenance
|
|
163
214
|
.command("step <step>")
|
|
164
215
|
.description(
|
|
165
216
|
"Run exactly one named maintenance step in isolation (bypasses that step's cadence guard). Use `maintenance steps` to list valid names.",
|
|
166
|
-
)
|
|
167
|
-
.action(
|
|
168
|
-
withExit(async (step: string, opts, cmd?: CommanderOptsParent) => {
|
|
169
|
-
const stepName = step?.trim();
|
|
170
|
-
const validNames = listMaintenanceSteps().map((s) => s.name);
|
|
171
|
-
if (!stepName || !validNames.includes(stepName)) {
|
|
172
|
-
console.error(`Unknown maintenance step: ${stepName || "(none)"}`);
|
|
173
|
-
console.error(`Valid steps (${validNames.length}):`);
|
|
174
|
-
for (const name of validNames) console.error(` ${name}`);
|
|
175
|
-
process.exitCode = 1;
|
|
176
|
-
return;
|
|
177
|
-
}
|
|
178
|
-
// Force so the targeted step actually runs even if its cadence guard has not expired (this
|
|
179
|
-
// command is an explicit "run this step now" diagnostic). include=[step] across both tiers
|
|
180
|
-
// means the orchestrator executes ONLY that step and no unrelated stages (#2028).
|
|
181
|
-
await runTier(["cycle", "nightly"], {
|
|
182
|
-
...opts,
|
|
183
|
-
include: stepName,
|
|
184
|
-
force: true,
|
|
185
|
-
verbose: !!opts?.verbose || readHybridMemVerbose(cmd),
|
|
186
|
-
});
|
|
187
|
-
}),
|
|
188
217
|
),
|
|
218
|
+
).action(
|
|
219
|
+
withExit(async (step: string, opts, cmd?: CommanderOptsParent) => {
|
|
220
|
+
const stepName = step?.trim();
|
|
221
|
+
const validNames = listMaintenanceSteps().map((s) => s.name);
|
|
222
|
+
if (!stepName || !validNames.includes(stepName)) {
|
|
223
|
+
console.error(`Unknown maintenance step: ${stepName || "(none)"}`);
|
|
224
|
+
console.error(`Valid steps (${validNames.length}):`);
|
|
225
|
+
for (const name of validNames) console.error(` ${name}`);
|
|
226
|
+
process.exitCode = 1;
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
// Force so the targeted step actually runs even if its cadence guard has not expired (this
|
|
230
|
+
// command is an explicit "run this step now" diagnostic). include=[step] across both tiers
|
|
231
|
+
// means the orchestrator executes ONLY that step and no unrelated stages (#2028). runTier()
|
|
232
|
+
// itself checks whether this forced, explicitly-included step actually executed (#2031).
|
|
233
|
+
await runTier(["cycle", "nightly"], {
|
|
234
|
+
...opts,
|
|
235
|
+
include: stepName,
|
|
236
|
+
force: true,
|
|
237
|
+
maxRuntimeMin: opts?.maxRuntimeMin ?? DEFAULT_STEP_MAX_RUNTIME_MIN,
|
|
238
|
+
verbose: !!opts?.verbose || readHybridMemVerbose(cmd),
|
|
239
|
+
});
|
|
240
|
+
}),
|
|
189
241
|
);
|
|
190
242
|
|
|
191
243
|
maintenance
|
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
import { formatTimestampUtcFromMs } from "../../../utils/dates.js";
|
|
2
2
|
import { describeCronStoreLocation, readOpenClawCronStore } from "../../../services/openclaw-cron-store.js";
|
|
3
|
+
import { formatStepLockDetail, listActiveStepLocks } from "../../../services/cron-guard.js";
|
|
3
4
|
import { relativeTime, withExit } from "../../shared.js";
|
|
5
|
+
import { collectMaintenanceSteps, summarizeMaintenanceLogAnalysis } from "../../../services/maintenance-log-analyzer.js";
|
|
4
6
|
import { collectMaintenanceInventory, renderMaintenanceInventoryMarkdown, renderMaintenanceInventoryText } from "../../../services/maintenance-inventory.js";
|
|
5
7
|
import { homedir } from "node:os";
|
|
6
8
|
import { join } from "node:path";
|
|
9
|
+
import { existsSync } from "node:fs";
|
|
7
10
|
//#region cli/commands/manage/register-maintenance-health.ts
|
|
8
11
|
/**
|
|
9
12
|
* Maintenance inventory, status, and cron-health subcommands (Issue #281).
|
|
10
13
|
*/
|
|
14
|
+
/** Lookback window for the log-health check folded into `maintenance status` (#2033). */
|
|
15
|
+
const STATUS_LOG_HEALTH_SINCE = "24h";
|
|
11
16
|
function registerMaintenanceHealthCommands(maintenance, cfg) {
|
|
12
17
|
maintenance.command("inventory").description("List host-crontab and gateway-cron maintenance jobs in one report.").option("--json", "Output as JSON").option("--format <format>", "Output format: text, markdown, json", "text").action(withExit(async (opts) => {
|
|
13
18
|
const requestedFormat = (opts?.format ?? "text").toLowerCase();
|
|
@@ -29,7 +34,7 @@ function registerMaintenanceHealthCommands(maintenance, cfg) {
|
|
|
29
34
|
}
|
|
30
35
|
console.log(renderMaintenanceInventoryText(report));
|
|
31
36
|
}));
|
|
32
|
-
maintenance.command("status").description("Show maintenance cron job health: nightly cycle, weekly backup, and any reliability issues.").option("--json", "Output as JSON").action(withExit(async (opts) => {
|
|
37
|
+
maintenance.command("status").description("Show maintenance cron job health: nightly cycle, weekly backup, and any reliability issues.").option("--json", "Output as JSON").option("-v, --verbose", "Also show log-health detail when there are no strict findings (active/stale locks are always shown when present)").action(withExit(async (opts) => {
|
|
33
38
|
const openclawDir = join(homedir(), ".openclaw");
|
|
34
39
|
const staleThresholdMs = (cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28) * 60 * 60 * 1e3;
|
|
35
40
|
const nightlyCronExpr = cfg.maintenance?.cronReliability?.nightlyCron ?? "0 3 * * *";
|
|
@@ -94,12 +99,27 @@ function registerMaintenanceHealthCommands(maintenance, cfg) {
|
|
|
94
99
|
});
|
|
95
100
|
}
|
|
96
101
|
const issues = results.filter((r) => r.issue);
|
|
97
|
-
|
|
102
|
+
const logRoot = join(openclawDir, "logs", "cron-hybrid-mem");
|
|
103
|
+
let logHealth = null;
|
|
104
|
+
if (existsSync(logRoot)) try {
|
|
105
|
+
logHealth = summarizeMaintenanceLogAnalysis(collectMaintenanceSteps(logRoot, STATUS_LOG_HEALTH_SINCE));
|
|
106
|
+
} catch {
|
|
107
|
+
logHealth = null;
|
|
108
|
+
}
|
|
109
|
+
const locks = listActiveStepLocks(openclawDir);
|
|
110
|
+
const schedulerHealthy = issues.length === 0;
|
|
111
|
+
const logHealthy = !logHealth?.strictFailed;
|
|
112
|
+
if (issues.length > 0 || logHealth?.strictFailed) process.exitCode = 1;
|
|
98
113
|
if (opts?.json) {
|
|
99
114
|
console.log(JSON.stringify({
|
|
100
|
-
ok:
|
|
115
|
+
ok: schedulerHealthy && logHealthy,
|
|
101
116
|
jobs: results,
|
|
102
|
-
issueCount: issues.length
|
|
117
|
+
issueCount: issues.length,
|
|
118
|
+
logHealth: logHealth ? {
|
|
119
|
+
...logHealth,
|
|
120
|
+
since: STATUS_LOG_HEALTH_SINCE
|
|
121
|
+
} : null,
|
|
122
|
+
locks
|
|
103
123
|
}, null, 2));
|
|
104
124
|
return;
|
|
105
125
|
}
|
|
@@ -115,11 +135,19 @@ function registerMaintenanceHealthCommands(maintenance, cfg) {
|
|
|
115
135
|
if (r.issue) console.log(` └─ ${r.issue}`);
|
|
116
136
|
}
|
|
117
137
|
console.log("");
|
|
118
|
-
if (
|
|
138
|
+
if (schedulerHealthy) console.log("✅ Scheduler freshness: all maintenance jobs healthy (cron cadence only).");
|
|
119
139
|
else {
|
|
120
|
-
console.log(`⚠️ ${issues.length} issue(s) detected. Run \`hybrid-mem verify --fix\` to repair.`);
|
|
140
|
+
console.log(`⚠️ ${issues.length} scheduler issue(s) detected. Run \`hybrid-mem verify --fix\` to repair.`);
|
|
121
141
|
if (issues.some((r) => r.isMissing)) console.log(" Missing jobs can be registered with: hybrid-mem install");
|
|
122
142
|
}
|
|
143
|
+
if (logHealth?.strictFailed) console.log(`⚠️ Log health: analyze-maintenance-logs reports strict findings in the last ${STATUS_LOG_HEALTH_SINCE} (${logHealth.summary}). Run \`hybrid-mem maintenance step analyze-maintenance-logs --force --verbose\` for detail.`);
|
|
144
|
+
else if (opts?.verbose) console.log(logHealth ? `✅ Log health: no strict findings in the last ${STATUS_LOG_HEALTH_SINCE} (${logHealth.summary}).` : `ℹ️ Log health: unchecked (no logs found under ${logRoot}).`);
|
|
145
|
+
if (locks.length > 0) {
|
|
146
|
+
console.log("");
|
|
147
|
+
console.log(`🔒 Active/stale maintenance step locks (${locks.length}):`);
|
|
148
|
+
for (const lock of locks) console.log(` step--${lock.stepName} ${formatStepLockDetail(lock)}`);
|
|
149
|
+
console.log(" A lock here will make `maintenance step <name> --force` report skipped_guard until it clears.");
|
|
150
|
+
}
|
|
123
151
|
}));
|
|
124
152
|
maintenance.command("cron-health").description("Check if expected cron jobs exist and have fired recently. Logs warnings for missing or stale jobs. Useful in heartbeat checks.").action(withExit(async () => {
|
|
125
153
|
const openclawDir = join(homedir(), ".openclaw");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"register-maintenance-health.js","names":[],"sources":["../../../../cli/commands/manage/register-maintenance-health.ts"],"sourcesContent":["/**\n * Maintenance inventory, status, and cron-health subcommands (Issue #281).\n */\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\n\nimport type { HybridMemoryConfig } from \"../../../config.js\";\nimport {\n collectMaintenanceInventory,\n renderMaintenanceInventoryMarkdown,\n renderMaintenanceInventoryText,\n} from \"../../../services/maintenance-inventory.js\";\nimport { formatTimestampUtcFromMs } from \"../../../utils/dates.js\";\nimport { readOpenClawCronStore, describeCronStoreLocation } from \"../../../services/openclaw-cron-store.js\";\nimport { type Chainable, relativeTime, withExit } from \"../../shared.js\";\n\nexport function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: HybridMemoryConfig): void {\n maintenance\n .command(\"inventory\")\n .description(\"List host-crontab and gateway-cron maintenance jobs in one report.\")\n .option(\"--json\", \"Output as JSON\")\n .option(\"--format <format>\", \"Output format: text, markdown, json\", \"text\")\n .action(\n withExit(async (opts?: { json?: boolean; format?: string }) => {\n const requestedFormat = (opts?.format ?? \"text\").toLowerCase();\n if (opts?.json && requestedFormat === \"markdown\") {\n throw new Error(\"Use either --json or --format markdown, not both.\");\n }\n const format = opts?.json ? \"json\" : requestedFormat;\n if (![\"text\", \"markdown\", \"json\"].includes(format)) {\n throw new Error(`Unsupported inventory format: ${opts?.format ?? \"\"}`);\n }\n\n const report = collectMaintenanceInventory(join(homedir(), \".openclaw\"));\n if (format === \"json\") {\n console.log(JSON.stringify(report, null, 2));\n return;\n }\n if (format === \"markdown\") {\n console.log(renderMaintenanceInventoryMarkdown(report));\n return;\n }\n console.log(renderMaintenanceInventoryText(report));\n }),\n );\n\n maintenance\n .command(\"status\")\n .description(\"Show maintenance cron job health: nightly cycle, weekly backup, and any reliability issues.\")\n .option(\"--json\", \"Output as JSON\")\n .action(\n withExit(async (opts?: { json?: boolean }) => {\n const openclawDir = join(homedir(), \".openclaw\");\n const staleThresholdMs = (cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28) * 60 * 60 * 1000;\n const nightlyCronExpr = cfg.maintenance?.cronReliability?.nightlyCron ?? \"0 3 * * *\";\n const weeklyBackupCronExpr = cfg.maintenance?.cronReliability?.weeklyBackupCron ?? \"0 4 * * 0\";\n const useConsolidatedCron = cfg.maintenance?.orchestrator?.consolidatedCronJobs !== false;\n\n type JobStatus = {\n name: string;\n pluginJobId: string;\n enabled: boolean;\n lastRunAt: string | null;\n nextRunAt: string | null;\n lastStatus: string | null;\n isStale: boolean;\n isMissing: boolean;\n configuredSchedule: string;\n issue?: string;\n };\n\n const jobsOfInterest: Array<{\n id: string;\n label: string;\n scheduleExpr: string;\n staleMs: number;\n }> = useConsolidatedCron\n ? [\n {\n id: \"hybrid-mem:maintenance-nightly\",\n label: \"maintenance-nightly\",\n scheduleExpr: nightlyCronExpr,\n staleMs: staleThresholdMs,\n },\n ]\n : [\n {\n id: \"hybrid-mem:nightly-distill\",\n label: \"nightly-memory-sweep (legacy)\",\n scheduleExpr: nightlyCronExpr,\n staleMs: staleThresholdMs,\n },\n ];\n\n const results: JobStatus[] = [];\n\n let cronStore: { jobs?: unknown[] } = { jobs: [] };\n try {\n cronStore = readOpenClawCronStore(openclawDir).store;\n } catch {\n // corrupt store — treat all as missing\n }\n\n const jobs = Array.isArray(cronStore.jobs) ? (cronStore.jobs as Array<Record<string, unknown>>) : [];\n\n for (const wanted of jobsOfInterest) {\n const found = jobs.find((j) => j && (j.pluginJobId === wanted.id || String(j.name ?? \"\") === wanted.label));\n\n if (!found) {\n results.push({\n name: wanted.label,\n pluginJobId: wanted.id,\n enabled: false,\n lastRunAt: null,\n nextRunAt: null,\n lastStatus: null,\n isStale: false,\n isMissing: true,\n configuredSchedule: wanted.scheduleExpr,\n issue: \"Job not found in cron store — run `hybrid-mem verify --fix` to install.\",\n });\n continue;\n }\n\n const enabled = found.enabled !== false;\n const state = found.state as\n | {\n nextRunAtMs?: number;\n lastRunAtMs?: number;\n lastStatus?: string;\n lastError?: string;\n }\n | undefined;\n const lastRunAtMs = state?.lastRunAtMs;\n const nextRunAtMs = state?.nextRunAtMs;\n const lastStatus = state?.lastStatus ?? null;\n\n const isStale = enabled && lastRunAtMs != null && Date.now() - lastRunAtMs > wanted.staleMs;\n const neverRan = enabled && lastRunAtMs == null;\n\n let issue: string | undefined;\n if (!enabled) {\n issue = \"Job is disabled.\";\n } else if (neverRan) {\n issue = \"Job has never run — check cron daemon is running.\";\n } else if (isStale) {\n const hoursSince = Math.floor((Date.now() - (lastRunAtMs ?? 0)) / 3600000);\n issue = `Job is stale — last run was ${hoursSince}h ago (threshold: ${Math.floor(\n wanted.staleMs / 3600000,\n )}h).`;\n } else if (lastStatus === \"error\") {\n issue = `Last run failed: ${state?.lastError ?? \"unknown error\"}`;\n }\n\n results.push({\n name: wanted.label,\n pluginJobId: wanted.id,\n enabled,\n lastRunAt: lastRunAtMs != null ? formatTimestampUtcFromMs(lastRunAtMs) : null,\n nextRunAt: nextRunAtMs != null ? formatTimestampUtcFromMs(nextRunAtMs) : null,\n lastStatus,\n isStale,\n isMissing: false,\n configuredSchedule: wanted.scheduleExpr,\n issue,\n });\n }\n\n const issues = results.filter((r) => r.issue);\n\n if (issues.length > 0) {\n process.exitCode = 1;\n }\n\n if (opts?.json) {\n console.log(\n JSON.stringify(\n {\n ok: issues.length === 0,\n jobs: results,\n issueCount: issues.length,\n },\n null,\n 2,\n ),\n );\n return;\n }\n\n console.log(\"Memory Maintenance Status (Issue #281)\");\n console.log(\"========================================\");\n console.log(`Cron store: ${describeCronStoreLocation(openclawDir)}`);\n console.log(`Stale threshold (daily): ${cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28}h`);\n console.log(\"\");\n\n for (const r of results) {\n const icon = r.isMissing ? \"❌\" : !r.enabled ? \"⏸ \" : r.issue ? \"⚠️ \" : \"✅\";\n const lastRun = r.lastRunAt\n ? `last: ${relativeTime(new Date(r.lastRunAt).getTime())} (${r.lastStatus ?? \"unknown\"})`\n : \"last: never\";\n const nextRun = r.nextRunAt ? `next: ${relativeTime(new Date(r.nextRunAt).getTime())}` : \"\";\n const timing = [lastRun, nextRun].filter(Boolean).join(\" \");\n console.log(\n `${icon} ${r.name.padEnd(32)} ${r.isMissing ? \"MISSING\" : r.enabled ? \"enabled \" : \"disabled\"} ${timing}`,\n );\n if (r.issue) {\n console.log(` └─ ${r.issue}`);\n }\n }\n\n console.log(\"\");\n if (issues.length === 0) {\n console.log(\"✅ All maintenance jobs healthy.\");\n } else {\n console.log(`⚠️ ${issues.length} issue(s) detected. Run \\`hybrid-mem verify --fix\\` to repair.`);\n if (issues.some((r) => r.isMissing)) {\n console.log(\" Missing jobs can be registered with: hybrid-mem install\");\n }\n }\n }),\n );\n\n maintenance\n .command(\"cron-health\")\n .description(\n \"Check if expected cron jobs exist and have fired recently. \" +\n \"Logs warnings for missing or stale jobs. Useful in heartbeat checks.\",\n )\n .action(\n withExit(async () => {\n const openclawDir = join(homedir(), \".openclaw\");\n const staleThresholdMs = (cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28) * 60 * 60 * 1000;\n const useConsolidatedCron = cfg.maintenance?.orchestrator?.consolidatedCronJobs !== false;\n const criticalJobs = useConsolidatedCron ? [\"hybrid-mem:maintenance-nightly\"] : [\"hybrid-mem:nightly-distill\"];\n\n let cronStore: { jobs?: unknown[] } = { jobs: [] };\n try {\n const snapshot = readOpenClawCronStore(openclawDir);\n cronStore = snapshot.store;\n } catch {\n console.warn(\"⚠ Could not read cron store — skipping health check.\");\n process.exitCode = 1;\n return;\n }\n\n const jobs = Array.isArray(cronStore.jobs) ? (cronStore.jobs as Array<Record<string, unknown>>) : [];\n let healthy = true;\n\n for (const id of criticalJobs) {\n const job = jobs.find((j) => j && j.pluginJobId === id);\n if (!job) {\n console.warn(`⚠ Maintenance job missing: ${id}. Run: hybrid-mem install`);\n healthy = false;\n continue;\n }\n if (job.enabled === false) {\n continue;\n }\n const state = job.state as { lastRunAtMs?: number; lastStatus?: string } | undefined;\n if (state?.lastRunAtMs != null && Date.now() - state.lastRunAtMs > staleThresholdMs) {\n const h = Math.floor((Date.now() - state.lastRunAtMs) / 3600000);\n console.warn(`⚠ Stale maintenance job: ${id} (last run ${h}h ago). Check cron daemon.`);\n healthy = false;\n }\n }\n\n if (healthy) {\n console.log(\"✓ Maintenance cron jobs healthy.\");\n } else {\n process.exitCode = 1;\n }\n }),\n );\n}\n"],"mappings":";;;;;;;;;;AAgBA,SAAgB,kCAAkC,aAAwB,KAA+B;CACvG,YACG,QAAQ,WAAW,CAAC,CACpB,YAAY,oEAAoE,CAAC,CACjF,OAAO,UAAU,gBAAgB,CAAC,CAClC,OAAO,qBAAqB,uCAAuC,MAAM,CAAC,CAC1E,OACC,SAAS,OAAO,SAA+C;EAC7D,MAAM,mBAAmB,MAAM,UAAU,OAAA,CAAQ,YAAY;EAC7D,IAAI,MAAM,QAAQ,oBAAoB,YACpC,MAAM,IAAI,MAAM,mDAAmD;EAErE,MAAM,SAAS,MAAM,OAAO,SAAS;EACrC,IAAI,CAAC;GAAC;GAAQ;GAAY;EAAM,CAAC,CAAC,SAAS,MAAM,GAC/C,MAAM,IAAI,MAAM,iCAAiC,MAAM,UAAU,IAAI;EAGvE,MAAM,SAAS,4BAA4B,KAAK,QAAQ,GAAG,WAAW,CAAC;EACvE,IAAI,WAAW,QAAQ;GACrB,QAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;GAC3C;EACF;EACA,IAAI,WAAW,YAAY;GACzB,QAAQ,IAAI,mCAAmC,MAAM,CAAC;GACtD;EACF;EACA,QAAQ,IAAI,+BAA+B,MAAM,CAAC;CACpD,CAAC,CACH;CAEF,YACG,QAAQ,QAAQ,CAAC,CACjB,YAAY,6FAA6F,CAAC,CAC1G,OAAO,UAAU,gBAAgB,CAAC,CAClC,OACC,SAAS,OAAO,SAA8B;EAC5C,MAAM,cAAc,KAAK,QAAQ,GAAG,WAAW;EAC/C,MAAM,oBAAoB,IAAI,aAAa,iBAAiB,uBAAuB,MAAM,KAAK,KAAK;EACnG,MAAM,kBAAkB,IAAI,aAAa,iBAAiB,eAAe;EAC5C,IAAI,aAAa,iBAAiB;EAgB/D,MAAM,iBAfsB,IAAI,aAAa,cAAc,yBAAyB,QAqBhF,CACE;GACE,IAAI;GACJ,OAAO;GACP,cAAc;GACd,SAAS;EACX,CACF,IACA,CACE;GACE,IAAI;GACJ,OAAO;GACP,cAAc;GACd,SAAS;EACX,CACF;EAEJ,MAAM,UAAuB,CAAC;EAE9B,IAAI,YAAkC,EAAE,MAAM,CAAC,EAAE;EACjD,IAAI;GACF,YAAY,sBAAsB,WAAW,CAAC,CAAC;EACjD,QAAQ,CAER;EAEA,MAAM,OAAO,MAAM,QAAQ,UAAU,IAAI,IAAK,UAAU,OAA0C,CAAC;EAEnG,KAAK,MAAM,UAAU,gBAAgB;GACnC,MAAM,QAAQ,KAAK,MAAM,MAAM,MAAM,EAAE,gBAAgB,OAAO,MAAM,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,MAAM;GAE1G,IAAI,CAAC,OAAO;IACV,QAAQ,KAAK;KACX,MAAM,OAAO;KACb,aAAa,OAAO;KACpB,SAAS;KACT,WAAW;KACX,WAAW;KACX,YAAY;KACZ,SAAS;KACT,WAAW;KACX,oBAAoB,OAAO;KAC3B,OAAO;IACT,CAAC;IACD;GACF;GAEA,MAAM,UAAU,MAAM,YAAY;GAClC,MAAM,QAAQ,MAAM;GAQpB,MAAM,cAAc,OAAO;GAC3B,MAAM,cAAc,OAAO;GAC3B,MAAM,aAAa,OAAO,cAAc;GAExC,MAAM,UAAU,WAAW,eAAe,QAAQ,KAAK,IAAI,IAAI,cAAc,OAAO;GACpF,MAAM,WAAW,WAAW,eAAe;GAE3C,IAAI;GACJ,IAAI,CAAC,SACH,QAAQ;QACH,IAAI,UACT,QAAQ;QACH,IAAI,SAET,QAAQ,+BADW,KAAK,OAAO,KAAK,IAAI,KAAK,eAAe,MAAM,IAClB,EAAE,oBAAoB,KAAK,MACzE,OAAO,UAAU,IACnB,EAAE;QACG,IAAI,eAAe,SACxB,QAAQ,oBAAoB,OAAO,aAAa;GAGlD,QAAQ,KAAK;IACX,MAAM,OAAO;IACb,aAAa,OAAO;IACpB;IACA,WAAW,eAAe,OAAO,yBAAyB,WAAW,IAAI;IACzE,WAAW,eAAe,OAAO,yBAAyB,WAAW,IAAI;IACzE;IACA;IACA,WAAW;IACX,oBAAoB,OAAO;IAC3B;GACF,CAAC;EACH;EAEA,MAAM,SAAS,QAAQ,QAAQ,MAAM,EAAE,KAAK;EAE5C,IAAI,OAAO,SAAS,GAClB,QAAQ,WAAW;EAGrB,IAAI,MAAM,MAAM;GACd,QAAQ,IACN,KAAK,UACH;IACE,IAAI,OAAO,WAAW;IACtB,MAAM;IACN,YAAY,OAAO;GACrB,GACA,MACA,CACF,CACF;GACA;EACF;EAEA,QAAQ,IAAI,wCAAwC;EACpD,QAAQ,IAAI,0CAA0C;EACtD,QAAQ,IAAI,eAAe,0BAA0B,WAAW,GAAG;EACnE,QAAQ,IAAI,4BAA4B,IAAI,aAAa,iBAAiB,uBAAuB,GAAG,EAAE;EACtG,QAAQ,IAAI,EAAE;EAEd,KAAK,MAAM,KAAK,SAAS;GACvB,MAAM,OAAO,EAAE,YAAY,MAAM,CAAC,EAAE,UAAU,OAAO,EAAE,QAAQ,QAAQ;GAKvE,MAAM,SAAS,CAJC,EAAE,YACd,SAAS,aAAa,IAAI,KAAK,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,cAAc,UAAU,KACrF,eACY,EAAE,YAAY,SAAS,aAAa,IAAI,KAAK,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,EACzD,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI;GAC3D,QAAQ,IACN,GAAG,KAAK,GAAG,EAAE,KAAK,OAAO,EAAE,EAAE,GAAG,EAAE,YAAY,YAAY,EAAE,UAAU,aAAa,WAAW,GAAG,QACnG;GACA,IAAI,EAAE,OACJ,QAAQ,IAAI,SAAS,EAAE,OAAO;EAElC;EAEA,QAAQ,IAAI,EAAE;EACd,IAAI,OAAO,WAAW,GACpB,QAAQ,IAAI,iCAAiC;OACxC;GACL,QAAQ,IAAI,OAAO,OAAO,OAAO,+DAA+D;GAChG,IAAI,OAAO,MAAM,MAAM,EAAE,SAAS,GAChC,QAAQ,IAAI,4DAA4D;EAE5E;CACF,CAAC,CACH;CAEF,YACG,QAAQ,aAAa,CAAC,CACtB,YACC,iIAEF,CAAC,CACA,OACC,SAAS,YAAY;EACnB,MAAM,cAAc,KAAK,QAAQ,GAAG,WAAW;EAC/C,MAAM,oBAAoB,IAAI,aAAa,iBAAiB,uBAAuB,MAAM,KAAK,KAAK;EAEnG,MAAM,eADsB,IAAI,aAAa,cAAc,yBAAyB,QACzC,CAAC,gCAAgC,IAAI,CAAC,4BAA4B;EAE7G,IAAI,YAAkC,EAAE,MAAM,CAAC,EAAE;EACjD,IAAI;GAEF,YADiB,sBAAsB,WACpB,CAAC,CAAC;EACvB,QAAQ;GACN,QAAQ,KAAK,sDAAsD;GACnE,QAAQ,WAAW;GACnB;EACF;EAEA,MAAM,OAAO,MAAM,QAAQ,UAAU,IAAI,IAAK,UAAU,OAA0C,CAAC;EACnG,IAAI,UAAU;EAEd,KAAK,MAAM,MAAM,cAAc;GAC7B,MAAM,MAAM,KAAK,MAAM,MAAM,KAAK,EAAE,gBAAgB,EAAE;GACtD,IAAI,CAAC,KAAK;IACR,QAAQ,KAAK,8BAA8B,GAAG,0BAA0B;IACxE,UAAU;IACV;GACF;GACA,IAAI,IAAI,YAAY,OAClB;GAEF,MAAM,QAAQ,IAAI;GAClB,IAAI,OAAO,eAAe,QAAQ,KAAK,IAAI,IAAI,MAAM,cAAc,kBAAkB;IACnF,MAAM,IAAI,KAAK,OAAO,KAAK,IAAI,IAAI,MAAM,eAAe,IAAO;IAC/D,QAAQ,KAAK,4BAA4B,GAAG,aAAa,EAAE,2BAA2B;IACtF,UAAU;GACZ;EACF;EAEA,IAAI,SACF,QAAQ,IAAI,kCAAkC;OAE9C,QAAQ,WAAW;CAEvB,CAAC,CACH;AACJ"}
|
|
1
|
+
{"version":3,"file":"register-maintenance-health.js","names":[],"sources":["../../../../cli/commands/manage/register-maintenance-health.ts"],"sourcesContent":["/**\n * Maintenance inventory, status, and cron-health subcommands (Issue #281).\n */\nimport { existsSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\n\nimport type { HybridMemoryConfig } from \"../../../config.js\";\nimport { formatStepLockDetail, listActiveStepLocks } from \"../../../services/cron-guard.js\";\nimport {\n collectMaintenanceInventory,\n renderMaintenanceInventoryMarkdown,\n renderMaintenanceInventoryText,\n} from \"../../../services/maintenance-inventory.js\";\nimport {\n collectMaintenanceSteps,\n summarizeMaintenanceLogAnalysis,\n} from \"../../../services/maintenance-log-analyzer.js\";\nimport { describeCronStoreLocation, readOpenClawCronStore } from \"../../../services/openclaw-cron-store.js\";\nimport { formatTimestampUtcFromMs } from \"../../../utils/dates.js\";\nimport { type Chainable, relativeTime, withExit } from \"../../shared.js\";\n\n/** Lookback window for the log-health check folded into `maintenance status` (#2033). */\nconst STATUS_LOG_HEALTH_SINCE = \"24h\";\n\nexport function registerMaintenanceHealthCommands(maintenance: Chainable, cfg: HybridMemoryConfig): void {\n maintenance\n .command(\"inventory\")\n .description(\"List host-crontab and gateway-cron maintenance jobs in one report.\")\n .option(\"--json\", \"Output as JSON\")\n .option(\"--format <format>\", \"Output format: text, markdown, json\", \"text\")\n .action(\n withExit(async (opts?: { json?: boolean; format?: string }) => {\n const requestedFormat = (opts?.format ?? \"text\").toLowerCase();\n if (opts?.json && requestedFormat === \"markdown\") {\n throw new Error(\"Use either --json or --format markdown, not both.\");\n }\n const format = opts?.json ? \"json\" : requestedFormat;\n if (![\"text\", \"markdown\", \"json\"].includes(format)) {\n throw new Error(`Unsupported inventory format: ${opts?.format ?? \"\"}`);\n }\n\n const report = collectMaintenanceInventory(join(homedir(), \".openclaw\"));\n if (format === \"json\") {\n console.log(JSON.stringify(report, null, 2));\n return;\n }\n if (format === \"markdown\") {\n console.log(renderMaintenanceInventoryMarkdown(report));\n return;\n }\n console.log(renderMaintenanceInventoryText(report));\n }),\n );\n\n maintenance\n .command(\"status\")\n .description(\"Show maintenance cron job health: nightly cycle, weekly backup, and any reliability issues.\")\n .option(\"--json\", \"Output as JSON\")\n .option(\n \"-v, --verbose\",\n \"Also show log-health detail when there are no strict findings (active/stale locks are always shown when present)\",\n )\n .action(\n withExit(async (opts?: { json?: boolean; verbose?: boolean }) => {\n const openclawDir = join(homedir(), \".openclaw\");\n const staleThresholdMs = (cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28) * 60 * 60 * 1000;\n const nightlyCronExpr = cfg.maintenance?.cronReliability?.nightlyCron ?? \"0 3 * * *\";\n const weeklyBackupCronExpr = cfg.maintenance?.cronReliability?.weeklyBackupCron ?? \"0 4 * * 0\";\n const useConsolidatedCron = cfg.maintenance?.orchestrator?.consolidatedCronJobs !== false;\n\n type JobStatus = {\n name: string;\n pluginJobId: string;\n enabled: boolean;\n lastRunAt: string | null;\n nextRunAt: string | null;\n lastStatus: string | null;\n isStale: boolean;\n isMissing: boolean;\n configuredSchedule: string;\n issue?: string;\n };\n\n const jobsOfInterest: Array<{\n id: string;\n label: string;\n scheduleExpr: string;\n staleMs: number;\n }> = useConsolidatedCron\n ? [\n {\n id: \"hybrid-mem:maintenance-nightly\",\n label: \"maintenance-nightly\",\n scheduleExpr: nightlyCronExpr,\n staleMs: staleThresholdMs,\n },\n ]\n : [\n {\n id: \"hybrid-mem:nightly-distill\",\n label: \"nightly-memory-sweep (legacy)\",\n scheduleExpr: nightlyCronExpr,\n staleMs: staleThresholdMs,\n },\n ];\n\n const results: JobStatus[] = [];\n\n let cronStore: { jobs?: unknown[] } = { jobs: [] };\n try {\n cronStore = readOpenClawCronStore(openclawDir).store;\n } catch {\n // corrupt store — treat all as missing\n }\n\n const jobs = Array.isArray(cronStore.jobs) ? (cronStore.jobs as Array<Record<string, unknown>>) : [];\n\n for (const wanted of jobsOfInterest) {\n const found = jobs.find((j) => j && (j.pluginJobId === wanted.id || String(j.name ?? \"\") === wanted.label));\n\n if (!found) {\n results.push({\n name: wanted.label,\n pluginJobId: wanted.id,\n enabled: false,\n lastRunAt: null,\n nextRunAt: null,\n lastStatus: null,\n isStale: false,\n isMissing: true,\n configuredSchedule: wanted.scheduleExpr,\n issue: \"Job not found in cron store — run `hybrid-mem verify --fix` to install.\",\n });\n continue;\n }\n\n const enabled = found.enabled !== false;\n const state = found.state as\n | {\n nextRunAtMs?: number;\n lastRunAtMs?: number;\n lastStatus?: string;\n lastError?: string;\n }\n | undefined;\n const lastRunAtMs = state?.lastRunAtMs;\n const nextRunAtMs = state?.nextRunAtMs;\n const lastStatus = state?.lastStatus ?? null;\n\n const isStale = enabled && lastRunAtMs != null && Date.now() - lastRunAtMs > wanted.staleMs;\n const neverRan = enabled && lastRunAtMs == null;\n\n let issue: string | undefined;\n if (!enabled) {\n issue = \"Job is disabled.\";\n } else if (neverRan) {\n issue = \"Job has never run — check cron daemon is running.\";\n } else if (isStale) {\n const hoursSince = Math.floor((Date.now() - (lastRunAtMs ?? 0)) / 3600000);\n issue = `Job is stale — last run was ${hoursSince}h ago (threshold: ${Math.floor(\n wanted.staleMs / 3600000,\n )}h).`;\n } else if (lastStatus === \"error\") {\n issue = `Last run failed: ${state?.lastError ?? \"unknown error\"}`;\n }\n\n results.push({\n name: wanted.label,\n pluginJobId: wanted.id,\n enabled,\n lastRunAt: lastRunAtMs != null ? formatTimestampUtcFromMs(lastRunAtMs) : null,\n nextRunAt: nextRunAtMs != null ? formatTimestampUtcFromMs(nextRunAtMs) : null,\n lastStatus,\n isStale,\n isMissing: false,\n configuredSchedule: wanted.scheduleExpr,\n issue,\n });\n }\n\n const issues = results.filter((r) => r.issue);\n\n // Log health (#2033): scheduler cadence looking healthy must not imply the maintenance\n // logs are clean — analyze-maintenance-logs can report strict findings for the same window\n // even when cron cadence looks fine, and an operator who stops at `status` would miss it.\n const logRoot = join(openclawDir, \"logs\", \"cron-hybrid-mem\");\n let logHealth: { summary: string; strictFailed: boolean; findingsCount: number } | null = null;\n if (existsSync(logRoot)) {\n try {\n const steps = collectMaintenanceSteps(logRoot, STATUS_LOG_HEALTH_SINCE);\n logHealth = summarizeMaintenanceLogAnalysis(steps);\n } catch {\n // Best-effort — a log-analysis failure must not break the cron-freshness check below.\n logHealth = null;\n }\n }\n\n // Active/stale step locks (#2031): surfaces what `maintenance step <name> --force` would\n // report as \"locked by a concurrent maintenance run\" instead of only discovering it there.\n const locks = listActiveStepLocks(openclawDir);\n\n const schedulerHealthy = issues.length === 0;\n const logHealthy = !logHealth?.strictFailed;\n\n if (issues.length > 0 || logHealth?.strictFailed) {\n process.exitCode = 1;\n }\n\n if (opts?.json) {\n console.log(\n JSON.stringify(\n {\n ok: schedulerHealthy && logHealthy,\n jobs: results,\n issueCount: issues.length,\n logHealth: logHealth ? { ...logHealth, since: STATUS_LOG_HEALTH_SINCE } : null,\n locks,\n },\n null,\n 2,\n ),\n );\n return;\n }\n\n console.log(\"Memory Maintenance Status (Issue #281)\");\n console.log(\"========================================\");\n console.log(`Cron store: ${describeCronStoreLocation(openclawDir)}`);\n console.log(`Stale threshold (daily): ${cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28}h`);\n console.log(\"\");\n\n for (const r of results) {\n const icon = r.isMissing ? \"❌\" : !r.enabled ? \"⏸ \" : r.issue ? \"⚠️ \" : \"✅\";\n const lastRun = r.lastRunAt\n ? `last: ${relativeTime(new Date(r.lastRunAt).getTime())} (${r.lastStatus ?? \"unknown\"})`\n : \"last: never\";\n const nextRun = r.nextRunAt ? `next: ${relativeTime(new Date(r.nextRunAt).getTime())}` : \"\";\n const timing = [lastRun, nextRun].filter(Boolean).join(\" \");\n console.log(\n `${icon} ${r.name.padEnd(32)} ${r.isMissing ? \"MISSING\" : r.enabled ? \"enabled \" : \"disabled\"} ${timing}`,\n );\n if (r.issue) {\n console.log(` └─ ${r.issue}`);\n }\n }\n\n console.log(\"\");\n if (schedulerHealthy) {\n console.log(\"✅ Scheduler freshness: all maintenance jobs healthy (cron cadence only).\");\n } else {\n console.log(`⚠️ ${issues.length} scheduler issue(s) detected. Run \\`hybrid-mem verify --fix\\` to repair.`);\n if (issues.some((r) => r.isMissing)) {\n console.log(\" Missing jobs can be registered with: hybrid-mem install\");\n }\n }\n\n if (logHealth?.strictFailed) {\n console.log(\n `⚠️ Log health: analyze-maintenance-logs reports strict findings in the last ${STATUS_LOG_HEALTH_SINCE} ` +\n `(${logHealth.summary}). Run \\`hybrid-mem maintenance step analyze-maintenance-logs --force --verbose\\` for detail.`,\n );\n } else if (opts?.verbose) {\n console.log(\n logHealth\n ? `✅ Log health: no strict findings in the last ${STATUS_LOG_HEALTH_SINCE} (${logHealth.summary}).`\n : `ℹ️ Log health: unchecked (no logs found under ${logRoot}).`,\n );\n }\n\n if (locks.length > 0) {\n console.log(\"\");\n console.log(`🔒 Active/stale maintenance step locks (${locks.length}):`);\n for (const lock of locks) {\n console.log(` step--${lock.stepName} ${formatStepLockDetail(lock)}`);\n }\n console.log(\n \" A lock here will make `maintenance step <name> --force` report skipped_guard until it clears.\",\n );\n }\n }),\n );\n\n maintenance\n .command(\"cron-health\")\n .description(\n \"Check if expected cron jobs exist and have fired recently. \" +\n \"Logs warnings for missing or stale jobs. Useful in heartbeat checks.\",\n )\n .action(\n withExit(async () => {\n const openclawDir = join(homedir(), \".openclaw\");\n const staleThresholdMs = (cfg.maintenance?.cronReliability?.staleThresholdHours ?? 28) * 60 * 60 * 1000;\n const useConsolidatedCron = cfg.maintenance?.orchestrator?.consolidatedCronJobs !== false;\n const criticalJobs = useConsolidatedCron ? [\"hybrid-mem:maintenance-nightly\"] : [\"hybrid-mem:nightly-distill\"];\n\n let cronStore: { jobs?: unknown[] } = { jobs: [] };\n try {\n const snapshot = readOpenClawCronStore(openclawDir);\n cronStore = snapshot.store;\n } catch {\n console.warn(\"⚠ Could not read cron store — skipping health check.\");\n process.exitCode = 1;\n return;\n }\n\n const jobs = Array.isArray(cronStore.jobs) ? (cronStore.jobs as Array<Record<string, unknown>>) : [];\n let healthy = true;\n\n for (const id of criticalJobs) {\n const job = jobs.find((j) => j && j.pluginJobId === id);\n if (!job) {\n console.warn(`⚠ Maintenance job missing: ${id}. Run: hybrid-mem install`);\n healthy = false;\n continue;\n }\n if (job.enabled === false) {\n continue;\n }\n const state = job.state as { lastRunAtMs?: number; lastStatus?: string } | undefined;\n if (state?.lastRunAtMs != null && Date.now() - state.lastRunAtMs > staleThresholdMs) {\n const h = Math.floor((Date.now() - state.lastRunAtMs) / 3600000);\n console.warn(`⚠ Stale maintenance job: ${id} (last run ${h}h ago). Check cron daemon.`);\n healthy = false;\n }\n }\n\n if (healthy) {\n console.log(\"✓ Maintenance cron jobs healthy.\");\n } else {\n process.exitCode = 1;\n }\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,MAAM,0BAA0B;AAEhC,SAAgB,kCAAkC,aAAwB,KAA+B;CACvG,YACG,QAAQ,WAAW,CAAC,CACpB,YAAY,oEAAoE,CAAC,CACjF,OAAO,UAAU,gBAAgB,CAAC,CAClC,OAAO,qBAAqB,uCAAuC,MAAM,CAAC,CAC1E,OACC,SAAS,OAAO,SAA+C;EAC7D,MAAM,mBAAmB,MAAM,UAAU,OAAA,CAAQ,YAAY;EAC7D,IAAI,MAAM,QAAQ,oBAAoB,YACpC,MAAM,IAAI,MAAM,mDAAmD;EAErE,MAAM,SAAS,MAAM,OAAO,SAAS;EACrC,IAAI,CAAC;GAAC;GAAQ;GAAY;EAAM,CAAC,CAAC,SAAS,MAAM,GAC/C,MAAM,IAAI,MAAM,iCAAiC,MAAM,UAAU,IAAI;EAGvE,MAAM,SAAS,4BAA4B,KAAK,QAAQ,GAAG,WAAW,CAAC;EACvE,IAAI,WAAW,QAAQ;GACrB,QAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;GAC3C;EACF;EACA,IAAI,WAAW,YAAY;GACzB,QAAQ,IAAI,mCAAmC,MAAM,CAAC;GACtD;EACF;EACA,QAAQ,IAAI,+BAA+B,MAAM,CAAC;CACpD,CAAC,CACH;CAEF,YACG,QAAQ,QAAQ,CAAC,CACjB,YAAY,6FAA6F,CAAC,CAC1G,OAAO,UAAU,gBAAgB,CAAC,CAClC,OACC,iBACA,kHACF,CAAC,CACA,OACC,SAAS,OAAO,SAAiD;EAC/D,MAAM,cAAc,KAAK,QAAQ,GAAG,WAAW;EAC/C,MAAM,oBAAoB,IAAI,aAAa,iBAAiB,uBAAuB,MAAM,KAAK,KAAK;EACnG,MAAM,kBAAkB,IAAI,aAAa,iBAAiB,eAAe;EAC5C,IAAI,aAAa,iBAAiB;EAgB/D,MAAM,iBAfsB,IAAI,aAAa,cAAc,yBAAyB,QAqBhF,CACE;GACE,IAAI;GACJ,OAAO;GACP,cAAc;GACd,SAAS;EACX,CACF,IACA,CACE;GACE,IAAI;GACJ,OAAO;GACP,cAAc;GACd,SAAS;EACX,CACF;EAEJ,MAAM,UAAuB,CAAC;EAE9B,IAAI,YAAkC,EAAE,MAAM,CAAC,EAAE;EACjD,IAAI;GACF,YAAY,sBAAsB,WAAW,CAAC,CAAC;EACjD,QAAQ,CAER;EAEA,MAAM,OAAO,MAAM,QAAQ,UAAU,IAAI,IAAK,UAAU,OAA0C,CAAC;EAEnG,KAAK,MAAM,UAAU,gBAAgB;GACnC,MAAM,QAAQ,KAAK,MAAM,MAAM,MAAM,EAAE,gBAAgB,OAAO,MAAM,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,MAAM;GAE1G,IAAI,CAAC,OAAO;IACV,QAAQ,KAAK;KACX,MAAM,OAAO;KACb,aAAa,OAAO;KACpB,SAAS;KACT,WAAW;KACX,WAAW;KACX,YAAY;KACZ,SAAS;KACT,WAAW;KACX,oBAAoB,OAAO;KAC3B,OAAO;IACT,CAAC;IACD;GACF;GAEA,MAAM,UAAU,MAAM,YAAY;GAClC,MAAM,QAAQ,MAAM;GAQpB,MAAM,cAAc,OAAO;GAC3B,MAAM,cAAc,OAAO;GAC3B,MAAM,aAAa,OAAO,cAAc;GAExC,MAAM,UAAU,WAAW,eAAe,QAAQ,KAAK,IAAI,IAAI,cAAc,OAAO;GACpF,MAAM,WAAW,WAAW,eAAe;GAE3C,IAAI;GACJ,IAAI,CAAC,SACH,QAAQ;QACH,IAAI,UACT,QAAQ;QACH,IAAI,SAET,QAAQ,+BADW,KAAK,OAAO,KAAK,IAAI,KAAK,eAAe,MAAM,IAClB,EAAE,oBAAoB,KAAK,MACzE,OAAO,UAAU,IACnB,EAAE;QACG,IAAI,eAAe,SACxB,QAAQ,oBAAoB,OAAO,aAAa;GAGlD,QAAQ,KAAK;IACX,MAAM,OAAO;IACb,aAAa,OAAO;IACpB;IACA,WAAW,eAAe,OAAO,yBAAyB,WAAW,IAAI;IACzE,WAAW,eAAe,OAAO,yBAAyB,WAAW,IAAI;IACzE;IACA;IACA,WAAW;IACX,oBAAoB,OAAO;IAC3B;GACF,CAAC;EACH;EAEA,MAAM,SAAS,QAAQ,QAAQ,MAAM,EAAE,KAAK;EAK5C,MAAM,UAAU,KAAK,aAAa,QAAQ,iBAAiB;EAC3D,IAAI,YAAsF;EAC1F,IAAI,WAAW,OAAO,GACpB,IAAI;GAEF,YAAY,gCADE,wBAAwB,SAAS,uBACC,CAAC;EACnD,QAAQ;GAEN,YAAY;EACd;EAKF,MAAM,QAAQ,oBAAoB,WAAW;EAE7C,MAAM,mBAAmB,OAAO,WAAW;EAC3C,MAAM,aAAa,CAAC,WAAW;EAE/B,IAAI,OAAO,SAAS,KAAK,WAAW,cAClC,QAAQ,WAAW;EAGrB,IAAI,MAAM,MAAM;GACd,QAAQ,IACN,KAAK,UACH;IACE,IAAI,oBAAoB;IACxB,MAAM;IACN,YAAY,OAAO;IACnB,WAAW,YAAY;KAAE,GAAG;KAAW,OAAO;IAAwB,IAAI;IAC1E;GACF,GACA,MACA,CACF,CACF;GACA;EACF;EAEA,QAAQ,IAAI,wCAAwC;EACpD,QAAQ,IAAI,0CAA0C;EACtD,QAAQ,IAAI,eAAe,0BAA0B,WAAW,GAAG;EACnE,QAAQ,IAAI,4BAA4B,IAAI,aAAa,iBAAiB,uBAAuB,GAAG,EAAE;EACtG,QAAQ,IAAI,EAAE;EAEd,KAAK,MAAM,KAAK,SAAS;GACvB,MAAM,OAAO,EAAE,YAAY,MAAM,CAAC,EAAE,UAAU,OAAO,EAAE,QAAQ,QAAQ;GAKvE,MAAM,SAAS,CAJC,EAAE,YACd,SAAS,aAAa,IAAI,KAAK,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,cAAc,UAAU,KACrF,eACY,EAAE,YAAY,SAAS,aAAa,IAAI,KAAK,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,EACzD,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI;GAC3D,QAAQ,IACN,GAAG,KAAK,GAAG,EAAE,KAAK,OAAO,EAAE,EAAE,GAAG,EAAE,YAAY,YAAY,EAAE,UAAU,aAAa,WAAW,GAAG,QACnG;GACA,IAAI,EAAE,OACJ,QAAQ,IAAI,SAAS,EAAE,OAAO;EAElC;EAEA,QAAQ,IAAI,EAAE;EACd,IAAI,kBACF,QAAQ,IAAI,0EAA0E;OACjF;GACL,QAAQ,IAAI,OAAO,OAAO,OAAO,yEAAyE;GAC1G,IAAI,OAAO,MAAM,MAAM,EAAE,SAAS,GAChC,QAAQ,IAAI,4DAA4D;EAE5E;EAEA,IAAI,WAAW,cACb,QAAQ,IACN,gFAAgF,wBAAwB,IAClG,UAAU,QAAQ,8FAC1B;OACK,IAAI,MAAM,SACf,QAAQ,IACN,YACI,gDAAgD,wBAAwB,IAAI,UAAU,QAAQ,MAC9F,kDAAkD,QAAQ,GAChE;EAGF,IAAI,MAAM,SAAS,GAAG;GACpB,QAAQ,IAAI,EAAE;GACd,QAAQ,IAAI,2CAA2C,MAAM,OAAO,GAAG;GACvE,KAAK,MAAM,QAAQ,OACjB,QAAQ,IAAI,YAAY,KAAK,SAAS,IAAI,qBAAqB,IAAI,GAAG;GAExE,QAAQ,IACN,kGACF;EACF;CACF,CAAC,CACH;CAEF,YACG,QAAQ,aAAa,CAAC,CACtB,YACC,iIAEF,CAAC,CACA,OACC,SAAS,YAAY;EACnB,MAAM,cAAc,KAAK,QAAQ,GAAG,WAAW;EAC/C,MAAM,oBAAoB,IAAI,aAAa,iBAAiB,uBAAuB,MAAM,KAAK,KAAK;EAEnG,MAAM,eADsB,IAAI,aAAa,cAAc,yBAAyB,QACzC,CAAC,gCAAgC,IAAI,CAAC,4BAA4B;EAE7G,IAAI,YAAkC,EAAE,MAAM,CAAC,EAAE;EACjD,IAAI;GAEF,YADiB,sBAAsB,WACpB,CAAC,CAAC;EACvB,QAAQ;GACN,QAAQ,KAAK,sDAAsD;GACnE,QAAQ,WAAW;GACnB;EACF;EAEA,MAAM,OAAO,MAAM,QAAQ,UAAU,IAAI,IAAK,UAAU,OAA0C,CAAC;EACnG,IAAI,UAAU;EAEd,KAAK,MAAM,MAAM,cAAc;GAC7B,MAAM,MAAM,KAAK,MAAM,MAAM,KAAK,EAAE,gBAAgB,EAAE;GACtD,IAAI,CAAC,KAAK;IACR,QAAQ,KAAK,8BAA8B,GAAG,0BAA0B;IACxE,UAAU;IACV;GACF;GACA,IAAI,IAAI,YAAY,OAClB;GAEF,MAAM,QAAQ,IAAI;GAClB,IAAI,OAAO,eAAe,QAAQ,KAAK,IAAI,IAAI,MAAM,cAAc,kBAAkB;IACnF,MAAM,IAAI,KAAK,OAAO,KAAK,IAAI,IAAI,MAAM,eAAe,IAAO;IAC/D,QAAQ,KAAK,4BAA4B,GAAG,aAAa,EAAE,2BAA2B;IACtF,UAAU;GACZ;EACF;EAEA,IAAI,SACF,QAAQ,IAAI,kCAAkC;OAE9C,QAAQ,WAAW;CAEvB,CAAC,CACH;AACJ"}
|
|
@@ -33,7 +33,8 @@ function registerMaintenanceOrchestratorCommands(maintenance, b) {
|
|
|
33
33
|
full: opts?.full
|
|
34
34
|
}
|
|
35
35
|
});
|
|
36
|
-
const
|
|
36
|
+
const parsedMaxRuntimeMin = opts?.maxRuntimeMin ? Number(opts.maxRuntimeMin) : void 0;
|
|
37
|
+
const maxRuntimeMs = parsedMaxRuntimeMin !== void 0 && Number.isFinite(parsedMaxRuntimeMin) && parsedMaxRuntimeMin > 0 ? parsedMaxRuntimeMin * 6e4 : void 0;
|
|
37
38
|
const log = opts?.json || !!opts?.summaryOut?.trim() || !!process.env.HM_SUMMARY?.trim() ? (m) => console.error(m) : (m) => console.log(m);
|
|
38
39
|
const warn = (m) => console.error(m);
|
|
39
40
|
const result = await runMaintenanceOrchestrator({
|
|
@@ -71,8 +72,21 @@ function registerMaintenanceOrchestratorCommands(maintenance, b) {
|
|
|
71
72
|
for (const line of lines) console.log(line);
|
|
72
73
|
}
|
|
73
74
|
if (result.exitCode !== 0) process.exitCode = result.exitCode;
|
|
75
|
+
const includeNames = parseStepList(opts?.include);
|
|
76
|
+
if ((opts?.force || opts?.full) && includeNames?.length && !opts?.allowSkip) {
|
|
77
|
+
const requested = includeNames.map((name) => ({
|
|
78
|
+
name,
|
|
79
|
+
result: result.steps.find((s) => s.name === name)
|
|
80
|
+
}));
|
|
81
|
+
if (requested.every((r) => !r.result || r.result.status.startsWith("skipped"))) {
|
|
82
|
+
const detail = requested.map((r) => r.result ? `${r.name}: ${r.result.status} — ${r.result.summary}` : `${r.name}: no result (wrong tier, excluded, or unregistered runner)`).join("; ");
|
|
83
|
+
console.error(`maintenance ${result.tierLabel}: selected step(s) did not run (${detail}). Pass --allow-skip to treat this as a non-strict status check.`);
|
|
84
|
+
if (!process.exitCode) process.exitCode = 3;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return result;
|
|
74
88
|
};
|
|
75
|
-
const commonOptions = (cmd) => cmd.option("--dry-run", "List steps and guard status without executing").option("--force", "Bypass per-step guards and scan watermarks (not feature gates)").option("--full", "Alias for --force").option("-v, --verbose", "Detailed per-step output").option("--include <steps>", "Comma-separated step names to run only").option("--exclude <steps>", "Comma-separated step names to skip").option("--max-runtime-min <n>", "Optional time budget in minutes").option("--json", "Emit orchestrator run summary as JSON").option("--summary-out <path>", "Write orchestrator run summary JSON to path");
|
|
89
|
+
const commonOptions = (cmd) => cmd.option("--dry-run", "List steps and guard status without executing").option("--force", "Bypass per-step guards and scan watermarks (not feature gates)").option("--full", "Alias for --force").option("-v, --verbose", "Detailed per-step output").option("--include <steps>", "Comma-separated step names to run only").option("--exclude <steps>", "Comma-separated step names to skip").option("--max-runtime-min <n>", "Optional time budget in minutes").option("--json", "Emit orchestrator run summary as JSON").option("--summary-out <path>", "Write orchestrator run summary JSON to path").option("--allow-skip", "With --include --force: exit 0 even if the selected step(s) didn't execute (locked/gated/guarded)");
|
|
76
90
|
commonOptions(maintenance.command("cycle").description("Run cycle-tier maintenance steps (local gateway-native work)").action(withExit(async (opts, cmd) => {
|
|
77
91
|
await runTier(["cycle"], {
|
|
78
92
|
...opts,
|
|
@@ -91,7 +105,8 @@ function registerMaintenanceOrchestratorCommands(maintenance, b) {
|
|
|
91
105
|
verbose: !!opts?.verbose || readHybridMemVerbose(cmd)
|
|
92
106
|
});
|
|
93
107
|
})));
|
|
94
|
-
|
|
108
|
+
const DEFAULT_STEP_MAX_RUNTIME_MIN = "15";
|
|
109
|
+
commonOptions(maintenance.command("step <step>").description("Run exactly one named maintenance step in isolation (bypasses that step's cadence guard). Use `maintenance steps` to list valid names.")).action(withExit(async (step, opts, cmd) => {
|
|
95
110
|
const stepName = step?.trim();
|
|
96
111
|
const validNames = listMaintenanceSteps().map((s) => s.name);
|
|
97
112
|
if (!stepName || !validNames.includes(stepName)) {
|
|
@@ -105,9 +120,10 @@ function registerMaintenanceOrchestratorCommands(maintenance, b) {
|
|
|
105
120
|
...opts,
|
|
106
121
|
include: stepName,
|
|
107
122
|
force: true,
|
|
123
|
+
maxRuntimeMin: opts?.maxRuntimeMin ?? DEFAULT_STEP_MAX_RUNTIME_MIN,
|
|
108
124
|
verbose: !!opts?.verbose || readHybridMemVerbose(cmd)
|
|
109
125
|
});
|
|
110
|
-
}))
|
|
126
|
+
}));
|
|
111
127
|
maintenance.command("steps").description("List registered maintenance steps with tier, guard interval, and cadence").option("--json", "Output JSON").action(withExit(async (opts) => {
|
|
112
128
|
const rows = listMaintenanceSteps().map((step) => {
|
|
113
129
|
const guardMs = resolveStepGuardIntervalMs(step, b.cfg);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"register-maintenance-orchestrator.js","names":[],"sources":["../../../../cli/commands/manage/register-maintenance-orchestrator.ts"],"sourcesContent":["/**\n * Maintenance orchestrator CLI: cycle, nightly, full, steps.\n */\n\nimport { existsSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { dirname, join } from \"node:path\";\nimport { atomicWriteFile } from \"../../../utils/atomic-write.js\";\nimport {\n effectiveCadenceLabel,\n formatMaintenanceSummary,\n listMaintenanceSteps,\n resolveStepGuardIntervalMs,\n runMaintenanceOrchestrator,\n toOrchestratorRunSummary,\n} from \"../../../services/maintenance-orchestrator.js\";\nimport { readStepGuardTimestampMs, stepGuardEligible } from \"../../../services/cron-guard.js\";\nimport { formatTimestampUtcFromMs } from \"../../../utils/dates.js\";\nimport { type CommanderOptsParent, readHybridMemVerbose } from \"../../global-verbose.js\";\nimport { type Chainable, withExit } from \"../../shared.js\";\nimport type { ManageBindings } from \"./bindings.js\";\nimport { buildCliMaintenanceRunners } from \"./maintenance-step-runners.js\";\n\nfunction parseStepList(raw?: string): string[] | undefined {\n if (!raw?.trim()) return undefined;\n return raw\n .split(\",\")\n .map((s) => s.trim())\n .filter(Boolean);\n}\n\nexport function registerMaintenanceOrchestratorCommands(maintenance: Chainable, b: ManageBindings): void {\n const openclawDir = join(homedir(), \".openclaw\");\n\n const runTier = async (\n tiers: Array<\"cycle\" | \"nightly\">,\n opts?: {\n dryRun?: boolean;\n force?: boolean;\n full?: boolean;\n verbose?: boolean;\n include?: string;\n exclude?: string;\n maxRuntimeMin?: string;\n json?: boolean;\n summaryOut?: string;\n },\n ) => {\n const verbose = !!opts?.verbose;\n const runStartedMs = Date.now();\n const runStartedIso = new Date(runStartedMs).toISOString();\n if (process.env.HM_RUN_ID) {\n process.env.HM_ORCHESTRATOR_RUN_ID = process.env.HM_RUN_ID;\n }\n const memoryDir = b.resolvedSqlitePath ? dirname(b.resolvedSqlitePath) : join(homedir(), \".openclaw\", \"memory\");\n const backfillMarker = join(memoryDir, \".backfill-decay-done\");\n const runners = buildCliMaintenanceRunners(b, {\n verbose,\n backfillDecayMarker: backfillMarker,\n scanOverrides: { force: opts?.force, full: opts?.full },\n });\n const maxRuntimeMs =\n opts?.maxRuntimeMin && Number.isFinite(Number(opts.maxRuntimeMin))\n ? Number(opts.maxRuntimeMin) * 60_000\n : undefined;\n\n const isJsonMode = opts?.json || !!opts?.summaryOut?.trim() || !!process.env.HM_SUMMARY?.trim();\n const log = isJsonMode ? (m: string) => console.error(m) : (m: string) => console.log(m);\n const warn = (m: string) => console.error(m);\n\n const result = await runMaintenanceOrchestrator(\n {\n cfg: b.cfg,\n runners,\n openclawDir,\n logger: { info: log, warn },\n oneTimeMarkerExists: (path) => existsSync(join(memoryDir, path)),\n journalDb: b.factsDb.getRawDb(),\n },\n {\n tiers,\n dryRun: opts?.dryRun,\n force: opts?.force || opts?.full,\n verbose,\n include: parseStepList(opts?.include),\n exclude: parseStepList(opts?.exclude),\n maxRuntimeMs,\n },\n );\n\n const orchestratorSummary = toOrchestratorRunSummary(result, {\n runId: result.runId,\n job: process.env.HM_JOB,\n startedAt: result.startedAt ?? runStartedIso,\n finishedAt: result.finishedAt,\n durationMs: result.durationMs ?? Date.now() - runStartedMs,\n });\n\n const summaryOutPath = opts?.summaryOut?.trim() || process.env.HM_SUMMARY?.trim();\n if (summaryOutPath) {\n atomicWriteFile(summaryOutPath, `${JSON.stringify(orchestratorSummary, null, 2)}\\n`);\n }\n\n if (opts?.json) {\n console.log(JSON.stringify(orchestratorSummary, null, 2));\n } else {\n const { summaryLine, lines } = formatMaintenanceSummary(result.tierLabel, result.steps);\n console.log(summaryLine);\n for (const line of lines) console.log(line);\n }\n if (result.exitCode !== 0) process.exitCode = result.exitCode;\n };\n\n const commonOptions = (cmd: Chainable) =>\n cmd\n .option(\"--dry-run\", \"List steps and guard status without executing\")\n .option(\"--force\", \"Bypass per-step guards and scan watermarks (not feature gates)\")\n .option(\"--full\", \"Alias for --force\")\n .option(\"-v, --verbose\", \"Detailed per-step output\")\n .option(\"--include <steps>\", \"Comma-separated step names to run only\")\n .option(\"--exclude <steps>\", \"Comma-separated step names to skip\")\n .option(\"--max-runtime-min <n>\", \"Optional time budget in minutes\")\n .option(\"--json\", \"Emit orchestrator run summary as JSON\")\n .option(\"--summary-out <path>\", \"Write orchestrator run summary JSON to path\");\n\n commonOptions(\n maintenance\n .command(\"cycle\")\n .description(\"Run cycle-tier maintenance steps (local gateway-native work)\")\n .action(\n withExit(async (opts, cmd?: CommanderOptsParent) => {\n await runTier([\"cycle\"], { ...opts, verbose: !!opts?.verbose || readHybridMemVerbose(cmd) });\n }),\n ),\n );\n\n commonOptions(\n maintenance\n .command(\"nightly\")\n .description(\"Run all non-cycle steps (staggered guards decide which execute)\")\n .action(\n withExit(async (opts, cmd?: CommanderOptsParent) => {\n await runTier([\"nightly\"], { ...opts, verbose: !!opts?.verbose || readHybridMemVerbose(cmd) });\n }),\n ),\n );\n\n commonOptions(\n maintenance\n .command(\"full\")\n .description(\"Run cycle + nightly tiers in sequence\")\n .action(\n withExit(async (opts, cmd?: CommanderOptsParent) => {\n await runTier([\"cycle\", \"nightly\"], { ...opts, verbose: !!opts?.verbose || readHybridMemVerbose(cmd) });\n }),\n ),\n );\n\n // `maintenance step <name>` runs exactly one named step in isolation (#2028). Named `step` (singular)\n // to avoid colliding with the `maintenance run` JobRun-inspection group and the `steps` list command.\n commonOptions(\n maintenance\n .command(\"step <step>\")\n .description(\n \"Run exactly one named maintenance step in isolation (bypasses that step's cadence guard). Use `maintenance steps` to list valid names.\",\n )\n .action(\n withExit(async (step: string, opts, cmd?: CommanderOptsParent) => {\n const stepName = step?.trim();\n const validNames = listMaintenanceSteps().map((s) => s.name);\n if (!stepName || !validNames.includes(stepName)) {\n console.error(`Unknown maintenance step: ${stepName || \"(none)\"}`);\n console.error(`Valid steps (${validNames.length}):`);\n for (const name of validNames) console.error(` ${name}`);\n process.exitCode = 1;\n return;\n }\n // Force so the targeted step actually runs even if its cadence guard has not expired (this\n // command is an explicit \"run this step now\" diagnostic). include=[step] across both tiers\n // means the orchestrator executes ONLY that step and no unrelated stages (#2028).\n await runTier([\"cycle\", \"nightly\"], {\n ...opts,\n include: stepName,\n force: true,\n verbose: !!opts?.verbose || readHybridMemVerbose(cmd),\n });\n }),\n ),\n );\n\n maintenance\n .command(\"steps\")\n .description(\"List registered maintenance steps with tier, guard interval, and cadence\")\n .option(\"--json\", \"Output JSON\")\n .action(\n withExit(async (opts?: { json?: boolean }) => {\n const rows = listMaintenanceSteps().map((step) => {\n const guardMs = resolveStepGuardIntervalMs(step, b.cfg);\n const guard = stepGuardEligible(step.name, guardMs, openclawDir);\n return {\n name: step.name,\n tier: step.tier,\n guardIntervalMs: guardMs,\n cadence: effectiveCadenceLabel(guardMs),\n llmTier: step.llmTier,\n dependsOn: step.dependsOn ?? [],\n featureGate: step.featureGate ? step.featureGate(b.cfg) : true,\n lastRunAt: guard.lastRunMs ? formatTimestampUtcFromMs(guard.lastRunMs) : null,\n nextEligibleAt: guard.nextEligibleMs ? formatTimestampUtcFromMs(guard.nextEligibleMs) : null,\n eligibleNow: guard.eligible,\n };\n });\n if (opts?.json) {\n console.log(JSON.stringify(rows, null, 2));\n return;\n }\n console.log(`Maintenance steps (${rows.length} registered):`);\n for (const row of rows) {\n const gate = row.featureGate ? \"\" : \" [gate:off]\";\n console.log(\n ` ${row.name.padEnd(28)} ${row.tier.padEnd(8)} ${row.cadence.padEnd(18)} llm=${row.llmTier}${gate}${row.eligibleNow ? \" *due*\" : \"\"}`,\n );\n }\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAuBA,SAAS,cAAc,KAAoC;CACzD,IAAI,CAAC,KAAK,KAAK,GAAG,OAAO,KAAA;CACzB,OAAO,IACJ,MAAM,GAAG,CAAC,CACV,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,CACpB,OAAO,OAAO;AACnB;AAEA,SAAgB,wCAAwC,aAAwB,GAAyB;CACvG,MAAM,cAAc,KAAK,QAAQ,GAAG,WAAW;CAE/C,MAAM,UAAU,OACd,OACA,SAWG;EACH,MAAM,UAAU,CAAC,CAAC,MAAM;EACxB,MAAM,eAAe,KAAK,IAAI;EAC9B,MAAM,gBAAgB,IAAI,KAAK,YAAY,CAAC,CAAC,YAAY;EACzD,IAAI,QAAQ,IAAI,WACd,QAAQ,IAAI,yBAAyB,QAAQ,IAAI;EAEnD,MAAM,YAAY,EAAE,qBAAqB,QAAQ,EAAE,kBAAkB,IAAI,KAAK,QAAQ,GAAG,aAAa,QAAQ;EAE9G,MAAM,UAAU,2BAA2B,GAAG;GAC5C;GACA,qBAHqB,KAAK,WAAW,sBAGH;GAClC,eAAe;IAAE,OAAO,MAAM;IAAO,MAAM,MAAM;GAAK;EACxD,CAAC;EACD,MAAM,eACJ,MAAM,iBAAiB,OAAO,SAAS,OAAO,KAAK,aAAa,CAAC,IAC7D,OAAO,KAAK,aAAa,IAAI,MAC7B,KAAA;EAGN,MAAM,MADa,MAAM,QAAQ,CAAC,CAAC,MAAM,YAAY,KAAK,KAAK,CAAC,CAAC,QAAQ,IAAI,YAAY,KAAK,KACpE,MAAc,QAAQ,MAAM,CAAC,KAAK,MAAc,QAAQ,IAAI,CAAC;EACvF,MAAM,QAAQ,MAAc,QAAQ,MAAM,CAAC;EAE3C,MAAM,SAAS,MAAM,2BACnB;GACE,KAAK,EAAE;GACP;GACA;GACA,QAAQ;IAAE,MAAM;IAAK;GAAK;GAC1B,sBAAsB,SAAS,WAAW,KAAK,WAAW,IAAI,CAAC;GAC/D,WAAW,EAAE,QAAQ,SAAS;EAChC,GACA;GACE;GACA,QAAQ,MAAM;GACd,OAAO,MAAM,SAAS,MAAM;GAC5B;GACA,SAAS,cAAc,MAAM,OAAO;GACpC,SAAS,cAAc,MAAM,OAAO;GACpC;EACF,CACF;EAEA,MAAM,sBAAsB,yBAAyB,QAAQ;GAC3D,OAAO,OAAO;GACd,KAAK,QAAQ,IAAI;GACjB,WAAW,OAAO,aAAa;GAC/B,YAAY,OAAO;GACnB,YAAY,OAAO,cAAc,KAAK,IAAI,IAAI;EAChD,CAAC;EAED,MAAM,iBAAiB,MAAM,YAAY,KAAK,KAAK,QAAQ,IAAI,YAAY,KAAK;EAChF,IAAI,gBACF,gBAAgB,gBAAgB,GAAG,KAAK,UAAU,qBAAqB,MAAM,CAAC,EAAE,GAAG;EAGrF,IAAI,MAAM,MACR,QAAQ,IAAI,KAAK,UAAU,qBAAqB,MAAM,CAAC,CAAC;OACnD;GACL,MAAM,EAAE,aAAa,UAAU,yBAAyB,OAAO,WAAW,OAAO,KAAK;GACtF,QAAQ,IAAI,WAAW;GACvB,KAAK,MAAM,QAAQ,OAAO,QAAQ,IAAI,IAAI;EAC5C;EACA,IAAI,OAAO,aAAa,GAAG,QAAQ,WAAW,OAAO;CACvD;CAEA,MAAM,iBAAiB,QACrB,IACG,OAAO,aAAa,+CAA+C,CAAC,CACpE,OAAO,WAAW,gEAAgE,CAAC,CACnF,OAAO,UAAU,mBAAmB,CAAC,CACrC,OAAO,iBAAiB,0BAA0B,CAAC,CACnD,OAAO,qBAAqB,wCAAwC,CAAC,CACrE,OAAO,qBAAqB,oCAAoC,CAAC,CACjE,OAAO,yBAAyB,iCAAiC,CAAC,CAClE,OAAO,UAAU,uCAAuC,CAAC,CACzD,OAAO,wBAAwB,6CAA6C;CAEjF,cACE,YACG,QAAQ,OAAO,CAAC,CAChB,YAAY,8DAA8D,CAAC,CAC3E,OACC,SAAS,OAAO,MAAM,QAA8B;EAClD,MAAM,QAAQ,CAAC,OAAO,GAAG;GAAE,GAAG;GAAM,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EAAE,CAAC;CAC7F,CAAC,CACH,CACJ;CAEA,cACE,YACG,QAAQ,SAAS,CAAC,CAClB,YAAY,iEAAiE,CAAC,CAC9E,OACC,SAAS,OAAO,MAAM,QAA8B;EAClD,MAAM,QAAQ,CAAC,SAAS,GAAG;GAAE,GAAG;GAAM,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EAAE,CAAC;CAC/F,CAAC,CACH,CACJ;CAEA,cACE,YACG,QAAQ,MAAM,CAAC,CACf,YAAY,uCAAuC,CAAC,CACpD,OACC,SAAS,OAAO,MAAM,QAA8B;EAClD,MAAM,QAAQ,CAAC,SAAS,SAAS,GAAG;GAAE,GAAG;GAAM,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EAAE,CAAC;CACxG,CAAC,CACH,CACJ;CAIA,cACE,YACG,QAAQ,aAAa,CAAC,CACtB,YACC,wIACF,CAAC,CACA,OACC,SAAS,OAAO,MAAc,MAAM,QAA8B;EAChE,MAAM,WAAW,MAAM,KAAK;EAC5B,MAAM,aAAa,qBAAqB,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;EAC3D,IAAI,CAAC,YAAY,CAAC,WAAW,SAAS,QAAQ,GAAG;GAC/C,QAAQ,MAAM,6BAA6B,YAAY,UAAU;GACjE,QAAQ,MAAM,gBAAgB,WAAW,OAAO,GAAG;GACnD,KAAK,MAAM,QAAQ,YAAY,QAAQ,MAAM,KAAK,MAAM;GACxD,QAAQ,WAAW;GACnB;EACF;EAIA,MAAM,QAAQ,CAAC,SAAS,SAAS,GAAG;GAClC,GAAG;GACH,SAAS;GACT,OAAO;GACP,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EACtD,CAAC;CACH,CAAC,CACH,CACJ;CAEA,YACG,QAAQ,OAAO,CAAC,CAChB,YAAY,0EAA0E,CAAC,CACvF,OAAO,UAAU,aAAa,CAAC,CAC/B,OACC,SAAS,OAAO,SAA8B;EAC5C,MAAM,OAAO,qBAAqB,CAAC,CAAC,KAAK,SAAS;GAChD,MAAM,UAAU,2BAA2B,MAAM,EAAE,GAAG;GACtD,MAAM,QAAQ,kBAAkB,KAAK,MAAM,SAAS,WAAW;GAC/D,OAAO;IACL,MAAM,KAAK;IACX,MAAM,KAAK;IACX,iBAAiB;IACjB,SAAS,sBAAsB,OAAO;IACtC,SAAS,KAAK;IACd,WAAW,KAAK,aAAa,CAAC;IAC9B,aAAa,KAAK,cAAc,KAAK,YAAY,EAAE,GAAG,IAAI;IAC1D,WAAW,MAAM,YAAY,yBAAyB,MAAM,SAAS,IAAI;IACzE,gBAAgB,MAAM,iBAAiB,yBAAyB,MAAM,cAAc,IAAI;IACxF,aAAa,MAAM;GACrB;EACF,CAAC;EACD,IAAI,MAAM,MAAM;GACd,QAAQ,IAAI,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;GACzC;EACF;EACA,QAAQ,IAAI,sBAAsB,KAAK,OAAO,cAAc;EAC5D,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,OAAO,IAAI,cAAc,KAAK;GACpC,QAAQ,IACN,KAAK,IAAI,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,KAAK,OAAO,CAAC,EAAE,GAAG,IAAI,QAAQ,OAAO,EAAE,EAAE,OAAO,IAAI,UAAU,OAAO,IAAI,cAAc,WAAW,IACpI;EACF;CACF,CAAC,CACH;AACJ"}
|
|
1
|
+
{"version":3,"file":"register-maintenance-orchestrator.js","names":[],"sources":["../../../../cli/commands/manage/register-maintenance-orchestrator.ts"],"sourcesContent":["/**\n * Maintenance orchestrator CLI: cycle, nightly, full, steps.\n */\n\nimport { existsSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { dirname, join } from \"node:path\";\nimport { readStepGuardTimestampMs, stepGuardEligible } from \"../../../services/cron-guard.js\";\nimport {\n effectiveCadenceLabel,\n formatMaintenanceSummary,\n listMaintenanceSteps,\n resolveStepGuardIntervalMs,\n runMaintenanceOrchestrator,\n toOrchestratorRunSummary,\n} from \"../../../services/maintenance-orchestrator.js\";\nimport { atomicWriteFile } from \"../../../utils/atomic-write.js\";\nimport { formatTimestampUtcFromMs } from \"../../../utils/dates.js\";\nimport { type CommanderOptsParent, readHybridMemVerbose } from \"../../global-verbose.js\";\nimport { type Chainable, withExit } from \"../../shared.js\";\nimport type { ManageBindings } from \"./bindings.js\";\nimport { buildCliMaintenanceRunners } from \"./maintenance-step-runners.js\";\n\nfunction parseStepList(raw?: string): string[] | undefined {\n if (!raw?.trim()) return undefined;\n return raw\n .split(\",\")\n .map((s) => s.trim())\n .filter(Boolean);\n}\n\nexport function registerMaintenanceOrchestratorCommands(maintenance: Chainable, b: ManageBindings): void {\n const openclawDir = join(homedir(), \".openclaw\");\n\n const runTier = async (\n tiers: Array<\"cycle\" | \"nightly\">,\n opts?: {\n dryRun?: boolean;\n force?: boolean;\n full?: boolean;\n verbose?: boolean;\n include?: string;\n exclude?: string;\n maxRuntimeMin?: string;\n json?: boolean;\n summaryOut?: string;\n allowSkip?: boolean;\n },\n ) => {\n const verbose = !!opts?.verbose;\n const runStartedMs = Date.now();\n const runStartedIso = new Date(runStartedMs).toISOString();\n if (process.env.HM_RUN_ID) {\n process.env.HM_ORCHESTRATOR_RUN_ID = process.env.HM_RUN_ID;\n }\n const memoryDir = b.resolvedSqlitePath ? dirname(b.resolvedSqlitePath) : join(homedir(), \".openclaw\", \"memory\");\n const backfillMarker = join(memoryDir, \".backfill-decay-done\");\n const runners = buildCliMaintenanceRunners(b, {\n verbose,\n backfillDecayMarker: backfillMarker,\n scanOverrides: { force: opts?.force, full: opts?.full },\n });\n // `> 0` (not just finite) — \"0\" is a truthy, finite string that would otherwise produce a\n // zero-minute budget, which the orchestrator's `elapsed >= maxRuntimeMs` check treats as\n // already-exceeded before the first step runs, deferring every step including the one the\n // operator asked for. Treat a non-positive value the same as \"no budget\", consistent with\n // guardIntervalMs <= 0 meaning \"always eligible\" elsewhere in this file.\n const parsedMaxRuntimeMin = opts?.maxRuntimeMin ? Number(opts.maxRuntimeMin) : undefined;\n const maxRuntimeMs =\n parsedMaxRuntimeMin !== undefined && Number.isFinite(parsedMaxRuntimeMin) && parsedMaxRuntimeMin > 0\n ? parsedMaxRuntimeMin * 60_000\n : undefined;\n\n const isJsonMode = opts?.json || !!opts?.summaryOut?.trim() || !!process.env.HM_SUMMARY?.trim();\n const log = isJsonMode ? (m: string) => console.error(m) : (m: string) => console.log(m);\n const warn = (m: string) => console.error(m);\n\n const result = await runMaintenanceOrchestrator(\n {\n cfg: b.cfg,\n runners,\n openclawDir,\n logger: { info: log, warn },\n oneTimeMarkerExists: (path) => existsSync(join(memoryDir, path)),\n journalDb: b.factsDb.getRawDb(),\n },\n {\n tiers,\n dryRun: opts?.dryRun,\n force: opts?.force || opts?.full,\n verbose,\n include: parseStepList(opts?.include),\n exclude: parseStepList(opts?.exclude),\n maxRuntimeMs,\n },\n );\n\n const orchestratorSummary = toOrchestratorRunSummary(result, {\n runId: result.runId,\n job: process.env.HM_JOB,\n startedAt: result.startedAt ?? runStartedIso,\n finishedAt: result.finishedAt,\n durationMs: result.durationMs ?? Date.now() - runStartedMs,\n });\n\n const summaryOutPath = opts?.summaryOut?.trim() || process.env.HM_SUMMARY?.trim();\n if (summaryOutPath) {\n atomicWriteFile(summaryOutPath, `${JSON.stringify(orchestratorSummary, null, 2)}\\n`);\n }\n\n if (opts?.json) {\n console.log(JSON.stringify(orchestratorSummary, null, 2));\n } else {\n const { summaryLine, lines } = formatMaintenanceSummary(result.tierLabel, result.steps);\n console.log(summaryLine);\n for (const line of lines) console.log(line);\n }\n if (result.exitCode !== 0) process.exitCode = result.exitCode;\n\n // An explicit, forced --include run (whether via the dedicated `step <name>` command or\n // `cycle`/`nightly`/`full --include x,y --force`) is a targeted \"run this now and verify it\"\n // diagnostic — computeExitCode() treats \"skipped\" as benign because it's normal cadence-guard\n // behavior in an ordinary full/nightly run, so it doesn't cover \"the step(s) I explicitly asked\n // for didn't execute\" (#2031). Scoped to include+force so ordinary unforced --include automation\n // (which can legitimately skip by cadence) keeps its existing exit-0-on-skip behavior.\n const includeNames = parseStepList(opts?.include);\n if ((opts?.force || opts?.full) && includeNames?.length && !opts?.allowSkip) {\n // Look up each requested name individually rather than filtering result.steps down to the\n // matches — a name that belongs to a tier not requested (e.g. `cycle --include <nightly-step>\n // --force`) or that --exclude also removed never produces a step result at all, so it would\n // otherwise vanish from `selected` entirely and slip past the \"every(...)\" check below with a\n // false \"didn't run\" report never firing (an even quieter version of the bug #2031 closes).\n const requested = includeNames.map((name) => ({ name, result: result.steps.find((s) => s.name === name) }));\n const didNotRun = requested.every((r) => !r.result || r.result.status.startsWith(\"skipped\"));\n if (didNotRun) {\n const detail = requested\n .map((r) =>\n r.result\n ? `${r.name}: ${r.result.status} — ${r.result.summary}`\n : `${r.name}: no result (wrong tier, excluded, or unregistered runner)`,\n )\n .join(\"; \");\n console.error(\n `maintenance ${result.tierLabel}: selected step(s) did not run (${detail}). ` +\n \"Pass --allow-skip to treat this as a non-strict status check.\",\n );\n if (!process.exitCode) process.exitCode = 3;\n }\n }\n\n return result;\n };\n\n const commonOptions = (cmd: Chainable) =>\n cmd\n .option(\"--dry-run\", \"List steps and guard status without executing\")\n .option(\"--force\", \"Bypass per-step guards and scan watermarks (not feature gates)\")\n .option(\"--full\", \"Alias for --force\")\n .option(\"-v, --verbose\", \"Detailed per-step output\")\n .option(\"--include <steps>\", \"Comma-separated step names to run only\")\n .option(\"--exclude <steps>\", \"Comma-separated step names to skip\")\n .option(\"--max-runtime-min <n>\", \"Optional time budget in minutes\")\n .option(\"--json\", \"Emit orchestrator run summary as JSON\")\n .option(\"--summary-out <path>\", \"Write orchestrator run summary JSON to path\")\n .option(\n \"--allow-skip\",\n \"With --include --force: exit 0 even if the selected step(s) didn't execute (locked/gated/guarded)\",\n );\n\n commonOptions(\n maintenance\n .command(\"cycle\")\n .description(\"Run cycle-tier maintenance steps (local gateway-native work)\")\n .action(\n withExit(async (opts, cmd?: CommanderOptsParent) => {\n await runTier([\"cycle\"], { ...opts, verbose: !!opts?.verbose || readHybridMemVerbose(cmd) });\n }),\n ),\n );\n\n commonOptions(\n maintenance\n .command(\"nightly\")\n .description(\"Run all non-cycle steps (staggered guards decide which execute)\")\n .action(\n withExit(async (opts, cmd?: CommanderOptsParent) => {\n await runTier([\"nightly\"], { ...opts, verbose: !!opts?.verbose || readHybridMemVerbose(cmd) });\n }),\n ),\n );\n\n commonOptions(\n maintenance\n .command(\"full\")\n .description(\"Run cycle + nightly tiers in sequence\")\n .action(\n withExit(async (opts, cmd?: CommanderOptsParent) => {\n await runTier([\"cycle\", \"nightly\"], { ...opts, verbose: !!opts?.verbose || readHybridMemVerbose(cmd) });\n }),\n ),\n );\n\n // `maintenance step <name>` runs exactly one named step in isolation (#2028). Named `step` (singular)\n // to avoid colliding with the `maintenance run` JobRun-inspection group and the `steps` list command.\n //\n // Bounded by default (#2032): this command is meant for isolated diagnostic/verification runs, so\n // unlike cycle/nightly/full it caps runtime unless the operator overrides --max-runtime-min. Long\n // CPU/LLM-bound steps (e.g. passive-observer scanning a large first-run backlog) already consult the\n // orchestrator-wide deadline between sessions/chunks, so this cap actually bounds wall-clock time.\n const DEFAULT_STEP_MAX_RUNTIME_MIN = \"15\";\n\n commonOptions(\n maintenance\n .command(\"step <step>\")\n .description(\n \"Run exactly one named maintenance step in isolation (bypasses that step's cadence guard). Use `maintenance steps` to list valid names.\",\n ),\n ).action(\n withExit(async (step: string, opts, cmd?: CommanderOptsParent) => {\n const stepName = step?.trim();\n const validNames = listMaintenanceSteps().map((s) => s.name);\n if (!stepName || !validNames.includes(stepName)) {\n console.error(`Unknown maintenance step: ${stepName || \"(none)\"}`);\n console.error(`Valid steps (${validNames.length}):`);\n for (const name of validNames) console.error(` ${name}`);\n process.exitCode = 1;\n return;\n }\n // Force so the targeted step actually runs even if its cadence guard has not expired (this\n // command is an explicit \"run this step now\" diagnostic). include=[step] across both tiers\n // means the orchestrator executes ONLY that step and no unrelated stages (#2028). runTier()\n // itself checks whether this forced, explicitly-included step actually executed (#2031).\n await runTier([\"cycle\", \"nightly\"], {\n ...opts,\n include: stepName,\n force: true,\n maxRuntimeMin: opts?.maxRuntimeMin ?? DEFAULT_STEP_MAX_RUNTIME_MIN,\n verbose: !!opts?.verbose || readHybridMemVerbose(cmd),\n });\n }),\n );\n\n maintenance\n .command(\"steps\")\n .description(\"List registered maintenance steps with tier, guard interval, and cadence\")\n .option(\"--json\", \"Output JSON\")\n .action(\n withExit(async (opts?: { json?: boolean }) => {\n const rows = listMaintenanceSteps().map((step) => {\n const guardMs = resolveStepGuardIntervalMs(step, b.cfg);\n const guard = stepGuardEligible(step.name, guardMs, openclawDir);\n return {\n name: step.name,\n tier: step.tier,\n guardIntervalMs: guardMs,\n cadence: effectiveCadenceLabel(guardMs),\n llmTier: step.llmTier,\n dependsOn: step.dependsOn ?? [],\n featureGate: step.featureGate ? step.featureGate(b.cfg) : true,\n lastRunAt: guard.lastRunMs ? formatTimestampUtcFromMs(guard.lastRunMs) : null,\n nextEligibleAt: guard.nextEligibleMs ? formatTimestampUtcFromMs(guard.nextEligibleMs) : null,\n eligibleNow: guard.eligible,\n };\n });\n if (opts?.json) {\n console.log(JSON.stringify(rows, null, 2));\n return;\n }\n console.log(`Maintenance steps (${rows.length} registered):`);\n for (const row of rows) {\n const gate = row.featureGate ? \"\" : \" [gate:off]\";\n console.log(\n ` ${row.name.padEnd(28)} ${row.tier.padEnd(8)} ${row.cadence.padEnd(18)} llm=${row.llmTier}${gate}${row.eligibleNow ? \" *due*\" : \"\"}`,\n );\n }\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAuBA,SAAS,cAAc,KAAoC;CACzD,IAAI,CAAC,KAAK,KAAK,GAAG,OAAO,KAAA;CACzB,OAAO,IACJ,MAAM,GAAG,CAAC,CACV,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,CACpB,OAAO,OAAO;AACnB;AAEA,SAAgB,wCAAwC,aAAwB,GAAyB;CACvG,MAAM,cAAc,KAAK,QAAQ,GAAG,WAAW;CAE/C,MAAM,UAAU,OACd,OACA,SAYG;EACH,MAAM,UAAU,CAAC,CAAC,MAAM;EACxB,MAAM,eAAe,KAAK,IAAI;EAC9B,MAAM,gBAAgB,IAAI,KAAK,YAAY,CAAC,CAAC,YAAY;EACzD,IAAI,QAAQ,IAAI,WACd,QAAQ,IAAI,yBAAyB,QAAQ,IAAI;EAEnD,MAAM,YAAY,EAAE,qBAAqB,QAAQ,EAAE,kBAAkB,IAAI,KAAK,QAAQ,GAAG,aAAa,QAAQ;EAE9G,MAAM,UAAU,2BAA2B,GAAG;GAC5C;GACA,qBAHqB,KAAK,WAAW,sBAGH;GAClC,eAAe;IAAE,OAAO,MAAM;IAAO,MAAM,MAAM;GAAK;EACxD,CAAC;EAMD,MAAM,sBAAsB,MAAM,gBAAgB,OAAO,KAAK,aAAa,IAAI,KAAA;EAC/E,MAAM,eACJ,wBAAwB,KAAA,KAAa,OAAO,SAAS,mBAAmB,KAAK,sBAAsB,IAC/F,sBAAsB,MACtB,KAAA;EAGN,MAAM,MADa,MAAM,QAAQ,CAAC,CAAC,MAAM,YAAY,KAAK,KAAK,CAAC,CAAC,QAAQ,IAAI,YAAY,KAAK,KACpE,MAAc,QAAQ,MAAM,CAAC,KAAK,MAAc,QAAQ,IAAI,CAAC;EACvF,MAAM,QAAQ,MAAc,QAAQ,MAAM,CAAC;EAE3C,MAAM,SAAS,MAAM,2BACnB;GACE,KAAK,EAAE;GACP;GACA;GACA,QAAQ;IAAE,MAAM;IAAK;GAAK;GAC1B,sBAAsB,SAAS,WAAW,KAAK,WAAW,IAAI,CAAC;GAC/D,WAAW,EAAE,QAAQ,SAAS;EAChC,GACA;GACE;GACA,QAAQ,MAAM;GACd,OAAO,MAAM,SAAS,MAAM;GAC5B;GACA,SAAS,cAAc,MAAM,OAAO;GACpC,SAAS,cAAc,MAAM,OAAO;GACpC;EACF,CACF;EAEA,MAAM,sBAAsB,yBAAyB,QAAQ;GAC3D,OAAO,OAAO;GACd,KAAK,QAAQ,IAAI;GACjB,WAAW,OAAO,aAAa;GAC/B,YAAY,OAAO;GACnB,YAAY,OAAO,cAAc,KAAK,IAAI,IAAI;EAChD,CAAC;EAED,MAAM,iBAAiB,MAAM,YAAY,KAAK,KAAK,QAAQ,IAAI,YAAY,KAAK;EAChF,IAAI,gBACF,gBAAgB,gBAAgB,GAAG,KAAK,UAAU,qBAAqB,MAAM,CAAC,EAAE,GAAG;EAGrF,IAAI,MAAM,MACR,QAAQ,IAAI,KAAK,UAAU,qBAAqB,MAAM,CAAC,CAAC;OACnD;GACL,MAAM,EAAE,aAAa,UAAU,yBAAyB,OAAO,WAAW,OAAO,KAAK;GACtF,QAAQ,IAAI,WAAW;GACvB,KAAK,MAAM,QAAQ,OAAO,QAAQ,IAAI,IAAI;EAC5C;EACA,IAAI,OAAO,aAAa,GAAG,QAAQ,WAAW,OAAO;EAQrD,MAAM,eAAe,cAAc,MAAM,OAAO;EAChD,KAAK,MAAM,SAAS,MAAM,SAAS,cAAc,UAAU,CAAC,MAAM,WAAW;GAM3E,MAAM,YAAY,aAAa,KAAK,UAAU;IAAE;IAAM,QAAQ,OAAO,MAAM,MAAM,MAAM,EAAE,SAAS,IAAI;GAAE,EAAE;GAE1G,IADkB,UAAU,OAAO,MAAM,CAAC,EAAE,UAAU,EAAE,OAAO,OAAO,WAAW,SAAS,CAC9E,GAAG;IACb,MAAM,SAAS,UACZ,KAAK,MACJ,EAAE,SACE,GAAG,EAAE,KAAK,IAAI,EAAE,OAAO,OAAO,KAAK,EAAE,OAAO,YAC5C,GAAG,EAAE,KAAK,2DAChB,CAAC,CACA,KAAK,IAAI;IACZ,QAAQ,MACN,eAAe,OAAO,UAAU,kCAAkC,OAAO,iEAE3E;IACA,IAAI,CAAC,QAAQ,UAAU,QAAQ,WAAW;GAC5C;EACF;EAEA,OAAO;CACT;CAEA,MAAM,iBAAiB,QACrB,IACG,OAAO,aAAa,+CAA+C,CAAC,CACpE,OAAO,WAAW,gEAAgE,CAAC,CACnF,OAAO,UAAU,mBAAmB,CAAC,CACrC,OAAO,iBAAiB,0BAA0B,CAAC,CACnD,OAAO,qBAAqB,wCAAwC,CAAC,CACrE,OAAO,qBAAqB,oCAAoC,CAAC,CACjE,OAAO,yBAAyB,iCAAiC,CAAC,CAClE,OAAO,UAAU,uCAAuC,CAAC,CACzD,OAAO,wBAAwB,6CAA6C,CAAC,CAC7E,OACC,gBACA,mGACF;CAEJ,cACE,YACG,QAAQ,OAAO,CAAC,CAChB,YAAY,8DAA8D,CAAC,CAC3E,OACC,SAAS,OAAO,MAAM,QAA8B;EAClD,MAAM,QAAQ,CAAC,OAAO,GAAG;GAAE,GAAG;GAAM,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EAAE,CAAC;CAC7F,CAAC,CACH,CACJ;CAEA,cACE,YACG,QAAQ,SAAS,CAAC,CAClB,YAAY,iEAAiE,CAAC,CAC9E,OACC,SAAS,OAAO,MAAM,QAA8B;EAClD,MAAM,QAAQ,CAAC,SAAS,GAAG;GAAE,GAAG;GAAM,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EAAE,CAAC;CAC/F,CAAC,CACH,CACJ;CAEA,cACE,YACG,QAAQ,MAAM,CAAC,CACf,YAAY,uCAAuC,CAAC,CACpD,OACC,SAAS,OAAO,MAAM,QAA8B;EAClD,MAAM,QAAQ,CAAC,SAAS,SAAS,GAAG;GAAE,GAAG;GAAM,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EAAE,CAAC;CACxG,CAAC,CACH,CACJ;CASA,MAAM,+BAA+B;CAErC,cACE,YACG,QAAQ,aAAa,CAAC,CACtB,YACC,wIACF,CACJ,CAAC,CAAC,OACA,SAAS,OAAO,MAAc,MAAM,QAA8B;EAChE,MAAM,WAAW,MAAM,KAAK;EAC5B,MAAM,aAAa,qBAAqB,CAAC,CAAC,KAAK,MAAM,EAAE,IAAI;EAC3D,IAAI,CAAC,YAAY,CAAC,WAAW,SAAS,QAAQ,GAAG;GAC/C,QAAQ,MAAM,6BAA6B,YAAY,UAAU;GACjE,QAAQ,MAAM,gBAAgB,WAAW,OAAO,GAAG;GACnD,KAAK,MAAM,QAAQ,YAAY,QAAQ,MAAM,KAAK,MAAM;GACxD,QAAQ,WAAW;GACnB;EACF;EAKA,MAAM,QAAQ,CAAC,SAAS,SAAS,GAAG;GAClC,GAAG;GACH,SAAS;GACT,OAAO;GACP,eAAe,MAAM,iBAAiB;GACtC,SAAS,CAAC,CAAC,MAAM,WAAW,qBAAqB,GAAG;EACtD,CAAC;CACH,CAAC,CACH;CAEA,YACG,QAAQ,OAAO,CAAC,CAChB,YAAY,0EAA0E,CAAC,CACvF,OAAO,UAAU,aAAa,CAAC,CAC/B,OACC,SAAS,OAAO,SAA8B;EAC5C,MAAM,OAAO,qBAAqB,CAAC,CAAC,KAAK,SAAS;GAChD,MAAM,UAAU,2BAA2B,MAAM,EAAE,GAAG;GACtD,MAAM,QAAQ,kBAAkB,KAAK,MAAM,SAAS,WAAW;GAC/D,OAAO;IACL,MAAM,KAAK;IACX,MAAM,KAAK;IACX,iBAAiB;IACjB,SAAS,sBAAsB,OAAO;IACtC,SAAS,KAAK;IACd,WAAW,KAAK,aAAa,CAAC;IAC9B,aAAa,KAAK,cAAc,KAAK,YAAY,EAAE,GAAG,IAAI;IAC1D,WAAW,MAAM,YAAY,yBAAyB,MAAM,SAAS,IAAI;IACzE,gBAAgB,MAAM,iBAAiB,yBAAyB,MAAM,cAAc,IAAI;IACxF,aAAa,MAAM;GACrB;EACF,CAAC;EACD,IAAI,MAAM,MAAM;GACd,QAAQ,IAAI,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;GACzC;EACF;EACA,QAAQ,IAAI,sBAAsB,KAAK,OAAO,cAAc;EAC5D,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,OAAO,IAAI,cAAc,KAAK;GACpC,QAAQ,IACN,KAAK,IAAI,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,KAAK,OAAO,CAAC,EAAE,GAAG,IAAI,QAAQ,OAAO,EAAE,EAAE,OAAO,IAAI,UAAU,OAAO,IAAI,cAAc,WAAW,IACpI;EACF;CACF,CAAC,CACH;AACJ"}
|
|
@@ -16,8 +16,8 @@ import { abortCredentialVaultWriteOnPointerDedupe, buildCredentialPointerText, e
|
|
|
16
16
|
import { classifyMemoryOperation, classifyMemoryOperationsBatch } from "../../services/classification.js";
|
|
17
17
|
import { extractStructuredFields } from "../../services/fact-extraction.js";
|
|
18
18
|
import { atomicWriteFile } from "../../utils/atomic-write.js";
|
|
19
|
-
import { isSubstantiveMemoryText, prepareMemoryTextForStorage } from "../../services/recalled-context-assembler.js";
|
|
20
19
|
import { peekCaptureDedupWindow, runCaptureStoreWithDedupWindow } from "../../services/capture-dedup.js";
|
|
20
|
+
import { isSubstantiveMemoryText, prepareMemoryTextForStorage } from "../../services/recalled-context-assembler.js";
|
|
21
21
|
import { isRecallContextSuperseded, shouldSuppressStaleLifecycleError } from "../../utils/registration-superseded.js";
|
|
22
22
|
import { getAutoCaptureExtractionConfidence, getAutoCaptureExtractionMethod, resolveCaptureProvenance } from "../../services/capture-provenance.js";
|
|
23
23
|
import { resolveDefaultStoreScope } from "../../services/default-store-scope.js";
|