idlewatch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +73 -0
- package/.github/workflows/ci.yml +99 -0
- package/.github/workflows/release-macos-trusted.yml +103 -0
- package/README.md +336 -0
- package/bin/idlewatch-agent.js +1053 -0
- package/docs/onboarding-external.md +58 -0
- package/docs/packaging/macos-dmg.md +199 -0
- package/docs/packaging/macos-launch-agent.md +70 -0
- package/docs/qa/archive/mac-qa-log-2026-02-17.md +5838 -0
- package/docs/qa/mac-qa-log.md +2864 -0
- package/docs/telemetry/idle-stale-policy.md +57 -0
- package/docs/telemetry/openclaw-mapping.md +80 -0
- package/package.json +76 -0
- package/scripts/build-dmg.sh +65 -0
- package/scripts/install-macos-launch-agent.sh +78 -0
- package/scripts/lib/telemetry-row-parser.mjs +100 -0
- package/scripts/package-macos.sh +228 -0
- package/scripts/uninstall-macos-launch-agent.sh +30 -0
- package/scripts/validate-all.sh +142 -0
- package/scripts/validate-bin.mjs +25 -0
- package/scripts/validate-dmg-checksum.sh +37 -0
- package/scripts/validate-dmg-install.sh +155 -0
- package/scripts/validate-dry-run-schema.mjs +257 -0
- package/scripts/validate-onboarding.mjs +63 -0
- package/scripts/validate-openclaw-cache-recovery-e2e.mjs +113 -0
- package/scripts/validate-openclaw-release-gates.mjs +51 -0
- package/scripts/validate-openclaw-stats-ingestion.mjs +372 -0
- package/scripts/validate-openclaw-usage-health.mjs +95 -0
- package/scripts/validate-packaged-artifact.mjs +233 -0
- package/scripts/validate-packaged-bundled-runtime.sh +191 -0
- package/scripts/validate-packaged-metadata.sh +43 -0
- package/scripts/validate-packaged-openclaw-cache-recovery-e2e.mjs +153 -0
- package/scripts/validate-packaged-openclaw-release-gates.mjs +72 -0
- package/scripts/validate-packaged-openclaw-stats-ingestion.mjs +402 -0
- package/scripts/validate-packaged-sourcemaps.mjs +82 -0
- package/scripts/validate-packaged-usage-alert-rate-e2e.mjs +98 -0
- package/scripts/validate-packaged-usage-probe-noise-e2e.mjs +87 -0
- package/scripts/validate-packaged-usage-recovery-e2e.mjs +90 -0
- package/scripts/validate-trusted-prereqs.sh +44 -0
- package/scripts/validate-usage-alert-rate-e2e.mjs +91 -0
- package/scripts/validate-usage-freshness-e2e.mjs +81 -0
- package/skill/SKILL.md +43 -0
- package/src/config.js +100 -0
- package/src/enrollment.js +176 -0
- package/src/gpu.js +115 -0
- package/src/memory.js +67 -0
- package/src/openclaw-cache.js +51 -0
- package/src/openclaw-usage.js +1020 -0
- package/src/telemetry-mapping.js +54 -0
- package/src/usage-alert.js +41 -0
- package/src/usage-freshness.js +31 -0
- package/test/config.test.mjs +112 -0
- package/test/fixtures/gpu-agx.txt +2 -0
- package/test/fixtures/gpu-iogpu.txt +2 -0
- package/test/fixtures/gpu-top-grep.txt +2 -0
- package/test/fixtures/openclaw-fleet-sample-v1.json +68 -0
- package/test/fixtures/openclaw-mixed-equal-score-status-vs-generic-iso-ts.txt +2 -0
- package/test/fixtures/openclaw-mixed-equal-score-status-vs-generic-newest.txt +2 -0
- package/test/fixtures/openclaw-mixed-equal-score-status-vs-generic-string-ts.txt +2 -0
- package/test/fixtures/openclaw-mixed-status-then-generic-output.txt +2 -0
- package/test/fixtures/openclaw-stats-current-wrapper.json +12 -0
- package/test/fixtures/openclaw-stats-current-wrapper2.json +15 -0
- package/test/fixtures/openclaw-stats-data-wrapper.json +21 -0
- package/test/fixtures/openclaw-stats-nested-session-wrapper.json +23 -0
- package/test/fixtures/openclaw-stats-payload-wrapper.json +1 -0
- package/test/fixtures/openclaw-stats-status-current-wrapper.json +19 -0
- package/test/fixtures/openclaw-stats.json +17 -0
- package/test/fixtures/openclaw-status-ansi-complex-noise.txt +3 -0
- package/test/fixtures/openclaw-status-ansi-noise.txt +2 -0
- package/test/fixtures/openclaw-status-control-noise.txt +1 -0
- package/test/fixtures/openclaw-status-data-wrapper.json +20 -0
- package/test/fixtures/openclaw-status-dcs-noise.txt +1 -0
- package/test/fixtures/openclaw-status-epoch-seconds.json +15 -0
- package/test/fixtures/openclaw-status-mixed-noise.txt +1 -0
- package/test/fixtures/openclaw-status-multi-json.txt +3 -0
- package/test/fixtures/openclaw-status-nested-recent.json +19 -0
- package/test/fixtures/openclaw-status-noisy-default-then-usage.txt +2 -0
- package/test/fixtures/openclaw-status-noisy.txt +3 -0
- package/test/fixtures/openclaw-status-osc-noise.txt +1 -0
- package/test/fixtures/openclaw-status-result-session.json +15 -0
- package/test/fixtures/openclaw-status-session-map-with-defaults.json +23 -0
- package/test/fixtures/openclaw-status-session-map.json +28 -0
- package/test/fixtures/openclaw-status-session-model-name.json +18 -0
- package/test/fixtures/openclaw-status-snake-session-wrapper.json +13 -0
- package/test/fixtures/openclaw-status-stats-current-sessions-snake-tokens.json +25 -0
- package/test/fixtures/openclaw-status-stats-current-sessions.json +28 -0
- package/test/fixtures/openclaw-status-stats-current-usage-time-camelcase.json +19 -0
- package/test/fixtures/openclaw-status-stats-session-default-model.json +27 -0
- package/test/fixtures/openclaw-status-status-wrapper.json +13 -0
- package/test/fixtures/openclaw-status-strings.json +38 -0
- package/test/fixtures/openclaw-status-ts-ms-alias.json +14 -0
- package/test/fixtures/openclaw-status-updated-at-ms-alias.json +14 -0
- package/test/fixtures/openclaw-status-usage-timestamp-ms-alias.json +14 -0
- package/test/fixtures/openclaw-status-usage-ts-alias.json +14 -0
- package/test/fixtures/openclaw-status-wrap-session-object.json +24 -0
- package/test/fixtures/openclaw-status.json +41 -0
- package/test/fixtures/openclaw-usage-model-name-generic.json +9 -0
- package/test/gpu.test.mjs +58 -0
- package/test/memory.test.mjs +35 -0
- package/test/openclaw-cache.test.mjs +48 -0
- package/test/openclaw-env.test.mjs +365 -0
- package/test/openclaw-usage.test.mjs +555 -0
- package/test/telemetry-mapping.test.mjs +69 -0
- package/test/telemetry-row-parser.test.mjs +44 -0
- package/test/usage-alert.test.mjs +73 -0
- package/test/usage-freshness.test.mjs +63 -0
- package/test/validate-dry-run-schema.test.mjs +146 -0
- package/tui/Cargo.lock +801 -0
- package/tui/Cargo.toml +11 -0
- package/tui/src/main.rs +368 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import { spawnSync } from 'node:child_process'
|
|
4
|
+
import { extractJsonCandidates } from './lib/telemetry-row-parser.mjs'
|
|
5
|
+
|
|
6
|
+
const command = process.argv[2] || 'node'
|
|
7
|
+
const args = process.argv.slice(3)
|
|
8
|
+
|
|
9
|
+
const TIMEOUT_MS_RAW = process.env.IDLEWATCH_DRY_RUN_TIMEOUT_MS
|
|
10
|
+
const DRY_RUN_TIMEOUT_MS = TIMEOUT_MS_RAW
|
|
11
|
+
? Number(TIMEOUT_MS_RAW)
|
|
12
|
+
: 15000
|
|
13
|
+
|
|
14
|
+
if (!Number.isFinite(DRY_RUN_TIMEOUT_MS) || DRY_RUN_TIMEOUT_MS <= 0) {
|
|
15
|
+
console.error('IDLEWATCH_DRY_RUN_TIMEOUT_MS must be a finite positive number when set')
|
|
16
|
+
process.exit(1)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function collectOutput(stderrBuffer, stdoutBuffer) {
|
|
20
|
+
const parts = [stdoutBuffer, stderrBuffer].filter(Boolean)
|
|
21
|
+
const output = parts.join('')
|
|
22
|
+
return String(output || '')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function run() {
|
|
26
|
+
const result = spawnSync(command, args, {
|
|
27
|
+
encoding: 'utf8',
|
|
28
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
29
|
+
timeout: DRY_RUN_TIMEOUT_MS,
|
|
30
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
31
|
+
killSignal: 'SIGINT'
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
const out = collectOutput(result.stdout, result.stderr)
|
|
35
|
+
const timedOut = result.error?.code === 'ETIMEDOUT' || result.signal === 'SIGINT'
|
|
36
|
+
const maybeCandidates = extractJsonCandidates(out)
|
|
37
|
+
|
|
38
|
+
if (timedOut) {
|
|
39
|
+
console.error(`dry-run timed out after ${DRY_RUN_TIMEOUT_MS}ms; attempting to validate captured output from partial row`)
|
|
40
|
+
} else if (result.error) {
|
|
41
|
+
if (!out.trim()) {
|
|
42
|
+
throw result.error
|
|
43
|
+
}
|
|
44
|
+
console.error(`dry-run process error: ${result.error.message}; validating captured output anyway`)
|
|
45
|
+
} else if (result.status !== 0 && !out.trim()) {
|
|
46
|
+
throw new Error(`dry-run exited with non-zero status ${result.status}`)
|
|
47
|
+
} else if (result.status !== 0) {
|
|
48
|
+
console.error(`dry-run exited with non-zero status ${result.status}; validating captured output anyway`)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
for (let i = maybeCandidates.length - 1; i >= 0; i -= 1) {
|
|
52
|
+
const candidate = maybeCandidates[i]
|
|
53
|
+
try {
|
|
54
|
+
const row = JSON.parse(candidate)
|
|
55
|
+
validateRow(row)
|
|
56
|
+
console.log(`dry-run schema ok (${command} ${args.join(' ')})`)
|
|
57
|
+
return
|
|
58
|
+
} catch {
|
|
59
|
+
// ignore and continue with the next JSON candidate
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
throw new Error('No telemetry JSON row found in dry-run output')
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
function assertNumberOrNull(value, field) {
|
|
68
|
+
assert.ok(value === null || Number.isFinite(value), `${field} must be a finite number or null`)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const REQUIRE_OPENCLAW_USAGE = process.env.IDLEWATCH_REQUIRE_OPENCLAW_USAGE === '1'
|
|
72
|
+
const MAX_OPENCLAW_USAGE_AGE_MS_RAW = process.env.IDLEWATCH_MAX_OPENCLAW_USAGE_AGE_MS
|
|
73
|
+
const MAX_OPENCLAW_USAGE_AGE_MS = MAX_OPENCLAW_USAGE_AGE_MS_RAW
|
|
74
|
+
? Number(MAX_OPENCLAW_USAGE_AGE_MS_RAW)
|
|
75
|
+
: null
|
|
76
|
+
|
|
77
|
+
if (MAX_OPENCLAW_USAGE_AGE_MS_RAW) {
|
|
78
|
+
assert.ok(Number.isFinite(MAX_OPENCLAW_USAGE_AGE_MS) && MAX_OPENCLAW_USAGE_AGE_MS > 0, 'IDLEWATCH_MAX_OPENCLAW_USAGE_AGE_MS must be a number > 0 when set')
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function validateRow(row) {
|
|
82
|
+
assert.equal(typeof row.host, 'string', 'host must be a string')
|
|
83
|
+
assert.ok(Number.isFinite(row.ts), 'ts must be a finite number')
|
|
84
|
+
|
|
85
|
+
assertNumberOrNull(row.cpuPct, 'cpuPct')
|
|
86
|
+
assertNumberOrNull(row.memPct, 'memPct')
|
|
87
|
+
assertNumberOrNull(row.memUsedPct, 'memUsedPct')
|
|
88
|
+
assertNumberOrNull(row.memPressurePct, 'memPressurePct')
|
|
89
|
+
assert.ok(['normal', 'warning', 'critical', 'unavailable'].includes(row.memPressureClass), 'memPressureClass invalid')
|
|
90
|
+
|
|
91
|
+
assertNumberOrNull(row.gpuPct, 'gpuPct')
|
|
92
|
+
assert.equal(typeof row.gpuSource, 'string', 'gpuSource must be string')
|
|
93
|
+
assert.ok(['high', 'medium', 'low', 'none'].includes(row.gpuConfidence), 'gpuConfidence invalid')
|
|
94
|
+
assertNumberOrNull(row.gpuSampleWindowMs, 'gpuSampleWindowMs')
|
|
95
|
+
|
|
96
|
+
assertNumberOrNull(row.tokensPerMin, 'tokensPerMin')
|
|
97
|
+
assert.ok(row.openclawModel === null || typeof row.openclawModel === 'string', 'openclawModel must be string or null')
|
|
98
|
+
assertNumberOrNull(row.openclawTotalTokens, 'openclawTotalTokens')
|
|
99
|
+
assert.ok(row.openclawSessionId === null || typeof row.openclawSessionId === 'string', 'openclawSessionId must be string or null')
|
|
100
|
+
assert.ok(row.openclawAgentId === null || typeof row.openclawAgentId === 'string', 'openclawAgentId must be string or null')
|
|
101
|
+
assertNumberOrNull(row.openclawUsageTs, 'openclawUsageTs')
|
|
102
|
+
assertNumberOrNull(row.openclawUsageAgeMs, 'openclawUsageAgeMs')
|
|
103
|
+
|
|
104
|
+
assert.equal(typeof row.source, 'object', 'source must exist')
|
|
105
|
+
const source = row.source
|
|
106
|
+
|
|
107
|
+
assert.equal(typeof row.schemaFamily, 'string', 'schemaFamily must be string')
|
|
108
|
+
assert.equal(typeof row.schemaVersion, 'string', 'schemaVersion must be string')
|
|
109
|
+
assert.ok(Array.isArray(row.schemaCompat), 'schemaCompat must be array')
|
|
110
|
+
assert.equal(typeof row.fleet, 'object', 'fleet must exist')
|
|
111
|
+
assert.equal(typeof row.fleet.host, 'string', 'fleet.host must be string')
|
|
112
|
+
assert.ok(Number.isFinite(row.fleet.collectedAtMs), 'fleet.collectedAtMs must be number')
|
|
113
|
+
assert.equal(typeof row.fleet.resources, 'object', 'fleet.resources must exist')
|
|
114
|
+
assertNumberOrNull(row.fleet.resources.cpuPct, 'fleet.resources.cpuPct')
|
|
115
|
+
assertNumberOrNull(row.fleet.resources.memUsedPct, 'fleet.resources.memUsedPct')
|
|
116
|
+
assertNumberOrNull(row.fleet.resources.memPressurePct, 'fleet.resources.memPressurePct')
|
|
117
|
+
assert.ok(['normal', 'warning', 'critical', 'unavailable'].includes(row.fleet.resources.memPressureClass), 'fleet.resources.memPressureClass invalid')
|
|
118
|
+
assert.equal(typeof row.fleet.usage, 'object', 'fleet.usage must exist')
|
|
119
|
+
assert.ok(row.fleet.usage.model === null || typeof row.fleet.usage.model === 'string', 'fleet.usage.model invalid')
|
|
120
|
+
assertNumberOrNull(row.fleet.usage.totalTokens, 'fleet.usage.totalTokens')
|
|
121
|
+
assertNumberOrNull(row.fleet.usage.tokensPerMin, 'fleet.usage.tokensPerMin')
|
|
122
|
+
assert.ok(row.fleet.usage.sessionId === null || typeof row.fleet.usage.sessionId === 'string', 'fleet.usage.sessionId invalid')
|
|
123
|
+
assert.ok(row.fleet.usage.agentId === null || typeof row.fleet.usage.agentId === 'string', 'fleet.usage.agentId invalid')
|
|
124
|
+
assertNumberOrNull(row.fleet.usage.usageTimestampMs, 'fleet.usage.usageTimestampMs')
|
|
125
|
+
assertNumberOrNull(row.fleet.usage.usageAgeMs, 'fleet.usage.usageAgeMs')
|
|
126
|
+
assert.ok(row.fleet.usage.freshnessState === null || ['fresh', 'aging', 'stale', 'unknown', 'disabled'].includes(row.fleet.usage.freshnessState), 'fleet.usage.freshnessState invalid')
|
|
127
|
+
assert.ok(['ok', 'stale', 'disabled', 'unavailable'].includes(row.fleet.usage.integrationStatus), 'fleet.usage.integrationStatus invalid')
|
|
128
|
+
assert.ok(['ok', 'disabled', 'unavailable'].includes(row.fleet.usage.ingestionStatus), 'fleet.usage.ingestionStatus invalid')
|
|
129
|
+
assert.ok(['fresh', 'aging', 'stale', 'unknown', 'disabled', 'unavailable'].includes(row.fleet.usage.activityStatus), 'fleet.usage.activityStatus invalid')
|
|
130
|
+
assert.ok(['ok', 'notice', 'warning', 'critical', 'off'].includes(row.fleet.usage.alertLevel), 'fleet.usage.alertLevel invalid')
|
|
131
|
+
assert.equal(typeof row.fleet.provenance, 'object', 'fleet.provenance must exist')
|
|
132
|
+
assert.equal(typeof row.fleet.provenance.collector, 'string', 'fleet.provenance.collector must be string')
|
|
133
|
+
assert.ok(row.fleet.provenance.collectorVersion === null || typeof row.fleet.provenance.collectorVersion === 'string', 'fleet.provenance.collectorVersion invalid')
|
|
134
|
+
assert.ok(['openclaw', 'disabled', 'unavailable'].includes(source.usage), 'source.usage invalid')
|
|
135
|
+
assert.ok(['ok', 'stale', 'disabled', 'unavailable'].includes(source.usageIntegrationStatus), 'source.usageIntegrationStatus invalid')
|
|
136
|
+
assert.ok(['ok', 'disabled', 'unavailable'].includes(source.usageIngestionStatus), 'source.usageIngestionStatus invalid')
|
|
137
|
+
assert.ok(['fresh', 'aging', 'stale', 'unknown', 'disabled', 'unavailable'].includes(source.usageFreshnessState), 'source.usageFreshnessState invalid')
|
|
138
|
+
assert.ok(['fresh', 'aging', 'stale', 'unknown', 'disabled', 'unavailable'].includes(source.usageActivityStatus), 'source.usageActivityStatus invalid')
|
|
139
|
+
assert.equal(typeof source.usageNearStale, 'boolean', 'source.usageNearStale must be boolean')
|
|
140
|
+
assert.equal(typeof source.usagePastStaleThreshold, 'boolean', 'source.usagePastStaleThreshold must be boolean')
|
|
141
|
+
assert.equal(typeof source.usageRefreshAttempted, 'boolean', 'source.usageRefreshAttempted must be boolean')
|
|
142
|
+
assert.equal(typeof source.usageRefreshRecovered, 'boolean', 'source.usageRefreshRecovered must be boolean')
|
|
143
|
+
assert.ok(Number.isInteger(source.usageRefreshAttempts) && source.usageRefreshAttempts >= 0, 'source.usageRefreshAttempts must be integer >= 0')
|
|
144
|
+
assert.ok(Number.isInteger(source.usageRefreshReprobes) && source.usageRefreshReprobes >= 0, 'source.usageRefreshReprobes must be integer >= 0')
|
|
145
|
+
assert.ok(Number.isFinite(source.usageRefreshDelayMs) && source.usageRefreshDelayMs >= 0, 'source.usageRefreshDelayMs must be number >= 0')
|
|
146
|
+
assert.ok(source.usageRefreshDurationMs === null || Number.isFinite(source.usageRefreshDurationMs), 'source.usageRefreshDurationMs must be number or null')
|
|
147
|
+
assert.equal(typeof source.usageRefreshOnNearStale, 'boolean', 'source.usageRefreshOnNearStale must be boolean')
|
|
148
|
+
assert.equal(typeof source.usageIdle, 'boolean', 'source.usageIdle must be boolean')
|
|
149
|
+
assert.ok(source.usageCommand === null || typeof source.usageCommand === 'string', 'source.usageCommand must be string or null')
|
|
150
|
+
assert.ok(['ok', 'fallback-cache', 'disabled', 'command-missing', 'command-error', 'parse-error', 'unavailable'].includes(source.usageProbeResult), 'source.usageProbeResult invalid')
|
|
151
|
+
assert.ok(Number.isInteger(source.usageProbeAttempts) && source.usageProbeAttempts >= 0, 'source.usageProbeAttempts must be integer >= 0')
|
|
152
|
+
assert.ok(source.usageProbeDurationMs === null || (Number.isFinite(source.usageProbeDurationMs) && source.usageProbeDurationMs >= 0), 'source.usageProbeDurationMs must be number or null')
|
|
153
|
+
assert.ok(Number.isInteger(source.usageProbeSweeps) && source.usageProbeSweeps >= 0, 'source.usageProbeSweeps must be integer >= 0')
|
|
154
|
+
assert.ok(Number.isInteger(source.usageProbeRetries) && source.usageProbeRetries >= 0, 'source.usageProbeRetries must be integer >= 0')
|
|
155
|
+
assert.ok(Number.isFinite(source.usageProbeTimeoutMs) && source.usageProbeTimeoutMs > 0, 'source.usageProbeTimeoutMs must be number > 0')
|
|
156
|
+
assert.ok(source.usageProbeError === null || typeof source.usageProbeError === 'string', 'source.usageProbeError must be string or null')
|
|
157
|
+
assert.equal(typeof source.usageUsedFallbackCache, 'boolean', 'source.usageUsedFallbackCache must be boolean')
|
|
158
|
+
assert.ok(source.usageFallbackCacheSource === null || ['memory', 'disk'].includes(source.usageFallbackCacheSource), 'source.usageFallbackCacheSource must be memory|disk|null')
|
|
159
|
+
assert.ok(['ok', 'notice', 'warning', 'critical', 'off'].includes(source.usageAlertLevel), 'source.usageAlertLevel invalid')
|
|
160
|
+
assert.ok(['healthy', 'activity-idle', 'activity-near-stale', 'activity-past-threshold', 'activity-stale', 'activity-no-new-usage', 'ingestion-unavailable', 'usage-disabled'].includes(source.usageAlertReason), 'source.usageAlertReason invalid')
|
|
161
|
+
assertNumberOrNull(source.usageFallbackCacheAgeMs, 'source.usageFallbackCacheAgeMs')
|
|
162
|
+
assert.ok(Number.isFinite(source.usageStaleMsThreshold), 'source.usageStaleMsThreshold must be number')
|
|
163
|
+
assert.ok(Number.isFinite(source.usageNearStaleMsThreshold), 'source.usageNearStaleMsThreshold must be number')
|
|
164
|
+
assert.ok(Number.isFinite(source.usageStaleGraceMs), 'source.usageStaleGraceMs must be number')
|
|
165
|
+
assert.ok(Number.isFinite(source.usageIdleAfterMsThreshold), 'source.usageIdleAfterMsThreshold must be number')
|
|
166
|
+
assert.ok(typeof source.memPressureSource === 'string', 'source.memPressureSource must be string')
|
|
167
|
+
|
|
168
|
+
assert.equal(row.fleet.usage.tokensPerMin, row.tokensPerMin, 'fleet usage tokensPerMin must mirror legacy field')
|
|
169
|
+
assert.equal(row.fleet.usage.totalTokens, row.openclawTotalTokens, 'fleet usage totalTokens must mirror legacy field')
|
|
170
|
+
assert.equal(row.fleet.usage.model, row.openclawModel, 'fleet usage model must mirror legacy field')
|
|
171
|
+
assert.equal(row.fleet.usage.sessionId, row.openclawSessionId, 'fleet usage sessionId must mirror legacy field')
|
|
172
|
+
assert.equal(row.fleet.usage.agentId, row.openclawAgentId, 'fleet usage.agentId must mirror legacy field')
|
|
173
|
+
|
|
174
|
+
if (source.usage === 'openclaw') {
|
|
175
|
+
assert.ok(row.openclawSessionId, 'openclawSessionId required when source.usage=openclaw')
|
|
176
|
+
assert.ok(row.openclawUsageTs, 'openclawUsageTs required when source.usage=openclaw')
|
|
177
|
+
assert.ok(source.usageFreshnessState, 'usageFreshnessState required when source.usage=openclaw')
|
|
178
|
+
assert.ok(['ok', 'fallback-cache'].includes(source.usageProbeResult), 'usageProbeResult must be ok or fallback-cache when source.usage=openclaw')
|
|
179
|
+
assert.equal(source.usageIngestionStatus, 'ok', 'usageIngestionStatus must be ok when source.usage=openclaw')
|
|
180
|
+
assert.ok(['fresh', 'aging', 'stale', 'unknown'].includes(source.usageActivityStatus), 'usageActivityStatus must reflect freshness when source.usage=openclaw')
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (source.usage === 'disabled') {
|
|
184
|
+
assert.equal(source.usageProbeResult, 'disabled', 'usageProbeResult must be disabled when source.usage=disabled')
|
|
185
|
+
assert.equal(source.usageIngestionStatus, 'disabled', 'usageIngestionStatus must be disabled when source.usage=disabled')
|
|
186
|
+
assert.equal(source.usageActivityStatus, 'disabled', 'usageActivityStatus must be disabled when source.usage=disabled')
|
|
187
|
+
assert.equal(source.usageFreshnessState, 'disabled', 'usageFreshnessState must be disabled when source.usage=disabled')
|
|
188
|
+
assert.equal(source.usageAlertLevel, 'off', 'usageAlertLevel must be off when source.usage=disabled')
|
|
189
|
+
assert.equal(source.usageAlertReason, 'usage-disabled', 'usageAlertReason must be usage-disabled when source.usage=disabled')
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (source.usageProbeAttempts === 0) {
|
|
193
|
+
assert.equal(source.usageProbeSweeps, 0, 'usageProbeSweeps must be 0 when no probe attempts ran')
|
|
194
|
+
} else {
|
|
195
|
+
assert.ok(source.usageProbeSweeps >= 1, 'usageProbeSweeps must be >= 1 when probe attempts run')
|
|
196
|
+
}
|
|
197
|
+
assert.ok(source.usageProbeSweeps <= source.usageProbeRetries + 1, 'usageProbeSweeps cannot exceed configured retries + 1')
|
|
198
|
+
if (source.usageRefreshRecovered) {
|
|
199
|
+
assert.equal(source.usageRefreshAttempted, true, 'usageRefreshRecovered implies usageRefreshAttempted')
|
|
200
|
+
}
|
|
201
|
+
if (source.usageRefreshAttempted) {
|
|
202
|
+
assert.ok(source.usageRefreshAttempts >= 1, 'usageRefreshAttempts must be >= 1 when refresh attempted')
|
|
203
|
+
assert.ok(source.usageRefreshAttempts <= source.usageRefreshReprobes + 1, 'usageRefreshAttempts cannot exceed configured reprobes + 1')
|
|
204
|
+
} else {
|
|
205
|
+
assert.equal(source.usageRefreshAttempts, 0, 'usageRefreshAttempts must be 0 when refresh not attempted')
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (source.usageAlertReason === 'activity-no-new-usage') {
|
|
209
|
+
assert.equal(source.usageIngestionStatus, 'ok', 'activity-no-new-usage requires ingestionStatus=ok')
|
|
210
|
+
assert.equal(source.usageActivityStatus, 'stale', 'activity-no-new-usage requires activityStatus=stale')
|
|
211
|
+
assert.equal(source.usageRefreshAttempted, true, 'activity-no-new-usage requires usageRefreshAttempted=true')
|
|
212
|
+
assert.equal(source.usageRefreshRecovered, false, 'activity-no-new-usage requires usageRefreshRecovered=false')
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (source.usage === 'unavailable') {
|
|
216
|
+
assert.ok(!['ok', 'fallback-cache'].includes(source.usageProbeResult), 'usageProbeResult must explain unavailable usage')
|
|
217
|
+
assert.ok(source.usageProbeAttempts > 0 || source.usageProbeResult === 'disabled', 'usageProbeAttempts should be > 0 when unavailable')
|
|
218
|
+
assert.equal(source.usageIngestionStatus, 'unavailable', 'usageIngestionStatus must be unavailable when source.usage=unavailable')
|
|
219
|
+
assert.equal(source.usageActivityStatus, 'unavailable', 'usageActivityStatus must be unavailable when source.usage=unavailable')
|
|
220
|
+
assert.equal(source.usageAlertLevel, 'critical', 'usageAlertLevel must be critical when source=unavailable')
|
|
221
|
+
assert.equal(source.usageAlertReason, 'ingestion-unavailable', 'usageAlertReason must be ingestion-unavailable when source=unavailable')
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (source.usageProbeResult === 'fallback-cache') {
|
|
225
|
+
assert.equal(source.usageUsedFallbackCache, true, 'usageUsedFallbackCache must be true for fallback-cache')
|
|
226
|
+
assert.ok(Number.isFinite(source.usageFallbackCacheAgeMs), 'usageFallbackCacheAgeMs required for fallback-cache')
|
|
227
|
+
assert.ok(['memory', 'disk'].includes(source.usageFallbackCacheSource), 'usageFallbackCacheSource required for fallback-cache')
|
|
228
|
+
} else {
|
|
229
|
+
assert.equal(source.usageUsedFallbackCache, false, 'usageUsedFallbackCache must be false unless fallback-cache')
|
|
230
|
+
assert.equal(source.usageFallbackCacheAgeMs, null, 'usageFallbackCacheAgeMs must be null unless fallback-cache')
|
|
231
|
+
assert.equal(source.usageFallbackCacheSource, null, 'usageFallbackCacheSource must be null unless fallback-cache')
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (source.usageIngestionStatus === 'ok') {
|
|
235
|
+
assert.ok(['ok', 'notice', 'warning'].includes(source.usageAlertLevel), 'ingestion ok cannot emit critical/off alert level')
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (REQUIRE_OPENCLAW_USAGE) {
|
|
239
|
+
assert.equal(source.usage, 'openclaw', 'IDLEWATCH_REQUIRE_OPENCLAW_USAGE=1 requires source.usage=openclaw')
|
|
240
|
+
assert.notEqual(source.usageIntegrationStatus, 'unavailable', 'OpenClaw usage must not be unavailable in strict mode')
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if (MAX_OPENCLAW_USAGE_AGE_MS !== null && source.usage === 'openclaw') {
|
|
244
|
+
assert.ok(Number.isFinite(row.openclawUsageAgeMs), 'openclawUsageAgeMs must be present when enforcing max age')
|
|
245
|
+
assert.ok(
|
|
246
|
+
row.openclawUsageAgeMs <= MAX_OPENCLAW_USAGE_AGE_MS,
|
|
247
|
+
`openclawUsageAgeMs (${row.openclawUsageAgeMs}) exceeds max allowed ${MAX_OPENCLAW_USAGE_AGE_MS}`
|
|
248
|
+
)
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
try {
|
|
253
|
+
run()
|
|
254
|
+
} catch (err) {
|
|
255
|
+
console.error(`dry-run schema validation failed: ${err.message}`)
|
|
256
|
+
process.exit(1)
|
|
257
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { spawnSync } from 'node:child_process'
|
|
5
|
+
|
|
6
|
+
const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..')
|
|
7
|
+
const binPath = path.join(repoRoot, 'bin', 'idlewatch-agent.js')
|
|
8
|
+
const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'idlewatch-onboarding-'))
|
|
9
|
+
|
|
10
|
+
try {
|
|
11
|
+
const sourceCreds = path.join(tmpRoot, 'service-account.json')
|
|
12
|
+
fs.writeFileSync(
|
|
13
|
+
sourceCreds,
|
|
14
|
+
JSON.stringify({
|
|
15
|
+
type: 'service_account',
|
|
16
|
+
project_id: 'idlewatch-test-project',
|
|
17
|
+
private_key_id: 'abc123',
|
|
18
|
+
private_key: '-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n',
|
|
19
|
+
client_email: 'idlewatch@idlewatch-test-project.iam.gserviceaccount.com',
|
|
20
|
+
client_id: '123'
|
|
21
|
+
})
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
const envOut = path.join(tmpRoot, 'generated.env')
|
|
25
|
+
const configDir = path.join(tmpRoot, 'config')
|
|
26
|
+
const run = spawnSync(process.execPath, [binPath, 'quickstart'], {
|
|
27
|
+
env: {
|
|
28
|
+
...process.env,
|
|
29
|
+
IDLEWATCH_ENROLL_NON_INTERACTIVE: '1',
|
|
30
|
+
IDLEWATCH_ENROLL_MODE: 'production',
|
|
31
|
+
IDLEWATCH_ENROLL_PROJECT_ID: 'idlewatch-test-project',
|
|
32
|
+
IDLEWATCH_ENROLL_SERVICE_ACCOUNT_FILE: sourceCreds,
|
|
33
|
+
IDLEWATCH_ENROLL_OUTPUT_ENV_FILE: envOut,
|
|
34
|
+
IDLEWATCH_ENROLL_CONFIG_DIR: configDir
|
|
35
|
+
},
|
|
36
|
+
encoding: 'utf8'
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
if (run.status !== 0) {
|
|
40
|
+
throw new Error(`quickstart failed\nstdout:\n${run.stdout}\nstderr:\n${run.stderr}`)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (!fs.existsSync(envOut)) {
|
|
44
|
+
throw new Error('quickstart did not create output env file')
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const envContent = fs.readFileSync(envOut, 'utf8')
|
|
48
|
+
if (!envContent.includes('FIREBASE_PROJECT_ID=idlewatch-test-project')) {
|
|
49
|
+
throw new Error('env file missing FIREBASE_PROJECT_ID')
|
|
50
|
+
}
|
|
51
|
+
if (!envContent.includes('FIREBASE_SERVICE_ACCOUNT_FILE=')) {
|
|
52
|
+
throw new Error('env file missing FIREBASE_SERVICE_ACCOUNT_FILE')
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const copiedCreds = path.join(configDir, 'credentials', 'idlewatch-test-project-service-account.json')
|
|
56
|
+
if (!fs.existsSync(copiedCreds)) {
|
|
57
|
+
throw new Error('quickstart did not copy service account file to secure credentials path')
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
console.log('onboarding validation passed')
|
|
61
|
+
} finally {
|
|
62
|
+
fs.rmSync(tmpRoot, { recursive: true, force: true })
|
|
63
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import { mkdtempSync, writeFileSync, chmodSync, rmSync, readFileSync } from 'node:fs'
|
|
4
|
+
import { readTelemetryJsonRow } from './lib/telemetry-row-parser.mjs'
|
|
5
|
+
import { tmpdir } from 'node:os'
|
|
6
|
+
import { join } from 'node:path'
|
|
7
|
+
import { execFileSync } from 'node:child_process'
|
|
8
|
+
|
|
9
|
+
const repoRoot = process.cwd()
|
|
10
|
+
const tempDir = mkdtempSync(join(tmpdir(), 'idlewatch-openclaw-cache-recover-'))
|
|
11
|
+
const mockBinPath = join(tempDir, 'openclaw-mock.sh')
|
|
12
|
+
const cachePath = join(tempDir, 'openclaw-last-good.json')
|
|
13
|
+
const callLog = join(tempDir, 'calls.txt')
|
|
14
|
+
const staleAgeMs = 90000
|
|
15
|
+
const staleUsageTs = Date.now() - staleAgeMs
|
|
16
|
+
|
|
17
|
+
function writeMockOpenClaw(path) {
|
|
18
|
+
const script = `#!/usr/bin/env bash
|
|
19
|
+
set -euo pipefail
|
|
20
|
+
calls_file="${callLog}"
|
|
21
|
+
if [[ ! -f "$calls_file" ]]; then
|
|
22
|
+
echo 0 > "$calls_file"
|
|
23
|
+
fi
|
|
24
|
+
n=$(cat "$calls_file")
|
|
25
|
+
next=$((n + 1))
|
|
26
|
+
echo "$next" > "$calls_file"
|
|
27
|
+
|
|
28
|
+
if [[ "$n" -lt 5 ]]; then
|
|
29
|
+
echo "openclaw temporary failure for recovery test" >&2
|
|
30
|
+
exit 1
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
now_ms=$(node -p 'Date.now()')
|
|
34
|
+
updated_at=$((now_ms - 2000))
|
|
35
|
+
cat <<JSON
|
|
36
|
+
{"sessions":{"defaults":{"model":"gpt-5.3-codex"},"recent":[{"sessionId":"cached-recover","agentId":"main","model":"gpt-5.3-codex","totalTokens":98765,"totalTokensFresh":true,"updatedAt":$updated_at,"updated_at":$updated_at}]},"ts":$now_ms}
|
|
37
|
+
JSON
|
|
38
|
+
`
|
|
39
|
+
|
|
40
|
+
writeFileSync(path, script, 'utf8')
|
|
41
|
+
chmodSync(path, 0o755)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function readRow(output) {
|
|
45
|
+
return readTelemetryJsonRow(output)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function run() {
|
|
49
|
+
const mockNow = Date.now()
|
|
50
|
+
const staleSnapshot = {
|
|
51
|
+
at: mockNow,
|
|
52
|
+
usage: {
|
|
53
|
+
model: 'gpt-5.3-codex',
|
|
54
|
+
totalTokens: 1111,
|
|
55
|
+
tokensPerMin: 45.0,
|
|
56
|
+
sessionId: 'cached-recover',
|
|
57
|
+
agentId: 'main',
|
|
58
|
+
usageTimestampMs: staleUsageTs,
|
|
59
|
+
sourceCommand: 'legacy-cache'
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
writeFileSync(cachePath, JSON.stringify(staleSnapshot), 'utf8')
|
|
64
|
+
writeMockOpenClaw(mockBinPath)
|
|
65
|
+
|
|
66
|
+
const env = {
|
|
67
|
+
...process.env,
|
|
68
|
+
IDLEWATCH_OPENCLAW_BIN: mockBinPath,
|
|
69
|
+
IDLEWATCH_OPENCLAW_BIN_STRICT: '1',
|
|
70
|
+
IDLEWATCH_OPENCLAW_LAST_GOOD_CACHE_PATH: cachePath,
|
|
71
|
+
IDLEWATCH_USAGE_STALE_MS: '60000',
|
|
72
|
+
IDLEWATCH_USAGE_STALE_GRACE_MS: '10000',
|
|
73
|
+
IDLEWATCH_USAGE_REFRESH_REPROBES: '1',
|
|
74
|
+
IDLEWATCH_USAGE_REFRESH_DELAY_MS: '0',
|
|
75
|
+
IDLEWATCH_OPENCLAW_PROBE_RETRIES: '0',
|
|
76
|
+
IDLEWATCH_USAGE_REFRESH_ON_NEAR_STALE: '1',
|
|
77
|
+
IDLEWATCH_INTERVAL_MS: '1000'
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const out = execFileSync('node', ['bin/idlewatch-agent.js', '--dry-run'], {
|
|
81
|
+
cwd: repoRoot,
|
|
82
|
+
encoding: 'utf8',
|
|
83
|
+
env,
|
|
84
|
+
stdio: ['ignore', 'pipe', 'pipe']
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
const row = readRow(out)
|
|
88
|
+
assert.equal(row?.source?.usage, 'openclaw', 'usage source should remain openclaw with fallback-cache path')
|
|
89
|
+
assert.equal(
|
|
90
|
+
row?.source?.usageProbeResult === 'fallback-cache' || row?.source?.usageProbeResult === 'ok',
|
|
91
|
+
true,
|
|
92
|
+
'expected fallback-cache or live probe result'
|
|
93
|
+
)
|
|
94
|
+
assert.equal(row?.source?.usageRefreshAttempted, true, 'stale cache path should trigger refresh attempts')
|
|
95
|
+
assert.equal(row?.source?.usageRefreshAttempts >= 1, true, 'expected at least one refresh attempt')
|
|
96
|
+
|
|
97
|
+
if (row?.source?.usageProbeResult === 'ok') {
|
|
98
|
+
assert.equal(row?.source?.usageFreshnessState, 'fresh', 'recovered probe should return fresh state')
|
|
99
|
+
assert.equal(row?.source?.usageRefreshRecovered, true, 'stale cache path should recover when probe succeeds')
|
|
100
|
+
assert.equal(row?.source?.usageAlertLevel, 'ok', 'activity should be healthy after recovery')
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const calls = Number(readFileSync(callLog, 'utf8').trim())
|
|
104
|
+
assert.ok(Number.isFinite(calls) && calls >= 2, `expected mock probe called at least twice, got ${calls}`)
|
|
105
|
+
|
|
106
|
+
console.log('validate-openclaw-cache-recovery-e2e: ok (fallback-cache sample recovered via forced reprobe)')
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
try {
|
|
110
|
+
run()
|
|
111
|
+
} finally {
|
|
112
|
+
rmSync(tempDir, { recursive: true, force: true })
|
|
113
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { spawnSync } from 'node:child_process'
|
|
3
|
+
|
|
4
|
+
const repoRoot = process.cwd()
|
|
5
|
+
|
|
6
|
+
const RELEASE_GATE_TIMEOUT_MS = process.env.IDLEWATCH_DRY_RUN_TIMEOUT_MS || '60000'
|
|
7
|
+
|
|
8
|
+
function shouldRequireOpenClaw(rawValue) {
|
|
9
|
+
const raw = String(rawValue ?? '1').trim().toLowerCase()
|
|
10
|
+
if (raw === '0' || raw === 'false' || raw === 'off' || raw === 'no') return false
|
|
11
|
+
if (raw === '1' || raw === 'true' || raw === 'on' || raw === 'yes') return true
|
|
12
|
+
return true
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function runValidator(name, extraEnv = {}) {
|
|
16
|
+
const requireOpenClawUsage = shouldRequireOpenClaw(process.env.IDLEWATCH_REQUIRE_OPENCLAW_USAGE)
|
|
17
|
+
const requireOpenClaw = requireOpenClawUsage ? '1' : '0'
|
|
18
|
+
|
|
19
|
+
const result = spawnSync('npm', ['run', name, '--silent'], {
|
|
20
|
+
cwd: repoRoot,
|
|
21
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
22
|
+
encoding: 'utf8',
|
|
23
|
+
env: {
|
|
24
|
+
...process.env,
|
|
25
|
+
IDLEWATCH_REQUIRE_OPENCLAW_USAGE: requireOpenClaw,
|
|
26
|
+
IDLEWATCH_DRY_RUN_TIMEOUT_MS: RELEASE_GATE_TIMEOUT_MS,
|
|
27
|
+
...extraEnv
|
|
28
|
+
}
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
const out = String(result.stdout || '') + String(result.stderr || '')
|
|
32
|
+
if (result.status !== 0) {
|
|
33
|
+
if (out.trim()) {
|
|
34
|
+
console.error(out.trim())
|
|
35
|
+
}
|
|
36
|
+
console.error(`Validator ${name} failed with exit code ${result.status}`)
|
|
37
|
+
process.exit(result.status)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
process.stdout.write(out)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function main() {
|
|
44
|
+
runValidator('validate:openclaw-usage-health')
|
|
45
|
+
runValidator('validate:openclaw-stats-ingestion')
|
|
46
|
+
runValidator('validate:openclaw-cache-recovery-e2e')
|
|
47
|
+
|
|
48
|
+
console.log('validate-openclaw-release-gates: ok (host OpenClaw checks for usage-health, stats fallback, and stale-cache recovery)')
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
main()
|