idlewatch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +73 -0
- package/.github/workflows/ci.yml +99 -0
- package/.github/workflows/release-macos-trusted.yml +103 -0
- package/README.md +336 -0
- package/bin/idlewatch-agent.js +1053 -0
- package/docs/onboarding-external.md +58 -0
- package/docs/packaging/macos-dmg.md +199 -0
- package/docs/packaging/macos-launch-agent.md +70 -0
- package/docs/qa/archive/mac-qa-log-2026-02-17.md +5838 -0
- package/docs/qa/mac-qa-log.md +2864 -0
- package/docs/telemetry/idle-stale-policy.md +57 -0
- package/docs/telemetry/openclaw-mapping.md +80 -0
- package/package.json +76 -0
- package/scripts/build-dmg.sh +65 -0
- package/scripts/install-macos-launch-agent.sh +78 -0
- package/scripts/lib/telemetry-row-parser.mjs +100 -0
- package/scripts/package-macos.sh +228 -0
- package/scripts/uninstall-macos-launch-agent.sh +30 -0
- package/scripts/validate-all.sh +142 -0
- package/scripts/validate-bin.mjs +25 -0
- package/scripts/validate-dmg-checksum.sh +37 -0
- package/scripts/validate-dmg-install.sh +155 -0
- package/scripts/validate-dry-run-schema.mjs +257 -0
- package/scripts/validate-onboarding.mjs +63 -0
- package/scripts/validate-openclaw-cache-recovery-e2e.mjs +113 -0
- package/scripts/validate-openclaw-release-gates.mjs +51 -0
- package/scripts/validate-openclaw-stats-ingestion.mjs +372 -0
- package/scripts/validate-openclaw-usage-health.mjs +95 -0
- package/scripts/validate-packaged-artifact.mjs +233 -0
- package/scripts/validate-packaged-bundled-runtime.sh +191 -0
- package/scripts/validate-packaged-metadata.sh +43 -0
- package/scripts/validate-packaged-openclaw-cache-recovery-e2e.mjs +153 -0
- package/scripts/validate-packaged-openclaw-release-gates.mjs +72 -0
- package/scripts/validate-packaged-openclaw-stats-ingestion.mjs +402 -0
- package/scripts/validate-packaged-sourcemaps.mjs +82 -0
- package/scripts/validate-packaged-usage-alert-rate-e2e.mjs +98 -0
- package/scripts/validate-packaged-usage-probe-noise-e2e.mjs +87 -0
- package/scripts/validate-packaged-usage-recovery-e2e.mjs +90 -0
- package/scripts/validate-trusted-prereqs.sh +44 -0
- package/scripts/validate-usage-alert-rate-e2e.mjs +91 -0
- package/scripts/validate-usage-freshness-e2e.mjs +81 -0
- package/skill/SKILL.md +43 -0
- package/src/config.js +100 -0
- package/src/enrollment.js +176 -0
- package/src/gpu.js +115 -0
- package/src/memory.js +67 -0
- package/src/openclaw-cache.js +51 -0
- package/src/openclaw-usage.js +1020 -0
- package/src/telemetry-mapping.js +54 -0
- package/src/usage-alert.js +41 -0
- package/src/usage-freshness.js +31 -0
- package/test/config.test.mjs +112 -0
- package/test/fixtures/gpu-agx.txt +2 -0
- package/test/fixtures/gpu-iogpu.txt +2 -0
- package/test/fixtures/gpu-top-grep.txt +2 -0
- package/test/fixtures/openclaw-fleet-sample-v1.json +68 -0
- package/test/fixtures/openclaw-mixed-equal-score-status-vs-generic-iso-ts.txt +2 -0
- package/test/fixtures/openclaw-mixed-equal-score-status-vs-generic-newest.txt +2 -0
- package/test/fixtures/openclaw-mixed-equal-score-status-vs-generic-string-ts.txt +2 -0
- package/test/fixtures/openclaw-mixed-status-then-generic-output.txt +2 -0
- package/test/fixtures/openclaw-stats-current-wrapper.json +12 -0
- package/test/fixtures/openclaw-stats-current-wrapper2.json +15 -0
- package/test/fixtures/openclaw-stats-data-wrapper.json +21 -0
- package/test/fixtures/openclaw-stats-nested-session-wrapper.json +23 -0
- package/test/fixtures/openclaw-stats-payload-wrapper.json +1 -0
- package/test/fixtures/openclaw-stats-status-current-wrapper.json +19 -0
- package/test/fixtures/openclaw-stats.json +17 -0
- package/test/fixtures/openclaw-status-ansi-complex-noise.txt +3 -0
- package/test/fixtures/openclaw-status-ansi-noise.txt +2 -0
- package/test/fixtures/openclaw-status-control-noise.txt +1 -0
- package/test/fixtures/openclaw-status-data-wrapper.json +20 -0
- package/test/fixtures/openclaw-status-dcs-noise.txt +1 -0
- package/test/fixtures/openclaw-status-epoch-seconds.json +15 -0
- package/test/fixtures/openclaw-status-mixed-noise.txt +1 -0
- package/test/fixtures/openclaw-status-multi-json.txt +3 -0
- package/test/fixtures/openclaw-status-nested-recent.json +19 -0
- package/test/fixtures/openclaw-status-noisy-default-then-usage.txt +2 -0
- package/test/fixtures/openclaw-status-noisy.txt +3 -0
- package/test/fixtures/openclaw-status-osc-noise.txt +1 -0
- package/test/fixtures/openclaw-status-result-session.json +15 -0
- package/test/fixtures/openclaw-status-session-map-with-defaults.json +23 -0
- package/test/fixtures/openclaw-status-session-map.json +28 -0
- package/test/fixtures/openclaw-status-session-model-name.json +18 -0
- package/test/fixtures/openclaw-status-snake-session-wrapper.json +13 -0
- package/test/fixtures/openclaw-status-stats-current-sessions-snake-tokens.json +25 -0
- package/test/fixtures/openclaw-status-stats-current-sessions.json +28 -0
- package/test/fixtures/openclaw-status-stats-current-usage-time-camelcase.json +19 -0
- package/test/fixtures/openclaw-status-stats-session-default-model.json +27 -0
- package/test/fixtures/openclaw-status-status-wrapper.json +13 -0
- package/test/fixtures/openclaw-status-strings.json +38 -0
- package/test/fixtures/openclaw-status-ts-ms-alias.json +14 -0
- package/test/fixtures/openclaw-status-updated-at-ms-alias.json +14 -0
- package/test/fixtures/openclaw-status-usage-timestamp-ms-alias.json +14 -0
- package/test/fixtures/openclaw-status-usage-ts-alias.json +14 -0
- package/test/fixtures/openclaw-status-wrap-session-object.json +24 -0
- package/test/fixtures/openclaw-status.json +41 -0
- package/test/fixtures/openclaw-usage-model-name-generic.json +9 -0
- package/test/gpu.test.mjs +58 -0
- package/test/memory.test.mjs +35 -0
- package/test/openclaw-cache.test.mjs +48 -0
- package/test/openclaw-env.test.mjs +365 -0
- package/test/openclaw-usage.test.mjs +555 -0
- package/test/telemetry-mapping.test.mjs +69 -0
- package/test/telemetry-row-parser.test.mjs +44 -0
- package/test/usage-alert.test.mjs +73 -0
- package/test/usage-freshness.test.mjs +63 -0
- package/test/validate-dry-run-schema.test.mjs +146 -0
- package/tui/Cargo.lock +801 -0
- package/tui/Cargo.toml +11 -0
- package/tui/src/main.rs +368 -0
|
@@ -0,0 +1,1053 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from 'fs'
|
|
3
|
+
import { accessSync, constants } from 'node:fs'
|
|
4
|
+
import os from 'os'
|
|
5
|
+
import path from 'path'
|
|
6
|
+
import process from 'process'
|
|
7
|
+
import { spawnSync } from 'node:child_process'
|
|
8
|
+
import { createRequire } from 'module'
|
|
9
|
+
import { parseOpenClawUsage } from '../src/openclaw-usage.js'
|
|
10
|
+
import { gpuSampleDarwin } from '../src/gpu.js'
|
|
11
|
+
import { memUsedPct, memoryPressureDarwin } from '../src/memory.js'
|
|
12
|
+
import { deriveUsageFreshness } from '../src/usage-freshness.js'
|
|
13
|
+
import { deriveUsageAlert } from '../src/usage-alert.js'
|
|
14
|
+
import { loadLastGoodUsageSnapshot, persistLastGoodUsageSnapshot } from '../src/openclaw-cache.js'
|
|
15
|
+
import { runEnrollmentWizard } from '../src/enrollment.js'
|
|
16
|
+
import { enrichWithOpenClawFleetTelemetry } from '../src/telemetry-mapping.js'
|
|
17
|
+
import pkg from '../package.json' with { type: 'json' }
|
|
18
|
+
|
|
19
|
+
function printHelp() {
|
|
20
|
+
console.log(`idlewatch-agent\n\nUsage:\n idlewatch-agent [quickstart] [--dry-run] [--once] [--help]\n\nOptions:\n quickstart Run first-run enrollment wizard and generate secure env config\n --dry-run Collect and print one telemetry sample, then exit without Firebase writes\n --once Collect and publish one telemetry sample, then exit\n --help Show this help message\n\nEnvironment:\n IDLEWATCH_HOST Optional custom host label (default: hostname)\n IDLEWATCH_INTERVAL_MS Sampling interval in ms (default: 10000)\n IDLEWATCH_LOCAL_LOG_PATH Optional NDJSON file path for local sample durability\n IDLEWATCH_OPENCLAW_USAGE OpenClaw usage lookup mode: auto|off (default: auto)\n IDLEWATCH_OPENCLAW_PROBE_TIMEOUT_MS OpenClaw command timeout per probe in ms (default: 2500)\n IDLEWATCH_OPENCLAW_PROBE_RETRIES Extra OpenClaw probe sweep retries after first pass (default: 1)\n IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES Max per-command OpenClaw probe output capture in bytes before truncation (default: 2097152 / 2MB)\n IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES_HARD_CAP Hard cap for auto-retry output capture escalation (default: 16777216 / 16MB)\n IDLEWATCH_USAGE_STALE_MS Mark OpenClaw usage stale beyond this age in ms (default: max(interval*3,60000))\n IDLEWATCH_USAGE_NEAR_STALE_MS Mark OpenClaw usage as aging beyond this age in ms (default: floor((stale+grace)*0.85))\n IDLEWATCH_USAGE_STALE_GRACE_MS Extra grace window before status becomes stale (default: min(interval,10000))\n IDLEWATCH_USAGE_REFRESH_REPROBES Forced uncached reprobes when usage crosses stale threshold (default: 1)\n IDLEWATCH_USAGE_REFRESH_DELAY_MS Delay between forced stale-threshold reprobes in ms (default: 250)\n IDLEWATCH_USAGE_REFRESH_ON_NEAR_STALE Trigger refresh when usage is near-stale: 1|0 (default: 1)\n IDLEWATCH_USAGE_IDLE_AFTER_MS Downgrade stale usage alerts to idle notice beyond this age in ms (default: 21600000)\n IDLEWATCH_OPENCLAW_LAST_GOOD_MAX_AGE_MS Reuse last successful usage snapshot after probe failures up to this age in ms\n IDLEWATCH_OPENCLAW_LAST_GOOD_CACHE_PATH Persist/reuse last successful usage snapshot across restarts (default: ~/.idlewatch/cache/<host>-openclaw-last-good.json)\n IDLEWATCH_REQUIRE_FIREBASE_WRITES Require Firebase publish path in --once mode: 1|0 (default: 0)\n FIREBASE_PROJECT_ID Firebase project id\n FIREBASE_SERVICE_ACCOUNT_FILE Path to service account JSON file (preferred for production)\n FIREBASE_SERVICE_ACCOUNT_JSON Raw JSON service account (supported, less secure than file path)\n FIREBASE_SERVICE_ACCOUNT_B64 Base64-encoded JSON service account (legacy)\n FIRESTORE_EMULATOR_HOST Optional Firestore emulator host; allows local writes without service-account creds\n IDLEWATCH_CLOUD_INGEST_URL Optional cloud ingest endpoint (e.g. https://idlewatch.com/api/ingest)\n IDLEWATCH_CLOUD_API_KEY Cloud API key from dashboard for device linking\n IDLEWATCH_REQUIRE_CLOUD_WRITES Require cloud publish path in --once mode: 1|0 (default: 0)\n`)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const require = createRequire(import.meta.url)
|
|
24
|
+
|
|
25
|
+
function parseEnvFileToObject(envFilePath) {
|
|
26
|
+
const raw = fs.readFileSync(envFilePath, 'utf8')
|
|
27
|
+
const env = {}
|
|
28
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
29
|
+
const trimmed = line.trim()
|
|
30
|
+
if (!trimmed || trimmed.startsWith('#')) continue
|
|
31
|
+
const idx = trimmed.indexOf('=')
|
|
32
|
+
if (idx <= 0) continue
|
|
33
|
+
const key = trimmed.slice(0, idx).trim()
|
|
34
|
+
const value = trimmed.slice(idx + 1).trim()
|
|
35
|
+
if (key) env[key] = value
|
|
36
|
+
}
|
|
37
|
+
return env
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function parseMonitorTargets(raw) {
|
|
41
|
+
const allowed = new Set(['cpu', 'memory', 'gpu', 'openclaw'])
|
|
42
|
+
const fallback = ['cpu', 'memory', 'openclaw', 'gpu']
|
|
43
|
+
|
|
44
|
+
if (!raw || typeof raw !== 'string') {
|
|
45
|
+
return new Set(fallback)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const parsed = raw
|
|
49
|
+
.split(',')
|
|
50
|
+
.map((item) => item.trim().toLowerCase())
|
|
51
|
+
.filter((item) => allowed.has(item))
|
|
52
|
+
|
|
53
|
+
if (parsed.length === 0) return new Set(fallback)
|
|
54
|
+
return new Set(parsed)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const argv = process.argv.slice(2)
|
|
58
|
+
const quickstartRequested = argv[0] === 'quickstart' || argv.includes('--quickstart')
|
|
59
|
+
const args = new Set(argv)
|
|
60
|
+
if (args.has('--help') || args.has('-h')) {
|
|
61
|
+
printHelp()
|
|
62
|
+
process.exit(0)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (quickstartRequested) {
|
|
66
|
+
try {
|
|
67
|
+
const result = await runEnrollmentWizard()
|
|
68
|
+
console.log(`Enrollment complete. Mode=${result.mode} envFile=${result.outputEnvFile}`)
|
|
69
|
+
|
|
70
|
+
const enrolledEnv = parseEnvFileToObject(result.outputEnvFile)
|
|
71
|
+
const onceRun = spawnSync(process.execPath, [process.argv[1], '--once'], {
|
|
72
|
+
stdio: 'inherit',
|
|
73
|
+
env: {
|
|
74
|
+
...process.env,
|
|
75
|
+
...enrolledEnv
|
|
76
|
+
}
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
if (onceRun.status === 0) {
|
|
80
|
+
console.log('✅ Initial telemetry sample sent successfully.')
|
|
81
|
+
process.exit(0)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
console.log('⚠️ Initial --once sample did not complete successfully.')
|
|
85
|
+
console.log(`You can retry with: set -a; source "${result.outputEnvFile}"; set +a && idlewatch-agent --once`)
|
|
86
|
+
process.exit(onceRun.status ?? 1)
|
|
87
|
+
} catch (err) {
|
|
88
|
+
console.error(`Enrollment failed: ${err.message}`)
|
|
89
|
+
process.exit(1)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const DRY_RUN = args.has('--dry-run')
|
|
94
|
+
const ONCE = args.has('--once')
|
|
95
|
+
const HOST = process.env.IDLEWATCH_HOST || os.hostname()
|
|
96
|
+
const SAFE_HOST = HOST.replace(/[^a-zA-Z0-9_.-]/g, '_')
|
|
97
|
+
const INTERVAL_MS = Number(process.env.IDLEWATCH_INTERVAL_MS || 10000)
|
|
98
|
+
const PROJECT_ID = process.env.FIREBASE_PROJECT_ID
|
|
99
|
+
const CREDS_FILE = process.env.FIREBASE_SERVICE_ACCOUNT_FILE
|
|
100
|
+
const CREDS_JSON = process.env.FIREBASE_SERVICE_ACCOUNT_JSON
|
|
101
|
+
const CREDS_B64 = process.env.FIREBASE_SERVICE_ACCOUNT_B64
|
|
102
|
+
const FIRESTORE_EMULATOR_HOST = process.env.FIRESTORE_EMULATOR_HOST
|
|
103
|
+
const OPENCLAW_USAGE_MODE = (process.env.IDLEWATCH_OPENCLAW_USAGE || 'auto').toLowerCase()
|
|
104
|
+
const MONITOR_TARGETS = parseMonitorTargets(process.env.IDLEWATCH_MONITOR_TARGETS)
|
|
105
|
+
const MONITOR_CPU = MONITOR_TARGETS.has('cpu')
|
|
106
|
+
const MONITOR_MEMORY = MONITOR_TARGETS.has('memory')
|
|
107
|
+
const MONITOR_GPU = MONITOR_TARGETS.has('gpu')
|
|
108
|
+
const MONITOR_OPENCLAW = MONITOR_TARGETS.has('openclaw')
|
|
109
|
+
const EFFECTIVE_OPENCLAW_MODE = MONITOR_OPENCLAW ? OPENCLAW_USAGE_MODE : 'off'
|
|
110
|
+
const REQUIRE_FIREBASE_WRITES = process.env.IDLEWATCH_REQUIRE_FIREBASE_WRITES === '1'
|
|
111
|
+
const CLOUD_INGEST_URL = process.env.IDLEWATCH_CLOUD_INGEST_URL
|
|
112
|
+
const CLOUD_API_KEY = process.env.IDLEWATCH_CLOUD_API_KEY
|
|
113
|
+
const REQUIRE_CLOUD_WRITES = process.env.IDLEWATCH_REQUIRE_CLOUD_WRITES === '1'
|
|
114
|
+
const OPENCLAW_PROBE_TIMEOUT_MS = Number(process.env.IDLEWATCH_OPENCLAW_PROBE_TIMEOUT_MS || 2500)
|
|
115
|
+
const OPENCLAW_PROBE_MAX_OUTPUT_BYTES = process.env.IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES
|
|
116
|
+
? Number(process.env.IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES)
|
|
117
|
+
: 2 * 1024 * 1024
|
|
118
|
+
const OPENCLAW_PROBE_MAX_OUTPUT_BYTES_HARD_CAP = process.env.IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES_HARD_CAP
|
|
119
|
+
? Number(process.env.IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES_HARD_CAP)
|
|
120
|
+
: 16 * 1024 * 1024
|
|
121
|
+
const OPENCLAW_BIN_STRICT = process.env.IDLEWATCH_OPENCLAW_BIN_STRICT === '1'
|
|
122
|
+
const OPENCLAW_PROBE_RETRIES = process.env.IDLEWATCH_OPENCLAW_PROBE_RETRIES
|
|
123
|
+
? Number(process.env.IDLEWATCH_OPENCLAW_PROBE_RETRIES)
|
|
124
|
+
: 1
|
|
125
|
+
const BASE_DIR = path.join(os.homedir(), '.idlewatch')
|
|
126
|
+
|
|
127
|
+
const LOCAL_LOG_PATH = process.env.IDLEWATCH_LOCAL_LOG_PATH
|
|
128
|
+
? path.resolve(process.env.IDLEWATCH_LOCAL_LOG_PATH)
|
|
129
|
+
: path.join(BASE_DIR, 'logs', `${SAFE_HOST}-metrics.ndjson`)
|
|
130
|
+
|
|
131
|
+
if (!Number.isFinite(INTERVAL_MS) || INTERVAL_MS <= 0) {
|
|
132
|
+
console.error(`Invalid IDLEWATCH_INTERVAL_MS: ${process.env.IDLEWATCH_INTERVAL_MS}. Expected a positive number.`)
|
|
133
|
+
process.exit(1)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (!Number.isFinite(OPENCLAW_PROBE_TIMEOUT_MS) || OPENCLAW_PROBE_TIMEOUT_MS <= 0) {
|
|
137
|
+
console.error(
|
|
138
|
+
`Invalid IDLEWATCH_OPENCLAW_PROBE_TIMEOUT_MS: ${process.env.IDLEWATCH_OPENCLAW_PROBE_TIMEOUT_MS}. Expected a positive number.`
|
|
139
|
+
)
|
|
140
|
+
process.exit(1)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (!Number.isFinite(OPENCLAW_PROBE_MAX_OUTPUT_BYTES) || OPENCLAW_PROBE_MAX_OUTPUT_BYTES <= 0) {
|
|
144
|
+
console.error(
|
|
145
|
+
`Invalid IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES: undefined. Expected a positive number.`
|
|
146
|
+
)
|
|
147
|
+
process.exit(1)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if (!Number.isFinite(OPENCLAW_PROBE_MAX_OUTPUT_BYTES_HARD_CAP) || OPENCLAW_PROBE_MAX_OUTPUT_BYTES_HARD_CAP < OPENCLAW_PROBE_MAX_OUTPUT_BYTES) {
|
|
151
|
+
console.error(
|
|
152
|
+
`Invalid IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES_HARD_CAP: ${process.env.IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES_HARD_CAP}. Must be a number >= IDLEWATCH_OPENCLAW_MAX_OUTPUT_BYTES.`
|
|
153
|
+
)
|
|
154
|
+
process.exit(1)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (!Number.isInteger(OPENCLAW_PROBE_RETRIES) || OPENCLAW_PROBE_RETRIES < 0) {
|
|
158
|
+
console.error(
|
|
159
|
+
`Invalid IDLEWATCH_OPENCLAW_PROBE_RETRIES: ${process.env.IDLEWATCH_OPENCLAW_PROBE_RETRIES}. Expected an integer >= 0.`
|
|
160
|
+
)
|
|
161
|
+
process.exit(1)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const USAGE_STALE_MS = process.env.IDLEWATCH_USAGE_STALE_MS
|
|
165
|
+
? Number(process.env.IDLEWATCH_USAGE_STALE_MS)
|
|
166
|
+
: Math.max(INTERVAL_MS * 3, 60000)
|
|
167
|
+
|
|
168
|
+
if (!Number.isFinite(USAGE_STALE_MS) || USAGE_STALE_MS <= 0) {
|
|
169
|
+
console.error(`Invalid IDLEWATCH_USAGE_STALE_MS: ${process.env.IDLEWATCH_USAGE_STALE_MS}. Expected a positive number.`)
|
|
170
|
+
process.exit(1)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const USAGE_STALE_GRACE_MS = process.env.IDLEWATCH_USAGE_STALE_GRACE_MS
|
|
174
|
+
? Number(process.env.IDLEWATCH_USAGE_STALE_GRACE_MS)
|
|
175
|
+
: Math.min(INTERVAL_MS, 10000)
|
|
176
|
+
|
|
177
|
+
if (!Number.isFinite(USAGE_STALE_GRACE_MS) || USAGE_STALE_GRACE_MS < 0) {
|
|
178
|
+
console.error(
|
|
179
|
+
`Invalid IDLEWATCH_USAGE_STALE_GRACE_MS: ${process.env.IDLEWATCH_USAGE_STALE_GRACE_MS}. Expected a non-negative number.`
|
|
180
|
+
)
|
|
181
|
+
process.exit(1)
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const USAGE_NEAR_STALE_MS = process.env.IDLEWATCH_USAGE_NEAR_STALE_MS
|
|
185
|
+
? Number(process.env.IDLEWATCH_USAGE_NEAR_STALE_MS)
|
|
186
|
+
: Math.floor((USAGE_STALE_MS + USAGE_STALE_GRACE_MS) * 0.85)
|
|
187
|
+
|
|
188
|
+
if (!Number.isFinite(USAGE_NEAR_STALE_MS) || USAGE_NEAR_STALE_MS < 0) {
|
|
189
|
+
console.error(
|
|
190
|
+
`Invalid IDLEWATCH_USAGE_NEAR_STALE_MS: ${process.env.IDLEWATCH_USAGE_NEAR_STALE_MS}. Expected a non-negative number.`
|
|
191
|
+
)
|
|
192
|
+
process.exit(1)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const USAGE_REFRESH_REPROBES = process.env.IDLEWATCH_USAGE_REFRESH_REPROBES
|
|
196
|
+
? Number(process.env.IDLEWATCH_USAGE_REFRESH_REPROBES)
|
|
197
|
+
: 1
|
|
198
|
+
|
|
199
|
+
if (!Number.isInteger(USAGE_REFRESH_REPROBES) || USAGE_REFRESH_REPROBES < 0) {
|
|
200
|
+
console.error(
|
|
201
|
+
`Invalid IDLEWATCH_USAGE_REFRESH_REPROBES: ${process.env.IDLEWATCH_USAGE_REFRESH_REPROBES}. Expected an integer >= 0.`
|
|
202
|
+
)
|
|
203
|
+
process.exit(1)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const USAGE_REFRESH_DELAY_MS = process.env.IDLEWATCH_USAGE_REFRESH_DELAY_MS
|
|
207
|
+
? Number(process.env.IDLEWATCH_USAGE_REFRESH_DELAY_MS)
|
|
208
|
+
: 250
|
|
209
|
+
|
|
210
|
+
if (!Number.isFinite(USAGE_REFRESH_DELAY_MS) || USAGE_REFRESH_DELAY_MS < 0) {
|
|
211
|
+
console.error(
|
|
212
|
+
`Invalid IDLEWATCH_USAGE_REFRESH_DELAY_MS: ${process.env.IDLEWATCH_USAGE_REFRESH_DELAY_MS}. Expected a non-negative number.`
|
|
213
|
+
)
|
|
214
|
+
process.exit(1)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const USAGE_REFRESH_ON_NEAR_STALE = process.env.IDLEWATCH_USAGE_REFRESH_ON_NEAR_STALE
|
|
218
|
+
? Number(process.env.IDLEWATCH_USAGE_REFRESH_ON_NEAR_STALE)
|
|
219
|
+
: 1
|
|
220
|
+
|
|
221
|
+
if (![0, 1].includes(USAGE_REFRESH_ON_NEAR_STALE)) {
|
|
222
|
+
console.error(
|
|
223
|
+
`Invalid IDLEWATCH_USAGE_REFRESH_ON_NEAR_STALE: ${process.env.IDLEWATCH_USAGE_REFRESH_ON_NEAR_STALE}. Expected 0 or 1.`
|
|
224
|
+
)
|
|
225
|
+
process.exit(1)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const USAGE_IDLE_AFTER_MS = process.env.IDLEWATCH_USAGE_IDLE_AFTER_MS
|
|
229
|
+
? Number(process.env.IDLEWATCH_USAGE_IDLE_AFTER_MS)
|
|
230
|
+
: 21600000
|
|
231
|
+
|
|
232
|
+
if (!Number.isFinite(USAGE_IDLE_AFTER_MS) || USAGE_IDLE_AFTER_MS <= 0) {
|
|
233
|
+
console.error(
|
|
234
|
+
`Invalid IDLEWATCH_USAGE_IDLE_AFTER_MS: ${process.env.IDLEWATCH_USAGE_IDLE_AFTER_MS}. Expected a positive number.`
|
|
235
|
+
)
|
|
236
|
+
process.exit(1)
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const OPENCLAW_LAST_GOOD_MAX_AGE_MS = process.env.IDLEWATCH_OPENCLAW_LAST_GOOD_MAX_AGE_MS
|
|
240
|
+
? Number(process.env.IDLEWATCH_OPENCLAW_LAST_GOOD_MAX_AGE_MS)
|
|
241
|
+
: Math.max(USAGE_STALE_MS + USAGE_STALE_GRACE_MS, 120000)
|
|
242
|
+
|
|
243
|
+
if (process.env.IDLEWATCH_OPENCLAW_LAST_GOOD_MAX_AGE_MS && (!Number.isFinite(OPENCLAW_LAST_GOOD_MAX_AGE_MS) || OPENCLAW_LAST_GOOD_MAX_AGE_MS <= 0)) {
|
|
244
|
+
console.error(
|
|
245
|
+
`Invalid IDLEWATCH_OPENCLAW_LAST_GOOD_MAX_AGE_MS: ${process.env.IDLEWATCH_OPENCLAW_LAST_GOOD_MAX_AGE_MS}. Expected a positive number.`
|
|
246
|
+
)
|
|
247
|
+
process.exit(1)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const OPENCLAW_LAST_GOOD_CACHE_PATH = process.env.IDLEWATCH_OPENCLAW_LAST_GOOD_CACHE_PATH
|
|
251
|
+
? path.resolve(process.env.IDLEWATCH_OPENCLAW_LAST_GOOD_CACHE_PATH)
|
|
252
|
+
: path.join(BASE_DIR, 'cache', `${SAFE_HOST}-openclaw-last-good.json`)
|
|
253
|
+
|
|
254
|
+
let appReady = false
|
|
255
|
+
let firebaseConfigError = null
|
|
256
|
+
let admin = null
|
|
257
|
+
|
|
258
|
+
function loadFirebaseAdmin() {
|
|
259
|
+
if (admin) return admin
|
|
260
|
+
try {
|
|
261
|
+
admin = require('firebase-admin')
|
|
262
|
+
return admin
|
|
263
|
+
} catch (err) {
|
|
264
|
+
firebaseConfigError = `Failed to load firebase-admin runtime dependency: ${err.message}`
|
|
265
|
+
return null
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (PROJECT_ID || CREDS_FILE || CREDS_JSON || CREDS_B64 || FIRESTORE_EMULATOR_HOST) {
|
|
270
|
+
if (!PROJECT_ID) {
|
|
271
|
+
firebaseConfigError =
|
|
272
|
+
'FIREBASE_PROJECT_ID is missing. Set FIREBASE_PROJECT_ID plus FIREBASE_SERVICE_ACCOUNT_FILE (preferred) or FIREBASE_SERVICE_ACCOUNT_JSON / FIREBASE_SERVICE_ACCOUNT_B64. For emulator-only mode, set FIREBASE_PROJECT_ID + FIRESTORE_EMULATOR_HOST.'
|
|
273
|
+
} else if (FIRESTORE_EMULATOR_HOST) {
|
|
274
|
+
const firebaseAdmin = loadFirebaseAdmin()
|
|
275
|
+
if (firebaseAdmin) {
|
|
276
|
+
try {
|
|
277
|
+
firebaseAdmin.initializeApp({ projectId: PROJECT_ID })
|
|
278
|
+
appReady = true
|
|
279
|
+
} catch (err) {
|
|
280
|
+
firebaseConfigError = `Failed to initialize Firebase emulator mode: ${err.message}`
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
} else if (!CREDS_FILE && !CREDS_JSON && !CREDS_B64) {
|
|
284
|
+
firebaseConfigError =
|
|
285
|
+
'Firebase credentials are missing. Set FIREBASE_SERVICE_ACCOUNT_FILE (preferred) or FIREBASE_SERVICE_ACCOUNT_JSON / FIREBASE_SERVICE_ACCOUNT_B64. For emulator-only mode, set FIRESTORE_EMULATOR_HOST.'
|
|
286
|
+
} else {
|
|
287
|
+
const firebaseAdmin = loadFirebaseAdmin()
|
|
288
|
+
if (firebaseAdmin) {
|
|
289
|
+
try {
|
|
290
|
+
const credsRaw = CREDS_FILE
|
|
291
|
+
? fs.readFileSync(path.resolve(CREDS_FILE), 'utf8')
|
|
292
|
+
: (CREDS_JSON || Buffer.from(CREDS_B64, 'base64').toString('utf8'))
|
|
293
|
+
const creds = JSON.parse(credsRaw)
|
|
294
|
+
firebaseAdmin.initializeApp({ credential: firebaseAdmin.credential.cert(creds), projectId: PROJECT_ID })
|
|
295
|
+
appReady = true
|
|
296
|
+
} catch (err) {
|
|
297
|
+
firebaseConfigError = `Failed to initialize Firebase credentials: ${err.message}`
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
if (firebaseConfigError) {
|
|
304
|
+
console.error(`Firebase configuration error: ${firebaseConfigError}`)
|
|
305
|
+
process.exit(1)
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
if (!appReady) {
|
|
309
|
+
console.error(
|
|
310
|
+
'Firebase is not configured. Running in local-only mode (stdout logging only). Set FIREBASE_PROJECT_ID + FIREBASE_SERVICE_ACCOUNT_FILE (preferred, or FIREBASE_SERVICE_ACCOUNT_JSON / FIREBASE_SERVICE_ACCOUNT_B64), or use FIREBASE_PROJECT_ID + FIRESTORE_EMULATOR_HOST for emulator writes.'
|
|
311
|
+
)
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if (REQUIRE_FIREBASE_WRITES && !appReady) {
|
|
315
|
+
console.error(
|
|
316
|
+
'IDLEWATCH_REQUIRE_FIREBASE_WRITES=1 requires Firebase to be configured. Set FIREBASE_PROJECT_ID with service-account creds (or FIRESTORE_EMULATOR_HOST for emulator mode).'
|
|
317
|
+
)
|
|
318
|
+
process.exit(1)
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function ensureDirFor(filePath) {
|
|
322
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true })
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function appendLocal(row) {
|
|
326
|
+
try {
|
|
327
|
+
ensureDirFor(LOCAL_LOG_PATH)
|
|
328
|
+
fs.appendFileSync(LOCAL_LOG_PATH, `${JSON.stringify(row)}\n`, 'utf8')
|
|
329
|
+
} catch (err) {
|
|
330
|
+
console.error(`Local log append failed (${LOCAL_LOG_PATH}): ${err.message}`)
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function snapshotCpuTimes() {
|
|
335
|
+
return os.cpus().map((c) => ({ ...c.times }))
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function cpuPctFromDeltas(previous, current) {
|
|
339
|
+
if (!previous || !current || previous.length !== current.length) return null
|
|
340
|
+
let idle = 0
|
|
341
|
+
let total = 0
|
|
342
|
+
for (let i = 0; i < previous.length; i++) {
|
|
343
|
+
const before = previous[i]
|
|
344
|
+
const after = current[i]
|
|
345
|
+
const didle = after.idle - before.idle
|
|
346
|
+
const dtotal =
|
|
347
|
+
(after.user - before.user) +
|
|
348
|
+
(after.nice - before.nice) +
|
|
349
|
+
(after.sys - before.sys) +
|
|
350
|
+
(after.irq - before.irq) +
|
|
351
|
+
didle
|
|
352
|
+
idle += didle
|
|
353
|
+
total += dtotal
|
|
354
|
+
}
|
|
355
|
+
if (total <= 0) return null
|
|
356
|
+
return Math.max(0, Math.min(100, Number((100 * (1 - idle / total)).toFixed(2))))
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
let previousCpuSnapshot = snapshotCpuTimes()
|
|
360
|
+
function cpuPct() {
|
|
361
|
+
const current = snapshotCpuTimes()
|
|
362
|
+
const pct = cpuPctFromDeltas(previousCpuSnapshot, current)
|
|
363
|
+
previousCpuSnapshot = current
|
|
364
|
+
return pct ?? 0
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const OPENCLAW_USAGE_TTL_MS = Math.max(INTERVAL_MS, 30000)
|
|
368
|
+
const MEM_PRESSURE_TTL_MS = Math.max(INTERVAL_MS, 30000)
|
|
369
|
+
let openClawUsageCache = {
|
|
370
|
+
at: 0,
|
|
371
|
+
value: {
|
|
372
|
+
usage: null,
|
|
373
|
+
probe: {
|
|
374
|
+
result: EFFECTIVE_OPENCLAW_MODE === 'off' ? 'disabled' : 'unavailable',
|
|
375
|
+
attempts: 0,
|
|
376
|
+
sweeps: 0,
|
|
377
|
+
command: null,
|
|
378
|
+
error: null,
|
|
379
|
+
usedFallbackCache: false,
|
|
380
|
+
fallbackAgeMs: null,
|
|
381
|
+
fallbackCacheSource: null
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
let lastGoodOpenClawUsage = (() => {
|
|
386
|
+
const cached = loadLastGoodUsageSnapshot(OPENCLAW_LAST_GOOD_CACHE_PATH)
|
|
387
|
+
if (!cached) return null
|
|
388
|
+
return { at: cached.at, usage: cached.usage, source: 'disk' }
|
|
389
|
+
})()
|
|
390
|
+
let preferredOpenClawProbe = null
|
|
391
|
+
let memPressureCache = { at: 0, value: { pct: null, cls: 'unavailable', source: 'unavailable' } }
|
|
392
|
+
|
|
393
|
+
function loadMemPressure() {
|
|
394
|
+
if (process.platform !== 'darwin') {
|
|
395
|
+
return { pct: null, cls: 'unavailable', source: 'unsupported' }
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
const now = Date.now()
|
|
399
|
+
if (now - memPressureCache.at < MEM_PRESSURE_TTL_MS) return memPressureCache.value
|
|
400
|
+
const sampled = memoryPressureDarwin()
|
|
401
|
+
memPressureCache = { at: now, value: sampled }
|
|
402
|
+
return sampled
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function gpuSampleNvidiaSmi() {
|
|
406
|
+
try {
|
|
407
|
+
const probe = spawnSync(
|
|
408
|
+
'nvidia-smi',
|
|
409
|
+
['--query-gpu=utilization.gpu', '--format=csv,noheader,nounits'],
|
|
410
|
+
{ encoding: 'utf8', timeout: 1500, maxBuffer: 256 * 1024 }
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
if (probe.error || probe.status !== 0) {
|
|
414
|
+
return { pct: null, source: 'nvidia-smi-unavailable', confidence: 'none', sampleWindowMs: null }
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const values = (probe.stdout || '')
|
|
418
|
+
.split(/\r?\n/)
|
|
419
|
+
.map((line) => Number(line.trim()))
|
|
420
|
+
.filter((value) => Number.isFinite(value) && value >= 0)
|
|
421
|
+
|
|
422
|
+
if (values.length === 0) {
|
|
423
|
+
return { pct: null, source: 'nvidia-smi-empty', confidence: 'none', sampleWindowMs: null }
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const average = values.reduce((sum, value) => sum + value, 0) / values.length
|
|
427
|
+
return {
|
|
428
|
+
pct: Math.max(0, Math.min(100, average)),
|
|
429
|
+
source: 'nvidia-smi',
|
|
430
|
+
confidence: 'medium',
|
|
431
|
+
sampleWindowMs: null
|
|
432
|
+
}
|
|
433
|
+
} catch {
|
|
434
|
+
return { pct: null, source: 'nvidia-smi-error', confidence: 'none', sampleWindowMs: null }
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function resolveOpenClawBinaries() {
|
|
439
|
+
const explicit = (process.env.IDLEWATCH_OPENCLAW_BIN?.trim()) || (process.env.IDLEWATCH_OPENCLAW_BIN_HINT?.trim())
|
|
440
|
+
const homeDir = process.env.HOME?.trim()
|
|
441
|
+
|
|
442
|
+
if (OPENCLAW_BIN_STRICT && explicit) {
|
|
443
|
+
return [explicit]
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const bins = [
|
|
447
|
+
explicit,
|
|
448
|
+
'/opt/homebrew/bin/openclaw',
|
|
449
|
+
'/usr/local/bin/openclaw',
|
|
450
|
+
'/usr/bin/openclaw',
|
|
451
|
+
'/usr/local/sbin/openclaw',
|
|
452
|
+
'/usr/sbin/openclaw',
|
|
453
|
+
homeDir ? `${homeDir}/.local/bin/openclaw` : null,
|
|
454
|
+
homeDir ? `${homeDir}/bin/openclaw` : null,
|
|
455
|
+
homeDir ? `${homeDir}/.npm-global/bin/openclaw` : null,
|
|
456
|
+
homeDir ? `${homeDir}/.nvm/versions/node/${process.version}/bin/openclaw` : null,
|
|
457
|
+
'/opt/homebrew/lib/node_modules/.bin/openclaw',
|
|
458
|
+
'openclaw'
|
|
459
|
+
].filter(Boolean)
|
|
460
|
+
|
|
461
|
+
const deduped = []
|
|
462
|
+
const seen = new Set()
|
|
463
|
+
for (const binPath of bins) {
|
|
464
|
+
if (seen.has(binPath)) continue
|
|
465
|
+
seen.add(binPath)
|
|
466
|
+
deduped.push(binPath)
|
|
467
|
+
}
|
|
468
|
+
return deduped
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function loadOpenClawUsage(forceRefresh = false) {
|
|
472
|
+
if (EFFECTIVE_OPENCLAW_MODE === 'off') {
|
|
473
|
+
return {
|
|
474
|
+
usage: null,
|
|
475
|
+
probe: {
|
|
476
|
+
result: 'disabled',
|
|
477
|
+
attempts: 0,
|
|
478
|
+
sweeps: 0,
|
|
479
|
+
command: null,
|
|
480
|
+
error: null,
|
|
481
|
+
usedFallbackCache: false,
|
|
482
|
+
fallbackAgeMs: null,
|
|
483
|
+
fallbackCacheSource: null,
|
|
484
|
+
durationMs: null
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
const now = Date.now()
|
|
490
|
+
if (!forceRefresh && now - openClawUsageCache.at < OPENCLAW_USAGE_TTL_MS) return openClawUsageCache.value
|
|
491
|
+
|
|
492
|
+
const binaries = resolveOpenClawBinaries()
|
|
493
|
+
const subcommands = [
|
|
494
|
+
['status', '--json'],
|
|
495
|
+
['usage', '--json'],
|
|
496
|
+
['session', 'status', '--json'],
|
|
497
|
+
['session_status', '--json'],
|
|
498
|
+
['stats', '--json']
|
|
499
|
+
]
|
|
500
|
+
|
|
501
|
+
const pathEntries = (process.env.PATH || '').split(':').filter(Boolean)
|
|
502
|
+
|
|
503
|
+
function isExecutable(candidate) {
|
|
504
|
+
try {
|
|
505
|
+
accessSync(candidate, constants.X_OK)
|
|
506
|
+
return true
|
|
507
|
+
} catch {
|
|
508
|
+
return false
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function hasPathExecutable(binName) {
|
|
513
|
+
for (const entry of pathEntries) {
|
|
514
|
+
const candidate = path.join(entry, binName)
|
|
515
|
+
if (isExecutable(candidate)) return true
|
|
516
|
+
}
|
|
517
|
+
return false
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function isBinaryAvailable(binPath) {
|
|
521
|
+
if (binPath.includes('/')) {
|
|
522
|
+
return isExecutable(binPath)
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
return hasPathExecutable(binPath)
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// Build an augmented PATH for probe subprocesses so that #!/usr/bin/env node
|
|
529
|
+
// scripts (like openclaw) can locate the node binary even when the packaged
|
|
530
|
+
// app runs with a restricted PATH (e.g. /usr/bin:/bin:/usr/sbin:/sbin).
|
|
531
|
+
const probeEnv = (() => {
|
|
532
|
+
const currentPath = process.env.PATH || ''
|
|
533
|
+
const extraDirs = new Set()
|
|
534
|
+
|
|
535
|
+
// Add directory of the running node binary (handles packaged + nvm setups)
|
|
536
|
+
const nodeDir = path.dirname(process.execPath)
|
|
537
|
+
if (nodeDir && nodeDir !== '.') extraDirs.add(nodeDir)
|
|
538
|
+
|
|
539
|
+
// Common Homebrew / system node locations
|
|
540
|
+
for (const dir of ['/opt/homebrew/bin', '/usr/local/bin', `${process.env.HOME || ''}/.local/bin`]) {
|
|
541
|
+
if (dir) extraDirs.add(dir)
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
const pathDirs = currentPath.split(':').filter(Boolean)
|
|
545
|
+
const augmented = [...new Set([...extraDirs, ...pathDirs])].join(':')
|
|
546
|
+
return { ...process.env, PATH: augmented }
|
|
547
|
+
})()
|
|
548
|
+
|
|
549
|
+
function runProbe(binPath, args) {
|
|
550
|
+
const startMs = Date.now()
|
|
551
|
+
let limit = OPENCLAW_PROBE_MAX_OUTPUT_BYTES
|
|
552
|
+
|
|
553
|
+
while (limit <= OPENCLAW_PROBE_MAX_OUTPUT_BYTES_HARD_CAP) {
|
|
554
|
+
try {
|
|
555
|
+
const result = spawnSync(binPath, args, {
|
|
556
|
+
encoding: 'utf8',
|
|
557
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
558
|
+
timeout: OPENCLAW_PROBE_TIMEOUT_MS,
|
|
559
|
+
maxBuffer: limit,
|
|
560
|
+
env: probeEnv
|
|
561
|
+
})
|
|
562
|
+
|
|
563
|
+
const stdoutPayload = typeof result.stdout === 'string' ? result.stdout.trim() : ''
|
|
564
|
+
const stderrPayload = typeof result.stderr === 'string' ? result.stderr.trim() : ''
|
|
565
|
+
const status = result.status === 0 ? 'ok' : 'ok-with-stderr'
|
|
566
|
+
const commandStatus = result.status
|
|
567
|
+
const combinedOutput = [stdoutPayload, stderrPayload].filter(Boolean).join('\n')
|
|
568
|
+
|
|
569
|
+
if (combinedOutput) {
|
|
570
|
+
return {
|
|
571
|
+
out: combinedOutput,
|
|
572
|
+
error: status === 'ok'
|
|
573
|
+
? null
|
|
574
|
+
: `command-exited-${String(commandStatus || 'nonzero')}: ${(stderrPayload || 'non-zero-exit').split('\n')[0].slice(0, 120)}`,
|
|
575
|
+
status,
|
|
576
|
+
maxBuffer: limit,
|
|
577
|
+
durationMs: Date.now() - startMs
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
if (result.error?.code === 'ENOENT') {
|
|
582
|
+
return {
|
|
583
|
+
out: null,
|
|
584
|
+
error: 'openclaw-not-found',
|
|
585
|
+
status: 'command-error',
|
|
586
|
+
maxBuffer: limit,
|
|
587
|
+
durationMs: Date.now() - startMs
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
if (result.status !== 0) {
|
|
592
|
+
return {
|
|
593
|
+
out: null,
|
|
594
|
+
error: `command-exited-${String(commandStatus || 'nonzero')}`,
|
|
595
|
+
status: 'command-error',
|
|
596
|
+
maxBuffer: limit,
|
|
597
|
+
durationMs: Date.now() - startMs
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
return { out: null, error: null, status: 'ok', maxBuffer: limit, durationMs: Date.now() - startMs }
|
|
602
|
+
} catch (err) {
|
|
603
|
+
const stdoutText = typeof err?.stdout === 'string' ? err.stdout : ''
|
|
604
|
+
const stderrText = typeof err?.stderr === 'string' ? err.stderr : ''
|
|
605
|
+
const stdoutPayload = stdoutText.trim()
|
|
606
|
+
const stderrPayload = stderrText.trim()
|
|
607
|
+
const cmdStatus = err?.status
|
|
608
|
+
|
|
609
|
+
if ((err?.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || err?.code === 'ENOBUFS') && limit < OPENCLAW_PROBE_MAX_OUTPUT_BYTES_HARD_CAP) {
|
|
610
|
+
limit = Math.min(limit * 2, OPENCLAW_PROBE_MAX_OUTPUT_BYTES_HARD_CAP)
|
|
611
|
+
continue
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
if (stdoutPayload || stderrPayload) {
|
|
615
|
+
const candidateOutput = [stdoutPayload, stderrPayload].filter(Boolean).join('\n')
|
|
616
|
+
return {
|
|
617
|
+
out: candidateOutput,
|
|
618
|
+
error: `command-exited-${String(cmdStatus || 'nonzero')}: ${(stderrPayload || 'non-zero-exit').split('\n')[0].slice(0, 120)}`,
|
|
619
|
+
status: 'ok-with-stderr',
|
|
620
|
+
maxBuffer: limit,
|
|
621
|
+
durationMs: Date.now() - startMs
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
if (err?.code === 'ENOENT') {
|
|
626
|
+
return { out: null, error: 'openclaw-not-found', status: 'command-error', maxBuffer: limit, durationMs: Date.now() - startMs }
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
return {
|
|
630
|
+
out: null,
|
|
631
|
+
error: err?.message ? String(err.message).split('\n')[0].slice(0, 180) : 'command-failed',
|
|
632
|
+
status: 'command-error',
|
|
633
|
+
maxBuffer: limit,
|
|
634
|
+
durationMs: Date.now() - startMs
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
return {
|
|
640
|
+
out: null,
|
|
641
|
+
error: 'command-failed: max-buffer-limit-reached',
|
|
642
|
+
status: 'command-error',
|
|
643
|
+
maxBuffer: limit,
|
|
644
|
+
durationMs: Date.now() - startMs
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
let attempts = 0
|
|
649
|
+
let sweeps = 0
|
|
650
|
+
let sawCommandError = false
|
|
651
|
+
let sawParseError = false
|
|
652
|
+
let sawCommandMissing = false
|
|
653
|
+
let lastError = null
|
|
654
|
+
|
|
655
|
+
const evaluateProbe = (binPath, cmdArgs, isPreferred = false) => {
|
|
656
|
+
const cmdText = `${binPath} ${cmdArgs.join(' ')}`
|
|
657
|
+
attempts += 1
|
|
658
|
+
const probeRun = runProbe(binPath, cmdArgs)
|
|
659
|
+
|
|
660
|
+
if (probeRun.out !== null) {
|
|
661
|
+
const parsed = parseOpenClawUsage(probeRun.out)
|
|
662
|
+
if (parsed) {
|
|
663
|
+
preferredOpenClawProbe = { binPath, args: cmdArgs }
|
|
664
|
+
const usage = { ...parsed, sourceCommand: cmdText }
|
|
665
|
+
const value = {
|
|
666
|
+
usage,
|
|
667
|
+
probe: {
|
|
668
|
+
result: 'ok',
|
|
669
|
+
attempts,
|
|
670
|
+
sweeps,
|
|
671
|
+
command: cmdText,
|
|
672
|
+
error: probeRun.status === 'ok-with-stderr' ? probeRun.error : null,
|
|
673
|
+
usedFallbackCache: false,
|
|
674
|
+
fallbackAgeMs: null,
|
|
675
|
+
fallbackCacheSource: null,
|
|
676
|
+
durationMs: probeRun.durationMs
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
lastGoodOpenClawUsage = { at: now, usage, source: 'memory' }
|
|
680
|
+
persistLastGoodUsageSnapshot(OPENCLAW_LAST_GOOD_CACHE_PATH, { at: now, usage })
|
|
681
|
+
openClawUsageCache = { at: now, value }
|
|
682
|
+
return value
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
sawParseError = true
|
|
686
|
+
lastError = 'unrecognized-json-shape'
|
|
687
|
+
preferredOpenClawProbe = isPreferred ? null : preferredOpenClawProbe
|
|
688
|
+
return null
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
if (probeRun.status === 'command-error') {
|
|
692
|
+
if (probeRun.error === 'openclaw-not-found') {
|
|
693
|
+
sawCommandMissing = true
|
|
694
|
+
lastError = probeRun.error
|
|
695
|
+
preferredOpenClawProbe = isPreferred ? null : preferredOpenClawProbe
|
|
696
|
+
return null
|
|
697
|
+
}
|
|
698
|
+
sawCommandError = true
|
|
699
|
+
lastError = probeRun.error
|
|
700
|
+
preferredOpenClawProbe = isPreferred ? null : preferredOpenClawProbe
|
|
701
|
+
return null
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
return null
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
if (preferredOpenClawProbe) {
|
|
708
|
+
sweeps = 1
|
|
709
|
+
if (!isBinaryAvailable(preferredOpenClawProbe.binPath)) {
|
|
710
|
+
preferredOpenClawProbe = null
|
|
711
|
+
} else {
|
|
712
|
+
const cachedResult = evaluateProbe(preferredOpenClawProbe.binPath, preferredOpenClawProbe.args, true)
|
|
713
|
+
if (cachedResult) return cachedResult
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
for (let sweep = 0; sweep <= OPENCLAW_PROBE_RETRIES; sweep++) {
|
|
718
|
+
sweeps = sweep + 1
|
|
719
|
+
let sweepHasPotentialExecutable = false
|
|
720
|
+
|
|
721
|
+
for (const binPath of binaries) {
|
|
722
|
+
const candidateExecutable = isBinaryAvailable(binPath)
|
|
723
|
+
if (!candidateExecutable) {
|
|
724
|
+
continue
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
sweepHasPotentialExecutable = true
|
|
728
|
+
for (const args of subcommands) {
|
|
729
|
+
const candidateResult = evaluateProbe(binPath, args)
|
|
730
|
+
if (candidateResult) return candidateResult
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
if (
|
|
735
|
+
sawCommandMissing &&
|
|
736
|
+
!sawCommandError &&
|
|
737
|
+
!sawParseError &&
|
|
738
|
+
!sweepHasPotentialExecutable
|
|
739
|
+
) {
|
|
740
|
+
break
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
const result = sawParseError
|
|
745
|
+
? 'parse-error'
|
|
746
|
+
: sawCommandError
|
|
747
|
+
? 'command-error'
|
|
748
|
+
: 'command-missing'
|
|
749
|
+
|
|
750
|
+
const fallbackAgeMs = lastGoodOpenClawUsage ? now - lastGoodOpenClawUsage.at : null
|
|
751
|
+
if (lastGoodOpenClawUsage && Number.isFinite(fallbackAgeMs) && fallbackAgeMs <= OPENCLAW_LAST_GOOD_MAX_AGE_MS) {
|
|
752
|
+
const value = {
|
|
753
|
+
usage: { ...lastGoodOpenClawUsage.usage, sourceCommand: `${lastGoodOpenClawUsage.usage.sourceCommand} (cached)` },
|
|
754
|
+
probe: {
|
|
755
|
+
result: 'fallback-cache',
|
|
756
|
+
attempts,
|
|
757
|
+
sweeps,
|
|
758
|
+
command: null,
|
|
759
|
+
error: lastError,
|
|
760
|
+
usedFallbackCache: true,
|
|
761
|
+
fallbackAgeMs,
|
|
762
|
+
fallbackCacheSource: lastGoodOpenClawUsage.source || 'memory',
|
|
763
|
+
durationMs: null
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
openClawUsageCache = { at: now, value }
|
|
767
|
+
return value
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
const value = {
|
|
771
|
+
usage: null,
|
|
772
|
+
probe: {
|
|
773
|
+
result,
|
|
774
|
+
attempts,
|
|
775
|
+
sweeps,
|
|
776
|
+
command: null,
|
|
777
|
+
error: lastError,
|
|
778
|
+
usedFallbackCache: false,
|
|
779
|
+
fallbackAgeMs: null,
|
|
780
|
+
fallbackCacheSource: null,
|
|
781
|
+
durationMs: null
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
openClawUsageCache = { at: now, value }
|
|
785
|
+
return value
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
async function publish(row, retries = 2) {
|
|
789
|
+
if (DRY_RUN) return false
|
|
790
|
+
|
|
791
|
+
if (CLOUD_INGEST_URL && CLOUD_API_KEY) {
|
|
792
|
+
let attempt = 0
|
|
793
|
+
while (attempt <= retries) {
|
|
794
|
+
try {
|
|
795
|
+
const response = await fetch(CLOUD_INGEST_URL, {
|
|
796
|
+
method: 'POST',
|
|
797
|
+
headers: {
|
|
798
|
+
'content-type': 'application/json',
|
|
799
|
+
'x-idlewatch-key': CLOUD_API_KEY
|
|
800
|
+
},
|
|
801
|
+
body: JSON.stringify(row)
|
|
802
|
+
})
|
|
803
|
+
if (!response.ok) {
|
|
804
|
+
throw new Error(`cloud_ingest_failed_${response.status}`)
|
|
805
|
+
}
|
|
806
|
+
return true
|
|
807
|
+
} catch (err) {
|
|
808
|
+
if (attempt >= retries) throw err
|
|
809
|
+
await new Promise((resolve) => setTimeout(resolve, 300 * (attempt + 1)))
|
|
810
|
+
attempt += 1
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
return false
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
if (!appReady) return false
|
|
817
|
+
const db = admin.firestore()
|
|
818
|
+
let attempt = 0
|
|
819
|
+
while (attempt <= retries) {
|
|
820
|
+
try {
|
|
821
|
+
await db.collection('metrics').add(row)
|
|
822
|
+
return true
|
|
823
|
+
} catch (err) {
|
|
824
|
+
if (attempt >= retries) throw err
|
|
825
|
+
await new Promise((resolve) => setTimeout(resolve, 300 * (attempt + 1)))
|
|
826
|
+
attempt += 1
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
return false
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
async function collectSample() {
|
|
833
|
+
const sampleStartMs = Date.now()
|
|
834
|
+
const openclawEnabled = EFFECTIVE_OPENCLAW_MODE !== 'off'
|
|
835
|
+
|
|
836
|
+
const disabledProbe = {
|
|
837
|
+
result: 'disabled',
|
|
838
|
+
attempts: 0,
|
|
839
|
+
sweeps: 0,
|
|
840
|
+
error: null,
|
|
841
|
+
durationMs: null,
|
|
842
|
+
usedFallbackCache: false,
|
|
843
|
+
fallbackAgeMs: null,
|
|
844
|
+
fallbackCacheSource: null
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
let usageProbe = openclawEnabled ? loadOpenClawUsage() : { usage: null, probe: disabledProbe }
|
|
848
|
+
let usage = usageProbe.usage
|
|
849
|
+
let usageFreshness = deriveUsageFreshness(usage, sampleStartMs, USAGE_STALE_MS, USAGE_NEAR_STALE_MS, USAGE_STALE_GRACE_MS)
|
|
850
|
+
let usageRefreshAttempted = false
|
|
851
|
+
let usageRefreshRecovered = false
|
|
852
|
+
let usageRefreshAttempts = 0
|
|
853
|
+
let usageRefreshDurationMs = null
|
|
854
|
+
let usageRefreshStartMs = null
|
|
855
|
+
|
|
856
|
+
const shouldRefreshForNearStale = USAGE_REFRESH_ON_NEAR_STALE === 1 && usageFreshness.isNearStale
|
|
857
|
+
const canRefreshFromCurrentState = usageProbe.probe.result === 'ok' || usageProbe.probe.result === 'fallback-cache'
|
|
858
|
+
|
|
859
|
+
if (openclawEnabled && usage && (usageFreshness.isPastStaleThreshold || shouldRefreshForNearStale) && canRefreshFromCurrentState) {
|
|
860
|
+
usageRefreshAttempted = true
|
|
861
|
+
usageRefreshStartMs = Date.now()
|
|
862
|
+
|
|
863
|
+
for (let attempt = 0; attempt <= USAGE_REFRESH_REPROBES; attempt++) {
|
|
864
|
+
usageRefreshAttempts += 1
|
|
865
|
+
if (attempt > 0 && USAGE_REFRESH_DELAY_MS > 0) {
|
|
866
|
+
await new Promise((resolve) => setTimeout(resolve, USAGE_REFRESH_DELAY_MS))
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
const refreshedUsageProbe = loadOpenClawUsage(true)
|
|
870
|
+
const refreshedUsage = refreshedUsageProbe.usage
|
|
871
|
+
const refreshedUsageTs = refreshedUsage?.usageTimestampMs
|
|
872
|
+
const previousUsageTs = usage?.usageTimestampMs
|
|
873
|
+
|
|
874
|
+
if (Number.isFinite(refreshedUsageTs) && (!Number.isFinite(previousUsageTs) || refreshedUsageTs > previousUsageTs)) {
|
|
875
|
+
usageProbe = refreshedUsageProbe
|
|
876
|
+
usage = refreshedUsage
|
|
877
|
+
usageFreshness = deriveUsageFreshness(usage, Date.now(), USAGE_STALE_MS, USAGE_NEAR_STALE_MS, USAGE_STALE_GRACE_MS)
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
if (!usageFreshness.isPastStaleThreshold) break
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
usageRefreshDurationMs = usageRefreshStartMs !== null ? Date.now() - usageRefreshStartMs : null
|
|
884
|
+
usageRefreshRecovered = usageFreshness.isPastStaleThreshold === false
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
const sampleAtMs = Date.now()
|
|
888
|
+
if (usage) {
|
|
889
|
+
usageFreshness = deriveUsageFreshness(usage, sampleAtMs, USAGE_STALE_MS, USAGE_NEAR_STALE_MS, USAGE_STALE_GRACE_MS)
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
const gpu = MONITOR_GPU
|
|
893
|
+
? (process.platform === 'darwin'
|
|
894
|
+
? gpuSampleDarwin()
|
|
895
|
+
: gpuSampleNvidiaSmi())
|
|
896
|
+
: { pct: null, source: 'disabled', confidence: 'none', sampleWindowMs: null }
|
|
897
|
+
|
|
898
|
+
const memPressure = MONITOR_MEMORY
|
|
899
|
+
? loadMemPressure()
|
|
900
|
+
: { pct: null, cls: 'disabled', source: 'disabled' }
|
|
901
|
+
|
|
902
|
+
const usedMemPct = MONITOR_MEMORY ? memUsedPct() : null
|
|
903
|
+
|
|
904
|
+
const usageIntegrationStatus = usage
|
|
905
|
+
? usageFreshness.isStale
|
|
906
|
+
? 'stale'
|
|
907
|
+
: usage?.integrationStatus === 'partial'
|
|
908
|
+
? 'ok'
|
|
909
|
+
: (usage?.integrationStatus ?? 'ok')
|
|
910
|
+
: (openclawEnabled ? 'unavailable' : 'disabled')
|
|
911
|
+
|
|
912
|
+
const source = {
|
|
913
|
+
monitorTargets: [...MONITOR_TARGETS],
|
|
914
|
+
usage: usage ? 'openclaw' : openclawEnabled ? 'unavailable' : 'disabled',
|
|
915
|
+
usageIntegrationStatus,
|
|
916
|
+
usageIngestionStatus: openclawEnabled
|
|
917
|
+
? usage && ['ok', 'fallback-cache'].includes(usageProbe.probe.result)
|
|
918
|
+
? 'ok'
|
|
919
|
+
: 'unavailable'
|
|
920
|
+
: 'disabled',
|
|
921
|
+
usageActivityStatus: usage
|
|
922
|
+
? usageFreshness.freshnessState
|
|
923
|
+
: (openclawEnabled ? 'unavailable' : 'disabled'),
|
|
924
|
+
usageProbeResult: usageProbe.probe.result,
|
|
925
|
+
usageProbeAttempts: usageProbe.probe.attempts,
|
|
926
|
+
usageProbeSweeps: usageProbe.probe.sweeps,
|
|
927
|
+
usageProbeTimeoutMs: OPENCLAW_PROBE_TIMEOUT_MS,
|
|
928
|
+
usageProbeRetries: OPENCLAW_PROBE_RETRIES,
|
|
929
|
+
usageProbeError: usageProbe.probe.error,
|
|
930
|
+
usageProbeDurationMs: usageProbe.probe.durationMs,
|
|
931
|
+
usageUsedFallbackCache: usageProbe.probe.usedFallbackCache,
|
|
932
|
+
usageFallbackCacheAgeMs: usageProbe.probe.fallbackAgeMs,
|
|
933
|
+
usageFallbackCacheSource: usageProbe.probe.fallbackCacheSource,
|
|
934
|
+
usageFreshnessState: openclawEnabled
|
|
935
|
+
? usage
|
|
936
|
+
? usageFreshness.freshnessState
|
|
937
|
+
: null
|
|
938
|
+
: 'disabled',
|
|
939
|
+
usageNearStale: usage ? usageFreshness.isNearStale : false,
|
|
940
|
+
usagePastStaleThreshold: usage ? usageFreshness.isPastStaleThreshold : false,
|
|
941
|
+
usageRefreshAttempted,
|
|
942
|
+
usageRefreshRecovered,
|
|
943
|
+
usageRefreshAttempts,
|
|
944
|
+
usageRefreshReprobes: USAGE_REFRESH_REPROBES,
|
|
945
|
+
usageRefreshDelayMs: USAGE_REFRESH_DELAY_MS,
|
|
946
|
+
usageRefreshDurationMs,
|
|
947
|
+
usageRefreshOnNearStale: USAGE_REFRESH_ON_NEAR_STALE === 1,
|
|
948
|
+
usageIdleAfterMsThreshold: USAGE_IDLE_AFTER_MS,
|
|
949
|
+
usageIdle: usage ? (Number.isFinite(usageFreshness.usageAgeMs) && usageFreshness.usageAgeMs >= USAGE_IDLE_AFTER_MS) : false,
|
|
950
|
+
usageCommand: usage?.sourceCommand ?? null,
|
|
951
|
+
usageStaleMsThreshold: USAGE_STALE_MS,
|
|
952
|
+
usageNearStaleMsThreshold: USAGE_NEAR_STALE_MS,
|
|
953
|
+
usageStaleGraceMs: USAGE_STALE_GRACE_MS,
|
|
954
|
+
memPressureSource: memPressure.source
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
const usageAlert = deriveUsageAlert(source, { usageAgeMs: usageFreshness.usageAgeMs, idleAfterMs: USAGE_IDLE_AFTER_MS })
|
|
958
|
+
source.usageAlertLevel = usageAlert.level
|
|
959
|
+
source.usageAlertReason = usageAlert.reason
|
|
960
|
+
|
|
961
|
+
const row = {
|
|
962
|
+
host: HOST,
|
|
963
|
+
ts: sampleAtMs,
|
|
964
|
+
cpuPct: MONITOR_CPU ? cpuPct() : null,
|
|
965
|
+
memPct: MONITOR_MEMORY ? usedMemPct : null,
|
|
966
|
+
memUsedPct: MONITOR_MEMORY ? usedMemPct : null,
|
|
967
|
+
memPressurePct: MONITOR_MEMORY ? memPressure.pct : null,
|
|
968
|
+
memPressureClass: MONITOR_MEMORY ? memPressure.cls : 'disabled',
|
|
969
|
+
gpuPct: MONITOR_GPU ? gpu.pct : null,
|
|
970
|
+
gpuSource: gpu.source,
|
|
971
|
+
gpuConfidence: gpu.confidence,
|
|
972
|
+
gpuSampleWindowMs: gpu.sampleWindowMs,
|
|
973
|
+
tokensPerMin: MONITOR_OPENCLAW ? (usage?.tokensPerMin ?? null) : null,
|
|
974
|
+
openclawModel: MONITOR_OPENCLAW ? (usage?.model ?? null) : null,
|
|
975
|
+
openclawTotalTokens: MONITOR_OPENCLAW ? (usage?.totalTokens ?? null) : null,
|
|
976
|
+
openclawSessionId: MONITOR_OPENCLAW ? (usage?.sessionId ?? null) : null,
|
|
977
|
+
openclawAgentId: MONITOR_OPENCLAW ? (usage?.agentId ?? null) : null,
|
|
978
|
+
openclawUsageTs: MONITOR_OPENCLAW ? (usage?.usageTimestampMs ?? null) : null,
|
|
979
|
+
openclawUsageAgeMs: MONITOR_OPENCLAW ? usageFreshness.usageAgeMs : null,
|
|
980
|
+
source
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
return enrichWithOpenClawFleetTelemetry(row, {
|
|
984
|
+
host: HOST,
|
|
985
|
+
collectedAtMs: sampleAtMs,
|
|
986
|
+
collector: 'idlewatch-agent',
|
|
987
|
+
collectorVersion: pkg.version
|
|
988
|
+
})
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
async function tick() {
|
|
992
|
+
const row = await collectSample()
|
|
993
|
+
console.log(JSON.stringify(row))
|
|
994
|
+
appendLocal(row)
|
|
995
|
+
const published = await publish(row)
|
|
996
|
+
if (REQUIRE_FIREBASE_WRITES && ONCE && !published) {
|
|
997
|
+
throw new Error('Firebase write was required but not executed. Check Firebase configuration and connectivity.')
|
|
998
|
+
}
|
|
999
|
+
if (REQUIRE_CLOUD_WRITES && ONCE && !published) {
|
|
1000
|
+
throw new Error('Cloud write was required but not executed. Check API key and cloud connectivity.')
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
let running = false
|
|
1005
|
+
let stopped = false
|
|
1006
|
+
let inflightTick = null
|
|
1007
|
+
|
|
1008
|
+
async function loop() {
|
|
1009
|
+
if (stopped || running) return
|
|
1010
|
+
running = true
|
|
1011
|
+
try {
|
|
1012
|
+
inflightTick = tick()
|
|
1013
|
+
await inflightTick
|
|
1014
|
+
} catch (e) {
|
|
1015
|
+
console.error(e.message)
|
|
1016
|
+
} finally {
|
|
1017
|
+
inflightTick = null
|
|
1018
|
+
running = false
|
|
1019
|
+
if (!stopped) setTimeout(loop, INTERVAL_MS)
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
async function gracefulShutdown(signal) {
|
|
1024
|
+
if (stopped) return
|
|
1025
|
+
stopped = true
|
|
1026
|
+
if (inflightTick) {
|
|
1027
|
+
console.log(`idlewatch-agent received ${signal}, waiting for in-flight sample…`)
|
|
1028
|
+
try { await inflightTick } catch { /* already logged */ }
|
|
1029
|
+
}
|
|
1030
|
+
console.log('idlewatch-agent stopped')
|
|
1031
|
+
process.exit(0)
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
process.on('SIGINT', () => gracefulShutdown('SIGINT'))
|
|
1035
|
+
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
|
|
1036
|
+
|
|
1037
|
+
if (DRY_RUN || ONCE) {
|
|
1038
|
+
const mode = DRY_RUN ? 'dry-run' : 'once'
|
|
1039
|
+
console.log(
|
|
1040
|
+
`idlewatch-agent ${mode} host=${HOST} intervalMs=${INTERVAL_MS} firebase=${appReady} localLog=${LOCAL_LOG_PATH}`
|
|
1041
|
+
)
|
|
1042
|
+
tick()
|
|
1043
|
+
.then(() => process.exit(0))
|
|
1044
|
+
.catch((e) => {
|
|
1045
|
+
console.error(e.message)
|
|
1046
|
+
process.exit(1)
|
|
1047
|
+
})
|
|
1048
|
+
} else {
|
|
1049
|
+
console.log(
|
|
1050
|
+
`idlewatch-agent started host=${HOST} intervalMs=${INTERVAL_MS} firebase=${appReady} localLog=${LOCAL_LOG_PATH} monitorTargets=${[...MONITOR_TARGETS].join(',')} openclawUsage=${EFFECTIVE_OPENCLAW_MODE}`
|
|
1051
|
+
)
|
|
1052
|
+
loop()
|
|
1053
|
+
}
|