@calltelemetry/cli 0.11.1 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cube.d.ts +8 -0
- package/dist/commands/cube.d.ts.map +1 -0
- package/dist/commands/cube.js +47 -0
- package/dist/commands/cube.js.map +1 -0
- package/dist/commands/proxy.d.ts +3 -0
- package/dist/commands/proxy.d.ts.map +1 -0
- package/dist/commands/proxy.js +94 -0
- package/dist/commands/proxy.js.map +1 -0
- package/dist/commands/update.d.ts.map +1 -1
- package/dist/commands/update.js +28 -2
- package/dist/commands/update.js.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/cube-diag.d.ts +26 -0
- package/dist/lib/cube-diag.d.ts.map +1 -0
- package/dist/lib/cube-diag.js +336 -0
- package/dist/lib/cube-diag.js.map +1 -0
- package/dist/lib/migration-016-seaweedfs-recovery.d.ts +25 -0
- package/dist/lib/migration-016-seaweedfs-recovery.d.ts.map +1 -0
- package/dist/lib/migration-016-seaweedfs-recovery.js +125 -0
- package/dist/lib/migration-016-seaweedfs-recovery.js.map +1 -0
- package/dist/lib/migrations.d.ts.map +1 -1
- package/dist/lib/migrations.js +6 -0
- package/dist/lib/migrations.js.map +1 -1
- package/dist/lib/post-upgrade-health.d.ts +34 -0
- package/dist/lib/post-upgrade-health.d.ts.map +1 -0
- package/dist/lib/post-upgrade-health.js +208 -0
- package/dist/lib/post-upgrade-health.js.map +1 -0
- package/dist/lib/proxy.d.ts +19 -0
- package/dist/lib/proxy.d.ts.map +1 -0
- package/dist/lib/proxy.js +104 -0
- package/dist/lib/proxy.js.map +1 -0
- package/dist/lib/rpc.d.ts +10 -1
- package/dist/lib/rpc.d.ts.map +1 -1
- package/dist/lib/rpc.js +14 -2
- package/dist/lib/rpc.js.map +1 -1
- package/dist/lib/update-steps.d.ts.map +1 -1
- package/dist/lib/update-steps.js +15 -0
- package/dist/lib/update-steps.js.map +1 -1
- package/dist/lib/version.d.ts +1 -1
- package/dist/lib/version.js +1 -1
- package/dist/shell/commands/cube.d.ts +8 -0
- package/dist/shell/commands/cube.d.ts.map +1 -0
- package/dist/shell/commands/cube.js +47 -0
- package/dist/shell/commands/cube.js.map +1 -0
- package/dist/shell/commands/registry.d.ts.map +1 -1
- package/dist/shell/commands/registry.js +20 -0
- package/dist/shell/commands/registry.js.map +1 -1
- package/dist/shell/commands/system.d.ts.map +1 -1
- package/dist/shell/commands/system.js +14 -0
- package/dist/shell/commands/system.js.map +1 -1
- package/dist/utils/parseSecs.d.ts +17 -0
- package/dist/utils/parseSecs.d.ts.map +1 -0
- package/dist/utils/parseSecs.js +33 -0
- package/dist/utils/parseSecs.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only collector for the "CUBE stuck in REGISTERED, never ONLINE" bug class.
|
|
3
|
+
*
|
|
4
|
+
* Builds one text bundle from three sources — Postgres, NATS KV/streams, and
|
|
5
|
+
* container logs — redacts it, and writes it to a single .log file. Each section
|
|
6
|
+
* is best-effort: a failure in one section is captured inline and never aborts
|
|
7
|
+
* the rest of the collection. Pure async — no React/Ink imports.
|
|
8
|
+
*
|
|
9
|
+
* The registered -> online transition requires a healthy SolicitXccProbing
|
|
10
|
+
* keepalive; this bundle surfaces the DB state, NATS KV view, and the log lines
|
|
11
|
+
* that explain why that keepalive never lands.
|
|
12
|
+
*/
|
|
13
|
+
import { writeFile, chmod } from 'node:fs/promises';
|
|
14
|
+
import { compose, composeExec, composePs } from './compose.js';
|
|
15
|
+
import { exec } from './exec.js';
|
|
16
|
+
import { rpc } from './rpc.js';
|
|
17
|
+
import { redactSecrets } from './redact.js';
|
|
18
|
+
import { DB_USER, DB_NAME, PG_ENV } from './db.js';
|
|
19
|
+
import { getPaths } from './paths.js';
|
|
20
|
+
import { debug } from './log.js';
|
|
21
|
+
const NATS_BOX_IMAGE = 'natsio/nats-box:0.14.5';
|
|
22
|
+
const NATS_URL = 'nats://nats:4222';
|
|
23
|
+
const DEFAULT_TIMEOUT = 10_000;
|
|
24
|
+
const RPC_TIMEOUT = 25_000;
|
|
25
|
+
/** Sleep helper for the heartbeat-trace loop. */
|
|
26
|
+
function sleep(ms) {
|
|
27
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Run a cube-diag RPC bounded by RPC_TIMEOUT. The budget is passed INTO `rpc()`
|
|
31
|
+
* (→ composeExec → execa `timeout`), so a wedged `docker compose exec` child is
|
|
32
|
+
* actually KILLED rather than abandoned by a Promise.race that leaves it alive.
|
|
33
|
+
*/
|
|
34
|
+
function rpcBounded(code) {
|
|
35
|
+
return rpc(code, RPC_TIMEOUT);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Elixir snapshot run via `onprem rpc` inside the web container. For every
|
|
39
|
+
* xcc-enabled cube it decodes the live cube_status KV and runs the SAME read
|
|
40
|
+
* path the product uses (ConnectionHealthState.derive_from_kv/3), then flags the
|
|
41
|
+
* probe-interval pathologies engineering looks for:
|
|
42
|
+
* CORRUPT_KV — observed_interval_ms set but last_event_at nil (won't relearn)
|
|
43
|
+
* MANUAL!=OBSERVED — configured interval differs from the learned one
|
|
44
|
+
* DB_OBS!=KV_OBS — the DB mirror and the live KV disagree on the cadence
|
|
45
|
+
*
|
|
46
|
+
* NOTE: this is a backtick template literal but contains NO `${...}` JS
|
|
47
|
+
* interpolation — only Elixir `#{...}`, which JS leaves untouched. Do not add a
|
|
48
|
+
* `${` anywhere in this string.
|
|
49
|
+
*/
|
|
50
|
+
const CUBE_DIAG_RPC_SNAPSHOT = `import Ecto.Query
|
|
51
|
+
now = DateTime.utc_now()
|
|
52
|
+
alias Cdrcisco.Cubes.{Cube, ConnectionStateKV, ConnectionHealthState}
|
|
53
|
+
cubes = Cdrcisco.Repo.all(from c in Cube, where: c.xcc_enabled == true, order_by: c.id)
|
|
54
|
+
if cubes == [], do: IO.puts("(no xcc-enabled cubes)")
|
|
55
|
+
Enum.each(cubes, fn c ->
|
|
56
|
+
out =
|
|
57
|
+
try do
|
|
58
|
+
raw = Cdrcisco.Liveness.KV.get("cube_status", "cube:#{c.id}:status")
|
|
59
|
+
dec =
|
|
60
|
+
case raw && ConnectionStateKV.decode(raw) do
|
|
61
|
+
{:ok, d} -> d
|
|
62
|
+
_ -> nil
|
|
63
|
+
end
|
|
64
|
+
row = ConnectionHealthState.derive_from_kv(c, dec, now)
|
|
65
|
+
kv_ms = dec && dec.observed_interval_ms
|
|
66
|
+
kv_le = dec && dec.last_event_at
|
|
67
|
+
kv_obs_s = (is_integer(kv_ms) and kv_ms > 0) && div(kv_ms, 1000) || nil
|
|
68
|
+
manual = c.xcc_provider_probing_interval
|
|
69
|
+
db_obs = c.xcc_observed_probing_interval
|
|
70
|
+
corrupt = is_integer(kv_ms) and kv_ms > 0 and is_nil(kv_le)
|
|
71
|
+
mism = is_integer(manual) and is_integer(db_obs) and manual != db_obs
|
|
72
|
+
div_obs = is_integer(db_obs) and is_integer(kv_obs_s) and db_obs != kv_obs_s
|
|
73
|
+
flags = []
|
|
74
|
+
flags = if corrupt, do: ["CORRUPT_KV(observed_ms set, last_event_at nil -> wont relearn)" | flags], else: flags
|
|
75
|
+
flags = if mism, do: ["MANUAL!=OBSERVED(#{manual} vs #{db_obs})" | flags], else: flags
|
|
76
|
+
flags = if div_obs, do: ["DB_OBS!=KV_OBS(#{db_obs} vs #{kv_obs_s})" | flags], else: flags
|
|
77
|
+
Enum.join([
|
|
78
|
+
"cube #{c.id} (#{c.name})",
|
|
79
|
+
" db : manual=#{inspect(manual)} observed=#{inspect(db_obs)} observed_at=#{inspect(c.xcc_observed_probing_interval_at)} online=#{inspect(c.xcc_online)} last_keepalive=#{inspect(c.xcc_last_keepalive)}",
|
|
80
|
+
" kv : state=#{inspect(dec && dec.state)} last_event=#{inspect(dec && dec.last_event)} last_event_at=#{inspect(kv_le)} attempt=#{inspect(dec && dec.attempt_count)} observed_ms=#{inspect(kv_ms)} probing_ms=#{inspect(dec && dec.probing_interval_ms)}",
|
|
81
|
+
" derived: state=#{inspect(row.state)} conf=#{inspect(row.status_confidence)} observed_sec=#{inspect(row.observed_probing_interval_sec)} window_sec=#{inspect(row.freshness_window_sec)} stale_after=#{inspect(row.stale_after_at)}",
|
|
82
|
+
" flags : #{if flags == [], do: "none", else: Enum.join(flags, " ")}"
|
|
83
|
+
], "\\n")
|
|
84
|
+
rescue
|
|
85
|
+
e -> "cube #{c.id}: ERROR #{Exception.message(e)}"
|
|
86
|
+
end
|
|
87
|
+
IO.puts(out)
|
|
88
|
+
end)
|
|
89
|
+
:ok`;
|
|
90
|
+
/**
|
|
91
|
+
* Compact one-line-per-cube trace sample, called repeatedly by the
|
|
92
|
+
* `--monitor <secs>` heartbeat trace to capture how the FSM state / learned
|
|
93
|
+
* cadence / freshness window evolve over time.
|
|
94
|
+
*
|
|
95
|
+
* NOTE: same `${...}`-free rule as CUBE_DIAG_RPC_SNAPSHOT above.
|
|
96
|
+
*/
|
|
97
|
+
const CUBE_DIAG_RPC_TRACE = `import Ecto.Query
|
|
98
|
+
now = DateTime.utc_now()
|
|
99
|
+
alias Cdrcisco.Cubes.{Cube, ConnectionStateKV, ConnectionHealthState}
|
|
100
|
+
ts = DateTime.to_iso8601(now)
|
|
101
|
+
for c <- Cdrcisco.Repo.all(from c in Cube, where: c.xcc_enabled == true, order_by: c.id) do
|
|
102
|
+
try do
|
|
103
|
+
raw = Cdrcisco.Liveness.KV.get("cube_status", "cube:#{c.id}:status")
|
|
104
|
+
dec =
|
|
105
|
+
case raw && ConnectionStateKV.decode(raw) do
|
|
106
|
+
{:ok, d} -> d
|
|
107
|
+
_ -> nil
|
|
108
|
+
end
|
|
109
|
+
row = ConnectionHealthState.derive_from_kv(c, dec, now)
|
|
110
|
+
IO.puts("#{ts}|cube=#{c.id}|kv_state=#{inspect(dec && dec.state)}|kv_obs_ms=#{inspect(dec && dec.observed_interval_ms)}|kv_last_event_at=#{inspect(dec && dec.last_event_at)}|attempt=#{inspect(dec && dec.attempt_count)}|obs_sec=#{inspect(row.observed_probing_interval_sec)}|win_sec=#{inspect(row.freshness_window_sec)}|state=#{row.state}|conf=#{row.status_confidence}")
|
|
111
|
+
rescue
|
|
112
|
+
e -> IO.puts("#{ts}|cube=#{c.id}|ERROR #{Exception.message(e)}")
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
:ok`;
|
|
116
|
+
/** Run a psql query in the db container, returning trimmed stdout. */
|
|
117
|
+
async function psql(sql) {
|
|
118
|
+
const result = await composeExec('db', ['psql', '-U', DB_USER, '-d', DB_NAME, '-c', sql], { env: PG_ENV, pipe: true, timeout: DEFAULT_TIMEOUT });
|
|
119
|
+
return (result.stdout ?? '').toString();
|
|
120
|
+
}
|
|
121
|
+
/** Run one collection step, capturing its output (or error) into a labeled block. */
|
|
122
|
+
async function section(title, fn) {
|
|
123
|
+
const lines = [`\n===== ${title} =====`];
|
|
124
|
+
try {
|
|
125
|
+
const body = await fn();
|
|
126
|
+
lines.push(body.trim().length > 0 ? body.trimEnd() : '(no output)');
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
130
|
+
lines.push(`[section failed] ${message}`);
|
|
131
|
+
}
|
|
132
|
+
return lines.join('\n');
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Resolve the `_ct` docker network the appliance services share, so the
|
|
136
|
+
* throwaway nats-box container can reach the `nats` service by name.
|
|
137
|
+
*
|
|
138
|
+
* Strategy: ask compose for the nats service's network (first line, first
|
|
139
|
+
* field). Fall back to `<project>_ct` derived from the install dir basename.
|
|
140
|
+
*/
|
|
141
|
+
async function resolveCtNetwork() {
|
|
142
|
+
try {
|
|
143
|
+
const result = await compose(['ps', '--format', '{{.Networks}}', 'nats'], { pipe: true });
|
|
144
|
+
const firstLine = (result.stdout ?? '').toString().split('\n').map(l => l.trim()).filter(Boolean)[0];
|
|
145
|
+
if (firstLine) {
|
|
146
|
+
const firstField = firstLine.split(',')[0]?.trim();
|
|
147
|
+
if (firstField)
|
|
148
|
+
return firstField;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
catch (err) {
|
|
152
|
+
debug(`cube-diag: compose network resolve failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
153
|
+
}
|
|
154
|
+
const { installDir } = getPaths();
|
|
155
|
+
const project = (installDir.split('/').filter(Boolean).pop() ?? 'docker-compose').toLowerCase();
|
|
156
|
+
return `${project}_ct`;
|
|
157
|
+
}
|
|
158
|
+
/** Run `nats` CLI inside a throwaway nats-box container on the appliance network. */
|
|
159
|
+
async function natsRun(net, natsArgs) {
|
|
160
|
+
const result = await exec('docker', ['run', '--rm', '--network', net, NATS_BOX_IMAGE, 'nats', '-s', NATS_URL, ...natsArgs], { timeout: DEFAULT_TIMEOUT, reject: false });
|
|
161
|
+
const stdout = (result.stdout ?? '').toString();
|
|
162
|
+
const stderr = (result.stderr ?? '').toString();
|
|
163
|
+
return [stdout, stderr.trim() ? `[stderr] ${stderr}` : ''].filter(Boolean).join('\n');
|
|
164
|
+
}
|
|
165
|
+
// STATIC sh script — NO host-side interpolation. The NATS URL and bucket are
|
|
166
|
+
// passed as `docker run -e` env vars (NATS_URL / KV_BUCKET) and referenced via
|
|
167
|
+
// `$NATS_URL` / `"$KV_BUCKET"` INSIDE the container, so a non-constant bucket can
|
|
168
|
+
// never break out of the shell string (repo rule: no shell string interpolation).
|
|
169
|
+
// `set -f` disables globbing and `while IFS= read` handles keys with whitespace /
|
|
170
|
+
// glob chars, so a stray `*` in CLI output can never glob-expand against the CWD.
|
|
171
|
+
const NATS_KV_DUMP_SCRIPT = 'set -f; nats -s "$NATS_URL" kv ls "$KV_BUCKET" 2>/dev/null | while IFS= read -r k; do ' +
|
|
172
|
+
'[ -n "$k" ] || continue; ' +
|
|
173
|
+
'echo "--- key: $k ---"; ' +
|
|
174
|
+
'nats -s "$NATS_URL" kv get "$KV_BUCKET" "$k" 2>&1; ' +
|
|
175
|
+
'echo "--- key (raw base64): $k ---"; ' +
|
|
176
|
+
'nats -s "$NATS_URL" kv get "$KV_BUCKET" "$k" --raw 2>/dev/null | base64; ' +
|
|
177
|
+
'done';
|
|
178
|
+
/** Dump every key in a NATS KV bucket (human get + raw base64), via an in-container sh loop. */
|
|
179
|
+
async function natsKvDump(net, bucket) {
|
|
180
|
+
const result = await exec('docker', [
|
|
181
|
+
'run', '--rm', '--network', net,
|
|
182
|
+
'-e', `NATS_URL=${NATS_URL}`,
|
|
183
|
+
'-e', `KV_BUCKET=${bucket}`,
|
|
184
|
+
NATS_BOX_IMAGE, 'sh', '-c', NATS_KV_DUMP_SCRIPT,
|
|
185
|
+
], { timeout: DEFAULT_TIMEOUT, reject: false });
|
|
186
|
+
const stdout = (result.stdout ?? '').toString();
|
|
187
|
+
const stderr = (result.stderr ?? '').toString();
|
|
188
|
+
// Surface truncation: a silently-cut KV dump would mislead diagnosis.
|
|
189
|
+
const timedOut = result.timedOut === true;
|
|
190
|
+
return [
|
|
191
|
+
stdout,
|
|
192
|
+
timedOut ? '[WARN: nats-box exec timed out — KV dump may be truncated]' : '',
|
|
193
|
+
stderr.trim() ? `[stderr] ${stderr}` : '',
|
|
194
|
+
].filter(Boolean).join('\n');
|
|
195
|
+
}
|
|
196
|
+
/** Get the first container id for a compose service (empty string if none). */
|
|
197
|
+
async function containerId(service) {
|
|
198
|
+
// composePs already has its own internal fallback; a second compose ps here
|
|
199
|
+
// would just repeat the same command, so on failure we return empty directly.
|
|
200
|
+
try {
|
|
201
|
+
const id = await composePs(service);
|
|
202
|
+
if (id)
|
|
203
|
+
return id.split('\n')[0].trim();
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
debug(`cube-diag: composePs ${service} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
207
|
+
}
|
|
208
|
+
return '';
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Tail container logs and keep only the lines matching the cube/xcc keepalive
|
|
212
|
+
* grep. When `opts.since` is given, uses `docker logs --since <window>` instead
|
|
213
|
+
* of `--tail`, mirroring the bash reference's 10-minute heartbeat window.
|
|
214
|
+
*/
|
|
215
|
+
async function logsFiltered(service, tailIn, keepTail, grep, opts) {
|
|
216
|
+
const id = await containerId(service);
|
|
217
|
+
if (!id)
|
|
218
|
+
return `(no ${service} container found)`;
|
|
219
|
+
const dockerArgs = opts?.since
|
|
220
|
+
? ['logs', '--since', opts.since, id]
|
|
221
|
+
: ['logs', '--tail', String(tailIn), id];
|
|
222
|
+
const result = await exec('docker', dockerArgs, {
|
|
223
|
+
timeout: DEFAULT_TIMEOUT,
|
|
224
|
+
reject: false,
|
|
225
|
+
});
|
|
226
|
+
const combined = `${(result.stdout ?? '').toString()}\n${(result.stderr ?? '').toString()}`;
|
|
227
|
+
let lines = combined.split('\n');
|
|
228
|
+
if (grep)
|
|
229
|
+
lines = lines.filter(l => grep.test(l));
|
|
230
|
+
return lines.slice(-keepTail).join('\n');
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Collect the full cube-diag bundle and write it to a single .log file.
|
|
234
|
+
*
|
|
235
|
+
* @param opts.outputPath Target log path. Defaults to ./cube-diag-<ISO>.log.
|
|
236
|
+
* @param opts.captureSecs Seconds to live-capture cube.> / alarms.> subjects (0 = skip).
|
|
237
|
+
* @param opts.monitorSecs Seconds to RPC-poll each cube's state/interval ~every 5s (0 = skip).
|
|
238
|
+
* @returns the path the bundle was written to.
|
|
239
|
+
*/
|
|
240
|
+
export async function collectCubeDiag(opts) {
|
|
241
|
+
const captureSecs = opts?.captureSecs ?? 0;
|
|
242
|
+
const monitorSecs = opts?.monitorSecs ?? 0;
|
|
243
|
+
const iso = new Date().toISOString().replace(/[:.]/g, '-');
|
|
244
|
+
const outputPath = opts?.outputPath ?? `./cube-diag-${iso}.log`;
|
|
245
|
+
const parts = [];
|
|
246
|
+
parts.push('===== CUBE DIAGNOSTIC BUNDLE =====', `Generated: ${new Date().toISOString()}`, 'Bug class: cube stuck REGISTERED, never ONLINE (registered->online requires a healthy SolicitXccProbing keepalive).');
|
|
247
|
+
// ── Database sections ──────────────────────────────────────────────────
|
|
248
|
+
parts.push(await section('DB: cubes', () => psql(`SELECT id, org_id, name, xcc_enabled, xcc_emulator, xcc_mode, xcc_hostname, ` +
|
|
249
|
+
`xcc_provider_probing_interval, xcc_observed_probing_interval, xcc_registration_id, ` +
|
|
250
|
+
`xcc_disconnect_code, xcc_online, xcc_last_keepalive, connection_expired_count, ` +
|
|
251
|
+
`reconnection_failed_count, unregistered_count, policy_id FROM cubes ORDER BY id;`)));
|
|
252
|
+
parts.push(await section('DB: cube_connection_health_state (authoritative)', () => psql(`SELECT cube_id, state, status_confidence, status_reason, last_keepalive_at, last_probe_at, ` +
|
|
253
|
+
`last_authorize_at, last_provider_status, observed_probing_interval_sec, freshness_window_sec, ` +
|
|
254
|
+
`stale_after_at, last_state_change_at, last_kv_sync_at FROM cube_connection_health_state ORDER BY cube_id;`)));
|
|
255
|
+
parts.push(await section('DB: cube_connection_health_events (recent 50)', () => psql(`SELECT cube_id, event_type, severity, source, transition_from, transition_to, inserted_at ` +
|
|
256
|
+
`FROM cube_connection_health_events ORDER BY inserted_at DESC LIMIT 50;`)));
|
|
257
|
+
parts.push(await section('DB: alarm_events (active cube/xcc/policy)', () => psql(`SELECT id, org_id, alarm_type, status, inserted_at, updated_at FROM alarm_events ` +
|
|
258
|
+
`WHERE status IN ('active','acknowledged') ` +
|
|
259
|
+
`AND (alarm_type ILIKE '%cube%' OR alarm_type ILIKE '%xcc%' OR alarm_type ILIKE '%policy%') ` +
|
|
260
|
+
`ORDER BY inserted_at DESC;`)));
|
|
261
|
+
parts.push(await section('DB: cube_event_logs (24h volume per cube)', () => psql(`SELECT cube_id, COUNT(*) AS events_24h FROM cube_event_logs ` +
|
|
262
|
+
`WHERE inserted_at > NOW() - INTERVAL '24 hours' GROUP BY cube_id ORDER BY cube_id;`)));
|
|
263
|
+
// ── NATS sections ──────────────────────────────────────────────────────
|
|
264
|
+
const net = await resolveCtNetwork();
|
|
265
|
+
parts.push(`\n===== NATS: resolved network = ${net} =====`);
|
|
266
|
+
parts.push(await section('NATS: kv ls', () => natsRun(net, ['kv', 'ls'])));
|
|
267
|
+
parts.push(await section('NATS: kv info cube_status', () => natsRun(net, ['kv', 'info', 'cube_status'])));
|
|
268
|
+
parts.push(await section('NATS: kv info policy_status', () => natsRun(net, ['kv', 'info', 'policy_status'])));
|
|
269
|
+
parts.push(await section('NATS: cube_status bucket dump', () => natsKvDump(net, 'cube_status')));
|
|
270
|
+
parts.push(await section('NATS: policy_status bucket dump', () => natsKvDump(net, 'policy_status')));
|
|
271
|
+
parts.push(await section('NATS: stream report', () => natsRun(net, ['stream', 'report'])));
|
|
272
|
+
// ── RPC-decoded interval + KV health snapshot (always runs) ─────────────
|
|
273
|
+
// Decodes each cube's live KV through the product's own read path and flags
|
|
274
|
+
// the interval pathologies (corrupt KV that won't relearn, manual-vs-observed
|
|
275
|
+
// mismatch, DB-vs-KV divergence). This is where a "shows 10s, router is 15s"
|
|
276
|
+
// stuck cube is caught.
|
|
277
|
+
parts.push(await section('cube interval + KV health (RPC-decoded)', () => rpcBounded(CUBE_DIAG_RPC_SNAPSHOT)));
|
|
278
|
+
// ── Optional RPC heartbeat trace ────────────────────────────────────────
|
|
279
|
+
// Polls the compact per-cube snapshot every ~5s for the window so engineering
|
|
280
|
+
// can watch the FSM state / learned cadence / freshness window evolve.
|
|
281
|
+
if (monitorSecs > 0) {
|
|
282
|
+
parts.push(await section(`RPC heartbeat trace (${monitorSecs}s, ~5s samples)`, async () => {
|
|
283
|
+
const samples = [];
|
|
284
|
+
const end = Date.now() + monitorSecs * 1000;
|
|
285
|
+
while (Date.now() < end) {
|
|
286
|
+
try {
|
|
287
|
+
const out = await rpcBounded(CUBE_DIAG_RPC_TRACE);
|
|
288
|
+
const traceLines = out
|
|
289
|
+
.split('\n')
|
|
290
|
+
.filter(l => /^[0-9]{4}-/.test(l));
|
|
291
|
+
samples.push(traceLines.length > 0 ? traceLines.join('\n') : '[trace sample failed]');
|
|
292
|
+
}
|
|
293
|
+
catch (err) {
|
|
294
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
295
|
+
samples.push(`[trace sample failed] ${message}`);
|
|
296
|
+
}
|
|
297
|
+
// Clamp the inter-sample sleep to the remaining window so the loop never
|
|
298
|
+
// overshoots the requested duration (e.g. `--monitor 6` must not run ~10s).
|
|
299
|
+
const remaining = end - Date.now();
|
|
300
|
+
if (remaining > 0)
|
|
301
|
+
await sleep(Math.min(5000, remaining));
|
|
302
|
+
}
|
|
303
|
+
return samples.join('\n');
|
|
304
|
+
}));
|
|
305
|
+
}
|
|
306
|
+
// ── Optional live capture ──────────────────────────────────────────────
|
|
307
|
+
if (captureSecs > 0) {
|
|
308
|
+
for (const subject of ['cube.>', 'alarms.>']) {
|
|
309
|
+
parts.push(await section(`NATS: live capture ${subject} (${captureSecs}s)`, async () => {
|
|
310
|
+
const result = await exec('timeout', [
|
|
311
|
+
String(captureSecs),
|
|
312
|
+
'docker', 'run', '--rm', '--network', net, NATS_BOX_IMAGE,
|
|
313
|
+
'nats', '-s', NATS_URL, 'sub', '--raw', subject,
|
|
314
|
+
], { timeout: (captureSecs + 5) * 1000, reject: false });
|
|
315
|
+
return [(result.stdout ?? '').toString(), (result.stderr ?? '').toString()].filter(Boolean).join('\n');
|
|
316
|
+
}));
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
// ── Log sections ───────────────────────────────────────────────────────
|
|
320
|
+
// Heartbeat timeline first: the failure_count progression, "did not refresh
|
|
321
|
+
// healthy" gate hits, and registration rotations are the stuck/flap signature.
|
|
322
|
+
const heartbeatGrep = /failure_count|did not refresh healthy|registrationID rotated|keepalive/i;
|
|
323
|
+
parts.push(await section('probe heartbeat timeline (failure_count / refresh / rotation, last 10m)', () => logsFiltered('web', 6000, 200, heartbeatGrep, { since: '10m' })));
|
|
324
|
+
const cubeGrep = /xcc|cube|keepalive|solicit|probing|register|authorize|reconnect|liveness|connection_state/i;
|
|
325
|
+
parts.push(await section('LOGS: web (filtered, tail 800)', () => logsFiltered('web', 6000, 800, cubeGrep)));
|
|
326
|
+
parts.push(await section('LOGS: nats (tail 300)', () => logsFiltered('nats', 300, 300, null)));
|
|
327
|
+
const bundle = redactSecrets(parts.join('\n') + '\n');
|
|
328
|
+
await writeFile(outputPath, bundle, { mode: 0o600 });
|
|
329
|
+
// `mode` only applies when writeFile CREATES the file. If outputPath already
|
|
330
|
+
// existed with broader perms, the write truncates but leaves the old mode, so
|
|
331
|
+
// the (potentially secret-bearing) bundle could stay world-readable. chmod
|
|
332
|
+
// unconditionally to clamp an existing file down to owner-only.
|
|
333
|
+
await chmod(outputPath, 0o600);
|
|
334
|
+
return outputPath;
|
|
335
|
+
}
|
|
336
|
+
//# sourceMappingURL=cube-diag.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cube-diag.js","sourceRoot":"","sources":["../../src/lib/cube-diag.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAEjC,MAAM,cAAc,GAAG,wBAAwB,CAAC;AAChD,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AACpC,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,MAAM,WAAW,GAAG,MAAM,CAAC;AAE3B,iDAAiD;AACjD,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAuC3B,CAAC;AAEL;;;;;;GAMG;AACH,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;IAkBxB,CAAC;AAEL,sEAAsE;AACtE,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,IAAI,EACJ,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,EACjD,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,CACtD,CAAC;IACF,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,OAAO,CAAC,KAAa,EAAE,EAAyB;IAC7D,MAAM,KAAK,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1F,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACrG,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YACnD,IAAI,UAAU;gBAAE,OAAO,UAAU,CAAC;QACpC,CAAC;IACH,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,KAAK,CAAC,8CAA8C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1G,CAAC;IACD,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC;IAChG,OAAO,GAAG,OAAO,KAAK,CAAC;AACzB,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,OAAO,CAAC,GAAW,EAAE,QAAkB;IACpD,MAAM,MAAM,GAAG,MAAM,IAAI,CACvB,QAAQ,EACR,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,QAAQ,CAAC,EACtF,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,CAC5C,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAChD,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACxF,CAAC;AAED,6EAA6E;AAC7E,+EAA+E;AAC/E,kFAAkF;AAClF,kFAAkF;AAClF,kFAAkF;AAClF,kFAAkF;AAClF,MAAM,mBAAmB,GACvB,wFAAwF;IACxF,2BAA2B;IAC3B,0BAA0B;IAC1B,qDAAqD;IACrD,uCAAuC;IACvC,2EAA2E;IAC3E,MAAM,CAAC;AAET,gGAAgG;AAChG,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,MAAc;IACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CACvB,QAAQ,EACR;QACE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG;QAC/B,IAAI,EAAE,YAAY,QAAQ,EAAE;QAC5B,IAAI,EAAE,aAAa,MAAM,EAAE;QAC3B,cAAc,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB;KAChD,EACD,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,CAC5C,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAChD,sEAAsE;IACtE,MAAM,QAAQ,GAAI,MAAiC,CAAC,QAAQ,KAAK,IAAI,CAAC;IACtE,OAAO;QACL,MAAM;QACN,QAAQ,CAAC,CAAC,CAAC,4DAA4D,CAAC,CAAC,CAAC,EAAE;QAC5E,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;KAC1C,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,+EAA+E;AAC/E,KAAK,UAAU,WAAW,CAAC,OAAe;IACxC,4EAA4E;IAC5E,8EAA8E;IAC9E,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,KAAK,CAAC,wBAAwB,OAAO,YAAY,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,MAAc,EACd,QAAgB,EAChB,IAAmB,EACnB,IAAyB;IAEzB,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,CAAC,EAAE;QAAE,OAAO,OAAO,OAAO,mBAAmB,CAAC;IAClD,MAAM,UAAU,GAAG,IAAI,EAAE,KAAK;QAC5B,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;QAC9C,OAAO,EAAE,eAAe;QACxB,MAAM,EAAE,KAAK;KACd,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IAC5F,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,IAAI;QAAE,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAA0E;IAC9G,MAAM,WAAW,GAAG,IAAI,EAAE,WAAW,IAAI,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,EAAE,WAAW,IAAI,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,IAAI,EAAE,UAAU,IAAI,eAAe,GAAG,MAAM,CAAC;IAEhE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CACR,oCAAoC,EACpC,cAAc,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EACxC,qHAAqH,CACtH,CAAC;IAEF,0EAA0E;IAC1E,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,CACzC,IAAI,CACF,8EAA8E;QAC9E,qFAAqF;QACrF,iFAAiF;QACjF,kFAAkF,CACnF,CACF,CAAC,CAAC;IAEH,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,kDAAkD,EAAE,GAAG,EAAE,CAChF,IAAI,CACF,6FAA6F;QAC7F,gGAAgG;QAChG,2GAA2G,CAC5G,CACF,CAAC,CAAC;IAEH,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,+CAA+C,EAAE,GAAG,EAAE,CAC7E,IAAI,CACF,4FAA4F;QAC5F,wEAAwE,CACzE,CACF,CAAC,CAAC;IAEH,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,2CAA2C,EAAE,GAAG,EAAE,CACzE,IAAI,CACF,mFAAmF;QACnF,4CAA4C;QAC5C,6FAA6F;QAC7F,4BAA4B,CAC7B,CACF,CAAC,CAAC;IAEH,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,2CAA2C,EAAE,GAAG,EAAE,CACzE,IAAI,CACF,8DAA8D;QAC9D,oFAAoF,CACrF,CACF,CAAC,CAAC;IAEH,0EAA0E;IAC1E,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC;IACrC,KAAK,CAAC,IAAI,CAAC,oCAAoC,GAAG,QAAQ,CAAC,CAAC;IAE5D,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,2BAA2B,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,6BAA6B,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,+BAA+B,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;IACjG,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,iCAAiC,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IACrG,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3F,2EAA2E;IAC3E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,wBAAwB;IACxB,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,yCAAyC,EAAE,GAAG,EAAE,CACvE,UAAU,CAAC,sBAAsB,CAAC,CACnC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,8EAA8E;IAC9E,uEAAuE;IACvE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,wBAAwB,WAAW,iBAAiB,EAAE,KAAK,IAAI,EAAE;YACxF,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,IAAI,CAAC;YAC5C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;gBACxB,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,mBAAmB,CAAC,CAAC;oBAClD,MAAM,UAAU,GAAG,GAAG;yBACnB,KAAK,CAAC,IAAI,CAAC;yBACX,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC;gBACxF,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACjE,OAAO,CAAC,IAAI,CAAC,yBAAyB,OAAO,EAAE,CAAC,CAAC;gBACnD,CAAC;gBACD,yEAAyE;gBACzE,4EAA4E;gBAC5E,MAAM,SAAS,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACnC,IAAI,SAAS,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,0EAA0E;IAC1E,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,MAAM,OAAO,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;YAC7C,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,sBAAsB,OAAO,KAAK,WAAW,IAAI,EAAE,KAAK,IAAI,EAAE;gBACrF,MAAM,MAAM,GAAG,MAAM,IAAI,CACvB,SAAS,EACT;oBACE,MAAM,CAAC,WAAW,CAAC;oBACnB,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,cAAc;oBACzD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO;iBAChD,EACD,EAAE,OAAO,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CACrD,CAAC;gBACF,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzG,CAAC,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,4EAA4E;IAC5E,+EAA+E;IAC/E,MAAM,aAAa,GAAG,yEAAyE,CAAC;IAChG,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,yEAAyE,EAAE,GAAG,EAAE,CACvG,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAChE,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,4FAA4F,CAAC;IAC9G,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,gCAAgC,EAAE,GAAG,EAAE,CAC9D,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CACzC,CAAC,CAAC;IACH,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,uBAAuB,EAAE,GAAG,EAAE,CACrD,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CACrC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IACtD,MAAM,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACrD,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,gEAAgE;IAChE,MAAM,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC/B,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 016: SeaweedFS mount + permission recovery.
|
|
3
|
+
*
|
|
4
|
+
* Two failure modes crash-loop SeaweedFS after an upgrade/restore and then
|
|
5
|
+
* cascade through `depends_on: service_healthy` to fail the whole upgrade:
|
|
6
|
+
*
|
|
7
|
+
* 1. Bind-mount dir-trap — `seaweedfs-s3.json` (and `nats.conf`) are FILE
|
|
8
|
+
* bind-mounts. If the source is missing when `docker compose up` runs,
|
|
9
|
+
* Docker auto-creates a DIRECTORY at the source and SeaweedFS dies with
|
|
10
|
+
* "fail to load config file /etc/seaweedfs/s3.json: ... is a directory".
|
|
11
|
+
* Migration 009 fixed the alertmanager/tempo/prometheus bind-mounts but not
|
|
12
|
+
* these two — this migration closes that gap for existing appliances.
|
|
13
|
+
*
|
|
14
|
+
* 2. Data-volume not writable — SeaweedFS does an explicit owner-write-bit
|
|
15
|
+
* check on its `-mdir` and fatals even as root:
|
|
16
|
+
* file_util.go: Folder /data Permission: -r-xr-xr-x
|
|
17
|
+
* server.go: Check Meta Folder (-mdir="/data") Writable: Not writable!
|
|
18
|
+
* A restore, a bad extraction, or a stray chmod/chattr can leave the volume
|
|
19
|
+
* read-only. Restore the owner-write bit (and clear an immutable flag).
|
|
20
|
+
*
|
|
21
|
+
* Runs as root (migrations run under the update flow's sudo). Best-effort and
|
|
22
|
+
* idempotent.
|
|
23
|
+
*/
|
|
24
|
+
export declare function migrateSeaweedfsRecovery(): Promise<boolean>;
|
|
25
|
+
//# sourceMappingURL=migration-016-seaweedfs-recovery.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migration-016-seaweedfs-recovery.d.ts","sourceRoot":"","sources":["../../src/lib/migration-016-seaweedfs-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAqGH,wBAAsB,wBAAwB,IAAI,OAAO,CAAC,OAAO,CAAC,CAMjE"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 016: SeaweedFS mount + permission recovery.
|
|
3
|
+
*
|
|
4
|
+
* Two failure modes crash-loop SeaweedFS after an upgrade/restore and then
|
|
5
|
+
* cascade through `depends_on: service_healthy` to fail the whole upgrade:
|
|
6
|
+
*
|
|
7
|
+
* 1. Bind-mount dir-trap — `seaweedfs-s3.json` (and `nats.conf`) are FILE
|
|
8
|
+
* bind-mounts. If the source is missing when `docker compose up` runs,
|
|
9
|
+
* Docker auto-creates a DIRECTORY at the source and SeaweedFS dies with
|
|
10
|
+
* "fail to load config file /etc/seaweedfs/s3.json: ... is a directory".
|
|
11
|
+
* Migration 009 fixed the alertmanager/tempo/prometheus bind-mounts but not
|
|
12
|
+
* these two — this migration closes that gap for existing appliances.
|
|
13
|
+
*
|
|
14
|
+
* 2. Data-volume not writable — SeaweedFS does an explicit owner-write-bit
|
|
15
|
+
* check on its `-mdir` and fatals even as root:
|
|
16
|
+
* file_util.go: Folder /data Permission: -r-xr-xr-x
|
|
17
|
+
* server.go: Check Meta Folder (-mdir="/data") Writable: Not writable!
|
|
18
|
+
* A restore, a bad extraction, or a stray chmod/chattr can leave the volume
|
|
19
|
+
* read-only. Restore the owner-write bit (and clear an immutable flag).
|
|
20
|
+
*
|
|
21
|
+
* Runs as root (migrations run under the update flow's sudo). Best-effort and
|
|
22
|
+
* idempotent.
|
|
23
|
+
*/
|
|
24
|
+
import { existsSync, statSync, rmSync, writeFileSync, chmodSync } from 'node:fs';
|
|
25
|
+
import { join } from 'node:path';
|
|
26
|
+
import { execa } from 'execa';
|
|
27
|
+
import { getPaths } from './paths.js';
|
|
28
|
+
import { readEnvKeys } from './env.js';
|
|
29
|
+
const NATS_CONF_CONTENT = `# Maximum payload
|
|
30
|
+
max_payload: 8MB
|
|
31
|
+
|
|
32
|
+
# HTTP monitoring port — required for Docker healthcheck (/healthz)
|
|
33
|
+
http_port: 8222
|
|
34
|
+
|
|
35
|
+
jetstream {
|
|
36
|
+
store_dir: "/jetstream/store"
|
|
37
|
+
max_memory_store: 1GB
|
|
38
|
+
max_file_store: 10GB
|
|
39
|
+
}
|
|
40
|
+
`;
|
|
41
|
+
function jsonEscape(value) {
|
|
42
|
+
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
43
|
+
}
|
|
44
|
+
function seaweedS3Content() {
|
|
45
|
+
const env = readEnvKeys();
|
|
46
|
+
// This CLI's S3 secret model uses S3_ROOT_USER / S3_ROOT_PASSWORD (see
|
|
47
|
+
// secrets.ts); fall back to the legacy S3_ACCESS_KEY_ID / S3_SECRET_ACCESS_KEY
|
|
48
|
+
// names, then to the bundle default, so a recovered config keeps SeaweedFS auth
|
|
49
|
+
// in sync with the configured credentials.
|
|
50
|
+
const ak = jsonEscape(env.S3_ROOT_USER || env.S3_ACCESS_KEY_ID || 'changeme');
|
|
51
|
+
const sk = jsonEscape(env.S3_ROOT_PASSWORD || env.S3_SECRET_ACCESS_KEY || 'changeme');
|
|
52
|
+
return `{
|
|
53
|
+
"identities": [
|
|
54
|
+
{
|
|
55
|
+
"name": "admin",
|
|
56
|
+
"credentials": [
|
|
57
|
+
{ "accessKey": "${ak}", "secretKey": "${sk}" }
|
|
58
|
+
],
|
|
59
|
+
"actions": ["Admin", "Read", "Write", "List", "Tagging"]
|
|
60
|
+
}
|
|
61
|
+
]
|
|
62
|
+
}
|
|
63
|
+
`;
|
|
64
|
+
}
|
|
65
|
+
/** Clobber a stray directory at a FILE bind-mount source and (re)create the file. */
|
|
66
|
+
function fixBindMountFile(filePath, content) {
|
|
67
|
+
let changed = false;
|
|
68
|
+
if (existsSync(filePath) && statSync(filePath).isDirectory()) {
|
|
69
|
+
rmSync(filePath, { recursive: true, force: true });
|
|
70
|
+
changed = true;
|
|
71
|
+
}
|
|
72
|
+
if (!existsSync(filePath)) {
|
|
73
|
+
writeFileSync(filePath, content);
|
|
74
|
+
changed = true;
|
|
75
|
+
}
|
|
76
|
+
return changed;
|
|
77
|
+
}
|
|
78
|
+
/** Restore the SeaweedFS data volume's owner-write bit when it is read-only.
|
|
79
|
+
* Fixes EVERY matching `*_seaweedfs-data` volume (a host can carry more than
|
|
80
|
+
* one, e.g. a shared ct-infra_* alongside the app's calltelemetry_*). */
|
|
81
|
+
async function fixDataVolumeWritable() {
|
|
82
|
+
let volumes = [];
|
|
83
|
+
try {
|
|
84
|
+
const { stdout } = await execa('docker', ['volume', 'ls', '-q'], { reject: false });
|
|
85
|
+
volumes = (stdout ?? '')
|
|
86
|
+
.split('\n')
|
|
87
|
+
.map(s => s.trim())
|
|
88
|
+
.filter(name => name === 'seaweedfs-data' || name.endsWith('_seaweedfs-data'));
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
if (volumes.length === 0)
|
|
94
|
+
return false;
|
|
95
|
+
let changed = false;
|
|
96
|
+
for (const vol of volumes) {
|
|
97
|
+
let mount = '';
|
|
98
|
+
try {
|
|
99
|
+
const { stdout } = await execa('docker', ['volume', 'inspect', '-f', '{{.Mountpoint}}', vol], { reject: false });
|
|
100
|
+
mount = (stdout ?? '').trim();
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (!mount || !existsSync(mount) || !statSync(mount).isDirectory())
|
|
106
|
+
continue;
|
|
107
|
+
// Clear an immutable flag where supported (no-op on overlay/unsupported fs).
|
|
108
|
+
await execa('chattr', ['-i', mount], { reject: false }).catch(() => undefined);
|
|
109
|
+
// SeaweedFS checks the owner-write MODE BIT explicitly, so guarantee it.
|
|
110
|
+
const mode = statSync(mount).mode & 0o777;
|
|
111
|
+
if ((mode & 0o200) === 0) {
|
|
112
|
+
chmodSync(mount, mode | 0o700);
|
|
113
|
+
changed = true;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return changed;
|
|
117
|
+
}
|
|
118
|
+
export async function migrateSeaweedfsRecovery() {
|
|
119
|
+
const { installDir } = getPaths();
|
|
120
|
+
const a = fixBindMountFile(join(installDir, 'seaweedfs-s3.json'), seaweedS3Content());
|
|
121
|
+
const b = fixBindMountFile(join(installDir, 'nats.conf'), NATS_CONF_CONTENT);
|
|
122
|
+
const c = await fixDataVolumeWritable();
|
|
123
|
+
return a || b || c;
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=migration-016-seaweedfs-recovery.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migration-016-seaweedfs-recovery.js","sourceRoot":"","sources":["../../src/lib/migration-016-seaweedfs-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,MAAM,iBAAiB,GAAG;;;;;;;;;;;CAWzB,CAAC;AAEF,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,gBAAgB;IACvB,MAAM,GAAG,GAAG,WAAW,EAAE,CAAC;IAC1B,uEAAuE;IACvE,+EAA+E;IAC/E,gFAAgF;IAChF,2CAA2C;IAC3C,MAAM,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,gBAAgB,IAAI,UAAU,CAAC,CAAC;IAC9E,MAAM,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,oBAAoB,IAAI,UAAU,CAAC,CAAC;IACtF,OAAO;;;;;0BAKiB,EAAE,oBAAoB,EAAE;;;;;;CAMjD,CAAC;AACF,CAAC;AAED,qFAAqF;AACrF,SAAS,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IACzD,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7D,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACjC,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;0EAE0E;AAC1E,KAAK,UAAU,qBAAqB;IAClC,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACpF,OAAO,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;aACrB,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAClB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,gBAAgB,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACnF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACjH,KAAK,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,SAAS;QAE7E,6EAA6E;QAC7E,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAE/E,yEAAyE;QACzE,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;QAC1C,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC;YAC/B,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB;IAC5C,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,EAAE,CAAC;IAClC,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,UAAU,EAAE,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACtF,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC7E,MAAM,CAAC,GAAG,MAAM,qBAAqB,EAAE,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migrations.d.ts","sourceRoot":"","sources":["../../src/lib/migrations.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,GAAG,SAAS,CAAC;CAC/B;AAED,UAAU,WAAW;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;CACrC;
|
|
1
|
+
{"version":3,"file":"migrations.d.ts","sourceRoot":"","sources":["../../src/lib/migrations.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,SAAS,GAAG,SAAS,CAAC;CAC/B;AAED,UAAU,WAAW;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;CACrC;AAuBD,eAAO,MAAM,UAAU,EAAE,SAAS,EAiFjC,CAAC;AAEF,wBAAgB,kBAAkB,IAAI,cAAc,CAcnD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI,CAG9D;AAED,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAGxD;AAED,wBAAsB,oBAAoB,CACxC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC1B,OAAO,CAAC;IAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA4C/C;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,MAAM,EACV,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAwC1E;AAED,wBAAgB,kBAAkB,IAAI,KAAK,CAAC;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;IACpD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC,CAgCD"}
|
package/dist/lib/migrations.js
CHANGED
|
@@ -25,6 +25,7 @@ import { migrateDbPoolSizes } from './migration-012-db-pool-sizes.js';
|
|
|
25
25
|
import { migrateGrafanaPassword } from './migration-013-grafana-password.js';
|
|
26
26
|
import { migrateDockerFirewallBackend } from './migration-014-docker-firewall-backend.js';
|
|
27
27
|
import { migrateDockerInternalNoProxy } from './migration-015-docker-internal-no-proxy.js';
|
|
28
|
+
import { migrateSeaweedfsRecovery } from './migration-016-seaweedfs-recovery.js';
|
|
28
29
|
export const MIGRATIONS = [
|
|
29
30
|
{
|
|
30
31
|
id: '001_cgnat_docker_subnets',
|
|
@@ -101,6 +102,11 @@ export const MIGRATIONS = [
|
|
|
101
102
|
description: 'Bypass any egress proxy for internal Docker traffic (prevents proxy from blocking SeaweedFS/ct-media startup)',
|
|
102
103
|
run: migrateDockerInternalNoProxy,
|
|
103
104
|
},
|
|
105
|
+
{
|
|
106
|
+
id: '016_seaweedfs_recovery',
|
|
107
|
+
description: 'Recover SeaweedFS s3.json/nats.conf bind-mount dir-trap + restore data-volume writable bit (prevents crash-loop on upgrade/restore)',
|
|
108
|
+
run: migrateSeaweedfsRecovery,
|
|
109
|
+
},
|
|
104
110
|
];
|
|
105
111
|
export function loadMigrationState() {
|
|
106
112
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/lib/migrations.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAuB7E,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;AACtC,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;AAExD,oBAAoB;AACpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,wCAAwC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,4BAA4B,EAAE,MAAM,6CAA6C,CAAC;
|
|
1
|
+
{"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/lib/migrations.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAuB7E,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;AACtC,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;AAExD,oBAAoB;AACpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,wCAAwC,CAAC;AAClF,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,4BAA4B,EAAE,MAAM,6CAA6C,CAAC;AAC3F,OAAO,EAAE,wBAAwB,EAAE,MAAM,uCAAuC,CAAC;AAEjF,MAAM,CAAC,MAAM,UAAU,GAAgB;IACrC;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,+CAA+C;QAC5D,GAAG,EAAE,mBAAmB;KACzB;IACD;QACE,EAAE,EAAE,eAAe;QACnB,WAAW,EAAE,kCAAkC;QAC/C,GAAG,EAAE,eAAe;KACrB;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sEAAsE;QACnF,GAAG,EAAE,aAAa;KACnB;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,wCAAwC;QACrD,GAAG,EAAE,oBAAoB;KAC1B;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,WAAW,EAAE,6CAA6C;QAC1D,GAAG,EAAE,cAAc;KACpB;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,oDAAoD;QACjE,GAAG,EAAE,sBAAsB;KAC5B;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,2DAA2D;QACxE,GAAG,EAAE,gBAAgB;KACtB;IACD;QACE,EAAE,EAAE,4BAA4B;QAChC,WAAW,EAAE,qEAAqE;QAClF,GAAG,EAAE,qBAAqB;KAC3B;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,sEAAsE;QACnF,GAAG,EAAE,qBAAqB;KAC3B;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,gEAAgE;QAC7E,GAAG,EAAE,iBAAiB;KACvB;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,mDAAmD;QAChE,GAAG,EAAE,wBAAwB;KAC9B;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,WAAW,EAAE,0DAA0D;QACvE,GAAG,EAAE,kBAAkB;KACxB;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,0DAA0D;QACvE,GAAG,EAAE,sBAAsB;KAC5B;IACD;QACE,EAAE,EAAE,6BAA6B;QACjC,WAAW,EAAE,4FAA4F;QACzG,GAAG,EAAE,4BAA4B;KAClC;IACD;QACE,EAAE,EAAE,8BAA8B;QAClC,WAAW,EAAE,+GAA+G;QAC5H,GAAG,EAAE,4BAA4B;KAClC;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,qIAAqI;QAClJ,GAAG,EAAE,wBAAwB;KAC9B;CACF,CAAC;AAEF,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QACvC,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE;SAC5B,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAqB;IACtD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,EAAU;IAC7C,MAAM,KAAK,GAAG,kBAAkB,EAAE,CAAC;IACnC,OAAO,EAAE,IAAI,KAAK,CAAC,SAAS,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAA2B;IAE3B,MAAM,KAAK,GAAG,kBAAkB,EAAE,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,GAAG,EAAE,CAAC,sBAAsB,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC;YAEtC,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG;gBAC9B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACxC,CAAC;YAEF,8CAA8C;YAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAElC,IAAI,OAAO,EAAE,CAAC;gBACZ,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACvB,GAAG,EAAE,CAAC,cAAc,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAC3B,GAAG,EAAE,CAAC,gCAAgC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG;gBAC3B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC5B,KAAK,EAAE,OAAO;aACf,CAAC;YACF,GAAG,EAAE,CAAC,YAAY,SAAS,CAAC,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;QACjD,CAAC;QAED,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,EAAU,EACV,GAA2B;IAE3B,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjC,CAAC;IAED,MAAM,KAAK,GAAG,kBAAkB,EAAE,CAAC;IAEnC,GAAG,EAAE,CAAC,sBAAsB,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC;QAEtC,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG;YAC9B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACxC,CAAC;QAEF,8CAA8C;QAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAElC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE1B,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,EAAE,CAAC,cAAc,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,GAAG,EAAE,CAAC,gCAAgC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;YACtD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG;YAC3B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,KAAK,EAAE,OAAO;SACf,CAAC;QACF,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC1B,GAAG,EAAE,CAAC,YAAY,SAAS,CAAC,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;QAC/C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB;IAOhC,MAAM,KAAK,GAAG,kBAAkB,EAAE,CAAC;IAEnC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAErC,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC;QACJ,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,MAAM,EAAE,OAAgB;gBACxB,EAAE,EAAE,SAAS,CAAC,EAAE;gBAChB,KAAK,EAAE,SAAS,CAAC,KAAK;aACvB,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,MAAM,EAAE,SAAkB;SAC3B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-upgrade health diagnostics.
|
|
3
|
+
*
|
|
4
|
+
* After an upgrade, enumerate every compose service and, for each container NOT
|
|
5
|
+
* in a healthy running state, capture its last 100 log lines to a single
|
|
6
|
+
* failed-pods-<ts>.log file (echoed to screen too) so the operator sees WHY it
|
|
7
|
+
* failed without hunting through `docker logs`. When the upgrade failed (a step
|
|
8
|
+
* threw) OR any container is unhealthy, also collect the full show-tech bundle
|
|
9
|
+
* for surrounding context and print its archive path.
|
|
10
|
+
*
|
|
11
|
+
* Best-effort and never throws — diagnostics must never mask the underlying
|
|
12
|
+
* upgrade failure.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* True when a container's status means it is NOT healthily running.
|
|
16
|
+
*
|
|
17
|
+
* A cleanly-exited container ("Exited (0)") is only treated as healthy when its
|
|
18
|
+
* service is a one-shot init (name matches {@link INIT_SERVICE_RE}); an
|
|
19
|
+
* Exited (0) long-running service is down and IS flagged. Exported for unit
|
|
20
|
+
* testing the matcher in isolation.
|
|
21
|
+
*/
|
|
22
|
+
export declare function isFailedContainerState(status: string, service?: string): boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Run the post-upgrade health capture. `upgradeFailed` should be true when an
|
|
25
|
+
* update step threw — in that case the show-tech bundle is collected even if the
|
|
26
|
+
* containers themselves report healthy (the failure is likely in
|
|
27
|
+
* migrations/RPC/healthz). Returns true if any failed container was detected.
|
|
28
|
+
*
|
|
29
|
+
* Never throws: a top-level guard converts any unexpected error into a warning.
|
|
30
|
+
*/
|
|
31
|
+
export declare function capturePostUpgradeHealth(reason: string, options?: {
|
|
32
|
+
upgradeFailed?: boolean;
|
|
33
|
+
}): Promise<boolean>;
|
|
34
|
+
//# sourceMappingURL=post-upgrade-health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"post-upgrade-health.d.ts","sourceRoot":"","sources":["../../src/lib/post-upgrade-health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AA+BH;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,SAAK,GAAG,OAAO,CAG5E;AA8ED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAO,GACxC,OAAO,CAAC,OAAO,CAAC,CAOlB"}
|